mirror of
https://gitea.com/gitea/runner.git
synced 2026-11-16 21:03:43 +00:00
Compare commits
15
Commits
f2fbd67962
...
v5.4.1
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
bb6d793cb9 | ||
|
|
95ef17d022 | ||
|
|
ee03687688 | ||
|
|
30e600d8f6 | ||
|
|
664978c819 | ||
|
|
aa56f46c89 | ||
|
|
7a9fccb842 | ||
|
|
7d6e1ff021 | ||
|
|
319567213d | ||
|
|
0093731295 | ||
|
|
3e1b4b42c2 | ||
|
|
dcba546cb3 | ||
|
|
142d9f8c88 | ||
|
|
54b2cb944c | ||
|
|
b65fd0f8f8 |
No files matched your search
@@ -34,6 +34,13 @@ jobs:
|
|||||||
with:
|
with:
|
||||||
go-version-file: "go.mod"
|
go-version-file: "go.mod"
|
||||||
check-latest: true
|
check-latest: true
|
||||||
|
- name: Import GPG key
|
||||||
|
id: import_gpg
|
||||||
|
uses: crazy-max/ghaction-import-gpg@2dc316deee8e90f13e1a351ab510b4d5bc0c82cd # v7
|
||||||
|
with:
|
||||||
|
gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }}
|
||||||
|
passphrase: ${{ secrets.PASSPHRASE }}
|
||||||
|
fingerprint: CC64B1DB67ABBEECAB24B6455FC346329753F4B0
|
||||||
- name: goreleaser
|
- name: goreleaser
|
||||||
uses: goreleaser/goreleaser-action@a4f02ab5355a589de60d917a659ce617ed19b3de # v7
|
uses: goreleaser/goreleaser-action@a4f02ab5355a589de60d917a659ce617ed19b3de # v7
|
||||||
with:
|
with:
|
||||||
@@ -47,6 +54,7 @@ jobs:
|
|||||||
R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
|
R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
|
||||||
GORELEASER_FORCE_TOKEN: "gitea"
|
GORELEASER_FORCE_TOKEN: "gitea"
|
||||||
GITEA_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
GITEA_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||||
|
GPG_FINGERPRINT: ${{ steps.import_gpg.outputs.fingerprint }}
|
||||||
|
|
||||||
release-image:
|
release-image:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
|
|||||||
@@ -1,21 +0,0 @@
|
|||||||
name: windows
|
|
||||||
on:
|
|
||||||
schedule:
|
|
||||||
- cron: '0 3 * * 1'
|
|
||||||
workflow_dispatch:
|
|
||||||
|
|
||||||
env:
|
|
||||||
DOCKER_CONFIG: /tmp/docker-noauth # see test.yml
|
|
||||||
|
|
||||||
jobs:
|
|
||||||
test:
|
|
||||||
name: test on windows
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
timeout-minutes: 180
|
|
||||||
steps:
|
|
||||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
|
||||||
- uses: ./.gitea/actions/go-setup
|
|
||||||
- name: prepare anonymous docker config
|
|
||||||
run: mkdir -p "$DOCKER_CONFIG" && echo '{}' > "$DOCKER_CONFIG/config.json"
|
|
||||||
- name: test on windows
|
|
||||||
run: make test-windows
|
|
||||||
@@ -57,6 +57,32 @@ jobs:
|
|||||||
make coverage-report
|
make coverage-report
|
||||||
cat .tmp/coverage.md >> "$GITHUB_STEP_SUMMARY"
|
cat .tmp/coverage.md >> "$GITHUB_STEP_SUMMARY"
|
||||||
|
|
||||||
|
windows:
|
||||||
|
name: test on windows
|
||||||
|
runs-on: ubuntu-latest-kvm
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||||
|
- uses: ./.gitea/actions/go-setup
|
||||||
|
- name: prepare anonymous docker config
|
||||||
|
run: mkdir -p "$DOCKER_CONFIG" && echo '{}' > "$DOCKER_CONFIG/config.json"
|
||||||
|
- name: windows vm cache key
|
||||||
|
id: windows-vm
|
||||||
|
run: echo "key=windows-vm-$(grep '^WINDOWS_TEST_IMAGE' Makefile | cat - scripts/test-windows.sh scripts/test-windows-setup.ps1 | sha256sum | cut -c1-16)" >> "$GITHUB_OUTPUT"
|
||||||
|
- uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
|
||||||
|
with:
|
||||||
|
path: ~/.cache/windows-vm
|
||||||
|
key: ${{ steps.windows-vm.outputs.key }}
|
||||||
|
- name: test on windows
|
||||||
|
id: test-windows
|
||||||
|
run: WINDOWS_TEST_CACHE="$HOME/.cache/windows-vm" make test-windows
|
||||||
|
env:
|
||||||
|
WINDOWS_TEST_TIMEOUT: 45
|
||||||
|
- if: always() && steps.test-windows.outputs.save-cache == 'true'
|
||||||
|
uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
|
||||||
|
with:
|
||||||
|
path: ~/.cache/windows-vm
|
||||||
|
key: ${{ steps.windows-vm.outputs.key }}
|
||||||
|
|
||||||
e2e:
|
e2e:
|
||||||
name: gitea compatibility (${{ matrix.gitea_image }})
|
name: gitea compatibility (${{ matrix.gitea_image }})
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
|
|||||||
@@ -39,6 +39,8 @@ linters:
|
|||||||
deny:
|
deny:
|
||||||
- pkg: golang.org/x/exp
|
- pkg: golang.org/x/exp
|
||||||
desc: it's experimental and unreliable
|
desc: it's experimental and unreliable
|
||||||
|
- pkg: unsafe
|
||||||
|
desc: use typed APIs instead
|
||||||
logging:
|
logging:
|
||||||
files:
|
files:
|
||||||
- $all
|
- $all
|
||||||
|
|||||||
@@ -80,6 +80,7 @@ builds:
|
|||||||
publishers:
|
publishers:
|
||||||
- name: cloudflare-r2
|
- name: cloudflare-r2
|
||||||
checksum: true
|
checksum: true
|
||||||
|
signature: true
|
||||||
extra_files:
|
extra_files:
|
||||||
- glob: ./**.xz
|
- glob: ./**.xz
|
||||||
- glob: ./**.sha256
|
- glob: ./**.sha256
|
||||||
@@ -100,6 +101,10 @@ checksum:
|
|||||||
extra_files:
|
extra_files:
|
||||||
- glob: ./**.xz
|
- glob: ./**.xz
|
||||||
|
|
||||||
|
signs:
|
||||||
|
- artifacts: checksum
|
||||||
|
args: ["--batch", "-u", "{{ .Env.GPG_FINGERPRINT }}", "--output", "${signature}", "--detach-sign", "${artifact}"]
|
||||||
|
|
||||||
snapshot:
|
snapshot:
|
||||||
version_template: "{{ .Branch }}-devel"
|
version_template: "{{ .Branch }}-devel"
|
||||||
|
|
||||||
|
|||||||
+9
-5
@@ -23,11 +23,14 @@ depending on the prefix:
|
|||||||
covers both forms: it escapes `%` first, so a literal `%3B` becomes `%253B` that the extra
|
covers both forms: it escapes `%` first, so a literal `%3B` becomes `%253B` that the extra
|
||||||
`##[…]` rules cannot match, and a raw `;` or `]` is never decoded. It is also what makes
|
`##[…]` rules cannot match, and a raw `;` or `]` is never decoded. It is also what makes
|
||||||
multi-line work, `\n` becomes `%0A` and the UI turns it back into a line break.
|
multi-line work, `\n` becomes `%0A` and the UI turns it back into a line break.
|
||||||
- **Forwarding a command from step output?** Leave the payload alone, it arrived escaped and is
|
- **Forwarding a command from step output?** Preserve its escaping unless masking changes it.
|
||||||
decoded once. Decoding here double-decodes and destroys multi-line.
|
Decoding for display happens in the UI.
|
||||||
- **No prefix?** Do not escape, and split multi-line values into one row each.
|
- **No prefix?** Do not escape, and split multi-line values into one row each.
|
||||||
- **Interpolating a secret?** Masking runs after escaping, so `AppendSecretMasker` registers the
|
- **Interpolating a secret?** Masking runs after escaping. Recognised command payloads are
|
||||||
encoded forms too.
|
normalised before masking to cover partially escaped values. Their original escaping is
|
||||||
|
preserved when no replacement occurs. `AppendSecretMasker` also registers encoded forms for
|
||||||
|
other output. Complete nonblank one-character values are masked, while one-byte fragments
|
||||||
|
of multiline values are skipped.
|
||||||
- Command *properties* also escape `%3A` and `%2C`, which the UI never decodes, so the command
|
- Command *properties* also escape `%3A` and `%2C`, which the UI never decodes, so the command
|
||||||
handler folds an annotation's location into its decoded message and escapes the result again.
|
handler folds an annotation's location into its decoded message and escapes the result again.
|
||||||
|
|
||||||
@@ -77,4 +80,5 @@ repository runners. A run path is `<workflow>@<ref>` and a log row is `<timestam
|
|||||||
The host must be amd64 and allow privileged containers. `WINDOWS_TEST_TIMEOUT` sets the minutes
|
The host must be amd64 and allow privileged containers. `WINDOWS_TEST_TIMEOUT` sets the minutes
|
||||||
allowed for installing Windows and running the tests, 165 by default. The VM uses KVM when the host
|
allowed for installing Windows and running the tests, 165 by default. The VM uses KVM when the host
|
||||||
has `/dev/kvm`, otherwise Windows is emulated and much slower. Docker-backed tests skip in the VM
|
has `/dev/kvm`, otherwise Windows is emulated and much slower. Docker-backed tests skip in the VM
|
||||||
and the race detector is off, it needs cgo. CI runs this weekly and on demand.
|
and the race detector is off, it needs cgo. `WINDOWS_TEST_CACHE` names a directory to keep the
|
||||||
|
installed Windows in, so later runs skip the install. CI runs this on every pull request.
|
||||||
+7
-5
@@ -19,14 +19,15 @@ RUN --mount=type=cache,target=/go/pkg/mod \
|
|||||||
### DIND VARIANT
|
### DIND VARIANT
|
||||||
#
|
#
|
||||||
#
|
#
|
||||||
FROM docker:29.9.2-dind AS dind
|
FROM docker:29.9.3-dind AS dind
|
||||||
|
|
||||||
ARG VERSION=dev
|
ARG VERSION=dev
|
||||||
|
|
||||||
LABEL org.opencontainers.image.source="https://gitea.com/gitea/runner"
|
LABEL org.opencontainers.image.source="https://gitea.com/gitea/runner"
|
||||||
LABEL org.opencontainers.image.version="${VERSION}"
|
LABEL org.opencontainers.image.version="${VERSION}"
|
||||||
|
|
||||||
RUN apk add --no-cache s6 bash git tzdata nftables
|
# upgrade first: the docker base image can lag behind Alpine security fixes
|
||||||
|
RUN apk upgrade --no-cache && apk add --no-cache s6 bash git tzdata nftables
|
||||||
|
|
||||||
COPY --from=builder /opt/src/runner/gitea-runner /usr/local/bin/gitea-runner
|
COPY --from=builder /opt/src/runner/gitea-runner /usr/local/bin/gitea-runner
|
||||||
COPY scripts/run.sh /usr/local/bin/run.sh
|
COPY scripts/run.sh /usr/local/bin/run.sh
|
||||||
@@ -41,7 +42,7 @@ ENTRYPOINT ["s6-svscan","/etc/s6"]
|
|||||||
### DIND-ROOTLESS VARIANT
|
### DIND-ROOTLESS VARIANT
|
||||||
#
|
#
|
||||||
#
|
#
|
||||||
FROM docker:29.9.2-dind-rootless AS dind-rootless
|
FROM docker:29.9.3-dind-rootless AS dind-rootless
|
||||||
|
|
||||||
ARG VERSION=dev
|
ARG VERSION=dev
|
||||||
|
|
||||||
@@ -49,7 +50,8 @@ LABEL org.opencontainers.image.source="https://gitea.com/gitea/runner"
|
|||||||
LABEL org.opencontainers.image.version="${VERSION}"
|
LABEL org.opencontainers.image.version="${VERSION}"
|
||||||
|
|
||||||
USER root
|
USER root
|
||||||
RUN apk add --no-cache s6 bash git tzdata nftables
|
# upgrade first: the docker base image can lag behind Alpine security fixes
|
||||||
|
RUN apk upgrade --no-cache && apk add --no-cache s6 bash git tzdata nftables
|
||||||
|
|
||||||
COPY --from=builder /opt/src/runner/gitea-runner /usr/local/bin/gitea-runner
|
COPY --from=builder /opt/src/runner/gitea-runner /usr/local/bin/gitea-runner
|
||||||
COPY scripts/run.sh /usr/local/bin/run.sh
|
COPY scripts/run.sh /usr/local/bin/run.sh
|
||||||
@@ -75,7 +77,7 @@ ARG VERSION=dev
|
|||||||
LABEL org.opencontainers.image.source="https://gitea.com/gitea/runner"
|
LABEL org.opencontainers.image.source="https://gitea.com/gitea/runner"
|
||||||
LABEL org.opencontainers.image.version="${VERSION}"
|
LABEL org.opencontainers.image.version="${VERSION}"
|
||||||
|
|
||||||
RUN apk add --no-cache tini bash git tzdata
|
RUN apk upgrade --no-cache && apk add --no-cache tini bash git tzdata
|
||||||
|
|
||||||
COPY --from=builder /opt/src/runner/gitea-runner /usr/local/bin/gitea-runner
|
COPY --from=builder /opt/src/runner/gitea-runner /usr/local/bin/gitea-runner
|
||||||
COPY scripts/run.sh /usr/local/bin/run.sh
|
COPY scripts/run.sh /usr/local/bin/run.sh
|
||||||
|
|||||||
@@ -153,7 +153,7 @@ test-dind: ## run the daemon-facing tests against the built dind image or podman
|
|||||||
|
|
||||||
.PHONY: test-windows
|
.PHONY: test-windows
|
||||||
test-windows: ## run the tests on Windows Server Core in a QEMU VM on the docker host
|
test-windows: ## run the tests on Windows Server Core in a QEMU VM on the docker host
|
||||||
@WINDOWS_TEST_IMAGE=$(WINDOWS_TEST_IMAGE) ./scripts/test-windows.sh
|
@GO=$(GO) WINDOWS_TEST_IMAGE=$(WINDOWS_TEST_IMAGE) ./scripts/test-windows.sh
|
||||||
|
|
||||||
E2E_CONCURRENCY ?= 8
|
E2E_CONCURRENCY ?= 8
|
||||||
|
|
||||||
|
|||||||
@@ -95,7 +95,7 @@ The recommended Docker-in-Docker image runs its own Docker daemon for jobs. Jobs
|
|||||||
|
|
||||||
```bash
|
```bash
|
||||||
docker run --privileged -e GITEA_INSTANCE_URL=https://your_gitea.com -e GITEA_RUNNER_REGISTRATION_TOKEN=<your_token> \
|
docker run --privileged -e GITEA_INSTANCE_URL=https://your_gitea.com -e GITEA_RUNNER_REGISTRATION_TOKEN=<your_token> \
|
||||||
-v "$PWD/data:/data" -v runner-docker:/var/lib/docker --name my_runner gitea/runner:nightly-dind
|
-v "$PWD/data:/data" -v runner-docker:/var/lib/docker --name my_runner gitea/runner:latest-dind
|
||||||
```
|
```
|
||||||
|
|
||||||
The volumes keep the registration and the image cache across container recreation. The image reads more variables, such as `GITEA_RUNNER_LABELS` and `CONFIG_FILE`, see [scripts/run.sh](scripts/run.sh) and the [Docker example](examples/docker/README.md).
|
The volumes keep the registration and the image cache across container recreation. The image reads more variables, such as `GITEA_RUNNER_LABELS` and `CONFIG_FILE`, see [scripts/run.sh](scripts/run.sh) and the [Docker example](examples/docker/README.md).
|
||||||
@@ -164,7 +164,7 @@ ubuntu-latest:docker://docker.gitea.com/runner-images:ubuntu-latest,macos:host
|
|||||||
|
|
||||||
[gitea/runner-images](https://gitea.com/gitea/runner-images) has suitable images. Jobs matching no label run in `runner.default_image`, or on the host when docker is unavailable.
|
[gitea/runner-images](https://gitea.com/gitea/runner-images) has suitable images. Jobs matching no label run in `runner.default_image`, or on the host when docker is unavailable.
|
||||||
|
|
||||||
Host jobs still need a Docker daemon for Docker actions and `docker://` steps. Windows hosts do not support those, as on GitHub. On Windows, a step's background processes end with the job unless it opts out, see [host mode](docs/host-mode.md).
|
Host jobs still need a Docker daemon for Docker actions and `docker://` steps. Windows hosts do not support those, as on GitHub. A step's background processes end with the job unless it opts out, see [host mode](docs/host-mode.md).
|
||||||
|
|
||||||
#### Registration vs config labels
|
#### Registration vs config labels
|
||||||
|
|
||||||
@@ -243,7 +243,7 @@ volumes:
|
|||||||
- ${GITEA_DOCKER_WORKSPACE:-.}/data:/app/data
|
- ${GITEA_DOCKER_WORKSPACE:-.}/data:/app/data
|
||||||
```
|
```
|
||||||
|
|
||||||
In most Linux container jobs, the containers, networks and volumes a job creates are removed when it ends, `docker compose` services included. Set `container.sweep: false` if jobs deploy services to the runner's Docker daemon.
|
Set `container.sweep: true` to remove the containers, networks and volumes most Linux container jobs create when they end, `docker compose` services included. Leave it off if jobs deploy services to the runner's Docker daemon.
|
||||||
|
|
||||||
### Built-in actions (`uses: builtin:<name>`)
|
### Built-in actions (`uses: builtin:<name>`)
|
||||||
|
|
||||||
|
|||||||
@@ -104,7 +104,8 @@ type Handler struct {
|
|||||||
stopSync context.CancelFunc
|
stopSync context.CancelFunc
|
||||||
syncing sync.WaitGroup
|
syncing sync.WaitGroup
|
||||||
|
|
||||||
outboundIP string
|
outboundIP string
|
||||||
|
instanceTransport http.RoundTripper
|
||||||
|
|
||||||
// internalSecret guards /_internal/{register,revoke}. When set, a remote
|
// internalSecret guards /_internal/{register,revoke}. When set, a remote
|
||||||
// runner can use these endpoints to pre-register per-job
|
// runner can use these endpoints to pre-register per-job
|
||||||
@@ -124,7 +125,7 @@ type Handler struct {
|
|||||||
|
|
||||||
policy Policy
|
policy Policy
|
||||||
|
|
||||||
// freeDisk is a field so tests can drive evictForFreeSpace without a full volume.
|
// freeDisk is a field so tests can drive makeRoom without a full volume.
|
||||||
freeDisk func(string) (uint64, error)
|
freeDisk func(string) (uint64, error)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -143,19 +144,21 @@ type Options struct {
|
|||||||
// in-process RegisterJob method directly.
|
// in-process RegisterJob method directly.
|
||||||
InternalSecret string
|
InternalSecret string
|
||||||
|
|
||||||
Policy Policy
|
Policy Policy
|
||||||
Logger *logging.Logger
|
Logger *logging.Logger
|
||||||
|
InstanceTransport http.RoundTripper // reaches the instance artifact requests are forwarded to, nil for http.DefaultTransport
|
||||||
}
|
}
|
||||||
|
|
||||||
// StartHandler starts an embedded cache or forwards cache requests to Upstream.
|
// StartHandler starts an embedded cache or forwards cache requests to Upstream.
|
||||||
func StartHandler(opts Options) (*Handler, error) {
|
func StartHandler(opts Options) (*Handler, error) {
|
||||||
dir, logger := opts.Dir, opts.Logger
|
dir, logger := opts.Dir, opts.Logger
|
||||||
h := &Handler{
|
h := &Handler{
|
||||||
creds: make(map[string]*credEntry),
|
creds: make(map[string]*credEntry),
|
||||||
internalSecret: opts.InternalSecret,
|
internalSecret: opts.InternalSecret,
|
||||||
policy: opts.Policy.withDefaults(),
|
policy: opts.Policy.withDefaults(),
|
||||||
freeDisk: disk.FreeBytes,
|
freeDisk: disk.FreeBytes,
|
||||||
dbLock: make(chan struct{}, 1),
|
dbLock: make(chan struct{}, 1),
|
||||||
|
instanceTransport: opts.InstanceTransport,
|
||||||
}
|
}
|
||||||
|
|
||||||
if logger == nil {
|
if logger == nil {
|
||||||
@@ -773,6 +776,7 @@ func (h *Handler) upload(w http.ResponseWriter, r *http.Request) {
|
|||||||
h.responseJSON(w, r, 400, err)
|
h.responseJSON(w, r, 400, err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
h.makeRoom(r.ContentLength)
|
||||||
if err := h.storage.Write(cache.ID, start, r.Body); err != nil {
|
if err := h.storage.Write(cache.ID, start, r.Body); err != nil {
|
||||||
h.responseJSON(w, r, 500, err)
|
h.responseJSON(w, r, 500, err)
|
||||||
return
|
return
|
||||||
@@ -875,11 +879,8 @@ func (h *Handler) commitCache(cache *Cache) error {
|
|||||||
if h.s3 != nil {
|
if h.s3 != nil {
|
||||||
return nil // lifecycle rules on the bucket bound its size
|
return nil // lifecycle rules on the bucket bound its size
|
||||||
}
|
}
|
||||||
// A commit is the only thing that grows the store, so the only thing that can push the
|
|
||||||
// volume under the floor.
|
|
||||||
h.evictRepo(db, cache.Repo)
|
h.evictRepo(db, cache.Repo)
|
||||||
h.evictTotal(db)
|
h.evictTotal(db)
|
||||||
h.evictForFreeSpace(db)
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1173,9 +1174,6 @@ func (h *Handler) touchCache(id uint64, requireIncomplete bool) error {
|
|||||||
defer db.Close()
|
defer db.Close()
|
||||||
|
|
||||||
cache, err := db.get(id)
|
cache, err := db.get(id)
|
||||||
if errors.Is(err, errCacheNotFound) {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -1260,23 +1258,29 @@ func (h *Handler) gcCache() {
|
|||||||
h.evictExpired(db)
|
h.evictExpired(db)
|
||||||
h.evictSuperseded(db)
|
h.evictSuperseded(db)
|
||||||
h.evictOversized(db)
|
h.evictOversized(db)
|
||||||
h.evictForFreeSpace(db)
|
_ = db.Close()
|
||||||
|
h.makeRoom(0)
|
||||||
}
|
}
|
||||||
|
|
||||||
// evictForFreeSpace bounds the volume itself, so it also covers bytes the cache never
|
// makeRoom evicts the least recently used entries to keep need bytes above the floor, as GitHub saves new over old.
|
||||||
// accounted for.
|
func (h *Handler) makeRoom(need int64) {
|
||||||
func (h *Handler) evictForFreeSpace(db *cacheDB) {
|
|
||||||
free, err := h.freeDisk(h.dir)
|
free, err := h.freeDisk(h.dir)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
h.logger.Debugf("free disk check: %v", err) // unsupported platform, treat as unavailable rather than full
|
h.logger.Debugf("free disk check: %v", err) // unsupported platform, treat as unavailable rather than full
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
if free >= uint64(h.policy.MinFreeDisk) {
|
shortfall := h.policy.MinFreeDisk + max(need, 0) - int64(free)
|
||||||
|
if h.s3 != nil || shortfall <= 0 {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
db, err := h.openDB()
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
defer db.Close()
|
||||||
|
|
||||||
caches := h.completedByUse(db)
|
caches := h.completedByUse(db)
|
||||||
total, shortfall := totalSize(caches), h.policy.MinFreeDisk-int64(free)
|
total := totalSize(caches)
|
||||||
if total <= shortfall {
|
if total <= shortfall {
|
||||||
// Say so, or shedding everything and still being short reads as the backstop working.
|
// Say so, or shedding everything and still being short reads as the backstop working.
|
||||||
h.logger.Warnf("cache volume is %d MiB short of the free space floor with only %d MiB of cache on it; something else is filling it", shortfall/miB, total/miB)
|
h.logger.Warnf("cache volume is %d MiB short of the free space floor with only %d MiB of cache on it; something else is filling it", shortfall/miB, total/miB)
|
||||||
|
|||||||
@@ -18,6 +18,7 @@ import (
|
|||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
"testing/iotest"
|
||||||
"testing/synctest"
|
"testing/synctest"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -673,7 +674,7 @@ func TestHandler_evictPolicy(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestHandler_evictForFreeSpace(t *testing.T) {
|
func TestHandler_makeRoom(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
free := func(n int64) func(string) (uint64, error) {
|
free := func(n int64) func(string) (uint64, error) {
|
||||||
return func(string) (uint64, error) { return uint64(n), nil }
|
return func(string) (uint64, error) { return uint64(n), nil }
|
||||||
@@ -682,14 +683,16 @@ func TestHandler_evictForFreeSpace(t *testing.T) {
|
|||||||
for _, tc := range []struct {
|
for _, tc := range []struct {
|
||||||
name string
|
name string
|
||||||
freeDisk func(string) (uint64, error)
|
freeDisk func(string) (uint64, error)
|
||||||
|
upload int64
|
||||||
kept []string
|
kept []string
|
||||||
}{
|
}{
|
||||||
{"ample free space evicts nothing", free(defaultMinFreeDisk), []string{"oldest", "middle", "newest"}},
|
{"ample free space evicts nothing", free(defaultMinFreeDisk), 0, []string{"oldest", "middle", "newest"}},
|
||||||
{"a small shortfall sheds one entry", free(defaultMinFreeDisk - 4*miB), []string{"middle", "newest"}},
|
{"room for an upload sheds the oldest entry", free(defaultMinFreeDisk), 4 * miB, []string{"middle", "newest"}},
|
||||||
{"a shortfall the cache cannot cover sheds all of it", free(0), nil},
|
{"a shortfall the cache cannot cover sheds all of it", free(0), 0, nil},
|
||||||
{
|
{
|
||||||
"an unreadable volume is treated as unavailable, not as full",
|
"an unreadable volume is treated as unavailable, not as full",
|
||||||
func(string) (uint64, error) { return 0, errors.New("unsupported") },
|
func(string) (uint64, error) { return 0, errors.New("unsupported") },
|
||||||
|
0,
|
||||||
[]string{"oldest", "middle", "newest"},
|
[]string{"oldest", "middle", "newest"},
|
||||||
},
|
},
|
||||||
} {
|
} {
|
||||||
@@ -702,11 +705,7 @@ func TestHandler_evictForFreeSpace(t *testing.T) {
|
|||||||
}
|
}
|
||||||
handler := newTestHandler(t, Policy{}, entries...)
|
handler := newTestHandler(t, Policy{}, entries...)
|
||||||
handler.freeDisk = tc.freeDisk
|
handler.freeDisk = tc.freeDisk
|
||||||
|
handler.makeRoom(tc.upload)
|
||||||
db, err := handler.openDB()
|
|
||||||
require.NoError(t, err)
|
|
||||||
handler.evictForFreeSpace(db)
|
|
||||||
require.NoError(t, db.Close())
|
|
||||||
|
|
||||||
assert.ElementsMatch(t, tc.kept, keptKeys(t, handler, entries))
|
assert.ElementsMatch(t, tc.kept, keptKeys(t, handler, entries))
|
||||||
})
|
})
|
||||||
@@ -852,6 +851,9 @@ func TestHandler_touchCacheSkipsFreshCompletedEntries(t *testing.T) {
|
|||||||
func newTestHandler(t *testing.T, policy Policy, entries ...*Cache) *Handler {
|
func newTestHandler(t *testing.T, policy Policy, entries ...*Cache) *Handler {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
handler := startTestHandler(t, Options{Policy: policy})
|
handler := startTestHandler(t, Options{Policy: policy})
|
||||||
|
if len(entries) == 0 {
|
||||||
|
return handler
|
||||||
|
}
|
||||||
|
|
||||||
db, err := handler.openDB()
|
db, err := handler.openDB()
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
@@ -888,6 +890,13 @@ func keptKeys(t *testing.T, handler *Handler, entries []*Cache) []string {
|
|||||||
return kept
|
return kept
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestStorageDropsFailedWrites(t *testing.T) {
|
||||||
|
storage, err := NewStorage(t.TempDir())
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Error(t, storage.Write(1, 0, io.MultiReader(strings.NewReader("partial"), iotest.ErrReader(io.ErrUnexpectedEOF))))
|
||||||
|
assert.NoFileExists(t, storage.tempName(1, 0))
|
||||||
|
}
|
||||||
|
|
||||||
func writeBlob(t *testing.T, handler *Handler, id uint64) {
|
func writeBlob(t *testing.T, handler *Handler, id uint64) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
require.NoError(t, handler.storage.Write(id, 0, strings.NewReader("a")))
|
require.NoError(t, handler.storage.Write(id, 0, strings.NewReader("a")))
|
||||||
@@ -934,6 +943,10 @@ func TestHandler_RejectsBearer(t *testing.T) {
|
|||||||
|
|
||||||
func TestHandler_UnregisterRevokes(t *testing.T) {
|
func TestHandler_UnregisterRevokes(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
handler, err := StartHandler(Options{Dir: filepath.Join(t.TempDir(), "artifactcache")})
|
||||||
|
require.NoError(t, err)
|
||||||
|
defer handler.Close()
|
||||||
|
|
||||||
for _, tc := range []struct {
|
for _, tc := range []struct {
|
||||||
name string
|
name string
|
||||||
token string
|
token string
|
||||||
@@ -944,9 +957,6 @@ func TestHandler_UnregisterRevokes(t *testing.T) {
|
|||||||
{"reference counted", "shared", "v", 2},
|
{"reference counted", "shared", "v", 2},
|
||||||
} {
|
} {
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
handler, err := StartHandler(Options{Dir: filepath.Join(t.TempDir(), "artifactcache")})
|
|
||||||
require.NoError(t, err)
|
|
||||||
defer handler.Close()
|
|
||||||
revokers := make([]func(), tc.registrations)
|
revokers := make([]func(), tc.registrations)
|
||||||
for i := range revokers {
|
for i := range revokers {
|
||||||
revokers[i] = handler.RegisterJob(tc.token, JobCredential{Repo: testRepo})
|
revokers[i] = handler.RegisterJob(tc.token, JobCredential{Repo: testRepo})
|
||||||
@@ -1043,9 +1053,8 @@ func TestHandler_ArtifactSignature(t *testing.T) {
|
|||||||
|
|
||||||
base := handler.ExternalURL() + apiPath
|
base := handler.ExternalURL() + apiPath
|
||||||
good := signArtifactURL(handler, 1)
|
good := signArtifactURL(handler, 1)
|
||||||
other, err := StartHandler(Options{Dir: filepath.Join(t.TempDir(), "artifactcache2")})
|
secret := bytes.Clone(handler.secret)
|
||||||
require.NoError(t, err)
|
secret[0] ^= 1
|
||||||
defer other.Close()
|
|
||||||
|
|
||||||
for _, tc := range []struct {
|
for _, tc := range []struct {
|
||||||
name string
|
name string
|
||||||
@@ -1054,7 +1063,7 @@ func TestHandler_ArtifactSignature(t *testing.T) {
|
|||||||
{"missing signature", fmt.Sprintf("%s/artifacts/%d", base, 1)},
|
{"missing signature", fmt.Sprintf("%s/artifacts/%d", base, 1)},
|
||||||
{"tampered signature", good[:len(good)-4] + "dead"},
|
{"tampered signature", good[:len(good)-4] + "dead"},
|
||||||
{"expired signature", handler.signedArtifactURL(JobCredential{}, 1, time.Now().Add(-time.Second))},
|
{"expired signature", handler.signedArtifactURL(JobCredential{}, 1, time.Now().Add(-time.Second))},
|
||||||
{"signature from a different server", base + strings.SplitN(signArtifactURL(other, 1), apiPath, 2)[1]},
|
{"signature from a different server", base + strings.SplitN(signArtifactURL(&Handler{secret: secret}, 1), apiPath, 2)[1]},
|
||||||
} {
|
} {
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
resp, err := testClient.Get(tc.url)
|
resp, err := testClient.Get(tc.url)
|
||||||
|
|||||||
@@ -218,6 +218,7 @@ func (h *Handler) v2UploadBlob(w http.ResponseWriter, r *http.Request) {
|
|||||||
h.responseJSON(w, r, http.StatusBadRequest, err)
|
h.responseJSON(w, r, http.StatusBadRequest, err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
h.makeRoom(r.ContentLength)
|
||||||
|
|
||||||
query := r.URL.Query()
|
query := r.URL.Query()
|
||||||
switch strings.ToLower(query.Get("comp")) {
|
switch strings.ToLower(query.Get("comp")) {
|
||||||
|
|||||||
@@ -14,6 +14,7 @@ import (
|
|||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
@@ -84,6 +85,8 @@ func TestCacheServiceV2RoundTrip(t *testing.T) {
|
|||||||
|
|
||||||
unsigned := fmt.Sprintf("%s%s/1", handler.ExternalURL(), blobPath)
|
unsigned := fmt.Sprintf("%s%s/1", handler.ExternalURL(), blobPath)
|
||||||
assert.Equal(t, http.StatusUnauthorized, putBlob(t, unsigned, content))
|
assert.Equal(t, http.StatusUnauthorized, putBlob(t, unsigned, content))
|
||||||
|
unreserved := handler.signedURL(JobCredential{Repo: testRepo}, blobPath, blobUploadPurpose, 1, time.Now().Add(time.Minute))
|
||||||
|
assert.Equal(t, http.StatusBadRequest, putBlob(t, unreserved, content))
|
||||||
|
|
||||||
finalized, uploadURL := saveV2(t, handler, "deps-v1", "abc123", content)
|
finalized, uploadURL := saveV2(t, handler, "deps-v1", "abc123", content)
|
||||||
require.Equal(t, true, finalized["ok"])
|
require.Equal(t, true, finalized["ok"])
|
||||||
|
|||||||
@@ -67,7 +67,7 @@ func (h *Handler) forwardOrNotFound(w http.ResponseWriter, r *http.Request) {
|
|||||||
h.twirpError(w, r, twirpInternal, fmt.Errorf("cache server cannot reach the artifact service at %s: %w", target, err))
|
h.twirpError(w, r, twirpInternal, fmt.Errorf("cache server cannot reach the artifact service at %s: %w", target, err))
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
if cred.InsecureTLS {
|
if proxy.Transport = h.instanceTransport; proxy.Transport == nil && cred.InsecureTLS { // the instance transport carries runner.insecure and the client cert
|
||||||
proxy.Transport = insecureTransport
|
proxy.Transport = insecureTransport
|
||||||
}
|
}
|
||||||
proxy.ServeHTTP(w, r)
|
proxy.ServeHTTP(w, r)
|
||||||
|
|||||||
@@ -133,3 +133,20 @@ func TestFrontResultsService(t *testing.T) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestFrontResultsServiceForwardsThroughTheInstanceTransport(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
gitea := httptest.NewTLSServer(http.HandlerFunc(func(writer http.ResponseWriter, _request *http.Request) { _, _ = io.WriteString(writer, "{}") }))
|
||||||
|
defer gitea.Close()
|
||||||
|
handler, err := StartHandler(Options{Dir: t.TempDir(), OutboundIP: "127.0.0.1", InstanceTransport: gitea.Client().Transport})
|
||||||
|
require.NoError(t, err)
|
||||||
|
defer handler.Close()
|
||||||
|
defer handler.RegisterJob("token", JobCredential{Results: gitea.URL})()
|
||||||
|
|
||||||
|
req, err := http.NewRequestWithContext(t.Context(), http.MethodPost, handler.ExternalURL()+artifactServicePath+"CreateArtifact", nil)
|
||||||
|
require.NoError(t, err)
|
||||||
|
resp, err := (&http.Client{Transport: &bearerTransport{token: "token"}}).Do(req)
|
||||||
|
require.NoError(t, err)
|
||||||
|
defer resp.Body.Close()
|
||||||
|
require.Equal(t, http.StatusOK, resp.StatusCode)
|
||||||
|
}
|
||||||
@@ -9,6 +9,7 @@ import (
|
|||||||
"crypto/sha256"
|
"crypto/sha256"
|
||||||
"encoding/hex"
|
"encoding/hex"
|
||||||
"encoding/json/v2"
|
"encoding/json/v2"
|
||||||
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"net/http"
|
"net/http"
|
||||||
@@ -51,9 +52,10 @@ func (s *Storage) writeFile(name string, reader io.Reader) error {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
defer file.Close()
|
|
||||||
|
|
||||||
_, err = io.Copy(file, reader)
|
_, err = io.Copy(file, reader)
|
||||||
|
if err = errors.Join(err, file.Close()); err != nil {
|
||||||
|
_ = os.Remove(name)
|
||||||
|
}
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -102,50 +104,46 @@ func (s *Storage) Commit(id uint64, size int64) (int64, error) {
|
|||||||
return 0, err
|
return 0, err
|
||||||
}
|
}
|
||||||
written, err := assemble(name, tempNames)
|
written, err := assemble(name, tempNames)
|
||||||
if err != nil {
|
if err == nil && size >= 0 && written != size { // old clients like actions/cache@v2 send no size
|
||||||
return 0, err
|
err = fmt.Errorf("broken file: %v != %v", written, size)
|
||||||
}
|
}
|
||||||
// If size is less than 0, it means the size is unknown.
|
if err != nil {
|
||||||
// We can't check the size of the file, just skip the check.
|
|
||||||
// It happens when the request comes from old versions of actions, like `actions/cache@v2`.
|
|
||||||
if size >= 0 && written != size {
|
|
||||||
_ = os.Remove(name)
|
_ = os.Remove(name)
|
||||||
return 0, fmt.Errorf("broken file: %v != %v", written, size)
|
return 0, err
|
||||||
}
|
}
|
||||||
return written, nil
|
return written, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// assemble concatenates the uploaded parts into name. A single part, which is what the v2 API
|
// assemble appends the parts into the first, deleting each once copied, so it never needs twice the space.
|
||||||
// produces below the client's block threshold, is already the whole archive and is moved.
|
|
||||||
func assemble(name string, tempNames []string) (int64, error) {
|
func assemble(name string, tempNames []string) (int64, error) {
|
||||||
if len(tempNames) == 1 {
|
if len(tempNames) == 0 {
|
||||||
info, err := os.Stat(tempNames[0])
|
return 0, errors.New("nothing was uploaded")
|
||||||
if err != nil {
|
|
||||||
return 0, err
|
|
||||||
}
|
|
||||||
return info.Size(), os.Rename(tempNames[0], name)
|
|
||||||
}
|
}
|
||||||
|
if err := os.Rename(tempNames[0], name); err != nil {
|
||||||
file, err := os.Create(name)
|
return 0, err
|
||||||
|
}
|
||||||
|
file, err := os.OpenFile(name, os.O_WRONLY|os.O_APPEND, 0)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return 0, err
|
return 0, err
|
||||||
}
|
}
|
||||||
defer file.Close()
|
defer file.Close()
|
||||||
|
for _, v := range tempNames[1:] {
|
||||||
var written int64
|
part, err := os.Open(v)
|
||||||
for _, v := range tempNames {
|
|
||||||
f, err := os.Open(v)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return 0, err
|
return 0, err
|
||||||
}
|
}
|
||||||
n, err := io.Copy(file, f)
|
_, err = io.Copy(file, part)
|
||||||
_ = f.Close()
|
_ = part.Close()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return 0, err
|
return 0, err
|
||||||
}
|
}
|
||||||
written += n
|
_ = os.Remove(v)
|
||||||
}
|
}
|
||||||
return written, nil
|
info, err := file.Stat()
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
return info.Size(), file.Close()
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *Storage) Serve(w http.ResponseWriter, r *http.Request, id uint64) {
|
func (s *Storage) Serve(w http.ResponseWriter, r *http.Request, id uint64) {
|
||||||
|
|||||||
@@ -7,7 +7,9 @@ package common
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"math/rand/v2"
|
||||||
"runtime/debug"
|
"runtime/debug"
|
||||||
|
"time"
|
||||||
|
|
||||||
"gitea.com/gitea/runner/act/common/logging"
|
"gitea.com/gitea/runner/act/common/logging"
|
||||||
)
|
)
|
||||||
@@ -158,6 +160,28 @@ func (e Executor) IfBool(conditional bool) Executor {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// RetryBackoff is the wait between Retry's attempts, random between 1 and 10 seconds like GitHub's runner.
|
||||||
|
var RetryBackoff = func() time.Duration { return time.Second + rand.N(9*time.Second) }
|
||||||
|
|
||||||
|
// Retry runs the executor up to attempts times until it succeeds.
|
||||||
|
func (e Executor) Retry(attempts int) Executor {
|
||||||
|
return func(ctx context.Context) error {
|
||||||
|
for attempt := 1; ; attempt++ {
|
||||||
|
err := e(ctx)
|
||||||
|
if err == nil || attempt == attempts || ctx.Err() != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
backoff := RetryBackoff()
|
||||||
|
Logger(ctx).Warnf("%v, retrying in %s", err, backoff.Round(time.Second))
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return err
|
||||||
|
case <-time.After(backoff):
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Finally adds an executor to run after other executor
|
// Finally adds an executor to run after other executor
|
||||||
func (e Executor) Finally(finally Executor) Executor {
|
func (e Executor) Finally(finally Executor) Executor {
|
||||||
return func(ctx context.Context) error {
|
return func(ctx context.Context) error {
|
||||||
|
|||||||
@@ -11,6 +11,7 @@ import (
|
|||||||
"sync"
|
"sync"
|
||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
"testing"
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
@@ -134,6 +135,31 @@ func TestExecutorConditionalsAndFinally(t *testing.T) {
|
|||||||
assert.Equal(t, []string{"if-true", "if-not", "if-bool", "main", "finally"}, calls)
|
assert.Equal(t, []string{"if-true", "if-not", "if-bool", "main", "finally"}, calls)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestExecutorRetryStopsOnSuccessOrAfterAllAttempts(t *testing.T) {
|
||||||
|
backoff := RetryBackoff
|
||||||
|
RetryBackoff = func() time.Duration { return 0 }
|
||||||
|
t.Cleanup(func() { RetryBackoff = backoff })
|
||||||
|
transient := errors.New("transient")
|
||||||
|
for _, tc := range []struct {
|
||||||
|
failures, calls int
|
||||||
|
err error
|
||||||
|
}{
|
||||||
|
{failures: 2, calls: 3},
|
||||||
|
{failures: 5, calls: 3, err: transient},
|
||||||
|
} {
|
||||||
|
var calls int
|
||||||
|
err := Executor(func(context.Context) error {
|
||||||
|
calls++
|
||||||
|
if calls <= tc.failures {
|
||||||
|
return transient
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}).Retry(3)(t.Context())
|
||||||
|
require.Equal(t, tc.err, err)
|
||||||
|
require.Equal(t, tc.calls, calls)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestExecutorFinallyReturnsFinallyErrorWithOriginal(t *testing.T) {
|
func TestExecutorFinallyReturnsFinallyErrorWithOriginal(t *testing.T) {
|
||||||
finalErr := errors.New("cleanup failed")
|
finalErr := errors.New("cleanup failed")
|
||||||
for _, mainErr := range []error{context.Canceled, nil} {
|
for _, mainErr := range []error{context.Canceled, nil} {
|
||||||
|
|||||||
@@ -36,7 +36,7 @@ func TestFetchAction(t *testing.T) {
|
|||||||
|
|
||||||
root := t.TempDir()
|
root := t.TempDir()
|
||||||
remote := filepath.Join(root, "org", "action")
|
remote := filepath.Join(root, "org", "action")
|
||||||
require.NoError(t, gitCmd("init", "--initial-branch=main", remote))
|
require.NoError(t, os.CopyFS(remote, os.DirFS(filepath.Join(gitFixtureDir, "work"))))
|
||||||
require.NoError(t, os.WriteFile(filepath.Join(remote, "action.yml"), []byte("name: action\n"), 0o644))
|
require.NoError(t, os.WriteFile(filepath.Join(remote, "action.yml"), []byte("name: action\n"), 0o644))
|
||||||
require.NoError(t, os.WriteFile(filepath.Join(remote, "run.sh"), []byte("#!/bin/sh\n"), 0o755))
|
require.NoError(t, os.WriteFile(filepath.Join(remote, "run.sh"), []byte("#!/bin/sh\n"), 0o755))
|
||||||
require.NoError(t, os.Symlink("action.yml", filepath.Join(remote, "link")))
|
require.NoError(t, os.Symlink("action.yml", filepath.Join(remote, "link")))
|
||||||
@@ -53,6 +53,7 @@ func TestFetchAction(t *testing.T) {
|
|||||||
var archiveRequests []http.Header
|
var archiveRequests []http.Header
|
||||||
var mirror *httptest.Server
|
var mirror *httptest.Server
|
||||||
archiveCache := map[string][]byte{}
|
archiveCache := map[string][]byte{}
|
||||||
|
revisions := map[string]string{}
|
||||||
requests, archives, failures := 0, "", 0
|
requests, archives, failures := 0, "", 0
|
||||||
handler := http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
|
handler := http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
|
||||||
mu.Lock()
|
mu.Lock()
|
||||||
@@ -79,7 +80,14 @@ func TestFetchAction(t *testing.T) {
|
|||||||
http.Redirect(writer, request, mirror.URL+request.URL.Path, http.StatusFound)
|
http.Redirect(writer, request, mirror.URL+request.URL.Path, http.StatusFound)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
sha, _ := gitOutput(request.Context(), remote, "rev-parse", strings.TrimSuffix(ref, ".tar.gz"))
|
sha, ok := revisions[ref]
|
||||||
|
if !ok {
|
||||||
|
var err error
|
||||||
|
sha, err = gitOutput(request.Context(), remote, "rev-parse", strings.TrimSuffix(ref, ".tar.gz"))
|
||||||
|
if err == nil {
|
||||||
|
revisions[ref] = sha
|
||||||
|
}
|
||||||
|
}
|
||||||
etag := fmt.Sprintf(`W/"%s-tar.gz-%t"`, sha, archives != "unprefixed")
|
etag := fmt.Sprintf(`W/"%s-tar.gz-%t"`, sha, archives != "unprefixed")
|
||||||
if request.Header.Get("If-None-Match") == etag {
|
if request.Header.Get("If-None-Match") == etag {
|
||||||
writer.WriteHeader(http.StatusNotModified)
|
writer.WriteHeader(http.StatusNotModified)
|
||||||
@@ -208,6 +216,9 @@ func TestFetchAction(t *testing.T) {
|
|||||||
require.NoError(t, gitCmd("-C", remote, "commit", "--allow-empty", "-m", "moved"))
|
require.NoError(t, gitCmd("-C", remote, "commit", "--allow-empty", "-m", "moved"))
|
||||||
require.NoError(t, gitCmd("-C", remote, "tag", "--force", "v1"))
|
require.NoError(t, gitCmd("-C", remote, "tag", "--force", "v1"))
|
||||||
expectedSha = gitRevParse(t, remote, "v1")
|
expectedSha = gitRevParse(t, remote, "v1")
|
||||||
|
mu.Lock()
|
||||||
|
clear(revisions)
|
||||||
|
mu.Unlock()
|
||||||
moved, movedSha, err := FetchAction(t.Context(), input)
|
moved, movedSha, err := FetchAction(t.Context(), input)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
assert.Equal(t, expectedSha, movedSha)
|
assert.Equal(t, expectedSha, movedSha)
|
||||||
@@ -253,7 +264,7 @@ func TestFetchAction(t *testing.T) {
|
|||||||
})
|
})
|
||||||
|
|
||||||
t.Run("downloads again over a tree whose files changed size or that has no record, not under a job copying it", func(t *testing.T) {
|
t.Run("downloads again over a tree whose files changed size or that has no record, not under a job copying it", func(t *testing.T) {
|
||||||
sha := gitRevParse(t, remote, "main")
|
sha := expectedSha
|
||||||
for _, mode := range []string{"prefixed", "none"} {
|
for _, mode := range []string{"prefixed", "none"} {
|
||||||
serve(mode, 0)
|
serve(mode, 0)
|
||||||
input := newInput(sha)
|
input := newInput(sha)
|
||||||
|
|||||||
+11
-6
@@ -185,6 +185,7 @@ type NewGitCloneExecutorInput struct {
|
|||||||
// For Gitea
|
// For Gitea
|
||||||
InsecureSkipTLS bool
|
InsecureSkipTLS bool
|
||||||
ClientCertFile, ClientKeyFile string // PEM, absolute as git runs in other directories
|
ClientCertFile, ClientKeyFile string // PEM, absolute as git runs in other directories
|
||||||
|
CAFile string // PEM, absolute, replaces git's CA bundle
|
||||||
Transport http.RoundTripper // for FetchAction's archive downloads, nil for http.DefaultTransport
|
Transport http.RoundTripper // for FetchAction's archive downloads, nil for http.DefaultTransport
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -389,12 +390,16 @@ func RemoteConfig(token string, insecureSkipTLS bool) []string {
|
|||||||
return config
|
return config
|
||||||
}
|
}
|
||||||
|
|
||||||
func ClientCertConfig(prefix, certFile, keyFile, executionOS string) []string {
|
// SSLConfig uses OpenSSL on Windows, as Schannel takes no PEM key and ignores sslCAInfo.
|
||||||
if certFile == "" {
|
func SSLConfig(prefix, certFile, keyFile, caFile, executionOS string) []string {
|
||||||
return nil
|
var config []string
|
||||||
|
if certFile != "" {
|
||||||
|
config = append(config, prefix+"sslCert="+certFile, prefix+"sslKey="+keyFile)
|
||||||
}
|
}
|
||||||
config := []string{prefix + "sslCert=" + certFile, prefix + "sslKey=" + keyFile}
|
if caFile != "" {
|
||||||
if strings.EqualFold(executionOS, "windows") {
|
config = append(config, prefix+"sslCAInfo="+caFile)
|
||||||
|
}
|
||||||
|
if config != nil && strings.EqualFold(executionOS, "windows") {
|
||||||
config = append(config, prefix+"sslBackend=openssl")
|
config = append(config, prefix+"sslBackend=openssl")
|
||||||
}
|
}
|
||||||
return config
|
return config
|
||||||
@@ -457,7 +462,7 @@ func runGitWithStdin(ctx context.Context, dir string, input *NewGitCloneExecutor
|
|||||||
config := []string{"core.autocrlf=false", "core.eol=lf"} // keep action files LF
|
config := []string{"core.autocrlf=false", "core.eol=lf"} // keep action files LF
|
||||||
if input != nil {
|
if input != nil {
|
||||||
config = append(config, RemoteConfig(input.Token, input.InsecureSkipTLS)...)
|
config = append(config, RemoteConfig(input.Token, input.InsecureSkipTLS)...)
|
||||||
config = append(config, ClientCertConfig("http.", input.ClientCertFile, input.ClientKeyFile, runtime.GOOS)...)
|
config = append(config, SSLConfig("http.", input.ClientCertFile, input.ClientKeyFile, input.CAFile, runtime.GOOS)...)
|
||||||
}
|
}
|
||||||
if dir != "" {
|
if dir != "" {
|
||||||
args = append([]string{"-C", dir}, args...)
|
args = append([]string{"-C", dir}, args...)
|
||||||
|
|||||||
+34
-40
@@ -54,18 +54,18 @@ func TestMain(m *testing.M) {
|
|||||||
os.Exit(code)
|
os.Exit(code)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestClientCertConfig(t *testing.T) {
|
func TestSSLConfig(t *testing.T) {
|
||||||
for _, testcase := range []struct {
|
for _, testcase := range []struct {
|
||||||
prefix, certFile, executionOS string
|
prefix, certFile, caFile, executionOS string
|
||||||
expected []string
|
expected []string
|
||||||
}{
|
}{
|
||||||
{"http.", "cert.pem", "windows", []string{"http.sslCert=cert.pem", "http.sslKey=key.pem", "http.sslBackend=openssl"}},
|
{"http.https://example.com/.", "cert.pem", "ca.pem", "Windows", []string{"http.https://example.com/.sslCert=cert.pem", "http.https://example.com/.sslKey=key.pem", "http.https://example.com/.sslCAInfo=ca.pem", "http.https://example.com/.sslBackend=openssl"}},
|
||||||
{"http.https://example.com/.", "cert.pem", "Windows", []string{"http.https://example.com/.sslCert=cert.pem", "http.https://example.com/.sslKey=key.pem", "http.https://example.com/.sslBackend=openssl"}},
|
{"http.", "", "ca.pem", "windows", []string{"http.sslCAInfo=ca.pem", "http.sslBackend=openssl"}},
|
||||||
{"http.", "cert.pem", "Linux", []string{"http.sslCert=cert.pem", "http.sslKey=key.pem"}},
|
{"http.", "cert.pem", "ca.pem", "Linux", []string{"http.sslCert=cert.pem", "http.sslKey=key.pem", "http.sslCAInfo=ca.pem"}},
|
||||||
{"http.", "", "Windows", nil},
|
{"http.", "", "", "Windows", nil},
|
||||||
} {
|
} {
|
||||||
t.Run(testcase.prefix+testcase.certFile+testcase.executionOS, func(t *testing.T) {
|
t.Run(testcase.prefix+testcase.certFile+testcase.caFile+testcase.executionOS, func(t *testing.T) {
|
||||||
assert.Equal(t, testcase.expected, ClientCertConfig(testcase.prefix, testcase.certFile, "key.pem", testcase.executionOS))
|
assert.Equal(t, testcase.expected, SSLConfig(testcase.prefix, testcase.certFile, "key.pem", testcase.caFile, testcase.executionOS))
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -112,8 +112,7 @@ func TestFindGitMetadataOfSHA256RepositoryUsesOrigin(t *testing.T) {
|
|||||||
t.Parallel()
|
t.Parallel()
|
||||||
basedir := t.TempDir()
|
basedir := t.TempDir()
|
||||||
const remoteURL = "https://github.com/owner/repo.git"
|
const remoteURL = "https://github.com/owner/repo.git"
|
||||||
require.NoError(t, gitCmd("init", "--object-format=sha256", "--initial-branch=main", basedir))
|
require.NoError(t, gitCmd("init", "--template=", "--object-format=sha256", "--initial-branch=main", basedir))
|
||||||
require.NoError(t, os.RemoveAll(filepath.Join(basedir, ".git", "hooks")))
|
|
||||||
require.NoError(t, gitCmd("-C", basedir, "commit", "--allow-empty", "-m", "init"))
|
require.NoError(t, gitCmd("-C", basedir, "commit", "--allow-empty", "-m", "init"))
|
||||||
require.NoError(t, gitCmd("-C", basedir, "remote", "add", "origin", remoteURL))
|
require.NoError(t, gitCmd("-C", basedir, "remote", "add", "origin", remoteURL))
|
||||||
require.NoError(t, gitCmd("-C", basedir, "config", "--add", "remote.origin.url", "https://github.com/other/repo.git"))
|
require.NoError(t, gitCmd("-C", basedir, "config", "--add", "remote.origin.url", "https://github.com/other/repo.git"))
|
||||||
@@ -205,16 +204,13 @@ func TestGitCloneExecutor(t *testing.T) {
|
|||||||
require.NoError(t, gitCmd("-C", workDir, "add", "action.sh", ".gitattributes"))
|
require.NoError(t, gitCmd("-C", workDir, "add", "action.sh", ".gitattributes"))
|
||||||
require.NoError(t, gitCmd("-C", workDir, "commit", "-m", "initial"))
|
require.NoError(t, gitCmd("-C", workDir, "commit", "-m", "initial"))
|
||||||
require.NoError(t, gitCmd("-C", workDir, "tag", "v2"))
|
require.NoError(t, gitCmd("-C", workDir, "tag", "v2"))
|
||||||
require.NoError(t, gitCmd("-C", workDir, "push", "-u", "origin", "main"))
|
|
||||||
require.NoError(t, gitCmd("-C", workDir, "push", "origin", "v2"))
|
|
||||||
|
|
||||||
require.NoError(t, gitCmd("-C", workDir, "checkout", "-b", "act-fails"))
|
require.NoError(t, gitCmd("-C", workDir, "checkout", "-b", "act-fails"))
|
||||||
require.NoError(t, gitCmd("-C", workDir, "commit", "--allow-empty", "-m", "branch-commit"))
|
require.NoError(t, gitCmd("-C", workDir, "commit", "--allow-empty", "-m", "branch-commit"))
|
||||||
require.NoError(t, gitCmd("-C", workDir, "push", "origin", "act-fails"))
|
|
||||||
|
|
||||||
fullSha := gitRevParse(t, workDir, "main")
|
fullSha := gitRevParse(t, workDir, "main")
|
||||||
|
|
||||||
require.NoError(t, gitCmd("-C", workDir, "push", "origin", "main:refs/heads/"+fullSha[:4]))
|
require.NoError(t, gitCmd("-C", workDir, "push", "-u", "origin", "main", "v2", "act-fails", "main:refs/heads/"+fullSha[:4]))
|
||||||
|
|
||||||
hostConfig := filepath.Join(t.TempDir(), "gitconfig")
|
hostConfig := filepath.Join(t.TempDir(), "gitconfig")
|
||||||
require.NoError(t, os.WriteFile(hostConfig, []byte("[core]\n\tautocrlf = true\n\teol = crlf\n[url "+strconv.Quote(remoteDir)+"]\n\tinsteadOf = https://example.invalid/action\n"), 0o644))
|
require.NoError(t, os.WriteFile(hostConfig, []byte("[core]\n\tautocrlf = true\n\teol = crlf\n[url "+strconv.Quote(remoteDir)+"]\n\tinsteadOf = https://example.invalid/action\n"), 0o644))
|
||||||
@@ -282,10 +278,10 @@ func TestGitCloneExecutor(t *testing.T) {
|
|||||||
func TestGitCloneExecutorReclonesWhenOriginURLChanges(t *testing.T) {
|
func TestGitCloneExecutorReclonesWhenOriginURLChanges(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
createRemote := func(message string) string {
|
createRemote := func(message string) string {
|
||||||
remoteDir, workDir := newGitRemote(t, message)
|
dir := t.TempDir()
|
||||||
require.NoError(t, gitCmd("-C", workDir, "push", "-u", "origin", "main"))
|
require.NoError(t, os.CopyFS(dir, os.DirFS(filepath.Join(gitFixtureDir, "work"))))
|
||||||
|
require.NoError(t, gitCmd("-C", dir, "commit", "--allow-empty", "-m", message))
|
||||||
return remoteDir
|
return dir
|
||||||
}
|
}
|
||||||
|
|
||||||
oldRemoteDir := createRemote("old-action")
|
oldRemoteDir := createRemote("old-action")
|
||||||
@@ -405,31 +401,30 @@ func TestGitCloneExecutorShallow(t *testing.T) {
|
|||||||
remoteDir, workDir := newGitRemote(t, "c1", "c2", "c3")
|
remoteDir, workDir := newGitRemote(t, "c1", "c2", "c3")
|
||||||
require.NoError(t, gitCmd("-C", workDir, "tag", "v1"))
|
require.NoError(t, gitCmd("-C", workDir, "tag", "v1"))
|
||||||
unadvertisedSHA := gitRevParse(t, workDir, "HEAD~1")
|
unadvertisedSHA := gitRevParse(t, workDir, "HEAD~1")
|
||||||
require.NoError(t, gitCmd("-C", workDir, "push", "-u", "origin", "main"))
|
require.NoError(t, gitCmd("-C", workDir, "push", "-u", "origin", "main", "v1"))
|
||||||
require.NoError(t, gitCmd("-C", workDir, "push", "origin", "v1"))
|
|
||||||
|
|
||||||
shallowMarker := func(dir string) string { return filepath.Join(dir, ".git", "shallow") }
|
shallowMarker := func(dir string) string { return filepath.Join(dir, ".git", "shallow") }
|
||||||
|
|
||||||
for name, ref := range map[string]string{"branch is cloned shallowly": "main", "tag is cloned shallowly": "v1"} {
|
for name, ref := range map[string]string{"branch is cloned shallowly": "main", "tag is cloned shallowly": "v1"} {
|
||||||
t.Run(name, func(t *testing.T) {
|
t.Run(name, func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
require.NoError(t, NewGitCloneExecutor(NewGitCloneExecutorInput{
|
require.NoError(t, NewGitCloneExecutor(NewGitCloneExecutorInput{
|
||||||
URL: remoteDir, Ref: ref, Dir: dir, Depth: 1,
|
URL: remoteDir, Ref: ref, Dir: dir, Depth: 1,
|
||||||
})(t.Context()))
|
})(t.Context()))
|
||||||
assert.FileExists(t, shallowMarker(dir), "clone should be shallow")
|
assert.FileExists(t, shallowMarker(dir), "clone should be shallow")
|
||||||
assert.Equal(t, 1, gitRevCount(t, dir), "only the tip commit should be present")
|
assert.Equal(t, []string{"c3"}, gitSubjects(t, dir), "only the tip commit should be present")
|
||||||
assert.Equal(t, "c3", gitHeadSubject(t, dir))
|
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
unadvertisedRemote := func(t *testing.T, allowUnadvertised bool) string {
|
unadvertisedRemote := func(t *testing.T, allowUnadvertised bool) string {
|
||||||
remote := filepath.Join(t.TempDir(), "remote.git")
|
remote := filepath.Join(t.TempDir(), "remote.git")
|
||||||
require.NoError(t, gitCmd("clone", "--bare", remoteDir, remote))
|
require.NoError(t, gitCmd("clone", "--bare", "--template=", "--config", "uploadpack.allowAnySHA1InWant="+strconv.FormatBool(allowUnadvertised), remoteDir, remote))
|
||||||
require.NoError(t, gitCmd("-C", remote, "config", "uploadpack.allowAnySHA1InWant", strconv.FormatBool(allowUnadvertised)))
|
|
||||||
return remote
|
return remote
|
||||||
}
|
}
|
||||||
|
|
||||||
t.Run("commit hash resolves when the remote accepts a direct shallow fetch", func(t *testing.T) {
|
t.Run("commit hash resolves when the remote accepts a direct shallow fetch", func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
require.NoError(t, NewGitCloneExecutor(NewGitCloneExecutorInput{
|
require.NoError(t, NewGitCloneExecutor(NewGitCloneExecutorInput{
|
||||||
URL: unadvertisedRemote(t, false), Ref: unadvertisedSHA, Dir: dir, Depth: 1,
|
URL: unadvertisedRemote(t, false), Ref: unadvertisedSHA, Dir: dir, Depth: 1,
|
||||||
@@ -438,6 +433,7 @@ func TestGitCloneExecutorShallow(t *testing.T) {
|
|||||||
})
|
})
|
||||||
|
|
||||||
t.Run("commit hash is fetched shallowly, moved to another hash, then reused without reaching the remote", func(t *testing.T) {
|
t.Run("commit hash is fetched shallowly, moved to another hash, then reused without reaching the remote", func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
remote := unadvertisedRemote(t, true)
|
remote := unadvertisedRemote(t, true)
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
cloneAt := func(ref string) error {
|
cloneAt := func(ref string) error {
|
||||||
@@ -448,7 +444,7 @@ func TestGitCloneExecutorShallow(t *testing.T) {
|
|||||||
|
|
||||||
require.NoError(t, cloneAt(unadvertisedSHA))
|
require.NoError(t, cloneAt(unadvertisedSHA))
|
||||||
assert.FileExists(t, shallowMarker(dir))
|
assert.FileExists(t, shallowMarker(dir))
|
||||||
assert.Equal(t, 1, gitRevCount(t, dir))
|
assert.Len(t, gitSubjects(t, dir), 1)
|
||||||
assert.Equal(t, unadvertisedSHA, gitRevParse(t, dir, "HEAD"))
|
assert.Equal(t, unadvertisedSHA, gitRevParse(t, dir, "HEAD"))
|
||||||
|
|
||||||
olderSHA := gitRevParse(t, workDir, "HEAD~2")
|
olderSHA := gitRevParse(t, workDir, "HEAD~2")
|
||||||
@@ -461,6 +457,8 @@ func TestGitCloneExecutorShallow(t *testing.T) {
|
|||||||
})
|
})
|
||||||
|
|
||||||
t.Run("moving hexadecimal-named branch updates while staying shallow", func(t *testing.T) {
|
t.Run("moving hexadecimal-named branch updates while staying shallow", func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
remoteDir, workDir := newGitRemote(t, "c2", "c3")
|
||||||
require.NoError(t, gitCmd("-C", workDir, "push", "origin", "main:deadbeef"))
|
require.NoError(t, gitCmd("-C", workDir, "push", "origin", "main:deadbeef"))
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
clone := NewGitCloneExecutor(NewGitCloneExecutorInput{
|
clone := NewGitCloneExecutor(NewGitCloneExecutorInput{
|
||||||
@@ -473,19 +471,17 @@ func TestGitCloneExecutorShallow(t *testing.T) {
|
|||||||
require.NoError(t, gitCmd("-C", workDir, "push", "origin", "main:deadbeef"))
|
require.NoError(t, gitCmd("-C", workDir, "push", "origin", "main:deadbeef"))
|
||||||
|
|
||||||
require.NoError(t, clone(t.Context()))
|
require.NoError(t, clone(t.Context()))
|
||||||
assert.Equal(t, "c4", gitHeadSubject(t, dir), "reused shallow clone should update to the new tip")
|
assert.Equal(t, []string{"c4"}, gitSubjects(t, dir), "reused shallow clone should update to the new tip")
|
||||||
assert.FileExists(t, shallowMarker(dir), "repo should remain shallow after update")
|
assert.FileExists(t, shallowMarker(dir), "repo should remain shallow after update")
|
||||||
assert.Equal(t, 1, gitRevCount(t, dir))
|
|
||||||
})
|
})
|
||||||
|
|
||||||
t.Run("sha256 full hash is cloned shallowly, reused without the remote, and its 40-character prefix is short", func(t *testing.T) {
|
t.Run("sha256 full hash is cloned shallowly, reused without the remote, and its 40-character prefix is short", func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
remote := filepath.Join(t.TempDir(), "remote.git")
|
remote := filepath.Join(t.TempDir(), "remote.git")
|
||||||
require.NoError(t, gitCmd("init", "--bare", "--object-format=sha256", "--initial-branch=main", remote))
|
|
||||||
work := t.TempDir()
|
work := t.TempDir()
|
||||||
require.NoError(t, gitCmd("init", "--object-format=sha256", "--initial-branch=main", work))
|
require.NoError(t, gitCmd("init", "--template=", "--object-format=sha256", "--initial-branch=main", work))
|
||||||
require.NoError(t, gitCmd("-C", work, "remote", "add", "origin", remote))
|
|
||||||
require.NoError(t, gitCmd("-C", work, "commit", "--allow-empty", "-m", "pinned"))
|
require.NoError(t, gitCmd("-C", work, "commit", "--allow-empty", "-m", "pinned"))
|
||||||
require.NoError(t, gitCmd("-C", work, "push", "origin", "main"))
|
require.NoError(t, gitCmd("clone", "--bare", "--template=", work, remote))
|
||||||
sha := gitRevParse(t, work, "HEAD")
|
sha := gitRevParse(t, work, "HEAD")
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
clone := NewGitCloneExecutor(NewGitCloneExecutorInput{URL: remote, Ref: sha, Dir: dir, Depth: 1})
|
clone := NewGitCloneExecutor(NewGitCloneExecutorInput{URL: remote, Ref: sha, Dir: dir, Depth: 1})
|
||||||
@@ -504,11 +500,11 @@ func TestGitCloneExecutorPinnedHashIgnoresShadowingRef(t *testing.T) {
|
|||||||
pinned := gitRevParse(t, workDir, "HEAD")
|
pinned := gitRevParse(t, workDir, "HEAD")
|
||||||
require.NoError(t, gitCmd("-C", workDir, "commit", "--allow-empty", "-m", "decoy"))
|
require.NoError(t, gitCmd("-C", workDir, "commit", "--allow-empty", "-m", "decoy"))
|
||||||
require.NoError(t, gitCmd("-C", workDir, "tag", pinned))
|
require.NoError(t, gitCmd("-C", workDir, "tag", pinned))
|
||||||
require.NoError(t, gitCmd("-C", workDir, "push", "-u", "origin", "main"))
|
require.NoError(t, gitCmd("-C", workDir, "push", "-u", "origin", "main", pinned))
|
||||||
require.NoError(t, gitCmd("-C", workDir, "push", "origin", pinned))
|
|
||||||
|
|
||||||
for name, depth := range map[string]int{"shallow": 1, "full clone": 0} {
|
for name, depth := range map[string]int{"shallow": 1, "full clone": 0} {
|
||||||
t.Run(name, func(t *testing.T) {
|
t.Run(name, func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
require.NoError(t, NewGitCloneExecutor(NewGitCloneExecutorInput{
|
require.NoError(t, NewGitCloneExecutor(NewGitCloneExecutorInput{
|
||||||
URL: remoteDir, Ref: pinned, Dir: dir, Depth: depth,
|
URL: remoteDir, Ref: pinned, Dir: dir, Depth: depth,
|
||||||
@@ -540,13 +536,11 @@ func gitRevParse(t *testing.T, dir, rev string) string {
|
|||||||
return strings.TrimSpace(string(out))
|
return strings.TrimSpace(string(out))
|
||||||
}
|
}
|
||||||
|
|
||||||
func gitRevCount(t *testing.T, dir string) int {
|
func gitSubjects(t *testing.T, dir string) []string {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
out, err := exec.Command("git", "-C", dir, "rev-list", "--count", "HEAD").Output()
|
out, err := exec.Command("git", "-C", dir, "log", "--format=%s").Output()
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
n, err := strconv.Atoi(strings.TrimSpace(string(out)))
|
return strings.Split(strings.TrimSpace(string(out)), "\n")
|
||||||
require.NoError(t, err)
|
|
||||||
return n
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func gitHeadSubject(t *testing.T, dir string) string {
|
func gitHeadSubject(t *testing.T, dir string) string {
|
||||||
@@ -680,7 +674,7 @@ func TestNewGitCloneExecutorFetchSendsTokenAndHonoursContext(t *testing.T) {
|
|||||||
})
|
})
|
||||||
|
|
||||||
dir := filepath.Join(t.TempDir(), "cached-action")
|
dir := filepath.Join(t.TempDir(), "cached-action")
|
||||||
require.NoError(t, gitCmd("init", dir))
|
require.NoError(t, os.CopyFS(dir, os.DirFS(filepath.Join(gitFixtureDir, "work"))))
|
||||||
require.NoError(t, gitCmd("-C", dir, "remote", "add", "origin", server.URL))
|
require.NoError(t, gitCmd("-C", dir, "remote", "add", "origin", server.URL))
|
||||||
|
|
||||||
ctx, cancel := context.WithCancel(t.Context())
|
ctx, cancel := context.WithCancel(t.Context())
|
||||||
|
|||||||
@@ -48,6 +48,7 @@ func NewDockerBuildExecutor(input NewDockerBuildExecutorInput) common.Executor {
|
|||||||
|
|
||||||
options := client.ImageBuildOptions{
|
options := client.ImageBuildOptions{
|
||||||
Tags: []string{input.ImageTag},
|
Tags: []string{input.ImageTag},
|
||||||
|
Labels: map[string]string{"com.gitea.runner.image": input.ImageTag}, // own image ID per tag, so removing one never deletes another job's identical build
|
||||||
Remove: true,
|
Remove: true,
|
||||||
AuthConfigs: LoadDockerAuthConfigs(ctx),
|
AuthConfigs: LoadDockerAuthConfigs(ctx),
|
||||||
Dockerfile: input.Dockerfile,
|
Dockerfile: input.Dockerfile,
|
||||||
|
|||||||
@@ -18,10 +18,12 @@ import (
|
|||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestDockerBuildReportsDaemonError(t *testing.T) {
|
func TestDockerBuildLabelsImageWithItsTagAndReportsDaemonError(t *testing.T) {
|
||||||
|
var labels string
|
||||||
daemon := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
|
daemon := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
|
||||||
writer.Header().Set("API-Version", "1.47")
|
writer.Header().Set("API-Version", "1.47")
|
||||||
if strings.HasSuffix(request.URL.Path, "/build") {
|
if strings.HasSuffix(request.URL.Path, "/build") {
|
||||||
|
labels = request.URL.Query().Get("labels")
|
||||||
writer.Header().Set("Content-Type", "application/json")
|
writer.Header().Set("Content-Type", "application/json")
|
||||||
writer.WriteHeader(http.StatusInternalServerError)
|
writer.WriteHeader(http.StatusInternalServerError)
|
||||||
_, _ = io.WriteString(writer, `{"message":"build rejected"}`)
|
_, _ = io.WriteString(writer, `{"message":"build rejected"}`)
|
||||||
@@ -35,15 +37,16 @@ func TestDockerBuildReportsDaemonError(t *testing.T) {
|
|||||||
require.ErrorContains(t, NewDockerBuildExecutor(NewDockerBuildExecutorInput{
|
require.ErrorContains(t, NewDockerBuildExecutor(NewDockerBuildExecutorInput{
|
||||||
ImageTag: "action:test", Dockerfile: "Dockerfile", BuildContext: strings.NewReader(""),
|
ImageTag: "action:test", Dockerfile: "Dockerfile", BuildContext: strings.NewReader(""),
|
||||||
})(t.Context()), "build rejected")
|
})(t.Context()), "build rejected")
|
||||||
|
require.JSONEq(t, `{"com.gitea.runner.image":"action:test"}`, labels)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestCreateBuildContextFollowsDockerignoreAndSymlinkedRootButKeepsDockerfile(t *testing.T) {
|
func TestCreateBuildContextFollowsDockerignoreAndSymlinkedRootButKeepsDockerfile(t *testing.T) {
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
|
require.NoError(t, os.Mkdir(filepath.Join(dir, "build"), 0o755))
|
||||||
for name, content := range map[string]string{
|
for name, content := range map[string]string{
|
||||||
".dockerignore": "*.log\nbuild\n!build/keep.txt\nDockerfile", "Dockerfile": "FROM scratch", "a.txt": "a",
|
".dockerignore": "*.log\nbuild\n!build/keep.txt\nDockerfile", "Dockerfile": "FROM scratch", "a.txt": "a",
|
||||||
"a.log": "log", "build/drop.txt": "drop", "build/keep.txt": "keep",
|
"a.log": "log", "build/drop.txt": "drop", "build/keep.txt": "keep",
|
||||||
} {
|
} {
|
||||||
require.NoError(t, os.MkdirAll(filepath.Join(dir, filepath.Dir(name)), 0o755))
|
|
||||||
require.NoError(t, os.WriteFile(filepath.Join(dir, name), []byte(content), 0o644))
|
require.NoError(t, os.WriteFile(filepath.Join(dir, name), []byte(content), 0o644))
|
||||||
}
|
}
|
||||||
require.NoError(t, os.Mkdir(filepath.Join(dir, "empty"), 0o755))
|
require.NoError(t, os.Mkdir(filepath.Join(dir, "empty"), 0o755))
|
||||||
|
|||||||
@@ -22,7 +22,7 @@ import (
|
|||||||
|
|
||||||
// NewDockerPullExecutor function to create a run executor for the container
|
// NewDockerPullExecutor function to create a run executor for the container
|
||||||
func NewDockerPullExecutor(input NewDockerPullExecutorInput) common.Executor {
|
func NewDockerPullExecutor(input NewDockerPullExecutorInput) common.Executor {
|
||||||
return func(ctx context.Context) error {
|
return common.Executor(func(ctx context.Context) error {
|
||||||
logger := common.Logger(ctx)
|
logger := common.Logger(ctx)
|
||||||
logger.Debugf("docker pull %v", input.Image)
|
logger.Debugf("docker pull %v", input.Image)
|
||||||
|
|
||||||
@@ -83,7 +83,7 @@ func NewDockerPullExecutor(input NewDockerPullExecutorInput) common.Executor {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
return fmt.Errorf("failed to pull image '%s' (%s): %w", imageRef, input.Platform, err)
|
return fmt.Errorf("failed to pull image '%s' (%s): %w", imageRef, input.Platform, err)
|
||||||
}
|
}).Retry(3)
|
||||||
}
|
}
|
||||||
|
|
||||||
func getImagePullOptions(ctx context.Context, input NewDockerPullExecutorInput) (client.ImagePullOptions, error) {
|
func getImagePullOptions(ctx context.Context, input NewDockerPullExecutorInput) (client.ImagePullOptions, error) {
|
||||||
|
|||||||
@@ -540,6 +540,9 @@ func TestRejectsMissingContainer(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestPublicCopyPipelineHandlesStaleID(t *testing.T) {
|
func TestPublicCopyPipelineHandlesStaleID(t *testing.T) {
|
||||||
|
source := t.TempDir()
|
||||||
|
require.NoError(t, os.WriteFile(filepath.Join(source, "x"), nil, 0o644))
|
||||||
|
|
||||||
for _, testcase := range []struct {
|
for _, testcase := range []struct {
|
||||||
name, destination, extractionRoot, entry string
|
name, destination, extractionRoot, entry string
|
||||||
directory bool
|
directory bool
|
||||||
@@ -567,8 +570,6 @@ func TestPublicCopyPipelineHandlesStaleID(t *testing.T) {
|
|||||||
|
|
||||||
cr := &containerReference{id: "stale", cli: client, input: &NewContainerInput{Name: "job-1"}}
|
cr := &containerReference{id: "stale", cli: client, input: &NewContainerInput{Name: "job-1"}}
|
||||||
if testcase.directory {
|
if testcase.directory {
|
||||||
source := t.TempDir()
|
|
||||||
require.NoError(t, os.WriteFile(filepath.Join(source, "x"), nil, 0o644))
|
|
||||||
require.NoError(t, cr.CopyDir(testcase.destination, source+string(filepath.Separator), false, true)(ctx))
|
require.NoError(t, cr.CopyDir(testcase.destination, source+string(filepath.Separator), false, true)(ctx))
|
||||||
} else {
|
} else {
|
||||||
require.NoError(t, cr.Copy(testcase.destination, &FileEntry{Name: "x", Mode: 0o644})(ctx))
|
require.NoError(t, cr.Copy(testcase.destination, &FileEntry{Name: "x", Mode: 0o644})(ctx))
|
||||||
@@ -1108,6 +1109,10 @@ func TestConnectTakesDaemonOSFromPing(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestContainerCopyModes(t *testing.T) {
|
func TestContainerCopyModes(t *testing.T) {
|
||||||
|
source := t.TempDir()
|
||||||
|
require.NoError(t, os.WriteFile(filepath.Join(source, "run.sh"), []byte("#!/bin/sh\n"), 0o666))
|
||||||
|
require.NoError(t, os.Chmod(filepath.Join(source, "run.sh"), 0o666))
|
||||||
|
|
||||||
for _, testcase := range []struct {
|
for _, testcase := range []struct {
|
||||||
hostOS, serverOS string
|
hostOS, serverOS string
|
||||||
expected int64
|
expected int64
|
||||||
@@ -1117,9 +1122,6 @@ func TestContainerCopyModes(t *testing.T) {
|
|||||||
{"linux", "linux", 0o666},
|
{"linux", "linux", 0o666},
|
||||||
} {
|
} {
|
||||||
t.Run(testcase.hostOS+"-"+testcase.serverOS, func(t *testing.T) {
|
t.Run(testcase.hostOS+"-"+testcase.serverOS, func(t *testing.T) {
|
||||||
source := t.TempDir()
|
|
||||||
require.NoError(t, os.WriteFile(filepath.Join(source, "run.sh"), []byte("#!/bin/sh\n"), 0o666))
|
|
||||||
require.NoError(t, os.Chmod(filepath.Join(source, "run.sh"), 0o666))
|
|
||||||
file := &FileEntry{Name: "run.sh", Mode: 0o600, Body: "#!/bin/sh\n"}
|
file := &FileEntry{Name: "run.sh", Mode: 0o600, Body: "#!/bin/sh\n"}
|
||||||
original := hostOS
|
original := hostOS
|
||||||
hostOS = testcase.hostOS
|
hostOS = testcase.hostOS
|
||||||
|
|||||||
@@ -57,7 +57,7 @@ func (e *HostEnvironment) processGroup(ctx context.Context) *process.Group {
|
|||||||
e.procGroupOnce.Do(func() {
|
e.procGroupOnce.Do(func() {
|
||||||
group, err := process.NewGroup()
|
group, err := process.NewGroup()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
common.Logger(ctx).Warnf("could not create the job's process group; processes a step leaves behind can only be reclaimed by the workspace scan: %v", err)
|
common.Logger(ctx).Warnf("could not create the job's process group, processes a step leaves behind may outlive the job: %v", err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
e.procGroup.Store(group)
|
e.procGroup.Store(group)
|
||||||
@@ -415,46 +415,6 @@ func removePathWithRetry(ctx context.Context, path string) error {
|
|||||||
return lastErr
|
return lastErr
|
||||||
}
|
}
|
||||||
|
|
||||||
func (e *HostEnvironment) terminateRunningProcesses(ctx context.Context) {
|
|
||||||
if runtime.GOOS != "windows" {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// Detached: exec.CommandContext won't start on a cancelled ctx, and a
|
|
||||||
// server cancel has already cancelled the parent ctx.
|
|
||||||
killCtx, killCancel := context.WithTimeout(context.Background(), 30*time.Second)
|
|
||||||
defer killCancel()
|
|
||||||
|
|
||||||
logger := common.Logger(ctx)
|
|
||||||
|
|
||||||
// Dirs we own; a process referencing one is a leftover. ToolCache is shared
|
|
||||||
// across jobs, and Workdir may be a caller-owned checkout.
|
|
||||||
owned := []string{e.Path, e.TmpDir}
|
|
||||||
if e.CleanWorkdir {
|
|
||||||
owned = append(owned, e.Workdir)
|
|
||||||
}
|
|
||||||
dirs := make([]string, 0, len(owned))
|
|
||||||
for _, d := range owned {
|
|
||||||
if d == "" {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
abs, err := filepath.Abs(d)
|
|
||||||
if err != nil {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
dirs = append(dirs, abs)
|
|
||||||
}
|
|
||||||
if len(dirs) == 0 {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
if killed, err := process.KillProcessesWithPaths(killCtx, dirs, e.TrackingID); err != nil {
|
|
||||||
logger.Debugf("workspace process cleanup reported errors: %v", err)
|
|
||||||
} else if killed > 0 {
|
|
||||||
logger.Debugf("terminated %d leftover workspace process(es)", killed)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// hostCleanupTimeout bounds each teardown phase so one stalled delete cannot
|
// hostCleanupTimeout bounds each teardown phase so one stalled delete cannot
|
||||||
// wedge the runner slot. A var so tests can shrink it.
|
// wedge the runner slot. A var so tests can shrink it.
|
||||||
var hostCleanupTimeout = 30 * time.Second
|
var hostCleanupTimeout = 30 * time.Second
|
||||||
@@ -463,12 +423,15 @@ func (e *HostEnvironment) Remove() common.Executor {
|
|||||||
return func(ctx context.Context) error {
|
return func(ctx context.Context) error {
|
||||||
logger := common.Logger(ctx)
|
logger := common.Logger(ctx)
|
||||||
|
|
||||||
// End lingering processes before removing the workspace; on Windows their
|
// End lingering processes before removing the workspace, on Windows their file locks block cleanup.
|
||||||
// file locks block cleanup. Closing the group is deterministic, the scan a net.
|
|
||||||
if err := e.procGroup.Load().Close(); err != nil {
|
if err := e.procGroup.Load().Close(); err != nil {
|
||||||
logger.Debugf("closing the job's process group failed: %v", err)
|
logger.Debugf("closing the job's process group failed: %v", err)
|
||||||
}
|
}
|
||||||
e.terminateRunningProcesses(ctx)
|
if killed, err := process.KillTrackedProcesses(e.TrackingID); err != nil {
|
||||||
|
logger.Debugf("killing the job's leftover processes reported errors: %v", err)
|
||||||
|
} else if killed > 0 {
|
||||||
|
logger.Debugf("killed %d leftover process(es) of the job", killed)
|
||||||
|
}
|
||||||
|
|
||||||
// Detach: a cancelled ctx would skip removePathWithRetry's retries,
|
// Detach: a cancelled ctx would skip removePathWithRetry's retries,
|
||||||
// which absorb Windows file-handle release lag after the kill above.
|
// which absorb Windows file-handle release lag after the kill above.
|
||||||
|
|||||||
@@ -32,6 +32,11 @@ func TestActionPlatformNames(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestCopyDir(t *testing.T) {
|
func TestCopyDir(t *testing.T) {
|
||||||
|
src := filepath.Join(t.TempDir(), "action")
|
||||||
|
require.NoError(t, os.MkdirAll(filepath.Join(src, ".git", "objects"), 0o700))
|
||||||
|
require.NoError(t, os.WriteFile(filepath.Join(src, ".git", "objects", "pack"), []byte("x"), 0o600))
|
||||||
|
require.NoError(t, os.WriteFile(filepath.Join(src, "action.yml"), []byte("name: x\n"), 0o600))
|
||||||
|
|
||||||
for _, test := range []struct {
|
for _, test := range []struct {
|
||||||
name string
|
name string
|
||||||
useGitIgnore, skipGitDir bool
|
useGitIgnore, skipGitDir bool
|
||||||
@@ -41,11 +46,6 @@ func TestCopyDir(t *testing.T) {
|
|||||||
{name: "unfiltered outside a git repository", useGitIgnore: true},
|
{name: "unfiltered outside a git repository", useGitIgnore: true},
|
||||||
} {
|
} {
|
||||||
t.Run(test.name, func(t *testing.T) {
|
t.Run(test.name, func(t *testing.T) {
|
||||||
src := filepath.Join(t.TempDir(), "action")
|
|
||||||
require.NoError(t, os.MkdirAll(filepath.Join(src, ".git", "objects"), 0o700))
|
|
||||||
require.NoError(t, os.WriteFile(filepath.Join(src, ".git", "objects", "pack"), []byte("x"), 0o600))
|
|
||||||
require.NoError(t, os.WriteFile(filepath.Join(src, "action.yml"), []byte("name: x\n"), 0o600))
|
|
||||||
|
|
||||||
dest := filepath.Join(t.TempDir(), "dest")
|
dest := filepath.Join(t.TempDir(), "dest")
|
||||||
require.NoError(t, (&HostEnvironment{}).CopyDir(dest, src+string(filepath.Separator), test.useGitIgnore, test.skipGitDir)(t.Context()))
|
require.NoError(t, (&HostEnvironment{}).CopyDir(dest, src+string(filepath.Separator), test.useGitIgnore, test.skipGitDir)(t.Context()))
|
||||||
|
|
||||||
|
|||||||
@@ -4,18 +4,15 @@
|
|||||||
package container
|
package container
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"bufio"
|
|
||||||
"context"
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"net"
|
"net"
|
||||||
"os"
|
"os"
|
||||||
"os/exec"
|
"os/exec"
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
"unsafe"
|
|
||||||
|
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
"golang.org/x/sys/windows"
|
"golang.org/x/sys/windows"
|
||||||
@@ -40,7 +37,6 @@ func TestHostEnvironmentTrackingCleanup(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
env["GITEA_TRACKING_HELPER"] = "step"
|
env["GITEA_TRACKING_HELPER"] = "step"
|
||||||
env["GITEA_TRACKING_WORKSPACE"] = workspace
|
|
||||||
env["GITEA_TRACKING_ADDRESS"] = listener.Addr().String()
|
env["GITEA_TRACKING_ADDRESS"] = listener.Addr().String()
|
||||||
if tracking == "default" {
|
if tracking == "default" {
|
||||||
env["RUNNER_TRACKING_ID"] = environment.TrackingID
|
env["RUNNER_TRACKING_ID"] = environment.TrackingID
|
||||||
@@ -52,23 +48,6 @@ func TestHostEnvironmentTrackingCleanup(t *testing.T) {
|
|||||||
go func() {
|
go func() {
|
||||||
result <- environment.Exec([]string{os.Args[0], "-test.run=^TestHostTrackingHelper$"}, env, "", workspace)(ctx)
|
result <- environment.Exec([]string{os.Args[0], "-test.run=^TestHostTrackingHelper$"}, env, "", workspace)(ctx)
|
||||||
}()
|
}()
|
||||||
step, err := listener.Accept()
|
|
||||||
require.NoError(t, err)
|
|
||||||
defer step.Close()
|
|
||||||
require.NoError(t, step.SetDeadline(time.Now().Add(20*time.Second)))
|
|
||||||
var stepPID int
|
|
||||||
_, err = fmt.Fscanln(step, &stepPID)
|
|
||||||
require.NoError(t, err)
|
|
||||||
stepHandle, err := windows.OpenProcess(windows.PROCESS_QUERY_LIMITED_INFORMATION, false, uint32(stepPID))
|
|
||||||
require.NoError(t, err)
|
|
||||||
defer func() { _ = windows.CloseHandle(stepHandle) }()
|
|
||||||
require.Eventually(t, func() bool {
|
|
||||||
var assigned int32
|
|
||||||
result, _, _ := windows.NewLazySystemDLL("kernel32.dll").NewProc("IsProcessInJob").Call(uintptr(stepHandle), 0, uintptr(unsafe.Pointer(&assigned)))
|
|
||||||
return result != 0 && assigned != 0
|
|
||||||
}, 5*time.Second, 10*time.Millisecond)
|
|
||||||
_, err = step.Write([]byte{1})
|
|
||||||
require.NoError(t, err)
|
|
||||||
var helpers []windows.Handle
|
var helpers []windows.Handle
|
||||||
for range 2 {
|
for range 2 {
|
||||||
connection, err := listener.Accept()
|
connection, err := listener.Accept()
|
||||||
@@ -114,23 +93,18 @@ func TestHostTrackingHelper(t *testing.T) {
|
|||||||
if mode == "" {
|
if mode == "" {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
if mode != "descendant" {
|
||||||
|
child := exec.Command(os.Args[0], "-test.run=^TestHostTrackingHelper$")
|
||||||
|
child.Env = append(os.Environ(), "GITEA_TRACKING_HELPER="+map[string]string{"step": "helper", "helper": "descendant"}[mode])
|
||||||
|
require.NoError(t, child.Start())
|
||||||
|
if mode == "step" {
|
||||||
|
os.Exit(0)
|
||||||
|
}
|
||||||
|
}
|
||||||
connection, err := net.Dial("tcp4", os.Getenv("GITEA_TRACKING_ADDRESS"))
|
connection, err := net.Dial("tcp4", os.Getenv("GITEA_TRACKING_ADDRESS"))
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
_, err = fmt.Fprintln(connection, os.Getpid())
|
_, err = fmt.Fprintln(connection, os.Getpid())
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
if mode == "step" {
|
|
||||||
_, err := bufio.NewReader(connection).ReadByte()
|
|
||||||
require.NoError(t, err)
|
|
||||||
}
|
|
||||||
if mode != "descendant" {
|
|
||||||
child := exec.Command(os.Args[0], "-test.run=^TestHostTrackingHelper$", filepath.Join(os.Getenv("GITEA_TRACKING_WORKSPACE"), "tracking-reference"))
|
|
||||||
child.Env = append(os.Environ(), "GITEA_TRACKING_HELPER="+map[string]string{"step": "helper", "helper": "descendant"}[mode])
|
|
||||||
require.NoError(t, child.Start())
|
|
||||||
if mode == "step" {
|
|
||||||
require.NoError(t, connection.Close())
|
|
||||||
os.Exit(0)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
_, err = io.Copy(io.Discard, connection)
|
_, err = io.Copy(io.Discard, connection)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
require.NoError(t, connection.Close())
|
require.NoError(t, connection.Close())
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ package container
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"bufio"
|
"bufio"
|
||||||
|
"bytes"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
@@ -15,14 +16,9 @@ import (
|
|||||||
"golang.org/x/text/encoding/unicode"
|
"golang.org/x/text/encoding/unicode"
|
||||||
)
|
)
|
||||||
|
|
||||||
func newTestHostEnv(t *testing.T) (*HostEnvironment, string) {
|
|
||||||
t.Helper()
|
|
||||||
e := &HostEnvironment{Path: t.TempDir()}
|
|
||||||
return e, filepath.Join(e.Path, "envfile")
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestParseEnvFile(t *testing.T) {
|
func TestParseEnvFile(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
hostEnv := &HostEnvironment{Path: t.TempDir()}
|
||||||
largeValue := strings.Repeat("x", 2*1024*1024)
|
largeValue := strings.Repeat("x", 2*1024*1024)
|
||||||
encodeUTF16 := func(endianness unicode.Endianness) string {
|
encodeUTF16 := func(endianness unicode.Endianness) string {
|
||||||
content, err := unicode.UTF16(endianness, unicode.UseBOM).NewEncoder().Bytes([]byte("FOO=bar\r\nMULTI<<EOF\r\nline1\r\nEOF\r\n"))
|
content, err := unicode.UTF16(endianness, unicode.UseBOM).NewEncoder().Bytes([]byte("FOO=bar\r\nMULTI<<EOF\r\nline1\r\nEOF\r\n"))
|
||||||
@@ -51,15 +47,15 @@ func TestParseEnvFile(t *testing.T) {
|
|||||||
{"UTF16 big endian", encodeUTF16(unicode.BigEndian), map[string]string{"FOO": "bar\r", "MULTI": "line1\r"}},
|
{"UTF16 big endian", encodeUTF16(unicode.BigEndian), map[string]string{"FOO": "bar\r", "MULTI": "line1\r"}},
|
||||||
} {
|
} {
|
||||||
t.Run(test.name, func(t *testing.T) {
|
t.Run(test.name, func(t *testing.T) {
|
||||||
e, envPath := newTestHostEnv(t)
|
envPath := filepath.Join(hostEnv.Path, test.name)
|
||||||
require.NoError(t, os.WriteFile(envPath, []byte(test.content), 0o600))
|
require.NoError(t, os.WriteFile(envPath, []byte(test.content), 0o600))
|
||||||
env := map[string]string{}
|
env := map[string]string{}
|
||||||
require.NoError(t, parseEnvFile(e, envPath, &env, false)(t.Context()))
|
require.NoError(t, parseEnvFile(hostEnv, envPath, &env, false)(t.Context()))
|
||||||
assert.Equal(t, test.want, env)
|
assert.Equal(t, test.want, env)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
t.Run("ordered entries preserve repeated names", func(t *testing.T) {
|
t.Run("ordered entries preserve repeated names", func(t *testing.T) {
|
||||||
hostEnv, envPath := newTestHostEnv(t)
|
envPath := filepath.Join(hostEnv.Path, "ordered")
|
||||||
require.NoError(t, os.WriteFile(envPath, []byte("FOO=first\r\nfoo<<EOF\r\nmiddle\r\nEOF\r\nFOO=last\r\n"), 0o600))
|
require.NoError(t, os.WriteFile(envPath, []byte("FOO=first\r\nfoo<<EOF\r\nmiddle\r\nEOF\r\nFOO=last\r\n"), 0o600))
|
||||||
var entries [][2]string
|
var entries [][2]string
|
||||||
require.NoError(t, ParseEnvFile(hostEnv, envPath, func(name, value string) {
|
require.NoError(t, ParseEnvFile(hostEnv, envPath, func(name, value string) {
|
||||||
@@ -70,7 +66,7 @@ func TestParseEnvFile(t *testing.T) {
|
|||||||
}, entries)
|
}, entries)
|
||||||
})
|
})
|
||||||
t.Run("Windows line endings", func(t *testing.T) {
|
t.Run("Windows line endings", func(t *testing.T) {
|
||||||
hostEnv, envPath := newTestHostEnv(t)
|
envPath := filepath.Join(hostEnv.Path, "windows")
|
||||||
require.NoError(t, os.WriteFile(envPath, []byte("FOO=bar\r\nMULTI<<EOF\r\nline1\r\nline2\nEOF\r\n"), 0o600))
|
require.NoError(t, os.WriteFile(envPath, []byte("FOO=bar\r\nMULTI<<EOF\r\nline1\r\nline2\nEOF\r\n"), 0o600))
|
||||||
env := map[string]string{}
|
env := map[string]string{}
|
||||||
require.NoError(t, parseEnvFile(hostEnv, envPath, &env, true)(t.Context()))
|
require.NoError(t, parseEnvFile(hostEnv, envPath, &env, true)(t.Context()))
|
||||||
@@ -80,16 +76,22 @@ func TestParseEnvFile(t *testing.T) {
|
|||||||
|
|
||||||
func TestParseEnvFileLineExceedsBufferReportsScannerError(t *testing.T) {
|
func TestParseEnvFileLineExceedsBufferReportsScannerError(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
e, envPath := newTestHostEnv(t)
|
hostEnv := &HostEnvironment{Path: t.TempDir()}
|
||||||
require.NoError(t, os.WriteFile(envPath, []byte("FOO<<EOF\n"+strings.Repeat("x", 17*1024*1024)+"\nEOF\n"), 0o600))
|
envPath := filepath.Join(hostEnv.Path, "envfile")
|
||||||
|
const header, footer = "FOO<<EOF\n", "\nEOF\n"
|
||||||
|
content := bytes.Repeat([]byte("x"), len(header)+17*1024*1024+len(footer))
|
||||||
|
copy(content, header)
|
||||||
|
copy(content[len(content)-len(footer):], footer)
|
||||||
|
require.NoError(t, os.WriteFile(envPath, content, 0o600))
|
||||||
|
|
||||||
env := map[string]string{}
|
env := map[string]string{}
|
||||||
err := parseEnvFile(e, envPath, &env, false)(t.Context())
|
err := parseEnvFile(hostEnv, envPath, &env, false)(t.Context())
|
||||||
require.ErrorIs(t, err, bufio.ErrTooLong)
|
require.ErrorIs(t, err, bufio.ErrTooLong)
|
||||||
assert.Contains(t, err.Error(), "reading env file")
|
assert.Contains(t, err.Error(), "reading env file")
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestParseEnvFileInvalidHeredoc(t *testing.T) {
|
func TestParseEnvFileInvalidHeredoc(t *testing.T) {
|
||||||
|
hostEnv := &HostEnvironment{Path: t.TempDir()}
|
||||||
for name, content := range map[string]string{
|
for name, content := range map[string]string{
|
||||||
"missing delimiter": "FOO<<EOF\nline1\nline2\n",
|
"missing delimiter": "FOO<<EOF\nline1\nline2\n",
|
||||||
"empty name": "<<EOF\nline1\nEOF\n",
|
"empty name": "<<EOF\nline1\nEOF\n",
|
||||||
@@ -97,11 +99,11 @@ func TestParseEnvFileInvalidHeredoc(t *testing.T) {
|
|||||||
"delimiter with carriage return": "FOO<<EOF\nline1\nEOF\r\n",
|
"delimiter with carriage return": "FOO<<EOF\nline1\nEOF\r\n",
|
||||||
} {
|
} {
|
||||||
t.Run(name, func(t *testing.T) {
|
t.Run(name, func(t *testing.T) {
|
||||||
e, envPath := newTestHostEnv(t)
|
envPath := filepath.Join(hostEnv.Path, name)
|
||||||
require.NoError(t, os.WriteFile(envPath, []byte("APPLIED=before-error\n"+content), 0o600))
|
require.NoError(t, os.WriteFile(envPath, []byte("APPLIED=before-error\n"+content), 0o600))
|
||||||
|
|
||||||
env := map[string]string{}
|
env := map[string]string{}
|
||||||
err := parseEnvFile(e, envPath, &env, false)(t.Context())
|
err := parseEnvFile(hostEnv, envPath, &env, false)(t.Context())
|
||||||
require.Error(t, err)
|
require.Error(t, err)
|
||||||
assert.Contains(t, err.Error(), "delimiter")
|
assert.Contains(t, err.Error(), "delimiter")
|
||||||
assert.Equal(t, map[string]string{"APPLIED": "before-error"}, env)
|
assert.Equal(t, map[string]string{"APPLIED": "before-error"}, env)
|
||||||
|
|||||||
@@ -28,10 +28,12 @@ func gitCmd(t *testing.T, args ...string) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestCollectFilesKeepsWhatGitTracksOrDoesNotIgnore(t *testing.T) {
|
func TestCollectFilesKeepsWhatGitTracksOrDoesNotIgnore(t *testing.T) {
|
||||||
repoDir := filepath.Join(t.TempDir(), "mygitrepo")
|
repoDir := t.TempDir()
|
||||||
for _, dir := range []string{repoDir, filepath.Join(repoDir, "sub"), filepath.Join(repoDir, "nested")} {
|
gitCmd(t, "init", "-q", "--template=", repoDir)
|
||||||
gitCmd(t, "init", "-q", dir)
|
for _, dir := range []string{"sub", "nested"} {
|
||||||
|
require.NoError(t, os.CopyFS(filepath.Join(repoDir, dir, ".git"), os.DirFS(filepath.Join(repoDir, ".git"))))
|
||||||
}
|
}
|
||||||
|
require.NoError(t, os.Mkdir(filepath.Join(repoDir, "ign"), 0o755))
|
||||||
for name, body := range map[string]string{
|
for name, body := range map[string]string{
|
||||||
".gitignore": ".*\nign/\n",
|
".gitignore": ".*\nign/\n",
|
||||||
".env": "",
|
".env": "",
|
||||||
@@ -43,15 +45,15 @@ func TestCollectFilesKeepsWhatGitTracksOrDoesNotIgnore(t *testing.T) {
|
|||||||
"sub/kept": "",
|
"sub/kept": "",
|
||||||
"nested/kept": "",
|
"nested/kept": "",
|
||||||
} {
|
} {
|
||||||
require.NoError(t, os.MkdirAll(filepath.Dir(filepath.Join(repoDir, name)), 0o755))
|
|
||||||
require.NoError(t, os.WriteFile(filepath.Join(repoDir, name), []byte(body), 0o644))
|
require.NoError(t, os.WriteFile(filepath.Join(repoDir, name), []byte(body), 0o644))
|
||||||
}
|
}
|
||||||
require.NoError(t, os.Mkdir(filepath.Join(repoDir, "uninitialized"), 0o755))
|
require.NoError(t, os.Mkdir(filepath.Join(repoDir, "uninitialized"), 0o755))
|
||||||
gitCmd(t, "-C", repoDir, "add", "-f", ".gitignore", "Renamed", "ign/tracked")
|
gitCmd(t, "-C", repoDir, "add", "-f", ".gitignore", "Renamed", "ign/tracked")
|
||||||
require.NoError(t, os.Rename(filepath.Join(repoDir, "Renamed"), filepath.Join(repoDir, "renamed")))
|
require.NoError(t, os.Rename(filepath.Join(repoDir, "Renamed"), filepath.Join(repoDir, "renamed")))
|
||||||
for _, gitlink := range []string{"sub", "uninitialized"} {
|
gitCmd(t, "-C", repoDir, "update-index", "--add",
|
||||||
gitCmd(t, "-C", repoDir, "update-index", "--add", "--cacheinfo", "160000,4b825dc642cb6eb9a060e54bf8d69288fbee4904,"+gitlink)
|
"--cacheinfo", "160000,4b825dc642cb6eb9a060e54bf8d69288fbee4904,sub",
|
||||||
}
|
"--cacheinfo", "160000,4b825dc642cb6eb9a060e54bf8d69288fbee4904,uninitialized",
|
||||||
|
)
|
||||||
expected := map[string]string{".gitignore": "", "ign/tracked": "", "nested/kept": "", "renamed": "", "sub/.gitignore": "", "sub/kept": ""}
|
expected := map[string]string{".gitignore": "", "ign/tracked": "", "nested/kept": "", "renamed": "", "sub/.gitignore": "", "sub/kept": ""}
|
||||||
if runtime.GOOS != "windows" {
|
if runtime.GOOS != "windows" {
|
||||||
require.NoError(t, os.Chmod(filepath.Join(repoDir, "renamed"), 0o777|os.ModeSticky))
|
require.NoError(t, os.Chmod(filepath.Join(repoDir, "renamed"), 0o777|os.ModeSticky))
|
||||||
|
|||||||
@@ -80,9 +80,6 @@ func lookPath(file string, env map[string]string) (string, error) {
|
|||||||
return "", &Error{file, err}
|
return "", &Error{file, err}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if f, err := findExecutable(filepath.Join(".", file), exts); err == nil {
|
|
||||||
return f, nil
|
|
||||||
}
|
|
||||||
path := getenv(env, "path")
|
path := getenv(env, "path")
|
||||||
for _, dir := range filepath.SplitList(path) {
|
for _, dir := range filepath.SplitList(path) {
|
||||||
if f, err := findExecutable(filepath.Join(dir, file), exts); err == nil {
|
if f, err := findExecutable(filepath.Join(dir, file), exts); err == nil {
|
||||||
|
|||||||
@@ -0,0 +1,20 @@
|
|||||||
|
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||||
|
// SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
package lookpath
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestLookPath2IgnoresTheCurrentDirectory(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
require.NoError(t, os.WriteFile(filepath.Join(dir, "tool.exe"), nil, 0o755))
|
||||||
|
t.Chdir(dir)
|
||||||
|
_, err := LookPath2("tool", map[string]string{"PATH": t.TempDir()})
|
||||||
|
require.EqualError(t, err, ErrNotFound.Error())
|
||||||
|
}
|
||||||
+18
-22
@@ -339,34 +339,30 @@ func execAsDocker(ctx context.Context, step actionStep, actionName, actionDir, b
|
|||||||
|
|
||||||
if !correctArchExists || rc.Config.ForceRebuild {
|
if !correctArchExists || rc.Config.ForceRebuild {
|
||||||
logger.Debugf("image '%s' for architecture '%s' will be built from context '%s", image, rc.Config.ContainerArchitecture, contextDir)
|
logger.Debugf("image '%s' for architecture '%s' will be built from context '%s", image, rc.Config.ContainerArchitecture, contextDir)
|
||||||
var buildContext io.ReadCloser
|
prepImage = common.Executor(func(ctx context.Context) error {
|
||||||
if localAction {
|
input := container.NewDockerBuildExecutorInput{
|
||||||
buildContext, err = rc.JobContainer.GetContainerArchive(ctx, contextDir+"/.")
|
ContextDir: contextDir,
|
||||||
if err != nil {
|
Dockerfile: fileName,
|
||||||
return err
|
ImageTag: image,
|
||||||
|
Platform: rc.Config.ContainerArchitecture,
|
||||||
|
BuildArgs: rc.proxyBuildArgs(),
|
||||||
}
|
}
|
||||||
defer buildContext.Close()
|
if localAction {
|
||||||
}
|
buildContext, err := rc.JobContainer.GetContainerArchive(ctx, contextDir+"/.")
|
||||||
prepImage = ContainerNewDockerBuildExecutor(container.NewDockerBuildExecutorInput{
|
if err != nil {
|
||||||
ContextDir: contextDir,
|
return err
|
||||||
Dockerfile: fileName,
|
}
|
||||||
ImageTag: image,
|
defer buildContext.Close()
|
||||||
BuildContext: buildContext,
|
input.BuildContext = buildContext
|
||||||
Platform: rc.Config.ContainerArchitecture,
|
} else {
|
||||||
BuildArgs: rc.proxyBuildArgs(),
|
unlock, err := git.AcquireCloneLock(ctx, actionDir) // held across the whole build, the daemon drains contextDir lazily
|
||||||
})
|
|
||||||
if buildContext == nil {
|
|
||||||
// Held across the whole build: the daemon drains contextDir lazily.
|
|
||||||
inner := prepImage
|
|
||||||
prepImage = func(ctx context.Context) error {
|
|
||||||
unlock, err := git.AcquireCloneLock(ctx, actionDir)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
defer unlock()
|
defer unlock()
|
||||||
return inner(ctx)
|
|
||||||
}
|
}
|
||||||
}
|
return ContainerNewDockerBuildExecutor(input)(ctx)
|
||||||
|
}).Retry(3)
|
||||||
} else {
|
} else {
|
||||||
logger.Debugf("image '%s' for architecture '%s' already exists", image, rc.Config.ContainerArchitecture)
|
logger.Debugf("image '%s' for architecture '%s' already exists", image, rc.Config.ContainerArchitecture)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -92,7 +92,7 @@ func TestCompositeActionParity(t *testing.T) {
|
|||||||
return "", "", assert.AnError
|
return "", "", assert.AnError
|
||||||
})
|
})
|
||||||
rc := &RunContext{
|
rc := &RunContext{
|
||||||
Config: &Config{GitHubInstance: "github.com", ActionCacheDir: t.TempDir(), PresetGitHubContext: &model.GithubContext{}},
|
Config: &Config{GitHubInstance: "github.com", PresetGitHubContext: &model.GithubContext{}},
|
||||||
Run: &model.Run{JobID: "job", Workflow: &model.Workflow{Jobs: map[string]*model.Job{"job": {}}}},
|
Run: &model.Run{JobID: "job", Workflow: &model.Workflow{Jobs: map[string]*model.Job{"job": {}}}},
|
||||||
JobContainer: &jobContainerMock{},
|
JobContainer: &jobContainerMock{},
|
||||||
}
|
}
|
||||||
|
|||||||
+17
-13
@@ -249,6 +249,7 @@ func TestActionRunner(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestGoActionUsesLinuxExecutablePaths(t *testing.T) {
|
func TestGoActionUsesLinuxExecutablePaths(t *testing.T) {
|
||||||
|
actionRoot := t.TempDir()
|
||||||
for _, stage := range []string{"main", "pre", "post"} {
|
for _, stage := range []string{"main", "pre", "post"} {
|
||||||
t.Run(stage, func(t *testing.T) {
|
t.Run(stage, func(t *testing.T) {
|
||||||
cm := &containerMock{}
|
cm := &containerMock{}
|
||||||
@@ -264,7 +265,7 @@ func TestGoActionUsesLinuxExecutablePaths(t *testing.T) {
|
|||||||
JobContainer: cm,
|
JobContainer: cm,
|
||||||
},
|
},
|
||||||
action: &model.Action{Runs: model.ActionRuns{Using: model.ActionRunsUsingGo, Main: "main.go", Pre: "pre.go", Post: "post.go"}},
|
action: &model.Action{Runs: model.ActionRuns{Using: model.ActionRunsUsingGo, Main: "main.go", Pre: "pre.go", Post: "post.go"}},
|
||||||
actionDir: t.TempDir(),
|
actionDir: filepath.Join(actionRoot, stage),
|
||||||
env: map[string]string{},
|
env: map[string]string{},
|
||||||
remoteAction: mustNewRemoteAction(t, "org/repo@ref"),
|
remoteAction: mustNewRemoteAction(t, "org/repo@ref"),
|
||||||
}
|
}
|
||||||
@@ -504,11 +505,21 @@ func TestExecAsDockerHoldsCloneLockForRemoteUncached(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestExecAsDockerImageChecks(t *testing.T) {
|
func TestExecAsDockerImageChecks(t *testing.T) {
|
||||||
|
actionRoot := t.TempDir()
|
||||||
originalExists, originalBuild, originalContainer := ContainerImageExistsLocally, ContainerNewDockerBuildExecutor, ContainerNewContainer
|
originalExists, originalBuild, originalContainer := ContainerImageExistsLocally, ContainerNewDockerBuildExecutor, ContainerNewContainer
|
||||||
t.Cleanup(func() {
|
t.Cleanup(func() {
|
||||||
ContainerImageExistsLocally, ContainerNewDockerBuildExecutor, ContainerNewContainer = originalExists, originalBuild, originalContainer
|
ContainerImageExistsLocally, ContainerNewDockerBuildExecutor, ContainerNewContainer = originalExists, originalBuild, originalContainer
|
||||||
})
|
})
|
||||||
ContainerNewContainer = func(*container.NewContainerInput) container.ExecutionsEnvironment { return fakeContainer{} }
|
ContainerNewContainer = func(*container.NewContainerInput) container.ExecutionsEnvironment { return fakeContainer{} }
|
||||||
|
var removed bool
|
||||||
|
fakeDockerDaemon(t, func(writer http.ResponseWriter, request *http.Request) {
|
||||||
|
if request.Method == http.MethodDelete {
|
||||||
|
removed = true
|
||||||
|
_, _ = io.WriteString(writer, `[]`)
|
||||||
|
} else {
|
||||||
|
_, _ = io.WriteString(writer, `{"Architecture":"aarch64"}`)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
for _, testcase := range []struct {
|
for _, testcase := range []struct {
|
||||||
name, platform string
|
name, platform string
|
||||||
@@ -529,18 +540,11 @@ func TestExecAsDockerImageChecks(t *testing.T) {
|
|||||||
checks = append(checks, platform)
|
checks = append(checks, platform)
|
||||||
return testcase.exists && (platform == "any" || testcase.matches), nil
|
return testcase.exists && (platform == "any" || testcase.matches), nil
|
||||||
}
|
}
|
||||||
var built, removed bool
|
var built bool
|
||||||
|
removed = false
|
||||||
ContainerNewDockerBuildExecutor = func(container.NewDockerBuildExecutorInput) common.Executor {
|
ContainerNewDockerBuildExecutor = func(container.NewDockerBuildExecutorInput) common.Executor {
|
||||||
return func(context.Context) error { built = true; return nil }
|
return func(context.Context) error { built = true; return nil }
|
||||||
}
|
}
|
||||||
fakeDockerDaemon(t, func(writer http.ResponseWriter, request *http.Request) {
|
|
||||||
if request.Method == http.MethodDelete {
|
|
||||||
removed = true
|
|
||||||
_, _ = io.WriteString(writer, `[]`)
|
|
||||||
} else {
|
|
||||||
_, _ = io.WriteString(writer, `{"Architecture":"aarch64"}`)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
step := &stepActionRemote{
|
step := &stepActionRemote{
|
||||||
Step: &model.Step{ID: "action"},
|
Step: &model.Step{ID: "action"},
|
||||||
RunContext: &RunContext{
|
RunContext: &RunContext{
|
||||||
@@ -555,7 +559,7 @@ func TestExecAsDockerImageChecks(t *testing.T) {
|
|||||||
action: &model.Action{Runs: model.ActionRuns{Using: "docker", Image: "Dockerfile"}},
|
action: &model.Action{Runs: model.ActionRuns{Using: "docker", Image: "Dockerfile"}},
|
||||||
env: map[string]string{},
|
env: map[string]string{},
|
||||||
}
|
}
|
||||||
dir := t.TempDir()
|
dir := filepath.Join(actionRoot, testcase.name)
|
||||||
require.NoError(t, execAsDocker(t.Context(), step, "action", dir, dir, false, stepStageMain))
|
require.NoError(t, execAsDocker(t.Context(), step, "action", dir, dir, false, stepStageMain))
|
||||||
assert.Equal(t, testcase.wantChecks, checks)
|
assert.Equal(t, testcase.wantChecks, checks)
|
||||||
assert.Equal(t, testcase.wantBuild, built)
|
assert.Equal(t, testcase.wantBuild, built)
|
||||||
@@ -727,9 +731,9 @@ func TestExecAsDockerStageEntrypoint(t *testing.T) {
|
|||||||
cm.On("Close").Return(func(context.Context) error { return nil })
|
cm.On("Close").Return(func(context.Context) error { return nil })
|
||||||
|
|
||||||
if tc.stage == stepStageMain {
|
if tc.stage == stepStageMain {
|
||||||
require.NoError(t, runActionImpl(step, t.TempDir(), step.remoteAction)(ctx))
|
require.NoError(t, runActionImpl(step, "", step.remoteAction)(ctx))
|
||||||
} else {
|
} else {
|
||||||
require.NoError(t, execAsDocker(ctx, step, "action", t.TempDir(), t.TempDir(), false, tc.stage))
|
require.NoError(t, execAsDocker(ctx, step, "action", "", "", false, tc.stage))
|
||||||
}
|
}
|
||||||
if tc.warning == "" {
|
if tc.warning == "" {
|
||||||
assert.Empty(t, hook.AllRecords())
|
assert.Empty(t, hook.AllRecords())
|
||||||
|
|||||||
@@ -193,6 +193,11 @@ func TestCommandLogWriter(t *testing.T) {
|
|||||||
log []string
|
log []string
|
||||||
masks []string
|
masks []string
|
||||||
}{
|
}{
|
||||||
|
{
|
||||||
|
name: "mask commands decode modern and legacy escapes once",
|
||||||
|
steps: [][]string{{"::add-mask::%25", "##[add-mask]alpha beta%3Bgamma%5Ddelta%250A"}},
|
||||||
|
masks: []string{"%", "alpha beta;gamma]delta%0A"},
|
||||||
|
},
|
||||||
{
|
{
|
||||||
name: "applied commands are hidden unless echoed",
|
name: "applied commands are hidden unless echoed",
|
||||||
steps: [][]string{{
|
steps: [][]string{{
|
||||||
|
|||||||
@@ -49,6 +49,7 @@ func TestDockerProxyMountedJob(t *testing.T) {
|
|||||||
ContainerNamePrefix: resourceName,
|
ContainerNamePrefix: resourceName,
|
||||||
ContainerDaemonSocket: dockerClient.DaemonHost(),
|
ContainerDaemonSocket: dockerClient.DaemonHost(),
|
||||||
ContainerMaxLifetime: 2 * time.Minute,
|
ContainerMaxLifetime: 2 * time.Minute,
|
||||||
|
Sweep: true,
|
||||||
ContainerOptions: "-v /usr/local/bin/docker:/usr/local/bin/docker:ro -v /usr/local/libexec/docker/cli-plugins:/usr/local/libexec/docker/cli-plugins:ro",
|
ContainerOptions: "-v /usr/local/bin/docker:/usr/local/bin/docker:ro -v /usr/local/libexec/docker/cli-plugins:/usr/local/libexec/docker/cli-plugins:ro",
|
||||||
ValidVolumes: []string{"/usr/local/bin/docker", "/usr/local/libexec/docker/cli-plugins"},
|
ValidVolumes: []string{"/usr/local/bin/docker", "/usr/local/libexec/docker/cli-plugins"},
|
||||||
Env: map[string]string{"PROXY_TEST_RESOURCE": resourceName, "PROXY_TEST_MODE": mode, "PROXY_TEST_IMAGE": baseImage, "COMPOSE_PROJECT_NAME": resourceName},
|
Env: map[string]string{"PROXY_TEST_RESOURCE": resourceName, "PROXY_TEST_MODE": mode, "PROXY_TEST_IMAGE": baseImage, "COMPOSE_PROJECT_NAME": resourceName},
|
||||||
|
|||||||
@@ -412,11 +412,10 @@ func TestGetEvaluatorInputsBoolean(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
for _, eventName := range []string{"workflow_call", "workflow_dispatch"} {
|
for _, eventName := range []string{"workflow_call", "workflow_dispatch"} {
|
||||||
|
wf, err := model.ReadWorkflow(strings.NewReader(workflow(eventName)))
|
||||||
|
require.NoError(t, err)
|
||||||
for _, table := range tables {
|
for _, table := range tables {
|
||||||
t.Run(eventName+"/"+table.name, func(t *testing.T) {
|
t.Run(eventName+"/"+table.name, func(t *testing.T) {
|
||||||
wf, err := model.ReadWorkflow(strings.NewReader(workflow(eventName)))
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
rc := &RunContext{
|
rc := &RunContext{
|
||||||
Config: &Config{Workdir: "."},
|
Config: &Config{Workdir: "."},
|
||||||
Run: &model.Run{JobID: "job1", Workflow: wf},
|
Run: &model.Run{JobID: "job1", Workflow: wf},
|
||||||
|
|||||||
@@ -993,6 +993,7 @@ func tarArchive(t *testing.T, entries ...tarEntry) []byte {
|
|||||||
buf := &bytes.Buffer{}
|
buf := &bytes.Buffer{}
|
||||||
tw := tar.NewWriter(buf)
|
tw := tar.NewWriter(buf)
|
||||||
for _, entry := range entries {
|
for _, entry := range entries {
|
||||||
|
buf.Grow(len(entry.body) + 3*512)
|
||||||
typeflag := entry.typeflag
|
typeflag := entry.typeflag
|
||||||
if typeflag == 0 {
|
if typeflag == 0 {
|
||||||
typeflag = tar.TypeReg
|
typeflag = tar.TypeReg
|
||||||
@@ -1009,7 +1010,7 @@ func tarArchive(t *testing.T, entries ...tarEntry) []byte {
|
|||||||
}
|
}
|
||||||
require.NoError(t, tw.WriteHeader(header))
|
require.NoError(t, tw.WriteHeader(header))
|
||||||
if typeflag == tar.TypeReg {
|
if typeflag == tar.TypeReg {
|
||||||
_, err := tw.Write([]byte(entry.body))
|
_, err := io.CopyN(tw, strings.NewReader(entry.body), int64(len(entry.body)))
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1149,12 +1150,12 @@ func TestTryUploadJobSummaryMasksSecrets(t *testing.T) {
|
|||||||
}))
|
}))
|
||||||
defer server.Close()
|
defer server.Close()
|
||||||
|
|
||||||
content, err := testcase.encoding.NewEncoder().Bytes([]byte(testcase.content))
|
content, err := testcase.encoding.NewEncoder().String(testcase.content)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
cm := &containerMock{}
|
cm := &containerMock{}
|
||||||
cm.On("GetContainerArchive", mock.Anything, "/var/run/act/workflow/step-summary-0.md").Return(
|
cm.On("GetContainerArchive", mock.Anything, "/var/run/act/workflow/step-summary-0.md").Return(
|
||||||
io.NopCloser(bytes.NewReader(tarArchive(t, tarEntry{
|
io.NopCloser(bytes.NewReader(tarArchive(t, tarEntry{
|
||||||
name: "step-summary-0.md", body: string(content),
|
name: "step-summary-0.md", body: content,
|
||||||
}))),
|
}))),
|
||||||
nil,
|
nil,
|
||||||
).Once()
|
).Once()
|
||||||
|
|||||||
@@ -53,7 +53,7 @@ func (rc *RunContext) runJobHook(ctx context.Context, hookPath, name string) err
|
|||||||
}
|
}
|
||||||
rawLogger.Infof("::endgroup::")
|
rawLogger.Infof("::endgroup::")
|
||||||
|
|
||||||
env := mergeMaps(rc.containerSpec.Env, rc.GetEnv())
|
env := rc.mergeEnv(rc.caEnv, rc.containerSpec.Env, rc.GetEnv())
|
||||||
rc.withGithubEnv(ctx, rc.getGithubContext(ctx), env)
|
rc.withGithubEnv(ctx, rc.getGithubContext(ctx), env)
|
||||||
rc.ApplyExtraPath(ctx, &env)
|
rc.ApplyExtraPath(ctx, &env)
|
||||||
|
|
||||||
|
|||||||
+37
-9
@@ -263,29 +263,27 @@ func AppendSecretMaskers(oldnew []string, secrets map[string]string) []string {
|
|||||||
return oldnew
|
return oldnew
|
||||||
}
|
}
|
||||||
|
|
||||||
// AppendSecretMasker registers v and each of its lines, as GitHub does.
|
// AppendSecretMasker skips one-byte line fragments to avoid masking punctuation from structured secrets.
|
||||||
func AppendSecretMasker(oldnew []string, v string) []string {
|
func AppendSecretMasker(oldnew []string, v string) []string {
|
||||||
ret := appendMaskedValue(oldnew, v)
|
ret := appendMaskedValue(oldnew, v)
|
||||||
for l := range strings.FieldsFuncSeq(v, func(r rune) bool { return r == '\r' || r == '\n' }) {
|
for l := range strings.FieldsFuncSeq(v, func(r rune) bool { return r == '\r' || r == '\n' }) {
|
||||||
ret = appendMaskedValue(ret, strings.TrimSpace(l))
|
if line := strings.TrimSpace(l); len(line) > 1 {
|
||||||
|
ret = appendMaskedValue(ret, line)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
return ret
|
return ret
|
||||||
}
|
}
|
||||||
|
|
||||||
// appendMaskedValue registers one value and every shape it takes on its way into a log.
|
|
||||||
func appendMaskedValue(ret []string, v string) []string {
|
func appendMaskedValue(ret []string, v string) []string {
|
||||||
// formatted JSON secrets could otherwise mask {,[,],} everywhere
|
if strings.TrimSpace(v) == "" || slices.Contains(ret, v) {
|
||||||
if len(strings.TrimSpace(v)) <= 1 || slices.Contains(ret, v) {
|
|
||||||
return ret
|
return ret
|
||||||
}
|
}
|
||||||
ret = append(ret, v, "***")
|
ret = append(ret, v, "***")
|
||||||
// command data reaches the log escaped, so "pass%word" also arrives as "pass%25word"
|
|
||||||
if strings.ContainsAny(v, "%\r\n") {
|
if strings.ContainsAny(v, "%\r\n") {
|
||||||
ret = append(ret, EscapeCommandData(v), "***")
|
ret = append(ret, EscapeCommandData(v), "***")
|
||||||
}
|
}
|
||||||
for _, encode := range secretValueEncoders {
|
for _, encode := range secretValueEncoders {
|
||||||
encoded := encode(v)
|
encoded := encode(v)
|
||||||
// An encoding that leaves the value unchanged is already masked above.
|
|
||||||
if encoded == v || len(encoded) <= 1 || slices.Contains(ret, encoded) {
|
if encoded == v || len(encoded) <= 1 || slices.Contains(ret, encoded) {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
@@ -319,7 +317,7 @@ func valueMasker(insecureSecrets bool, oldnew []string) entryProcessor {
|
|||||||
masks := Masks(ctx)
|
masks := Masks(ctx)
|
||||||
|
|
||||||
if len(*masks) == 0 {
|
if len(*masks) == 0 {
|
||||||
r.Message = defReplacer.Replace(r.Message)
|
r.Message = MaskLog(r.Message, defReplacer)
|
||||||
return r
|
return r
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -339,12 +337,42 @@ func valueMasker(insecureSecrets bool, oldnew []string) entryProcessor {
|
|||||||
cmasker := replacer
|
cmasker := replacer
|
||||||
mu.Unlock()
|
mu.Unlock()
|
||||||
|
|
||||||
r.Message = cmasker.Replace(r.Message)
|
r.Message = MaskLog(r.Message, cmasker)
|
||||||
|
|
||||||
return r
|
return r
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// MaskLog decodes command data before masking to cover partially escaped values.
|
||||||
|
func MaskLog(message string, replacer *strings.Replacer) string {
|
||||||
|
var command, data string
|
||||||
|
var found bool
|
||||||
|
unescape := UnescapeCommandData
|
||||||
|
if strings.HasPrefix(message, "##[") {
|
||||||
|
command, data, found = strings.Cut(message[3:], "]")
|
||||||
|
unescape = UnescapeLegacyCommand
|
||||||
|
} else if strings.HasPrefix(message, "::") {
|
||||||
|
command, data, found = strings.Cut(message[2:], "::")
|
||||||
|
if name, _, properties := strings.Cut(command, " "); properties {
|
||||||
|
switch name {
|
||||||
|
case "error", "warning", "notice", "debug":
|
||||||
|
command = name
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
switch command {
|
||||||
|
case "error", "warning", "notice", "debug", "group", "endgroup", "add-matcher":
|
||||||
|
if found {
|
||||||
|
trimmedData := strings.TrimRight(data, "\r\n")
|
||||||
|
normalized := message[:len(message)-len(data)] + EscapeCommandData(unescape(trimmedData)) + data[len(trimmedData):]
|
||||||
|
if masked := replacer.Replace(normalized); masked != normalized {
|
||||||
|
return masked
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return replacer.Replace(message)
|
||||||
|
}
|
||||||
|
|
||||||
type maskedHandler struct {
|
type maskedHandler struct {
|
||||||
logging.Handler
|
logging.Handler
|
||||||
masker entryProcessor
|
masker entryProcessor
|
||||||
|
|||||||
@@ -28,6 +28,7 @@ func TestValueMasker(t *testing.T) {
|
|||||||
secrets map[string]string
|
secrets map[string]string
|
||||||
masks []string
|
masks []string
|
||||||
disallowed []string
|
disallowed []string
|
||||||
|
want string
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
name: "Multiline Private Key",
|
name: "Multiline Private Key",
|
||||||
@@ -49,6 +50,60 @@ func TestValueMasker(t *testing.T) {
|
|||||||
secrets: map[string]string{"TOKEN": "pass%word"},
|
secrets: map[string]string{"TOKEN": "pass%word"},
|
||||||
disallowed: []string{"pass%25word"},
|
disallowed: []string{"pass%25word"},
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
name: "one-character mask",
|
||||||
|
lines: "%",
|
||||||
|
masks: []string{"%"},
|
||||||
|
want: "***",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "fully escaped legacy secret",
|
||||||
|
lines: "##[group]alpha beta%3Bgamma%5Ddelta",
|
||||||
|
secrets: map[string]string{"TOKEN": "alpha beta;gamma]delta"},
|
||||||
|
want: "##[group]***",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "partially escaped legacy mask",
|
||||||
|
lines: "##[group]alpha beta%3Bgamma]delta",
|
||||||
|
masks: []string{"alpha beta;gamma]delta"},
|
||||||
|
want: "##[group]***",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "partially escaped modern mask preserves unrelated escapes",
|
||||||
|
lines: "::group::alpha%25beta%gamma tail%250A%0A",
|
||||||
|
masks: []string{"alpha%beta%gamma"},
|
||||||
|
want: "::group::*** tail%250A%0A",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "command escapes are decoded once",
|
||||||
|
lines: "##[group]alpha%253Bbeta%5Dgamma",
|
||||||
|
masks: []string{"alpha;beta]gamma"},
|
||||||
|
want: "##[group]alpha%253Bbeta%5Dgamma",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "plain output preserves escapes",
|
||||||
|
lines: "alpha beta%3Bgamma%5Ddelta",
|
||||||
|
masks: []string{"alpha beta;gamma]delta"},
|
||||||
|
want: "alpha beta%3Bgamma%5Ddelta",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "command display preserves literal escapes around a mask",
|
||||||
|
lines: "##[command]alpha tail%3Bvalue%5D",
|
||||||
|
masks: []string{"alpha"},
|
||||||
|
want: "##[command]*** tail%3Bvalue%5D",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "unknown modern commands preserve literal escapes",
|
||||||
|
lines: "::unknown::alpha tail%25value",
|
||||||
|
masks: []string{"alpha"},
|
||||||
|
want: "::unknown::*** tail%25value",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "masks can span command headers and data",
|
||||||
|
lines: "::group::hidden other",
|
||||||
|
masks: []string{"group::hidden", "other"},
|
||||||
|
want: "::*** ***",
|
||||||
|
},
|
||||||
}
|
}
|
||||||
for _, entry := range table {
|
for _, entry := range table {
|
||||||
t.Run(entry.name, func(t *testing.T) {
|
t.Run(entry.name, func(t *testing.T) {
|
||||||
@@ -56,6 +111,9 @@ func TestValueMasker(t *testing.T) {
|
|||||||
masker := valueMasker(false, AppendSecretMaskers(nil, entry.secrets))
|
masker := valueMasker(false, AppendSecretMaskers(nil, entry.secrets))
|
||||||
for line := range strings.SplitSeq(entry.lines, "\n") {
|
for line := range strings.SplitSeq(entry.lines, "\n") {
|
||||||
lentry := masker(ctx, logging.Record{Message: line})
|
lentry := masker(ctx, logging.Record{Message: line})
|
||||||
|
if entry.want != "" {
|
||||||
|
assert.Equal(t, entry.want, lentry.Message)
|
||||||
|
}
|
||||||
for _, line := range entry.disallowed {
|
for _, line := range entry.disallowed {
|
||||||
assert.NotContains(t, lentry.Message, line)
|
assert.NotContains(t, lentry.Message, line)
|
||||||
}
|
}
|
||||||
@@ -139,6 +197,7 @@ func TestValueMaskerEncodedMasks(t *testing.T) {
|
|||||||
{"encoded: " + base64.StdEncoding.EncodeToString([]byte(" s3cr3t ")), "encoded: ***"},
|
{"encoded: " + base64.StdEncoding.EncodeToString([]byte(" s3cr3t ")), "encoded: ***"},
|
||||||
{"encoded: " + base64.StdEncoding.EncodeToString([]byte("s3cr3t")), "encoded: ***"},
|
{"encoded: " + base64.StdEncoding.EncodeToString([]byte("s3cr3t")), "encoded: ***"},
|
||||||
{"encoded: " + base64.StdEncoding.EncodeToString([]byte("first")), "encoded: ***"},
|
{"encoded: " + base64.StdEncoding.EncodeToString([]byte("first")), "encoded: ***"},
|
||||||
|
{"##[group]s3cr3t value\r\n", "##[group]***\r\n"},
|
||||||
} {
|
} {
|
||||||
entry := masker(WithMasks(t.Context(), &masks), logging.Record{Message: tc.line})
|
entry := masker(WithMasks(t.Context(), &masks), logging.Record{Message: tc.line})
|
||||||
assert.Equal(t, tc.want, entry.Message, tc.line)
|
assert.Equal(t, tc.want, entry.Message, tc.line)
|
||||||
@@ -194,17 +253,15 @@ func TestValueMaskerCachedReplacerSeesNewMasks(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestAppendSecretMaskerSkipsUselessEncodings(t *testing.T) {
|
func TestAppendSecretMaskerSkipsUselessEncodings(t *testing.T) {
|
||||||
// A token with no character an escape would touch only gains its base64 forms:
|
|
||||||
// JSON, query and path escaping all leave it unchanged.
|
|
||||||
pairs := AppendSecretMasker(nil, "plaintoken")
|
pairs := AppendSecretMasker(nil, "plaintoken")
|
||||||
assert.Equal(t, []string{
|
assert.Equal(t, []string{
|
||||||
"plaintoken", "***", "cGxhaW50b2tlbg==", "***", "bGFpbnRva2Vu", "***", "YWludG9rZW4=", "***",
|
"plaintoken", "***", "cGxhaW50b2tlbg==", "***", "bGFpbnRva2Vu", "***", "YWludG9rZW4=", "***",
|
||||||
"aW50b2tl", "***", "YWludG9r", "***", "bGFpbnRv", "***",
|
"aW50b2tl", "***", "YWludG9r", "***", "bGFpbnRv", "***",
|
||||||
}, pairs)
|
}, pairs)
|
||||||
|
|
||||||
// Too short to mask.
|
assert.Contains(t, AppendSecretMasker(nil, "x"), "x")
|
||||||
assert.Empty(t, AppendSecretMasker(nil, "x"))
|
|
||||||
assert.Empty(t, AppendSecretMasker(nil, " \t"))
|
assert.Empty(t, AppendSecretMasker(nil, " \t"))
|
||||||
|
assert.NotContains(t, AppendSecretMasker(nil, "{\nvalue\n}"), "{")
|
||||||
assert.NotContains(t, AppendSecretMasker(nil, `"123456"`), "123456")
|
assert.NotContains(t, AppendSecretMasker(nil, `"123456"`), "123456")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -4,6 +4,7 @@
|
|||||||
package runner
|
package runner
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
"encoding/base64"
|
"encoding/base64"
|
||||||
"io"
|
"io"
|
||||||
@@ -209,14 +210,14 @@ func TestPatchedBundleAcrossActions(t *testing.T) {
|
|||||||
assert.Equal(t, data, out, "an untouched bundle must come back byte for byte")
|
assert.Equal(t, data, out, "an untouched bundle must come back byte for byte")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
assert.NotContains(t, string(out), ".LOCALHOST", "a copy of the gate was missed")
|
assert.False(t, bytes.Contains(out, []byte(".LOCALHOST")), "a copy of the gate was missed")
|
||||||
|
|
||||||
if !strings.Contains(string(data), CacheServiceV2Env) {
|
if !bytes.Contains(data, []byte(CacheServiceV2Env)) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
assert.NotZero(t, strings.Count(string(out), "(process.env."+cacheURLEnv+"||process.env"),
|
assert.NotZero(t, bytes.Count(out, []byte("(process.env."+cacheURLEnv+"||process.env")),
|
||||||
"the cache service URL was not redirected")
|
"the cache service URL was not redirected")
|
||||||
assert.Equal(t, strings.Count(string(data), resultsURLEnv), strings.Count(string(out), resultsURLEnv),
|
assert.Equal(t, bytes.Count(data, []byte(resultsURLEnv)), bytes.Count(out, []byte(resultsURLEnv)),
|
||||||
"a read of the results URL was lost, it must stay as the fallback")
|
"a read of the results URL was lost, it must stay as the fallback")
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
@@ -311,18 +312,18 @@ func TestCodeSplitActionFindsTheCacheService(t *testing.T) {
|
|||||||
requireHostTools(t, "node", "rustc")
|
requireHostTools(t, "node", "rustc")
|
||||||
|
|
||||||
dir := tempDirPath(t)
|
dir := tempDirPath(t)
|
||||||
|
require.NoError(t, os.Mkdir(filepath.Join(dir, "dist"), 0o755))
|
||||||
for _, name := range []string{
|
for _, name := range []string{
|
||||||
"action.yml", "dist/restore.js", "dist/save.js", "dist/cleanup-BWEbZ6YT.js",
|
"action.yml", "dist/restore.js", "dist/save.js", "dist/cleanup-BWEbZ6YT.js",
|
||||||
"dist/cache-CTfQoCTL.js", "dist/cache-DC26rUaF.js", "dist/state-cjs-CkvFUaNy.js",
|
"dist/cache-CTfQoCTL.js", "dist/cache-DC26rUaF.js", "dist/state-cjs-CkvFUaNy.js",
|
||||||
} {
|
} {
|
||||||
body, err := os.ReadFile(bundleFromGitHub(t, "Swatinem/rust-cache", "v2.9.2", name))
|
body, err := os.ReadFile(bundleFromGitHub(t, "Swatinem/rust-cache", "v2.9.2", name))
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
require.NoError(t, os.MkdirAll(filepath.Dir(filepath.Join(dir, name)), 0o755))
|
|
||||||
require.NoError(t, os.WriteFile(filepath.Join(dir, name), body, 0o600))
|
require.NoError(t, os.WriteFile(filepath.Join(dir, name), body, 0o600))
|
||||||
}
|
}
|
||||||
body, err := os.ReadFile(filepath.Join(dir, "action.yml"))
|
body, err := os.ReadFile(filepath.Join(dir, "action.yml"))
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
action, err := model.ReadAction(strings.NewReader(string(body)))
|
action, err := model.ReadAction(bytes.NewReader(body))
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
inputs := map[string]string{"lookup-only": "true", "add-rust-environment-hash-key": "false", "cache-bin": "false"}
|
inputs := map[string]string{"lookup-only": "true", "add-rust-environment-hash-key": "false", "cache-bin": "false"}
|
||||||
for name, input := range action.Inputs {
|
for name, input := range action.Inputs {
|
||||||
|
|||||||
@@ -4,6 +4,7 @@
|
|||||||
package runner
|
package runner
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
"os"
|
"os"
|
||||||
"os/exec"
|
"os/exec"
|
||||||
@@ -182,9 +183,9 @@ func TestPatchedBundleSurvivesInsideAStringLiteral(t *testing.T) {
|
|||||||
out, patched := patchedBundle([]byte(embedded))
|
out, patched := patchedBundle([]byte(embedded))
|
||||||
require.True(t, patched)
|
require.True(t, patched)
|
||||||
|
|
||||||
file := filepath.Join(t.TempDir(), "bundle.js")
|
cmd := exec.CommandContext(t.Context(), "node", "--check", "-")
|
||||||
require.NoError(t, os.WriteFile(file, out, 0o600))
|
cmd.Stdin = bytes.NewReader(out)
|
||||||
checked, err := exec.CommandContext(t.Context(), "node", "--check", file).CombinedOutput()
|
checked, err := cmd.CombinedOutput()
|
||||||
require.NoError(t, err, "%s", checked)
|
require.NoError(t, err, "%s", checked)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -256,6 +257,14 @@ func TestActionScriptPaths(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestPatchActionsAtTheContainerCopy(t *testing.T) {
|
func TestPatchActionsAtTheContainerCopy(t *testing.T) {
|
||||||
|
actionDir := t.TempDir()
|
||||||
|
script := filepath.Join(actionDir, "dist", "cache.js")
|
||||||
|
require.NoError(t, os.MkdirAll(filepath.Dir(script), 0o755))
|
||||||
|
require.NoError(t, os.WriteFile(filepath.Join(filepath.Dir(script), "index.js"), []byte("require('./cache.js')"), 0o600))
|
||||||
|
require.NoError(t, os.WriteFile(script, []byte(gateTSC), 0o600))
|
||||||
|
info, err := os.Stat(script)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
copiedBundles := func(t *testing.T, noPatch bool) ([]string, []*container.FileEntry, int64) {
|
copiedBundles := func(t *testing.T, noPatch bool) ([]string, []*container.FileEntry, int64) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
@@ -264,19 +273,12 @@ func TestPatchActionsAtTheContainerCopy(t *testing.T) {
|
|||||||
Step: &model.Step{Uses: "owner/repo/sub@v1"},
|
Step: &model.Step{Uses: "owner/repo/sub@v1"},
|
||||||
remoteAction: &remoteAction{Org: "owner", Repo: "repo", Path: "sub", Ref: "v1"},
|
remoteAction: &remoteAction{Org: "owner", Repo: "repo", Path: "sub", Ref: "v1"},
|
||||||
action: &model.Action{Runs: model.ActionRuns{Using: "node20", Main: "../dist/index.js"}},
|
action: &model.Action{Runs: model.ActionRuns{Using: "node20", Main: "../dist/index.js"}},
|
||||||
actionDir: t.TempDir(),
|
actionDir: actionDir,
|
||||||
RunContext: &RunContext{
|
RunContext: &RunContext{
|
||||||
Config: &Config{ActionCacheDir: t.TempDir(), NoActionPatch: noPatch},
|
Config: &Config{NoActionPatch: noPatch},
|
||||||
JobContainer: cm,
|
JobContainer: cm,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
script := filepath.Join(sar.actionDir, "dist", "cache.js")
|
|
||||||
require.NoError(t, os.MkdirAll(filepath.Dir(script), 0o755))
|
|
||||||
require.NoError(t, os.WriteFile(filepath.Join(filepath.Dir(script), "index.js"), []byte("require('./cache.js')"), 0o600))
|
|
||||||
require.NoError(t, os.WriteFile(script, []byte(gateTSC), 0o600))
|
|
||||||
info, err := os.Stat(script)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
var calls []string
|
var calls []string
|
||||||
var copied []*container.FileEntry
|
var copied []*container.FileEntry
|
||||||
cm.On("CopyDir", "/var/run/act/actions/repo/", sar.actionDir+"/", false, true).Return(func(context.Context) error {
|
cm.On("CopyDir", "/var/run/act/actions/repo/", sar.actionDir+"/", false, true).Return(func(context.Context) error {
|
||||||
|
|||||||
@@ -197,7 +197,7 @@ func (c Config) withInstanceAuth(input git.NewGitCloneExecutorInput, token strin
|
|||||||
}
|
}
|
||||||
input.ClientCertFile, input.ClientKeyFile = c.ClientCertFile, c.ClientKeyFile
|
input.ClientCertFile, input.ClientKeyFile = c.ClientCertFile, c.ClientKeyFile
|
||||||
if onInstanceHost(u.Host, c.GitHubInstance) {
|
if onInstanceHost(u.Host, c.GitHubInstance) {
|
||||||
input.InsecureSkipTLS, input.Transport = c.InsecureSkipTLS, c.InstanceTransport
|
input.InsecureSkipTLS, input.CAFile, input.Transport = c.InsecureSkipTLS, c.CAFile, c.InstanceTransport
|
||||||
}
|
}
|
||||||
if u.User == nil {
|
if u.User == nil {
|
||||||
input.Token = token
|
input.Token = token
|
||||||
|
|||||||
@@ -33,24 +33,17 @@ func TestReusableWorkflowCachedBranchRefRefreshes(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
remoteDir := t.TempDir()
|
remoteDir := t.TempDir()
|
||||||
gitMust(t, "", "init", "--bare", "--initial-branch=master", remoteDir)
|
gitMust(t, "", "init", "--template=", "--initial-branch=master", remoteDir)
|
||||||
|
|
||||||
workDir := t.TempDir()
|
|
||||||
gitMust(t, "", "clone", remoteDir, workDir)
|
|
||||||
gitMust(t, workDir, "config", "user.email", "test@test")
|
|
||||||
gitMust(t, workDir, "config", "user.name", "test")
|
|
||||||
gitMust(t, workDir, "checkout", "-b", "master")
|
|
||||||
|
|
||||||
const workflowPath = ".gitea/workflows/reusable.yml"
|
const workflowPath = ".gitea/workflows/reusable.yml"
|
||||||
tmpl := func(tag string) string {
|
tmpl := func(tag string) string {
|
||||||
return "name: reusable\non:\n workflow_call:\njobs:\n build:\n runs-on: ubuntu-latest\n steps:\n - run: echo " + tag + "\n"
|
return "name: reusable\non:\n workflow_call:\njobs:\n build:\n runs-on: ubuntu-latest\n steps:\n - run: echo " + tag + "\n"
|
||||||
}
|
}
|
||||||
|
|
||||||
require.NoError(t, os.MkdirAll(filepath.Join(workDir, ".gitea/workflows"), 0o755))
|
require.NoError(t, os.MkdirAll(filepath.Join(remoteDir, ".gitea/workflows"), 0o755))
|
||||||
require.NoError(t, os.WriteFile(filepath.Join(workDir, workflowPath), []byte(tmpl("v1")), 0o644))
|
require.NoError(t, os.WriteFile(filepath.Join(remoteDir, workflowPath), []byte(tmpl("v1")), 0o644))
|
||||||
gitMust(t, workDir, "add", workflowPath)
|
gitMust(t, remoteDir, "add", workflowPath)
|
||||||
gitMust(t, workDir, "commit", "-m", "v1")
|
gitMust(t, remoteDir, "commit", "-m", "v1")
|
||||||
gitMust(t, workDir, "push", "-u", "origin", "master")
|
|
||||||
|
|
||||||
rc := &RunContext{
|
rc := &RunContext{
|
||||||
Config: &Config{Env: testCheckoutEnv()},
|
Config: &Config{Env: testCheckoutEnv()},
|
||||||
@@ -70,9 +63,8 @@ func TestReusableWorkflowCachedBranchRefRefreshes(t *testing.T) {
|
|||||||
require.Equal(t, tmpl("v1"), string(got))
|
require.Equal(t, tmpl("v1"), string(got))
|
||||||
|
|
||||||
// Branch tip moves; cache key (cacheDir) does not.
|
// Branch tip moves; cache key (cacheDir) does not.
|
||||||
require.NoError(t, os.WriteFile(filepath.Join(workDir, workflowPath), []byte(tmpl("v2")), 0o644))
|
require.NoError(t, os.WriteFile(filepath.Join(remoteDir, workflowPath), []byte(tmpl("v2")), 0o644))
|
||||||
gitMust(t, workDir, "commit", "-am", "v2")
|
gitMust(t, remoteDir, "commit", "-am", "v2")
|
||||||
gitMust(t, workDir, "push", "origin", "master")
|
|
||||||
|
|
||||||
require.NoError(t, cloneRemoteReusableWorkflow(rc, remoteDir, "master", cacheDir)(t.Context()))
|
require.NoError(t, cloneRemoteReusableWorkflow(rc, remoteDir, "master", cacheDir)(t.Context()))
|
||||||
got, err = os.ReadFile(filepath.Join(cacheDir, workflowPath))
|
got, err = os.ReadFile(filepath.Join(cacheDir, workflowPath))
|
||||||
@@ -82,7 +74,7 @@ func TestReusableWorkflowCachedBranchRefRefreshes(t *testing.T) {
|
|||||||
|
|
||||||
func TestNewReusableWorkflowExecutorHoldsCloneLock(t *testing.T) {
|
func TestNewReusableWorkflowExecutorHoldsCloneLock(t *testing.T) {
|
||||||
workflowDir := t.TempDir()
|
workflowDir := t.TempDir()
|
||||||
require.NoError(t, os.WriteFile(filepath.Join(workflowDir, "reusable.yml"), []byte(":"), 0o644))
|
require.NoError(t, os.WriteFile(filepath.Join(workflowDir, "reusable.yml"), []byte("["), 0o644))
|
||||||
|
|
||||||
unlock, err := git.AcquireCloneLock(t.Context(), workflowDir)
|
unlock, err := git.AcquireCloneLock(t.Context(), workflowDir)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
@@ -118,7 +110,7 @@ func TestNewLocalReusableWorkflowExecutorFindsSameRepositoryPaths(t *testing.T)
|
|||||||
workdir := filepath.Join(t.TempDir(), "workspace")
|
workdir := filepath.Join(t.TempDir(), "workspace")
|
||||||
require.NoError(t, os.MkdirAll(filepath.Join(workdir, ".gitea", "workflows"), 0o755))
|
require.NoError(t, os.MkdirAll(filepath.Join(workdir, ".gitea", "workflows"), 0o755))
|
||||||
require.NoError(t, os.Mkdir(workdir+".lock", 0o755))
|
require.NoError(t, os.Mkdir(workdir+".lock", 0o755))
|
||||||
require.NoError(t, os.WriteFile(filepath.Join(workdir, ".gitea", "workflows", "reusable.yml"), []byte(":"), 0o644))
|
require.NoError(t, os.WriteFile(filepath.Join(workdir, ".gitea", "workflows", "reusable.yml"), []byte("["), 0o644))
|
||||||
|
|
||||||
for _, uses := range []string{"./.gitea/workflows/reusable.yml", "$/.gitea/workflows/reusable.yml"} {
|
for _, uses := range []string{"./.gitea/workflows/reusable.yml", "$/.gitea/workflows/reusable.yml"} {
|
||||||
rc := &RunContext{
|
rc := &RunContext{
|
||||||
@@ -130,7 +122,7 @@ func TestNewLocalReusableWorkflowExecutorFindsSameRepositoryPaths(t *testing.T)
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestCloneRemoteReusableWorkflowSendsInstanceTokenAndClientCert(t *testing.T) {
|
func TestCloneRemoteReusableWorkflowSendsInstanceTokenAndClientCertTrustingCA(t *testing.T) {
|
||||||
var gotCert bool
|
var gotCert bool
|
||||||
var gotToken string
|
var gotToken string
|
||||||
server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
@@ -150,12 +142,12 @@ func TestCloneRemoteReusableWorkflowSendsInstanceTokenAndClientCert(t *testing.T
|
|||||||
require.NoError(t, os.WriteFile(keyFile, pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: keyDER}), 0o600))
|
require.NoError(t, os.WriteFile(keyFile, pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: keyDER}), 0o600))
|
||||||
rc := &RunContext{
|
rc := &RunContext{
|
||||||
Config: &Config{
|
Config: &Config{
|
||||||
Env: testCheckoutEnv(),
|
Env: testCheckoutEnv(),
|
||||||
GitHubInstance: server.URL,
|
GitHubInstance: server.URL,
|
||||||
Secrets: map[string]string{"GITEA_TOKEN": "token"},
|
Secrets: map[string]string{"GITEA_TOKEN": "token"},
|
||||||
InsecureSkipTLS: true,
|
ClientCertFile: certFile,
|
||||||
ClientCertFile: certFile,
|
ClientKeyFile: keyFile,
|
||||||
ClientKeyFile: keyFile,
|
CAFile: certFile,
|
||||||
},
|
},
|
||||||
Run: &model.Run{JobID: "j1", Workflow: &model.Workflow{Jobs: map[string]*model.Job{"j1": {}}}},
|
Run: &model.Run{JobID: "j1", Workflow: &model.Workflow{Jobs: map[string]*model.Job{"j1": {}}}},
|
||||||
}
|
}
|
||||||
@@ -188,11 +180,12 @@ func TestInstanceAuthAndTransportOnlyApplyToTheInstance(t *testing.T) {
|
|||||||
InsecureSkipTLS: true,
|
InsecureSkipTLS: true,
|
||||||
ClientCertFile: "/cert",
|
ClientCertFile: "/cert",
|
||||||
ClientKeyFile: "/key",
|
ClientKeyFile: "/key",
|
||||||
|
CAFile: "/ca",
|
||||||
InstanceTransport: &http.Transport{},
|
InstanceTransport: &http.Transport{},
|
||||||
}
|
}
|
||||||
want := git.NewGitCloneExecutorInput{URL: tt.cloneURL, InsecureSkipTLS: tt.insecure}
|
want := git.NewGitCloneExecutorInput{URL: tt.cloneURL, InsecureSkipTLS: tt.insecure}
|
||||||
if tt.insecure {
|
if tt.insecure {
|
||||||
want.Transport = conf.InstanceTransport
|
want.CAFile, want.Transport = conf.CAFile, conf.InstanceTransport
|
||||||
}
|
}
|
||||||
if tt.token {
|
if tt.token {
|
||||||
want.Token = "token"
|
want.Token = "token"
|
||||||
@@ -213,7 +206,7 @@ func TestInstanceAuthAndTransportOnlyApplyToTheInstance(t *testing.T) {
|
|||||||
|
|
||||||
func gitMust(t *testing.T, dir string, args ...string) {
|
func gitMust(t *testing.T, dir string, args ...string) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
cmd := exec.Command("git", args...)
|
cmd := exec.Command("git", append([]string{"-c", "user.name=test", "-c", "user.email=test@test"}, args...)...)
|
||||||
if dir != "" {
|
if dir != "" {
|
||||||
cmd.Dir = dir
|
cmd.Dir = dir
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -68,6 +68,7 @@ type RunContext struct {
|
|||||||
kubernetes bool
|
kubernetes bool
|
||||||
containerSpec model.ContainerSpec // container:, resolved once with platformImage
|
containerSpec model.ContainerSpec // container:, resolved once with platformImage
|
||||||
containerEnv yaml.Node // container.env of the kept containerSpec, decoded once the job env resolves
|
containerEnv yaml.Node // container.env of the kept containerSpec, decoded once the job env resolves
|
||||||
|
caEnv map[string]string // NODE_EXTRA_CA_CERTS for the runner CA, beneath any env the job and its image set
|
||||||
JobName string
|
JobName string
|
||||||
ActionPath string
|
ActionPath string
|
||||||
Parent *RunContext
|
Parent *RunContext
|
||||||
@@ -278,11 +279,37 @@ func (rc *RunContext) validVolumes() []string {
|
|||||||
if rc.dockerProxy != nil {
|
if rc.dockerProxy != nil {
|
||||||
volumes = append(volumes, rc.dockerProxy.Socket)
|
volumes = append(volumes, rc.dockerProxy.Socket)
|
||||||
}
|
}
|
||||||
|
volumes = append(volumes, rc.Config.containerVolumeSources()...)
|
||||||
// TODO: add a new configuration to control whether the docker daemon can be mounted
|
// TODO: add a new configuration to control whether the docker daemon can be mounted
|
||||||
return append(volumes, name, name+"-env",
|
return append(volumes, name, name+"-env",
|
||||||
getDockerDaemonSocketMountPath(rc.containerDaemonSocket()))
|
getDockerDaemonSocketMountPath(rc.containerDaemonSocket()))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// containerVolumeSources lets workflows mount the container.volumes sources with no more access than they give.
|
||||||
|
func (c *Config) containerVolumeSources() []string {
|
||||||
|
var sources []string
|
||||||
|
for _, spec := range c.ContainerVolumes {
|
||||||
|
parsed, _ := container.ParseVolume(spec) // validated on config load
|
||||||
|
if parsed.ReadOnly {
|
||||||
|
parsed.Source += ":ro"
|
||||||
|
}
|
||||||
|
sources = append(sources, parsed.Source)
|
||||||
|
}
|
||||||
|
return sources
|
||||||
|
}
|
||||||
|
|
||||||
|
// mountVolumes sanitizes volumes and adds the container.volumes on the targets they leave unclaimed.
|
||||||
|
func (rc *RunContext) mountVolumes(ctx context.Context, validVolumes, volumes []string) ([]string, map[string]string, map[string]bool) {
|
||||||
|
binds, mounts, claimed := splitVolumes(container.SanitizeBinds(ctx, validVolumes, volumes))
|
||||||
|
for _, spec := range rc.Config.ContainerVolumes {
|
||||||
|
if parsed, _ := container.ParseVolume(spec); !claimed[parsed.Target] { // validated on config load
|
||||||
|
claimed[parsed.Target] = true
|
||||||
|
binds = append(binds, spec)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return binds, mounts, claimed
|
||||||
|
}
|
||||||
|
|
||||||
func (rc *RunContext) workdirMountRoot() string {
|
func (rc *RunContext) workdirMountRoot() string {
|
||||||
if rc.Config.PresetGitHubContext != nil {
|
if rc.Config.PresetGitHubContext != nil {
|
||||||
return filepath.Dir(filepath.Dir(rc.Config.Workdir)) // only the daemon presets, its workdir is <parent>/<owner>/<repo>
|
return filepath.Dir(filepath.Dir(rc.Config.Workdir)) // only the daemon presets, its workdir is <parent>/<owner>/<repo>
|
||||||
@@ -357,7 +384,7 @@ func (rc *RunContext) GetBindsAndMounts(ctx context.Context) ([]string, map[stri
|
|||||||
ext := container.LinuxContainerEnvironmentExtensions{}
|
ext := container.LinuxContainerEnvironmentExtensions{}
|
||||||
|
|
||||||
// the runner's own mounts below yield to the targets the job claims
|
// the runner's own mounts below yield to the targets the job claims
|
||||||
binds, mounts, claimed := splitVolumes(container.SanitizeBinds(ctx, rc.validVolumes(), rc.containerSpec.Volumes))
|
binds, mounts, claimed := rc.mountVolumes(ctx, rc.validVolumes(), rc.containerSpec.Volumes)
|
||||||
|
|
||||||
if rc.containerDaemonSocket() != "-" && !claimed["/var/run/docker.sock"] {
|
if rc.containerDaemonSocket() != "-" && !claimed["/var/run/docker.sock"] {
|
||||||
binds = append(binds, rc.jobDockerSocket()+":/var/run/docker.sock")
|
binds = append(binds, rc.jobDockerSocket()+":/var/run/docker.sock")
|
||||||
@@ -705,7 +732,12 @@ func (rc *RunContext) jobServices(ctx context.Context, networkName string) (map[
|
|||||||
entrypoint = []string{spec.Entrypoint}
|
entrypoint = []string{spec.Entrypoint}
|
||||||
}
|
}
|
||||||
|
|
||||||
serviceBinds, serviceMounts, _ := splitVolumes(spec.Volumes)
|
serviceBinds, serviceMounts, _ := splitVolumes(spec.Volumes) // as given on Kubernetes, which rejects them
|
||||||
|
validVolumes := rc.Config.ValidVolumes // not validVolumes(), a service gets no docker socket
|
||||||
|
if !rc.kubernetes {
|
||||||
|
validVolumes = append(slices.Clone(validVolumes), rc.Config.containerVolumeSources()...)
|
||||||
|
serviceBinds, serviceMounts, _ = rc.mountVolumes(ctx, validVolumes, spec.Volumes)
|
||||||
|
}
|
||||||
|
|
||||||
exposedPorts, portBindings, err := nat.ParsePortSpecs(spec.Ports)
|
exposedPorts, portBindings, err := nat.ParsePortSpecs(spec.Ports)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -734,7 +766,7 @@ func (rc *RunContext) jobServices(ctx context.Context, networkName string) (map[
|
|||||||
NetworkAliases: []string{serviceID},
|
NetworkAliases: []string{serviceID},
|
||||||
ExposedPorts: exposedPorts,
|
ExposedPorts: exposedPorts,
|
||||||
PortBindings: portBindings,
|
PortBindings: portBindings,
|
||||||
ValidVolumes: rc.Config.ValidVolumes, // not validVolumes(), a service gets no docker socket
|
ValidVolumes: validVolumes,
|
||||||
AllocatePTY: rc.Config.AllocatePTY,
|
AllocatePTY: rc.Config.AllocatePTY,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -778,7 +810,7 @@ func (rc *RunContext) cleanupJobResources(networkName string, createAndDeleteNet
|
|||||||
errs = append(errs, runAll(ctx, containers))
|
errs = append(errs, runAll(ctx, containers))
|
||||||
ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), jobCleanupTimeout)
|
ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), jobCleanupTimeout)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
if !common.Dryrun(ctx) && !rc.Config.NoSweep && (preclean || rc.hadDockerProxy) {
|
if !common.Dryrun(ctx) && rc.Config.Sweep && (preclean || rc.hadDockerProxy) {
|
||||||
errs = append(errs, container.RemoveDockerJobResources(ctx, rc.jobContainerName())) // after the containers, whose endpoints would block removing the job networks
|
errs = append(errs, container.RemoveDockerJobResources(ctx, rc.jobContainerName())) // after the containers, whose endpoints would block removing the job networks
|
||||||
}
|
}
|
||||||
var rest []common.Executor
|
var rest []common.Executor
|
||||||
@@ -1119,10 +1151,35 @@ func (svc *serviceContainer) healthOutputSuffix() string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (rc *RunContext) copyWorkflowFiles() common.Executor {
|
func (rc *RunContext) copyWorkflowFiles() common.Executor {
|
||||||
return rc.JobContainer.Copy(rc.JobContainer.GetActPath()+"/",
|
return func(ctx context.Context) error {
|
||||||
&container.FileEntry{Name: "workflow/event.json", Mode: 0o644, Body: rc.EventJSON},
|
files := []*container.FileEntry{
|
||||||
&container.FileEntry{Name: "workflow/envs.txt", Mode: 0o666},
|
{Name: "workflow/event.json", Mode: 0o644, Body: rc.EventJSON},
|
||||||
)
|
{Name: "workflow/envs.txt", Mode: 0o666},
|
||||||
|
}
|
||||||
|
if rc.Config.CAFile != "" {
|
||||||
|
ca, err := os.ReadFile(rc.Config.CAFile)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("read runner CA file: %w", err)
|
||||||
|
}
|
||||||
|
files = append(files, &container.FileEntry{Name: "ca.crt", Mode: 0o644, Body: string(ca)})
|
||||||
|
imageEnv := map[string]string{}
|
||||||
|
if err = rc.JobContainer.UpdateFromImageEnv(&imageEnv)(ctx); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if _, set := imageEnv["NODE_EXTRA_CA_CERTS"]; !set {
|
||||||
|
rc.caEnv = map[string]string{"NODE_EXTRA_CA_CERTS": rc.jobCAFile()}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return rc.JobContainer.Copy(rc.JobContainer.GetActPath()+"/", files...)(ctx)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// jobCAFile is where copyWorkflowFiles puts the runner CA in the job, empty without one.
|
||||||
|
func (rc *RunContext) jobCAFile() string {
|
||||||
|
if rc.Config.CAFile == "" {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return rc.JobContainer.GetActPath() + "/ca.crt"
|
||||||
}
|
}
|
||||||
|
|
||||||
// captureJobContainerInfo is a convenience: failing to describe the container must not
|
// captureJobContainerInfo is a convenience: failing to describe the container must not
|
||||||
|
|||||||
@@ -277,25 +277,38 @@ jobs:
|
|||||||
require.Equal(t, [2]string{"", ""}, credentials["redis:latest"])
|
require.Equal(t, [2]string{"", ""}, credentials["redis:latest"])
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestStartJobContainerGivesServicesTheirVolumes(t *testing.T) {
|
func TestStartJobContainerMountsWorkflowVolumesAndContainerVolumesOnUnclaimedTargets(t *testing.T) {
|
||||||
redis := startJobContainerInputs(t, `
|
const ca, certs = "/host/ca.crt:/etc/ssl/certs/ca-certificates.crt:ro", "/host/certs:/certs"
|
||||||
|
inputs := startJobContainerInputs(t, `
|
||||||
jobs:
|
jobs:
|
||||||
job:
|
job:
|
||||||
|
container:
|
||||||
|
image: node:24
|
||||||
|
volumes:
|
||||||
|
- /job/certs:/certs
|
||||||
services:
|
services:
|
||||||
redis:
|
redis:
|
||||||
image: redis:latest
|
image: redis:latest
|
||||||
volumes:
|
volumes:
|
||||||
- data:/data
|
- data:/data
|
||||||
- data:/backup
|
- data:/backup
|
||||||
`, &Config{ValidVolumes: []string{"data"}})[0]
|
- /blocked/ca.crt:/etc/ssl/certs/ca-certificates.crt:ro
|
||||||
|
- blocked:/certs
|
||||||
|
`, &Config{ValidVolumes: []string{"data", "/job/certs"}, ContainerVolumes: []string{ca, certs}})
|
||||||
|
require.Len(t, inputs, 2)
|
||||||
|
redis, job := inputs[0], inputs[1]
|
||||||
|
|
||||||
require.Equal(t, "redis:latest", redis.Image)
|
require.Equal(t, "redis:latest", redis.Image)
|
||||||
require.Equal(t, []string{"data"}, redis.ValidVolumes)
|
require.Equal(t, []string{"data", "/job/certs", "/host/ca.crt:ro", "/host/certs"}, redis.ValidVolumes)
|
||||||
require.Equal(t, map[string]string{"data": "/data"}, redis.Mounts)
|
require.Equal(t, map[string]string{"data": "/data"}, redis.Mounts)
|
||||||
require.Equal(t, []string{"data:/backup"}, redis.Binds)
|
require.Equal(t, []string{"data:/backup", ca, certs}, redis.Binds)
|
||||||
require.Empty(t, redis.WorkingDir)
|
require.Empty(t, redis.WorkingDir)
|
||||||
require.Nil(t, redis.Entrypoint)
|
require.Nil(t, redis.Entrypoint)
|
||||||
require.Nil(t, redis.Cmd)
|
require.Nil(t, redis.Cmd)
|
||||||
|
assert.Subset(t, job.Binds, []string{"/job/certs:/certs", ca})
|
||||||
|
assert.NotContains(t, job.Binds, certs)
|
||||||
|
assert.Subset(t, job.ValidVolumes, []string{"/job/certs", "/host/ca.crt:ro", "/host/certs"})
|
||||||
|
assert.NotContains(t, job.ValidVolumes, "/host/ca.crt", "a workflow must not mount a read-only source writable")
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestStartJobContainerBuildsEvaluatedJobAndServiceInputs(t *testing.T) {
|
func TestStartJobContainerBuildsEvaluatedJobAndServiceInputs(t *testing.T) {
|
||||||
@@ -304,7 +317,7 @@ jobs:
|
|||||||
job:
|
job:
|
||||||
container: ${{ fromJSON('{"image":"docker://node:20","options":"--label ${{ github.job }}","credentials":{"username":"user","password":"${{ vars.TITLE }}"}}') }}
|
container: ${{ fromJSON('{"image":"docker://node:20","options":"--label ${{ github.job }}","credentials":{"username":"user","password":"${{ vars.TITLE }}"}}') }}
|
||||||
services: ${{ fromJSON('{"redis":{"image":"docker://redis:latest","volumes":["data:/data"],"env":{"TITLE":"${{ github.job }}"},"credentials":{"username":"user","password":"${{ github.job }}"},"entrypoint":"/entry.sh","command":"redis-server --port 6380"}}') }}
|
services: ${{ fromJSON('{"redis":{"image":"docker://redis:latest","volumes":["data:/data"],"env":{"TITLE":"${{ github.job }}"},"credentials":{"username":"user","password":"${{ github.job }}"},"entrypoint":"/entry.sh","command":"redis-server --port 6380"}}') }}
|
||||||
`, &Config{})
|
`, &Config{ValidVolumes: []string{"data"}})
|
||||||
redis, job := inputs[0], inputs[1]
|
redis, job := inputs[0], inputs[1]
|
||||||
|
|
||||||
require.Equal(t, "redis:latest", redis.Image)
|
require.Equal(t, "redis:latest", redis.Image)
|
||||||
@@ -712,7 +725,7 @@ func TestCleanupJobResources(t *testing.T) {
|
|||||||
service.On("Remove").Return(removeAlongside(serviceRemoving, jobRemoving, removeError)).Once()
|
service.On("Remove").Return(removeAlongside(serviceRemoving, jobRemoving, removeError)).Once()
|
||||||
service.On("Close").Return(func(context.Context) error { return closeError }).Once()
|
service.On("Close").Return(func(context.Context) error { return closeError }).Once()
|
||||||
rc := &RunContext{
|
rc := &RunContext{
|
||||||
Config: &Config{CacheContainer: "cache-container", NoSweep: name == "kept proxy"},
|
Config: &Config{CacheContainer: "cache-container", Sweep: name != "kept proxy"},
|
||||||
Run: &model.Run{Workflow: &model.Workflow{Name: "wf"}, JobID: "job"},
|
Run: &model.Run{Workflow: &model.Workflow{Name: "wf"}, JobID: "job"},
|
||||||
JobContainer: job,
|
JobContainer: job,
|
||||||
serviceContainers: []*serviceContainer{{name: "svc", container: service}},
|
serviceContainers: []*serviceContainer{{name: "svc", container: service}},
|
||||||
@@ -1450,6 +1463,43 @@ func TestGetJobContextReportsContainers(t *testing.T) {
|
|||||||
}, jobContext.Services)
|
}, jobContext.Services)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type imageEnvContainer struct {
|
||||||
|
*container.HostEnvironment
|
||||||
|
imageEnv map[string]string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c imageEnvContainer) UpdateFromImageEnv(env *map[string]string) common.Executor {
|
||||||
|
return func(_ctx context.Context) error { maps.Copy(*env, c.imageEnv); return nil }
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCopyWorkflowFilesHandsNodeTheRunnerCAUnlessTheJobOrImageSetsIt(t *testing.T) {
|
||||||
|
caFile, actPath := filepath.Join(t.TempDir(), "ca.crt"), t.TempDir()
|
||||||
|
require.NoError(t, os.WriteFile(caFile, []byte("PEM\n"), 0o600))
|
||||||
|
own := map[string]string{"NODE_EXTRA_CA_CERTS": "/own.pem"}
|
||||||
|
stepEnv := func(caFile string, jobEnv, containerEnv, imageEnv map[string]string) map[string]string {
|
||||||
|
rc := &RunContext{
|
||||||
|
Config: &Config{CAFile: caFile},
|
||||||
|
Run: &model.Run{JobID: "job", Workflow: &model.Workflow{Jobs: map[string]*model.Job{"job": {}}}},
|
||||||
|
Env: maps.Clone(jobEnv),
|
||||||
|
JobContainer: imageEnvContainer{&container.HostEnvironment{Path: t.TempDir(), ActPath: actPath, StdOut: io.Discard}, imageEnv},
|
||||||
|
}
|
||||||
|
rc.containerSpec.Env = containerEnv
|
||||||
|
require.NoError(t, rc.copyWorkflowFiles()(t.Context()))
|
||||||
|
env := map[string]string{}
|
||||||
|
require.NoError(t, setupEnv(t.Context(), &stepRun{RunContext: rc, Step: &model.Step{}, env: env}))
|
||||||
|
return env
|
||||||
|
}
|
||||||
|
|
||||||
|
assert.Equal(t, filepath.ToSlash(actPath)+"/ca.crt", stepEnv(caFile, nil, nil, nil)["NODE_EXTRA_CA_CERTS"])
|
||||||
|
ca, err := os.ReadFile(filepath.Join(actPath, "ca.crt"))
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, "PEM\n", string(ca))
|
||||||
|
assert.Equal(t, "/own.pem", stepEnv(caFile, own, nil, nil)["NODE_EXTRA_CA_CERTS"])
|
||||||
|
assert.Equal(t, "/own.pem", stepEnv(caFile, nil, own, nil)["NODE_EXTRA_CA_CERTS"])
|
||||||
|
assert.NotContains(t, stepEnv(caFile, nil, nil, own), "NODE_EXTRA_CA_CERTS")
|
||||||
|
assert.NotContains(t, stepEnv("", nil, nil, nil), "NODE_EXTRA_CA_CERTS")
|
||||||
|
}
|
||||||
|
|
||||||
func TestCaptureJobContainerInfoExportsDockerWorkspace(t *testing.T) {
|
func TestCaptureJobContainerInfoExportsDockerWorkspace(t *testing.T) {
|
||||||
workdir := t.TempDir()
|
workdir := t.TempDir()
|
||||||
job := &containerMock{}
|
job := &containerMock{}
|
||||||
@@ -1616,21 +1666,21 @@ func TestRunContextUpdateExtraPath(t *testing.T) {
|
|||||||
longPath := strings.Repeat("x", 64*1024+1)
|
longPath := strings.Repeat("x", 64*1024+1)
|
||||||
for _, testcase := range []struct {
|
for _, testcase := range []struct {
|
||||||
name string
|
name string
|
||||||
content []byte
|
content string
|
||||||
want string
|
want string
|
||||||
wantErr bool
|
wantErr bool
|
||||||
}{
|
}{
|
||||||
{name: "UTF-8 BOM", content: []byte("\xef\xbb\xbf/utf8/tools\n"), want: "/utf8/tools"},
|
{name: "UTF-8 BOM", content: "\xef\xbb\xbf/utf8/tools\n", want: "/utf8/tools"},
|
||||||
{name: "UTF-16 BOM", content: []byte{0xff, 0xfe, 'C', 0, ':', 0, '\\', 0, 't', 0, 'o', 0, 'o', 0, 'l', 0, 's', 0, '\n', 0}, want: `C:\tools`},
|
{name: "UTF-16 BOM", content: "\xff\xfeC\x00:\x00\\\x00t\x00o\x00o\x00l\x00s\x00\n\x00", want: `C:\tools`},
|
||||||
{name: "over 64 KiB", content: []byte(longPath + "\n"), want: longPath},
|
{name: "over 64 KiB", content: longPath + "\n", want: longPath},
|
||||||
{name: "over 16 MiB", content: []byte(strings.Repeat("x", 16*1024*1024+1)), wantErr: true},
|
{name: "over 16 MiB", content: strings.Repeat("x", 16*1024*1024+1), wantErr: true},
|
||||||
} {
|
} {
|
||||||
t.Run(testcase.name, func(t *testing.T) {
|
t.Run(testcase.name, func(t *testing.T) {
|
||||||
logger := logging.New(logging.DiscardHandler)
|
logger := logging.New(logging.DiscardHandler)
|
||||||
ctx := common.WithLogger(t.Context(), logger)
|
ctx := common.WithLogger(t.Context(), logger)
|
||||||
jobContainer := &containerMock{}
|
jobContainer := &containerMock{}
|
||||||
jobContainer.On("GetContainerArchive", mock.Anything, "/github/path").
|
jobContainer.On("GetContainerArchive", mock.Anything, "/github/path").
|
||||||
Return(io.NopCloser(bytes.NewReader(tarArchive(t, tarEntry{name: "path", body: string(testcase.content)}))), nil).Once()
|
Return(io.NopCloser(bytes.NewReader(tarArchive(t, tarEntry{name: "path", body: testcase.content}))), nil).Once()
|
||||||
defer jobContainer.AssertExpectations(t)
|
defer jobContainer.AssertExpectations(t)
|
||||||
rc := &RunContext{JobContainer: jobContainer}
|
rc := &RunContext{JobContainer: jobContainer}
|
||||||
|
|
||||||
|
|||||||
@@ -83,13 +83,15 @@ type Config struct {
|
|||||||
InstanceTransport http.RoundTripper
|
InstanceTransport http.RoundTripper
|
||||||
JobLoggerLevel *logging.Level // the level of job logger
|
JobLoggerLevel *logging.Level // the level of job logger
|
||||||
ValidVolumes []string // only volumes (and bind mounts) in this slice can be mounted on the job container or service containers
|
ValidVolumes []string // only volumes (and bind mounts) in this slice can be mounted on the job container or service containers
|
||||||
|
ContainerVolumes []string // mounted into every job, service and docker action container
|
||||||
SharedToolCache bool // one tool cache for all jobs instead of one per job
|
SharedToolCache bool // one tool cache for all jobs instead of one per job
|
||||||
InsecureSkipTLS bool // whether to skip verifying TLS certificate of the Gitea instance
|
InsecureSkipTLS bool // whether to skip verifying TLS certificate of the Gitea instance
|
||||||
ClientCertFile, ClientKeyFile string // PEM client certificate for the Gitea instance
|
ClientCertFile, ClientKeyFile string // PEM client certificate for the Gitea instance
|
||||||
|
CAFile string // PEM CA certificates the Gitea instance's certificate chains to
|
||||||
MaxParallel int // max parallel jobs to run across all workflows (0 = no limit, uses CPU count)
|
MaxParallel int // max parallel jobs to run across all workflows (0 = no limit, uses CPU count)
|
||||||
AllocatePTY bool // allocate a pseudo-TTY for each step's process
|
AllocatePTY bool // allocate a pseudo-TTY for each step's process
|
||||||
ServiceReadyTimeout time.Duration // how long a job waits for its service containers to report healthy (0 uses the default)
|
ServiceReadyTimeout time.Duration // how long a job waits for its service containers to report healthy (0 uses the default)
|
||||||
NoSweep bool // keep the containers, networks and volumes a job creates through its docker proxy
|
Sweep bool // remove the containers, networks and volumes a job creates through its docker proxy
|
||||||
RunnerName string // name this runner registered with, reported as `runner.name`, defaults to the hostname
|
RunnerName string // name this runner registered with, reported as `runner.name`, defaults to the hostname
|
||||||
JobStartedHook string // script run inside the job environment before the job's first step; ACTIONS_RUNNER_HOOK_JOB_STARTED is read from Env when empty
|
JobStartedHook string // script run inside the job environment before the job's first step; ACTIONS_RUNNER_HOOK_JOB_STARTED is read from Env when empty
|
||||||
JobCompletedHook string // script run inside the job environment after the job's last step; ACTIONS_RUNNER_HOOK_JOB_COMPLETED is read from Env when empty
|
JobCompletedHook string // script run inside the job environment after the job's last step; ACTIONS_RUNNER_HOOK_JOB_COMPLETED is read from Env when empty
|
||||||
|
|||||||
+1
-1
@@ -263,7 +263,7 @@ func setupEnv(ctx context.Context, step step) error {
|
|||||||
|
|
||||||
env := step.getEnv()
|
env := step.getEnv()
|
||||||
// container env is the image's baseline, which job env and $GITHUB_ENV override
|
// container env is the image's baseline, which job env and $GITHUB_ENV override
|
||||||
mergeIntoMap(step, env, rc.containerSpec.Env, rc.GetEnv())
|
mergeIntoMap(step, env, rc.caEnv, rc.containerSpec.Env, rc.GetEnv())
|
||||||
rc.withGithubEnv(ctx, getStepGithubContext(ctx, step), *env)
|
rc.withGithubEnv(ctx, getStepGithubContext(ctx, step), *env)
|
||||||
|
|
||||||
var err error
|
var err error
|
||||||
|
|||||||
Loaded 50 of 102 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user