mirror of
https://gitea.com/gitea/runner.git
synced 2026-11-13 03:13:42 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
aa56f46c89 | ||
|
|
7a9fccb842 | ||
|
|
7d6e1ff021 | ||
|
|
319567213d | ||
|
|
0093731295 | ||
|
|
3e1b4b42c2 | ||
|
|
dcba546cb3 | ||
|
|
142d9f8c88 | ||
|
|
54b2cb944c | ||
|
|
b65fd0f8f8 | ||
|
|
f2fbd67962 | ||
|
|
d6f5b68a0d |
No files matched your search
@@ -34,6 +34,13 @@ jobs:
|
||||
with:
|
||||
go-version-file: "go.mod"
|
||||
check-latest: true
|
||||
- name: Import GPG key
|
||||
id: import_gpg
|
||||
uses: crazy-max/ghaction-import-gpg@2dc316deee8e90f13e1a351ab510b4d5bc0c82cd # v7
|
||||
with:
|
||||
gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }}
|
||||
passphrase: ${{ secrets.PASSPHRASE }}
|
||||
fingerprint: CC64B1DB67ABBEECAB24B6455FC346329753F4B0
|
||||
- name: goreleaser
|
||||
uses: goreleaser/goreleaser-action@a4f02ab5355a589de60d917a659ce617ed19b3de # v7
|
||||
with:
|
||||
@@ -47,6 +54,7 @@ jobs:
|
||||
R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
|
||||
GORELEASER_FORCE_TOKEN: "gitea"
|
||||
GITEA_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
GPG_FINGERPRINT: ${{ steps.import_gpg.outputs.fingerprint }}
|
||||
|
||||
release-image:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
@@ -57,6 +57,32 @@ jobs:
|
||||
make coverage-report
|
||||
cat .tmp/coverage.md >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
windows:
|
||||
name: test on windows
|
||||
runs-on: ubuntu-latest-kvm
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
- uses: ./.gitea/actions/go-setup
|
||||
- name: prepare anonymous docker config
|
||||
run: mkdir -p "$DOCKER_CONFIG" && echo '{}' > "$DOCKER_CONFIG/config.json"
|
||||
- name: windows vm cache key
|
||||
id: windows-vm
|
||||
run: echo "key=windows-vm-$(grep '^WINDOWS_TEST_IMAGE' Makefile | cat - scripts/test-windows.sh scripts/test-windows-setup.ps1 | sha256sum | cut -c1-16)" >> "$GITHUB_OUTPUT"
|
||||
- uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
|
||||
with:
|
||||
path: ~/.cache/windows-vm
|
||||
key: ${{ steps.windows-vm.outputs.key }}
|
||||
- name: test on windows
|
||||
id: test-windows
|
||||
run: WINDOWS_TEST_CACHE="$HOME/.cache/windows-vm" make test-windows
|
||||
env:
|
||||
WINDOWS_TEST_TIMEOUT: 45
|
||||
- if: always() && steps.test-windows.outputs.save-cache == 'true'
|
||||
uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
|
||||
with:
|
||||
path: ~/.cache/windows-vm
|
||||
key: ${{ steps.windows-vm.outputs.key }}
|
||||
|
||||
e2e:
|
||||
name: gitea compatibility (${{ matrix.gitea_image }})
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
/gitea-runner
|
||||
/gitea-runner.exe
|
||||
.env
|
||||
!/act/runner/testdata/secrets/.env
|
||||
.runner
|
||||
|
||||
@@ -39,6 +39,8 @@ linters:
|
||||
deny:
|
||||
- pkg: golang.org/x/exp
|
||||
desc: it's experimental and unreliable
|
||||
- pkg: unsafe
|
||||
desc: use typed APIs instead
|
||||
logging:
|
||||
files:
|
||||
- $all
|
||||
|
||||
+5
-2
@@ -34,8 +34,6 @@ builds:
|
||||
goarch: s390x
|
||||
- goos: windows
|
||||
goarch: arm
|
||||
- goos: windows
|
||||
goarch: arm64
|
||||
- goos: freebsd
|
||||
goarch: s390x
|
||||
- goos: freebsd
|
||||
@@ -82,6 +80,7 @@ builds:
|
||||
publishers:
|
||||
- name: cloudflare-r2
|
||||
checksum: true
|
||||
signature: true
|
||||
extra_files:
|
||||
- glob: ./**.xz
|
||||
- glob: ./**.sha256
|
||||
@@ -102,6 +101,10 @@ checksum:
|
||||
extra_files:
|
||||
- glob: ./**.xz
|
||||
|
||||
signs:
|
||||
- artifacts: checksum
|
||||
args: ["--batch", "-u", "{{ .Env.GPG_FINGERPRINT }}", "--output", "${signature}", "--detach-sign", "${artifact}"]
|
||||
|
||||
snapshot:
|
||||
version_template: "{{ .Branch }}-devel"
|
||||
|
||||
|
||||
@@ -69,3 +69,13 @@ Docker and Kubernetes.
|
||||
|
||||
The suite shares one Gitea, regular runner and k3s cluster. Cache and ephemeral scenarios use isolated
|
||||
repository runners. A run path is `<workflow>@<ref>` and a log row is `<timestamp>Z <payload>`.
|
||||
|
||||
## Windows tests
|
||||
|
||||
`make test-windows` cross-compiles the test binaries, installs Windows Server 2025 Core in a
|
||||
[dockur/windows](https://github.com/dockur/windows) QEMU VM on the docker host and runs them there.
|
||||
The host must be amd64 and allow privileged containers. `WINDOWS_TEST_TIMEOUT` sets the minutes
|
||||
allowed for installing Windows and running the tests, 165 by default. The VM uses KVM when the host
|
||||
has `/dev/kvm`, otherwise Windows is emulated and much slower. Docker-backed tests skip in the VM
|
||||
and the race detector is off, it needs cgo. `WINDOWS_TEST_CACHE` names a directory to keep the
|
||||
installed Windows in, so later runs skip the install. CI runs this on every pull request.
|
||||
@@ -1,6 +1,6 @@
|
||||
DIST := dist
|
||||
EXECUTABLE := gitea-runner
|
||||
GO ?= go
|
||||
EXECUTABLE := gitea-runner$(shell $(GO) env GOEXE)
|
||||
HAS_GO = $(shell hash $(GO) > /dev/null 2>&1 && echo "GO" || echo "NOGO" )
|
||||
|
||||
PODMAN_TEST_IMAGE ?= quay.io/containers/podman:v5.8.9-immutable@sha256:87f42f37fa85b421664c30b8f5f9cfc23920057f5e5fcadf0b6f68d9edc6d415 # renovate: datasource=docker
|
||||
@@ -8,6 +8,7 @@ E2E_JOB_IMAGE ?= docker.gitea.com/runner-images:ubuntu-latest@sha256:657378e5cfd
|
||||
SERVICE_IMAGE ?= nginx:1.31.7-alpine@sha256:892bb2152464f0c5d8972593a5f5afaf8054552580b699d95f8ff4af08c2d33d # renovate: datasource=docker
|
||||
E2E_GITEA_IMAGE ?= gitea/gitea:main-nightly
|
||||
E2E_K3S_IMAGE ?= rancher/k3s:v1.37.1-k3s1@sha256:ca7f37d993d82ef0dcdcfecb2e0e2618ea541dbaffc620c8cedebe01a82acd0d # renovate: datasource=docker
|
||||
WINDOWS_TEST_IMAGE ?= dockurr/windows:6.06@sha256:16fd7c39a3e48bb946d29c59dc6015942e5b900a1fa5d7e5552053bfc48ee2ea # renovate: datasource=docker
|
||||
|
||||
GOFILES := $(shell find . -type f -name "*.go" -o -name "go.mod" ! -name "generated.*")
|
||||
|
||||
@@ -150,6 +151,10 @@ coverage-report: ## turn coverage.txt from `make test` into .tmp/coverage.md
|
||||
test-dind: ## run the daemon-facing tests against the built dind image or podman (TARGET=dind|dind-rootless|podman)
|
||||
@PODMAN_TEST_IMAGE=$(PODMAN_TEST_IMAGE) ./scripts/test-dind.sh $(TARGET)
|
||||
|
||||
.PHONY: test-windows
|
||||
test-windows: ## run the tests on Windows Server Core in a QEMU VM on the docker host
|
||||
@GO=$(GO) WINDOWS_TEST_IMAGE=$(WINDOWS_TEST_IMAGE) ./scripts/test-windows.sh
|
||||
|
||||
E2E_CONCURRENCY ?= 8
|
||||
|
||||
.PHONY: test-e2e
|
||||
|
||||
@@ -104,7 +104,8 @@ type Handler struct {
|
||||
stopSync context.CancelFunc
|
||||
syncing sync.WaitGroup
|
||||
|
||||
outboundIP string
|
||||
outboundIP string
|
||||
instanceTransport http.RoundTripper
|
||||
|
||||
// internalSecret guards /_internal/{register,revoke}. When set, a remote
|
||||
// runner can use these endpoints to pre-register per-job
|
||||
@@ -124,7 +125,7 @@ type Handler struct {
|
||||
|
||||
policy Policy
|
||||
|
||||
// freeDisk is a field so tests can drive evictForFreeSpace without a full volume.
|
||||
// freeDisk is a field so tests can drive makeRoom without a full volume.
|
||||
freeDisk func(string) (uint64, error)
|
||||
}
|
||||
|
||||
@@ -143,19 +144,21 @@ type Options struct {
|
||||
// in-process RegisterJob method directly.
|
||||
InternalSecret string
|
||||
|
||||
Policy Policy
|
||||
Logger *logging.Logger
|
||||
Policy Policy
|
||||
Logger *logging.Logger
|
||||
InstanceTransport http.RoundTripper // reaches the instance artifact requests are forwarded to, nil for http.DefaultTransport
|
||||
}
|
||||
|
||||
// StartHandler starts an embedded cache or forwards cache requests to Upstream.
|
||||
func StartHandler(opts Options) (*Handler, error) {
|
||||
dir, logger := opts.Dir, opts.Logger
|
||||
h := &Handler{
|
||||
creds: make(map[string]*credEntry),
|
||||
internalSecret: opts.InternalSecret,
|
||||
policy: opts.Policy.withDefaults(),
|
||||
freeDisk: disk.FreeBytes,
|
||||
dbLock: make(chan struct{}, 1),
|
||||
creds: make(map[string]*credEntry),
|
||||
internalSecret: opts.InternalSecret,
|
||||
policy: opts.Policy.withDefaults(),
|
||||
freeDisk: disk.FreeBytes,
|
||||
dbLock: make(chan struct{}, 1),
|
||||
instanceTransport: opts.InstanceTransport,
|
||||
}
|
||||
|
||||
if logger == nil {
|
||||
@@ -773,6 +776,7 @@ func (h *Handler) upload(w http.ResponseWriter, r *http.Request) {
|
||||
h.responseJSON(w, r, 400, err)
|
||||
return
|
||||
}
|
||||
h.makeRoom(r.ContentLength)
|
||||
if err := h.storage.Write(cache.ID, start, r.Body); err != nil {
|
||||
h.responseJSON(w, r, 500, err)
|
||||
return
|
||||
@@ -875,11 +879,8 @@ func (h *Handler) commitCache(cache *Cache) error {
|
||||
if h.s3 != nil {
|
||||
return nil // lifecycle rules on the bucket bound its size
|
||||
}
|
||||
// A commit is the only thing that grows the store, so the only thing that can push the
|
||||
// volume under the floor.
|
||||
h.evictRepo(db, cache.Repo)
|
||||
h.evictTotal(db)
|
||||
h.evictForFreeSpace(db)
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -1173,9 +1174,6 @@ func (h *Handler) touchCache(id uint64, requireIncomplete bool) error {
|
||||
defer db.Close()
|
||||
|
||||
cache, err := db.get(id)
|
||||
if errors.Is(err, errCacheNotFound) {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -1260,23 +1258,29 @@ func (h *Handler) gcCache() {
|
||||
h.evictExpired(db)
|
||||
h.evictSuperseded(db)
|
||||
h.evictOversized(db)
|
||||
h.evictForFreeSpace(db)
|
||||
_ = db.Close()
|
||||
h.makeRoom(0)
|
||||
}
|
||||
|
||||
// evictForFreeSpace bounds the volume itself, so it also covers bytes the cache never
|
||||
// accounted for.
|
||||
func (h *Handler) evictForFreeSpace(db *cacheDB) {
|
||||
// makeRoom evicts the least recently used entries to keep need bytes above the floor, as GitHub saves new over old.
|
||||
func (h *Handler) makeRoom(need int64) {
|
||||
free, err := h.freeDisk(h.dir)
|
||||
if err != nil {
|
||||
h.logger.Debugf("free disk check: %v", err) // unsupported platform, treat as unavailable rather than full
|
||||
return
|
||||
}
|
||||
if free >= uint64(h.policy.MinFreeDisk) {
|
||||
shortfall := h.policy.MinFreeDisk + max(need, 0) - int64(free)
|
||||
if h.s3 != nil || shortfall <= 0 {
|
||||
return
|
||||
}
|
||||
db, err := h.openDB()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
caches := h.completedByUse(db)
|
||||
total, shortfall := totalSize(caches), h.policy.MinFreeDisk-int64(free)
|
||||
total := totalSize(caches)
|
||||
if total <= shortfall {
|
||||
// Say so, or shedding everything and still being short reads as the backstop working.
|
||||
h.logger.Warnf("cache volume is %d MiB short of the free space floor with only %d MiB of cache on it; something else is filling it", shortfall/miB, total/miB)
|
||||
|
||||
@@ -18,6 +18,7 @@ import (
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"testing/iotest"
|
||||
"testing/synctest"
|
||||
"time"
|
||||
|
||||
@@ -673,7 +674,7 @@ func TestHandler_evictPolicy(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandler_evictForFreeSpace(t *testing.T) {
|
||||
func TestHandler_makeRoom(t *testing.T) {
|
||||
t.Parallel()
|
||||
free := func(n int64) func(string) (uint64, error) {
|
||||
return func(string) (uint64, error) { return uint64(n), nil }
|
||||
@@ -682,14 +683,16 @@ func TestHandler_evictForFreeSpace(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
freeDisk func(string) (uint64, error)
|
||||
upload int64
|
||||
kept []string
|
||||
}{
|
||||
{"ample free space evicts nothing", free(defaultMinFreeDisk), []string{"oldest", "middle", "newest"}},
|
||||
{"a small shortfall sheds one entry", free(defaultMinFreeDisk - 4*miB), []string{"middle", "newest"}},
|
||||
{"a shortfall the cache cannot cover sheds all of it", free(0), nil},
|
||||
{"ample free space evicts nothing", free(defaultMinFreeDisk), 0, []string{"oldest", "middle", "newest"}},
|
||||
{"room for an upload sheds the oldest entry", free(defaultMinFreeDisk), 4 * miB, []string{"middle", "newest"}},
|
||||
{"a shortfall the cache cannot cover sheds all of it", free(0), 0, nil},
|
||||
{
|
||||
"an unreadable volume is treated as unavailable, not as full",
|
||||
func(string) (uint64, error) { return 0, errors.New("unsupported") },
|
||||
0,
|
||||
[]string{"oldest", "middle", "newest"},
|
||||
},
|
||||
} {
|
||||
@@ -702,11 +705,7 @@ func TestHandler_evictForFreeSpace(t *testing.T) {
|
||||
}
|
||||
handler := newTestHandler(t, Policy{}, entries...)
|
||||
handler.freeDisk = tc.freeDisk
|
||||
|
||||
db, err := handler.openDB()
|
||||
require.NoError(t, err)
|
||||
handler.evictForFreeSpace(db)
|
||||
require.NoError(t, db.Close())
|
||||
handler.makeRoom(tc.upload)
|
||||
|
||||
assert.ElementsMatch(t, tc.kept, keptKeys(t, handler, entries))
|
||||
})
|
||||
@@ -888,6 +887,13 @@ func keptKeys(t *testing.T, handler *Handler, entries []*Cache) []string {
|
||||
return kept
|
||||
}
|
||||
|
||||
func TestStorageDropsFailedWrites(t *testing.T) {
|
||||
storage, err := NewStorage(t.TempDir())
|
||||
require.NoError(t, err)
|
||||
require.Error(t, storage.Write(1, 0, io.MultiReader(strings.NewReader("partial"), iotest.ErrReader(io.ErrUnexpectedEOF))))
|
||||
assert.NoFileExists(t, storage.tempName(1, 0))
|
||||
}
|
||||
|
||||
func writeBlob(t *testing.T, handler *Handler, id uint64) {
|
||||
t.Helper()
|
||||
require.NoError(t, handler.storage.Write(id, 0, strings.NewReader("a")))
|
||||
|
||||
@@ -218,6 +218,7 @@ func (h *Handler) v2UploadBlob(w http.ResponseWriter, r *http.Request) {
|
||||
h.responseJSON(w, r, http.StatusBadRequest, err)
|
||||
return
|
||||
}
|
||||
h.makeRoom(r.ContentLength)
|
||||
|
||||
query := r.URL.Query()
|
||||
switch strings.ToLower(query.Get("comp")) {
|
||||
|
||||
@@ -14,6 +14,7 @@ import (
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
@@ -84,6 +85,8 @@ func TestCacheServiceV2RoundTrip(t *testing.T) {
|
||||
|
||||
unsigned := fmt.Sprintf("%s%s/1", handler.ExternalURL(), blobPath)
|
||||
assert.Equal(t, http.StatusUnauthorized, putBlob(t, unsigned, content))
|
||||
unreserved := handler.signedURL(JobCredential{Repo: testRepo}, blobPath, blobUploadPurpose, 1, time.Now().Add(time.Minute))
|
||||
assert.Equal(t, http.StatusBadRequest, putBlob(t, unreserved, content))
|
||||
|
||||
finalized, uploadURL := saveV2(t, handler, "deps-v1", "abc123", content)
|
||||
require.Equal(t, true, finalized["ok"])
|
||||
|
||||
@@ -67,7 +67,7 @@ func (h *Handler) forwardOrNotFound(w http.ResponseWriter, r *http.Request) {
|
||||
h.twirpError(w, r, twirpInternal, fmt.Errorf("cache server cannot reach the artifact service at %s: %w", target, err))
|
||||
},
|
||||
}
|
||||
if cred.InsecureTLS {
|
||||
if proxy.Transport = h.instanceTransport; proxy.Transport == nil && cred.InsecureTLS { // the instance transport carries runner.insecure and the client cert
|
||||
proxy.Transport = insecureTransport
|
||||
}
|
||||
proxy.ServeHTTP(w, r)
|
||||
|
||||
@@ -133,3 +133,20 @@ func TestFrontResultsService(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestFrontResultsServiceForwardsThroughTheInstanceTransport(t *testing.T) {
|
||||
t.Parallel()
|
||||
gitea := httptest.NewTLSServer(http.HandlerFunc(func(writer http.ResponseWriter, _request *http.Request) { _, _ = io.WriteString(writer, "{}") }))
|
||||
defer gitea.Close()
|
||||
handler, err := StartHandler(Options{Dir: t.TempDir(), OutboundIP: "127.0.0.1", InstanceTransport: gitea.Client().Transport})
|
||||
require.NoError(t, err)
|
||||
defer handler.Close()
|
||||
defer handler.RegisterJob("token", JobCredential{Results: gitea.URL})()
|
||||
|
||||
req, err := http.NewRequestWithContext(t.Context(), http.MethodPost, handler.ExternalURL()+artifactServicePath+"CreateArtifact", nil)
|
||||
require.NoError(t, err)
|
||||
resp, err := (&http.Client{Transport: &bearerTransport{token: "token"}}).Do(req)
|
||||
require.NoError(t, err)
|
||||
defer resp.Body.Close()
|
||||
require.Equal(t, http.StatusOK, resp.StatusCode)
|
||||
}
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json/v2"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
@@ -51,9 +52,10 @@ func (s *Storage) writeFile(name string, reader io.Reader) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
_, err = io.Copy(file, reader)
|
||||
if err = errors.Join(err, file.Close()); err != nil {
|
||||
_ = os.Remove(name)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -102,50 +104,46 @@ func (s *Storage) Commit(id uint64, size int64) (int64, error) {
|
||||
return 0, err
|
||||
}
|
||||
written, err := assemble(name, tempNames)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
if err == nil && size >= 0 && written != size { // old clients like actions/cache@v2 send no size
|
||||
err = fmt.Errorf("broken file: %v != %v", written, size)
|
||||
}
|
||||
// If size is less than 0, it means the size is unknown.
|
||||
// We can't check the size of the file, just skip the check.
|
||||
// It happens when the request comes from old versions of actions, like `actions/cache@v2`.
|
||||
if size >= 0 && written != size {
|
||||
if err != nil {
|
||||
_ = os.Remove(name)
|
||||
return 0, fmt.Errorf("broken file: %v != %v", written, size)
|
||||
return 0, err
|
||||
}
|
||||
return written, nil
|
||||
}
|
||||
|
||||
// assemble concatenates the uploaded parts into name. A single part, which is what the v2 API
|
||||
// produces below the client's block threshold, is already the whole archive and is moved.
|
||||
// assemble appends the parts into the first, deleting each once copied, so it never needs twice the space.
|
||||
func assemble(name string, tempNames []string) (int64, error) {
|
||||
if len(tempNames) == 1 {
|
||||
info, err := os.Stat(tempNames[0])
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return info.Size(), os.Rename(tempNames[0], name)
|
||||
if len(tempNames) == 0 {
|
||||
return 0, errors.New("nothing was uploaded")
|
||||
}
|
||||
|
||||
file, err := os.Create(name)
|
||||
if err := os.Rename(tempNames[0], name); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
file, err := os.OpenFile(name, os.O_WRONLY|os.O_APPEND, 0)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
var written int64
|
||||
for _, v := range tempNames {
|
||||
f, err := os.Open(v)
|
||||
for _, v := range tempNames[1:] {
|
||||
part, err := os.Open(v)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
n, err := io.Copy(file, f)
|
||||
_ = f.Close()
|
||||
_, err = io.Copy(file, part)
|
||||
_ = part.Close()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
written += n
|
||||
_ = os.Remove(v)
|
||||
}
|
||||
return written, nil
|
||||
info, err := file.Stat()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return info.Size(), file.Close()
|
||||
}
|
||||
|
||||
func (s *Storage) Serve(w http.ResponseWriter, r *http.Request, id uint64) {
|
||||
|
||||
@@ -281,7 +281,7 @@ func TestSafeResolve(t *testing.T) {
|
||||
|
||||
for name, tc := range tests {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
require.Equal(t, tc.want, safeResolve(baseDir, tc.input))
|
||||
require.Equal(t, filepath.FromSlash(tc.want), safeResolve(baseDir, tc.input))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7,7 +7,9 @@ package common
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"math/rand/v2"
|
||||
"runtime/debug"
|
||||
"time"
|
||||
|
||||
"gitea.com/gitea/runner/act/common/logging"
|
||||
)
|
||||
@@ -158,6 +160,28 @@ func (e Executor) IfBool(conditional bool) Executor {
|
||||
})
|
||||
}
|
||||
|
||||
// RetryBackoff is the wait between Retry's attempts, random between 1 and 10 seconds like GitHub's runner.
|
||||
var RetryBackoff = func() time.Duration { return time.Second + rand.N(9*time.Second) }
|
||||
|
||||
// Retry runs the executor up to attempts times until it succeeds.
|
||||
func (e Executor) Retry(attempts int) Executor {
|
||||
return func(ctx context.Context) error {
|
||||
for attempt := 1; ; attempt++ {
|
||||
err := e(ctx)
|
||||
if err == nil || attempt == attempts || ctx.Err() != nil {
|
||||
return err
|
||||
}
|
||||
backoff := RetryBackoff()
|
||||
Logger(ctx).Warnf("%v, retrying in %s", err, backoff.Round(time.Second))
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return err
|
||||
case <-time.After(backoff):
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Finally adds an executor to run after other executor
|
||||
func (e Executor) Finally(finally Executor) Executor {
|
||||
return func(ctx context.Context) error {
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
@@ -134,6 +135,31 @@ func TestExecutorConditionalsAndFinally(t *testing.T) {
|
||||
assert.Equal(t, []string{"if-true", "if-not", "if-bool", "main", "finally"}, calls)
|
||||
}
|
||||
|
||||
func TestExecutorRetryStopsOnSuccessOrAfterAllAttempts(t *testing.T) {
|
||||
backoff := RetryBackoff
|
||||
RetryBackoff = func() time.Duration { return 0 }
|
||||
t.Cleanup(func() { RetryBackoff = backoff })
|
||||
transient := errors.New("transient")
|
||||
for _, tc := range []struct {
|
||||
failures, calls int
|
||||
err error
|
||||
}{
|
||||
{failures: 2, calls: 3},
|
||||
{failures: 5, calls: 3, err: transient},
|
||||
} {
|
||||
var calls int
|
||||
err := Executor(func(context.Context) error {
|
||||
calls++
|
||||
if calls <= tc.failures {
|
||||
return transient
|
||||
}
|
||||
return nil
|
||||
}).Retry(3)(t.Context())
|
||||
require.Equal(t, tc.err, err)
|
||||
require.Equal(t, tc.calls, calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecutorFinallyReturnsFinallyErrorWithOriginal(t *testing.T) {
|
||||
finalErr := errors.New("cleanup failed")
|
||||
for _, mainErr := range []error{context.Canceled, nil} {
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"math/rand/v2"
|
||||
"net"
|
||||
@@ -29,6 +30,10 @@ import (
|
||||
|
||||
var actionFetches sync.Map
|
||||
|
||||
var symlink = (*os.Root).Symlink
|
||||
|
||||
var symlinkFallback = runtime.GOOS == "windows"
|
||||
|
||||
const treeSizes = ".sizes" // beside the tree rather than in it, which is copied into job containers as it is
|
||||
|
||||
type treeFile struct {
|
||||
@@ -229,6 +234,8 @@ func extractTree(ctx context.Context, archive io.Reader, dir string, strip bool)
|
||||
|
||||
sha := ""
|
||||
sizes := map[string]int64{}
|
||||
links := map[string]string{}
|
||||
var deferred []string
|
||||
reader := tar.NewReader(archive)
|
||||
for {
|
||||
header, err := reader.Next()
|
||||
@@ -257,9 +264,8 @@ func extractTree(ctx context.Context, archive io.Reader, dir string, strip bool)
|
||||
case tar.TypeReg:
|
||||
sizes[name], err = writeTreeFile(root, name, header.Mode, reader)
|
||||
case tar.TypeSymlink:
|
||||
if err = root.Symlink(header.Linkname, name); err != nil && runtime.GOOS == "windows" {
|
||||
sizes[name], err = writeTreeFile(root, name, 0, strings.NewReader(header.Linkname))
|
||||
}
|
||||
links[filepath.Clean(name)] = header.Linkname
|
||||
deferred = append(deferred, filepath.Clean(name))
|
||||
}
|
||||
if err != nil {
|
||||
return "", err
|
||||
@@ -268,12 +274,166 @@ func extractTree(ctx context.Context, archive io.Reader, dir string, strip bool)
|
||||
if !isFullObjectID(sha) {
|
||||
return "", errors.New("the archive names no commit")
|
||||
}
|
||||
linksByParent := map[string][]string{}
|
||||
if symlinkFallback {
|
||||
for name := range links {
|
||||
linksByParent[filepath.Dir(name)] = append(linksByParent[filepath.Dir(name)], filepath.Base(name))
|
||||
}
|
||||
}
|
||||
for requireTarget := symlinkFallback; len(deferred) > 0; {
|
||||
unresolved := deferred[:0]
|
||||
for _, name := range deferred {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if requireTarget {
|
||||
if _, err := root.Stat(filepath.Dir(name) + string(filepath.Separator) + links[name]); err != nil {
|
||||
unresolved = append(unresolved, name)
|
||||
continue
|
||||
}
|
||||
}
|
||||
if err := symlink(root, links[name], name); err == nil || !symlinkFallback {
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
continue
|
||||
}
|
||||
if err := copyTreeLink(ctx, root, name, links, linksByParent, sizes); err != nil {
|
||||
return "", fmt.Errorf("materialize symlink %q: %w", filepath.ToSlash(name), err)
|
||||
}
|
||||
}
|
||||
if len(unresolved) == len(deferred) {
|
||||
requireTarget = false
|
||||
}
|
||||
deferred = unresolved
|
||||
}
|
||||
if err := root.Close(); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return sha, installTree(ctx, dir, sha, temp, sizes)
|
||||
}
|
||||
|
||||
func resolveTreeLink(name string, links map[string]string) (string, bool) {
|
||||
if !filepath.IsLocal(name) {
|
||||
return "", false
|
||||
}
|
||||
parts := strings.Split(name, string(filepath.Separator))
|
||||
var resolved []string
|
||||
for count := 0; len(parts) > 0; {
|
||||
part := parts[0]
|
||||
parts = parts[1:]
|
||||
switch part {
|
||||
case "", ".":
|
||||
continue
|
||||
case "..":
|
||||
if len(resolved) == 0 {
|
||||
return "", false
|
||||
}
|
||||
resolved = resolved[:len(resolved)-1]
|
||||
continue
|
||||
}
|
||||
target, ok := links[filepath.Join(append(resolved, part)...)]
|
||||
if !ok {
|
||||
resolved = append(resolved, part)
|
||||
continue
|
||||
}
|
||||
count++
|
||||
if target == "" || os.IsPathSeparator(target[0]) || filepath.VolumeName(target) != "" || count > 255 {
|
||||
return "", false
|
||||
}
|
||||
parts = append(strings.Split(filepath.FromSlash(target), string(filepath.Separator)), parts...)
|
||||
}
|
||||
return filepath.Join(append([]string{"."}, resolved...)...), true
|
||||
}
|
||||
|
||||
var (
|
||||
errTreeLinkTarget = errors.New("unmaterializable symlink target")
|
||||
errTreeLinkCycle = fmt.Errorf("cyclic %w", errTreeLinkTarget)
|
||||
)
|
||||
|
||||
func copyTreeLink(ctx context.Context, root *os.Root, name string, links map[string]string, linksByParent map[string][]string, sizes map[string]int64) error {
|
||||
writeLinkText := func(destination, target string) error { // as git writes links it cannot create
|
||||
if err := root.RemoveAll(destination); err != nil {
|
||||
return err
|
||||
}
|
||||
for written := range sizes {
|
||||
if written == destination || strings.HasPrefix(written, destination+string(filepath.Separator)) {
|
||||
delete(sizes, written)
|
||||
}
|
||||
}
|
||||
size, err := writeTreeFile(root, destination, 0o644, strings.NewReader(target))
|
||||
sizes[destination] = size
|
||||
return err
|
||||
}
|
||||
active := map[string]bool{}
|
||||
var copyTarget func(string, string) error
|
||||
copyTarget = func(source, destination string) error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
source, ok := resolveTreeLink(source, links)
|
||||
if !ok {
|
||||
return errTreeLinkTarget
|
||||
}
|
||||
file, err := root.Open(source)
|
||||
if err != nil {
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return errTreeLinkTarget
|
||||
}
|
||||
return err
|
||||
}
|
||||
defer file.Close()
|
||||
info, err := file.Stat()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if info.Mode().IsRegular() {
|
||||
sizes[destination], err = writeTreeFile(root, destination, int64(info.Mode().Perm()), file)
|
||||
return err
|
||||
}
|
||||
if !info.IsDir() {
|
||||
return errTreeLinkTarget
|
||||
}
|
||||
if active[source] {
|
||||
return errTreeLinkCycle
|
||||
}
|
||||
active[source] = true
|
||||
defer delete(active, source)
|
||||
entries, err := file.ReadDir(-1)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
children := map[string]bool{}
|
||||
for _, entry := range entries {
|
||||
children[entry.Name()] = true
|
||||
}
|
||||
for _, child := range linksByParent[source] {
|
||||
children[child] = true
|
||||
}
|
||||
if err := root.MkdirAll(destination, info.Mode().Perm()); err != nil {
|
||||
return err
|
||||
}
|
||||
for child := range children {
|
||||
childSource, childDestination := filepath.Join(source, child), filepath.Join(destination, child)
|
||||
err := copyTarget(childSource, childDestination)
|
||||
if target, isLink := links[childSource]; isLink && errors.Is(err, errTreeLinkTarget) && !errors.Is(err, errTreeLinkCycle) {
|
||||
err = writeLinkText(childDestination, target)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if err := copyTarget(name, name); !errors.Is(err, errTreeLinkTarget) {
|
||||
return err
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
return writeLinkText(name, links[name])
|
||||
}
|
||||
|
||||
var lockTree = AcquireCloneLock // replaced in tests to see installTree wait for a job holding the tree
|
||||
|
||||
// installTree moves temp into place as dir/sha, replacing a tree that failed treeIntact.
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/cgi"
|
||||
@@ -40,6 +41,7 @@ func TestFetchAction(t *testing.T) {
|
||||
require.NoError(t, os.WriteFile(filepath.Join(remote, "run.sh"), []byte("#!/bin/sh\n"), 0o755))
|
||||
require.NoError(t, os.Symlink("action.yml", filepath.Join(remote, "link")))
|
||||
require.NoError(t, gitCmd("-C", remote, "add", "."))
|
||||
require.NoError(t, gitCmd("-C", remote, "update-index", "--chmod=+x", "run.sh"))
|
||||
require.NoError(t, gitCmd("-C", remote, "commit", "-m", "initial"))
|
||||
require.NoError(t, gitCmd("-C", remote, "tag", "v1"))
|
||||
expectedSha := gitRevParse(t, remote, "main")
|
||||
@@ -192,7 +194,9 @@ func TestFetchAction(t *testing.T) {
|
||||
assert.NoDirExists(t, filepath.Join(tree, ".git"))
|
||||
info, err := os.Stat(filepath.Join(tree, "run.sh"))
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, os.FileMode(0o755), info.Mode().Perm())
|
||||
if runtime.GOOS != "windows" {
|
||||
assert.EqualValues(t, 0o755, info.Mode().Perm())
|
||||
}
|
||||
link, err := os.Readlink(filepath.Join(tree, "link"))
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "action.yml", link)
|
||||
@@ -391,6 +395,79 @@ func TestExtractTreeConfinesEntries(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractTreeCopiesDeferredLinks(t *testing.T) {
|
||||
originalSymlink, originalFallback := symlink, symlinkFallback
|
||||
symlink = func(_root *os.Root, _target, _name string) error { return errors.New("symlink privilege unavailable") }
|
||||
symlinkFallback = true
|
||||
t.Cleanup(func() { symlink, symlinkFallback = originalSymlink, originalFallback })
|
||||
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
links [][2]string
|
||||
files []string
|
||||
text bool
|
||||
}{
|
||||
{name: "file chain", links: [][2]string{{"index.js", "entry.js"}, {"entry.js", "dist/index.js"}}, files: []string{"index.js", "entry.js"}},
|
||||
{name: "directory chain", links: [][2]string{{"lib", "bundle"}, {"bundle", "dist"}, {"dist/entry.js", "index.js"}}, files: []string{"lib/index.js", "lib/entry.js", "lib/nested/index.js", "bundle/index.js", "bundle/entry.js", "bundle/nested/index.js"}},
|
||||
{name: "directory in target path", links: [][2]string{{"index.js", "bundle/nested/index.js"}, {"bundle", "dist"}}, files: []string{"index.js", "bundle/nested/index.js"}},
|
||||
{name: "parent after directory link", links: [][2]string{{"index.js", "bundle/../index.js"}, {"bundle", "dist/nested"}}, files: []string{"index.js", "bundle/index.js"}},
|
||||
{name: "outside", links: [][2]string{{"index.js", "../outside.js"}}, text: true},
|
||||
{name: "absolute", links: [][2]string{{"index.js", t.TempDir()}}, text: true},
|
||||
{name: "dangling", links: [][2]string{{"index.js", "missing.js"}}, text: true},
|
||||
{name: "dangling child of directory link", links: [][2]string{{"lib", "dist"}, {"dist/optional", "missing.js"}}, files: []string{"lib/index.js", "lib/nested/index.js"}},
|
||||
{name: "cycle", links: [][2]string{{"index.js", "entry.js"}, {"entry.js", "index.js"}}, text: true},
|
||||
{name: "directory cycle", links: [][2]string{{"dist/loop", "."}}, text: true},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
sha := strings.Repeat("a", 40)
|
||||
var archive bytes.Buffer
|
||||
writer := tar.NewWriter(&archive)
|
||||
require.NoError(t, writer.WriteHeader(&tar.Header{Typeflag: tar.TypeXGlobalHeader, PAXRecords: map[string]string{"comment": sha}}))
|
||||
require.NoError(t, writer.WriteHeader(&tar.Header{Name: "dist", Typeflag: tar.TypeDir, Mode: 0o755}))
|
||||
require.NoError(t, writer.WriteHeader(&tar.Header{Name: "dist/nested", Typeflag: tar.TypeDir, Mode: 0o755}))
|
||||
for _, link := range test.links {
|
||||
require.NoError(t, writer.WriteHeader(&tar.Header{Name: link[0], Typeflag: tar.TypeSymlink, Linkname: link[1]}))
|
||||
}
|
||||
content := "console.log('action');\n"
|
||||
for _, name := range []string{"dist/index.js", "dist/nested/index.js"} {
|
||||
require.NoError(t, writer.WriteHeader(&tar.Header{Name: name, Typeflag: tar.TypeReg, Mode: 0o755, Size: int64(len(content))}))
|
||||
_, err := writer.Write([]byte(content))
|
||||
require.NoError(t, err)
|
||||
}
|
||||
require.NoError(t, writer.Close())
|
||||
|
||||
dir := t.TempDir()
|
||||
_, err := extractTree(t.Context(), &archive, dir, false)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, treeIntact(dir, sha))
|
||||
if test.text {
|
||||
for _, link := range test.links {
|
||||
data, err := os.ReadFile(filepath.Join(dir, sha, link[0]))
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, link[1], string(data))
|
||||
info, err := os.Lstat(filepath.Join(dir, sha, link[0]))
|
||||
require.NoError(t, err)
|
||||
assert.True(t, info.Mode().IsRegular())
|
||||
}
|
||||
}
|
||||
for _, name := range test.files {
|
||||
data, err := os.ReadFile(filepath.Join(dir, sha, name))
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, content, string(data))
|
||||
info, err := os.Lstat(filepath.Join(dir, sha, name))
|
||||
require.NoError(t, err)
|
||||
assert.True(t, info.Mode().IsRegular())
|
||||
if runtime.GOOS != "windows" {
|
||||
assert.Equal(t, os.FileMode(0o755), info.Mode().Perm())
|
||||
}
|
||||
require.NoError(t, os.WriteFile(filepath.Join(dir, sha, name), nil, 0o644))
|
||||
assert.False(t, treeIntact(dir, sha), name)
|
||||
require.NoError(t, os.WriteFile(filepath.Join(dir, sha, name), []byte(content), 0o755))
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestTreeIntactFindsNamesThatAreNotUTF8(t *testing.T) {
|
||||
if runtime.GOOS != "linux" {
|
||||
t.Skip("macOS and Windows store no name that is not UTF-8")
|
||||
@@ -410,3 +487,76 @@ func TestTreeIntactFindsNamesThatAreNotUTF8(t *testing.T) {
|
||||
require.FileExists(t, filepath.Join(dir, sha, "caf\xe9.js"))
|
||||
assert.True(t, treeIntact(dir, sha))
|
||||
}
|
||||
|
||||
func TestExtractTreeCreatesLinksInPlatformOrder(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("requires symlink privileges")
|
||||
}
|
||||
original, originalFallback := symlink, symlinkFallback
|
||||
var created []string
|
||||
symlink = func(root *os.Root, target, name string) error {
|
||||
created = append(created, name)
|
||||
return root.Symlink(target, name)
|
||||
}
|
||||
t.Cleanup(func() { symlink, symlinkFallback = original, originalFallback })
|
||||
|
||||
var archive bytes.Buffer
|
||||
writer := tar.NewWriter(&archive)
|
||||
require.NoError(t, writer.WriteHeader(&tar.Header{Typeflag: tar.TypeXGlobalHeader, PAXRecords: map[string]string{"comment": strings.Repeat("a", 40)}}))
|
||||
for _, entry := range []*tar.Header{
|
||||
{Name: "a", Typeflag: tar.TypeSymlink, Linkname: "b"},
|
||||
{Name: "b", Typeflag: tar.TypeSymlink, Linkname: "dist"},
|
||||
{Name: "dangling", Typeflag: tar.TypeSymlink, Linkname: "missing"},
|
||||
{Name: "dist", Typeflag: tar.TypeDir, Mode: 0o755},
|
||||
} {
|
||||
require.NoError(t, writer.WriteHeader(entry))
|
||||
}
|
||||
require.NoError(t, writer.Close())
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
fallback bool
|
||||
want []string
|
||||
}{
|
||||
{name: "native", want: []string{"a", "b", "dangling"}},
|
||||
{name: "fallback", fallback: true, want: []string{"b", "a", "dangling"}},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
symlinkFallback = test.fallback
|
||||
created = nil
|
||||
_, err := extractTree(t.Context(), bytes.NewReader(archive.Bytes()), t.TempDir(), false)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, test.want, created)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCopyTreeLinkCancellationBetweenEntries(t *testing.T) {
|
||||
root, err := os.OpenRoot(t.TempDir())
|
||||
require.NoError(t, err)
|
||||
defer root.Close()
|
||||
require.NoError(t, root.Mkdir("source", 0o755))
|
||||
for _, name := range []string{"first", "second"} {
|
||||
_, err := writeTreeFile(root, filepath.Join("source", name), 0o644, strings.NewReader("content"))
|
||||
require.NoError(t, err)
|
||||
}
|
||||
sizes := map[string]int64{}
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
defer cancel()
|
||||
observed := &treeCopyContext{Context: ctx, check: func() {
|
||||
if len(sizes) == 1 {
|
||||
cancel()
|
||||
}
|
||||
}}
|
||||
require.ErrorIs(t, copyTreeLink(observed, root, "copy", map[string]string{"copy": "source"}, nil, sizes), context.Canceled)
|
||||
assert.Len(t, sizes, 1)
|
||||
}
|
||||
|
||||
type treeCopyContext struct {
|
||||
context.Context
|
||||
check func()
|
||||
}
|
||||
|
||||
func (ctx *treeCopyContext) Err() error {
|
||||
ctx.check()
|
||||
return ctx.Context.Err()
|
||||
}
|
||||
+37
-8
@@ -5,6 +5,7 @@
|
||||
package git
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
@@ -27,6 +28,7 @@ import (
|
||||
"gitea.com/gitea/runner/act/common"
|
||||
"gitea.com/gitea/runner/act/common/logging"
|
||||
"gitea.com/gitea/runner/internal/pkg/lock"
|
||||
"gitea.com/gitea/runner/internal/pkg/process"
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -183,6 +185,7 @@ type NewGitCloneExecutorInput struct {
|
||||
// For Gitea
|
||||
InsecureSkipTLS bool
|
||||
ClientCertFile, ClientKeyFile string // PEM, absolute as git runs in other directories
|
||||
CAFile string // PEM, absolute, replaces git's CA bundle
|
||||
Transport http.RoundTripper // for FetchAction's archive downloads, nil for http.DefaultTransport
|
||||
}
|
||||
|
||||
@@ -387,6 +390,21 @@ func RemoteConfig(token string, insecureSkipTLS bool) []string {
|
||||
return config
|
||||
}
|
||||
|
||||
// SSLConfig uses OpenSSL on Windows, as Schannel takes no PEM key and ignores sslCAInfo.
|
||||
func SSLConfig(prefix, certFile, keyFile, caFile, executionOS string) []string {
|
||||
var config []string
|
||||
if certFile != "" {
|
||||
config = append(config, prefix+"sslCert="+certFile, prefix+"sslKey="+keyFile)
|
||||
}
|
||||
if caFile != "" {
|
||||
config = append(config, prefix+"sslCAInfo="+caFile)
|
||||
}
|
||||
if config != nil && strings.EqualFold(executionOS, "windows") {
|
||||
config = append(config, prefix+"sslBackend=openssl")
|
||||
}
|
||||
return config
|
||||
}
|
||||
|
||||
// ConfigEnv applies `-c` style entries through the environment, keeping secrets out of argv.
|
||||
func ConfigEnv(config ...string) map[string]string {
|
||||
env := map[string]string{"GIT_CONFIG_COUNT": strconv.Itoa(len(config))}
|
||||
@@ -444,9 +462,7 @@ func runGitWithStdin(ctx context.Context, dir string, input *NewGitCloneExecutor
|
||||
config := []string{"core.autocrlf=false", "core.eol=lf"} // keep action files LF
|
||||
if input != nil {
|
||||
config = append(config, RemoteConfig(input.Token, input.InsecureSkipTLS)...)
|
||||
if input.ClientCertFile != "" {
|
||||
config = append(config, "http.sslCert="+input.ClientCertFile, "http.sslKey="+input.ClientKeyFile)
|
||||
}
|
||||
config = append(config, SSLConfig("http.", input.ClientCertFile, input.ClientKeyFile, input.CAFile, runtime.GOOS)...)
|
||||
}
|
||||
if dir != "" {
|
||||
args = append([]string{"-C", dir}, args...)
|
||||
@@ -461,15 +477,28 @@ func runGitWithStdin(ctx context.Context, dir string, input *NewGitCloneExecutor
|
||||
cmd.Env = append(cmd.Env, key+"="+value)
|
||||
}
|
||||
cmd.Env = append(cmd.Env, "GIT_TERMINAL_PROMPT=0")
|
||||
if runtime.GOOS != "windows" {
|
||||
var treeKill *process.TreeKill
|
||||
if runtime.GOOS == "windows" {
|
||||
treeKill = process.NewTreeKill(cmd) // a cancelled git leaves its remote helper holding the repository otherwise
|
||||
} else {
|
||||
cmd.Cancel = func() error { return cmd.Process.Signal(os.Interrupt) } // lets git remove its lock files
|
||||
}
|
||||
cmd.WaitDelay = time.Second
|
||||
var stdout bytes.Buffer
|
||||
var stderr strings.Builder
|
||||
cmd.Stdout = &stdout
|
||||
cmd.Stderr = &stderr
|
||||
output, err := cmd.Output()
|
||||
if err != nil {
|
||||
return output, fmt.Errorf("git %s: %w: %s", strings.Join(args, " "), err, strings.TrimSpace(stderr.String()))
|
||||
err := cmd.Start()
|
||||
if err == nil {
|
||||
if treeKill != nil {
|
||||
if killer, captureErr := treeKill.Capture(cmd.Process); captureErr == nil {
|
||||
defer killer.Close()
|
||||
}
|
||||
}
|
||||
err = cmd.Wait()
|
||||
}
|
||||
return output, nil
|
||||
if err != nil {
|
||||
return stdout.Bytes(), fmt.Errorf("git %s: %w: %s", strings.Join(args, " "), err, strings.TrimSpace(stderr.String()))
|
||||
}
|
||||
return stdout.Bytes(), nil
|
||||
}
|
||||
@@ -54,6 +54,22 @@ func TestMain(m *testing.M) {
|
||||
os.Exit(code)
|
||||
}
|
||||
|
||||
func TestSSLConfig(t *testing.T) {
|
||||
for _, testcase := range []struct {
|
||||
prefix, certFile, caFile, executionOS string
|
||||
expected []string
|
||||
}{
|
||||
{"http.https://example.com/.", "cert.pem", "ca.pem", "Windows", []string{"http.https://example.com/.sslCert=cert.pem", "http.https://example.com/.sslKey=key.pem", "http.https://example.com/.sslCAInfo=ca.pem", "http.https://example.com/.sslBackend=openssl"}},
|
||||
{"http.", "", "ca.pem", "windows", []string{"http.sslCAInfo=ca.pem", "http.sslBackend=openssl"}},
|
||||
{"http.", "cert.pem", "ca.pem", "Linux", []string{"http.sslCert=cert.pem", "http.sslKey=key.pem", "http.sslCAInfo=ca.pem"}},
|
||||
{"http.", "", "", "Windows", nil},
|
||||
} {
|
||||
t.Run(testcase.prefix+testcase.certFile+testcase.caFile+testcase.executionOS, func(t *testing.T) {
|
||||
assert.Equal(t, testcase.expected, SSLConfig(testcase.prefix, testcase.certFile, "key.pem", testcase.caFile, testcase.executionOS))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestFindGitSlug(t *testing.T) {
|
||||
for _, testcase := range []struct {
|
||||
remoteURL string
|
||||
@@ -185,7 +201,8 @@ func TestGitFindRef(t *testing.T) {
|
||||
func TestGitCloneExecutor(t *testing.T) {
|
||||
remoteDir, workDir := newGitRemote(t)
|
||||
require.NoError(t, os.WriteFile(filepath.Join(workDir, "action.sh"), []byte("#!/bin/sh\necho hi\n"), 0o755))
|
||||
require.NoError(t, gitCmd("-C", workDir, "add", "action.sh"))
|
||||
require.NoError(t, os.WriteFile(filepath.Join(workDir, ".gitattributes"), []byte("*.sh text\n"), 0o644))
|
||||
require.NoError(t, gitCmd("-C", workDir, "add", "action.sh", ".gitattributes"))
|
||||
require.NoError(t, gitCmd("-C", workDir, "commit", "-m", "initial"))
|
||||
require.NoError(t, gitCmd("-C", workDir, "tag", "v2"))
|
||||
require.NoError(t, gitCmd("-C", workDir, "push", "-u", "origin", "main"))
|
||||
@@ -200,7 +217,7 @@ func TestGitCloneExecutor(t *testing.T) {
|
||||
require.NoError(t, gitCmd("-C", workDir, "push", "origin", "main:refs/heads/"+fullSha[:4]))
|
||||
|
||||
hostConfig := filepath.Join(t.TempDir(), "gitconfig")
|
||||
require.NoError(t, os.WriteFile(hostConfig, []byte("[core]\n\tautocrlf = true\n[url \""+remoteDir+"\"]\n\tinsteadOf = https://example.invalid/action\n"), 0o644))
|
||||
require.NoError(t, os.WriteFile(hostConfig, []byte("[core]\n\tautocrlf = true\n\teol = crlf\n[url "+strconv.Quote(remoteDir)+"]\n\tinsteadOf = https://example.invalid/action\n"), 0o644))
|
||||
t.Setenv("GIT_CONFIG_GLOBAL", hostConfig)
|
||||
t.Setenv("GIT_DIR", t.TempDir())
|
||||
|
||||
@@ -676,8 +693,8 @@ func TestNewGitCloneExecutorFetchSendsTokenAndHonoursContext(t *testing.T) {
|
||||
|
||||
select {
|
||||
case <-reached:
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatal("the executor never reached the remote")
|
||||
case err := <-done:
|
||||
t.Fatalf("the executor finished before reaching the remote: %v", err)
|
||||
}
|
||||
assert.Equal(t, "Basic "+base64.StdEncoding.EncodeToString([]byte("token:secret")), authorization)
|
||||
cancel()
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package common
|
||||
|
||||
import "strings"
|
||||
|
||||
// LookupFold finds name in values, ignoring case where the environment does, as GitHub's inputs always do.
|
||||
func LookupFold[V any](values map[string]V, name string, caseInsensitive bool) (V, bool) {
|
||||
if value, ok := values[name]; ok || !caseInsensitive {
|
||||
return value, ok
|
||||
}
|
||||
for key, value := range values {
|
||||
if strings.EqualFold(key, name) {
|
||||
return value, true
|
||||
}
|
||||
}
|
||||
var zero V
|
||||
return zero, false
|
||||
}
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"path"
|
||||
"runtime"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
@@ -20,6 +21,8 @@ import (
|
||||
"github.com/moby/moby/api/types/container"
|
||||
)
|
||||
|
||||
var hostOS = runtime.GOOS // replaced in tests
|
||||
|
||||
// ExitCodeError reports a non-zero process exit code from a container command.
|
||||
type ExitCodeError int
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
// Copyright 2021 The nektos/act Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd))
|
||||
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd || freebsd))
|
||||
|
||||
package container
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
//go:build !windows && !(WITHOUT_DOCKER || !(linux || darwin || netbsd))
|
||||
//go:build !windows && !(WITHOUT_DOCKER || !(linux || darwin || netbsd || freebsd))
|
||||
|
||||
package container
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
// Copyright 2020 The nektos/act Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd))
|
||||
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd || freebsd))
|
||||
|
||||
package container
|
||||
|
||||
@@ -48,6 +48,7 @@ func NewDockerBuildExecutor(input NewDockerBuildExecutorInput) common.Executor {
|
||||
|
||||
options := client.ImageBuildOptions{
|
||||
Tags: []string{input.ImageTag},
|
||||
Labels: map[string]string{"com.gitea.runner.image": input.ImageTag}, // own image ID per tag, so removing one never deletes another job's identical build
|
||||
Remove: true,
|
||||
AuthConfigs: LoadDockerAuthConfigs(ctx),
|
||||
Dockerfile: input.Dockerfile,
|
||||
@@ -125,6 +126,10 @@ func createBuildContext(ctx context.Context, contextDir, relDockerfile string) (
|
||||
}
|
||||
|
||||
func writeBuildContext(ctx context.Context, w io.Writer, contextDir string, includes []string, pm *patternmatcher.PatternMatcher) error {
|
||||
contextDir, err := filepath.EvalSymlinks(contextDir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tw := tar.NewWriter(w)
|
||||
seen := map[string]bool{}
|
||||
hardlinks := map[uint64]string{}
|
||||
@@ -132,8 +137,7 @@ func writeBuildContext(ctx context.Context, w io.Writer, contextDir string, incl
|
||||
for _, include := range includes {
|
||||
include = filepath.FromSlash(include)
|
||||
var parents []parentMatch
|
||||
root := strings.TrimSuffix(contextDir, string(filepath.Separator)) + string(filepath.Separator) + include // unlike filepath.Join, follows a symlinked context dir as docker build does
|
||||
err := filepath.WalkDir(root, func(file string, entry fs.DirEntry, err error) error {
|
||||
err := filepath.WalkDir(filepath.Join(contextDir, include), func(file string, entry fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return leaveOut(ctx, file, err)
|
||||
}
|
||||
@@ -210,7 +214,7 @@ func addBuildContextEntry(ctx context.Context, tw *tar.Writer, file, rel string,
|
||||
return leaveOut(ctx, file, err)
|
||||
}
|
||||
}
|
||||
header, err := tar.FileInfoHeader(noNameLookups{info}, link)
|
||||
header, err := tar.FileInfoHeader(noNameLookups{info}, filepath.ToSlash(link))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd))
|
||||
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd || freebsd))
|
||||
|
||||
package container
|
||||
|
||||
@@ -18,10 +18,12 @@ import (
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestDockerBuildReportsDaemonError(t *testing.T) {
|
||||
func TestDockerBuildLabelsImageWithItsTagAndReportsDaemonError(t *testing.T) {
|
||||
var labels string
|
||||
daemon := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
|
||||
writer.Header().Set("API-Version", "1.47")
|
||||
if strings.HasSuffix(request.URL.Path, "/build") {
|
||||
labels = request.URL.Query().Get("labels")
|
||||
writer.Header().Set("Content-Type", "application/json")
|
||||
writer.WriteHeader(http.StatusInternalServerError)
|
||||
_, _ = io.WriteString(writer, `{"message":"build rejected"}`)
|
||||
@@ -35,6 +37,7 @@ func TestDockerBuildReportsDaemonError(t *testing.T) {
|
||||
require.ErrorContains(t, NewDockerBuildExecutor(NewDockerBuildExecutorInput{
|
||||
ImageTag: "action:test", Dockerfile: "Dockerfile", BuildContext: strings.NewReader(""),
|
||||
})(t.Context()), "build rejected")
|
||||
require.JSONEq(t, `{"com.gitea.runner.image":"action:test"}`, labels)
|
||||
}
|
||||
|
||||
func TestCreateBuildContextFollowsDockerignoreAndSymlinkedRootButKeepsDockerfile(t *testing.T) {
|
||||
@@ -50,7 +53,7 @@ func TestCreateBuildContextFollowsDockerignoreAndSymlinkedRootButKeepsDockerfile
|
||||
|
||||
contexts := []string{dir}
|
||||
if link := filepath.Join(t.TempDir(), "link"); os.Symlink(dir, link) == nil {
|
||||
contexts = append(contexts, link+string(filepath.Separator))
|
||||
contexts = append(contexts, link, link+string(filepath.Separator))
|
||||
}
|
||||
|
||||
for _, contextDir := range contexts {
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
//go:build !WITHOUT_DOCKER && (linux || darwin || netbsd)
|
||||
//go:build !WITHOUT_DOCKER && (linux || darwin || netbsd || freebsd)
|
||||
|
||||
package container
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
//go:build !WITHOUT_DOCKER && (linux || darwin || netbsd)
|
||||
//go:build !WITHOUT_DOCKER && (linux || darwin || netbsd || freebsd)
|
||||
|
||||
package container
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
// Copyright 2022 The nektos/act Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd))
|
||||
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd || freebsd))
|
||||
|
||||
// This file is based on https://github.com/docker/cli/blob/4a63305d74332de5ceba7fcbccbc3cbb7412f5ba/cli/command/container/opts.go with:
|
||||
// * appended with license information
|
||||
@@ -351,7 +351,7 @@ type containerConfig struct {
|
||||
// parse parses the args for the specified command and generates a Config,
|
||||
// a HostConfig and returns them with the specified command.
|
||||
// If the specified args are not valid, it will return an error.
|
||||
func parse(flags *pflag.FlagSet, copts *containerOptions, serverOS string) (*containerConfig, error) { //nolint:gocyclo,unparam // verbatim copy from docker/cli
|
||||
func parse(flags *pflag.FlagSet, copts *containerOptions, serverOS string) (*containerConfig, error) { //nolint:gocyclo // verbatim copy from docker/cli
|
||||
var (
|
||||
attachStdin = copts.attach.Get("stdin")
|
||||
attachStdout = copts.attach.Get("stdout")
|
||||
@@ -724,6 +724,10 @@ func parse(flags *pflag.FlagSet, copts *containerOptions, serverOS string) (*con
|
||||
}, nil
|
||||
}
|
||||
|
||||
func isUserDefinedNetwork(mode container.NetworkMode) bool { // Docker's helper uses the host OS instead of the daemon's.
|
||||
return !mode.IsDefault() && mode != network.NetworkBridge && mode != network.NetworkHost && !mode.IsNone() && !mode.IsContainer()
|
||||
}
|
||||
|
||||
// parseNetworkOpts converts --network advanced options to endpoint-specs, and combines
|
||||
// them with the old --network-alias and --links. If returns an error if conflicting options
|
||||
// are found.
|
||||
@@ -752,7 +756,7 @@ func parseNetworkOpts(copts *containerOptions) (map[string]*network.EndpointSett
|
||||
}
|
||||
|
||||
for i, n := range copts.netMode.Value() {
|
||||
if container.NetworkMode(n.Target).IsUserDefined() {
|
||||
if isUserDefinedNetwork(container.NetworkMode(n.Target)) {
|
||||
hasUserDefined = true
|
||||
} else {
|
||||
hasNonUserDefined = true
|
||||
@@ -814,7 +818,7 @@ func applyContainerOptions(n *opts.NetworkAttachmentOpts, copts *containerOption
|
||||
}
|
||||
// For a user-defined network, "--link" is an endpoint option, it creates an alias. But,
|
||||
// for the default bridge it defines a legacy-link.
|
||||
if container.NetworkMode(n.Target).IsUserDefined() && copts.links.Len() > 0 {
|
||||
if isUserDefinedNetwork(container.NetworkMode(n.Target)) && copts.links.Len() > 0 {
|
||||
n.Links = slices.Clone(copts.links.GetSlice())
|
||||
}
|
||||
if copts.ipv4Address != nil {
|
||||
@@ -840,7 +844,7 @@ func parseNetworkAttachmentOpt(ep opts.NetworkAttachmentOpts) (*network.Endpoint
|
||||
if strings.TrimSpace(ep.Target) == "" {
|
||||
return nil, errors.New("no name set for network")
|
||||
}
|
||||
if !container.NetworkMode(ep.Target).IsUserDefined() {
|
||||
if !isUserDefinedNetwork(container.NetworkMode(ep.Target)) {
|
||||
if len(ep.Aliases) > 0 {
|
||||
return nil, errors.New("network-scoped aliases are only supported for user-defined networks")
|
||||
}
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd))
|
||||
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd || freebsd))
|
||||
|
||||
package container
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
// Copyright 2020 The nektos/act Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd))
|
||||
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd || freebsd))
|
||||
|
||||
package container
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
// Copyright 2020 The nektos/act Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd))
|
||||
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd || freebsd))
|
||||
|
||||
package container
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
// Copyright 2023 The nektos/act Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd))
|
||||
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd || freebsd))
|
||||
|
||||
package container
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
// Copyright 2025 The nektos/act Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd))
|
||||
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd || freebsd))
|
||||
|
||||
package container
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd))
|
||||
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd || freebsd))
|
||||
|
||||
package container
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd))
|
||||
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd || freebsd))
|
||||
|
||||
package container
|
||||
|
||||
@@ -484,6 +484,10 @@ type probeClient struct {
|
||||
containers []container.Summary
|
||||
}
|
||||
|
||||
func (c *probeClient) Info(_ctx context.Context, _opts mobyclient.InfoOptions) (mobyclient.SystemInfoResult, error) {
|
||||
return mobyclient.SystemInfoResult{}, nil
|
||||
}
|
||||
|
||||
func (c *probeClient) Close() error {
|
||||
return nil
|
||||
}
|
||||
@@ -766,6 +770,9 @@ func TestDaemonSeesDir(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestRunnerContainerWorkdir(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("a runner container is Linux")
|
||||
}
|
||||
workdir := t.TempDir()
|
||||
t.Chdir(workdir)
|
||||
volume := container.MountPoint{Type: mount.TypeVolume, Source: "/daemon/_data", Destination: filepath.Dir(workdir)}
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
//go:build !WITHOUT_DOCKER && (linux || darwin || netbsd)
|
||||
//go:build !WITHOUT_DOCKER && (linux || darwin || netbsd || freebsd)
|
||||
|
||||
package container
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
// Copyright 2020 The nektos/act Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd))
|
||||
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd || freebsd))
|
||||
|
||||
package container
|
||||
|
||||
@@ -22,7 +22,7 @@ import (
|
||||
|
||||
// NewDockerPullExecutor function to create a run executor for the container
|
||||
func NewDockerPullExecutor(input NewDockerPullExecutorInput) common.Executor {
|
||||
return func(ctx context.Context) error {
|
||||
return common.Executor(func(ctx context.Context) error {
|
||||
logger := common.Logger(ctx)
|
||||
logger.Debugf("docker pull %v", input.Image)
|
||||
|
||||
@@ -83,7 +83,7 @@ func NewDockerPullExecutor(input NewDockerPullExecutorInput) common.Executor {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("failed to pull image '%s' (%s): %w", imageRef, input.Platform, err)
|
||||
}
|
||||
}).Retry(3)
|
||||
}
|
||||
|
||||
func getImagePullOptions(ctx context.Context, input NewDockerPullExecutorInput) (client.ImagePullOptions, error) {
|
||||
|
||||
+39
-16
@@ -2,7 +2,7 @@
|
||||
// Copyright 2020 The nektos/act Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd))
|
||||
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd || freebsd))
|
||||
|
||||
package container
|
||||
|
||||
@@ -366,6 +366,7 @@ type containerReference struct {
|
||||
id string
|
||||
input *NewContainerInput
|
||||
pullPolicy string
|
||||
daemonOS string
|
||||
UID int
|
||||
GID int
|
||||
// attachDone is closed by the attach() streaming goroutine once it has
|
||||
@@ -375,13 +376,18 @@ type containerReference struct {
|
||||
LinuxContainerEnvironmentExtensions
|
||||
}
|
||||
|
||||
func GetDockerClient(ctx context.Context) (cli client.APIClient, err error) {
|
||||
func GetDockerClient(ctx context.Context) (client.APIClient, error) {
|
||||
cli, _, err := newDockerClient(ctx)
|
||||
return cli, err
|
||||
}
|
||||
|
||||
func newDockerClient(ctx context.Context) (cli client.APIClient, daemonOS string, err error) {
|
||||
dockerHost := os.Getenv("DOCKER_HOST")
|
||||
|
||||
if strings.HasPrefix(dockerHost, "ssh://") {
|
||||
var args []string
|
||||
if args, err = sshArgs(dockerHost); err != nil {
|
||||
return nil, err
|
||||
return nil, "", err
|
||||
}
|
||||
cli, err = client.New(
|
||||
client.WithHost("http://docker.example.com"), // the dialer below ignores the address
|
||||
@@ -391,16 +397,17 @@ func GetDockerClient(ctx context.Context) (cli client.APIClient, err error) {
|
||||
cli, err = client.New(client.FromEnv)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to connect to docker daemon: %w", err)
|
||||
return nil, "", fmt.Errorf("failed to connect to docker daemon: %w", err)
|
||||
}
|
||||
// Best-effort API version negotiation, matching the old client.NegotiateAPIVersion
|
||||
// behaviour. Ping failures here are non-fatal: the connection is exercised again on
|
||||
// the first real call, and the client falls back to the daemon's default API version.
|
||||
if _, err := cli.Ping(ctx, client.PingOptions{NegotiateAPIVersion: true}); err != nil {
|
||||
ping, err := cli.Ping(ctx, client.PingOptions{NegotiateAPIVersion: true})
|
||||
if err != nil {
|
||||
common.Logger(ctx).Warnf("docker daemon ping during version negotiation failed, continuing: %v", err)
|
||||
}
|
||||
|
||||
return cli, nil
|
||||
return cli, ping.OSType, nil
|
||||
}
|
||||
|
||||
func GetHostInfo(ctx context.Context) (system.Info, error) {
|
||||
@@ -440,16 +447,32 @@ func RunnerArch(ctx context.Context) string {
|
||||
return info.Architecture
|
||||
}
|
||||
|
||||
func (cr *containerReference) targetOS(ctx context.Context) string {
|
||||
if cr.daemonOS == "" {
|
||||
cr.daemonOS = "linux"
|
||||
if cr.cli != nil {
|
||||
if result, err := cr.cli.Info(ctx, client.InfoOptions{}); err == nil && result.Info.OSType != "" {
|
||||
cr.daemonOS = result.Info.OSType
|
||||
}
|
||||
}
|
||||
}
|
||||
return cr.daemonOS
|
||||
}
|
||||
|
||||
func (cr *containerReference) markExecutable(ctx context.Context) bool {
|
||||
return hostOS == "windows" && cr.targetOS(ctx) == "linux"
|
||||
}
|
||||
|
||||
func (cr *containerReference) connect() common.Executor {
|
||||
return func(ctx context.Context) error {
|
||||
if cr.cli != nil {
|
||||
return nil
|
||||
}
|
||||
cli, err := GetDockerClient(ctx)
|
||||
cli, daemonOS, err := newDockerClient(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
cr.cli = cli
|
||||
cr.cli, cr.daemonOS = cli, daemonOS
|
||||
return nil
|
||||
}
|
||||
}
|
||||
@@ -459,6 +482,7 @@ func (cr *containerReference) Close() common.Executor {
|
||||
if cr.cli != nil {
|
||||
err := cr.cli.Close()
|
||||
cr.cli = nil
|
||||
cr.daemonOS = ""
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to close client: %w", err)
|
||||
}
|
||||
@@ -603,7 +627,8 @@ func (cr *containerReference) mergeContainerConfigs(ctx context.Context, config
|
||||
copts.privileged = false
|
||||
}
|
||||
|
||||
containerConfig, err := parse(flags, copts, runtime.GOOS)
|
||||
serverOS := cr.targetOS(ctx)
|
||||
containerConfig, err := parse(flags, copts, serverOS)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot process container options: '%s': '%w'", options, err)
|
||||
}
|
||||
@@ -618,7 +643,7 @@ func (cr *containerReference) mergeContainerConfigs(ctx context.Context, config
|
||||
|
||||
// For Gitea, forcing --privileged off is not enough, other options reach the host too
|
||||
if !hostConfig.Privileged {
|
||||
trusted, err := parseOptionsHostConfig(cr.input.RunnerOptions)
|
||||
trusted, err := parseOptionsHostConfig(cr.input.RunnerOptions, serverOS)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -731,9 +756,7 @@ func (cr *containerReference) create(capAdd, capDrop []string) common.Executor {
|
||||
cr.sanitizeConfig(ctx, hostConfig)
|
||||
|
||||
var networkingConfig *network.NetworkingConfig
|
||||
n := hostConfig.NetworkMode
|
||||
// IsUserDefined and IsHost are broken on windows
|
||||
if n.IsUserDefined() && n != "host" && len(input.NetworkAliases) > 0 {
|
||||
if isUserDefinedNetwork(hostConfig.NetworkMode) && len(input.NetworkAliases) > 0 {
|
||||
networkingConfig = &network.NetworkingConfig{
|
||||
EndpointsConfig: map[string]*network.EndpointSettings{
|
||||
input.NetworkMode: {Aliases: input.NetworkAliases},
|
||||
@@ -919,7 +942,7 @@ func (cr *containerReference) copyDir(dstPath, srcPath string, useGitIgnore, ski
|
||||
if actPath := cr.GetActPath(); strings.HasPrefix(dstPath+"/", actPath+"/") {
|
||||
extractionRoot = actPath // keeps tar entries from crossing the image's /var/run symlink
|
||||
}
|
||||
if err := writeDirTar(ctx, tarFile, path.Join("/", strings.TrimPrefix(dstPath, extractionRoot)), srcPath, useGitIgnore, skipGitDir, cr.UID, cr.GID); err != nil {
|
||||
if err := writeDirTar(ctx, tarFile, path.Join("/", strings.TrimPrefix(dstPath, extractionRoot)), srcPath, useGitIgnore, skipGitDir, cr.UID, cr.GID, cr.markExecutable(ctx)); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -1090,12 +1113,12 @@ func resetOption[T any](logger *logging.Logger, option string, field *T, trusted
|
||||
|
||||
// parseOptionsHostConfig parses one options string on its own, to see what it alone asks for.
|
||||
// Even "" goes through the parser, or its empty slices and maps would differ from a real parse.
|
||||
func parseOptionsHostConfig(options string) (*container.HostConfig, error) {
|
||||
func parseOptionsHostConfig(options, serverOS string) (*container.HostConfig, error) {
|
||||
flags, copts, _, err := parseContainerOptions(options)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
containerConfig, err := parse(flags, copts, runtime.GOOS)
|
||||
containerConfig, err := parse(flags, copts, serverOS)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot process container options: '%s': '%w'", options, err)
|
||||
}
|
||||
|
||||
@@ -27,10 +27,12 @@ import (
|
||||
"gitea.com/gitea/runner/act/common/logging/logtest"
|
||||
|
||||
cerrdefs "github.com/containerd/errdefs"
|
||||
"github.com/kballard/go-shellquote"
|
||||
"github.com/moby/moby/api/pkg/stdcopy"
|
||||
"github.com/moby/moby/api/types/container"
|
||||
"github.com/moby/moby/api/types/mount"
|
||||
"github.com/moby/moby/api/types/network"
|
||||
"github.com/moby/moby/api/types/system"
|
||||
mobyclient "github.com/moby/moby/client"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/mock"
|
||||
@@ -65,6 +67,14 @@ func TestDockerEnvironmentUsesContainerPath(t *testing.T) {
|
||||
type mockDockerClient struct {
|
||||
mobyclient.APIClient
|
||||
mock.Mock
|
||||
info system.Info
|
||||
infoErr error
|
||||
infoCalls int
|
||||
}
|
||||
|
||||
func (m *mockDockerClient) Info(_ctx context.Context, _opts mobyclient.InfoOptions) (mobyclient.SystemInfoResult, error) {
|
||||
m.infoCalls++
|
||||
return mobyclient.SystemInfoResult{Info: m.info}, m.infoErr
|
||||
}
|
||||
|
||||
func (m *mockDockerClient) ServerVersion(ctx context.Context, opts mobyclient.ServerVersionOptions) (mobyclient.ServerVersionResult, error) {
|
||||
@@ -689,11 +699,11 @@ func TestMergeContainerConfigsDropsVolumeDriversFromWorkflows(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestMergeContainerConfigsKeepsTheRunnersFilesAndEnvToItself(t *testing.T) {
|
||||
hostFile := filepath.Join(t.TempDir(), "host.env")
|
||||
hostFile := filepath.Join(t.TempDir(), "host file.env")
|
||||
require.NoError(t, os.WriteFile(hostFile, []byte("STOLEN=from-the-host\n"), 0o600))
|
||||
t.Setenv("RUNNER_SECRET", "s3cr3t")
|
||||
|
||||
for _, option := range []string{"--env-file " + hostFile, "--label-file " + hostFile} {
|
||||
for _, option := range []string{shellquote.Join("--env-file", hostFile), shellquote.Join("--label-file", hostFile)} {
|
||||
logger := logging.New(logtest.New())
|
||||
cr := &containerReference{input: &NewContainerInput{NetworkMode: "bridge", WorkflowOptions: option}}
|
||||
|
||||
@@ -933,6 +943,7 @@ func TestContainerInfoFromInspect(t *testing.T) {
|
||||
"/etc/hosts": "",
|
||||
"/etc/hostname": "",
|
||||
}, info.Mounts)
|
||||
assert.Equal(t, "/var/lib/docker/volumes/job/_data/subdir", info.DaemonPath("/workspace/owner/repo/subdir"))
|
||||
})
|
||||
}
|
||||
|
||||
@@ -998,6 +1009,39 @@ func TestDockerCreateKeepsInputEntrypoint(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestDockerCreateNetworkAliases(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
networkMode string
|
||||
wantAliases bool
|
||||
}{
|
||||
{"job-network", true},
|
||||
{"host", false},
|
||||
{"bridge", false},
|
||||
{"default", false},
|
||||
{"none", false},
|
||||
{"container:x", false},
|
||||
} {
|
||||
t.Run(test.networkMode, func(t *testing.T) {
|
||||
cr := &containerReference{
|
||||
input: &NewContainerInput{NetworkMode: test.networkMode, NetworkAliases: []string{"service"}},
|
||||
cli: &probeClient{create: func(_ctx context.Context, opts mobyclient.ContainerCreateOptions) (mobyclient.ContainerCreateResult, error) {
|
||||
assert.Equal(t, container.NetworkMode(test.networkMode), opts.HostConfig.NetworkMode)
|
||||
if test.wantAliases {
|
||||
assert.Equal(t, &network.NetworkingConfig{EndpointsConfig: map[string]*network.EndpointSettings{
|
||||
"job-network": {Aliases: []string{"service"}},
|
||||
}}, opts.NetworkingConfig)
|
||||
} else {
|
||||
assert.Nil(t, opts.NetworkingConfig)
|
||||
}
|
||||
return mobyclient.ContainerCreateResult{ID: "created"}, nil
|
||||
}},
|
||||
}
|
||||
require.NoError(t, cr.create(nil, nil)(t.Context()))
|
||||
assert.Equal(t, "created", cr.id)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestMergeContainerConfigsKeepsNetworkAliasesFromOptions(t *testing.T) {
|
||||
logger := logging.New(logtest.New())
|
||||
cr := &containerReference{input: &NewContainerInput{
|
||||
@@ -1019,3 +1063,92 @@ func TestSupportsContainerImagePlatformDaemonError(t *testing.T) {
|
||||
require.ErrorContains(t, err, "cannot connect to the Docker daemon")
|
||||
assert.False(t, supported)
|
||||
}
|
||||
|
||||
func TestContainerOptionsUseDaemonOS(t *testing.T) {
|
||||
for _, testcase := range []struct {
|
||||
name string
|
||||
serverOS string
|
||||
infoErr error
|
||||
device string
|
||||
expected container.DeviceMapping
|
||||
}{
|
||||
{name: "Linux daemon", serverOS: "linux", device: "/dev/kvm:/dev/kvm:rwm", expected: container.DeviceMapping{PathOnHost: "/dev/kvm", PathInContainer: "/dev/kvm", CgroupPermissions: "rwm"}},
|
||||
{name: "unknown daemon", device: "/dev/kvm:/dev/kvm:rwm", expected: container.DeviceMapping{PathOnHost: "/dev/kvm", PathInContainer: "/dev/kvm", CgroupPermissions: "rwm"}},
|
||||
{name: "unavailable daemon info", infoErr: errors.New("unavailable"), device: "/dev/kvm:/dev/kvm:rwm", expected: container.DeviceMapping{PathOnHost: "/dev/kvm", PathInContainer: "/dev/kvm", CgroupPermissions: "rwm"}},
|
||||
{name: "Windows daemon", serverOS: "windows", device: "class/01234567-89ab-cdef-0123-456789abcdef", expected: container.DeviceMapping{PathOnHost: "class/01234567-89ab-cdef-0123-456789abcdef"}},
|
||||
} {
|
||||
t.Run(testcase.name, func(t *testing.T) {
|
||||
cli := &mockDockerClient{info: system.Info{OSType: testcase.serverOS}, infoErr: testcase.infoErr}
|
||||
cr := &containerReference{cli: cli, input: &NewContainerInput{RunnerOptions: "--device=" + testcase.device, WorkflowOptions: "--device=" + testcase.device, NetworkMode: "bridge"}}
|
||||
for range 2 {
|
||||
hostConfig := &container.HostConfig{}
|
||||
require.NoError(t, cr.mergeContainerConfigs(t.Context(), &container.Config{}, hostConfig))
|
||||
assert.Equal(t, []container.DeviceMapping{testcase.expected}, hostConfig.Devices)
|
||||
}
|
||||
assert.Equal(t, 1, cli.infoCalls)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestConnectTakesDaemonOSFromPing(t *testing.T) {
|
||||
var infos atomic.Int32
|
||||
daemon := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
|
||||
writer.Header().Set("API-Version", "1.47")
|
||||
writer.Header().Set("OSType", "windows")
|
||||
if strings.HasSuffix(request.URL.Path, "/info") {
|
||||
infos.Add(1)
|
||||
}
|
||||
}))
|
||||
t.Cleanup(daemon.Close)
|
||||
t.Setenv("DOCKER_HOST", daemon.URL)
|
||||
cr := &containerReference{input: &NewContainerInput{}}
|
||||
require.NoError(t, cr.connect()(t.Context()))
|
||||
assert.Equal(t, "windows", cr.targetOS(t.Context()))
|
||||
assert.Zero(t, infos.Load())
|
||||
}
|
||||
|
||||
func TestContainerCopyModes(t *testing.T) {
|
||||
for _, testcase := range []struct {
|
||||
hostOS, serverOS string
|
||||
expected int64
|
||||
}{
|
||||
{"windows", "linux", 0o755},
|
||||
{"windows", "windows", 0o666},
|
||||
{"linux", "linux", 0o666},
|
||||
} {
|
||||
t.Run(testcase.hostOS+"-"+testcase.serverOS, func(t *testing.T) {
|
||||
source := t.TempDir()
|
||||
require.NoError(t, os.WriteFile(filepath.Join(source, "run.sh"), []byte("#!/bin/sh\n"), 0o666))
|
||||
require.NoError(t, os.Chmod(filepath.Join(source, "run.sh"), 0o666))
|
||||
file := &FileEntry{Name: "run.sh", Mode: 0o600, Body: "#!/bin/sh\n"}
|
||||
original := hostOS
|
||||
hostOS = testcase.hostOS
|
||||
t.Cleanup(func() { hostOS = original })
|
||||
cli := &mockDockerClient{info: system.Info{OSType: testcase.serverOS}}
|
||||
var modes []int64
|
||||
cli.On("CopyToContainer", mock.Anything, "job", mock.Anything).
|
||||
Run(func(args mock.Arguments) {
|
||||
reader := tar.NewReader(args.Get(2).(mobyclient.CopyToContainerOptions).Content)
|
||||
header, err := reader.Next()
|
||||
require.NoError(t, err)
|
||||
modes = append(modes, header.Mode)
|
||||
assert.Equal(t, "run.sh", filepath.Base(header.Name))
|
||||
content, err := io.ReadAll(reader)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, file.Body, string(content))
|
||||
_, err = reader.Next()
|
||||
require.ErrorIs(t, err, io.EOF)
|
||||
}).Return(mobyclient.CopyToContainerResult{}, nil).Twice()
|
||||
cr := &containerReference{cli: cli, id: "job", input: &NewContainerInput{}}
|
||||
require.NoError(t, cr.copyContent("/workspace", file)(t.Context()))
|
||||
require.NoError(t, cr.copyDir("/workspace", source+string(filepath.Separator), false, true)(t.Context()))
|
||||
assert.Equal(t, []int64{0o600, testcase.expected}, modes)
|
||||
if testcase.hostOS == "windows" {
|
||||
assert.Equal(t, 1, cli.infoCalls)
|
||||
} else {
|
||||
assert.Zero(t, cli.infoCalls)
|
||||
}
|
||||
cli.AssertExpectations(t)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,7 +1,7 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd))
|
||||
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd || freebsd))
|
||||
|
||||
package container
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd))
|
||||
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd || freebsd))
|
||||
|
||||
package container
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
//go:build !WITHOUT_DOCKER && (darwin || netbsd)
|
||||
//go:build !WITHOUT_DOCKER && (darwin || netbsd || freebsd)
|
||||
|
||||
package container
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
// Copyright 2023 The nektos/act Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
//go:build WITHOUT_DOCKER || !(linux || darwin || windows || netbsd)
|
||||
//go:build WITHOUT_DOCKER || !(linux || darwin || windows || netbsd || freebsd)
|
||||
|
||||
package container
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
// Copyright 2020 The nektos/act Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd))
|
||||
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd || freebsd))
|
||||
|
||||
package container
|
||||
|
||||
|
||||
@@ -4,7 +4,18 @@
|
||||
|
||||
package container
|
||||
|
||||
import "context"
|
||||
import (
|
||||
"context"
|
||||
"path"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func JoinPath(elem ...string) string { // path.Join would collapse the leading // of a UNC act path on a Windows host
|
||||
if len(elem) > 0 && strings.HasPrefix(elem[0], "//") {
|
||||
return "/" + path.Join(elem...)
|
||||
}
|
||||
return path.Join(elem...)
|
||||
}
|
||||
|
||||
type ExecutionsEnvironment interface {
|
||||
Container
|
||||
|
||||
@@ -40,7 +40,8 @@ type HostEnvironment struct {
|
||||
// when Workdir points at a caller-owned checkout (e.g. `act` local mode).
|
||||
CleanWorkdir bool
|
||||
ActPath string
|
||||
CleanUp func()
|
||||
Root string
|
||||
TrackingID string // RUNNER_TRACKING_ID of the job, a step with another value keeps its leftovers
|
||||
StdOut io.Writer
|
||||
AllocatePTY bool // allocate a pseudo-TTY for each step's process
|
||||
|
||||
@@ -56,7 +57,7 @@ func (e *HostEnvironment) processGroup(ctx context.Context) *process.Group {
|
||||
e.procGroupOnce.Do(func() {
|
||||
group, err := process.NewGroup()
|
||||
if err != nil {
|
||||
common.Logger(ctx).Warnf("could not create the job's process group; processes a step leaves behind can only be reclaimed by the workspace scan: %v", err)
|
||||
common.Logger(ctx).Warnf("could not create the job's process group, processes a step leaves behind may outlive the job: %v", err)
|
||||
return
|
||||
}
|
||||
e.procGroup.Store(group)
|
||||
@@ -261,6 +262,7 @@ func (e *HostEnvironment) exec(ctx context.Context, command []string, cmdline st
|
||||
// Flush any buffered, not-yet-newline-terminated trailing line, as the docker backend
|
||||
// does in waitForCommand, so the final line of a command's output is not lost.
|
||||
defer common.FlushWriter(e.StdOut)
|
||||
process.UTF8Console()
|
||||
|
||||
f, err := lookupPathHost(command[0], env, e.StdOut)
|
||||
if err != nil {
|
||||
@@ -270,7 +272,12 @@ func (e *HostEnvironment) exec(ctx context.Context, command []string, cmdline st
|
||||
cmd.Path = f
|
||||
cmd.Args = command
|
||||
cmd.Stdout = e.StdOut
|
||||
cmd.Env = getEnvListFromMap(env)
|
||||
trackingID := e.TrackingID
|
||||
if value, ok := common.LookupFold(env, "RUNNER_TRACKING_ID", e.IsEnvironmentCaseInsensitive()); ok {
|
||||
trackingID = value
|
||||
}
|
||||
tracked := trackingID == e.TrackingID
|
||||
cmd.Env = append(getEnvListFromMap(env), "RUNNER_TRACKING_ID="+trackingID)
|
||||
cmd.Stderr = e.StdOut
|
||||
cmd.Dir = wd
|
||||
cmd.SysProcAttr = process.SysProcAttr(cmdline, false)
|
||||
@@ -278,6 +285,7 @@ func (e *HostEnvironment) exec(ctx context.Context, command []string, cmdline st
|
||||
// Kills the step's whole tree on cancellation and bounds the post-exit I/O
|
||||
// wait, so an orphan holding cmd's stdout pipe cannot hang cmd.Wait().
|
||||
treeKill := process.NewTreeKill(cmd)
|
||||
treeKill.KeepLeftovers = !tracked
|
||||
|
||||
var ppty *os.File
|
||||
var tty *os.File
|
||||
@@ -311,10 +319,10 @@ func (e *HostEnvironment) exec(ctx context.Context, command []string, cmdline st
|
||||
if err := cmd.Start(); err != nil {
|
||||
return err
|
||||
}
|
||||
// Assign before the step's Killer so the step's job nests inside the group's;
|
||||
// cancellation still scopes to this step's tree.
|
||||
if err := e.processGroup(ctx).Assign(cmd.Process); err != nil {
|
||||
common.Logger(ctx).Warnf("could not assign the step's process to the job's process group; a process it leaves behind may outlive the job: %v", err)
|
||||
if tracked {
|
||||
if err := e.processGroup(ctx).Assign(cmd.Process); err != nil {
|
||||
common.Logger(ctx).Warnf("could not assign the step's process to the job's process group, a process it leaves behind may outlive the job: %v", err)
|
||||
}
|
||||
}
|
||||
if k, kerr := treeKill.Capture(cmd.Process); kerr != nil {
|
||||
common.Logger(ctx).Warnf("process tree kill setup failed, falling back to single-process kill: %v", kerr)
|
||||
@@ -365,9 +373,7 @@ func (e *HostEnvironment) UpdateFromEnv(srcPath string, env *map[string]string)
|
||||
// removeAll is a var so tests can substitute a blocking stub.
|
||||
var removeAll = os.RemoveAll
|
||||
|
||||
// removeAllWithContext returns once the delete finishes or ctx is cancelled. On
|
||||
// cancellation the goroutine leaks: a delete inside a syscall cannot be
|
||||
// interrupted (see runWithTimeout).
|
||||
// Returns on cancel while an uninterruptible delete keeps running, freeing the runner's capacity slot.
|
||||
func removeAllWithContext(ctx context.Context, path string) error {
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- removeAll(path) }()
|
||||
@@ -409,129 +415,22 @@ func removePathWithRetry(ctx context.Context, path string) error {
|
||||
return lastErr
|
||||
}
|
||||
|
||||
// buildWindowsWorkspaceKillScript builds a PowerShell command that taskkills
|
||||
// every process tree whose ExecutablePath or CommandLine references one of the
|
||||
// given workspace dirs, releasing file handles for cleanup. Win32_Process
|
||||
// exposes both fields (Get-Process doesn't, wmic is deprecated); matching is on
|
||||
// the dir+separator prefix via ordinal String methods, so a name-prefix sibling
|
||||
// (job1 vs job10) is spared and path metacharacters stay literal.
|
||||
func buildWindowsWorkspaceKillScript(dirs []string) string {
|
||||
quoted := make([]string, len(dirs))
|
||||
for i, d := range dirs {
|
||||
// Single-quoted PowerShell literal; escape ' by doubling it.
|
||||
quoted[i] = "'" + strings.ReplaceAll(d, "'", "''") + "'"
|
||||
}
|
||||
|
||||
return `$paths = @(` + strings.Join(quoted, ",") + `)
|
||||
$selfPid = $PID
|
||||
Get-CimInstance Win32_Process -ErrorAction SilentlyContinue | Where-Object {
|
||||
if ($_.ProcessId -eq $selfPid) { return $false }
|
||||
foreach ($p in $paths) {
|
||||
$prefix = $p + '\'
|
||||
if ($_.ExecutablePath -and $_.ExecutablePath.StartsWith($prefix, [System.StringComparison]::OrdinalIgnoreCase)) { return $true }
|
||||
if ($_.CommandLine -and $_.CommandLine.IndexOf($prefix, [System.StringComparison]::OrdinalIgnoreCase) -ge 0) { return $true }
|
||||
}
|
||||
return $false
|
||||
} | ForEach-Object {
|
||||
& taskkill.exe /PID $_.ProcessId /T /F 2>$null | Out-Null
|
||||
}
|
||||
`
|
||||
}
|
||||
|
||||
func (e *HostEnvironment) terminateRunningProcesses(ctx context.Context) {
|
||||
if runtime.GOOS != "windows" {
|
||||
return
|
||||
}
|
||||
|
||||
// Detached: exec.CommandContext won't start on a cancelled ctx, and a
|
||||
// server cancel has already cancelled the parent ctx.
|
||||
killCtx, killCancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer killCancel()
|
||||
|
||||
logger := common.Logger(ctx)
|
||||
|
||||
// Dirs we own; a process referencing one is a leftover. ToolCache is shared
|
||||
// across jobs, and Workdir may be a caller-owned checkout.
|
||||
owned := []string{e.Path, e.TmpDir}
|
||||
if e.CleanWorkdir {
|
||||
owned = append(owned, e.Workdir)
|
||||
}
|
||||
dirs := make([]string, 0, len(owned))
|
||||
for _, d := range owned {
|
||||
if d == "" {
|
||||
continue
|
||||
}
|
||||
abs, err := filepath.Abs(d)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
dirs = append(dirs, abs)
|
||||
}
|
||||
if len(dirs) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
script := buildWindowsWorkspaceKillScript(dirs)
|
||||
|
||||
cmd := exec.CommandContext(killCtx, "powershell.exe", "-NoProfile", "-NonInteractive", "-Command", script)
|
||||
out, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
logger.Debugf("workspace process-tree kill via PowerShell failed: %v output=%s", err, strings.TrimSpace(string(out)))
|
||||
}
|
||||
|
||||
// Win32_Process exposes no working directory, so the scan above misses a
|
||||
// process that merely runs in a workspace dir while pinning a handle on it.
|
||||
if killed, err := process.KillProcessesWithCWDUnder(killCtx, dirs); err != nil {
|
||||
logger.Debugf("workspace process kill by working directory reported errors: %v", err)
|
||||
} else if killed > 0 {
|
||||
logger.Debugf("terminated %d leftover process(es) by workspace working directory", killed)
|
||||
}
|
||||
}
|
||||
|
||||
// hostCleanupTimeout bounds each teardown phase so one stalled delete cannot
|
||||
// wedge the runner slot. A var so tests can shrink it.
|
||||
var hostCleanupTimeout = 30 * time.Second
|
||||
|
||||
// runWithTimeout returns context.DeadlineExceeded once timeout elapses, leaking
|
||||
// the goroutine: a delete blocked in a syscall (AV filter driver, dead network
|
||||
// mount) cannot be interrupted, and leaking scratch state beats losing the
|
||||
// runner's capacity slot forever. The idle stale-dir sweep reclaims it later.
|
||||
func runWithTimeout(fn func(), timeout time.Duration) error {
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
defer close(done)
|
||||
fn()
|
||||
}()
|
||||
timer := time.NewTimer(timeout)
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case <-done:
|
||||
return nil
|
||||
case <-timer.C:
|
||||
return context.DeadlineExceeded
|
||||
}
|
||||
}
|
||||
|
||||
func (e *HostEnvironment) Remove() common.Executor {
|
||||
return func(ctx context.Context) error {
|
||||
logger := common.Logger(ctx)
|
||||
|
||||
// End lingering processes before removing the workspace; on Windows their
|
||||
// file locks block cleanup. Closing the group is deterministic, the scan a net.
|
||||
// End lingering processes before removing the workspace, on Windows their file locks block cleanup.
|
||||
if err := e.procGroup.Load().Close(); err != nil {
|
||||
logger.Debugf("closing the job's process group failed: %v", err)
|
||||
}
|
||||
e.terminateRunningProcesses(ctx)
|
||||
|
||||
// Removes per-job misc state only, never the toolcache root. Bounded because
|
||||
// CleanUp is a caller-supplied, typically unbounded os.RemoveAll.
|
||||
if e.CleanUp != nil {
|
||||
logger.Debugf("running host environment cleanup callback")
|
||||
if err := runWithTimeout(e.CleanUp, hostCleanupTimeout); err != nil {
|
||||
logger.Warnf("host environment cleanup did not finish within %s; continuing job completion, scratch state may be leaked and is reclaimed by the idle stale-dir sweep", hostCleanupTimeout)
|
||||
} else {
|
||||
logger.Debugf("host environment cleanup callback finished")
|
||||
}
|
||||
if killed, err := process.KillTrackedProcesses(e.TrackingID); err != nil {
|
||||
logger.Debugf("killing the job's leftover processes reported errors: %v", err)
|
||||
} else if killed > 0 {
|
||||
logger.Debugf("killed %d leftover process(es) of the job", killed)
|
||||
}
|
||||
|
||||
// Detach: a cancelled ctx would skip removePathWithRetry's retries,
|
||||
@@ -540,8 +439,8 @@ func (e *HostEnvironment) Remove() common.Executor {
|
||||
defer rmCancel()
|
||||
|
||||
var errs []error
|
||||
if err := removePathWithRetry(rmCtx, e.Path); err != nil {
|
||||
logger.Warnf("failed to remove host misc state %s: %v", e.Path, err)
|
||||
if err := removePathWithRetry(rmCtx, cmp.Or(e.Root, e.Path)); err != nil {
|
||||
logger.Warnf("failed to remove host misc state %s: %v", cmp.Or(e.Root, e.Path), err)
|
||||
errs = append(errs, err)
|
||||
}
|
||||
if e.CleanWorkdir {
|
||||
@@ -613,6 +512,7 @@ func goOsToActionOs(os string) string {
|
||||
"linux": "Linux",
|
||||
"darwin": "macOS",
|
||||
"windows": "Windows",
|
||||
"freebsd": "FreeBSD",
|
||||
}[os], os)
|
||||
}
|
||||
|
||||
|
||||
@@ -27,7 +27,7 @@ import (
|
||||
var _ ExecutionsEnvironment = &HostEnvironment{}
|
||||
|
||||
func TestActionPlatformNames(t *testing.T) {
|
||||
assert.Equal(t, []string{"Linux", "macOS", "Windows"}, []string{goOsToActionOs("linux"), goOsToActionOs("darwin"), goOsToActionOs("windows")})
|
||||
assert.Equal(t, []string{"Linux", "macOS", "Windows", "FreeBSD"}, []string{goOsToActionOs("linux"), goOsToActionOs("darwin"), goOsToActionOs("windows"), goOsToActionOs("freebsd")})
|
||||
assert.Equal(t, []string{"X86", "X64", "ARM", "ARM64"}, []string{goArchToActionArch("386"), goArchToActionArch("amd64"), goArchToActionArch("arm"), goArchToActionArch("arm64")})
|
||||
}
|
||||
|
||||
@@ -161,10 +161,12 @@ func TestHostEnvironmentRemoveWorkdir(t *testing.T) {
|
||||
Path: filepath.Join(miscRoot, "hostexecutor"),
|
||||
Workdir: workdir,
|
||||
CleanWorkdir: owned,
|
||||
CleanUp: func() { _ = os.RemoveAll(miscRoot) },
|
||||
Root: miscRoot,
|
||||
}
|
||||
require.NoError(t, os.MkdirAll(e.Path, 0o700))
|
||||
require.NoError(t, os.MkdirAll(filepath.Join(miscRoot, "tmp"), 0o700))
|
||||
require.NoError(t, e.Remove()(t.Context()))
|
||||
assert.NoDirExists(t, miscRoot)
|
||||
_, err := os.Stat(workdir)
|
||||
if owned {
|
||||
assert.ErrorIs(t, err, os.ErrNotExist)
|
||||
@@ -201,87 +203,38 @@ func TestRemoveAllWithContextDoesNotHangOnStuckDelete(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestHostEnvironmentRemoveDoesNotHang(t *testing.T) {
|
||||
for _, phase := range []string{"CleanUp callback", "path removal"} {
|
||||
t.Run(phase, func(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
origTimeout := hostCleanupTimeout
|
||||
hostCleanupTimeout = 100 * time.Millisecond
|
||||
t.Cleanup(func() { hostCleanupTimeout = origTimeout })
|
||||
release, stubDone := make(chan struct{}), make(chan struct{})
|
||||
block := func() {
|
||||
defer close(stubDone)
|
||||
<-release
|
||||
}
|
||||
e := &HostEnvironment{Path: filepath.Join(t.TempDir(), "misc", "hostexecutor"), StdOut: os.Stdout}
|
||||
if phase == "CleanUp callback" {
|
||||
e.CleanUp = block
|
||||
} else {
|
||||
origRemoveAll := removeAll
|
||||
removeAll = func(string) error {
|
||||
block()
|
||||
return nil
|
||||
}
|
||||
t.Cleanup(func() { removeAll = origRemoveAll })
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
close(release)
|
||||
<-stubDone
|
||||
})
|
||||
require.NoError(t, os.MkdirAll(e.Path, 0o700))
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- e.Remove()(common.WithLogger(t.Context(), logging.Default())) }()
|
||||
select {
|
||||
case err := <-done:
|
||||
require.NoError(t, err)
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatalf("Remove() hung on a stuck %s", phase)
|
||||
}
|
||||
})
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
origTimeout, origRemoveAll := hostCleanupTimeout, removeAll
|
||||
hostCleanupTimeout = 100 * time.Millisecond
|
||||
release, stubDone := make(chan struct{}), make(chan struct{})
|
||||
removeAll = func(string) error {
|
||||
defer close(stubDone)
|
||||
<-release
|
||||
return nil
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
close(release)
|
||||
<-stubDone
|
||||
hostCleanupTimeout, removeAll = origTimeout, origRemoveAll
|
||||
})
|
||||
e := &HostEnvironment{Path: filepath.Join(t.TempDir(), "misc", "hostexecutor"), StdOut: os.Stdout}
|
||||
require.NoError(t, os.MkdirAll(e.Path, 0o700))
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- e.Remove()(common.WithLogger(t.Context(), logging.Default())) }()
|
||||
select {
|
||||
case err := <-done:
|
||||
require.NoError(t, err)
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatal("Remove() hung on a stuck path removal")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestJoinPath(t *testing.T) {
|
||||
assert.Equal(t, "//server/share/act/workflow/envs.txt", JoinPath("//server/share/act", "workflow/envs.txt"))
|
||||
assert.Equal(t, "/var/run/act/workflow/envs.txt", JoinPath("/var/run/act/", "workflow", "envs.txt"))
|
||||
for _, root := range []string{"//server/share/act", "/var/run/act", "C:/act"} {
|
||||
assert.Equal(t, root+"/actions/key/main.go.out", JoinPath(root+"/actions/key", "main.go.out"))
|
||||
assert.Equal(t, root+"/workflow", JoinPath(root+"/workflow/outputcmd.txt", ".."))
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildWindowsWorkspaceKillScript(t *testing.T) {
|
||||
t.Run("single dir", func(t *testing.T) {
|
||||
s := buildWindowsWorkspaceKillScript([]string{`C:\workspace\job1`})
|
||||
assert.Contains(t, s, `$paths = @('C:\workspace\job1')`)
|
||||
assert.Contains(t, s, "$selfPid = $PID")
|
||||
assert.Contains(t, s, "$_.ProcessId -eq $selfPid")
|
||||
assert.Contains(t, s, `$prefix = $p + '\'`)
|
||||
assert.Contains(t, s, "$_.ExecutablePath.StartsWith($prefix")
|
||||
assert.Contains(t, s, "$_.CommandLine.IndexOf($prefix")
|
||||
assert.Contains(t, s, "taskkill.exe /PID $_.ProcessId /T /F")
|
||||
})
|
||||
|
||||
t.Run("multiple dirs comma-separated", func(t *testing.T) {
|
||||
s := buildWindowsWorkspaceKillScript([]string{
|
||||
`C:\work\path`,
|
||||
`C:\work\workdir`,
|
||||
`C:\Users\runner\AppData\Local\Temp\job-42`,
|
||||
})
|
||||
assert.Contains(t, s, `'C:\work\path'`)
|
||||
assert.Contains(t, s, `'C:\work\workdir'`)
|
||||
assert.Contains(t, s, `'C:\Users\runner\AppData\Local\Temp\job-42'`)
|
||||
assert.Contains(t, s, `'C:\work\path','C:\work\workdir',`)
|
||||
})
|
||||
|
||||
t.Run("path with single quote is escaped", func(t *testing.T) {
|
||||
s := buildWindowsWorkspaceKillScript([]string{`C:\work\it's\path`})
|
||||
assert.Contains(t, s, `'C:\work\it''s\path'`)
|
||||
assert.NotContains(t, s, `'C:\work\it's\path'`)
|
||||
})
|
||||
|
||||
t.Run("path with wildcard metacharacters is matched literally", func(t *testing.T) {
|
||||
s := buildWindowsWorkspaceKillScript([]string{`C:\work\[job]?1`})
|
||||
assert.Contains(t, s, `'C:\work\[job]?1'`)
|
||||
assert.NotContains(t, s, "-like")
|
||||
assert.Contains(t, s, "StartsWith")
|
||||
assert.Contains(t, s, "IndexOf")
|
||||
})
|
||||
|
||||
t.Run("empty dir list still produces a valid script", func(t *testing.T) {
|
||||
s := buildWindowsWorkspaceKillScript(nil)
|
||||
assert.Contains(t, s, "$paths = @()")
|
||||
assert.Contains(t, s, "Get-CimInstance Win32_Process")
|
||||
})
|
||||
}
|
||||
Loaded 50 of 144 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user