mirror of
https://gitea.com/gitea/runner.git
synced 2026-11-16 12:53:42 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
6e42621b4a | ||
|
|
228effab64 | ||
|
|
a46d3f3eda | ||
|
|
2e72cf902a | ||
|
|
f4741df90d | ||
|
|
261ab374ce | ||
|
|
ca8e9c3c64 | ||
|
|
d8d4a4657d | ||
|
|
5fd2f82735 | ||
|
|
c6567296a0 | ||
|
|
e3607b576e | ||
|
|
ffd80101a3 | ||
|
|
b712c91c2d | ||
|
|
d77e216674 | ||
|
|
a76ccd7020 |
No files matched your search
@@ -21,7 +21,7 @@ jobs:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
- uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7
|
||||
with:
|
||||
node-version: 24
|
||||
node-version: 26
|
||||
- run: make lint-pr-title
|
||||
env:
|
||||
PR_TITLE: ${{ github.event.pull_request.title }}
|
||||
@@ -25,7 +25,7 @@ jobs:
|
||||
# daemon retains them between runs, and their digest pins skip the registry when it has them.
|
||||
- name: pre-pull test images
|
||||
env:
|
||||
TEST_JOB_IMAGE: node:24-bookworm-slim@sha256:55fb5f5a527ea2c5c4a95f963e48ef90aa38c7b346054fdc8ee9957a2b55d205 # renovate: datasource=docker
|
||||
TEST_JOB_IMAGE: node:26-bookworm-slim@sha256:bc0a2afd3b74ce12222fc8bab99271e66257f021078f5f4a5d687a77d280c1d9 # renovate: datasource=docker
|
||||
TEST_SERVICE_IMAGE: nginx:alpine@sha256:a3554c0d0b48177b1413c26e01f695334428020d9df01243fbae3e11cfd17417 # renovate: datasource=docker
|
||||
run: |
|
||||
for image in "$TEST_JOB_IMAGE" "$TEST_SERVICE_IMAGE"; do
|
||||
|
||||
+10
-2
@@ -57,7 +57,15 @@ Each lives in `internal/pkg/action/<name>` and is registered in `builtinActions`
|
||||
|
||||
`make test-e2e` runs the runner against `E2E_GITEA_IMAGE`. It defaults to the nightly image.
|
||||
It requires Docker and is excluded from `make test`. CI runs stable and nightly variants in
|
||||
parallel.
|
||||
parallel. Outside a container it runs `go test` in a Linux container on its own network, as CI does,
|
||||
so both take the same paths. `E2E_ARGS` passes extra flags, e.g. `make test-e2e E2E_ARGS='-run TestCompatibility/dispatch'`.
|
||||
|
||||
The suite shares one Gitea and regular runner. Cache and ephemeral scenarios use isolated
|
||||
The `cli_*` scenarios build the binary and drive `register` and `daemon` as a user would, including `--once`,
|
||||
`--labels` and shutdown with a running job. `failure_paths` covers failed, skipped and tolerated jobs, `job_timeout`
|
||||
a job killed at the runner's deadline. `label_routing` and `capacity` cover scheduling, `reconnect` drops the
|
||||
connection to Gitea through a proxy, and `host_mode` and `reusable_workflow` cover host labels and `workflow_call`.
|
||||
`dispatch`, `failure_paths` and `reusable_workflow` run in every mode: through the daemon and `exec`, each on
|
||||
Docker and Kubernetes.
|
||||
|
||||
The suite shares one Gitea, regular runner and k3s cluster. Cache and ephemeral scenarios use isolated
|
||||
repository runners. A run path is `<workflow>@<ref>` and a log row is `<timestamp>Z <payload>`.
|
||||
+2
-2
@@ -19,7 +19,7 @@ RUN --mount=type=cache,target=/go/pkg/mod \
|
||||
### DIND VARIANT
|
||||
#
|
||||
#
|
||||
FROM docker:29.9.1-dind AS dind
|
||||
FROM docker:29.9.2-dind AS dind
|
||||
|
||||
ARG VERSION=dev
|
||||
|
||||
@@ -41,7 +41,7 @@ ENTRYPOINT ["s6-svscan","/etc/s6"]
|
||||
### DIND-ROOTLESS VARIANT
|
||||
#
|
||||
#
|
||||
FROM docker:29.9.1-dind-rootless AS dind-rootless
|
||||
FROM docker:29.9.2-dind-rootless AS dind-rootless
|
||||
|
||||
ARG VERSION=dev
|
||||
|
||||
|
||||
@@ -4,7 +4,7 @@ GO ?= go
|
||||
HAS_GO = $(shell hash $(GO) > /dev/null 2>&1 && echo "GO" || echo "NOGO" )
|
||||
|
||||
PODMAN_TEST_IMAGE ?= quay.io/containers/podman:v5.8.9-immutable@sha256:87f42f37fa85b421664c30b8f5f9cfc23920057f5e5fcadf0b6f68d9edc6d415 # renovate: datasource=docker
|
||||
E2E_JOB_IMAGE ?= docker.gitea.com/runner-images:ubuntu-latest@sha256:8856b200466e9a5af9cc0d4de4ca96f73547833bcdddf9a567189ff772c4d372 # renovate: datasource=docker
|
||||
E2E_JOB_IMAGE ?= docker.gitea.com/runner-images:ubuntu-latest@sha256:657378e5cfd898e36f820459c60ef8e69be8fcda9fd354e4381a8b654d091bbc # renovate: datasource=docker
|
||||
SERVICE_IMAGE ?= nginx:1.31.7-alpine@sha256:892bb2152464f0c5d8972593a5f5afaf8054552580b699d95f8ff4af08c2d33d # renovate: datasource=docker
|
||||
E2E_GITEA_IMAGE ?= gitea/gitea:main-nightly
|
||||
E2E_K3S_IMAGE ?= rancher/k3s:v1.37.1-k3s1@sha256:ca7f37d993d82ef0dcdcfecb2e0e2618ea541dbaffc620c8cedebe01a82acd0d # renovate: datasource=docker
|
||||
@@ -154,7 +154,7 @@ E2E_CONCURRENCY ?= 8
|
||||
|
||||
.PHONY: test-e2e
|
||||
test-e2e: ## run Gitea compatibility tests against E2E_GITEA_IMAGE
|
||||
@E2E_CONCURRENCY=$(E2E_CONCURRENCY) E2E_GITEA_IMAGE=$(E2E_GITEA_IMAGE) E2E_K3S_IMAGE=$(E2E_K3S_IMAGE) E2E_JOB_IMAGE=$(E2E_JOB_IMAGE) GO=$(GO) SERVICE_IMAGE=$(SERVICE_IMAGE) ./tools/test-e2e.sh
|
||||
@E2E_CONCURRENCY=$(E2E_CONCURRENCY) E2E_GITEA_IMAGE=$(E2E_GITEA_IMAGE) E2E_K3S_IMAGE=$(E2E_K3S_IMAGE) E2E_JOB_IMAGE=$(E2E_JOB_IMAGE) GO=$(GO) SERVICE_IMAGE=$(SERVICE_IMAGE) ./tools/test-e2e.sh $(E2E_ARGS)
|
||||
|
||||
.PHONY: install
|
||||
install: $(GOFILES) ## install the runner binary via `go install`
|
||||
|
||||
@@ -303,6 +303,8 @@ Use `uses: builtin:checkout` to check out a repository without downloading `acti
|
||||
|
||||
Each runner starts its own cache server, so runners do not share cached entries. A runner in a container on the jobs' Docker daemon attaches itself to each job network, so jobs reach its cache without extra setup.
|
||||
|
||||
As on GitHub, a job restores caches saved for its own ref, its pull request's base branch and the default branch, so a cache saved by a pull request is only restored by that pull request.
|
||||
|
||||
**Sharing a cache between runners**
|
||||
|
||||
Run one `gitea-runner cache-server` and point every runner at it, using the same secret everywhere, for example from `openssl rand -hex 32`:
|
||||
|
||||
@@ -6,7 +6,5 @@
|
||||
//
|
||||
// Inspired by https://github.com/sp-ricard-valverde/github-act-cache-server
|
||||
//
|
||||
// TODO: Authorization
|
||||
// TODO: Restrictions for accessing a cache, see https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows#restrictions-for-accessing-a-cache
|
||||
// TODO: Force deleting cache entries, see https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows#force-deleting-cache-entries
|
||||
package artifactcache
|
||||
@@ -59,6 +59,10 @@ type credKey struct{}
|
||||
type JobCredential struct {
|
||||
Repo string `json:"repo"`
|
||||
|
||||
Ref string `json:"ref"`
|
||||
BaseRef string `json:"base_ref"`
|
||||
DefaultRef string `json:"default_ref"`
|
||||
|
||||
// Results is the instance whose artifact service this server forwards for the job, and
|
||||
// InsecureTLS how the runner reaches it; see results.go.
|
||||
Results string `json:"-"`
|
||||
@@ -68,6 +72,10 @@ type JobCredential struct {
|
||||
PublicURL string `json:"public_url"`
|
||||
}
|
||||
|
||||
func (cred JobCredential) owns(cache *Cache) bool {
|
||||
return cache.Repo == cred.Repo && cache.Ref == cred.Ref
|
||||
}
|
||||
|
||||
// credEntry holds a registered job's credential along with an active
|
||||
// registration count. RegisterJob is reference-counted so that if two tasks
|
||||
// briefly share an ACTIONS_RUNTIME_TOKEN — e.g. a runner that retries a task
|
||||
@@ -653,7 +661,7 @@ func (h *Handler) find(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
cache, err := h.lookupCache(db, cred.Repo, keys, version)
|
||||
cache, err := h.lookupCache(db, cred, keys, version)
|
||||
if err != nil {
|
||||
h.responseJSON(w, r, 500, err)
|
||||
return
|
||||
@@ -669,11 +677,10 @@ func (h *Handler) find(w http.ResponseWriter, r *http.Request) {
|
||||
})
|
||||
}
|
||||
|
||||
// lookupCache returns the entry to restore for these keys, or (nil, nil) when there is none:
|
||||
// either nothing matched, or the match had lost its blob to a prune, in which case the dangling
|
||||
// entry is dropped on the way out.
|
||||
func (h *Handler) lookupCache(db *cacheDB, repo string, keys []string, version string) (*Cache, error) {
|
||||
cache, err := findCache(db, repo, keys, version)
|
||||
// lookupCache returns the entry to restore or (nil, nil), dropping matches whose blob was pruned.
|
||||
func (h *Handler) lookupCache(db *cacheDB, cred JobCredential, keys []string, version string) (*Cache, error) {
|
||||
for {
|
||||
cache, err := findCache(db, cred, keys, version)
|
||||
if err != nil || cache == nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -681,14 +688,16 @@ func (h *Handler) lookupCache(db *cacheDB, repo string, keys []string, version s
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !ok {
|
||||
h.deleteCache(db, cache)
|
||||
return nil, nil //nolint:nilnil // absence is not an error here
|
||||
}
|
||||
if ok {
|
||||
// Handing out a download URL counts as access, or eviction could drop the entry between
|
||||
// this call and the GET that follows it.
|
||||
h.touch(db, cache)
|
||||
return cache, nil
|
||||
}
|
||||
if !h.deleteCache(db, cache) {
|
||||
return nil, nil //nolint:nilnil // absence is not an error here
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// POST /_apis/artifactcache/caches
|
||||
@@ -700,7 +709,7 @@ func (h *Handler) reserve(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
cache := &Cache{Repo: cred.Repo, Key: api.Key, Version: api.Version, Size: api.Size}
|
||||
cache := &Cache{Repo: cred.Repo, Ref: cred.Ref, Key: api.Key, Version: api.Version, Size: api.Size}
|
||||
if cache.Size == 0 {
|
||||
cache.Size = -1
|
||||
}
|
||||
@@ -749,7 +758,7 @@ func (h *Handler) upload(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
if cache.Repo != cred.Repo {
|
||||
if !cred.owns(cache) {
|
||||
h.responseJSON(w, r, 403, fmt.Errorf("cache %d: forbidden", id))
|
||||
return
|
||||
}
|
||||
@@ -808,7 +817,7 @@ func (h *Handler) commit(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
if cache.Repo != cred.Repo {
|
||||
if !cred.owns(cache) {
|
||||
h.responseJSON(w, r, 403, fmt.Errorf("cache %d: forbidden", id))
|
||||
return
|
||||
}
|
||||
@@ -1071,27 +1080,37 @@ func (h *Handler) signedArtifactURL(cred JobCredential, cacheID uint64, exp time
|
||||
return h.signedURL(cred, apiPath+"/artifacts", "", cacheID, exp)
|
||||
}
|
||||
|
||||
// if not found, return (nil, nil) instead of an error.
|
||||
func findCache(db *cacheDB, repo string, keys []string, version string) (*Cache, error) {
|
||||
for _, prefix := range keys {
|
||||
// if a key in the list matches exactly, don't return partial matches
|
||||
exact, err := findExactCache(db, repo, prefix, version, true)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
// findCache tries every key in one ref before the next ref, and returns (nil, nil) on a miss.
|
||||
func findCache(db *cacheDB, cred JobCredential, keys []string, version string) (*Cache, error) {
|
||||
refs := []string{cred.Ref}
|
||||
for _, ref := range []string{cred.BaseRef, cred.DefaultRef} {
|
||||
if ref != "" && !slices.Contains(refs, ref) {
|
||||
refs = append(refs, ref)
|
||||
}
|
||||
if exact != nil {
|
||||
return exact, nil
|
||||
}
|
||||
caches, err := db.find(func(cache *Cache) bool {
|
||||
return cache.Repo == repo && strings.HasPrefix(cache.Key, prefix) && cache.Version == version && cache.Complete
|
||||
return cache.Repo == cred.Repo && cache.Version == version && cache.Complete && slices.Contains(refs, cache.Ref) &&
|
||||
slices.ContainsFunc(keys, func(prefix string) bool { return strings.HasPrefix(cache.Key, prefix) })
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("find cache: %w", err)
|
||||
}
|
||||
if cache := latest(caches, (*Cache).createdAtNano); cache != nil {
|
||||
for _, ref := range refs {
|
||||
for _, prefix := range keys {
|
||||
var exact, partial []*Cache
|
||||
for _, cache := range caches {
|
||||
if cache.Ref == ref && cache.Key == prefix {
|
||||
exact = append(exact, cache)
|
||||
} else if cache.Ref == ref && strings.HasPrefix(cache.Key, prefix) {
|
||||
partial = append(partial, cache)
|
||||
}
|
||||
}
|
||||
// if a key in the list matches exactly, don't return partial matches
|
||||
if cache := cmp.Or(latest(exact, (*Cache).createdAtNano), latest(partial, (*Cache).createdAtNano)); cache != nil {
|
||||
return cache, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil, nil //nolint:nilnil // pre-existing issue from nektos/act
|
||||
}
|
||||
|
||||
@@ -1111,9 +1130,9 @@ func latest(caches []*Cache, stamp func(*Cache) int64) *Cache {
|
||||
// A completed entry is the one to restore, sorted by when it was written. An incomplete one is a
|
||||
// reservation being uploaded to, sorted by when it was last written to, because the upload route
|
||||
// touches UsedAt on every part.
|
||||
func findExactCache(db *cacheDB, repo, key, version string, complete bool) (*Cache, error) {
|
||||
func findExactCache(db *cacheDB, cred JobCredential, key, version string, complete bool) (*Cache, error) {
|
||||
caches, err := db.find(func(cache *Cache) bool {
|
||||
return cache.Key == key && cache.Repo == repo && cache.Version == version && cache.Complete == complete
|
||||
return cred.owns(cache) && cache.Key == key && cache.Version == version && cache.Complete == complete
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("find cache: %w", err)
|
||||
@@ -1280,13 +1299,12 @@ func (h *Handler) evictExpired(db *cacheDB) {
|
||||
h.sweep(db, func(cache *Cache) bool { return cache.UsedAt < expired })
|
||||
}
|
||||
|
||||
// evictSuperseded removes entries a newer one with the same key and version replaced. The
|
||||
// grouping includes Repo so two repos sharing a (key, version) do not evict each other.
|
||||
// evictSuperseded removes entries a newer one with the same repo, ref, key and version replaced.
|
||||
func (h *Handler) evictSuperseded(db *cacheDB) {
|
||||
type entry struct{ repo, key, version string }
|
||||
type entry struct{ repo, ref, key, version string }
|
||||
groups := make(map[entry][]*Cache)
|
||||
for _, cache := range h.caches(db, func(cache *Cache) bool { return cache.Complete }) {
|
||||
group := entry{cache.Repo, cache.Key, cache.Version}
|
||||
group := entry{cache.Repo, cache.Ref, cache.Key, cache.Version}
|
||||
groups[group] = append(groups[group], cache)
|
||||
}
|
||||
for _, caches := range groups {
|
||||
|
||||
@@ -759,11 +759,55 @@ func TestHandler_PrefixUsesNewestReservationWithinSecond(t *testing.T) {
|
||||
db, err := handler.openDB()
|
||||
require.NoError(t, err)
|
||||
defer db.Close()
|
||||
cache, err := findCache(db, testRepo, []string{"deps-"}, "v")
|
||||
cache, err := findCache(db, JobCredential{Repo: testRepo}, []string{"deps-"}, "v")
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, newer, cache)
|
||||
}
|
||||
|
||||
func TestHandler_RestoreRefs(t *testing.T) {
|
||||
t.Parallel()
|
||||
const main, release, pull = "refs/heads/main", "refs/heads/release", "refs/pull/1/head"
|
||||
now := time.Now().Unix()
|
||||
entry := func(ref, key string, age int64) *Cache {
|
||||
return &Cache{Repo: testRepo, Ref: ref, Key: key, Version: "v", Complete: true, CreatedAt: now - 3600 + age, UsedAt: now}
|
||||
}
|
||||
exactOnMain := entry(main, "deps-exact", -1)
|
||||
onRelease := entry(release, "deps-release", 0)
|
||||
onMain := entry(main, "deps-main", 1)
|
||||
onPull := entry(pull, "deps-pull", 2)
|
||||
unscoped := entry("", "deps-unscoped", 3)
|
||||
handler := newTestHandler(t, Policy{}, exactOnMain, onRelease, onMain, onPull, unscoped, entry("refs/pull/2/head", "deps-lost", 4))
|
||||
for _, cache := range []*Cache{exactOnMain, onRelease, onMain, onPull, unscoped} {
|
||||
writeBlob(t, handler, cache.ID)
|
||||
}
|
||||
|
||||
db, err := handler.openDB()
|
||||
require.NoError(t, err)
|
||||
defer db.Close()
|
||||
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
cred JobCredential
|
||||
keys []string
|
||||
want *Cache
|
||||
}{
|
||||
{"the default branch never restores what a pull request wrote", JobCredential{Ref: main, DefaultRef: main}, []string{"deps-"}, onMain},
|
||||
{"a pull request restores its own entry first", JobCredential{Ref: pull, BaseRef: release, DefaultRef: main}, []string{"deps-"}, onPull},
|
||||
{"a pull request falls back past a lost blob to its base branch before the default branch", JobCredential{Ref: "refs/pull/2/head", BaseRef: release, DefaultRef: main}, []string{"deps-"}, onRelease},
|
||||
{"a branch does not restore a sibling branch", JobCredential{Ref: "refs/heads/other", DefaultRef: main}, []string{"deps-"}, onMain},
|
||||
{"a prefix in the own ref wins over an exact key in the default branch", JobCredential{Ref: release, DefaultRef: main}, []string{"deps-exact", "deps-"}, onRelease},
|
||||
{"a job with a ref never restores entries saved without one", JobCredential{Ref: "refs/heads/other"}, []string{"deps-"}, nil},
|
||||
{"a job without a ref only restores entries saved without one", JobCredential{}, []string{"deps-"}, unscoped},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
tc.cred.Repo = testRepo
|
||||
cache, err := handler.lookupCache(db, tc.cred, tc.keys, "v")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, tc.want, cache)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandler_evictOnCommit(t *testing.T) {
|
||||
t.Parallel()
|
||||
full := &Cache{Repo: testRepo, Key: "full", Version: "v", Complete: true, Size: 4 * miB, UsedAt: time.Now().Add(-time.Hour).Unix()}
|
||||
@@ -938,6 +982,7 @@ func TestHandler_CrossRepoIsolation(t *testing.T) {
|
||||
defer handler.Close()
|
||||
handler.RegisterJob("token-a", JobCredential{Repo: "owner/repoA"})
|
||||
handler.RegisterJob("token-b", JobCredential{Repo: "owner/repoB"})
|
||||
handler.RegisterJob("token-a-pull", JobCredential{Repo: "owner/repoA", Ref: "refs/pull/1/head"})
|
||||
|
||||
base := handler.ExternalURL() + apiPath
|
||||
key := "shared-key"
|
||||
@@ -972,13 +1017,20 @@ func TestHandler_CrossRepoIsolation(t *testing.T) {
|
||||
resp.Body.Close()
|
||||
assert.Equal(t, http.StatusOK, resp.StatusCode)
|
||||
|
||||
for _, client := range []*http.Client{clientB, {Transport: &bearerTransport{token: "token-a-pull"}}} {
|
||||
req, err = http.NewRequest(http.MethodPatch, fmt.Sprintf("%s/caches/%d", base, id), bytes.NewReader([]byte("poison")))
|
||||
require.NoError(t, err)
|
||||
req.Header.Set("Content-Range", "bytes 0-5/*")
|
||||
resp, err = clientB.Do(req)
|
||||
resp, err = client.Do(req)
|
||||
require.NoError(t, err)
|
||||
resp.Body.Close()
|
||||
assert.Equal(t, http.StatusForbidden, resp.StatusCode)
|
||||
|
||||
resp, err = client.Post(fmt.Sprintf("%s/caches/%d", base, id), "", nil)
|
||||
require.NoError(t, err)
|
||||
resp.Body.Close()
|
||||
assert.Equal(t, http.StatusForbidden, resp.StatusCode)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandler_ArtifactSignature(t *testing.T) {
|
||||
@@ -1083,49 +1135,20 @@ func TestHandler_ArtifactSignatureDownload(t *testing.T) {
|
||||
assert.Equal(t, content, body)
|
||||
}
|
||||
|
||||
func TestHandler_GC_PerRepoDedup(t *testing.T) {
|
||||
func TestHandler_GC_DedupPerRepoAndRef(t *testing.T) {
|
||||
t.Parallel()
|
||||
dir := filepath.Join(t.TempDir(), "artifactcache")
|
||||
handler, err := StartHandler(Options{Dir: dir})
|
||||
require.NoError(t, err)
|
||||
defer handler.Close()
|
||||
handler.RegisterJob("tok-a", JobCredential{Repo: "owner/repoA"})
|
||||
handler.RegisterJob("tok-b", JobCredential{Repo: "owner/repoB"})
|
||||
|
||||
key := "shared-dedup-key"
|
||||
version := "c19da02a2bd7e77277f1ac29ab45c09b7d46a4ee758284e26bb3045ad11d9d20"
|
||||
|
||||
db, err := handler.openDB()
|
||||
require.NoError(t, err)
|
||||
now := time.Now().Unix()
|
||||
stale := time.Now().Add(-inUseGrace - time.Minute).Unix()
|
||||
a := &Cache{Repo: "owner/repoA", Key: key, Version: version, Complete: true, CreatedAt: stale, UsedAt: stale, Size: 1}
|
||||
b := &Cache{Repo: "owner/repoB", Key: key, Version: version, Complete: true, CreatedAt: now, UsedAt: now, Size: 1}
|
||||
require.NoError(t, handler.insertCache(db, a))
|
||||
require.NoError(t, handler.insertCache(db, b))
|
||||
require.NoError(t, handler.storage.Write(a.ID, 0, strings.NewReader("a")))
|
||||
_, err = handler.storage.Commit(a.ID, 1)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, handler.storage.Write(b.ID, 0, strings.NewReader("b")))
|
||||
_, err = handler.storage.Commit(b.ID, 1)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, db.Close())
|
||||
|
||||
entries := []*Cache{
|
||||
{Repo: "owner/repoA", Key: "shared", Version: "v", Complete: true, CreatedAt: stale, UsedAt: stale, Size: 1},
|
||||
{Repo: "owner/repoB", Key: "shared", Version: "v", Complete: true, CreatedAt: now, UsedAt: now, Size: 1},
|
||||
{Repo: "owner/repoA", Ref: "refs/pull/1/head", Key: "shared", Version: "v", Complete: true, CreatedAt: now, UsedAt: now, Size: 1},
|
||||
}
|
||||
handler := newTestHandler(t, Policy{}, entries...)
|
||||
handler.gcAt = time.Time{}
|
||||
handler.gcCache()
|
||||
|
||||
db, err = handler.openDB()
|
||||
require.NoError(t, err)
|
||||
defer db.Close()
|
||||
after, err := db.find(func(cache *Cache) bool { return cache.Key == key && cache.Version == version })
|
||||
require.NoError(t, err)
|
||||
|
||||
repos := make(map[string]bool)
|
||||
for _, c := range after {
|
||||
repos[c.Repo] = true
|
||||
}
|
||||
assert.True(t, repos["owner/repoA"], "repoA's cache must survive dedup against repoB")
|
||||
assert.True(t, repos["owner/repoB"], "repoB's cache must survive dedup against repoA")
|
||||
assert.Len(t, keptKeys(t, handler, entries), len(entries))
|
||||
}
|
||||
|
||||
func TestHandler_InternalAPI_Disabled(t *testing.T) {
|
||||
@@ -1190,10 +1213,10 @@ func TestHandler_InternalAPI_AuthAndUsage(t *testing.T) {
|
||||
}
|
||||
|
||||
assert.Equal(t, http.StatusUnauthorized, probe("via-internal-api"))
|
||||
assert.Equal(t, http.StatusOK, post("/_internal/register", secret, `{"token":"via-internal-api","repo":"owner/repo","public_url":"http://runner.example","results":"https://gitea.example","insecure_tls":true}`))
|
||||
assert.Equal(t, http.StatusOK, post("/_internal/register", secret, `{"token":"via-internal-api","repo":"owner/repo","ref":"refs/pull/1/head","base_ref":"refs/heads/release","default_ref":"refs/heads/main","public_url":"http://runner.example","results":"https://gitea.example","insecure_tls":true}`))
|
||||
cred, ok := handler.lookupCredential("via-internal-api")
|
||||
require.True(t, ok)
|
||||
assert.Equal(t, JobCredential{Repo: testRepo, PublicURL: "http://runner.example"}, cred)
|
||||
assert.Equal(t, JobCredential{Repo: testRepo, Ref: "refs/pull/1/head", BaseRef: "refs/heads/release", DefaultRef: "refs/heads/main", PublicURL: "http://runner.example"}, cred)
|
||||
assert.NotEqual(t, http.StatusUnauthorized, probe("via-internal-api"))
|
||||
assert.Equal(t, http.StatusOK, post("/_internal/revoke", secret, `{"token":"via-internal-api"}`))
|
||||
assert.Equal(t, http.StatusUnauthorized, probe("via-internal-api"))
|
||||
|
||||
@@ -73,7 +73,7 @@ func (h *Handler) v2CreateCacheEntry(w http.ResponseWriter, r *http.Request) {
|
||||
// An exact (key, version) match means the entry is already cached; the client then skips
|
||||
// the upload. A prefix match must not count here, or a shorter key would be reported as
|
||||
// existing and silently never saved.
|
||||
if existing, err := findExactCache(db, cred.Repo, req.Key, req.Version, true); err != nil {
|
||||
if existing, err := findExactCache(db, cred, req.Key, req.Version, true); err != nil {
|
||||
h.twirpError(w, r, twirpInternal, err)
|
||||
return
|
||||
} else if existing != nil {
|
||||
@@ -84,7 +84,7 @@ func (h *Handler) v2CreateCacheEntry(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
// A second live reservation is finalized by whichever job calls last, against the other's upload.
|
||||
owner := hashedToken(bearerToken(r))
|
||||
if pending, err := findExactCache(db, cred.Repo, req.Key, req.Version, false); err != nil {
|
||||
if pending, err := findExactCache(db, cred, req.Key, req.Version, false); err != nil {
|
||||
h.twirpError(w, r, twirpInternal, err)
|
||||
return
|
||||
} else if pending != nil && pending.UsedAt > time.Now().Add(-uploadStallTimeout).Unix() {
|
||||
@@ -103,6 +103,7 @@ func (h *Handler) v2CreateCacheEntry(w http.ResponseWriter, r *http.Request) {
|
||||
now := time.Now()
|
||||
cache := &Cache{
|
||||
Repo: cred.Repo,
|
||||
Ref: cred.Ref,
|
||||
Key: req.Key,
|
||||
Version: req.Version,
|
||||
Owner: owner,
|
||||
@@ -137,13 +138,13 @@ func (h *Handler) v2FinalizeCacheEntryUpload(w http.ResponseWriter, r *http.Requ
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
cache, err := findExactCache(db, cred.Repo, req.Key, req.Version, false)
|
||||
cache, err := findExactCache(db, cred, req.Key, req.Version, false)
|
||||
if err == nil && cache != nil && cache.Owner != hashedToken(bearerToken(r)) {
|
||||
cache = nil // not the reservation this job made, so not this job's to commit
|
||||
}
|
||||
if err == nil && cache == nil {
|
||||
// A retry whose first response was lost finds the entry already committed.
|
||||
cache, err = findExactCache(db, cred.Repo, req.Key, req.Version, true)
|
||||
cache, err = findExactCache(db, cred, req.Key, req.Version, true)
|
||||
}
|
||||
if err != nil {
|
||||
h.twirpError(w, r, twirpInternal, err)
|
||||
@@ -186,7 +187,7 @@ func (h *Handler) v2GetCacheEntryDownloadURL(w http.ResponseWriter, r *http.Requ
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
cache, err := h.lookupCache(db, cred.Repo, req.keys(), req.Version)
|
||||
cache, err := h.lookupCache(db, cred, req.keys(), req.Version)
|
||||
if err != nil {
|
||||
h.twirpError(w, r, twirpInternal, err)
|
||||
return
|
||||
|
||||
@@ -231,6 +231,24 @@ func TestCacheServiceV2Lookups(t *testing.T) {
|
||||
got := v2Call(t, handler, otherClient, "GetCacheEntryDownloadURL", map[string]any{"key": "deps-abc", "version": "v1"})
|
||||
assert.Equal(t, false, got["ok"])
|
||||
})
|
||||
|
||||
t.Run("each ref saves and restores its own entry for the same key", func(t *testing.T) {
|
||||
clients := map[string]*http.Client{"x": testClient}
|
||||
for _, ref := range []string{"y", "z"} {
|
||||
handler.RegisterJob(ref, JobCredential{Repo: testRepo, Ref: "refs/pull/" + ref + "/head"})
|
||||
clients[ref] = &http.Client{Transport: &bearerTransport{token: ref}}
|
||||
created := v2Call(t, handler, clients[ref], "CreateCacheEntry", map[string]any{"key": "deps-abc", "version": "v1"})
|
||||
require.Equal(t, true, created["ok"])
|
||||
require.Equal(t, http.StatusCreated, putBlob(t, created["signed_upload_url"].(string), []byte(ref)))
|
||||
}
|
||||
for _, ref := range []string{"y", "z"} {
|
||||
require.Equal(t, true, v2Call(t, handler, clients[ref], "FinalizeCacheEntryUpload", map[string]any{"key": "deps-abc", "version": "v1", "size_bytes": 1})["ok"])
|
||||
}
|
||||
for content, client := range clients {
|
||||
got := v2Call(t, handler, client, "GetCacheEntryDownloadURL", map[string]any{"key": "deps-abc", "version": "v1"})
|
||||
assert.Equal(t, content, string(getURL(t, got["signed_download_url"].(string))))
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestCacheServiceV2RefusalsAreTwirp(t *testing.T) {
|
||||
|
||||
@@ -15,6 +15,7 @@ type Request struct {
|
||||
type Cache struct {
|
||||
ID uint64 `json:"id"`
|
||||
Repo string `json:"repo"`
|
||||
Ref string `json:"ref,omitzero"`
|
||||
Key string `json:"key"`
|
||||
Version string `json:"version"`
|
||||
Size int64 `json:"cacheSize"`
|
||||
|
||||
@@ -218,6 +218,7 @@ func Serve(ctx context.Context, artifactPath, addr, port string) context.CancelF
|
||||
logger.Debugf("Artifacts base path '%s'", artifactPath)
|
||||
uploads(router, artifactPath)
|
||||
downloads(router, artifactPath)
|
||||
registerV4(router, artifactPath)
|
||||
|
||||
server := &http.Server{
|
||||
Addr: net.JoinHostPort(addr, port),
|
||||
|
||||
@@ -28,6 +28,7 @@ func TestArtifactFlow(t *testing.T) {
|
||||
router := http.NewServeMux()
|
||||
uploads(router, artifactPath)
|
||||
downloads(router, artifactPath)
|
||||
registerV4(router, artifactPath)
|
||||
server := httptest.NewServer(router)
|
||||
defer server.Close()
|
||||
|
||||
@@ -174,6 +175,92 @@ func TestArtifactFlow(t *testing.T) {
|
||||
require.Equal(t, http.StatusOK, status)
|
||||
require.Equal(t, content, string(data))
|
||||
})
|
||||
|
||||
t.Run("v4", func(t *testing.T) {
|
||||
const artifactRef = `"workflow_run_backend_id":"7","name":"50% art@v1"`
|
||||
call := func(method, body string) (int, map[string]any) {
|
||||
status, data := request(t, http.MethodPost, baseURL+artifactServicePath+"/"+method, strings.NewReader(body), nil)
|
||||
var out map[string]any
|
||||
require.NoError(t, json.Unmarshal(data, &out), string(data))
|
||||
return status, out
|
||||
}
|
||||
put := func(rawURL, body string) int {
|
||||
status, _ := request(t, http.MethodPut, rawURL, strings.NewReader(body), nil)
|
||||
return status
|
||||
}
|
||||
download := func() (string, http.Header) {
|
||||
status, signed := call("GetSignedArtifactURL", "{"+artifactRef+"}")
|
||||
require.Equal(t, http.StatusOK, status, signed)
|
||||
resp, err := client.Get(signed["signedUrl"].(string))
|
||||
require.NoError(t, err)
|
||||
defer resp.Body.Close()
|
||||
require.Equal(t, http.StatusOK, resp.StatusCode)
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
require.NoError(t, err)
|
||||
return string(body), resp.Header
|
||||
}
|
||||
|
||||
status, created := call("CreateArtifact", "{"+artifactRef+"}")
|
||||
require.Equal(t, http.StatusOK, status, created)
|
||||
upload := created["signedUploadUrl"].(string)
|
||||
require.Equal(t, http.StatusCreated, put(upload+"&comp=block&blockid=Yg", "world"))
|
||||
require.Equal(t, http.StatusCreated, put(upload+"&comp=block&blockid=YQ", "hello "))
|
||||
for attempt := range 2 {
|
||||
require.Equal(t, http.StatusCreated, put(upload+"&comp=blocklist", `<?xml version="1.0"?><BlockList><Latest>YQ</Latest><Latest>Yg</Latest></BlockList>`), "commit attempt %d", attempt)
|
||||
}
|
||||
|
||||
digest := "sha256:b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9"
|
||||
status, _ = call("FinalizeArtifact", "{"+artifactRef+`,"size":"10"}`)
|
||||
require.Equal(t, http.StatusBadRequest, status)
|
||||
status, finalized := call("FinalizeArtifact", "{"+artifactRef+`,"size":"11","hash":"`+digest+`"}`)
|
||||
require.Equal(t, http.StatusOK, status, finalized)
|
||||
id := finalized["artifactId"].(string)
|
||||
status, refinalized := call("FinalizeArtifact", "{"+artifactRef+`,"size":"10"}`)
|
||||
require.Equal(t, http.StatusOK, status)
|
||||
require.Equal(t, id, refinalized["artifactId"])
|
||||
status, conflict := call("CreateArtifact", "{"+artifactRef+"}")
|
||||
require.Equal(t, http.StatusConflict, status)
|
||||
require.Equal(t, "already_exists", conflict["code"])
|
||||
|
||||
status, listed := call("ListArtifacts", `{"workflowRunBackendId":"7","idFilter":"`+id+`"}`)
|
||||
require.Equal(t, http.StatusOK, status)
|
||||
artifacts := listed["artifacts"].([]any)
|
||||
require.Len(t, artifacts, 1)
|
||||
artifact := artifacts[0].(map[string]any)
|
||||
require.Regexp(t, `^\d{4}-\d\d-\d\dT\d\d:\d\d:\d\dZ$`, artifact["createdAt"])
|
||||
delete(artifact, "createdAt")
|
||||
require.Equal(t, map[string]any{"workflowRunBackendId": "7", "name": "50% art@v1", "size": "11", "digest": digest, "databaseId": id}, artifact)
|
||||
_, other := call("ListArtifacts", `{"workflow_run_backend_id":"8"}`)
|
||||
require.Empty(t, other["artifacts"])
|
||||
|
||||
body, header := download()
|
||||
require.Equal(t, "hello world", body)
|
||||
require.Equal(t, "application/zip", header.Get("Content-Type"))
|
||||
require.Equal(t, "attachment; filename*=UTF-8''50%25%20art%40v1.zip", header.Get("Content-Disposition"))
|
||||
|
||||
status, _ = call("DeleteArtifact", "{"+artifactRef+"}")
|
||||
require.Equal(t, http.StatusOK, status)
|
||||
status, missing := call("GetSignedArtifactURL", "{"+artifactRef+"}")
|
||||
require.Equal(t, http.StatusNotFound, status)
|
||||
require.Equal(t, "not_found", missing["code"])
|
||||
|
||||
status, created = call("CreateArtifact", "{"+artifactRef+"}")
|
||||
require.Equal(t, http.StatusOK, status, created)
|
||||
status, _ = request(t, http.MethodPut, created["signedUploadUrl"].(string), strings.NewReader("gz"), http.Header{"X-Ms-Blob-Content-Type": {"application/gzip"}})
|
||||
require.Equal(t, http.StatusCreated, status)
|
||||
status, _ = call("FinalizeArtifact", "{"+artifactRef+`,"size":"2"}`)
|
||||
require.Equal(t, http.StatusOK, status)
|
||||
body, header = download()
|
||||
require.Equal(t, "gz", body)
|
||||
require.Equal(t, "application/gzip", header.Get("Content-Type"))
|
||||
require.Equal(t, "attachment; filename*=UTF-8''50%25%20art%40v1", header.Get("Content-Disposition"))
|
||||
|
||||
rerun := http.NewServeMux()
|
||||
registerV4(rerun, artifactPath)
|
||||
recorder := httptest.NewRecorder()
|
||||
rerun.ServeHTTP(recorder, httptest.NewRequest(http.MethodPost, artifactServicePath+"/CreateArtifact", strings.NewReader("{"+artifactRef+"}")))
|
||||
require.Equal(t, http.StatusOK, recorder.Code, recorder.Body.String())
|
||||
})
|
||||
}
|
||||
|
||||
func TestSafeResolve(t *testing.T) {
|
||||
|
||||
@@ -0,0 +1,329 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package artifacts
|
||||
|
||||
import (
|
||||
"cmp"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json/v2"
|
||||
"encoding/xml"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
type serverV4 struct { // the subset of Gitea's v4 artifact service that the toolkit uses, unauthenticated like the v3 routes beside it
|
||||
dir string
|
||||
mu sync.Mutex
|
||||
lastID int64
|
||||
created map[string]bool
|
||||
}
|
||||
|
||||
const (
|
||||
zipMimeType = "application/zip"
|
||||
maxTwirpBytes = 1 << 20
|
||||
)
|
||||
|
||||
func registerV4(router *http.ServeMux, baseDir string) {
|
||||
server := &serverV4{dir: filepath.Join(baseDir, "v4"), lastID: time.Now().UnixMilli(), created: map[string]bool{}} // IDs stay unique across invocations sharing baseDir
|
||||
router.HandleFunc("POST "+artifactServicePath+"/CreateArtifact", twirp(server.createArtifact))
|
||||
router.HandleFunc("POST "+artifactServicePath+"/FinalizeArtifact", twirp(server.finalizeArtifact))
|
||||
router.HandleFunc("POST "+artifactServicePath+"/ListArtifacts", twirp(server.listArtifacts))
|
||||
router.HandleFunc("POST "+artifactServicePath+"/GetSignedArtifactURL", twirp(server.getSignedArtifactURL))
|
||||
router.HandleFunc("POST "+artifactServicePath+"/DeleteArtifact", twirp(server.deleteArtifact))
|
||||
router.HandleFunc("PUT "+artifactServicePath+"/UploadArtifact", server.uploadArtifact)
|
||||
router.HandleFunc("GET "+artifactServicePath+"/DownloadArtifact", server.downloadArtifact)
|
||||
}
|
||||
|
||||
type twirpError struct {
|
||||
status int
|
||||
code string
|
||||
msg string
|
||||
}
|
||||
|
||||
func (e *twirpError) Error() string { return e.msg }
|
||||
|
||||
func notFound(name string) error {
|
||||
return &twirpError{http.StatusNotFound, "not_found", fmt.Sprintf("artifact %q not found", name)}
|
||||
}
|
||||
|
||||
func badRequest(format string, args ...any) error {
|
||||
return &twirpError{http.StatusBadRequest, "invalid_argument", fmt.Sprintf(format, args...)}
|
||||
}
|
||||
|
||||
func twirp[Req, Resp any](handle func(*http.Request, Req) (Resp, error)) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
var req Req
|
||||
var resp Resp
|
||||
err := json.UnmarshalRead(io.LimitReader(r.Body, maxTwirpBytes), &req, json.MatchCaseInsensitiveNames(true)) // also matches the toolkit's proto field names
|
||||
if err != nil {
|
||||
err = badRequest("malformed request: %v", err)
|
||||
} else {
|
||||
resp, err = handle(r, req)
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
if err != nil {
|
||||
terr := &twirpError{http.StatusInternalServerError, "internal", err.Error()}
|
||||
errors.As(err, &terr)
|
||||
w.WriteHeader(terr.status)
|
||||
_ = json.MarshalWrite(w, map[string]string{"code": terr.code, "msg": terr.msg})
|
||||
return
|
||||
}
|
||||
_ = json.MarshalWrite(w, resp)
|
||||
}
|
||||
}
|
||||
|
||||
func fileName(name string) string { // safe on any filesystem for any client-chosen name, case-insensitive ones included
|
||||
sum := sha256.Sum256([]byte(name))
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
func (s *serverV4) runDir(runID string) string {
|
||||
return filepath.Join(s.dir, fileName(runID))
|
||||
}
|
||||
|
||||
func (s *serverV4) artifactDir(runID, name string) string {
|
||||
return filepath.Join(s.runDir(runID), fileName(name))
|
||||
}
|
||||
|
||||
func (s *serverV4) urlDir(r *http.Request) string {
|
||||
return s.artifactDir(r.URL.Query().Get("run"), r.URL.Query().Get("name"))
|
||||
}
|
||||
|
||||
func readMeta(dir string) (*monolithArtifact, error) {
|
||||
data, err := os.ReadFile(filepath.Join(dir, "meta.json"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var meta monolithArtifact
|
||||
return &meta, json.Unmarshal(data, &meta)
|
||||
}
|
||||
|
||||
func writeMeta(dir string, meta *monolithArtifact) error {
|
||||
data, err := json.Marshal(meta)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return os.WriteFile(filepath.Join(dir, "meta.json"), data, 0o644)
|
||||
}
|
||||
|
||||
func blobURL(r *http.Request, endpoint, runID, name string) string {
|
||||
return "http://" + r.Host + artifactServicePath + "/" + endpoint + "?" + url.Values{"run": {runID}, "name": {name}}.Encode()
|
||||
}
|
||||
|
||||
func (s *serverV4) createArtifact(r *http.Request, req createArtifactRequest) (createArtifactResponse, error) { // an exec invocation re-runs its workflows, so it replaces earlier invocations' artifacts but conflicts with its own
|
||||
dir := s.artifactDir(req.WorkflowRunBackendID, req.Name)
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.created[dir] {
|
||||
return createArtifactResponse{}, &twirpError{http.StatusConflict, "already_exists", "an artifact with this name already exists on the workflow run"}
|
||||
}
|
||||
if err := os.RemoveAll(dir); err != nil {
|
||||
return createArtifactResponse{}, err
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Join(dir, "blocks"), 0o755); err != nil {
|
||||
return createArtifactResponse{}, err
|
||||
}
|
||||
s.lastID++
|
||||
if err := writeMeta(dir, &monolithArtifact{
|
||||
WorkflowRunBackendID: req.WorkflowRunBackendID,
|
||||
DatabaseID: s.lastID,
|
||||
Name: req.Name,
|
||||
CreatedAt: time.Now().UTC().Truncate(time.Second),
|
||||
}); err != nil {
|
||||
return createArtifactResponse{}, err
|
||||
}
|
||||
s.created[dir] = true
|
||||
return createArtifactResponse{Ok: true, SignedUploadURL: blobURL(r, "UploadArtifact", req.WorkflowRunBackendID, req.Name)}, nil
|
||||
}
|
||||
|
||||
type blockList struct {
|
||||
Blocks []string `xml:",any"` // Latest, Committed and Uncommitted all name a staged block
|
||||
}
|
||||
|
||||
func (s *serverV4) uploadArtifact(w http.ResponseWriter, r *http.Request) {
|
||||
dir := s.urlDir(r)
|
||||
if meta, err := readMeta(dir); err != nil || meta.Digest != "" {
|
||||
http.Error(w, "no artifact awaits an upload", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
contentType := r.Header.Get("x-ms-blob-content-type")
|
||||
var err error
|
||||
switch comp := r.URL.Query().Get("comp"); comp {
|
||||
case "block":
|
||||
err = writeFile(filepath.Join(dir, "blocks", fileName(r.URL.Query().Get("blockid"))), r.Body)
|
||||
case "blocklist":
|
||||
err = commitBlob(dir, contentType, func(out io.Writer) error { return joinBlocks(out, dir, r.Body) })
|
||||
case "": // Put Blob, which docker/build-push-action uses
|
||||
err = commitBlob(dir, contentType, func(out io.Writer) error {
|
||||
_, err := io.Copy(out, r.Body)
|
||||
return err
|
||||
})
|
||||
default:
|
||||
http.Error(w, "unsupported comp "+comp, http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusCreated)
|
||||
}
|
||||
|
||||
func writeFile(name string, body io.Reader) error {
|
||||
file, err := os.Create(name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = io.Copy(file, body)
|
||||
return errors.Join(err, file.Close())
|
||||
}
|
||||
|
||||
func commitBlob(dir, contentType string, write func(io.Writer) error) error { // replaces the blob whole and keeps its content type, as Azure does
|
||||
out, err := os.CreateTemp(dir, "blob")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err = errors.Join(write(out), out.Close()); err == nil {
|
||||
err = os.Rename(out.Name(), filepath.Join(dir, "blob"))
|
||||
}
|
||||
if err != nil {
|
||||
return errors.Join(err, os.Remove(out.Name()))
|
||||
}
|
||||
return os.WriteFile(filepath.Join(dir, "content-type"), []byte(contentType), 0o644)
|
||||
}
|
||||
|
||||
func joinBlocks(out io.Writer, dir string, body io.Reader) error { // keeps the blocks for finalizeArtifact, so that a retried commit can join them again
|
||||
var list blockList
|
||||
if err := xml.NewDecoder(io.LimitReader(body, maxTwirpBytes)).Decode(&list); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, id := range list.Blocks {
|
||||
if err := appendBlock(out, filepath.Join(dir, "blocks", fileName(id))); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func appendBlock(out io.Writer, name string) error {
|
||||
in, err := os.Open(name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer in.Close()
|
||||
_, err = io.Copy(out, in)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *serverV4) finalizeArtifact(_ *http.Request, req finalizeArtifactRequest) (finalizeArtifactResponse, error) {
|
||||
dir := s.artifactDir(req.WorkflowRunBackendID, req.Name)
|
||||
meta, err := readMeta(dir)
|
||||
if err != nil {
|
||||
return finalizeArtifactResponse{}, notFound(req.Name)
|
||||
} else if meta.Digest != "" {
|
||||
return finalizeArtifactResponse{Ok: true, ArtifactID: meta.DatabaseID}, nil
|
||||
}
|
||||
blob, err := os.Open(filepath.Join(dir, "blob"))
|
||||
if err != nil {
|
||||
return finalizeArtifactResponse{}, badRequest("artifact %q has no upload", req.Name)
|
||||
}
|
||||
defer blob.Close()
|
||||
hash := sha256.New()
|
||||
size, err := io.Copy(hash, blob)
|
||||
if err != nil {
|
||||
return finalizeArtifactResponse{}, err
|
||||
}
|
||||
digest := "sha256:" + hex.EncodeToString(hash.Sum(nil))
|
||||
if size != req.Size {
|
||||
return finalizeArtifactResponse{}, badRequest("size %d does not match the %d bytes uploaded", req.Size, size)
|
||||
}
|
||||
if req.Hash != "" && req.Hash != digest {
|
||||
return finalizeArtifactResponse{}, badRequest("hash %s does not match the uploaded %s", req.Hash, digest)
|
||||
}
|
||||
s.mu.Lock() // only after hashing, so finalizing one large artifact does not stall the others
|
||||
defer s.mu.Unlock()
|
||||
if current, err := readMeta(dir); err != nil || current.DatabaseID != meta.DatabaseID {
|
||||
return finalizeArtifactResponse{}, notFound(req.Name)
|
||||
}
|
||||
meta.Size, meta.Digest = size, digest
|
||||
if err := writeMeta(dir, meta); err != nil {
|
||||
return finalizeArtifactResponse{}, err
|
||||
}
|
||||
return finalizeArtifactResponse{Ok: true, ArtifactID: meta.DatabaseID}, os.RemoveAll(filepath.Join(dir, "blocks"))
|
||||
}
|
||||
|
||||
func (s *serverV4) listArtifacts(_ *http.Request, req listArtifactsRequest) (listArtifactsResponse, error) {
|
||||
run := s.runDir(req.WorkflowRunBackendID)
|
||||
entries, err := os.ReadDir(run)
|
||||
if err != nil && !errors.Is(err, fs.ErrNotExist) {
|
||||
return listArtifactsResponse{}, err
|
||||
}
|
||||
var resp listArtifactsResponse
|
||||
for _, entry := range entries {
|
||||
meta, err := readMeta(filepath.Join(run, entry.Name()))
|
||||
if err != nil || meta.Digest == "" ||
|
||||
req.NameFilter != "" && req.NameFilter != meta.Name || req.IDFilter != 0 && req.IDFilter != meta.DatabaseID {
|
||||
continue
|
||||
}
|
||||
resp.Artifacts = append(resp.Artifacts, *meta)
|
||||
}
|
||||
slices.SortFunc(resp.Artifacts, func(left, right monolithArtifact) int { return cmp.Compare(left.DatabaseID, right.DatabaseID) })
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
func (s *serverV4) getSignedArtifactURL(r *http.Request, req getSignedArtifactURLRequest) (getSignedArtifactURLResponse, error) {
|
||||
if meta, err := readMeta(s.artifactDir(req.WorkflowRunBackendID, req.Name)); err != nil || meta.Digest == "" {
|
||||
return getSignedArtifactURLResponse{}, notFound(req.Name)
|
||||
}
|
||||
return getSignedArtifactURLResponse{SignedURL: blobURL(r, "DownloadArtifact", req.WorkflowRunBackendID, req.Name)}, nil
|
||||
}
|
||||
|
||||
func (s *serverV4) downloadArtifact(w http.ResponseWriter, r *http.Request) {
|
||||
dir := s.urlDir(r)
|
||||
meta, err := readMeta(dir)
|
||||
if err != nil || meta.Digest == "" {
|
||||
http.Error(w, "artifact not found", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
blob, err := os.Open(filepath.Join(dir, "blob"))
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
defer blob.Close()
|
||||
data, _ := os.ReadFile(filepath.Join(dir, "content-type"))
|
||||
contentType, filename := string(data), meta.Name
|
||||
if !strings.Contains(contentType, "/") {
|
||||
contentType = zipMimeType
|
||||
}
|
||||
if contentType == zipMimeType && !strings.HasSuffix(filename, ".zip") {
|
||||
filename += ".zip"
|
||||
}
|
||||
w.Header().Set("Content-Type", contentType)
|
||||
w.Header().Set("Content-Disposition", "attachment; filename*=UTF-8''"+strings.ReplaceAll(url.QueryEscape(filename), "+", "%20")) // RFC 5987 attr-chars only, which the toolkit and mime.ParseMediaType both accept
|
||||
http.ServeContent(w, r, "", meta.CreatedAt, blob)
|
||||
}
|
||||
|
||||
func (s *serverV4) deleteArtifact(_ *http.Request, req deleteArtifactRequest) (deleteArtifactResponse, error) {
|
||||
dir := s.artifactDir(req.WorkflowRunBackendID, req.Name)
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
meta, err := readMeta(dir)
|
||||
if err != nil {
|
||||
return deleteArtifactResponse{}, notFound(req.Name)
|
||||
}
|
||||
delete(s.created, dir)
|
||||
return deleteArtifactResponse{Ok: true, ArtifactID: meta.DatabaseID}, os.RemoveAll(dir)
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package artifacts
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"encoding/base64"
|
||||
"encoding/json/v2"
|
||||
"time"
|
||||
)
|
||||
|
||||
func LocalRuntimeToken(run string, lifetime time.Duration) string { // an unsecured JWT with the claims the toolkit and buildx read, its run hashed for scp and its jti an unguessable bearer
|
||||
now := time.Now()
|
||||
claims, _ := json.Marshal(map[string]any{
|
||||
"scp": "Actions.Results:" + fileName(run) + ":1",
|
||||
"ac": `[{"Scope":"","Permission":3}]`,
|
||||
"nbf": now.Unix(),
|
||||
"exp": now.Add(lifetime).Unix(),
|
||||
"jti": rand.Text(),
|
||||
})
|
||||
return base64.RawURLEncoding.EncodeToString([]byte(`{"alg":"none","typ":"JWT"}`)) + "." + base64.RawURLEncoding.EncodeToString(claims) + "."
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package artifacts
|
||||
|
||||
import "time"
|
||||
|
||||
const artifactServicePath = "/twirp/github.actions.results.api.v1.ArtifactService"
|
||||
|
||||
type ( // the ArtifactService messages in the protobuf JSON mapping, which writes int64 as strings
|
||||
createArtifactResponse struct {
|
||||
Ok bool `json:"ok"`
|
||||
SignedUploadURL string `json:"signedUploadUrl"`
|
||||
}
|
||||
|
||||
finalizeArtifactRequest struct {
|
||||
WorkflowRunBackendID string
|
||||
Name string
|
||||
Size int64 `json:",string"`
|
||||
Hash string
|
||||
}
|
||||
finalizeArtifactResponse struct {
|
||||
Ok bool `json:"ok"`
|
||||
ArtifactID int64 `json:"artifactId,string"`
|
||||
}
|
||||
|
||||
listArtifactsRequest struct {
|
||||
WorkflowRunBackendID string
|
||||
NameFilter string
|
||||
IDFilter int64 `json:",string"`
|
||||
}
|
||||
listArtifactsResponse struct {
|
||||
Artifacts []monolithArtifact `json:"artifacts"`
|
||||
}
|
||||
monolithArtifact struct {
|
||||
WorkflowRunBackendID string `json:"workflowRunBackendId"` // the toolkit sends it back to GetSignedArtifactURL
|
||||
DatabaseID int64 `json:"databaseId,string"`
|
||||
Name string `json:"name"`
|
||||
Size int64 `json:"size,string"`
|
||||
CreatedAt time.Time `json:"createdAt"` // whole seconds, the toolkit rejects the trimmed fractions json writes
|
||||
Digest string `json:"digest"` // empty until the upload is finalized
|
||||
}
|
||||
|
||||
getSignedArtifactURLRequest struct {
|
||||
WorkflowRunBackendID string
|
||||
Name string
|
||||
}
|
||||
getSignedArtifactURLResponse struct {
|
||||
SignedURL string `json:"signedUrl"`
|
||||
}
|
||||
|
||||
createArtifactRequest = getSignedArtifactURLRequest
|
||||
deleteArtifactRequest = getSignedArtifactURLRequest
|
||||
deleteArtifactResponse = finalizeArtifactResponse
|
||||
)
|
||||
@@ -41,6 +41,8 @@ const (
|
||||
maxCreateBody = 8 << 20
|
||||
|
||||
dockerProxyProbeTimeout = 5 * time.Second
|
||||
dockerProxyProbeWorkerTimeout = time.Minute
|
||||
dockerProxyProbeTarget = "/gitea-runner-probe"
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -48,7 +50,7 @@ var (
|
||||
rawStreamPath = regexp.MustCompile(`^(/v[0-9.]+)?/(containers/[^/]+/attach|exec/[^/]+/start)$`)
|
||||
)
|
||||
|
||||
var dockerProxyDirectories, dockerProxyProbes sync.Map
|
||||
var dockerProxyProbes sync.Map
|
||||
|
||||
type dockerProxyDirectory struct {
|
||||
dir, daemonDir string
|
||||
@@ -68,7 +70,7 @@ func NewDockerProxy(ctx context.Context, job string) *DockerProxy {
|
||||
}
|
||||
probeCtx, cancel := context.WithTimeout(ctx, dockerProxyProbeTimeout)
|
||||
defer cancel()
|
||||
cli, err := GetDockerClient(probeCtx)
|
||||
cli, err := client.New(client.FromEnv) // negotiates on first request, so a cached probe costs no round trip
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
@@ -83,53 +85,43 @@ func NewDockerProxy(ctx context.Context, job string) *DockerProxy {
|
||||
}
|
||||
workdir, _ := os.Getwd()
|
||||
key := [3]string{cli.DaemonHost(), workdir, os.TempDir()}
|
||||
cached, _ := dockerProxyDirectories.Load(key)
|
||||
directory, seen := cached.(dockerProxyDirectory)
|
||||
seen = seen && os.SameFile(directory.socket, socket) // daemon replacement invalidates cached paths
|
||||
if !seen {
|
||||
directory, seen, err = probeDockerProxyDirectory(probeCtx, cli, key, socket)
|
||||
}
|
||||
probe, err := probeDockerProxyDirectory(probeCtx, cli, key, socket)
|
||||
if err != nil {
|
||||
common.Logger(ctx).Infof("docker proxy probe failed, jobs get the daemon socket directly: %v", err)
|
||||
return nil
|
||||
}
|
||||
if !seen {
|
||||
if !probe.seen {
|
||||
common.Logger(ctx).Infof("the docker daemon cannot reach the runner's temporary or working directory, jobs get the daemon socket directly")
|
||||
return nil
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
return nil
|
||||
}
|
||||
proxy, err := StartDockerProxy(daemonSocket, directory.dir, job)
|
||||
proxy, err := StartDockerProxy(daemonSocket, probe.directory.dir, job)
|
||||
if err != nil {
|
||||
dockerProxyDirectories.Delete(key)
|
||||
dockerProxyProbes.CompareAndDelete(key, probe)
|
||||
common.Logger(ctx).Warnf("docker proxy not started, the job gets the daemon socket directly: %v", err)
|
||||
return nil
|
||||
}
|
||||
dockerProxyDirectories.Store(key, directory)
|
||||
proxy.Socket = directory.daemonDir + strings.TrimPrefix(proxy.Socket, directory.dir)
|
||||
proxy.Socket = probe.directory.daemonDir + strings.TrimPrefix(proxy.Socket, probe.directory.dir)
|
||||
return proxy
|
||||
}
|
||||
|
||||
func probeDockerProxyDirectory(ctx context.Context, cli client.APIClient, key [3]string, socket os.FileInfo) (dockerProxyDirectory, bool, error) {
|
||||
func probeDockerProxyDirectory(ctx context.Context, cli client.APIClient, key [3]string, socket os.FileInfo) (*dockerProxyProbe, error) {
|
||||
for {
|
||||
probe := &dockerProxyProbe{directory: dockerProxyDirectory{socket: socket}, done: make(chan struct{})}
|
||||
actual, loaded := dockerProxyProbes.LoadOrStore(key, probe)
|
||||
if loaded {
|
||||
probe, _ = actual.(*dockerProxyProbe)
|
||||
if !os.SameFile(probe.directory.socket, socket) {
|
||||
if !os.SameFile(probe.directory.socket, socket) { // daemon replacement invalidates cached paths
|
||||
dockerProxyProbes.CompareAndDelete(key, probe)
|
||||
continue
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return dockerProxyDirectory{}, false, ctx.Err()
|
||||
case <-probe.done:
|
||||
}
|
||||
if ctx.Err() == nil && (errors.Is(probe.err, context.Canceled) || errors.Is(probe.err, context.DeadlineExceeded)) {
|
||||
continue
|
||||
}
|
||||
} else {
|
||||
go func() { // outlives callers' deadlines so a slow daemon still gets its result cached
|
||||
defer cli.Close()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), dockerProxyProbeWorkerTimeout)
|
||||
defer cancel()
|
||||
probe.directory.dir, probe.directory.daemonDir = runnerContainerWorkdir(ctx, cli)
|
||||
probe.seen = probe.directory.daemonDir != ""
|
||||
if !probe.seen {
|
||||
@@ -138,10 +130,24 @@ func probeDockerProxyDirectory(ctx context.Context, cli client.APIClient, key [3
|
||||
probe.seen, probe.err = daemonSeesDir(ctx, cli, probe.directory.dir, probe.directory.daemonDir)
|
||||
}
|
||||
}
|
||||
if probe.seen {
|
||||
dockerProxyProbes.LoadOrStore(key, probe)
|
||||
} else {
|
||||
dockerProxyProbes.CompareAndDelete(key, probe)
|
||||
close(probe.done)
|
||||
}
|
||||
return probe.directory, probe.seen, probe.err
|
||||
close(probe.done)
|
||||
removeStaleProbeContainers(cli)
|
||||
}()
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return &dockerProxyProbe{}, ctx.Err()
|
||||
case <-probe.done:
|
||||
}
|
||||
if ctx.Err() == nil && errors.Is(probe.err, context.DeadlineExceeded) {
|
||||
continue
|
||||
}
|
||||
return probe, probe.err
|
||||
}
|
||||
}
|
||||
|
||||
@@ -200,7 +206,7 @@ func daemonSeesDir(ctx context.Context, cli client.APIClient, dir, daemonDir str
|
||||
created, err = cli.ContainerCreate(ctx, client.ContainerCreateOptions{
|
||||
Config: &container.Config{Image: image.ID, Cmd: []string{"true"}},
|
||||
HostConfig: &container.HostConfig{Mounts: []mount.Mount{
|
||||
{Type: mount.TypeBind, Source: daemonDir, Target: "/gitea-runner-probe", ReadOnly: true}, // not the marker itself, podman creates a missing bind source where docker rejects it
|
||||
{Type: mount.TypeBind, Source: daemonDir, Target: dockerProxyProbeTarget, ReadOnly: true}, // not the marker itself, podman creates a missing bind source where docker rejects it
|
||||
}},
|
||||
})
|
||||
if !cerrdefs.IsNotFound(err) {
|
||||
@@ -213,7 +219,7 @@ func daemonSeesDir(ctx context.Context, cli client.APIClient, dir, daemonDir str
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
seen, err := containerSeesMarker(ctx, cli, dir, created.ID, "/gitea-runner-probe")
|
||||
seen, err := containerSeesMarker(ctx, cli, dir, created.ID, dockerProxyProbeTarget)
|
||||
cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), dockerProxyProbeTimeout)
|
||||
defer cancel()
|
||||
if _, removeErr := cli.ContainerRemove(cleanupCtx, created.ID, client.ContainerRemoveOptions{Force: true, RemoveVolumes: true}); removeErr != nil {
|
||||
@@ -222,6 +228,36 @@ func daemonSeesDir(ctx context.Context, cli client.APIClient, dir, daemonDir str
|
||||
return seen, err
|
||||
}
|
||||
|
||||
// removeStaleProbeContainers removes probe containers left by abandoned creates once any probe in flight when listed has ended.
|
||||
func removeStaleProbeContainers(cli client.APIClient) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), dockerProxyProbeTimeout)
|
||||
defer cancel()
|
||||
listed, err := cli.ContainerList(ctx, client.ContainerListOptions{
|
||||
All: true,
|
||||
Filters: make(client.Filters).Add("status", string(container.StateCreated)).Add("volume", dockerProxyProbeTarget),
|
||||
})
|
||||
if err != nil {
|
||||
common.Logger(ctx).Warnf("listing stale docker proxy probe containers failed: %v", err)
|
||||
return
|
||||
}
|
||||
listed.Items = slices.DeleteFunc(listed.Items, func(stale container.Summary) bool {
|
||||
return stale.State != container.StateCreated || !slices.ContainsFunc(stale.Mounts, func(mountPoint container.MountPoint) bool {
|
||||
return mountPoint.Type == mount.TypeBind && !mountPoint.RW && mountPoint.Destination == dockerProxyProbeTarget
|
||||
})
|
||||
})
|
||||
if len(listed.Items) == 0 {
|
||||
return
|
||||
}
|
||||
time.Sleep(dockerProxyProbeWorkerTimeout + dockerProxyProbeTimeout)
|
||||
for _, stale := range listed.Items {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), dockerProxyProbeTimeout)
|
||||
if err := (&containerReference{cli: cli, id: stale.ID}).remove()(ctx); err != nil {
|
||||
common.Logger(ctx).Warnf("removing stale docker proxy probe container failed: %v", err)
|
||||
}
|
||||
cancel()
|
||||
}
|
||||
}
|
||||
|
||||
// StartDockerProxy serves a job's docker socket in dir, labelling what the job creates through it.
|
||||
func StartDockerProxy(daemonSocket, dir, job string) (*DockerProxy, error) {
|
||||
info, err := os.Stat(daemonSocket)
|
||||
|
||||
@@ -94,6 +94,8 @@ func TestDockerProxy(t *testing.T) {
|
||||
return
|
||||
}
|
||||
_, _ = w.Write([]byte(`[{"Id":"probe-image"}]`))
|
||||
case strings.HasSuffix(r.URL.Path, "/containers/json"):
|
||||
_, _ = w.Write([]byte(`[]`))
|
||||
case strings.HasSuffix(r.URL.Path, "/json"):
|
||||
http.NotFound(w, r)
|
||||
case strings.HasSuffix(r.URL.Path, "/archive"):
|
||||
@@ -235,7 +237,8 @@ func TestDockerProxy(t *testing.T) {
|
||||
t.Skip("automatic proxy discovery requires Linux")
|
||||
}
|
||||
t.Setenv("DOCKER_HOST", "unix://"+daemonSocket)
|
||||
t.Setenv("TMPDIR", shortTempDir(t))
|
||||
tempDir := shortTempDir(t)
|
||||
t.Setenv("TMPDIR", tempDir)
|
||||
require.Nil(t, NewDockerProxy(t.Context(), "failed"))
|
||||
for _, job := range []string{"first", "second"} {
|
||||
proxy := NewDockerProxy(t.Context(), job)
|
||||
@@ -243,12 +246,10 @@ func TestDockerProxy(t *testing.T) {
|
||||
require.NoError(t, proxy.Close(context.Background()))
|
||||
}
|
||||
assert.EqualValues(t, 2, probes.Load())
|
||||
dockerProxyDirectories.Range(func(key, value any) bool {
|
||||
directory, _ := value.(dockerProxyDirectory)
|
||||
directory.socket, _ = os.Stat(t.TempDir())
|
||||
dockerProxyDirectories.Store(key, directory)
|
||||
return true
|
||||
})
|
||||
workdir, err := os.Getwd()
|
||||
require.NoError(t, err)
|
||||
cached, _ := dockerProxyProbes.Load([3]string{"unix://" + daemonSocket, workdir, tempDir})
|
||||
cached.(*dockerProxyProbe).directory.socket, _ = os.Stat(t.TempDir())
|
||||
proxy := NewDockerProxy(t.Context(), "replaced daemon")
|
||||
require.NotNil(t, proxy)
|
||||
require.NoError(t, proxy.Close(context.Background()))
|
||||
@@ -476,10 +477,23 @@ func BenchmarkDockerProxy(b *testing.B) {
|
||||
|
||||
type probeClient struct {
|
||||
mobyclient.APIClient
|
||||
create func(mobyclient.ContainerCreateOptions) (mobyclient.ContainerCreateResult, error)
|
||||
create func(context.Context, mobyclient.ContainerCreateOptions) (mobyclient.ContainerCreateResult, error)
|
||||
remove func(context.Context, string, mobyclient.ContainerRemoveOptions) (mobyclient.ContainerRemoveResult, error)
|
||||
stat func(string) error
|
||||
mounts []container.MountPoint
|
||||
containers []container.Summary
|
||||
}
|
||||
|
||||
func (c *probeClient) Close() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *probeClient) ContainerList(context.Context, mobyclient.ContainerListOptions) (mobyclient.ContainerListResult, error) {
|
||||
return mobyclient.ContainerListResult{Items: c.containers}, nil
|
||||
}
|
||||
|
||||
func (c *probeClient) ContainerKill(context.Context, string, mobyclient.ContainerKillOptions) (mobyclient.ContainerKillResult, error) {
|
||||
return mobyclient.ContainerKillResult{}, cerrdefs.ErrConflict
|
||||
}
|
||||
|
||||
func (c *probeClient) ContainerInspect(context.Context, string, mobyclient.ContainerInspectOptions) (mobyclient.ContainerInspectResult, error) {
|
||||
@@ -490,8 +504,8 @@ func (c *probeClient) ImageList(context.Context, mobyclient.ImageListOptions) (m
|
||||
return mobyclient.ImageListResult{Items: []image.Summary{{ID: "removed-image"}, {ID: "probe-image"}}}, nil
|
||||
}
|
||||
|
||||
func (c *probeClient) ContainerCreate(_ context.Context, opts mobyclient.ContainerCreateOptions) (mobyclient.ContainerCreateResult, error) {
|
||||
return c.create(opts)
|
||||
func (c *probeClient) ContainerCreate(ctx context.Context, opts mobyclient.ContainerCreateOptions) (mobyclient.ContainerCreateResult, error) {
|
||||
return c.create(ctx, opts)
|
||||
}
|
||||
|
||||
func (c *probeClient) ContainerRemove(ctx context.Context, id string, opts mobyclient.ContainerRemoveOptions) (mobyclient.ContainerRemoveResult, error) {
|
||||
@@ -518,7 +532,7 @@ func TestDockerProxyDirectoryProbes(t *testing.T) {
|
||||
var calls, removals atomic.Int32
|
||||
visible := false
|
||||
cli := &probeClient{
|
||||
create: func(_opts mobyclient.ContainerCreateOptions) (mobyclient.ContainerCreateResult, error) {
|
||||
create: func(_ctx context.Context, _opts mobyclient.ContainerCreateOptions) (mobyclient.ContainerCreateResult, error) {
|
||||
calls.Add(1)
|
||||
<-release
|
||||
return mobyclient.ContainerCreateResult{ID: "probe"}, createErr
|
||||
@@ -538,23 +552,18 @@ func TestDockerProxyDirectoryProbes(t *testing.T) {
|
||||
results := make(chan result, 8)
|
||||
for range cap(results) {
|
||||
go func() {
|
||||
_, seen, err := probeDockerProxyDirectory(t.Context(), cli, key, socket)
|
||||
results <- result{seen, err}
|
||||
probe, err := probeDockerProxyDirectory(t.Context(), cli, key, socket)
|
||||
results <- result{probe.seen, err}
|
||||
}()
|
||||
}
|
||||
synctest.Wait()
|
||||
assert.EqualValues(t, 1, calls.Load())
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
cancel()
|
||||
_, seen, err := probeDockerProxyDirectory(ctx, cli, key, socket)
|
||||
require.ErrorIs(t, err, context.Canceled)
|
||||
assert.False(t, seen)
|
||||
replacement, err := os.Stat(t.TempDir())
|
||||
require.NoError(t, err)
|
||||
replaced := make(chan result, 1)
|
||||
go func() {
|
||||
_, seen, err := probeDockerProxyDirectory(t.Context(), cli, key, replacement)
|
||||
replaced <- result{seen, err}
|
||||
probe, err := probeDockerProxyDirectory(t.Context(), cli, key, replacement)
|
||||
replaced <- result{probe.seen, err}
|
||||
}()
|
||||
synctest.Wait()
|
||||
assert.EqualValues(t, 2, calls.Load())
|
||||
@@ -574,11 +583,11 @@ func TestDockerProxyDirectoryProbes(t *testing.T) {
|
||||
assert.Zero(t, removals.Load())
|
||||
}
|
||||
createErr, visible = nil, true
|
||||
directory, seen, err := probeDockerProxyDirectory(t.Context(), cli, key, socket)
|
||||
probe, err := probeDockerProxyDirectory(t.Context(), cli, key, socket)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, seen)
|
||||
assert.Equal(t, dir, directory.dir)
|
||||
assert.Equal(t, dir, directory.daemonDir)
|
||||
assert.True(t, probe.seen)
|
||||
assert.Equal(t, dir, probe.directory.dir)
|
||||
assert.Equal(t, dir, probe.directory.daemonDir)
|
||||
assert.EqualValues(t, 3, calls.Load())
|
||||
entries, err := os.ReadDir(dir)
|
||||
require.NoError(t, err)
|
||||
@@ -586,7 +595,7 @@ func TestDockerProxyDirectoryProbes(t *testing.T) {
|
||||
})
|
||||
})
|
||||
}
|
||||
t.Run("cancelled probe does not cancel another caller", func(t *testing.T) {
|
||||
t.Run("cancelled caller leaves the probe to finish for the others", func(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
socket, err := os.Stat(t.TempDir())
|
||||
@@ -595,49 +604,105 @@ func TestDockerProxyDirectoryProbes(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
defer cancel()
|
||||
release := make(chan struct{})
|
||||
calls := 0
|
||||
var calls, removals atomic.Int32
|
||||
cli := &probeClient{
|
||||
create: func(_opts mobyclient.ContainerCreateOptions) (mobyclient.ContainerCreateResult, error) {
|
||||
calls++
|
||||
if calls == 1 {
|
||||
create: func(ctx context.Context, _opts mobyclient.ContainerCreateOptions) (mobyclient.ContainerCreateResult, error) {
|
||||
calls.Add(1)
|
||||
<-release
|
||||
return mobyclient.ContainerCreateResult{}, ctx.Err()
|
||||
} else {
|
||||
return mobyclient.ContainerCreateResult{ID: "probe"}, nil
|
||||
return mobyclient.ContainerCreateResult{ID: "probe"}, ctx.Err()
|
||||
},
|
||||
stat: func(_path string) error { return nil },
|
||||
remove: func(_ctx context.Context, _id string, _opts mobyclient.ContainerRemoveOptions) (mobyclient.ContainerRemoveResult, error) {
|
||||
removals.Add(1)
|
||||
return mobyclient.ContainerRemoveResult{}, nil
|
||||
},
|
||||
}
|
||||
leader, follower, late := make(chan result, 1), make(chan result, 1), make(chan result, 1)
|
||||
go func() {
|
||||
probe, err := probeDockerProxyDirectory(ctx, cli, key, socket)
|
||||
leader <- result{probe.seen, err}
|
||||
}()
|
||||
synctest.Wait()
|
||||
go func() {
|
||||
probe, err := probeDockerProxyDirectory(t.Context(), cli, key, socket)
|
||||
follower <- result{probe.seen, err}
|
||||
}()
|
||||
synctest.Wait()
|
||||
cancel()
|
||||
leaderResult := <-leader
|
||||
require.ErrorIs(t, leaderResult.err, context.Canceled)
|
||||
assert.False(t, leaderResult.seen)
|
||||
go func() {
|
||||
probe, err := probeDockerProxyDirectory(t.Context(), cli, key, socket)
|
||||
late <- result{probe.seen, err}
|
||||
}()
|
||||
synctest.Wait()
|
||||
close(release)
|
||||
for _, results := range []chan result{follower, late} {
|
||||
result := <-results
|
||||
require.NoError(t, result.err)
|
||||
assert.True(t, result.seen)
|
||||
}
|
||||
assert.EqualValues(t, 1, calls.Load())
|
||||
assert.EqualValues(t, 1, removals.Load())
|
||||
_, cached := dockerProxyProbes.Load(key)
|
||||
assert.True(t, cached)
|
||||
})
|
||||
})
|
||||
t.Run("caller joining a probe that ran out of time retries", func(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
socket, err := os.Stat(t.TempDir())
|
||||
require.NoError(t, err)
|
||||
var calls atomic.Int32
|
||||
cli := &probeClient{
|
||||
create: func(ctx context.Context, _opts mobyclient.ContainerCreateOptions) (mobyclient.ContainerCreateResult, error) {
|
||||
if calls.Add(1) == 1 {
|
||||
<-ctx.Done()
|
||||
return mobyclient.ContainerCreateResult{}, ctx.Err()
|
||||
}
|
||||
return mobyclient.ContainerCreateResult{ID: "probe"}, nil
|
||||
},
|
||||
stat: func(_path string) error { return nil },
|
||||
remove: func(_ctx context.Context, _id string, _opts mobyclient.ContainerRemoveOptions) (mobyclient.ContainerRemoveResult, error) {
|
||||
return mobyclient.ContainerRemoveResult{}, nil
|
||||
},
|
||||
}
|
||||
leader, follower := make(chan result, 1), make(chan result, 1)
|
||||
go func() {
|
||||
_, seen, err := probeDockerProxyDirectory(ctx, cli, key, socket)
|
||||
leader <- result{seen, err}
|
||||
}()
|
||||
synctest.Wait()
|
||||
go func() {
|
||||
_, seen, err := probeDockerProxyDirectory(t.Context(), cli, key, socket)
|
||||
follower <- result{seen, err}
|
||||
}()
|
||||
synctest.Wait()
|
||||
assert.Equal(t, 1, calls)
|
||||
cancel()
|
||||
close(release)
|
||||
leaderResult, followerResult := <-leader, <-follower
|
||||
require.ErrorIs(t, leaderResult.err, context.Canceled)
|
||||
assert.False(t, leaderResult.seen)
|
||||
require.NoError(t, followerResult.err)
|
||||
assert.True(t, followerResult.seen)
|
||||
assert.Equal(t, 2, calls)
|
||||
probe, err := probeDockerProxyDirectory(t.Context(), cli, [3]string{"daemon", dir, dir}, socket)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, probe.seen)
|
||||
assert.EqualValues(t, 2, calls.Load())
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
func TestRemoveStaleProbeContainers(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
probeMount := container.MountPoint{Type: mount.TypeBind, Destination: "/gitea-runner-probe"}
|
||||
writableMount := container.MountPoint{Type: mount.TypeBind, Destination: "/gitea-runner-probe", RW: true}
|
||||
var removed []string
|
||||
cli := &probeClient{
|
||||
containers: []container.Summary{
|
||||
{ID: "abandoned", State: container.StateCreated, Mounts: []container.MountPoint{probeMount}},
|
||||
{ID: "foreign", State: container.StateCreated, Mounts: []container.MountPoint{writableMount}},
|
||||
{ID: "started", State: container.StateExited, Mounts: []container.MountPoint{probeMount}},
|
||||
},
|
||||
remove: func(_ctx context.Context, id string, _opts mobyclient.ContainerRemoveOptions) (mobyclient.ContainerRemoveResult, error) {
|
||||
removed = append(removed, id)
|
||||
return mobyclient.ContainerRemoveResult{}, nil
|
||||
},
|
||||
}
|
||||
start := time.Now()
|
||||
removeStaleProbeContainers(cli)
|
||||
assert.Equal(t, []string{"abandoned"}, removed)
|
||||
assert.GreaterOrEqual(t, time.Since(start), dockerProxyProbeWorkerTimeout+dockerProxyProbeTimeout)
|
||||
})
|
||||
}
|
||||
|
||||
func TestDaemonSeesDir(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
markers := make(map[string]bool)
|
||||
|
||||
for _, testCase := range []struct {
|
||||
name string
|
||||
daemonDir string
|
||||
@@ -656,7 +721,7 @@ func TestDaemonSeesDir(t *testing.T) {
|
||||
removed := false
|
||||
var attemptedImages []string
|
||||
cli := &probeClient{
|
||||
create: func(opts mobyclient.ContainerCreateOptions) (mobyclient.ContainerCreateResult, error) {
|
||||
create: func(_ctx context.Context, opts mobyclient.ContainerCreateOptions) (mobyclient.ContainerCreateResult, error) {
|
||||
assert.Equal(t, []mount.Mount{{Type: mount.TypeBind, Source: testCase.daemonDir, Target: "/gitea-runner-probe", ReadOnly: true}}, opts.HostConfig.Mounts)
|
||||
attemptedImages = append(attemptedImages, opts.Config.Image)
|
||||
if opts.Config.Image == "removed-image" {
|
||||
@@ -692,6 +757,7 @@ func TestDaemonSeesDir(t *testing.T) {
|
||||
assert.Equal(t, []string{"removed-image", "probe-image"}, attemptedImages)
|
||||
assert.Equal(t, testCase.seen, seen)
|
||||
assert.Equal(t, testCase.removed, removed)
|
||||
|
||||
entries, err := os.ReadDir(dir)
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, entries)
|
||||
|
||||
@@ -980,7 +980,7 @@ func TestDockerCreateKeepsInputEntrypoint(t *testing.T) {
|
||||
RunnerOptions: "--entrypoint=runner-entrypoint",
|
||||
WorkflowOptions: "--entrypoint=" + optionEntrypoint + " --init",
|
||||
},
|
||||
cli: &probeClient{create: func(opts mobyclient.ContainerCreateOptions) (mobyclient.ContainerCreateResult, error) {
|
||||
cli: &probeClient{create: func(_ctx context.Context, opts mobyclient.ContainerCreateOptions) (mobyclient.ContainerCreateResult, error) {
|
||||
if len(entrypoint) > 0 {
|
||||
assert.Equal(t, entrypoint, opts.Config.Entrypoint)
|
||||
} else {
|
||||
|
||||
+95
-10
@@ -7,12 +7,15 @@ import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"cmp"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"maps"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
@@ -23,6 +26,7 @@ import (
|
||||
"time"
|
||||
|
||||
"gitea.com/gitea/runner/act/common"
|
||||
"gitea.com/gitea/runner/act/common/logging"
|
||||
|
||||
"github.com/kballard/go-shellquote"
|
||||
"github.com/spf13/pflag"
|
||||
@@ -36,6 +40,7 @@ type KubernetesOptions struct {
|
||||
MaxLifetime time.Duration
|
||||
ForcePull bool
|
||||
RunnerUUID string
|
||||
Transfer *KubernetesTransfer
|
||||
}
|
||||
|
||||
// KubernetesPod runs a job container and its service containers as one pod.
|
||||
@@ -46,6 +51,10 @@ type KubernetesPod struct {
|
||||
job *kubernetesContainer
|
||||
services []*kubernetesContainer
|
||||
arch, path string
|
||||
ips []net.IP
|
||||
transfer *transferServer
|
||||
transferImage string
|
||||
transferHelper string // empty when copies go through the API server
|
||||
}
|
||||
|
||||
type kubernetesContainer struct {
|
||||
@@ -55,6 +64,7 @@ type kubernetesContainer struct {
|
||||
name string
|
||||
input *NewContainerInput
|
||||
probe map[string]any
|
||||
gzip bool // the container can gunzip, so copies into it go compressed
|
||||
}
|
||||
|
||||
type containerStatus struct {
|
||||
@@ -74,6 +84,7 @@ type podStatus struct {
|
||||
Spec struct{ Containers []struct{} }
|
||||
Status struct {
|
||||
Phase, Reason, Message string
|
||||
PodIPs []struct{ IP string }
|
||||
Conditions []struct{ Type, Status, Message string }
|
||||
InitContainerStatuses, ContainerStatuses []containerStatus
|
||||
}
|
||||
@@ -129,14 +140,24 @@ func (p *KubernetesPod) create(ctx context.Context) error {
|
||||
labels[runnerUUIDLabel] = p.options.RunnerUUID
|
||||
}
|
||||
metadata := map[string]any{"name": name, "labels": labels}
|
||||
var runnerImage string
|
||||
if hostname, err := os.Hostname(); p.client.inCluster && err == nil {
|
||||
var runner struct{ Metadata struct{ Name, UID string } }
|
||||
var runner struct {
|
||||
Metadata struct{ Name, UID string }
|
||||
Spec struct{ Containers []struct{ Image string } }
|
||||
}
|
||||
if err := p.client.do(ctx, http.MethodGet, p.client.path("pods/"+hostname), nil, &runner); err == nil {
|
||||
metadata["ownerReferences"] = []any{map[string]any{"apiVersion": "v1", "kind": "Pod", "name": runner.Metadata.Name, "uid": runner.Metadata.UID}}
|
||||
if len(runner.Spec.Containers) == 1 {
|
||||
runnerImage = runner.Spec.Containers[0].Image
|
||||
}
|
||||
} else {
|
||||
common.Logger(ctx).Debugf("The runner is not pod %s, job pods get no owner: %v", hostname, err)
|
||||
}
|
||||
}
|
||||
if err := p.setupTransfer(runnerImage); err != nil {
|
||||
return err
|
||||
}
|
||||
secretData := map[string]string{}
|
||||
manifest, err := p.manifest(name, secretData, metadata)
|
||||
if err != nil {
|
||||
@@ -205,6 +226,18 @@ func (p *KubernetesPod) manifest(secret string, secretData map[string]string, me
|
||||
}
|
||||
}
|
||||
spec := map[string]any{"restartPolicy": "Never", "containers": containers, "volumes": volumes}
|
||||
if p.transferHelper != "" {
|
||||
transfer := map[string]any{
|
||||
"name": "gitea-runner-transfer", "image": p.transferImage, "terminationMessagePolicy": "FallbackToLogsOnError",
|
||||
"command": []string{"gitea-runner", KubernetesTransferCommand, "install", p.transferHelper},
|
||||
"volumeMounts": []any{map[string]any{"name": "act", "mountPath": p.job.GetActPath()}},
|
||||
"securityContext": map[string]any{"allowPrivilegeEscalation": false, "capabilities": map[string]any{"drop": []string{"ALL"}}},
|
||||
}
|
||||
if p.options.ForcePull {
|
||||
transfer["imagePullPolicy"] = "Always"
|
||||
}
|
||||
spec["initContainers"] = []any{transfer}
|
||||
}
|
||||
if len(hostnames) > 0 {
|
||||
spec["hostAliases"] = []any{map[string]any{"ip": "127.0.0.1", "hostnames": hostnames}}
|
||||
}
|
||||
@@ -341,7 +374,13 @@ func (p *KubernetesPod) waitStarted(ctx context.Context) error {
|
||||
return err
|
||||
}
|
||||
if pod.Status.Phase == "Failed" {
|
||||
return fmt.Errorf("pod %s failed: %s", p.name, cmp.Or(pod.Status.Message, pod.Status.Reason))
|
||||
reason := cmp.Or(pod.Status.Message, pod.Status.Reason)
|
||||
for _, status := range append(pod.Status.InitContainerStatuses, pod.Status.ContainerStatuses...) {
|
||||
if terminated := status.State.Terminated; reason == "" && terminated != nil && terminated.ExitCode != 0 {
|
||||
reason = fmt.Sprintf("container %s exited with %d: %s", status.Name, terminated.ExitCode, strings.TrimSpace(terminated.Message))
|
||||
}
|
||||
}
|
||||
return fmt.Errorf("pod %s failed: %s", p.name, reason)
|
||||
}
|
||||
started := len(pod.Status.ContainerStatuses) == len(pod.Spec.Containers)
|
||||
for _, status := range append(pod.Status.InitContainerStatuses, pod.Status.ContainerStatuses...) {
|
||||
@@ -354,6 +393,9 @@ func (p *KubernetesPod) waitStarted(ctx context.Context) error {
|
||||
}
|
||||
}
|
||||
if started {
|
||||
for _, ip := range pod.Status.PodIPs {
|
||||
p.ips = append(p.ips, net.ParseIP(ip.IP))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
for _, condition := range pod.Status.Conditions {
|
||||
@@ -391,8 +433,11 @@ func (c *kubernetesContainer) Start(_ bool) common.Executor {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
machine, path, _ := strings.Cut(output, "\n")
|
||||
c.pod.arch, c.pod.path = goArchToActionArch(machine), path
|
||||
probed := strings.SplitN(output, "\n", 3)
|
||||
for len(probed) < 3 {
|
||||
probed = append(probed, "")
|
||||
}
|
||||
c.pod.arch, c.gzip, c.pod.path = goArchToActionArch(probed[0]), probed[1] == "gzip", probed[2]
|
||||
return nil
|
||||
}
|
||||
}
|
||||
@@ -403,7 +448,7 @@ func (p *KubernetesPod) probe(ctx context.Context, container string) (string, er
|
||||
deadline := time.Now().Add(time.Minute)
|
||||
for delay := 100 * time.Millisecond; ; delay = min(2*delay, 2*time.Second) {
|
||||
var stdout bytes.Buffer
|
||||
err := p.client.exec(ctx, p.name, container, []string{"sh", "-c", `printf '%s\n%s' "$(uname -m)" "$PATH"`}, nil, &stdout, io.Discard)
|
||||
err := p.client.exec(ctx, p.name, container, []string{"sh", "-c", `printf '%s\n%s\n%s' "$(uname -m)" "$(command -v gzip >/dev/null && echo gzip)" "$PATH"`}, nil, &stdout, io.Discard)
|
||||
var exitErr ExitCodeError
|
||||
var apiErr *apiError
|
||||
if err == nil || ctx.Err() != nil || errors.As(err, &exitErr) || time.Now().After(deadline) ||
|
||||
@@ -452,26 +497,66 @@ func (c *kubernetesContainer) Exec(command []string, env map[string]string, _, w
|
||||
}
|
||||
}
|
||||
|
||||
func (c *kubernetesContainer) extract(ctx context.Context, destPath string, write func(io.Writer) error) error {
|
||||
// compressCopyMin is the size from which Copy compresses and uses the transfer helper: below it that saves next to nothing on the wire and costs a process in the pod.
|
||||
const compressCopyMin = 64 << 10
|
||||
|
||||
// extract streams the tar write produces into destPath, gzipped with compress, as copied actions shrink about fivefold and API server traffic often crosses zones.
|
||||
func (c *kubernetesContainer) extract(ctx context.Context, destPath string, compress bool, write func(io.Writer) error) error {
|
||||
if compress && c.pod.transferHelper != "" {
|
||||
request, done := c.pod.transfer.offer(c.pod.ips, write)
|
||||
stdin, _ := json.Marshal(request)
|
||||
var stderr bytes.Buffer
|
||||
err := c.pod.client.exec(ctx, c.pod.name, c.name, []string{c.pod.transferHelper, KubernetesTransferCommand, "fetch", destPath}, bytes.NewReader(stdin), io.Discard, &stderr)
|
||||
taken, writeErr := done()
|
||||
if err == nil || ctx.Err() != nil {
|
||||
return err
|
||||
}
|
||||
if taken || !c.pod.options.Transfer.Fallback {
|
||||
return fmt.Errorf("copy to %s over the pod network: %w: %s", destPath, errors.Join(writeErr, err), strings.TrimSpace(stderr.String()))
|
||||
}
|
||||
c.pod.transferHelper = "" // a pod that cannot reach the runner once will not later
|
||||
logging.Warnf("Copying into pod %s over the pod network failed, copying through the API server instead: %v: %s", c.pod.name, err, strings.TrimSpace(stderr.String()))
|
||||
}
|
||||
compress = compress && c.gzip
|
||||
reader, writer := io.Pipe()
|
||||
defer reader.Close()
|
||||
go func() { writer.CloseWithError(write(writer)) }()
|
||||
go func() {
|
||||
if compress {
|
||||
writer.CloseWithError(writeGzip(writer, write))
|
||||
} else {
|
||||
writer.CloseWithError(write(writer))
|
||||
}
|
||||
}()
|
||||
script := `mkdir -p "$1" && tar -xf - -C "$1"`
|
||||
if compress {
|
||||
script = `mkdir -p "$1" && gzip -dc | tar -xf - -C "$1"` // gzip -dc rather than tar -z, which GNU tar runs gzip for anyway
|
||||
}
|
||||
var stderr bytes.Buffer
|
||||
if err := c.pod.client.exec(ctx, c.pod.name, c.name, []string{"sh", "-c", `mkdir -p "$1" && tar -xf - -C "$1"`, "sh", destPath}, reader, io.Discard, &stderr); err != nil {
|
||||
if err := c.pod.client.exec(ctx, c.pod.name, c.name, []string{"sh", "-c", script, "sh", destPath}, reader, io.Discard, &stderr); err != nil {
|
||||
return fmt.Errorf("extract to %s: %w: %s", destPath, err, strings.TrimSpace(stderr.String()))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// writeGzip gzips what write produces into w, at a level fast enough to not slow copies down and nearly as small as the default.
|
||||
func writeGzip(w io.Writer, write func(io.Writer) error) error {
|
||||
compressed, _ := gzip.NewWriterLevel(w, gzip.BestSpeed)
|
||||
return errors.Join(write(compressed), compressed.Close())
|
||||
}
|
||||
|
||||
func (c *kubernetesContainer) Copy(destPath string, files ...*FileEntry) common.Executor {
|
||||
return func(ctx context.Context) error {
|
||||
return c.extract(ctx, destPath, func(writer io.Writer) error { return writeFilesTar(ctx, writer, 0, 0, files...) })
|
||||
size := 0
|
||||
for _, file := range files {
|
||||
size += len(file.Body)
|
||||
}
|
||||
return c.extract(ctx, destPath, size >= compressCopyMin, func(writer io.Writer) error { return writeFilesTar(ctx, writer, 0, 0, files...) })
|
||||
}
|
||||
}
|
||||
|
||||
func (c *kubernetesContainer) CopyDir(destPath, srcPath string, useGitIgnore, skipGitDir bool) common.Executor {
|
||||
return func(ctx context.Context) error {
|
||||
return c.extract(ctx, destPath, func(writer io.Writer) error {
|
||||
return c.extract(ctx, destPath, true, func(writer io.Writer) error { // directories are actions and workspaces, megabytes of compressible sources
|
||||
return writeDirTar(ctx, writer, "/", srcPath, useGitIgnore, skipGitDir, 0, 0)
|
||||
})
|
||||
}
|
||||
|
||||
@@ -260,6 +260,7 @@ func TestKubernetesPodRunsJobThroughAPI(t *testing.T) {
|
||||
)
|
||||
require.NoError(t, job.Start(false)(ctx))
|
||||
assert.Len(t, cluster.requests["exec"], 3)
|
||||
assert.True(t, pod.job.gzip)
|
||||
|
||||
stepEnv := map[string]string{"INPUT_WHO-TO-GREET": "it's\nme", "SECRET": "hunter2", "UID": "5"}
|
||||
require.NoError(t, job.Exec([]string{"printenv", "SECRET"}, stepEnv, "", "")(ctx))
|
||||
@@ -276,7 +277,9 @@ func TestKubernetesPodRunsJobThroughAPI(t *testing.T) {
|
||||
assert.NotContains(t, query, "hunter2")
|
||||
}
|
||||
|
||||
lastExec := func() string { return cluster.requests["exec"][len(cluster.requests["exec"])-1] }
|
||||
require.NoError(t, job.Copy(workdir+"/act/", &FileEntry{Name: "workflow/event.json", Mode: 0o644, Body: `{"ref":"main"}`})(ctx))
|
||||
assert.NotContains(t, lastExec(), "gzip")
|
||||
archive, err := job.GetContainerArchive(ctx, workdir+"/act/workflow/event.json")
|
||||
require.NoError(t, err)
|
||||
reader := tar.NewReader(archive)
|
||||
@@ -289,6 +292,12 @@ func TestKubernetesPodRunsJobThroughAPI(t *testing.T) {
|
||||
require.NoError(t, archive.Close())
|
||||
_, err = job.GetContainerArchive(ctx, workdir+"/missing")
|
||||
require.Error(t, err)
|
||||
large := strings.Repeat("a", compressCopyMin)
|
||||
require.NoError(t, job.Copy(workdir+"/act/", &FileEntry{Name: "workflow/large", Mode: 0o644, Body: large})(ctx))
|
||||
assert.Contains(t, lastExec(), url.QueryEscape("gzip -dc |"))
|
||||
content, err = os.ReadFile(workdir + "/act/workflow/large")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, large, string(content))
|
||||
|
||||
actionDir := filepath.Join(t.TempDir(), "action")
|
||||
require.NoError(t, os.MkdirAll(actionDir, 0o755))
|
||||
@@ -297,7 +306,8 @@ func TestKubernetesPodRunsJobThroughAPI(t *testing.T) {
|
||||
script, err := os.ReadFile(workdir + "/act/actions/sha/index.js")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "main()", string(script))
|
||||
assert.Contains(t, cluster.requests["exec"][len(cluster.requests["exec"])-1], "command="+url.QueryEscape(workdir+"/act/actions/sha/"))
|
||||
assert.Contains(t, lastExec(), "command="+url.QueryEscape(workdir+"/act/actions/sha/"))
|
||||
assert.Contains(t, lastExec(), url.QueryEscape("gzip -dc |"))
|
||||
|
||||
pathEnv := map[string]string{}
|
||||
require.NoError(t, job.UpdateFromImageEnv(&pathEnv)(ctx))
|
||||
@@ -321,6 +331,38 @@ func TestKubernetesPodRunsJobThroughAPI(t *testing.T) {
|
||||
assert.Empty(t, cluster.requests["DELETE /api/v1/namespaces/ci/secrets/new"])
|
||||
}
|
||||
|
||||
func TestKubernetesCopiesUncompressedWithoutGzip(t *testing.T) {
|
||||
t.Parallel()
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("uses POSIX shell")
|
||||
}
|
||||
ctx := t.Context()
|
||||
workdir, bin := t.TempDir(), t.TempDir()
|
||||
for _, tool := range []string{"uname", "tar", "mkdir"} { // all the job container needs but gzip, as on a minimal image
|
||||
found, err := exec.LookPath(tool)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, os.Symlink(found, filepath.Join(bin, tool)))
|
||||
}
|
||||
kubeconfig, cluster := fakeKubernetes(t, `{"spec":{"containers":[{}]},"status":{"phase":"Running","containerStatuses":[
|
||||
{"name":"job","ready":true,"state":{"running":{}}}]}}`, "PATH="+bin)
|
||||
pod, err := NewKubernetesPod(KubernetesOptions{Kubeconfig: kubeconfig})
|
||||
require.NoError(t, err)
|
||||
job := pod.JobContainer(&NewContainerInput{Image: "node", WorkingDir: workdir})
|
||||
require.NoError(t, job.Create(nil, nil)(ctx))
|
||||
require.NoError(t, job.Start(false)(ctx))
|
||||
assert.False(t, pod.job.gzip)
|
||||
assert.Equal(t, bin, pod.path)
|
||||
|
||||
actionDir := filepath.Join(t.TempDir(), "action")
|
||||
require.NoError(t, os.MkdirAll(actionDir, 0o755))
|
||||
require.NoError(t, os.WriteFile(filepath.Join(actionDir, "index.js"), []byte("main()"), 0o644))
|
||||
require.NoError(t, job.CopyDir(workdir+"/act/actions/sha/", actionDir+"/", false, false)(ctx))
|
||||
script, err := os.ReadFile(workdir + "/act/actions/sha/index.js")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "main()", string(script))
|
||||
assert.NotContains(t, cluster.requests["exec"][len(cluster.requests["exec"])-1], "gzip")
|
||||
}
|
||||
|
||||
func TestKubernetesPodFailsBeforeSteps(t *testing.T) {
|
||||
kubeconfig, _ := fakeKubernetes(t, `{"spec":{"containers":[{}]},"status":{"containerStatuses":[
|
||||
{"name":"job","state":{"waiting":{"reason":"ErrImagePull","message":"no such image"}}}]}}`)
|
||||
@@ -361,6 +403,13 @@ func TestKubernetesPodFailsBeforeSteps(t *testing.T) {
|
||||
job := pod.JobContainer(&NewContainerInput{Image: "missing"})
|
||||
require.NoError(t, job.Create(nil, nil)(t.Context()))
|
||||
require.ErrorContains(t, job.Start(false)(t.Context()), "ErrImagePull: no such image")
|
||||
failed, _ := fakeKubernetes(t, `{"spec":{"containers":[{}]},"status":{"phase":"Failed","initContainerStatuses":[
|
||||
{"name":"gitea-runner-transfer","state":{"terminated":{"exitCode":1,"message":"exec format error\n"}}}]}}`)
|
||||
pod, err = NewKubernetesPod(KubernetesOptions{Kubeconfig: failed})
|
||||
require.NoError(t, err)
|
||||
job = pod.JobContainer(&NewContainerInput{Image: "node"})
|
||||
require.NoError(t, job.Create(nil, nil)(t.Context()))
|
||||
require.ErrorContains(t, job.Start(false)(t.Context()), "container gitea-runner-transfer exited with 1: exec format error")
|
||||
|
||||
plugin := filepath.Join(t.TempDir(), "kubeconfig")
|
||||
require.NoError(t, os.WriteFile(plugin, []byte(`
|
||||
|
||||
@@ -0,0 +1,248 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package container
|
||||
|
||||
import (
|
||||
"cmp"
|
||||
"context"
|
||||
"crypto/ecdsa"
|
||||
"crypto/elliptic"
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"math/big"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"gitea.com/gitea/runner/act/common"
|
||||
"gitea.com/gitea/runner/act/common/logging"
|
||||
)
|
||||
|
||||
// KubernetesTransfer has job pods fetch large copies from the runner over the pod network instead of through the API server.
|
||||
type KubernetesTransfer struct {
|
||||
Host string // empty uses the outbound IP
|
||||
Port uint16 // 0 picks a free port
|
||||
Image string // has gitea-runner on its PATH, for an init container to copy into each job pod
|
||||
Fallback bool // copy through the API server when a job pod cannot reach the runner, else fail
|
||||
}
|
||||
|
||||
// KubernetesTransferCommand is the hidden runner subcommand job pods run the transfer helper with.
|
||||
const KubernetesTransferCommand = "kubernetes-transfer"
|
||||
|
||||
// transferRequest goes to the helper on the exec stream's stdin, keeping the token out of the command line and the audit log.
|
||||
type transferRequest struct {
|
||||
URL string `json:"url"`
|
||||
Pin string `json:"pin"` // hex SHA-256 of the runner's certificate
|
||||
}
|
||||
|
||||
type transferServer struct {
|
||||
base string
|
||||
pin string
|
||||
mu sync.Mutex
|
||||
pending map[string]transferOffer
|
||||
}
|
||||
|
||||
type transferOffer struct {
|
||||
clients []net.IP
|
||||
write func(io.Writer) error
|
||||
result chan error
|
||||
}
|
||||
|
||||
var (
|
||||
transferServersMu sync.Mutex
|
||||
transferServers = map[string]*transferServer{}
|
||||
)
|
||||
|
||||
// startTransferServer returns the runner's server for these settings, started on first use.
|
||||
func startTransferServer(host string, port uint16) (*transferServer, error) {
|
||||
transferServersMu.Lock()
|
||||
defer transferServersMu.Unlock()
|
||||
configured := net.JoinHostPort(host, strconv.Itoa(int(port)))
|
||||
if server, ok := transferServers[configured]; ok {
|
||||
return server, nil
|
||||
}
|
||||
if host == "" {
|
||||
ip := common.GetOutboundIP()
|
||||
if ip == nil {
|
||||
return nil, errors.New("no outbound IP for job pods to reach the runner at, set kubernetes.transfer.host")
|
||||
}
|
||||
host = ip.String()
|
||||
}
|
||||
privateKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
template := &x509.Certificate{SerialNumber: big.NewInt(1)} // trusted by its pin alone, so no CA, names or validity
|
||||
certificate, err := x509.CreateCertificate(rand.Reader, template, template, &privateKey.PublicKey, privateKey)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
listener, err := net.Listen("tcp", net.JoinHostPort(host, strconv.Itoa(int(port))))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("listen for job pod transfers: %w", err)
|
||||
}
|
||||
pin := sha256.Sum256(certificate)
|
||||
server := &transferServer{base: "https://" + listener.Addr().String() + "/", pin: hex.EncodeToString(pin[:]), pending: map[string]transferOffer{}}
|
||||
httpServer := &http.Server{
|
||||
Handler: server,
|
||||
TLSConfig: &tls.Config{Certificates: []tls.Certificate{{Certificate: [][]byte{certificate}, PrivateKey: privateKey}}, MinVersion: tls.VersionTLS13},
|
||||
ReadHeaderTimeout: 10 * time.Second,
|
||||
IdleTimeout: 30 * time.Second,
|
||||
}
|
||||
go func() { _ = httpServer.ServeTLS(listener, "", "") }()
|
||||
transferServers[configured] = server
|
||||
return server, nil
|
||||
}
|
||||
|
||||
// offer serves write once to one of clients until done, which reports whether a client took it and what write failed with.
|
||||
func (s *transferServer) offer(clients []net.IP, write func(io.Writer) error) (request transferRequest, done func() (taken bool, err error)) {
|
||||
token := rand.Text()
|
||||
result := make(chan error, 1)
|
||||
s.mu.Lock()
|
||||
s.pending[token] = transferOffer{clients: clients, write: write, result: result}
|
||||
s.mu.Unlock()
|
||||
return transferRequest{URL: s.base + token, Pin: s.pin}, func() (bool, error) {
|
||||
s.mu.Lock()
|
||||
_, pending := s.pending[token]
|
||||
delete(s.pending, token)
|
||||
s.mu.Unlock()
|
||||
select {
|
||||
case err := <-result:
|
||||
return true, err
|
||||
default:
|
||||
return !pending, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *transferServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
host, _, _ := net.SplitHostPort(r.RemoteAddr)
|
||||
token := strings.TrimPrefix(r.URL.Path, "/")
|
||||
s.mu.Lock()
|
||||
offer, ok := s.pending[token]
|
||||
ok = ok && r.Method == http.MethodGet && slices.ContainsFunc(offer.clients, net.ParseIP(host).Equal)
|
||||
if ok {
|
||||
delete(s.pending, token)
|
||||
}
|
||||
s.mu.Unlock()
|
||||
if !ok {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Encoding", "gzip")
|
||||
err := writeGzip(w, offer.write)
|
||||
offer.result <- err
|
||||
if err != nil {
|
||||
panic(http.ErrAbortHandler) // a cut-off body fails the fetch, a short tar might not
|
||||
}
|
||||
}
|
||||
|
||||
// FetchKubernetesTransfer reads a transferRequest from request and extracts the tar it points to into dest.
|
||||
func FetchKubernetesTransfer(ctx context.Context, request io.Reader, dest string) error {
|
||||
var transfer transferRequest
|
||||
if err := json.NewDecoder(request).Decode(&transfer); err != nil {
|
||||
return fmt.Errorf("read transfer request: %w", err)
|
||||
}
|
||||
client := &http.Client{Transport: &http.Transport{
|
||||
DialContext: (&net.Dialer{Timeout: 10 * time.Second}).DialContext,
|
||||
TLSHandshakeTimeout: 10 * time.Second,
|
||||
ResponseHeaderTimeout: time.Minute,
|
||||
TLSClientConfig: &tls.Config{
|
||||
MinVersion: tls.VersionTLS13,
|
||||
InsecureSkipVerify: true, // VerifyConnection checks the pin instead
|
||||
VerifyConnection: func(state tls.ConnectionState) error {
|
||||
if sum := sha256.Sum256(state.PeerCertificates[0].Raw); hex.EncodeToString(sum[:]) != transfer.Pin {
|
||||
return errors.New("runner certificate does not match the pin")
|
||||
}
|
||||
return nil
|
||||
},
|
||||
},
|
||||
}}
|
||||
httpRequest, err := http.NewRequestWithContext(ctx, http.MethodGet, transfer.URL, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
response, err := client.Do(httpRequest)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer response.Body.Close()
|
||||
if response.StatusCode != http.StatusOK {
|
||||
return fmt.Errorf("fetch: %s", response.Status)
|
||||
}
|
||||
if err := os.MkdirAll(dest, 0o777); err != nil {
|
||||
return err
|
||||
}
|
||||
cmd := exec.CommandContext(ctx, "tar", "-xf", "-", "-C", dest)
|
||||
cmd.Stdin, cmd.Stdout, cmd.Stderr = response.Body, os.Stderr, os.Stderr
|
||||
if err := cmd.Run(); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = io.Copy(io.Discard, response.Body) // tar stops at the first zero blocks, a cut-off body after them still fails
|
||||
return err
|
||||
}
|
||||
|
||||
// InstallKubernetesTransferHelper copies the running binary to target.
|
||||
func InstallKubernetesTransferHelper(target string) error {
|
||||
executable, err := os.Executable()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
source, err := os.Open(executable)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer source.Close()
|
||||
if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
_ = os.Remove(target) // a file left in a reused act volume would keep its mode
|
||||
file, err := os.OpenFile(target, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o755)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = io.Copy(file, source)
|
||||
return errors.Join(err, file.Close())
|
||||
}
|
||||
|
||||
// setupTransfer turns on transfers for the pod, with the runner pod's image unless one is set.
|
||||
func (p *KubernetesPod) setupTransfer(runnerImage string) error {
|
||||
settings := p.options.Transfer
|
||||
if settings == nil {
|
||||
return nil
|
||||
}
|
||||
image := cmp.Or(settings.Image, runnerImage)
|
||||
var err error
|
||||
var server *transferServer
|
||||
if image == "" {
|
||||
err = errors.New("set kubernetes.transfer.image, the runner is no single-container pod in kubernetes.namespace to take it from")
|
||||
} else {
|
||||
server, err = startTransferServer(settings.Host, settings.Port)
|
||||
}
|
||||
if err != nil {
|
||||
if !settings.Fallback {
|
||||
return fmt.Errorf("kubernetes.transfer: %w", err)
|
||||
}
|
||||
logging.Warnf("Job pod copies go through the API server: %v", err)
|
||||
return nil
|
||||
}
|
||||
p.transfer, p.transferImage = server, image
|
||||
p.transferHelper = path.Join(p.job.GetActPath(), "bin", "gitea-runner")
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,185 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package container
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"crypto/tls"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
const transferHelperEnv = "GITEA_RUNNER_TEST_TRANSFER_HELPER"
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
if os.Getenv(transferHelperEnv) == "1" && len(os.Args) == 4 && os.Args[1] == KubernetesTransferCommand && os.Args[2] == "fetch" {
|
||||
if err := FetchKubernetesTransfer(context.Background(), os.Stdin, os.Args[3]); err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
os.Exit(0)
|
||||
}
|
||||
os.Exit(m.Run())
|
||||
}
|
||||
|
||||
func fetch(t *testing.T, request transferRequest) error {
|
||||
t.Helper()
|
||||
body, err := json.Marshal(request)
|
||||
require.NoError(t, err)
|
||||
return FetchKubernetesTransfer(t.Context(), bytes.NewReader(body), t.TempDir())
|
||||
}
|
||||
|
||||
func TestTransferServer(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("extracts with tar")
|
||||
}
|
||||
server, err := startTransferServer("127.0.0.1", 0)
|
||||
require.NoError(t, err)
|
||||
again, err := startTransferServer("127.0.0.1", 0)
|
||||
require.NoError(t, err)
|
||||
assert.Same(t, server, again)
|
||||
loopback := []net.IP{net.ParseIP("127.0.0.1")}
|
||||
nothing := func(io.Writer) error { return nil }
|
||||
|
||||
t.Run("serves an offer gzipped once to an IP of its job pod", func(t *testing.T) {
|
||||
request, done := server.offer([]net.IP{net.ParseIP("fd00::1"), loopback[0]}, nothing)
|
||||
client := &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, DisableCompression: true}}
|
||||
response, err := client.Get(request.URL)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, response.Body.Close())
|
||||
assert.Equal(t, "gzip", response.Header.Get("Content-Encoding"))
|
||||
require.ErrorContains(t, fetch(t, request), "404")
|
||||
taken, _ := done()
|
||||
assert.True(t, taken)
|
||||
})
|
||||
|
||||
t.Run("another pod can neither take nor block an offer", func(t *testing.T) {
|
||||
request, done := server.offer([]net.IP{net.ParseIP("10.1.2.3")}, nothing)
|
||||
require.ErrorContains(t, fetch(t, request), "404")
|
||||
taken, _ := done()
|
||||
assert.False(t, taken)
|
||||
})
|
||||
|
||||
t.Run("a failed write fails the fetch even past the end blocks tar stops at, and done reports it", func(t *testing.T) {
|
||||
request, done := server.offer(loopback, func(w io.Writer) error {
|
||||
_, _ = w.Write(make([]byte, 1024))
|
||||
_, _ = io.CopyN(w, rand.Reader, 1<<20)
|
||||
return errors.New("source gone")
|
||||
})
|
||||
require.Error(t, fetch(t, request))
|
||||
taken, err := done()
|
||||
assert.True(t, taken)
|
||||
require.EqualError(t, err, "source gone")
|
||||
})
|
||||
|
||||
t.Run("the helper refuses a runner whose certificate does not match its pin", func(t *testing.T) {
|
||||
request, done := server.offer(loopback, nothing)
|
||||
request.Pin = strings.Repeat("0", 64)
|
||||
require.ErrorContains(t, fetch(t, request), "does not match the pin")
|
||||
taken, _ := done()
|
||||
assert.False(t, taken)
|
||||
})
|
||||
}
|
||||
|
||||
func TestKubernetesPodTransferImage(t *testing.T) {
|
||||
newPod := func(image string, fallback bool) *KubernetesPod {
|
||||
pod := &KubernetesPod{options: KubernetesOptions{Transfer: &KubernetesTransfer{Host: "127.0.0.1", Image: image, Fallback: fallback}}}
|
||||
pod.JobContainer(&NewContainerInput{})
|
||||
return pod
|
||||
}
|
||||
|
||||
pod := newPod("", false)
|
||||
require.NoError(t, pod.setupTransfer("gitea/runner:latest"))
|
||||
assert.Equal(t, "gitea/runner:latest", pod.transferImage)
|
||||
|
||||
pod = newPod("registry.example/runner:1", false)
|
||||
require.NoError(t, pod.setupTransfer("gitea/runner:latest"))
|
||||
assert.Equal(t, "registry.example/runner:1", pod.transferImage)
|
||||
|
||||
pod = newPod("", true)
|
||||
require.NoError(t, pod.setupTransfer(""))
|
||||
assert.Empty(t, pod.transferHelper)
|
||||
|
||||
pod = newPod("", false)
|
||||
require.ErrorContains(t, pod.setupTransfer(""), "kubernetes.transfer.image")
|
||||
}
|
||||
|
||||
func TestKubernetesPodCopiesLargeFilesOverPodNetwork(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("uses POSIX shell")
|
||||
}
|
||||
ctx := t.Context()
|
||||
workdir, err := filepath.EvalSymlinks(t.TempDir())
|
||||
require.NoError(t, err)
|
||||
kubeconfig, cluster := fakeKubernetes(t, `{"spec":{"containers":[{}]},"status":{"phase":"Running","podIPs":[{"ip":"127.0.0.1"}],"containerStatuses":[
|
||||
{"name":"job","ready":true,"state":{"running":{}}}]}}`, transferHelperEnv+"=1")
|
||||
pod, err := NewKubernetesPod(KubernetesOptions{Kubeconfig: kubeconfig, ForcePull: true, Transfer: &KubernetesTransfer{Host: "127.0.0.1", Image: "gitea/runner:latest", Fallback: true}})
|
||||
require.NoError(t, err)
|
||||
job := pod.JobContainer(&NewContainerInput{Name: "job", Image: "node", WorkingDir: workdir, Stdout: io.Discard, Stderr: io.Discard})
|
||||
require.NoError(t, job.Create(nil, nil)(ctx))
|
||||
|
||||
var manifest struct {
|
||||
Spec struct {
|
||||
InitContainers []struct {
|
||||
Image, ImagePullPolicy string
|
||||
Command []string
|
||||
VolumeMounts []struct{ Name, MountPath string }
|
||||
}
|
||||
}
|
||||
}
|
||||
require.NoError(t, json.Unmarshal([]byte(cluster.requests["POST /api/v1/namespaces/ci/pods"][0]), &manifest))
|
||||
require.Len(t, manifest.Spec.InitContainers, 1)
|
||||
initContainer := manifest.Spec.InitContainers[0]
|
||||
assert.Equal(t, "gitea/runner:latest", initContainer.Image)
|
||||
assert.Equal(t, "Always", initContainer.ImagePullPolicy)
|
||||
assert.Equal(t, []string{"gitea-runner", KubernetesTransferCommand, "install", "/var/run/act/bin/gitea-runner"}, initContainer.Command)
|
||||
assert.Equal(t, []struct{ Name, MountPath string }{{"act", "/var/run/act"}}, initContainer.VolumeMounts)
|
||||
require.NoError(t, job.Start(false)(ctx))
|
||||
|
||||
helper := filepath.Join(t.TempDir(), "bin", "gitea-runner")
|
||||
require.NoError(t, InstallKubernetesTransferHelper(helper))
|
||||
pod.transferHelper = helper
|
||||
lastExec := func() string { return cluster.requests["exec"][len(cluster.requests["exec"])-1] }
|
||||
large := &FileEntry{Name: "large", Mode: 0o644, Body: strings.Repeat("a", compressCopyMin)}
|
||||
copyLarge := func(dir string) error { return job.Copy(filepath.Join(workdir, dir)+"/", large)(ctx) }
|
||||
require.NoError(t, copyLarge("transferred"))
|
||||
content, err := os.ReadFile(filepath.Join(workdir, "transferred", "large"))
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, large.Body, string(content))
|
||||
assert.Contains(t, lastExec(), "command="+KubernetesTransferCommand)
|
||||
assert.NotContains(t, lastExec(), "127.0.0.1")
|
||||
|
||||
require.NoError(t, job.Copy(workdir+"/act/", &FileEntry{Name: "event.json", Mode: 0o644, Body: `{}`})(ctx))
|
||||
assert.NotContains(t, lastExec(), "command="+KubernetesTransferCommand)
|
||||
|
||||
require.NoError(t, os.WriteFile(filepath.Join(workdir, "file"), nil, 0o644))
|
||||
require.ErrorContains(t, copyLarge("file"), "over the pod network")
|
||||
assert.Equal(t, helper, pod.transferHelper)
|
||||
|
||||
unreachable := filepath.Join(workdir, "unreachable")
|
||||
require.NoError(t, os.WriteFile(unreachable, []byte("#!/bin/sh\necho 'connection refused' >&2\nexit 1\n"), 0o755))
|
||||
pod.transferHelper = unreachable
|
||||
pod.options.Transfer.Fallback = false
|
||||
require.ErrorContains(t, copyLarge("direct"), "connection refused")
|
||||
assert.NoFileExists(t, filepath.Join(workdir, "direct", "large"))
|
||||
|
||||
pod.options.Transfer.Fallback = true
|
||||
require.NoError(t, copyLarge("auto"))
|
||||
assert.FileExists(t, filepath.Join(workdir, "auto", "large"))
|
||||
assert.Empty(t, pod.transferHelper)
|
||||
}
|
||||
@@ -180,9 +180,9 @@ func runActionImpl(step actionStep, actionDir string, remoteAction *remoteAction
|
||||
|
||||
action := step.getActionModel()
|
||||
|
||||
rc.withGithubEnv(ctx, step.getGithubContext(ctx), *step.getEnv())
|
||||
rc.withGithubEnv(ctx, getStepGithubContext(ctx, step), *step.getEnv())
|
||||
populateEnvsFromSavedState(step.getEnv(), step, rc)
|
||||
if err := populateEnvsFromInput(ctx, step.getEnv(), action, rc); err != nil {
|
||||
if err := populateEnvsFromInput(ctx, step, action); err != nil {
|
||||
return err
|
||||
}
|
||||
warnUnexpectedInputs(ctx, action, stepModel.With)
|
||||
@@ -283,7 +283,7 @@ func (rc *RunContext) dockerActionImage(ctx context.Context, step actionStep, ac
|
||||
}
|
||||
image, ok := job.dockerActionImages[step.getStepModel()]
|
||||
if !ok {
|
||||
image = dockerActionImageTag(step.getGithubContext(ctx).Repository, actionName, localAction) + "-" + strings.ToLower(rand.Text())
|
||||
image = dockerActionImageTag(getStepGithubContext(ctx, step).Repository, actionName, localAction) + "-" + strings.ToLower(rand.Text())
|
||||
if !common.Dryrun(ctx) { // a dry run builds nothing to remove
|
||||
job.dockerActionImages[step.getStepModel()] = image
|
||||
}
|
||||
@@ -437,7 +437,7 @@ func evalDockerEnv(ctx context.Context, step step, action *model.Action) (*expre
|
||||
for key := range step.getStepModel().With {
|
||||
providedInputs[strings.ToLower(key)] = true
|
||||
}
|
||||
eval := rc.NewExpressionEvaluator(ctx)
|
||||
eval := rc.NewStepExpressionEvaluator(ctx, step)
|
||||
for k, input := range action.Inputs {
|
||||
if providedInputs[strings.ToLower(k)] {
|
||||
continue
|
||||
@@ -456,6 +456,7 @@ func evalDockerEnv(ctx context.Context, step step, action *model.Action) (*expre
|
||||
}
|
||||
}
|
||||
mergeIntoMap(step, step.getEnv(), runsEnv, maps.Clone(*step.getEnv()))
|
||||
aliasGiteaEnv(*step.getEnv())
|
||||
return ee, nil
|
||||
}
|
||||
|
||||
@@ -509,8 +510,9 @@ func inputEnvKey(inputID string) string {
|
||||
return "INPUT_" + inputEnvPattern.ReplaceAllString(strings.ToUpper(inputID), "_")
|
||||
}
|
||||
|
||||
func populateEnvsFromInput(ctx context.Context, env *map[string]string, action *model.Action, rc *RunContext) error {
|
||||
eval := rc.NewExpressionEvaluator(ctx)
|
||||
func populateEnvsFromInput(ctx context.Context, step step, action *model.Action) error {
|
||||
env := step.getEnv()
|
||||
eval := step.getRunContext().NewStepExpressionEvaluator(ctx, step)
|
||||
for inputID, input := range action.Inputs {
|
||||
envKey := inputEnvKey(inputID)
|
||||
if _, ok := (*env)[envKey]; !ok {
|
||||
@@ -652,7 +654,7 @@ func runPreStep(step actionStep) common.Executor {
|
||||
x := action.Runs.Using
|
||||
if !x.IsComposite() {
|
||||
// defaults in pre steps were missing, however provided inputs are available
|
||||
if err := populateEnvsFromInput(ctx, step.getEnv(), action, rc); err != nil {
|
||||
if err := populateEnvsFromInput(ctx, step, action); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
@@ -57,9 +57,6 @@ func evaluateCompositeInputAndEnv(ctx context.Context, parent *RunContext, step
|
||||
env[envKey] = value
|
||||
}
|
||||
}
|
||||
gh := step.getGithubContext(ctx)
|
||||
env["GITHUB_ACTION_REPOSITORY"] = gh.ActionRepository
|
||||
env["GITHUB_ACTION_REF"] = gh.ActionRef
|
||||
|
||||
return env, nil
|
||||
}
|
||||
@@ -110,6 +107,7 @@ func newCompositeRunContext(ctx context.Context, parent *RunContext, step action
|
||||
kubernetes: parent.kubernetes,
|
||||
jobIndex: parent.jobIndex,
|
||||
jobTotal: parent.jobTotal,
|
||||
runtimeToken: parent.runtimeToken,
|
||||
}
|
||||
compositerc.setCompositeActionEnv(env)
|
||||
compositerc.ExprEval = compositerc.NewExpressionEvaluator(ctx)
|
||||
|
||||
@@ -19,6 +19,35 @@ import (
|
||||
)
|
||||
|
||||
func TestCompositeActionParity(t *testing.T) {
|
||||
t.Run("input defaults use the step identity and docker env keeps the GITEA aliases", func(t *testing.T) {
|
||||
for _, uses := range []string{"./action", "own/action@v2"} {
|
||||
t.Run(uses, func(t *testing.T) {
|
||||
rc := &RunContext{
|
||||
Config: &Config{PresetGitHubContext: &model.GithubContext{}},
|
||||
Run: &model.Run{JobID: "job", Workflow: &model.Workflow{Jobs: map[string]*model.Job{"job": {}}}},
|
||||
Env: map[string]string{"GITHUB_ACTION_REPOSITORY": "other/composite", "GITHUB_ACTION_REF": "v1"},
|
||||
}
|
||||
var currentStep step = &stepActionLocal{RunContext: rc, Step: &model.Step{Uses: uses}, env: map[string]string{}}
|
||||
want := "|"
|
||||
if uses == "own/action@v2" {
|
||||
currentStep = &stepActionRemote{RunContext: rc, Step: &model.Step{Uses: uses}, env: map[string]string{}, remoteAction: mustNewRemoteAction(t, uses)}
|
||||
want = "own/action|v2"
|
||||
}
|
||||
action := &model.Action{
|
||||
Inputs: map[string]model.Input{"identity": {Default: "${{ github.action_repository }}|${{ github.action_ref }}"}},
|
||||
Runs: model.ActionRuns{Env: map[string]string{"GITHUB_WORKFLOW_SHA": "runs-sha"}},
|
||||
}
|
||||
require.NoError(t, setupEnv(t.Context(), currentStep))
|
||||
require.NoError(t, populateEnvsFromInput(t.Context(), currentStep, action))
|
||||
assert.Equal(t, want, (*currentStep.getEnv())["INPUT_IDENTITY"])
|
||||
_, err := evalDockerEnv(t.Context(), currentStep, action)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, want, (*currentStep.getEnv())["identity"])
|
||||
assert.Equal(t, "runs-sha", (*currentStep.getEnv())["GITEA_WORKFLOW_SHA"])
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("inherits contexts without leaking inputs", func(t *testing.T) {
|
||||
hook := logtest.NewWithLevel(logging.LevelWarn)
|
||||
ctx := common.WithLogger(t.Context(), logging.New(hook))
|
||||
@@ -54,7 +83,7 @@ func TestCompositeActionParity(t *testing.T) {
|
||||
assert.NotContains(t, composite.Env, "INPUT_EXTRA")
|
||||
|
||||
nestedEnv := composite.GetEnv()
|
||||
require.NoError(t, populateEnvsFromInput(ctx, &nestedEnv, &model.Action{Inputs: map[string]model.Input{"shared": {Default: "inner-default"}}}, composite))
|
||||
require.NoError(t, populateEnvsFromInput(ctx, &stepActionLocal{RunContext: composite, env: nestedEnv}, &model.Action{Inputs: map[string]model.Input{"shared": {Default: "inner-default"}}}))
|
||||
assert.Equal(t, "inner-default", nestedEnv["INPUT_SHARED"])
|
||||
})
|
||||
|
||||
|
||||
@@ -90,18 +90,18 @@ var hashfiles string
|
||||
|
||||
// NewStepExpressionEvaluator creates a new evaluator with the `inputs` of the enclosing workflow or composite action
|
||||
func (rc *RunContext) NewStepExpressionEvaluator(ctx context.Context, step step) *ExpressionEvaluator {
|
||||
return rc.newStepExpressionEvaluator(ctx, step, rc.actionInputs, rc.getJobContext())
|
||||
return rc.newStepExpressionEvaluator(ctx, step, rc.actionInputs, rc.getJobContext(), getStepGithubContext(ctx, step))
|
||||
}
|
||||
|
||||
// NewActionInputsExpressionEvaluator creates a new evaluator with the step's own with: values as `inputs`
|
||||
func (rc *RunContext) NewActionInputsExpressionEvaluator(ctx context.Context, step step) *ExpressionEvaluator {
|
||||
return rc.newStepExpressionEvaluator(ctx, step, inputsFromEnv(*step.getEnv()), rc.getJobContext())
|
||||
return rc.newStepExpressionEvaluator(ctx, step, inputsFromEnv(*step.getEnv()), rc.getJobContext(), getStepGithubContext(ctx, step))
|
||||
}
|
||||
|
||||
func (rc *RunContext) newStepExpressionEvaluator(ctx context.Context, step step, stepInputs map[string]any, jobContext *model.JobContext) *ExpressionEvaluator {
|
||||
func (rc *RunContext) newStepExpressionEvaluator(ctx context.Context, step step, stepInputs map[string]any, jobContext *model.JobContext, github *model.GithubContext) *ExpressionEvaluator {
|
||||
// todo: cleanup EvaluationEnvironment creation
|
||||
ee := &exprparser.EvaluationEnvironment{
|
||||
Github: step.getGithubContext(ctx),
|
||||
Github: github,
|
||||
Env: *step.getEnv(),
|
||||
Job: jobContext,
|
||||
Steps: rc.StepResults,
|
||||
@@ -112,7 +112,7 @@ func (rc *RunContext) newStepExpressionEvaluator(ctx context.Context, step step,
|
||||
Needs: exprparser.NeedsContext(rc.Run),
|
||||
// todo: should be unavailable
|
||||
// but required to interpolate/evaluate the inputs in actions/composite
|
||||
Inputs: getEvaluatorInputs(rc, stepInputs, rc.getGithubContext(ctx)),
|
||||
Inputs: getEvaluatorInputs(rc, stepInputs, github),
|
||||
HashFiles: getHashFilesFunction(ctx, rc),
|
||||
}
|
||||
ee.Runner = rc.getRunnerContext(ctx)
|
||||
@@ -332,5 +332,10 @@ func getWorkflowSecrets(rc *RunContext) map[string]string {
|
||||
if rc.caller != nil {
|
||||
return rc.workflowCallSecrets
|
||||
}
|
||||
if rc.runtimeToken == "" {
|
||||
return rc.Config.Secrets
|
||||
}
|
||||
secrets := map[string]string{"ACTIONS_RUNTIME_TOKEN": rc.runtimeToken} // as the daemon exposes its token
|
||||
maps.Copy(secrets, rc.Config.Secrets)
|
||||
return secrets
|
||||
}
|
||||
@@ -504,7 +504,7 @@ jobs:
|
||||
assert.Equal(t, "workflow", env["INPUT_FALLBACK"])
|
||||
}
|
||||
|
||||
func TestResolveWorkflowCallToleratesMismatchesAndPassesInheritedSecretsVerbatim(t *testing.T) {
|
||||
func TestResolveWorkflowCallToleratesMismatchesAndPassesInheritedSecretsAndRuntimeToken(t *testing.T) {
|
||||
callerWorkflow, err := model.ReadWorkflow(strings.NewReader(`
|
||||
jobs:
|
||||
call:
|
||||
@@ -527,7 +527,11 @@ jobs:
|
||||
require.NoError(t, err)
|
||||
notCallable, err := model.ReadWorkflow(strings.NewReader("on: push\njobs:\n test: {}\n"))
|
||||
require.NoError(t, err)
|
||||
config := &Config{Secrets: map[string]string{"token": "${{ github.token }}"}, PresetGitHubContext: &model.GithubContext{}}
|
||||
config := &Config{
|
||||
Secrets: map[string]string{"token": "${{ github.token }}"},
|
||||
PresetGitHubContext: &model.GithubContext{},
|
||||
RuntimeTokens: map[*model.Workflow]string{callerWorkflow: "caller-run"},
|
||||
}
|
||||
resolve := func(callerJobID string, workflow *model.Workflow) *RunContext {
|
||||
parent, err := (&runnerImpl{config: config}).newRunContext(t.Context(), &model.Run{Workflow: callerWorkflow, JobID: callerJobID}, nil)
|
||||
require.NoError(t, err)
|
||||
@@ -539,9 +543,10 @@ jobs:
|
||||
called := resolve("call", callee)
|
||||
assert.Equal(t, map[string]any{"flag": true}, called.workflowCallInputs)
|
||||
assert.Equal(t, map[string]string{"undeclared": "${{ github.token }}"}, called.workflowCallSecrets)
|
||||
assert.Equal(t, "caller-run", called.GetEnv()["ACTIONS_RUNTIME_TOKEN"])
|
||||
inherited := resolve("inherit", notCallable)
|
||||
assert.Empty(t, inherited.workflowCallInputs)
|
||||
assert.Equal(t, config.Secrets, inherited.workflowCallSecrets)
|
||||
assert.Equal(t, map[string]string{"token": "${{ github.token }}", "ACTIONS_RUNTIME_TOKEN": "caller-run"}, inherited.workflowCallSecrets)
|
||||
}
|
||||
|
||||
func TestJobNameMasksSecrets(t *testing.T) {
|
||||
|
||||
@@ -111,7 +111,7 @@ func printPrepareActions(rc *RunContext, preparers []actionPreparer) common.Exec
|
||||
if !ok || strings.Contains(action.Step.Uses, "${{") || action.action != nil {
|
||||
continue
|
||||
}
|
||||
github := action.getGithubContext(common.WithLogger(ctx, logging.New(logging.DiscardHandler)))
|
||||
github := action.RunContext.getGithubContext(common.WithLogger(ctx, logging.New(logging.DiscardHandler)))
|
||||
remote, err := newRemoteAction(action.Step.Uses, github, action.RunContext.Config.GitHubInstance)
|
||||
if err != nil || action.skipsLocalCheckout(remote, github) {
|
||||
continue
|
||||
|
||||
@@ -1049,21 +1049,23 @@ func TestApplyJobTimeout(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
yaml string
|
||||
wantTimeout bool
|
||||
want time.Duration
|
||||
}{
|
||||
{"empty", "runs-on: ubuntu-latest", false},
|
||||
{"integer", "timeout-minutes: 5\nruns-on: ubuntu-latest", true},
|
||||
{"zero ignored", "timeout-minutes: 0\nruns-on: ubuntu-latest", false},
|
||||
{"non-numeric ignored", "timeout-minutes: abc\nruns-on: ubuntu-latest", false},
|
||||
{"empty", "runs-on: ubuntu-latest", 0},
|
||||
{"integer", "timeout-minutes: 5\nruns-on: ubuntu-latest", 5 * time.Minute},
|
||||
{"zero ignored", "timeout-minutes: 0\nruns-on: ubuntu-latest", 0},
|
||||
{"non-numeric ignored", "timeout-minutes: abc\nruns-on: ubuntu-latest", 0},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
rc := makeTestRC(t, tc.yaml)
|
||||
ctx := t.Context()
|
||||
newCtx, cancel := applyJobTimeout(ctx, rc, rc.Run.Job())
|
||||
newCtx, cancel := applyJobTimeout(t.Context(), rc, rc.Run.Job())
|
||||
defer cancel()
|
||||
_, hasDeadline := newCtx.Deadline()
|
||||
assert.Equal(t, tc.wantTimeout, hasDeadline)
|
||||
deadline, hasDeadline := newCtx.Deadline()
|
||||
assert.Equal(t, tc.want != 0, hasDeadline)
|
||||
if hasDeadline {
|
||||
assert.WithinDuration(t, time.Now().Add(tc.want), deadline, time.Minute)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -84,8 +84,7 @@ func (rc *RunContext) setupHookFileCommands(ctx context.Context, env map[string]
|
||||
actPath := rc.JobContainer.GetActPath()
|
||||
env["GITHUB_ENV"] = path.Join(actPath, hookEnvFileCommand)
|
||||
env["GITHUB_PATH"] = path.Join(actPath, hookPathFileCommand)
|
||||
env["GITEA_ENV"] = env["GITHUB_ENV"]
|
||||
env["GITEA_PATH"] = env["GITHUB_PATH"]
|
||||
aliasGiteaEnv(env)
|
||||
|
||||
return rc.JobContainer.Copy(actPath,
|
||||
&container.FileEntry{Name: hookEnvFileCommand, Mode: 0o666},
|
||||
|
||||
@@ -121,6 +121,7 @@ func TestRunJobHook(t *testing.T) {
|
||||
assert.Equal(t, "job", jobContainer.env["GITHUB_JOB"])
|
||||
assert.Equal(t, "/var/run/act/workflow/hook-envs.txt", jobContainer.env["GITHUB_ENV"])
|
||||
assert.Equal(t, "/var/run/act/workflow/hook-path.txt", jobContainer.env["GITHUB_PATH"])
|
||||
assert.Equal(t, jobContainer.env["GITHUB_PATH"], jobContainer.env["GITEA_PATH"])
|
||||
})
|
||||
|
||||
t.Run("each hook has its own command state", func(t *testing.T) {
|
||||
|
||||
@@ -23,13 +23,14 @@ func newLocalReusableWorkflowExecutor(rc *RunContext) common.Executor {
|
||||
return common.NewErrorExecutor(err)
|
||||
}
|
||||
|
||||
if !rc.Config.NoSkipCheckout {
|
||||
preset := rc.Config.PresetGitHubContext
|
||||
if preset == nil {
|
||||
// resolve the local workflow against the workspace root, not the process
|
||||
// working directory, so it is found regardless of where the runner is invoked
|
||||
return newReusableWorkflowExecutor(rc, rc.Config.Workdir, localWorkflow.Path, false)
|
||||
}
|
||||
|
||||
uses := fmt.Sprintf("%s/%s@%s", rc.Config.PresetGitHubContext.Repository, localWorkflow.Path, rc.Config.PresetGitHubContext.Sha)
|
||||
uses := fmt.Sprintf("%s/%s@%s", preset.Repository, localWorkflow.Path, preset.Sha)
|
||||
|
||||
remoteReusableWorkflow, err := newRemoteReusableWorkflow(rc.Config.GitHubInstance, uses)
|
||||
if err != nil {
|
||||
|
||||
@@ -122,7 +122,7 @@ func TestNewLocalReusableWorkflowExecutorFindsSameRepositoryPaths(t *testing.T)
|
||||
|
||||
for _, uses := range []string{"./.gitea/workflows/reusable.yml", "$/.gitea/workflows/reusable.yml"} {
|
||||
rc := &RunContext{
|
||||
Config: &Config{Workdir: workdir},
|
||||
Config: &Config{Workdir: workdir, NoSkipCheckout: true},
|
||||
Run: &model.Run{JobID: "job", Workflow: &model.Workflow{Jobs: map[string]*model.Job{"job": {Uses: uses}}}},
|
||||
}
|
||||
err := newLocalReusableWorkflowExecutor(rc)(t.Context())
|
||||
|
||||
@@ -81,6 +81,7 @@ type RunContext struct {
|
||||
jobRunDefaults model.RunDefaults // defaults.run, resolved once at job setup as GitHub does
|
||||
platformImage string // container.image or the runs-on pick, resolved once by isEnabled
|
||||
jobIndex, jobTotal int // strategy.job-index and job-total, total 0 when unknown
|
||||
runtimeToken string // ACTIONS_RUNTIME_TOKEN of the workflow run, shared with its called workflows and composite actions
|
||||
|
||||
jobEnv map[string]string // workflow and job env as evaluated for the container and service specs
|
||||
dockerActionImages map[*model.Step]string // Dockerfile action images built for this job, removed when it ends
|
||||
@@ -193,6 +194,9 @@ func (rc *RunContext) GetEnv() map[string]string {
|
||||
rc.Env = mergeMaps(rc.Run.Workflow.Env, job.Environment(), rc.Config.Env)
|
||||
}
|
||||
}
|
||||
if rc.runtimeToken != "" {
|
||||
rc.Env["ACTIONS_RUNTIME_TOKEN"] = rc.runtimeToken // as the daemon's Config.Env carries its token
|
||||
}
|
||||
}
|
||||
if !rc.Config.DisableActEnv {
|
||||
rc.Env["ACT"] = "true"
|
||||
@@ -1598,6 +1602,23 @@ func isLocalCheckout(ghc *model.GithubContext, step *model.Step) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
// aliasGiteaEnv mirrors the runner's GITHUB_* variables as GITEA_*, call it after the last write to them.
|
||||
func aliasGiteaEnv(env map[string]string) {
|
||||
for _, suffix := range []string{
|
||||
"ACTION", "ACTION_PATH", "ACTION_REF", "ACTION_REPOSITORY", "ACTIONS", "ACTOR", "ACTOR_ID", "API_URL", "BASE_REF",
|
||||
"ENV", "EVENT_NAME", "EVENT_PATH", "GRAPHQL_URL", "HEAD_REF", "JOB", "OUTPUT", "PATH", "REF", "REF_NAME",
|
||||
"REF_PROTECTED", "REF_TYPE", "REPOSITORY", "REPOSITORY_ID", "REPOSITORY_OWNER", "REPOSITORY_OWNER_ID",
|
||||
"RETENTION_DAYS", "RUN_ATTEMPT", "RUN_ID", "RUN_NUMBER", "SERVER_URL", "SHA", "STATE", "STEP_SUMMARY",
|
||||
"TRIGGERING_ACTOR", "WORKFLOW", "WORKFLOW_REF", "WORKFLOW_SHA", "WORKSPACE",
|
||||
} {
|
||||
if value, ok := env["GITHUB_"+suffix]; ok {
|
||||
env["GITEA_"+suffix] = value
|
||||
} else {
|
||||
delete(env, "GITEA_"+suffix)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (rc *RunContext) withGithubEnv(ctx context.Context, github *model.GithubContext, env map[string]string) {
|
||||
if _, ok := env["CI"]; !ok {
|
||||
env["CI"] = "true"
|
||||
@@ -1650,6 +1671,7 @@ func (rc *RunContext) withGithubEnv(ctx context.Context, github *model.GithubCon
|
||||
|
||||
env["GITHUB_GRAPHQL_URL"] = ""
|
||||
env["GITHUB_RUN_ATTEMPT"] = github.RunAttempt
|
||||
aliasGiteaEnv(env)
|
||||
|
||||
env["RUNNER_NAME"] = rc.Config.RunnerName
|
||||
env["RUNNER_ENVIRONMENT"] = "self-hosted"
|
||||
@@ -1662,7 +1684,7 @@ func (rc *RunContext) withGithubEnv(ctx context.Context, github *model.GithubCon
|
||||
|
||||
if rc.Config.ArtifactServerPath != "" {
|
||||
env["ACTIONS_RUNTIME_URL"] = cmp.Or(os.Getenv("ACTIONS_RUNTIME_URL"), "http://"+net.JoinHostPort(rc.Config.ArtifactServerAddr, rc.Config.ArtifactServerPort)+"/")
|
||||
env["ACTIONS_RUNTIME_TOKEN"] = cmp.Or(os.Getenv("ACTIONS_RUNTIME_TOKEN"), "token")
|
||||
env["ACTIONS_RUNTIME_TOKEN"] = cmp.Or(rc.runtimeToken, os.Getenv("ACTIONS_RUNTIME_TOKEN"), "token")
|
||||
}
|
||||
|
||||
if imageOS := rc.imageOS(ctx); imageOS != "" {
|
||||
|
||||
@@ -877,25 +877,25 @@ func TestGetGitHubContext(t *testing.T) {
|
||||
RepositoryID: "34",
|
||||
RepositoryOwnerID: "56",
|
||||
WorkflowRef: "owner/repo/.gitea/workflows/test.yml@refs/heads/main",
|
||||
WorkflowSha: "workflow-sha",
|
||||
RefProtected: true,
|
||||
}
|
||||
ghc = rc.getGithubContext(t.Context())
|
||||
assert.Equal(t, "preset-actor", ghc.Actor)
|
||||
assert.Equal(t, "triggerer", ghc.TriggeringActor)
|
||||
assert.Equal(t, "job1", ghc.Job)
|
||||
env := map[string]string{}
|
||||
env := map[string]string{"GITHUB_WORKFLOW_SHA": "workflow-sha"}
|
||||
rc.withGithubEnv(t.Context(), ghc, env)
|
||||
for name, value := range map[string]string{
|
||||
"GITHUB_ACTOR_ID": "12",
|
||||
"GITHUB_TRIGGERING_ACTOR": "triggerer",
|
||||
"GITHUB_REPOSITORY_ID": "34",
|
||||
"GITHUB_REPOSITORY_OWNER_ID": "56",
|
||||
"GITHUB_WORKFLOW_REF": "owner/repo/.gitea/workflows/test.yml@refs/heads/main",
|
||||
"GITHUB_WORKFLOW_SHA": "workflow-sha",
|
||||
"GITHUB_REF_PROTECTED": "true",
|
||||
for suffix, value := range map[string]string{
|
||||
"ACTOR_ID": "12",
|
||||
"TRIGGERING_ACTOR": "triggerer",
|
||||
"REPOSITORY_ID": "34",
|
||||
"REPOSITORY_OWNER_ID": "56",
|
||||
"WORKFLOW_REF": "owner/repo/.gitea/workflows/test.yml@refs/heads/main",
|
||||
"WORKFLOW_SHA": "workflow-sha",
|
||||
"REF_PROTECTED": "true",
|
||||
} {
|
||||
assert.Equal(t, value, env[name], name)
|
||||
assert.Equal(t, value, env["GITHUB_"+suffix], suffix)
|
||||
assert.Equal(t, value, env["GITEA_"+suffix], suffix)
|
||||
}
|
||||
ghc.RefProtected = false
|
||||
rc.withGithubEnv(t.Context(), ghc, env)
|
||||
@@ -922,12 +922,12 @@ func TestGetGitHubContext(t *testing.T) {
|
||||
github := rc.getGithubContext(t.Context())
|
||||
assert.Equal(t, testcase.serverURL, github.ServerURL)
|
||||
assert.Equal(t, testcase.apiURL, github.APIURL)
|
||||
env := map[string]string{}
|
||||
env := map[string]string{"GITEA_SERVER_URL": "stale", "GITEA_API_URL": "stale"}
|
||||
rc.withGithubEnv(t.Context(), github, env)
|
||||
_, exported := env["GITHUB_SERVER_URL"]
|
||||
assert.Equal(t, testcase.serverURL != "", exported)
|
||||
assert.Equal(t, testcase.serverURL, env["GITHUB_SERVER_URL"])
|
||||
assert.Equal(t, testcase.apiURL, env["GITHUB_API_URL"])
|
||||
assert.Equal(t, testcase.serverURL, env["GITEA_SERVER_URL"])
|
||||
assert.Equal(t, testcase.apiURL, env["GITEA_API_URL"])
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1631,4 +1631,8 @@ func TestRunContextWithGithubEnvRunnerValues(t *testing.T) {
|
||||
rc.withGithubEnv(ctx, &model.GithubContext{}, env)
|
||||
assert.Equal(t, ci, env["CI"])
|
||||
}
|
||||
|
||||
rc.runtimeToken = "run-token"
|
||||
newTestCompositeRunContext(t, rc).withGithubEnv(ctx, &model.GithubContext{}, env)
|
||||
assert.Equal(t, "run-token", env["ACTIONS_RUNTIME_TOKEN"])
|
||||
}
|
||||
@@ -41,6 +41,7 @@ type Config struct {
|
||||
Env map[string]string // env for containers
|
||||
Secrets map[string]string // list of secrets
|
||||
ExtraMasks []string // values to hide that are not in Secrets, such as the proxy password
|
||||
RuntimeTokens map[*model.Workflow]string // ACTIONS_RUNTIME_TOKEN per workflow run, as each is its own artifact run
|
||||
Vars map[string]string // list of vars
|
||||
Token string // GitHub token
|
||||
InsecureSecrets bool // switch hiding output when printing to terminal
|
||||
@@ -355,6 +356,11 @@ func (runner *runnerImpl) newCombinationRunContext(ctx context.Context, run *mod
|
||||
caller: runner.caller,
|
||||
jobIndex: index,
|
||||
jobTotal: total,
|
||||
|
||||
runtimeToken: runner.config.RuntimeTokens[run.Workflow],
|
||||
}
|
||||
if runner.caller != nil {
|
||||
rc.runtimeToken = runner.caller.runContext.runtimeToken
|
||||
}
|
||||
if err := rc.resolveWorkflowCall(ctx); err != nil {
|
||||
return nil, err
|
||||
|
||||
+18
-12
@@ -27,12 +27,24 @@ type step interface {
|
||||
post() common.Executor
|
||||
|
||||
getRunContext() *RunContext
|
||||
getGithubContext(ctx context.Context) *model.GithubContext
|
||||
getStepModel() *model.Step
|
||||
getEnv() *map[string]string
|
||||
getIfExpression(context context.Context, stage stepStage) string
|
||||
}
|
||||
|
||||
func stepActionIdentity(step step) (repository, ref string) {
|
||||
if action, ok := step.(interface{ actionIdentity() (string, string) }); ok {
|
||||
return action.actionIdentity()
|
||||
}
|
||||
return "", ""
|
||||
}
|
||||
|
||||
func getStepGithubContext(ctx context.Context, step step) *model.GithubContext {
|
||||
github := step.getRunContext().getGithubContext(ctx)
|
||||
github.ActionRepository, github.ActionRef = stepActionIdentity(step)
|
||||
return github
|
||||
}
|
||||
|
||||
type stepStage int
|
||||
|
||||
const (
|
||||
@@ -92,6 +104,8 @@ func runStepExecutor(step step, stage stepStage, executor common.Executor) commo
|
||||
runStep, err = isStepEnabled(ctx, ifExpression, step, stage)
|
||||
}
|
||||
if err == nil && runStep {
|
||||
env := *step.getEnv()
|
||||
env["GITHUB_ACTION_REPOSITORY"], env["GITHUB_ACTION_REF"] = stepActionIdentity(step)
|
||||
err = setupInputs(ctx, step)
|
||||
}
|
||||
if err != nil {
|
||||
@@ -143,15 +157,7 @@ func runStepExecutor(step step, stage stepStage, executor common.Executor) commo
|
||||
stepSummaryIndex := topRC.CurrentStepIndex
|
||||
summaryFileCommand := path.Join("workflow", "step-summary-"+strconv.Itoa(stepSummaryIndex)+".md")
|
||||
(*step.getEnv())["GITHUB_STEP_SUMMARY"] = path.Join(actPath, summaryFileCommand)
|
||||
|
||||
{
|
||||
// For Gitea
|
||||
(*step.getEnv())["GITEA_OUTPUT"] = (*step.getEnv())["GITHUB_OUTPUT"]
|
||||
(*step.getEnv())["GITEA_STATE"] = (*step.getEnv())["GITHUB_STATE"]
|
||||
(*step.getEnv())["GITEA_PATH"] = (*step.getEnv())["GITHUB_PATH"]
|
||||
(*step.getEnv())["GITEA_ENV"] = (*step.getEnv())["GITHUB_ENV"]
|
||||
(*step.getEnv())["GITEA_STEP_SUMMARY"] = (*step.getEnv())["GITHUB_STEP_SUMMARY"]
|
||||
}
|
||||
aliasGiteaEnv(*step.getEnv())
|
||||
|
||||
// Reset the per-phase file-command files. GITHUB_STEP_SUMMARY is intentionally
|
||||
// excluded here and initialized below at most once per step so writes from later
|
||||
@@ -253,7 +259,7 @@ func setupEnv(ctx context.Context, step step) error {
|
||||
env := step.getEnv()
|
||||
// container env is the image's baseline, which job env and $GITHUB_ENV override
|
||||
mergeIntoMap(step, env, rc.containerSpec.Env, rc.GetEnv())
|
||||
rc.withGithubEnv(ctx, step.getGithubContext(ctx), *env)
|
||||
rc.withGithubEnv(ctx, getStepGithubContext(ctx, step), *env)
|
||||
|
||||
var err error
|
||||
exprEval := rc.NewExpressionEvaluator(ctx)
|
||||
@@ -302,7 +308,7 @@ func isStepEnabled(ctx context.Context, expr string, step step, stage stepStage)
|
||||
if rc.Parent != nil && stage == stepStageMain && jobContext.Status != "cancelled" {
|
||||
jobContext.Status = rc.ownStatus()
|
||||
}
|
||||
runStep, err := EvalBool(ctx, rc.newStepExpressionEvaluator(ctx, step, rc.actionInputs, jobContext), expr, defaultStatusCheck)
|
||||
runStep, err := EvalBool(ctx, rc.newStepExpressionEvaluator(ctx, step, rc.actionInputs, jobContext, rc.getGithubContext(ctx)), expr, defaultStatusCheck)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("if-expression %q evaluation failed: %s", expr, err)
|
||||
}
|
||||
|
||||
@@ -39,7 +39,7 @@ func (sab *stepActionBuiltin) main() common.Executor {
|
||||
rc.ApplyExtraPath(ctx, &sab.env)
|
||||
return sab.run(ctx, &action.Context{
|
||||
Container: rc.JobContainer,
|
||||
Github: rc.getGithubContext(ctx),
|
||||
Github: getStepGithubContext(ctx, sab),
|
||||
Inputs: sab.Step.With,
|
||||
HostWorkdir: rc.Config.Workdir,
|
||||
Workspace: rc.JobContainer.ToContainerPath(rc.Config.Workdir),
|
||||
@@ -58,10 +58,6 @@ func (sab *stepActionBuiltin) post() common.Executor { return common.NewPipeline
|
||||
|
||||
func (sab *stepActionBuiltin) getRunContext() *RunContext { return sab.RunContext }
|
||||
|
||||
func (sab *stepActionBuiltin) getGithubContext(ctx context.Context) *model.GithubContext {
|
||||
return sab.RunContext.getGithubContext(ctx)
|
||||
}
|
||||
|
||||
func (sab *stepActionBuiltin) getStepModel() *model.Step { return sab.Step }
|
||||
|
||||
func (sab *stepActionBuiltin) getEnv() *map[string]string { return &sab.env }
|
||||
|
||||
@@ -95,10 +95,6 @@ func (sal *stepActionLocal) getRunContext() *RunContext {
|
||||
return sal.RunContext
|
||||
}
|
||||
|
||||
func (sal *stepActionLocal) getGithubContext(ctx context.Context) *model.GithubContext {
|
||||
return sal.getRunContext().getGithubContext(ctx)
|
||||
}
|
||||
|
||||
func (sal *stepActionLocal) getStepModel() *model.Step {
|
||||
return sal.Step
|
||||
}
|
||||
|
||||
@@ -18,6 +18,7 @@ import (
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/mock"
|
||||
"github.com/stretchr/testify/require"
|
||||
"go.yaml.in/yaml/v4"
|
||||
)
|
||||
|
||||
type stepActionMocks struct {
|
||||
@@ -53,7 +54,7 @@ func TestStepActionLocalTest(t *testing.T) {
|
||||
readAction: salm.readAction,
|
||||
runAction: salm.runAction,
|
||||
RunContext: &RunContext{
|
||||
StepResults: map[string]*model.StepResult{},
|
||||
StepResults: map[string]*model.StepResult{"previous": {Outputs: map[string]string{"value": "previous"}}},
|
||||
ExprEval: &expressionEvaluator{},
|
||||
Config: &Config{
|
||||
Workdir: "/tmp",
|
||||
@@ -68,25 +69,42 @@ func TestStepActionLocalTest(t *testing.T) {
|
||||
},
|
||||
},
|
||||
JobContainer: cm,
|
||||
Env: map[string]string{"GITHUB_ACTION_REPOSITORY": "other/action", "GITHUB_ACTION_REF": "v1"},
|
||||
Matrix: map[string]any{"value": "matrix"},
|
||||
actionInputs: map[string]any{"shared": "input"},
|
||||
},
|
||||
Step: &model.Step{
|
||||
ID: "1",
|
||||
Uses: "./path/to/action",
|
||||
If: yaml.Node{Value: "github.action_repository == 'other/action' && github.action_ref == 'v1' && " +
|
||||
"env.STEP_VALUE == 'step' && env.GITHUB_ACTION_REPOSITORY == 'forged/repo' && env.GITHUB_ACTION_REF == 'forged-ref' && " +
|
||||
"steps.previous.outputs.value == 'previous' && inputs.shared == 'input' && matrix.value == 'matrix'"},
|
||||
},
|
||||
}
|
||||
require.NoError(t, sal.Step.Env.Encode(map[string]string{"STEP_VALUE": "step", "GITHUB_ACTION_REPOSITORY": "forged/repo", "GITHUB_ACTION_REF": "forged-ref"}))
|
||||
require.NoError(t, sal.Step.RawWith.Encode(map[string]string{"identity": "${{ github.action_repository }}|${{ github.action_ref }}"}))
|
||||
|
||||
salm.On("readAction", sal.Step, filepath.Clean("/tmp/path/to/action"), "", mock.Anything, mock.Anything).
|
||||
Return(&model.Action{}, nil)
|
||||
|
||||
expectStepFileCommands(t, cm)
|
||||
|
||||
salm.On("runAction", sal, filepath.Clean("/tmp/path/to/action"), (*remoteAction)(nil)).Return(noopExecutor)
|
||||
salm.On("runAction", sal, filepath.Clean("/tmp/path/to/action"), (*remoteAction)(nil)).Return(func(ctx context.Context) error {
|
||||
sal.RunContext.setOutput(ctx, map[string]string{"name": "value"}, "ran")
|
||||
return nil
|
||||
})
|
||||
|
||||
err := sal.pre()(ctx)
|
||||
require.NoError(t, err)
|
||||
|
||||
err = sal.main()(ctx)
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, sal.env["GITHUB_ACTION_REPOSITORY"])
|
||||
assert.Empty(t, sal.env["GITHUB_ACTION_REF"])
|
||||
assert.Equal(t, "|", sal.Step.With["identity"])
|
||||
assert.Equal(t, "ran", sal.RunContext.StepResults["1"].Outputs["value"])
|
||||
assert.Equal(t, "other/action", sal.RunContext.Env["GITHUB_ACTION_REPOSITORY"])
|
||||
assert.Equal(t, "v1", sal.RunContext.Env["GITHUB_ACTION_REF"])
|
||||
|
||||
cm.AssertExpectations(t)
|
||||
salm.AssertExpectations(t)
|
||||
|
||||
@@ -75,7 +75,7 @@ func (sar *stepActionRemote) prepareActionExecutor() common.Executor {
|
||||
}
|
||||
sar.Step.Uses = uses
|
||||
|
||||
github := sar.getGithubContext(ctx) // read before remoteAction is set, so `$/` resolves against the enclosing action
|
||||
github := sar.RunContext.getGithubContext(ctx)
|
||||
if sar.remoteAction, err = newRemoteAction(sar.Step.Uses, github, sar.RunContext.Config.GitHubInstance); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -153,7 +153,7 @@ func (sar *stepActionRemote) main() common.Executor {
|
||||
return common.NewPipelineExecutor(
|
||||
sar.prepareActionExecutor(),
|
||||
runStepExecutor(sar, stepStageMain, func(ctx context.Context) error {
|
||||
github := sar.getGithubContext(ctx)
|
||||
github := getStepGithubContext(ctx, sar)
|
||||
if sar.skipsLocalCheckout(sar.remoteAction, github) {
|
||||
printRunActionHeader(ctx, sar.Step, nil, sar.env, sar.RunContext)
|
||||
if sar.RunContext.Config.BindWorkdir {
|
||||
@@ -181,17 +181,11 @@ func (sar *stepActionRemote) getRunContext() *RunContext {
|
||||
return sar.RunContext
|
||||
}
|
||||
|
||||
func (sar *stepActionRemote) getGithubContext(ctx context.Context) *model.GithubContext {
|
||||
ghc := sar.getRunContext().getGithubContext(ctx)
|
||||
|
||||
// extend github context if we already have an initialized remoteAction
|
||||
remoteAction := sar.remoteAction
|
||||
if remoteAction != nil {
|
||||
ghc.ActionRepository = fmt.Sprintf("%s/%s", remoteAction.Org, remoteAction.Repo)
|
||||
ghc.ActionRef = remoteAction.Ref
|
||||
func (sar *stepActionRemote) actionIdentity() (string, string) {
|
||||
if sar.remoteAction == nil {
|
||||
return "", ""
|
||||
}
|
||||
|
||||
return ghc
|
||||
return sar.remoteAction.Org + "/" + sar.remoteAction.Repo, sar.remoteAction.Ref
|
||||
}
|
||||
|
||||
func (sar *stepActionRemote) getStepModel() *model.Step {
|
||||
@@ -205,7 +199,7 @@ func (sar *stepActionRemote) getEnv() *map[string]string {
|
||||
func (sar *stepActionRemote) getIfExpression(ctx context.Context, stage stepStage) string {
|
||||
switch stage {
|
||||
case stepStagePre:
|
||||
github := sar.getGithubContext(ctx)
|
||||
github := getStepGithubContext(ctx, sar)
|
||||
if sar.skipsLocalCheckout(sar.remoteAction, github) {
|
||||
// skip local checkout pre step
|
||||
return "false"
|
||||
|
||||
@@ -76,7 +76,7 @@ func TestStepActionRemote(t *testing.T) {
|
||||
status model.StepStatus
|
||||
runError error
|
||||
}{
|
||||
{name: "run-successful", status: model.StepStatusSuccess},
|
||||
{name: "run-successful", condition: "github.action_repository == 'other/composite' && github.action_ref == 'v2'", status: model.StepStatusSuccess},
|
||||
{name: "run-skipped", condition: "false", status: model.StepStatusSkipped},
|
||||
{name: "run-error", status: model.StepStatusFailure, runError: assert.AnError},
|
||||
} {
|
||||
@@ -110,6 +110,7 @@ func TestStepActionRemote(t *testing.T) {
|
||||
},
|
||||
StepResults: map[string]*model.StepResult{},
|
||||
JobContainer: cm,
|
||||
Env: map[string]string{"GITHUB_ACTION_REPOSITORY": "other/composite", "GITHUB_ACTION_REF": "v2"},
|
||||
},
|
||||
Step: &model.Step{ID: "step", Uses: "remote/action@v1", If: yaml.Node{Value: tt.condition}},
|
||||
readAction: sarm.readAction,
|
||||
@@ -127,6 +128,8 @@ func TestStepActionRemote(t *testing.T) {
|
||||
require.NoError(t, sar.pre()(ctx))
|
||||
assert.Equal(t, tt.runError, sar.main()(ctx))
|
||||
assert.True(t, clonedAction)
|
||||
assert.Equal(t, "remote/action", sar.env["GITHUB_ACTION_REPOSITORY"])
|
||||
assert.Equal(t, "v1", sar.env["GITHUB_ACTION_REF"])
|
||||
assert.Equal(t, &model.StepResult{Conclusion: tt.status, Outcome: tt.status, Outputs: map[string]string{}}, sar.RunContext.StepResults["step"])
|
||||
|
||||
sarm.AssertExpectations(t)
|
||||
@@ -182,7 +185,7 @@ func TestStepActionRemotePrepare(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
root, err := prepare(job, "org/repo@v1")
|
||||
require.NoError(t, err)
|
||||
nested, err := prepare(&RunContext{Config: job.Config, Run: job.Run, JobContainer: job.JobContainer, Parent: job}, "org/repo/save@v1")
|
||||
nested, err := prepare(&RunContext{Config: job.Config, Run: job.Run, JobContainer: job.JobContainer, Parent: job, Env: map[string]string{"GITHUB_ACTION_REPOSITORY": "org/repo", "GITHUB_ACTION_REF": "v1"}}, "$/save")
|
||||
require.NoError(t, err)
|
||||
other, err := prepare(job, "org/repo@v2")
|
||||
require.NoError(t, err)
|
||||
@@ -346,9 +349,6 @@ func Test_newRemoteActionSelfRepo(t *testing.T) {
|
||||
},
|
||||
},
|
||||
{name: "unknown instance", action: "$/.gitea/actions/build", github: &model.GithubContext{Repository: "owner/workflow-repo", Sha: "abc123"}},
|
||||
{name: "empty path", action: "$/", github: workflow},
|
||||
{name: "ref suffix is not allowed", action: "$/.gitea/actions/build@v1", github: workflow},
|
||||
{name: "path traversal", action: "$/../escape", github: workflow},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
|
||||
@@ -35,10 +35,6 @@ func (sd *stepDocker) getRunContext() *RunContext {
|
||||
return sd.RunContext
|
||||
}
|
||||
|
||||
func (sd *stepDocker) getGithubContext(ctx context.Context) *model.GithubContext {
|
||||
return sd.getRunContext().getGithubContext(ctx)
|
||||
}
|
||||
|
||||
func (sd *stepDocker) getStepModel() *model.Step {
|
||||
return sd.Step
|
||||
}
|
||||
|
||||
@@ -14,6 +14,7 @@ import (
|
||||
"gitea.dev/actionslib/pkg/model"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"go.yaml.in/yaml/v4"
|
||||
)
|
||||
|
||||
func TestStepDockerMain(t *testing.T) {
|
||||
@@ -57,13 +58,16 @@ func TestStepDockerMain(t *testing.T) {
|
||||
},
|
||||
},
|
||||
JobContainer: cm,
|
||||
Env: map[string]string{"GITHUB_ACTION_REPOSITORY": "other/composite", "GITHUB_ACTION_REF": "v1"},
|
||||
},
|
||||
Step: &model.Step{
|
||||
ID: "1",
|
||||
Uses: "docker://node:14",
|
||||
WorkingDirectory: "workdir",
|
||||
If: yaml.Node{Value: "github.action_repository == 'other/composite' && github.action_ref == 'v1'"},
|
||||
},
|
||||
}
|
||||
require.NoError(t, sd.Step.RawWith.Encode(map[string]string{"args": "identity:${{ github.action_repository }}:${{ github.action_ref }}"}))
|
||||
sd.RunContext.ExprEval = sd.RunContext.NewExpressionEvaluator(ctx)
|
||||
|
||||
cm.On("Pull", false).Return(noopExecutor)
|
||||
@@ -80,6 +84,11 @@ func TestStepDockerMain(t *testing.T) {
|
||||
require.NoError(t, sd.main()(ctx))
|
||||
|
||||
assert.Equal(t, "node:14", input.Image)
|
||||
assert.Equal(t, []string{"identity::"}, input.Cmd)
|
||||
assert.Contains(t, input.Env, "GITHUB_ACTION_REPOSITORY=")
|
||||
assert.Contains(t, input.Env, "GITHUB_ACTION_REF=")
|
||||
assert.Equal(t, "other/composite", sd.RunContext.Env["GITHUB_ACTION_REPOSITORY"])
|
||||
assert.Equal(t, "v1", sd.RunContext.Env["GITHUB_ACTION_REF"])
|
||||
|
||||
assert.Empty(t, input.Username)
|
||||
assert.Empty(t, input.Password)
|
||||
|
||||
@@ -222,10 +222,6 @@ func (sr *stepRun) getRunContext() *RunContext {
|
||||
return sr.RunContext
|
||||
}
|
||||
|
||||
func (sr *stepRun) getGithubContext(ctx context.Context) *model.GithubContext {
|
||||
return sr.getRunContext().getGithubContext(ctx)
|
||||
}
|
||||
|
||||
func (sr *stepRun) getStepModel() *model.Step {
|
||||
return sr.Step
|
||||
}
|
||||
|
||||
@@ -19,6 +19,7 @@ import (
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/mock"
|
||||
"github.com/stretchr/testify/require"
|
||||
"go.yaml.in/yaml/v4"
|
||||
)
|
||||
|
||||
type shellContainerMock struct{ *containerMock }
|
||||
@@ -36,7 +37,6 @@ func TestStepRun(t *testing.T) {
|
||||
sr := &stepRun{
|
||||
RunContext: &RunContext{
|
||||
StepResults: map[string]*model.StepResult{},
|
||||
ExprEval: &expressionEvaluator{},
|
||||
Config: &Config{Env: testCheckoutEnv()},
|
||||
Run: &model.Run{
|
||||
JobID: "1",
|
||||
@@ -48,19 +48,33 @@ func TestStepRun(t *testing.T) {
|
||||
},
|
||||
JobContainer: cm,
|
||||
jobRunDefaults: model.RunDefaults{Shell: "bash"},
|
||||
Env: map[string]string{"GITHUB_ACTION_REPOSITORY": "other/composite", "GITHUB_ACTION_REF": "v1"},
|
||||
},
|
||||
Step: &model.Step{
|
||||
ID: "1",
|
||||
Run: "cmd",
|
||||
Run: "cmd${{ github.action_repository }}${{ github.action_ref }}",
|
||||
WorkingDirectory: "workdir",
|
||||
If: yaml.Node{Value: "github.action_repository == 'other/composite' && github.action_ref == 'v1'"},
|
||||
},
|
||||
}
|
||||
sr.RunContext.ExprEval = sr.RunContext.NewExpressionEvaluator(t.Context())
|
||||
require.NoError(t, sr.Step.Env.Encode(map[string]string{
|
||||
"GITHUB_ACTION_REPOSITORY": "forged/repo", "GITHUB_ACTION_REF": "forged-ref",
|
||||
"GITEA_ACTION_REPOSITORY": "forged/repo", "GITEA_ACTION_REF": "forged-ref",
|
||||
}))
|
||||
|
||||
cm.On("Copy", "/var/run/act", []*container.FileEntry{fileEntry}).Return(noopExecutor)
|
||||
cm.On("Exec", []string{"bash", "--noprofile", "--norc", "-e", "-o", "pipefail", "/var/run/act/workflow/1.sh"}, mock.AnythingOfType("map[string]string"), "", "workdir").Return(noopExecutor)
|
||||
|
||||
expectStepFileCommands(t, cm)
|
||||
require.NoError(t, sr.main()(t.Context()))
|
||||
assert.Empty(t, sr.env["GITHUB_ACTION_REPOSITORY"])
|
||||
assert.Empty(t, sr.env["GITHUB_ACTION_REF"])
|
||||
assert.Empty(t, sr.env["GITEA_ACTION_REPOSITORY"])
|
||||
assert.Empty(t, sr.env["GITEA_ACTION_REF"])
|
||||
assert.Equal(t, sr.env["GITHUB_STEP_SUMMARY"], sr.env["GITEA_STEP_SUMMARY"])
|
||||
assert.Equal(t, "other/composite", sr.RunContext.Env["GITHUB_ACTION_REPOSITORY"])
|
||||
assert.Equal(t, "v1", sr.RunContext.Env["GITHUB_ACTION_REF"])
|
||||
|
||||
cm.AssertExpectations(t)
|
||||
}
|
||||
|
||||
+12
-10
@@ -155,18 +155,14 @@ func TestSetupEnv(t *testing.T) {
|
||||
require.NoError(t, setupEnv(t.Context(), testStep))
|
||||
require.NoError(t, setupInputs(t.Context(), testStep))
|
||||
|
||||
delete(env, "GITHUB_REF")
|
||||
delete(env, "GITHUB_REF_NAME")
|
||||
delete(env, "GITHUB_REF_TYPE")
|
||||
delete(env, "GITHUB_SHA")
|
||||
delete(env, "GITHUB_WORKSPACE")
|
||||
delete(env, "GITHUB_REPOSITORY")
|
||||
delete(env, "GITHUB_REPOSITORY_OWNER")
|
||||
delete(env, "GITHUB_ACTOR")
|
||||
for _, suffix := range []string{"REF", "REF_NAME", "REF_TYPE", "SHA", "WORKSPACE", "REPOSITORY", "REPOSITORY_OWNER", "ACTOR"} {
|
||||
delete(env, "GITHUB_"+suffix)
|
||||
delete(env, "GITEA_"+suffix)
|
||||
}
|
||||
delete(env, "RUNNER_NAME")
|
||||
delete(env, "RUNNER_WORKSPACE")
|
||||
|
||||
assert.Equal(t, map[string]string{
|
||||
expected := map[string]string{
|
||||
"ACT": "true",
|
||||
"ACT_SKIP_CHECKOUT": "true",
|
||||
"CI": "true",
|
||||
@@ -193,7 +189,13 @@ func TestSetupEnv(t *testing.T) {
|
||||
"RUNNER_ENVIRONMENT": "self-hosted",
|
||||
"RUNNER_PERFLOG": "/dev/null",
|
||||
"RUNNER_TRACKING_ID": "",
|
||||
}, env)
|
||||
}
|
||||
for name, value := range expected {
|
||||
if suffix, ok := strings.CutPrefix(name, "GITHUB_"); ok {
|
||||
expected["GITEA_"+suffix] = value
|
||||
}
|
||||
}
|
||||
assert.Equal(t, expected, env)
|
||||
|
||||
for _, expression := range []string{"inputs.args", "env.ARGS"} {
|
||||
t.Run("deferred inputs from "+expression, func(t *testing.T) {
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# Container image that runs your code
|
||||
FROM node:24-bookworm-slim
|
||||
FROM node:26-bookworm-slim
|
||||
|
||||
# Copies your code file from your action repository to the filesystem path `/` of the container
|
||||
COPY entrypoint.sh /entrypoint.sh
|
||||
|
||||
Loaded 50 of 97 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user