mirror of
https://gitea.com/gitea/runner.git
synced 2026-11-09 00:30:28 +00:00
Compare commits
17
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e67697a7c4 | ||
|
|
d34e4cf11a | ||
|
|
8e999cd039 | ||
|
|
cd748053b0 | ||
|
|
9bb72bbe5c | ||
|
|
79faf5e55d | ||
|
|
993389d428 | ||
|
|
db76fd5159 | ||
|
|
c8b16f4792 | ||
|
|
02446620df | ||
|
|
8c8950d9ef | ||
|
|
b88d111cc5 | ||
|
|
bb6d793cb9 | ||
|
|
95ef17d022 | ||
|
|
ee03687688 | ||
|
|
30e600d8f6 | ||
|
|
664978c819 |
No files matched your search
+7
-4
@@ -23,11 +23,14 @@ depending on the prefix:
|
||||
covers both forms: it escapes `%` first, so a literal `%3B` becomes `%253B` that the extra
|
||||
`##[…]` rules cannot match, and a raw `;` or `]` is never decoded. It is also what makes
|
||||
multi-line work, `\n` becomes `%0A` and the UI turns it back into a line break.
|
||||
- **Forwarding a command from step output?** Leave the payload alone, it arrived escaped and is
|
||||
decoded once. Decoding here double-decodes and destroys multi-line.
|
||||
- **Forwarding a command from step output?** Preserve its escaping unless masking changes it.
|
||||
Decoding for display happens in the UI.
|
||||
- **No prefix?** Do not escape, and split multi-line values into one row each.
|
||||
- **Interpolating a secret?** Masking runs after escaping, so `AppendSecretMasker` registers the
|
||||
encoded forms too.
|
||||
- **Interpolating a secret?** Masking runs after escaping. Recognised command payloads are
|
||||
normalised before masking to cover partially escaped values. Their original escaping is
|
||||
preserved when no replacement occurs. `AppendSecretMasker` also registers encoded forms for
|
||||
other output. Complete nonblank one-character values are masked, while one-byte fragments
|
||||
of multiline values are skipped.
|
||||
- Command *properties* also escape `%3A` and `%2C`, which the UI never decodes, so the command
|
||||
handler folds an annotation's location into its decoded message and escapes the result again.
|
||||
|
||||
|
||||
+7
-5
@@ -19,14 +19,15 @@ RUN --mount=type=cache,target=/go/pkg/mod \
|
||||
### DIND VARIANT
|
||||
#
|
||||
#
|
||||
FROM docker:29.9.2-dind AS dind
|
||||
FROM docker:29.9.3-dind AS dind
|
||||
|
||||
ARG VERSION=dev
|
||||
|
||||
LABEL org.opencontainers.image.source="https://gitea.com/gitea/runner"
|
||||
LABEL org.opencontainers.image.version="${VERSION}"
|
||||
|
||||
RUN apk add --no-cache s6 bash git tzdata nftables
|
||||
# upgrade first: the docker base image can lag behind Alpine security fixes
|
||||
RUN apk upgrade --no-cache && apk add --no-cache s6 bash git tzdata nftables
|
||||
|
||||
COPY --from=builder /opt/src/runner/gitea-runner /usr/local/bin/gitea-runner
|
||||
COPY scripts/run.sh /usr/local/bin/run.sh
|
||||
@@ -41,7 +42,7 @@ ENTRYPOINT ["s6-svscan","/etc/s6"]
|
||||
### DIND-ROOTLESS VARIANT
|
||||
#
|
||||
#
|
||||
FROM docker:29.9.2-dind-rootless AS dind-rootless
|
||||
FROM docker:29.9.3-dind-rootless AS dind-rootless
|
||||
|
||||
ARG VERSION=dev
|
||||
|
||||
@@ -49,7 +50,8 @@ LABEL org.opencontainers.image.source="https://gitea.com/gitea/runner"
|
||||
LABEL org.opencontainers.image.version="${VERSION}"
|
||||
|
||||
USER root
|
||||
RUN apk add --no-cache s6 bash git tzdata nftables
|
||||
# upgrade first: the docker base image can lag behind Alpine security fixes
|
||||
RUN apk upgrade --no-cache && apk add --no-cache s6 bash git tzdata nftables
|
||||
|
||||
COPY --from=builder /opt/src/runner/gitea-runner /usr/local/bin/gitea-runner
|
||||
COPY scripts/run.sh /usr/local/bin/run.sh
|
||||
@@ -75,7 +77,7 @@ ARG VERSION=dev
|
||||
LABEL org.opencontainers.image.source="https://gitea.com/gitea/runner"
|
||||
LABEL org.opencontainers.image.version="${VERSION}"
|
||||
|
||||
RUN apk add --no-cache tini bash git tzdata
|
||||
RUN apk upgrade --no-cache && apk add --no-cache tini bash git tzdata
|
||||
|
||||
COPY --from=builder /opt/src/runner/gitea-runner /usr/local/bin/gitea-runner
|
||||
COPY scripts/run.sh /usr/local/bin/run.sh
|
||||
|
||||
@@ -4,10 +4,10 @@ EXECUTABLE := gitea-runner$(shell $(GO) env GOEXE)
|
||||
HAS_GO = $(shell hash $(GO) > /dev/null 2>&1 && echo "GO" || echo "NOGO" )
|
||||
|
||||
PODMAN_TEST_IMAGE ?= quay.io/containers/podman:v5.8.9-immutable@sha256:87f42f37fa85b421664c30b8f5f9cfc23920057f5e5fcadf0b6f68d9edc6d415 # renovate: datasource=docker
|
||||
E2E_JOB_IMAGE ?= docker.gitea.com/runner-images:ubuntu-latest@sha256:657378e5cfd898e36f820459c60ef8e69be8fcda9fd354e4381a8b654d091bbc # renovate: datasource=docker
|
||||
E2E_JOB_IMAGE ?= docker.gitea.com/runner-images:ubuntu-latest@sha256:b96ba465021b1a4c6a1c9e79220c2b9bfad0f29b09dbb528bbfb35411443a3a4 # renovate: datasource=docker
|
||||
SERVICE_IMAGE ?= nginx:1.31.7-alpine@sha256:892bb2152464f0c5d8972593a5f5afaf8054552580b699d95f8ff4af08c2d33d # renovate: datasource=docker
|
||||
E2E_GITEA_IMAGE ?= gitea/gitea:main-nightly
|
||||
E2E_K3S_IMAGE ?= rancher/k3s:v1.37.1-k3s1@sha256:ca7f37d993d82ef0dcdcfecb2e0e2618ea541dbaffc620c8cedebe01a82acd0d # renovate: datasource=docker
|
||||
E2E_K3S_IMAGE ?= rancher/k3s:v1.37.2-k3s1@sha256:21ea6e0fac4e3930c2ce8aa40ddd41a8b18cb3e9a0222c030dc005e7e88c4332 # renovate: datasource=docker
|
||||
WINDOWS_TEST_IMAGE ?= dockurr/windows:6.06@sha256:16fd7c39a3e48bb946d29c59dc6015942e5b900a1fa5d7e5552053bfc48ee2ea # renovate: datasource=docker
|
||||
|
||||
GOFILES := $(shell find . -type f -name "*.go" -o -name "go.mod" ! -name "generated.*")
|
||||
|
||||
@@ -243,11 +243,11 @@ volumes:
|
||||
- ${GITEA_DOCKER_WORKSPACE:-.}/data:/app/data
|
||||
```
|
||||
|
||||
In most Linux container jobs, the containers, networks and volumes a job creates are removed when it ends, `docker compose` services included. Set `container.sweep: false` if jobs deploy services to the runner's Docker daemon.
|
||||
Set `container.sweep: true` to remove the containers, networks and volumes most Linux container jobs create when they end, `docker compose` services included. Leave it off if jobs deploy services to the runner's Docker daemon.
|
||||
|
||||
### Built-in actions (`uses: builtin:<name>`)
|
||||
|
||||
`uses: builtin:checkout` checks out a repository without downloading `actions/checkout` or needing Node in the job image, see [docs/builtin-actions.md](docs/builtin-actions.md).
|
||||
`uses: builtin:checkout`, `builtin:upload-artifact` and `builtin:download-artifact` replace their `actions/` counterparts without an action download or Node in the job image, see [docs/builtin-actions.md](docs/builtin-actions.md).
|
||||
|
||||
### Caching (`actions/cache`)
|
||||
|
||||
|
||||
@@ -104,7 +104,9 @@ type Handler struct {
|
||||
stopSync context.CancelFunc
|
||||
syncing sync.WaitGroup
|
||||
|
||||
outboundIPMu sync.RWMutex
|
||||
outboundIP string
|
||||
detectOutboundIP func() net.IP // nil when the address is configured
|
||||
instanceTransport http.RoundTripper
|
||||
|
||||
// internalSecret guards /_internal/{register,revoke}. When set, a remote
|
||||
@@ -158,6 +160,7 @@ func StartHandler(opts Options) (*Handler, error) {
|
||||
policy: opts.Policy.withDefaults(),
|
||||
freeDisk: disk.FreeBytes,
|
||||
dbLock: make(chan struct{}, 1),
|
||||
outboundIP: opts.OutboundIP,
|
||||
instanceTransport: opts.InstanceTransport,
|
||||
}
|
||||
|
||||
@@ -167,12 +170,11 @@ func StartHandler(opts Options) (*Handler, error) {
|
||||
logger = logger.With("module", "artifactcache")
|
||||
h.logger = logger
|
||||
|
||||
if opts.OutboundIP != "" {
|
||||
h.outboundIP = opts.OutboundIP
|
||||
} else if ip := common.GetOutboundIP(); ip == nil {
|
||||
return nil, errors.New("unable to determine outbound IP address")
|
||||
} else {
|
||||
h.outboundIP = ip.String()
|
||||
if opts.OutboundIP == "" {
|
||||
h.detectOutboundIP = common.GetOutboundIP
|
||||
if h.RefreshExternalURL() == "" {
|
||||
logger.Warn("unable to determine an outbound IP address for the cache server yet, set cache.host to configure one")
|
||||
}
|
||||
}
|
||||
|
||||
if opts.Upstream != "" {
|
||||
@@ -304,11 +306,36 @@ func (h *Handler) cacheForwarder(upstream string) (http.Handler, error) {
|
||||
}), nil
|
||||
}
|
||||
|
||||
// ExternalURL is "" while no address is known.
|
||||
func (h *Handler) ExternalURL() string {
|
||||
// TODO: make the external url configurable if necessary
|
||||
if h == nil {
|
||||
return ""
|
||||
}
|
||||
h.outboundIPMu.RLock()
|
||||
defer h.outboundIPMu.RUnlock()
|
||||
if h.outboundIP == "" {
|
||||
return ""
|
||||
}
|
||||
return "http://" + net.JoinHostPort(h.outboundIP, strconv.Itoa(h.port))
|
||||
}
|
||||
|
||||
// RefreshExternalURL re-detects an unconfigured address, which can change after startup.
|
||||
func (h *Handler) RefreshExternalURL() string {
|
||||
if h != nil && h.detectOutboundIP != nil {
|
||||
if ip := h.detectOutboundIP(); ip != nil {
|
||||
h.outboundIPMu.Lock()
|
||||
if address := ip.String(); address != h.outboundIP {
|
||||
h.outboundIP = address
|
||||
if ip.To4() == nil {
|
||||
h.logger.Warnf("detected IPv6 address %s for the cache server, jobs on IPv4-only networks cannot reach it, set cache.host if they fail to", address)
|
||||
}
|
||||
}
|
||||
h.outboundIPMu.Unlock()
|
||||
}
|
||||
}
|
||||
return h.ExternalURL()
|
||||
}
|
||||
|
||||
func (h *Handler) baseURL(cred JobCredential) string {
|
||||
if base := strings.TrimRight(cred.PublicURL, "/"); base != "" {
|
||||
return base
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
@@ -22,6 +23,8 @@ import (
|
||||
"testing/synctest"
|
||||
"time"
|
||||
|
||||
"gitea.com/gitea/runner/act/common/logging/logtest"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"go.etcd.io/bbolt"
|
||||
@@ -50,7 +53,23 @@ func signArtifactURL(h *Handler, id int64) string {
|
||||
|
||||
func TestHandler_ExternalURL(t *testing.T) {
|
||||
t.Parallel()
|
||||
assert.Equal(t, "http://[2001:db8::1]:8080", (&Handler{outboundIP: "2001:db8::1", port: 8080}).ExternalURL())
|
||||
assert.Empty(t, (*Handler)(nil).RefreshExternalURL())
|
||||
assert.Equal(t, "http://cache.example:8080", (&Handler{outboundIP: "cache.example", port: 8080}).RefreshExternalURL())
|
||||
|
||||
logger, logs := logtest.NewLogger()
|
||||
var ip net.IP
|
||||
handler := &Handler{port: 8080, logger: logger, detectOutboundIP: func() net.IP { return ip }}
|
||||
assert.Empty(t, handler.RefreshExternalURL())
|
||||
ip = net.ParseIP("2001:db8::1")
|
||||
assert.Equal(t, "http://[2001:db8::1]:8080", handler.RefreshExternalURL())
|
||||
assert.Equal(t, "http://[2001:db8::1]:8080", handler.RefreshExternalURL())
|
||||
ip = net.ParseIP("192.0.2.1")
|
||||
assert.Equal(t, "http://[2001:db8::1]:8080", handler.ExternalURL())
|
||||
assert.Equal(t, "http://192.0.2.1:8080", handler.RefreshExternalURL())
|
||||
ip = nil
|
||||
assert.Equal(t, "http://192.0.2.1:8080", handler.RefreshExternalURL())
|
||||
require.Len(t, logs.AllRecords(), 1)
|
||||
assert.Contains(t, logs.LastRecord().Message, "IPv6")
|
||||
}
|
||||
|
||||
func TestHandler_Upstream(t *testing.T) {
|
||||
@@ -851,6 +870,9 @@ func TestHandler_touchCacheSkipsFreshCompletedEntries(t *testing.T) {
|
||||
func newTestHandler(t *testing.T, policy Policy, entries ...*Cache) *Handler {
|
||||
t.Helper()
|
||||
handler := startTestHandler(t, Options{Policy: policy})
|
||||
if len(entries) == 0 {
|
||||
return handler
|
||||
}
|
||||
|
||||
db, err := handler.openDB()
|
||||
require.NoError(t, err)
|
||||
@@ -940,6 +962,10 @@ func TestHandler_RejectsBearer(t *testing.T) {
|
||||
|
||||
func TestHandler_UnregisterRevokes(t *testing.T) {
|
||||
t.Parallel()
|
||||
handler, err := StartHandler(Options{Dir: filepath.Join(t.TempDir(), "artifactcache")})
|
||||
require.NoError(t, err)
|
||||
defer handler.Close()
|
||||
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
token string
|
||||
@@ -950,9 +976,6 @@ func TestHandler_UnregisterRevokes(t *testing.T) {
|
||||
{"reference counted", "shared", "v", 2},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
handler, err := StartHandler(Options{Dir: filepath.Join(t.TempDir(), "artifactcache")})
|
||||
require.NoError(t, err)
|
||||
defer handler.Close()
|
||||
revokers := make([]func(), tc.registrations)
|
||||
for i := range revokers {
|
||||
revokers[i] = handler.RegisterJob(tc.token, JobCredential{Repo: testRepo})
|
||||
@@ -1049,9 +1072,8 @@ func TestHandler_ArtifactSignature(t *testing.T) {
|
||||
|
||||
base := handler.ExternalURL() + apiPath
|
||||
good := signArtifactURL(handler, 1)
|
||||
other, err := StartHandler(Options{Dir: filepath.Join(t.TempDir(), "artifactcache2")})
|
||||
require.NoError(t, err)
|
||||
defer other.Close()
|
||||
secret := bytes.Clone(handler.secret)
|
||||
secret[0] ^= 1
|
||||
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
@@ -1060,7 +1082,7 @@ func TestHandler_ArtifactSignature(t *testing.T) {
|
||||
{"missing signature", fmt.Sprintf("%s/artifacts/%d", base, 1)},
|
||||
{"tampered signature", good[:len(good)-4] + "dead"},
|
||||
{"expired signature", handler.signedArtifactURL(JobCredential{}, 1, time.Now().Add(-time.Second))},
|
||||
{"signature from a different server", base + strings.SplitN(signArtifactURL(other, 1), apiPath, 2)[1]},
|
||||
{"signature from a different server", base + strings.SplitN(signArtifactURL(&Handler{secret: secret}, 1), apiPath, 2)[1]},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
resp, err := testClient.Get(tc.url)
|
||||
|
||||
@@ -218,7 +218,7 @@ func Serve(ctx context.Context, artifactPath, addr, port string) context.CancelF
|
||||
logger.Debugf("Artifacts base path '%s'", artifactPath)
|
||||
uploads(router, artifactPath)
|
||||
downloads(router, artifactPath)
|
||||
registerV4(router, artifactPath)
|
||||
RegisterV4(router, artifactPath)
|
||||
|
||||
server := &http.Server{
|
||||
Addr: net.JoinHostPort(addr, port),
|
||||
|
||||
@@ -28,7 +28,7 @@ func TestArtifactFlow(t *testing.T) {
|
||||
router := http.NewServeMux()
|
||||
uploads(router, artifactPath)
|
||||
downloads(router, artifactPath)
|
||||
registerV4(router, artifactPath)
|
||||
RegisterV4(router, artifactPath)
|
||||
server := httptest.NewServer(router)
|
||||
defer server.Close()
|
||||
|
||||
@@ -256,7 +256,7 @@ func TestArtifactFlow(t *testing.T) {
|
||||
require.Equal(t, "attachment; filename*=UTF-8''50%25%20art%40v1", header.Get("Content-Disposition"))
|
||||
|
||||
rerun := http.NewServeMux()
|
||||
registerV4(rerun, artifactPath)
|
||||
RegisterV4(rerun, artifactPath)
|
||||
recorder := httptest.NewRecorder()
|
||||
rerun.ServeHTTP(recorder, httptest.NewRequest(http.MethodPost, artifactServicePath+"/CreateArtifact", strings.NewReader("{"+artifactRef+"}")))
|
||||
require.Equal(t, http.StatusOK, recorder.Code, recorder.Body.String())
|
||||
|
||||
@@ -35,7 +35,7 @@ const (
|
||||
maxTwirpBytes = 1 << 20
|
||||
)
|
||||
|
||||
func registerV4(router *http.ServeMux, baseDir string) {
|
||||
func RegisterV4(router *http.ServeMux, baseDir string) {
|
||||
server := &serverV4{dir: filepath.Join(baseDir, "v4"), lastID: time.Now().UnixMilli(), created: map[string]bool{}} // IDs stay unique across invocations sharing baseDir
|
||||
router.HandleFunc("POST "+artifactServicePath+"/CreateArtifact", twirp(server.createArtifact))
|
||||
router.HandleFunc("POST "+artifactServicePath+"/FinalizeArtifact", twirp(server.finalizeArtifact))
|
||||
|
||||
@@ -36,7 +36,7 @@ func TestFetchAction(t *testing.T) {
|
||||
|
||||
root := t.TempDir()
|
||||
remote := filepath.Join(root, "org", "action")
|
||||
require.NoError(t, gitCmd("init", "--initial-branch=main", remote))
|
||||
require.NoError(t, os.CopyFS(remote, os.DirFS(filepath.Join(gitFixtureDir, "work"))))
|
||||
require.NoError(t, os.WriteFile(filepath.Join(remote, "action.yml"), []byte("name: action\n"), 0o644))
|
||||
require.NoError(t, os.WriteFile(filepath.Join(remote, "run.sh"), []byte("#!/bin/sh\n"), 0o755))
|
||||
require.NoError(t, os.Symlink("action.yml", filepath.Join(remote, "link")))
|
||||
@@ -53,6 +53,7 @@ func TestFetchAction(t *testing.T) {
|
||||
var archiveRequests []http.Header
|
||||
var mirror *httptest.Server
|
||||
archiveCache := map[string][]byte{}
|
||||
revisions := map[string]string{}
|
||||
requests, archives, failures := 0, "", 0
|
||||
handler := http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
|
||||
mu.Lock()
|
||||
@@ -79,7 +80,14 @@ func TestFetchAction(t *testing.T) {
|
||||
http.Redirect(writer, request, mirror.URL+request.URL.Path, http.StatusFound)
|
||||
return
|
||||
}
|
||||
sha, _ := gitOutput(request.Context(), remote, "rev-parse", strings.TrimSuffix(ref, ".tar.gz"))
|
||||
sha, ok := revisions[ref]
|
||||
if !ok {
|
||||
var err error
|
||||
sha, err = gitOutput(request.Context(), remote, "rev-parse", strings.TrimSuffix(ref, ".tar.gz"))
|
||||
if err == nil {
|
||||
revisions[ref] = sha
|
||||
}
|
||||
}
|
||||
etag := fmt.Sprintf(`W/"%s-tar.gz-%t"`, sha, archives != "unprefixed")
|
||||
if request.Header.Get("If-None-Match") == etag {
|
||||
writer.WriteHeader(http.StatusNotModified)
|
||||
@@ -208,6 +216,9 @@ func TestFetchAction(t *testing.T) {
|
||||
require.NoError(t, gitCmd("-C", remote, "commit", "--allow-empty", "-m", "moved"))
|
||||
require.NoError(t, gitCmd("-C", remote, "tag", "--force", "v1"))
|
||||
expectedSha = gitRevParse(t, remote, "v1")
|
||||
mu.Lock()
|
||||
clear(revisions)
|
||||
mu.Unlock()
|
||||
moved, movedSha, err := FetchAction(t.Context(), input)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, expectedSha, movedSha)
|
||||
@@ -253,7 +264,7 @@ func TestFetchAction(t *testing.T) {
|
||||
})
|
||||
|
||||
t.Run("downloads again over a tree whose files changed size or that has no record, not under a job copying it", func(t *testing.T) {
|
||||
sha := gitRevParse(t, remote, "main")
|
||||
sha := expectedSha
|
||||
for _, mode := range []string{"prefixed", "none"} {
|
||||
serve(mode, 0)
|
||||
input := newInput(sha)
|
||||
|
||||
+25
-31
@@ -112,8 +112,7 @@ func TestFindGitMetadataOfSHA256RepositoryUsesOrigin(t *testing.T) {
|
||||
t.Parallel()
|
||||
basedir := t.TempDir()
|
||||
const remoteURL = "https://github.com/owner/repo.git"
|
||||
require.NoError(t, gitCmd("init", "--object-format=sha256", "--initial-branch=main", basedir))
|
||||
require.NoError(t, os.RemoveAll(filepath.Join(basedir, ".git", "hooks")))
|
||||
require.NoError(t, gitCmd("init", "--template=", "--object-format=sha256", "--initial-branch=main", basedir))
|
||||
require.NoError(t, gitCmd("-C", basedir, "commit", "--allow-empty", "-m", "init"))
|
||||
require.NoError(t, gitCmd("-C", basedir, "remote", "add", "origin", remoteURL))
|
||||
require.NoError(t, gitCmd("-C", basedir, "config", "--add", "remote.origin.url", "https://github.com/other/repo.git"))
|
||||
@@ -205,16 +204,13 @@ func TestGitCloneExecutor(t *testing.T) {
|
||||
require.NoError(t, gitCmd("-C", workDir, "add", "action.sh", ".gitattributes"))
|
||||
require.NoError(t, gitCmd("-C", workDir, "commit", "-m", "initial"))
|
||||
require.NoError(t, gitCmd("-C", workDir, "tag", "v2"))
|
||||
require.NoError(t, gitCmd("-C", workDir, "push", "-u", "origin", "main"))
|
||||
require.NoError(t, gitCmd("-C", workDir, "push", "origin", "v2"))
|
||||
|
||||
require.NoError(t, gitCmd("-C", workDir, "checkout", "-b", "act-fails"))
|
||||
require.NoError(t, gitCmd("-C", workDir, "commit", "--allow-empty", "-m", "branch-commit"))
|
||||
require.NoError(t, gitCmd("-C", workDir, "push", "origin", "act-fails"))
|
||||
|
||||
fullSha := gitRevParse(t, workDir, "main")
|
||||
|
||||
require.NoError(t, gitCmd("-C", workDir, "push", "origin", "main:refs/heads/"+fullSha[:4]))
|
||||
require.NoError(t, gitCmd("-C", workDir, "push", "-u", "origin", "main", "v2", "act-fails", "main:refs/heads/"+fullSha[:4]))
|
||||
|
||||
hostConfig := filepath.Join(t.TempDir(), "gitconfig")
|
||||
require.NoError(t, os.WriteFile(hostConfig, []byte("[core]\n\tautocrlf = true\n\teol = crlf\n[url "+strconv.Quote(remoteDir)+"]\n\tinsteadOf = https://example.invalid/action\n"), 0o644))
|
||||
@@ -282,10 +278,10 @@ func TestGitCloneExecutor(t *testing.T) {
|
||||
func TestGitCloneExecutorReclonesWhenOriginURLChanges(t *testing.T) {
|
||||
t.Parallel()
|
||||
createRemote := func(message string) string {
|
||||
remoteDir, workDir := newGitRemote(t, message)
|
||||
require.NoError(t, gitCmd("-C", workDir, "push", "-u", "origin", "main"))
|
||||
|
||||
return remoteDir
|
||||
dir := t.TempDir()
|
||||
require.NoError(t, os.CopyFS(dir, os.DirFS(filepath.Join(gitFixtureDir, "work"))))
|
||||
require.NoError(t, gitCmd("-C", dir, "commit", "--allow-empty", "-m", message))
|
||||
return dir
|
||||
}
|
||||
|
||||
oldRemoteDir := createRemote("old-action")
|
||||
@@ -405,31 +401,30 @@ func TestGitCloneExecutorShallow(t *testing.T) {
|
||||
remoteDir, workDir := newGitRemote(t, "c1", "c2", "c3")
|
||||
require.NoError(t, gitCmd("-C", workDir, "tag", "v1"))
|
||||
unadvertisedSHA := gitRevParse(t, workDir, "HEAD~1")
|
||||
require.NoError(t, gitCmd("-C", workDir, "push", "-u", "origin", "main"))
|
||||
require.NoError(t, gitCmd("-C", workDir, "push", "origin", "v1"))
|
||||
require.NoError(t, gitCmd("-C", workDir, "push", "-u", "origin", "main", "v1"))
|
||||
|
||||
shallowMarker := func(dir string) string { return filepath.Join(dir, ".git", "shallow") }
|
||||
|
||||
for name, ref := range map[string]string{"branch is cloned shallowly": "main", "tag is cloned shallowly": "v1"} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
dir := t.TempDir()
|
||||
require.NoError(t, NewGitCloneExecutor(NewGitCloneExecutorInput{
|
||||
URL: remoteDir, Ref: ref, Dir: dir, Depth: 1,
|
||||
})(t.Context()))
|
||||
assert.FileExists(t, shallowMarker(dir), "clone should be shallow")
|
||||
assert.Equal(t, 1, gitRevCount(t, dir), "only the tip commit should be present")
|
||||
assert.Equal(t, "c3", gitHeadSubject(t, dir))
|
||||
assert.Equal(t, []string{"c3"}, gitSubjects(t, dir), "only the tip commit should be present")
|
||||
})
|
||||
}
|
||||
|
||||
unadvertisedRemote := func(t *testing.T, allowUnadvertised bool) string {
|
||||
remote := filepath.Join(t.TempDir(), "remote.git")
|
||||
require.NoError(t, gitCmd("clone", "--bare", remoteDir, remote))
|
||||
require.NoError(t, gitCmd("-C", remote, "config", "uploadpack.allowAnySHA1InWant", strconv.FormatBool(allowUnadvertised)))
|
||||
require.NoError(t, gitCmd("clone", "--bare", "--template=", "--config", "uploadpack.allowAnySHA1InWant="+strconv.FormatBool(allowUnadvertised), remoteDir, remote))
|
||||
return remote
|
||||
}
|
||||
|
||||
t.Run("commit hash resolves when the remote accepts a direct shallow fetch", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
dir := t.TempDir()
|
||||
require.NoError(t, NewGitCloneExecutor(NewGitCloneExecutorInput{
|
||||
URL: unadvertisedRemote(t, false), Ref: unadvertisedSHA, Dir: dir, Depth: 1,
|
||||
@@ -438,6 +433,7 @@ func TestGitCloneExecutorShallow(t *testing.T) {
|
||||
})
|
||||
|
||||
t.Run("commit hash is fetched shallowly, moved to another hash, then reused without reaching the remote", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
remote := unadvertisedRemote(t, true)
|
||||
dir := t.TempDir()
|
||||
cloneAt := func(ref string) error {
|
||||
@@ -448,7 +444,7 @@ func TestGitCloneExecutorShallow(t *testing.T) {
|
||||
|
||||
require.NoError(t, cloneAt(unadvertisedSHA))
|
||||
assert.FileExists(t, shallowMarker(dir))
|
||||
assert.Equal(t, 1, gitRevCount(t, dir))
|
||||
assert.Len(t, gitSubjects(t, dir), 1)
|
||||
assert.Equal(t, unadvertisedSHA, gitRevParse(t, dir, "HEAD"))
|
||||
|
||||
olderSHA := gitRevParse(t, workDir, "HEAD~2")
|
||||
@@ -461,6 +457,8 @@ func TestGitCloneExecutorShallow(t *testing.T) {
|
||||
})
|
||||
|
||||
t.Run("moving hexadecimal-named branch updates while staying shallow", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
remoteDir, workDir := newGitRemote(t, "c2", "c3")
|
||||
require.NoError(t, gitCmd("-C", workDir, "push", "origin", "main:deadbeef"))
|
||||
dir := t.TempDir()
|
||||
clone := NewGitCloneExecutor(NewGitCloneExecutorInput{
|
||||
@@ -473,19 +471,17 @@ func TestGitCloneExecutorShallow(t *testing.T) {
|
||||
require.NoError(t, gitCmd("-C", workDir, "push", "origin", "main:deadbeef"))
|
||||
|
||||
require.NoError(t, clone(t.Context()))
|
||||
assert.Equal(t, "c4", gitHeadSubject(t, dir), "reused shallow clone should update to the new tip")
|
||||
assert.Equal(t, []string{"c4"}, gitSubjects(t, dir), "reused shallow clone should update to the new tip")
|
||||
assert.FileExists(t, shallowMarker(dir), "repo should remain shallow after update")
|
||||
assert.Equal(t, 1, gitRevCount(t, dir))
|
||||
})
|
||||
|
||||
t.Run("sha256 full hash is cloned shallowly, reused without the remote, and its 40-character prefix is short", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
remote := filepath.Join(t.TempDir(), "remote.git")
|
||||
require.NoError(t, gitCmd("init", "--bare", "--object-format=sha256", "--initial-branch=main", remote))
|
||||
work := t.TempDir()
|
||||
require.NoError(t, gitCmd("init", "--object-format=sha256", "--initial-branch=main", work))
|
||||
require.NoError(t, gitCmd("-C", work, "remote", "add", "origin", remote))
|
||||
require.NoError(t, gitCmd("init", "--template=", "--object-format=sha256", "--initial-branch=main", work))
|
||||
require.NoError(t, gitCmd("-C", work, "commit", "--allow-empty", "-m", "pinned"))
|
||||
require.NoError(t, gitCmd("-C", work, "push", "origin", "main"))
|
||||
require.NoError(t, gitCmd("clone", "--bare", "--template=", work, remote))
|
||||
sha := gitRevParse(t, work, "HEAD")
|
||||
dir := t.TempDir()
|
||||
clone := NewGitCloneExecutor(NewGitCloneExecutorInput{URL: remote, Ref: sha, Dir: dir, Depth: 1})
|
||||
@@ -504,11 +500,11 @@ func TestGitCloneExecutorPinnedHashIgnoresShadowingRef(t *testing.T) {
|
||||
pinned := gitRevParse(t, workDir, "HEAD")
|
||||
require.NoError(t, gitCmd("-C", workDir, "commit", "--allow-empty", "-m", "decoy"))
|
||||
require.NoError(t, gitCmd("-C", workDir, "tag", pinned))
|
||||
require.NoError(t, gitCmd("-C", workDir, "push", "-u", "origin", "main"))
|
||||
require.NoError(t, gitCmd("-C", workDir, "push", "origin", pinned))
|
||||
require.NoError(t, gitCmd("-C", workDir, "push", "-u", "origin", "main", pinned))
|
||||
|
||||
for name, depth := range map[string]int{"shallow": 1, "full clone": 0} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
dir := t.TempDir()
|
||||
require.NoError(t, NewGitCloneExecutor(NewGitCloneExecutorInput{
|
||||
URL: remoteDir, Ref: pinned, Dir: dir, Depth: depth,
|
||||
@@ -540,13 +536,11 @@ func gitRevParse(t *testing.T, dir, rev string) string {
|
||||
return strings.TrimSpace(string(out))
|
||||
}
|
||||
|
||||
func gitRevCount(t *testing.T, dir string) int {
|
||||
func gitSubjects(t *testing.T, dir string) []string {
|
||||
t.Helper()
|
||||
out, err := exec.Command("git", "-C", dir, "rev-list", "--count", "HEAD").Output()
|
||||
out, err := exec.Command("git", "-C", dir, "log", "--format=%s").Output()
|
||||
require.NoError(t, err)
|
||||
n, err := strconv.Atoi(strings.TrimSpace(string(out)))
|
||||
require.NoError(t, err)
|
||||
return n
|
||||
return strings.Split(strings.TrimSpace(string(out)), "\n")
|
||||
}
|
||||
|
||||
func gitHeadSubject(t *testing.T, dir string) string {
|
||||
@@ -680,7 +674,7 @@ func TestNewGitCloneExecutorFetchSendsTokenAndHonoursContext(t *testing.T) {
|
||||
})
|
||||
|
||||
dir := filepath.Join(t.TempDir(), "cached-action")
|
||||
require.NoError(t, gitCmd("init", dir))
|
||||
require.NoError(t, os.CopyFS(dir, os.DirFS(filepath.Join(gitFixtureDir, "work"))))
|
||||
require.NoError(t, gitCmd("-C", dir, "remote", "add", "origin", server.URL))
|
||||
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
|
||||
@@ -6,6 +6,7 @@ package common
|
||||
|
||||
import (
|
||||
"net"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
@@ -55,19 +56,22 @@ func GetOutboundIP() net.IP {
|
||||
}
|
||||
|
||||
func preferredIP(ips []interfaceIP) net.IP {
|
||||
ips = slices.DeleteFunc(ips, func(ip interfaceIP) bool {
|
||||
return len(ip.Name) == 15 && strings.HasPrefix(ip.Name, "br-") && strings.Trim(ip.Name[3:], "0123456789abcdef") == "" // a Docker network's bridge, removed with the network
|
||||
})
|
||||
if len(ips) == 0 {
|
||||
return nil
|
||||
}
|
||||
sort.Slice(ips, func(i, j int) bool {
|
||||
first, second := ips[i], ips[j]
|
||||
|
||||
// ethernet is preferred
|
||||
if vi, vj := strings.HasPrefix(first.Name, "e"), strings.HasPrefix(second.Name, "e"); vi != vj {
|
||||
// IPv4 is preferred, as job networks are IPv4-only by default
|
||||
if vi, vj := first.To4() != nil, second.To4() != nil; vi != vj {
|
||||
return vi
|
||||
}
|
||||
|
||||
// IPv4 is preferred
|
||||
if vi, vj := first.To4() != nil, second.To4() != nil; vi != vj {
|
||||
// ethernet is preferred
|
||||
if vi, vj := strings.HasPrefix(first.Name, "e"), strings.HasPrefix(second.Name, "e"); vi != vj {
|
||||
return vi
|
||||
}
|
||||
|
||||
|
||||
@@ -13,8 +13,9 @@ import (
|
||||
func TestPreferredIP(t *testing.T) {
|
||||
assert.Nil(t, preferredIP(nil))
|
||||
assert.Equal(t, net.ParseIP("2001:db8::1"), preferredIP([]interfaceIP{{IP: net.ParseIP("2001:db8::1")}}))
|
||||
assert.Equal(t, net.ParseIP("2001:db8::1"), preferredIP([]interfaceIP{
|
||||
assert.Equal(t, net.ParseIP("192.0.2.1"), preferredIP([]interfaceIP{
|
||||
{IP: net.ParseIP("192.0.2.1"), Name: "wlan0"},
|
||||
{IP: net.ParseIP("2001:db8::1"), Name: "eth0"},
|
||||
{IP: net.ParseIP("172.18.0.1"), Name: "br-0123456789ab"},
|
||||
}))
|
||||
}
|
||||
@@ -5,6 +5,7 @@
|
||||
package container
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
@@ -14,6 +15,7 @@ import (
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"gitea.com/gitea/runner/act/common"
|
||||
|
||||
@@ -23,6 +25,9 @@ import (
|
||||
|
||||
var hostOS = runtime.GOOS // replaced in tests
|
||||
|
||||
// DrainGracePeriod caps the wait for container output to drain so the log tail is not truncated
|
||||
const DrainGracePeriod = 2 * time.Second
|
||||
|
||||
// ExitCodeError reports a non-zero process exit code from a container command.
|
||||
type ExitCodeError int
|
||||
|
||||
@@ -30,6 +35,38 @@ func (e ExitCodeError) Error() string {
|
||||
return fmt.Sprintf("Process completed with exit code %d.", int(e))
|
||||
}
|
||||
|
||||
type commandOutput struct {
|
||||
*io.PipeReader
|
||||
cancel context.CancelFunc
|
||||
done chan struct{}
|
||||
err error
|
||||
}
|
||||
|
||||
func (output *commandOutput) Close() error {
|
||||
_, _ = io.Copy(io.Discard, output.PipeReader)
|
||||
output.PipeReader.Close()
|
||||
output.cancel()
|
||||
<-output.done
|
||||
return output.err
|
||||
}
|
||||
|
||||
func streamOutput(ctx context.Context, run func(context.Context, io.Writer, io.Writer) error) io.ReadCloser {
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
reader, writer := io.Pipe()
|
||||
output := &commandOutput{PipeReader: reader, cancel: cancel, done: make(chan struct{})}
|
||||
go func() {
|
||||
defer close(output.done)
|
||||
defer cancel()
|
||||
var stderr bytes.Buffer
|
||||
output.err = run(ctx, writer, &stderr)
|
||||
if output.err != nil && strings.TrimSpace(stderr.String()) != "" {
|
||||
output.err = fmt.Errorf("%w: %s", output.err, strings.TrimSpace(stderr.String()))
|
||||
}
|
||||
writer.CloseWithError(output.err)
|
||||
}()
|
||||
return output
|
||||
}
|
||||
|
||||
// NewContainerInput the input for the New function
|
||||
type NewContainerInput struct {
|
||||
Image string
|
||||
@@ -130,7 +167,9 @@ type Container interface {
|
||||
Create(capAdd, capDrop []string) common.Executor
|
||||
Copy(destPath string, files ...*FileEntry) common.Executor
|
||||
CopyDir(destPath, srcPath string, useGitIgnore, skipGitDir bool) common.Executor
|
||||
ExecOutput(ctx context.Context, command []string) (io.ReadCloser, error)
|
||||
GetContainerArchive(ctx context.Context, srcPath string) (io.ReadCloser, error)
|
||||
CopyTarStream(ctx context.Context, destPath string, write func(*TarWriter) error) error
|
||||
Inspect(ctx context.Context) (*Info, error)
|
||||
DumpLogs(ctx context.Context) error
|
||||
Pull(forcePull bool) common.Executor
|
||||
|
||||
@@ -42,11 +42,11 @@ func TestDockerBuildLabelsImageWithItsTagAndReportsDaemonError(t *testing.T) {
|
||||
|
||||
func TestCreateBuildContextFollowsDockerignoreAndSymlinkedRootButKeepsDockerfile(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
require.NoError(t, os.Mkdir(filepath.Join(dir, "build"), 0o755))
|
||||
for name, content := range map[string]string{
|
||||
".dockerignore": "*.log\nbuild\n!build/keep.txt\nDockerfile", "Dockerfile": "FROM scratch", "a.txt": "a",
|
||||
"a.log": "log", "build/drop.txt": "drop", "build/keep.txt": "keep",
|
||||
} {
|
||||
require.NoError(t, os.MkdirAll(filepath.Join(dir, filepath.Dir(name)), 0o755))
|
||||
require.NoError(t, os.WriteFile(filepath.Join(dir, name), []byte(content), 0o644))
|
||||
}
|
||||
require.NoError(t, os.Mkdir(filepath.Join(dir, "empty"), 0o755))
|
||||
|
||||
+83
-13
@@ -41,13 +41,6 @@ import (
|
||||
specs "github.com/opencontainers/image-spec/specs-go/v1"
|
||||
)
|
||||
|
||||
// drainGracePeriod bounds how long we wait for an output-copy goroutine to
|
||||
// finish draining a container's output before returning, so that neither a
|
||||
// cancellation (waitForCommand) nor a normal container exit (wait) truncates
|
||||
// the tail of the log. It is a safety bound: in the common case the stream
|
||||
// reaches EOF and the goroutine returns well before this elapses.
|
||||
const drainGracePeriod = 2 * time.Second
|
||||
|
||||
// NewContainer creates a reference to a container
|
||||
func NewContainer(input *NewContainerInput) ExecutionsEnvironment {
|
||||
// Resolved up front because the image pull runs before the container is created.
|
||||
@@ -146,6 +139,47 @@ func (cr *containerReference) CopyDir(destPath, srcPath string, useGitIgnore, sk
|
||||
).IfNot(common.Dryrun)
|
||||
}
|
||||
|
||||
func (cr *containerReference) ExecOutput(ctx context.Context, command []string) (io.ReadCloser, error) {
|
||||
if common.Dryrun(ctx) {
|
||||
return nil, errors.New("DRYRUN is not supported in ExecOutput")
|
||||
}
|
||||
if err := cr.connect()(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := cr.find()(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if cr.id == "" {
|
||||
return nil, cr.missingContainerError("exec %v", command)
|
||||
}
|
||||
return streamOutput(ctx, func(ctx context.Context, stdout, stderr io.Writer) error {
|
||||
created, err := cr.cli.ExecCreate(ctx, cr.id, client.ExecCreateOptions{
|
||||
Cmd: command, AttachStdout: true, AttachStderr: true,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
attached, err := cr.cli.ExecAttach(ctx, created.ID, client.ExecAttachOptions{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer attached.Close()
|
||||
stop := context.AfterFunc(ctx, func() { attached.Close() })
|
||||
defer stop()
|
||||
if _, err := stdcopy.StdCopy(stdout, stderr, attached.Reader); err != nil {
|
||||
return err
|
||||
}
|
||||
inspected, err := cr.cli.ExecInspect(ctx, created.ID, client.ExecInspectOptions{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if inspected.ExitCode != 0 {
|
||||
return ExitCodeError(inspected.ExitCode)
|
||||
}
|
||||
return nil
|
||||
}), nil
|
||||
}
|
||||
|
||||
func (cr *containerReference) GetContainerArchive(ctx context.Context, srcPath string) (io.ReadCloser, error) {
|
||||
if common.Dryrun(ctx) {
|
||||
return nil, errors.New("DRYRUN is not supported in GetContainerArchive")
|
||||
@@ -861,6 +895,7 @@ func (cr *containerReference) tryReadID(opt string, cbk func(id int)) common.Exe
|
||||
return nil
|
||||
}
|
||||
defer resp.Close()
|
||||
defer context.AfterFunc(ctx, resp.Close)()
|
||||
|
||||
sid, err := resp.Reader.ReadString('\n')
|
||||
if err != nil {
|
||||
@@ -906,7 +941,7 @@ func (cr *containerReference) waitForCommand(ctx context.Context, resp client.Hi
|
||||
if err != nil {
|
||||
logger.Debugf("Failed to drain command output after cancellation: %v", err)
|
||||
}
|
||||
case <-time.After(drainGracePeriod):
|
||||
case <-time.After(DrainGracePeriod):
|
||||
logger.Warn("Timed out draining command output after cancellation")
|
||||
}
|
||||
|
||||
@@ -938,10 +973,7 @@ func (cr *containerReference) copyDir(dstPath, srcPath string, useGitIgnore, ski
|
||||
logger.Error(err)
|
||||
}
|
||||
}()
|
||||
extractionRoot := "/"
|
||||
if actPath := cr.GetActPath(); strings.HasPrefix(dstPath+"/", actPath+"/") {
|
||||
extractionRoot = actPath // keeps tar entries from crossing the image's /var/run symlink
|
||||
}
|
||||
extractionRoot := cr.extractionRoot(dstPath)
|
||||
if err := writeDirTar(ctx, tarFile, path.Join("/", strings.TrimPrefix(dstPath, extractionRoot)), srcPath, useGitIgnore, skipGitDir, cr.UID, cr.GID, cr.markExecutable(ctx)); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -954,6 +986,44 @@ func (cr *containerReference) copyDir(dstPath, srcPath string, useGitIgnore, ski
|
||||
}
|
||||
}
|
||||
|
||||
func (cr *containerReference) extractionRoot(dstPath string) string {
|
||||
if actPath := cr.GetActPath(); strings.HasPrefix(dstPath+"/", actPath+"/") {
|
||||
return actPath // keeps tar entries from crossing the image's /var/run symlink
|
||||
}
|
||||
return "/"
|
||||
}
|
||||
|
||||
func (cr *containerReference) CopyTarStream(ctx context.Context, destPath string, write func(*TarWriter) error) error {
|
||||
if common.Dryrun(ctx) {
|
||||
return nil
|
||||
}
|
||||
if err := cr.connect()(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := cr.find()(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
if cr.id == "" {
|
||||
return cr.missingContainerError("copy to %s", destPath)
|
||||
}
|
||||
destPath = path.Clean(destPath)
|
||||
root := cr.extractionRoot(destPath)
|
||||
rel := func(fullPath string) string { return strings.TrimPrefix(strings.TrimPrefix(fullPath, root), "/") }
|
||||
var parents []string // sent as entries so they belong to the job user, the daemon creates missing ones as root
|
||||
for dir := destPath; dir != root && dir != "/" && dir != "."; dir = path.Dir(dir) {
|
||||
if _, err := cr.cli.ContainerStatPath(ctx, cr.id, client.ContainerStatPathOptions{Path: dir}); err == nil {
|
||||
break
|
||||
} else if !cerrdefs.IsNotFound(err) {
|
||||
return fmt.Errorf("stat %s: %w", dir, err)
|
||||
}
|
||||
parents = append(parents, rel(dir))
|
||||
}
|
||||
slices.Reverse(parents)
|
||||
content := pipeTar(cr.UID, cr.GID, rel(destPath), parents, write)
|
||||
defer content.Close()
|
||||
return cr.upload(ctx, root, content)
|
||||
}
|
||||
|
||||
func (cr *containerReference) copyContent(dstPath string, files ...*FileEntry) common.Executor {
|
||||
return func(ctx context.Context) error {
|
||||
if cr.id == "" {
|
||||
@@ -1046,7 +1116,7 @@ func (cr *containerReference) wait() common.Executor {
|
||||
if cr.attachDone != nil {
|
||||
select {
|
||||
case <-cr.attachDone:
|
||||
case <-time.After(drainGracePeriod):
|
||||
case <-time.After(DrainGracePeriod):
|
||||
logger.Warn("Timed out draining container output")
|
||||
}
|
||||
cr.attachDone = nil
|
||||
|
||||
@@ -122,6 +122,11 @@ func (m *mockDockerClient) CopyToContainer(ctx context.Context, id string, optio
|
||||
return args.Get(0).(mobyclient.CopyToContainerResult), args.Error(1)
|
||||
}
|
||||
|
||||
func (m *mockDockerClient) ContainerStatPath(ctx context.Context, id string, options mobyclient.ContainerStatPathOptions) (mobyclient.ContainerStatPathResult, error) {
|
||||
args := m.Called(ctx, id, options)
|
||||
return args.Get(0).(mobyclient.ContainerStatPathResult), args.Error(1)
|
||||
}
|
||||
|
||||
func (m *mockDockerClient) ContainerInspect(ctx context.Context, id string, opts mobyclient.ContainerInspectOptions) (mobyclient.ContainerInspectResult, error) {
|
||||
args := m.Called(ctx, id, opts)
|
||||
return args.Get(0).(mobyclient.ContainerInspectResult), args.Error(1)
|
||||
@@ -255,6 +260,28 @@ func TestDockerExecFailure(t *testing.T) {
|
||||
assert.Equal(t, ExitCodeError(1), exitErr)
|
||||
assert.Equal(t, "Process completed with exit code 1.", err.Error())
|
||||
|
||||
client.On("ContainerInspect", ctx, "123", mobyclient.ContainerInspectOptions{}).
|
||||
Return(mobyclient.ContainerInspectResult{}, nil)
|
||||
client.On("ExecCreate", mock.Anything, "123", mobyclient.ExecCreateOptions{
|
||||
Cmd: []string{"printf", `a\b`}, AttachStdout: true, AttachStderr: true,
|
||||
}).Return(mobyclient.ExecCreateResult{ID: "output"}, nil)
|
||||
client.On("ExecAttach", mock.Anything, "output", mobyclient.ExecAttachOptions{}).
|
||||
Return(mobyclient.ExecAttachResult{
|
||||
Conn: &mockConn{},
|
||||
Reader: bufio.NewReader(bytes.NewReader(append(stdcopyFrame(stdcopy.Stdout, "a\x00b"), stdcopyFrame(stdcopy.Stderr, " failed \n")...))),
|
||||
}, nil)
|
||||
client.On("ExecInspect", mock.Anything, "output", mobyclient.ExecInspectOptions{}).
|
||||
Return(mobyclient.ExecInspectResult{ExitCode: 3}, nil)
|
||||
output, err := cr.ExecOutput(ctx, []string{"printf", `a\b`})
|
||||
require.NoError(t, err)
|
||||
content, err := io.ReadAll(output)
|
||||
require.ErrorAs(t, err, &exitErr)
|
||||
assert.Equal(t, ExitCodeError(3), exitErr)
|
||||
assert.Equal(t, "Process completed with exit code 3.: failed", err.Error())
|
||||
assert.Equal(t, "a\x00b", string(content))
|
||||
require.ErrorAs(t, output.Close(), &exitErr)
|
||||
assert.Equal(t, ExitCodeError(3), exitErr)
|
||||
|
||||
client.AssertExpectations(t)
|
||||
}
|
||||
|
||||
@@ -351,6 +378,21 @@ func TestDockerAttachStreamClosesWithItsContext(t *testing.T) {
|
||||
client.AssertExpectations(t)
|
||||
}
|
||||
|
||||
func TestDockerReadIDStreamClosesWithItsContext(t *testing.T) {
|
||||
conn, peer := net.Pipe()
|
||||
t.Cleanup(func() { _ = peer.Close() })
|
||||
client := &mockDockerClient{}
|
||||
client.On("ExecCreate", mock.Anything, "123", mock.AnythingOfType("client.ExecCreateOptions")).
|
||||
Return(mobyclient.ExecCreateResult{ID: "id"}, nil)
|
||||
client.On("ExecAttach", mock.Anything, "id", mobyclient.ExecAttachOptions{}).
|
||||
Return(mobyclient.ExecAttachResult{HijackedResponse: mobyclient.NewHijackedResponse(conn, "")}, nil)
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
cancel()
|
||||
|
||||
require.NoError(t, (&containerReference{id: "123", cli: client}).tryReadID("-u", func(int) { t.Error("read an ID after cancellation") })(ctx))
|
||||
client.AssertExpectations(t)
|
||||
}
|
||||
|
||||
func TestDockerWaitFailure(t *testing.T) {
|
||||
ctx := t.Context()
|
||||
statusCh := make(chan container.WaitResponse, 1)
|
||||
@@ -528,6 +570,8 @@ func TestRejectsMissingContainer(t *testing.T) {
|
||||
check("exec", cr.exec([]string{"echo"}, nil, "", "")(ctx))
|
||||
_, err := cr.GetContainerArchive(ctx, "/var/run/act/x")
|
||||
check("GetContainerArchive", err)
|
||||
_, err = cr.ExecOutput(ctx, []string{"true"})
|
||||
check("ExecOutput", err)
|
||||
_, err = cr.Inspect(ctx)
|
||||
check("Inspect", err)
|
||||
|
||||
@@ -539,7 +583,50 @@ func TestRejectsMissingContainer(t *testing.T) {
|
||||
check("Inspect after removal", err)
|
||||
}
|
||||
|
||||
func TestCopyTarStreamOwnsMissingParentsAndRejectsEscapes(t *testing.T) {
|
||||
ctx := t.Context()
|
||||
client := &mockDockerClient{}
|
||||
client.On("ContainerInspect", ctx, "job", mobyclient.ContainerInspectOptions{}).
|
||||
Return(mobyclient.ContainerInspectResult{Container: container.InspectResponse{ID: "job", Name: "/job"}}, nil)
|
||||
stat := func(p string, err error) {
|
||||
client.On("ContainerStatPath", ctx, "job", mobyclient.ContainerStatPathOptions{Path: p}).
|
||||
Return(mobyclient.ContainerStatPathResult{}, err).Once()
|
||||
}
|
||||
stat("/w/out/a", cerrdefs.ErrNotFound)
|
||||
stat("/w/out", cerrdefs.ErrNotFound)
|
||||
stat("/w", nil)
|
||||
var entries []string
|
||||
client.On("CopyToContainer", ctx, "job", mock.Anything).Run(func(args mock.Arguments) {
|
||||
opts := args.Get(2).(mobyclient.CopyToContainerOptions)
|
||||
assert.Equal(t, "/", opts.DestinationPath)
|
||||
reader := tar.NewReader(opts.Content)
|
||||
for {
|
||||
header, err := reader.Next()
|
||||
if errors.Is(err, io.EOF) {
|
||||
return
|
||||
}
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, []any{1001, 1002, ""}, []any{header.Uid, header.Gid, header.Uname}, header.Name)
|
||||
entries = append(entries, header.Name)
|
||||
}
|
||||
}).Return(mobyclient.CopyToContainerResult{}, nil).Once()
|
||||
|
||||
cr := &containerReference{id: "job", cli: client, input: &NewContainerInput{Name: "job"}, UID: 1001, GID: 1002}
|
||||
require.NoError(t, cr.CopyTarStream(ctx, "/w/out/a/", func(tw *TarWriter) error {
|
||||
return tw.WriteHeader(&tar.Header{Typeflag: tar.TypeReg, Name: "dir/f.txt", Mode: 0o644, Uname: "nobody"})
|
||||
}))
|
||||
assert.Equal(t, []string{"w/out/", "w/out/a/", "w/out/a/dir/", "w/out/a/dir/f.txt"}, entries)
|
||||
client.AssertExpectations(t)
|
||||
|
||||
require.ErrorContains(t, writeTarStream(io.Discard, 0, 0, "w/out/a", nil, func(tw *TarWriter) error {
|
||||
return tw.WriteHeader(&tar.Header{Typeflag: tar.TypeReg, Name: "../../../etc/passwd"})
|
||||
}), "escapes")
|
||||
}
|
||||
|
||||
func TestPublicCopyPipelineHandlesStaleID(t *testing.T) {
|
||||
source := t.TempDir()
|
||||
require.NoError(t, os.WriteFile(filepath.Join(source, "x"), nil, 0o644))
|
||||
|
||||
for _, testcase := range []struct {
|
||||
name, destination, extractionRoot, entry string
|
||||
directory bool
|
||||
@@ -567,8 +654,6 @@ func TestPublicCopyPipelineHandlesStaleID(t *testing.T) {
|
||||
|
||||
cr := &containerReference{id: "stale", cli: client, input: &NewContainerInput{Name: "job-1"}}
|
||||
if testcase.directory {
|
||||
source := t.TempDir()
|
||||
require.NoError(t, os.WriteFile(filepath.Join(source, "x"), nil, 0o644))
|
||||
require.NoError(t, cr.CopyDir(testcase.destination, source+string(filepath.Separator), false, true)(ctx))
|
||||
} else {
|
||||
require.NoError(t, cr.Copy(testcase.destination, &FileEntry{Name: "x", Mode: 0o644})(ctx))
|
||||
@@ -1108,6 +1193,10 @@ func TestConnectTakesDaemonOSFromPing(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestContainerCopyModes(t *testing.T) {
|
||||
source := t.TempDir()
|
||||
require.NoError(t, os.WriteFile(filepath.Join(source, "run.sh"), []byte("#!/bin/sh\n"), 0o666))
|
||||
require.NoError(t, os.Chmod(filepath.Join(source, "run.sh"), 0o666))
|
||||
|
||||
for _, testcase := range []struct {
|
||||
hostOS, serverOS string
|
||||
expected int64
|
||||
@@ -1117,9 +1206,6 @@ func TestContainerCopyModes(t *testing.T) {
|
||||
{"linux", "linux", 0o666},
|
||||
} {
|
||||
t.Run(testcase.hostOS+"-"+testcase.serverOS, func(t *testing.T) {
|
||||
source := t.TempDir()
|
||||
require.NoError(t, os.WriteFile(filepath.Join(source, "run.sh"), []byte("#!/bin/sh\n"), 0o666))
|
||||
require.NoError(t, os.Chmod(filepath.Join(source, "run.sh"), 0o666))
|
||||
file := &FileEntry{Name: "run.sh", Mode: 0o600, Body: "#!/bin/sh\n"}
|
||||
original := hostOS
|
||||
hostOS = testcase.hostOS
|
||||
|
||||
@@ -6,7 +6,6 @@ package container
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"cmp"
|
||||
"context"
|
||||
"errors"
|
||||
@@ -17,6 +16,7 @@ import (
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
@@ -116,18 +116,39 @@ func (e *HostEnvironment) Inspect(_ context.Context) (*Info, error) {
|
||||
return &Info{Health: HealthNone, Ports: map[string]string{}}, nil
|
||||
}
|
||||
|
||||
func (e *HostEnvironment) ExecOutput(ctx context.Context, command []string) (io.ReadCloser, error) {
|
||||
return streamOutput(ctx, func(ctx context.Context, stdout, stderr io.Writer) error {
|
||||
cmd := exec.CommandContext(ctx, command[0], command[1:]...)
|
||||
cmd.Dir, cmd.Stdout, cmd.Stderr = e.Workdir, stdout, stderr
|
||||
if err := cmd.Run(); err != nil {
|
||||
if exit, ok := errors.AsType[*exec.ExitError](err); ok {
|
||||
return ExitCodeError(exit.ExitCode())
|
||||
}
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}), nil
|
||||
}
|
||||
|
||||
func (e *HostEnvironment) GetContainerArchive(ctx context.Context, srcPath string) (io.ReadCloser, error) {
|
||||
buf := &bytes.Buffer{}
|
||||
tw := tar.NewWriter(buf)
|
||||
defer tw.Close()
|
||||
srcPath = filepath.Clean(srcPath)
|
||||
fi, err := os.Lstat(srcPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
tc := &filecollector.TarCollector{
|
||||
TarWriter: tw,
|
||||
}
|
||||
reader, writer := io.Pipe()
|
||||
go func() {
|
||||
tw := tar.NewWriter(writer)
|
||||
err := writeHostArchive(ctx, &filecollector.TarCollector{TarWriter: tw}, srcPath, fi)
|
||||
if err == nil {
|
||||
err = tw.Close()
|
||||
}
|
||||
writer.CloseWithError(err)
|
||||
}()
|
||||
return startedArchive(reader)
|
||||
}
|
||||
|
||||
func writeHostArchive(ctx context.Context, tc *filecollector.TarCollector, srcPath string, fi fs.FileInfo) error {
|
||||
if fi.IsDir() {
|
||||
srcPrefix := srcPath
|
||||
if !strings.HasSuffix(srcPrefix, string(filepath.Separator)) {
|
||||
@@ -138,31 +159,74 @@ func (e *HostEnvironment) GetContainerArchive(ctx context.Context, srcPath strin
|
||||
SrcPrefix: srcPrefix,
|
||||
Handler: tc,
|
||||
}
|
||||
err = filepath.Walk(srcPath, fc.CollectFiles(ctx))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return filepath.Walk(srcPath, fc.CollectFiles(ctx))
|
||||
}
|
||||
var f io.ReadCloser
|
||||
var linkname string
|
||||
var err error
|
||||
if fi.Mode()&fs.ModeSymlink != 0 {
|
||||
if linkname, err = os.Readlink(srcPath); err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
var f io.ReadCloser
|
||||
var linkname string
|
||||
if fi.Mode()&fs.ModeSymlink != 0 {
|
||||
linkname, err = os.Readlink(srcPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if f, err = os.Open(srcPath); err != nil {
|
||||
return err
|
||||
}
|
||||
defer f.Close()
|
||||
}
|
||||
return tc.WriteFile(fi.Name(), fi, linkname, f)
|
||||
}
|
||||
|
||||
func (e *HostEnvironment) CopyTarStream(ctx context.Context, destPath string, write func(*TarWriter) error) error {
|
||||
if err := os.MkdirAll(destPath, 0o777); err != nil {
|
||||
return err
|
||||
}
|
||||
root, err := os.OpenRoot(destPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer root.Close()
|
||||
content := pipeTar(0, 0, "", nil, write)
|
||||
defer content.Close()
|
||||
tr := tar.NewReader(content)
|
||||
var dirs []*tar.Header
|
||||
for {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
hdr, err := tr.Next()
|
||||
if errors.Is(err, io.EOF) {
|
||||
break
|
||||
} else if err != nil {
|
||||
return err
|
||||
}
|
||||
hdr.Name = filepath.FromSlash(strings.TrimSuffix(hdr.Name, "/"))
|
||||
if hdr.Typeflag == tar.TypeDir {
|
||||
err = root.MkdirAll(hdr.Name, 0o777) // modes apply after the contents, so read-only directories can be filled
|
||||
dirs = append(dirs, hdr)
|
||||
} else {
|
||||
f, err = os.Open(srcPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer f.Close()
|
||||
err = writeRootFile(root, hdr.Name, hdr.FileInfo().Mode().Perm(), tr)
|
||||
}
|
||||
err := tc.WriteFile(fi.Name(), fi, linkname, f)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return err
|
||||
}
|
||||
}
|
||||
return io.NopCloser(buf), nil
|
||||
for _, dir := range slices.Backward(dirs) {
|
||||
if err := root.Chmod(dir.Name, dir.FileInfo().Mode().Perm()); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func writeRootFile(root *os.Root, name string, mode fs.FileMode, body io.Reader) error {
|
||||
_ = root.Remove(name) // replaces read-only files and applies mode, as the daemon and tar do
|
||||
file, err := root.OpenFile(name, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, mode)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = io.Copy(file, body)
|
||||
return errors.Join(err, file.Close())
|
||||
}
|
||||
|
||||
func (e *HostEnvironment) Pull(_ bool) common.Executor {
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
@@ -32,6 +33,11 @@ func TestActionPlatformNames(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestCopyDir(t *testing.T) {
|
||||
src := filepath.Join(t.TempDir(), "action")
|
||||
require.NoError(t, os.MkdirAll(filepath.Join(src, ".git", "objects"), 0o700))
|
||||
require.NoError(t, os.WriteFile(filepath.Join(src, ".git", "objects", "pack"), []byte("x"), 0o600))
|
||||
require.NoError(t, os.WriteFile(filepath.Join(src, "action.yml"), []byte("name: x\n"), 0o600))
|
||||
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
useGitIgnore, skipGitDir bool
|
||||
@@ -41,11 +47,6 @@ func TestCopyDir(t *testing.T) {
|
||||
{name: "unfiltered outside a git repository", useGitIgnore: true},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
src := filepath.Join(t.TempDir(), "action")
|
||||
require.NoError(t, os.MkdirAll(filepath.Join(src, ".git", "objects"), 0o700))
|
||||
require.NoError(t, os.WriteFile(filepath.Join(src, ".git", "objects", "pack"), []byte("x"), 0o600))
|
||||
require.NoError(t, os.WriteFile(filepath.Join(src, "action.yml"), []byte("name: x\n"), 0o600))
|
||||
|
||||
dest := filepath.Join(t.TempDir(), "dest")
|
||||
require.NoError(t, (&HostEnvironment{}).CopyDir(dest, src+string(filepath.Separator), test.useGitIgnore, test.skipGitDir)(t.Context()))
|
||||
|
||||
@@ -76,7 +77,54 @@ func TestGetContainerArchive(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, expectedContent, content)
|
||||
_, err = reader.Next()
|
||||
assert.ErrorIs(t, err, io.EOF)
|
||||
require.ErrorIs(t, err, io.EOF)
|
||||
require.NoError(t, archive.Close())
|
||||
|
||||
_, err = e.GetContainerArchive(ctx, filepath.Join(e.Path, "missing"))
|
||||
assert.ErrorIs(t, err, os.ErrNotExist)
|
||||
}
|
||||
|
||||
func TestHostCopyTarStream(t *testing.T) {
|
||||
dest := filepath.Join(t.TempDir(), "out")
|
||||
write := func(name string, mode int64, body string) func(*TarWriter) error {
|
||||
return func(tw *TarWriter) error {
|
||||
if err := tw.WriteHeader(&tar.Header{Typeflag: tar.TypeReg, Name: name, Mode: mode, Size: int64(len(body))}); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err := tw.Write([]byte(body))
|
||||
return err
|
||||
}
|
||||
}
|
||||
e := &HostEnvironment{}
|
||||
require.NoError(t, e.CopyTarStream(t.Context(), dest, write("dir/run.sh", 0o755, "#!/bin/sh\n")))
|
||||
content, err := os.ReadFile(filepath.Join(dest, "dir", "run.sh"))
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "#!/bin/sh\n", string(content))
|
||||
if runtime.GOOS != "windows" {
|
||||
require.NoError(t, os.Chmod(filepath.Join(dest, "dir", "run.sh"), 0o444))
|
||||
require.NoError(t, e.CopyTarStream(t.Context(), dest, func(tw *TarWriter) error {
|
||||
if err := tw.WriteHeader(&tar.Header{Typeflag: tar.TypeDir, Name: "dir/", Mode: 0o555}); err != nil {
|
||||
return err
|
||||
}
|
||||
return write("dir/run.sh", 0o755, "")(tw)
|
||||
}))
|
||||
t.Cleanup(func() { require.NoError(t, os.Chmod(filepath.Join(dest, "dir"), 0o755)) })
|
||||
for name, mode := range map[string]os.FileMode{"dir": 0o555, filepath.Join("dir", "run.sh"): 0o755} {
|
||||
info, err := os.Stat(filepath.Join(dest, name))
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, mode, info.Mode().Perm(), name)
|
||||
}
|
||||
require.NoError(t, os.Symlink(t.TempDir(), filepath.Join(dest, "link")))
|
||||
require.ErrorContains(t, e.CopyTarStream(t.Context(), dest, write("link/x", 0o644, "x")), "escapes")
|
||||
}
|
||||
|
||||
require.ErrorContains(t, e.CopyTarStream(t.Context(), dest, write("../escape", 0o644, "x")), "escapes")
|
||||
assert.NoFileExists(t, filepath.Join(filepath.Dir(dest), "escape"))
|
||||
require.Error(t, e.CopyTarStream(t.Context(), dest, func(_tw *TarWriter) error { return fmt.Errorf("read body: %w", io.EOF) }))
|
||||
canceled, cancel := context.WithCancel(t.Context())
|
||||
cancel()
|
||||
require.ErrorIs(t, e.CopyTarStream(canceled, dest, write("late", 0o644, "x")), context.Canceled)
|
||||
assert.NoFileExists(t, filepath.Join(dest, "late"))
|
||||
}
|
||||
|
||||
func TestHostEnvironmentExecExitCode(t *testing.T) {
|
||||
@@ -84,6 +132,21 @@ func TestHostEnvironmentExecExitCode(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("uses POSIX shell")
|
||||
}
|
||||
for _, read := range []bool{true, false} {
|
||||
output, err := (&HostEnvironment{AllocatePTY: true}).ExecOutput(t.Context(), []string{"sh", "-c", `test ! -t 1; printf 'out\000'; printf ' failed \n' >&2; exit 3`})
|
||||
require.NoError(t, err)
|
||||
if read {
|
||||
content, err := io.ReadAll(output)
|
||||
var exit ExitCodeError
|
||||
require.ErrorAs(t, err, &exit)
|
||||
assert.Equal(t, ExitCodeError(3), exit)
|
||||
assert.Equal(t, "out\x00", string(content))
|
||||
assert.Equal(t, "Process completed with exit code 3.: failed", err.Error())
|
||||
}
|
||||
var exit ExitCodeError
|
||||
require.ErrorAs(t, output.Close(), &exit)
|
||||
assert.Equal(t, ExitCodeError(3), exit)
|
||||
}
|
||||
for name, allocatePTY := range map[string]bool{"without PTY": false, "with PTY": true} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
+13
-18
@@ -4,7 +4,6 @@
|
||||
package container
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"cmp"
|
||||
"compress/gzip"
|
||||
@@ -554,6 +553,10 @@ func (c *kubernetesContainer) Copy(destPath string, files ...*FileEntry) common.
|
||||
}
|
||||
}
|
||||
|
||||
func (c *kubernetesContainer) CopyTarStream(ctx context.Context, destPath string, write func(*TarWriter) error) error {
|
||||
return c.extract(ctx, destPath, true, func(writer io.Writer) error { return writeTarStream(writer, 0, 0, "", nil, write) }) // streams carry large content like artifacts
|
||||
}
|
||||
|
||||
func (c *kubernetesContainer) CopyDir(destPath, srcPath string, useGitIgnore, skipGitDir bool) common.Executor {
|
||||
return func(ctx context.Context) error {
|
||||
return c.extract(ctx, destPath, true, func(writer io.Writer) error { // directories are actions and workspaces, megabytes of compressible sources
|
||||
@@ -562,26 +565,18 @@ func (c *kubernetesContainer) CopyDir(destPath, srcPath string, useGitIgnore, sk
|
||||
}
|
||||
}
|
||||
|
||||
// GetContainerArchive streams the archive, checking the path first as tar writes an empty archive for a missing one.
|
||||
func (c *kubernetesContainer) ExecOutput(ctx context.Context, command []string) (io.ReadCloser, error) {
|
||||
return streamOutput(ctx, func(ctx context.Context, stdout, stderr io.Writer) error {
|
||||
return c.pod.client.exec(ctx, c.pod.name, c.name, command, nil, stdout, stderr)
|
||||
}), nil
|
||||
}
|
||||
|
||||
func (c *kubernetesContainer) GetContainerArchive(ctx context.Context, srcPath string) (io.ReadCloser, error) {
|
||||
reader, writer := io.Pipe()
|
||||
go func() {
|
||||
var stderr bytes.Buffer
|
||||
err := c.pod.client.exec(ctx, c.pod.name, c.name, []string{"sh", "-c", `test -e "$1" -o -L "$1" && exec tar -cf - -C "$2" "$3"`, "sh", srcPath, path.Dir(srcPath), path.Base(srcPath)}, nil, writer, &stderr)
|
||||
if err != nil {
|
||||
err = fmt.Errorf("archive %s: %w: %s", srcPath, err, strings.TrimSpace(stderr.String()))
|
||||
}
|
||||
writer.CloseWithError(err)
|
||||
}()
|
||||
buffered := bufio.NewReader(reader)
|
||||
if _, err := buffered.Peek(1); err != nil {
|
||||
reader.Close()
|
||||
reader, err := c.ExecOutput(ctx, []string{"sh", "-c", `test -e "$1" -o -L "$1" && exec tar -cf - -C "$2" "$3"`, "sh", srcPath, path.Dir(srcPath), path.Base(srcPath)})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return struct {
|
||||
io.Reader
|
||||
io.Closer
|
||||
}{buffered, reader}, nil
|
||||
return startedArchive(reader)
|
||||
}
|
||||
|
||||
func (c *kubernetesContainer) UpdateFromEnv(srcPath string, env *map[string]string) common.Executor {
|
||||
|
||||
@@ -273,6 +273,17 @@ func TestKubernetesPodRunsJobThroughAPI(t *testing.T) {
|
||||
assert.Equal(t, "hunter2\nit's\nme\n5\n"+workdir+"\n", jobLog.String())
|
||||
assert.Equal(t, ExitCodeError(3), job.Exec([]string{"sh", "-c", "echo failing >&2; exit 3"}, nil, "", "")(ctx))
|
||||
assert.Contains(t, jobLog.String(), "failing\n")
|
||||
output, err := job.ExecOutput(ctx, []string{"sh", "-c", `printf 'out\000'; printf ' failed \n' >&2; exit 3`})
|
||||
require.NoError(t, err)
|
||||
content, err := io.ReadAll(output)
|
||||
var exit ExitCodeError
|
||||
require.ErrorAs(t, err, &exit)
|
||||
assert.Equal(t, ExitCodeError(3), exit)
|
||||
assert.Contains(t, err.Error(), ": failed")
|
||||
assert.Equal(t, "out\x00", string(content))
|
||||
require.ErrorAs(t, output.Close(), &exit)
|
||||
assert.Equal(t, ExitCodeError(3), exit)
|
||||
|
||||
for _, query := range cluster.requests["exec"] {
|
||||
assert.NotContains(t, query, "hunter2")
|
||||
}
|
||||
@@ -285,7 +296,7 @@ func TestKubernetesPodRunsJobThroughAPI(t *testing.T) {
|
||||
reader := tar.NewReader(archive)
|
||||
header, err := reader.Next()
|
||||
require.NoError(t, err)
|
||||
content, err := io.ReadAll(reader)
|
||||
content, err = io.ReadAll(reader)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "event.json", header.Name)
|
||||
assert.JSONEq(t, `{"ref":"main"}`, string(content))
|
||||
@@ -309,6 +320,18 @@ func TestKubernetesPodRunsJobThroughAPI(t *testing.T) {
|
||||
assert.Contains(t, lastExec(), "command="+url.QueryEscape(workdir+"/act/actions/sha/"))
|
||||
assert.Contains(t, lastExec(), url.QueryEscape("gzip -dc |"))
|
||||
|
||||
require.NoError(t, job.CopyTarStream(ctx, workdir+"/download/new", func(tw *TarWriter) error {
|
||||
if err := tw.WriteHeader(&tar.Header{Typeflag: tar.TypeReg, Name: "dir/data.txt", Mode: 0o644, Size: 7}); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err := tw.Write([]byte("payload"))
|
||||
return err
|
||||
}))
|
||||
downloaded, err := os.ReadFile(workdir + "/download/new/dir/data.txt")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "payload", string(downloaded))
|
||||
assert.Contains(t, lastExec(), url.QueryEscape("gzip -dc |"))
|
||||
|
||||
pathEnv := map[string]string{}
|
||||
require.NoError(t, job.UpdateFromImageEnv(&pathEnv)(ctx))
|
||||
assert.Equal(t, path, pathEnv["PATH"])
|
||||
|
||||
@@ -5,6 +5,7 @@ package container
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
@@ -15,14 +16,9 @@ import (
|
||||
"golang.org/x/text/encoding/unicode"
|
||||
)
|
||||
|
||||
func newTestHostEnv(t *testing.T) (*HostEnvironment, string) {
|
||||
t.Helper()
|
||||
e := &HostEnvironment{Path: t.TempDir()}
|
||||
return e, filepath.Join(e.Path, "envfile")
|
||||
}
|
||||
|
||||
func TestParseEnvFile(t *testing.T) {
|
||||
t.Parallel()
|
||||
hostEnv := &HostEnvironment{Path: t.TempDir()}
|
||||
largeValue := strings.Repeat("x", 2*1024*1024)
|
||||
encodeUTF16 := func(endianness unicode.Endianness) string {
|
||||
content, err := unicode.UTF16(endianness, unicode.UseBOM).NewEncoder().Bytes([]byte("FOO=bar\r\nMULTI<<EOF\r\nline1\r\nEOF\r\n"))
|
||||
@@ -51,15 +47,15 @@ func TestParseEnvFile(t *testing.T) {
|
||||
{"UTF16 big endian", encodeUTF16(unicode.BigEndian), map[string]string{"FOO": "bar\r", "MULTI": "line1\r"}},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
e, envPath := newTestHostEnv(t)
|
||||
envPath := filepath.Join(hostEnv.Path, test.name)
|
||||
require.NoError(t, os.WriteFile(envPath, []byte(test.content), 0o600))
|
||||
env := map[string]string{}
|
||||
require.NoError(t, parseEnvFile(e, envPath, &env, false)(t.Context()))
|
||||
require.NoError(t, parseEnvFile(hostEnv, envPath, &env, false)(t.Context()))
|
||||
assert.Equal(t, test.want, env)
|
||||
})
|
||||
}
|
||||
t.Run("ordered entries preserve repeated names", func(t *testing.T) {
|
||||
hostEnv, envPath := newTestHostEnv(t)
|
||||
envPath := filepath.Join(hostEnv.Path, "ordered")
|
||||
require.NoError(t, os.WriteFile(envPath, []byte("FOO=first\r\nfoo<<EOF\r\nmiddle\r\nEOF\r\nFOO=last\r\n"), 0o600))
|
||||
var entries [][2]string
|
||||
require.NoError(t, ParseEnvFile(hostEnv, envPath, func(name, value string) {
|
||||
@@ -70,7 +66,7 @@ func TestParseEnvFile(t *testing.T) {
|
||||
}, entries)
|
||||
})
|
||||
t.Run("Windows line endings", func(t *testing.T) {
|
||||
hostEnv, envPath := newTestHostEnv(t)
|
||||
envPath := filepath.Join(hostEnv.Path, "windows")
|
||||
require.NoError(t, os.WriteFile(envPath, []byte("FOO=bar\r\nMULTI<<EOF\r\nline1\r\nline2\nEOF\r\n"), 0o600))
|
||||
env := map[string]string{}
|
||||
require.NoError(t, parseEnvFile(hostEnv, envPath, &env, true)(t.Context()))
|
||||
@@ -80,16 +76,22 @@ func TestParseEnvFile(t *testing.T) {
|
||||
|
||||
func TestParseEnvFileLineExceedsBufferReportsScannerError(t *testing.T) {
|
||||
t.Parallel()
|
||||
e, envPath := newTestHostEnv(t)
|
||||
require.NoError(t, os.WriteFile(envPath, []byte("FOO<<EOF\n"+strings.Repeat("x", 17*1024*1024)+"\nEOF\n"), 0o600))
|
||||
hostEnv := &HostEnvironment{Path: t.TempDir()}
|
||||
envPath := filepath.Join(hostEnv.Path, "envfile")
|
||||
const header, footer = "FOO<<EOF\n", "\nEOF\n"
|
||||
content := bytes.Repeat([]byte("x"), len(header)+17*1024*1024+len(footer))
|
||||
copy(content, header)
|
||||
copy(content[len(content)-len(footer):], footer)
|
||||
require.NoError(t, os.WriteFile(envPath, content, 0o600))
|
||||
|
||||
env := map[string]string{}
|
||||
err := parseEnvFile(e, envPath, &env, false)(t.Context())
|
||||
err := parseEnvFile(hostEnv, envPath, &env, false)(t.Context())
|
||||
require.ErrorIs(t, err, bufio.ErrTooLong)
|
||||
assert.Contains(t, err.Error(), "reading env file")
|
||||
}
|
||||
|
||||
func TestParseEnvFileInvalidHeredoc(t *testing.T) {
|
||||
hostEnv := &HostEnvironment{Path: t.TempDir()}
|
||||
for name, content := range map[string]string{
|
||||
"missing delimiter": "FOO<<EOF\nline1\nline2\n",
|
||||
"empty name": "<<EOF\nline1\nEOF\n",
|
||||
@@ -97,11 +99,11 @@ func TestParseEnvFileInvalidHeredoc(t *testing.T) {
|
||||
"delimiter with carriage return": "FOO<<EOF\nline1\nEOF\r\n",
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
e, envPath := newTestHostEnv(t)
|
||||
envPath := filepath.Join(hostEnv.Path, name)
|
||||
require.NoError(t, os.WriteFile(envPath, []byte("APPLIED=before-error\n"+content), 0o600))
|
||||
|
||||
env := map[string]string{}
|
||||
err := parseEnvFile(e, envPath, &env, false)(t.Context())
|
||||
err := parseEnvFile(hostEnv, envPath, &env, false)(t.Context())
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "delimiter")
|
||||
assert.Equal(t, map[string]string{"APPLIED": "before-error"}, env)
|
||||
|
||||
@@ -6,8 +6,13 @@ package container
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bufio"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
@@ -15,6 +20,84 @@ import (
|
||||
"gitea.com/gitea/runner/act/filecollector"
|
||||
)
|
||||
|
||||
// TarWriter takes directories and files named relative to the destination, owned by the job user.
|
||||
type TarWriter struct {
|
||||
writer *tar.Writer
|
||||
uid, gid int
|
||||
prefix string
|
||||
dirs map[string]bool
|
||||
}
|
||||
|
||||
func (w *TarWriter) Write(data []byte) (int, error) {
|
||||
return w.writer.Write(data)
|
||||
}
|
||||
|
||||
func (w *TarWriter) WriteHeader(hdr *tar.Header) error {
|
||||
name := strings.TrimSuffix(hdr.Name, "/")
|
||||
if !fs.ValidPath(name) {
|
||||
return fmt.Errorf("tar entry %q escapes the destination", hdr.Name)
|
||||
}
|
||||
if hdr.Typeflag != tar.TypeDir && hdr.Typeflag != tar.TypeReg {
|
||||
return fmt.Errorf("tar entry %q: unsupported type %q", hdr.Name, hdr.Typeflag)
|
||||
}
|
||||
for index := range len(name) {
|
||||
if dir := name[:index]; name[index] == '/' && !w.dirs[dir] {
|
||||
if err := w.write(&tar.Header{Typeflag: tar.TypeDir, Name: path.Join(w.prefix, dir), Mode: 0o755}); err != nil {
|
||||
return err
|
||||
}
|
||||
w.dirs[dir] = true
|
||||
}
|
||||
}
|
||||
if hdr.Typeflag == tar.TypeDir {
|
||||
w.dirs[name] = true
|
||||
}
|
||||
entry := *hdr
|
||||
entry.Name = path.Join(w.prefix, name)
|
||||
return w.write(&entry)
|
||||
}
|
||||
|
||||
func (w *TarWriter) write(hdr *tar.Header) error {
|
||||
hdr.Uid, hdr.Gid, hdr.Uname, hdr.Gname = w.uid, w.gid, "", "" // tar running as root prefers names over ids
|
||||
if hdr.Typeflag == tar.TypeDir {
|
||||
hdr.Name += "/"
|
||||
}
|
||||
return w.writer.WriteHeader(hdr)
|
||||
}
|
||||
|
||||
func writeTarStream(dst io.Writer, uid, gid int, prefix string, parents []string, write func(*TarWriter) error) error {
|
||||
tw := &TarWriter{writer: tar.NewWriter(dst), uid: uid, gid: gid, prefix: prefix, dirs: map[string]bool{}}
|
||||
for _, name := range parents {
|
||||
if err := tw.write(&tar.Header{Typeflag: tar.TypeDir, Name: name, Mode: 0o755}); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err := write(tw); errors.Is(err, io.EOF) {
|
||||
return fmt.Errorf("tar stream: %v", err) // %v, as a wrapped io.EOF reads as the archive's end
|
||||
} else if err != nil {
|
||||
return err
|
||||
}
|
||||
return tw.writer.Close()
|
||||
}
|
||||
|
||||
func pipeTar(uid, gid int, prefix string, parents []string, write func(*TarWriter) error) *io.PipeReader {
|
||||
reader, writer := io.Pipe()
|
||||
go func() { writer.CloseWithError(writeTarStream(writer, uid, gid, prefix, parents, write)) }()
|
||||
return reader
|
||||
}
|
||||
|
||||
// startedArchive returns reader once its first byte arrives, so failing to start archiving errors here, not on read.
|
||||
func startedArchive(reader io.ReadCloser) (io.ReadCloser, error) {
|
||||
buffered := bufio.NewReader(reader)
|
||||
if _, err := buffered.Peek(1); err != nil {
|
||||
reader.Close()
|
||||
return nil, err
|
||||
}
|
||||
return struct {
|
||||
io.Reader
|
||||
io.Closer
|
||||
}{buffered, reader}, nil
|
||||
}
|
||||
|
||||
func writeFilesTar(ctx context.Context, w io.Writer, uid, gid int, files ...*FileEntry) error {
|
||||
tw := tar.NewWriter(w)
|
||||
for _, file := range files {
|
||||
|
||||
@@ -28,10 +28,12 @@ func gitCmd(t *testing.T, args ...string) {
|
||||
}
|
||||
|
||||
func TestCollectFilesKeepsWhatGitTracksOrDoesNotIgnore(t *testing.T) {
|
||||
repoDir := filepath.Join(t.TempDir(), "mygitrepo")
|
||||
for _, dir := range []string{repoDir, filepath.Join(repoDir, "sub"), filepath.Join(repoDir, "nested")} {
|
||||
gitCmd(t, "init", "-q", dir)
|
||||
repoDir := t.TempDir()
|
||||
gitCmd(t, "init", "-q", "--template=", repoDir)
|
||||
for _, dir := range []string{"sub", "nested"} {
|
||||
require.NoError(t, os.CopyFS(filepath.Join(repoDir, dir, ".git"), os.DirFS(filepath.Join(repoDir, ".git"))))
|
||||
}
|
||||
require.NoError(t, os.Mkdir(filepath.Join(repoDir, "ign"), 0o755))
|
||||
for name, body := range map[string]string{
|
||||
".gitignore": ".*\nign/\n",
|
||||
".env": "",
|
||||
@@ -43,15 +45,15 @@ func TestCollectFilesKeepsWhatGitTracksOrDoesNotIgnore(t *testing.T) {
|
||||
"sub/kept": "",
|
||||
"nested/kept": "",
|
||||
} {
|
||||
require.NoError(t, os.MkdirAll(filepath.Dir(filepath.Join(repoDir, name)), 0o755))
|
||||
require.NoError(t, os.WriteFile(filepath.Join(repoDir, name), []byte(body), 0o644))
|
||||
}
|
||||
require.NoError(t, os.Mkdir(filepath.Join(repoDir, "uninitialized"), 0o755))
|
||||
gitCmd(t, "-C", repoDir, "add", "-f", ".gitignore", "Renamed", "ign/tracked")
|
||||
require.NoError(t, os.Rename(filepath.Join(repoDir, "Renamed"), filepath.Join(repoDir, "renamed")))
|
||||
for _, gitlink := range []string{"sub", "uninitialized"} {
|
||||
gitCmd(t, "-C", repoDir, "update-index", "--add", "--cacheinfo", "160000,4b825dc642cb6eb9a060e54bf8d69288fbee4904,"+gitlink)
|
||||
}
|
||||
gitCmd(t, "-C", repoDir, "update-index", "--add",
|
||||
"--cacheinfo", "160000,4b825dc642cb6eb9a060e54bf8d69288fbee4904,sub",
|
||||
"--cacheinfo", "160000,4b825dc642cb6eb9a060e54bf8d69288fbee4904,uninitialized",
|
||||
)
|
||||
expected := map[string]string{".gitignore": "", "ign/tracked": "", "nested/kept": "", "renamed": "", "sub/.gitignore": "", "sub/kept": ""}
|
||||
if runtime.GOOS != "windows" {
|
||||
require.NoError(t, os.Chmod(filepath.Join(repoDir, "renamed"), 0o777|os.ModeSticky))
|
||||
|
||||
@@ -472,7 +472,7 @@ func newStepContainer(ctx context.Context, step step, image string, cmd, entrypo
|
||||
|
||||
envList = append(envList, rc.runnerEnv(ctx)...)
|
||||
|
||||
binds, mounts := rc.GetBindsAndMounts(ctx)
|
||||
binds, mounts, _ := rc.GetBindsAndMounts(ctx)
|
||||
networkMode := "container:" + rc.jobContainerName()
|
||||
if rc.IsHostEnv() {
|
||||
networkMode = "default"
|
||||
|
||||
@@ -11,6 +11,8 @@ import (
|
||||
"gitea.com/gitea/runner/act/common/git"
|
||||
"gitea.com/gitea/runner/act/common/logging"
|
||||
"gitea.com/gitea/runner/act/common/logging/logtest"
|
||||
"gitea.com/gitea/runner/act/container"
|
||||
"gitea.com/gitea/runner/internal/pkg/action"
|
||||
|
||||
"gitea.dev/actionslib/pkg/model"
|
||||
"github.com/stretchr/testify/assert"
|
||||
@@ -92,7 +94,7 @@ func TestCompositeActionParity(t *testing.T) {
|
||||
return "", "", assert.AnError
|
||||
})
|
||||
rc := &RunContext{
|
||||
Config: &Config{GitHubInstance: "github.com", ActionCacheDir: t.TempDir(), PresetGitHubContext: &model.GithubContext{}},
|
||||
Config: &Config{GitHubInstance: "github.com", PresetGitHubContext: &model.GithubContext{}},
|
||||
Run: &model.Run{JobID: "job", Workflow: &model.Workflow{Jobs: map[string]*model.Job{"job": {}}}},
|
||||
JobContainer: &jobContainerMock{},
|
||||
}
|
||||
@@ -182,3 +184,21 @@ func newTestCompositeRunContext(t *testing.T, parent *RunContext) *RunContext {
|
||||
require.NoError(t, err)
|
||||
return composite
|
||||
}
|
||||
|
||||
func TestBuiltinActionGetsLowercaseInputsAndTheJobTokenInComposites(t *testing.T) {
|
||||
job := makeTestRC(t, "runs-on: ubuntu-latest")
|
||||
job.Config.Secrets = map[string]string{"ACTIONS_RUNTIME_TOKEN": "daemon-token"}
|
||||
job.JobContainer = &container.HostEnvironment{Path: t.TempDir(), ActPath: t.TempDir(), Workdir: t.TempDir()}
|
||||
composite := newTestCompositeRunContext(t, job)
|
||||
composite.JobContainer = job.JobContainer
|
||||
for _, rc := range []*RunContext{job, composite} {
|
||||
var got *action.Context
|
||||
sab := &stepActionBuiltin{Step: &model.Step{ID: "x", With: map[string]string{"Name": "x"}}, RunContext: rc, run: func(_ctx context.Context, c *action.Context) error {
|
||||
got = c
|
||||
return nil
|
||||
}}
|
||||
require.NoError(t, sab.main()(t.Context()))
|
||||
assert.Equal(t, "daemon-token", got.RuntimeToken)
|
||||
assert.Equal(t, map[string]string{"name": "x"}, got.Inputs)
|
||||
}
|
||||
}
|
||||
+17
-13
@@ -249,6 +249,7 @@ func TestActionRunner(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestGoActionUsesLinuxExecutablePaths(t *testing.T) {
|
||||
actionRoot := t.TempDir()
|
||||
for _, stage := range []string{"main", "pre", "post"} {
|
||||
t.Run(stage, func(t *testing.T) {
|
||||
cm := &containerMock{}
|
||||
@@ -264,7 +265,7 @@ func TestGoActionUsesLinuxExecutablePaths(t *testing.T) {
|
||||
JobContainer: cm,
|
||||
},
|
||||
action: &model.Action{Runs: model.ActionRuns{Using: model.ActionRunsUsingGo, Main: "main.go", Pre: "pre.go", Post: "post.go"}},
|
||||
actionDir: t.TempDir(),
|
||||
actionDir: filepath.Join(actionRoot, stage),
|
||||
env: map[string]string{},
|
||||
remoteAction: mustNewRemoteAction(t, "org/repo@ref"),
|
||||
}
|
||||
@@ -504,11 +505,21 @@ func TestExecAsDockerHoldsCloneLockForRemoteUncached(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestExecAsDockerImageChecks(t *testing.T) {
|
||||
actionRoot := t.TempDir()
|
||||
originalExists, originalBuild, originalContainer := ContainerImageExistsLocally, ContainerNewDockerBuildExecutor, ContainerNewContainer
|
||||
t.Cleanup(func() {
|
||||
ContainerImageExistsLocally, ContainerNewDockerBuildExecutor, ContainerNewContainer = originalExists, originalBuild, originalContainer
|
||||
})
|
||||
ContainerNewContainer = func(*container.NewContainerInput) container.ExecutionsEnvironment { return fakeContainer{} }
|
||||
var removed bool
|
||||
fakeDockerDaemon(t, func(writer http.ResponseWriter, request *http.Request) {
|
||||
if request.Method == http.MethodDelete {
|
||||
removed = true
|
||||
_, _ = io.WriteString(writer, `[]`)
|
||||
} else {
|
||||
_, _ = io.WriteString(writer, `{"Architecture":"aarch64"}`)
|
||||
}
|
||||
})
|
||||
|
||||
for _, testcase := range []struct {
|
||||
name, platform string
|
||||
@@ -529,18 +540,11 @@ func TestExecAsDockerImageChecks(t *testing.T) {
|
||||
checks = append(checks, platform)
|
||||
return testcase.exists && (platform == "any" || testcase.matches), nil
|
||||
}
|
||||
var built, removed bool
|
||||
var built bool
|
||||
removed = false
|
||||
ContainerNewDockerBuildExecutor = func(container.NewDockerBuildExecutorInput) common.Executor {
|
||||
return func(context.Context) error { built = true; return nil }
|
||||
}
|
||||
fakeDockerDaemon(t, func(writer http.ResponseWriter, request *http.Request) {
|
||||
if request.Method == http.MethodDelete {
|
||||
removed = true
|
||||
_, _ = io.WriteString(writer, `[]`)
|
||||
} else {
|
||||
_, _ = io.WriteString(writer, `{"Architecture":"aarch64"}`)
|
||||
}
|
||||
})
|
||||
step := &stepActionRemote{
|
||||
Step: &model.Step{ID: "action"},
|
||||
RunContext: &RunContext{
|
||||
@@ -555,7 +559,7 @@ func TestExecAsDockerImageChecks(t *testing.T) {
|
||||
action: &model.Action{Runs: model.ActionRuns{Using: "docker", Image: "Dockerfile"}},
|
||||
env: map[string]string{},
|
||||
}
|
||||
dir := t.TempDir()
|
||||
dir := filepath.Join(actionRoot, testcase.name)
|
||||
require.NoError(t, execAsDocker(t.Context(), step, "action", dir, dir, false, stepStageMain))
|
||||
assert.Equal(t, testcase.wantChecks, checks)
|
||||
assert.Equal(t, testcase.wantBuild, built)
|
||||
@@ -727,9 +731,9 @@ func TestExecAsDockerStageEntrypoint(t *testing.T) {
|
||||
cm.On("Close").Return(func(context.Context) error { return nil })
|
||||
|
||||
if tc.stage == stepStageMain {
|
||||
require.NoError(t, runActionImpl(step, t.TempDir(), step.remoteAction)(ctx))
|
||||
require.NoError(t, runActionImpl(step, "", step.remoteAction)(ctx))
|
||||
} else {
|
||||
require.NoError(t, execAsDocker(ctx, step, "action", t.TempDir(), t.TempDir(), false, tc.stage))
|
||||
require.NoError(t, execAsDocker(ctx, step, "action", "", "", false, tc.stage))
|
||||
}
|
||||
if tc.warning == "" {
|
||||
assert.Empty(t, hook.AllRecords())
|
||||
|
||||
@@ -118,7 +118,11 @@ func (rc *RunContext) commandHandler(ctx context.Context) common.LineHandler {
|
||||
rawLogger.Infof("%s", formatAnnotation(command, kvPairs, arg))
|
||||
return false
|
||||
case "group", "endgroup":
|
||||
return true
|
||||
if strings.HasPrefix(line, "::"+command+"::") || strings.HasPrefix(line, "##["+command+"]") {
|
||||
return true
|
||||
}
|
||||
rawLogger.Infof("::%s::%s", command, EscapeCommandData(arg))
|
||||
return false
|
||||
}
|
||||
return echoed
|
||||
}
|
||||
|
||||
@@ -193,6 +193,11 @@ func TestCommandLogWriter(t *testing.T) {
|
||||
log []string
|
||||
masks []string
|
||||
}{
|
||||
{
|
||||
name: "mask commands decode modern and legacy escapes once",
|
||||
steps: [][]string{{"::add-mask::%25", "##[add-mask]alpha beta%3Bgamma%5Ddelta%250A"}},
|
||||
masks: []string{"%", "alpha beta;gamma]delta%0A"},
|
||||
},
|
||||
{
|
||||
name: "applied commands are hidden unless echoed",
|
||||
steps: [][]string{{
|
||||
@@ -235,10 +240,23 @@ func TestCommandLogWriter(t *testing.T) {
|
||||
"::notice::title: m", "::error::x: m", "::error::x:1: m", "::notice::std::vector", "::error::x;y: m]",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "groups normalize unsupported prefixes and preserve recognized prefixes",
|
||||
steps: [][]string{{
|
||||
"::group::a%3Bb%5Dc%250A", "::endgroup::", "##[group]a%3Bb%5Dc%250A", "##[endgroup]",
|
||||
"::GrOuP::a%25b%0Ac", "::EnDgRoUp::", " \t::group::indented", " \t::endgroup::",
|
||||
"prefix ##[GrOuP]a%3Bb%5Dc%253B%0A", "prefix ##[EnDgRoUp]",
|
||||
}},
|
||||
log: []string{
|
||||
"::group::a%3Bb%5Dc%250A", "::endgroup::", "##[group]a%3Bb%5Dc%250A", "##[endgroup]",
|
||||
"::group::a%25b%0Ac", "::endgroup::", "::group::indented", "::endgroup::",
|
||||
"::group::a;b]c%253B%0A", "::endgroup::",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "stop-commands keeps lines verbatim until resumed",
|
||||
steps: [][]string{{"::stop-commands::resume-token", "::add-mask::kept", "::set-output name=a::b", "::RESUME-TOKEN::", "::add-mask::masked"}},
|
||||
log: []string{"::stop-commands::resume-token", "::add-mask::kept", "::set-output name=a::b", "::RESUME-TOKEN::"},
|
||||
steps: [][]string{{"::stop-commands::resume-token", "::add-mask::kept", "::set-output name=a::b", " \t::GrOuP::kept%3B", "prefix ##[EnDgRoUp]", "::RESUME-TOKEN::", "::add-mask::masked"}},
|
||||
log: []string{"::stop-commands::resume-token", "::add-mask::kept", "::set-output name=a::b", " \t::GrOuP::kept%3B", "prefix ##[EnDgRoUp]", "::RESUME-TOKEN::"},
|
||||
masks: []string{"resume-token", "masked"},
|
||||
},
|
||||
{
|
||||
|
||||
@@ -66,6 +66,12 @@ func (cm *containerMock) Exec(command []string, env map[string]string, user, wor
|
||||
return cm.Called(command, env, user, workdir).Get(0).(func(context.Context) error)
|
||||
}
|
||||
|
||||
func (cm *containerMock) ExecOutput(ctx context.Context, command []string) (io.ReadCloser, error) {
|
||||
args := cm.Called(ctx, command)
|
||||
output, _ := args.Get(0).(io.ReadCloser)
|
||||
return output, args.Error(1)
|
||||
}
|
||||
|
||||
func (cm *containerMock) GetContainerArchive(ctx context.Context, srcPath string) (io.ReadCloser, error) {
|
||||
args := cm.Called(ctx, srcPath)
|
||||
err, _ := args.Get(1).(error)
|
||||
@@ -82,3 +88,12 @@ func (cm *containerMock) Inspect(ctx context.Context) (*container.Info, error) {
|
||||
err, _ := args.Get(1).(error)
|
||||
return info, err
|
||||
}
|
||||
|
||||
func (cm *containerMock) CopyTarStream(ctx context.Context, destPath string, write func(*container.TarWriter) error) error {
|
||||
return cm.Called(ctx, destPath, write).Error(0)
|
||||
}
|
||||
|
||||
func (cm *containerMock) ReplaceLogWriter(stdout, stderr io.Writer) (io.Writer, io.Writer) {
|
||||
args := cm.Called(stdout, stderr)
|
||||
return args.Get(0).(io.Writer), args.Get(1).(io.Writer)
|
||||
}
|
||||
@@ -49,6 +49,7 @@ func TestDockerProxyMountedJob(t *testing.T) {
|
||||
ContainerNamePrefix: resourceName,
|
||||
ContainerDaemonSocket: dockerClient.DaemonHost(),
|
||||
ContainerMaxLifetime: 2 * time.Minute,
|
||||
Sweep: true,
|
||||
ContainerOptions: "-v /usr/local/bin/docker:/usr/local/bin/docker:ro -v /usr/local/libexec/docker/cli-plugins:/usr/local/libexec/docker/cli-plugins:ro",
|
||||
ValidVolumes: []string{"/usr/local/bin/docker", "/usr/local/libexec/docker/cli-plugins"},
|
||||
Env: map[string]string{"PROXY_TEST_RESOURCE": resourceName, "PROXY_TEST_MODE": mode, "PROXY_TEST_IMAGE": baseImage, "COMPOSE_PROJECT_NAME": resourceName},
|
||||
|
||||
@@ -136,7 +136,7 @@ func getHashFilesFunction(ctx context.Context, rc *RunContext) func(v []reflect.
|
||||
followSymlink := false
|
||||
|
||||
for i, p := range v {
|
||||
s := p.String()
|
||||
s := exprparser.CoerceToString(p)
|
||||
if i == 0 && strings.HasPrefix(s, "--") {
|
||||
if strings.EqualFold(s, "--follow-symbolic-links") {
|
||||
followSymlink = true
|
||||
|
||||
@@ -178,7 +178,7 @@ func (c *hashFilesContainer) ReplaceLogWriter(stdout, stderr io.Writer) (io.Writ
|
||||
}
|
||||
|
||||
func TestGetHashFilesFunction(t *testing.T) {
|
||||
const failureMessage = "hashFiles('dir''file, *.sum') failed. Fail to hash files under directory '/workspace'"
|
||||
const failureMessage = "hashFiles('dir''file, *.sum, 123, true, ') failed. Fail to hash files under directory '/workspace'"
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
output string
|
||||
@@ -192,7 +192,7 @@ func TestGetHashFilesFunction(t *testing.T) {
|
||||
{name: "success", output: "patterns __OUTPUT__wrong__OUTPUT__ echoed\n__OUTPUT__digest__OUTPUT__", hash: "digest"},
|
||||
{name: "no matches", output: "__OUTPUT____OUTPUT__"},
|
||||
{name: "script failure", output: "Cannot find: node in PATH\n__OUTPUT__digest__OUTPUT__", execErr: container.ExitCodeError(1), err: failureMessage},
|
||||
{name: "timeout", execErr: context.DeadlineExceeded, err: "hashFiles('dir''file, *.sum') couldn't finish within 120 seconds"},
|
||||
{name: "timeout", execErr: context.DeadlineExceeded, err: "hashFiles('dir''file, *.sum, 123, true, ') couldn't finish within 120 seconds"},
|
||||
{name: "job deadline", execErr: context.DeadlineExceeded, expired: true, err: context.DeadlineExceeded.Error()},
|
||||
{name: "node missing", execErr: fmt.Errorf("wrapped: %w", container.ExitCodeError(127)), warning: true},
|
||||
{name: "node not executable", execErr: container.ExitCodeError(126), warning: true},
|
||||
@@ -205,7 +205,7 @@ func TestGetHashFilesFunction(t *testing.T) {
|
||||
jobContainer := &hashFilesContainer{stdout: stdout, stderr: stderr, copyErr: test.copyErr}
|
||||
jobContainer.execute = func(command []string, env map[string]string, user, workdir string) common.Executor {
|
||||
assert.Equal(t, []string{"node", "/var/run/act/workflow/hashfiles/index.js"}, command)
|
||||
assert.Equal(t, map[string]string{"patterns": "dir'file\n*.sum", "followSymbolicLinks": "true"}, env)
|
||||
assert.Equal(t, map[string]string{"patterns": "dir'file\n*.sum\n123\ntrue\n", "followSymbolicLinks": "true"}, env)
|
||||
assert.Empty(t, user)
|
||||
assert.Empty(t, workdir)
|
||||
return func(_ctx context.Context) error {
|
||||
@@ -223,7 +223,7 @@ func TestGetHashFilesFunction(t *testing.T) {
|
||||
result, err := getHashFilesFunction(ctx, &RunContext{
|
||||
Config: &Config{Workdir: "/workspace"},
|
||||
JobContainer: jobContainer,
|
||||
})([]reflect.Value{reflect.ValueOf("--follow-symbolic-links"), reflect.ValueOf("dir'file"), reflect.ValueOf("*.sum")})
|
||||
})([]reflect.Value{reflect.ValueOf("--follow-symbolic-links"), reflect.ValueOf("dir'file"), reflect.ValueOf("*.sum"), reflect.ValueOf(float64(123)), reflect.ValueOf(true), {}})
|
||||
if test.err == "" {
|
||||
require.NoError(t, err)
|
||||
} else {
|
||||
@@ -412,11 +412,10 @@ func TestGetEvaluatorInputsBoolean(t *testing.T) {
|
||||
}
|
||||
|
||||
for _, eventName := range []string{"workflow_call", "workflow_dispatch"} {
|
||||
wf, err := model.ReadWorkflow(strings.NewReader(workflow(eventName)))
|
||||
require.NoError(t, err)
|
||||
for _, table := range tables {
|
||||
t.Run(eventName+"/"+table.name, func(t *testing.T) {
|
||||
wf, err := model.ReadWorkflow(strings.NewReader(workflow(eventName)))
|
||||
require.NoError(t, err)
|
||||
|
||||
rc := &RunContext{
|
||||
Config: &Config{Workdir: "."},
|
||||
Run: &model.Run{JobID: "job1", Workflow: wf},
|
||||
|
||||
@@ -993,6 +993,7 @@ func tarArchive(t *testing.T, entries ...tarEntry) []byte {
|
||||
buf := &bytes.Buffer{}
|
||||
tw := tar.NewWriter(buf)
|
||||
for _, entry := range entries {
|
||||
buf.Grow(len(entry.body) + 3*512)
|
||||
typeflag := entry.typeflag
|
||||
if typeflag == 0 {
|
||||
typeflag = tar.TypeReg
|
||||
@@ -1009,7 +1010,7 @@ func tarArchive(t *testing.T, entries ...tarEntry) []byte {
|
||||
}
|
||||
require.NoError(t, tw.WriteHeader(header))
|
||||
if typeflag == tar.TypeReg {
|
||||
_, err := tw.Write([]byte(entry.body))
|
||||
_, err := io.CopyN(tw, strings.NewReader(entry.body), int64(len(entry.body)))
|
||||
require.NoError(t, err)
|
||||
}
|
||||
}
|
||||
@@ -1149,12 +1150,12 @@ func TestTryUploadJobSummaryMasksSecrets(t *testing.T) {
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
content, err := testcase.encoding.NewEncoder().Bytes([]byte(testcase.content))
|
||||
content, err := testcase.encoding.NewEncoder().String(testcase.content)
|
||||
require.NoError(t, err)
|
||||
cm := &containerMock{}
|
||||
cm.On("GetContainerArchive", mock.Anything, "/var/run/act/workflow/step-summary-0.md").Return(
|
||||
io.NopCloser(bytes.NewReader(tarArchive(t, tarEntry{
|
||||
name: "step-summary-0.md", body: string(content),
|
||||
name: "step-summary-0.md", body: content,
|
||||
}))),
|
||||
nil,
|
||||
).Once()
|
||||
|
||||
@@ -71,6 +71,11 @@ func (c *hookContainer) UpdateFromEnv(_ string, env *map[string]string) common.E
|
||||
}
|
||||
}
|
||||
|
||||
func (c *hookContainer) ExecOutput(_ctx context.Context, command []string) (io.ReadCloser, error) {
|
||||
c.cmd = command
|
||||
return io.NopCloser(bytes.NewBufferString(c.output)), c.err
|
||||
}
|
||||
|
||||
func (c *hookContainer) GetContainerArchive(context.Context, string) (io.ReadCloser, error) {
|
||||
return io.NopCloser(bytes.NewReader(c.pathTar)), nil
|
||||
}
|
||||
|
||||
+37
-9
@@ -263,29 +263,27 @@ func AppendSecretMaskers(oldnew []string, secrets map[string]string) []string {
|
||||
return oldnew
|
||||
}
|
||||
|
||||
// AppendSecretMasker registers v and each of its lines, as GitHub does.
|
||||
// AppendSecretMasker skips one-byte line fragments to avoid masking punctuation from structured secrets.
|
||||
func AppendSecretMasker(oldnew []string, v string) []string {
|
||||
ret := appendMaskedValue(oldnew, v)
|
||||
for l := range strings.FieldsFuncSeq(v, func(r rune) bool { return r == '\r' || r == '\n' }) {
|
||||
ret = appendMaskedValue(ret, strings.TrimSpace(l))
|
||||
if line := strings.TrimSpace(l); len(line) > 1 {
|
||||
ret = appendMaskedValue(ret, line)
|
||||
}
|
||||
}
|
||||
return ret
|
||||
}
|
||||
|
||||
// appendMaskedValue registers one value and every shape it takes on its way into a log.
|
||||
func appendMaskedValue(ret []string, v string) []string {
|
||||
// formatted JSON secrets could otherwise mask {,[,],} everywhere
|
||||
if len(strings.TrimSpace(v)) <= 1 || slices.Contains(ret, v) {
|
||||
if strings.TrimSpace(v) == "" || slices.Contains(ret, v) {
|
||||
return ret
|
||||
}
|
||||
ret = append(ret, v, "***")
|
||||
// command data reaches the log escaped, so "pass%word" also arrives as "pass%25word"
|
||||
if strings.ContainsAny(v, "%\r\n") {
|
||||
ret = append(ret, EscapeCommandData(v), "***")
|
||||
}
|
||||
for _, encode := range secretValueEncoders {
|
||||
encoded := encode(v)
|
||||
// An encoding that leaves the value unchanged is already masked above.
|
||||
if encoded == v || len(encoded) <= 1 || slices.Contains(ret, encoded) {
|
||||
continue
|
||||
}
|
||||
@@ -319,7 +317,7 @@ func valueMasker(insecureSecrets bool, oldnew []string) entryProcessor {
|
||||
masks := Masks(ctx)
|
||||
|
||||
if len(*masks) == 0 {
|
||||
r.Message = defReplacer.Replace(r.Message)
|
||||
r.Message = MaskLog(r.Message, defReplacer)
|
||||
return r
|
||||
}
|
||||
|
||||
@@ -339,12 +337,42 @@ func valueMasker(insecureSecrets bool, oldnew []string) entryProcessor {
|
||||
cmasker := replacer
|
||||
mu.Unlock()
|
||||
|
||||
r.Message = cmasker.Replace(r.Message)
|
||||
r.Message = MaskLog(r.Message, cmasker)
|
||||
|
||||
return r
|
||||
}
|
||||
}
|
||||
|
||||
// MaskLog decodes command data before masking to cover partially escaped values.
|
||||
func MaskLog(message string, replacer *strings.Replacer) string {
|
||||
var command, data string
|
||||
var found bool
|
||||
unescape := UnescapeCommandData
|
||||
if strings.HasPrefix(message, "##[") {
|
||||
command, data, found = strings.Cut(message[3:], "]")
|
||||
unescape = UnescapeLegacyCommand
|
||||
} else if strings.HasPrefix(message, "::") {
|
||||
command, data, found = strings.Cut(message[2:], "::")
|
||||
if name, _, properties := strings.Cut(command, " "); properties {
|
||||
switch name {
|
||||
case "error", "warning", "notice", "debug":
|
||||
command = name
|
||||
}
|
||||
}
|
||||
}
|
||||
switch command {
|
||||
case "error", "warning", "notice", "debug", "group", "endgroup", "add-matcher":
|
||||
if found {
|
||||
trimmedData := strings.TrimRight(data, "\r\n")
|
||||
normalized := message[:len(message)-len(data)] + EscapeCommandData(unescape(trimmedData)) + data[len(trimmedData):]
|
||||
if masked := replacer.Replace(normalized); masked != normalized {
|
||||
return masked
|
||||
}
|
||||
}
|
||||
}
|
||||
return replacer.Replace(message)
|
||||
}
|
||||
|
||||
type maskedHandler struct {
|
||||
logging.Handler
|
||||
masker entryProcessor
|
||||
|
||||
@@ -28,6 +28,7 @@ func TestValueMasker(t *testing.T) {
|
||||
secrets map[string]string
|
||||
masks []string
|
||||
disallowed []string
|
||||
want string
|
||||
}{
|
||||
{
|
||||
name: "Multiline Private Key",
|
||||
@@ -49,6 +50,60 @@ func TestValueMasker(t *testing.T) {
|
||||
secrets: map[string]string{"TOKEN": "pass%word"},
|
||||
disallowed: []string{"pass%25word"},
|
||||
},
|
||||
{
|
||||
name: "one-character mask",
|
||||
lines: "%",
|
||||
masks: []string{"%"},
|
||||
want: "***",
|
||||
},
|
||||
{
|
||||
name: "fully escaped legacy secret",
|
||||
lines: "##[group]alpha beta%3Bgamma%5Ddelta",
|
||||
secrets: map[string]string{"TOKEN": "alpha beta;gamma]delta"},
|
||||
want: "##[group]***",
|
||||
},
|
||||
{
|
||||
name: "partially escaped legacy mask",
|
||||
lines: "##[group]alpha beta%3Bgamma]delta",
|
||||
masks: []string{"alpha beta;gamma]delta"},
|
||||
want: "##[group]***",
|
||||
},
|
||||
{
|
||||
name: "partially escaped modern mask preserves unrelated escapes",
|
||||
lines: "::group::alpha%25beta%gamma tail%250A%0A",
|
||||
masks: []string{"alpha%beta%gamma"},
|
||||
want: "::group::*** tail%250A%0A",
|
||||
},
|
||||
{
|
||||
name: "command escapes are decoded once",
|
||||
lines: "##[group]alpha%253Bbeta%5Dgamma",
|
||||
masks: []string{"alpha;beta]gamma"},
|
||||
want: "##[group]alpha%253Bbeta%5Dgamma",
|
||||
},
|
||||
{
|
||||
name: "plain output preserves escapes",
|
||||
lines: "alpha beta%3Bgamma%5Ddelta",
|
||||
masks: []string{"alpha beta;gamma]delta"},
|
||||
want: "alpha beta%3Bgamma%5Ddelta",
|
||||
},
|
||||
{
|
||||
name: "command display preserves literal escapes around a mask",
|
||||
lines: "##[command]alpha tail%3Bvalue%5D",
|
||||
masks: []string{"alpha"},
|
||||
want: "##[command]*** tail%3Bvalue%5D",
|
||||
},
|
||||
{
|
||||
name: "unknown modern commands preserve literal escapes",
|
||||
lines: "::unknown::alpha tail%25value",
|
||||
masks: []string{"alpha"},
|
||||
want: "::unknown::*** tail%25value",
|
||||
},
|
||||
{
|
||||
name: "masks can span command headers and data",
|
||||
lines: "::group::hidden other",
|
||||
masks: []string{"group::hidden", "other"},
|
||||
want: "::*** ***",
|
||||
},
|
||||
}
|
||||
for _, entry := range table {
|
||||
t.Run(entry.name, func(t *testing.T) {
|
||||
@@ -56,6 +111,9 @@ func TestValueMasker(t *testing.T) {
|
||||
masker := valueMasker(false, AppendSecretMaskers(nil, entry.secrets))
|
||||
for line := range strings.SplitSeq(entry.lines, "\n") {
|
||||
lentry := masker(ctx, logging.Record{Message: line})
|
||||
if entry.want != "" {
|
||||
assert.Equal(t, entry.want, lentry.Message)
|
||||
}
|
||||
for _, line := range entry.disallowed {
|
||||
assert.NotContains(t, lentry.Message, line)
|
||||
}
|
||||
@@ -139,6 +197,7 @@ func TestValueMaskerEncodedMasks(t *testing.T) {
|
||||
{"encoded: " + base64.StdEncoding.EncodeToString([]byte(" s3cr3t ")), "encoded: ***"},
|
||||
{"encoded: " + base64.StdEncoding.EncodeToString([]byte("s3cr3t")), "encoded: ***"},
|
||||
{"encoded: " + base64.StdEncoding.EncodeToString([]byte("first")), "encoded: ***"},
|
||||
{"##[group]s3cr3t value\r\n", "##[group]***\r\n"},
|
||||
} {
|
||||
entry := masker(WithMasks(t.Context(), &masks), logging.Record{Message: tc.line})
|
||||
assert.Equal(t, tc.want, entry.Message, tc.line)
|
||||
@@ -194,17 +253,15 @@ func TestValueMaskerCachedReplacerSeesNewMasks(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestAppendSecretMaskerSkipsUselessEncodings(t *testing.T) {
|
||||
// A token with no character an escape would touch only gains its base64 forms:
|
||||
// JSON, query and path escaping all leave it unchanged.
|
||||
pairs := AppendSecretMasker(nil, "plaintoken")
|
||||
assert.Equal(t, []string{
|
||||
"plaintoken", "***", "cGxhaW50b2tlbg==", "***", "bGFpbnRva2Vu", "***", "YWludG9rZW4=", "***",
|
||||
"aW50b2tl", "***", "YWludG9r", "***", "bGFpbnRv", "***",
|
||||
}, pairs)
|
||||
|
||||
// Too short to mask.
|
||||
assert.Empty(t, AppendSecretMasker(nil, "x"))
|
||||
assert.Contains(t, AppendSecretMasker(nil, "x"), "x")
|
||||
assert.Empty(t, AppendSecretMasker(nil, " \t"))
|
||||
assert.NotContains(t, AppendSecretMasker(nil, "{\nvalue\n}"), "{")
|
||||
assert.NotContains(t, AppendSecretMasker(nil, `"123456"`), "123456")
|
||||
}
|
||||
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
package runner
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"io"
|
||||
@@ -209,14 +210,14 @@ func TestPatchedBundleAcrossActions(t *testing.T) {
|
||||
assert.Equal(t, data, out, "an untouched bundle must come back byte for byte")
|
||||
return
|
||||
}
|
||||
assert.NotContains(t, string(out), ".LOCALHOST", "a copy of the gate was missed")
|
||||
assert.False(t, bytes.Contains(out, []byte(".LOCALHOST")), "a copy of the gate was missed")
|
||||
|
||||
if !strings.Contains(string(data), CacheServiceV2Env) {
|
||||
if !bytes.Contains(data, []byte(CacheServiceV2Env)) {
|
||||
return
|
||||
}
|
||||
assert.NotZero(t, strings.Count(string(out), "(process.env."+cacheURLEnv+"||process.env"),
|
||||
assert.NotZero(t, bytes.Count(out, []byte("(process.env."+cacheURLEnv+"||process.env")),
|
||||
"the cache service URL was not redirected")
|
||||
assert.Equal(t, strings.Count(string(data), resultsURLEnv), strings.Count(string(out), resultsURLEnv),
|
||||
assert.Equal(t, bytes.Count(data, []byte(resultsURLEnv)), bytes.Count(out, []byte(resultsURLEnv)),
|
||||
"a read of the results URL was lost, it must stay as the fallback")
|
||||
})
|
||||
}
|
||||
@@ -311,18 +312,18 @@ func TestCodeSplitActionFindsTheCacheService(t *testing.T) {
|
||||
requireHostTools(t, "node", "rustc")
|
||||
|
||||
dir := tempDirPath(t)
|
||||
require.NoError(t, os.Mkdir(filepath.Join(dir, "dist"), 0o755))
|
||||
for _, name := range []string{
|
||||
"action.yml", "dist/restore.js", "dist/save.js", "dist/cleanup-BWEbZ6YT.js",
|
||||
"dist/cache-CTfQoCTL.js", "dist/cache-DC26rUaF.js", "dist/state-cjs-CkvFUaNy.js",
|
||||
} {
|
||||
body, err := os.ReadFile(bundleFromGitHub(t, "Swatinem/rust-cache", "v2.9.2", name))
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, os.MkdirAll(filepath.Dir(filepath.Join(dir, name)), 0o755))
|
||||
require.NoError(t, os.WriteFile(filepath.Join(dir, name), body, 0o600))
|
||||
}
|
||||
body, err := os.ReadFile(filepath.Join(dir, "action.yml"))
|
||||
require.NoError(t, err)
|
||||
action, err := model.ReadAction(strings.NewReader(string(body)))
|
||||
action, err := model.ReadAction(bytes.NewReader(body))
|
||||
require.NoError(t, err)
|
||||
inputs := map[string]string{"lookup-only": "true", "add-rust-environment-hash-key": "false", "cache-bin": "false"}
|
||||
for name, input := range action.Inputs {
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
package runner
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"os"
|
||||
"os/exec"
|
||||
@@ -182,9 +183,9 @@ func TestPatchedBundleSurvivesInsideAStringLiteral(t *testing.T) {
|
||||
out, patched := patchedBundle([]byte(embedded))
|
||||
require.True(t, patched)
|
||||
|
||||
file := filepath.Join(t.TempDir(), "bundle.js")
|
||||
require.NoError(t, os.WriteFile(file, out, 0o600))
|
||||
checked, err := exec.CommandContext(t.Context(), "node", "--check", file).CombinedOutput()
|
||||
cmd := exec.CommandContext(t.Context(), "node", "--check", "-")
|
||||
cmd.Stdin = bytes.NewReader(out)
|
||||
checked, err := cmd.CombinedOutput()
|
||||
require.NoError(t, err, "%s", checked)
|
||||
}
|
||||
|
||||
@@ -256,6 +257,14 @@ func TestActionScriptPaths(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestPatchActionsAtTheContainerCopy(t *testing.T) {
|
||||
actionDir := t.TempDir()
|
||||
script := filepath.Join(actionDir, "dist", "cache.js")
|
||||
require.NoError(t, os.MkdirAll(filepath.Dir(script), 0o755))
|
||||
require.NoError(t, os.WriteFile(filepath.Join(filepath.Dir(script), "index.js"), []byte("require('./cache.js')"), 0o600))
|
||||
require.NoError(t, os.WriteFile(script, []byte(gateTSC), 0o600))
|
||||
info, err := os.Stat(script)
|
||||
require.NoError(t, err)
|
||||
|
||||
copiedBundles := func(t *testing.T, noPatch bool) ([]string, []*container.FileEntry, int64) {
|
||||
t.Helper()
|
||||
|
||||
@@ -264,19 +273,12 @@ func TestPatchActionsAtTheContainerCopy(t *testing.T) {
|
||||
Step: &model.Step{Uses: "owner/repo/sub@v1"},
|
||||
remoteAction: &remoteAction{Org: "owner", Repo: "repo", Path: "sub", Ref: "v1"},
|
||||
action: &model.Action{Runs: model.ActionRuns{Using: "node20", Main: "../dist/index.js"}},
|
||||
actionDir: t.TempDir(),
|
||||
actionDir: actionDir,
|
||||
RunContext: &RunContext{
|
||||
Config: &Config{ActionCacheDir: t.TempDir(), NoActionPatch: noPatch},
|
||||
Config: &Config{NoActionPatch: noPatch},
|
||||
JobContainer: cm,
|
||||
},
|
||||
}
|
||||
script := filepath.Join(sar.actionDir, "dist", "cache.js")
|
||||
require.NoError(t, os.MkdirAll(filepath.Dir(script), 0o755))
|
||||
require.NoError(t, os.WriteFile(filepath.Join(filepath.Dir(script), "index.js"), []byte("require('./cache.js')"), 0o600))
|
||||
require.NoError(t, os.WriteFile(script, []byte(gateTSC), 0o600))
|
||||
info, err := os.Stat(script)
|
||||
require.NoError(t, err)
|
||||
|
||||
var calls []string
|
||||
var copied []*container.FileEntry
|
||||
cm.On("CopyDir", "/var/run/act/actions/repo/", sar.actionDir+"/", false, true).Return(func(context.Context) error {
|
||||
|
||||
@@ -33,24 +33,17 @@ func TestReusableWorkflowCachedBranchRefRefreshes(t *testing.T) {
|
||||
}
|
||||
|
||||
remoteDir := t.TempDir()
|
||||
gitMust(t, "", "init", "--bare", "--initial-branch=master", remoteDir)
|
||||
|
||||
workDir := t.TempDir()
|
||||
gitMust(t, "", "clone", remoteDir, workDir)
|
||||
gitMust(t, workDir, "config", "user.email", "test@test")
|
||||
gitMust(t, workDir, "config", "user.name", "test")
|
||||
gitMust(t, workDir, "checkout", "-b", "master")
|
||||
gitMust(t, "", "init", "--template=", "--initial-branch=master", remoteDir)
|
||||
|
||||
const workflowPath = ".gitea/workflows/reusable.yml"
|
||||
tmpl := func(tag string) string {
|
||||
return "name: reusable\non:\n workflow_call:\njobs:\n build:\n runs-on: ubuntu-latest\n steps:\n - run: echo " + tag + "\n"
|
||||
}
|
||||
|
||||
require.NoError(t, os.MkdirAll(filepath.Join(workDir, ".gitea/workflows"), 0o755))
|
||||
require.NoError(t, os.WriteFile(filepath.Join(workDir, workflowPath), []byte(tmpl("v1")), 0o644))
|
||||
gitMust(t, workDir, "add", workflowPath)
|
||||
gitMust(t, workDir, "commit", "-m", "v1")
|
||||
gitMust(t, workDir, "push", "-u", "origin", "master")
|
||||
require.NoError(t, os.MkdirAll(filepath.Join(remoteDir, ".gitea/workflows"), 0o755))
|
||||
require.NoError(t, os.WriteFile(filepath.Join(remoteDir, workflowPath), []byte(tmpl("v1")), 0o644))
|
||||
gitMust(t, remoteDir, "add", workflowPath)
|
||||
gitMust(t, remoteDir, "commit", "-m", "v1")
|
||||
|
||||
rc := &RunContext{
|
||||
Config: &Config{Env: testCheckoutEnv()},
|
||||
@@ -70,9 +63,8 @@ func TestReusableWorkflowCachedBranchRefRefreshes(t *testing.T) {
|
||||
require.Equal(t, tmpl("v1"), string(got))
|
||||
|
||||
// Branch tip moves; cache key (cacheDir) does not.
|
||||
require.NoError(t, os.WriteFile(filepath.Join(workDir, workflowPath), []byte(tmpl("v2")), 0o644))
|
||||
gitMust(t, workDir, "commit", "-am", "v2")
|
||||
gitMust(t, workDir, "push", "origin", "master")
|
||||
require.NoError(t, os.WriteFile(filepath.Join(remoteDir, workflowPath), []byte(tmpl("v2")), 0o644))
|
||||
gitMust(t, remoteDir, "commit", "-am", "v2")
|
||||
|
||||
require.NoError(t, cloneRemoteReusableWorkflow(rc, remoteDir, "master", cacheDir)(t.Context()))
|
||||
got, err = os.ReadFile(filepath.Join(cacheDir, workflowPath))
|
||||
@@ -214,7 +206,7 @@ func TestInstanceAuthAndTransportOnlyApplyToTheInstance(t *testing.T) {
|
||||
|
||||
func gitMust(t *testing.T, dir string, args ...string) {
|
||||
t.Helper()
|
||||
cmd := exec.Command("git", args...)
|
||||
cmd := exec.Command("git", append([]string{"-c", "user.name=test", "-c", "user.email=test@test"}, args...)...)
|
||||
if dir != "" {
|
||||
cmd.Dir = dir
|
||||
}
|
||||
|
||||
@@ -30,6 +30,7 @@ import (
|
||||
"time"
|
||||
|
||||
"gitea.com/gitea/runner/act/common"
|
||||
"gitea.com/gitea/runner/act/common/logging"
|
||||
"gitea.com/gitea/runner/act/container"
|
||||
"gitea.com/gitea/runner/act/ghcontext"
|
||||
"gitea.com/gitea/runner/internal/pkg/lock"
|
||||
@@ -75,6 +76,7 @@ type RunContext struct {
|
||||
actionInputs map[string]any // inputs of the composite action this runs, nil for a job
|
||||
Masks []string
|
||||
cleanUpJobContainer common.Executor
|
||||
taskWorkspace string
|
||||
deferVolumeCleanup func(common.Executor)
|
||||
caller *caller // job calling this RunContext (reusable workflows)
|
||||
workflowCallInputs map[string]any // the caller's with:, resolved once by resolveWorkflowCall
|
||||
@@ -378,8 +380,8 @@ func splitVolumes(specs []string) ([]string, map[string]string, map[string]bool)
|
||||
return binds, mounts, targets
|
||||
}
|
||||
|
||||
// Returns the binds and mounts for the container, resolving paths as appopriate
|
||||
func (rc *RunContext) GetBindsAndMounts(ctx context.Context) ([]string, map[string]string) {
|
||||
// Returns the container's binds, mounts and bound task workspace, resolving paths as appropriate
|
||||
func (rc *RunContext) GetBindsAndMounts(ctx context.Context) ([]string, map[string]string, string) {
|
||||
name := rc.jobContainerName()
|
||||
ext := container.LinuxContainerEnvironmentExtensions{}
|
||||
|
||||
@@ -399,12 +401,16 @@ func (rc *RunContext) GetBindsAndMounts(ctx context.Context) ([]string, map[stri
|
||||
}
|
||||
mounts[name+"-env"] = ext.GetActPath() // runner-internal, never overridable
|
||||
|
||||
taskWorkspace := ""
|
||||
if workdir := ext.ToContainerPath(rc.Config.Workdir); !claimed[workdir] {
|
||||
source := rc.workdirMountRoot()
|
||||
target := ext.ToContainerPath(source)
|
||||
if claimed[target] {
|
||||
source, target = rc.Config.Workdir, workdir
|
||||
}
|
||||
if rc.Config.BindWorkdir && rc.Config.PresetGitHubContext != nil {
|
||||
taskWorkspace = target
|
||||
}
|
||||
if rc.Config.BindWorkdir {
|
||||
bindModifiers := ""
|
||||
if runtime.GOOS == "darwin" {
|
||||
@@ -419,7 +425,7 @@ func (rc *RunContext) GetBindsAndMounts(ctx context.Context) ([]string, map[stri
|
||||
}
|
||||
}
|
||||
|
||||
return binds, mounts
|
||||
return binds, mounts, taskWorkspace
|
||||
}
|
||||
|
||||
func (rc *RunContext) startHostEnvironment() common.Executor {
|
||||
@@ -672,7 +678,8 @@ func (rc *RunContext) startJobContainer() common.Executor {
|
||||
return err
|
||||
}
|
||||
rc.startDockerProxy(ctx)
|
||||
containerInput.Binds, containerInput.Mounts = rc.GetBindsAndMounts(ctx)
|
||||
var taskWorkspace string
|
||||
containerInput.Binds, containerInput.Mounts, taskWorkspace = rc.GetBindsAndMounts(ctx)
|
||||
containerInput.ValidVolumes = rc.validVolumes()
|
||||
|
||||
rc.jobNetworkName = networkName
|
||||
@@ -687,6 +694,7 @@ func (rc *RunContext) startJobContainer() common.Executor {
|
||||
func(ctx context.Context) error { return rc.createJobVolumes(ctx, containerInput.Mounts) },
|
||||
rc.JobContainer.Create(rc.Config.ContainerCapAdd, rc.Config.ContainerCapDrop),
|
||||
rc.JobContainer.Start(false),
|
||||
func(context.Context) error { rc.taskWorkspace = taskWorkspace; return nil },
|
||||
rc.captureJobContainerInfo(),
|
||||
rc.copyWorkflowFiles(),
|
||||
rc.waitForServiceContainers(),
|
||||
@@ -801,7 +809,10 @@ func (rc *RunContext) cleanupJobResources(networkName string, createAndDeleteNet
|
||||
errs := []error{rc.closeDockerProxy(ctx)}
|
||||
var containers []common.Executor
|
||||
if rc.JobContainer != nil {
|
||||
containers = append(containers, rc.JobContainer.Remove())
|
||||
containers = append(containers, func(ctx context.Context) error {
|
||||
rc.emptyTaskWorkspace(ctx)
|
||||
return rc.JobContainer.Remove()(ctx)
|
||||
})
|
||||
}
|
||||
if len(rc.serviceContainers) > 0 {
|
||||
logger.Infof("Cleaning up services for job %s", rc.JobName)
|
||||
@@ -810,7 +821,7 @@ func (rc *RunContext) cleanupJobResources(networkName string, createAndDeleteNet
|
||||
errs = append(errs, runAll(ctx, containers))
|
||||
ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), jobCleanupTimeout)
|
||||
defer cancel()
|
||||
if !common.Dryrun(ctx) && !rc.Config.NoSweep && (preclean || rc.hadDockerProxy) {
|
||||
if !common.Dryrun(ctx) && rc.Config.Sweep && (preclean || rc.hadDockerProxy) {
|
||||
errs = append(errs, container.RemoveDockerJobResources(ctx, rc.jobContainerName())) // after the containers, whose endpoints would block removing the job networks
|
||||
}
|
||||
var rest []common.Executor
|
||||
@@ -831,6 +842,42 @@ func (rc *RunContext) cleanupJobResources(networkName string, createAndDeleteNet
|
||||
}
|
||||
}
|
||||
|
||||
// exits 3 unless workspace $1 is safe to empty per mountinfo $2
|
||||
const emptyTaskWorkspaceScript = `case "$1" in /?*/?*) ;; *) exit 3 ;; esac
|
||||
mounted=
|
||||
while read -r _ _ _ _ point _; do
|
||||
case "$point" in "$1") mounted=1 ;; "$1"/*) exit 3 ;; esac
|
||||
done <"$2"
|
||||
[ "$mounted" ] || exit 3
|
||||
rm -rf "$1"/* "$1"/.[!.]* "$1"/..?*`
|
||||
|
||||
var runnerCanRemoveJobFiles = runtime.GOOS != "linux" || os.Geteuid() == 0 // root, or a desktop VM mapping file ownership, replaced in tests
|
||||
|
||||
// runs as root with root-owned PATH dirs only, skipped for called workflows sharing the workspace
|
||||
func (rc *RunContext) emptyTaskWorkspace(ctx context.Context) {
|
||||
if rc.taskWorkspace == "" || rc.caller != nil || runnerCanRemoveJobFiles {
|
||||
return
|
||||
}
|
||||
budget := 30 * time.Second
|
||||
if deadline, ok := ctx.Deadline(); ok {
|
||||
budget = min(budget, time.Until(deadline)/2-container.DrainGracePeriod)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(ctx, budget)
|
||||
defer cancel()
|
||||
stdout, stderr := rc.JobContainer.ReplaceLogWriter(io.Discard, io.Discard)
|
||||
defer rc.JobContainer.ReplaceLogWriter(stdout, stderr)
|
||||
err := rc.JobContainer.Exec(
|
||||
[]string{"sh", "-c", emptyTaskWorkspaceScript, "sh", rc.taskWorkspace, "/proc/self/mountinfo"},
|
||||
map[string]string{"PATH": "/usr/sbin:/usr/bin:/sbin:/bin"}, "0", "/",
|
||||
)(common.WithLogger(ctx, logging.New(logging.DiscardHandler)))
|
||||
switch {
|
||||
case errors.Is(err, container.ExitCodeError(3)):
|
||||
logging.Infof("not emptying workspace %s of %s: under two levels deep, not a mountpoint as given, or a mount below it", rc.taskWorkspace, rc.jobContainerName())
|
||||
case err != nil:
|
||||
logging.Infof("could not empty workspace %s of %s: %v", rc.taskWorkspace, rc.jobContainerName(), err)
|
||||
}
|
||||
}
|
||||
|
||||
func runAll(ctx context.Context, executors []common.Executor) error {
|
||||
errs := make([]error, len(executors))
|
||||
var wg sync.WaitGroup
|
||||
|
||||
+132
-17
@@ -16,6 +16,7 @@ import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"slices"
|
||||
@@ -455,7 +456,7 @@ func TestRunContext_GetBindsAndMounts(t *testing.T) {
|
||||
t.Run(testcase.name+testBindSuffix, func(t *testing.T) {
|
||||
rctemplate.Config.Workdir = testcase.name
|
||||
rctemplate.Config.BindWorkdir = bindWorkDir
|
||||
gotbind, gotmount := rctemplate.GetBindsAndMounts(t.Context())
|
||||
gotbind, gotmount, _ := rctemplate.GetBindsAndMounts(t.Context())
|
||||
|
||||
if bindWorkDir {
|
||||
fullBind := testcase.name + ":" + testcase.want
|
||||
@@ -507,7 +508,7 @@ func TestRunContext_GetBindsAndMounts(t *testing.T) {
|
||||
rc.ExprEval = rc.NewExpressionEvaluator(t.Context())
|
||||
require.NoError(t, rc.resolvePlatformImage(t.Context()))
|
||||
|
||||
gotbind, gotmount := rc.GetBindsAndMounts(t.Context())
|
||||
gotbind, gotmount, _ := rc.GetBindsAndMounts(t.Context())
|
||||
assert.Contains(t, gotbind, "/host/mame/roms:/root/.mame/roms:ro")
|
||||
assert.NotContains(t, gotbind, "${{ secrets.MAME }}")
|
||||
assert.NotContains(t, gotmount, "${{ secrets.MAME }}")
|
||||
@@ -522,7 +523,7 @@ func TestRunContext_GetBindsAndMounts(t *testing.T) {
|
||||
rc.containerSpec.Volumes = testcase.volumes
|
||||
rc.Run.JobID = "job1"
|
||||
|
||||
gotbind, gotmount := rc.GetBindsAndMounts(t.Context())
|
||||
gotbind, gotmount, _ := rc.GetBindsAndMounts(t.Context())
|
||||
|
||||
if len(testcase.wantbind) > 0 {
|
||||
assert.Contains(t, gotbind, testcase.wantbind)
|
||||
@@ -551,12 +552,12 @@ func TestRunContext_GetBindsAndMounts(t *testing.T) {
|
||||
t.Run("ToolCacheMount", func(t *testing.T) {
|
||||
rc := newRunContext(&Config{})
|
||||
|
||||
_, gotmount := rc.GetBindsAndMounts(t.Context())
|
||||
_, gotmount, _ := rc.GetBindsAndMounts(t.Context())
|
||||
assert.NotContains(t, gotmount, sharedToolCacheVolume)
|
||||
|
||||
rc.Config.SharedToolCache = true
|
||||
rc.containerSpec.Volumes = []string{"/unapproved:" + container.DefaultToolCache}
|
||||
_, gotmount = rc.GetBindsAndMounts(t.Context())
|
||||
_, gotmount, _ = rc.GetBindsAndMounts(t.Context())
|
||||
assert.Equal(t, container.DefaultToolCache, gotmount[sharedToolCacheVolume])
|
||||
})
|
||||
|
||||
@@ -569,16 +570,26 @@ func TestRunContext_GetBindsAndMounts(t *testing.T) {
|
||||
Config: &Config{BindWorkdir: true, Workdir: "/workspace/1/owner/repo", PresetGitHubContext: &model.GithubContext{}, ValidVolumes: []string{"claimed"}},
|
||||
}
|
||||
|
||||
gotbind, _ := rc.GetBindsAndMounts(t.Context())
|
||||
gotbind, _, workspace := rc.GetBindsAndMounts(t.Context())
|
||||
assert.True(t, slices.ContainsFunc(gotbind, func(bind string) bool { return strings.HasPrefix(bind, "/workspace/1:/workspace/1") }), gotbind)
|
||||
assert.Equal(t, "/workspace/1", workspace)
|
||||
|
||||
rc.Config.BindWorkdir = false
|
||||
_, gotmount := rc.GetBindsAndMounts(t.Context())
|
||||
_, gotmount, workspace := rc.GetBindsAndMounts(t.Context())
|
||||
assert.Equal(t, "/workspace/1", gotmount[rc.jobContainerName()])
|
||||
assert.Empty(t, workspace)
|
||||
|
||||
rc.containerSpec.Volumes = []string{"claimed:/workspace/1"}
|
||||
_, gotmount = rc.GetBindsAndMounts(t.Context())
|
||||
_, gotmount, _ = rc.GetBindsAndMounts(t.Context())
|
||||
assert.Equal(t, "/workspace/1/owner/repo", gotmount[rc.jobContainerName()])
|
||||
rc.Config.BindWorkdir = true
|
||||
_, _, workspace = rc.GetBindsAndMounts(t.Context())
|
||||
assert.Equal(t, "/workspace/1/owner/repo", workspace)
|
||||
|
||||
rc.containerSpec.Volumes = nil
|
||||
rc.Config.PresetGitHubContext = nil
|
||||
_, _, workspace = rc.GetBindsAndMounts(t.Context())
|
||||
assert.Empty(t, workspace)
|
||||
})
|
||||
}
|
||||
|
||||
@@ -611,7 +622,7 @@ func TestRunContextValidVolumes(t *testing.T) {
|
||||
rc.JobContainer = hostEnv
|
||||
rc.Config.BindWorkdir = false
|
||||
rc.Config.SharedToolCache = true
|
||||
binds, mounts := rc.GetBindsAndMounts(t.Context())
|
||||
binds, mounts, _ := rc.GetBindsAndMounts(t.Context())
|
||||
assert.Subset(t, binds, []string{hostEnv.Path + ":" + hostEnv.Path, hostEnv.GetActPath() + ":" + hostEnv.GetActPath()})
|
||||
assert.Contains(t, mounts, sharedToolCacheVolume)
|
||||
assert.Equal(t, binds, container.SanitizeBinds(t.Context(), rc.validVolumes(), binds))
|
||||
@@ -683,6 +694,110 @@ func TestCleanupJobVolumesReapsAbandonedDeferredCleanup(t *testing.T) {
|
||||
assert.ElementsMatch(t, []string{"became-active", "remove-failed", "foreign", "fresh", "unknown-age"}, slices.Collect(maps.Keys(volumes)))
|
||||
}
|
||||
|
||||
func TestCleanupJobResourcesEmptiesTaskWorkspace(t *testing.T) {
|
||||
defer func(previous bool) { runnerCanRemoveJobFiles = previous }(runnerCanRemoveJobFiles)
|
||||
for _, tt := range []struct {
|
||||
name, workspace string
|
||||
caller *caller
|
||||
runnerCanRemove bool
|
||||
execErr error
|
||||
log string
|
||||
}{
|
||||
{name: "refusal is logged", workspace: "/workspace/1", execErr: container.ExitCodeError(3), log: "not emptying workspace /workspace/1"},
|
||||
{name: "failure does not fail the job", workspace: "/workspace/1", execErr: errors.New("no rm"), log: "could not empty workspace /workspace/1"},
|
||||
{name: "no runner bind"},
|
||||
{name: "called workflow shares it", workspace: "/workspace/1", caller: &caller{}},
|
||||
{name: "runner removes it itself", workspace: "/workspace/1", runnerCanRemove: true},
|
||||
} {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
runnerCanRemoveJobFiles = tt.runnerCanRemove
|
||||
processLog := logtest.SetDefault(t)
|
||||
var jobLog bytes.Buffer
|
||||
var calls []string
|
||||
job := &containerMock{}
|
||||
if tt.log != "" {
|
||||
job.On("ReplaceLogWriter", io.Discard, io.Discard).Return(&jobLog, &jobLog).Once()
|
||||
job.On("ReplaceLogWriter", &jobLog, &jobLog).Return(io.Discard, io.Discard).Once()
|
||||
job.On("Exec", mock.MatchedBy(func(command []string) bool { return command[4] == tt.workspace }), map[string]string{"PATH": "/usr/sbin:/usr/bin:/sbin:/bin"}, "0", "/").Return(func(ctx context.Context) error {
|
||||
calls = append(calls, "exec")
|
||||
deadline, ok := ctx.Deadline()
|
||||
assert.True(t, ok)
|
||||
assert.LessOrEqual(t, time.Until(deadline), 10*time.Second-container.DrainGracePeriod)
|
||||
common.Logger(ctx).Infof("workspace script output")
|
||||
return tt.execErr
|
||||
})
|
||||
}
|
||||
job.On("Remove").Return(func(context.Context) error {
|
||||
calls = append(calls, "remove")
|
||||
return nil
|
||||
})
|
||||
rc := &RunContext{
|
||||
Config: &Config{},
|
||||
Run: &model.Run{Workflow: &model.Workflow{Name: "workflow"}, JobID: "job"},
|
||||
JobContainer: job,
|
||||
taskWorkspace: tt.workspace,
|
||||
caller: tt.caller,
|
||||
deferVolumeCleanup: func(common.Executor) {},
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(t.Context(), 20*time.Second)
|
||||
defer cancel()
|
||||
ctx = common.WithLogger(ctx, logging.New(logging.NewLevelHandler(newJobLogHandler(&jobLog, cyan), logging.LevelInfo)))
|
||||
require.NoError(t, rc.cleanupJobResources("", false, false)(ctx))
|
||||
job.AssertExpectations(t)
|
||||
assert.NotContains(t, jobLog.String(), "workspace")
|
||||
if tt.log == "" {
|
||||
assert.Equal(t, []string{"remove"}, calls)
|
||||
return
|
||||
}
|
||||
assert.Equal(t, []string{"exec", "remove"}, calls)
|
||||
assert.True(t, slices.ContainsFunc(processLog.AllRecords(), func(record logging.Record) bool {
|
||||
return strings.HasPrefix(record.Message, tt.log)
|
||||
}), processLog.AllRecords())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestEmptyTaskWorkspaceScript(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("the script runs in the linux job container")
|
||||
}
|
||||
run := func(t *testing.T, workspace string, mountpoints ...string) error {
|
||||
t.Helper()
|
||||
var mountinfo strings.Builder
|
||||
for _, mountpoint := range mountpoints {
|
||||
fmt.Fprintf(&mountinfo, "36 35 98:0 /src %s rw - ext4 /dev/sda1 rw\n", mountpoint)
|
||||
}
|
||||
info := filepath.Join(t.TempDir(), "mountinfo")
|
||||
require.NoError(t, os.WriteFile(info, []byte(mountinfo.String()), 0o600))
|
||||
return exec.Command("sh", "-c", emptyTaskWorkspaceScript, "sh", workspace, info).Run()
|
||||
}
|
||||
|
||||
workspace := t.TempDir()
|
||||
for _, file := range []string{"owner/repo/file", ".hidden", "..odd"} {
|
||||
require.NoError(t, os.MkdirAll(filepath.Dir(filepath.Join(workspace, file)), 0o755))
|
||||
require.NoError(t, os.WriteFile(filepath.Join(workspace, file), nil, 0o600))
|
||||
}
|
||||
require.NoError(t, run(t, workspace, workspace))
|
||||
entries, err := os.ReadDir(workspace)
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, entries)
|
||||
|
||||
for _, tt := range []struct {
|
||||
name, workspace string
|
||||
mountpoints []string
|
||||
}{
|
||||
{"a mount below it", "/nonexistent/1", []string{"/nonexistent/1", "/nonexistent/1/owner/repo/cache"}},
|
||||
{"a workspace that is not a mountpoint", "/nonexistent/1", []string{"/nonexistent/2"}},
|
||||
{"a top-level directory", "/nonexistent", []string{"/nonexistent"}},
|
||||
} {
|
||||
t.Run("refuses "+tt.name, func(t *testing.T) {
|
||||
var exitErr *exec.ExitError
|
||||
require.ErrorAs(t, run(t, tt.workspace, tt.mountpoints...), &exitErr)
|
||||
assert.Equal(t, 3, exitErr.ExitCode())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemoveDockerActionImagesByTag(t *testing.T) {
|
||||
var removed []string
|
||||
fakeDockerDaemon(t, func(writer http.ResponseWriter, request *http.Request) {
|
||||
@@ -725,7 +840,7 @@ func TestCleanupJobResources(t *testing.T) {
|
||||
service.On("Remove").Return(removeAlongside(serviceRemoving, jobRemoving, removeError)).Once()
|
||||
service.On("Close").Return(func(context.Context) error { return closeError }).Once()
|
||||
rc := &RunContext{
|
||||
Config: &Config{CacheContainer: "cache-container", NoSweep: name == "kept proxy"},
|
||||
Config: &Config{CacheContainer: "cache-container", Sweep: name != "kept proxy"},
|
||||
Run: &model.Run{Workflow: &model.Workflow{Name: "wf"}, JobID: "job"},
|
||||
JobContainer: job,
|
||||
serviceContainers: []*serviceContainer{{name: "svc", container: service}},
|
||||
@@ -1490,7 +1605,7 @@ func TestCopyWorkflowFilesHandsNodeTheRunnerCAUnlessTheJobOrImageSetsIt(t *testi
|
||||
return env
|
||||
}
|
||||
|
||||
assert.Equal(t, actPath+"/ca.crt", stepEnv(caFile, nil, nil, nil)["NODE_EXTRA_CA_CERTS"])
|
||||
assert.Equal(t, filepath.ToSlash(actPath)+"/ca.crt", stepEnv(caFile, nil, nil, nil)["NODE_EXTRA_CA_CERTS"])
|
||||
ca, err := os.ReadFile(filepath.Join(actPath, "ca.crt"))
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "PEM\n", string(ca))
|
||||
@@ -1666,21 +1781,21 @@ func TestRunContextUpdateExtraPath(t *testing.T) {
|
||||
longPath := strings.Repeat("x", 64*1024+1)
|
||||
for _, testcase := range []struct {
|
||||
name string
|
||||
content []byte
|
||||
content string
|
||||
want string
|
||||
wantErr bool
|
||||
}{
|
||||
{name: "UTF-8 BOM", content: []byte("\xef\xbb\xbf/utf8/tools\n"), want: "/utf8/tools"},
|
||||
{name: "UTF-16 BOM", content: []byte{0xff, 0xfe, 'C', 0, ':', 0, '\\', 0, 't', 0, 'o', 0, 'o', 0, 'l', 0, 's', 0, '\n', 0}, want: `C:\tools`},
|
||||
{name: "over 64 KiB", content: []byte(longPath + "\n"), want: longPath},
|
||||
{name: "over 16 MiB", content: []byte(strings.Repeat("x", 16*1024*1024+1)), wantErr: true},
|
||||
{name: "UTF-8 BOM", content: "\xef\xbb\xbf/utf8/tools\n", want: "/utf8/tools"},
|
||||
{name: "UTF-16 BOM", content: "\xff\xfeC\x00:\x00\\\x00t\x00o\x00o\x00l\x00s\x00\n\x00", want: `C:\tools`},
|
||||
{name: "over 64 KiB", content: longPath + "\n", want: longPath},
|
||||
{name: "over 16 MiB", content: strings.Repeat("x", 16*1024*1024+1), wantErr: true},
|
||||
} {
|
||||
t.Run(testcase.name, func(t *testing.T) {
|
||||
logger := logging.New(logging.DiscardHandler)
|
||||
ctx := common.WithLogger(t.Context(), logger)
|
||||
jobContainer := &containerMock{}
|
||||
jobContainer.On("GetContainerArchive", mock.Anything, "/github/path").
|
||||
Return(io.NopCloser(bytes.NewReader(tarArchive(t, tarEntry{name: "path", body: string(testcase.content)}))), nil).Once()
|
||||
Return(io.NopCloser(bytes.NewReader(tarArchive(t, tarEntry{name: "path", body: testcase.content}))), nil).Once()
|
||||
defer jobContainer.AssertExpectations(t)
|
||||
rc := &RunContext{JobContainer: jobContainer}
|
||||
|
||||
|
||||
@@ -81,6 +81,7 @@ type Config struct {
|
||||
PlatformPicker func(labels []string) string
|
||||
KubernetesPicker func(labels []string) container.KubernetesOptions
|
||||
InstanceTransport http.RoundTripper
|
||||
ResultsURL string // the results service as the runner reaches it, never the job's ACTIONS_RESULTS_URL, which a step can rewrite
|
||||
JobLoggerLevel *logging.Level // the level of job logger
|
||||
ValidVolumes []string // only volumes (and bind mounts) in this slice can be mounted on the job container or service containers
|
||||
ContainerVolumes []string // mounted into every job, service and docker action container
|
||||
@@ -91,7 +92,7 @@ type Config struct {
|
||||
MaxParallel int // max parallel jobs to run across all workflows (0 = no limit, uses CPU count)
|
||||
AllocatePTY bool // allocate a pseudo-TTY for each step's process
|
||||
ServiceReadyTimeout time.Duration // how long a job waits for its service containers to report healthy (0 uses the default)
|
||||
NoSweep bool // keep the containers, networks and volumes a job creates through its docker proxy
|
||||
Sweep bool // remove the containers, networks and volumes a job creates through its docker proxy
|
||||
RunnerName string // name this runner registered with, reported as `runner.name`, defaults to the hostname
|
||||
JobStartedHook string // script run inside the job environment before the job's first step; ACTIONS_RUNNER_HOOK_JOB_STARTED is read from Env when empty
|
||||
JobCompletedHook string // script run inside the job environment after the job's last step; ACTIONS_RUNNER_HOOK_JOB_COMPLETED is read from Env when empty
|
||||
|
||||
@@ -4,17 +4,24 @@
|
||||
package runner
|
||||
|
||||
import (
|
||||
"cmp"
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"gitea.com/gitea/runner/act/common"
|
||||
"gitea.com/gitea/runner/internal/pkg/action"
|
||||
"gitea.com/gitea/runner/internal/pkg/action/checkout"
|
||||
"gitea.com/gitea/runner/internal/pkg/action/downloadartifact"
|
||||
"gitea.com/gitea/runner/internal/pkg/action/uploadartifact"
|
||||
|
||||
"gitea.dev/actionslib/pkg/model"
|
||||
)
|
||||
|
||||
var builtinActions = map[string]func(context.Context, *action.Context) error{
|
||||
"checkout": checkout.Main,
|
||||
"checkout": checkout.Main,
|
||||
"download-artifact": downloadartifact.Main,
|
||||
"upload-artifact": uploadartifact.Main,
|
||||
}
|
||||
|
||||
type stepActionBuiltin struct {
|
||||
@@ -34,13 +41,18 @@ func (sab *stepActionBuiltin) main() common.Executor {
|
||||
}
|
||||
|
||||
printRunActionHeader(ctx, sab.Step, nil, sab.env, sab.RunContext)
|
||||
rawLogger := common.Logger(ctx).With(rawOutputField, true)
|
||||
|
||||
rc := sab.RunContext
|
||||
rc.ApplyExtraPath(ctx, &sab.env)
|
||||
inputs := make(map[string]string, len(sab.Step.With))
|
||||
for name, value := range sab.Step.With {
|
||||
inputs[strings.ToLower(name)] = value // input names are case-insensitive on GitHub
|
||||
}
|
||||
return sab.run(ctx, &action.Context{
|
||||
Container: rc.JobContainer,
|
||||
Github: getStepGithubContext(ctx, sab),
|
||||
Inputs: sab.Step.With,
|
||||
Inputs: inputs,
|
||||
HostWorkdir: rc.Config.Workdir,
|
||||
Workspace: rc.JobContainer.ToContainerPath(rc.Config.Workdir),
|
||||
BindWorkdir: rc.Config.BindWorkdir,
|
||||
@@ -51,6 +63,13 @@ func (sab *stepActionBuiltin) main() common.Executor {
|
||||
ClientKeyFile: rc.Config.ClientKeyFile,
|
||||
CAFile: rc.jobCAFile(),
|
||||
Env: sab.env,
|
||||
ResultsURL: rc.Config.ResultsURL,
|
||||
RuntimeToken: cmp.Or(rc.runtimeToken, rc.topLevelRunContext().Config.Secrets["ACTIONS_RUNTIME_TOKEN"]), // composite contexts have no secrets
|
||||
Transport: rc.Config.InstanceTransport,
|
||||
Infof: rawLogger.Infof,
|
||||
Warning: func(format string, args ...any) {
|
||||
rawLogger.Infof("%s", formatAnnotation("warning", nil, fmt.Sprintf(format, args...)))
|
||||
},
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
@@ -35,14 +35,8 @@ func setFetch(t *testing.T, fetch func(context.Context, git.NewGitCloneExecutorI
|
||||
func TestShortSHAActionRejected(t *testing.T) {
|
||||
actionRoot := t.TempDir()
|
||||
repo := filepath.Join(actionRoot, "actions", "hello-world-docker-action")
|
||||
require.NoError(t, os.MkdirAll(repo, 0o755))
|
||||
gitMust(t, "", "init", "--initial-branch=main", repo)
|
||||
gitMust(t, repo, "config", "user.email", "test@test")
|
||||
gitMust(t, repo, "config", "user.name", "test")
|
||||
require.NoError(t, os.WriteFile(filepath.Join(repo, "action.yml"),
|
||||
[]byte("name: hello\nruns:\n using: node24\n main: index.js\n"), 0o644))
|
||||
gitMust(t, repo, "add", ".")
|
||||
gitMust(t, repo, "commit", "-m", "initial")
|
||||
gitMust(t, "", "init", "--template=", "--initial-branch=main", repo)
|
||||
gitMust(t, repo, "commit", "--allow-empty", "-m", "initial")
|
||||
output, err := exec.Command("git", "-C", repo, "rev-parse", "--short=7", "HEAD").Output()
|
||||
require.NoError(t, err)
|
||||
|
||||
@@ -393,7 +387,7 @@ func TestStepActionRemotePreResolvesDownloadedCommit(t *testing.T) {
|
||||
[]byte("name: setup-go\nruns:\n using: node20\n main: index.js\n"), 0o600))
|
||||
|
||||
for _, args := range [][]string{
|
||||
{"init", "--initial-branch=main", actionDir},
|
||||
{"init", "--template=", "--initial-branch=main", actionDir},
|
||||
{"-C", actionDir, "add", "action.yml"},
|
||||
{"-C", actionDir, "-c", "user.name=runner", "-c", "[email protected]", "-c", "commit.gpgsign=false", "commit", "-m", "action"},
|
||||
} {
|
||||
|
||||
@@ -4,6 +4,8 @@ Built-in actions ship with the runner, so a job needs no action download and no
|
||||
|
||||
```yaml
|
||||
- uses: builtin:checkout
|
||||
- uses: builtin:upload-artifact
|
||||
- uses: builtin:download-artifact
|
||||
```
|
||||
|
||||
## checkout
|
||||
@@ -37,3 +39,47 @@ A branch is checked out as a local branch tracking `origin`, a tag or commit as
|
||||
Persisted credentials end with the job. `token` is only sent to `github.server_url`, while `ssh-key` is offered to any SSH host, as with `actions/checkout`. The runner's `client_cert_file` is only offered to `github.server_url` and only during the step, so later steps do not get it. Its `ca_file` replaces the job image's CA bundle for `github.server_url`, in later steps too when credentials persist. Without `ssh-key`, submodule URLs of the form `git@<server host>:` are fetched from `github.server_url` with `token`. With `ssh-key`, the repository is fetched from the SSH URL in the event's `repository.ssh_url`, or from the server's host on port 22.
|
||||
|
||||
With `gitea-runner exec`, checking out the workflow's own repository copies your local working directory, unless `--no-skip-checkout` or `ssh-key` is set.
|
||||
|
||||
## upload-artifact
|
||||
|
||||
Works like `actions/upload-artifact` v7 with these inputs. Any other input logs a warning, as on GitHub. It needs Gitea 1.22.4 or later, and 1.26 for `archive: false`.
|
||||
|
||||
| Input | Default |
|
||||
| --- | --- |
|
||||
| `name` | `artifact`, ignored with `archive: false`, which names the artifact after the file |
|
||||
| `path` | required, files, directories or patterns, one per line, `!` excludes |
|
||||
| `if-no-files-found` | `warn`, also `error` or `ignore` |
|
||||
| `retention-days` | the server's default, at most the repository's retention limit |
|
||||
| `compression-level` | `6`, from `0` for none to `9` |
|
||||
| `overwrite` | `false`, `true` deletes an artifact of the same name first |
|
||||
| `include-hidden-files` | `false`, `true` uploads files and directories whose name starts with `.` |
|
||||
| `archive` | `true`, `false` uploads a single file as is |
|
||||
|
||||
Patterns, symlinks and the artifact's root directory follow `actions/upload-artifact`. `ACTIONS_ARTIFACT_UPLOAD_CONCURRENCY` sets upload concurrency (default `5`), and `ACTIONS_ARTIFACT_UPLOAD_TIMEOUT_MS` sets the upload stall timeout (default `300000`). The step outputs `artifact-id`, `artifact-digest` and `artifact-url`. On Gitea 1.26 and later, the `artifact-url` is the artifact's link on the run page for this attempt, where `actions/upload-artifact` gives a link only GitHub serves.
|
||||
|
||||
Container jobs list the search paths in the job container and transfer only the selected files, gzipped when the image has gzip, requiring `sh`, `find` and `tar` in the image. On Kubernetes these files still pass through the API server, unlike `actions/upload-artifact`, which uploads from the pod.
|
||||
|
||||
## download-artifact
|
||||
|
||||
Works like `actions/download-artifact` v8 with these inputs. Any other input logs a warning, as on GitHub.
|
||||
|
||||
| Input | Default |
|
||||
| --- | --- |
|
||||
| `name` | empty, every artifact of the run |
|
||||
| `artifact-ids` | empty, comma-separated IDs instead of `name` |
|
||||
| `path` | workspace root, `~` is the job user's home |
|
||||
| `pattern` | empty, a glob the artifact names must match, without the extglobs and brace ranges of minimatch, ignored with `name` or `artifact-ids` |
|
||||
| `merge-multiple` | `false`, several artifacts go to `path/<name>`, `true` puts them all in `path` |
|
||||
| `github-token` | empty, a token to download from another run or repository |
|
||||
| `repository` | `github.repository`, with `github-token` |
|
||||
| `run-id` | `github.run_id`, with `github-token` |
|
||||
| `skip-decompress` | `false`, `true` keeps a zipped artifact as `<name>.zip` |
|
||||
| `digest-mismatch` | `error`, also `ignore`, `info` or `warn` |
|
||||
|
||||
ZIP entries become regular files with current modification times, so files lose their executable bit and symlinks become files containing their targets. `digest-mismatch` only applies when the server reports a digest, which Gitea does not. The step outputs `download-path`.
|
||||
|
||||
The runner downloads each artifact to a temporary file on the runner because the zip is read out of order, then streams the files into the job container. On Kubernetes, the stream goes through the API server, gzipped when the image has gzip, or over the pod network with `kubernetes.transfer.mode` set to `auto` or `direct`, while `actions/download-artifact` downloads from the pod directly.
|
||||
|
||||
`github-token` needs Gitea 1.24 or later and is only sent to the instance the runner is registered with. Downloads from another run paginate through its artifacts and skip expired ones, where `actions/download-artifact` sees only Gitea's first 30 and fails on an expired newest copy. `run-id` must be positive, since Gitea treats nonpositive IDs as a request for artifacts from every run. Artifact names and ZIP entries that would leave the destination are rejected.
|
||||
|
||||
With `gitea-runner exec`, both actions use the local artifact server of `--artifact-server-path`, which serves no other runs for `github-token`.
|
||||
+36
-47
@@ -90,20 +90,29 @@ container:
|
||||
type cliDaemon struct {
|
||||
cmd *exec.Cmd
|
||||
log *syncBuffer
|
||||
exited chan error
|
||||
exited chan struct{}
|
||||
err error
|
||||
}
|
||||
|
||||
func (c *cliRunner) startDaemon(t *testing.T, args ...string) *cliDaemon {
|
||||
t.Helper()
|
||||
d := &cliDaemon{log: &syncBuffer{}, exited: make(chan error, 1)}
|
||||
d := &cliDaemon{log: &syncBuffer{}, exited: make(chan struct{})}
|
||||
d.cmd = exec.Command(c.bin, append([]string{"daemon", "-c", c.cfgFile}, args...)...)
|
||||
d.cmd.Stdout, d.cmd.Stderr = d.log, d.log
|
||||
if err := d.cmd.Start(); err != nil {
|
||||
t.Fatalf("start daemon: %v", err)
|
||||
}
|
||||
go func() { d.exited <- d.cmd.Wait() }()
|
||||
go func() {
|
||||
d.err = d.cmd.Wait()
|
||||
close(d.exited)
|
||||
}()
|
||||
t.Cleanup(func() {
|
||||
_ = d.cmd.Process.Kill() // no-op once the daemon has exited
|
||||
_ = d.cmd.Process.Signal(syscall.SIGTERM)
|
||||
select {
|
||||
case <-d.exited:
|
||||
case <-time.After(30 * time.Second):
|
||||
_ = d.cmd.Process.Kill()
|
||||
}
|
||||
})
|
||||
return d
|
||||
}
|
||||
@@ -112,8 +121,8 @@ func (c *cliRunner) startDaemon(t *testing.T, args ...string) *cliDaemon {
|
||||
func (d *cliDaemon) wait(t *testing.T, timeout time.Duration) error {
|
||||
t.Helper()
|
||||
select {
|
||||
case err := <-d.exited:
|
||||
return err
|
||||
case <-d.exited:
|
||||
return d.err
|
||||
case <-time.After(timeout):
|
||||
t.Fatalf("daemon still running after %s\n%s", timeout, d.log)
|
||||
return nil
|
||||
@@ -140,8 +149,8 @@ func (d *cliDaemon) requireSuccess(t *testing.T, c *cliRunner, runID int64) {
|
||||
// testCLIRegisterDaemon drives the built binary the way a user does: register, then daemon.
|
||||
func testCLIRegisterDaemon(t *testing.T) {
|
||||
t.Parallel()
|
||||
c := newCLIRunner(t, "e2e-cli", "")
|
||||
d := c.startDaemon(t)
|
||||
c := newCLIRunner(t, "e2e-registered", "")
|
||||
d := c.startDaemon(t, "--labels", "e2e-cli:docker://"+os.Getenv("E2E_JOB_IMAGE"))
|
||||
|
||||
pushWorkflow(t, c.api, c.repo, "cli.yml")
|
||||
wfRun := waitForRun(t, c.api, c.repo)
|
||||
@@ -150,6 +159,22 @@ func testCLIRegisterDaemon(t *testing.T) {
|
||||
t.Errorf("job logs are missing the marker:\n%s", logs)
|
||||
}
|
||||
|
||||
runners, err := c.api.Runners(t.Context(), c.repo)
|
||||
if err != nil {
|
||||
t.Fatalf("list runners: %v", err)
|
||||
}
|
||||
var labels []string
|
||||
for _, runner := range runners {
|
||||
if runner.Name == c.repo {
|
||||
for _, label := range runner.Labels {
|
||||
labels = append(labels, label.Name)
|
||||
}
|
||||
}
|
||||
}
|
||||
if !slices.Equal(labels, []string{"e2e-cli"}) {
|
||||
t.Errorf("runner %q has labels %v on the server, want only the --labels override", c.repo, labels)
|
||||
}
|
||||
|
||||
d.terminate(t)
|
||||
if err := d.wait(t, 30*time.Second); err != nil {
|
||||
t.Errorf("daemon did not exit cleanly on SIGTERM: %v\n%s", err, d.log)
|
||||
@@ -186,36 +211,6 @@ func testCLIOnce(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// testCLILabelsOverride checks that daemon --labels replaces the labels given at registration.
|
||||
func testCLILabelsOverride(t *testing.T) {
|
||||
t.Parallel()
|
||||
c := newCLIRunner(t, "e2e-registered", "")
|
||||
d := c.startDaemon(t, "--labels", "e2e-override:docker://"+os.Getenv("E2E_JOB_IMAGE"))
|
||||
|
||||
pushWorkflow(t, c.api, c.repo, "cli_labels.yml")
|
||||
wfRun := waitForRun(t, c.api, c.repo)
|
||||
d.requireSuccess(t, c, wfRun.ID)
|
||||
|
||||
runners, err := c.api.Runners(t.Context(), c.repo)
|
||||
if err != nil {
|
||||
t.Fatalf("list runners: %v", err)
|
||||
}
|
||||
for _, runner := range runners {
|
||||
if runner.Name != c.repo {
|
||||
continue
|
||||
}
|
||||
var labels []string
|
||||
for _, label := range runner.Labels {
|
||||
labels = append(labels, label.Name)
|
||||
}
|
||||
if !slices.Equal(labels, []string{"e2e-override"}) {
|
||||
t.Errorf("runner has labels %v on the server, want only the --labels override", labels)
|
||||
}
|
||||
return
|
||||
}
|
||||
t.Errorf("runner %q not found among %+v", c.repo, runners)
|
||||
}
|
||||
|
||||
// testCLIShutdownDrains checks that SIGTERM lets a running job finish within shutdown_timeout.
|
||||
func testCLIShutdownDrains(t *testing.T) {
|
||||
t.Parallel()
|
||||
@@ -282,10 +277,6 @@ func testExecArtifacts(t *testing.T) {
|
||||
t.Parallel()
|
||||
ctx := t.Context()
|
||||
|
||||
bin, err := runnerBinary()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
workflow, err := os.ReadFile("testdata/workflows/exec_artifact.yml")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -309,11 +300,9 @@ func testExecArtifacts(t *testing.T) {
|
||||
}
|
||||
|
||||
artifacts := t.TempDir()
|
||||
args := []string{"exec", "-C", workdir, "-W", workflows, "-i", os.Getenv("E2E_JOB_IMAGE"), "--artifact-server-path", artifacts, "--network", fixture.network}
|
||||
cmd := exec.CommandContext(ctx, bin, args...)
|
||||
cmd.Env = slices.DeleteFunc(os.Environ(), func(kv string) bool { return strings.HasPrefix(kv, "ACTIONS_") })
|
||||
if out, err := cmd.CombinedOutput(); err != nil {
|
||||
t.Fatalf("exec: %v\n%s", err, out)
|
||||
stdout, stderr, err := runExec(t, nil, "-C", workdir, "-W", workflows, "-i", os.Getenv("E2E_JOB_IMAGE"), "--artifact-server-path", artifacts, "--network", fixture.network)
|
||||
if err != nil {
|
||||
t.Fatalf("exec: %v\n%s%s", err, stdout, stderr)
|
||||
}
|
||||
if blobs, _ := filepath.Glob(filepath.Join(artifacts, "v4", "*", "*", "blob")); len(blobs) != 2 {
|
||||
t.Fatalf("want one v4 artifact blob per workflow run, got %v", blobs)
|
||||
|
||||
@@ -37,7 +37,6 @@ func TestCompatibility(t *testing.T) {
|
||||
t.Run("job_timeout", testJobTimeout)
|
||||
t.Run("cli_register_daemon", testCLIRegisterDaemon)
|
||||
t.Run("cli_once", testCLIOnce)
|
||||
t.Run("cli_labels_override", testCLILabelsOverride)
|
||||
t.Run("cli_shutdown_drains", testCLIShutdownDrains)
|
||||
t.Run("cli_shutdown_cancels", testCLIShutdownCancels)
|
||||
t.Run("host_mode", testHostMode)
|
||||
|
||||
+14
-12
@@ -80,11 +80,11 @@ func selfNetwork(ctx context.Context, cli mobyclient.APIClient) (string, string,
|
||||
return "", "", false
|
||||
}
|
||||
|
||||
func StartGitea(ctx context.Context, cli mobyclient.APIClient) (*GiteaFixture, error) {
|
||||
closeClient := true
|
||||
func StartGitea(ctx context.Context, cli mobyclient.APIClient) (_ *GiteaFixture, err error) {
|
||||
f := &GiteaFixture{cli: cli}
|
||||
defer func() {
|
||||
if closeClient {
|
||||
_ = cli.Close()
|
||||
if err != nil {
|
||||
_ = f.Close()
|
||||
}
|
||||
}()
|
||||
|
||||
@@ -123,26 +123,21 @@ func StartGitea(ctx context.Context, cli mobyclient.APIClient) (*GiteaFixture, e
|
||||
return nil, fmt.Errorf("create gitea container: %w", err)
|
||||
}
|
||||
|
||||
f := &GiteaFixture{cli: cli, id: resp.ID, image: image, baseURL: baseURL, network: sharedNet, selfIP: selfIP}
|
||||
closeClient = false
|
||||
f.id, f.image, f.baseURL, f.network, f.selfIP = resp.ID, image, baseURL, sharedNet, selfIP
|
||||
|
||||
if _, err := cli.ContainerStart(ctx, f.id, mobyclient.ContainerStartOptions{}); err != nil {
|
||||
_ = f.Close(ctx)
|
||||
return nil, fmt.Errorf("start gitea container: %w", err)
|
||||
}
|
||||
|
||||
if err := f.waitHealthy(ctx); err != nil {
|
||||
_ = f.Close(ctx)
|
||||
return nil, fmt.Errorf("gitea did not become healthy: %w", err)
|
||||
}
|
||||
|
||||
if err := f.bootstrapAdmin(ctx); err != nil {
|
||||
_ = f.Close(ctx)
|
||||
return nil, fmt.Errorf("bootstrap gitea admin: %w", err)
|
||||
}
|
||||
|
||||
if err := f.readVersion(ctx); err != nil {
|
||||
_ = f.Close(ctx)
|
||||
return nil, fmt.Errorf("read gitea version: %w", err)
|
||||
}
|
||||
|
||||
@@ -249,6 +244,7 @@ func containerExec(ctx context.Context, cli mobyclient.APIClient, id, user strin
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer context.AfterFunc(ctx, attached.Close)()
|
||||
defer attached.Close()
|
||||
if stdin != nil {
|
||||
go func() {
|
||||
@@ -258,7 +254,11 @@ func containerExec(ctx context.Context, cli mobyclient.APIClient, id, user strin
|
||||
}
|
||||
|
||||
var out bytes.Buffer
|
||||
if _, err := stdcopy.StdCopy(&out, &out, attached.Reader); err != nil {
|
||||
_, err = stdcopy.StdCopy(&out, &out, attached.Reader)
|
||||
if ctx.Err() != nil {
|
||||
return "", ctx.Err()
|
||||
}
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
@@ -291,10 +291,12 @@ func (f *GiteaFixture) doJSON(ctx context.Context, method, path string, reqBody,
|
||||
return api.doJSON(ctx, method, path, reqBody, respBody)
|
||||
}
|
||||
|
||||
func (f *GiteaFixture) Close(ctx context.Context) error {
|
||||
func (f *GiteaFixture) Close() error {
|
||||
if f.id == "" {
|
||||
return f.cli.Close()
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), cleanupTimeout)
|
||||
defer cancel()
|
||||
_, removeErr := f.cli.ContainerRemove(ctx, f.id, mobyclient.ContainerRemoveOptions{Force: true, RemoveVolumes: true})
|
||||
return errors.Join(removeErr, f.cli.Close())
|
||||
}
|
||||
+13
-7
@@ -55,7 +55,9 @@ func checkClusterLeaks() error {
|
||||
if err != nil {
|
||||
return nil // the tests that needed the cluster already failed
|
||||
}
|
||||
if out, err := containerExec(context.Background(), dockerCLI, cluster.id, "", nil,
|
||||
ctx, cancel := context.WithTimeout(context.Background(), cleanupTimeout)
|
||||
defer cancel()
|
||||
if out, err := containerExec(ctx, dockerCLI, cluster.id, "", nil,
|
||||
"kubectl", "wait", "--for=delete", "pods,secrets", "--selector=app.kubernetes.io/managed-by=gitea-runner", "--timeout=1m",
|
||||
); err != nil {
|
||||
return fmt.Errorf("job pods or secrets left behind: %w: %s", err, out)
|
||||
@@ -72,7 +74,7 @@ type kubeCluster struct {
|
||||
var (
|
||||
k3s = sync.OnceValues(startK3s) // independent of Gitea, so it can boot alongside it
|
||||
clusterStarted atomic.Bool
|
||||
clusterCleanup []func()
|
||||
clusterCleanup []func(context.Context)
|
||||
sharedCluster = sync.OnceValues(func() (*kubeCluster, error) {
|
||||
cluster, err := k3s()
|
||||
if err != nil {
|
||||
@@ -109,7 +111,9 @@ func stopCluster() {
|
||||
}
|
||||
_, _ = k3s()
|
||||
for _, cleanup := range slices.Backward(clusterCleanup) {
|
||||
cleanup()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), cleanupTimeout)
|
||||
cleanup(ctx)
|
||||
cancel()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -138,8 +142,8 @@ func startK3s() (*kubeCluster, error) {
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create k3s container: %w", err)
|
||||
}
|
||||
clusterCleanup = append(clusterCleanup, func() {
|
||||
_, _ = cli.ContainerRemove(context.Background(), created.ID, mobyclient.ContainerRemoveOptions{Force: true, RemoveVolumes: true})
|
||||
clusterCleanup = append(clusterCleanup, func(ctx context.Context) {
|
||||
_, _ = cli.ContainerRemove(ctx, created.ID, mobyclient.ContainerRemoveOptions{Force: true, RemoveVolumes: true})
|
||||
})
|
||||
if _, err := cli.ContainerStart(ctx, created.ID, mobyclient.ContainerStartOptions{}); err != nil {
|
||||
return nil, fmt.Errorf("start k3s container: %w", err)
|
||||
@@ -170,7 +174,9 @@ func startK3s() (*kubeCluster, error) {
|
||||
inspected, inspectErr := cli.ContainerInspect(ctx, created.ID, mobyclient.ContainerInspectOptions{})
|
||||
if inspectErr != nil || !inspected.Container.State.Running || ctx.Err() != nil {
|
||||
var logs bytes.Buffer
|
||||
if reader, logsErr := cli.ContainerLogs(context.Background(), created.ID, mobyclient.ContainerLogsOptions{ShowStdout: true, ShowStderr: true, Tail: "30"}); logsErr == nil {
|
||||
logsCtx, logsCancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer logsCancel()
|
||||
if reader, logsErr := cli.ContainerLogs(logsCtx, created.ID, mobyclient.ContainerLogsOptions{ShowStdout: true, ShowStderr: true, Tail: "30"}); logsErr == nil {
|
||||
_, _ = stdcopy.StdCopy(&logs, &logs, reader)
|
||||
_ = reader.Close()
|
||||
}
|
||||
@@ -226,7 +232,7 @@ func preloadImages(ctx context.Context, k3sID string, images ...string) ([]strin
|
||||
if _, err := dockerCLI.ImageTag(ctx, mobyclient.ImageTagOptions{Source: image, Target: alias}); err != nil {
|
||||
return nil, fmt.Errorf("tag %s: %w", image, err)
|
||||
}
|
||||
clusterCleanup = append(clusterCleanup, func() { _, _ = dockerCLI.ImageRemove(context.Background(), alias, mobyclient.ImageRemoveOptions{}) })
|
||||
clusterCleanup = append(clusterCleanup, func(ctx context.Context) { _, _ = dockerCLI.ImageRemove(ctx, alias, mobyclient.ImageRemoveOptions{}) })
|
||||
aliases = append(aliases, alias)
|
||||
}
|
||||
archive, err := dockerCLI.ImageSave(ctx, aliases)
|
||||
|
||||
+6
-2
@@ -31,6 +31,8 @@ import (
|
||||
|
||||
const runTimeout = 3 * time.Minute
|
||||
|
||||
const cleanupTimeout = 2 * time.Minute
|
||||
|
||||
var nonRepoChars = regexp.MustCompile(`[^a-zA-Z0-9._-]+`)
|
||||
|
||||
var fixture *GiteaFixture
|
||||
@@ -42,8 +44,10 @@ var binDir string
|
||||
var dockerCLI mobyclient.APIClient
|
||||
|
||||
var runnerBinary = sync.OnceValues(func() (string, error) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), runTimeout)
|
||||
defer cancel()
|
||||
bin := filepath.Join(binDir, "gitea-runner")
|
||||
if out, err := exec.Command(cmp.Or(os.Getenv("GO"), "go"), "build", "-o", bin, "..").CombinedOutput(); err != nil {
|
||||
if out, err := exec.CommandContext(ctx, cmp.Or(os.Getenv("GO"), "go"), "build", "-o", bin, "..").CombinedOutput(); err != nil {
|
||||
return "", fmt.Errorf("build runner: %w\n%s", err, out)
|
||||
}
|
||||
return bin, nil
|
||||
@@ -80,7 +84,7 @@ func runSuite(m *testing.M) int {
|
||||
return 1
|
||||
}
|
||||
fixture = f
|
||||
defer func() { _ = fixture.Close(context.Background()) }()
|
||||
defer func() { _ = fixture.Close() }()
|
||||
|
||||
fmt.Fprintf(os.Stderr, "gitea fixture: image=%s version=%s\n", fixture.image, fixture.version)
|
||||
|
||||
|
||||
+29
-15
@@ -9,6 +9,7 @@ import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"cmp"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"os"
|
||||
@@ -138,14 +139,31 @@ func daemonWorkflow(t *testing.T, m mode, spec workflowSpec) *workflowResult {
|
||||
return result
|
||||
}
|
||||
|
||||
func execWorkflow(t *testing.T, m mode, spec workflowSpec) *workflowResult {
|
||||
func runExec(t *testing.T, env []string, args ...string) (stdout, stderr []byte, err error) {
|
||||
t.Helper()
|
||||
ctx := t.Context()
|
||||
|
||||
bin, err := runnerBinary()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(t.Context(), runTimeout)
|
||||
defer cancel()
|
||||
cmd := exec.CommandContext(ctx, bin, append([]string{"exec"}, args...)...)
|
||||
cmd.Env = append(slices.DeleteFunc(os.Environ(), func(kv string) bool { return strings.HasPrefix(kv, "ACTIONS_") }), env...)
|
||||
cmd.Cancel = func() error { return cmd.Process.Signal(os.Interrupt) }
|
||||
cmd.WaitDelay = cleanupTimeout
|
||||
var stdoutBuf, stderrBuf bytes.Buffer
|
||||
cmd.Stdout, cmd.Stderr = &stdoutBuf, &stderrBuf
|
||||
err = cmd.Run()
|
||||
if ctx.Err() != nil {
|
||||
t.Fatalf("exec did not finish within %s\n%s%s", runTimeout, stdoutBuf.Bytes(), stderrBuf.Bytes())
|
||||
}
|
||||
return stdoutBuf.Bytes(), stderrBuf.Bytes(), err
|
||||
}
|
||||
|
||||
func execWorkflow(t *testing.T, m mode, spec workflowSpec) *workflowResult {
|
||||
t.Helper()
|
||||
ctx := t.Context()
|
||||
|
||||
workdir := t.TempDir()
|
||||
workflows := filepath.Join(workdir, ".gitea", "workflows")
|
||||
if err := os.MkdirAll(workflows, 0o755); err != nil {
|
||||
@@ -170,12 +188,12 @@ func execWorkflow(t *testing.T, m mode, spec workflowSpec) *workflowResult {
|
||||
}
|
||||
}
|
||||
|
||||
args := []string{"exec", "--json", "-C", workdir, "-W", workflows, "--artifact-server-path", t.TempDir(), "--artifact-server-port", "0"}
|
||||
env := slices.DeleteFunc(os.Environ(), func(kv string) bool { return strings.HasPrefix(kv, "ACTIONS_") })
|
||||
args := []string{"--json", "-C", workdir, "-W", workflows, "--artifact-server-path", t.TempDir(), "--artifact-server-port", "0"}
|
||||
var env []string
|
||||
if m.kubernetes {
|
||||
cluster := requireCluster(t)
|
||||
args = append(args, "-i", "kubernetes://"+cluster.jobImage)
|
||||
env = append(env, "KUBECONFIG="+cluster.kubernetes.Kubeconfig)
|
||||
env = []string{"KUBECONFIG=" + cluster.kubernetes.Kubeconfig}
|
||||
} else {
|
||||
args = append(args, "-i", os.Getenv("E2E_JOB_IMAGE"), "--network", fixture.network)
|
||||
}
|
||||
@@ -185,21 +203,17 @@ func execWorkflow(t *testing.T, m mode, spec workflowSpec) *workflowResult {
|
||||
args = append(args, "--input", name+"="+value)
|
||||
}
|
||||
}
|
||||
cmd := exec.CommandContext(ctx, bin, append(args, spec.execArgs...)...)
|
||||
cmd.Env = env
|
||||
var stdout, stderr bytes.Buffer
|
||||
cmd.Stdout, cmd.Stderr = &stdout, &stderr
|
||||
runErr := cmd.Run()
|
||||
stdout, stderr, runErr := runExec(t, env, append(args, spec.execArgs...)...)
|
||||
var exitErr *exec.ExitError
|
||||
if runErr != nil && !errors.As(runErr, &exitErr) {
|
||||
t.Fatalf("exec: %v\n%s", runErr, stderr.String())
|
||||
t.Fatalf("exec: %v\n%s", runErr, stderr)
|
||||
}
|
||||
|
||||
result := &workflowResult{conclusion: "success", jobs: map[string]string{}}
|
||||
if runErr != nil {
|
||||
result.conclusion = "failure"
|
||||
}
|
||||
scanner := bufio.NewScanner(&stdout)
|
||||
scanner := bufio.NewScanner(bytes.NewReader(stdout))
|
||||
scanner.Buffer(nil, 1<<20)
|
||||
for scanner.Scan() {
|
||||
var entry struct {
|
||||
@@ -220,8 +234,8 @@ func execWorkflow(t *testing.T, m mode, spec workflowSpec) *workflowResult {
|
||||
}
|
||||
}
|
||||
}
|
||||
if result.conclusion != "success" && len(result.jobs) == 0 {
|
||||
t.Fatalf("exec ran no job: %v\n%s", runErr, stderr.String())
|
||||
if len(result.jobs) == 0 {
|
||||
t.Fatalf("exec ran no job: %v\n%s", runErr, stderr)
|
||||
}
|
||||
return result
|
||||
}
|
||||
Vendored
+49
@@ -26,3 +26,52 @@ jobs:
|
||||
name: e2e-artifact-${{ github.run_id }}
|
||||
path: got
|
||||
- run: grep -q artifact-payload got/data.txt
|
||||
builtin-upload:
|
||||
runs-on: e2e-artifact
|
||||
steps:
|
||||
- run: |
|
||||
mkdir -p out/sub
|
||||
echo "builtin-payload" > out/data.txt
|
||||
echo "nested" > out/sub/nested.txt
|
||||
echo "hidden" > out/.hidden
|
||||
echo "raw" > report.txt
|
||||
- id: zipped
|
||||
uses: builtin:upload-artifact
|
||||
with:
|
||||
name: e2e-builtin-${{ github.run_id }}
|
||||
path: out
|
||||
- run: test -n "${{ steps.zipped.outputs.artifact-id }}" && test -n "${{ steps.zipped.outputs.artifact-digest }}"
|
||||
- uses: builtin:upload-artifact
|
||||
with:
|
||||
path: report.txt
|
||||
archive: false
|
||||
js-downloads-builtin:
|
||||
needs: builtin-upload
|
||||
runs-on: e2e-artifact
|
||||
steps:
|
||||
- uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: e2e-builtin-${{ github.run_id }}
|
||||
path: got
|
||||
- run: grep -q builtin-payload got/data.txt && grep -q nested got/sub/nested.txt && test ! -e got/.hidden
|
||||
- uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: report.txt
|
||||
- run: grep -q raw report.txt
|
||||
builtin-downloads-js:
|
||||
needs: upload
|
||||
runs-on: e2e-artifact
|
||||
steps:
|
||||
- id: own
|
||||
uses: builtin:download-artifact
|
||||
with:
|
||||
name: e2e-artifact-${{ github.run_id }}
|
||||
path: got
|
||||
- run: grep -q artifact-payload got/data.txt && test "${{ steps.own.outputs.download-path }}" = "$GITHUB_WORKSPACE/got"
|
||||
- uses: builtin:download-artifact
|
||||
with:
|
||||
name: e2e-artifact-${{ github.run_id }}
|
||||
path: rest
|
||||
github-token: ${{ github.token }}
|
||||
run-id: ${{ github.run_id }}
|
||||
- run: grep -q artifact-payload rest/data.txt
|
||||
-7
@@ -1,7 +0,0 @@
|
||||
name: cli-labels
|
||||
on: push
|
||||
jobs:
|
||||
hello:
|
||||
runs-on: e2e-override
|
||||
steps:
|
||||
- run: echo e2e-override-ran
|
||||
@@ -12,3 +12,15 @@ docker run --privileged \
|
||||
```
|
||||
|
||||
The volumes keep the runner's registration and the job image cache. See [image flavours](../../README.md#image-flavours) for alternatives.
|
||||
|
||||
#### Pull job images from a registry with a private CA
|
||||
|
||||
The bundled Docker daemon pulls job images, so it needs to trust the registry's CA itself. `runner.ca_file` doesn't cover this. Mount the CA where the daemon looks for it, naming the directory after the registry's host and port:
|
||||
|
||||
```sh
|
||||
--mount "type=bind,src=$PWD/ca.crt,dst=/etc/docker/certs.d/registry.example.com:5000/ca.crt,readonly"
|
||||
```
|
||||
|
||||
Use `--mount`, since `-v` rejects a container path with a port in it.
|
||||
|
||||
On `latest-dind-rootless`, use `/home/rootless/.config/docker/certs.d/registry.example.com:5000/ca.crt` as the target.
|
||||
@@ -12,7 +12,7 @@ require (
|
||||
github.com/containerd/errdefs v1.0.0
|
||||
github.com/creack/pty v1.1.24
|
||||
github.com/distribution/reference v0.6.0
|
||||
github.com/docker/cli v29.9.2+incompatible
|
||||
github.com/docker/cli v29.9.3+incompatible
|
||||
github.com/docker/go-connections v0.8.2
|
||||
github.com/docker/go-units v0.5.0
|
||||
github.com/joho/godotenv v1.5.1
|
||||
|
||||
@@ -21,8 +21,8 @@ github.com/creack/pty v1.1.24 h1:bJrF4RRfyJnbTJqzRLHzcGaZK1NeM5kTC9jGgovnR1s=
|
||||
github.com/creack/pty v1.1.24/go.mod h1:08sCNb52WyoAwi2QDyzUCTgcvVFhUzewun7wtTfvcwE=
|
||||
github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk=
|
||||
github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
|
||||
github.com/docker/cli v29.9.2+incompatible h1:trup/vUxx5uUo2t+87Cd9AmITR+DvKUe4H2tTtZN8FY=
|
||||
github.com/docker/cli v29.9.2+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
|
||||
github.com/docker/cli v29.9.3+incompatible h1:SBaMp7ecohadEHMDrUJg22t9I3awqsAVhnJ7GOcWpfE=
|
||||
github.com/docker/cli v29.9.3+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
|
||||
github.com/docker/go-connections v0.8.2 h1:9d9DEokx5W7qlr0shkFXIxEA7+TTD+llwzcykSlxMNc=
|
||||
github.com/docker/go-connections v0.8.2/go.mod h1:no1qkHdjq7kLMGUXYAduOhYPSJxxvgWBh7ogVvptn3Q=
|
||||
github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
|
||||
|
||||
@@ -128,6 +128,11 @@ func (*daemonTestService) UpdateLog(_ context.Context, req *runnerv1.UpdateLogRe
|
||||
func TestDaemonRetriesDeclareOutagesAndRemovesOnlyConsumedEphemeralRegistration(t *testing.T) {
|
||||
defer func(delay time.Duration) { declareRetryDelay = delay }(declareRetryDelay)
|
||||
declareRetryDelay = 0
|
||||
dir := t.TempDir()
|
||||
regFile := filepath.Join(dir, "registration.json")
|
||||
configFile := filepath.Join(dir, "config.yaml")
|
||||
require.NoError(t, os.WriteFile(configFile, []byte("runner:\n file: "+regFile+"\n idle_cleanup_interval: 0s\ncache:\n enabled: false\n"), 0o600))
|
||||
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
ephemeral, beforeTask bool
|
||||
@@ -154,10 +159,6 @@ func TestDaemonRetriesDeclareOutagesAndRemovesOnlyConsumedEphemeralRegistration(
|
||||
connecthttp.Mount(mux, rpcServer, connecthttp.WithReadMaxBytes(0))
|
||||
server := httptest.NewServer(http.StripPrefix("/api/actions", mux))
|
||||
defer server.Close()
|
||||
dir := t.TempDir()
|
||||
regFile := filepath.Join(dir, "registration.json")
|
||||
configFile := filepath.Join(dir, "config.yaml")
|
||||
require.NoError(t, os.WriteFile(configFile, []byte("runner:\n file: "+regFile+"\n idle_cleanup_interval: 0s\ncache:\n enabled: false\n"), 0o600))
|
||||
require.NoError(t, config.SaveRegistration(regFile, &config.Registration{Address: server.URL, UUID: "test", Name: "test", Token: "test", Labels: []string{"host:host"}, Ephemeral: tc.ephemeral}))
|
||||
err := runDaemon(ctx, &daemonArgs{Once: true}, &configFile)(nil, nil)
|
||||
if tc.declareErr == nil && tc.fetchErr == nil || tc.beforeTask {
|
||||
|
||||
@@ -188,10 +188,14 @@ func (i *executeArgs) eventJSON() (string, error) {
|
||||
func (i *executeArgs) LoadEnvs() map[string]string {
|
||||
envs := parseKVAndFile(i.envs, i.Envfile())
|
||||
|
||||
envs["ACTIONS_CACHE_URL"] = i.cacheHandler.ExternalURL() + "/"
|
||||
cacheURL := i.cacheHandler.ExternalURL()
|
||||
if cacheURL == "" {
|
||||
return envs
|
||||
}
|
||||
envs["ACTIONS_CACHE_URL"] = cacheURL + "/"
|
||||
// The same server answers cache v2, which docker buildx reads from the results origin alone.
|
||||
if envs["ACTIONS_RESULTS_URL"] == "" {
|
||||
envs["ACTIONS_RESULTS_URL"] = cmp.Or(os.Getenv("ACTIONS_RESULTS_URL"), i.cacheHandler.ExternalURL())
|
||||
envs["ACTIONS_RESULTS_URL"] = cmp.Or(os.Getenv("ACTIONS_RESULTS_URL"), cacheURL)
|
||||
}
|
||||
envs[runner.CacheServiceV2Env] = "true"
|
||||
|
||||
@@ -209,6 +213,10 @@ func parseKVAndFile(rawKVs []string, filePath string) map[string]string {
|
||||
return result
|
||||
}
|
||||
|
||||
func (i *executeArgs) artifactServerURL() string {
|
||||
return "http://" + net.JoinHostPort(i.artifactServerAddr, i.artifactServerPort)
|
||||
}
|
||||
|
||||
// Workdir returns path to workdir
|
||||
func (i *executeArgs) Workdir() string {
|
||||
return i.resolve(".")
|
||||
@@ -382,6 +390,7 @@ func (i *executeArgs) runnerConfig(eventName string, env, proxyEnv map[string]st
|
||||
ArtifactServerPath: i.artifactServerPath,
|
||||
ArtifactServerPort: i.artifactServerPort,
|
||||
ArtifactServerAddr: i.artifactServerAddr,
|
||||
ResultsURL: i.artifactServerURL(),
|
||||
NoSkipCheckout: i.noSkipCheckout,
|
||||
EventName: eventName,
|
||||
EventJSON: eventJSON,
|
||||
@@ -513,7 +522,7 @@ func runExec(ctx context.Context, execArgs *executeArgs) func(cmd *cobra.Command
|
||||
token := cmp.Or(inheritedToken, artifacts.LocalRuntimeToken(workflowRun, maxLifetime))
|
||||
config.RuntimeTokens[run.Workflow] = token
|
||||
config.ExtraMasks = append(config.ExtraMasks, token)
|
||||
handler.RegisterJob(token, artifactcache.JobCredential{Repo: "__local/__exec", Results: "http://" + net.JoinHostPort(execArgs.artifactServerAddr, execArgs.artifactServerPort)})
|
||||
handler.RegisterJob(token, artifactcache.JobCredential{Repo: "__local/__exec", Results: execArgs.artifactServerURL()})
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -75,40 +75,35 @@ func TestExecuteArgsLoadInputs(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestExecuteArgsEventJSON(t *testing.T) {
|
||||
payload := func(content string) string {
|
||||
path := filepath.Join(t.TempDir(), "event.json")
|
||||
require.NoError(t, os.WriteFile(path, []byte(content), 0o600))
|
||||
return path
|
||||
}
|
||||
|
||||
dir := t.TempDir()
|
||||
tests := []struct {
|
||||
name string
|
||||
eventpath string
|
||||
inputs []string
|
||||
expected string
|
||||
name string
|
||||
payload string
|
||||
inputs []string
|
||||
expected string
|
||||
}{
|
||||
{
|
||||
name: "keeps a payload without inputs to merge",
|
||||
eventpath: payload(`{"inputs": {"kept": "from_event"}}`),
|
||||
expected: `{"inputs": {"kept": "from_event"}}`,
|
||||
name: "keeps a payload without inputs to merge",
|
||||
payload: `{"inputs": {"kept": "from_event"}}`,
|
||||
expected: `{"inputs": {"kept": "from_event"}}`,
|
||||
},
|
||||
{
|
||||
name: "overrides the event inputs it names, keeping the rest",
|
||||
eventpath: payload(`{"inputs": {"kept": "from_event", "named": "from_event"}}`),
|
||||
inputs: []string{"named=from_cli"},
|
||||
expected: `{"inputs": {"kept": "from_event", "named": "from_cli"}}`,
|
||||
name: "overrides the event inputs it names, keeping the rest",
|
||||
payload: `{"inputs": {"kept": "from_event", "named": "from_event"}}`,
|
||||
inputs: []string{"named=from_cli"},
|
||||
expected: `{"inputs": {"kept": "from_event", "named": "from_cli"}}`,
|
||||
},
|
||||
{
|
||||
name: "adds inputs to a payload carrying none",
|
||||
eventpath: payload(`{"repository": {"name": "test-repo"}}`),
|
||||
inputs: []string{"flag=val"},
|
||||
expected: `{"repository": {"name": "test-repo"}, "inputs": {"flag": "val"}}`,
|
||||
name: "adds inputs to a payload carrying none",
|
||||
payload: `{"repository": {"name": "test-repo"}}`,
|
||||
inputs: []string{"flag=val"},
|
||||
expected: `{"repository": {"name": "test-repo"}, "inputs": {"flag": "val"}}`,
|
||||
},
|
||||
{
|
||||
name: "handles a null payload",
|
||||
eventpath: payload("null"),
|
||||
inputs: []string{"flag=val"},
|
||||
expected: `{"inputs": {"flag": "val"}}`,
|
||||
name: "handles a null payload",
|
||||
payload: "null",
|
||||
inputs: []string{"flag=val"},
|
||||
expected: `{"inputs": {"flag": "val"}}`,
|
||||
},
|
||||
{
|
||||
name: "builds a payload without an event file",
|
||||
@@ -119,7 +114,12 @@ func TestExecuteArgsEventJSON(t *testing.T) {
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
eventJSON, err := (&executeArgs{eventpath: tt.eventpath, inputs: tt.inputs}).eventJSON()
|
||||
var eventpath string
|
||||
if tt.payload != "" {
|
||||
eventpath = filepath.Join(dir, tt.name+".json")
|
||||
require.NoError(t, os.WriteFile(eventpath, []byte(tt.payload), 0o600))
|
||||
}
|
||||
eventJSON, err := (&executeArgs{eventpath: eventpath, inputs: tt.inputs}).eventJSON()
|
||||
require.NoError(t, err)
|
||||
require.JSONEq(t, tt.expected, eventJSON)
|
||||
})
|
||||
@@ -275,21 +275,13 @@ func workflowForList(name, file, event, jobID, jobName string) *model.Workflow {
|
||||
return &model.Workflow{
|
||||
Name: name,
|
||||
File: file,
|
||||
RawOn: rawOnNode(event),
|
||||
RawOn: yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: event},
|
||||
Jobs: map[string]*model.Job{
|
||||
jobID: {Name: jobName},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func rawOnNode(event string) yaml.Node {
|
||||
var node yaml.Node
|
||||
if err := yaml.Unmarshal([]byte(event), &node); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return *node.Content[0]
|
||||
}
|
||||
|
||||
func TestExecuteArgsLoadEnvsResultsOrigin(t *testing.T) {
|
||||
t.Setenv("ACTIONS_RESULTS_URL", "") // the default is only supplied when nothing else does
|
||||
handler, err := artifactcache.StartHandler(artifactcache.Options{Dir: t.TempDir(), OutboundIP: "127.0.0.1"})
|
||||
|
||||
@@ -89,6 +89,7 @@ const registerTokenEnvVar = "GITEA_RUNNER_REGISTRATION_TOKEN"
|
||||
|
||||
var defaultLabels = []string{
|
||||
"ubuntu-latest:docker://docker.gitea.com/runner-images:ubuntu-latest",
|
||||
"ubuntu-26.04:docker://docker.gitea.com/runner-images:ubuntu-26.04",
|
||||
"ubuntu-24.04:docker://docker.gitea.com/runner-images:ubuntu-24.04",
|
||||
"ubuntu-22.04:docker://docker.gitea.com/runner-images:ubuntu-22.04",
|
||||
}
|
||||
|
||||
@@ -27,16 +27,9 @@ func TestRunPostTaskScriptSkippedWhenEmpty(t *testing.T) {
|
||||
r := &Runner{
|
||||
cfg: &config.Config{},
|
||||
}
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
defer cancel()
|
||||
|
||||
taskCtx, err := structpb.NewStruct(map[string]any{})
|
||||
require.NoError(t, err)
|
||||
task := &runnerv1.Task{Id: 1, Context: taskCtx}
|
||||
reporter := report.NewReporter(ctx, cancel, nil, task, r.cfg)
|
||||
|
||||
require.NotPanics(t, func() {
|
||||
r.runPostTaskScript(ctx, reporter, task, "/workspace/owner/repo")
|
||||
r.runPostTaskScript(t.Context(), nil, nil, "/workspace/owner/repo")
|
||||
})
|
||||
}
|
||||
|
||||
@@ -59,9 +52,7 @@ func TestRunPostTaskScriptNonZeroExitDoesNotPanic(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
defer cancel()
|
||||
|
||||
taskCtx, err := structpb.NewStruct(map[string]any{})
|
||||
require.NoError(t, err)
|
||||
task := &runnerv1.Task{Id: 1, Context: taskCtx}
|
||||
task := &runnerv1.Task{Id: 1, Context: &structpb.Struct{}}
|
||||
reporter := report.NewReporter(ctx, cancel, nil, task, cfg)
|
||||
|
||||
require.NotPanics(t, func() {
|
||||
@@ -146,9 +137,7 @@ func TestRunPostTaskScriptIntegration(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
defer cancel()
|
||||
|
||||
taskCtx, err := structpb.NewStruct(map[string]any{})
|
||||
require.NoError(t, err)
|
||||
task := &runnerv1.Task{Id: 11, Context: taskCtx}
|
||||
task := &runnerv1.Task{Id: 11, Context: &structpb.Struct{}}
|
||||
reporter := report.NewReporter(ctx, cancel, nil, task, cfg)
|
||||
setReporterJobResult(t, reporter, runnerv1.Result_RESULT_SUCCESS)
|
||||
|
||||
|
||||
@@ -55,7 +55,7 @@ func JobProxyEnv(jobEnvs map[string]string, cacheURL string, serviceNames []stri
|
||||
setPair("https_proxy", "HTTPS_PROXY", cfg.HTTPSProxy)
|
||||
|
||||
// no_proxy is merged rather than replaced: the hosts above are structural, and an
|
||||
// operator cannot list the cache server's startup-assigned address in advance.
|
||||
// operator cannot list the cache server's detected address in advance.
|
||||
noProxy := appendNoProxy(cfg.NoProxy, direct...)
|
||||
for _, name := range []string{"no_proxy", "NO_PROXY"} {
|
||||
if existing, ok := jobEnvs[name]; ok {
|
||||
|
||||
+44
-27
@@ -73,7 +73,7 @@ type Runner struct {
|
||||
cacheHandler *artifactcache.Handler
|
||||
capabilities string
|
||||
|
||||
isolatedCacheContainer func() string
|
||||
isolatedCacheContainer func(cacheURL string) string
|
||||
instanceTransport http.RoundTripper
|
||||
|
||||
runningTasks sync.Map
|
||||
@@ -118,7 +118,6 @@ func NewRunner(cfg *config.Config, reg *config.Registration, cli client.Client)
|
||||
// go on
|
||||
} else {
|
||||
cacheHandler = handler
|
||||
envs["ACTIONS_CACHE_URL"] = handler.ExternalURL() + "/"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -141,7 +140,7 @@ func NewRunner(cfg *config.Config, reg *config.Registration, cli client.Client)
|
||||
now: time.Now,
|
||||
runHealthCheck: executeHealthCheck,
|
||||
}
|
||||
runner.isolatedCacheContainer = sync.OnceValue(runner.detectIsolatedCacheContainer)
|
||||
runner.isolatedCacheContainer = lastResult(runner.detectIsolatedCacheContainer)
|
||||
runner.instanceTransport = instanceTransport
|
||||
return runner
|
||||
}
|
||||
@@ -196,7 +195,7 @@ func (r *Runner) cleanupOrphanDockerResources(ctx context.Context) {
|
||||
return
|
||||
}
|
||||
cutoff := r.now().Add(-orphanDockerResourceAge)
|
||||
if err := removeOrphanNetworks(ctx, r.uuid, r.isolatedCacheContainer(), cutoff); err != nil {
|
||||
if err := removeOrphanNetworks(ctx, r.uuid, r.isolatedCacheContainer(r.cacheHandler.ExternalURL()), cutoff); err != nil {
|
||||
logging.Warnf("failed to clean up networks left behind by earlier jobs: %v", err)
|
||||
}
|
||||
if err := removeOrphanJobVolumes(ctx, r.uuid, cutoff); err != nil {
|
||||
@@ -485,7 +484,7 @@ func (r *Runner) run(ctx context.Context, task *runnerv1.Task, reporter *report.
|
||||
return fallbackPlatform()
|
||||
}
|
||||
platform := platformPicker(job.RunsOn())
|
||||
cacheURL, cacheContainer := r.cacheForJob(job, platform)
|
||||
cacheURL, cacheContainer := r.cacheForJob(job, platform, reporter.Logf)
|
||||
if cacheURL != "" {
|
||||
envs["ACTIONS_CACHE_URL"] = cacheURL + "/"
|
||||
}
|
||||
@@ -612,7 +611,7 @@ func (r *Runner) run(ctx context.Context, task *runnerv1.Task, reporter *report.
|
||||
},
|
||||
ContainerOptions: r.cfg.Container.Options,
|
||||
ServiceReadyTimeout: r.cfg.Container.ServiceReadyTimeout,
|
||||
NoSweep: r.cfg.Container.Sweep != nil && !*r.cfg.Container.Sweep,
|
||||
Sweep: r.cfg.Container.Sweep,
|
||||
ContainerDaemonSocket: r.cfg.Container.DockerHost,
|
||||
Privileged: r.cfg.Container.Privileged,
|
||||
DefaultActionInstance: r.getDefaultActionsURL(task),
|
||||
@@ -629,6 +628,7 @@ func (r *Runner) run(ctx context.Context, task *runnerv1.Task, reporter *report.
|
||||
ClientKeyFile: r.cfg.Runner.ClientKeyFile,
|
||||
CAFile: r.cfg.Runner.CAFile,
|
||||
InstanceTransport: r.instanceTransport,
|
||||
ResultsURL: strings.TrimSuffix(r.client.Address(), "/"),
|
||||
RunnerName: r.name,
|
||||
KubernetesPicker: r.kubernetesOptions,
|
||||
}
|
||||
@@ -663,11 +663,24 @@ func (r *Runner) run(ctx context.Context, task *runnerv1.Task, reporter *report.
|
||||
return execErr
|
||||
}
|
||||
|
||||
func (r *Runner) builtInCacheURL() string {
|
||||
if r.cacheHandler == nil {
|
||||
return ""
|
||||
func lastResult(compute func(key string) (string, bool)) func(key string) string {
|
||||
var (
|
||||
mu sync.Mutex
|
||||
lastKey string
|
||||
result string
|
||||
)
|
||||
return func(key string) string {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if key == lastKey {
|
||||
return result
|
||||
}
|
||||
value, ok := compute(key)
|
||||
if ok {
|
||||
lastKey, result = key, value
|
||||
}
|
||||
return value
|
||||
}
|
||||
return r.envs["ACTIONS_CACHE_URL"]
|
||||
}
|
||||
|
||||
func (r *Runner) setResultsService(envs map[string]string, resultsURL string) {
|
||||
@@ -698,7 +711,7 @@ func (r *Runner) instanceOutOfReach(instance string) bool {
|
||||
// Safe with an empty token (older Gitea did not issue one).
|
||||
// It also returns the URL to advertise as ACTIONS_RESULTS_URL.
|
||||
func (r *Runner) registerCacheForTask(token string, cred artifactcache.JobCredential, reporter *report.Reporter) (func(), string) {
|
||||
if token == "" || r.cacheHandler == nil {
|
||||
if token == "" || cred.PublicURL == "" {
|
||||
return func() {}, ""
|
||||
}
|
||||
cred.Results = r.envs["ACTIONS_RESULTS_URL"] // the instance as the job would reach it
|
||||
@@ -929,35 +942,39 @@ func warnIgnoredCacheSecret(cfg *config.Config) {
|
||||
}
|
||||
|
||||
// cacheForJob returns the cache address for the job, and the container to attach to its network to reach it there.
|
||||
func (r *Runner) cacheForJob(job *model.Job, platform string) (string, string) {
|
||||
func (r *Runner) cacheForJob(job *model.Job, platform string, logf func(string, ...any)) (string, string) {
|
||||
cacheURL := r.cacheHandler.RefreshExternalURL()
|
||||
jobContainer := job.Container()
|
||||
cacheContainer := ""
|
||||
if !strings.HasPrefix(platform, labels.SchemeKubernetes+"://") && (jobContainer != nil && jobContainer.Image != "" || platform != labels.SelfHostedPlatform) {
|
||||
cacheContainer = r.isolatedCacheContainer()
|
||||
if cacheURL == "" || strings.HasPrefix(platform, labels.SchemeKubernetes+"://") || (jobContainer == nil || jobContainer.Image == "") && platform == labels.SelfHostedPlatform {
|
||||
return cacheURL, ""
|
||||
}
|
||||
if cacheContainer == "" {
|
||||
return strings.TrimSuffix(r.builtInCacheURL(), "/"), ""
|
||||
if cacheContainer := r.isolatedCacheContainer(cacheURL); cacheContainer != "" {
|
||||
isolatedURL, _ := url.Parse(cacheURL)
|
||||
isolatedURL.Host = net.JoinHostPort(cacheContainer, isolatedURL.Port())
|
||||
return isolatedURL.String(), cacheContainer
|
||||
}
|
||||
cacheURL, _ := url.Parse(r.cacheHandler.ExternalURL())
|
||||
cacheURL.Host = net.JoinHostPort(cacheContainer, cacheURL.Port())
|
||||
return cacheURL.String(), cacheContainer
|
||||
network, enableIPv6 := r.cfg.Container.Network, r.cfg.Container.NetworkCreateOptions.EnableIPv6
|
||||
if addr, err := netip.ParseAddr(hostOf(cacheURL)); err == nil && addr.Is6() && r.cfg.Cache.Host == "" &&
|
||||
network != "host" && (network != "" || enableIPv6 == nil || !*enableIPv6) {
|
||||
logf("::warning::%s", runner.EscapeCommandData("The runner's cache server is at the IPv6 address "+addr.String()+
|
||||
", so cache and artifact steps fail if this job's network has no IPv6. Set cache.host to an address the job reaches, or enable IPv6 on the job network."))
|
||||
}
|
||||
return cacheURL, ""
|
||||
}
|
||||
|
||||
func (r *Runner) detectIsolatedCacheContainer() string {
|
||||
if r.cacheHandler == nil {
|
||||
return ""
|
||||
}
|
||||
addr, err := netip.ParseAddr(hostOf(r.cacheHandler.ExternalURL()))
|
||||
func (r *Runner) detectIsolatedCacheContainer(cacheURL string) (string, bool) {
|
||||
addr, err := netip.ParseAddr(hostOf(cacheURL))
|
||||
if err != nil {
|
||||
return ""
|
||||
return "", true
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
cacheContainer, err := container.IsolatedCacheContainer(ctx, addr, r.cfg.Container.Network)
|
||||
if err != nil {
|
||||
logging.Warnf("cannot check whether jobs reach the cache server: %v", err)
|
||||
return "", false
|
||||
}
|
||||
return cacheContainer
|
||||
return cacheContainer, true
|
||||
}
|
||||
|
||||
// taskLogFields returns key/value pairs for logging.With.
|
||||
|
||||
@@ -155,12 +155,11 @@ func TestRunner_ExternalCacheServer_RegisterRevoke(t *testing.T) {
|
||||
}, &config.Registration{}, cli)
|
||||
t.Cleanup(func() { _ = r.Close() })
|
||||
require.NotNil(t, r.cacheHandler)
|
||||
require.Equal(t, r.cacheHandler.ExternalURL()+"/", r.envs["ACTIONS_CACHE_URL"])
|
||||
|
||||
const publicURL = "http://a1b2c3d4e5f6:8088"
|
||||
token := "external-task-token"
|
||||
repo := "owner/repoX"
|
||||
base := r.envs["ACTIONS_CACHE_URL"] + "_apis/artifactcache"
|
||||
base := r.cacheHandler.ExternalURL() + "/_apis/artifactcache"
|
||||
probe := func() int {
|
||||
resp := cacheRequest(t, token, http.MethodGet, base+"/cache?keys=k&version=v", "", "", "")
|
||||
resp.Body.Close()
|
||||
@@ -213,7 +212,7 @@ func TestRunner_ExternalCacheServer_RegisterRevoke(t *testing.T) {
|
||||
require.NoError(t, decodeJSON(resp, &hit))
|
||||
require.True(t, strings.HasPrefix(hit.ArchiveLocation, publicURL+"/"), hit.ArchiveLocation)
|
||||
|
||||
unregisterOther, _ := r.registerCacheForTask("external-other-token", artifactcache.JobCredential{Repo: repo, Ref: "refs/heads/other"}, nil)
|
||||
unregisterOther, _ := r.registerCacheForTask("external-other-token", artifactcache.JobCredential{Repo: repo, Ref: "refs/heads/other", PublicURL: publicURL}, nil)
|
||||
defer unregisterOther()
|
||||
resp = cacheRequest(t, "external-other-token", http.MethodGet, base+"/cache?keys=ext-key&version=v", "", "", "")
|
||||
resp.Body.Close()
|
||||
|
||||
@@ -21,8 +21,7 @@ import (
|
||||
|
||||
func TestRunnerCleanupStaleTaskDirs(t *testing.T) {
|
||||
now := time.Date(2026, time.April, 29, 20, 0, 0, 0, time.UTC)
|
||||
workdirRoot := filepath.Join(t.TempDir(), "workspace")
|
||||
require.NoError(t, os.MkdirAll(workdirRoot, 0o700))
|
||||
workdirRoot := t.TempDir()
|
||||
|
||||
oldTask := filepath.Join(workdirRoot, "1001")
|
||||
freshTask := filepath.Join(workdirRoot, "1002")
|
||||
@@ -59,8 +58,7 @@ func TestRunnerCleanupStaleTaskDirs(t *testing.T) {
|
||||
// be reclaimed, while the shared tool_cache and operator data are preserved.
|
||||
func TestRunnerOnIdleCleansStaleHostScratchDirs(t *testing.T) {
|
||||
now := time.Date(2026, time.April, 29, 20, 0, 0, 0, time.UTC)
|
||||
hostRoot := filepath.Join(t.TempDir(), "act")
|
||||
require.NoError(t, os.MkdirAll(hostRoot, 0o700))
|
||||
hostRoot := t.TempDir()
|
||||
|
||||
staleScratch := filepath.Join(hostRoot, "0123456789abcdef") // 16 hex
|
||||
freshScratch := filepath.Join(hostRoot, "fedcba9876543210")
|
||||
@@ -118,8 +116,7 @@ func TestRunnerCleanupStaleTaskDirsMissingRoot(t *testing.T) {
|
||||
|
||||
func TestRunnerCleanupStaleTaskDirsHonorsContext(t *testing.T) {
|
||||
now := time.Date(2026, time.April, 29, 20, 0, 0, 0, time.UTC)
|
||||
workdirRoot := filepath.Join(t.TempDir(), "workspace")
|
||||
require.NoError(t, os.MkdirAll(workdirRoot, 0o700))
|
||||
workdirRoot := t.TempDir()
|
||||
|
||||
for i := 1001; i <= 1003; i++ {
|
||||
dir := filepath.Join(workdirRoot, strconv.Itoa(i))
|
||||
@@ -344,7 +341,7 @@ func TestRunnerOnIdleRemovesOrphanNetworks(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
|
||||
r := &Runner{uuid: "runner-1", cfg: cfg, labels: labels.Labels{kubernetesLabel}, now: func() time.Time { return now }}
|
||||
r.isolatedCacheContainer = func() string { return "a1b2c3d4e5f6" }
|
||||
r.isolatedCacheContainer = func(_cacheURL string) string { return "a1b2c3d4e5f6" }
|
||||
r.OnIdle(t.Context())
|
||||
assert.Equal(t, []string{"runner-1"}, swept)
|
||||
assert.Equal(t, swept, sweptVolumes)
|
||||
@@ -356,7 +353,7 @@ func TestRunnerOnIdleRemovesOrphanNetworks(t *testing.T) {
|
||||
dockerReachable = func(context.Context) bool { return false }
|
||||
t.Cleanup(func() { dockerReachable = origDockerReachable })
|
||||
hostOnly := &Runner{uuid: "runner-2", cfg: &config.Config{Runner: cfg.Runner}, now: func() time.Time { return now }}
|
||||
hostOnly.isolatedCacheContainer = func() string { return "" }
|
||||
hostOnly.isolatedCacheContainer = func(_cacheURL string) string { return "" }
|
||||
hostOnly.OnIdle(t.Context())
|
||||
assert.Equal(t, []string{"runner-1"}, swept)
|
||||
assert.Equal(t, swept, sweptVolumes)
|
||||
|
||||
@@ -306,7 +306,7 @@ func TestNewRunnerLeavesProxyToTheTask(t *testing.T) {
|
||||
|
||||
require.NotContains(t, r.envs, "http_proxy")
|
||||
require.NotContains(t, r.envs, "no_proxy")
|
||||
proxyEnv := JobProxyEnv(r.envs, r.builtInCacheURL(), nil)
|
||||
proxyEnv := JobProxyEnv(r.envs, r.cacheHandler.ExternalURL(), nil)
|
||||
assert.Contains(t, proxyEnv["no_proxy"], "runner.local")
|
||||
assert.NotContains(t, proxyEnv["no_proxy"], "cache.local")
|
||||
}
|
||||
@@ -355,7 +355,7 @@ func TestNewRunnerCacheServiceV2(t *testing.T) {
|
||||
defer resp.Body.Close()
|
||||
|
||||
assert.Equal(t, http.StatusOK, resp.StatusCode, "the advertised results service serves no cache service")
|
||||
assert.Equal(t, r.envs["ACTIONS_CACHE_URL"], r.builtInCacheURL(), "the address only the runner knows is bypassed for the operator")
|
||||
assert.NotContains(t, r.envs, "ACTIONS_CACHE_URL")
|
||||
|
||||
envs := r.cloneEnvs()
|
||||
r.setResultsService(envs, resultsURL)
|
||||
@@ -390,10 +390,10 @@ func TestNewRunnerCacheServiceV2(t *testing.T) {
|
||||
|
||||
workflow, err := model.ReadWorkflow(strings.NewReader(`jobs: {native: {runs-on: native}, linux: {runs-on: linux}, containerized: {runs-on: native, container: alpine}, empty: {runs-on: native, container: ""}}`))
|
||||
require.NoError(t, err)
|
||||
r.isolatedCacheContainer = func() string { return "a1b2c3d4e5f6" }
|
||||
r.isolatedCacheContainer = func(_cacheURL string) string { return "a1b2c3d4e5f6" }
|
||||
pickPlatform := func(runsOn []string) string { return map[string]string{"native": labels.SelfHostedPlatform}[runsOn[0]] }
|
||||
for job, isolated := range map[string]bool{"native": false, "linux": true, "containerized": true, "empty": false} {
|
||||
cacheURL, cacheContainer := r.cacheForJob(workflow.GetJob(job), pickPlatform(workflow.GetJob(job).RunsOn()))
|
||||
cacheURL, cacheContainer := r.cacheForJob(workflow.GetJob(job), pickPlatform(workflow.GetJob(job).RunsOn()), t.Logf)
|
||||
if isolated {
|
||||
assert.Equal(t, "a1b2c3d4e5f6", cacheContainer, job)
|
||||
assert.Equal(t, strings.Replace(r.cacheHandler.ExternalURL(), "127.0.0.1", cacheContainer, 1), cacheURL, job)
|
||||
@@ -404,6 +404,41 @@ func TestNewRunnerCacheServiceV2(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestCacheForJobIPv6WarningAndIsolationRecheck(t *testing.T) {
|
||||
handler, err := artifactcache.StartHandler(artifactcache.Options{OutboundIP: "2001:db8::1", Upstream: "http://cache.invalid"})
|
||||
require.NoError(t, err)
|
||||
defer handler.Close()
|
||||
var checked []string
|
||||
r := &Runner{cfg: &config.Config{}, cacheHandler: handler, isolatedCacheContainer: lastResult(func(cacheURL string) (string, bool) {
|
||||
checked = append(checked, cacheURL)
|
||||
return "", cacheURL != "http://unreachable"
|
||||
})}
|
||||
warnings := 0
|
||||
logf := func(_format string, _args ...any) { warnings++ }
|
||||
workflow, err := model.ReadWorkflow(strings.NewReader(`jobs: {native: {runs-on: native}, containerized: {runs-on: native, container: alpine}}`))
|
||||
require.NoError(t, err)
|
||||
for _, job := range []string{"native", "containerized", "containerized"} {
|
||||
cacheURL, _ := r.cacheForJob(workflow.GetJob(job), labels.SelfHostedPlatform, logf)
|
||||
assert.Equal(t, handler.ExternalURL(), cacheURL)
|
||||
}
|
||||
assert.Equal(t, 2, warnings)
|
||||
assert.Equal(t, []string{handler.ExternalURL()}, checked)
|
||||
|
||||
r.cfg.Container.NetworkCreateOptions.EnableIPv6 = new(true)
|
||||
r.cacheForJob(workflow.GetJob("containerized"), labels.SelfHostedPlatform, logf)
|
||||
assert.Equal(t, 2, warnings)
|
||||
r.cfg.Container.Network = "bridge"
|
||||
r.cacheForJob(workflow.GetJob("containerized"), labels.SelfHostedPlatform, logf)
|
||||
assert.Equal(t, 3, warnings)
|
||||
r.cfg.Cache.Host = "2001:db8::1"
|
||||
r.cacheForJob(workflow.GetJob("containerized"), labels.SelfHostedPlatform, logf)
|
||||
assert.Equal(t, 3, warnings)
|
||||
|
||||
r.isolatedCacheContainer("http://unreachable")
|
||||
r.isolatedCacheContainer("http://unreachable")
|
||||
assert.Len(t, checked, 3)
|
||||
}
|
||||
|
||||
func TestRestoreCalledWorkflowContext(t *testing.T) {
|
||||
newTask := func(workflowCall map[string]any) *runnerv1.Task {
|
||||
values := map[string]any{
|
||||
|
||||
@@ -96,6 +96,7 @@ func TestSetupLinesInputs(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestPrettyOSName(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
for _, tt := range []struct {
|
||||
name string
|
||||
osRelease string
|
||||
@@ -109,7 +110,7 @@ func TestPrettyOSName(t *testing.T) {
|
||||
{"missing file", "", "", true},
|
||||
} {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "os-release")
|
||||
path := filepath.Join(dir, tt.name)
|
||||
if !tt.missing {
|
||||
require.NoError(t, os.WriteFile(path, []byte(tt.osRelease), 0o600))
|
||||
}
|
||||
|
||||
@@ -5,6 +5,19 @@
|
||||
package action
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"unicode"
|
||||
|
||||
"gitea.com/gitea/runner/act/container"
|
||||
|
||||
"gitea.dev/actionslib/pkg/model"
|
||||
@@ -24,4 +37,138 @@ type Context struct {
|
||||
ClientKeyFile string
|
||||
CAFile string // the runner CA as seen inside Container
|
||||
Env map[string]string
|
||||
ResultsURL string // the results service as the runner reaches it
|
||||
RuntimeToken string
|
||||
Transport http.RoundTripper // carries the instance's TLS settings, nil for the default
|
||||
Infof func(format string, args ...any)
|
||||
Warning func(format string, args ...any)
|
||||
}
|
||||
|
||||
func (c *Context) ApplyDefaults(pairs ...string) {
|
||||
var valid, unexpected []string
|
||||
for i := 0; i+1 < len(pairs); i += 2 {
|
||||
valid = append(valid, pairs[i])
|
||||
if _, set := c.Inputs[pairs[i]]; !set && pairs[i+1] != "" {
|
||||
c.Inputs[pairs[i]] = pairs[i+1]
|
||||
}
|
||||
}
|
||||
for name := range c.Inputs {
|
||||
if !slices.Contains(valid, name) {
|
||||
unexpected = append(unexpected, name)
|
||||
}
|
||||
}
|
||||
if len(unexpected) > 0 {
|
||||
slices.Sort(unexpected)
|
||||
c.Warning("Unexpected input(s) '%s', valid inputs are ['%s']", strings.Join(unexpected, "', '"), strings.Join(valid, "', '"))
|
||||
}
|
||||
}
|
||||
|
||||
// Input is the trimmed input, as @actions/core getInput returns it.
|
||||
func (c *Context) Input(name string) string {
|
||||
return strings.TrimSpace(c.Inputs[name])
|
||||
}
|
||||
|
||||
// BoolInput accepts what @actions/core getBooleanInput does.
|
||||
func (c *Context) BoolInput(name string) (bool, error) {
|
||||
switch c.Input(name) {
|
||||
case "true", "True", "TRUE":
|
||||
return true, nil
|
||||
case "false", "False", "FALSE":
|
||||
return false, nil
|
||||
}
|
||||
return false, Failf("Input does not meet YAML 1.2 \"Core Schema\" specification: %s\nSupport boolean input list: `true | True | TRUE | false | False | FALSE`", name)
|
||||
}
|
||||
|
||||
func (c *Context) SetOutputs(ctx context.Context, pairs ...string) error {
|
||||
output := c.Env["GITHUB_OUTPUT"]
|
||||
if output == "" {
|
||||
return nil
|
||||
}
|
||||
var body strings.Builder
|
||||
for i := 0; i+1 < len(pairs); i += 2 {
|
||||
if strings.ContainsAny(pairs[i+1], "\r\n") {
|
||||
return fmt.Errorf("output %s spans several lines", pairs[i])
|
||||
}
|
||||
body.WriteString(pairs[i] + "=" + pairs[i+1] + "\n")
|
||||
}
|
||||
return c.Container.Copy(c.Dir(output), &container.FileEntry{Name: c.Base(output), Mode: 0o666, Body: body.String()})(ctx)
|
||||
}
|
||||
|
||||
func (c *Context) Host() bool {
|
||||
_, host := c.Container.(*container.HostEnvironment)
|
||||
return host
|
||||
}
|
||||
|
||||
func (c *Context) Windows() bool {
|
||||
return c.Host() && runtime.GOOS == "windows"
|
||||
}
|
||||
|
||||
func (c *Context) Join(elem ...string) string {
|
||||
if c.Windows() {
|
||||
return filepath.Join(elem...)
|
||||
}
|
||||
return path.Join(elem...)
|
||||
}
|
||||
|
||||
func (c *Context) Dir(p string) string {
|
||||
if c.Windows() {
|
||||
return filepath.Dir(p)
|
||||
}
|
||||
return path.Dir(p)
|
||||
}
|
||||
|
||||
func (c *Context) Base(p string) string {
|
||||
if c.Windows() {
|
||||
return filepath.Base(p)
|
||||
}
|
||||
return path.Base(p)
|
||||
}
|
||||
|
||||
func (c *Context) Resolve(p string) string {
|
||||
if c.Windows() && filepath.IsAbs(p) || !c.Windows() && path.IsAbs(p) {
|
||||
return c.Join(p)
|
||||
}
|
||||
return c.Join(c.Workspace, p)
|
||||
}
|
||||
|
||||
// Home is the job user's home directory, what os.homedir() returns to a JavaScript action.
|
||||
func (c *Context) Home(ctx context.Context) (string, error) {
|
||||
if home := c.Env["HOME"]; home != "" {
|
||||
return home, nil
|
||||
}
|
||||
if home := c.Env["USERPROFILE"]; c.Windows() && home != "" {
|
||||
return home, nil
|
||||
}
|
||||
if c.Host() {
|
||||
return os.UserHomeDir()
|
||||
}
|
||||
var out strings.Builder
|
||||
stdout, stderr := c.Container.ReplaceLogWriter(&out, &out)
|
||||
err := c.Container.Exec([]string{"sh", "-c", `printf %s "$HOME"`}, nil, "", "")(ctx)
|
||||
c.Container.ReplaceLogWriter(stdout, stderr)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("cannot expand ~ without HOME: %w", err)
|
||||
}
|
||||
if out.Len() == 0 {
|
||||
return "", errors.New("cannot expand ~, the job user has no HOME")
|
||||
}
|
||||
return out.String(), nil
|
||||
}
|
||||
|
||||
// Failf is fmt.Errorf for messages worded as the JavaScript action words them, which the error string lint rejects.
|
||||
func Failf(format string, args ...any) error {
|
||||
return fmt.Errorf(format, args...)
|
||||
}
|
||||
|
||||
// ParseInt reads a leading integer the way JavaScript's parseInt does.
|
||||
func ParseInt(s string) (int, error) {
|
||||
s = strings.TrimLeftFunc(s, unicode.IsSpace)
|
||||
end := 0
|
||||
if end < len(s) && (s[end] == '-' || s[end] == '+') {
|
||||
end++
|
||||
}
|
||||
for end < len(s) && s[end] >= '0' && s[end] <= '9' {
|
||||
end++
|
||||
}
|
||||
return strconv.Atoi(s[:end])
|
||||
}
|
||||
@@ -0,0 +1,184 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package artifact
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"golang.org/x/sync/errgroup"
|
||||
)
|
||||
|
||||
// BlockSize is the toolkit's upload chunk.
|
||||
const BlockSize = 8 << 20
|
||||
|
||||
type Uploaded struct {
|
||||
Size int64
|
||||
Digest string // sha256:<hex> for Finalize
|
||||
}
|
||||
|
||||
// Upload puts body to a signed URL as parallel Azure blocks, failing when upload progress stalls for stall.
|
||||
func (c *Client) Upload(ctx context.Context, signedURL, contentType string, body io.Reader, concurrency int, stall time.Duration) (result Uploaded, err error) {
|
||||
ctx, cancel := context.WithCancelCause(ctx)
|
||||
defer func() {
|
||||
if ctx.Err() != nil {
|
||||
err = context.Cause(ctx)
|
||||
}
|
||||
cancel(nil)
|
||||
}()
|
||||
stall = max(stall, time.Millisecond)
|
||||
timer := time.AfterFunc(stall, func() { cancel(errors.New("upload progress stalled")) })
|
||||
defer timer.Stop()
|
||||
hash := sha256.New()
|
||||
reader := io.TeeReader(body, hash)
|
||||
group, blockCtx := errgroup.WithContext(ctx)
|
||||
if closer, ok := body.(io.ReadCloser); ok {
|
||||
stop := context.AfterFunc(blockCtx, func() { closer.Close() })
|
||||
defer stop()
|
||||
}
|
||||
buffers := make(chan []byte, concurrency)
|
||||
prefix := rand.Text()
|
||||
var ids []string
|
||||
var uploaded atomic.Int64
|
||||
for index := 0; ; index++ {
|
||||
var buf []byte
|
||||
if index < concurrency {
|
||||
buf = make([]byte, BlockSize)
|
||||
} else {
|
||||
select {
|
||||
case buf = <-buffers:
|
||||
case <-blockCtx.Done():
|
||||
return Uploaded{}, group.Wait()
|
||||
}
|
||||
}
|
||||
n, readErr := io.ReadFull(reader, buf)
|
||||
if errors.Is(readErr, io.EOF) {
|
||||
break
|
||||
} else if readErr != nil && !errors.Is(readErr, io.ErrUnexpectedEOF) {
|
||||
if blockCtx.Err() == nil {
|
||||
cancel(readErr)
|
||||
}
|
||||
if err := group.Wait(); err != nil {
|
||||
return Uploaded{}, err
|
||||
}
|
||||
return Uploaded{}, readErr
|
||||
}
|
||||
id := base64.StdEncoding.EncodeToString(fmt.Appendf(nil, "%s-%08d", prefix, index)) // fixed width as Azure needs, unique as Gitea before 1.25.5 shares a run's blocks
|
||||
ids = append(ids, id)
|
||||
group.Go(func() error {
|
||||
defer func() { buffers <- buf }()
|
||||
if err := c.put(blockCtx, signedURL+"&comp=block&blockid="+url.QueryEscape(id), "x-ms-blob-type", "BlockBlob", buf[:n]); err != nil {
|
||||
return err
|
||||
}
|
||||
timer.Reset(stall)
|
||||
c.Logf("Uploaded bytes %d", uploaded.Add(int64(n)))
|
||||
return nil
|
||||
})
|
||||
if readErr != nil {
|
||||
break
|
||||
}
|
||||
}
|
||||
if err := group.Wait(); err != nil {
|
||||
return Uploaded{}, err
|
||||
}
|
||||
var list strings.Builder
|
||||
list.WriteString("<BlockList>")
|
||||
for _, id := range ids {
|
||||
list.WriteString("<Latest>" + id + "</Latest>")
|
||||
}
|
||||
list.WriteString("</BlockList>")
|
||||
if err := c.put(ctx, signedURL+"&comp=blocklist", "x-ms-blob-content-type", contentType, []byte(list.String())); err != nil {
|
||||
return Uploaded{}, err
|
||||
}
|
||||
return Uploaded{Size: uploaded.Load(), Digest: "sha256:" + hex.EncodeToString(hash.Sum(nil))}, nil
|
||||
}
|
||||
|
||||
func (c *Client) put(ctx context.Context, signedURL, header, value string, body []byte) error {
|
||||
resp, err := c.Retry.do(ctx, c.Logf, "blob upload", func() (*http.Response, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPut, signedURL, bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.Header.Set(header, value)
|
||||
resp, err := c.blob.Do(req)
|
||||
return resp, scrub(err)
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return resp.Body.Close()
|
||||
}
|
||||
|
||||
// Download GETs a signed URL, a read failing when nothing arrives for stall.
|
||||
func (c *Client) Download(ctx context.Context, signedURL string, stall time.Duration) (*http.Response, error) {
|
||||
ctx, cancel := context.WithCancelCause(ctx)
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, signedURL, nil)
|
||||
if err != nil {
|
||||
cancel(nil)
|
||||
return nil, err
|
||||
}
|
||||
resp, err := c.blob.Do(req)
|
||||
if err != nil {
|
||||
cancel(nil)
|
||||
return nil, scrub(err)
|
||||
}
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
resp.Body.Close()
|
||||
cancel(nil)
|
||||
return nil, fmt.Errorf("unexpected HTTP response from blob storage: %d %s", resp.StatusCode, http.StatusText(resp.StatusCode))
|
||||
}
|
||||
resp.Body = &stallReader{body: resp.Body, timer: time.AfterFunc(stall, func() {
|
||||
cancel(fmt.Errorf("blob storage chunk did not respond in %s", stall))
|
||||
}), stall: stall, cancel: cancel, ctx: ctx}
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
func scrub(err error) error { // the signature of a signed URL is a credential
|
||||
if urlErr, ok := errors.AsType[*url.Error](err); ok {
|
||||
if parsed, parseErr := url.Parse(urlErr.URL); parseErr == nil {
|
||||
parsed.RawQuery = ""
|
||||
urlErr.URL = parsed.String()
|
||||
} else {
|
||||
urlErr.URL = "<unparsable URL>"
|
||||
}
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
type stallReader struct {
|
||||
body io.ReadCloser
|
||||
timer *time.Timer
|
||||
stall time.Duration
|
||||
cancel context.CancelCauseFunc
|
||||
ctx context.Context
|
||||
}
|
||||
|
||||
func (r *stallReader) Read(p []byte) (int, error) {
|
||||
n, err := r.body.Read(p)
|
||||
if n > 0 {
|
||||
r.timer.Reset(r.stall)
|
||||
}
|
||||
if err != nil && !errors.Is(err, io.EOF) && context.Cause(r.ctx) != nil {
|
||||
err = context.Cause(r.ctx)
|
||||
}
|
||||
return n, err
|
||||
}
|
||||
|
||||
func (r *stallReader) Close() error {
|
||||
r.timer.Stop()
|
||||
r.cancel(nil)
|
||||
return r.body.Close()
|
||||
}
|
||||
@@ -0,0 +1,264 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
// Package artifact talks to the v4 results service of the job's run.
|
||||
package artifact
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"cmp"
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json/v2"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"math"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type Artifact struct {
|
||||
ID int64 `json:"databaseId,string"`
|
||||
Name string `json:"name"`
|
||||
Size int64 `json:"size,string"`
|
||||
Digest string `json:"digest"`
|
||||
}
|
||||
|
||||
type Retry struct {
|
||||
Attempts int
|
||||
Base time.Duration
|
||||
Multiplier float64
|
||||
MaxWait time.Duration
|
||||
}
|
||||
|
||||
var DefaultRetry = Retry{Attempts: 5, Base: 8 * time.Second, Multiplier: 1.5, MaxWait: 2 * time.Minute}
|
||||
|
||||
type Client struct {
|
||||
base *url.URL
|
||||
token string
|
||||
ids backendIDsMessage
|
||||
api, blob *http.Client
|
||||
Retry Retry
|
||||
Logf func(format string, args ...any)
|
||||
}
|
||||
|
||||
// New reaches base through transport, which carries the instance's TLS settings, and signed URLs elsewhere directly.
|
||||
func New(base, token string, transport http.RoundTripper) (*Client, error) {
|
||||
if base == "" {
|
||||
return nil, errors.New("no artifact service is configured for this runner")
|
||||
}
|
||||
parsed, err := url.Parse(base)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if token == "" {
|
||||
return nil, errors.New("ACTIONS_RUNTIME_TOKEN is not set")
|
||||
}
|
||||
ids, err := backendIDs(token)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &Client{
|
||||
base: parsed, token: token, ids: ids,
|
||||
api: &http.Client{Transport: transport, Timeout: 3 * time.Minute},
|
||||
blob: &http.Client{Transport: blobTransport{instance: parsed.Host, transport: transport}},
|
||||
Retry: DefaultRetry,
|
||||
Logf: func(string, ...any) {},
|
||||
}, nil
|
||||
}
|
||||
|
||||
// backendIDs reads the unverified scp claim of a runtime token, as the toolkit does.
|
||||
func backendIDs(token string) (backendIDsMessage, error) {
|
||||
parts := strings.Split(token, ".")
|
||||
if len(parts) != 3 {
|
||||
return backendIDsMessage{}, errors.New("the runtime token is not a JWT, artifacts v4 need Gitea 1.22 or later")
|
||||
}
|
||||
payload, err := base64.RawURLEncoding.DecodeString(strings.TrimRight(parts[1], "="))
|
||||
if err != nil {
|
||||
return backendIDsMessage{}, err
|
||||
}
|
||||
var claims struct {
|
||||
Scp string `json:"scp"`
|
||||
}
|
||||
if err := json.Unmarshal(payload, &claims); err != nil {
|
||||
return backendIDsMessage{}, err
|
||||
}
|
||||
for scope := range strings.FieldsSeq(claims.Scp) {
|
||||
if fields := strings.Split(scope, ":"); fields[0] == "Actions.Results" && len(fields) == 3 {
|
||||
return backendIDsMessage{fields[1], fields[2]}, nil
|
||||
}
|
||||
}
|
||||
return backendIDsMessage{}, errors.New("the runtime token has no Actions.Results scope")
|
||||
}
|
||||
|
||||
type blobTransport struct { // the instance's TLS settings only for the instance, as a signed URL can point at object storage
|
||||
instance string
|
||||
transport http.RoundTripper
|
||||
}
|
||||
|
||||
func (t blobTransport) RoundTrip(req *http.Request) (*http.Response, error) {
|
||||
if req.URL.Host == t.instance && t.transport != nil {
|
||||
return t.transport.RoundTrip(req)
|
||||
}
|
||||
return http.DefaultTransport.RoundTrip(req)
|
||||
}
|
||||
|
||||
type StatusError struct {
|
||||
Status int
|
||||
Msg string
|
||||
}
|
||||
|
||||
func (e *StatusError) Error() string {
|
||||
if e.Msg == "" {
|
||||
return fmt.Sprintf("(%d) %s", e.Status, http.StatusText(e.Status))
|
||||
}
|
||||
return fmt.Sprintf("(%d) %s: %s", e.Status, http.StatusText(e.Status), e.Msg)
|
||||
}
|
||||
|
||||
func IsNotFound(err error) bool {
|
||||
var status *StatusError
|
||||
return errors.As(err, &status) && status.Status == http.StatusNotFound
|
||||
}
|
||||
|
||||
func (r Retry) do(ctx context.Context, logf func(string, ...any), what string, send func() (*http.Response, error)) (*http.Response, error) {
|
||||
var waited time.Duration
|
||||
for attempt := 0; ; attempt++ {
|
||||
resp, err := send()
|
||||
var wait time.Duration
|
||||
if err == nil {
|
||||
if resp.StatusCode >= 200 && resp.StatusCode < 400 {
|
||||
return resp, nil
|
||||
}
|
||||
body, _ := io.ReadAll(io.LimitReader(resp.Body, 4096))
|
||||
resp.Body.Close()
|
||||
err = &StatusError{Status: resp.StatusCode, Msg: twirpMessage(body)}
|
||||
switch resp.StatusCode {
|
||||
case http.StatusTooManyRequests, http.StatusInternalServerError, http.StatusBadGateway, http.StatusServiceUnavailable, http.StatusGatewayTimeout:
|
||||
default:
|
||||
return nil, err
|
||||
}
|
||||
if seconds, parseErr := strconv.Atoi(resp.Header.Get("Retry-After")); parseErr == nil && seconds > 0 && resp.StatusCode == http.StatusTooManyRequests {
|
||||
wait = time.Duration(seconds) * time.Second
|
||||
}
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
if attempt+1 >= r.Attempts {
|
||||
return nil, fmt.Errorf("%s failed after %d attempts: %w", what, r.Attempts, err)
|
||||
}
|
||||
if wait == 0 {
|
||||
wait = time.Duration(float64(r.Base) * math.Pow(r.Multiplier, float64(attempt)))
|
||||
}
|
||||
if waited+wait > r.MaxWait {
|
||||
return nil, fmt.Errorf("%s: a retry wait of %s would exceed %s: %w", what, wait, r.MaxWait, err)
|
||||
}
|
||||
logf("Attempt %d of %d failed with error: %v. Retrying request in %d ms...", attempt+1, r.Attempts, err, wait.Milliseconds())
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
case <-time.After(wait):
|
||||
}
|
||||
waited += wait
|
||||
}
|
||||
}
|
||||
|
||||
// twirpMessage is the msg of a twirp error, the message of a Gitea API one, or the body Gitea answers in plain text.
|
||||
func twirpMessage(body []byte) string {
|
||||
var twirpErr struct {
|
||||
Msg string `json:"msg"`
|
||||
Message string `json:"message"`
|
||||
}
|
||||
if json.Unmarshal(body, &twirpErr) == nil && cmp.Or(twirpErr.Msg, twirpErr.Message) != "" {
|
||||
return cmp.Or(twirpErr.Msg, twirpErr.Message)
|
||||
}
|
||||
return strings.TrimSpace(string(body))
|
||||
}
|
||||
|
||||
type ( // the ArtifactService messages in the protobuf JSON mapping, requests with the proto names the toolkit sends
|
||||
backendIDsMessage struct {
|
||||
WorkflowRunBackendID string `json:"workflow_run_backend_id"`
|
||||
WorkflowJobRunBackendID string `json:"workflow_job_run_backend_id"`
|
||||
}
|
||||
namedRequest struct {
|
||||
backendIDsMessage
|
||||
Name string `json:"name"`
|
||||
}
|
||||
createRequest struct {
|
||||
namedRequest
|
||||
ExpiresAt *time.Time `json:"expires_at,omitempty"`
|
||||
Version int32 `json:"version"`
|
||||
MimeType string `json:"mime_type,omitempty"` // Gitea before 1.26 rejects the field
|
||||
}
|
||||
finalizeRequest struct {
|
||||
namedRequest
|
||||
Size int64 `json:"size,string"`
|
||||
Hash string `json:"hash"`
|
||||
}
|
||||
listRequest struct {
|
||||
backendIDsMessage
|
||||
NameFilter string `json:"name_filter,omitempty"`
|
||||
}
|
||||
idResponse struct {
|
||||
ArtifactID int64 `json:"artifactId,string"`
|
||||
}
|
||||
)
|
||||
|
||||
func call[Resp any](ctx context.Context, c *Client, method string, req any) (Resp, error) {
|
||||
var resp Resp
|
||||
body, err := json.Marshal(req)
|
||||
if err != nil {
|
||||
return resp, err
|
||||
}
|
||||
endpoint := c.base.JoinPath("/twirp/github.actions.results.api.v1.ArtifactService", method).String()
|
||||
httpResp, err := c.Retry.do(ctx, c.Logf, method, func() (*http.Response, error) {
|
||||
httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
httpReq.Header.Set("Content-Type", "application/json")
|
||||
httpReq.Header.Set("Authorization", "Bearer "+c.token)
|
||||
return c.api.Do(httpReq)
|
||||
})
|
||||
if err != nil {
|
||||
return resp, fmt.Errorf("failed to %s: %w", method, err)
|
||||
}
|
||||
defer httpResp.Body.Close()
|
||||
err = json.UnmarshalRead(httpResp.Body, &resp)
|
||||
return resp, err
|
||||
}
|
||||
|
||||
func (c *Client) Create(ctx context.Context, name, mimeType string, expiresAt *time.Time) (string, error) {
|
||||
resp, err := call[struct {
|
||||
SignedUploadURL string `json:"signedUploadUrl"`
|
||||
}](ctx, c, "CreateArtifact", createRequest{namedRequest{c.ids, name}, expiresAt, 7, mimeType})
|
||||
return resp.SignedUploadURL, err
|
||||
}
|
||||
|
||||
func (c *Client) Finalize(ctx context.Context, name string, size int64, digest string) (int64, error) {
|
||||
resp, err := call[idResponse](ctx, c, "FinalizeArtifact", finalizeRequest{namedRequest{c.ids, name}, size, digest})
|
||||
return resp.ArtifactID, err
|
||||
}
|
||||
|
||||
func (c *Client) List(ctx context.Context, name string) ([]Artifact, error) {
|
||||
resp, err := call[struct {
|
||||
Artifacts []Artifact `json:"artifacts"`
|
||||
}](ctx, c, "ListArtifacts", listRequest{c.ids, name})
|
||||
return resp.Artifacts, err
|
||||
}
|
||||
|
||||
func (c *Client) SignedURL(ctx context.Context, name string) (string, error) {
|
||||
resp, err := call[struct {
|
||||
SignedURL string `json:"signedUrl"`
|
||||
}](ctx, c, "GetSignedArtifactURL", namedRequest{c.ids, name})
|
||||
return resp.SignedURL, err
|
||||
}
|
||||
|
||||
func (c *Client) Delete(ctx context.Context, name string) (int64, error) {
|
||||
resp, err := call[idResponse](ctx, c, "DeleteArtifact", namedRequest{c.ids, name})
|
||||
return resp.ArtifactID, err
|
||||
}
|
||||
@@ -0,0 +1,182 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package artifact
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.com/gitea/runner/act/artifacts"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestClientRoundTrip(t *testing.T) {
|
||||
router := http.NewServeMux()
|
||||
artifacts.RegisterV4(router, t.TempDir())
|
||||
var blocks atomic.Int32
|
||||
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
|
||||
if request.URL.Query().Get("comp") == "block" {
|
||||
blocks.Add(1)
|
||||
}
|
||||
router.ServeHTTP(writer, request)
|
||||
}))
|
||||
defer server.Close()
|
||||
client, err := New(server.URL, artifacts.LocalRuntimeToken("7", time.Hour), nil)
|
||||
require.NoError(t, err)
|
||||
for _, content := range [][]byte{nil, bytes.Repeat([]byte("payload"), 2*BlockSize/7+123)} {
|
||||
sum := sha256.Sum256(content)
|
||||
uploadURL, err := client.Create(t.Context(), "art", "text/plain", nil)
|
||||
require.NoError(t, err)
|
||||
uploaded, err := client.Upload(t.Context(), uploadURL, "text/plain", bytes.NewReader(content), 2, time.Minute)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, int32((len(content)+BlockSize-1)/BlockSize), blocks.Swap(0))
|
||||
assert.Equal(t, Uploaded{Size: int64(len(content)), Digest: "sha256:" + hex.EncodeToString(sum[:])}, uploaded)
|
||||
id, err := client.Finalize(t.Context(), "art", uploaded.Size, uploaded.Digest)
|
||||
require.NoError(t, err)
|
||||
listed, err := client.List(t.Context(), "art")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, []Artifact{{ID: id, Name: "art", Size: int64(len(content)), Digest: uploaded.Digest}}, listed)
|
||||
signed, err := client.SignedURL(t.Context(), "art")
|
||||
require.NoError(t, err)
|
||||
resp, err := client.Download(t.Context(), signed, time.Minute)
|
||||
require.NoError(t, err)
|
||||
got, err := io.ReadAll(resp.Body)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, resp.Body.Close())
|
||||
assert.True(t, bytes.Equal(content, got))
|
||||
deleted, err := client.Delete(t.Context(), "art")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, id, deleted)
|
||||
_, err = client.SignedURL(t.Context(), "art")
|
||||
assert.True(t, IsNotFound(err), err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestClientRetries(t *testing.T) {
|
||||
var calls atomic.Int32
|
||||
var blockIDs []string
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method == http.MethodPut {
|
||||
blockIDs = append(blockIDs, r.URL.Query().Get("blockid"))
|
||||
w.WriteHeader(http.StatusCreated)
|
||||
return
|
||||
}
|
||||
body, _ := io.ReadAll(r.Body)
|
||||
assert.NotContains(t, string(body), "mime_type", "older Gitea rejects the field")
|
||||
assert.True(t, strings.HasPrefix(r.Header.Get("Authorization"), "Bearer ey"), "the runtime token is the bearer")
|
||||
switch calls.Add(1) {
|
||||
case 1:
|
||||
w.Header().Set("Retry-After", "0")
|
||||
w.WriteHeader(http.StatusTooManyRequests)
|
||||
case 2:
|
||||
http.Error(w, "Error create or get artifact", http.StatusInternalServerError)
|
||||
case 3:
|
||||
w.WriteHeader(http.StatusFound)
|
||||
_, _ = io.WriteString(w, `{"ok":true,"signedUploadUrl":"http://blob/u?sig=x"}`)
|
||||
default:
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
_, _ = io.WriteString(w, `{"code":"invalid_argument","msg":"bad name"}`)
|
||||
}
|
||||
}))
|
||||
defer server.Close()
|
||||
client, err := New(server.URL, artifacts.LocalRuntimeToken("7", time.Hour), nil)
|
||||
require.NoError(t, err)
|
||||
client.Retry = Retry{Attempts: 3}
|
||||
var logged []string
|
||||
client.Logf = func(format string, args ...any) { logged = append(logged, fmt.Sprintf(format, args...)) }
|
||||
|
||||
uploadURL, err := client.Create(t.Context(), "art", "", nil)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "http://blob/u?sig=x", uploadURL)
|
||||
assert.Contains(t, logged[1], "Error create or get artifact")
|
||||
|
||||
_, err = client.Create(t.Context(), "art", "", nil)
|
||||
require.ErrorContains(t, err, "(400) Bad Request: bad name")
|
||||
assert.Equal(t, int32(4), calls.Load(), "a 400 is not retried")
|
||||
|
||||
for range 2 {
|
||||
_, err = client.Upload(t.Context(), server.URL+"/u?sig=secret", "application/zip", strings.NewReader("x"), 1, time.Minute)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
assert.NotEqual(t, blockIDs[0], blockIDs[2], "each upload names its blocks anew")
|
||||
server.Close()
|
||||
_, err = client.Upload(t.Context(), server.URL+"/u?sig=secret", "application/zip", strings.NewReader("x"), 1, time.Minute)
|
||||
require.Error(t, err)
|
||||
assert.NotContains(t, err.Error()+strings.Join(logged, ""), "secret")
|
||||
}
|
||||
|
||||
func TestTransferStalls(t *testing.T) {
|
||||
for _, test := range []struct{ name, want string }{
|
||||
{"api", "Client.Timeout"},
|
||||
{"failed upload", "(400) Bad Request"},
|
||||
{"source", "upload progress stalled"},
|
||||
{"commit", "upload progress stalled"},
|
||||
{"download", "did not respond"},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
|
||||
_, _ = io.Copy(io.Discard, request.Body)
|
||||
if test.name == "failed upload" {
|
||||
writer.WriteHeader(http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if test.name == "download" {
|
||||
_, _ = io.WriteString(writer, "partial")
|
||||
writer.(http.Flusher).Flush()
|
||||
}
|
||||
<-request.Context().Done()
|
||||
}))
|
||||
defer server.Close()
|
||||
client, err := New(server.URL, artifacts.LocalRuntimeToken("7", time.Hour), nil)
|
||||
require.NoError(t, err)
|
||||
client.Retry = Retry{Attempts: 1}
|
||||
ctx, cancel := context.WithTimeout(t.Context(), time.Second)
|
||||
defer cancel()
|
||||
switch test.name {
|
||||
case "api":
|
||||
assert.Equal(t, 3*time.Minute, client.api.Timeout)
|
||||
client.api.Timeout = time.Millisecond
|
||||
_, err = client.Create(ctx, "art", "", nil)
|
||||
case "download":
|
||||
var resp *http.Response
|
||||
resp, err = client.Download(ctx, server.URL, time.Millisecond)
|
||||
require.NoError(t, err)
|
||||
defer resp.Body.Close()
|
||||
_, err = io.ReadAll(resp.Body)
|
||||
case "commit":
|
||||
_, err = client.Upload(ctx, server.URL+"?sig=x", "text/plain", strings.NewReader(""), 1, time.Millisecond)
|
||||
default:
|
||||
reader, writer := io.Pipe()
|
||||
defer reader.Close()
|
||||
defer writer.Close()
|
||||
done := make(chan error, 1)
|
||||
stall := time.Millisecond
|
||||
if test.name == "failed upload" {
|
||||
stall = time.Minute
|
||||
go func() {
|
||||
_, err := writer.Write(make([]byte, BlockSize))
|
||||
done <- err
|
||||
}()
|
||||
}
|
||||
_, err = client.Upload(ctx, server.URL+"?sig=x", "text/plain", reader, 2, stall)
|
||||
if test.name == "failed upload" {
|
||||
require.NoError(t, <-done)
|
||||
}
|
||||
}
|
||||
require.ErrorContains(t, err, test.want)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,117 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package artifact
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json/v2"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"time"
|
||||
)
|
||||
|
||||
type REST struct {
|
||||
base *url.URL
|
||||
token string
|
||||
api *http.Client
|
||||
blob *http.Client
|
||||
Retry Retry
|
||||
Logf func(format string, args ...any)
|
||||
}
|
||||
|
||||
// NewREST takes the instance as the runner reaches it, never the job's GITHUB_API_URL, so the token only goes there.
|
||||
func NewREST(instance, token string, transport http.RoundTripper) (*REST, error) {
|
||||
parsed, err := url.Parse(instance)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &REST{
|
||||
base: parsed.JoinPath("api", "v1"),
|
||||
token: token,
|
||||
api: &http.Client{Transport: transport, CheckRedirect: func(*http.Request, []*http.Request) error {
|
||||
return http.ErrUseLastResponse // the redirect target is signed and must not get the token
|
||||
}},
|
||||
blob: &http.Client{Transport: blobTransport{instance: parsed.Host, transport: transport}},
|
||||
Retry: DefaultRetry,
|
||||
Logf: func(string, ...any) {},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (r *REST) get(ctx context.Context, what string, endpoint *url.URL) (*http.Response, error) {
|
||||
return r.Retry.do(ctx, r.Logf, what, func() (*http.Response, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint.String(), nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.Header.Set("Authorization", "token "+r.token)
|
||||
req.Header.Set("Accept", "application/json")
|
||||
return r.api.Do(req)
|
||||
})
|
||||
}
|
||||
|
||||
func (r *REST) List(ctx context.Context, owner, repo string, runID int64, name string) ([]Artifact, error) {
|
||||
if name == "" {
|
||||
r.Logf("Fetching artifact list for workflow run %d in repository %s/%s", runID, owner, repo)
|
||||
}
|
||||
var list []Artifact
|
||||
seen := 0
|
||||
for page := 1; ; page++ {
|
||||
endpoint := r.base.JoinPath("repos", owner, repo, "actions", "runs", strconv.FormatInt(runID, 10), "artifacts")
|
||||
query := url.Values{"page": {strconv.Itoa(page)}, "limit": {"50"}}
|
||||
if name != "" {
|
||||
query.Set("name", name)
|
||||
}
|
||||
endpoint.RawQuery = query.Encode()
|
||||
resp, err := r.get(ctx, "list artifacts", endpoint)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var body struct {
|
||||
TotalCount int `json:"total_count"`
|
||||
Artifacts []struct {
|
||||
ID int64 `json:"id"`
|
||||
Name string `json:"name"`
|
||||
SizeInBytes int64 `json:"size_in_bytes"`
|
||||
Expired bool `json:"expired"`
|
||||
Digest string `json:"digest"`
|
||||
} `json:"artifacts"`
|
||||
}
|
||||
err = json.UnmarshalRead(resp.Body, &body)
|
||||
resp.Body.Close()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list artifacts: %w", err)
|
||||
}
|
||||
for _, listed := range body.Artifacts {
|
||||
if !listed.Expired {
|
||||
list = append(list, Artifact{ID: listed.ID, Name: listed.Name, Size: listed.SizeInBytes, Digest: listed.Digest})
|
||||
}
|
||||
}
|
||||
if seen += len(body.Artifacts); len(body.Artifacts) == 0 || seen >= body.TotalCount {
|
||||
return list, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (r *REST) DownloadURL(ctx context.Context, owner, repo string, id int64) (string, error) {
|
||||
resp, err := r.get(ctx, "download artifact", r.base.JoinPath("repos", owner, repo, "actions", "artifacts", strconv.FormatInt(id, 10), "zip"))
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("unable to download artifact: %w", err)
|
||||
}
|
||||
resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusFound {
|
||||
return "", fmt.Errorf("unable to download artifact. Unexpected status: %d", resp.StatusCode)
|
||||
}
|
||||
location, err := resp.Location()
|
||||
if err != nil {
|
||||
return "", errors.New("unable to redirect to artifact download url")
|
||||
}
|
||||
return location.String(), nil
|
||||
}
|
||||
|
||||
func (r *REST) Download(ctx context.Context, signedURL string, stall time.Duration) (*http.Response, error) {
|
||||
return (&Client{blob: r.blob}).Download(ctx, signedURL, stall)
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package artifact
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestRESTListsEveryPageAndSkipsExpired(t *testing.T) {
|
||||
var pages []string
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
assert.Equal(t, "/api/v1/repos/own%20er/repo/actions/runs/12/artifacts", r.URL.EscapedPath())
|
||||
assert.Equal(t, "token secret", r.Header.Get("Authorization"))
|
||||
assert.Empty(t, r.URL.Query().Get("per_page"))
|
||||
pages = append(pages, r.URL.Query().Get("page")+"/"+r.URL.Query().Get("limit"))
|
||||
page, _ := strconv.Atoi(r.URL.Query().Get("page"))
|
||||
const maxResponseItems = 20
|
||||
var entries []string
|
||||
for id := (page-1)*maxResponseItems + 1; id <= min(page*maxResponseItems, 53); id++ {
|
||||
entries = append(entries, fmt.Sprintf(`{"id":%d,"name":"a%d","size_in_bytes":%d,"expired":%t}`, id, id, id, id == 2))
|
||||
}
|
||||
_, _ = io.WriteString(w, `{"total_count":53,"artifacts":[`+strings.Join(entries, ",")+`]}`)
|
||||
}))
|
||||
defer server.Close()
|
||||
rest, err := NewREST(server.URL, "secret", nil)
|
||||
require.NoError(t, err)
|
||||
list, err := rest.List(t.Context(), "own er", "repo", 12, "")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, []string{"1/50", "2/50", "3/50"}, pages)
|
||||
assert.Len(t, list, 52)
|
||||
assert.Equal(t, Artifact{ID: 3, Name: "a3", Size: 3}, list[1])
|
||||
}
|
||||
@@ -59,7 +59,7 @@ func TestCheckout(t *testing.T) {
|
||||
t.Run("history", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
work := t.TempDir()
|
||||
runGit(t, "init", "--initial-branch=main", work)
|
||||
runGit(t, "init", "--template=", "--initial-branch=main", work)
|
||||
runGit(t, "-C", work, "commit", "--allow-empty", "-m", "c1")
|
||||
sha = runGit(t, "-C", work, "rev-parse", "HEAD")
|
||||
runGit(t, "-C", work, "tag", "v1")
|
||||
@@ -77,7 +77,7 @@ func TestCheckout(t *testing.T) {
|
||||
t.Run("sha256", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
sha256Repo := filepath.Join(server, "org", "sha256")
|
||||
runGit(t, "init", "--object-format=sha256", "--initial-branch=main", sha256Repo)
|
||||
runGit(t, "init", "--template=", "--object-format=sha256", "--initial-branch=main", sha256Repo)
|
||||
runGit(t, "-C", sha256Repo, "commit", "--allow-empty", "-m", "s1")
|
||||
runGit(t, "-C", sha256Repo, "checkout", "--detach")
|
||||
})
|
||||
@@ -267,21 +267,41 @@ exec "$LFS" "$@"
|
||||
assert.Contains(t, string(outputs), "ref="+cmp.Or(tc.eventRef, "refs/heads/main")+"\n")
|
||||
return
|
||||
}
|
||||
if tc.head == "" {
|
||||
assert.Empty(t, gitOutput(dst, "symbolic-ref", "-q", "HEAD"))
|
||||
} else {
|
||||
assert.Equal(t, "refs/heads/"+tc.head, gitOutput(dst, "symbolic-ref", "-q", "HEAD"))
|
||||
assert.Equal(t, "origin/"+tc.head, gitOutput(dst, "rev-parse", "--abbrev-ref", "@{u}"))
|
||||
headArgs := []string{"-C", dst, "rev-parse", "HEAD", "--symbolic-full-name", "HEAD"}
|
||||
if tc.head != "" {
|
||||
headArgs = append(headArgs, "--abbrev-ref", "@{u}")
|
||||
}
|
||||
assert.Contains(t, string(outputs), "commit="+gitOutput(dst, "rev-parse", "HEAD")+"\n")
|
||||
head := strings.Fields(runGit(t, headArgs...))
|
||||
if tc.head == "" {
|
||||
require.Len(t, head, 2)
|
||||
assert.Equal(t, "HEAD", head[1])
|
||||
} else {
|
||||
require.Len(t, head, 3)
|
||||
assert.Equal(t, "refs/heads/"+tc.head, head[1])
|
||||
assert.Equal(t, "origin/"+tc.head, head[2])
|
||||
}
|
||||
assert.Contains(t, string(outputs), "commit="+head[0]+"\n")
|
||||
if inputRef := tc.inputs["ref"]; inputRef != "" && inputRef != sha {
|
||||
assert.Contains(t, string(outputs), "ref="+inputRef+"\n")
|
||||
}
|
||||
assert.Equal(t, tc.inputs["sparse-checkout"] != "", gitOutput(dst, "config", "--local", "extensions.worktreeConfig") == "true")
|
||||
persisted := tc.inputs["persist-credentials"] != "false"
|
||||
for _, repo := range repos {
|
||||
var credentialIncludes []string
|
||||
var worktreeConfig string
|
||||
for entry := range strings.SplitSeq(runGit(t, "-C", repo, "config", "--local", "--no-includes", "--null", "--list"), "\x00") {
|
||||
key, value, _ := strings.Cut(entry, "\n")
|
||||
switch key {
|
||||
case "extensions.worktreeconfig":
|
||||
worktreeConfig = value
|
||||
case strings.ToLower(credentialsInclude):
|
||||
credentialIncludes = append(credentialIncludes, value)
|
||||
}
|
||||
}
|
||||
if repo == dst {
|
||||
assert.Equal(t, tc.inputs["sparse-checkout"] != "", strings.TrimSpace(worktreeConfig) == "true")
|
||||
}
|
||||
assert.Equal(t, tc.inputs["clean"] == "false", fileExists(filepath.Join(repo, "untracked")), repo)
|
||||
assert.Len(t, strings.Fields(gitOutput(repo, "config", "--local", "--get-all", credentialsInclude)), 1, repo)
|
||||
assert.Len(t, strings.Fields(strings.Join(credentialIncludes, "\n")), 1, repo)
|
||||
headers, _ := exec.Command("git", "-C", repo, "config", "--get-all", "http."+serverURL+"/.extraheader").Output()
|
||||
assert.Equal(t, persisted, string(headers) == "\nAuthorization: Basic dG9rZW46dG9r\n", repo)
|
||||
assert.Equal(t, persisted, gitOutput(repo, "config", "http."+serverURL+"/.sslcainfo") != "", repo)
|
||||
@@ -345,15 +365,16 @@ func TestCheckoutTrustsTheRunnerCAAndOffersClientCertOnlyDuringTheStep(t *testin
|
||||
|
||||
func publish(t *testing.T, server, name string, files, submodules map[string]string) {
|
||||
work := t.TempDir()
|
||||
runGit(t, "init", "--initial-branch=main", work)
|
||||
runGit(t, "init", "--template=", "--initial-branch=main", work)
|
||||
for file, content := range files {
|
||||
require.NoError(t, os.MkdirAll(filepath.Dir(filepath.Join(work, file)), 0o755))
|
||||
require.NoError(t, os.WriteFile(filepath.Join(work, file), []byte(content), 0o644))
|
||||
}
|
||||
runGit(t, "-C", work, "add", ".")
|
||||
var moduleConfig []byte
|
||||
for path, url := range submodules {
|
||||
runGit(t, "-C", work, "config", "-f", ".gitmodules", "submodule."+path+".path", path)
|
||||
runGit(t, "-C", work, "config", "-f", ".gitmodules", "submodule."+path+".url", url)
|
||||
moduleConfig = fmt.Appendf(moduleConfig, "[submodule %q]\n\tpath = %q\n\turl = %q\n", path, path, url)
|
||||
require.NoError(t, os.WriteFile(filepath.Join(work, ".gitmodules"), moduleConfig, 0o644))
|
||||
commit := runGit(t, "--git-dir", filepath.Join(server, name, url), "rev-parse", "main")
|
||||
runGit(t, "-C", work, "update-index", "--add", "--cacheinfo", "160000,"+commit+","+path)
|
||||
runGit(t, "-C", work, "add", ".gitmodules")
|
||||
|
||||
@@ -0,0 +1,174 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
// Package downloadartifact works like actions/download-artifact v8.
|
||||
package downloadartifact
|
||||
|
||||
import (
|
||||
"cmp"
|
||||
"context"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.com/gitea/runner/act/common"
|
||||
"gitea.com/gitea/runner/act/container"
|
||||
"gitea.com/gitea/runner/internal/pkg/action"
|
||||
"gitea.com/gitea/runner/internal/pkg/action/artifact"
|
||||
|
||||
"golang.org/x/sync/errgroup"
|
||||
)
|
||||
|
||||
var digestBehaviors = []string{"ignore", "info", "warn", "error"}
|
||||
|
||||
const downloadAttempts = 5
|
||||
|
||||
func Main(ctx context.Context, c *action.Context) error {
|
||||
if err := run(ctx, c); err != nil {
|
||||
return action.Failf("Unable to download artifact(s): %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func run(ctx context.Context, c *action.Context) error {
|
||||
c.ApplyDefaults("name", "", "artifact-ids", "", "path", "", "pattern", "", "merge-multiple", "false", "github-token", "",
|
||||
"repository", c.Github.Repository, "run-id", c.Github.RunID, "skip-decompress", "false", "digest-mismatch", "error")
|
||||
name, ids, pattern, token := c.Input("name"), c.Input("artifact-ids"), c.Input("pattern"), c.Input("github-token")
|
||||
mergeMultiple, err := c.BoolInput("merge-multiple")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
skipDecompress, err := c.BoolInput("skip-decompress")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
digestMismatch := cmp.Or(c.Input("digest-mismatch"), "error")
|
||||
if !slices.Contains(digestBehaviors, digestMismatch) {
|
||||
return action.Failf("Invalid value for 'digest-mismatch': '%s'. Valid options are: %s", digestMismatch, strings.Join(digestBehaviors, ", "))
|
||||
}
|
||||
dest := cmp.Or(c.Input("path"), c.Env["GITHUB_WORKSPACE"], c.Workspace)
|
||||
if strings.HasPrefix(dest, "~") {
|
||||
home, err := c.Home(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
dest = home + dest[1:]
|
||||
}
|
||||
dest = c.Resolve(dest)
|
||||
if name != "" && ids != "" {
|
||||
return action.Failf("Inputs 'name' and 'artifact-ids' cannot be used together. Please specify only one.")
|
||||
}
|
||||
|
||||
src, err := newSource(c, token)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
artifacts, err := src.list(ctx, name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
slices.SortStableFunc(artifacts, func(a, b artifact.Artifact) int { return cmp.Compare(b.ID, a.ID) })
|
||||
seen := map[string]bool{}
|
||||
artifacts = slices.DeleteFunc(artifacts, func(a artifact.Artifact) bool {
|
||||
duplicate := seen[a.Name]
|
||||
seen[a.Name] = true
|
||||
return duplicate
|
||||
})
|
||||
if name == "" {
|
||||
c.Infof("Found %d artifact(s)", len(artifacts))
|
||||
}
|
||||
switch {
|
||||
case name != "":
|
||||
if len(artifacts) == 0 {
|
||||
return action.Failf("Artifact not found for name: %s\nPlease ensure that your artifact is not expired and the artifact was uploaded using a compatible version of toolkit/upload-artifact.", name)
|
||||
}
|
||||
artifacts = artifacts[:1]
|
||||
case ids != "":
|
||||
var wanted []int64
|
||||
for id := range strings.SplitSeq(ids, ",") {
|
||||
if id = strings.TrimSpace(id); id == "" {
|
||||
continue
|
||||
}
|
||||
parsed, err := action.ParseInt(id)
|
||||
if err != nil {
|
||||
return action.Failf("Invalid artifact ID: '%s'. Must be a number.", id)
|
||||
}
|
||||
wanted = append(wanted, int64(parsed))
|
||||
}
|
||||
if len(wanted) == 0 {
|
||||
return action.Failf("No valid artifact IDs provided in 'artifact-ids' input")
|
||||
}
|
||||
artifacts = slices.DeleteFunc(artifacts, func(a artifact.Artifact) bool { return !slices.Contains(wanted, a.ID) })
|
||||
if len(artifacts) == 0 {
|
||||
return action.Failf("None of the provided artifact IDs were found")
|
||||
}
|
||||
case pattern != "":
|
||||
artifacts = slices.DeleteFunc(artifacts, func(a artifact.Artifact) bool { return !matchName(pattern, a.Name) })
|
||||
}
|
||||
for _, art := range artifacts {
|
||||
if art.Name == "" || art.Name == "." || art.Name == ".." || strings.ContainsAny(art.Name, `/\`) || c.Windows() && !filepath.IsLocal(art.Name) {
|
||||
return action.Failf("Artifact name is not valid: %s", art.Name)
|
||||
}
|
||||
}
|
||||
|
||||
mismatches := make([]string, len(artifacts))
|
||||
group, groupCtx := errgroup.WithContext(ctx)
|
||||
group.SetLimit(5)
|
||||
for i, art := range artifacts {
|
||||
target := dest
|
||||
if name == "" && !mergeMultiple && len(artifacts) > 1 {
|
||||
target = c.Join(dest, art.Name)
|
||||
}
|
||||
group.Go(func() error {
|
||||
mismatch, err := fetch(groupCtx, c, src, art, target, skipDecompress)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if mismatch {
|
||||
mismatches[i] = art.Name
|
||||
message := fmt.Sprintf("Artifact '%s' digest validation failed. Please verify the integrity of the artifact.", art.Name)
|
||||
switch digestMismatch {
|
||||
case "info":
|
||||
c.Infof("%s", message)
|
||||
case "warn":
|
||||
c.Warning("%s", message)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
if err := group.Wait(); err != nil {
|
||||
return err
|
||||
}
|
||||
mismatches = slices.DeleteFunc(mismatches, func(name string) bool { return name == "" })
|
||||
if len(mismatches) > 0 && digestMismatch == "error" {
|
||||
return action.Failf("Digest validation failed for artifact(s): %s. Use 'digest-mismatch: warn' to continue on mismatch.", strings.Join(mismatches, ", "))
|
||||
}
|
||||
c.Infof("Total of %d artifact(s) downloaded", len(artifacts))
|
||||
return c.SetOutputs(ctx, "download-path", dest)
|
||||
}
|
||||
|
||||
func fetch(ctx context.Context, c *action.Context, src *source, art artifact.Artifact, target string, skipDecompress bool) (bool, error) {
|
||||
if err := c.Container.CopyTarStream(ctx, target, func(*container.TarWriter) error { return nil }); err != nil {
|
||||
return false, err
|
||||
}
|
||||
signed, err := src.signedURL(ctx, art)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
for attempt := 1; attempt <= downloadAttempts; attempt++ {
|
||||
digest, err := fetchOnce(ctx, c, src, signed, target, skipDecompress)
|
||||
if err == nil {
|
||||
return art.Digest != "" && art.Digest != digest, nil
|
||||
}
|
||||
common.Logger(ctx).Debugf("Failed to download artifact after %d retries due to %v. Retrying in 5 seconds...", attempt, err)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return false, ctx.Err()
|
||||
case <-time.After(5 * time.Second):
|
||||
}
|
||||
}
|
||||
return false, action.Failf("Unable to download and extract artifact: Artifact download failed after %d retries.", downloadAttempts)
|
||||
}
|
||||
@@ -0,0 +1,316 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package downloadartifact
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"maps"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"testing/synctest"
|
||||
"time"
|
||||
|
||||
"gitea.com/gitea/runner/act/artifacts"
|
||||
"gitea.com/gitea/runner/act/container"
|
||||
"gitea.com/gitea/runner/internal/pkg/action"
|
||||
"gitea.com/gitea/runner/internal/pkg/action/artifact"
|
||||
|
||||
"gitea.dev/actionslib/pkg/model"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
type zipEntry struct {
|
||||
name, body string
|
||||
mode os.FileMode
|
||||
}
|
||||
|
||||
func zipOf(t *testing.T, entries ...zipEntry) []byte {
|
||||
t.Helper()
|
||||
var buf bytes.Buffer
|
||||
zw := zip.NewWriter(&buf)
|
||||
for _, e := range entries {
|
||||
header := &zip.FileHeader{Name: e.name, Method: zip.Deflate}
|
||||
header.SetMode(e.mode)
|
||||
w, err := zw.CreateHeader(header)
|
||||
require.NoError(t, err)
|
||||
_, err = io.WriteString(w, e.body)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
require.NoError(t, zw.Close())
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
type harness struct {
|
||||
t *testing.T
|
||||
server, store string
|
||||
client *artifact.Client
|
||||
workspace string
|
||||
env map[string]string
|
||||
mu sync.Mutex
|
||||
logs, annotation []string
|
||||
}
|
||||
|
||||
func newHarness(t *testing.T) *harness {
|
||||
t.Helper()
|
||||
store := t.TempDir()
|
||||
router := http.NewServeMux()
|
||||
artifacts.RegisterV4(router, store)
|
||||
server := httptest.NewServer(router)
|
||||
t.Cleanup(server.Close)
|
||||
client, err := artifact.New(server.URL, artifacts.LocalRuntimeToken("7", time.Hour), nil)
|
||||
require.NoError(t, err)
|
||||
return &harness{t: t, server: server.URL, store: store, client: client, workspace: t.TempDir()}
|
||||
}
|
||||
|
||||
func (h *harness) upload(name, mimeType string, content []byte) int64 {
|
||||
h.t.Helper()
|
||||
ctx := h.t.Context()
|
||||
signed, err := h.client.Create(ctx, name, mimeType, nil)
|
||||
require.NoError(h.t, err)
|
||||
uploaded, err := h.client.Upload(ctx, signed, mimeType, bytes.NewReader(content), 2, time.Minute)
|
||||
require.NoError(h.t, err)
|
||||
id, err := h.client.Finalize(ctx, name, uploaded.Size, uploaded.Digest)
|
||||
require.NoError(h.t, err)
|
||||
return id
|
||||
}
|
||||
|
||||
func (h *harness) run(inputs map[string]string) error {
|
||||
h.logs, h.annotation = nil, nil
|
||||
env := map[string]string{"GITHUB_OUTPUT": filepath.Join(h.workspace, "output")}
|
||||
maps.Copy(env, h.env)
|
||||
return Main(h.t.Context(), &action.Context{
|
||||
Container: &container.HostEnvironment{},
|
||||
Github: &model.GithubContext{Repository: "owner/repo", RunID: "7"},
|
||||
Inputs: inputs,
|
||||
Workspace: h.workspace,
|
||||
Env: env,
|
||||
ResultsURL: h.server,
|
||||
RuntimeToken: artifacts.LocalRuntimeToken("7", time.Hour),
|
||||
Infof: func(format string, args ...any) {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
h.logs = append(h.logs, fmt.Sprintf(format, args...))
|
||||
},
|
||||
Warning: func(format string, args ...any) {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
h.annotation = append(h.annotation, "warning: "+fmt.Sprintf(format, args...))
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
func (h *harness) read(rel string) string {
|
||||
h.t.Helper()
|
||||
content, err := os.ReadFile(filepath.Join(h.workspace, filepath.FromSlash(rel)))
|
||||
require.NoError(h.t, err)
|
||||
return string(content)
|
||||
}
|
||||
|
||||
func TestDownload(t *testing.T) {
|
||||
h := newHarness(t)
|
||||
first := h.upload("alpha", "", zipOf(t, zipEntry{name: "a.txt", body: "a1"}))
|
||||
_, err := h.client.Delete(t.Context(), "alpha")
|
||||
require.NoError(t, err)
|
||||
archive := zipOf(t,
|
||||
zipEntry{name: "run.sh", body: "#!/bin/sh\n", mode: 0o755},
|
||||
zipEntry{name: "empty/", mode: os.ModeDir | 0o755},
|
||||
zipEntry{name: "nested/../a.txt", body: "a2"},
|
||||
zipEntry{name: "empty.txt"},
|
||||
)
|
||||
second := h.upload("alpha", "", archive)
|
||||
h.upload("beta ü.txt", "text/plain", []byte("plain text"))
|
||||
h.upload(".hidden", "", zipOf(t, zipEntry{name: "h.txt", body: "h"}))
|
||||
|
||||
for _, test := range []struct {
|
||||
inputs map[string]string
|
||||
files map[string]string
|
||||
}{
|
||||
{map[string]string{"name": "alpha", "path": "named"}, map[string]string{"named/a.txt": "a2", "named/run.sh": "#!/bin/sh\n", "named/empty.txt": ""}},
|
||||
{map[string]string{"path": "all"}, map[string]string{"all/alpha/a.txt": "a2", "all/beta ü.txt/beta ü.txt": "plain text", "all/.hidden/h.txt": "h"}},
|
||||
{map[string]string{"path": "merged", "merge-multiple": "true"}, map[string]string{"merged/a.txt": "a2", "merged/beta ü.txt": "plain text", "merged/h.txt": "h"}},
|
||||
{map[string]string{"path": "pattern", "pattern": "*"}, map[string]string{"pattern/alpha/a.txt": "a2", "pattern/beta ü.txt/beta ü.txt": "plain text"}},
|
||||
{map[string]string{"path": "ids", "artifact-ids": fmt.Sprintf("%dtrailing, %d", second, first), "pattern": "ignored"}, map[string]string{"ids/a.txt": "a2"}},
|
||||
{map[string]string{"name": "alpha", "path": "kept", "skip-decompress": "true"}, map[string]string{"kept/alpha.zip": string(archive)}},
|
||||
{map[string]string{"path": "none", "pattern": "nope-*", "repository": "ignored", "run-id": "invalid"}, nil},
|
||||
} {
|
||||
require.NoError(t, h.run(test.inputs))
|
||||
assert.Contains(t, h.read("output"), "download-path="+filepath.Join(h.workspace, test.inputs["path"])+"\n")
|
||||
for name, body := range test.files {
|
||||
assert.Equal(t, body, h.read(name))
|
||||
}
|
||||
}
|
||||
assert.Contains(t, h.logs, "Total of 0 artifact(s) downloaded")
|
||||
assert.NoDirExists(t, filepath.Join(h.workspace, "pattern", ".hidden"))
|
||||
assert.DirExists(t, filepath.Join(h.workspace, "named", "empty"))
|
||||
if runtime.GOOS != "windows" {
|
||||
info, err := os.Stat(filepath.Join(h.workspace, "named", "run.sh"))
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, os.FileMode(0o644), info.Mode())
|
||||
assert.WithinDuration(t, time.Now(), info.ModTime(), time.Minute)
|
||||
}
|
||||
for _, test := range []struct {
|
||||
inputs map[string]string
|
||||
error string
|
||||
}{
|
||||
{map[string]string{"artifact-ids": strconv.FormatInt(first, 10)}, "None of the provided artifact IDs were found"},
|
||||
{map[string]string{"name": "gone"}, "Artifact not found for name: gone"},
|
||||
{map[string]string{"name": "alpha", "artifact-ids": "1"}, "cannot be used together"},
|
||||
{map[string]string{"artifact-ids": "x"}, "Invalid artifact ID: 'x'"},
|
||||
{map[string]string{"artifact-ids": " , "}, "No valid artifact IDs provided"},
|
||||
{map[string]string{"digest-mismatch": "loud"}, "Valid options are: ignore, info, warn, error"},
|
||||
{map[string]string{"merge-multiple": "yes"}, "Core Schema"},
|
||||
} {
|
||||
require.ErrorContains(t, h.run(test.inputs), test.error)
|
||||
}
|
||||
h.env = map[string]string{"GITHUB_WORKSPACE": filepath.Join(h.workspace, "overridden")}
|
||||
require.NoError(t, h.run(map[string]string{"name": "beta ü.txt"}))
|
||||
assert.Equal(t, "plain text", h.read("overridden/beta ü.txt"))
|
||||
h.env = map[string]string{"HOME": filepath.Join(h.workspace, "home")}
|
||||
require.NoError(t, h.run(map[string]string{"name": "beta ü.txt", "path": "~/tilde", "retention-days": "1"}))
|
||||
assert.Equal(t, "plain text", h.read("home/tilde/beta ü.txt"))
|
||||
assert.Equal(t, []string{"warning: Unexpected input(s) 'retention-days', valid inputs are ['name', 'artifact-ids', 'path', 'pattern', 'merge-multiple', 'github-token', 'repository', 'run-id', 'skip-decompress', 'digest-mismatch']"}, h.annotation)
|
||||
h.upload("..", "", archive)
|
||||
require.ErrorContains(t, h.run(map[string]string{}), "Artifact name is not valid: ..")
|
||||
}
|
||||
|
||||
func TestDownloadFormats(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
contentType, url string
|
||||
zip bool
|
||||
}{
|
||||
{"application/zip", "raw", true},
|
||||
{"Application/X-Zip-Compressed; charset=binary", "raw", true},
|
||||
{"application/zip-compressed", "raw", true},
|
||||
{"", "FILE.ZIP?sig=x", true},
|
||||
{"", "raw?name=file.zip", false},
|
||||
} {
|
||||
assert.Equal(t, test.zip, isZip(&http.Response{Header: http.Header{"Content-Type": {test.contentType}}}, "https://example.test/"+test.url))
|
||||
}
|
||||
for _, test := range []struct {
|
||||
pattern, name string
|
||||
want bool
|
||||
}{
|
||||
{"a*", "alpha", true},
|
||||
{"*", ".hidden", false},
|
||||
{"!alpha", "beta", true},
|
||||
{"#alpha", "alpha", false},
|
||||
} {
|
||||
assert.Equal(t, test.want, matchName(test.pattern, test.name))
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloadDigestMismatch(t *testing.T) {
|
||||
h := newHarness(t)
|
||||
h.upload("art", "", zipOf(t, zipEntry{name: "f.txt", body: "original"}))
|
||||
blobs, err := filepath.Glob(filepath.Join(h.store, "v4", "*", "*", "blob"))
|
||||
require.NoError(t, err)
|
||||
require.Len(t, blobs, 1)
|
||||
tampered := zipOf(t, zipEntry{name: "f.txt", body: "tampered"})
|
||||
require.NoError(t, os.WriteFile(blobs[0], tampered, 0o644))
|
||||
|
||||
for _, behavior := range []string{"", "ignore", "info", "warn", "error"} {
|
||||
err = h.run(map[string]string{"name": "art", "path": "digest", "digest-mismatch": behavior})
|
||||
if behavior == "" || behavior == "error" {
|
||||
require.ErrorContains(t, err, "Digest validation failed for artifact(s): art. Use 'digest-mismatch: warn' to continue on mismatch.")
|
||||
} else {
|
||||
require.NoError(t, err)
|
||||
}
|
||||
assert.Equal(t, "tampered", h.read("digest/f.txt"))
|
||||
if behavior == "warn" {
|
||||
assert.Equal(t, []string{"warning: Artifact 'art' digest validation failed. Please verify the integrity of the artifact."}, h.annotation)
|
||||
} else {
|
||||
assert.Empty(t, h.annotation)
|
||||
}
|
||||
if behavior == "info" {
|
||||
assert.Contains(t, h.logs, "Artifact 'art' digest validation failed. Please verify the integrity of the artifact.")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloadFromAnotherRun(t *testing.T) {
|
||||
h := &harness{t: t, workspace: t.TempDir()}
|
||||
archive := zipOf(t, zipEntry{name: "other.txt", body: "from run 12"})
|
||||
var expired atomic.Bool
|
||||
var zipCalls atomic.Int32
|
||||
blob := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
assert.Empty(t, r.Header.Get("Authorization"))
|
||||
w.Header().Set("Content-Type", "application/zip")
|
||||
_, _ = w.Write(archive)
|
||||
}))
|
||||
defer blob.Close()
|
||||
gitea := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.URL.Path {
|
||||
case "/api/v1/repos/someone/else/actions/runs/12/artifacts":
|
||||
assert.Equal(t, "token pat", r.Header.Get("Authorization"))
|
||||
_, _ = io.WriteString(w, `{"total_count":3,"artifacts":[
|
||||
{"id":10,"name":"dist","size_in_bytes":1,"expired":false},
|
||||
{"id":30,"name":"dist","size_in_bytes":1,"expired":true},
|
||||
{"id":20,"name":"dist","size_in_bytes":1,"expired":false}]}`)
|
||||
case "/api/v1/repos/someone/else/actions/artifacts/20/zip":
|
||||
zipCalls.Add(1)
|
||||
if expired.Load() {
|
||||
http.Error(w, "Artifact has expired", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
http.Redirect(w, r, blob.URL+"/raw/20?sig=abc", http.StatusFound)
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
}))
|
||||
defer gitea.Close()
|
||||
h.server = gitea.URL
|
||||
|
||||
inputs := map[string]string{"github-token": "pat", "repository": "someone/else/ignored", "run-id": "12", "path": "other"}
|
||||
require.NoError(t, h.run(inputs))
|
||||
assert.Equal(t, "from run 12", h.read("other/other.txt"))
|
||||
assert.Contains(t, h.logs, "Found 1 artifact(s)")
|
||||
expired.Store(true)
|
||||
require.ErrorContains(t, h.run(inputs), "Artifact has expired")
|
||||
assert.Equal(t, int32(2), zipCalls.Load(), "an API error is not retried")
|
||||
require.ErrorContains(t, h.run(map[string]string{"github-token": "pat", "repository": "nobody"}), "Invalid repository: 'nobody'. Must be in format owner/repo")
|
||||
require.ErrorContains(t, h.run(map[string]string{"github-token": "pat", "run-id": "0"}), "Invalid run-id: '0'")
|
||||
}
|
||||
|
||||
func TestDownloadRetries(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
attempts := 0
|
||||
target := filepath.Join(t.TempDir(), "download")
|
||||
started := time.Now()
|
||||
_, err := fetch(t.Context(), &action.Context{Container: &container.HostEnvironment{}}, &source{
|
||||
signedURL: func(context.Context, artifact.Artifact) (string, error) {
|
||||
assert.DirExists(t, target)
|
||||
return "https://example.test/artifact.zip", nil
|
||||
},
|
||||
download: func(context.Context, string, time.Duration) (*http.Response, error) {
|
||||
attempts++
|
||||
return nil, errors.New("connection failed")
|
||||
},
|
||||
}, artifact.Artifact{}, target, false)
|
||||
require.EqualError(t, err, "Unable to download and extract artifact: Artifact download failed after 5 retries.")
|
||||
assert.Equal(t, 5, attempts)
|
||||
assert.Equal(t, 25*time.Second, time.Since(started))
|
||||
})
|
||||
}
|
||||
|
||||
func TestDownloadRejectsEscapingZipPaths(t *testing.T) {
|
||||
for _, name := range []string{"../escape.txt", "a/../../escape.txt", "/absolute.txt", `..\escape.txt`} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
archive := zipOf(t, zipEntry{name: name, body: "escape"})
|
||||
require.ErrorContains(t, extractZip(t.Context(), &action.Context{Container: &container.HostEnvironment{}}, bytes.NewReader(archive), int64(len(archive)), t.TempDir()), "Artifact entry path is not valid")
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,142 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package downloadartifact
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"archive/zip"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"io"
|
||||
"mime"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.com/gitea/runner/act/container"
|
||||
"gitea.com/gitea/runner/internal/pkg/action"
|
||||
|
||||
"github.com/bmatcuk/doublestar/v4"
|
||||
)
|
||||
|
||||
func fetchOnce(ctx context.Context, c *action.Context, src *source, signed, target string, skipDecompress bool) (string, error) {
|
||||
resp, err := src.download(ctx, signed, 30*time.Second)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
file, err := os.CreateTemp("", "artifact-*")
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer os.Remove(file.Name())
|
||||
defer file.Close()
|
||||
hash := sha256.New()
|
||||
size, err := io.Copy(io.MultiWriter(file, hash), resp.Body)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
digest := "sha256:" + hex.EncodeToString(hash.Sum(nil))
|
||||
if isZip(resp, signed) && !skipDecompress {
|
||||
return digest, extractZip(ctx, c, file, size, target)
|
||||
}
|
||||
fileName := rawFileName(resp.Header.Get("Content-Disposition"))
|
||||
if fileName == ".." || strings.ContainsAny(fileName, `/\`) || c.Windows() && !filepath.IsLocal(fileName) {
|
||||
return "", action.Failf("Artifact filename is not valid: %s", fileName)
|
||||
}
|
||||
if _, err := file.Seek(0, io.SeekStart); err != nil {
|
||||
return digest, err
|
||||
}
|
||||
return digest, c.Container.CopyTarStream(ctx, target, func(tw *container.TarWriter) error {
|
||||
if err := tw.WriteHeader(&tar.Header{Typeflag: tar.TypeReg, Name: fileName, Mode: 0o644, Size: size, ModTime: time.Now()}); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err := io.Copy(tw, file)
|
||||
return err
|
||||
})
|
||||
}
|
||||
|
||||
var zipTypes = []string{"application/zip", "application/x-zip-compressed", "application/zip-compressed"}
|
||||
|
||||
func isZip(resp *http.Response, signed string) bool {
|
||||
mediaType, _, _ := strings.Cut(resp.Header.Get("Content-Type"), ";")
|
||||
mediaType = strings.ToLower(strings.TrimSpace(mediaType))
|
||||
signedURL, _ := url.Parse(signed)
|
||||
return slices.Contains(zipTypes, mediaType) || signedURL != nil && strings.HasSuffix(strings.ToLower(signedURL.Path), ".zip")
|
||||
}
|
||||
|
||||
func rawFileName(disposition string) string {
|
||||
_, params, err := mime.ParseMediaType(disposition)
|
||||
name := params["filename"] // filename* arrives decoded in here
|
||||
if err != nil || name == "" {
|
||||
return "artifact"
|
||||
}
|
||||
if unescaped, err := url.PathUnescape(name); err == nil && !strings.Contains(strings.ToLower(disposition), "filename*") {
|
||||
name = unescaped
|
||||
}
|
||||
if name = path.Base(name); name == "/" || name == "." {
|
||||
return "artifact"
|
||||
}
|
||||
return name
|
||||
}
|
||||
|
||||
func extractZip(ctx context.Context, c *action.Context, file io.ReaderAt, size int64, target string) error {
|
||||
reader, err := zip.NewReader(file, size)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
now := time.Now()
|
||||
return c.Container.CopyTarStream(ctx, target, func(tw *container.TarWriter) error {
|
||||
for _, entry := range reader.File {
|
||||
name := path.Clean(strings.ReplaceAll(entry.Name, `\`, "/"))
|
||||
if name == "." {
|
||||
continue
|
||||
}
|
||||
if path.IsAbs(name) || name == ".." || strings.HasPrefix(name, "../") || c.Windows() && !filepath.IsLocal(name) {
|
||||
return action.Failf("Artifact entry path is not valid: %s", entry.Name)
|
||||
}
|
||||
header := &tar.Header{Typeflag: tar.TypeReg, Name: name, Mode: 0o644, Size: int64(entry.UncompressedSize64), ModTime: now}
|
||||
if entry.FileInfo().IsDir() {
|
||||
header.Typeflag, header.Name, header.Mode, header.Size = tar.TypeDir, name+"/", 0o755, 0
|
||||
}
|
||||
if err := tw.WriteHeader(header); err != nil {
|
||||
return err
|
||||
}
|
||||
body, err := entry.Open()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = io.Copy(tw, body)
|
||||
if err := errors.Join(err, body.Close()); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func matchName(pattern, name string) bool {
|
||||
if strings.HasPrefix(pattern, "#") {
|
||||
return false
|
||||
}
|
||||
negate := false
|
||||
for strings.HasPrefix(pattern, "!") {
|
||||
negate, pattern = !negate, pattern[1:]
|
||||
}
|
||||
matched, err := doublestar.Match(pattern, name)
|
||||
if err != nil {
|
||||
matched = pattern == name
|
||||
}
|
||||
if strings.HasPrefix(name, ".") && !strings.HasPrefix(pattern, ".") {
|
||||
matched = false
|
||||
}
|
||||
return matched != negate
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package downloadartifact
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.com/gitea/runner/internal/pkg/action"
|
||||
"gitea.com/gitea/runner/internal/pkg/action/artifact"
|
||||
)
|
||||
|
||||
type source struct {
|
||||
list func(context.Context, string) ([]artifact.Artifact, error)
|
||||
signedURL func(context.Context, artifact.Artifact) (string, error)
|
||||
download func(context.Context, string, time.Duration) (*http.Response, error)
|
||||
}
|
||||
|
||||
func newSource(c *action.Context, token string) (*source, error) {
|
||||
if token == "" {
|
||||
client, err := artifact.New(c.ResultsURL, c.RuntimeToken, c.Transport)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
client.Logf = c.Infof
|
||||
return &source{
|
||||
list: client.List,
|
||||
signedURL: func(ctx context.Context, art artifact.Artifact) (string, error) {
|
||||
return client.SignedURL(ctx, art.Name)
|
||||
},
|
||||
download: client.Download,
|
||||
}, nil
|
||||
}
|
||||
parts := append(strings.Split(c.Input("repository"), "/"), "")
|
||||
owner, repo := parts[0], parts[1]
|
||||
if owner == "" || repo == "" {
|
||||
return nil, action.Failf("Invalid repository: '%s'. Must be in format owner/repo", c.Input("repository"))
|
||||
}
|
||||
runID, err := action.ParseInt(c.Input("run-id"))
|
||||
if err != nil || runID <= 0 { // Gitea lists the artifacts of every run for those
|
||||
return nil, action.Failf("Invalid run-id: '%s'. Must be a number.", c.Input("run-id"))
|
||||
}
|
||||
rest, err := artifact.NewREST(c.ResultsURL, token, c.Transport)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rest.Logf = c.Infof
|
||||
return &source{
|
||||
list: func(ctx context.Context, name string) ([]artifact.Artifact, error) {
|
||||
return rest.List(ctx, owner, repo, int64(runID), name)
|
||||
},
|
||||
signedURL: func(ctx context.Context, art artifact.Artifact) (string, error) {
|
||||
return rest.DownloadURL(ctx, owner, repo, art.ID)
|
||||
},
|
||||
download: rest.Download,
|
||||
}, nil
|
||||
}
|
||||
@@ -0,0 +1,239 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
// Package glob ports the pattern semantics of the JavaScript @actions/glob 0.6.1 package, as
|
||||
// bundled in actions/upload-artifact, without walking a filesystem. Malformed bracket expressions
|
||||
// and some non-ASCII case folding on Windows can match differently.
|
||||
package glob
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"path"
|
||||
"strings"
|
||||
"sync"
|
||||
)
|
||||
|
||||
type Options struct {
|
||||
Workdir string // absolute; relative patterns resolve against it
|
||||
Home func() (string, error) // replaces ~, called once and only when a pattern uses it
|
||||
Windows bool // Windows path flavour
|
||||
ImplicitDescendants bool
|
||||
}
|
||||
|
||||
type MatchKind int
|
||||
|
||||
const (
|
||||
MatchNone MatchKind = 0
|
||||
MatchDirectory MatchKind = 1
|
||||
MatchFile MatchKind = 2
|
||||
MatchAll = MatchDirectory | MatchFile
|
||||
)
|
||||
|
||||
type Globber struct {
|
||||
patterns []*pattern
|
||||
searchPaths []string
|
||||
}
|
||||
|
||||
// Parse parses newline separated patterns like DefaultGlobber.create.
|
||||
func Parse(patterns string, o Options) (*Globber, error) {
|
||||
w := flavour{o.Windows}
|
||||
env := parseEnv{w: w, cwd: w.normalizeSeparators(o.Workdir)}
|
||||
if o.Home != nil {
|
||||
env.home = sync.OnceValues(o.Home)
|
||||
}
|
||||
if env.cwd == "" || !w.hasAbsoluteRoot(env.cwd) {
|
||||
return nil, fmt.Errorf("workdir '%s' is not an absolute path", o.Workdir)
|
||||
}
|
||||
if o.Windows {
|
||||
patterns = strings.ReplaceAll(patterns, "\r\n", "\n")
|
||||
patterns = strings.ReplaceAll(patterns, "\r", "\n")
|
||||
}
|
||||
|
||||
var parsed []*pattern
|
||||
for line := range strings.SplitSeq(patterns, "\n") {
|
||||
line = jsTrim(line)
|
||||
if line == "" || strings.HasPrefix(line, "#") {
|
||||
continue
|
||||
}
|
||||
p, err := newPattern(line, false, env)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
parsed = append(parsed, p)
|
||||
}
|
||||
|
||||
g := &Globber{searchPaths: searchPaths(parsed, w)}
|
||||
for _, p := range parsed {
|
||||
g.patterns = append(g.patterns, p)
|
||||
if o.ImplicitDescendants && (p.trailingSeparator || p.segments[len(p.segments)-1] != "**") {
|
||||
s := jsTrim(w.joinSegments(append(append([]string(nil), p.segments...), "**")))
|
||||
if p.negate {
|
||||
s = "!" + s
|
||||
}
|
||||
implicit, err := newPattern(s, true, env)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
g.patterns = append(g.patterns, implicit)
|
||||
}
|
||||
}
|
||||
return g, nil
|
||||
}
|
||||
|
||||
// SearchPaths returns the literal prefix of each include pattern, without duplicates and
|
||||
// paths nested under another search path.
|
||||
func (g *Globber) SearchPaths() []string {
|
||||
return append([]string(nil), g.searchPaths...)
|
||||
}
|
||||
|
||||
// Match ORs in each include pattern and masks out each exclude pattern, in order.
|
||||
func (g *Globber) Match(p string) MatchKind {
|
||||
result := MatchNone
|
||||
for _, pat := range g.patterns {
|
||||
if pat.negate {
|
||||
result &^= pat.match(p)
|
||||
} else {
|
||||
result |= pat.match(p)
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// PartialMatch reports whether an include pattern may match descendants of p.
|
||||
func (g *Globber) PartialMatch(p string) bool {
|
||||
for _, pat := range g.patterns {
|
||||
if !pat.negate && pat.partialMatch(p) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func searchPaths(patterns []*pattern, w flavour) []string {
|
||||
key := func(p *pattern) string {
|
||||
if w.windows {
|
||||
return strings.ToUpper(p.searchPath)
|
||||
}
|
||||
return p.searchPath
|
||||
}
|
||||
const candidate, included = 1, 2
|
||||
seen := map[string]int{}
|
||||
for _, p := range patterns {
|
||||
if !p.negate {
|
||||
seen[key(p)] = candidate
|
||||
}
|
||||
}
|
||||
var result []string
|
||||
for _, p := range patterns {
|
||||
if p.negate {
|
||||
continue
|
||||
}
|
||||
k := key(p)
|
||||
if seen[k] == included {
|
||||
continue
|
||||
}
|
||||
foundAncestor := false
|
||||
for temp, parent := k, w.dirname(k); parent != temp; temp, parent = parent, w.dirname(parent) {
|
||||
if seen[parent] != 0 {
|
||||
foundAncestor = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !foundAncestor {
|
||||
result = append(result, p.searchPath)
|
||||
seen[k] = included
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// LeastCommonAncestor ports getMultiPathLCA from upload-artifact's search.ts, which becomes the
|
||||
// artifact root when there are several search paths. Segments compare case-sensitively there,
|
||||
// even on Windows.
|
||||
func LeastCommonAncestor(paths []string, windows bool) string {
|
||||
switch len(paths) {
|
||||
case 0:
|
||||
return ""
|
||||
case 1:
|
||||
return paths[0]
|
||||
}
|
||||
w := flavour{windows}
|
||||
sep := w.sep()
|
||||
split := make([][]string, len(paths))
|
||||
smallest := -1
|
||||
for i, p := range paths {
|
||||
if w.windows {
|
||||
p = w.normalizeSeparators(p)
|
||||
} else {
|
||||
p = path.Clean(p)
|
||||
}
|
||||
split[i] = strings.Split(p, sep)
|
||||
if smallest < 0 || len(split[i]) < smallest {
|
||||
smallest = len(split[i])
|
||||
}
|
||||
}
|
||||
|
||||
var common []string
|
||||
if strings.HasPrefix(paths[0], sep) {
|
||||
common = append(common, sep)
|
||||
}
|
||||
for i := 0; i < smallest; i++ {
|
||||
same := true
|
||||
for _, s := range split[1:] {
|
||||
if s[i] != split[0][i] {
|
||||
same = false
|
||||
break
|
||||
}
|
||||
}
|
||||
if !same {
|
||||
break
|
||||
}
|
||||
common = append(common, split[0][i])
|
||||
}
|
||||
return w.nodeJoin(common)
|
||||
}
|
||||
|
||||
// nodeJoin ports Node's path.join for the already normalized segments LeastCommonAncestor produces.
|
||||
func (w flavour) nodeJoin(parts []string) string {
|
||||
var nonEmpty []string
|
||||
for _, p := range parts {
|
||||
if p != "" {
|
||||
nonEmpty = append(nonEmpty, p)
|
||||
}
|
||||
}
|
||||
if len(nonEmpty) == 0 {
|
||||
return "."
|
||||
}
|
||||
joined := strings.Join(nonEmpty, w.sep())
|
||||
if !w.windows {
|
||||
return path.Clean(joined)
|
||||
}
|
||||
first := nonEmpty[0]
|
||||
needsReplace, slashCount := true, 0
|
||||
if w.isSep(first[0]) {
|
||||
slashCount++
|
||||
if len(first) > 1 && w.isSep(first[1]) {
|
||||
slashCount++
|
||||
if len(first) > 2 {
|
||||
if w.isSep(first[2]) {
|
||||
slashCount++
|
||||
} else {
|
||||
needsReplace = false
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if needsReplace {
|
||||
for slashCount < len(joined) && w.isSep(joined[slashCount]) {
|
||||
slashCount++
|
||||
}
|
||||
if slashCount >= 2 {
|
||||
joined = `\` + joined[slashCount:]
|
||||
}
|
||||
}
|
||||
// path.win32.normalize turns a bare drive into the drive relative "C:.".
|
||||
if reDriveOnly.MatchString(joined) {
|
||||
return joined + "."
|
||||
}
|
||||
return joined
|
||||
}
|
||||
@@ -0,0 +1,216 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package glob
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
var (
|
||||
posix = Options{Workdir: "/work/cwd-with-[and]", Home: home("/home/home-with-[and]")}
|
||||
windows = Options{Workdir: `C:\work\cwd-with-[and]`, Home: home(`C:\Users\home-with-[and]`), Windows: true}
|
||||
)
|
||||
|
||||
func home(dir string) func() (string, error) {
|
||||
return func() (string, error) { return dir, nil }
|
||||
}
|
||||
|
||||
func implicit(o Options) Options {
|
||||
o.ImplicitDescendants = true
|
||||
return o
|
||||
}
|
||||
|
||||
func TestHomeResolvedLazilyOnce(t *testing.T) {
|
||||
calls := 0
|
||||
o := Options{Workdir: "/w", Home: func() (string, error) { calls++; return "/h", nil }}
|
||||
_, err := Parse("~/a\n! ~/b", o)
|
||||
require.NoError(t, err)
|
||||
_, err = Parse("/a", o)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 1, calls)
|
||||
}
|
||||
|
||||
func TestMatch(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
patterns string
|
||||
opts Options
|
||||
want map[string]MatchKind
|
||||
}{
|
||||
{"negate markers toggle", "/a/*.txt\n!!/a/two.txt\n!/a/one.txt\n!!!/a/three.txt", posix, map[string]MatchKind{
|
||||
"/a/two.txt": MatchAll, "/a/one.txt": MatchNone, "/a/three.txt": MatchNone, "/a/x.txt": MatchAll,
|
||||
}},
|
||||
{"home is escaped", "~/m*", posix, map[string]MatchKind{
|
||||
"/home/home-with-[and]/match": MatchAll, "/home/home-with-[and]/not-match": MatchNone, "/home/home-with-a/match": MatchNone,
|
||||
}},
|
||||
{"workdir is escaped", "m*", posix, map[string]MatchKind{
|
||||
"/work/cwd-with-[and]/match": MatchAll, "/work/cwd-with-[and]/not-match": MatchNone,
|
||||
}},
|
||||
{"workdir is escaped on windows", "m*\nC:n*\n" + `\work\cwd-with-[[]and]\o*`, windows, map[string]MatchKind{
|
||||
`C:\work\cwd-with-[and]\match`: MatchAll, `C:\work\cwd-with-[and]\nope`: MatchAll, `C:\work\cwd-with-[and]\ok`: MatchAll, `C:\work\cwd-with-[and]\x`: MatchNone,
|
||||
}},
|
||||
{"globstar matches the directory before it", "/foo/bar/**", posix, map[string]MatchKind{
|
||||
"/": MatchNone, "/foo": MatchNone, "/foo/bar": MatchAll, "/foo/bar/baz": MatchAll,
|
||||
}},
|
||||
{"globstar matches root", "/**", posix, map[string]MatchKind{"/": MatchAll}},
|
||||
{"globstar matches root on windows", `C:\**`, windows, map[string]MatchKind{`C:\`: MatchAll}},
|
||||
{"case sensitive on posix", "/Foo/**/Baz", posix, map[string]MatchKind{
|
||||
"/Foo/Baz": MatchAll, "/Foo/bAZ": MatchNone, "/fOO/bar/Baz": MatchNone,
|
||||
}},
|
||||
{"case insensitive on windows", "C:/Foo/**/Baz", windows, map[string]MatchKind{
|
||||
`C:\Foo\Baz`: MatchAll, `c:\Foo\bAZ`: MatchAll, "C:/fOO/bar/bAZ": MatchAll, `C:\Foo\Bar`: MatchNone,
|
||||
}},
|
||||
{"leading dot segment", ".\n./foo\n.bar", posix, map[string]MatchKind{
|
||||
"/work/cwd-with-[and]": MatchAll, "/work/cwd-with-[and]/foo": MatchAll, "/work/cwd-with-[and]/.bar": MatchAll,
|
||||
"/work/cwd-with-[and]/bar": MatchNone, "/work/cwd-with-[and]bar": MatchNone,
|
||||
}},
|
||||
{"leading tilde segment", "~\n~/foo\n~bar", posix, map[string]MatchKind{
|
||||
"/home/home-with-[and]": MatchAll, "/home/home-with-[and]/foo": MatchAll, "/work/cwd-with-[and]/~bar": MatchAll,
|
||||
"/home/home-with-[and]/bar": MatchNone, "/home/home-with-[and]bar": MatchNone,
|
||||
}},
|
||||
{"windows relative roots", "C:\nC:foo\nD:\nD:bar\n" + `\work\cwd-with-[[]and]\baz`, windows, map[string]MatchKind{
|
||||
`C:\work\cwd-with-[and]`: MatchAll, `C:\work\cwd-with-[and]\foo`: MatchAll, `C:\foo`: MatchNone,
|
||||
`D:\`: MatchDirectory, `D:\bar`: MatchAll, `D:\foo`: MatchNone, `C:\work\cwd-with-[and]\baz`: MatchAll,
|
||||
}},
|
||||
{"windows unc", `\\srv\share\a\*`, windows, map[string]MatchKind{`\\srv\share\a\b`: MatchAll, `//SRV/share/a/b`: MatchAll, `\\srv\other\a\b`: MatchNone}},
|
||||
{"trailing separator matches directories only", "/foo/**/", posix, map[string]MatchKind{
|
||||
"/": MatchNone, "/foo/": MatchDirectory, "/foo/bar": MatchDirectory, "/foo/bar/baz": MatchDirectory,
|
||||
}},
|
||||
{"backslash is a separator on windows", `C:\foo\`, windows, map[string]MatchKind{`C:\foo`: MatchDirectory}},
|
||||
{"backslash escapes on posix", `/foo\*/b\[a]r`, posix, map[string]MatchKind{"/foo*/b[a]r": MatchAll, "/fooo/bar": MatchNone}},
|
||||
{"interleaved excludes", "/**/*.proj\n/**/README.txt\n!/**/solution2/**\n/**/*.sln\n!/**/proj2/README.txt", posix, map[string]MatchKind{
|
||||
"/solution1/proj1/proj1.proj": MatchAll, "/solution1/proj1/README.txt": MatchAll, "/solution1/proj2/README.txt": MatchNone,
|
||||
"/solution1/solution1.sln": MatchAll, "/solution2/proj1/proj1.proj": MatchNone, "/solution2/solution2.sln": MatchAll,
|
||||
}},
|
||||
{"exclude directories", "C:/foo/**\n!C:/foo/**/", windows, map[string]MatchKind{
|
||||
`C:\`: MatchNone, `C:\foo`: MatchFile, `C:\foo\bar`: MatchFile,
|
||||
}},
|
||||
{"dot is on, brace and extglob are off", "/a/*\n/b/{x,y}\n/c/+(x)\n/d/#e", posix, map[string]MatchKind{
|
||||
"/a/.hidden": MatchAll, "/a/.": MatchNone, "/b/{x,y}": MatchAll, "/b/x": MatchNone, "/c/+(x)": MatchAll, "/c/x": MatchNone, "/d/#e": MatchAll,
|
||||
}},
|
||||
{"character classes", "/a/[!x]\n/b/[a-c]z\n/c/[z-a]\n/d/[]]", posix, map[string]MatchKind{
|
||||
"/a/y": MatchAll, "/a/x": MatchNone, "/b/bz": MatchAll, "/b/dz": MatchNone, "/c/[z-a]": MatchAll, "/c/z": MatchNone, "/d/]": MatchAll,
|
||||
}},
|
||||
{"unclosed class splices undefined like minimatch 3", "C:/foo/b[\\!]r/b*", windows, map[string]MatchKind{
|
||||
"C:/foo/b[undefined/!]r/baz": MatchAll,
|
||||
}},
|
||||
{"implicit descendants", "/foo\n/bar/**\n/dir/\n!/foo/skip", implicit(posix), map[string]MatchKind{
|
||||
"/foo": MatchAll, "/foo/a/b": MatchAll, "/foo/skip": MatchNone, "/foo/skip/x": MatchNone,
|
||||
"/bar": MatchAll, "/dir": MatchDirectory, "/dir/file": MatchAll,
|
||||
}},
|
||||
{"no implicit descendants", "/foo", posix, map[string]MatchKind{"/foo": MatchAll, "/foo/a": MatchNone}},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
g, err := Parse(tt.patterns, tt.opts)
|
||||
require.NoError(t, err)
|
||||
for p, want := range tt.want {
|
||||
assert.Equal(t, want, g.Match(p), p)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPartialMatch(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
patterns string
|
||||
opts Options
|
||||
want map[string]bool
|
||||
}{
|
||||
{"skips excludes", "/search1/foo/**\n/search2/bar/**\n!/search2/bar/**\n!/search3/baz/**", posix, map[string]bool{
|
||||
"/search1": true, "/search1/foo": true, "/search2": true, "/search2/bar": true, "/search3": false, "/search3/bar": false,
|
||||
}},
|
||||
{"root", "/foo/**", posix, map[string]bool{"/": true}},
|
||||
{"root on windows", "C:\\foo\\**\nd:\\foo\\**", windows, map[string]bool{`c:\`: true, `D:\`: true, `E:\`: false}},
|
||||
{"case", "/Foo/Bar/**/Baz", posix, map[string]bool{"/Foo": true, "/fOO": false}},
|
||||
{"case on windows", "C:/Foo/Bar/**/Baz", windows, map[string]bool{`C:\Foo`: true, `C:\fOO`: true, `C:\Other`: false}},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
g, err := Parse(tt.patterns, tt.opts)
|
||||
require.NoError(t, err)
|
||||
for p, want := range tt.want {
|
||||
assert.Equal(t, want, g.PartialMatch(p), p)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSearchPaths(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
patterns string
|
||||
opts Options
|
||||
want []string
|
||||
}{
|
||||
{"omits excludes", "/search1/foo/**\n/search2/bar/**\n!/search3/baz/**", posix, []string{"/search1/foo", "/search2/bar"}},
|
||||
{
|
||||
"omits descendants", "/search1/foo/**\n/search1/foo/bar/**\n/search2/foo/bar\n/search2/**\n/search3/foo/bar/**\n/search3/foo/bar/**", posix,
|
||||
[]string{"/search1/foo", "/search2", "/search3/foo/bar"},
|
||||
},
|
||||
{
|
||||
"omits descendants on windows", "C:\\Search1\\Foo\\**\nC:\\sEARCH1\\fOO\\bar\\**\nC:\\sEARCH1\\foo\\bar\nC:\\Search2\\**\nC:\\Search3\\Foo\\Bar\\**\nC:\\sEARCH3\\fOO\\bAR\\**", windows,
|
||||
[]string{`C:\Search1\Foo`, `C:\Search2`, `C:\Search3\Foo\Bar`},
|
||||
},
|
||||
{"skips comments and blank lines", "# /nope\n\n /a/* \n", posix, []string{"/a"}},
|
||||
{"splits carriage returns on windows", "C:/a/*\r\nC:/b/*\rC:/c/*", windows, []string{`C:\a`, `C:\b`, `C:\c`}},
|
||||
{
|
||||
"unescapes literal segments", "/foo/b[a]r/b*\n/x/b[*]r/b*\n/y/b[!]r/b*\n/z/b[]r/b*\n/q/b[aA]r/b*\n/e/b\\[a]r/b*\n/f/b[\\]]r/b*", posix,
|
||||
[]string{"/foo/bar", "/x/b*r", "/y/b!r", "/z/b[]r", "/q", "/e/b[a]r", "/f/b]r"},
|
||||
},
|
||||
{"unescapes literal segments on windows", "C:/foo/b\\[a]r/b*\nC:/x/b[?]r/b*", windows, []string{`C:\foo\b\ar`, `C:\x\b?r`}},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
g, err := Parse(tt.patterns, tt.opts)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, tt.want, g.SearchPaths())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseErrors(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
patterns string
|
||||
opts Options
|
||||
}{
|
||||
{"empty after negation", "!", posix},
|
||||
{"dot segment", "/a/./b", posix},
|
||||
{"dot dot segment", "a/../b", posix},
|
||||
{"glob in unc root", `\\srv\sh*re\a`, windows},
|
||||
{"tilde without home", "~/a", Options{Workdir: "/w"}},
|
||||
{"relative workdir", "a", Options{Workdir: "w"}},
|
||||
{"drive relative with unc workdir", "C:a", Options{Workdir: `\\srv\share`, Windows: true}},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
_, err := Parse(tt.patterns, tt.opts)
|
||||
assert.Error(t, err)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLeastCommonAncestor(t *testing.T) {
|
||||
tests := []struct {
|
||||
paths []string
|
||||
windows bool
|
||||
want string
|
||||
}{
|
||||
{[]string{"/a/b/c", "/a/b/d", "/a/b"}, false, "/a/b"},
|
||||
{[]string{"/foo", "/bar"}, false, "/"},
|
||||
{[]string{"/a"}, false, "/a"},
|
||||
{[]string{`C:\a\b`, `C:\a\c`}, true, `C:\a`},
|
||||
{[]string{`C:\a`, `c:\a\b`}, true, "."},
|
||||
{[]string{`C:\a`, `C:\b`}, true, "C:."},
|
||||
{[]string{`\\srv\share\a`, `\\srv\share\b`}, true, `\srv\share`},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
assert.Equal(t, tt.want, LeastCommonAncestor(tt.paths, tt.windows), tt.paths)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,534 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package glob
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"slices"
|
||||
"unicode"
|
||||
"unicode/utf16"
|
||||
)
|
||||
|
||||
// This ports the subset of minimatch 3.1.5 that @actions/glob 0.6.1 uses: options dot, nobrace,
|
||||
// noext, nocomment and nonegate, with nocase on Windows. Like JS regexps without the u flag it
|
||||
// works on UTF-16 code units.
|
||||
|
||||
const maxGlobstarRecursion = 200
|
||||
|
||||
type tokKind uint8
|
||||
|
||||
const (
|
||||
tokLit tokKind = iota
|
||||
tokAny // [^/]
|
||||
tokStar // [^/]*?
|
||||
tokClass // [...]
|
||||
tokNonEmpty // (?=.)
|
||||
tokNotDot // (?!(?:^|\/)\.{1,2}(?:$|\/))
|
||||
)
|
||||
|
||||
type unitRange struct{ lo, hi uint16 }
|
||||
|
||||
type tok struct {
|
||||
kind tokKind
|
||||
ch uint16
|
||||
neg bool
|
||||
ranges []unitRange
|
||||
}
|
||||
|
||||
type mmSegment struct {
|
||||
globstar bool
|
||||
magic bool
|
||||
nocase bool
|
||||
literal string
|
||||
toks []tok
|
||||
}
|
||||
|
||||
type classItem struct {
|
||||
ch uint16
|
||||
escaped bool
|
||||
}
|
||||
|
||||
func isReSpecial(c uint16) bool {
|
||||
switch c {
|
||||
case '(', ')', '.', '*', '{', '}', '+', '?', '[', ']', '^', '$', '\\', '!':
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func compileMinimatch(pattern string, nocase bool) ([]mmSegment, error) {
|
||||
if len(utf16.Encode([]rune(pattern))) > 64*1024 {
|
||||
return nil, errors.New("pattern is too long")
|
||||
}
|
||||
parts := splitRuns(jsTrim(pattern), '/')
|
||||
segs := make([]mmSegment, len(parts))
|
||||
for i, part := range parts {
|
||||
segs[i] = compileSegment(part, nocase)
|
||||
}
|
||||
return segs, nil
|
||||
}
|
||||
|
||||
func compileSegment(s string, nocase bool) mmSegment {
|
||||
if s == "**" {
|
||||
return mmSegment{globstar: true}
|
||||
}
|
||||
if s == "" {
|
||||
return mmSegment{}
|
||||
}
|
||||
p := utf16.Encode([]rune(s))
|
||||
toks, magic := parseSegment(p, nocase)
|
||||
if !magic {
|
||||
return mmSegment{literal: globUnescape(s)}
|
||||
}
|
||||
return mmSegment{magic: true, nocase: nocase, toks: toks}
|
||||
}
|
||||
|
||||
// globUnescape replaces \x with x like s.replace(/\\(.)/g, '$1'); JS . excludes line terminators.
|
||||
func globUnescape(s string) string {
|
||||
u := utf16.Encode([]rune(s))
|
||||
out := make([]uint16, 0, len(u))
|
||||
for i := 0; i < len(u); i++ {
|
||||
if u[i] == '\\' && i+1 < len(u) && !isLineTerminator(u[i+1]) {
|
||||
i++
|
||||
}
|
||||
out = append(out, u[i])
|
||||
}
|
||||
return string(utf16.Decode(out))
|
||||
}
|
||||
|
||||
// parseSegment ports minimatch's parse(), emitting tokens instead of a regexp source.
|
||||
func parseSegment(p []uint16, nocase bool) (toks []tok, hasMagic bool) {
|
||||
hasMagic = nocase
|
||||
escaping := false
|
||||
var stateChar uint16
|
||||
inClass := false
|
||||
classStart := -1
|
||||
var class []classItem
|
||||
lit := func(c uint16) { toks = append(toks, tok{kind: tokLit, ch: c}) }
|
||||
clearStateChar := func() {
|
||||
switch stateChar {
|
||||
case 0:
|
||||
return
|
||||
case '*':
|
||||
toks = append(toks, tok{kind: tokStar})
|
||||
hasMagic = true
|
||||
case '?':
|
||||
toks = append(toks, tok{kind: tokAny})
|
||||
hasMagic = true
|
||||
default:
|
||||
lit(stateChar)
|
||||
}
|
||||
stateChar = 0
|
||||
}
|
||||
fallback := func(cs []uint16, closing bool) {
|
||||
lit('[')
|
||||
// parse() returns a non-array for "" and "**", so minimatch splices in the text "undefined".
|
||||
if len(cs) == 0 || (len(cs) == 2 && cs[0] == '*' && cs[1] == '*') {
|
||||
for _, c := range "undefined" {
|
||||
lit(uint16(c))
|
||||
}
|
||||
if closing {
|
||||
lit(']')
|
||||
}
|
||||
inClass = false
|
||||
return
|
||||
}
|
||||
subToks, subMagic := parseSegment(cs, nocase)
|
||||
toks = append(toks, subToks...)
|
||||
if closing {
|
||||
lit(']')
|
||||
}
|
||||
hasMagic = hasMagic || subMagic
|
||||
inClass = false
|
||||
}
|
||||
|
||||
for i := range p {
|
||||
c := p[i]
|
||||
if escaping && isReSpecial(c) {
|
||||
if inClass {
|
||||
class = append(class, classItem{c, true})
|
||||
} else {
|
||||
lit(c)
|
||||
}
|
||||
escaping = false
|
||||
continue
|
||||
}
|
||||
switch c {
|
||||
case '\\':
|
||||
clearStateChar()
|
||||
escaping = true
|
||||
case '?', '*', '+', '@', '!':
|
||||
if inClass {
|
||||
if c == '!' && i == classStart+1 {
|
||||
c = '^'
|
||||
}
|
||||
class = append(class, classItem{c, false})
|
||||
continue
|
||||
}
|
||||
clearStateChar()
|
||||
stateChar = c
|
||||
clearStateChar() // noext
|
||||
case '(':
|
||||
if inClass {
|
||||
class = append(class, classItem{c, false})
|
||||
} else {
|
||||
lit(c)
|
||||
}
|
||||
case ')':
|
||||
if inClass {
|
||||
class = append(class, classItem{c, true})
|
||||
} else {
|
||||
lit(c)
|
||||
}
|
||||
case '|':
|
||||
if inClass {
|
||||
class = append(class, classItem{c, true})
|
||||
} else {
|
||||
lit(c)
|
||||
}
|
||||
escaping = false
|
||||
case '[':
|
||||
clearStateChar()
|
||||
if inClass {
|
||||
class = append(class, classItem{c, true})
|
||||
continue
|
||||
}
|
||||
inClass = true
|
||||
classStart = i
|
||||
class = class[:0]
|
||||
case ']':
|
||||
if i == classStart+1 || !inClass {
|
||||
if inClass {
|
||||
class = append(class, classItem{c, true})
|
||||
} else {
|
||||
lit(c)
|
||||
}
|
||||
escaping = false
|
||||
continue
|
||||
}
|
||||
t, ok := buildClass(class)
|
||||
if !ok {
|
||||
fallback(p[classStart+1:i], true)
|
||||
continue
|
||||
}
|
||||
toks = append(toks, t)
|
||||
hasMagic = true
|
||||
inClass = false
|
||||
default:
|
||||
clearStateChar()
|
||||
if inClass {
|
||||
class = append(class, classItem{c, !escaping && isReSpecial(c) && c != '^'})
|
||||
} else {
|
||||
lit(c)
|
||||
}
|
||||
escaping = false
|
||||
}
|
||||
}
|
||||
|
||||
if inClass {
|
||||
fallback(p[classStart+1:], false)
|
||||
}
|
||||
clearStateChar()
|
||||
if escaping {
|
||||
lit('\\')
|
||||
}
|
||||
|
||||
addPatternStart := len(toks) > 0 && (toks[0].kind == tokStar || toks[0].kind == tokAny || toks[0].kind == tokClass)
|
||||
if len(toks) > 0 && hasMagic {
|
||||
toks = append([]tok{{kind: tokNonEmpty}}, toks...)
|
||||
}
|
||||
if addPatternStart && p[0] != '.' {
|
||||
toks = append([]tok{{kind: tokNotDot}}, toks...)
|
||||
}
|
||||
return toks, hasMagic
|
||||
}
|
||||
|
||||
// buildClass turns the class body minimatch emits into ranges, ok is false for an out of order range like [z-a], which matches literally.
|
||||
func buildClass(items []classItem) (tok, bool) {
|
||||
t := tok{kind: tokClass}
|
||||
if len(items) > 0 && items[0].ch == '^' && !items[0].escaped {
|
||||
t.neg = true
|
||||
items = items[1:]
|
||||
}
|
||||
for i := 0; i < len(items); {
|
||||
lo := items[i].ch
|
||||
if i+2 < len(items) && items[i+1].ch == '-' && !items[i+1].escaped {
|
||||
hi := items[i+2].ch
|
||||
if lo > hi {
|
||||
return t, false
|
||||
}
|
||||
t.ranges = append(t.ranges, unitRange{lo, hi})
|
||||
i += 3
|
||||
continue
|
||||
}
|
||||
t.ranges = append(t.ranges, unitRange{lo, lo})
|
||||
i++
|
||||
}
|
||||
return t, true
|
||||
}
|
||||
|
||||
func isLineTerminator(c uint16) bool {
|
||||
return c == '\n' || c == '\r' || c == 0x2028 || c == 0x2029
|
||||
}
|
||||
|
||||
// canonicalize ports the ES Canonicalize() used by non-unicode case-insensitive regexps.
|
||||
func canonicalize(c uint16) uint16 {
|
||||
if utf16.IsSurrogate(rune(c)) {
|
||||
return c
|
||||
}
|
||||
up := unicode.ToUpper(rune(c))
|
||||
if up > 0xffff || (c >= 128 && up < 128) {
|
||||
return c
|
||||
}
|
||||
return uint16(up)
|
||||
}
|
||||
|
||||
func (t *tok) classMatch(c uint16, nocase bool) bool {
|
||||
in := func(v uint16) bool {
|
||||
for _, r := range t.ranges {
|
||||
if v >= r.lo && v <= r.hi {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
hit := in(c)
|
||||
if !hit && nocase && !utf16.IsSurrogate(rune(c)) {
|
||||
want := canonicalize(c)
|
||||
for f := unicode.SimpleFold(rune(c)); f != rune(c); f = unicode.SimpleFold(f) {
|
||||
if f <= 0xffff && canonicalize(uint16(f)) == want && in(uint16(f)) {
|
||||
hit = true
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
return hit != t.neg
|
||||
}
|
||||
|
||||
func (s *mmSegment) match(f string) bool {
|
||||
if !s.magic {
|
||||
return f == s.literal
|
||||
}
|
||||
u := utf16.Encode([]rune(f))
|
||||
n := len(u) + 1
|
||||
memo := make([]int8, (len(s.toks)+1)*n)
|
||||
var rec func(ti, pos int) bool
|
||||
rec = func(ti, pos int) bool {
|
||||
if ti == len(s.toks) {
|
||||
return pos == len(u)
|
||||
}
|
||||
key := ti*n + pos
|
||||
if memo[key] != 0 {
|
||||
return memo[key] > 0
|
||||
}
|
||||
t := &s.toks[ti]
|
||||
hit := false
|
||||
switch t.kind {
|
||||
case tokLit:
|
||||
hit = pos < len(u) && (u[pos] == t.ch || (s.nocase && canonicalize(u[pos]) == canonicalize(t.ch))) && rec(ti+1, pos+1)
|
||||
case tokAny:
|
||||
hit = pos < len(u) && u[pos] != '/' && rec(ti+1, pos+1)
|
||||
case tokClass:
|
||||
hit = pos < len(u) && t.classMatch(u[pos], s.nocase) && rec(ti+1, pos+1)
|
||||
case tokStar:
|
||||
for j := pos; !hit; j++ {
|
||||
hit = rec(ti+1, j)
|
||||
if j == len(u) || u[j] == '/' {
|
||||
break
|
||||
}
|
||||
}
|
||||
case tokNonEmpty:
|
||||
hit = pos < len(u) && !isLineTerminator(u[pos]) && rec(ti+1, pos)
|
||||
case tokNotDot:
|
||||
hit = !dotSegmentAhead(u, pos) && rec(ti+1, pos)
|
||||
}
|
||||
memo[key] = -1
|
||||
if hit {
|
||||
memo[key] = 1
|
||||
}
|
||||
return hit
|
||||
}
|
||||
return rec(0, 0)
|
||||
}
|
||||
|
||||
// dotSegmentAhead evaluates (?:^|\/)\.{1,2}(?:$|\/) at pos.
|
||||
func dotSegmentAhead(u []uint16, pos int) bool {
|
||||
i := pos
|
||||
if pos != 0 {
|
||||
if i >= len(u) || u[i] != '/' {
|
||||
return false
|
||||
}
|
||||
i++
|
||||
}
|
||||
for dots := 0; dots < 2 && i < len(u) && u[i] == '.'; dots++ {
|
||||
i++
|
||||
if i == len(u) || u[i] == '/' {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// matchOne ports Minimatch.prototype.matchOne for the single pattern set nobrace produces.
|
||||
func matchOne(file []string, pattern []mmSegment, partial bool) bool {
|
||||
for i := range pattern {
|
||||
if pattern[i].globstar {
|
||||
return matchGlobstar(file, pattern, partial, 0, 0)
|
||||
}
|
||||
}
|
||||
return matchOneFrom(file, pattern, partial, 0)
|
||||
}
|
||||
|
||||
func clampSlice(s []string, from, to int) []string {
|
||||
to = min(to, len(s))
|
||||
from = min(from, to)
|
||||
return s[from:to]
|
||||
}
|
||||
|
||||
func matchOneFrom(file []string, pattern []mmSegment, partial bool, fi int) bool {
|
||||
fl, pl, pi := len(file), len(pattern), 0
|
||||
for ; fi < fl && pi < pl; fi, pi = fi+1, pi+1 {
|
||||
if pattern[pi].globstar || !pattern[pi].match(file[fi]) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case fi == fl && pi == pl:
|
||||
return true
|
||||
case fi == fl:
|
||||
return partial
|
||||
case pi == pl:
|
||||
return fi == fl-1 && file[fi] == ""
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func isDotOrDotDot(f string) bool {
|
||||
return f == "." || f == ".."
|
||||
}
|
||||
|
||||
func matchGlobstar(file []string, pattern []mmSegment, partial bool, fileIndex, patternIndex int) bool {
|
||||
firstgs, lastgs := -1, -1
|
||||
for i := patternIndex; i < len(pattern); i++ {
|
||||
if pattern[i].globstar {
|
||||
firstgs = i
|
||||
break
|
||||
}
|
||||
}
|
||||
for i, seg := range slices.Backward(pattern) {
|
||||
if seg.globstar {
|
||||
lastgs = i
|
||||
break
|
||||
}
|
||||
}
|
||||
head := pattern[patternIndex:firstgs]
|
||||
var body, tail []mmSegment
|
||||
if partial {
|
||||
body = pattern[firstgs+1:]
|
||||
} else {
|
||||
body = pattern[firstgs+1 : max(lastgs, firstgs+1)]
|
||||
tail = pattern[lastgs+1:]
|
||||
}
|
||||
|
||||
if len(head) > 0 {
|
||||
if !matchOneFrom(clampSlice(file, fileIndex, fileIndex+len(head)), head, partial, 0) {
|
||||
return false
|
||||
}
|
||||
fileIndex += len(head)
|
||||
}
|
||||
|
||||
fileTailMatch := 0
|
||||
if len(tail) > 0 {
|
||||
if len(tail)+fileIndex > len(file) {
|
||||
return false
|
||||
}
|
||||
tailStart := len(file) - len(tail)
|
||||
if matchOneFrom(file, tail, partial, tailStart) {
|
||||
fileTailMatch = len(tail)
|
||||
} else {
|
||||
// a/**/* matching a/b/
|
||||
if file[len(file)-1] != "" || fileIndex+len(tail) == len(file) {
|
||||
return false
|
||||
}
|
||||
tailStart--
|
||||
if !matchOneFrom(file, tail, partial, tailStart) {
|
||||
return false
|
||||
}
|
||||
fileTailMatch = len(tail) + 1
|
||||
}
|
||||
}
|
||||
|
||||
if len(body) == 0 {
|
||||
sawSome := fileTailMatch > 0
|
||||
for i := fileIndex; i < len(file)-fileTailMatch; i++ {
|
||||
sawSome = true
|
||||
if isDotOrDotDot(file[i]) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return partial || sawSome
|
||||
}
|
||||
|
||||
type section struct {
|
||||
body []mmSegment
|
||||
after int
|
||||
}
|
||||
sections := []section{{}}
|
||||
nonGsParts := 0
|
||||
nonGsPartsSums := []int{0}
|
||||
for _, b := range body {
|
||||
if b.globstar {
|
||||
nonGsPartsSums = append(nonGsPartsSums, nonGsParts)
|
||||
sections = append(sections, section{})
|
||||
} else {
|
||||
sections[len(sections)-1].body = append(sections[len(sections)-1].body, b)
|
||||
nonGsParts++
|
||||
}
|
||||
}
|
||||
idx := len(sections) - 1
|
||||
fileLength := len(file) - fileTailMatch
|
||||
for si := range sections {
|
||||
sections[si].after = fileLength - (nonGsPartsSums[idx] + len(sections[si].body))
|
||||
idx--
|
||||
}
|
||||
|
||||
// minimatch returns false (no match), null (no match, stop trying) or true.
|
||||
const (
|
||||
triFalse = iota
|
||||
triNull
|
||||
triTrue
|
||||
)
|
||||
var bodySections func(fileIndex, bodyIndex, depth int, sawTail bool) int
|
||||
bodySections = func(fileIndex, bodyIndex, depth int, sawTail bool) int {
|
||||
if bodyIndex >= len(sections) {
|
||||
for i := fileIndex; i < len(file); i++ {
|
||||
sawTail = true
|
||||
if isDotOrDotDot(file[i]) {
|
||||
return triFalse
|
||||
}
|
||||
}
|
||||
if sawTail {
|
||||
return triTrue
|
||||
}
|
||||
return triFalse
|
||||
}
|
||||
bs := sections[bodyIndex]
|
||||
for ; fileIndex <= bs.after; fileIndex++ {
|
||||
m := matchOneFrom(clampSlice(file, 0, fileIndex+len(bs.body)), bs.body, partial, fileIndex)
|
||||
if m && depth < maxGlobstarRecursion {
|
||||
if sub := bodySections(fileIndex+len(bs.body), bodyIndex+1, depth+1, sawTail); sub != triFalse {
|
||||
return sub
|
||||
}
|
||||
}
|
||||
if fileIndex < len(file) && isDotOrDotDot(file[fileIndex]) {
|
||||
return triFalse
|
||||
}
|
||||
}
|
||||
if partial {
|
||||
return triTrue
|
||||
}
|
||||
return triNull
|
||||
}
|
||||
return bodySections(fileIndex, 0, 0, fileTailMatch > 0) == triTrue
|
||||
}
|
||||
@@ -0,0 +1,284 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package glob
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// flavour selects Windows or POSIX path semantics; the helpers port @actions/glob internal-path-helper.ts.
|
||||
type flavour struct{ windows bool }
|
||||
|
||||
var (
|
||||
reUNCPrefix = regexp.MustCompile(`^\\\\+[^\\]`)
|
||||
reBackslashRun = regexp.MustCompile(`\\\\+`)
|
||||
reSlashRun = regexp.MustCompile(`//+`)
|
||||
reDriveRoot = regexp.MustCompile(`^[A-Za-z]:\\$`)
|
||||
reDriveAbs = regexp.MustCompile(`^[A-Za-z]:\\`)
|
||||
reDrive = regexp.MustCompile(`^[A-Za-z]:`)
|
||||
reDriveOnly = regexp.MustCompile(`^[A-Za-z]:$`)
|
||||
reDriveRel = regexp.MustCompile(`^[A-Za-z]:[^\\/]|^[A-Za-z]:$`)
|
||||
reRelRoot = regexp.MustCompile(`^\\$|^\\[^\\]`)
|
||||
reRelRootOnly = regexp.MustCompile(`^\\[^\\]`)
|
||||
reDriveNoSep = regexp.MustCompile(`^[A-Za-z]:[^\\]`)
|
||||
reUNCRoot = regexp.MustCompile(`^\\\\[^\\]+(\\[^\\]+)?$`)
|
||||
reUNCRootSlash = regexp.MustCompile(`^\\\\[^\\]+\\[^\\]+\\$`)
|
||||
)
|
||||
|
||||
func (w flavour) sep() string {
|
||||
if w.windows {
|
||||
return `\`
|
||||
}
|
||||
return "/"
|
||||
}
|
||||
|
||||
func (w flavour) isSep(c byte) bool {
|
||||
return c == '/' || (w.windows && c == '\\')
|
||||
}
|
||||
|
||||
func (w flavour) normalizeSeparators(p string) string {
|
||||
if w.windows {
|
||||
p = strings.ReplaceAll(p, "/", `\`)
|
||||
prefix := ""
|
||||
if reUNCPrefix.MatchString(p) {
|
||||
prefix = `\`
|
||||
}
|
||||
return prefix + reBackslashRun.ReplaceAllLiteralString(p, `\`)
|
||||
}
|
||||
return reSlashRun.ReplaceAllLiteralString(p, "/")
|
||||
}
|
||||
|
||||
func (w flavour) safeTrimTrailingSeparator(p string) string {
|
||||
if p == "" {
|
||||
return ""
|
||||
}
|
||||
p = w.normalizeSeparators(p)
|
||||
if !strings.HasSuffix(p, w.sep()) || p == w.sep() || (w.windows && reDriveRoot.MatchString(p)) {
|
||||
return p
|
||||
}
|
||||
return p[:len(p)-1]
|
||||
}
|
||||
|
||||
func (w flavour) hasAbsoluteRoot(p string) bool {
|
||||
p = w.normalizeSeparators(p)
|
||||
if w.windows {
|
||||
return strings.HasPrefix(p, `\\`) || reDriveAbs.MatchString(p)
|
||||
}
|
||||
return strings.HasPrefix(p, "/")
|
||||
}
|
||||
|
||||
func (w flavour) hasRoot(p string) bool {
|
||||
p = w.normalizeSeparators(p)
|
||||
if w.windows {
|
||||
return strings.HasPrefix(p, `\`) || reDrive.MatchString(p)
|
||||
}
|
||||
return strings.HasPrefix(p, "/")
|
||||
}
|
||||
|
||||
func (w flavour) dirname(p string) string {
|
||||
p = w.safeTrimTrailingSeparator(p)
|
||||
if w.windows && reUNCRoot.MatchString(p) {
|
||||
return p
|
||||
}
|
||||
result := w.nodeDirname(p)
|
||||
if w.windows && reUNCRootSlash.MatchString(result) {
|
||||
result = w.safeTrimTrailingSeparator(result)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// ensureAbsoluteRoot resolves Windows drive and root relative paths against cwd like process.cwd().
|
||||
func (w flavour) ensureAbsoluteRoot(root, itemPath, cwd string) (string, error) {
|
||||
if root == "" || itemPath == "" {
|
||||
return "", errors.New("ensureAbsoluteRoot parameters must not be empty")
|
||||
}
|
||||
if w.hasAbsoluteRoot(itemPath) {
|
||||
return itemPath, nil
|
||||
}
|
||||
if w.windows {
|
||||
if reDriveRel.MatchString(itemPath) {
|
||||
if !reDriveAbs.MatchString(cwd) {
|
||||
return "", errors.New("expected current directory to start with an absolute drive root, actual '" + cwd + "'")
|
||||
}
|
||||
if strings.EqualFold(itemPath[:1], cwd[:1]) {
|
||||
if len(itemPath) == 2 {
|
||||
return itemPath[:1] + `:\` + cwd[3:], nil
|
||||
}
|
||||
if !strings.HasSuffix(cwd, `\`) {
|
||||
cwd += `\`
|
||||
}
|
||||
return itemPath[:1] + `:\` + cwd[3:] + itemPath[2:], nil
|
||||
}
|
||||
return itemPath[:1] + `:\` + itemPath[2:], nil
|
||||
} else if reRelRoot.MatchString(w.normalizeSeparators(itemPath)) {
|
||||
if !reDriveAbs.MatchString(cwd) {
|
||||
return "", errors.New("expected current directory to start with an absolute drive root, actual '" + cwd + "'")
|
||||
}
|
||||
return cwd[:1] + `:\` + itemPath[1:], nil
|
||||
}
|
||||
}
|
||||
if !w.hasAbsoluteRoot(root) {
|
||||
return "", errors.New("ensureAbsoluteRoot parameter 'root' must have an absolute root")
|
||||
}
|
||||
if !strings.HasSuffix(root, "/") && (!w.windows || !strings.HasSuffix(root, `\`)) {
|
||||
root += w.sep()
|
||||
}
|
||||
return root + itemPath, nil
|
||||
}
|
||||
|
||||
// splitPath ports internal-path.ts Path(string).segments.
|
||||
func (w flavour) splitPath(p string) []string {
|
||||
p = w.safeTrimTrailingSeparator(p)
|
||||
if !w.hasRoot(p) {
|
||||
return strings.Split(p, w.sep())
|
||||
}
|
||||
var segments []string
|
||||
remaining := p
|
||||
for dir := w.dirname(remaining); dir != remaining; dir = w.dirname(remaining) {
|
||||
segments = append([]string{w.nodeBasename(remaining)}, segments...)
|
||||
remaining = dir
|
||||
}
|
||||
return append([]string{remaining}, segments...)
|
||||
}
|
||||
|
||||
// joinSegments ports internal-path.ts Path(string[]).toString().
|
||||
func (w flavour) joinSegments(segments []string) string {
|
||||
result := w.normalizeSeparators(segments[0])
|
||||
if w.hasRoot(result) {
|
||||
result = w.safeTrimTrailingSeparator(result)
|
||||
}
|
||||
skipSlash := strings.HasSuffix(result, w.sep()) || (w.windows && reDriveOnly.MatchString(result))
|
||||
for _, s := range segments[1:] {
|
||||
if skipSlash {
|
||||
skipSlash = false
|
||||
} else {
|
||||
result += w.sep()
|
||||
}
|
||||
result += w.normalizeSeparators(s)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func isDeviceRoot(c byte) bool {
|
||||
return (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z')
|
||||
}
|
||||
|
||||
// nodeDirname ports Node's path.posix.dirname and path.win32.dirname.
|
||||
func (w flavour) nodeDirname(p string) string {
|
||||
n := len(p)
|
||||
if n == 0 {
|
||||
return "."
|
||||
}
|
||||
if !w.windows {
|
||||
hasRoot := p[0] == '/'
|
||||
end := -1
|
||||
matchedSlash := true
|
||||
for i := n - 1; i >= 1; i-- {
|
||||
if p[i] == '/' {
|
||||
if !matchedSlash {
|
||||
end = i
|
||||
break
|
||||
}
|
||||
} else {
|
||||
matchedSlash = false
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case end == -1 && hasRoot:
|
||||
return "/"
|
||||
case end == -1:
|
||||
return "."
|
||||
case hasRoot && end == 1:
|
||||
return "//"
|
||||
}
|
||||
return p[:end]
|
||||
}
|
||||
if n == 1 {
|
||||
if w.isSep(p[0]) {
|
||||
return p
|
||||
}
|
||||
return "."
|
||||
}
|
||||
rootEnd, offset := -1, 0
|
||||
if w.isSep(p[0]) {
|
||||
rootEnd, offset = 1, 1
|
||||
if w.isSep(p[1]) {
|
||||
j := 2
|
||||
last := j
|
||||
for j < n && !w.isSep(p[j]) {
|
||||
j++
|
||||
}
|
||||
if j < n && j != last {
|
||||
last = j
|
||||
for j < n && w.isSep(p[j]) {
|
||||
j++
|
||||
}
|
||||
if j < n && j != last {
|
||||
last = j
|
||||
for j < n && !w.isSep(p[j]) {
|
||||
j++
|
||||
}
|
||||
if j == n {
|
||||
return p
|
||||
}
|
||||
if j != last {
|
||||
rootEnd, offset = j+1, j+1
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} else if isDeviceRoot(p[0]) && p[1] == ':' {
|
||||
rootEnd = 2
|
||||
if n > 2 && w.isSep(p[2]) {
|
||||
rootEnd = 3
|
||||
}
|
||||
offset = rootEnd
|
||||
}
|
||||
end := -1
|
||||
matchedSlash := true
|
||||
for i := n - 1; i >= offset; i-- {
|
||||
if w.isSep(p[i]) {
|
||||
if !matchedSlash {
|
||||
end = i
|
||||
break
|
||||
}
|
||||
} else {
|
||||
matchedSlash = false
|
||||
}
|
||||
}
|
||||
if end == -1 {
|
||||
if rootEnd == -1 {
|
||||
return "."
|
||||
}
|
||||
end = rootEnd
|
||||
}
|
||||
return p[:end]
|
||||
}
|
||||
|
||||
// nodeBasename ports Node's path.posix.basename and path.win32.basename without the ext argument.
|
||||
func (w flavour) nodeBasename(p string) string {
|
||||
start := 0
|
||||
if w.windows && len(p) >= 2 && isDeviceRoot(p[0]) && p[1] == ':' {
|
||||
start = 2
|
||||
}
|
||||
from, end := start, -1
|
||||
matchedSlash := true
|
||||
for i := len(p) - 1; i >= start; i-- {
|
||||
if w.isSep(p[i]) {
|
||||
if !matchedSlash {
|
||||
from = i + 1
|
||||
break
|
||||
}
|
||||
} else if end == -1 {
|
||||
matchedSlash = false
|
||||
end = i + 1
|
||||
}
|
||||
}
|
||||
if end == -1 {
|
||||
return ""
|
||||
}
|
||||
return p[from:end]
|
||||
}
|
||||
@@ -0,0 +1,269 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package glob
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"unicode/utf16"
|
||||
)
|
||||
|
||||
// pattern ports @actions/glob internal-pattern.ts.
|
||||
type pattern struct {
|
||||
negate bool
|
||||
searchPath string
|
||||
segments []string
|
||||
trailingSeparator bool
|
||||
implicit bool
|
||||
rootLiteral string
|
||||
mm []mmSegment
|
||||
w flavour
|
||||
}
|
||||
|
||||
type parseEnv struct {
|
||||
w flavour
|
||||
cwd string
|
||||
home func() (string, error)
|
||||
}
|
||||
|
||||
func newPattern(s string, implicit bool, env parseEnv) (*pattern, error) {
|
||||
w := env.w
|
||||
p := &pattern{implicit: implicit, w: w}
|
||||
s = jsTrim(s)
|
||||
for strings.HasPrefix(s, "!") {
|
||||
p.negate = !p.negate
|
||||
s = jsTrim(s[1:])
|
||||
}
|
||||
|
||||
s, err := fixupPattern(s, env)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
p.segments = w.splitPath(s)
|
||||
p.trailingSeparator = strings.HasSuffix(w.normalizeSeparators(s), w.sep())
|
||||
s = w.safeTrimTrailingSeparator(s)
|
||||
|
||||
var searchSegments []string
|
||||
for _, seg := range p.segments {
|
||||
lit := getLiteral(seg, w)
|
||||
if lit == "" {
|
||||
break
|
||||
}
|
||||
searchSegments = append(searchSegments, lit)
|
||||
}
|
||||
p.searchPath = w.joinSegments(searchSegments)
|
||||
p.rootLiteral = searchSegments[0]
|
||||
|
||||
if w.windows {
|
||||
s = strings.ReplaceAll(s, `\`, "/")
|
||||
}
|
||||
p.mm, err = compileMinimatch(s, w.windows)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return p, nil
|
||||
}
|
||||
|
||||
func fixupPattern(s string, env parseEnv) (string, error) {
|
||||
w := env.w
|
||||
if s == "" {
|
||||
return "", errors.New("pattern cannot be empty")
|
||||
}
|
||||
segments := w.splitPath(s)
|
||||
for i, seg := range segments {
|
||||
if lit := getLiteral(seg, w); (lit == "." && i != 0) || lit == ".." {
|
||||
return "", fmt.Errorf("invalid pattern '%s': relative pathing '.' and '..' is not allowed", s)
|
||||
}
|
||||
}
|
||||
if w.hasRoot(s) && getLiteral(segments[0], w) == "" {
|
||||
return "", fmt.Errorf("invalid pattern '%s': root segment must not contain globs", s)
|
||||
}
|
||||
|
||||
s = w.normalizeSeparators(s)
|
||||
sep := w.sep()
|
||||
var root string
|
||||
var err error
|
||||
switch {
|
||||
case s == "." || strings.HasPrefix(s, "."+sep):
|
||||
s = globEscape(env.cwd, w) + s[1:]
|
||||
case s == "~" || strings.HasPrefix(s, "~"+sep):
|
||||
var home string
|
||||
if env.home != nil {
|
||||
if home, err = env.home(); err != nil {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
if home == "" {
|
||||
return "", errors.New("unable to determine HOME directory")
|
||||
}
|
||||
if home = w.normalizeSeparators(home); !w.hasAbsoluteRoot(home) {
|
||||
return "", fmt.Errorf("expected HOME directory to be a rooted path, actual '%s'", home)
|
||||
}
|
||||
s = globEscape(home, w) + s[1:]
|
||||
case w.windows && (reDriveOnly.MatchString(s) || reDriveNoSep.MatchString(s)):
|
||||
if root, err = w.ensureAbsoluteRoot(`C:\dummy-root`, s[:2], env.cwd); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if len(s) > 2 && !strings.HasSuffix(root, `\`) {
|
||||
root += `\`
|
||||
}
|
||||
s = globEscape(root, w) + s[2:]
|
||||
case w.windows && (s == `\` || reRelRootOnly.MatchString(s)):
|
||||
if root, err = w.ensureAbsoluteRoot(`C:\dummy-root`, `\`, env.cwd); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if !strings.HasSuffix(root, `\`) {
|
||||
root += `\`
|
||||
}
|
||||
s = globEscape(root, w) + s[1:]
|
||||
default:
|
||||
if s, err = w.ensureAbsoluteRoot(globEscape(env.cwd, w), s, env.cwd); err != nil {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
return w.normalizeSeparators(s), nil
|
||||
}
|
||||
|
||||
func (p *pattern) match(itemPath string) MatchKind {
|
||||
w := p.w
|
||||
if p.segments[len(p.segments)-1] == "**" {
|
||||
itemPath = w.normalizeSeparators(itemPath)
|
||||
// Without the trailing separator minimatch would not match the directory preceding the globstar.
|
||||
if !strings.HasSuffix(itemPath, w.sep()) && !p.implicit {
|
||||
itemPath += w.sep()
|
||||
}
|
||||
} else {
|
||||
itemPath = w.safeTrimTrailingSeparator(itemPath)
|
||||
}
|
||||
if w.windows {
|
||||
itemPath = strings.ReplaceAll(itemPath, `\`, "/")
|
||||
}
|
||||
if !matchOne(splitRuns(itemPath, '/'), p.mm, false) {
|
||||
return MatchNone
|
||||
}
|
||||
if p.trailingSeparator {
|
||||
return MatchDirectory
|
||||
}
|
||||
return MatchAll
|
||||
}
|
||||
|
||||
func (p *pattern) partialMatch(itemPath string) bool {
|
||||
w := p.w
|
||||
itemPath = w.safeTrimTrailingSeparator(itemPath)
|
||||
// The toolkit tests roots with an unanchored regexp because minimatch mishandles them.
|
||||
if w.dirname(itemPath) == itemPath {
|
||||
if w.windows {
|
||||
return strings.Contains(strings.ToUpper(itemPath), strings.ToUpper(p.rootLiteral))
|
||||
}
|
||||
return strings.Contains(itemPath, p.rootLiteral)
|
||||
}
|
||||
sep := byte('/')
|
||||
if w.windows {
|
||||
sep = '\\'
|
||||
}
|
||||
return matchOne(splitRuns(itemPath, sep), p.mm, true)
|
||||
}
|
||||
|
||||
// globEscape ports Pattern.globEscape.
|
||||
func globEscape(s string, w flavour) string {
|
||||
if !w.windows {
|
||||
s = strings.ReplaceAll(s, `\`, `\\`)
|
||||
}
|
||||
var b strings.Builder
|
||||
for i := 0; i < len(s); i++ {
|
||||
if s[i] == '[' && closesBeforeSlash(s[i+1:]) {
|
||||
b.WriteString("[[]")
|
||||
continue
|
||||
}
|
||||
b.WriteByte(s[i])
|
||||
}
|
||||
s = strings.ReplaceAll(b.String(), "?", "[?]")
|
||||
return strings.ReplaceAll(s, "*", "[*]")
|
||||
}
|
||||
|
||||
// closesBeforeSlash reports whether s matches the lookahead `[^/]+\]`.
|
||||
func closesBeforeSlash(s string) bool {
|
||||
for i := 0; i < len(s) && s[i] != '/'; i++ {
|
||||
if s[i] == ']' && i > 0 {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// getLiteral ports Pattern.getLiteral, returning "" when the segment contains a glob.
|
||||
func getLiteral(segment string, w flavour) string {
|
||||
seg := utf16.Encode([]rune(segment))
|
||||
var lit []uint16
|
||||
for i := 0; i < len(seg); i++ {
|
||||
c := seg[i]
|
||||
switch {
|
||||
case c == '\\' && !w.windows && i+1 < len(seg):
|
||||
i++
|
||||
lit = append(lit, seg[i])
|
||||
continue
|
||||
case c == '*' || c == '?':
|
||||
return ""
|
||||
case c == '[' && i+1 < len(seg):
|
||||
var set []uint16
|
||||
closed := -1
|
||||
for i2 := i + 1; i2 < len(seg); i2++ {
|
||||
c2 := seg[i2]
|
||||
if c2 == '\\' && !w.windows && i2+1 < len(seg) {
|
||||
i2++
|
||||
set = append(set, seg[i2])
|
||||
continue
|
||||
} else if c2 == ']' {
|
||||
closed = i2
|
||||
break
|
||||
}
|
||||
set = append(set, c2)
|
||||
}
|
||||
if closed >= 0 {
|
||||
if len(set) > 1 {
|
||||
return ""
|
||||
}
|
||||
if len(set) == 1 {
|
||||
lit = append(lit, set...)
|
||||
i = closed
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
lit = append(lit, c)
|
||||
}
|
||||
return string(utf16.Decode(lit))
|
||||
}
|
||||
|
||||
func isJSWhitespace(r rune) bool {
|
||||
switch r {
|
||||
case '\t', '\n', '\v', '\f', '\r', ' ', 0xa0, 0x1680, 0x2028, 0x2029, 0x202f, 0x205f, 0x3000, 0xfeff:
|
||||
return true
|
||||
}
|
||||
return r >= 0x2000 && r <= 0x200a
|
||||
}
|
||||
|
||||
// jsTrim matches String.prototype.trim, which differs from strings.TrimSpace on U+0085 and U+FEFF.
|
||||
func jsTrim(s string) string {
|
||||
return strings.TrimFunc(s, isJSWhitespace)
|
||||
}
|
||||
|
||||
// splitRuns splits like JS s.split(/c+/).
|
||||
func splitRuns(s string, c byte) []string {
|
||||
parts := []string{}
|
||||
start := 0
|
||||
for i := 0; i < len(s); {
|
||||
if s[i] != c {
|
||||
i++
|
||||
continue
|
||||
}
|
||||
parts = append(parts, s[start:i])
|
||||
for i < len(s) && s[i] == c {
|
||||
i++
|
||||
}
|
||||
start = i
|
||||
}
|
||||
return append(parts, s[start:])
|
||||
}
|
||||
@@ -0,0 +1,120 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package uploadartifact
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bufio"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path"
|
||||
"strings"
|
||||
|
||||
"gitea.com/gitea/runner/internal/pkg/action"
|
||||
)
|
||||
|
||||
const fetchScript = `cd / && if command -v gzip >/dev/null 2>&1; then exec tar -czhf - -- "$@"; fi; exec tar -chf - -- "$@"`
|
||||
|
||||
func readFiles(ctx context.Context, c *action.Context, files []found, visit func(found) error) error {
|
||||
if c.Host() {
|
||||
for _, file := range files {
|
||||
body, err := os.Open(file.source)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
info, err := body.Stat()
|
||||
if err == nil {
|
||||
file.body, file.mode, file.mtime = body, info.Mode(), info.ModTime()
|
||||
err = visit(file)
|
||||
}
|
||||
body.Close()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
for len(files) > 0 {
|
||||
command := []string{"sh", "-c", fetchScript, "sh"}
|
||||
selected := map[string][]found{}
|
||||
size, count := 0, 0
|
||||
for _, file := range files {
|
||||
name := "./" + strings.TrimPrefix(file.source, "/")
|
||||
if size+len(name)+1 >= 32*1024 {
|
||||
if count == 0 {
|
||||
return fmt.Errorf("artifact path exceeds container command limit: %s", file.path)
|
||||
}
|
||||
break
|
||||
}
|
||||
count++
|
||||
if _, exists := selected[path.Clean(name)]; !exists {
|
||||
command = append(command, name)
|
||||
size += len(name) + 1
|
||||
}
|
||||
selected[path.Clean(name)] = append(selected[path.Clean(name)], file)
|
||||
}
|
||||
output, err := c.Container.ExecOutput(ctx, command)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
retry, err := readArchive(output, selected, visit)
|
||||
closeErr := output.Close()
|
||||
if err != nil {
|
||||
return err
|
||||
} else if closeErr != nil {
|
||||
return closeErr
|
||||
}
|
||||
files = append(retry, files[count:]...)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func readArchive(output io.Reader, selected map[string][]found, visit func(found) error) ([]found, error) {
|
||||
buffered := bufio.NewReader(output)
|
||||
var content io.Reader = buffered
|
||||
if magic, _ := buffered.Peek(2); len(magic) == 2 && magic[0] == 0x1f && magic[1] == 0x8b {
|
||||
compressed, err := gzip.NewReader(buffered)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer compressed.Close()
|
||||
compressed.Multistream(false) // bsdtar pads its gzip output with zeros to the block size
|
||||
content = compressed
|
||||
}
|
||||
reader := tar.NewReader(content)
|
||||
var retry []found
|
||||
for {
|
||||
header, err := reader.Next()
|
||||
if errors.Is(err, io.EOF) {
|
||||
break
|
||||
} else if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
name := path.Clean(header.Name)
|
||||
files := selected[name]
|
||||
delete(selected, name)
|
||||
if header.Typeflag == tar.TypeLink {
|
||||
retry = append(retry, files...)
|
||||
continue
|
||||
}
|
||||
if len(files) == 0 {
|
||||
continue
|
||||
}
|
||||
retry = append(retry, files[1:]...)
|
||||
file := files[0]
|
||||
file.body, file.mode, file.mtime = reader, header.FileInfo().Mode(), header.ModTime
|
||||
if err := visit(file); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
_, err := io.Copy(io.Discard, content)
|
||||
if err == nil {
|
||||
_, err = io.Copy(io.Discard, buffered)
|
||||
}
|
||||
return retry, err
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package uploadartifact
|
||||
|
||||
import (
|
||||
"cmp"
|
||||
"mime"
|
||||
"path"
|
||||
)
|
||||
|
||||
func mimeTypeOf(name string) string {
|
||||
return cmp.Or(mime.TypeByExtension(path.Ext(name)), "application/octet-stream")
|
||||
}
|
||||
@@ -0,0 +1,258 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
// Package uploadartifact works like actions/upload-artifact v7.
|
||||
package uploadartifact
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"cmp"
|
||||
"compress/flate"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.com/gitea/runner/internal/pkg/action"
|
||||
"gitea.com/gitea/runner/internal/pkg/action/artifact"
|
||||
"gitea.com/gitea/runner/internal/pkg/action/glob"
|
||||
)
|
||||
|
||||
type inputs struct {
|
||||
name, path, ifNoFilesFound string
|
||||
retentionDays, compressionLevel int
|
||||
overwrite, includeHidden, archiveIt bool
|
||||
}
|
||||
|
||||
func Main(ctx context.Context, c *action.Context) error {
|
||||
c.ApplyDefaults("name", "artifact", "path", "", "if-no-files-found", "warn", "retention-days", "", "compression-level", "6",
|
||||
"overwrite", "false", "include-hidden-files", "false", "archive", "true")
|
||||
in, err := readInputs(c)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
g, err := glob.Parse(in.path, glob.Options{
|
||||
Workdir: c.Workspace,
|
||||
Home: func() (string, error) { return c.Home(ctx) },
|
||||
Windows: c.Windows(),
|
||||
ImplicitDescendants: true,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
files, err := selectFiles(ctx, c, g, in.includeHidden)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
collected := &collected{files: files, root: glob.LeastCommonAncestor(g.SearchPaths(), c.Windows())}
|
||||
if len(files) == 1 && files[0].path == collected.root {
|
||||
collected.root = c.Dir(collected.root)
|
||||
}
|
||||
if len(collected.files) == 0 {
|
||||
message := fmt.Sprintf("No files were found with the provided path: %s. No artifacts will be uploaded.", in.path)
|
||||
switch in.ifNoFilesFound {
|
||||
case "warn":
|
||||
c.Warning("%s", message)
|
||||
case "error":
|
||||
return action.Failf("%s", message)
|
||||
default:
|
||||
c.Infof("%s", message)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if !in.archiveIt && len(collected.files) > 1 {
|
||||
return action.Failf("When 'archive' is set to false, only a single file can be uploaded. Found %d files to upload.", len(collected.files))
|
||||
}
|
||||
return upload(ctx, c, in, collected)
|
||||
}
|
||||
|
||||
func readInputs(c *action.Context) (*inputs, error) {
|
||||
in := &inputs{name: c.Input("name"), path: c.Input("path"), ifNoFilesFound: c.Input("if-no-files-found"), compressionLevel: 6}
|
||||
if in.path == "" {
|
||||
return nil, action.Failf("Input required and not supplied: path")
|
||||
}
|
||||
var err error
|
||||
if in.overwrite, err = c.BoolInput("overwrite"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if in.includeHidden, err = c.BoolInput("include-hidden-files"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if in.archiveIt, err = c.BoolInput("archive"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if in.ifNoFilesFound != "warn" && in.ifNoFilesFound != "error" && in.ifNoFilesFound != "ignore" {
|
||||
return nil, action.Failf("Unrecognized if-no-files-found input. Provided: %s. Available options: warn,error,ignore", in.ifNoFilesFound)
|
||||
}
|
||||
if days := c.Input("retention-days"); days != "" {
|
||||
if in.retentionDays, err = action.ParseInt(days); err != nil {
|
||||
return nil, action.Failf("Invalid retention-days")
|
||||
}
|
||||
}
|
||||
if level := c.Input("compression-level"); level != "" {
|
||||
if in.compressionLevel, err = action.ParseInt(level); err != nil {
|
||||
return nil, action.Failf("Invalid compression-level")
|
||||
}
|
||||
if in.compressionLevel < 0 || in.compressionLevel > 9 {
|
||||
return nil, action.Failf("Invalid compression-level. Valid values are 0-9")
|
||||
}
|
||||
}
|
||||
return in, nil
|
||||
}
|
||||
|
||||
type collected struct {
|
||||
files []found
|
||||
root string
|
||||
}
|
||||
|
||||
func (got *collected) write(ctx context.Context, c *action.Context, in *inputs, output io.Writer) error {
|
||||
if !in.archiveIt {
|
||||
return readFiles(ctx, c, got.files, func(file found) error {
|
||||
_, err := io.Copy(output, file.body)
|
||||
return err
|
||||
})
|
||||
}
|
||||
deflater, err := flate.NewWriter(nil, in.compressionLevel)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
writer := zip.NewWriter(output)
|
||||
writer.RegisterCompressor(zip.Deflate, func(output io.Writer) (io.WriteCloser, error) {
|
||||
deflater.Reset(output)
|
||||
return deflater, nil
|
||||
})
|
||||
if err := readFiles(ctx, c, got.files, func(file found) error {
|
||||
header := &zip.FileHeader{
|
||||
Name: strings.TrimLeft(strings.ReplaceAll(strings.TrimPrefix(file.path, got.root), `\`, "/"), "/"),
|
||||
Method: zip.Deflate, Modified: file.mtime.UTC(),
|
||||
}
|
||||
if in.compressionLevel == 0 {
|
||||
header.Method = zip.Store
|
||||
}
|
||||
header.SetMode(file.mode & (fs.ModePerm | fs.ModeSetuid | fs.ModeSetgid | fs.ModeSticky))
|
||||
entry, err := writer.CreateHeader(header)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = io.Copy(entry, file.body)
|
||||
return err
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
return writer.Close()
|
||||
}
|
||||
|
||||
const invalidPathChars = "\":<>|*?\r\n"
|
||||
|
||||
func upload(ctx context.Context, c *action.Context, in *inputs, got *collected) error {
|
||||
client, err := artifact.New(c.ResultsURL, c.RuntimeToken, c.Transport)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
client.Logf = c.Infof
|
||||
if in.overwrite {
|
||||
if id, err := client.Delete(ctx, in.name); err == nil {
|
||||
c.Infof("Artifact '%s' (ID: %d) deleted", in.name, id)
|
||||
} else if !artifact.IsNotFound(err) {
|
||||
c.Infof("Unable to delete artifact: %v", err) // best effort, as the JavaScript action
|
||||
}
|
||||
}
|
||||
name, fileName, mimeType := in.name, in.name+".zip", ""
|
||||
if !in.archiveIt {
|
||||
name, fileName, mimeType = c.Base(got.files[0].path), c.Base(got.files[0].path), mimeTypeOf(got.files[0].path)
|
||||
}
|
||||
if name == "" {
|
||||
return action.Failf("Provided artifact name input during validation is empty")
|
||||
}
|
||||
if strings.ContainsAny(name, invalidPathChars+`\/`) {
|
||||
return action.Failf("The artifact name is not valid: %s", name)
|
||||
}
|
||||
if in.archiveIt {
|
||||
for _, file := range got.files {
|
||||
if rel := strings.TrimPrefix(file.path, got.root); strings.ContainsAny(rel, invalidPathChars) {
|
||||
return action.Failf("The path for one of the files in artifact is not valid: %s", rel)
|
||||
}
|
||||
}
|
||||
}
|
||||
signed, err := client.Create(ctx, name, mimeType, expiresAt(c, in.retentionDays))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
concurrency, err := uploadConcurrency(c)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
stall, err := uploadTimeout(c)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
c.Infof("Uploading artifact: %s", fileName)
|
||||
reader, writer := io.Pipe()
|
||||
defer reader.Close()
|
||||
streamCtx, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
defer close(done)
|
||||
writer.CloseWithError(got.write(streamCtx, c, in, writer))
|
||||
}()
|
||||
uploaded, err := client.Upload(ctx, signed, cmp.Or(mimeType, "application/zip"), reader, concurrency, stall)
|
||||
reader.CloseWithError(err)
|
||||
cancel()
|
||||
<-done
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
id, err := client.Finalize(ctx, name, uploaded.Size, uploaded.Digest)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
c.Infof("Artifact %s has been successfully uploaded! Final size is %d bytes. Artifact ID is %d", in.name, uploaded.Size, id)
|
||||
ghc := c.Github
|
||||
link := strings.TrimSuffix(ghc.ServerURL, "/") + "/" + ghc.Repository + "/actions/runs/" + ghc.RunID + "/artifacts/" + url.PathEscape(name)
|
||||
if ghc.RunAttempt != "" {
|
||||
link += "?attempt=" + url.QueryEscape(ghc.RunAttempt)
|
||||
}
|
||||
c.Infof("Artifact download URL: %s", link)
|
||||
return c.SetOutputs(ctx, "artifact-id", strconv.FormatInt(id, 10), "artifact-digest", strings.TrimPrefix(uploaded.Digest, "sha256:"), "artifact-url", link)
|
||||
}
|
||||
|
||||
func expiresAt(c *action.Context, days int) *time.Time {
|
||||
if days == 0 {
|
||||
return nil
|
||||
}
|
||||
if limit, err := action.ParseInt(c.Github.RetentionDays); err == nil && limit != 0 && limit < days {
|
||||
c.Warning("Retention days cannot be greater than the maximum allowed retention set within the repository. Using %d instead.", limit)
|
||||
days = limit
|
||||
}
|
||||
expires := time.Now().AddDate(0, 0, days)
|
||||
return &expires
|
||||
}
|
||||
|
||||
func uploadConcurrency(c *action.Context) (int, error) {
|
||||
override := c.Env["ACTIONS_ARTIFACT_UPLOAD_CONCURRENCY"]
|
||||
if override == "" {
|
||||
return 5, nil
|
||||
}
|
||||
concurrency, err := action.ParseInt(override)
|
||||
if err != nil || concurrency < 1 {
|
||||
return 0, action.Failf("Invalid value set for ACTIONS_ARTIFACT_UPLOAD_CONCURRENCY env variable")
|
||||
}
|
||||
return concurrency, nil
|
||||
}
|
||||
|
||||
func uploadTimeout(c *action.Context) (time.Duration, error) {
|
||||
value := c.Env["ACTIONS_ARTIFACT_UPLOAD_TIMEOUT_MS"]
|
||||
if value == "" {
|
||||
return 5 * time.Minute, nil
|
||||
}
|
||||
ms, err := action.ParseInt(value)
|
||||
if err != nil {
|
||||
return 0, action.Failf("Invalid value set for ACTIONS_ARTIFACT_UPLOAD_TIMEOUT_MS env variable")
|
||||
}
|
||||
return time.Duration(ms) * time.Millisecond, nil
|
||||
}
|
||||
@@ -0,0 +1,280 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package uploadartifact
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"maps"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.com/gitea/runner/act/artifacts"
|
||||
"gitea.com/gitea/runner/act/container"
|
||||
"gitea.com/gitea/runner/internal/pkg/action"
|
||||
"gitea.com/gitea/runner/internal/pkg/action/artifact"
|
||||
|
||||
"gitea.dev/actionslib/pkg/model"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
type localContainer struct {
|
||||
*container.HostEnvironment
|
||||
bin string
|
||||
}
|
||||
|
||||
func (c localContainer) ExecOutput(ctx context.Context, command []string) (io.ReadCloser, error) {
|
||||
if c.bin != "" {
|
||||
command = append([]string{"env", "PATH=" + c.bin}, command...)
|
||||
}
|
||||
return c.HostEnvironment.ExecOutput(ctx, command)
|
||||
}
|
||||
|
||||
type harness struct {
|
||||
t *testing.T
|
||||
server string
|
||||
client *artifact.Client
|
||||
workspace string
|
||||
env container.ExecutionsEnvironment
|
||||
logs, annotation []string
|
||||
retentionDays string
|
||||
variables map[string]string
|
||||
output string
|
||||
}
|
||||
|
||||
func newHarness(t *testing.T, env container.ExecutionsEnvironment) *harness {
|
||||
t.Helper()
|
||||
router := http.NewServeMux()
|
||||
artifacts.RegisterV4(router, t.TempDir())
|
||||
server := httptest.NewServer(router)
|
||||
t.Cleanup(server.Close)
|
||||
client, err := artifact.New(server.URL, artifacts.LocalRuntimeToken("7", time.Hour), nil)
|
||||
require.NoError(t, err)
|
||||
workspace, err := filepath.EvalSymlinks(t.TempDir())
|
||||
require.NoError(t, err)
|
||||
return &harness{t: t, server: server.URL, client: client, workspace: workspace, env: env}
|
||||
}
|
||||
|
||||
func (h *harness) write(rel, content string, mode os.FileMode) {
|
||||
h.t.Helper()
|
||||
p := filepath.Join(h.workspace, filepath.FromSlash(rel))
|
||||
require.NoError(h.t, os.MkdirAll(filepath.Dir(p), 0o755))
|
||||
require.NoError(h.t, os.WriteFile(p, []byte(content), mode))
|
||||
}
|
||||
|
||||
func (h *harness) run(inputs map[string]string) error {
|
||||
h.logs, h.annotation = nil, nil
|
||||
h.output = filepath.Join(h.t.TempDir(), "output")
|
||||
variables := map[string]string{"GITHUB_OUTPUT": h.output}
|
||||
maps.Copy(variables, h.variables)
|
||||
return Main(h.t.Context(), &action.Context{
|
||||
Container: h.env,
|
||||
Github: &model.GithubContext{ServerURL: "https://gitea.example", Repository: "owner/repo", RunID: "7", RunAttempt: "2", RetentionDays: h.retentionDays},
|
||||
Inputs: inputs,
|
||||
Workspace: h.workspace,
|
||||
Env: variables,
|
||||
ResultsURL: h.server,
|
||||
RuntimeToken: artifacts.LocalRuntimeToken("7", time.Hour),
|
||||
Infof: func(format string, args ...any) { h.logs = append(h.logs, fmt.Sprintf(format, args...)) },
|
||||
Warning: func(format string, args ...any) {
|
||||
h.annotation = append(h.annotation, "warning: "+fmt.Sprintf(format, args...))
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
func (h *harness) fetch(name string) ([]byte, map[string]string) {
|
||||
h.t.Helper()
|
||||
signed, err := h.client.SignedURL(h.t.Context(), name)
|
||||
require.NoError(h.t, err)
|
||||
resp, err := h.client.Download(h.t.Context(), signed, time.Minute)
|
||||
require.NoError(h.t, err)
|
||||
defer resp.Body.Close()
|
||||
content, err := io.ReadAll(resp.Body)
|
||||
require.NoError(h.t, err)
|
||||
reader, err := zip.NewReader(bytes.NewReader(content), int64(len(content)))
|
||||
if err != nil {
|
||||
return content, nil
|
||||
}
|
||||
entries := map[string]string{}
|
||||
for _, f := range reader.File {
|
||||
body, err := f.Open()
|
||||
require.NoError(h.t, err)
|
||||
data, err := io.ReadAll(body)
|
||||
require.NoError(h.t, err)
|
||||
require.NoError(h.t, body.Close())
|
||||
entries[f.Name] = string(data)
|
||||
}
|
||||
return content, entries
|
||||
}
|
||||
|
||||
func TestUploadSelectsWhatTheToolkitSelects(t *testing.T) {
|
||||
envs := map[string]container.ExecutionsEnvironment{"host": &container.HostEnvironment{}}
|
||||
if runtime.GOOS != "windows" {
|
||||
envs["container gzip"] = localContainer{HostEnvironment: &container.HostEnvironment{}}
|
||||
bin := t.TempDir()
|
||||
for _, name := range []string{"sh", "find", "readlink", "tar"} {
|
||||
binary, err := exec.LookPath(name)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, os.Symlink(binary, filepath.Join(bin, name)))
|
||||
}
|
||||
envs["container plain"] = localContainer{HostEnvironment: &container.HostEnvironment{}, bin: bin}
|
||||
}
|
||||
for envName, env := range envs {
|
||||
t.Run(envName, func(t *testing.T) {
|
||||
h := newHarness(t, env)
|
||||
h.write("dist/a.txt", "a", 0o644)
|
||||
h.write("dist/run.sh", "#!/bin/sh\n", 0o755)
|
||||
h.write("dist/sub/b.txt", "b", 0o644)
|
||||
h.write("dist/empty", "", 0o644)
|
||||
require.NoError(t, os.Mkdir(filepath.Join(h.workspace, "dist", "emptydir"), 0o755))
|
||||
h.write("dist/.hidden", "h", 0o644)
|
||||
h.write("dist/.git/config", "c", 0o644)
|
||||
h.write("logs/x.log", "x", 0o644)
|
||||
h.write(".store/target.txt", "hidden target", 0o644)
|
||||
if runtime.GOOS != "windows" {
|
||||
require.NoError(t, os.Symlink("a.txt", filepath.Join(h.workspace, "dist", "link.txt")))
|
||||
require.NoError(t, os.Symlink("sub", filepath.Join(h.workspace, "dist", "linkdir")))
|
||||
require.NoError(t, os.Symlink("../.store", filepath.Join(h.workspace, "dist", "alias")))
|
||||
require.NoError(t, os.Symlink("missing", filepath.Join(h.workspace, "dist", "broken")))
|
||||
require.NoError(t, os.Symlink("..", filepath.Join(h.workspace, "dist", "loop")))
|
||||
require.NoError(t, os.Link(filepath.Join(h.workspace, "dist", "a.txt"), filepath.Join(h.workspace, "dist", "hard.txt")))
|
||||
}
|
||||
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
inputs map[string]string
|
||||
expected map[string]string
|
||||
links map[string]string
|
||||
}{
|
||||
{
|
||||
name: "a directory",
|
||||
inputs: map[string]string{"path": "dist"},
|
||||
expected: map[string]string{"a.txt": "a", "run.sh": "#!/bin/sh\n", "sub/b.txt": "b", "empty": ""},
|
||||
links: map[string]string{"link.txt": "a", "hard.txt": "a", "linkdir/b.txt": "b", "loop/logs/x.log": "x", "alias/target.txt": "hidden target"},
|
||||
},
|
||||
{
|
||||
name: "hidden files included",
|
||||
inputs: map[string]string{"path": "dist/", "include-hidden-files": "true"},
|
||||
expected: map[string]string{"a.txt": "a", "run.sh": "#!/bin/sh\n", "sub/b.txt": "b", ".hidden": "h", ".git/config": "c", "empty": ""},
|
||||
links: map[string]string{"link.txt": "a", "hard.txt": "a", "linkdir/b.txt": "b", "loop/logs/x.log": "x", "loop/.store/target.txt": "hidden target", "alias/target.txt": "hidden target"},
|
||||
},
|
||||
{
|
||||
name: "only exclusions and comments",
|
||||
inputs: map[string]string{"path": "!dist\n# dist"},
|
||||
expected: map[string]string{},
|
||||
},
|
||||
{
|
||||
name: "several patterns share their common ancestor as root",
|
||||
inputs: map[string]string{"path": "dist/**/*.txt\n!dist/sub/**\n# a comment\nlogs/x.log"},
|
||||
expected: map[string]string{"dist/a.txt": "a", "logs/x.log": "x"},
|
||||
links: map[string]string{"dist/link.txt": "a", "dist/hard.txt": "a", "dist/linkdir/b.txt": "b", "dist/alias/target.txt": "hidden target"},
|
||||
},
|
||||
{
|
||||
name: "hidden root skipped",
|
||||
inputs: map[string]string{"path": ".store"},
|
||||
expected: map[string]string{},
|
||||
},
|
||||
{
|
||||
name: "a single file keeps no directory",
|
||||
inputs: map[string]string{"path": h.workspace + "/dist/sub/b.txt"},
|
||||
expected: map[string]string{"b.txt": "b"},
|
||||
},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
inputs := map[string]string{"name": "art", "overwrite": "true"}
|
||||
maps.Copy(inputs, test.inputs)
|
||||
require.NoError(t, h.run(inputs))
|
||||
expected := test.expected
|
||||
if runtime.GOOS != "windows" {
|
||||
maps.Copy(expected, test.links)
|
||||
}
|
||||
if len(expected) == 0 {
|
||||
assert.Contains(t, h.annotation, "warning: No files were found with the provided path: "+test.inputs["path"]+". No artifacts will be uploaded.")
|
||||
return
|
||||
}
|
||||
_, entries := h.fetch("art")
|
||||
assert.Equal(t, expected, entries)
|
||||
})
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadOptions(t *testing.T) {
|
||||
h := newHarness(t, &container.HostEnvironment{})
|
||||
h.write("out/run.sh", "#!/bin/sh\n", 0o755)
|
||||
h.write("out/data.txt", strings.Repeat("data", 1000), 0o644)
|
||||
h.variables = map[string]string{"ACTIONS_ARTIFACT_UPLOAD_CONCURRENCY": " 2 workers", "ACTIONS_ARTIFACT_UPLOAD_TIMEOUT_MS": " 60000ms"}
|
||||
h.retentionDays = "5"
|
||||
require.NoError(t, h.run(map[string]string{"name": "packed", "path": "out", "compression-level": "0", "retention-days": "10 days"}))
|
||||
assert.Equal(t, []string{"warning: Retention days cannot be greater than the maximum allowed retention set within the repository. Using 5 instead."}, h.annotation)
|
||||
|
||||
content, _ := h.fetch("packed")
|
||||
reader, err := zip.NewReader(bytes.NewReader(content), int64(len(content)))
|
||||
require.NoError(t, err)
|
||||
for _, f := range reader.File {
|
||||
assert.Equal(t, zip.Store, f.Method, f.Name)
|
||||
if runtime.GOOS != "windows" && f.Name == "run.sh" {
|
||||
assert.Equal(t, os.FileMode(0o755), f.Mode().Perm())
|
||||
}
|
||||
}
|
||||
|
||||
list, err := h.client.List(t.Context(), "")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, list, 1)
|
||||
outputs, err := os.ReadFile(h.output)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, fmt.Sprintf("artifact-id=%d\nartifact-digest=%s\nartifact-url=https://gitea.example/owner/repo/actions/runs/7/artifacts/packed?attempt=2\n", list[0].ID, strings.TrimPrefix(list[0].Digest, "sha256:")), string(outputs))
|
||||
|
||||
h.write("report.html", "<html></html>", 0o644)
|
||||
|
||||
require.NoError(t, h.run(map[string]string{"name": "packed", "path": "report.html", "archive": "false", "overwrite": "true"}))
|
||||
list, err = h.client.List(t.Context(), "packed")
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, list)
|
||||
content, entries := h.fetch("report.html")
|
||||
assert.Nil(t, entries)
|
||||
assert.Equal(t, "<html></html>", string(content))
|
||||
assert.Contains(t, h.logs, "Uploading artifact: report.html")
|
||||
assert.Contains(t, strings.Join(h.logs, "\n"), "Artifact packed has been successfully uploaded!")
|
||||
require.ErrorContains(t, h.run(map[string]string{"path": "out", "archive": "false"}), "When 'archive' is set to false, only a single file can be uploaded. Found 2 files to upload.")
|
||||
|
||||
require.NoError(t, h.run(map[string]string{"path": "nothing/here"}))
|
||||
assert.Equal(t, []string{"warning: No files were found with the provided path: nothing/here. No artifacts will be uploaded."}, h.annotation)
|
||||
require.NoError(t, h.run(map[string]string{"path": "nothing", "if-no-files-found": "ignore"}))
|
||||
assert.Contains(t, h.logs, "No files were found with the provided path: nothing. No artifacts will be uploaded.")
|
||||
require.ErrorContains(t, h.run(map[string]string{"path": "nothing", "if-no-files-found": "error"}), "No files were found")
|
||||
|
||||
for _, test := range []struct {
|
||||
inputs map[string]string
|
||||
want string
|
||||
}{
|
||||
{map[string]string{"name": "a/b", "path": "out/data.txt"}, "The artifact name is not valid: a/b"},
|
||||
{map[string]string{}, "Input required and not supplied: path"},
|
||||
{map[string]string{"name": "", "path": "out/data.txt"}, "Provided artifact name input during validation is empty"},
|
||||
{map[string]string{"path": "out/data.txt", "overwrite": ""}, "Core Schema"},
|
||||
{map[string]string{"path": "out/data.txt", "compression-level": "10"}, "Invalid compression-level. Valid values are 0-9"},
|
||||
{map[string]string{"path": "out/data.txt", "compression-level": "+-1"}, "Invalid compression-level"},
|
||||
{map[string]string{"path": "out/data.txt", "if-no-files-found": "fail"}, "Unrecognized if-no-files-found input"},
|
||||
} {
|
||||
require.ErrorContains(t, h.run(test.inputs), test.want)
|
||||
}
|
||||
require.NoError(t, h.run(map[string]string{"path": "out/data.txt", "retention": "1", "overwrite": "true"}))
|
||||
assert.Equal(t, []string{"warning: Unexpected input(s) 'retention', valid inputs are ['name', 'path', 'if-no-files-found', 'retention-days', 'compression-level', 'overwrite', 'include-hidden-files', 'archive']"}, h.annotation)
|
||||
if runtime.GOOS != "windows" {
|
||||
h.write("bad/a:b.txt", "x", 0o644)
|
||||
require.ErrorContains(t, h.run(map[string]string{"path": "bad"}), "The path for one of the files in artifact is not valid: /a:b.txt")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,178 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package uploadartifact
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.com/gitea/runner/act/container"
|
||||
"gitea.com/gitea/runner/internal/pkg/action"
|
||||
"gitea.com/gitea/runner/internal/pkg/action/glob"
|
||||
)
|
||||
|
||||
type found struct {
|
||||
path, source string
|
||||
mode fs.FileMode
|
||||
mtime time.Time
|
||||
body io.Reader
|
||||
}
|
||||
|
||||
type treeEntry struct {
|
||||
name, target string
|
||||
dir bool
|
||||
}
|
||||
|
||||
type tree struct {
|
||||
real string
|
||||
entries map[string][]treeEntry
|
||||
}
|
||||
|
||||
const listScript = `test -e "$1" || exit 1
|
||||
[ -d "$1" ] || { printf 'f\0'; exit 0; }
|
||||
cd "$1" || exit 2
|
||||
shift
|
||||
pwd -P; printf '\0'
|
||||
find . "$@" -type d ! -name . -print0; printf '\0'
|
||||
find . "$@" -type l -exec sh -c 'for l; do printf "%s\0" "$l"; readlink "$l"; printf "\0"; done' sh {} +; printf '\0'
|
||||
find . "$@" ! -type d ! -type l -print0`
|
||||
|
||||
func list(ctx context.Context, c *action.Context, source string, includeHidden bool) (*tree, error) {
|
||||
if c.Host() {
|
||||
info, err := os.Stat(source)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
resolved, err := filepath.EvalSymlinks(source)
|
||||
if err != nil || !info.IsDir() {
|
||||
return &tree{real: resolved}, err
|
||||
}
|
||||
entries, err := os.ReadDir(resolved)
|
||||
result := &tree{real: resolved, entries: map[string][]treeEntry{".": nil}}
|
||||
for _, entry := range entries {
|
||||
result.entries["."] = append(result.entries["."], treeEntry{name: entry.Name(), dir: entry.IsDir()})
|
||||
}
|
||||
return result, err
|
||||
}
|
||||
command := []string{"sh", "-c", listScript, "sh", source}
|
||||
if !includeHidden {
|
||||
command = append(command, "!", "-name", ".", "-name", ".*", "-prune", "-o")
|
||||
}
|
||||
output, err := c.Container.ExecOutput(ctx, command)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer output.Close()
|
||||
data, err := io.ReadAll(output)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := output.Close(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if string(data) == "f\x00" {
|
||||
return &tree{real: source}, nil
|
||||
}
|
||||
fields := strings.Split(string(data), "\x00")
|
||||
if len(fields) < 4 {
|
||||
return nil, fmt.Errorf("invalid directory listing for %s", source)
|
||||
}
|
||||
result := &tree{real: strings.TrimSuffix(fields[0], "\n"), entries: map[string][]treeEntry{".": nil}}
|
||||
section := 0
|
||||
for index := 1; index < len(fields)-1; index++ {
|
||||
name := fields[index]
|
||||
if name == "" {
|
||||
section++
|
||||
continue
|
||||
}
|
||||
entry := treeEntry{name: path.Base(name), dir: section == 0}
|
||||
switch section {
|
||||
case 0:
|
||||
result.entries[path.Clean(name)] = nil
|
||||
case 1:
|
||||
index++
|
||||
if index >= len(fields)-1 {
|
||||
return nil, fmt.Errorf("invalid symlink listing for %s", source)
|
||||
}
|
||||
entry.target = strings.TrimSuffix(fields[index], "\n")
|
||||
}
|
||||
result.entries[path.Dir(name)] = append(result.entries[path.Dir(name)], entry)
|
||||
}
|
||||
for _, entries := range result.entries {
|
||||
slices.SortFunc(entries, func(left, right treeEntry) int { return strings.Compare(left.name, right.name) })
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func selectFiles(ctx context.Context, c *action.Context, globber *glob.Globber, includeHidden bool) ([]found, error) {
|
||||
var files []found
|
||||
join := path.Join
|
||||
if c.Windows() {
|
||||
join = filepath.Join
|
||||
}
|
||||
var walk func(string, string, []string, *tree, string) error
|
||||
walk = func(virtual, source string, chain []string, listed *tree, relative string) error {
|
||||
match := globber.Match(virtual)
|
||||
if match == glob.MatchNone && !globber.PartialMatch(virtual) || !includeHidden && strings.HasPrefix(c.Base(virtual), ".") {
|
||||
return nil
|
||||
}
|
||||
if listed == nil {
|
||||
var err error
|
||||
listed, err = list(ctx, c, source, includeHidden)
|
||||
var exit container.ExitCodeError
|
||||
if errors.Is(err, fs.ErrNotExist) || errors.As(err, &exit) && exit == 1 {
|
||||
return nil
|
||||
} else if err != nil {
|
||||
return err
|
||||
}
|
||||
relative = "."
|
||||
}
|
||||
if listed.entries == nil {
|
||||
if match&glob.MatchFile != 0 {
|
||||
files = append(files, found{path: virtual, source: listed.real})
|
||||
}
|
||||
return nil
|
||||
}
|
||||
resolved := join(listed.real, relative)
|
||||
if slices.Contains(chain, resolved) {
|
||||
return nil
|
||||
}
|
||||
chain = append(slices.Clip(chain), resolved)
|
||||
for _, entry := range listed.entries[relative] {
|
||||
child := join(virtual, entry.name)
|
||||
source := join(resolved, entry.name)
|
||||
var subtree *tree
|
||||
switch {
|
||||
case entry.target != "":
|
||||
source = entry.target
|
||||
if !path.IsAbs(source) {
|
||||
source = path.Join(resolved, source)
|
||||
}
|
||||
case entry.dir && !c.Host():
|
||||
subtree = listed
|
||||
case !entry.dir && !c.Host():
|
||||
subtree = &tree{real: source}
|
||||
}
|
||||
if err := walk(child, source, chain, subtree, path.Join(relative, entry.name)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
for _, root := range globber.SearchPaths() {
|
||||
if err := walk(root, root, nil, nil, "."); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return files, nil
|
||||
}
|
||||
@@ -119,6 +119,7 @@ runner:
|
||||
# If it's empty when execute `daemon`, will use labels in `.runner` file.
|
||||
#labels:
|
||||
# - "ubuntu-latest:docker://docker.gitea.com/runner-images:ubuntu-latest"
|
||||
# - "ubuntu-26.04:docker://docker.gitea.com/runner-images:ubuntu-26.04"
|
||||
# - "ubuntu-24.04:docker://docker.gitea.com/runner-images:ubuntu-24.04"
|
||||
# - "ubuntu-22.04:docker://docker.gitea.com/runner-images:ubuntu-22.04"
|
||||
# Allocate a pseudo-TTY for each step's process. Applies to both host and docker backends.
|
||||
@@ -306,8 +307,8 @@ container:
|
||||
# healthy. A negative value (e.g. -1s) starts the steps without waiting.
|
||||
#service_ready_timeout: 5m
|
||||
# Remove the containers, networks and volumes a job creates through its Docker proxy when
|
||||
# the job ends. Set to false when jobs deploy services to the runner's Docker daemon.
|
||||
#sweep: true
|
||||
# the job ends. Leave off when jobs deploy services to the runner's Docker daemon.
|
||||
#sweep: false
|
||||
|
||||
host:
|
||||
# The parent directory of a job's working directory.
|
||||
|
||||
@@ -190,7 +190,7 @@ type Container struct {
|
||||
DockerTimeout time.Duration `yaml:"docker_timeout"` // Timeout to wait for the docker daemon to be reachable, if docker is required by require_docker or runner
|
||||
BindWorkdir bool `yaml:"bind_workdir"` // BindWorkdir mounts the workspace from a host directory instead of a Docker volume.
|
||||
ServiceReadyTimeout time.Duration `yaml:"service_ready_timeout"` // ServiceReadyTimeout bounds how long a job waits for a service container that declares a healthcheck to report healthy. Negative disables waiting.
|
||||
Sweep *bool `yaml:"sweep"` // Sweep removes the containers, networks and volumes a job creates through its Docker proxy when the job ends. Unset means enabled.
|
||||
Sweep bool `yaml:"sweep"` // Sweep removes the containers, networks and volumes a job creates through its Docker proxy when the job ends.
|
||||
}
|
||||
|
||||
// Values of Runner.ToolCacheMode: the runner mounts no tool cache, or one that every job reuses.
|
||||
|
||||
@@ -6,7 +6,6 @@ package process
|
||||
import (
|
||||
"os/exec"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
@@ -23,16 +22,13 @@ func TestGroupClosePropagatesToOrphan(t *testing.T) {
|
||||
childPID := ready()
|
||||
|
||||
require.NoError(t, cmd.Wait())
|
||||
require.Eventually(t, func() bool {
|
||||
return !processAlive(cmd.Process.Pid)
|
||||
}, 20*time.Second, 200*time.Millisecond, "the step process should have exited on its own")
|
||||
require.True(t, cmd.ProcessState.Exited(), "the step process should have exited on its own")
|
||||
|
||||
require.True(t, processAlive(childPID), "orphan should outlive its parent")
|
||||
|
||||
waitForExit := newProcessExitWaiter(t, childPID)
|
||||
require.NoError(t, group.Close())
|
||||
require.Eventually(t, func() bool {
|
||||
return !processAlive(childPID)
|
||||
}, 20*time.Second, 200*time.Millisecond, "closing the job must terminate the orphan")
|
||||
waitForExit()
|
||||
}
|
||||
|
||||
// TestGroupNestsWithKiller: assigned to the group first and the step's Killer
|
||||
@@ -53,10 +49,9 @@ func TestGroupNestsWithKiller(t *testing.T) {
|
||||
require.NoError(t, err, "the step's job object must nest inside the group's")
|
||||
defer killer.Close()
|
||||
|
||||
waitForExit := newProcessExitWaiter(t, cmd.Process.Pid)
|
||||
require.NoError(t, killer.Kill())
|
||||
require.Eventually(t, func() bool {
|
||||
return !processAlive(cmd.Process.Pid)
|
||||
}, 20*time.Second, 200*time.Millisecond, "cancelling the step should still kill its tree")
|
||||
waitForExit()
|
||||
}
|
||||
|
||||
// TestGroupNilSafe covers the fallback path: when the job object cannot be
|
||||
|
||||
@@ -9,8 +9,6 @@ import (
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"syscall"
|
||||
"testing"
|
||||
@@ -46,23 +44,8 @@ func processAlive(pid int) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
// TestKillerKillsTree verifies that a process group captured by the killer is
|
||||
// terminated together with a child the process spawns afterwards. This mirrors
|
||||
// a step or post-task script that launches a child which spawns further
|
||||
// processes, where cancelling must take down the whole tree, not just the
|
||||
// direct child.
|
||||
func TestKillerKillsTree(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
pidFile := filepath.Join(dir, "child.pid")
|
||||
|
||||
// Parent shell backgrounds a long-lived child (writing its PID to a file)
|
||||
// and then sleeps. With job control off (non-interactive sh) the backgrounded
|
||||
// child stays in the parent's process group, so the group kill must reach it.
|
||||
script := fmt.Sprintf(`sleep 600 & echo $! > %q; sleep 600`, pidFile)
|
||||
cmd := exec.Command("/bin/sh", "-c", script)
|
||||
// Launch as its own process-group leader, exactly like a real process does
|
||||
// (see SysProcAttr), so the killer's PGID == the process PID.
|
||||
cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
|
||||
cmd, ready := newTestProcess(t, "parent", false)
|
||||
require.NoError(t, cmd.Start())
|
||||
t.Cleanup(func() {
|
||||
_ = syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL)
|
||||
@@ -73,25 +56,10 @@ func TestKillerKillsTree(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
defer killer.Close()
|
||||
|
||||
// Wait for the backgrounded child PID to be reported.
|
||||
var childPID int
|
||||
require.Eventually(t, func() bool {
|
||||
b, e := os.ReadFile(pidFile)
|
||||
if e != nil {
|
||||
return false
|
||||
}
|
||||
s := strings.TrimSpace(string(b))
|
||||
if s == "" {
|
||||
return false
|
||||
}
|
||||
childPID, _ = strconv.Atoi(s)
|
||||
return childPID > 0 && processAlive(childPID)
|
||||
}, 20*time.Second, 100*time.Millisecond, "child process should start")
|
||||
childPID := ready()
|
||||
require.True(t, processAlive(childPID), "child process should start")
|
||||
|
||||
// Killing the group must terminate both the parent and the backgrounded child.
|
||||
require.NoError(t, killer.Kill())
|
||||
// Reap the parent so it does not linger as a zombie (which would still report
|
||||
// as alive); SIGKILL makes Wait return promptly.
|
||||
_ = cmd.Wait()
|
||||
|
||||
require.Eventually(t, func() bool {
|
||||
|
||||
@@ -5,7 +5,6 @@ package process
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
"golang.org/x/sys/windows"
|
||||
@@ -25,6 +24,23 @@ func processAlive(pid int) bool {
|
||||
return code == stillActive
|
||||
}
|
||||
|
||||
func newProcessExitWaiter(t *testing.T, pids ...int) func() {
|
||||
t.Helper()
|
||||
handles := make([]windows.Handle, 0, len(pids))
|
||||
for _, pid := range pids {
|
||||
handle, err := windows.OpenProcess(windows.SYNCHRONIZE, false, uint32(pid))
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { require.NoError(t, windows.CloseHandle(handle)) })
|
||||
handles = append(handles, handle)
|
||||
}
|
||||
return func() {
|
||||
t.Helper()
|
||||
result, err := windows.WaitForMultipleObjects(handles, true, 20000)
|
||||
require.NoError(t, err)
|
||||
require.EqualValues(t, windows.WAIT_OBJECT_0, result, "processes should exit: %v", pids)
|
||||
}
|
||||
}
|
||||
|
||||
func TestKillerKillsTree(t *testing.T) {
|
||||
cmd, ready := newTestProcess(t, "parent", false)
|
||||
require.NoError(t, cmd.Start())
|
||||
@@ -39,9 +55,7 @@ func TestKillerKillsTree(t *testing.T) {
|
||||
childPID := ready()
|
||||
require.True(t, processAlive(childPID), "child process should start")
|
||||
|
||||
waitForExit := newProcessExitWaiter(t, cmd.Process.Pid, childPID)
|
||||
require.NoError(t, killer.Kill())
|
||||
|
||||
require.Eventually(t, func() bool {
|
||||
return !processAlive(cmd.Process.Pid) && !processAlive(childPID)
|
||||
}, 20*time.Second, 200*time.Millisecond, "parent and child should both be terminated")
|
||||
waitForExit()
|
||||
}
|
||||
@@ -960,7 +960,7 @@ func logRow(entry logging.Record) *runnerv1.LogRow {
|
||||
|
||||
// mask repairs the content first, so a secret the repair itself spells out is still caught.
|
||||
func (r *Reporter) mask(content string) string {
|
||||
return r.logReplacer.Replace(strings.ToValidUTF8(content, "?"))
|
||||
return runner.MaskLog(strings.ToValidUTF8(content, "?"), r.logReplacer)
|
||||
}
|
||||
|
||||
// AddMask deliberately leaves the process-wide masker alone. Its entries come from every live
|
||||
|
||||
@@ -173,11 +173,9 @@ func TestReporter_RunDaemonClose_Race(t *testing.T) {
|
||||
)
|
||||
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
taskCtx, err := structpb.NewStruct(map[string]any{})
|
||||
require.NoError(t, err)
|
||||
cfg, _ := config.LoadDefault("")
|
||||
reporter := NewReporter(ctx, cancel, client, &runnerv1.Task{
|
||||
Context: taskCtx,
|
||||
Context: &structpb.Struct{},
|
||||
}, cfg)
|
||||
reporter.ResetSteps(newTestSteps(1))
|
||||
|
||||
@@ -474,10 +472,8 @@ func TestReporter_ServerCancelStillFlushesFinal(t *testing.T) {
|
||||
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
defer cancel()
|
||||
taskCtx, err := structpb.NewStruct(map[string]any{})
|
||||
require.NoError(t, err)
|
||||
cfg, _ := config.LoadDefault("")
|
||||
reporter := NewReporter(ctx, cancel, client, &runnerv1.Task{Context: taskCtx}, cfg)
|
||||
reporter := NewReporter(ctx, cancel, client, &runnerv1.Task{Context: &structpb.Struct{}}, cfg)
|
||||
reporter.ResetSteps(newTestSteps(1))
|
||||
|
||||
reporter.stateMu.Lock()
|
||||
@@ -527,10 +523,8 @@ func TestReporter_CloseReportsCancelledOnCanceledCtx(t *testing.T) {
|
||||
)
|
||||
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
taskCtx, err := structpb.NewStruct(map[string]any{})
|
||||
require.NoError(t, err)
|
||||
cfg, _ := config.LoadDefault("")
|
||||
reporter := NewReporter(ctx, cancel, client, &runnerv1.Task{Context: taskCtx}, cfg)
|
||||
reporter := NewReporter(ctx, cancel, client, &runnerv1.Task{Context: &structpb.Struct{}}, cfg)
|
||||
reporter.ResetSteps(newTestSteps(1))
|
||||
|
||||
cancel()
|
||||
@@ -671,19 +665,15 @@ func TestReporter_SetOutputs(t *testing.T) {
|
||||
require.True(t, ok)
|
||||
assert.Equal(t, "caf\xff", got.value)
|
||||
|
||||
maxValue := strings.Repeat("v", maxOutputValueLen)
|
||||
r.SetOutputs(map[string]string{"atlimit": maxValue})
|
||||
r.SetOutputs(map[string]string{"atlimit": longValue[:maxOutputValueLen]})
|
||||
got, ok = r.outputs["atlimit"]
|
||||
require.True(t, ok)
|
||||
assert.Len(t, got.value, maxOutputValueLen)
|
||||
}
|
||||
|
||||
func TestReporter_DebugSettingsAreNotMasked(t *testing.T) {
|
||||
taskCtx, err := structpb.NewStruct(map[string]any{})
|
||||
require.NoError(t, err)
|
||||
|
||||
reporter := NewReporter(t.Context(), nil, nil, &runnerv1.Task{
|
||||
Context: taskCtx,
|
||||
Context: &structpb.Struct{},
|
||||
Secrets: map[string]string{"ACTIONS_STEP_DEBUG": "true", "ACTIONS_RUNNER_DEBUG": "true", "TOKEN": "s3cr3t"},
|
||||
}, &config.Config{})
|
||||
defer deregisterGlobalMasks(reporter)
|
||||
@@ -760,8 +750,7 @@ func TestReporter_ParseResult(t *testing.T) {
|
||||
func TestReporter_masksEncodedSecrets(t *testing.T) {
|
||||
secret := `p@ss w"rd/1`
|
||||
r := &Reporter{logReplacer: strings.NewReplacer()}
|
||||
r.oldnew = runner.AppendSecretMasker(r.oldnew, secret)
|
||||
r.logReplacer = strings.NewReplacer(r.oldnew...)
|
||||
r.AddMask(secret)
|
||||
|
||||
for _, line := range []string{
|
||||
"token: " + secret,
|
||||
@@ -773,6 +762,17 @@ func TestReporter_masksEncodedSecrets(t *testing.T) {
|
||||
assert.NotContains(t, masked, secret)
|
||||
assert.NotContains(t, masked, base64.StdEncoding.EncodeToString([]byte(secret)))
|
||||
}
|
||||
|
||||
r.AddMask("alpha beta;gamma]delta")
|
||||
r.AddMask("%")
|
||||
for _, test := range []struct{ line, want string }{
|
||||
{"##[group]alpha beta%3Bgamma%5Ddelta", "##[group]***"},
|
||||
{"##[group]alpha beta%3Bgamma]delta", "##[group]***"},
|
||||
{"%", "***"},
|
||||
} {
|
||||
r.appendLogRow(logRow(logging.Record{Message: test.line}))
|
||||
assert.Equal(t, test.want, r.logRows[len(r.logRows)-1].Content)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReporter_AckIndexBeyondBuffer(t *testing.T) {
|
||||
@@ -861,10 +861,8 @@ func TestReporter_ServerFailureResultIsReportedAsFailure(t *testing.T) {
|
||||
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
defer cancel()
|
||||
taskCtx, err := structpb.NewStruct(map[string]any{})
|
||||
require.NoError(t, err)
|
||||
cfg, _ := config.LoadDefault("")
|
||||
reporter := NewReporter(ctx, cancel, client, &runnerv1.Task{Context: taskCtx}, cfg)
|
||||
reporter := NewReporter(ctx, cancel, client, &runnerv1.Task{Context: &structpb.Struct{}}, cfg)
|
||||
close(reporter.daemon)
|
||||
|
||||
require.NoError(t, reporter.ReportState(false))
|
||||
@@ -1031,9 +1029,7 @@ func newTestReporter(t *testing.T, client *mocks.Client, cfg *config.Config) *Re
|
||||
t.Helper()
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
t.Cleanup(cancel)
|
||||
taskCtx, err := structpb.NewStruct(map[string]any{})
|
||||
require.NoError(t, err)
|
||||
return NewReporter(ctx, cancel, client, &runnerv1.Task{Context: taskCtx}, cfg)
|
||||
return NewReporter(ctx, cancel, client, &runnerv1.Task{Context: &structpb.Struct{}}, cfg)
|
||||
}
|
||||
|
||||
func newTestSteps(n int) []*model.Step {
|
||||
@@ -1051,11 +1047,9 @@ func TestReporter_ReportedSteps(t *testing.T) {
|
||||
return &runnerv1.UpdateLogResponse{AckIndex: req.Index + int64(len(req.Rows))}, nil
|
||||
})
|
||||
client.On("UpdateTask", mock.Anything, mock.Anything).Maybe().Return(&runnerv1.UpdateTaskResponse{}, nil)
|
||||
taskCtx, err := structpb.NewStruct(map[string]any{})
|
||||
require.NoError(t, err)
|
||||
cfg, _ := config.LoadDefault("")
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
reporter := NewReporter(ctx, cancel, client, &runnerv1.Task{Context: taskCtx, Secrets: map[string]string{"TOKEN": "s3cr3t"}}, cfg)
|
||||
reporter := NewReporter(ctx, cancel, client, &runnerv1.Task{Context: &structpb.Struct{}, Secrets: map[string]string{"TOKEN": "s3cr3t"}}, cfg)
|
||||
reporter.RunDaemon()
|
||||
reporter.Logf("runner version")
|
||||
reporter.ResetSteps(steps)
|
||||
|
||||
@@ -117,7 +117,8 @@ func TestExportRetriesAndMatchesOTLP(t *testing.T) {
|
||||
certificate := server.TLS.Certificates[0]
|
||||
key, err := x509.MarshalPKCS8PrivateKey(certificate.PrivateKey)
|
||||
require.NoError(t, err)
|
||||
certificatePath, keyPath := filepath.Join(t.TempDir(), "cert.pem"), filepath.Join(t.TempDir(), "key.pem")
|
||||
dir := t.TempDir()
|
||||
certificatePath, keyPath := filepath.Join(dir, "cert.pem"), filepath.Join(dir, "key.pem")
|
||||
require.NoError(t, os.WriteFile(certificatePath, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certificate.Certificate[0]}), 0o600))
|
||||
require.NoError(t, os.WriteFile(keyPath, pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: key}), 0o600))
|
||||
setEnv(t, map[string]string{
|
||||
|
||||
Reference in new issue
Block a user