From d77e216674e99c8c8761d546d9dab864378280fa Mon Sep 17 00:00:00 2001 From: bircni Date: Wed, 28 Oct 2026 03:52:53 +0000 Subject: [PATCH] fix(exec): serve the v4 artifact API so upload-artifact v4 and later work (#1341) `exec` forwarded the toolkit's ArtifactService calls to an instance it never registered, and its random runtime token held no backend IDs, so `actions/upload-artifact` and `actions/download-artifact` v4 and later failed. The local artifact server now serves the v4 API, including the single Put Blob uploads docker/build-push-action uses. `exec` gives each workflow its own unsigned JWT runtime token and artifact run, so artifact names conflict and carry over between runs as on GitHub. Artifacts default to a temporary directory instead of the workspace that jobs copy. Co-authored-by: silverwind Co-authored-by: Lunny Xiao Reviewed-on: https://gitea.com/gitea/runner/pulls/1341 Reviewed-by: Lunny Xiao Reviewed-by: silverwind <2021+silverwind@noreply.gitea.com> Co-authored-by: bircni --- act/artifacts/server.go | 1 + act/artifacts/server_test.go | 87 ++++++ act/artifacts/server_v4.go | 329 +++++++++++++++++++++++ act/artifacts/token.go | 23 ++ act/artifacts/twirp.go | 55 ++++ act/runner/action_composite.go | 1 + act/runner/expression.go | 7 +- act/runner/expression_test.go | 11 +- act/runner/run_context.go | 6 +- act/runner/run_context_test.go | 4 + act/runner/runner.go | 6 + e2e/cli_test.go | 53 +++- e2e/compatibility_test.go | 1 + e2e/main_test.go | 20 ++ e2e/testdata/workflows/exec_artifact.yml | 15 ++ internal/app/cmd/exec.go | 41 ++- 16 files changed, 630 insertions(+), 30 deletions(-) create mode 100644 act/artifacts/server_v4.go create mode 100644 act/artifacts/token.go create mode 100644 act/artifacts/twirp.go create mode 100644 e2e/testdata/workflows/exec_artifact.yml diff --git a/act/artifacts/server.go b/act/artifacts/server.go index 7df8e3c5..f2292c06 100644 --- a/act/artifacts/server.go +++ b/act/artifacts/server.go @@ -218,6 +218,7 @@ func Serve(ctx context.Context, artifactPath, addr, port string) context.CancelF logger.Debugf("Artifacts base path '%s'", artifactPath) uploads(router, artifactPath) downloads(router, artifactPath) + registerV4(router, artifactPath) server := &http.Server{ Addr: net.JoinHostPort(addr, port), diff --git a/act/artifacts/server_test.go b/act/artifacts/server_test.go index 4be02f23..f4fafa9a 100644 --- a/act/artifacts/server_test.go +++ b/act/artifacts/server_test.go @@ -28,6 +28,7 @@ func TestArtifactFlow(t *testing.T) { router := http.NewServeMux() uploads(router, artifactPath) downloads(router, artifactPath) + registerV4(router, artifactPath) server := httptest.NewServer(router) defer server.Close() @@ -174,6 +175,92 @@ func TestArtifactFlow(t *testing.T) { require.Equal(t, http.StatusOK, status) require.Equal(t, content, string(data)) }) + + t.Run("v4", func(t *testing.T) { + const artifactRef = `"workflow_run_backend_id":"7","name":"50% art@v1"` + call := func(method, body string) (int, map[string]any) { + status, data := request(t, http.MethodPost, baseURL+artifactServicePath+"/"+method, strings.NewReader(body), nil) + var out map[string]any + require.NoError(t, json.Unmarshal(data, &out), string(data)) + return status, out + } + put := func(rawURL, body string) int { + status, _ := request(t, http.MethodPut, rawURL, strings.NewReader(body), nil) + return status + } + download := func() (string, http.Header) { + status, signed := call("GetSignedArtifactURL", "{"+artifactRef+"}") + require.Equal(t, http.StatusOK, status, signed) + resp, err := client.Get(signed["signedUrl"].(string)) + require.NoError(t, err) + defer resp.Body.Close() + require.Equal(t, http.StatusOK, resp.StatusCode) + body, err := io.ReadAll(resp.Body) + require.NoError(t, err) + return string(body), resp.Header + } + + status, created := call("CreateArtifact", "{"+artifactRef+"}") + require.Equal(t, http.StatusOK, status, created) + upload := created["signedUploadUrl"].(string) + require.Equal(t, http.StatusCreated, put(upload+"&comp=block&blockid=Yg", "world")) + require.Equal(t, http.StatusCreated, put(upload+"&comp=block&blockid=YQ", "hello ")) + for attempt := range 2 { + require.Equal(t, http.StatusCreated, put(upload+"&comp=blocklist", `YQYg`), "commit attempt %d", attempt) + } + + digest := "sha256:b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9" + status, _ = call("FinalizeArtifact", "{"+artifactRef+`,"size":"10"}`) + require.Equal(t, http.StatusBadRequest, status) + status, finalized := call("FinalizeArtifact", "{"+artifactRef+`,"size":"11","hash":"`+digest+`"}`) + require.Equal(t, http.StatusOK, status, finalized) + id := finalized["artifactId"].(string) + status, refinalized := call("FinalizeArtifact", "{"+artifactRef+`,"size":"10"}`) + require.Equal(t, http.StatusOK, status) + require.Equal(t, id, refinalized["artifactId"]) + status, conflict := call("CreateArtifact", "{"+artifactRef+"}") + require.Equal(t, http.StatusConflict, status) + require.Equal(t, "already_exists", conflict["code"]) + + status, listed := call("ListArtifacts", `{"workflowRunBackendId":"7","idFilter":"`+id+`"}`) + require.Equal(t, http.StatusOK, status) + artifacts := listed["artifacts"].([]any) + require.Len(t, artifacts, 1) + artifact := artifacts[0].(map[string]any) + require.Regexp(t, `^\d{4}-\d\d-\d\dT\d\d:\d\d:\d\dZ$`, artifact["createdAt"]) + delete(artifact, "createdAt") + require.Equal(t, map[string]any{"workflowRunBackendId": "7", "name": "50% art@v1", "size": "11", "digest": digest, "databaseId": id}, artifact) + _, other := call("ListArtifacts", `{"workflow_run_backend_id":"8"}`) + require.Empty(t, other["artifacts"]) + + body, header := download() + require.Equal(t, "hello world", body) + require.Equal(t, "application/zip", header.Get("Content-Type")) + require.Equal(t, "attachment; filename*=UTF-8''50%25%20art%40v1.zip", header.Get("Content-Disposition")) + + status, _ = call("DeleteArtifact", "{"+artifactRef+"}") + require.Equal(t, http.StatusOK, status) + status, missing := call("GetSignedArtifactURL", "{"+artifactRef+"}") + require.Equal(t, http.StatusNotFound, status) + require.Equal(t, "not_found", missing["code"]) + + status, created = call("CreateArtifact", "{"+artifactRef+"}") + require.Equal(t, http.StatusOK, status, created) + status, _ = request(t, http.MethodPut, created["signedUploadUrl"].(string), strings.NewReader("gz"), http.Header{"X-Ms-Blob-Content-Type": {"application/gzip"}}) + require.Equal(t, http.StatusCreated, status) + status, _ = call("FinalizeArtifact", "{"+artifactRef+`,"size":"2"}`) + require.Equal(t, http.StatusOK, status) + body, header = download() + require.Equal(t, "gz", body) + require.Equal(t, "application/gzip", header.Get("Content-Type")) + require.Equal(t, "attachment; filename*=UTF-8''50%25%20art%40v1", header.Get("Content-Disposition")) + + rerun := http.NewServeMux() + registerV4(rerun, artifactPath) + recorder := httptest.NewRecorder() + rerun.ServeHTTP(recorder, httptest.NewRequest(http.MethodPost, artifactServicePath+"/CreateArtifact", strings.NewReader("{"+artifactRef+"}"))) + require.Equal(t, http.StatusOK, recorder.Code, recorder.Body.String()) + }) } func TestSafeResolve(t *testing.T) { diff --git a/act/artifacts/server_v4.go b/act/artifacts/server_v4.go new file mode 100644 index 00000000..d2943c17 --- /dev/null +++ b/act/artifacts/server_v4.go @@ -0,0 +1,329 @@ +// Copyright 2026 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package artifacts + +import ( + "cmp" + "crypto/sha256" + "encoding/hex" + "encoding/json/v2" + "encoding/xml" + "errors" + "fmt" + "io" + "io/fs" + "net/http" + "net/url" + "os" + "path/filepath" + "slices" + "strings" + "sync" + "time" +) + +type serverV4 struct { // the subset of Gitea's v4 artifact service that the toolkit uses, unauthenticated like the v3 routes beside it + dir string + mu sync.Mutex + lastID int64 + created map[string]bool +} + +const ( + zipMimeType = "application/zip" + maxTwirpBytes = 1 << 20 +) + +func registerV4(router *http.ServeMux, baseDir string) { + server := &serverV4{dir: filepath.Join(baseDir, "v4"), lastID: time.Now().UnixMilli(), created: map[string]bool{}} // IDs stay unique across invocations sharing baseDir + router.HandleFunc("POST "+artifactServicePath+"/CreateArtifact", twirp(server.createArtifact)) + router.HandleFunc("POST "+artifactServicePath+"/FinalizeArtifact", twirp(server.finalizeArtifact)) + router.HandleFunc("POST "+artifactServicePath+"/ListArtifacts", twirp(server.listArtifacts)) + router.HandleFunc("POST "+artifactServicePath+"/GetSignedArtifactURL", twirp(server.getSignedArtifactURL)) + router.HandleFunc("POST "+artifactServicePath+"/DeleteArtifact", twirp(server.deleteArtifact)) + router.HandleFunc("PUT "+artifactServicePath+"/UploadArtifact", server.uploadArtifact) + router.HandleFunc("GET "+artifactServicePath+"/DownloadArtifact", server.downloadArtifact) +} + +type twirpError struct { + status int + code string + msg string +} + +func (e *twirpError) Error() string { return e.msg } + +func notFound(name string) error { + return &twirpError{http.StatusNotFound, "not_found", fmt.Sprintf("artifact %q not found", name)} +} + +func badRequest(format string, args ...any) error { + return &twirpError{http.StatusBadRequest, "invalid_argument", fmt.Sprintf(format, args...)} +} + +func twirp[Req, Resp any](handle func(*http.Request, Req) (Resp, error)) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + var req Req + var resp Resp + err := json.UnmarshalRead(io.LimitReader(r.Body, maxTwirpBytes), &req, json.MatchCaseInsensitiveNames(true)) // also matches the toolkit's proto field names + if err != nil { + err = badRequest("malformed request: %v", err) + } else { + resp, err = handle(r, req) + } + w.Header().Set("Content-Type", "application/json") + if err != nil { + terr := &twirpError{http.StatusInternalServerError, "internal", err.Error()} + errors.As(err, &terr) + w.WriteHeader(terr.status) + _ = json.MarshalWrite(w, map[string]string{"code": terr.code, "msg": terr.msg}) + return + } + _ = json.MarshalWrite(w, resp) + } +} + +func fileName(name string) string { // safe on any filesystem for any client-chosen name, case-insensitive ones included + sum := sha256.Sum256([]byte(name)) + return hex.EncodeToString(sum[:]) +} + +func (s *serverV4) runDir(runID string) string { + return filepath.Join(s.dir, fileName(runID)) +} + +func (s *serverV4) artifactDir(runID, name string) string { + return filepath.Join(s.runDir(runID), fileName(name)) +} + +func (s *serverV4) urlDir(r *http.Request) string { + return s.artifactDir(r.URL.Query().Get("run"), r.URL.Query().Get("name")) +} + +func readMeta(dir string) (*monolithArtifact, error) { + data, err := os.ReadFile(filepath.Join(dir, "meta.json")) + if err != nil { + return nil, err + } + var meta monolithArtifact + return &meta, json.Unmarshal(data, &meta) +} + +func writeMeta(dir string, meta *monolithArtifact) error { + data, err := json.Marshal(meta) + if err != nil { + return err + } + return os.WriteFile(filepath.Join(dir, "meta.json"), data, 0o644) +} + +func blobURL(r *http.Request, endpoint, runID, name string) string { + return "http://" + r.Host + artifactServicePath + "/" + endpoint + "?" + url.Values{"run": {runID}, "name": {name}}.Encode() +} + +func (s *serverV4) createArtifact(r *http.Request, req createArtifactRequest) (createArtifactResponse, error) { // an exec invocation re-runs its workflows, so it replaces earlier invocations' artifacts but conflicts with its own + dir := s.artifactDir(req.WorkflowRunBackendID, req.Name) + s.mu.Lock() + defer s.mu.Unlock() + if s.created[dir] { + return createArtifactResponse{}, &twirpError{http.StatusConflict, "already_exists", "an artifact with this name already exists on the workflow run"} + } + if err := os.RemoveAll(dir); err != nil { + return createArtifactResponse{}, err + } + if err := os.MkdirAll(filepath.Join(dir, "blocks"), 0o755); err != nil { + return createArtifactResponse{}, err + } + s.lastID++ + if err := writeMeta(dir, &monolithArtifact{ + WorkflowRunBackendID: req.WorkflowRunBackendID, + DatabaseID: s.lastID, + Name: req.Name, + CreatedAt: time.Now().UTC().Truncate(time.Second), + }); err != nil { + return createArtifactResponse{}, err + } + s.created[dir] = true + return createArtifactResponse{Ok: true, SignedUploadURL: blobURL(r, "UploadArtifact", req.WorkflowRunBackendID, req.Name)}, nil +} + +type blockList struct { + Blocks []string `xml:",any"` // Latest, Committed and Uncommitted all name a staged block +} + +func (s *serverV4) uploadArtifact(w http.ResponseWriter, r *http.Request) { + dir := s.urlDir(r) + if meta, err := readMeta(dir); err != nil || meta.Digest != "" { + http.Error(w, "no artifact awaits an upload", http.StatusNotFound) + return + } + contentType := r.Header.Get("x-ms-blob-content-type") + var err error + switch comp := r.URL.Query().Get("comp"); comp { + case "block": + err = writeFile(filepath.Join(dir, "blocks", fileName(r.URL.Query().Get("blockid"))), r.Body) + case "blocklist": + err = commitBlob(dir, contentType, func(out io.Writer) error { return joinBlocks(out, dir, r.Body) }) + case "": // Put Blob, which docker/build-push-action uses + err = commitBlob(dir, contentType, func(out io.Writer) error { + _, err := io.Copy(out, r.Body) + return err + }) + default: + http.Error(w, "unsupported comp "+comp, http.StatusBadRequest) + return + } + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + w.WriteHeader(http.StatusCreated) +} + +func writeFile(name string, body io.Reader) error { + file, err := os.Create(name) + if err != nil { + return err + } + _, err = io.Copy(file, body) + return errors.Join(err, file.Close()) +} + +func commitBlob(dir, contentType string, write func(io.Writer) error) error { // replaces the blob whole and keeps its content type, as Azure does + out, err := os.CreateTemp(dir, "blob") + if err != nil { + return err + } + if err = errors.Join(write(out), out.Close()); err == nil { + err = os.Rename(out.Name(), filepath.Join(dir, "blob")) + } + if err != nil { + return errors.Join(err, os.Remove(out.Name())) + } + return os.WriteFile(filepath.Join(dir, "content-type"), []byte(contentType), 0o644) +} + +func joinBlocks(out io.Writer, dir string, body io.Reader) error { // keeps the blocks for finalizeArtifact, so that a retried commit can join them again + var list blockList + if err := xml.NewDecoder(io.LimitReader(body, maxTwirpBytes)).Decode(&list); err != nil { + return err + } + for _, id := range list.Blocks { + if err := appendBlock(out, filepath.Join(dir, "blocks", fileName(id))); err != nil { + return err + } + } + return nil +} + +func appendBlock(out io.Writer, name string) error { + in, err := os.Open(name) + if err != nil { + return err + } + defer in.Close() + _, err = io.Copy(out, in) + return err +} + +func (s *serverV4) finalizeArtifact(_ *http.Request, req finalizeArtifactRequest) (finalizeArtifactResponse, error) { + dir := s.artifactDir(req.WorkflowRunBackendID, req.Name) + meta, err := readMeta(dir) + if err != nil { + return finalizeArtifactResponse{}, notFound(req.Name) + } else if meta.Digest != "" { + return finalizeArtifactResponse{Ok: true, ArtifactID: meta.DatabaseID}, nil + } + blob, err := os.Open(filepath.Join(dir, "blob")) + if err != nil { + return finalizeArtifactResponse{}, badRequest("artifact %q has no upload", req.Name) + } + defer blob.Close() + hash := sha256.New() + size, err := io.Copy(hash, blob) + if err != nil { + return finalizeArtifactResponse{}, err + } + digest := "sha256:" + hex.EncodeToString(hash.Sum(nil)) + if size != req.Size { + return finalizeArtifactResponse{}, badRequest("size %d does not match the %d bytes uploaded", req.Size, size) + } + if req.Hash != "" && req.Hash != digest { + return finalizeArtifactResponse{}, badRequest("hash %s does not match the uploaded %s", req.Hash, digest) + } + s.mu.Lock() // only after hashing, so finalizing one large artifact does not stall the others + defer s.mu.Unlock() + if current, err := readMeta(dir); err != nil || current.DatabaseID != meta.DatabaseID { + return finalizeArtifactResponse{}, notFound(req.Name) + } + meta.Size, meta.Digest = size, digest + if err := writeMeta(dir, meta); err != nil { + return finalizeArtifactResponse{}, err + } + return finalizeArtifactResponse{Ok: true, ArtifactID: meta.DatabaseID}, os.RemoveAll(filepath.Join(dir, "blocks")) +} + +func (s *serverV4) listArtifacts(_ *http.Request, req listArtifactsRequest) (listArtifactsResponse, error) { + run := s.runDir(req.WorkflowRunBackendID) + entries, err := os.ReadDir(run) + if err != nil && !errors.Is(err, fs.ErrNotExist) { + return listArtifactsResponse{}, err + } + var resp listArtifactsResponse + for _, entry := range entries { + meta, err := readMeta(filepath.Join(run, entry.Name())) + if err != nil || meta.Digest == "" || + req.NameFilter != "" && req.NameFilter != meta.Name || req.IDFilter != 0 && req.IDFilter != meta.DatabaseID { + continue + } + resp.Artifacts = append(resp.Artifacts, *meta) + } + slices.SortFunc(resp.Artifacts, func(left, right monolithArtifact) int { return cmp.Compare(left.DatabaseID, right.DatabaseID) }) + return resp, nil +} + +func (s *serverV4) getSignedArtifactURL(r *http.Request, req getSignedArtifactURLRequest) (getSignedArtifactURLResponse, error) { + if meta, err := readMeta(s.artifactDir(req.WorkflowRunBackendID, req.Name)); err != nil || meta.Digest == "" { + return getSignedArtifactURLResponse{}, notFound(req.Name) + } + return getSignedArtifactURLResponse{SignedURL: blobURL(r, "DownloadArtifact", req.WorkflowRunBackendID, req.Name)}, nil +} + +func (s *serverV4) downloadArtifact(w http.ResponseWriter, r *http.Request) { + dir := s.urlDir(r) + meta, err := readMeta(dir) + if err != nil || meta.Digest == "" { + http.Error(w, "artifact not found", http.StatusNotFound) + return + } + blob, err := os.Open(filepath.Join(dir, "blob")) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + defer blob.Close() + data, _ := os.ReadFile(filepath.Join(dir, "content-type")) + contentType, filename := string(data), meta.Name + if !strings.Contains(contentType, "/") { + contentType = zipMimeType + } + if contentType == zipMimeType && !strings.HasSuffix(filename, ".zip") { + filename += ".zip" + } + w.Header().Set("Content-Type", contentType) + w.Header().Set("Content-Disposition", "attachment; filename*=UTF-8''"+strings.ReplaceAll(url.QueryEscape(filename), "+", "%20")) // RFC 5987 attr-chars only, which the toolkit and mime.ParseMediaType both accept + http.ServeContent(w, r, "", meta.CreatedAt, blob) +} + +func (s *serverV4) deleteArtifact(_ *http.Request, req deleteArtifactRequest) (deleteArtifactResponse, error) { + dir := s.artifactDir(req.WorkflowRunBackendID, req.Name) + s.mu.Lock() + defer s.mu.Unlock() + meta, err := readMeta(dir) + if err != nil { + return deleteArtifactResponse{}, notFound(req.Name) + } + delete(s.created, dir) + return deleteArtifactResponse{Ok: true, ArtifactID: meta.DatabaseID}, os.RemoveAll(dir) +} diff --git a/act/artifacts/token.go b/act/artifacts/token.go new file mode 100644 index 00000000..89bbbfb0 --- /dev/null +++ b/act/artifacts/token.go @@ -0,0 +1,23 @@ +// Copyright 2026 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package artifacts + +import ( + "crypto/rand" + "encoding/base64" + "encoding/json/v2" + "time" +) + +func LocalRuntimeToken(run string, lifetime time.Duration) string { // an unsecured JWT with the claims the toolkit and buildx read, its run hashed for scp and its jti an unguessable bearer + now := time.Now() + claims, _ := json.Marshal(map[string]any{ + "scp": "Actions.Results:" + fileName(run) + ":1", + "ac": `[{"Scope":"","Permission":3}]`, + "nbf": now.Unix(), + "exp": now.Add(lifetime).Unix(), + "jti": rand.Text(), + }) + return base64.RawURLEncoding.EncodeToString([]byte(`{"alg":"none","typ":"JWT"}`)) + "." + base64.RawURLEncoding.EncodeToString(claims) + "." +} diff --git a/act/artifacts/twirp.go b/act/artifacts/twirp.go new file mode 100644 index 00000000..d3e1b435 --- /dev/null +++ b/act/artifacts/twirp.go @@ -0,0 +1,55 @@ +// Copyright 2026 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package artifacts + +import "time" + +const artifactServicePath = "/twirp/github.actions.results.api.v1.ArtifactService" + +type ( // the ArtifactService messages in the protobuf JSON mapping, which writes int64 as strings + createArtifactResponse struct { + Ok bool `json:"ok"` + SignedUploadURL string `json:"signedUploadUrl"` + } + + finalizeArtifactRequest struct { + WorkflowRunBackendID string + Name string + Size int64 `json:",string"` + Hash string + } + finalizeArtifactResponse struct { + Ok bool `json:"ok"` + ArtifactID int64 `json:"artifactId,string"` + } + + listArtifactsRequest struct { + WorkflowRunBackendID string + NameFilter string + IDFilter int64 `json:",string"` + } + listArtifactsResponse struct { + Artifacts []monolithArtifact `json:"artifacts"` + } + monolithArtifact struct { + WorkflowRunBackendID string `json:"workflowRunBackendId"` // the toolkit sends it back to GetSignedArtifactURL + DatabaseID int64 `json:"databaseId,string"` + Name string `json:"name"` + Size int64 `json:"size,string"` + CreatedAt time.Time `json:"createdAt"` // whole seconds, the toolkit rejects the trimmed fractions json writes + Digest string `json:"digest"` // empty until the upload is finalized + } + + getSignedArtifactURLRequest struct { + WorkflowRunBackendID string + Name string + } + getSignedArtifactURLResponse struct { + SignedURL string `json:"signedUrl"` + } + + createArtifactRequest = getSignedArtifactURLRequest + deleteArtifactRequest = getSignedArtifactURLRequest + deleteArtifactResponse = finalizeArtifactResponse +) diff --git a/act/runner/action_composite.go b/act/runner/action_composite.go index 3b3eb3b5..2ec1385f 100644 --- a/act/runner/action_composite.go +++ b/act/runner/action_composite.go @@ -110,6 +110,7 @@ func newCompositeRunContext(ctx context.Context, parent *RunContext, step action kubernetes: parent.kubernetes, jobIndex: parent.jobIndex, jobTotal: parent.jobTotal, + runtimeToken: parent.runtimeToken, } compositerc.setCompositeActionEnv(env) compositerc.ExprEval = compositerc.NewExpressionEvaluator(ctx) diff --git a/act/runner/expression.go b/act/runner/expression.go index 7e55edd3..1115cf46 100644 --- a/act/runner/expression.go +++ b/act/runner/expression.go @@ -332,5 +332,10 @@ func getWorkflowSecrets(rc *RunContext) map[string]string { if rc.caller != nil { return rc.workflowCallSecrets } - return rc.Config.Secrets + if rc.runtimeToken == "" { + return rc.Config.Secrets + } + secrets := map[string]string{"ACTIONS_RUNTIME_TOKEN": rc.runtimeToken} // as the daemon exposes its token + maps.Copy(secrets, rc.Config.Secrets) + return secrets } diff --git a/act/runner/expression_test.go b/act/runner/expression_test.go index a765489e..5e36c84a 100644 --- a/act/runner/expression_test.go +++ b/act/runner/expression_test.go @@ -504,7 +504,7 @@ jobs: assert.Equal(t, "workflow", env["INPUT_FALLBACK"]) } -func TestResolveWorkflowCallToleratesMismatchesAndPassesInheritedSecretsVerbatim(t *testing.T) { +func TestResolveWorkflowCallToleratesMismatchesAndPassesInheritedSecretsAndRuntimeToken(t *testing.T) { callerWorkflow, err := model.ReadWorkflow(strings.NewReader(` jobs: call: @@ -527,7 +527,11 @@ jobs: require.NoError(t, err) notCallable, err := model.ReadWorkflow(strings.NewReader("on: push\njobs:\n test: {}\n")) require.NoError(t, err) - config := &Config{Secrets: map[string]string{"token": "${{ github.token }}"}, PresetGitHubContext: &model.GithubContext{}} + config := &Config{ + Secrets: map[string]string{"token": "${{ github.token }}"}, + PresetGitHubContext: &model.GithubContext{}, + RuntimeTokens: map[*model.Workflow]string{callerWorkflow: "caller-run"}, + } resolve := func(callerJobID string, workflow *model.Workflow) *RunContext { parent, err := (&runnerImpl{config: config}).newRunContext(t.Context(), &model.Run{Workflow: callerWorkflow, JobID: callerJobID}, nil) require.NoError(t, err) @@ -539,9 +543,10 @@ jobs: called := resolve("call", callee) assert.Equal(t, map[string]any{"flag": true}, called.workflowCallInputs) assert.Equal(t, map[string]string{"undeclared": "${{ github.token }}"}, called.workflowCallSecrets) + assert.Equal(t, "caller-run", called.GetEnv()["ACTIONS_RUNTIME_TOKEN"]) inherited := resolve("inherit", notCallable) assert.Empty(t, inherited.workflowCallInputs) - assert.Equal(t, config.Secrets, inherited.workflowCallSecrets) + assert.Equal(t, map[string]string{"token": "${{ github.token }}", "ACTIONS_RUNTIME_TOKEN": "caller-run"}, inherited.workflowCallSecrets) } func TestJobNameMasksSecrets(t *testing.T) { diff --git a/act/runner/run_context.go b/act/runner/run_context.go index 7b1cfcee..9b3ecb45 100644 --- a/act/runner/run_context.go +++ b/act/runner/run_context.go @@ -81,6 +81,7 @@ type RunContext struct { jobRunDefaults model.RunDefaults // defaults.run, resolved once at job setup as GitHub does platformImage string // container.image or the runs-on pick, resolved once by isEnabled jobIndex, jobTotal int // strategy.job-index and job-total, total 0 when unknown + runtimeToken string // ACTIONS_RUNTIME_TOKEN of the workflow run, shared with its called workflows and composite actions jobEnv map[string]string // workflow and job env as evaluated for the container and service specs dockerActionImages map[*model.Step]string // Dockerfile action images built for this job, removed when it ends @@ -193,6 +194,9 @@ func (rc *RunContext) GetEnv() map[string]string { rc.Env = mergeMaps(rc.Run.Workflow.Env, job.Environment(), rc.Config.Env) } } + if rc.runtimeToken != "" { + rc.Env["ACTIONS_RUNTIME_TOKEN"] = rc.runtimeToken // as the daemon's Config.Env carries its token + } } if !rc.Config.DisableActEnv { rc.Env["ACT"] = "true" @@ -1662,7 +1666,7 @@ func (rc *RunContext) withGithubEnv(ctx context.Context, github *model.GithubCon if rc.Config.ArtifactServerPath != "" { env["ACTIONS_RUNTIME_URL"] = cmp.Or(os.Getenv("ACTIONS_RUNTIME_URL"), "http://"+net.JoinHostPort(rc.Config.ArtifactServerAddr, rc.Config.ArtifactServerPort)+"/") - env["ACTIONS_RUNTIME_TOKEN"] = cmp.Or(os.Getenv("ACTIONS_RUNTIME_TOKEN"), "token") + env["ACTIONS_RUNTIME_TOKEN"] = cmp.Or(rc.runtimeToken, os.Getenv("ACTIONS_RUNTIME_TOKEN"), "token") } if imageOS := rc.imageOS(ctx); imageOS != "" { diff --git a/act/runner/run_context_test.go b/act/runner/run_context_test.go index 06ce5426..3f91357c 100644 --- a/act/runner/run_context_test.go +++ b/act/runner/run_context_test.go @@ -1631,4 +1631,8 @@ func TestRunContextWithGithubEnvRunnerValues(t *testing.T) { rc.withGithubEnv(ctx, &model.GithubContext{}, env) assert.Equal(t, ci, env["CI"]) } + + rc.runtimeToken = "run-token" + newTestCompositeRunContext(t, rc).withGithubEnv(ctx, &model.GithubContext{}, env) + assert.Equal(t, "run-token", env["ACTIONS_RUNTIME_TOKEN"]) } diff --git a/act/runner/runner.go b/act/runner/runner.go index de1b2e51..1542ce13 100644 --- a/act/runner/runner.go +++ b/act/runner/runner.go @@ -41,6 +41,7 @@ type Config struct { Env map[string]string // env for containers Secrets map[string]string // list of secrets ExtraMasks []string // values to hide that are not in Secrets, such as the proxy password + RuntimeTokens map[*model.Workflow]string // ACTIONS_RUNTIME_TOKEN per workflow run, as each is its own artifact run Vars map[string]string // list of vars Token string // GitHub token InsecureSecrets bool // switch hiding output when printing to terminal @@ -355,6 +356,11 @@ func (runner *runnerImpl) newCombinationRunContext(ctx context.Context, run *mod caller: runner.caller, jobIndex: index, jobTotal: total, + + runtimeToken: runner.config.RuntimeTokens[run.Workflow], + } + if runner.caller != nil { + rc.runtimeToken = runner.caller.runContext.runtimeToken } if err := rc.resolveWorkflowCall(ctx); err != nil { return nil, err diff --git a/e2e/cli_test.go b/e2e/cli_test.go index f66b9a25..eb89eb3c 100644 --- a/e2e/cli_test.go +++ b/e2e/cli_test.go @@ -12,6 +12,8 @@ import ( "os" "os/exec" "path/filepath" + "slices" + "strings" "sync" "syscall" "testing" @@ -47,9 +49,9 @@ func testCLIRegisterDaemon(t *testing.T) { } dir := t.TempDir() - bin := filepath.Join(dir, "gitea-runner") - if out, err := exec.CommandContext(ctx, "go", "build", "-o", bin, "..").CombinedOutput(); err != nil { - t.Fatalf("build runner: %v\n%s", err, out) + bin, err := runnerBinary() + if err != nil { + t.Fatal(err) } cfgFile := filepath.Join(dir, "config.yaml") @@ -119,3 +121,48 @@ func requireSuccessOrDumpDaemon(t *testing.T, api *GiteaAPI, repo string, runID requireSuccess(t, api, repo, runID) } } + +func testExecArtifacts(t *testing.T) { + t.Parallel() + ctx := t.Context() + + bin, err := runnerBinary() + if err != nil { + t.Fatal(err) + } + workflow, err := os.ReadFile("testdata/workflows/exec_artifact.yml") + if err != nil { + t.Fatal(err) + } + workflows := t.TempDir() + for _, name := range []string{"a.yml", "b.yml"} { + if err := os.WriteFile(filepath.Join(workflows, name), workflow, 0o644); err != nil { + t.Fatal(err) + } + } + + workdir := t.TempDir() + for _, args := range [][]string{ + {"init", "-q"}, + {"remote", "add", "origin", "https://example.com/e2e/exec.git"}, + {"-c", "user.name=e2e", "-c", "user.email=e2e@example.com", "commit", "-q", "--allow-empty", "-m", "init"}, + } { + if out, err := exec.CommandContext(ctx, "git", append([]string{"-C", workdir}, args...)...).CombinedOutput(); err != nil { + t.Fatalf("git %v: %v\n%s", args, err, out) + } + } + + artifacts := t.TempDir() + args := []string{"exec", "-C", workdir, "-W", workflows, "-i", os.Getenv("E2E_JOB_IMAGE"), "--artifact-server-path", artifacts} + if fixture.network != "" { + args = append(args, "--network", fixture.network) + } + cmd := exec.CommandContext(ctx, bin, args...) + cmd.Env = slices.DeleteFunc(os.Environ(), func(kv string) bool { return strings.HasPrefix(kv, "ACTIONS_") }) + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("exec: %v\n%s", err, out) + } + if blobs, _ := filepath.Glob(filepath.Join(artifacts, "v4", "*", "*", "blob")); len(blobs) != 2 { + t.Fatalf("want one v4 artifact blob per workflow run, got %v", blobs) + } +} diff --git a/e2e/compatibility_test.go b/e2e/compatibility_test.go index 9d0229b1..d0c47971 100644 --- a/e2e/compatibility_test.go +++ b/e2e/compatibility_test.go @@ -30,6 +30,7 @@ func TestCompatibility(t *testing.T) { t.Run("cancellation_and_log_streaming", testRunCancellation) t.Run("dispatch", testWorkflowDispatch) t.Run("builtin_checkout", testBuiltinCheckout) + t.Run("exec_artifacts", testExecArtifacts) t.Run("moving_action_ref", testMovingActionRef) t.Run("ephemeral", testEphemeralRunner) t.Run("failure_paths", testFailurePaths) diff --git a/e2e/main_test.go b/e2e/main_test.go index 814e3e73..770bd3b5 100644 --- a/e2e/main_test.go +++ b/e2e/main_test.go @@ -6,11 +6,15 @@ package e2e import ( + "cmp" "context" "fmt" "os" + "os/exec" + "path/filepath" "regexp" "strings" + "sync" "testing" "time" @@ -25,6 +29,16 @@ var fixture *GiteaFixture var skipReason string +var binDir string + +var runnerBinary = sync.OnceValues(func() (string, error) { + bin := filepath.Join(binDir, "gitea-runner") + if out, err := exec.Command(cmp.Or(os.Getenv("GO"), "go"), "build", "-o", bin, "..").CombinedOutput(); err != nil { + return "", fmt.Errorf("build runner: %w\n%s", err, out) + } + return bin, nil +}) + func TestMain(m *testing.M) { os.Exit(runSuite(m)) } @@ -47,6 +61,12 @@ func runSuite(m *testing.M) int { fixture = f defer func() { _ = fixture.Close(context.Background()) }() + if binDir, err = os.MkdirTemp("", "gitea-runner-e2e-"); err != nil { + fmt.Fprintf(os.Stderr, "create binary dir: %v\n", err) + return 1 + } + defer os.RemoveAll(binDir) + fmt.Fprintf(os.Stderr, "gitea fixture: image=%s version=%s\n", fixture.image, fixture.version) return m.Run() diff --git a/e2e/testdata/workflows/exec_artifact.yml b/e2e/testdata/workflows/exec_artifact.yml new file mode 100644 index 00000000..7708cbe9 --- /dev/null +++ b/e2e/testdata/workflows/exec_artifact.yml @@ -0,0 +1,15 @@ +on: push +jobs: + roundtrip: + runs-on: ubuntu-latest + steps: + - run: mkdir -p out && echo exec-payload > out/data.txt + - uses: actions/upload-artifact@v7 + with: + name: exec-artifact + path: out/data.txt + - uses: actions/download-artifact@v8 + with: + name: exec-artifact + path: got + - run: grep -q exec-payload got/data.txt diff --git a/internal/app/cmd/exec.go b/internal/app/cmd/exec.go index 8e6a7f78..0dc41305 100644 --- a/internal/app/cmd/exec.go +++ b/internal/app/cmd/exec.go @@ -7,12 +7,11 @@ package cmd import ( "cmp" "context" - "crypto/rand" - "encoding/hex" "encoding/json/v2" "errors" "fmt" "maps" + "net" "os" "path/filepath" "slices" @@ -481,24 +480,7 @@ func runExec(ctx context.Context, execArgs *executeArgs) func(cmd *cobra.Command execArgs.artifactServerPath = tempDir } - // Register ACTIONS_RUNTIME_TOKEN against local cache server env := execArgs.LoadEnvs() - const actionsRuntimeTokenEnvName = "ACTIONS_RUNTIME_TOKEN" - actionsRuntimeToken := env[actionsRuntimeTokenEnvName] - if actionsRuntimeToken == "" { - actionsRuntimeToken = os.Getenv(actionsRuntimeTokenEnvName) - } - if actionsRuntimeToken == "" { - tmpBranch := make([]byte, 12) - if _, err := rand.Read(tmpBranch); err != nil { - actionsRuntimeToken = "token" - } else { - actionsRuntimeToken = hex.EncodeToString(tmpBranch) - } - env[actionsRuntimeTokenEnvName] = actionsRuntimeToken - os.Setenv(actionsRuntimeTokenEnvName, actionsRuntimeToken) - } - handler.RegisterJob(actionsRuntimeToken, artifactcache.JobCredential{Repo: "__local/__exec"}) // no service aliases: exec builds one config for the whole plan run.BypassProxyForDockerHost(os.Getenv("DOCKER_HOST")) @@ -517,7 +499,22 @@ func runExec(ctx context.Context, execArgs *executeArgs) func(cmd *cobra.Command } config.Env["ACT_EXEC"] = "true" - config.Secrets[actionsRuntimeTokenEnvName] = actionsRuntimeToken + inheritedToken := cmp.Or(env["ACTIONS_RUNTIME_TOKEN"], os.Getenv("ACTIONS_RUNTIME_TOKEN")) + config.RuntimeTokens = map[*model.Workflow]string{} + sameFile := map[string]int{} + for _, stage := range plan.Stages { + for _, run := range stage.Runs { + if config.RuntimeTokens[run.Workflow] != "" { + continue + } + sameFile[run.Workflow.File]++ + workflowRun := fmt.Sprintf("%s/%s/%d", cmp.Or(env["GITHUB_RUN_ID"], "1"), run.Workflow.File, sameFile[run.Workflow.File]) // each workflow its own run as on GitHub, also when file names repeat across directories + token := cmp.Or(inheritedToken, artifacts.LocalRuntimeToken(workflowRun, maxLifetime)) + config.RuntimeTokens[run.Workflow] = token + config.ExtraMasks = append(config.ExtraMasks, token) + handler.RegisterJob(token, artifactcache.JobCredential{Repo: "__local/__exec", Results: "http://" + net.JoinHostPort(execArgs.artifactServerAddr, execArgs.artifactServerPort)}) + } + } config.Token = cmp.Or(config.Secrets["GITEA_TOKEN"], config.Secrets["GITHUB_TOKEN"]) @@ -580,9 +577,9 @@ func loadExecCmd(ctx context.Context) *cobra.Command { execCmd.Flags().StringArrayVarP(&execArg.containerCapAdd, "container-cap-add", "", []string{}, "kernel capabilities to add to the workflow containers (e.g. --container-cap-add SYS_PTRACE)") execCmd.Flags().StringArrayVarP(&execArg.containerCapDrop, "container-cap-drop", "", []string{}, "kernel capabilities to remove from the workflow containers (e.g. --container-cap-drop SYS_PTRACE)") execCmd.Flags().StringVarP(&execArg.containerOptions, "container-opts", "", "", "container options") - execCmd.PersistentFlags().StringVarP(&execArg.artifactServerPath, "artifact-server-path", "", ".", "Defines the path where the artifact server stores uploads and retrieves downloads from. If not specified the artifact server will not start.") + execCmd.PersistentFlags().StringVarP(&execArg.artifactServerPath, "artifact-server-path", "", "", "Defines the path where the artifact server stores uploads and retrieves downloads from, outside the workspace that jobs copy. Defaults to a temporary directory removed on exit.") execCmd.PersistentFlags().StringVarP(&execArg.artifactServerAddr, "artifact-server-addr", "", "", "Defines the address where the artifact server listens") - execCmd.PersistentFlags().StringVarP(&execArg.artifactServerPort, "artifact-server-port", "", "34567", "Defines the port where the artifact server listens (will only bind to localhost).") + execCmd.PersistentFlags().StringVarP(&execArg.artifactServerPort, "artifact-server-port", "", "34567", "Defines the port where the artifact server listens") execCmd.PersistentFlags().StringVarP(&execArg.defaultActionsURL, "default-actions-url", "", "https://github.com", "Defines the default url of action instance.") execCmd.PersistentFlags().BoolVarP(&execArg.noSkipCheckout, "no-skip-checkout", "", false, "Do not skip actions/checkout") execCmd.PersistentFlags().BoolVarP(&execArg.debug, "debug", "d", false, "enable debug log")