mirror of
https://gitea.com/gitea/runner.git
synced 2026-11-04 22:37:55 +00:00
fix(exec): serve the v4 artifact API so upload-artifact v4 and later work (#1341)
`exec` forwarded the toolkit's ArtifactService calls to an instance it never registered, and its random runtime token held no backend IDs, so `actions/upload-artifact` and `actions/download-artifact` v4 and later failed. The local artifact server now serves the v4 API, including the single Put Blob uploads docker/build-push-action uses. `exec` gives each workflow its own unsigned JWT runtime token and artifact run, so artifact names conflict and carry over between runs as on GitHub. Artifacts default to a temporary directory instead of the workspace that jobs copy. Co-authored-by: silverwind <[email protected]> Co-authored-by: Lunny Xiao <[email protected]> Reviewed-on: https://gitea.com/gitea/runner/pulls/1341 Reviewed-by: Lunny Xiao <[email protected]> Reviewed-by: silverwind <[email protected]> Co-authored-by: bircni <[email protected]>
This commit is contained in:
1 parent
a76ccd7020
commit
d77e216674
16 files changed
+630
-30
No files matched your search
@@ -218,6 +218,7 @@ func Serve(ctx context.Context, artifactPath, addr, port string) context.CancelF
|
||||
logger.Debugf("Artifacts base path '%s'", artifactPath)
|
||||
uploads(router, artifactPath)
|
||||
downloads(router, artifactPath)
|
||||
registerV4(router, artifactPath)
|
||||
|
||||
server := &http.Server{
|
||||
Addr: net.JoinHostPort(addr, port),
|
||||
|
||||
@@ -28,6 +28,7 @@ func TestArtifactFlow(t *testing.T) {
|
||||
router := http.NewServeMux()
|
||||
uploads(router, artifactPath)
|
||||
downloads(router, artifactPath)
|
||||
registerV4(router, artifactPath)
|
||||
server := httptest.NewServer(router)
|
||||
defer server.Close()
|
||||
|
||||
@@ -174,6 +175,92 @@ func TestArtifactFlow(t *testing.T) {
|
||||
require.Equal(t, http.StatusOK, status)
|
||||
require.Equal(t, content, string(data))
|
||||
})
|
||||
|
||||
t.Run("v4", func(t *testing.T) {
|
||||
const artifactRef = `"workflow_run_backend_id":"7","name":"50% art@v1"`
|
||||
call := func(method, body string) (int, map[string]any) {
|
||||
status, data := request(t, http.MethodPost, baseURL+artifactServicePath+"/"+method, strings.NewReader(body), nil)
|
||||
var out map[string]any
|
||||
require.NoError(t, json.Unmarshal(data, &out), string(data))
|
||||
return status, out
|
||||
}
|
||||
put := func(rawURL, body string) int {
|
||||
status, _ := request(t, http.MethodPut, rawURL, strings.NewReader(body), nil)
|
||||
return status
|
||||
}
|
||||
download := func() (string, http.Header) {
|
||||
status, signed := call("GetSignedArtifactURL", "{"+artifactRef+"}")
|
||||
require.Equal(t, http.StatusOK, status, signed)
|
||||
resp, err := client.Get(signed["signedUrl"].(string))
|
||||
require.NoError(t, err)
|
||||
defer resp.Body.Close()
|
||||
require.Equal(t, http.StatusOK, resp.StatusCode)
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
require.NoError(t, err)
|
||||
return string(body), resp.Header
|
||||
}
|
||||
|
||||
status, created := call("CreateArtifact", "{"+artifactRef+"}")
|
||||
require.Equal(t, http.StatusOK, status, created)
|
||||
upload := created["signedUploadUrl"].(string)
|
||||
require.Equal(t, http.StatusCreated, put(upload+"&comp=block&blockid=Yg", "world"))
|
||||
require.Equal(t, http.StatusCreated, put(upload+"&comp=block&blockid=YQ", "hello "))
|
||||
for attempt := range 2 {
|
||||
require.Equal(t, http.StatusCreated, put(upload+"&comp=blocklist", `<?xml version="1.0"?><BlockList><Latest>YQ</Latest><Latest>Yg</Latest></BlockList>`), "commit attempt %d", attempt)
|
||||
}
|
||||
|
||||
digest := "sha256:b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9"
|
||||
status, _ = call("FinalizeArtifact", "{"+artifactRef+`,"size":"10"}`)
|
||||
require.Equal(t, http.StatusBadRequest, status)
|
||||
status, finalized := call("FinalizeArtifact", "{"+artifactRef+`,"size":"11","hash":"`+digest+`"}`)
|
||||
require.Equal(t, http.StatusOK, status, finalized)
|
||||
id := finalized["artifactId"].(string)
|
||||
status, refinalized := call("FinalizeArtifact", "{"+artifactRef+`,"size":"10"}`)
|
||||
require.Equal(t, http.StatusOK, status)
|
||||
require.Equal(t, id, refinalized["artifactId"])
|
||||
status, conflict := call("CreateArtifact", "{"+artifactRef+"}")
|
||||
require.Equal(t, http.StatusConflict, status)
|
||||
require.Equal(t, "already_exists", conflict["code"])
|
||||
|
||||
status, listed := call("ListArtifacts", `{"workflowRunBackendId":"7","idFilter":"`+id+`"}`)
|
||||
require.Equal(t, http.StatusOK, status)
|
||||
artifacts := listed["artifacts"].([]any)
|
||||
require.Len(t, artifacts, 1)
|
||||
artifact := artifacts[0].(map[string]any)
|
||||
require.Regexp(t, `^\d{4}-\d\d-\d\dT\d\d:\d\d:\d\dZ$`, artifact["createdAt"])
|
||||
delete(artifact, "createdAt")
|
||||
require.Equal(t, map[string]any{"workflowRunBackendId": "7", "name": "50% art@v1", "size": "11", "digest": digest, "databaseId": id}, artifact)
|
||||
_, other := call("ListArtifacts", `{"workflow_run_backend_id":"8"}`)
|
||||
require.Empty(t, other["artifacts"])
|
||||
|
||||
body, header := download()
|
||||
require.Equal(t, "hello world", body)
|
||||
require.Equal(t, "application/zip", header.Get("Content-Type"))
|
||||
require.Equal(t, "attachment; filename*=UTF-8''50%25%20art%40v1.zip", header.Get("Content-Disposition"))
|
||||
|
||||
status, _ = call("DeleteArtifact", "{"+artifactRef+"}")
|
||||
require.Equal(t, http.StatusOK, status)
|
||||
status, missing := call("GetSignedArtifactURL", "{"+artifactRef+"}")
|
||||
require.Equal(t, http.StatusNotFound, status)
|
||||
require.Equal(t, "not_found", missing["code"])
|
||||
|
||||
status, created = call("CreateArtifact", "{"+artifactRef+"}")
|
||||
require.Equal(t, http.StatusOK, status, created)
|
||||
status, _ = request(t, http.MethodPut, created["signedUploadUrl"].(string), strings.NewReader("gz"), http.Header{"X-Ms-Blob-Content-Type": {"application/gzip"}})
|
||||
require.Equal(t, http.StatusCreated, status)
|
||||
status, _ = call("FinalizeArtifact", "{"+artifactRef+`,"size":"2"}`)
|
||||
require.Equal(t, http.StatusOK, status)
|
||||
body, header = download()
|
||||
require.Equal(t, "gz", body)
|
||||
require.Equal(t, "application/gzip", header.Get("Content-Type"))
|
||||
require.Equal(t, "attachment; filename*=UTF-8''50%25%20art%40v1", header.Get("Content-Disposition"))
|
||||
|
||||
rerun := http.NewServeMux()
|
||||
registerV4(rerun, artifactPath)
|
||||
recorder := httptest.NewRecorder()
|
||||
rerun.ServeHTTP(recorder, httptest.NewRequest(http.MethodPost, artifactServicePath+"/CreateArtifact", strings.NewReader("{"+artifactRef+"}")))
|
||||
require.Equal(t, http.StatusOK, recorder.Code, recorder.Body.String())
|
||||
})
|
||||
}
|
||||
|
||||
func TestSafeResolve(t *testing.T) {
|
||||
|
||||
@@ -0,0 +1,329 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package artifacts
|
||||
|
||||
import (
|
||||
"cmp"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json/v2"
|
||||
"encoding/xml"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
type serverV4 struct { // the subset of Gitea's v4 artifact service that the toolkit uses, unauthenticated like the v3 routes beside it
|
||||
dir string
|
||||
mu sync.Mutex
|
||||
lastID int64
|
||||
created map[string]bool
|
||||
}
|
||||
|
||||
const (
|
||||
zipMimeType = "application/zip"
|
||||
maxTwirpBytes = 1 << 20
|
||||
)
|
||||
|
||||
func registerV4(router *http.ServeMux, baseDir string) {
|
||||
server := &serverV4{dir: filepath.Join(baseDir, "v4"), lastID: time.Now().UnixMilli(), created: map[string]bool{}} // IDs stay unique across invocations sharing baseDir
|
||||
router.HandleFunc("POST "+artifactServicePath+"/CreateArtifact", twirp(server.createArtifact))
|
||||
router.HandleFunc("POST "+artifactServicePath+"/FinalizeArtifact", twirp(server.finalizeArtifact))
|
||||
router.HandleFunc("POST "+artifactServicePath+"/ListArtifacts", twirp(server.listArtifacts))
|
||||
router.HandleFunc("POST "+artifactServicePath+"/GetSignedArtifactURL", twirp(server.getSignedArtifactURL))
|
||||
router.HandleFunc("POST "+artifactServicePath+"/DeleteArtifact", twirp(server.deleteArtifact))
|
||||
router.HandleFunc("PUT "+artifactServicePath+"/UploadArtifact", server.uploadArtifact)
|
||||
router.HandleFunc("GET "+artifactServicePath+"/DownloadArtifact", server.downloadArtifact)
|
||||
}
|
||||
|
||||
type twirpError struct {
|
||||
status int
|
||||
code string
|
||||
msg string
|
||||
}
|
||||
|
||||
func (e *twirpError) Error() string { return e.msg }
|
||||
|
||||
func notFound(name string) error {
|
||||
return &twirpError{http.StatusNotFound, "not_found", fmt.Sprintf("artifact %q not found", name)}
|
||||
}
|
||||
|
||||
func badRequest(format string, args ...any) error {
|
||||
return &twirpError{http.StatusBadRequest, "invalid_argument", fmt.Sprintf(format, args...)}
|
||||
}
|
||||
|
||||
func twirp[Req, Resp any](handle func(*http.Request, Req) (Resp, error)) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
var req Req
|
||||
var resp Resp
|
||||
err := json.UnmarshalRead(io.LimitReader(r.Body, maxTwirpBytes), &req, json.MatchCaseInsensitiveNames(true)) // also matches the toolkit's proto field names
|
||||
if err != nil {
|
||||
err = badRequest("malformed request: %v", err)
|
||||
} else {
|
||||
resp, err = handle(r, req)
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
if err != nil {
|
||||
terr := &twirpError{http.StatusInternalServerError, "internal", err.Error()}
|
||||
errors.As(err, &terr)
|
||||
w.WriteHeader(terr.status)
|
||||
_ = json.MarshalWrite(w, map[string]string{"code": terr.code, "msg": terr.msg})
|
||||
return
|
||||
}
|
||||
_ = json.MarshalWrite(w, resp)
|
||||
}
|
||||
}
|
||||
|
||||
func fileName(name string) string { // safe on any filesystem for any client-chosen name, case-insensitive ones included
|
||||
sum := sha256.Sum256([]byte(name))
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
func (s *serverV4) runDir(runID string) string {
|
||||
return filepath.Join(s.dir, fileName(runID))
|
||||
}
|
||||
|
||||
func (s *serverV4) artifactDir(runID, name string) string {
|
||||
return filepath.Join(s.runDir(runID), fileName(name))
|
||||
}
|
||||
|
||||
func (s *serverV4) urlDir(r *http.Request) string {
|
||||
return s.artifactDir(r.URL.Query().Get("run"), r.URL.Query().Get("name"))
|
||||
}
|
||||
|
||||
func readMeta(dir string) (*monolithArtifact, error) {
|
||||
data, err := os.ReadFile(filepath.Join(dir, "meta.json"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var meta monolithArtifact
|
||||
return &meta, json.Unmarshal(data, &meta)
|
||||
}
|
||||
|
||||
func writeMeta(dir string, meta *monolithArtifact) error {
|
||||
data, err := json.Marshal(meta)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return os.WriteFile(filepath.Join(dir, "meta.json"), data, 0o644)
|
||||
}
|
||||
|
||||
func blobURL(r *http.Request, endpoint, runID, name string) string {
|
||||
return "http://" + r.Host + artifactServicePath + "/" + endpoint + "?" + url.Values{"run": {runID}, "name": {name}}.Encode()
|
||||
}
|
||||
|
||||
func (s *serverV4) createArtifact(r *http.Request, req createArtifactRequest) (createArtifactResponse, error) { // an exec invocation re-runs its workflows, so it replaces earlier invocations' artifacts but conflicts with its own
|
||||
dir := s.artifactDir(req.WorkflowRunBackendID, req.Name)
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.created[dir] {
|
||||
return createArtifactResponse{}, &twirpError{http.StatusConflict, "already_exists", "an artifact with this name already exists on the workflow run"}
|
||||
}
|
||||
if err := os.RemoveAll(dir); err != nil {
|
||||
return createArtifactResponse{}, err
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Join(dir, "blocks"), 0o755); err != nil {
|
||||
return createArtifactResponse{}, err
|
||||
}
|
||||
s.lastID++
|
||||
if err := writeMeta(dir, &monolithArtifact{
|
||||
WorkflowRunBackendID: req.WorkflowRunBackendID,
|
||||
DatabaseID: s.lastID,
|
||||
Name: req.Name,
|
||||
CreatedAt: time.Now().UTC().Truncate(time.Second),
|
||||
}); err != nil {
|
||||
return createArtifactResponse{}, err
|
||||
}
|
||||
s.created[dir] = true
|
||||
return createArtifactResponse{Ok: true, SignedUploadURL: blobURL(r, "UploadArtifact", req.WorkflowRunBackendID, req.Name)}, nil
|
||||
}
|
||||
|
||||
type blockList struct {
|
||||
Blocks []string `xml:",any"` // Latest, Committed and Uncommitted all name a staged block
|
||||
}
|
||||
|
||||
func (s *serverV4) uploadArtifact(w http.ResponseWriter, r *http.Request) {
|
||||
dir := s.urlDir(r)
|
||||
if meta, err := readMeta(dir); err != nil || meta.Digest != "" {
|
||||
http.Error(w, "no artifact awaits an upload", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
contentType := r.Header.Get("x-ms-blob-content-type")
|
||||
var err error
|
||||
switch comp := r.URL.Query().Get("comp"); comp {
|
||||
case "block":
|
||||
err = writeFile(filepath.Join(dir, "blocks", fileName(r.URL.Query().Get("blockid"))), r.Body)
|
||||
case "blocklist":
|
||||
err = commitBlob(dir, contentType, func(out io.Writer) error { return joinBlocks(out, dir, r.Body) })
|
||||
case "": // Put Blob, which docker/build-push-action uses
|
||||
err = commitBlob(dir, contentType, func(out io.Writer) error {
|
||||
_, err := io.Copy(out, r.Body)
|
||||
return err
|
||||
})
|
||||
default:
|
||||
http.Error(w, "unsupported comp "+comp, http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusCreated)
|
||||
}
|
||||
|
||||
func writeFile(name string, body io.Reader) error {
|
||||
file, err := os.Create(name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = io.Copy(file, body)
|
||||
return errors.Join(err, file.Close())
|
||||
}
|
||||
|
||||
func commitBlob(dir, contentType string, write func(io.Writer) error) error { // replaces the blob whole and keeps its content type, as Azure does
|
||||
out, err := os.CreateTemp(dir, "blob")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err = errors.Join(write(out), out.Close()); err == nil {
|
||||
err = os.Rename(out.Name(), filepath.Join(dir, "blob"))
|
||||
}
|
||||
if err != nil {
|
||||
return errors.Join(err, os.Remove(out.Name()))
|
||||
}
|
||||
return os.WriteFile(filepath.Join(dir, "content-type"), []byte(contentType), 0o644)
|
||||
}
|
||||
|
||||
func joinBlocks(out io.Writer, dir string, body io.Reader) error { // keeps the blocks for finalizeArtifact, so that a retried commit can join them again
|
||||
var list blockList
|
||||
if err := xml.NewDecoder(io.LimitReader(body, maxTwirpBytes)).Decode(&list); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, id := range list.Blocks {
|
||||
if err := appendBlock(out, filepath.Join(dir, "blocks", fileName(id))); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func appendBlock(out io.Writer, name string) error {
|
||||
in, err := os.Open(name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer in.Close()
|
||||
_, err = io.Copy(out, in)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *serverV4) finalizeArtifact(_ *http.Request, req finalizeArtifactRequest) (finalizeArtifactResponse, error) {
|
||||
dir := s.artifactDir(req.WorkflowRunBackendID, req.Name)
|
||||
meta, err := readMeta(dir)
|
||||
if err != nil {
|
||||
return finalizeArtifactResponse{}, notFound(req.Name)
|
||||
} else if meta.Digest != "" {
|
||||
return finalizeArtifactResponse{Ok: true, ArtifactID: meta.DatabaseID}, nil
|
||||
}
|
||||
blob, err := os.Open(filepath.Join(dir, "blob"))
|
||||
if err != nil {
|
||||
return finalizeArtifactResponse{}, badRequest("artifact %q has no upload", req.Name)
|
||||
}
|
||||
defer blob.Close()
|
||||
hash := sha256.New()
|
||||
size, err := io.Copy(hash, blob)
|
||||
if err != nil {
|
||||
return finalizeArtifactResponse{}, err
|
||||
}
|
||||
digest := "sha256:" + hex.EncodeToString(hash.Sum(nil))
|
||||
if size != req.Size {
|
||||
return finalizeArtifactResponse{}, badRequest("size %d does not match the %d bytes uploaded", req.Size, size)
|
||||
}
|
||||
if req.Hash != "" && req.Hash != digest {
|
||||
return finalizeArtifactResponse{}, badRequest("hash %s does not match the uploaded %s", req.Hash, digest)
|
||||
}
|
||||
s.mu.Lock() // only after hashing, so finalizing one large artifact does not stall the others
|
||||
defer s.mu.Unlock()
|
||||
if current, err := readMeta(dir); err != nil || current.DatabaseID != meta.DatabaseID {
|
||||
return finalizeArtifactResponse{}, notFound(req.Name)
|
||||
}
|
||||
meta.Size, meta.Digest = size, digest
|
||||
if err := writeMeta(dir, meta); err != nil {
|
||||
return finalizeArtifactResponse{}, err
|
||||
}
|
||||
return finalizeArtifactResponse{Ok: true, ArtifactID: meta.DatabaseID}, os.RemoveAll(filepath.Join(dir, "blocks"))
|
||||
}
|
||||
|
||||
func (s *serverV4) listArtifacts(_ *http.Request, req listArtifactsRequest) (listArtifactsResponse, error) {
|
||||
run := s.runDir(req.WorkflowRunBackendID)
|
||||
entries, err := os.ReadDir(run)
|
||||
if err != nil && !errors.Is(err, fs.ErrNotExist) {
|
||||
return listArtifactsResponse{}, err
|
||||
}
|
||||
var resp listArtifactsResponse
|
||||
for _, entry := range entries {
|
||||
meta, err := readMeta(filepath.Join(run, entry.Name()))
|
||||
if err != nil || meta.Digest == "" ||
|
||||
req.NameFilter != "" && req.NameFilter != meta.Name || req.IDFilter != 0 && req.IDFilter != meta.DatabaseID {
|
||||
continue
|
||||
}
|
||||
resp.Artifacts = append(resp.Artifacts, *meta)
|
||||
}
|
||||
slices.SortFunc(resp.Artifacts, func(left, right monolithArtifact) int { return cmp.Compare(left.DatabaseID, right.DatabaseID) })
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
func (s *serverV4) getSignedArtifactURL(r *http.Request, req getSignedArtifactURLRequest) (getSignedArtifactURLResponse, error) {
|
||||
if meta, err := readMeta(s.artifactDir(req.WorkflowRunBackendID, req.Name)); err != nil || meta.Digest == "" {
|
||||
return getSignedArtifactURLResponse{}, notFound(req.Name)
|
||||
}
|
||||
return getSignedArtifactURLResponse{SignedURL: blobURL(r, "DownloadArtifact", req.WorkflowRunBackendID, req.Name)}, nil
|
||||
}
|
||||
|
||||
func (s *serverV4) downloadArtifact(w http.ResponseWriter, r *http.Request) {
|
||||
dir := s.urlDir(r)
|
||||
meta, err := readMeta(dir)
|
||||
if err != nil || meta.Digest == "" {
|
||||
http.Error(w, "artifact not found", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
blob, err := os.Open(filepath.Join(dir, "blob"))
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
defer blob.Close()
|
||||
data, _ := os.ReadFile(filepath.Join(dir, "content-type"))
|
||||
contentType, filename := string(data), meta.Name
|
||||
if !strings.Contains(contentType, "/") {
|
||||
contentType = zipMimeType
|
||||
}
|
||||
if contentType == zipMimeType && !strings.HasSuffix(filename, ".zip") {
|
||||
filename += ".zip"
|
||||
}
|
||||
w.Header().Set("Content-Type", contentType)
|
||||
w.Header().Set("Content-Disposition", "attachment; filename*=UTF-8''"+strings.ReplaceAll(url.QueryEscape(filename), "+", "%20")) // RFC 5987 attr-chars only, which the toolkit and mime.ParseMediaType both accept
|
||||
http.ServeContent(w, r, "", meta.CreatedAt, blob)
|
||||
}
|
||||
|
||||
func (s *serverV4) deleteArtifact(_ *http.Request, req deleteArtifactRequest) (deleteArtifactResponse, error) {
|
||||
dir := s.artifactDir(req.WorkflowRunBackendID, req.Name)
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
meta, err := readMeta(dir)
|
||||
if err != nil {
|
||||
return deleteArtifactResponse{}, notFound(req.Name)
|
||||
}
|
||||
delete(s.created, dir)
|
||||
return deleteArtifactResponse{Ok: true, ArtifactID: meta.DatabaseID}, os.RemoveAll(dir)
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package artifacts
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"encoding/base64"
|
||||
"encoding/json/v2"
|
||||
"time"
|
||||
)
|
||||
|
||||
func LocalRuntimeToken(run string, lifetime time.Duration) string { // an unsecured JWT with the claims the toolkit and buildx read, its run hashed for scp and its jti an unguessable bearer
|
||||
now := time.Now()
|
||||
claims, _ := json.Marshal(map[string]any{
|
||||
"scp": "Actions.Results:" + fileName(run) + ":1",
|
||||
"ac": `[{"Scope":"","Permission":3}]`,
|
||||
"nbf": now.Unix(),
|
||||
"exp": now.Add(lifetime).Unix(),
|
||||
"jti": rand.Text(),
|
||||
})
|
||||
return base64.RawURLEncoding.EncodeToString([]byte(`{"alg":"none","typ":"JWT"}`)) + "." + base64.RawURLEncoding.EncodeToString(claims) + "."
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package artifacts
|
||||
|
||||
import "time"
|
||||
|
||||
const artifactServicePath = "/twirp/github.actions.results.api.v1.ArtifactService"
|
||||
|
||||
type ( // the ArtifactService messages in the protobuf JSON mapping, which writes int64 as strings
|
||||
createArtifactResponse struct {
|
||||
Ok bool `json:"ok"`
|
||||
SignedUploadURL string `json:"signedUploadUrl"`
|
||||
}
|
||||
|
||||
finalizeArtifactRequest struct {
|
||||
WorkflowRunBackendID string
|
||||
Name string
|
||||
Size int64 `json:",string"`
|
||||
Hash string
|
||||
}
|
||||
finalizeArtifactResponse struct {
|
||||
Ok bool `json:"ok"`
|
||||
ArtifactID int64 `json:"artifactId,string"`
|
||||
}
|
||||
|
||||
listArtifactsRequest struct {
|
||||
WorkflowRunBackendID string
|
||||
NameFilter string
|
||||
IDFilter int64 `json:",string"`
|
||||
}
|
||||
listArtifactsResponse struct {
|
||||
Artifacts []monolithArtifact `json:"artifacts"`
|
||||
}
|
||||
monolithArtifact struct {
|
||||
WorkflowRunBackendID string `json:"workflowRunBackendId"` // the toolkit sends it back to GetSignedArtifactURL
|
||||
DatabaseID int64 `json:"databaseId,string"`
|
||||
Name string `json:"name"`
|
||||
Size int64 `json:"size,string"`
|
||||
CreatedAt time.Time `json:"createdAt"` // whole seconds, the toolkit rejects the trimmed fractions json writes
|
||||
Digest string `json:"digest"` // empty until the upload is finalized
|
||||
}
|
||||
|
||||
getSignedArtifactURLRequest struct {
|
||||
WorkflowRunBackendID string
|
||||
Name string
|
||||
}
|
||||
getSignedArtifactURLResponse struct {
|
||||
SignedURL string `json:"signedUrl"`
|
||||
}
|
||||
|
||||
createArtifactRequest = getSignedArtifactURLRequest
|
||||
deleteArtifactRequest = getSignedArtifactURLRequest
|
||||
deleteArtifactResponse = finalizeArtifactResponse
|
||||
)
|
||||
@@ -110,6 +110,7 @@ func newCompositeRunContext(ctx context.Context, parent *RunContext, step action
|
||||
kubernetes: parent.kubernetes,
|
||||
jobIndex: parent.jobIndex,
|
||||
jobTotal: parent.jobTotal,
|
||||
runtimeToken: parent.runtimeToken,
|
||||
}
|
||||
compositerc.setCompositeActionEnv(env)
|
||||
compositerc.ExprEval = compositerc.NewExpressionEvaluator(ctx)
|
||||
|
||||
@@ -332,5 +332,10 @@ func getWorkflowSecrets(rc *RunContext) map[string]string {
|
||||
if rc.caller != nil {
|
||||
return rc.workflowCallSecrets
|
||||
}
|
||||
return rc.Config.Secrets
|
||||
if rc.runtimeToken == "" {
|
||||
return rc.Config.Secrets
|
||||
}
|
||||
secrets := map[string]string{"ACTIONS_RUNTIME_TOKEN": rc.runtimeToken} // as the daemon exposes its token
|
||||
maps.Copy(secrets, rc.Config.Secrets)
|
||||
return secrets
|
||||
}
|
||||
@@ -504,7 +504,7 @@ jobs:
|
||||
assert.Equal(t, "workflow", env["INPUT_FALLBACK"])
|
||||
}
|
||||
|
||||
func TestResolveWorkflowCallToleratesMismatchesAndPassesInheritedSecretsVerbatim(t *testing.T) {
|
||||
func TestResolveWorkflowCallToleratesMismatchesAndPassesInheritedSecretsAndRuntimeToken(t *testing.T) {
|
||||
callerWorkflow, err := model.ReadWorkflow(strings.NewReader(`
|
||||
jobs:
|
||||
call:
|
||||
@@ -527,7 +527,11 @@ jobs:
|
||||
require.NoError(t, err)
|
||||
notCallable, err := model.ReadWorkflow(strings.NewReader("on: push\njobs:\n test: {}\n"))
|
||||
require.NoError(t, err)
|
||||
config := &Config{Secrets: map[string]string{"token": "${{ github.token }}"}, PresetGitHubContext: &model.GithubContext{}}
|
||||
config := &Config{
|
||||
Secrets: map[string]string{"token": "${{ github.token }}"},
|
||||
PresetGitHubContext: &model.GithubContext{},
|
||||
RuntimeTokens: map[*model.Workflow]string{callerWorkflow: "caller-run"},
|
||||
}
|
||||
resolve := func(callerJobID string, workflow *model.Workflow) *RunContext {
|
||||
parent, err := (&runnerImpl{config: config}).newRunContext(t.Context(), &model.Run{Workflow: callerWorkflow, JobID: callerJobID}, nil)
|
||||
require.NoError(t, err)
|
||||
@@ -539,9 +543,10 @@ jobs:
|
||||
called := resolve("call", callee)
|
||||
assert.Equal(t, map[string]any{"flag": true}, called.workflowCallInputs)
|
||||
assert.Equal(t, map[string]string{"undeclared": "${{ github.token }}"}, called.workflowCallSecrets)
|
||||
assert.Equal(t, "caller-run", called.GetEnv()["ACTIONS_RUNTIME_TOKEN"])
|
||||
inherited := resolve("inherit", notCallable)
|
||||
assert.Empty(t, inherited.workflowCallInputs)
|
||||
assert.Equal(t, config.Secrets, inherited.workflowCallSecrets)
|
||||
assert.Equal(t, map[string]string{"token": "${{ github.token }}", "ACTIONS_RUNTIME_TOKEN": "caller-run"}, inherited.workflowCallSecrets)
|
||||
}
|
||||
|
||||
func TestJobNameMasksSecrets(t *testing.T) {
|
||||
|
||||
@@ -81,6 +81,7 @@ type RunContext struct {
|
||||
jobRunDefaults model.RunDefaults // defaults.run, resolved once at job setup as GitHub does
|
||||
platformImage string // container.image or the runs-on pick, resolved once by isEnabled
|
||||
jobIndex, jobTotal int // strategy.job-index and job-total, total 0 when unknown
|
||||
runtimeToken string // ACTIONS_RUNTIME_TOKEN of the workflow run, shared with its called workflows and composite actions
|
||||
|
||||
jobEnv map[string]string // workflow and job env as evaluated for the container and service specs
|
||||
dockerActionImages map[*model.Step]string // Dockerfile action images built for this job, removed when it ends
|
||||
@@ -193,6 +194,9 @@ func (rc *RunContext) GetEnv() map[string]string {
|
||||
rc.Env = mergeMaps(rc.Run.Workflow.Env, job.Environment(), rc.Config.Env)
|
||||
}
|
||||
}
|
||||
if rc.runtimeToken != "" {
|
||||
rc.Env["ACTIONS_RUNTIME_TOKEN"] = rc.runtimeToken // as the daemon's Config.Env carries its token
|
||||
}
|
||||
}
|
||||
if !rc.Config.DisableActEnv {
|
||||
rc.Env["ACT"] = "true"
|
||||
@@ -1662,7 +1666,7 @@ func (rc *RunContext) withGithubEnv(ctx context.Context, github *model.GithubCon
|
||||
|
||||
if rc.Config.ArtifactServerPath != "" {
|
||||
env["ACTIONS_RUNTIME_URL"] = cmp.Or(os.Getenv("ACTIONS_RUNTIME_URL"), "http://"+net.JoinHostPort(rc.Config.ArtifactServerAddr, rc.Config.ArtifactServerPort)+"/")
|
||||
env["ACTIONS_RUNTIME_TOKEN"] = cmp.Or(os.Getenv("ACTIONS_RUNTIME_TOKEN"), "token")
|
||||
env["ACTIONS_RUNTIME_TOKEN"] = cmp.Or(rc.runtimeToken, os.Getenv("ACTIONS_RUNTIME_TOKEN"), "token")
|
||||
}
|
||||
|
||||
if imageOS := rc.imageOS(ctx); imageOS != "" {
|
||||
|
||||
@@ -1631,4 +1631,8 @@ func TestRunContextWithGithubEnvRunnerValues(t *testing.T) {
|
||||
rc.withGithubEnv(ctx, &model.GithubContext{}, env)
|
||||
assert.Equal(t, ci, env["CI"])
|
||||
}
|
||||
|
||||
rc.runtimeToken = "run-token"
|
||||
newTestCompositeRunContext(t, rc).withGithubEnv(ctx, &model.GithubContext{}, env)
|
||||
assert.Equal(t, "run-token", env["ACTIONS_RUNTIME_TOKEN"])
|
||||
}
|
||||
@@ -41,6 +41,7 @@ type Config struct {
|
||||
Env map[string]string // env for containers
|
||||
Secrets map[string]string // list of secrets
|
||||
ExtraMasks []string // values to hide that are not in Secrets, such as the proxy password
|
||||
RuntimeTokens map[*model.Workflow]string // ACTIONS_RUNTIME_TOKEN per workflow run, as each is its own artifact run
|
||||
Vars map[string]string // list of vars
|
||||
Token string // GitHub token
|
||||
InsecureSecrets bool // switch hiding output when printing to terminal
|
||||
@@ -355,6 +356,11 @@ func (runner *runnerImpl) newCombinationRunContext(ctx context.Context, run *mod
|
||||
caller: runner.caller,
|
||||
jobIndex: index,
|
||||
jobTotal: total,
|
||||
|
||||
runtimeToken: runner.config.RuntimeTokens[run.Workflow],
|
||||
}
|
||||
if runner.caller != nil {
|
||||
rc.runtimeToken = runner.caller.runContext.runtimeToken
|
||||
}
|
||||
if err := rc.resolveWorkflowCall(ctx); err != nil {
|
||||
return nil, err
|
||||
|
||||
+50
-3
@@ -12,6 +12,8 @@ import (
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"syscall"
|
||||
"testing"
|
||||
@@ -47,9 +49,9 @@ func testCLIRegisterDaemon(t *testing.T) {
|
||||
}
|
||||
|
||||
dir := t.TempDir()
|
||||
bin := filepath.Join(dir, "gitea-runner")
|
||||
if out, err := exec.CommandContext(ctx, "go", "build", "-o", bin, "..").CombinedOutput(); err != nil {
|
||||
t.Fatalf("build runner: %v\n%s", err, out)
|
||||
bin, err := runnerBinary()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
cfgFile := filepath.Join(dir, "config.yaml")
|
||||
@@ -119,3 +121,48 @@ func requireSuccessOrDumpDaemon(t *testing.T, api *GiteaAPI, repo string, runID
|
||||
requireSuccess(t, api, repo, runID)
|
||||
}
|
||||
}
|
||||
|
||||
func testExecArtifacts(t *testing.T) {
|
||||
t.Parallel()
|
||||
ctx := t.Context()
|
||||
|
||||
bin, err := runnerBinary()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
workflow, err := os.ReadFile("testdata/workflows/exec_artifact.yml")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
workflows := t.TempDir()
|
||||
for _, name := range []string{"a.yml", "b.yml"} {
|
||||
if err := os.WriteFile(filepath.Join(workflows, name), workflow, 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
workdir := t.TempDir()
|
||||
for _, args := range [][]string{
|
||||
{"init", "-q"},
|
||||
{"remote", "add", "origin", "https://example.com/e2e/exec.git"},
|
||||
{"-c", "user.name=e2e", "-c", "[email protected]", "commit", "-q", "--allow-empty", "-m", "init"},
|
||||
} {
|
||||
if out, err := exec.CommandContext(ctx, "git", append([]string{"-C", workdir}, args...)...).CombinedOutput(); err != nil {
|
||||
t.Fatalf("git %v: %v\n%s", args, err, out)
|
||||
}
|
||||
}
|
||||
|
||||
artifacts := t.TempDir()
|
||||
args := []string{"exec", "-C", workdir, "-W", workflows, "-i", os.Getenv("E2E_JOB_IMAGE"), "--artifact-server-path", artifacts}
|
||||
if fixture.network != "" {
|
||||
args = append(args, "--network", fixture.network)
|
||||
}
|
||||
cmd := exec.CommandContext(ctx, bin, args...)
|
||||
cmd.Env = slices.DeleteFunc(os.Environ(), func(kv string) bool { return strings.HasPrefix(kv, "ACTIONS_") })
|
||||
if out, err := cmd.CombinedOutput(); err != nil {
|
||||
t.Fatalf("exec: %v\n%s", err, out)
|
||||
}
|
||||
if blobs, _ := filepath.Glob(filepath.Join(artifacts, "v4", "*", "*", "blob")); len(blobs) != 2 {
|
||||
t.Fatalf("want one v4 artifact blob per workflow run, got %v", blobs)
|
||||
}
|
||||
}
|
||||
@@ -30,6 +30,7 @@ func TestCompatibility(t *testing.T) {
|
||||
t.Run("cancellation_and_log_streaming", testRunCancellation)
|
||||
t.Run("dispatch", testWorkflowDispatch)
|
||||
t.Run("builtin_checkout", testBuiltinCheckout)
|
||||
t.Run("exec_artifacts", testExecArtifacts)
|
||||
t.Run("moving_action_ref", testMovingActionRef)
|
||||
t.Run("ephemeral", testEphemeralRunner)
|
||||
t.Run("failure_paths", testFailurePaths)
|
||||
|
||||
@@ -6,11 +6,15 @@
|
||||
package e2e
|
||||
|
||||
import (
|
||||
"cmp"
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -25,6 +29,16 @@ var fixture *GiteaFixture
|
||||
|
||||
var skipReason string
|
||||
|
||||
var binDir string
|
||||
|
||||
var runnerBinary = sync.OnceValues(func() (string, error) {
|
||||
bin := filepath.Join(binDir, "gitea-runner")
|
||||
if out, err := exec.Command(cmp.Or(os.Getenv("GO"), "go"), "build", "-o", bin, "..").CombinedOutput(); err != nil {
|
||||
return "", fmt.Errorf("build runner: %w\n%s", err, out)
|
||||
}
|
||||
return bin, nil
|
||||
})
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
os.Exit(runSuite(m))
|
||||
}
|
||||
@@ -47,6 +61,12 @@ func runSuite(m *testing.M) int {
|
||||
fixture = f
|
||||
defer func() { _ = fixture.Close(context.Background()) }()
|
||||
|
||||
if binDir, err = os.MkdirTemp("", "gitea-runner-e2e-"); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "create binary dir: %v\n", err)
|
||||
return 1
|
||||
}
|
||||
defer os.RemoveAll(binDir)
|
||||
|
||||
fmt.Fprintf(os.Stderr, "gitea fixture: image=%s version=%s\n", fixture.image, fixture.version)
|
||||
|
||||
return m.Run()
|
||||
|
||||
+15
@@ -0,0 +1,15 @@
|
||||
on: push
|
||||
jobs:
|
||||
roundtrip:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- run: mkdir -p out && echo exec-payload > out/data.txt
|
||||
- uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: exec-artifact
|
||||
path: out/data.txt
|
||||
- uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: exec-artifact
|
||||
path: got
|
||||
- run: grep -q exec-payload got/data.txt
|
||||
+19
-22
@@ -7,12 +7,11 @@ package cmd
|
||||
import (
|
||||
"cmp"
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"encoding/json/v2"
|
||||
"errors"
|
||||
"fmt"
|
||||
"maps"
|
||||
"net"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
@@ -481,24 +480,7 @@ func runExec(ctx context.Context, execArgs *executeArgs) func(cmd *cobra.Command
|
||||
execArgs.artifactServerPath = tempDir
|
||||
}
|
||||
|
||||
// Register ACTIONS_RUNTIME_TOKEN against local cache server
|
||||
env := execArgs.LoadEnvs()
|
||||
const actionsRuntimeTokenEnvName = "ACTIONS_RUNTIME_TOKEN"
|
||||
actionsRuntimeToken := env[actionsRuntimeTokenEnvName]
|
||||
if actionsRuntimeToken == "" {
|
||||
actionsRuntimeToken = os.Getenv(actionsRuntimeTokenEnvName)
|
||||
}
|
||||
if actionsRuntimeToken == "" {
|
||||
tmpBranch := make([]byte, 12)
|
||||
if _, err := rand.Read(tmpBranch); err != nil {
|
||||
actionsRuntimeToken = "token"
|
||||
} else {
|
||||
actionsRuntimeToken = hex.EncodeToString(tmpBranch)
|
||||
}
|
||||
env[actionsRuntimeTokenEnvName] = actionsRuntimeToken
|
||||
os.Setenv(actionsRuntimeTokenEnvName, actionsRuntimeToken)
|
||||
}
|
||||
handler.RegisterJob(actionsRuntimeToken, artifactcache.JobCredential{Repo: "__local/__exec"})
|
||||
|
||||
// no service aliases: exec builds one config for the whole plan
|
||||
run.BypassProxyForDockerHost(os.Getenv("DOCKER_HOST"))
|
||||
@@ -517,7 +499,22 @@ func runExec(ctx context.Context, execArgs *executeArgs) func(cmd *cobra.Command
|
||||
}
|
||||
|
||||
config.Env["ACT_EXEC"] = "true"
|
||||
config.Secrets[actionsRuntimeTokenEnvName] = actionsRuntimeToken
|
||||
inheritedToken := cmp.Or(env["ACTIONS_RUNTIME_TOKEN"], os.Getenv("ACTIONS_RUNTIME_TOKEN"))
|
||||
config.RuntimeTokens = map[*model.Workflow]string{}
|
||||
sameFile := map[string]int{}
|
||||
for _, stage := range plan.Stages {
|
||||
for _, run := range stage.Runs {
|
||||
if config.RuntimeTokens[run.Workflow] != "" {
|
||||
continue
|
||||
}
|
||||
sameFile[run.Workflow.File]++
|
||||
workflowRun := fmt.Sprintf("%s/%s/%d", cmp.Or(env["GITHUB_RUN_ID"], "1"), run.Workflow.File, sameFile[run.Workflow.File]) // each workflow its own run as on GitHub, also when file names repeat across directories
|
||||
token := cmp.Or(inheritedToken, artifacts.LocalRuntimeToken(workflowRun, maxLifetime))
|
||||
config.RuntimeTokens[run.Workflow] = token
|
||||
config.ExtraMasks = append(config.ExtraMasks, token)
|
||||
handler.RegisterJob(token, artifactcache.JobCredential{Repo: "__local/__exec", Results: "http://" + net.JoinHostPort(execArgs.artifactServerAddr, execArgs.artifactServerPort)})
|
||||
}
|
||||
}
|
||||
|
||||
config.Token = cmp.Or(config.Secrets["GITEA_TOKEN"], config.Secrets["GITHUB_TOKEN"])
|
||||
|
||||
@@ -580,9 +577,9 @@ func loadExecCmd(ctx context.Context) *cobra.Command {
|
||||
execCmd.Flags().StringArrayVarP(&execArg.containerCapAdd, "container-cap-add", "", []string{}, "kernel capabilities to add to the workflow containers (e.g. --container-cap-add SYS_PTRACE)")
|
||||
execCmd.Flags().StringArrayVarP(&execArg.containerCapDrop, "container-cap-drop", "", []string{}, "kernel capabilities to remove from the workflow containers (e.g. --container-cap-drop SYS_PTRACE)")
|
||||
execCmd.Flags().StringVarP(&execArg.containerOptions, "container-opts", "", "", "container options")
|
||||
execCmd.PersistentFlags().StringVarP(&execArg.artifactServerPath, "artifact-server-path", "", ".", "Defines the path where the artifact server stores uploads and retrieves downloads from. If not specified the artifact server will not start.")
|
||||
execCmd.PersistentFlags().StringVarP(&execArg.artifactServerPath, "artifact-server-path", "", "", "Defines the path where the artifact server stores uploads and retrieves downloads from, outside the workspace that jobs copy. Defaults to a temporary directory removed on exit.")
|
||||
execCmd.PersistentFlags().StringVarP(&execArg.artifactServerAddr, "artifact-server-addr", "", "", "Defines the address where the artifact server listens")
|
||||
execCmd.PersistentFlags().StringVarP(&execArg.artifactServerPort, "artifact-server-port", "", "34567", "Defines the port where the artifact server listens (will only bind to localhost).")
|
||||
execCmd.PersistentFlags().StringVarP(&execArg.artifactServerPort, "artifact-server-port", "", "34567", "Defines the port where the artifact server listens")
|
||||
execCmd.PersistentFlags().StringVarP(&execArg.defaultActionsURL, "default-actions-url", "", "https://github.com", "Defines the default url of action instance.")
|
||||
execCmd.PersistentFlags().BoolVarP(&execArg.noSkipCheckout, "no-skip-checkout", "", false, "Do not skip actions/checkout")
|
||||
execCmd.PersistentFlags().BoolVarP(&execArg.debug, "debug", "d", false, "enable debug log")
|
||||
|
||||
Reference in new issue
Block a user