mirror of
https://gitea.com/gitea/runner.git
synced 2026-11-08 08:10:30 +00:00
enhance: add builtin:download-artifact (#1344)
Co-authored-by: bircni <[email protected]>
This commit is contained in:
1 parent
02446620df
commit
c8b16f4792
11 files changed
+912
-4
No files matched your search
@@ -103,6 +103,13 @@ func (c *Context) Windows() bool {
|
||||
return c.Host() && runtime.GOOS == "windows"
|
||||
}
|
||||
|
||||
func (c *Context) Join(elem ...string) string {
|
||||
if c.Windows() {
|
||||
return filepath.Join(elem...)
|
||||
}
|
||||
return path.Join(elem...)
|
||||
}
|
||||
|
||||
func (c *Context) Dir(p string) string {
|
||||
if c.Windows() {
|
||||
return filepath.Dir(p)
|
||||
@@ -117,6 +124,13 @@ func (c *Context) Base(p string) string {
|
||||
return path.Base(p)
|
||||
}
|
||||
|
||||
func (c *Context) Resolve(p string) string {
|
||||
if c.Windows() && filepath.IsAbs(p) || !c.Windows() && path.IsAbs(p) {
|
||||
return c.Join(p)
|
||||
}
|
||||
return c.Join(c.Workspace, p)
|
||||
}
|
||||
|
||||
// Home is the job user's home directory, what os.homedir() returns to a JavaScript action.
|
||||
func (c *Context) Home(ctx context.Context) (string, error) {
|
||||
if home := c.Env["HOME"]; home != "" {
|
||||
|
||||
@@ -0,0 +1,117 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package artifact
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json/v2"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"time"
|
||||
)
|
||||
|
||||
type REST struct {
|
||||
base *url.URL
|
||||
token string
|
||||
api *http.Client
|
||||
blob *http.Client
|
||||
Retry Retry
|
||||
Logf func(format string, args ...any)
|
||||
}
|
||||
|
||||
// NewREST takes the instance as the runner reaches it, never the job's GITHUB_API_URL, so the token only goes there.
|
||||
func NewREST(instance, token string, transport http.RoundTripper) (*REST, error) {
|
||||
parsed, err := url.Parse(instance)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &REST{
|
||||
base: parsed.JoinPath("api", "v1"),
|
||||
token: token,
|
||||
api: &http.Client{Transport: transport, CheckRedirect: func(*http.Request, []*http.Request) error {
|
||||
return http.ErrUseLastResponse // the redirect target is signed and must not get the token
|
||||
}},
|
||||
blob: &http.Client{Transport: blobTransport{instance: parsed.Host, transport: transport}},
|
||||
Retry: DefaultRetry,
|
||||
Logf: func(string, ...any) {},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (r *REST) get(ctx context.Context, what string, endpoint *url.URL) (*http.Response, error) {
|
||||
return r.Retry.do(ctx, r.Logf, what, func() (*http.Response, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint.String(), nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.Header.Set("Authorization", "token "+r.token)
|
||||
req.Header.Set("Accept", "application/json")
|
||||
return r.api.Do(req)
|
||||
})
|
||||
}
|
||||
|
||||
func (r *REST) List(ctx context.Context, owner, repo string, runID int64, name string) ([]Artifact, error) {
|
||||
if name == "" {
|
||||
r.Logf("Fetching artifact list for workflow run %d in repository %s/%s", runID, owner, repo)
|
||||
}
|
||||
var list []Artifact
|
||||
seen := 0
|
||||
for page := 1; ; page++ {
|
||||
endpoint := r.base.JoinPath("repos", owner, repo, "actions", "runs", strconv.FormatInt(runID, 10), "artifacts")
|
||||
query := url.Values{"page": {strconv.Itoa(page)}, "limit": {"50"}}
|
||||
if name != "" {
|
||||
query.Set("name", name)
|
||||
}
|
||||
endpoint.RawQuery = query.Encode()
|
||||
resp, err := r.get(ctx, "list artifacts", endpoint)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var body struct {
|
||||
TotalCount int `json:"total_count"`
|
||||
Artifacts []struct {
|
||||
ID int64 `json:"id"`
|
||||
Name string `json:"name"`
|
||||
SizeInBytes int64 `json:"size_in_bytes"`
|
||||
Expired bool `json:"expired"`
|
||||
Digest string `json:"digest"`
|
||||
} `json:"artifacts"`
|
||||
}
|
||||
err = json.UnmarshalRead(resp.Body, &body)
|
||||
resp.Body.Close()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list artifacts: %w", err)
|
||||
}
|
||||
for _, listed := range body.Artifacts {
|
||||
if !listed.Expired {
|
||||
list = append(list, Artifact{ID: listed.ID, Name: listed.Name, Size: listed.SizeInBytes, Digest: listed.Digest})
|
||||
}
|
||||
}
|
||||
if seen += len(body.Artifacts); len(body.Artifacts) == 0 || seen >= body.TotalCount {
|
||||
return list, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (r *REST) DownloadURL(ctx context.Context, owner, repo string, id int64) (string, error) {
|
||||
resp, err := r.get(ctx, "download artifact", r.base.JoinPath("repos", owner, repo, "actions", "artifacts", strconv.FormatInt(id, 10), "zip"))
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("unable to download artifact: %w", err)
|
||||
}
|
||||
resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusFound {
|
||||
return "", fmt.Errorf("unable to download artifact. Unexpected status: %d", resp.StatusCode)
|
||||
}
|
||||
location, err := resp.Location()
|
||||
if err != nil {
|
||||
return "", errors.New("unable to redirect to artifact download url")
|
||||
}
|
||||
return location.String(), nil
|
||||
}
|
||||
|
||||
func (r *REST) Download(ctx context.Context, signedURL string, stall time.Duration) (*http.Response, error) {
|
||||
return (&Client{blob: r.blob}).Download(ctx, signedURL, stall)
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package artifact
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestRESTListsEveryPageAndSkipsExpired(t *testing.T) {
|
||||
var pages []string
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
assert.Equal(t, "/api/v1/repos/own%20er/repo/actions/runs/12/artifacts", r.URL.EscapedPath())
|
||||
assert.Equal(t, "token secret", r.Header.Get("Authorization"))
|
||||
assert.Empty(t, r.URL.Query().Get("per_page"))
|
||||
pages = append(pages, r.URL.Query().Get("page")+"/"+r.URL.Query().Get("limit"))
|
||||
page, _ := strconv.Atoi(r.URL.Query().Get("page"))
|
||||
const maxResponseItems = 20
|
||||
var entries []string
|
||||
for id := (page-1)*maxResponseItems + 1; id <= min(page*maxResponseItems, 53); id++ {
|
||||
entries = append(entries, fmt.Sprintf(`{"id":%d,"name":"a%d","size_in_bytes":%d,"expired":%t}`, id, id, id, id == 2))
|
||||
}
|
||||
_, _ = io.WriteString(w, `{"total_count":53,"artifacts":[`+strings.Join(entries, ",")+`]}`)
|
||||
}))
|
||||
defer server.Close()
|
||||
rest, err := NewREST(server.URL, "secret", nil)
|
||||
require.NoError(t, err)
|
||||
list, err := rest.List(t.Context(), "own er", "repo", 12, "")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, []string{"1/50", "2/50", "3/50"}, pages)
|
||||
assert.Len(t, list, 52)
|
||||
assert.Equal(t, Artifact{ID: 3, Name: "a3", Size: 3}, list[1])
|
||||
}
|
||||
@@ -0,0 +1,174 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
// Package downloadartifact works like actions/download-artifact v8.
|
||||
package downloadartifact
|
||||
|
||||
import (
|
||||
"cmp"
|
||||
"context"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.com/gitea/runner/act/common"
|
||||
"gitea.com/gitea/runner/act/container"
|
||||
"gitea.com/gitea/runner/internal/pkg/action"
|
||||
"gitea.com/gitea/runner/internal/pkg/action/artifact"
|
||||
|
||||
"golang.org/x/sync/errgroup"
|
||||
)
|
||||
|
||||
var digestBehaviors = []string{"ignore", "info", "warn", "error"}
|
||||
|
||||
const downloadAttempts = 5
|
||||
|
||||
func Main(ctx context.Context, c *action.Context) error {
|
||||
if err := run(ctx, c); err != nil {
|
||||
return action.Failf("Unable to download artifact(s): %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func run(ctx context.Context, c *action.Context) error {
|
||||
c.ApplyDefaults("name", "", "artifact-ids", "", "path", "", "pattern", "", "merge-multiple", "false", "github-token", "",
|
||||
"repository", c.Github.Repository, "run-id", c.Github.RunID, "skip-decompress", "false", "digest-mismatch", "error")
|
||||
name, ids, pattern, token := c.Input("name"), c.Input("artifact-ids"), c.Input("pattern"), c.Input("github-token")
|
||||
mergeMultiple, err := c.BoolInput("merge-multiple")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
skipDecompress, err := c.BoolInput("skip-decompress")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
digestMismatch := cmp.Or(c.Input("digest-mismatch"), "error")
|
||||
if !slices.Contains(digestBehaviors, digestMismatch) {
|
||||
return action.Failf("Invalid value for 'digest-mismatch': '%s'. Valid options are: %s", digestMismatch, strings.Join(digestBehaviors, ", "))
|
||||
}
|
||||
dest := cmp.Or(c.Input("path"), c.Env["GITHUB_WORKSPACE"], c.Workspace)
|
||||
if strings.HasPrefix(dest, "~") {
|
||||
home, err := c.Home(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
dest = home + dest[1:]
|
||||
}
|
||||
dest = c.Resolve(dest)
|
||||
if name != "" && ids != "" {
|
||||
return action.Failf("Inputs 'name' and 'artifact-ids' cannot be used together. Please specify only one.")
|
||||
}
|
||||
|
||||
src, err := newSource(c, token)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
artifacts, err := src.list(ctx, name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
slices.SortStableFunc(artifacts, func(a, b artifact.Artifact) int { return cmp.Compare(b.ID, a.ID) })
|
||||
seen := map[string]bool{}
|
||||
artifacts = slices.DeleteFunc(artifacts, func(a artifact.Artifact) bool {
|
||||
duplicate := seen[a.Name]
|
||||
seen[a.Name] = true
|
||||
return duplicate
|
||||
})
|
||||
if name == "" {
|
||||
c.Infof("Found %d artifact(s)", len(artifacts))
|
||||
}
|
||||
switch {
|
||||
case name != "":
|
||||
if len(artifacts) == 0 {
|
||||
return action.Failf("Artifact not found for name: %s\nPlease ensure that your artifact is not expired and the artifact was uploaded using a compatible version of toolkit/upload-artifact.", name)
|
||||
}
|
||||
artifacts = artifacts[:1]
|
||||
case ids != "":
|
||||
var wanted []int64
|
||||
for id := range strings.SplitSeq(ids, ",") {
|
||||
if id = strings.TrimSpace(id); id == "" {
|
||||
continue
|
||||
}
|
||||
parsed, err := action.ParseInt(id)
|
||||
if err != nil {
|
||||
return action.Failf("Invalid artifact ID: '%s'. Must be a number.", id)
|
||||
}
|
||||
wanted = append(wanted, int64(parsed))
|
||||
}
|
||||
if len(wanted) == 0 {
|
||||
return action.Failf("No valid artifact IDs provided in 'artifact-ids' input")
|
||||
}
|
||||
artifacts = slices.DeleteFunc(artifacts, func(a artifact.Artifact) bool { return !slices.Contains(wanted, a.ID) })
|
||||
if len(artifacts) == 0 {
|
||||
return action.Failf("None of the provided artifact IDs were found")
|
||||
}
|
||||
case pattern != "":
|
||||
artifacts = slices.DeleteFunc(artifacts, func(a artifact.Artifact) bool { return !matchName(pattern, a.Name) })
|
||||
}
|
||||
for _, art := range artifacts {
|
||||
if art.Name == "" || art.Name == "." || art.Name == ".." || strings.ContainsAny(art.Name, `/\`) || c.Windows() && !filepath.IsLocal(art.Name) {
|
||||
return action.Failf("Artifact name is not valid: %s", art.Name)
|
||||
}
|
||||
}
|
||||
|
||||
mismatches := make([]string, len(artifacts))
|
||||
group, groupCtx := errgroup.WithContext(ctx)
|
||||
group.SetLimit(5)
|
||||
for i, art := range artifacts {
|
||||
target := dest
|
||||
if name == "" && !mergeMultiple && len(artifacts) > 1 {
|
||||
target = c.Join(dest, art.Name)
|
||||
}
|
||||
group.Go(func() error {
|
||||
mismatch, err := fetch(groupCtx, c, src, art, target, skipDecompress)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if mismatch {
|
||||
mismatches[i] = art.Name
|
||||
message := fmt.Sprintf("Artifact '%s' digest validation failed. Please verify the integrity of the artifact.", art.Name)
|
||||
switch digestMismatch {
|
||||
case "info":
|
||||
c.Infof("%s", message)
|
||||
case "warn":
|
||||
c.Warning("%s", message)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
if err := group.Wait(); err != nil {
|
||||
return err
|
||||
}
|
||||
mismatches = slices.DeleteFunc(mismatches, func(name string) bool { return name == "" })
|
||||
if len(mismatches) > 0 && digestMismatch == "error" {
|
||||
return action.Failf("Digest validation failed for artifact(s): %s. Use 'digest-mismatch: warn' to continue on mismatch.", strings.Join(mismatches, ", "))
|
||||
}
|
||||
c.Infof("Total of %d artifact(s) downloaded", len(artifacts))
|
||||
return c.SetOutputs(ctx, "download-path", dest)
|
||||
}
|
||||
|
||||
func fetch(ctx context.Context, c *action.Context, src *source, art artifact.Artifact, target string, skipDecompress bool) (bool, error) {
|
||||
if err := c.Container.CopyTarStream(ctx, target, func(*container.TarWriter) error { return nil }); err != nil {
|
||||
return false, err
|
||||
}
|
||||
signed, err := src.signedURL(ctx, art)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
for attempt := 1; attempt <= downloadAttempts; attempt++ {
|
||||
digest, err := fetchOnce(ctx, c, src, signed, target, skipDecompress)
|
||||
if err == nil {
|
||||
return art.Digest != "" && art.Digest != digest, nil
|
||||
}
|
||||
common.Logger(ctx).Debugf("Failed to download artifact after %d retries due to %v. Retrying in 5 seconds...", attempt, err)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return false, ctx.Err()
|
||||
case <-time.After(5 * time.Second):
|
||||
}
|
||||
}
|
||||
return false, action.Failf("Unable to download and extract artifact: Artifact download failed after %d retries.", downloadAttempts)
|
||||
}
|
||||
@@ -0,0 +1,316 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package downloadartifact
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"maps"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"testing/synctest"
|
||||
"time"
|
||||
|
||||
"gitea.com/gitea/runner/act/artifacts"
|
||||
"gitea.com/gitea/runner/act/container"
|
||||
"gitea.com/gitea/runner/internal/pkg/action"
|
||||
"gitea.com/gitea/runner/internal/pkg/action/artifact"
|
||||
|
||||
"gitea.dev/actionslib/pkg/model"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
type zipEntry struct {
|
||||
name, body string
|
||||
mode os.FileMode
|
||||
}
|
||||
|
||||
func zipOf(t *testing.T, entries ...zipEntry) []byte {
|
||||
t.Helper()
|
||||
var buf bytes.Buffer
|
||||
zw := zip.NewWriter(&buf)
|
||||
for _, e := range entries {
|
||||
header := &zip.FileHeader{Name: e.name, Method: zip.Deflate}
|
||||
header.SetMode(e.mode)
|
||||
w, err := zw.CreateHeader(header)
|
||||
require.NoError(t, err)
|
||||
_, err = io.WriteString(w, e.body)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
require.NoError(t, zw.Close())
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
type harness struct {
|
||||
t *testing.T
|
||||
server, store string
|
||||
client *artifact.Client
|
||||
workspace string
|
||||
env map[string]string
|
||||
mu sync.Mutex
|
||||
logs, annotation []string
|
||||
}
|
||||
|
||||
func newHarness(t *testing.T) *harness {
|
||||
t.Helper()
|
||||
store := t.TempDir()
|
||||
router := http.NewServeMux()
|
||||
artifacts.RegisterV4(router, store)
|
||||
server := httptest.NewServer(router)
|
||||
t.Cleanup(server.Close)
|
||||
client, err := artifact.New(server.URL, artifacts.LocalRuntimeToken("7", time.Hour), nil)
|
||||
require.NoError(t, err)
|
||||
return &harness{t: t, server: server.URL, store: store, client: client, workspace: t.TempDir()}
|
||||
}
|
||||
|
||||
func (h *harness) upload(name, mimeType string, content []byte) int64 {
|
||||
h.t.Helper()
|
||||
ctx := h.t.Context()
|
||||
signed, err := h.client.Create(ctx, name, mimeType, nil)
|
||||
require.NoError(h.t, err)
|
||||
uploaded, err := h.client.Upload(ctx, signed, mimeType, bytes.NewReader(content), 2, time.Minute)
|
||||
require.NoError(h.t, err)
|
||||
id, err := h.client.Finalize(ctx, name, uploaded.Size, uploaded.Digest)
|
||||
require.NoError(h.t, err)
|
||||
return id
|
||||
}
|
||||
|
||||
func (h *harness) run(inputs map[string]string) error {
|
||||
h.logs, h.annotation = nil, nil
|
||||
env := map[string]string{"GITHUB_OUTPUT": filepath.Join(h.workspace, "output")}
|
||||
maps.Copy(env, h.env)
|
||||
return Main(h.t.Context(), &action.Context{
|
||||
Container: &container.HostEnvironment{},
|
||||
Github: &model.GithubContext{Repository: "owner/repo", RunID: "7"},
|
||||
Inputs: inputs,
|
||||
Workspace: h.workspace,
|
||||
Env: env,
|
||||
ResultsURL: h.server,
|
||||
RuntimeToken: artifacts.LocalRuntimeToken("7", time.Hour),
|
||||
Infof: func(format string, args ...any) {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
h.logs = append(h.logs, fmt.Sprintf(format, args...))
|
||||
},
|
||||
Warning: func(format string, args ...any) {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
h.annotation = append(h.annotation, "warning: "+fmt.Sprintf(format, args...))
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
func (h *harness) read(rel string) string {
|
||||
h.t.Helper()
|
||||
content, err := os.ReadFile(filepath.Join(h.workspace, filepath.FromSlash(rel)))
|
||||
require.NoError(h.t, err)
|
||||
return string(content)
|
||||
}
|
||||
|
||||
func TestDownload(t *testing.T) {
|
||||
h := newHarness(t)
|
||||
first := h.upload("alpha", "", zipOf(t, zipEntry{name: "a.txt", body: "a1"}))
|
||||
_, err := h.client.Delete(t.Context(), "alpha")
|
||||
require.NoError(t, err)
|
||||
archive := zipOf(t,
|
||||
zipEntry{name: "run.sh", body: "#!/bin/sh\n", mode: 0o755},
|
||||
zipEntry{name: "empty/", mode: os.ModeDir | 0o755},
|
||||
zipEntry{name: "nested/../a.txt", body: "a2"},
|
||||
zipEntry{name: "empty.txt"},
|
||||
)
|
||||
second := h.upload("alpha", "", archive)
|
||||
h.upload("beta ü.txt", "text/plain", []byte("plain text"))
|
||||
h.upload(".hidden", "", zipOf(t, zipEntry{name: "h.txt", body: "h"}))
|
||||
|
||||
for _, test := range []struct {
|
||||
inputs map[string]string
|
||||
files map[string]string
|
||||
}{
|
||||
{map[string]string{"name": "alpha", "path": "named"}, map[string]string{"named/a.txt": "a2", "named/run.sh": "#!/bin/sh\n", "named/empty.txt": ""}},
|
||||
{map[string]string{"path": "all"}, map[string]string{"all/alpha/a.txt": "a2", "all/beta ü.txt/beta ü.txt": "plain text", "all/.hidden/h.txt": "h"}},
|
||||
{map[string]string{"path": "merged", "merge-multiple": "true"}, map[string]string{"merged/a.txt": "a2", "merged/beta ü.txt": "plain text", "merged/h.txt": "h"}},
|
||||
{map[string]string{"path": "pattern", "pattern": "*"}, map[string]string{"pattern/alpha/a.txt": "a2", "pattern/beta ü.txt/beta ü.txt": "plain text"}},
|
||||
{map[string]string{"path": "ids", "artifact-ids": fmt.Sprintf("%dtrailing, %d", second, first), "pattern": "ignored"}, map[string]string{"ids/a.txt": "a2"}},
|
||||
{map[string]string{"name": "alpha", "path": "kept", "skip-decompress": "true"}, map[string]string{"kept/alpha.zip": string(archive)}},
|
||||
{map[string]string{"path": "none", "pattern": "nope-*", "repository": "ignored", "run-id": "invalid"}, nil},
|
||||
} {
|
||||
require.NoError(t, h.run(test.inputs))
|
||||
assert.Contains(t, h.read("output"), "download-path="+filepath.Join(h.workspace, test.inputs["path"])+"\n")
|
||||
for name, body := range test.files {
|
||||
assert.Equal(t, body, h.read(name))
|
||||
}
|
||||
}
|
||||
assert.Contains(t, h.logs, "Total of 0 artifact(s) downloaded")
|
||||
assert.NoDirExists(t, filepath.Join(h.workspace, "pattern", ".hidden"))
|
||||
assert.DirExists(t, filepath.Join(h.workspace, "named", "empty"))
|
||||
if runtime.GOOS != "windows" {
|
||||
info, err := os.Stat(filepath.Join(h.workspace, "named", "run.sh"))
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, os.FileMode(0o644), info.Mode())
|
||||
assert.WithinDuration(t, time.Now(), info.ModTime(), time.Minute)
|
||||
}
|
||||
for _, test := range []struct {
|
||||
inputs map[string]string
|
||||
error string
|
||||
}{
|
||||
{map[string]string{"artifact-ids": strconv.FormatInt(first, 10)}, "None of the provided artifact IDs were found"},
|
||||
{map[string]string{"name": "gone"}, "Artifact not found for name: gone"},
|
||||
{map[string]string{"name": "alpha", "artifact-ids": "1"}, "cannot be used together"},
|
||||
{map[string]string{"artifact-ids": "x"}, "Invalid artifact ID: 'x'"},
|
||||
{map[string]string{"artifact-ids": " , "}, "No valid artifact IDs provided"},
|
||||
{map[string]string{"digest-mismatch": "loud"}, "Valid options are: ignore, info, warn, error"},
|
||||
{map[string]string{"merge-multiple": "yes"}, "Core Schema"},
|
||||
} {
|
||||
require.ErrorContains(t, h.run(test.inputs), test.error)
|
||||
}
|
||||
h.env = map[string]string{"GITHUB_WORKSPACE": filepath.Join(h.workspace, "overridden")}
|
||||
require.NoError(t, h.run(map[string]string{"name": "beta ü.txt"}))
|
||||
assert.Equal(t, "plain text", h.read("overridden/beta ü.txt"))
|
||||
h.env = map[string]string{"HOME": filepath.Join(h.workspace, "home")}
|
||||
require.NoError(t, h.run(map[string]string{"name": "beta ü.txt", "path": "~/tilde", "retention-days": "1"}))
|
||||
assert.Equal(t, "plain text", h.read("home/tilde/beta ü.txt"))
|
||||
assert.Equal(t, []string{"warning: Unexpected input(s) 'retention-days', valid inputs are ['name', 'artifact-ids', 'path', 'pattern', 'merge-multiple', 'github-token', 'repository', 'run-id', 'skip-decompress', 'digest-mismatch']"}, h.annotation)
|
||||
h.upload("..", "", archive)
|
||||
require.ErrorContains(t, h.run(map[string]string{}), "Artifact name is not valid: ..")
|
||||
}
|
||||
|
||||
func TestDownloadFormats(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
contentType, url string
|
||||
zip bool
|
||||
}{
|
||||
{"application/zip", "raw", true},
|
||||
{"Application/X-Zip-Compressed; charset=binary", "raw", true},
|
||||
{"application/zip-compressed", "raw", true},
|
||||
{"", "FILE.ZIP?sig=x", true},
|
||||
{"", "raw?name=file.zip", false},
|
||||
} {
|
||||
assert.Equal(t, test.zip, isZip(&http.Response{Header: http.Header{"Content-Type": {test.contentType}}}, "https://example.test/"+test.url))
|
||||
}
|
||||
for _, test := range []struct {
|
||||
pattern, name string
|
||||
want bool
|
||||
}{
|
||||
{"a*", "alpha", true},
|
||||
{"*", ".hidden", false},
|
||||
{"!alpha", "beta", true},
|
||||
{"#alpha", "alpha", false},
|
||||
} {
|
||||
assert.Equal(t, test.want, matchName(test.pattern, test.name))
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloadDigestMismatch(t *testing.T) {
|
||||
h := newHarness(t)
|
||||
h.upload("art", "", zipOf(t, zipEntry{name: "f.txt", body: "original"}))
|
||||
blobs, err := filepath.Glob(filepath.Join(h.store, "v4", "*", "*", "blob"))
|
||||
require.NoError(t, err)
|
||||
require.Len(t, blobs, 1)
|
||||
tampered := zipOf(t, zipEntry{name: "f.txt", body: "tampered"})
|
||||
require.NoError(t, os.WriteFile(blobs[0], tampered, 0o644))
|
||||
|
||||
for _, behavior := range []string{"", "ignore", "info", "warn", "error"} {
|
||||
err = h.run(map[string]string{"name": "art", "path": "digest", "digest-mismatch": behavior})
|
||||
if behavior == "" || behavior == "error" {
|
||||
require.ErrorContains(t, err, "Digest validation failed for artifact(s): art. Use 'digest-mismatch: warn' to continue on mismatch.")
|
||||
} else {
|
||||
require.NoError(t, err)
|
||||
}
|
||||
assert.Equal(t, "tampered", h.read("digest/f.txt"))
|
||||
if behavior == "warn" {
|
||||
assert.Equal(t, []string{"warning: Artifact 'art' digest validation failed. Please verify the integrity of the artifact."}, h.annotation)
|
||||
} else {
|
||||
assert.Empty(t, h.annotation)
|
||||
}
|
||||
if behavior == "info" {
|
||||
assert.Contains(t, h.logs, "Artifact 'art' digest validation failed. Please verify the integrity of the artifact.")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloadFromAnotherRun(t *testing.T) {
|
||||
h := &harness{t: t, workspace: t.TempDir()}
|
||||
archive := zipOf(t, zipEntry{name: "other.txt", body: "from run 12"})
|
||||
var expired atomic.Bool
|
||||
var zipCalls atomic.Int32
|
||||
blob := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
assert.Empty(t, r.Header.Get("Authorization"))
|
||||
w.Header().Set("Content-Type", "application/zip")
|
||||
_, _ = w.Write(archive)
|
||||
}))
|
||||
defer blob.Close()
|
||||
gitea := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.URL.Path {
|
||||
case "/api/v1/repos/someone/else/actions/runs/12/artifacts":
|
||||
assert.Equal(t, "token pat", r.Header.Get("Authorization"))
|
||||
_, _ = io.WriteString(w, `{"total_count":3,"artifacts":[
|
||||
{"id":10,"name":"dist","size_in_bytes":1,"expired":false},
|
||||
{"id":30,"name":"dist","size_in_bytes":1,"expired":true},
|
||||
{"id":20,"name":"dist","size_in_bytes":1,"expired":false}]}`)
|
||||
case "/api/v1/repos/someone/else/actions/artifacts/20/zip":
|
||||
zipCalls.Add(1)
|
||||
if expired.Load() {
|
||||
http.Error(w, "Artifact has expired", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
http.Redirect(w, r, blob.URL+"/raw/20?sig=abc", http.StatusFound)
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
}))
|
||||
defer gitea.Close()
|
||||
h.server = gitea.URL
|
||||
|
||||
inputs := map[string]string{"github-token": "pat", "repository": "someone/else/ignored", "run-id": "12", "path": "other"}
|
||||
require.NoError(t, h.run(inputs))
|
||||
assert.Equal(t, "from run 12", h.read("other/other.txt"))
|
||||
assert.Contains(t, h.logs, "Found 1 artifact(s)")
|
||||
expired.Store(true)
|
||||
require.ErrorContains(t, h.run(inputs), "Artifact has expired")
|
||||
assert.Equal(t, int32(2), zipCalls.Load(), "an API error is not retried")
|
||||
require.ErrorContains(t, h.run(map[string]string{"github-token": "pat", "repository": "nobody"}), "Invalid repository: 'nobody'. Must be in format owner/repo")
|
||||
require.ErrorContains(t, h.run(map[string]string{"github-token": "pat", "run-id": "0"}), "Invalid run-id: '0'")
|
||||
}
|
||||
|
||||
func TestDownloadRetries(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
attempts := 0
|
||||
target := filepath.Join(t.TempDir(), "download")
|
||||
started := time.Now()
|
||||
_, err := fetch(t.Context(), &action.Context{Container: &container.HostEnvironment{}}, &source{
|
||||
signedURL: func(context.Context, artifact.Artifact) (string, error) {
|
||||
assert.DirExists(t, target)
|
||||
return "https://example.test/artifact.zip", nil
|
||||
},
|
||||
download: func(context.Context, string, time.Duration) (*http.Response, error) {
|
||||
attempts++
|
||||
return nil, errors.New("connection failed")
|
||||
},
|
||||
}, artifact.Artifact{}, target, false)
|
||||
require.EqualError(t, err, "Unable to download and extract artifact: Artifact download failed after 5 retries.")
|
||||
assert.Equal(t, 5, attempts)
|
||||
assert.Equal(t, 25*time.Second, time.Since(started))
|
||||
})
|
||||
}
|
||||
|
||||
func TestDownloadRejectsEscapingZipPaths(t *testing.T) {
|
||||
for _, name := range []string{"../escape.txt", "a/../../escape.txt", "/absolute.txt", `..\escape.txt`} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
archive := zipOf(t, zipEntry{name: name, body: "escape"})
|
||||
require.ErrorContains(t, extractZip(t.Context(), &action.Context{Container: &container.HostEnvironment{}}, bytes.NewReader(archive), int64(len(archive)), t.TempDir()), "Artifact entry path is not valid")
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,142 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package downloadartifact
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"archive/zip"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"io"
|
||||
"mime"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.com/gitea/runner/act/container"
|
||||
"gitea.com/gitea/runner/internal/pkg/action"
|
||||
|
||||
"github.com/bmatcuk/doublestar/v4"
|
||||
)
|
||||
|
||||
func fetchOnce(ctx context.Context, c *action.Context, src *source, signed, target string, skipDecompress bool) (string, error) {
|
||||
resp, err := src.download(ctx, signed, 30*time.Second)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
file, err := os.CreateTemp("", "artifact-*")
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer os.Remove(file.Name())
|
||||
defer file.Close()
|
||||
hash := sha256.New()
|
||||
size, err := io.Copy(io.MultiWriter(file, hash), resp.Body)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
digest := "sha256:" + hex.EncodeToString(hash.Sum(nil))
|
||||
if isZip(resp, signed) && !skipDecompress {
|
||||
return digest, extractZip(ctx, c, file, size, target)
|
||||
}
|
||||
fileName := rawFileName(resp.Header.Get("Content-Disposition"))
|
||||
if fileName == ".." || strings.ContainsAny(fileName, `/\`) || c.Windows() && !filepath.IsLocal(fileName) {
|
||||
return "", action.Failf("Artifact filename is not valid: %s", fileName)
|
||||
}
|
||||
if _, err := file.Seek(0, io.SeekStart); err != nil {
|
||||
return digest, err
|
||||
}
|
||||
return digest, c.Container.CopyTarStream(ctx, target, func(tw *container.TarWriter) error {
|
||||
if err := tw.WriteHeader(&tar.Header{Typeflag: tar.TypeReg, Name: fileName, Mode: 0o644, Size: size, ModTime: time.Now()}); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err := io.Copy(tw, file)
|
||||
return err
|
||||
})
|
||||
}
|
||||
|
||||
var zipTypes = []string{"application/zip", "application/x-zip-compressed", "application/zip-compressed"}
|
||||
|
||||
func isZip(resp *http.Response, signed string) bool {
|
||||
mediaType, _, _ := strings.Cut(resp.Header.Get("Content-Type"), ";")
|
||||
mediaType = strings.ToLower(strings.TrimSpace(mediaType))
|
||||
signedURL, _ := url.Parse(signed)
|
||||
return slices.Contains(zipTypes, mediaType) || signedURL != nil && strings.HasSuffix(strings.ToLower(signedURL.Path), ".zip")
|
||||
}
|
||||
|
||||
func rawFileName(disposition string) string {
|
||||
_, params, err := mime.ParseMediaType(disposition)
|
||||
name := params["filename"] // filename* arrives decoded in here
|
||||
if err != nil || name == "" {
|
||||
return "artifact"
|
||||
}
|
||||
if unescaped, err := url.PathUnescape(name); err == nil && !strings.Contains(strings.ToLower(disposition), "filename*") {
|
||||
name = unescaped
|
||||
}
|
||||
if name = path.Base(name); name == "/" || name == "." {
|
||||
return "artifact"
|
||||
}
|
||||
return name
|
||||
}
|
||||
|
||||
func extractZip(ctx context.Context, c *action.Context, file io.ReaderAt, size int64, target string) error {
|
||||
reader, err := zip.NewReader(file, size)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
now := time.Now()
|
||||
return c.Container.CopyTarStream(ctx, target, func(tw *container.TarWriter) error {
|
||||
for _, entry := range reader.File {
|
||||
name := path.Clean(strings.ReplaceAll(entry.Name, `\`, "/"))
|
||||
if name == "." {
|
||||
continue
|
||||
}
|
||||
if path.IsAbs(name) || name == ".." || strings.HasPrefix(name, "../") || c.Windows() && !filepath.IsLocal(name) {
|
||||
return action.Failf("Artifact entry path is not valid: %s", entry.Name)
|
||||
}
|
||||
header := &tar.Header{Typeflag: tar.TypeReg, Name: name, Mode: 0o644, Size: int64(entry.UncompressedSize64), ModTime: now}
|
||||
if entry.FileInfo().IsDir() {
|
||||
header.Typeflag, header.Name, header.Mode, header.Size = tar.TypeDir, name+"/", 0o755, 0
|
||||
}
|
||||
if err := tw.WriteHeader(header); err != nil {
|
||||
return err
|
||||
}
|
||||
body, err := entry.Open()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = io.Copy(tw, body)
|
||||
if err := errors.Join(err, body.Close()); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func matchName(pattern, name string) bool {
|
||||
if strings.HasPrefix(pattern, "#") {
|
||||
return false
|
||||
}
|
||||
negate := false
|
||||
for strings.HasPrefix(pattern, "!") {
|
||||
negate, pattern = !negate, pattern[1:]
|
||||
}
|
||||
matched, err := doublestar.Match(pattern, name)
|
||||
if err != nil {
|
||||
matched = pattern == name
|
||||
}
|
||||
if strings.HasPrefix(name, ".") && !strings.HasPrefix(pattern, ".") {
|
||||
matched = false
|
||||
}
|
||||
return matched != negate
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package downloadartifact
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.com/gitea/runner/internal/pkg/action"
|
||||
"gitea.com/gitea/runner/internal/pkg/action/artifact"
|
||||
)
|
||||
|
||||
type source struct {
|
||||
list func(context.Context, string) ([]artifact.Artifact, error)
|
||||
signedURL func(context.Context, artifact.Artifact) (string, error)
|
||||
download func(context.Context, string, time.Duration) (*http.Response, error)
|
||||
}
|
||||
|
||||
func newSource(c *action.Context, token string) (*source, error) {
|
||||
if token == "" {
|
||||
client, err := artifact.New(c.ResultsURL, c.RuntimeToken, c.Transport)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
client.Logf = c.Infof
|
||||
return &source{
|
||||
list: client.List,
|
||||
signedURL: func(ctx context.Context, art artifact.Artifact) (string, error) {
|
||||
return client.SignedURL(ctx, art.Name)
|
||||
},
|
||||
download: client.Download,
|
||||
}, nil
|
||||
}
|
||||
parts := append(strings.Split(c.Input("repository"), "/"), "")
|
||||
owner, repo := parts[0], parts[1]
|
||||
if owner == "" || repo == "" {
|
||||
return nil, action.Failf("Invalid repository: '%s'. Must be in format owner/repo", c.Input("repository"))
|
||||
}
|
||||
runID, err := action.ParseInt(c.Input("run-id"))
|
||||
if err != nil || runID <= 0 { // Gitea lists the artifacts of every run for those
|
||||
return nil, action.Failf("Invalid run-id: '%s'. Must be a number.", c.Input("run-id"))
|
||||
}
|
||||
rest, err := artifact.NewREST(c.ResultsURL, token, c.Transport)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rest.Logf = c.Infof
|
||||
return &source{
|
||||
list: func(ctx context.Context, name string) ([]artifact.Artifact, error) {
|
||||
return rest.List(ctx, owner, repo, int64(runID), name)
|
||||
},
|
||||
signedURL: func(ctx context.Context, art artifact.Artifact) (string, error) {
|
||||
return rest.DownloadURL(ctx, owner, repo, art.ID)
|
||||
},
|
||||
download: rest.Download,
|
||||
}, nil
|
||||
}
|
||||
Reference in new issue
Block a user