diff --git a/README.md b/README.md index 3da6adc3..0c6d2f4f 100644 --- a/README.md +++ b/README.md @@ -247,7 +247,7 @@ Set `container.sweep: true` to remove the containers, networks and volumes most ### Built-in actions (`uses: builtin:`) -`uses: builtin:checkout` checks out a repository without downloading `actions/checkout` or needing Node in the job image, see [docs/builtin-actions.md](docs/builtin-actions.md). +`uses: builtin:checkout` and `builtin:upload-artifact` replace their `actions/` counterparts without an action download or Node in the job image, see [docs/builtin-actions.md](docs/builtin-actions.md). ### Caching (`actions/cache`) diff --git a/act/artifacts/server.go b/act/artifacts/server.go index f2292c06..9301b5d0 100644 --- a/act/artifacts/server.go +++ b/act/artifacts/server.go @@ -218,7 +218,7 @@ func Serve(ctx context.Context, artifactPath, addr, port string) context.CancelF logger.Debugf("Artifacts base path '%s'", artifactPath) uploads(router, artifactPath) downloads(router, artifactPath) - registerV4(router, artifactPath) + RegisterV4(router, artifactPath) server := &http.Server{ Addr: net.JoinHostPort(addr, port), diff --git a/act/artifacts/server_test.go b/act/artifacts/server_test.go index 7d63dd3c..a03708e3 100644 --- a/act/artifacts/server_test.go +++ b/act/artifacts/server_test.go @@ -28,7 +28,7 @@ func TestArtifactFlow(t *testing.T) { router := http.NewServeMux() uploads(router, artifactPath) downloads(router, artifactPath) - registerV4(router, artifactPath) + RegisterV4(router, artifactPath) server := httptest.NewServer(router) defer server.Close() @@ -256,7 +256,7 @@ func TestArtifactFlow(t *testing.T) { require.Equal(t, "attachment; filename*=UTF-8''50%25%20art%40v1", header.Get("Content-Disposition")) rerun := http.NewServeMux() - registerV4(rerun, artifactPath) + RegisterV4(rerun, artifactPath) recorder := httptest.NewRecorder() rerun.ServeHTTP(recorder, httptest.NewRequest(http.MethodPost, artifactServicePath+"/CreateArtifact", strings.NewReader("{"+artifactRef+"}"))) require.Equal(t, http.StatusOK, recorder.Code, recorder.Body.String()) diff --git a/act/artifacts/server_v4.go b/act/artifacts/server_v4.go index d2943c17..64d3c451 100644 --- a/act/artifacts/server_v4.go +++ b/act/artifacts/server_v4.go @@ -35,7 +35,7 @@ const ( maxTwirpBytes = 1 << 20 ) -func registerV4(router *http.ServeMux, baseDir string) { +func RegisterV4(router *http.ServeMux, baseDir string) { server := &serverV4{dir: filepath.Join(baseDir, "v4"), lastID: time.Now().UnixMilli(), created: map[string]bool{}} // IDs stay unique across invocations sharing baseDir router.HandleFunc("POST "+artifactServicePath+"/CreateArtifact", twirp(server.createArtifact)) router.HandleFunc("POST "+artifactServicePath+"/FinalizeArtifact", twirp(server.finalizeArtifact)) diff --git a/act/container/container_types.go b/act/container/container_types.go index 0f399dc4..fe1abb9d 100644 --- a/act/container/container_types.go +++ b/act/container/container_types.go @@ -5,6 +5,7 @@ package container import ( + "bytes" "context" "errors" "fmt" @@ -30,6 +31,38 @@ func (e ExitCodeError) Error() string { return fmt.Sprintf("Process completed with exit code %d.", int(e)) } +type commandOutput struct { + *io.PipeReader + cancel context.CancelFunc + done chan struct{} + err error +} + +func (output *commandOutput) Close() error { + _, _ = io.Copy(io.Discard, output.PipeReader) + output.PipeReader.Close() + output.cancel() + <-output.done + return output.err +} + +func streamOutput(ctx context.Context, run func(context.Context, io.Writer, io.Writer) error) io.ReadCloser { + ctx, cancel := context.WithCancel(ctx) + reader, writer := io.Pipe() + output := &commandOutput{PipeReader: reader, cancel: cancel, done: make(chan struct{})} + go func() { + defer close(output.done) + defer cancel() + var stderr bytes.Buffer + output.err = run(ctx, writer, &stderr) + if output.err != nil && strings.TrimSpace(stderr.String()) != "" { + output.err = fmt.Errorf("%w: %s", output.err, strings.TrimSpace(stderr.String())) + } + writer.CloseWithError(output.err) + }() + return output +} + // NewContainerInput the input for the New function type NewContainerInput struct { Image string @@ -130,6 +163,7 @@ type Container interface { Create(capAdd, capDrop []string) common.Executor Copy(destPath string, files ...*FileEntry) common.Executor CopyDir(destPath, srcPath string, useGitIgnore, skipGitDir bool) common.Executor + ExecOutput(ctx context.Context, command []string) (io.ReadCloser, error) GetContainerArchive(ctx context.Context, srcPath string) (io.ReadCloser, error) CopyTarStream(ctx context.Context, destPath string, write func(*TarWriter) error) error Inspect(ctx context.Context) (*Info, error) diff --git a/act/container/docker_run.go b/act/container/docker_run.go index 303683f4..7f56956d 100644 --- a/act/container/docker_run.go +++ b/act/container/docker_run.go @@ -146,6 +146,47 @@ func (cr *containerReference) CopyDir(destPath, srcPath string, useGitIgnore, sk ).IfNot(common.Dryrun) } +func (cr *containerReference) ExecOutput(ctx context.Context, command []string) (io.ReadCloser, error) { + if common.Dryrun(ctx) { + return nil, errors.New("DRYRUN is not supported in ExecOutput") + } + if err := cr.connect()(ctx); err != nil { + return nil, err + } + if err := cr.find()(ctx); err != nil { + return nil, err + } + if cr.id == "" { + return nil, cr.missingContainerError("exec %v", command) + } + return streamOutput(ctx, func(ctx context.Context, stdout, stderr io.Writer) error { + created, err := cr.cli.ExecCreate(ctx, cr.id, client.ExecCreateOptions{ + Cmd: command, AttachStdout: true, AttachStderr: true, + }) + if err != nil { + return err + } + attached, err := cr.cli.ExecAttach(ctx, created.ID, client.ExecAttachOptions{}) + if err != nil { + return err + } + defer attached.Close() + stop := context.AfterFunc(ctx, func() { attached.Close() }) + defer stop() + if _, err := stdcopy.StdCopy(stdout, stderr, attached.Reader); err != nil { + return err + } + inspected, err := cr.cli.ExecInspect(ctx, created.ID, client.ExecInspectOptions{}) + if err != nil { + return err + } + if inspected.ExitCode != 0 { + return ExitCodeError(inspected.ExitCode) + } + return nil + }), nil +} + func (cr *containerReference) GetContainerArchive(ctx context.Context, srcPath string) (io.ReadCloser, error) { if common.Dryrun(ctx) { return nil, errors.New("DRYRUN is not supported in GetContainerArchive") diff --git a/act/container/docker_run_test.go b/act/container/docker_run_test.go index 628c3954..c1fbaa61 100644 --- a/act/container/docker_run_test.go +++ b/act/container/docker_run_test.go @@ -260,6 +260,28 @@ func TestDockerExecFailure(t *testing.T) { assert.Equal(t, ExitCodeError(1), exitErr) assert.Equal(t, "Process completed with exit code 1.", err.Error()) + client.On("ContainerInspect", ctx, "123", mobyclient.ContainerInspectOptions{}). + Return(mobyclient.ContainerInspectResult{}, nil) + client.On("ExecCreate", mock.Anything, "123", mobyclient.ExecCreateOptions{ + Cmd: []string{"printf", `a\b`}, AttachStdout: true, AttachStderr: true, + }).Return(mobyclient.ExecCreateResult{ID: "output"}, nil) + client.On("ExecAttach", mock.Anything, "output", mobyclient.ExecAttachOptions{}). + Return(mobyclient.ExecAttachResult{ + Conn: &mockConn{}, + Reader: bufio.NewReader(bytes.NewReader(append(stdcopyFrame(stdcopy.Stdout, "a\x00b"), stdcopyFrame(stdcopy.Stderr, " failed \n")...))), + }, nil) + client.On("ExecInspect", mock.Anything, "output", mobyclient.ExecInspectOptions{}). + Return(mobyclient.ExecInspectResult{ExitCode: 3}, nil) + output, err := cr.ExecOutput(ctx, []string{"printf", `a\b`}) + require.NoError(t, err) + content, err := io.ReadAll(output) + require.ErrorAs(t, err, &exitErr) + assert.Equal(t, ExitCodeError(3), exitErr) + assert.Equal(t, "Process completed with exit code 3.: failed", err.Error()) + assert.Equal(t, "a\x00b", string(content)) + require.ErrorAs(t, output.Close(), &exitErr) + assert.Equal(t, ExitCodeError(3), exitErr) + client.AssertExpectations(t) } @@ -533,6 +555,8 @@ func TestRejectsMissingContainer(t *testing.T) { check("exec", cr.exec([]string{"echo"}, nil, "", "")(ctx)) _, err := cr.GetContainerArchive(ctx, "/var/run/act/x") check("GetContainerArchive", err) + _, err = cr.ExecOutput(ctx, []string{"true"}) + check("ExecOutput", err) _, err = cr.Inspect(ctx) check("Inspect", err) diff --git a/act/container/host_environment.go b/act/container/host_environment.go index 18b97d90..20d45701 100644 --- a/act/container/host_environment.go +++ b/act/container/host_environment.go @@ -116,6 +116,20 @@ func (e *HostEnvironment) Inspect(_ context.Context) (*Info, error) { return &Info{Health: HealthNone, Ports: map[string]string{}}, nil } +func (e *HostEnvironment) ExecOutput(ctx context.Context, command []string) (io.ReadCloser, error) { + return streamOutput(ctx, func(ctx context.Context, stdout, stderr io.Writer) error { + cmd := exec.CommandContext(ctx, command[0], command[1:]...) + cmd.Dir, cmd.Stdout, cmd.Stderr = e.Workdir, stdout, stderr + if err := cmd.Run(); err != nil { + if exit, ok := errors.AsType[*exec.ExitError](err); ok { + return ExitCodeError(exit.ExitCode()) + } + return err + } + return nil + }), nil +} + func (e *HostEnvironment) GetContainerArchive(ctx context.Context, srcPath string) (io.ReadCloser, error) { srcPath = filepath.Clean(srcPath) fi, err := os.Lstat(srcPath) diff --git a/act/container/host_environment_test.go b/act/container/host_environment_test.go index 5bcd4a3c..96ca86e2 100644 --- a/act/container/host_environment_test.go +++ b/act/container/host_environment_test.go @@ -132,6 +132,21 @@ func TestHostEnvironmentExecExitCode(t *testing.T) { if runtime.GOOS == "windows" { t.Skip("uses POSIX shell") } + for _, read := range []bool{true, false} { + output, err := (&HostEnvironment{AllocatePTY: true}).ExecOutput(t.Context(), []string{"sh", "-c", `test ! -t 1; printf 'out\000'; printf ' failed \n' >&2; exit 3`}) + require.NoError(t, err) + if read { + content, err := io.ReadAll(output) + var exit ExitCodeError + require.ErrorAs(t, err, &exit) + assert.Equal(t, ExitCodeError(3), exit) + assert.Equal(t, "out\x00", string(content)) + assert.Equal(t, "Process completed with exit code 3.: failed", err.Error()) + } + var exit ExitCodeError + require.ErrorAs(t, output.Close(), &exit) + assert.Equal(t, ExitCodeError(3), exit) + } for name, allocatePTY := range map[string]bool{"without PTY": false, "with PTY": true} { t.Run(name, func(t *testing.T) { t.Parallel() diff --git a/act/container/kubernetes.go b/act/container/kubernetes.go index 9f554ff4..8294261b 100644 --- a/act/container/kubernetes.go +++ b/act/container/kubernetes.go @@ -565,17 +565,17 @@ func (c *kubernetesContainer) CopyDir(destPath, srcPath string, useGitIgnore, sk } } -// GetContainerArchive streams the archive, checking the path first as tar writes an empty archive for a missing one. +func (c *kubernetesContainer) ExecOutput(ctx context.Context, command []string) (io.ReadCloser, error) { + return streamOutput(ctx, func(ctx context.Context, stdout, stderr io.Writer) error { + return c.pod.client.exec(ctx, c.pod.name, c.name, command, nil, stdout, stderr) + }), nil +} + func (c *kubernetesContainer) GetContainerArchive(ctx context.Context, srcPath string) (io.ReadCloser, error) { - reader, writer := io.Pipe() - go func() { - var stderr bytes.Buffer - err := c.pod.client.exec(ctx, c.pod.name, c.name, []string{"sh", "-c", `test -e "$1" -o -L "$1" && exec tar -cf - -C "$2" "$3"`, "sh", srcPath, path.Dir(srcPath), path.Base(srcPath)}, nil, writer, &stderr) - if err != nil { - err = fmt.Errorf("archive %s: %w: %s", srcPath, err, strings.TrimSpace(stderr.String())) - } - writer.CloseWithError(err) - }() + reader, err := c.ExecOutput(ctx, []string{"sh", "-c", `test -e "$1" -o -L "$1" && exec tar -cf - -C "$2" "$3"`, "sh", srcPath, path.Dir(srcPath), path.Base(srcPath)}) + if err != nil { + return nil, err + } return startedArchive(reader) } diff --git a/act/container/kubernetes_test.go b/act/container/kubernetes_test.go index 48c731c4..c1f80ada 100644 --- a/act/container/kubernetes_test.go +++ b/act/container/kubernetes_test.go @@ -273,6 +273,17 @@ func TestKubernetesPodRunsJobThroughAPI(t *testing.T) { assert.Equal(t, "hunter2\nit's\nme\n5\n"+workdir+"\n", jobLog.String()) assert.Equal(t, ExitCodeError(3), job.Exec([]string{"sh", "-c", "echo failing >&2; exit 3"}, nil, "", "")(ctx)) assert.Contains(t, jobLog.String(), "failing\n") + output, err := job.ExecOutput(ctx, []string{"sh", "-c", `printf 'out\000'; printf ' failed \n' >&2; exit 3`}) + require.NoError(t, err) + content, err := io.ReadAll(output) + var exit ExitCodeError + require.ErrorAs(t, err, &exit) + assert.Equal(t, ExitCodeError(3), exit) + assert.Contains(t, err.Error(), ": failed") + assert.Equal(t, "out\x00", string(content)) + require.ErrorAs(t, output.Close(), &exit) + assert.Equal(t, ExitCodeError(3), exit) + for _, query := range cluster.requests["exec"] { assert.NotContains(t, query, "hunter2") } @@ -285,7 +296,7 @@ func TestKubernetesPodRunsJobThroughAPI(t *testing.T) { reader := tar.NewReader(archive) header, err := reader.Next() require.NoError(t, err) - content, err := io.ReadAll(reader) + content, err = io.ReadAll(reader) require.NoError(t, err) assert.Equal(t, "event.json", header.Name) assert.JSONEq(t, `{"ref":"main"}`, string(content)) diff --git a/act/container/tar.go b/act/container/tar.go index f36a6329..36b58b22 100644 --- a/act/container/tar.go +++ b/act/container/tar.go @@ -86,7 +86,7 @@ func pipeTar(uid, gid int, prefix string, parents []string, write func(*TarWrite } // startedArchive returns reader once its first byte arrives, so failing to start archiving errors here, not on read. -func startedArchive(reader *io.PipeReader) (io.ReadCloser, error) { +func startedArchive(reader io.ReadCloser) (io.ReadCloser, error) { buffered := bufio.NewReader(reader) if _, err := buffered.Peek(1); err != nil { reader.Close() diff --git a/act/runner/action_composite_test.go b/act/runner/action_composite_test.go index a9408bad..9a36ad65 100644 --- a/act/runner/action_composite_test.go +++ b/act/runner/action_composite_test.go @@ -11,6 +11,8 @@ import ( "gitea.com/gitea/runner/act/common/git" "gitea.com/gitea/runner/act/common/logging" "gitea.com/gitea/runner/act/common/logging/logtest" + "gitea.com/gitea/runner/act/container" + "gitea.com/gitea/runner/internal/pkg/action" "gitea.dev/actionslib/pkg/model" "github.com/stretchr/testify/assert" @@ -182,3 +184,21 @@ func newTestCompositeRunContext(t *testing.T, parent *RunContext) *RunContext { require.NoError(t, err) return composite } + +func TestBuiltinActionGetsLowercaseInputsAndTheJobTokenInComposites(t *testing.T) { + job := makeTestRC(t, "runs-on: ubuntu-latest") + job.Config.Secrets = map[string]string{"ACTIONS_RUNTIME_TOKEN": "daemon-token"} + job.JobContainer = &container.HostEnvironment{Path: t.TempDir(), ActPath: t.TempDir(), Workdir: t.TempDir()} + composite := newTestCompositeRunContext(t, job) + composite.JobContainer = job.JobContainer + for _, rc := range []*RunContext{job, composite} { + var got *action.Context + sab := &stepActionBuiltin{Step: &model.Step{ID: "x", With: map[string]string{"Name": "x"}}, RunContext: rc, run: func(_ctx context.Context, c *action.Context) error { + got = c + return nil + }} + require.NoError(t, sab.main()(t.Context())) + assert.Equal(t, "daemon-token", got.RuntimeToken) + assert.Equal(t, map[string]string{"name": "x"}, got.Inputs) + } +} diff --git a/act/runner/container_mock_test.go b/act/runner/container_mock_test.go index 4d585396..30317c40 100644 --- a/act/runner/container_mock_test.go +++ b/act/runner/container_mock_test.go @@ -66,6 +66,12 @@ func (cm *containerMock) Exec(command []string, env map[string]string, user, wor return cm.Called(command, env, user, workdir).Get(0).(func(context.Context) error) } +func (cm *containerMock) ExecOutput(ctx context.Context, command []string) (io.ReadCloser, error) { + args := cm.Called(ctx, command) + output, _ := args.Get(0).(io.ReadCloser) + return output, args.Error(1) +} + func (cm *containerMock) GetContainerArchive(ctx context.Context, srcPath string) (io.ReadCloser, error) { args := cm.Called(ctx, srcPath) err, _ := args.Get(1).(error) diff --git a/act/runner/job_hooks_test.go b/act/runner/job_hooks_test.go index 56ab5ab5..69d38266 100644 --- a/act/runner/job_hooks_test.go +++ b/act/runner/job_hooks_test.go @@ -71,6 +71,11 @@ func (c *hookContainer) UpdateFromEnv(_ string, env *map[string]string) common.E } } +func (c *hookContainer) ExecOutput(_ctx context.Context, command []string) (io.ReadCloser, error) { + c.cmd = command + return io.NopCloser(bytes.NewBufferString(c.output)), c.err +} + func (c *hookContainer) GetContainerArchive(context.Context, string) (io.ReadCloser, error) { return io.NopCloser(bytes.NewReader(c.pathTar)), nil } diff --git a/act/runner/runner.go b/act/runner/runner.go index 18e60476..240f9331 100644 --- a/act/runner/runner.go +++ b/act/runner/runner.go @@ -81,6 +81,7 @@ type Config struct { PlatformPicker func(labels []string) string KubernetesPicker func(labels []string) container.KubernetesOptions InstanceTransport http.RoundTripper + ResultsURL string // the results service as the runner reaches it, never the job's ACTIONS_RESULTS_URL, which a step can rewrite JobLoggerLevel *logging.Level // the level of job logger ValidVolumes []string // only volumes (and bind mounts) in this slice can be mounted on the job container or service containers ContainerVolumes []string // mounted into every job, service and docker action container diff --git a/act/runner/step_action_builtin.go b/act/runner/step_action_builtin.go index d129147a..e259e942 100644 --- a/act/runner/step_action_builtin.go +++ b/act/runner/step_action_builtin.go @@ -4,17 +4,22 @@ package runner import ( + "cmp" "context" + "fmt" + "strings" "gitea.com/gitea/runner/act/common" "gitea.com/gitea/runner/internal/pkg/action" "gitea.com/gitea/runner/internal/pkg/action/checkout" + "gitea.com/gitea/runner/internal/pkg/action/uploadartifact" "gitea.dev/actionslib/pkg/model" ) var builtinActions = map[string]func(context.Context, *action.Context) error{ - "checkout": checkout.Main, + "checkout": checkout.Main, + "upload-artifact": uploadartifact.Main, } type stepActionBuiltin struct { @@ -34,13 +39,18 @@ func (sab *stepActionBuiltin) main() common.Executor { } printRunActionHeader(ctx, sab.Step, nil, sab.env, sab.RunContext) + rawLogger := common.Logger(ctx).With(rawOutputField, true) rc := sab.RunContext rc.ApplyExtraPath(ctx, &sab.env) + inputs := make(map[string]string, len(sab.Step.With)) + for name, value := range sab.Step.With { + inputs[strings.ToLower(name)] = value // input names are case-insensitive on GitHub + } return sab.run(ctx, &action.Context{ Container: rc.JobContainer, Github: getStepGithubContext(ctx, sab), - Inputs: sab.Step.With, + Inputs: inputs, HostWorkdir: rc.Config.Workdir, Workspace: rc.JobContainer.ToContainerPath(rc.Config.Workdir), BindWorkdir: rc.Config.BindWorkdir, @@ -51,6 +61,13 @@ func (sab *stepActionBuiltin) main() common.Executor { ClientKeyFile: rc.Config.ClientKeyFile, CAFile: rc.jobCAFile(), Env: sab.env, + ResultsURL: rc.Config.ResultsURL, + RuntimeToken: cmp.Or(rc.runtimeToken, rc.topLevelRunContext().Config.Secrets["ACTIONS_RUNTIME_TOKEN"]), // composite contexts have no secrets + Transport: rc.Config.InstanceTransport, + Infof: rawLogger.Infof, + Warning: func(format string, args ...any) { + rawLogger.Infof("%s", formatAnnotation("warning", nil, fmt.Sprintf(format, args...))) + }, }) }) } diff --git a/docs/builtin-actions.md b/docs/builtin-actions.md index b33a977f..f9757ed0 100644 --- a/docs/builtin-actions.md +++ b/docs/builtin-actions.md @@ -4,6 +4,7 @@ Built-in actions ship with the runner, so a job needs no action download and no ```yaml - uses: builtin:checkout +- uses: builtin:upload-artifact ``` ## checkout @@ -37,3 +38,24 @@ A branch is checked out as a local branch tracking `origin`, a tag or commit as Persisted credentials end with the job. `token` is only sent to `github.server_url`, while `ssh-key` is offered to any SSH host, as with `actions/checkout`. The runner's `client_cert_file` is only offered to `github.server_url` and only during the step, so later steps do not get it. Its `ca_file` replaces the job image's CA bundle for `github.server_url`, in later steps too when credentials persist. Without `ssh-key`, submodule URLs of the form `git@:` are fetched from `github.server_url` with `token`. With `ssh-key`, the repository is fetched from the SSH URL in the event's `repository.ssh_url`, or from the server's host on port 22. With `gitea-runner exec`, checking out the workflow's own repository copies your local working directory, unless `--no-skip-checkout` or `ssh-key` is set. + +## upload-artifact + +Works like `actions/upload-artifact` v7 with these inputs. Any other input logs a warning, as on GitHub. It needs Gitea 1.22.4 or later, and 1.26 for `archive: false`. + +| Input | Default | +| --- | --- | +| `name` | `artifact`, ignored with `archive: false`, which names the artifact after the file | +| `path` | required, files, directories or patterns, one per line, `!` excludes | +| `if-no-files-found` | `warn`, also `error` or `ignore` | +| `retention-days` | the server's default, at most the repository's retention limit | +| `compression-level` | `6`, from `0` for none to `9` | +| `overwrite` | `false`, `true` deletes an artifact of the same name first | +| `include-hidden-files` | `false`, `true` uploads files and directories whose name starts with `.` | +| `archive` | `true`, `false` uploads a single file as is | + +Patterns, symlinks and the artifact's root directory follow `actions/upload-artifact`. `ACTIONS_ARTIFACT_UPLOAD_CONCURRENCY` sets upload concurrency (default `5`), and `ACTIONS_ARTIFACT_UPLOAD_TIMEOUT_MS` sets the upload stall timeout (default `300000`). The step outputs `artifact-id`, `artifact-digest` and `artifact-url`. On Gitea 1.26 and later, the `artifact-url` is the artifact's link on the run page for this attempt, where `actions/upload-artifact` gives a link only GitHub serves. + +Container jobs list the search paths in the job container and transfer only the selected files, gzipped when the image has gzip, requiring `sh`, `find` and `tar` in the image. On Kubernetes these files still pass through the API server, unlike `actions/upload-artifact`, which uploads from the pod. + +With `gitea-runner exec`, it uses the local artifact server of `--artifact-server-path`. diff --git a/e2e/testdata/workflows/artifact.yml b/e2e/testdata/workflows/artifact.yml index 6d88c6d2..b97e258e 100644 --- a/e2e/testdata/workflows/artifact.yml +++ b/e2e/testdata/workflows/artifact.yml @@ -26,3 +26,35 @@ jobs: name: e2e-artifact-${{ github.run_id }} path: got - run: grep -q artifact-payload got/data.txt + builtin-upload: + runs-on: e2e-artifact + steps: + - run: | + mkdir -p out/sub + echo "builtin-payload" > out/data.txt + echo "nested" > out/sub/nested.txt + echo "hidden" > out/.hidden + echo "raw" > report.txt + - id: zipped + uses: builtin:upload-artifact + with: + name: e2e-builtin-${{ github.run_id }} + path: out + - run: test -n "${{ steps.zipped.outputs.artifact-id }}" && test -n "${{ steps.zipped.outputs.artifact-digest }}" + - uses: builtin:upload-artifact + with: + path: report.txt + archive: false + js-downloads-builtin: + needs: builtin-upload + runs-on: e2e-artifact + steps: + - uses: actions/download-artifact@v8 + with: + name: e2e-builtin-${{ github.run_id }} + path: got + - run: grep -q builtin-payload got/data.txt && grep -q nested got/sub/nested.txt && test ! -e got/.hidden + - uses: actions/download-artifact@v8 + with: + name: report.txt + - run: grep -q raw report.txt diff --git a/internal/app/cmd/exec.go b/internal/app/cmd/exec.go index 7f1f9804..7f4edfae 100644 --- a/internal/app/cmd/exec.go +++ b/internal/app/cmd/exec.go @@ -209,6 +209,10 @@ func parseKVAndFile(rawKVs []string, filePath string) map[string]string { return result } +func (i *executeArgs) artifactServerURL() string { + return "http://" + net.JoinHostPort(i.artifactServerAddr, i.artifactServerPort) +} + // Workdir returns path to workdir func (i *executeArgs) Workdir() string { return i.resolve(".") @@ -382,6 +386,7 @@ func (i *executeArgs) runnerConfig(eventName string, env, proxyEnv map[string]st ArtifactServerPath: i.artifactServerPath, ArtifactServerPort: i.artifactServerPort, ArtifactServerAddr: i.artifactServerAddr, + ResultsURL: i.artifactServerURL(), NoSkipCheckout: i.noSkipCheckout, EventName: eventName, EventJSON: eventJSON, @@ -513,7 +518,7 @@ func runExec(ctx context.Context, execArgs *executeArgs) func(cmd *cobra.Command token := cmp.Or(inheritedToken, artifacts.LocalRuntimeToken(workflowRun, maxLifetime)) config.RuntimeTokens[run.Workflow] = token config.ExtraMasks = append(config.ExtraMasks, token) - handler.RegisterJob(token, artifactcache.JobCredential{Repo: "__local/__exec", Results: "http://" + net.JoinHostPort(execArgs.artifactServerAddr, execArgs.artifactServerPort)}) + handler.RegisterJob(token, artifactcache.JobCredential{Repo: "__local/__exec", Results: execArgs.artifactServerURL()}) } } diff --git a/internal/app/run/runner.go b/internal/app/run/runner.go index 55553d83..66a15fa1 100644 --- a/internal/app/run/runner.go +++ b/internal/app/run/runner.go @@ -629,6 +629,7 @@ func (r *Runner) run(ctx context.Context, task *runnerv1.Task, reporter *report. ClientKeyFile: r.cfg.Runner.ClientKeyFile, CAFile: r.cfg.Runner.CAFile, InstanceTransport: r.instanceTransport, + ResultsURL: strings.TrimSuffix(r.client.Address(), "/"), RunnerName: r.name, KubernetesPicker: r.kubernetesOptions, } diff --git a/internal/pkg/action/action.go b/internal/pkg/action/action.go index 5aca21d4..ffe2ba82 100644 --- a/internal/pkg/action/action.go +++ b/internal/pkg/action/action.go @@ -5,6 +5,19 @@ package action import ( + "context" + "errors" + "fmt" + "net/http" + "os" + "path" + "path/filepath" + "runtime" + "slices" + "strconv" + "strings" + "unicode" + "gitea.com/gitea/runner/act/container" "gitea.dev/actionslib/pkg/model" @@ -24,4 +37,124 @@ type Context struct { ClientKeyFile string CAFile string // the runner CA as seen inside Container Env map[string]string + ResultsURL string // the results service as the runner reaches it + RuntimeToken string + Transport http.RoundTripper // carries the instance's TLS settings, nil for the default + Infof func(format string, args ...any) + Warning func(format string, args ...any) +} + +func (c *Context) ApplyDefaults(pairs ...string) { + var valid, unexpected []string + for i := 0; i+1 < len(pairs); i += 2 { + valid = append(valid, pairs[i]) + if _, set := c.Inputs[pairs[i]]; !set && pairs[i+1] != "" { + c.Inputs[pairs[i]] = pairs[i+1] + } + } + for name := range c.Inputs { + if !slices.Contains(valid, name) { + unexpected = append(unexpected, name) + } + } + if len(unexpected) > 0 { + slices.Sort(unexpected) + c.Warning("Unexpected input(s) '%s', valid inputs are ['%s']", strings.Join(unexpected, "', '"), strings.Join(valid, "', '")) + } +} + +// Input is the trimmed input, as @actions/core getInput returns it. +func (c *Context) Input(name string) string { + return strings.TrimSpace(c.Inputs[name]) +} + +// BoolInput accepts what @actions/core getBooleanInput does. +func (c *Context) BoolInput(name string) (bool, error) { + switch c.Input(name) { + case "true", "True", "TRUE": + return true, nil + case "false", "False", "FALSE": + return false, nil + } + return false, Failf("Input does not meet YAML 1.2 \"Core Schema\" specification: %s\nSupport boolean input list: `true | True | TRUE | false | False | FALSE`", name) +} + +func (c *Context) SetOutputs(ctx context.Context, pairs ...string) error { + output := c.Env["GITHUB_OUTPUT"] + if output == "" { + return nil + } + var body strings.Builder + for i := 0; i+1 < len(pairs); i += 2 { + if strings.ContainsAny(pairs[i+1], "\r\n") { + return fmt.Errorf("output %s spans several lines", pairs[i]) + } + body.WriteString(pairs[i] + "=" + pairs[i+1] + "\n") + } + return c.Container.Copy(c.Dir(output), &container.FileEntry{Name: c.Base(output), Mode: 0o666, Body: body.String()})(ctx) +} + +func (c *Context) Host() bool { + _, host := c.Container.(*container.HostEnvironment) + return host +} + +func (c *Context) Windows() bool { + return c.Host() && runtime.GOOS == "windows" +} + +func (c *Context) Dir(p string) string { + if c.Windows() { + return filepath.Dir(p) + } + return path.Dir(p) +} + +func (c *Context) Base(p string) string { + if c.Windows() { + return filepath.Base(p) + } + return path.Base(p) +} + +// Home is the job user's home directory, what os.homedir() returns to a JavaScript action. +func (c *Context) Home(ctx context.Context) (string, error) { + if home := c.Env["HOME"]; home != "" { + return home, nil + } + if home := c.Env["USERPROFILE"]; c.Windows() && home != "" { + return home, nil + } + if c.Host() { + return os.UserHomeDir() + } + var out strings.Builder + stdout, stderr := c.Container.ReplaceLogWriter(&out, &out) + err := c.Container.Exec([]string{"sh", "-c", `printf %s "$HOME"`}, nil, "", "")(ctx) + c.Container.ReplaceLogWriter(stdout, stderr) + if err != nil { + return "", fmt.Errorf("cannot expand ~ without HOME: %w", err) + } + if out.Len() == 0 { + return "", errors.New("cannot expand ~, the job user has no HOME") + } + return out.String(), nil +} + +// Failf is fmt.Errorf for messages worded as the JavaScript action words them, which the error string lint rejects. +func Failf(format string, args ...any) error { + return fmt.Errorf(format, args...) +} + +// ParseInt reads a leading integer the way JavaScript's parseInt does. +func ParseInt(s string) (int, error) { + s = strings.TrimLeftFunc(s, unicode.IsSpace) + end := 0 + if end < len(s) && (s[end] == '-' || s[end] == '+') { + end++ + } + for end < len(s) && s[end] >= '0' && s[end] <= '9' { + end++ + } + return strconv.Atoi(s[:end]) } diff --git a/internal/pkg/action/artifact/blob.go b/internal/pkg/action/artifact/blob.go new file mode 100644 index 00000000..2dfdb9d7 --- /dev/null +++ b/internal/pkg/action/artifact/blob.go @@ -0,0 +1,184 @@ +// Copyright 2026 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package artifact + +import ( + "bytes" + "context" + "crypto/rand" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "sync/atomic" + "time" + + "golang.org/x/sync/errgroup" +) + +// BlockSize is the toolkit's upload chunk. +const BlockSize = 8 << 20 + +type Uploaded struct { + Size int64 + Digest string // sha256: for Finalize +} + +// Upload puts body to a signed URL as parallel Azure blocks, failing when upload progress stalls for stall. +func (c *Client) Upload(ctx context.Context, signedURL, contentType string, body io.Reader, concurrency int, stall time.Duration) (result Uploaded, err error) { + ctx, cancel := context.WithCancelCause(ctx) + defer func() { + if ctx.Err() != nil { + err = context.Cause(ctx) + } + cancel(nil) + }() + stall = max(stall, time.Millisecond) + timer := time.AfterFunc(stall, func() { cancel(errors.New("upload progress stalled")) }) + defer timer.Stop() + hash := sha256.New() + reader := io.TeeReader(body, hash) + group, blockCtx := errgroup.WithContext(ctx) + if closer, ok := body.(io.ReadCloser); ok { + stop := context.AfterFunc(blockCtx, func() { closer.Close() }) + defer stop() + } + buffers := make(chan []byte, concurrency) + prefix := rand.Text() + var ids []string + var uploaded atomic.Int64 + for index := 0; ; index++ { + var buf []byte + if index < concurrency { + buf = make([]byte, BlockSize) + } else { + select { + case buf = <-buffers: + case <-blockCtx.Done(): + return Uploaded{}, group.Wait() + } + } + n, readErr := io.ReadFull(reader, buf) + if errors.Is(readErr, io.EOF) { + break + } else if readErr != nil && !errors.Is(readErr, io.ErrUnexpectedEOF) { + if blockCtx.Err() == nil { + cancel(readErr) + } + if err := group.Wait(); err != nil { + return Uploaded{}, err + } + return Uploaded{}, readErr + } + id := base64.StdEncoding.EncodeToString(fmt.Appendf(nil, "%s-%08d", prefix, index)) // fixed width as Azure needs, unique as Gitea before 1.25.5 shares a run's blocks + ids = append(ids, id) + group.Go(func() error { + defer func() { buffers <- buf }() + if err := c.put(blockCtx, signedURL+"&comp=block&blockid="+url.QueryEscape(id), "x-ms-blob-type", "BlockBlob", buf[:n]); err != nil { + return err + } + timer.Reset(stall) + c.Logf("Uploaded bytes %d", uploaded.Add(int64(n))) + return nil + }) + if readErr != nil { + break + } + } + if err := group.Wait(); err != nil { + return Uploaded{}, err + } + var list strings.Builder + list.WriteString("") + for _, id := range ids { + list.WriteString("" + id + "") + } + list.WriteString("") + if err := c.put(ctx, signedURL+"&comp=blocklist", "x-ms-blob-content-type", contentType, []byte(list.String())); err != nil { + return Uploaded{}, err + } + return Uploaded{Size: uploaded.Load(), Digest: "sha256:" + hex.EncodeToString(hash.Sum(nil))}, nil +} + +func (c *Client) put(ctx context.Context, signedURL, header, value string, body []byte) error { + resp, err := c.Retry.do(ctx, c.Logf, "blob upload", func() (*http.Response, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodPut, signedURL, bytes.NewReader(body)) + if err != nil { + return nil, err + } + req.Header.Set(header, value) + resp, err := c.blob.Do(req) + return resp, scrub(err) + }) + if err != nil { + return err + } + return resp.Body.Close() +} + +// Download GETs a signed URL, a read failing when nothing arrives for stall. +func (c *Client) Download(ctx context.Context, signedURL string, stall time.Duration) (*http.Response, error) { + ctx, cancel := context.WithCancelCause(ctx) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, signedURL, nil) + if err != nil { + cancel(nil) + return nil, err + } + resp, err := c.blob.Do(req) + if err != nil { + cancel(nil) + return nil, scrub(err) + } + if resp.StatusCode != http.StatusOK { + resp.Body.Close() + cancel(nil) + return nil, fmt.Errorf("unexpected HTTP response from blob storage: %d %s", resp.StatusCode, http.StatusText(resp.StatusCode)) + } + resp.Body = &stallReader{body: resp.Body, timer: time.AfterFunc(stall, func() { + cancel(fmt.Errorf("blob storage chunk did not respond in %s", stall)) + }), stall: stall, cancel: cancel, ctx: ctx} + return resp, nil +} + +func scrub(err error) error { // the signature of a signed URL is a credential + if urlErr, ok := errors.AsType[*url.Error](err); ok { + if parsed, parseErr := url.Parse(urlErr.URL); parseErr == nil { + parsed.RawQuery = "" + urlErr.URL = parsed.String() + } else { + urlErr.URL = "" + } + } + return err +} + +type stallReader struct { + body io.ReadCloser + timer *time.Timer + stall time.Duration + cancel context.CancelCauseFunc + ctx context.Context +} + +func (r *stallReader) Read(p []byte) (int, error) { + n, err := r.body.Read(p) + if n > 0 { + r.timer.Reset(r.stall) + } + if err != nil && !errors.Is(err, io.EOF) && context.Cause(r.ctx) != nil { + err = context.Cause(r.ctx) + } + return n, err +} + +func (r *stallReader) Close() error { + r.timer.Stop() + r.cancel(nil) + return r.body.Close() +} diff --git a/internal/pkg/action/artifact/client.go b/internal/pkg/action/artifact/client.go new file mode 100644 index 00000000..91b5fc4d --- /dev/null +++ b/internal/pkg/action/artifact/client.go @@ -0,0 +1,264 @@ +// Copyright 2026 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +// Package artifact talks to the v4 results service of the job's run. +package artifact + +import ( + "bytes" + "cmp" + "context" + "encoding/base64" + "encoding/json/v2" + "errors" + "fmt" + "io" + "math" + "net/http" + "net/url" + "strconv" + "strings" + "time" +) + +type Artifact struct { + ID int64 `json:"databaseId,string"` + Name string `json:"name"` + Size int64 `json:"size,string"` + Digest string `json:"digest"` +} + +type Retry struct { + Attempts int + Base time.Duration + Multiplier float64 + MaxWait time.Duration +} + +var DefaultRetry = Retry{Attempts: 5, Base: 8 * time.Second, Multiplier: 1.5, MaxWait: 2 * time.Minute} + +type Client struct { + base *url.URL + token string + ids backendIDsMessage + api, blob *http.Client + Retry Retry + Logf func(format string, args ...any) +} + +// New reaches base through transport, which carries the instance's TLS settings, and signed URLs elsewhere directly. +func New(base, token string, transport http.RoundTripper) (*Client, error) { + if base == "" { + return nil, errors.New("no artifact service is configured for this runner") + } + parsed, err := url.Parse(base) + if err != nil { + return nil, err + } + if token == "" { + return nil, errors.New("ACTIONS_RUNTIME_TOKEN is not set") + } + ids, err := backendIDs(token) + if err != nil { + return nil, err + } + return &Client{ + base: parsed, token: token, ids: ids, + api: &http.Client{Transport: transport, Timeout: 3 * time.Minute}, + blob: &http.Client{Transport: blobTransport{instance: parsed.Host, transport: transport}}, + Retry: DefaultRetry, + Logf: func(string, ...any) {}, + }, nil +} + +// backendIDs reads the unverified scp claim of a runtime token, as the toolkit does. +func backendIDs(token string) (backendIDsMessage, error) { + parts := strings.Split(token, ".") + if len(parts) != 3 { + return backendIDsMessage{}, errors.New("the runtime token is not a JWT, artifacts v4 need Gitea 1.22 or later") + } + payload, err := base64.RawURLEncoding.DecodeString(strings.TrimRight(parts[1], "=")) + if err != nil { + return backendIDsMessage{}, err + } + var claims struct { + Scp string `json:"scp"` + } + if err := json.Unmarshal(payload, &claims); err != nil { + return backendIDsMessage{}, err + } + for scope := range strings.FieldsSeq(claims.Scp) { + if fields := strings.Split(scope, ":"); fields[0] == "Actions.Results" && len(fields) == 3 { + return backendIDsMessage{fields[1], fields[2]}, nil + } + } + return backendIDsMessage{}, errors.New("the runtime token has no Actions.Results scope") +} + +type blobTransport struct { // the instance's TLS settings only for the instance, as a signed URL can point at object storage + instance string + transport http.RoundTripper +} + +func (t blobTransport) RoundTrip(req *http.Request) (*http.Response, error) { + if req.URL.Host == t.instance && t.transport != nil { + return t.transport.RoundTrip(req) + } + return http.DefaultTransport.RoundTrip(req) +} + +type StatusError struct { + Status int + Msg string +} + +func (e *StatusError) Error() string { + if e.Msg == "" { + return fmt.Sprintf("(%d) %s", e.Status, http.StatusText(e.Status)) + } + return fmt.Sprintf("(%d) %s: %s", e.Status, http.StatusText(e.Status), e.Msg) +} + +func IsNotFound(err error) bool { + var status *StatusError + return errors.As(err, &status) && status.Status == http.StatusNotFound +} + +func (r Retry) do(ctx context.Context, logf func(string, ...any), what string, send func() (*http.Response, error)) (*http.Response, error) { + var waited time.Duration + for attempt := 0; ; attempt++ { + resp, err := send() + var wait time.Duration + if err == nil { + if resp.StatusCode >= 200 && resp.StatusCode < 400 { + return resp, nil + } + body, _ := io.ReadAll(io.LimitReader(resp.Body, 4096)) + resp.Body.Close() + err = &StatusError{Status: resp.StatusCode, Msg: twirpMessage(body)} + switch resp.StatusCode { + case http.StatusTooManyRequests, http.StatusInternalServerError, http.StatusBadGateway, http.StatusServiceUnavailable, http.StatusGatewayTimeout: + default: + return nil, err + } + if seconds, parseErr := strconv.Atoi(resp.Header.Get("Retry-After")); parseErr == nil && seconds > 0 && resp.StatusCode == http.StatusTooManyRequests { + wait = time.Duration(seconds) * time.Second + } + } + if ctx.Err() != nil { + return nil, ctx.Err() + } + if attempt+1 >= r.Attempts { + return nil, fmt.Errorf("%s failed after %d attempts: %w", what, r.Attempts, err) + } + if wait == 0 { + wait = time.Duration(float64(r.Base) * math.Pow(r.Multiplier, float64(attempt))) + } + if waited+wait > r.MaxWait { + return nil, fmt.Errorf("%s: a retry wait of %s would exceed %s: %w", what, wait, r.MaxWait, err) + } + logf("Attempt %d of %d failed with error: %v. Retrying request in %d ms...", attempt+1, r.Attempts, err, wait.Milliseconds()) + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-time.After(wait): + } + waited += wait + } +} + +// twirpMessage is the msg of a twirp error, the message of a Gitea API one, or the body Gitea answers in plain text. +func twirpMessage(body []byte) string { + var twirpErr struct { + Msg string `json:"msg"` + Message string `json:"message"` + } + if json.Unmarshal(body, &twirpErr) == nil && cmp.Or(twirpErr.Msg, twirpErr.Message) != "" { + return cmp.Or(twirpErr.Msg, twirpErr.Message) + } + return strings.TrimSpace(string(body)) +} + +type ( // the ArtifactService messages in the protobuf JSON mapping, requests with the proto names the toolkit sends + backendIDsMessage struct { + WorkflowRunBackendID string `json:"workflow_run_backend_id"` + WorkflowJobRunBackendID string `json:"workflow_job_run_backend_id"` + } + namedRequest struct { + backendIDsMessage + Name string `json:"name"` + } + createRequest struct { + namedRequest + ExpiresAt *time.Time `json:"expires_at,omitempty"` + Version int32 `json:"version"` + MimeType string `json:"mime_type,omitempty"` // Gitea before 1.26 rejects the field + } + finalizeRequest struct { + namedRequest + Size int64 `json:"size,string"` + Hash string `json:"hash"` + } + listRequest struct { + backendIDsMessage + NameFilter string `json:"name_filter,omitempty"` + } + idResponse struct { + ArtifactID int64 `json:"artifactId,string"` + } +) + +func call[Resp any](ctx context.Context, c *Client, method string, req any) (Resp, error) { + var resp Resp + body, err := json.Marshal(req) + if err != nil { + return resp, err + } + endpoint := c.base.JoinPath("/twirp/github.actions.results.api.v1.ArtifactService", method).String() + httpResp, err := c.Retry.do(ctx, c.Logf, method, func() (*http.Response, error) { + httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, bytes.NewReader(body)) + if err != nil { + return nil, err + } + httpReq.Header.Set("Content-Type", "application/json") + httpReq.Header.Set("Authorization", "Bearer "+c.token) + return c.api.Do(httpReq) + }) + if err != nil { + return resp, fmt.Errorf("failed to %s: %w", method, err) + } + defer httpResp.Body.Close() + err = json.UnmarshalRead(httpResp.Body, &resp) + return resp, err +} + +func (c *Client) Create(ctx context.Context, name, mimeType string, expiresAt *time.Time) (string, error) { + resp, err := call[struct { + SignedUploadURL string `json:"signedUploadUrl"` + }](ctx, c, "CreateArtifact", createRequest{namedRequest{c.ids, name}, expiresAt, 7, mimeType}) + return resp.SignedUploadURL, err +} + +func (c *Client) Finalize(ctx context.Context, name string, size int64, digest string) (int64, error) { + resp, err := call[idResponse](ctx, c, "FinalizeArtifact", finalizeRequest{namedRequest{c.ids, name}, size, digest}) + return resp.ArtifactID, err +} + +func (c *Client) List(ctx context.Context, name string) ([]Artifact, error) { + resp, err := call[struct { + Artifacts []Artifact `json:"artifacts"` + }](ctx, c, "ListArtifacts", listRequest{c.ids, name}) + return resp.Artifacts, err +} + +func (c *Client) SignedURL(ctx context.Context, name string) (string, error) { + resp, err := call[struct { + SignedURL string `json:"signedUrl"` + }](ctx, c, "GetSignedArtifactURL", namedRequest{c.ids, name}) + return resp.SignedURL, err +} + +func (c *Client) Delete(ctx context.Context, name string) (int64, error) { + resp, err := call[idResponse](ctx, c, "DeleteArtifact", namedRequest{c.ids, name}) + return resp.ArtifactID, err +} diff --git a/internal/pkg/action/artifact/client_test.go b/internal/pkg/action/artifact/client_test.go new file mode 100644 index 00000000..529db5e0 --- /dev/null +++ b/internal/pkg/action/artifact/client_test.go @@ -0,0 +1,182 @@ +// Copyright 2026 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package artifact + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" + + "gitea.com/gitea/runner/act/artifacts" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestClientRoundTrip(t *testing.T) { + router := http.NewServeMux() + artifacts.RegisterV4(router, t.TempDir()) + var blocks atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + if request.URL.Query().Get("comp") == "block" { + blocks.Add(1) + } + router.ServeHTTP(writer, request) + })) + defer server.Close() + client, err := New(server.URL, artifacts.LocalRuntimeToken("7", time.Hour), nil) + require.NoError(t, err) + for _, content := range [][]byte{nil, bytes.Repeat([]byte("payload"), 2*BlockSize/7+123)} { + sum := sha256.Sum256(content) + uploadURL, err := client.Create(t.Context(), "art", "text/plain", nil) + require.NoError(t, err) + uploaded, err := client.Upload(t.Context(), uploadURL, "text/plain", bytes.NewReader(content), 2, time.Minute) + require.NoError(t, err) + assert.Equal(t, int32((len(content)+BlockSize-1)/BlockSize), blocks.Swap(0)) + assert.Equal(t, Uploaded{Size: int64(len(content)), Digest: "sha256:" + hex.EncodeToString(sum[:])}, uploaded) + id, err := client.Finalize(t.Context(), "art", uploaded.Size, uploaded.Digest) + require.NoError(t, err) + listed, err := client.List(t.Context(), "art") + require.NoError(t, err) + assert.Equal(t, []Artifact{{ID: id, Name: "art", Size: int64(len(content)), Digest: uploaded.Digest}}, listed) + signed, err := client.SignedURL(t.Context(), "art") + require.NoError(t, err) + resp, err := client.Download(t.Context(), signed, time.Minute) + require.NoError(t, err) + got, err := io.ReadAll(resp.Body) + require.NoError(t, err) + require.NoError(t, resp.Body.Close()) + assert.True(t, bytes.Equal(content, got)) + deleted, err := client.Delete(t.Context(), "art") + require.NoError(t, err) + assert.Equal(t, id, deleted) + _, err = client.SignedURL(t.Context(), "art") + assert.True(t, IsNotFound(err), err) + } +} + +func TestClientRetries(t *testing.T) { + var calls atomic.Int32 + var blockIDs []string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodPut { + blockIDs = append(blockIDs, r.URL.Query().Get("blockid")) + w.WriteHeader(http.StatusCreated) + return + } + body, _ := io.ReadAll(r.Body) + assert.NotContains(t, string(body), "mime_type", "older Gitea rejects the field") + assert.True(t, strings.HasPrefix(r.Header.Get("Authorization"), "Bearer ey"), "the runtime token is the bearer") + switch calls.Add(1) { + case 1: + w.Header().Set("Retry-After", "0") + w.WriteHeader(http.StatusTooManyRequests) + case 2: + http.Error(w, "Error create or get artifact", http.StatusInternalServerError) + case 3: + w.WriteHeader(http.StatusFound) + _, _ = io.WriteString(w, `{"ok":true,"signedUploadUrl":"http://blob/u?sig=x"}`) + default: + w.WriteHeader(http.StatusBadRequest) + _, _ = io.WriteString(w, `{"code":"invalid_argument","msg":"bad name"}`) + } + })) + defer server.Close() + client, err := New(server.URL, artifacts.LocalRuntimeToken("7", time.Hour), nil) + require.NoError(t, err) + client.Retry = Retry{Attempts: 3} + var logged []string + client.Logf = func(format string, args ...any) { logged = append(logged, fmt.Sprintf(format, args...)) } + + uploadURL, err := client.Create(t.Context(), "art", "", nil) + require.NoError(t, err) + assert.Equal(t, "http://blob/u?sig=x", uploadURL) + assert.Contains(t, logged[1], "Error create or get artifact") + + _, err = client.Create(t.Context(), "art", "", nil) + require.ErrorContains(t, err, "(400) Bad Request: bad name") + assert.Equal(t, int32(4), calls.Load(), "a 400 is not retried") + + for range 2 { + _, err = client.Upload(t.Context(), server.URL+"/u?sig=secret", "application/zip", strings.NewReader("x"), 1, time.Minute) + require.NoError(t, err) + } + assert.NotEqual(t, blockIDs[0], blockIDs[2], "each upload names its blocks anew") + server.Close() + _, err = client.Upload(t.Context(), server.URL+"/u?sig=secret", "application/zip", strings.NewReader("x"), 1, time.Minute) + require.Error(t, err) + assert.NotContains(t, err.Error()+strings.Join(logged, ""), "secret") +} + +func TestTransferStalls(t *testing.T) { + for _, test := range []struct{ name, want string }{ + {"api", "Client.Timeout"}, + {"failed upload", "(400) Bad Request"}, + {"source", "upload progress stalled"}, + {"commit", "upload progress stalled"}, + {"download", "did not respond"}, + } { + t.Run(test.name, func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + _, _ = io.Copy(io.Discard, request.Body) + if test.name == "failed upload" { + writer.WriteHeader(http.StatusBadRequest) + return + } + if test.name == "download" { + _, _ = io.WriteString(writer, "partial") + writer.(http.Flusher).Flush() + } + <-request.Context().Done() + })) + defer server.Close() + client, err := New(server.URL, artifacts.LocalRuntimeToken("7", time.Hour), nil) + require.NoError(t, err) + client.Retry = Retry{Attempts: 1} + ctx, cancel := context.WithTimeout(t.Context(), time.Second) + defer cancel() + switch test.name { + case "api": + assert.Equal(t, 3*time.Minute, client.api.Timeout) + client.api.Timeout = time.Millisecond + _, err = client.Create(ctx, "art", "", nil) + case "download": + var resp *http.Response + resp, err = client.Download(ctx, server.URL, time.Millisecond) + require.NoError(t, err) + defer resp.Body.Close() + _, err = io.ReadAll(resp.Body) + case "commit": + _, err = client.Upload(ctx, server.URL+"?sig=x", "text/plain", strings.NewReader(""), 1, time.Millisecond) + default: + reader, writer := io.Pipe() + defer reader.Close() + defer writer.Close() + done := make(chan error, 1) + stall := time.Millisecond + if test.name == "failed upload" { + stall = time.Minute + go func() { + _, err := writer.Write(make([]byte, BlockSize)) + done <- err + }() + } + _, err = client.Upload(ctx, server.URL+"?sig=x", "text/plain", reader, 2, stall) + if test.name == "failed upload" { + require.NoError(t, <-done) + } + } + require.ErrorContains(t, err, test.want) + }) + } +} diff --git a/internal/pkg/action/uploadartifact/content.go b/internal/pkg/action/uploadartifact/content.go new file mode 100644 index 00000000..d432d15a --- /dev/null +++ b/internal/pkg/action/uploadartifact/content.go @@ -0,0 +1,120 @@ +// Copyright 2026 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package uploadartifact + +import ( + "archive/tar" + "bufio" + "compress/gzip" + "context" + "errors" + "fmt" + "io" + "os" + "path" + "strings" + + "gitea.com/gitea/runner/internal/pkg/action" +) + +const fetchScript = `cd / && if command -v gzip >/dev/null 2>&1; then exec tar -czhf - -- "$@"; fi; exec tar -chf - -- "$@"` + +func readFiles(ctx context.Context, c *action.Context, files []found, visit func(found) error) error { + if c.Host() { + for _, file := range files { + body, err := os.Open(file.source) + if err != nil { + return err + } + info, err := body.Stat() + if err == nil { + file.body, file.mode, file.mtime = body, info.Mode(), info.ModTime() + err = visit(file) + } + body.Close() + if err != nil { + return err + } + } + return nil + } + for len(files) > 0 { + command := []string{"sh", "-c", fetchScript, "sh"} + selected := map[string][]found{} + size, count := 0, 0 + for _, file := range files { + name := "./" + strings.TrimPrefix(file.source, "/") + if size+len(name)+1 >= 32*1024 { + if count == 0 { + return fmt.Errorf("artifact path exceeds container command limit: %s", file.path) + } + break + } + count++ + if _, exists := selected[path.Clean(name)]; !exists { + command = append(command, name) + size += len(name) + 1 + } + selected[path.Clean(name)] = append(selected[path.Clean(name)], file) + } + output, err := c.Container.ExecOutput(ctx, command) + if err != nil { + return err + } + retry, err := readArchive(output, selected, visit) + closeErr := output.Close() + if err != nil { + return err + } else if closeErr != nil { + return closeErr + } + files = append(retry, files[count:]...) + } + return nil +} + +func readArchive(output io.Reader, selected map[string][]found, visit func(found) error) ([]found, error) { + buffered := bufio.NewReader(output) + var content io.Reader = buffered + if magic, _ := buffered.Peek(2); len(magic) == 2 && magic[0] == 0x1f && magic[1] == 0x8b { + compressed, err := gzip.NewReader(buffered) + if err != nil { + return nil, err + } + defer compressed.Close() + compressed.Multistream(false) // bsdtar pads its gzip output with zeros to the block size + content = compressed + } + reader := tar.NewReader(content) + var retry []found + for { + header, err := reader.Next() + if errors.Is(err, io.EOF) { + break + } else if err != nil { + return nil, err + } + name := path.Clean(header.Name) + files := selected[name] + delete(selected, name) + if header.Typeflag == tar.TypeLink { + retry = append(retry, files...) + continue + } + if len(files) == 0 { + continue + } + retry = append(retry, files[1:]...) + file := files[0] + file.body, file.mode, file.mtime = reader, header.FileInfo().Mode(), header.ModTime + if err := visit(file); err != nil { + return nil, err + } + } + _, err := io.Copy(io.Discard, content) + if err == nil { + _, err = io.Copy(io.Discard, buffered) + } + return retry, err +} diff --git a/internal/pkg/action/uploadartifact/mime.go b/internal/pkg/action/uploadartifact/mime.go new file mode 100644 index 00000000..9dc7fa9e --- /dev/null +++ b/internal/pkg/action/uploadartifact/mime.go @@ -0,0 +1,14 @@ +// Copyright 2026 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package uploadartifact + +import ( + "cmp" + "mime" + "path" +) + +func mimeTypeOf(name string) string { + return cmp.Or(mime.TypeByExtension(path.Ext(name)), "application/octet-stream") +} diff --git a/internal/pkg/action/uploadartifact/upload.go b/internal/pkg/action/uploadartifact/upload.go new file mode 100644 index 00000000..6b165b6b --- /dev/null +++ b/internal/pkg/action/uploadartifact/upload.go @@ -0,0 +1,258 @@ +// Copyright 2026 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +// Package uploadartifact works like actions/upload-artifact v7. +package uploadartifact + +import ( + "archive/zip" + "cmp" + "compress/flate" + "context" + "fmt" + "io" + "io/fs" + "net/url" + "strconv" + "strings" + "time" + + "gitea.com/gitea/runner/internal/pkg/action" + "gitea.com/gitea/runner/internal/pkg/action/artifact" + "gitea.com/gitea/runner/internal/pkg/action/glob" +) + +type inputs struct { + name, path, ifNoFilesFound string + retentionDays, compressionLevel int + overwrite, includeHidden, archiveIt bool +} + +func Main(ctx context.Context, c *action.Context) error { + c.ApplyDefaults("name", "artifact", "path", "", "if-no-files-found", "warn", "retention-days", "", "compression-level", "6", + "overwrite", "false", "include-hidden-files", "false", "archive", "true") + in, err := readInputs(c) + if err != nil { + return err + } + g, err := glob.Parse(in.path, glob.Options{ + Workdir: c.Workspace, + Home: func() (string, error) { return c.Home(ctx) }, + Windows: c.Windows(), + ImplicitDescendants: true, + }) + if err != nil { + return err + } + files, err := selectFiles(ctx, c, g, in.includeHidden) + if err != nil { + return err + } + collected := &collected{files: files, root: glob.LeastCommonAncestor(g.SearchPaths(), c.Windows())} + if len(files) == 1 && files[0].path == collected.root { + collected.root = c.Dir(collected.root) + } + if len(collected.files) == 0 { + message := fmt.Sprintf("No files were found with the provided path: %s. No artifacts will be uploaded.", in.path) + switch in.ifNoFilesFound { + case "warn": + c.Warning("%s", message) + case "error": + return action.Failf("%s", message) + default: + c.Infof("%s", message) + } + return nil + } + if !in.archiveIt && len(collected.files) > 1 { + return action.Failf("When 'archive' is set to false, only a single file can be uploaded. Found %d files to upload.", len(collected.files)) + } + return upload(ctx, c, in, collected) +} + +func readInputs(c *action.Context) (*inputs, error) { + in := &inputs{name: c.Input("name"), path: c.Input("path"), ifNoFilesFound: c.Input("if-no-files-found"), compressionLevel: 6} + if in.path == "" { + return nil, action.Failf("Input required and not supplied: path") + } + var err error + if in.overwrite, err = c.BoolInput("overwrite"); err != nil { + return nil, err + } + if in.includeHidden, err = c.BoolInput("include-hidden-files"); err != nil { + return nil, err + } + if in.archiveIt, err = c.BoolInput("archive"); err != nil { + return nil, err + } + if in.ifNoFilesFound != "warn" && in.ifNoFilesFound != "error" && in.ifNoFilesFound != "ignore" { + return nil, action.Failf("Unrecognized if-no-files-found input. Provided: %s. Available options: warn,error,ignore", in.ifNoFilesFound) + } + if days := c.Input("retention-days"); days != "" { + if in.retentionDays, err = action.ParseInt(days); err != nil { + return nil, action.Failf("Invalid retention-days") + } + } + if level := c.Input("compression-level"); level != "" { + if in.compressionLevel, err = action.ParseInt(level); err != nil { + return nil, action.Failf("Invalid compression-level") + } + if in.compressionLevel < 0 || in.compressionLevel > 9 { + return nil, action.Failf("Invalid compression-level. Valid values are 0-9") + } + } + return in, nil +} + +type collected struct { + files []found + root string +} + +func (got *collected) write(ctx context.Context, c *action.Context, in *inputs, output io.Writer) error { + if !in.archiveIt { + return readFiles(ctx, c, got.files, func(file found) error { + _, err := io.Copy(output, file.body) + return err + }) + } + deflater, err := flate.NewWriter(nil, in.compressionLevel) + if err != nil { + return err + } + writer := zip.NewWriter(output) + writer.RegisterCompressor(zip.Deflate, func(output io.Writer) (io.WriteCloser, error) { + deflater.Reset(output) + return deflater, nil + }) + if err := readFiles(ctx, c, got.files, func(file found) error { + header := &zip.FileHeader{ + Name: strings.TrimLeft(strings.ReplaceAll(strings.TrimPrefix(file.path, got.root), `\`, "/"), "/"), + Method: zip.Deflate, Modified: file.mtime.UTC(), + } + if in.compressionLevel == 0 { + header.Method = zip.Store + } + header.SetMode(file.mode & (fs.ModePerm | fs.ModeSetuid | fs.ModeSetgid | fs.ModeSticky)) + entry, err := writer.CreateHeader(header) + if err != nil { + return err + } + _, err = io.Copy(entry, file.body) + return err + }); err != nil { + return err + } + return writer.Close() +} + +const invalidPathChars = "\":<>|*?\r\n" + +func upload(ctx context.Context, c *action.Context, in *inputs, got *collected) error { + client, err := artifact.New(c.ResultsURL, c.RuntimeToken, c.Transport) + if err != nil { + return err + } + client.Logf = c.Infof + if in.overwrite { + if id, err := client.Delete(ctx, in.name); err == nil { + c.Infof("Artifact '%s' (ID: %d) deleted", in.name, id) + } else if !artifact.IsNotFound(err) { + c.Infof("Unable to delete artifact: %v", err) // best effort, as the JavaScript action + } + } + name, fileName, mimeType := in.name, in.name+".zip", "" + if !in.archiveIt { + name, fileName, mimeType = c.Base(got.files[0].path), c.Base(got.files[0].path), mimeTypeOf(got.files[0].path) + } + if name == "" { + return action.Failf("Provided artifact name input during validation is empty") + } + if strings.ContainsAny(name, invalidPathChars+`\/`) { + return action.Failf("The artifact name is not valid: %s", name) + } + if in.archiveIt { + for _, file := range got.files { + if rel := strings.TrimPrefix(file.path, got.root); strings.ContainsAny(rel, invalidPathChars) { + return action.Failf("The path for one of the files in artifact is not valid: %s", rel) + } + } + } + signed, err := client.Create(ctx, name, mimeType, expiresAt(c, in.retentionDays)) + if err != nil { + return err + } + concurrency, err := uploadConcurrency(c) + if err != nil { + return err + } + stall, err := uploadTimeout(c) + if err != nil { + return err + } + c.Infof("Uploading artifact: %s", fileName) + reader, writer := io.Pipe() + defer reader.Close() + streamCtx, cancel := context.WithCancel(ctx) + defer cancel() + done := make(chan struct{}) + go func() { + defer close(done) + writer.CloseWithError(got.write(streamCtx, c, in, writer)) + }() + uploaded, err := client.Upload(ctx, signed, cmp.Or(mimeType, "application/zip"), reader, concurrency, stall) + reader.CloseWithError(err) + cancel() + <-done + if err != nil { + return err + } + id, err := client.Finalize(ctx, name, uploaded.Size, uploaded.Digest) + if err != nil { + return err + } + c.Infof("Artifact %s has been successfully uploaded! Final size is %d bytes. Artifact ID is %d", in.name, uploaded.Size, id) + ghc := c.Github + link := strings.TrimSuffix(ghc.ServerURL, "/") + "/" + ghc.Repository + "/actions/runs/" + ghc.RunID + "/artifacts/" + url.PathEscape(name) + if ghc.RunAttempt != "" { + link += "?attempt=" + url.QueryEscape(ghc.RunAttempt) + } + c.Infof("Artifact download URL: %s", link) + return c.SetOutputs(ctx, "artifact-id", strconv.FormatInt(id, 10), "artifact-digest", strings.TrimPrefix(uploaded.Digest, "sha256:"), "artifact-url", link) +} + +func expiresAt(c *action.Context, days int) *time.Time { + if days == 0 { + return nil + } + if limit, err := action.ParseInt(c.Github.RetentionDays); err == nil && limit != 0 && limit < days { + c.Warning("Retention days cannot be greater than the maximum allowed retention set within the repository. Using %d instead.", limit) + days = limit + } + expires := time.Now().AddDate(0, 0, days) + return &expires +} + +func uploadConcurrency(c *action.Context) (int, error) { + override := c.Env["ACTIONS_ARTIFACT_UPLOAD_CONCURRENCY"] + if override == "" { + return 5, nil + } + concurrency, err := action.ParseInt(override) + if err != nil || concurrency < 1 { + return 0, action.Failf("Invalid value set for ACTIONS_ARTIFACT_UPLOAD_CONCURRENCY env variable") + } + return concurrency, nil +} + +func uploadTimeout(c *action.Context) (time.Duration, error) { + value := c.Env["ACTIONS_ARTIFACT_UPLOAD_TIMEOUT_MS"] + if value == "" { + return 5 * time.Minute, nil + } + ms, err := action.ParseInt(value) + if err != nil { + return 0, action.Failf("Invalid value set for ACTIONS_ARTIFACT_UPLOAD_TIMEOUT_MS env variable") + } + return time.Duration(ms) * time.Millisecond, nil +} diff --git a/internal/pkg/action/uploadartifact/upload_test.go b/internal/pkg/action/uploadartifact/upload_test.go new file mode 100644 index 00000000..3d785836 --- /dev/null +++ b/internal/pkg/action/uploadartifact/upload_test.go @@ -0,0 +1,280 @@ +// Copyright 2026 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package uploadartifact + +import ( + "archive/zip" + "bytes" + "context" + "fmt" + "io" + "maps" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" + "time" + + "gitea.com/gitea/runner/act/artifacts" + "gitea.com/gitea/runner/act/container" + "gitea.com/gitea/runner/internal/pkg/action" + "gitea.com/gitea/runner/internal/pkg/action/artifact" + + "gitea.dev/actionslib/pkg/model" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type localContainer struct { + *container.HostEnvironment + bin string +} + +func (c localContainer) ExecOutput(ctx context.Context, command []string) (io.ReadCloser, error) { + if c.bin != "" { + command = append([]string{"env", "PATH=" + c.bin}, command...) + } + return c.HostEnvironment.ExecOutput(ctx, command) +} + +type harness struct { + t *testing.T + server string + client *artifact.Client + workspace string + env container.ExecutionsEnvironment + logs, annotation []string + retentionDays string + variables map[string]string + output string +} + +func newHarness(t *testing.T, env container.ExecutionsEnvironment) *harness { + t.Helper() + router := http.NewServeMux() + artifacts.RegisterV4(router, t.TempDir()) + server := httptest.NewServer(router) + t.Cleanup(server.Close) + client, err := artifact.New(server.URL, artifacts.LocalRuntimeToken("7", time.Hour), nil) + require.NoError(t, err) + workspace, err := filepath.EvalSymlinks(t.TempDir()) + require.NoError(t, err) + return &harness{t: t, server: server.URL, client: client, workspace: workspace, env: env} +} + +func (h *harness) write(rel, content string, mode os.FileMode) { + h.t.Helper() + p := filepath.Join(h.workspace, filepath.FromSlash(rel)) + require.NoError(h.t, os.MkdirAll(filepath.Dir(p), 0o755)) + require.NoError(h.t, os.WriteFile(p, []byte(content), mode)) +} + +func (h *harness) run(inputs map[string]string) error { + h.logs, h.annotation = nil, nil + h.output = filepath.Join(h.t.TempDir(), "output") + variables := map[string]string{"GITHUB_OUTPUT": h.output} + maps.Copy(variables, h.variables) + return Main(h.t.Context(), &action.Context{ + Container: h.env, + Github: &model.GithubContext{ServerURL: "https://gitea.example", Repository: "owner/repo", RunID: "7", RunAttempt: "2", RetentionDays: h.retentionDays}, + Inputs: inputs, + Workspace: h.workspace, + Env: variables, + ResultsURL: h.server, + RuntimeToken: artifacts.LocalRuntimeToken("7", time.Hour), + Infof: func(format string, args ...any) { h.logs = append(h.logs, fmt.Sprintf(format, args...)) }, + Warning: func(format string, args ...any) { + h.annotation = append(h.annotation, "warning: "+fmt.Sprintf(format, args...)) + }, + }) +} + +func (h *harness) fetch(name string) ([]byte, map[string]string) { + h.t.Helper() + signed, err := h.client.SignedURL(h.t.Context(), name) + require.NoError(h.t, err) + resp, err := h.client.Download(h.t.Context(), signed, time.Minute) + require.NoError(h.t, err) + defer resp.Body.Close() + content, err := io.ReadAll(resp.Body) + require.NoError(h.t, err) + reader, err := zip.NewReader(bytes.NewReader(content), int64(len(content))) + if err != nil { + return content, nil + } + entries := map[string]string{} + for _, f := range reader.File { + body, err := f.Open() + require.NoError(h.t, err) + data, err := io.ReadAll(body) + require.NoError(h.t, err) + require.NoError(h.t, body.Close()) + entries[f.Name] = string(data) + } + return content, entries +} + +func TestUploadSelectsWhatTheToolkitSelects(t *testing.T) { + envs := map[string]container.ExecutionsEnvironment{"host": &container.HostEnvironment{}} + if runtime.GOOS != "windows" { + envs["container gzip"] = localContainer{HostEnvironment: &container.HostEnvironment{}} + bin := t.TempDir() + for _, name := range []string{"sh", "find", "readlink", "tar"} { + binary, err := exec.LookPath(name) + require.NoError(t, err) + require.NoError(t, os.Symlink(binary, filepath.Join(bin, name))) + } + envs["container plain"] = localContainer{HostEnvironment: &container.HostEnvironment{}, bin: bin} + } + for envName, env := range envs { + t.Run(envName, func(t *testing.T) { + h := newHarness(t, env) + h.write("dist/a.txt", "a", 0o644) + h.write("dist/run.sh", "#!/bin/sh\n", 0o755) + h.write("dist/sub/b.txt", "b", 0o644) + h.write("dist/empty", "", 0o644) + require.NoError(t, os.Mkdir(filepath.Join(h.workspace, "dist", "emptydir"), 0o755)) + h.write("dist/.hidden", "h", 0o644) + h.write("dist/.git/config", "c", 0o644) + h.write("logs/x.log", "x", 0o644) + h.write(".store/target.txt", "hidden target", 0o644) + if runtime.GOOS != "windows" { + require.NoError(t, os.Symlink("a.txt", filepath.Join(h.workspace, "dist", "link.txt"))) + require.NoError(t, os.Symlink("sub", filepath.Join(h.workspace, "dist", "linkdir"))) + require.NoError(t, os.Symlink("../.store", filepath.Join(h.workspace, "dist", "alias"))) + require.NoError(t, os.Symlink("missing", filepath.Join(h.workspace, "dist", "broken"))) + require.NoError(t, os.Symlink("..", filepath.Join(h.workspace, "dist", "loop"))) + require.NoError(t, os.Link(filepath.Join(h.workspace, "dist", "a.txt"), filepath.Join(h.workspace, "dist", "hard.txt"))) + } + + for _, test := range []struct { + name string + inputs map[string]string + expected map[string]string + links map[string]string + }{ + { + name: "a directory", + inputs: map[string]string{"path": "dist"}, + expected: map[string]string{"a.txt": "a", "run.sh": "#!/bin/sh\n", "sub/b.txt": "b", "empty": ""}, + links: map[string]string{"link.txt": "a", "hard.txt": "a", "linkdir/b.txt": "b", "loop/logs/x.log": "x", "alias/target.txt": "hidden target"}, + }, + { + name: "hidden files included", + inputs: map[string]string{"path": "dist/", "include-hidden-files": "true"}, + expected: map[string]string{"a.txt": "a", "run.sh": "#!/bin/sh\n", "sub/b.txt": "b", ".hidden": "h", ".git/config": "c", "empty": ""}, + links: map[string]string{"link.txt": "a", "hard.txt": "a", "linkdir/b.txt": "b", "loop/logs/x.log": "x", "loop/.store/target.txt": "hidden target", "alias/target.txt": "hidden target"}, + }, + { + name: "only exclusions and comments", + inputs: map[string]string{"path": "!dist\n# dist"}, + expected: map[string]string{}, + }, + { + name: "several patterns share their common ancestor as root", + inputs: map[string]string{"path": "dist/**/*.txt\n!dist/sub/**\n# a comment\nlogs/x.log"}, + expected: map[string]string{"dist/a.txt": "a", "logs/x.log": "x"}, + links: map[string]string{"dist/link.txt": "a", "dist/hard.txt": "a", "dist/linkdir/b.txt": "b", "dist/alias/target.txt": "hidden target"}, + }, + { + name: "hidden root skipped", + inputs: map[string]string{"path": ".store"}, + expected: map[string]string{}, + }, + { + name: "a single file keeps no directory", + inputs: map[string]string{"path": h.workspace + "/dist/sub/b.txt"}, + expected: map[string]string{"b.txt": "b"}, + }, + } { + t.Run(test.name, func(t *testing.T) { + inputs := map[string]string{"name": "art", "overwrite": "true"} + maps.Copy(inputs, test.inputs) + require.NoError(t, h.run(inputs)) + expected := test.expected + if runtime.GOOS != "windows" { + maps.Copy(expected, test.links) + } + if len(expected) == 0 { + assert.Contains(t, h.annotation, "warning: No files were found with the provided path: "+test.inputs["path"]+". No artifacts will be uploaded.") + return + } + _, entries := h.fetch("art") + assert.Equal(t, expected, entries) + }) + } + }) + } +} + +func TestUploadOptions(t *testing.T) { + h := newHarness(t, &container.HostEnvironment{}) + h.write("out/run.sh", "#!/bin/sh\n", 0o755) + h.write("out/data.txt", strings.Repeat("data", 1000), 0o644) + h.variables = map[string]string{"ACTIONS_ARTIFACT_UPLOAD_CONCURRENCY": " 2 workers", "ACTIONS_ARTIFACT_UPLOAD_TIMEOUT_MS": " 60000ms"} + h.retentionDays = "5" + require.NoError(t, h.run(map[string]string{"name": "packed", "path": "out", "compression-level": "0", "retention-days": "10 days"})) + assert.Equal(t, []string{"warning: Retention days cannot be greater than the maximum allowed retention set within the repository. Using 5 instead."}, h.annotation) + + content, _ := h.fetch("packed") + reader, err := zip.NewReader(bytes.NewReader(content), int64(len(content))) + require.NoError(t, err) + for _, f := range reader.File { + assert.Equal(t, zip.Store, f.Method, f.Name) + if runtime.GOOS != "windows" && f.Name == "run.sh" { + assert.Equal(t, os.FileMode(0o755), f.Mode().Perm()) + } + } + + list, err := h.client.List(t.Context(), "") + require.NoError(t, err) + require.Len(t, list, 1) + outputs, err := os.ReadFile(h.output) + require.NoError(t, err) + assert.Equal(t, fmt.Sprintf("artifact-id=%d\nartifact-digest=%s\nartifact-url=https://gitea.example/owner/repo/actions/runs/7/artifacts/packed?attempt=2\n", list[0].ID, strings.TrimPrefix(list[0].Digest, "sha256:")), string(outputs)) + + h.write("report.html", "", 0o644) + + require.NoError(t, h.run(map[string]string{"name": "packed", "path": "report.html", "archive": "false", "overwrite": "true"})) + list, err = h.client.List(t.Context(), "packed") + require.NoError(t, err) + assert.Empty(t, list) + content, entries := h.fetch("report.html") + assert.Nil(t, entries) + assert.Equal(t, "", string(content)) + assert.Contains(t, h.logs, "Uploading artifact: report.html") + assert.Contains(t, strings.Join(h.logs, "\n"), "Artifact packed has been successfully uploaded!") + require.ErrorContains(t, h.run(map[string]string{"path": "out", "archive": "false"}), "When 'archive' is set to false, only a single file can be uploaded. Found 2 files to upload.") + + require.NoError(t, h.run(map[string]string{"path": "nothing/here"})) + assert.Equal(t, []string{"warning: No files were found with the provided path: nothing/here. No artifacts will be uploaded."}, h.annotation) + require.NoError(t, h.run(map[string]string{"path": "nothing", "if-no-files-found": "ignore"})) + assert.Contains(t, h.logs, "No files were found with the provided path: nothing. No artifacts will be uploaded.") + require.ErrorContains(t, h.run(map[string]string{"path": "nothing", "if-no-files-found": "error"}), "No files were found") + + for _, test := range []struct { + inputs map[string]string + want string + }{ + {map[string]string{"name": "a/b", "path": "out/data.txt"}, "The artifact name is not valid: a/b"}, + {map[string]string{}, "Input required and not supplied: path"}, + {map[string]string{"name": "", "path": "out/data.txt"}, "Provided artifact name input during validation is empty"}, + {map[string]string{"path": "out/data.txt", "overwrite": ""}, "Core Schema"}, + {map[string]string{"path": "out/data.txt", "compression-level": "10"}, "Invalid compression-level. Valid values are 0-9"}, + {map[string]string{"path": "out/data.txt", "compression-level": "+-1"}, "Invalid compression-level"}, + {map[string]string{"path": "out/data.txt", "if-no-files-found": "fail"}, "Unrecognized if-no-files-found input"}, + } { + require.ErrorContains(t, h.run(test.inputs), test.want) + } + require.NoError(t, h.run(map[string]string{"path": "out/data.txt", "retention": "1", "overwrite": "true"})) + assert.Equal(t, []string{"warning: Unexpected input(s) 'retention', valid inputs are ['name', 'path', 'if-no-files-found', 'retention-days', 'compression-level', 'overwrite', 'include-hidden-files', 'archive']"}, h.annotation) + if runtime.GOOS != "windows" { + h.write("bad/a:b.txt", "x", 0o644) + require.ErrorContains(t, h.run(map[string]string{"path": "bad"}), "The path for one of the files in artifact is not valid: /a:b.txt") + } +} diff --git a/internal/pkg/action/uploadartifact/walk.go b/internal/pkg/action/uploadartifact/walk.go new file mode 100644 index 00000000..ebecddad --- /dev/null +++ b/internal/pkg/action/uploadartifact/walk.go @@ -0,0 +1,178 @@ +// Copyright 2026 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package uploadartifact + +import ( + "context" + "errors" + "fmt" + "io" + "io/fs" + "os" + "path" + "path/filepath" + "slices" + "strings" + "time" + + "gitea.com/gitea/runner/act/container" + "gitea.com/gitea/runner/internal/pkg/action" + "gitea.com/gitea/runner/internal/pkg/action/glob" +) + +type found struct { + path, source string + mode fs.FileMode + mtime time.Time + body io.Reader +} + +type treeEntry struct { + name, target string + dir bool +} + +type tree struct { + real string + entries map[string][]treeEntry +} + +const listScript = `test -e "$1" || exit 1 +[ -d "$1" ] || { printf 'f\0'; exit 0; } +cd "$1" || exit 2 +shift +pwd -P; printf '\0' +find . "$@" -type d ! -name . -print0; printf '\0' +find . "$@" -type l -exec sh -c 'for l; do printf "%s\0" "$l"; readlink "$l"; printf "\0"; done' sh {} +; printf '\0' +find . "$@" ! -type d ! -type l -print0` + +func list(ctx context.Context, c *action.Context, source string, includeHidden bool) (*tree, error) { + if c.Host() { + info, err := os.Stat(source) + if err != nil { + return nil, err + } + resolved, err := filepath.EvalSymlinks(source) + if err != nil || !info.IsDir() { + return &tree{real: resolved}, err + } + entries, err := os.ReadDir(resolved) + result := &tree{real: resolved, entries: map[string][]treeEntry{".": nil}} + for _, entry := range entries { + result.entries["."] = append(result.entries["."], treeEntry{name: entry.Name(), dir: entry.IsDir()}) + } + return result, err + } + command := []string{"sh", "-c", listScript, "sh", source} + if !includeHidden { + command = append(command, "!", "-name", ".", "-name", ".*", "-prune", "-o") + } + output, err := c.Container.ExecOutput(ctx, command) + if err != nil { + return nil, err + } + defer output.Close() + data, err := io.ReadAll(output) + if err != nil { + return nil, err + } + if err := output.Close(); err != nil { + return nil, err + } + if string(data) == "f\x00" { + return &tree{real: source}, nil + } + fields := strings.Split(string(data), "\x00") + if len(fields) < 4 { + return nil, fmt.Errorf("invalid directory listing for %s", source) + } + result := &tree{real: strings.TrimSuffix(fields[0], "\n"), entries: map[string][]treeEntry{".": nil}} + section := 0 + for index := 1; index < len(fields)-1; index++ { + name := fields[index] + if name == "" { + section++ + continue + } + entry := treeEntry{name: path.Base(name), dir: section == 0} + switch section { + case 0: + result.entries[path.Clean(name)] = nil + case 1: + index++ + if index >= len(fields)-1 { + return nil, fmt.Errorf("invalid symlink listing for %s", source) + } + entry.target = strings.TrimSuffix(fields[index], "\n") + } + result.entries[path.Dir(name)] = append(result.entries[path.Dir(name)], entry) + } + for _, entries := range result.entries { + slices.SortFunc(entries, func(left, right treeEntry) int { return strings.Compare(left.name, right.name) }) + } + return result, nil +} + +func selectFiles(ctx context.Context, c *action.Context, globber *glob.Globber, includeHidden bool) ([]found, error) { + var files []found + join := path.Join + if c.Windows() { + join = filepath.Join + } + var walk func(string, string, []string, *tree, string) error + walk = func(virtual, source string, chain []string, listed *tree, relative string) error { + match := globber.Match(virtual) + if match == glob.MatchNone && !globber.PartialMatch(virtual) || !includeHidden && strings.HasPrefix(c.Base(virtual), ".") { + return nil + } + if listed == nil { + var err error + listed, err = list(ctx, c, source, includeHidden) + var exit container.ExitCodeError + if errors.Is(err, fs.ErrNotExist) || errors.As(err, &exit) && exit == 1 { + return nil + } else if err != nil { + return err + } + relative = "." + } + if listed.entries == nil { + if match&glob.MatchFile != 0 { + files = append(files, found{path: virtual, source: listed.real}) + } + return nil + } + resolved := join(listed.real, relative) + if slices.Contains(chain, resolved) { + return nil + } + chain = append(slices.Clip(chain), resolved) + for _, entry := range listed.entries[relative] { + child := join(virtual, entry.name) + source := join(resolved, entry.name) + var subtree *tree + switch { + case entry.target != "": + source = entry.target + if !path.IsAbs(source) { + source = path.Join(resolved, source) + } + case entry.dir && !c.Host(): + subtree = listed + case !entry.dir && !c.Host(): + subtree = &tree{real: source} + } + if err := walk(child, source, chain, subtree, path.Join(relative, entry.name)); err != nil { + return err + } + } + return nil + } + for _, root := range globber.SearchPaths() { + if err := walk(root, root, nil, nil, "."); err != nil { + return nil, err + } + } + return files, nil +}