enhance: add builtin:upload-artifact (#1343)

Co-authored-by: bircni <[email protected]>

act/artifacts/server.go

@@ -218,7 +218,7 @@ func Serve(ctx context.Context, artifactPath, addr, port string) context.CancelF
 	logger.Debugf("Artifacts base path '%s'", artifactPath)
 	uploads(router, artifactPath)
 	downloads(router, artifactPath)
-	registerV4(router, artifactPath)
+	RegisterV4(router, artifactPath)
 
 	server := &http.Server{
 		Addr:              net.JoinHostPort(addr, port),

act/artifacts/server_test.go

@@ -28,7 +28,7 @@ func TestArtifactFlow(t *testing.T) {
 	router := http.NewServeMux()
 	uploads(router, artifactPath)
 	downloads(router, artifactPath)
-	registerV4(router, artifactPath)
+	RegisterV4(router, artifactPath)
 	server := httptest.NewServer(router)
 	defer server.Close()
 
@@ -256,7 +256,7 @@ func TestArtifactFlow(t *testing.T) {
 		require.Equal(t, "attachment; filename*=UTF-8''50%25%20art%40v1", header.Get("Content-Disposition"))
 
 		rerun := http.NewServeMux()
-		registerV4(rerun, artifactPath)
+		RegisterV4(rerun, artifactPath)
 		recorder := httptest.NewRecorder()
 		rerun.ServeHTTP(recorder, httptest.NewRequest(http.MethodPost, artifactServicePath+"/CreateArtifact", strings.NewReader("{"+artifactRef+"}")))
 		require.Equal(t, http.StatusOK, recorder.Code, recorder.Body.String())

act/artifacts/server_v4.go

@@ -35,7 +35,7 @@ const (
 	maxTwirpBytes = 1 << 20
 )
 
-func registerV4(router *http.ServeMux, baseDir string) {
+func RegisterV4(router *http.ServeMux, baseDir string) {
 	server := &serverV4{dir: filepath.Join(baseDir, "v4"), lastID: time.Now().UnixMilli(), created: map[string]bool{}} // IDs stay unique across invocations sharing baseDir
 	router.HandleFunc("POST "+artifactServicePath+"/CreateArtifact", twirp(server.createArtifact))
 	router.HandleFunc("POST "+artifactServicePath+"/FinalizeArtifact", twirp(server.finalizeArtifact))

act/container/container_types.go

@@ -5,6 +5,7 @@
 package container
 
 import (
+	"bytes"
 	"context"
 	"errors"
 	"fmt"
@@ -30,6 +31,38 @@ func (e ExitCodeError) Error() string {
 	return fmt.Sprintf("Process completed with exit code %d.", int(e))
 }
 
+type commandOutput struct {
+	*io.PipeReader
+	cancel context.CancelFunc
+	done   chan struct{}
+	err    error
+}
+
+func (output *commandOutput) Close() error {
+	_, _ = io.Copy(io.Discard, output.PipeReader)
+	output.PipeReader.Close()
+	output.cancel()
+	<-output.done
+	return output.err
+}
+
+func streamOutput(ctx context.Context, run func(context.Context, io.Writer, io.Writer) error) io.ReadCloser {
+	ctx, cancel := context.WithCancel(ctx)
+	reader, writer := io.Pipe()
+	output := &commandOutput{PipeReader: reader, cancel: cancel, done: make(chan struct{})}
+	go func() {
+		defer close(output.done)
+		defer cancel()
+		var stderr bytes.Buffer
+		output.err = run(ctx, writer, &stderr)
+		if output.err != nil && strings.TrimSpace(stderr.String()) != "" {
+			output.err = fmt.Errorf("%w: %s", output.err, strings.TrimSpace(stderr.String()))
+		}
+		writer.CloseWithError(output.err)
+	}()
+	return output
+}
+
 // NewContainerInput the input for the New function
 type NewContainerInput struct {
 	Image           string
@@ -130,6 +163,7 @@ type Container interface {
 	Create(capAdd, capDrop []string) common.Executor
 	Copy(destPath string, files ...*FileEntry) common.Executor
 	CopyDir(destPath, srcPath string, useGitIgnore, skipGitDir bool) common.Executor
+	ExecOutput(ctx context.Context, command []string) (io.ReadCloser, error)
 	GetContainerArchive(ctx context.Context, srcPath string) (io.ReadCloser, error)
 	CopyTarStream(ctx context.Context, destPath string, write func(*TarWriter) error) error
 	Inspect(ctx context.Context) (*Info, error)

act/container/docker_run.go

@@ -146,6 +146,47 @@ func (cr *containerReference) CopyDir(destPath, srcPath string, useGitIgnore, sk
 	).IfNot(common.Dryrun)
 }
 
+func (cr *containerReference) ExecOutput(ctx context.Context, command []string) (io.ReadCloser, error) {
+	if common.Dryrun(ctx) {
+		return nil, errors.New("DRYRUN is not supported in ExecOutput")
+	}
+	if err := cr.connect()(ctx); err != nil {
+		return nil, err
+	}
+	if err := cr.find()(ctx); err != nil {
+		return nil, err
+	}
+	if cr.id == "" {
+		return nil, cr.missingContainerError("exec %v", command)
+	}
+	return streamOutput(ctx, func(ctx context.Context, stdout, stderr io.Writer) error {
+		created, err := cr.cli.ExecCreate(ctx, cr.id, client.ExecCreateOptions{
+			Cmd: command, AttachStdout: true, AttachStderr: true,
+		})
+		if err != nil {
+			return err
+		}
+		attached, err := cr.cli.ExecAttach(ctx, created.ID, client.ExecAttachOptions{})
+		if err != nil {
+			return err
+		}
+		defer attached.Close()
+		stop := context.AfterFunc(ctx, func() { attached.Close() })
+		defer stop()
+		if _, err := stdcopy.StdCopy(stdout, stderr, attached.Reader); err != nil {
+			return err
+		}
+		inspected, err := cr.cli.ExecInspect(ctx, created.ID, client.ExecInspectOptions{})
+		if err != nil {
+			return err
+		}
+		if inspected.ExitCode != 0 {
+			return ExitCodeError(inspected.ExitCode)
+		}
+		return nil
+	}), nil
+}
+
 func (cr *containerReference) GetContainerArchive(ctx context.Context, srcPath string) (io.ReadCloser, error) {
 	if common.Dryrun(ctx) {
 		return nil, errors.New("DRYRUN is not supported in GetContainerArchive")

act/container/docker_run_test.go

@@ -260,6 +260,28 @@ func TestDockerExecFailure(t *testing.T) {
 	assert.Equal(t, ExitCodeError(1), exitErr)
 	assert.Equal(t, "Process completed with exit code 1.", err.Error())
 
+	client.On("ContainerInspect", ctx, "123", mobyclient.ContainerInspectOptions{}).
+		Return(mobyclient.ContainerInspectResult{}, nil)
+	client.On("ExecCreate", mock.Anything, "123", mobyclient.ExecCreateOptions{
+		Cmd: []string{"printf", `a\b`}, AttachStdout: true, AttachStderr: true,
+	}).Return(mobyclient.ExecCreateResult{ID: "output"}, nil)
+	client.On("ExecAttach", mock.Anything, "output", mobyclient.ExecAttachOptions{}).
+		Return(mobyclient.ExecAttachResult{
+			Conn:   &mockConn{},
+			Reader: bufio.NewReader(bytes.NewReader(append(stdcopyFrame(stdcopy.Stdout, "a\x00b"), stdcopyFrame(stdcopy.Stderr, " failed \n")...))),
+		}, nil)
+	client.On("ExecInspect", mock.Anything, "output", mobyclient.ExecInspectOptions{}).
+		Return(mobyclient.ExecInspectResult{ExitCode: 3}, nil)
+	output, err := cr.ExecOutput(ctx, []string{"printf", `a\b`})
+	require.NoError(t, err)
+	content, err := io.ReadAll(output)
+	require.ErrorAs(t, err, &exitErr)
+	assert.Equal(t, ExitCodeError(3), exitErr)
+	assert.Equal(t, "Process completed with exit code 3.: failed", err.Error())
+	assert.Equal(t, "a\x00b", string(content))
+	require.ErrorAs(t, output.Close(), &exitErr)
+	assert.Equal(t, ExitCodeError(3), exitErr)
+
 	client.AssertExpectations(t)
 }
 
@@ -533,6 +555,8 @@ func TestRejectsMissingContainer(t *testing.T) {
 	check("exec", cr.exec([]string{"echo"}, nil, "", "")(ctx))
 	_, err := cr.GetContainerArchive(ctx, "/var/run/act/x")
 	check("GetContainerArchive", err)
+	_, err = cr.ExecOutput(ctx, []string{"true"})
+	check("ExecOutput", err)
 	_, err = cr.Inspect(ctx)
 	check("Inspect", err)
 

act/container/host_environment.go

@@ -116,6 +116,20 @@ func (e *HostEnvironment) Inspect(_ context.Context) (*Info, error) {
 	return &Info{Health: HealthNone, Ports: map[string]string{}}, nil
 }
 
+func (e *HostEnvironment) ExecOutput(ctx context.Context, command []string) (io.ReadCloser, error) {
+	return streamOutput(ctx, func(ctx context.Context, stdout, stderr io.Writer) error {
+		cmd := exec.CommandContext(ctx, command[0], command[1:]...)
+		cmd.Dir, cmd.Stdout, cmd.Stderr = e.Workdir, stdout, stderr
+		if err := cmd.Run(); err != nil {
+			if exit, ok := errors.AsType[*exec.ExitError](err); ok {
+				return ExitCodeError(exit.ExitCode())
+			}
+			return err
+		}
+		return nil
+	}), nil
+}
+
 func (e *HostEnvironment) GetContainerArchive(ctx context.Context, srcPath string) (io.ReadCloser, error) {
 	srcPath = filepath.Clean(srcPath)
 	fi, err := os.Lstat(srcPath)

act/container/host_environment_test.go

@@ -132,6 +132,21 @@ func TestHostEnvironmentExecExitCode(t *testing.T) {
 	if runtime.GOOS == "windows" {
 		t.Skip("uses POSIX shell")
 	}
+	for _, read := range []bool{true, false} {
+		output, err := (&HostEnvironment{AllocatePTY: true}).ExecOutput(t.Context(), []string{"sh", "-c", `test ! -t 1; printf 'out\000'; printf ' failed \n' >&2; exit 3`})
+		require.NoError(t, err)
+		if read {
+			content, err := io.ReadAll(output)
+			var exit ExitCodeError
+			require.ErrorAs(t, err, &exit)
+			assert.Equal(t, ExitCodeError(3), exit)
+			assert.Equal(t, "out\x00", string(content))
+			assert.Equal(t, "Process completed with exit code 3.: failed", err.Error())
+		}
+		var exit ExitCodeError
+		require.ErrorAs(t, output.Close(), &exit)
+		assert.Equal(t, ExitCodeError(3), exit)
+	}
 	for name, allocatePTY := range map[string]bool{"without PTY": false, "with PTY": true} {
 		t.Run(name, func(t *testing.T) {
 			t.Parallel()

act/container/kubernetes.go

@@ -565,17 +565,17 @@ func (c *kubernetesContainer) CopyDir(destPath, srcPath string, useGitIgnore, sk
 	}
 }
 
-// GetContainerArchive streams the archive, checking the path first as tar writes an empty archive for a missing one.
+func (c *kubernetesContainer) ExecOutput(ctx context.Context, command []string) (io.ReadCloser, error) {
+	return streamOutput(ctx, func(ctx context.Context, stdout, stderr io.Writer) error {
+		return c.pod.client.exec(ctx, c.pod.name, c.name, command, nil, stdout, stderr)
+	}), nil
+}
+
 func (c *kubernetesContainer) GetContainerArchive(ctx context.Context, srcPath string) (io.ReadCloser, error) {
-	reader, writer := io.Pipe()
-	go func() {
-		var stderr bytes.Buffer
-		err := c.pod.client.exec(ctx, c.pod.name, c.name, []string{"sh", "-c", `test -e "$1" -o -L "$1" && exec tar -cf - -C "$2" "$3"`, "sh", srcPath, path.Dir(srcPath), path.Base(srcPath)}, nil, writer, &stderr)
-		if err != nil {
-			err = fmt.Errorf("archive %s: %w: %s", srcPath, err, strings.TrimSpace(stderr.String()))
-		}
-		writer.CloseWithError(err)
-	}()
+	reader, err := c.ExecOutput(ctx, []string{"sh", "-c", `test -e "$1" -o -L "$1" && exec tar -cf - -C "$2" "$3"`, "sh", srcPath, path.Dir(srcPath), path.Base(srcPath)})
+	if err != nil {
+		return nil, err
+	}
 	return startedArchive(reader)
 }
 

act/container/kubernetes_test.go

@@ -273,6 +273,17 @@ func TestKubernetesPodRunsJobThroughAPI(t *testing.T) {
 	assert.Equal(t, "hunter2\nit's\nme\n5\n"+workdir+"\n", jobLog.String())
 	assert.Equal(t, ExitCodeError(3), job.Exec([]string{"sh", "-c", "echo failing >&2; exit 3"}, nil, "", "")(ctx))
 	assert.Contains(t, jobLog.String(), "failing\n")
+	output, err := job.ExecOutput(ctx, []string{"sh", "-c", `printf 'out\000'; printf ' failed \n' >&2; exit 3`})
+	require.NoError(t, err)
+	content, err := io.ReadAll(output)
+	var exit ExitCodeError
+	require.ErrorAs(t, err, &exit)
+	assert.Equal(t, ExitCodeError(3), exit)
+	assert.Contains(t, err.Error(), ": failed")
+	assert.Equal(t, "out\x00", string(content))
+	require.ErrorAs(t, output.Close(), &exit)
+	assert.Equal(t, ExitCodeError(3), exit)
+
 	for _, query := range cluster.requests["exec"] {
 		assert.NotContains(t, query, "hunter2")
 	}
@@ -285,7 +296,7 @@ func TestKubernetesPodRunsJobThroughAPI(t *testing.T) {
 	reader := tar.NewReader(archive)
 	header, err := reader.Next()
 	require.NoError(t, err)
-	content, err := io.ReadAll(reader)
+	content, err = io.ReadAll(reader)
 	require.NoError(t, err)
 	assert.Equal(t, "event.json", header.Name)
 	assert.JSONEq(t, `{"ref":"main"}`, string(content))

act/container/tar.go

@@ -86,7 +86,7 @@ func pipeTar(uid, gid int, prefix string, parents []string, write func(*TarWrite
 }
 
 // startedArchive returns reader once its first byte arrives, so failing to start archiving errors here, not on read.
-func startedArchive(reader *io.PipeReader) (io.ReadCloser, error) {
+func startedArchive(reader io.ReadCloser) (io.ReadCloser, error) {
 	buffered := bufio.NewReader(reader)
 	if _, err := buffered.Peek(1); err != nil {
 		reader.Close()

act/runner/action_composite_test.go

@@ -11,6 +11,8 @@ import (
 	"gitea.com/gitea/runner/act/common/git"
 	"gitea.com/gitea/runner/act/common/logging"
 	"gitea.com/gitea/runner/act/common/logging/logtest"
+	"gitea.com/gitea/runner/act/container"
+	"gitea.com/gitea/runner/internal/pkg/action"
 
 	"gitea.dev/actionslib/pkg/model"
 	"github.com/stretchr/testify/assert"
@@ -182,3 +184,21 @@ func newTestCompositeRunContext(t *testing.T, parent *RunContext) *RunContext {
 	require.NoError(t, err)
 	return composite
 }
+
+func TestBuiltinActionGetsLowercaseInputsAndTheJobTokenInComposites(t *testing.T) {
+	job := makeTestRC(t, "runs-on: ubuntu-latest")
+	job.Config.Secrets = map[string]string{"ACTIONS_RUNTIME_TOKEN": "daemon-token"}
+	job.JobContainer = &container.HostEnvironment{Path: t.TempDir(), ActPath: t.TempDir(), Workdir: t.TempDir()}
+	composite := newTestCompositeRunContext(t, job)
+	composite.JobContainer = job.JobContainer
+	for _, rc := range []*RunContext{job, composite} {
+		var got *action.Context
+		sab := &stepActionBuiltin{Step: &model.Step{ID: "x", With: map[string]string{"Name": "x"}}, RunContext: rc, run: func(_ctx context.Context, c *action.Context) error {
+			got = c
+			return nil
+		}}
+		require.NoError(t, sab.main()(t.Context()))
+		assert.Equal(t, "daemon-token", got.RuntimeToken)
+		assert.Equal(t, map[string]string{"name": "x"}, got.Inputs)
+	}
+}

act/runner/container_mock_test.go

@@ -66,6 +66,12 @@ func (cm *containerMock) Exec(command []string, env map[string]string, user, wor
 	return cm.Called(command, env, user, workdir).Get(0).(func(context.Context) error)
 }
 
+func (cm *containerMock) ExecOutput(ctx context.Context, command []string) (io.ReadCloser, error) {
+	args := cm.Called(ctx, command)
+	output, _ := args.Get(0).(io.ReadCloser)
+	return output, args.Error(1)
+}
+
 func (cm *containerMock) GetContainerArchive(ctx context.Context, srcPath string) (io.ReadCloser, error) {
 	args := cm.Called(ctx, srcPath)
 	err, _ := args.Get(1).(error)

act/runner/job_hooks_test.go

@@ -71,6 +71,11 @@ func (c *hookContainer) UpdateFromEnv(_ string, env *map[string]string) common.E
 	}
 }
 
+func (c *hookContainer) ExecOutput(_ctx context.Context, command []string) (io.ReadCloser, error) {
+	c.cmd = command
+	return io.NopCloser(bytes.NewBufferString(c.output)), c.err
+}
+
 func (c *hookContainer) GetContainerArchive(context.Context, string) (io.ReadCloser, error) {
 	return io.NopCloser(bytes.NewReader(c.pathTar)), nil
 }

act/runner/runner.go

@@ -81,6 +81,7 @@ type Config struct {
 	PlatformPicker                    func(labels []string) string
 	KubernetesPicker                  func(labels []string) container.KubernetesOptions
 	InstanceTransport                 http.RoundTripper
+	ResultsURL                        string         // the results service as the runner reaches it, never the job's ACTIONS_RESULTS_URL, which a step can rewrite
 	JobLoggerLevel                    *logging.Level // the level of job logger
 	ValidVolumes                      []string       // only volumes (and bind mounts) in this slice can be mounted on the job container or service containers
 	ContainerVolumes                  []string       // mounted into every job, service and docker action container

act/runner/step_action_builtin.go

@@ -4,17 +4,22 @@
 package runner
 
 import (
+	"cmp"
 	"context"
+	"fmt"
+	"strings"
 
 	"gitea.com/gitea/runner/act/common"
 	"gitea.com/gitea/runner/internal/pkg/action"
 	"gitea.com/gitea/runner/internal/pkg/action/checkout"
+	"gitea.com/gitea/runner/internal/pkg/action/uploadartifact"
 
 	"gitea.dev/actionslib/pkg/model"
 )
 
 var builtinActions = map[string]func(context.Context, *action.Context) error{
-	"checkout": checkout.Main,
+	"checkout":        checkout.Main,
+	"upload-artifact": uploadartifact.Main,
 }
 
 type stepActionBuiltin struct {
@@ -34,13 +39,18 @@ func (sab *stepActionBuiltin) main() common.Executor {
 		}
 
 		printRunActionHeader(ctx, sab.Step, nil, sab.env, sab.RunContext)
+		rawLogger := common.Logger(ctx).With(rawOutputField, true)
 
 		rc := sab.RunContext
 		rc.ApplyExtraPath(ctx, &sab.env)
+		inputs := make(map[string]string, len(sab.Step.With))
+		for name, value := range sab.Step.With {
+			inputs[strings.ToLower(name)] = value // input names are case-insensitive on GitHub
+		}
 		return sab.run(ctx, &action.Context{
 			Container:       rc.JobContainer,
 			Github:          getStepGithubContext(ctx, sab),
-			Inputs:          sab.Step.With,
+			Inputs:          inputs,
 			HostWorkdir:     rc.Config.Workdir,
 			Workspace:       rc.JobContainer.ToContainerPath(rc.Config.Workdir),
 			BindWorkdir:     rc.Config.BindWorkdir,
@@ -51,6 +61,13 @@ func (sab *stepActionBuiltin) main() common.Executor {
 			ClientKeyFile:   rc.Config.ClientKeyFile,
 			CAFile:          rc.jobCAFile(),
 			Env:             sab.env,
+			ResultsURL:      rc.Config.ResultsURL,
+			RuntimeToken:    cmp.Or(rc.runtimeToken, rc.topLevelRunContext().Config.Secrets["ACTIONS_RUNTIME_TOKEN"]), // composite contexts have no secrets
+			Transport:       rc.Config.InstanceTransport,
+			Infof:           rawLogger.Infof,
+			Warning: func(format string, args ...any) {
+				rawLogger.Infof("%s", formatAnnotation("warning", nil, fmt.Sprintf(format, args...)))
+			},
 		})
 	})
 }