fix: align runner behaviours (#1338)

Contains a number of small bug fixes where jobs behaved differently from GitHub or failed silently.

1. Hooks fail on a refused `::set-env::` or `::add-path::` and no longer inherit the last step's identity
1. Docker build, exec and wait errors are returned instead of swallowed, token-only registry credentials work, logs of `--tty` services decode
1. Env files split lines like GitHub on each OS, so a carriage return is kept on Linux and CRLF ends a line on Windows, empty heredoc names or delimiters are rejected
1. Post steps run after their pre step even when main was skipped, composites see undeclared inputs, `runs.args` wins without `with.args` being parsed
1. Docker actions skip evaluating defaults of inputs given in `with` and read `entryPoint` and `args` in any case, a lowercase `dockerfile` is found, undeclared inputs get GitHub's warning
1. A user-set `CI` is kept, `GITHUB_REPOSITORY_ID` and the other missing `GITHUB_*` variables are exported
1. `timeout-minutes` can read the step's own `env`, a step-level `ACTIONS_ALLOW_UNSECURE_COMMANDS: false` beats a job-level `true` while the runner's own opt-in still wins, `pwsh` quotes the script path
1. A step `env:` given as one expression such as `${{ fromJSON(matrix.env) }}` is applied, `hashFiles` fails with GitHub's messages on script errors and after 120 seconds, a job image without node still gets an empty hash
1. A step whose command files cannot be reset fails instead of importing stale ones, a timeout between steps fails the job instead of reporting success
1. One named volume can be mounted at several paths, host-mode PTY steps keep their last output when they fail, host scratch directories are removed when job setup fails
1. Cancelling a step right after it starts kills its whole process tree
1. Cache prefix restores pick the newest entry created within the same second, also on S3, cache commits with a wrong size are rejected
1. Artifact chunks are written at their offset, download URLs are escaped, listing a run without artifacts no longer crashes
1. Poll backoff reaches `fetch_interval_max`, proxy passwords are masked in more forms, secrets are also masked in step names and in their quoted form in the runner log, a failed final report is logged
1. On Windows hosts a differently cased unsecure-commands opt-in or proxy variable counts, and an early cancel no longer leaks a job object handle
1. Interactive `register` prompts for a missing token and stops on SIGTERM, `exec --list --detect-event` no longer panics without events, `config set` works on `cache.s3` keys

Reviewed-on: https://gitea.com/gitea/runner/pulls/1338
Reviewed-by: bircni <[email protected]>
Co-authored-by: silverwind <[email protected]>

internal/app/cmd/exec.go

@@ -326,11 +326,10 @@ func runExecList(planner model.WorkflowPlanner, execArgs *executeArgs) error {
 		// collect all events from loaded workflows
 		events := planner.GetEvents()
 
-		// set default event type to first event from many available
-		// this way user dont have to specify the event.
-		logging.Infof("Using first detected workflow event for filtering: %s", events[0])
-
-		filterEventName = events[0]
+		if len(events) > 0 {
+			logging.Infof("Using first detected workflow event for filtering: %s", events[0])
+			filterEventName = events[0]
+		}
 	}
 
 	var err error

internal/app/cmd/exec_test.go

@@ -187,6 +187,9 @@ func TestRunExecListUsesJobEventAndAllPlans(t *testing.T) {
 		require.NoError(t, runExecList(planner, &executeArgs{event: "push"}))
 		require.NoError(t, runExecList(planner, &executeArgs{autodetectEvent: true}))
 		require.NoError(t, runExecList(planner, &executeArgs{}))
+		emptyPlanner, err := model.NewWorkflowPlanner(t.TempDir(), false)
+		require.NoError(t, err)
+		require.NoError(t, runExecList(emptyPlanner, &executeArgs{autodetectEvent: true}))
 	})
 
 	require.Contains(t, out, "Build")

internal/app/cmd/register.go

@@ -9,7 +9,6 @@ import (
 	"errors"
 	"fmt"
 	"os"
-	"os/signal"
 	goruntime "runtime"
 	"strings"
 
@@ -48,25 +47,17 @@ func runRegister(ctx context.Context, regArgs *registerArgs, configFile *string)
 		}
 
 		if regArgs.NoInteractive {
-			if err := registerNoInteractive(ctx, *configFile, regArgs); err != nil {
-				return err
-			}
-		} else {
-			go func() {
-				if err := registerInteractive(ctx, *configFile, regArgs); err != nil {
-					logging.Error(err)
-					os.Exit(1)
-					return
-				}
-				os.Exit(0)
-			}()
-
-			c := make(chan os.Signal, 1)
-			signal.Notify(c, os.Interrupt)
-			<-c
+			return registerNoInteractive(ctx, *configFile, regArgs)
 		}
 
-		return nil
+		result := make(chan error, 1)
+		go func() { result <- registerInteractive(ctx, *configFile, regArgs) }()
+		select {
+		case err := <-result:
+			return err
+		case <-ctx.Done():
+			return nil
+		}
 	}
 }
 
@@ -214,7 +205,7 @@ func initInputs(regArgs *registerArgs) (*registerInputs, error) {
 		token = regArgs.Token
 	default:
 		token = os.Getenv(registerTokenEnvVar)
-		if token == "" {
+		if token == "" && regArgs.NoInteractive {
 			return nil, fmt.Errorf("missing token, token-file argument, or %s environment variable", registerTokenEnvVar)
 		}
 	}

internal/app/cmd/register_test.go

@@ -4,6 +4,8 @@
 package cmd
 
 import (
+	"bytes"
+	"context"
 	"net/http"
 	"net/http/httptest"
 	"os"
@@ -11,11 +13,43 @@ import (
 	"sync/atomic"
 	"testing"
 
+	"gitea.com/gitea/runner/act/common/logging"
 	"gitea.com/gitea/runner/internal/pkg/config"
 
 	"github.com/stretchr/testify/require"
 )
 
+type cancelRegistrationPrompt struct {
+	cancel context.CancelFunc
+}
+
+func (w cancelRegistrationPrompt) Write(data []byte) (int, error) {
+	if bytes.Contains(data, []byte("Enter the Gitea instance URL")) {
+		w.cancel()
+	}
+	return len(data), nil
+}
+
+func TestRunRegisterCancelsWhilePrompting(t *testing.T) {
+	t.Setenv(registerTokenEnvVar, "")
+	t.Chdir(t.TempDir())
+	stdin, input, err := os.Pipe()
+	require.NoError(t, err)
+	previousStdin := os.Stdin
+	os.Stdin = stdin
+	t.Cleanup(func() {
+		_ = input.Close()
+		_ = stdin.Close()
+		os.Stdin = previousStdin
+	})
+	ctx, cancel := context.WithCancel(t.Context())
+	defer cancel()
+	previousOutput := logging.Output()
+	logging.SetOutput(cancelRegistrationPrompt{cancel})
+	t.Cleanup(func() { logging.SetOutput(previousOutput) })
+	require.NoError(t, runRegister(ctx, &registerArgs{}, new(string))(nil, nil))
+}
+
 func TestRegisterNonInteractiveReturnsLabelValidationError(t *testing.T) {
 	err := registerNoInteractive(t.Context(), "", &registerArgs{
 		Labels:       "ubuntu:host,,broken",
@@ -148,6 +182,7 @@ func TestRegisterInputsAssignToNext(t *testing.T) {
 }
 
 func TestInitInputs(t *testing.T) {
+	t.Setenv(registerTokenEnvVar, "")
 	for _, tc := range []struct {
 		name, tokenFile, wantErr string
 		env                      *string
@@ -164,11 +199,12 @@ func TestInitInputs(t *testing.T) {
 				t.Setenv(registerTokenEnvVar, *tc.env)
 			}
 			_, err := initInputs(&registerArgs{
-				InstanceAddr: "http://localhost:3000",
-				TokenFile:    tc.tokenFile,
-				RunnerName:   "runner",
-				Ephemeral:    true,
-				Labels:       " ubuntu:host , ubuntu:docker://node:18 ",
+				NoInteractive: true,
+				InstanceAddr:  "http://localhost:3000",
+				TokenFile:     tc.tokenFile,
+				RunnerName:    "runner",
+				Ephemeral:     true,
+				Labels:        " ubuntu:host , ubuntu:docker://node:18 ",
 			})
 			require.EqualError(t, err, tc.wantErr)
 		})
@@ -178,12 +214,16 @@ func TestInitInputs(t *testing.T) {
 		name, address, token, runnerName, wantToken string
 		file                                        bool
 	}{
+		{name: "interactive token prompt", address: "http://localhost:3000", runnerName: "runner"},
 		{name: "valid token", address: " http://localhost:3000 ", token: "from-plain-arg\n", runnerName: "runner\n", wantToken: "from-plain-arg"},
 		{name: "valid token file", address: "http://localhost:3000", runnerName: "runner", file: true, wantToken: "from-file"},
 		{name: "token from environment variable", address: "http://localhost:3000", runnerName: "runner", wantToken: "from-env"},
 	} {
 		t.Run(tc.name, func(t *testing.T) {
 			t.Setenv(registerTokenEnvVar, "from-env")
+			if tc.wantToken == "" {
+				t.Setenv(registerTokenEnvVar, "")
+			}
 			var tokenFile string
 			if tc.file {
 				tokenFile = filepath.Join(t.TempDir(), "from-file")

internal/app/poll/poller.go

@@ -313,10 +313,14 @@ func (p *Poller) calculateInterval(s *workerState) time.Duration {
 		return base
 	}
 
-	// Capped exponential backoff: base * 2^(n-1), max shift=5 so multiplier <= 32
-	shift := min(n-1, 5)
-	interval := base * time.Duration(int64(1)<<shift)
-	return min(interval, maxInterval)
+	interval := base
+	for range n - 1 {
+		if interval >= maxInterval-interval {
+			return maxInterval
+		}
+		interval *= 2
+	}
+	return interval
 }
 
 // addJitter adds +/- 20% random jitter to the given duration to avoid thundering herd.

internal/app/poll/poller_test.go

@@ -140,6 +140,9 @@ func TestPoller_CalculateInterval(t *testing.T) {
 			assert.Equal(t, tc.wantInterval, p.calculateInterval(s))
 		})
 	}
+
+	cfg.Runner.FetchIntervalMax = 2 * time.Minute
+	assert.Equal(t, 2*time.Minute, p.calculateInterval(&workerState{consecutiveEmpty: 7}))
 }
 
 func atomicMax(target *atomic.Int64, val int64) {

internal/app/run/proxy.go

@@ -111,15 +111,20 @@ func WarnIfDaemonHasNoProxy(ctx context.Context) {
 
 // proxyPasswords returns the passwords embedded in the runner's proxy URLs, to mask before
 // a job echoes its environment.
-func proxyPasswords() []string {
-	cfg := proxyFromEnv()
-	if cfg == nil {
-		return nil
+func proxyPasswords(envs map[string]string) []string {
+	cfg := httpproxy.FromEnvironment()
+	raws := []string{cfg.HTTPProxy, cfg.HTTPSProxy}
+	for name, value := range envs {
+		if strings.EqualFold(name, "http_proxy") || strings.EqualFold(name, "https_proxy") { // Windows reads env names in any case
+			raws = append(raws, value)
+		}
 	}
-
 	var passwords []string
-	for _, raw := range []string{cfg.HTTPProxy, cfg.HTTPSProxy} {
+	for _, raw := range raws {
 		parsed, err := url.Parse(raw)
+		if err != nil || parsed.Scheme == "" || parsed.Host == "" {
+			parsed, err = url.Parse("http://" + raw)
+		}
 		if err != nil {
 			continue
 		}

internal/app/run/proxy_test.go

@@ -169,10 +169,19 @@ func TestProxyPasswords(t *testing.T) {
 	clearProxyEnv(t)
 	t.Setenv("http_proxy", "http://user:hunter2@proxy:3128")
 	t.Setenv("https_proxy", "http://user:s3cret@proxy:3128")
-	assert.Equal(t, []string{"hunter2", "s3cret"}, proxyPasswords())
+	assert.Equal(t, []string{"hunter2", "s3cret"}, proxyPasswords(nil))
+
+	t.Setenv("http_proxy", "user:hunter2@proxy:3128")
+	t.Setenv("https_proxy", "user:s3cret@proxy:3128")
+	assert.Equal(t, []string{"hunter2", "s3cret"}, proxyPasswords(nil))
 
 	t.Setenv("https_proxy", "http://proxy:3128")
-	assert.Equal(t, []string{"hunter2"}, proxyPasswords())
+	assert.Equal(t, []string{"hunter2"}, proxyPasswords(nil))
+
+	clearProxyEnv(t)
+	assert.Equal(t, []string{"configured"}, proxyPasswords(map[string]string{
+		"Http_Proxy": "http://user:configured@proxy:3128",
+	}))
 }
 
 func TestAppendNoProxy(t *testing.T) {

internal/app/run/runner.go

@@ -287,7 +287,7 @@ func (r *Runner) cleanupStaleDirs(ctx context.Context, root string, match func(n
 	}
 }
 
-func (r *Runner) Run(ctx context.Context, task *runnerv1.Task) error {
+func (r *Runner) Run(ctx context.Context, task *runnerv1.Task) (err error) {
 	if _, ok := r.runningTasks.LoadOrStore(task.Id, struct{}{}); ok {
 		return fmt.Errorf("task %d is already running", task.Id)
 	}
@@ -302,7 +302,7 @@ func (r *Runner) Run(ctx context.Context, task *runnerv1.Task) error {
 	defer cancel()
 	ctx, endJob := telemetry.StartJob(ctx, task)
 	// A proxy URL may carry credentials, and every job is given it; keep them out of the log.
-	reporter := report.NewReporter(ctx, cancel, r.client, task, r.cfg, proxyPasswords()...)
+	reporter := report.NewReporter(ctx, cancel, r.client, task, r.cfg, proxyPasswords(r.envs)...)
 	var volumeCleanup []common.Executor
 	var volumeCleanupMu sync.Mutex
 	if r.cfg.Runner.PostTaskScript == "" {
@@ -318,7 +318,7 @@ func (r *Runner) Run(ctx context.Context, task *runnerv1.Task) error {
 		if runErr != nil {
 			lastWords = runErr.Error()
 		}
-		_ = reporter.Close(lastWords)
+		err = reporter.Close(lastWords)
 		if err := cleanupJobVolumes(ctx, volumeCleanup); err != nil {
 			logging.Warnf("task %d volume cleanup after reporting: %v", task.Id, err)
 		}
@@ -584,7 +584,7 @@ func (r *Runner) run(ctx context.Context, task *runnerv1.Task, reporter *report.
 		Env:                  envs,
 		ProxyEnv:             proxyEnv,
 		Secrets:              task.Secrets,
-		ExtraMasks:           append(proxyPasswords(), preset.Token),
+		ExtraMasks:           append(proxyPasswords(r.envs), preset.Token),
 		GitHubInstance:       strings.TrimSuffix(r.client.Address(), "/"),
 		NoSkipCheckout:       true,
 		DisableActEnv:        r.cfg.Runner.SetActEnv != nil && !*r.cfg.Runner.SetActEnv,

internal/app/run/runner_test.go

@@ -6,6 +6,7 @@ package run
 import (
 	"context"
 	"encoding/base64"
+	"errors"
 	"io"
 	"net/http"
 	"net/http/httptest"
@@ -187,6 +188,29 @@ func TestCleanupJobVolumesJoinsErrorsAfterCancellation(t *testing.T) {
 	require.ErrorIs(t, err, io.ErrClosedPipe)
 }
 
+func TestRunnerReturnsMaskedFinalReportingError(t *testing.T) {
+	cfg, err := config.LoadDefault("")
+	require.NoError(t, err)
+	cfg.Cache.Enabled = new(false)
+	cfg.Runner.ReportCloseTimeout = time.Millisecond
+	cli := clientmocks.NewClient(t)
+	cli.AddressValue = "https://gitea.example/"
+	reportErr := errors.New("reporting unavailable for hunter2")
+	cli.On("UpdateLog", mock.Anything, mock.Anything).Return(nil, reportErr)
+	cli.On("UpdateTask", mock.Anything, mock.Anything).Return(nil, reportErr)
+	r := NewRunner(cfg, &config.Registration{}, cli)
+
+	err = r.Run(t.Context(), &runnerv1.Task{
+		Context:         &structpb.Struct{},
+		WorkflowPayload: []byte("["),
+		Secrets:         map[string]string{"PASSWORD": "hunter2"},
+	})
+
+	require.ErrorContains(t, err, "reporting unavailable for ***")
+	assert.NotContains(t, err.Error(), "hunter2")
+	assert.Zero(t, r.RunningCount())
+}
+
 func TestNewRunnerInitializesLabelsAndEnvironment(t *testing.T) {
 	cacheEnabled := false
 	cfg := &config.Config{}

internal/pkg/config/edit.go

@@ -460,6 +460,9 @@ func resolvePath(segments []string) (*fieldInfo, error) {
 					strings.Join(segments[:i+1], "."), strings.Join(yamlNames(typ), ", "))
 			}
 			typ = field.Type
+			if typ.Kind() == reflect.Pointer {
+				typ = typ.Elem()
+			}
 		case reflect.Map:
 			// The segment names a user-defined entry, so the walk ends here.
 			if i != len(segments)-1 {

internal/pkg/config/edit_test.go

@@ -219,6 +219,11 @@ func TestGetValue(t *testing.T) {
 	_, err = GetValue(file, "metrics.addr")
 	require.Error(t, err)
 	assert.Contains(t, err.Error(), "is not set")
+
+	require.NoError(t, SetValue(file, "cache.s3.endpoint", "https://cache.example"))
+	value, err = GetValue(file, "cache.s3.endpoint")
+	require.NoError(t, err)
+	assert.Equal(t, "https://cache.example", value)
 }
 
 func TestEditValuesFileHandling(t *testing.T) {

internal/pkg/process/killer_unix_test.go

@@ -98,3 +98,32 @@ func TestKillerKillsTree(t *testing.T) {
 		return !processAlive(childPID)
 	}, 20*time.Second, 100*time.Millisecond, "backgrounded child should be terminated")
 }
+
+func TestTreeKillCancelBeforeCaptureKillsTree(t *testing.T) {
+	stdin, input, err := os.Pipe()
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = stdin.Close() })
+	t.Cleanup(func() { _ = input.Close() })
+	cmd := exec.CommandContext(t.Context(), "/bin/sh", "-c", "cat <&0 & echo $!; read value")
+	cmd.Stdin = stdin
+	cmd.SysProcAttr = SysProcAttr("", false)
+	output, err := cmd.StdoutPipe()
+	require.NoError(t, err)
+	treeKill := NewTreeKill(cmd)
+	require.NoError(t, cmd.Start())
+	t.Cleanup(func() {
+		_ = syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL)
+		_ = cmd.Wait()
+	})
+	var childPID int
+	_, err = fmt.Fscan(output, &childPID)
+	require.NoError(t, err)
+	require.True(t, processAlive(childPID))
+
+	require.NoError(t, cmd.Cancel())
+	killer, err := treeKill.Capture(cmd.Process)
+	require.NoError(t, err)
+	defer killer.Close()
+	require.Error(t, cmd.Wait())
+	require.Eventually(t, func() bool { return !processAlive(childPID) }, 5*time.Second, 10*time.Millisecond)
+}

internal/pkg/process/treekill.go

@@ -44,6 +44,10 @@ func NewTreeKill(cmd *exec.Cmd) *TreeKill {
 			return k.Kill()
 		}
 		if cmd.Process != nil {
+			if killer, err := NewKiller(cmd.Process); err == nil {
+				defer killer.Close()
+				return killer.Kill()
+			}
 			return cmd.Process.Kill()
 		}
 		return nil

internal/pkg/report/globalmask.go

@@ -4,6 +4,8 @@
 package report
 
 import (
+	"slices"
+	"strconv"
 	"strings"
 	"sync"
 	"sync/atomic"
@@ -22,7 +24,13 @@ var (
 func registerGlobalMasks(r *Reporter) {
 	globalMu.Lock()
 	defer globalMu.Unlock()
-	globalMasks[r] = r.oldnew
+	masks := slices.Clip(r.oldnew)
+	for i := 0; i < len(r.oldnew); i += 2 { // the console handler Go-quotes values with special characters
+		if quoted := strconv.Quote(r.oldnew[i]); quoted[1:len(quoted)-1] != r.oldnew[i] {
+			masks = append(masks, quoted[1:len(quoted)-1], r.oldnew[i+1])
+		}
+	}
+	globalMasks[r] = masks
 	rebuildGlobalReplacer()
 }
 

internal/pkg/report/globalmask_test.go

@@ -9,6 +9,7 @@ import (
 	"testing"
 
 	"gitea.com/gitea/runner/act/common/logging"
+	"gitea.com/gitea/runner/act/runner"
 
 	"github.com/stretchr/testify/assert"
 )
@@ -61,4 +62,10 @@ func TestGlobalMasks(t *testing.T) {
 	writers.Wait()
 	assert.Contains(t, format("saw runtime-secret"), "runtime-secret")
 	assert.Equal(t, "saw *** ***", third.mask("saw runtime-secret another-secret"))
+
+	secret := "pass\a\v\x1b\u00a0word"
+	register(runner.AppendSecretMasker(nil, secret)...)
+	out.Reset()
+	logging.With("token", secret).Info(secret)
+	assert.Equal(t, "level=info msg=\"***\" token=\"***\"\n", out.String())
 }

internal/pkg/report/reporter.go

@@ -279,6 +279,9 @@ func (r *Reporter) enterReportedStep(step *runnerv1.StepState, timestamp time.Ti
 // cleanup is left running unless closing, as it still logs after the job result. Caller holds stateMu.
 func (r *Reporter) finishReportedSteps(jobResult runnerv1.Result, timestamp time.Time, closing bool) {
 	for _, v := range r.state.ReportedSteps {
+		if closing && v.StartedAt != nil && v.StoppedAt == nil {
+			v.StoppedAt = timestamppb.New(timestamp)
+		}
 		if v.Result != runnerv1.Result_RESULT_UNSPECIFIED || !closing && v.StartedAt != nil && v.Stage == runnerv1.StepStage_STEP_STAGE_CLEANUP {
 			continue
 		}
@@ -716,6 +719,7 @@ func (r *Reporter) Close(lastWords string) error {
 	trailer := fmt.Sprintf("task %d finished: %s", r.state.Id, metrics.ResultToStatusLabel(r.state.Result))
 	if err != nil {
 		trailer += fmt.Sprintf(", the final flush to Gitea failed: %v", err)
+		err = errors.New(r.mask(err.Error())) // the caller logs it after this task's masks leave the global masker
 	}
 	r.jobLog.close(r.mask(trailer))
 	r.stateMu.Unlock()

internal/pkg/report/reporter_test.go

@@ -1197,6 +1197,18 @@ func TestReporter_ReportedSteps(t *testing.T) {
 		assert.Equal(t, []runnerv1.Result{failure, skipped, success}, []runnerv1.Result{got[0].result, got[1].result, got[2].result})
 	})
 
+	t.Run("cleanup failure gets a stop timestamp", func(t *testing.T) {
+		reporter, _ := newReporter(t, newTestSteps(1))
+		send(t, reporter, logging.LevelError, "##[error]job completed hook failed", map[string]any{"stage": runner.StageComplete})
+		send(t, reporter, logging.LevelInfo, "Job failed", map[string]any{"stage": runner.StageComplete, "jobResult": model.StepStatusFailure})
+		require.NoError(t, reporter.Close(""))
+
+		step := reporter.state.ReportedSteps[len(reporter.state.ReportedSteps)-1]
+		assert.Equal(t, failure, step.Result)
+		require.NotNil(t, step.StartedAt)
+		assert.NotNil(t, step.StoppedAt)
+	})
+
 	t.Run("step not reached in a succeeding job", func(t *testing.T) {
 		reporter, _ := newReporter(t, newTestSteps(1))
 		send(t, reporter, logging.LevelInfo, "Job succeeded", map[string]any{"stage": runner.StageComplete, "jobResult": model.StepStatusSuccess})