fix: align runner behaviours (#1338)

Contains a number of small bug fixes where jobs behaved differently from GitHub or failed silently.

1. Hooks fail on a refused `::set-env::` or `::add-path::` and no longer inherit the last step's identity
1. Docker build, exec and wait errors are returned instead of swallowed, token-only registry credentials work, logs of `--tty` services decode
1. Env files split lines like GitHub on each OS, so a carriage return is kept on Linux and CRLF ends a line on Windows, empty heredoc names or delimiters are rejected
1. Post steps run after their pre step even when main was skipped, composites see undeclared inputs, `runs.args` wins without `with.args` being parsed
1. Docker actions skip evaluating defaults of inputs given in `with` and read `entryPoint` and `args` in any case, a lowercase `dockerfile` is found, undeclared inputs get GitHub's warning
1. A user-set `CI` is kept, `GITHUB_REPOSITORY_ID` and the other missing `GITHUB_*` variables are exported
1. `timeout-minutes` can read the step's own `env`, a step-level `ACTIONS_ALLOW_UNSECURE_COMMANDS: false` beats a job-level `true` while the runner's own opt-in still wins, `pwsh` quotes the script path
1. A step `env:` given as one expression such as `${{ fromJSON(matrix.env) }}` is applied, `hashFiles` fails with GitHub's messages on script errors and after 120 seconds, a job image without node still gets an empty hash
1. A step whose command files cannot be reset fails instead of importing stale ones, a timeout between steps fails the job instead of reporting success
1. One named volume can be mounted at several paths, host-mode PTY steps keep their last output when they fail, host scratch directories are removed when job setup fails
1. Cancelling a step right after it starts kills its whole process tree
1. Cache prefix restores pick the newest entry created within the same second, also on S3, cache commits with a wrong size are rejected
1. Artifact chunks are written at their offset, download URLs are escaped, listing a run without artifacts no longer crashes
1. Poll backoff reaches `fetch_interval_max`, proxy passwords are masked in more forms, secrets are also masked in step names and in their quoted form in the runner log, a failed final report is logged
1. On Windows hosts a differently cased unsecure-commands opt-in or proxy variable counts, and an early cancel no longer leaks a job object handle
1. Interactive `register` prompts for a missing token and stops on SIGTERM, `exec --list --detect-event` no longer panics without events, `config set` works on `cache.s3` keys

Reviewed-on: https://gitea.com/gitea/runner/pulls/1338
Reviewed-by: bircni <[email protected]>
Co-authored-by: silverwind <[email protected]>

act/artifacts/server.go

@@ -13,6 +13,7 @@ import (
 	"io/fs"
 	"net"
 	"net/http"
+	"net/url"
 	"os"
 	"path/filepath"
 	"strings"
@@ -84,16 +85,25 @@ func uploads(router *http.ServeMux, baseDir string) {
 		if err := os.MkdirAll(filepath.Dir(safePath), os.ModePerm); err != nil {
 			panic(err)
 		}
-		flags := os.O_CREATE | os.O_WRONLY | os.O_TRUNC
-		appendUpload := req.Header.Get("Content-Range")
-		if appendUpload != "" && !strings.HasPrefix(appendUpload, "bytes 0-") {
-			flags = os.O_CREATE | os.O_WRONLY | os.O_APPEND
+		var start int64
+		if contentRange := req.Header.Get("Content-Range"); contentRange != "" {
+			if _, err := fmt.Sscanf(contentRange, "bytes %d-", &start); err != nil || start < 0 {
+				http.Error(w, "invalid content range", http.StatusBadRequest)
+				return
+			}
+		}
+		flags := os.O_CREATE | os.O_WRONLY
+		if start == 0 {
+			flags |= os.O_TRUNC
 		}
 		file, err := os.OpenFile(safePath, flags, 0o644)
 		if err != nil {
 			panic(err)
 		}
 		defer file.Close()
+		if _, err := file.Seek(start, io.SeekStart); err != nil {
+			panic(err)
+		}
 		if req.Body == nil {
 			panic(errors.New("no body given"))
 		}
@@ -120,7 +130,7 @@ func downloads(router *http.ServeMux, baseDir string) {
 		runID := req.PathValue("runId")
 
 		entries, err := os.ReadDir(safeResolve(baseDir, runID))
-		if err != nil {
+		if err != nil && !os.IsNotExist(err) {
 			panic(err)
 		}
 
@@ -145,6 +155,9 @@ func downloads(router *http.ServeMux, baseDir string) {
 
 		var files []ContainerItem
 		err := filepath.WalkDir(safePath, func(path string, entry fs.DirEntry, err error) error {
+			if err != nil {
+				return err
+			}
 			if !entry.IsDir() {
 				rel, err := filepath.Rel(safePath, path)
 				if err != nil {
@@ -157,12 +170,12 @@ func downloads(router *http.ServeMux, baseDir string) {
 				files = append(files, ContainerItem{
 					Path:            filepath.ToSlash(filepath.Join(itemPath, rel)),
 					ItemType:        "file",
-					ContentLocation: fmt.Sprintf("http://%s/artifact/%s/%s/%s", req.Host, container, itemPath, filepath.ToSlash(rel)),
+					ContentLocation: (&url.URL{Scheme: "http", Host: req.Host, Path: "/artifact/" + container + "/" + itemPath + "/" + filepath.ToSlash(rel)}).String(),
 				})
 			}
 			return nil
 		})
-		if err != nil {
+		if err != nil && !os.IsNotExist(err) {
 			panic(err)
 		}
 

act/artifacts/server_test.go

@@ -48,8 +48,17 @@ func TestArtifactFlow(t *testing.T) {
 		return resp.StatusCode, data
 	}
 
+	t.Run("no-artifacts", func(t *testing.T) {
+		status, data := request(t, http.MethodGet, baseURL+"/_apis/pipelines/workflows/0/artifacts", nil, nil)
+		require.Equal(t, http.StatusOK, status, string(data))
+		var list NamedFileContainerResourceURLResponse
+		require.NoError(t, json.Unmarshal(data, &list))
+		require.Zero(t, list.Count)
+		require.Empty(t, list.Value)
+	})
+
 	t.Run("upload-and-download", func(t *testing.T) {
-		const runID, item, content = "1", "my-artifact/data.txt", "hello artifact\n"
+		const runID, item, content = "1", "my-artifact/data#1%25.txt", "hello artifact\n"
 
 		status, data := request(t, http.MethodPost, baseURL+"/_apis/pipelines/workflows/"+runID+"/artifacts", nil, nil)
 		require.Equal(t, http.StatusOK, status, string(data))
@@ -81,13 +90,13 @@ func TestArtifactFlow(t *testing.T) {
 		require.NoError(t, json.Unmarshal(data, &items))
 		require.Len(t, items.Value, 1)
 		require.Equal(t, "file", items.Value[0].ItemType)
-		require.Equal(t, "my-artifact/data.txt", items.Value[0].Path)
+		require.Equal(t, item, items.Value[0].Path)
 
 		status, data = request(t, http.MethodGet, items.Value[0].ContentLocation, nil, nil)
 		require.Equal(t, http.StatusOK, status)
 		require.Equal(t, content, string(data))
 
-		stored, err := os.ReadFile(filepath.Join(artifactPath, runID, "my-artifact", "data.txt"))
+		stored, err := os.ReadFile(filepath.Join(artifactPath, runID, filepath.FromSlash(item)))
 		require.NoError(t, err)
 		require.Equal(t, content, string(stored))
 	})
@@ -95,16 +104,25 @@ func TestArtifactFlow(t *testing.T) {
 	t.Run("content-range", func(t *testing.T) {
 		const rawURL = "/upload/4?itemPath=chunks.txt"
 		status, data := request(t, http.MethodPut, baseURL+rawURL, strings.NewReader("first"),
-			http.Header{"Content-Range": []string{"bytes 0-4/11"}})
+			http.Header{"Content-Range": []string{"bytes 0-4/12"}})
 		require.Equal(t, http.StatusOK, status, string(data))
 
-		status, data = request(t, http.MethodPut, baseURL+rawURL, strings.NewReader("-second"),
-			http.Header{"Content-Range": []string{"bytes 5-11/11"}})
-		require.Equal(t, http.StatusOK, status, string(data))
+		for range 2 {
+			status, data = request(t, http.MethodPut, baseURL+rawURL, strings.NewReader("-second"),
+				http.Header{"Content-Range": []string{"bytes 5-11/12"}})
+			require.Equal(t, http.StatusOK, status, string(data))
+		}
 
 		stored, err := os.ReadFile(filepath.Join(artifactPath, "4", "chunks.txt"))
 		require.NoError(t, err)
 		require.Equal(t, "first-second", string(stored))
+
+		status, data = request(t, http.MethodPut, baseURL+rawURL, strings.NewReader(""),
+			http.Header{"Content-Range": []string{"bytes 0--1/0"}})
+		require.Equal(t, http.StatusOK, status, string(data))
+		stored, err = os.ReadFile(filepath.Join(artifactPath, "4", "chunks.txt"))
+		require.NoError(t, err)
+		require.Empty(t, stored)
 	})
 
 	t.Run("gzip-roundtrip", func(t *testing.T) {