mirror of
https://gitea.com/gitea/runner.git
synced 2026-11-04 14:35:32 +00:00
silverwind
fe2a66dc25
Contains a number of small bug fixes where jobs behaved differently from GitHub or failed silently.
1. Hooks fail on a refused `::set-env::` or `::add-path::` and no longer inherit the last step's identity
1. Docker build, exec and wait errors are returned instead of swallowed, token-only registry credentials work, logs of `--tty` services decode
1. Env files split lines like GitHub on each OS, so a carriage return is kept on Linux and CRLF ends a line on Windows, empty heredoc names or delimiters are rejected
1. Post steps run after their pre step even when main was skipped, composites see undeclared inputs, `runs.args` wins without `with.args` being parsed
1. Docker actions skip evaluating defaults of inputs given in `with` and read `entryPoint` and `args` in any case, a lowercase `dockerfile` is found, undeclared inputs get GitHub's warning
1. A user-set `CI` is kept, `GITHUB_REPOSITORY_ID` and the other missing `GITHUB_*` variables are exported
1. `timeout-minutes` can read the step's own `env`, a step-level `ACTIONS_ALLOW_UNSECURE_COMMANDS: false` beats a job-level `true` while the runner's own opt-in still wins, `pwsh` quotes the script path
1. A step `env:` given as one expression such as `${{ fromJSON(matrix.env) }}` is applied, `hashFiles` fails with GitHub's messages on script errors and after 120 seconds, a job image without node still gets an empty hash
1. A step whose command files cannot be reset fails instead of importing stale ones, a timeout between steps fails the job instead of reporting success
1. One named volume can be mounted at several paths, host-mode PTY steps keep their last output when they fail, host scratch directories are removed when job setup fails
1. Cancelling a step right after it starts kills its whole process tree
1. Cache prefix restores pick the newest entry created within the same second, also on S3, cache commits with a wrong size are rejected
1. Artifact chunks are written at their offset, download URLs are escaped, listing a run without artifacts no longer crashes
1. Poll backoff reaches `fetch_interval_max`, proxy passwords are masked in more forms, secrets are also masked in step names and in their quoted form in the runner log, a failed final report is logged
1. On Windows hosts a differently cased unsecure-commands opt-in or proxy variable counts, and an early cancel no longer leaks a job object handle
1. Interactive `register` prompts for a missing token and stops on SIGTERM, `exec --list --detect-event` no longer panics without events, `config set` works on `cache.s3` keys
Reviewed-on: https://gitea.com/gitea/runner/pulls/1338
Reviewed-by: bircni <[email protected]>
Co-authored-by: silverwind <[email protected]>
207 lines
7.4 KiB
Go
207 lines
7.4 KiB
Go
// Copyright 2023 The Gitea Authors. All rights reserved.
|
|
// Copyright 2021 The nektos/act Authors. All rights reserved.
|
|
// SPDX-License-Identifier: MIT
|
|
|
|
package artifacts
|
|
|
|
import (
|
|
"bytes"
|
|
"compress/gzip"
|
|
"encoding/json/v2"
|
|
"io"
|
|
"maps"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"net/url"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/stretchr/testify/require"
|
|
)
|
|
|
|
func TestArtifactFlow(t *testing.T) {
|
|
artifactPath := t.TempDir()
|
|
|
|
router := http.NewServeMux()
|
|
uploads(router, artifactPath)
|
|
downloads(router, artifactPath)
|
|
server := httptest.NewServer(router)
|
|
defer server.Close()
|
|
|
|
baseURL := server.URL
|
|
client := server.Client()
|
|
client.Timeout = 5 * time.Second
|
|
|
|
request := func(t *testing.T, method, rawURL string, body io.Reader, header http.Header) (int, []byte) {
|
|
t.Helper()
|
|
req, err := http.NewRequest(method, rawURL, body)
|
|
require.NoError(t, err)
|
|
maps.Copy(req.Header, header)
|
|
resp, err := client.Do(req)
|
|
require.NoError(t, err)
|
|
defer resp.Body.Close()
|
|
data, err := io.ReadAll(resp.Body)
|
|
require.NoError(t, err)
|
|
return resp.StatusCode, data
|
|
}
|
|
|
|
t.Run("no-artifacts", func(t *testing.T) {
|
|
status, data := request(t, http.MethodGet, baseURL+"/_apis/pipelines/workflows/0/artifacts", nil, nil)
|
|
require.Equal(t, http.StatusOK, status, string(data))
|
|
var list NamedFileContainerResourceURLResponse
|
|
require.NoError(t, json.Unmarshal(data, &list))
|
|
require.Zero(t, list.Count)
|
|
require.Empty(t, list.Value)
|
|
})
|
|
|
|
t.Run("upload-and-download", func(t *testing.T) {
|
|
const runID, item, content = "1", "my-artifact/data#1%25.txt", "hello artifact\n"
|
|
|
|
status, data := request(t, http.MethodPost, baseURL+"/_apis/pipelines/workflows/"+runID+"/artifacts", nil, nil)
|
|
require.Equal(t, http.StatusOK, status, string(data))
|
|
var prep FileContainerResourceURL
|
|
require.NoError(t, json.Unmarshal(data, &prep))
|
|
require.Equal(t, baseURL+"/upload/"+runID, prep.FileContainerResourceURL)
|
|
|
|
status, data = request(t, http.MethodPut, prep.FileContainerResourceURL+"?itemPath="+url.QueryEscape(item), strings.NewReader(content), nil)
|
|
require.Equal(t, http.StatusOK, status, string(data))
|
|
var msg ResponseMessage
|
|
require.NoError(t, json.Unmarshal(data, &msg))
|
|
require.Equal(t, "success", msg.Message)
|
|
|
|
status, data = request(t, http.MethodPatch, baseURL+"/_apis/pipelines/workflows/"+runID+"/artifacts", nil, nil)
|
|
require.Equal(t, http.StatusOK, status, string(data))
|
|
require.NoError(t, json.Unmarshal(data, &msg))
|
|
require.Equal(t, "success", msg.Message)
|
|
|
|
status, data = request(t, http.MethodGet, baseURL+"/_apis/pipelines/workflows/"+runID+"/artifacts", nil, nil)
|
|
require.Equal(t, http.StatusOK, status, string(data))
|
|
var list NamedFileContainerResourceURLResponse
|
|
require.NoError(t, json.Unmarshal(data, &list))
|
|
require.Equal(t, 1, list.Count)
|
|
require.Equal(t, "my-artifact", list.Value[0].Name)
|
|
|
|
status, data = request(t, http.MethodGet, list.Value[0].FileContainerResourceURL+"?itemPath=my-artifact", nil, nil)
|
|
require.Equal(t, http.StatusOK, status, string(data))
|
|
var items ContainerItemResponse
|
|
require.NoError(t, json.Unmarshal(data, &items))
|
|
require.Len(t, items.Value, 1)
|
|
require.Equal(t, "file", items.Value[0].ItemType)
|
|
require.Equal(t, item, items.Value[0].Path)
|
|
|
|
status, data = request(t, http.MethodGet, items.Value[0].ContentLocation, nil, nil)
|
|
require.Equal(t, http.StatusOK, status)
|
|
require.Equal(t, content, string(data))
|
|
|
|
stored, err := os.ReadFile(filepath.Join(artifactPath, runID, filepath.FromSlash(item)))
|
|
require.NoError(t, err)
|
|
require.Equal(t, content, string(stored))
|
|
})
|
|
|
|
t.Run("content-range", func(t *testing.T) {
|
|
const rawURL = "/upload/4?itemPath=chunks.txt"
|
|
status, data := request(t, http.MethodPut, baseURL+rawURL, strings.NewReader("first"),
|
|
http.Header{"Content-Range": []string{"bytes 0-4/12"}})
|
|
require.Equal(t, http.StatusOK, status, string(data))
|
|
|
|
for range 2 {
|
|
status, data = request(t, http.MethodPut, baseURL+rawURL, strings.NewReader("-second"),
|
|
http.Header{"Content-Range": []string{"bytes 5-11/12"}})
|
|
require.Equal(t, http.StatusOK, status, string(data))
|
|
}
|
|
|
|
stored, err := os.ReadFile(filepath.Join(artifactPath, "4", "chunks.txt"))
|
|
require.NoError(t, err)
|
|
require.Equal(t, "first-second", string(stored))
|
|
|
|
status, data = request(t, http.MethodPut, baseURL+rawURL, strings.NewReader(""),
|
|
http.Header{"Content-Range": []string{"bytes 0--1/0"}})
|
|
require.Equal(t, http.StatusOK, status, string(data))
|
|
stored, err = os.ReadFile(filepath.Join(artifactPath, "4", "chunks.txt"))
|
|
require.NoError(t, err)
|
|
require.Empty(t, stored)
|
|
})
|
|
|
|
t.Run("gzip-roundtrip", func(t *testing.T) {
|
|
const runID, item, content = "2", "logs/app.log", "compressed payload\n"
|
|
|
|
var buf bytes.Buffer
|
|
gz := gzip.NewWriter(&buf)
|
|
_, err := gz.Write([]byte(content))
|
|
require.NoError(t, err)
|
|
require.NoError(t, gz.Close())
|
|
|
|
status, data := request(t, http.MethodPut, baseURL+"/upload/"+runID+"?itemPath="+url.QueryEscape(item),
|
|
&buf, http.Header{"Content-Encoding": []string{"gzip"}})
|
|
require.Equal(t, http.StatusOK, status, string(data))
|
|
|
|
// stored compressed, with the server's gzip marker suffix
|
|
_, err = os.Stat(filepath.Join(artifactPath, runID, "logs", "app.log.gz__"))
|
|
require.NoError(t, err)
|
|
|
|
status, data = request(t, http.MethodGet, baseURL+"/download/"+runID+"?itemPath=logs", nil, nil)
|
|
require.Equal(t, http.StatusOK, status, string(data))
|
|
var items ContainerItemResponse
|
|
require.NoError(t, json.Unmarshal(data, &items))
|
|
require.Len(t, items.Value, 1)
|
|
require.Equal(t, "logs/app.log", items.Value[0].Path)
|
|
|
|
status, data = request(t, http.MethodGet, items.Value[0].ContentLocation, nil, nil)
|
|
require.Equal(t, http.StatusOK, status)
|
|
require.Equal(t, content, string(data))
|
|
})
|
|
|
|
// GHSL-2023-004: an itemPath that climbs out of the run directory must be neutralised so the
|
|
// blob cannot be written outside the artifact root.
|
|
t.Run("GHSL-2023-004", func(t *testing.T) {
|
|
const runID, content = "3", "contained\n"
|
|
|
|
status, data := request(t, http.MethodPut, baseURL+"/upload/"+runID+"?itemPath="+url.QueryEscape("../../escape.txt"),
|
|
strings.NewReader(content), nil)
|
|
require.Equal(t, http.StatusOK, status, string(data))
|
|
|
|
stored, err := os.ReadFile(filepath.Join(artifactPath, runID, "escape.txt"))
|
|
require.NoError(t, err)
|
|
require.Equal(t, content, string(stored))
|
|
|
|
_, err = os.Stat(filepath.Join(filepath.Dir(artifactPath), "escape.txt"))
|
|
require.True(t, os.IsNotExist(err), "upload escaped the artifact root")
|
|
|
|
status, data = request(t, http.MethodGet, baseURL+"/artifact/"+runID+"/escape.txt", nil, nil)
|
|
require.Equal(t, http.StatusOK, status)
|
|
require.Equal(t, content, string(data))
|
|
})
|
|
}
|
|
|
|
func TestSafeResolve(t *testing.T) {
|
|
baseDir := "/foo/bar"
|
|
|
|
tests := map[string]struct {
|
|
input string
|
|
want string
|
|
}{
|
|
"simple": {input: "baz", want: "/foo/bar/baz"},
|
|
"nested": {input: "baz/blue", want: "/foo/bar/baz/blue"},
|
|
"dots in middle": {input: "baz/../../blue", want: "/foo/bar/blue"},
|
|
"leading dots": {input: "../../parent", want: "/foo/bar/parent"},
|
|
"root path": {input: "/root", want: "/foo/bar/root"},
|
|
"root": {input: "/", want: "/foo/bar"},
|
|
"empty": {input: "", want: "/foo/bar"},
|
|
}
|
|
|
|
for name, tc := range tests {
|
|
t.Run(name, func(t *testing.T) {
|
|
require.Equal(t, tc.want, safeResolve(baseDir, tc.input))
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestServeEmptyArtifactPathReturnsCancelableNoop(t *testing.T) {
|
|
cancel := Serve(t.Context(), "", "127.0.0.1", "0")
|
|
require.NotNil(t, cancel)
|
|
cancel()
|
|
}
|