mirror of
https://gitea.com/gitea/runner.git
synced 2026-11-17 05:13:42 +00:00
fix: stop a stalled docker daemon from hanging jobs and the e2e suite (#1404)
When the docker daemon stopped answering on https://gitea.com/gitea/runner/actions/runs/1204971, unbounded exec runs hung the e2e suite until the 20m `go test` panic, which skipped teardown. Exec runs and teardown are now bounded, and attached exec streams, including the runner's user ID probe, close with their context. CLI daemons now stop with SIGTERM, so job volumes no longer leak. _Written by Claude (Opus 5.5)._ --------- Co-authored-by: bircni <[email protected]> Reviewed-on: https://gitea.com/gitea/runner/pulls/1404 Reviewed-by: bircni <[email protected]> Co-authored-by: silverwind <[email protected]>
This commit is contained in:
1 parent
8e999cd039
commit
d34e4cf11a
9 files changed
+114
-91
No files matched your search
@@ -895,6 +895,7 @@ func (cr *containerReference) tryReadID(opt string, cbk func(id int)) common.Exe
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
defer resp.Close()
|
defer resp.Close()
|
||||||
|
defer context.AfterFunc(ctx, resp.Close)()
|
||||||
|
|
||||||
sid, err := resp.Reader.ReadString('\n')
|
sid, err := resp.Reader.ReadString('\n')
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -378,6 +378,21 @@ func TestDockerAttachStreamClosesWithItsContext(t *testing.T) {
|
|||||||
client.AssertExpectations(t)
|
client.AssertExpectations(t)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestDockerReadIDStreamClosesWithItsContext(t *testing.T) {
|
||||||
|
conn, peer := net.Pipe()
|
||||||
|
t.Cleanup(func() { _ = peer.Close() })
|
||||||
|
client := &mockDockerClient{}
|
||||||
|
client.On("ExecCreate", mock.Anything, "123", mock.AnythingOfType("client.ExecCreateOptions")).
|
||||||
|
Return(mobyclient.ExecCreateResult{ID: "id"}, nil)
|
||||||
|
client.On("ExecAttach", mock.Anything, "id", mobyclient.ExecAttachOptions{}).
|
||||||
|
Return(mobyclient.ExecAttachResult{HijackedResponse: mobyclient.NewHijackedResponse(conn, "")}, nil)
|
||||||
|
ctx, cancel := context.WithCancel(t.Context())
|
||||||
|
cancel()
|
||||||
|
|
||||||
|
require.NoError(t, (&containerReference{id: "123", cli: client}).tryReadID("-u", func(int) { t.Error("read an ID after cancellation") })(ctx))
|
||||||
|
client.AssertExpectations(t)
|
||||||
|
}
|
||||||
|
|
||||||
func TestDockerWaitFailure(t *testing.T) {
|
func TestDockerWaitFailure(t *testing.T) {
|
||||||
ctx := t.Context()
|
ctx := t.Context()
|
||||||
statusCh := make(chan container.WaitResponse, 1)
|
statusCh := make(chan container.WaitResponse, 1)
|
||||||
|
|||||||
+36
-47
@@ -90,20 +90,29 @@ container:
|
|||||||
type cliDaemon struct {
|
type cliDaemon struct {
|
||||||
cmd *exec.Cmd
|
cmd *exec.Cmd
|
||||||
log *syncBuffer
|
log *syncBuffer
|
||||||
exited chan error
|
exited chan struct{}
|
||||||
|
err error
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *cliRunner) startDaemon(t *testing.T, args ...string) *cliDaemon {
|
func (c *cliRunner) startDaemon(t *testing.T, args ...string) *cliDaemon {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
d := &cliDaemon{log: &syncBuffer{}, exited: make(chan error, 1)}
|
d := &cliDaemon{log: &syncBuffer{}, exited: make(chan struct{})}
|
||||||
d.cmd = exec.Command(c.bin, append([]string{"daemon", "-c", c.cfgFile}, args...)...)
|
d.cmd = exec.Command(c.bin, append([]string{"daemon", "-c", c.cfgFile}, args...)...)
|
||||||
d.cmd.Stdout, d.cmd.Stderr = d.log, d.log
|
d.cmd.Stdout, d.cmd.Stderr = d.log, d.log
|
||||||
if err := d.cmd.Start(); err != nil {
|
if err := d.cmd.Start(); err != nil {
|
||||||
t.Fatalf("start daemon: %v", err)
|
t.Fatalf("start daemon: %v", err)
|
||||||
}
|
}
|
||||||
go func() { d.exited <- d.cmd.Wait() }()
|
go func() {
|
||||||
|
d.err = d.cmd.Wait()
|
||||||
|
close(d.exited)
|
||||||
|
}()
|
||||||
t.Cleanup(func() {
|
t.Cleanup(func() {
|
||||||
_ = d.cmd.Process.Kill() // no-op once the daemon has exited
|
_ = d.cmd.Process.Signal(syscall.SIGTERM)
|
||||||
|
select {
|
||||||
|
case <-d.exited:
|
||||||
|
case <-time.After(30 * time.Second):
|
||||||
|
_ = d.cmd.Process.Kill()
|
||||||
|
}
|
||||||
})
|
})
|
||||||
return d
|
return d
|
||||||
}
|
}
|
||||||
@@ -112,8 +121,8 @@ func (c *cliRunner) startDaemon(t *testing.T, args ...string) *cliDaemon {
|
|||||||
func (d *cliDaemon) wait(t *testing.T, timeout time.Duration) error {
|
func (d *cliDaemon) wait(t *testing.T, timeout time.Duration) error {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
select {
|
select {
|
||||||
case err := <-d.exited:
|
case <-d.exited:
|
||||||
return err
|
return d.err
|
||||||
case <-time.After(timeout):
|
case <-time.After(timeout):
|
||||||
t.Fatalf("daemon still running after %s\n%s", timeout, d.log)
|
t.Fatalf("daemon still running after %s\n%s", timeout, d.log)
|
||||||
return nil
|
return nil
|
||||||
@@ -140,8 +149,8 @@ func (d *cliDaemon) requireSuccess(t *testing.T, c *cliRunner, runID int64) {
|
|||||||
// testCLIRegisterDaemon drives the built binary the way a user does: register, then daemon.
|
// testCLIRegisterDaemon drives the built binary the way a user does: register, then daemon.
|
||||||
func testCLIRegisterDaemon(t *testing.T) {
|
func testCLIRegisterDaemon(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
c := newCLIRunner(t, "e2e-cli", "")
|
c := newCLIRunner(t, "e2e-registered", "")
|
||||||
d := c.startDaemon(t)
|
d := c.startDaemon(t, "--labels", "e2e-cli:docker://"+os.Getenv("E2E_JOB_IMAGE"))
|
||||||
|
|
||||||
pushWorkflow(t, c.api, c.repo, "cli.yml")
|
pushWorkflow(t, c.api, c.repo, "cli.yml")
|
||||||
wfRun := waitForRun(t, c.api, c.repo)
|
wfRun := waitForRun(t, c.api, c.repo)
|
||||||
@@ -150,6 +159,22 @@ func testCLIRegisterDaemon(t *testing.T) {
|
|||||||
t.Errorf("job logs are missing the marker:\n%s", logs)
|
t.Errorf("job logs are missing the marker:\n%s", logs)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
runners, err := c.api.Runners(t.Context(), c.repo)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("list runners: %v", err)
|
||||||
|
}
|
||||||
|
var labels []string
|
||||||
|
for _, runner := range runners {
|
||||||
|
if runner.Name == c.repo {
|
||||||
|
for _, label := range runner.Labels {
|
||||||
|
labels = append(labels, label.Name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !slices.Equal(labels, []string{"e2e-cli"}) {
|
||||||
|
t.Errorf("runner %q has labels %v on the server, want only the --labels override", c.repo, labels)
|
||||||
|
}
|
||||||
|
|
||||||
d.terminate(t)
|
d.terminate(t)
|
||||||
if err := d.wait(t, 30*time.Second); err != nil {
|
if err := d.wait(t, 30*time.Second); err != nil {
|
||||||
t.Errorf("daemon did not exit cleanly on SIGTERM: %v\n%s", err, d.log)
|
t.Errorf("daemon did not exit cleanly on SIGTERM: %v\n%s", err, d.log)
|
||||||
@@ -186,36 +211,6 @@ func testCLIOnce(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// testCLILabelsOverride checks that daemon --labels replaces the labels given at registration.
|
|
||||||
func testCLILabelsOverride(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
c := newCLIRunner(t, "e2e-registered", "")
|
|
||||||
d := c.startDaemon(t, "--labels", "e2e-override:docker://"+os.Getenv("E2E_JOB_IMAGE"))
|
|
||||||
|
|
||||||
pushWorkflow(t, c.api, c.repo, "cli_labels.yml")
|
|
||||||
wfRun := waitForRun(t, c.api, c.repo)
|
|
||||||
d.requireSuccess(t, c, wfRun.ID)
|
|
||||||
|
|
||||||
runners, err := c.api.Runners(t.Context(), c.repo)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("list runners: %v", err)
|
|
||||||
}
|
|
||||||
for _, runner := range runners {
|
|
||||||
if runner.Name != c.repo {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
var labels []string
|
|
||||||
for _, label := range runner.Labels {
|
|
||||||
labels = append(labels, label.Name)
|
|
||||||
}
|
|
||||||
if !slices.Equal(labels, []string{"e2e-override"}) {
|
|
||||||
t.Errorf("runner has labels %v on the server, want only the --labels override", labels)
|
|
||||||
}
|
|
||||||
return
|
|
||||||
}
|
|
||||||
t.Errorf("runner %q not found among %+v", c.repo, runners)
|
|
||||||
}
|
|
||||||
|
|
||||||
// testCLIShutdownDrains checks that SIGTERM lets a running job finish within shutdown_timeout.
|
// testCLIShutdownDrains checks that SIGTERM lets a running job finish within shutdown_timeout.
|
||||||
func testCLIShutdownDrains(t *testing.T) {
|
func testCLIShutdownDrains(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
@@ -282,10 +277,6 @@ func testExecArtifacts(t *testing.T) {
|
|||||||
t.Parallel()
|
t.Parallel()
|
||||||
ctx := t.Context()
|
ctx := t.Context()
|
||||||
|
|
||||||
bin, err := runnerBinary()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
workflow, err := os.ReadFile("testdata/workflows/exec_artifact.yml")
|
workflow, err := os.ReadFile("testdata/workflows/exec_artifact.yml")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
@@ -309,11 +300,9 @@ func testExecArtifacts(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
artifacts := t.TempDir()
|
artifacts := t.TempDir()
|
||||||
args := []string{"exec", "-C", workdir, "-W", workflows, "-i", os.Getenv("E2E_JOB_IMAGE"), "--artifact-server-path", artifacts, "--network", fixture.network}
|
stdout, stderr, err := runExec(t, nil, "-C", workdir, "-W", workflows, "-i", os.Getenv("E2E_JOB_IMAGE"), "--artifact-server-path", artifacts, "--network", fixture.network)
|
||||||
cmd := exec.CommandContext(ctx, bin, args...)
|
if err != nil {
|
||||||
cmd.Env = slices.DeleteFunc(os.Environ(), func(kv string) bool { return strings.HasPrefix(kv, "ACTIONS_") })
|
t.Fatalf("exec: %v\n%s%s", err, stdout, stderr)
|
||||||
if out, err := cmd.CombinedOutput(); err != nil {
|
|
||||||
t.Fatalf("exec: %v\n%s", err, out)
|
|
||||||
}
|
}
|
||||||
if blobs, _ := filepath.Glob(filepath.Join(artifacts, "v4", "*", "*", "blob")); len(blobs) != 2 {
|
if blobs, _ := filepath.Glob(filepath.Join(artifacts, "v4", "*", "*", "blob")); len(blobs) != 2 {
|
||||||
t.Fatalf("want one v4 artifact blob per workflow run, got %v", blobs)
|
t.Fatalf("want one v4 artifact blob per workflow run, got %v", blobs)
|
||||||
|
|||||||
@@ -37,7 +37,6 @@ func TestCompatibility(t *testing.T) {
|
|||||||
t.Run("job_timeout", testJobTimeout)
|
t.Run("job_timeout", testJobTimeout)
|
||||||
t.Run("cli_register_daemon", testCLIRegisterDaemon)
|
t.Run("cli_register_daemon", testCLIRegisterDaemon)
|
||||||
t.Run("cli_once", testCLIOnce)
|
t.Run("cli_once", testCLIOnce)
|
||||||
t.Run("cli_labels_override", testCLILabelsOverride)
|
|
||||||
t.Run("cli_shutdown_drains", testCLIShutdownDrains)
|
t.Run("cli_shutdown_drains", testCLIShutdownDrains)
|
||||||
t.Run("cli_shutdown_cancels", testCLIShutdownCancels)
|
t.Run("cli_shutdown_cancels", testCLIShutdownCancels)
|
||||||
t.Run("host_mode", testHostMode)
|
t.Run("host_mode", testHostMode)
|
||||||
|
|||||||
+14
-12
@@ -80,11 +80,11 @@ func selfNetwork(ctx context.Context, cli mobyclient.APIClient) (string, string,
|
|||||||
return "", "", false
|
return "", "", false
|
||||||
}
|
}
|
||||||
|
|
||||||
func StartGitea(ctx context.Context, cli mobyclient.APIClient) (*GiteaFixture, error) {
|
func StartGitea(ctx context.Context, cli mobyclient.APIClient) (_ *GiteaFixture, err error) {
|
||||||
closeClient := true
|
f := &GiteaFixture{cli: cli}
|
||||||
defer func() {
|
defer func() {
|
||||||
if closeClient {
|
if err != nil {
|
||||||
_ = cli.Close()
|
_ = f.Close()
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
|
|
||||||
@@ -123,26 +123,21 @@ func StartGitea(ctx context.Context, cli mobyclient.APIClient) (*GiteaFixture, e
|
|||||||
return nil, fmt.Errorf("create gitea container: %w", err)
|
return nil, fmt.Errorf("create gitea container: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
f := &GiteaFixture{cli: cli, id: resp.ID, image: image, baseURL: baseURL, network: sharedNet, selfIP: selfIP}
|
f.id, f.image, f.baseURL, f.network, f.selfIP = resp.ID, image, baseURL, sharedNet, selfIP
|
||||||
closeClient = false
|
|
||||||
|
|
||||||
if _, err := cli.ContainerStart(ctx, f.id, mobyclient.ContainerStartOptions{}); err != nil {
|
if _, err := cli.ContainerStart(ctx, f.id, mobyclient.ContainerStartOptions{}); err != nil {
|
||||||
_ = f.Close(ctx)
|
|
||||||
return nil, fmt.Errorf("start gitea container: %w", err)
|
return nil, fmt.Errorf("start gitea container: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := f.waitHealthy(ctx); err != nil {
|
if err := f.waitHealthy(ctx); err != nil {
|
||||||
_ = f.Close(ctx)
|
|
||||||
return nil, fmt.Errorf("gitea did not become healthy: %w", err)
|
return nil, fmt.Errorf("gitea did not become healthy: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := f.bootstrapAdmin(ctx); err != nil {
|
if err := f.bootstrapAdmin(ctx); err != nil {
|
||||||
_ = f.Close(ctx)
|
|
||||||
return nil, fmt.Errorf("bootstrap gitea admin: %w", err)
|
return nil, fmt.Errorf("bootstrap gitea admin: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := f.readVersion(ctx); err != nil {
|
if err := f.readVersion(ctx); err != nil {
|
||||||
_ = f.Close(ctx)
|
|
||||||
return nil, fmt.Errorf("read gitea version: %w", err)
|
return nil, fmt.Errorf("read gitea version: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -249,6 +244,7 @@ func containerExec(ctx context.Context, cli mobyclient.APIClient, id, user strin
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
|
defer context.AfterFunc(ctx, attached.Close)()
|
||||||
defer attached.Close()
|
defer attached.Close()
|
||||||
if stdin != nil {
|
if stdin != nil {
|
||||||
go func() {
|
go func() {
|
||||||
@@ -258,7 +254,11 @@ func containerExec(ctx context.Context, cli mobyclient.APIClient, id, user strin
|
|||||||
}
|
}
|
||||||
|
|
||||||
var out bytes.Buffer
|
var out bytes.Buffer
|
||||||
if _, err := stdcopy.StdCopy(&out, &out, attached.Reader); err != nil {
|
_, err = stdcopy.StdCopy(&out, &out, attached.Reader)
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
return "", ctx.Err()
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -291,10 +291,12 @@ func (f *GiteaFixture) doJSON(ctx context.Context, method, path string, reqBody,
|
|||||||
return api.doJSON(ctx, method, path, reqBody, respBody)
|
return api.doJSON(ctx, method, path, reqBody, respBody)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (f *GiteaFixture) Close(ctx context.Context) error {
|
func (f *GiteaFixture) Close() error {
|
||||||
if f.id == "" {
|
if f.id == "" {
|
||||||
return f.cli.Close()
|
return f.cli.Close()
|
||||||
}
|
}
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), cleanupTimeout)
|
||||||
|
defer cancel()
|
||||||
_, removeErr := f.cli.ContainerRemove(ctx, f.id, mobyclient.ContainerRemoveOptions{Force: true, RemoveVolumes: true})
|
_, removeErr := f.cli.ContainerRemove(ctx, f.id, mobyclient.ContainerRemoveOptions{Force: true, RemoveVolumes: true})
|
||||||
return errors.Join(removeErr, f.cli.Close())
|
return errors.Join(removeErr, f.cli.Close())
|
||||||
}
|
}
|
||||||
+13
-7
@@ -55,7 +55,9 @@ func checkClusterLeaks() error {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil // the tests that needed the cluster already failed
|
return nil // the tests that needed the cluster already failed
|
||||||
}
|
}
|
||||||
if out, err := containerExec(context.Background(), dockerCLI, cluster.id, "", nil,
|
ctx, cancel := context.WithTimeout(context.Background(), cleanupTimeout)
|
||||||
|
defer cancel()
|
||||||
|
if out, err := containerExec(ctx, dockerCLI, cluster.id, "", nil,
|
||||||
"kubectl", "wait", "--for=delete", "pods,secrets", "--selector=app.kubernetes.io/managed-by=gitea-runner", "--timeout=1m",
|
"kubectl", "wait", "--for=delete", "pods,secrets", "--selector=app.kubernetes.io/managed-by=gitea-runner", "--timeout=1m",
|
||||||
); err != nil {
|
); err != nil {
|
||||||
return fmt.Errorf("job pods or secrets left behind: %w: %s", err, out)
|
return fmt.Errorf("job pods or secrets left behind: %w: %s", err, out)
|
||||||
@@ -72,7 +74,7 @@ type kubeCluster struct {
|
|||||||
var (
|
var (
|
||||||
k3s = sync.OnceValues(startK3s) // independent of Gitea, so it can boot alongside it
|
k3s = sync.OnceValues(startK3s) // independent of Gitea, so it can boot alongside it
|
||||||
clusterStarted atomic.Bool
|
clusterStarted atomic.Bool
|
||||||
clusterCleanup []func()
|
clusterCleanup []func(context.Context)
|
||||||
sharedCluster = sync.OnceValues(func() (*kubeCluster, error) {
|
sharedCluster = sync.OnceValues(func() (*kubeCluster, error) {
|
||||||
cluster, err := k3s()
|
cluster, err := k3s()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -109,7 +111,9 @@ func stopCluster() {
|
|||||||
}
|
}
|
||||||
_, _ = k3s()
|
_, _ = k3s()
|
||||||
for _, cleanup := range slices.Backward(clusterCleanup) {
|
for _, cleanup := range slices.Backward(clusterCleanup) {
|
||||||
cleanup()
|
ctx, cancel := context.WithTimeout(context.Background(), cleanupTimeout)
|
||||||
|
cleanup(ctx)
|
||||||
|
cancel()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -138,8 +142,8 @@ func startK3s() (*kubeCluster, error) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("create k3s container: %w", err)
|
return nil, fmt.Errorf("create k3s container: %w", err)
|
||||||
}
|
}
|
||||||
clusterCleanup = append(clusterCleanup, func() {
|
clusterCleanup = append(clusterCleanup, func(ctx context.Context) {
|
||||||
_, _ = cli.ContainerRemove(context.Background(), created.ID, mobyclient.ContainerRemoveOptions{Force: true, RemoveVolumes: true})
|
_, _ = cli.ContainerRemove(ctx, created.ID, mobyclient.ContainerRemoveOptions{Force: true, RemoveVolumes: true})
|
||||||
})
|
})
|
||||||
if _, err := cli.ContainerStart(ctx, created.ID, mobyclient.ContainerStartOptions{}); err != nil {
|
if _, err := cli.ContainerStart(ctx, created.ID, mobyclient.ContainerStartOptions{}); err != nil {
|
||||||
return nil, fmt.Errorf("start k3s container: %w", err)
|
return nil, fmt.Errorf("start k3s container: %w", err)
|
||||||
@@ -170,7 +174,9 @@ func startK3s() (*kubeCluster, error) {
|
|||||||
inspected, inspectErr := cli.ContainerInspect(ctx, created.ID, mobyclient.ContainerInspectOptions{})
|
inspected, inspectErr := cli.ContainerInspect(ctx, created.ID, mobyclient.ContainerInspectOptions{})
|
||||||
if inspectErr != nil || !inspected.Container.State.Running || ctx.Err() != nil {
|
if inspectErr != nil || !inspected.Container.State.Running || ctx.Err() != nil {
|
||||||
var logs bytes.Buffer
|
var logs bytes.Buffer
|
||||||
if reader, logsErr := cli.ContainerLogs(context.Background(), created.ID, mobyclient.ContainerLogsOptions{ShowStdout: true, ShowStderr: true, Tail: "30"}); logsErr == nil {
|
logsCtx, logsCancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||||
|
defer logsCancel()
|
||||||
|
if reader, logsErr := cli.ContainerLogs(logsCtx, created.ID, mobyclient.ContainerLogsOptions{ShowStdout: true, ShowStderr: true, Tail: "30"}); logsErr == nil {
|
||||||
_, _ = stdcopy.StdCopy(&logs, &logs, reader)
|
_, _ = stdcopy.StdCopy(&logs, &logs, reader)
|
||||||
_ = reader.Close()
|
_ = reader.Close()
|
||||||
}
|
}
|
||||||
@@ -226,7 +232,7 @@ func preloadImages(ctx context.Context, k3sID string, images ...string) ([]strin
|
|||||||
if _, err := dockerCLI.ImageTag(ctx, mobyclient.ImageTagOptions{Source: image, Target: alias}); err != nil {
|
if _, err := dockerCLI.ImageTag(ctx, mobyclient.ImageTagOptions{Source: image, Target: alias}); err != nil {
|
||||||
return nil, fmt.Errorf("tag %s: %w", image, err)
|
return nil, fmt.Errorf("tag %s: %w", image, err)
|
||||||
}
|
}
|
||||||
clusterCleanup = append(clusterCleanup, func() { _, _ = dockerCLI.ImageRemove(context.Background(), alias, mobyclient.ImageRemoveOptions{}) })
|
clusterCleanup = append(clusterCleanup, func(ctx context.Context) { _, _ = dockerCLI.ImageRemove(ctx, alias, mobyclient.ImageRemoveOptions{}) })
|
||||||
aliases = append(aliases, alias)
|
aliases = append(aliases, alias)
|
||||||
}
|
}
|
||||||
archive, err := dockerCLI.ImageSave(ctx, aliases)
|
archive, err := dockerCLI.ImageSave(ctx, aliases)
|
||||||
|
|||||||
+6
-2
@@ -31,6 +31,8 @@ import (
|
|||||||
|
|
||||||
const runTimeout = 3 * time.Minute
|
const runTimeout = 3 * time.Minute
|
||||||
|
|
||||||
|
const cleanupTimeout = 2 * time.Minute
|
||||||
|
|
||||||
var nonRepoChars = regexp.MustCompile(`[^a-zA-Z0-9._-]+`)
|
var nonRepoChars = regexp.MustCompile(`[^a-zA-Z0-9._-]+`)
|
||||||
|
|
||||||
var fixture *GiteaFixture
|
var fixture *GiteaFixture
|
||||||
@@ -42,8 +44,10 @@ var binDir string
|
|||||||
var dockerCLI mobyclient.APIClient
|
var dockerCLI mobyclient.APIClient
|
||||||
|
|
||||||
var runnerBinary = sync.OnceValues(func() (string, error) {
|
var runnerBinary = sync.OnceValues(func() (string, error) {
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), runTimeout)
|
||||||
|
defer cancel()
|
||||||
bin := filepath.Join(binDir, "gitea-runner")
|
bin := filepath.Join(binDir, "gitea-runner")
|
||||||
if out, err := exec.Command(cmp.Or(os.Getenv("GO"), "go"), "build", "-o", bin, "..").CombinedOutput(); err != nil {
|
if out, err := exec.CommandContext(ctx, cmp.Or(os.Getenv("GO"), "go"), "build", "-o", bin, "..").CombinedOutput(); err != nil {
|
||||||
return "", fmt.Errorf("build runner: %w\n%s", err, out)
|
return "", fmt.Errorf("build runner: %w\n%s", err, out)
|
||||||
}
|
}
|
||||||
return bin, nil
|
return bin, nil
|
||||||
@@ -80,7 +84,7 @@ func runSuite(m *testing.M) int {
|
|||||||
return 1
|
return 1
|
||||||
}
|
}
|
||||||
fixture = f
|
fixture = f
|
||||||
defer func() { _ = fixture.Close(context.Background()) }()
|
defer func() { _ = fixture.Close() }()
|
||||||
|
|
||||||
fmt.Fprintf(os.Stderr, "gitea fixture: image=%s version=%s\n", fixture.image, fixture.version)
|
fmt.Fprintf(os.Stderr, "gitea fixture: image=%s version=%s\n", fixture.image, fixture.version)
|
||||||
|
|
||||||
|
|||||||
+29
-15
@@ -9,6 +9,7 @@ import (
|
|||||||
"bufio"
|
"bufio"
|
||||||
"bytes"
|
"bytes"
|
||||||
"cmp"
|
"cmp"
|
||||||
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
"os"
|
"os"
|
||||||
@@ -138,14 +139,31 @@ func daemonWorkflow(t *testing.T, m mode, spec workflowSpec) *workflowResult {
|
|||||||
return result
|
return result
|
||||||
}
|
}
|
||||||
|
|
||||||
func execWorkflow(t *testing.T, m mode, spec workflowSpec) *workflowResult {
|
func runExec(t *testing.T, env []string, args ...string) (stdout, stderr []byte, err error) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
ctx := t.Context()
|
|
||||||
|
|
||||||
bin, err := runnerBinary()
|
bin, err := runnerBinary()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
ctx, cancel := context.WithTimeout(t.Context(), runTimeout)
|
||||||
|
defer cancel()
|
||||||
|
cmd := exec.CommandContext(ctx, bin, append([]string{"exec"}, args...)...)
|
||||||
|
cmd.Env = append(slices.DeleteFunc(os.Environ(), func(kv string) bool { return strings.HasPrefix(kv, "ACTIONS_") }), env...)
|
||||||
|
cmd.Cancel = func() error { return cmd.Process.Signal(os.Interrupt) }
|
||||||
|
cmd.WaitDelay = cleanupTimeout
|
||||||
|
var stdoutBuf, stderrBuf bytes.Buffer
|
||||||
|
cmd.Stdout, cmd.Stderr = &stdoutBuf, &stderrBuf
|
||||||
|
err = cmd.Run()
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
t.Fatalf("exec did not finish within %s\n%s%s", runTimeout, stdoutBuf.Bytes(), stderrBuf.Bytes())
|
||||||
|
}
|
||||||
|
return stdoutBuf.Bytes(), stderrBuf.Bytes(), err
|
||||||
|
}
|
||||||
|
|
||||||
|
func execWorkflow(t *testing.T, m mode, spec workflowSpec) *workflowResult {
|
||||||
|
t.Helper()
|
||||||
|
ctx := t.Context()
|
||||||
|
|
||||||
workdir := t.TempDir()
|
workdir := t.TempDir()
|
||||||
workflows := filepath.Join(workdir, ".gitea", "workflows")
|
workflows := filepath.Join(workdir, ".gitea", "workflows")
|
||||||
if err := os.MkdirAll(workflows, 0o755); err != nil {
|
if err := os.MkdirAll(workflows, 0o755); err != nil {
|
||||||
@@ -170,12 +188,12 @@ func execWorkflow(t *testing.T, m mode, spec workflowSpec) *workflowResult {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
args := []string{"exec", "--json", "-C", workdir, "-W", workflows, "--artifact-server-path", t.TempDir(), "--artifact-server-port", "0"}
|
args := []string{"--json", "-C", workdir, "-W", workflows, "--artifact-server-path", t.TempDir(), "--artifact-server-port", "0"}
|
||||||
env := slices.DeleteFunc(os.Environ(), func(kv string) bool { return strings.HasPrefix(kv, "ACTIONS_") })
|
var env []string
|
||||||
if m.kubernetes {
|
if m.kubernetes {
|
||||||
cluster := requireCluster(t)
|
cluster := requireCluster(t)
|
||||||
args = append(args, "-i", "kubernetes://"+cluster.jobImage)
|
args = append(args, "-i", "kubernetes://"+cluster.jobImage)
|
||||||
env = append(env, "KUBECONFIG="+cluster.kubernetes.Kubeconfig)
|
env = []string{"KUBECONFIG=" + cluster.kubernetes.Kubeconfig}
|
||||||
} else {
|
} else {
|
||||||
args = append(args, "-i", os.Getenv("E2E_JOB_IMAGE"), "--network", fixture.network)
|
args = append(args, "-i", os.Getenv("E2E_JOB_IMAGE"), "--network", fixture.network)
|
||||||
}
|
}
|
||||||
@@ -185,21 +203,17 @@ func execWorkflow(t *testing.T, m mode, spec workflowSpec) *workflowResult {
|
|||||||
args = append(args, "--input", name+"="+value)
|
args = append(args, "--input", name+"="+value)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
cmd := exec.CommandContext(ctx, bin, append(args, spec.execArgs...)...)
|
stdout, stderr, runErr := runExec(t, env, append(args, spec.execArgs...)...)
|
||||||
cmd.Env = env
|
|
||||||
var stdout, stderr bytes.Buffer
|
|
||||||
cmd.Stdout, cmd.Stderr = &stdout, &stderr
|
|
||||||
runErr := cmd.Run()
|
|
||||||
var exitErr *exec.ExitError
|
var exitErr *exec.ExitError
|
||||||
if runErr != nil && !errors.As(runErr, &exitErr) {
|
if runErr != nil && !errors.As(runErr, &exitErr) {
|
||||||
t.Fatalf("exec: %v\n%s", runErr, stderr.String())
|
t.Fatalf("exec: %v\n%s", runErr, stderr)
|
||||||
}
|
}
|
||||||
|
|
||||||
result := &workflowResult{conclusion: "success", jobs: map[string]string{}}
|
result := &workflowResult{conclusion: "success", jobs: map[string]string{}}
|
||||||
if runErr != nil {
|
if runErr != nil {
|
||||||
result.conclusion = "failure"
|
result.conclusion = "failure"
|
||||||
}
|
}
|
||||||
scanner := bufio.NewScanner(&stdout)
|
scanner := bufio.NewScanner(bytes.NewReader(stdout))
|
||||||
scanner.Buffer(nil, 1<<20)
|
scanner.Buffer(nil, 1<<20)
|
||||||
for scanner.Scan() {
|
for scanner.Scan() {
|
||||||
var entry struct {
|
var entry struct {
|
||||||
@@ -220,8 +234,8 @@ func execWorkflow(t *testing.T, m mode, spec workflowSpec) *workflowResult {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if result.conclusion != "success" && len(result.jobs) == 0 {
|
if len(result.jobs) == 0 {
|
||||||
t.Fatalf("exec ran no job: %v\n%s", runErr, stderr.String())
|
t.Fatalf("exec ran no job: %v\n%s", runErr, stderr)
|
||||||
}
|
}
|
||||||
return result
|
return result
|
||||||
}
|
}
|
||||||
-7
@@ -1,7 +0,0 @@
|
|||||||
name: cli-labels
|
|
||||||
on: push
|
|
||||||
jobs:
|
|
||||||
hello:
|
|
||||||
runs-on: e2e-override
|
|
||||||
steps:
|
|
||||||
- run: echo e2e-override-ran
|
|
||||||
Reference in new issue
Block a user