test(e2e): speed up and cover exec and Kubernetes modes (#1363)

Follow-up to https://gitea.com/gitea/runner/pulls/1362, whose crash e2e missed because it only drove the daemon on Docker. - `dispatch`, `failure_paths` and `reusable_workflow` run through the daemon and `exec`, each on Docker and one shared k3s cluster. - Fixed sleeps become gates the test releases, the job timeout uses a 15s runner deadline, k3s boots alongside Gitea, Gitea's data is on tmpfs. - Outside a container, `make test-e2e` runs `go test` in a Linux container on its own network like CI, so local runs take the same paths. Local wall time: | | Before | After | |---|---|---| | Suite | 84s | 27–35s | | `job_timeout` | 64s | 17s | | `reconnect` | 22s | 4.5s | | `capacity/serial` | 20s | 2s | | `cli_shutdown_drains` | 14s | 2.3s | | `kubernetes` | 31s | 21–23s | _Written by Claude (Opus 5.5)._ --------- Co-authored-by: bircni <[email protected]> Reviewed-on: https://gitea.com/gitea/runner/pulls/1363 Reviewed-by: bircni <[email protected]> Co-authored-by: silverwind <[email protected]>
2026-11-09 00:30:28 +00:00 · 2026-10-31 10:47:35 +00:00
1 parent 2e72cf902a
commit a46d3f3eda
@@ -9,15 +9,16 @@ import (
 	"bufio"
 	"bytes"
 	"context"
+	"errors"
 	"fmt"
 	"io"
-	"net"
-	"net/netip"
 	"net/url"
 	"os"
 	"path/filepath"
-	"strconv"
+	"slices"
 	"strings"
+	"sync"
+	"sync/atomic"
 	"testing"
 	"time"

@@ -32,125 +33,168 @@ import (
 func testKubernetes(t *testing.T) {
 	t.Parallel()
 	ctx := t.Context()
-	k3sID, kubernetes, images := startK3s(t)
+	cluster := requireCluster(t)

 	api, repo := newScenario(t)
-	if err := api.CreateVariable(ctx, repo, "E2E_SERVICE_IMAGE", images[1]); err != nil {
+	if err := api.CreateVariable(ctx, repo, "E2E_SERVICE_IMAGE", cluster.serviceImage); err != nil {
 		t.Fatalf("create service image variable: %v", err)
 	}
-	startRunner(t, repo, "e2e-kubernetes", runnerOptions{image: images[0], kubernetes: kubernetes})
+	startRunner(t, repo, "e2e-kubernetes", runnerOptions{image: cluster.jobImage, kubernetes: cluster.kubernetes})
 	pushWorkflow(t, api, repo, "kubernetes.yml")

 	wfRun := waitForRun(t, api, repo)
 	requireSuccess(t, api, repo, wfRun.ID)
+}

-	if out, err := containerExec(ctx, fixture.cli, k3sID, "", nil,
+// checkClusterLeaks runs after every test, so a pod or secret any of them left behind fails the suite.
+func checkClusterLeaks() error {
+	if !clusterStarted.Load() {
+		return nil
+	}
+	cluster, err := k3s()
+	if err != nil {
+		return nil // the tests that needed the cluster already failed
+	}
+	if out, err := containerExec(context.Background(), dockerCLI, cluster.id, "", nil,
 		"kubectl", "wait", "--for=delete", "pods,secrets", "--selector=app.kubernetes.io/managed-by=gitea-runner", "--timeout=1m",
 	); err != nil {
-		t.Errorf("job pods or secrets left behind: %v: %s", err, out)
+		return fmt.Errorf("job pods or secrets left behind: %w: %s", err, out)
+	}
+	return nil
+}
+
+type kubeCluster struct {
+	id                     string
+	kubernetes             *config.Kubernetes
+	jobImage, serviceImage string
+}
+
+var (
+	k3s            = sync.OnceValues(startK3s) // independent of Gitea, so it can boot alongside it
+	clusterStarted atomic.Bool
+	clusterCleanup []func()
+	sharedCluster  = sync.OnceValues(func() (*kubeCluster, error) {
+		cluster, err := k3s()
+		if err != nil {
+			return nil, err
+		}
+		gitea, err := fixture.cli.ContainerInspect(context.Background(), fixture.id, mobyclient.ContainerInspectOptions{})
+		if err != nil {
+			return nil, fmt.Errorf("inspect gitea container: %w", err)
+		}
+		giteaURL, err := url.Parse(fixture.baseURL)
+		if err != nil {
+			return nil, fmt.Errorf("parse gitea url: %w", err)
+		}
+		cluster.kubernetes.PodTemplate = map[string]any{"spec": map[string]any{"hostAliases": []any{map[string]any{
+			"ip":        gitea.Container.NetworkSettings.Networks[fixture.network].IPAddress.String(),
+			"hostnames": []any{giteaURL.Hostname()},
+		}}}}
+		return cluster, nil
+	})
+)
+
+func requireCluster(t *testing.T) *kubeCluster {
+	t.Helper()
+	cluster, err := sharedCluster()
+	if err != nil {
+		t.Fatal(err)
+	}
+	return cluster
+}
+
+func stopCluster() {
+	if !clusterStarted.Load() {
+		return
+	}
+	_, _ = k3s()
+	for _, cleanup := range slices.Backward(clusterCleanup) {
+		cleanup()
 	}
 }

-func startK3s(t *testing.T) (string, *config.Kubernetes, []string) {
-	t.Helper()
-	ctx := t.Context()
-	cli := fixture.cli
+func startK3s() (*kubeCluster, error) {
+	clusterStarted.Store(true)
+	ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute)
+	defer cancel()
+	cli := dockerCLI
+	sharedNet, _, ok := selfNetwork(ctx, cli)
+	if !ok {
+		return nil, errors.New("e2e tests must run in a container on a user-defined network")
+	}

 	name := fmt.Sprintf("gitea-runner-e2e-k3s-%d", time.Now().UnixNano())
-	apiPort := network.MustParsePort("6443/tcp")
 	server := "https://" + name + ":6443"
-	hostConfig := &apicontainer.HostConfig{Privileged: true}
-	var netConfig *network.NetworkingConfig
-	if fixture.network != "" {
-		netConfig = &network.NetworkingConfig{EndpointsConfig: map[string]*network.EndpointSettings{fixture.network: {}}}
-	} else {
-		loopback := netip.MustParseAddr("127.0.0.1")
-		port, err := freeHostPort(loopback.String())
-		if err != nil {
-			t.Fatalf("find a free host port: %v", err)
-		}
-		server = "https://" + net.JoinHostPort(loopback.String(), strconv.Itoa(port))
-		hostConfig.PortBindings = network.PortMap{apiPort: []network.PortBinding{{HostIP: loopback, HostPort: strconv.Itoa(port)}}}
-	}

 	created, err := cli.ContainerCreate(ctx, mobyclient.ContainerCreateOptions{
 		Config: &apicontainer.Config{
-			Image:        os.Getenv("E2E_K3S_IMAGE"),
-			Cmd:          []string{"server", "--disable=traefik,metrics-server,servicelb,local-storage,coredns", "--tls-san=" + name, "--kubelet-arg=fail-cgroupv1=false"},
-			ExposedPorts: network.PortSet{apiPort: struct{}{}},
+			Image: os.Getenv("E2E_K3S_IMAGE"),
+			Cmd:   []string{"server", "--disable=traefik,metrics-server,servicelb,local-storage,coredns", "--tls-san=" + name, "--kubelet-arg=fail-cgroupv1=false"},
 		},
-		HostConfig:       hostConfig,
-		NetworkingConfig: netConfig,
+		HostConfig:       &apicontainer.HostConfig{Privileged: true},
+		NetworkingConfig: &network.NetworkingConfig{EndpointsConfig: map[string]*network.EndpointSettings{sharedNet: {}}},
 		Name:             name,
 	})
 	if err != nil {
-		t.Fatalf("create k3s container: %v", err)
+		return nil, fmt.Errorf("create k3s container: %w", err)
 	}
-	t.Cleanup(func() {
+	clusterCleanup = append(clusterCleanup, func() {
 		_, _ = cli.ContainerRemove(context.Background(), created.ID, mobyclient.ContainerRemoveOptions{Force: true, RemoveVolumes: true})
 	})
 	if _, err := cli.ContainerStart(ctx, created.ID, mobyclient.ContainerStartOptions{}); err != nil {
-		t.Fatalf("start k3s container: %v", err)
+		return nil, fmt.Errorf("start k3s container: %w", err)
 	}

-	containerdCtx, stopWaiting := context.WithTimeout(ctx, 2*time.Minute)
-	defer stopWaiting()
-	if err := waitForContainerLog(containerdCtx, created.ID, "containerd is now running"); err != nil {
-		t.Fatalf("wait for k3s containerd: %v", err)
+	if err := waitForContainerLog(ctx, created.ID, "containerd is now running"); err != nil {
+		return nil, fmt.Errorf("wait for k3s containerd: %w", err)
+	}
+	sandboxPulled := make(chan struct{})
+	go func() { // first pods would otherwise pull the sandbox image themselves
+		defer close(sandboxPulled)
+		pullCtx, cancel := context.WithTimeout(ctx, 30*time.Second)
+		defer cancel()
+		_, _ = containerExec(pullCtx, cli, created.ID, "", nil, "sh", "-c",
+			`ctr --namespace=k8s.io images pull "docker.io/$(sed -n 's/^ *sandbox = "\(.*\)"$/\1/p' /var/lib/rancher/k3s/agent/etc/containerd/config.toml)"`)
+	}()
+	images, err := preloadImages(ctx, created.ID, os.Getenv("E2E_JOB_IMAGE"), os.Getenv("SERVICE_IMAGE"))
+	<-sandboxPulled
+	if err != nil {
+		return nil, err
 	}
-	images := preloadImages(t, created.ID, os.Getenv("E2E_JOB_IMAGE"), os.Getenv("SERVICE_IMAGE"))

-	waitCtx, cancel := context.WithTimeout(ctx, 2*time.Minute)
-	defer cancel()
 	for {
-		_, err := containerExec(waitCtx, cli, created.ID, "", nil, "kubectl", "get", "serviceaccount", "default")
+		_, err := containerExec(ctx, cli, created.ID, "", nil, "kubectl", "get", "serviceaccount", "default")
 		if err == nil {
 			break
 		}
 		inspected, inspectErr := cli.ContainerInspect(ctx, created.ID, mobyclient.ContainerInspectOptions{})
-		if inspectErr != nil || !inspected.Container.State.Running || waitCtx.Err() != nil {
+		if inspectErr != nil || !inspected.Container.State.Running || ctx.Err() != nil {
 			var logs bytes.Buffer
-			if reader, logsErr := cli.ContainerLogs(ctx, created.ID, mobyclient.ContainerLogsOptions{ShowStdout: true, ShowStderr: true, Tail: "30"}); logsErr == nil {
+			if reader, logsErr := cli.ContainerLogs(context.Background(), created.ID, mobyclient.ContainerLogsOptions{ShowStdout: true, ShowStderr: true, Tail: "30"}); logsErr == nil {
 				_, _ = stdcopy.StdCopy(&logs, &logs, reader)
 				_ = reader.Close()
 			}
-			t.Fatalf("k3s did not start: %v, state %+v %v, last log lines:\n%s", err, inspected.Container.State, inspectErr, logs.String())
+			return nil, fmt.Errorf("k3s did not start: %w, state %+v %v, last log lines:\n%s", err, inspected.Container.State, inspectErr, logs.String())
 		}
 		select {
-		case <-waitCtx.Done():
+		case <-ctx.Done():
 		case <-time.After(pollInterval):
 		}
 	}

 	kubeconfig, err := containerExec(ctx, cli, created.ID, "", nil, "cat", "/etc/rancher/k3s/k3s.yaml")
 	if err != nil {
-		t.Fatalf("read kubeconfig: %v", err)
+		return nil, fmt.Errorf("read kubeconfig: %w", err)
 	}
-	path := filepath.Join(t.TempDir(), "kubeconfig")
+	path := filepath.Join(binDir, "kubeconfig")
 	if err := os.WriteFile(path, []byte(strings.ReplaceAll(kubeconfig, "https://127.0.0.1:6443", server)), 0o600); err != nil {
-		t.Fatalf("write kubeconfig: %v", err)
+		return nil, fmt.Errorf("write kubeconfig: %w", err)
 	}
-	kubernetes := &config.Kubernetes{Kubeconfig: path, Namespace: "default"}
-
-	if fixture.network != "" {
-		gitea, err := cli.ContainerInspect(ctx, fixture.id, mobyclient.ContainerInspectOptions{})
-		if err != nil {
-			t.Fatalf("inspect gitea container: %v", err)
-		}
-		giteaURL, err := url.Parse(fixture.baseURL)
-		if err != nil {
-			t.Fatalf("parse gitea url: %v", err)
-		}
-		kubernetes.PodTemplate = map[string]any{"spec": map[string]any{"hostAliases": []any{map[string]any{
-			"ip":        gitea.Container.NetworkSettings.Networks[fixture.network].IPAddress.String(),
-			"hostnames": []any{giteaURL.Hostname()},
-		}}}}
-	}
-	return created.ID, kubernetes, images
+	return &kubeCluster{id: created.ID, kubernetes: &config.Kubernetes{Kubeconfig: path, Namespace: "default"}, jobImage: images[0], serviceImage: images[1]}, nil
 }

 func waitForContainerLog(ctx context.Context, id, message string) error {
-	logs, err := fixture.cli.ContainerLogs(ctx, id, mobyclient.ContainerLogsOptions{ShowStdout: true, ShowStderr: true, Follow: true})
+	logs, err := dockerCLI.ContainerLogs(ctx, id, mobyclient.ContainerLogsOptions{ShowStdout: true, ShowStderr: true, Follow: true})
 	if err != nil {
 		return err
 	}
@@ -175,25 +219,23 @@ func waitForContainerLog(ctx context.Context, id, message string) error {
 	return fmt.Errorf("container exited before logging %q", message)
 }

-func preloadImages(t *testing.T, k3sID string, images ...string) []string {
-	t.Helper()
-	ctx := t.Context()
+func preloadImages(ctx context.Context, k3sID string, images ...string) ([]string, error) {
 	var aliases []string
 	for index, image := range images {
 		alias := fmt.Sprintf("gitea-runner-e2e/preload-%d:%d", index, time.Now().UnixNano())
-		if _, err := fixture.cli.ImageTag(ctx, mobyclient.ImageTagOptions{Source: image, Target: alias}); err != nil {
-			t.Fatalf("tag %s: %v", image, err)
+		if _, err := dockerCLI.ImageTag(ctx, mobyclient.ImageTagOptions{Source: image, Target: alias}); err != nil {
+			return nil, fmt.Errorf("tag %s: %w", image, err)
 		}
-		t.Cleanup(func() { _, _ = fixture.cli.ImageRemove(context.Background(), alias, mobyclient.ImageRemoveOptions{}) })
+		clusterCleanup = append(clusterCleanup, func() { _, _ = dockerCLI.ImageRemove(context.Background(), alias, mobyclient.ImageRemoveOptions{}) })
 		aliases = append(aliases, alias)
 	}
-	archive, err := fixture.cli.ImageSave(ctx, aliases)
+	archive, err := dockerCLI.ImageSave(ctx, aliases)
 	if err != nil {
-		t.Fatalf("save %v: %v", images, err)
+		return nil, fmt.Errorf("save %v: %w", images, err)
 	}
 	defer archive.Close()
-	if out, err := containerExec(ctx, fixture.cli, k3sID, "", archive, "ctr", "--namespace=k8s.io", "images", "import", "-"); err != nil {
-		t.Fatalf("import %v into k3s: %v: %s", images, err, out)
+	if out, err := containerExec(ctx, dockerCLI, k3sID, "", archive, "ctr", "--namespace=k8s.io", "images", "import", "-"); err != nil {
+		return nil, fmt.Errorf("import %v into k3s: %w: %s", images, err, out)
 	}
-	return aliases
+	return aliases, nil
 }