feat: add Kubernetes backend (#1260)

Adds a Kubernetes backend. A `name:kubernetes://image` label runs the job as one pod through the Kubernetes API, with services as containers of the same pod. Step env is exported by a script sent over exec stdin, keeping it out of the pod spec and process arguments, and service env goes into a per-job Secret. Docker actions and container options are rejected. See `docs/kubernetes.md`.

Operators shape job pods with `kubernetes.pod_template`, with per-label `pod_templates` merged in `runs-on` order and a `$services` entry for service containers. The idle cleanup removes pods and Secrets a runner left behind.

The client calls the Kubernetes API directly and runs step commands over `coder/websocket` (same dependency used in gitea) instead of pulling in the heavyweight `kubernetes/client-go`. Includes an e2e test against k3s.

Replaces: https://gitea.com/gitea/runner/pulls/1000
Fixes: https://gitea.com/gitea/runner/issues/31

Assisted-by: Claude Code:claude-opus-5-5
Co-authored-by: Zettat123 <[email protected]>
Co-authored-by: bircni <[email protected]>
Reviewed-on: https://gitea.com/gitea/runner/pulls/1260
Reviewed-by: Lunny Xiao <[email protected]>
Reviewed-by: Zettat123 <[email protected]>
Co-authored-by: silverwind <[email protected]>

internal/app/cmd/exec.go

@@ -618,7 +618,7 @@ func loadExecCmd(ctx context.Context) *cobra.Command {
 	execCmd.PersistentFlags().BoolVarP(&execArg.noSkipCheckout, "no-skip-checkout", "", false, "Do not skip actions/checkout")
 	execCmd.PersistentFlags().BoolVarP(&execArg.debug, "debug", "d", false, "enable debug log")
 	execCmd.PersistentFlags().BoolVarP(&execArg.dryrun, "dryrun", "n", false, "dryrun mode")
-	execCmd.PersistentFlags().StringVarP(&execArg.image, "image", "i", config.DefaultImage, "Docker image to use. Use \"-self-hosted\" to run directly on the host.")
+	execCmd.PersistentFlags().StringVarP(&execArg.image, "image", "i", config.DefaultImage, "Docker image to use. Use \"kubernetes://<image>\" to run it as a pod, \"-self-hosted\" to run directly on the host.")
 	execCmd.PersistentFlags().StringVarP(&execArg.toolCacheMode, "tool-cache-mode", "", config.ToolCacheModeNone, "What to mount at RUNNER_TOOL_CACHE: none, or shared to reuse one tool cache across runs")
 	execCmd.PersistentFlags().StringVarP(&execArg.network, "network", "", "", "Specify the network to which the container will connect")
 	execCmd.PersistentFlags().StringVarP(&execArg.githubInstance, "gitea-instance", "", "", "Gitea instance to use.")

internal/app/run/runner.go

@@ -153,6 +153,8 @@ var removeOrphanNetworks = container.RemoveOrphanNetworks
 
 var removeOrphanJobVolumes = container.RemoveOrphanJobVolumes
 
+var removeOrphanKubernetesResources = container.RemoveOrphanKubernetesResources
+
 // OnIdle performs lightweight maintenance during polling idle windows.
 // It runs synchronously on the poller goroutine; shouldRunIdleCleanup
 // throttles invocations to runner.idle_cleanup_interval so the impact on
@@ -175,6 +177,13 @@ func (r *Runner) OnIdle(ctx context.Context) {
 		r.cleanupStaleDirs(ctx, hostRoot, isHostScratchDir)
 	}
 	r.cleanupOrphanDockerResources(ctx)
+	if r.uuid != "" && r.usesKubernetes() {
+		ctx, cancel := context.WithTimeout(ctx, time.Minute)
+		defer cancel()
+		if err := removeOrphanKubernetesResources(ctx, r.kubernetesOptions(nil), r.now().Add(-r.cfg.Runner.WorkdirCleanupAge)); err != nil {
+			log.Warnf("failed to clean up job pods left behind by earlier jobs: %v", err)
+		}
+	}
 }
 
 func (r *Runner) cleanupOrphanDockerResources(ctx context.Context) {
@@ -402,10 +411,36 @@ func (r *Runner) requiresDocker() bool {
 	return r.labels.RequireDocker() || r.cfg.Container.RequireDocker
 }
 
+func (r *Runner) usesKubernetes() bool {
+	return slices.ContainsFunc(r.labels, func(label *labels.Label) bool { return label.Schema == labels.SchemeKubernetes })
+}
+
+// kubernetesOptions layers the pod templates of the runner labels a job runs on, in runs-on order, over the shared one.
+func (r *Runner) kubernetesOptions(runsOn []string) container.KubernetesOptions {
+	options := container.KubernetesOptions{
+		Kubeconfig:   r.cfg.Kubernetes.Kubeconfig,
+		Namespace:    r.cfg.Kubernetes.Namespace,
+		PodTemplates: []map[string]any{r.cfg.Kubernetes.PodTemplate},
+		RunnerUUID:   r.uuid,
+	}
+	names := r.labels.Names()
+	for index, name := range runsOn {
+		if template, ok := r.cfg.Kubernetes.PodTemplates[name]; ok && slices.Contains(names, name) && !slices.Contains(runsOn[:index], name) {
+			options.PodTemplates = append(options.PodTemplates, template)
+		}
+	}
+	return options
+}
+
 // fallbackPlatform is where a job runs whose runs-on matches no label, as any job without a
 // runs-on does, since Gitea sends those to every runner.
 func (r *Runner) fallbackPlatform(ctx context.Context) string {
-	if r.requiresDocker() || dockerReachable(ctx) {
+	switch {
+	case r.requiresDocker():
+		return r.cfg.Runner.DefaultImage
+	case r.usesKubernetes():
+		return labels.SchemeKubernetes + "://" + r.cfg.Runner.DefaultImage
+	case dockerReachable(ctx):
 		return r.cfg.Runner.DefaultImage
 	}
 	return labels.SelfHostedPlatform
@@ -449,7 +484,8 @@ func (r *Runner) run(ctx context.Context, task *runnerv1.Task, reporter *report.
 		}
 		return fallbackPlatform()
 	}
-	cacheURL, cacheContainer := r.cacheForJob(job, platformPicker)
+	platform := platformPicker(job.RunsOn())
+	cacheURL, cacheContainer := r.cacheForJob(job, platform)
 	if cacheURL != "" {
 		envs["ACTIONS_CACHE_URL"] = cacheURL + "/"
 	}
@@ -458,7 +494,13 @@ func (r *Runner) run(ctx context.Context, task *runnerv1.Task, reporter *report.
 	// act reaches them by their workflow key.
 	var services map[string]yaml.Node
 	_ = job.RawServices.Decode(&services) // a whole-value expression names no service before evaluation
-	proxyEnv := JobProxyEnv(envs, cacheURL, slices.Sorted(maps.Keys(services)))
+	serviceNames := slices.Sorted(maps.Keys(services))
+	if strings.HasPrefix(platform, labels.SchemeKubernetes+"://") {
+		for index, name := range serviceNames {
+			serviceNames[index] = container.KubernetesServiceName(name)
+		}
+	}
+	proxyEnv := JobProxyEnv(envs, cacheURL, serviceNames)
 	maps.Copy(envs, proxyEnv)
 
 	if r.capabilities != "" {
@@ -585,6 +627,7 @@ func (r *Runner) run(ctx context.Context, task *runnerv1.Task, reporter *report.
 		SharedToolCache:                   r.cfg.Runner.ToolCacheMode == config.ToolCacheModeShared,
 		InsecureSkipTLS:                   r.cfg.Runner.Insecure,
 		RunnerName:                        r.name,
+		KubernetesPicker:                  r.kubernetesOptions,
 	}
 
 	rr, err := runner.New(runnerConfig)
@@ -867,10 +910,10 @@ func warnIgnoredCacheSecret(cfg *config.Config) {
 }
 
 // cacheForJob returns the cache address for the job, and the container to attach to its network to reach it there.
-func (r *Runner) cacheForJob(job *model.Job, pickPlatform func([]string) string) (string, string) {
+func (r *Runner) cacheForJob(job *model.Job, platform string) (string, string) {
 	jobContainer := job.Container()
 	cacheContainer := ""
-	if jobContainer != nil && jobContainer.Image != "" || pickPlatform(job.RunsOn()) != labels.SelfHostedPlatform {
+	if !strings.HasPrefix(platform, labels.SchemeKubernetes+"://") && (jobContainer != nil && jobContainer.Image != "" || platform != labels.SelfHostedPlatform) {
 		cacheContainer = r.isolatedCacheContainer()
 	}
 	if cacheContainer == "" {

internal/app/run/runner_idle_cleanup_test.go

@@ -11,7 +11,9 @@ import (
 	"testing"
 	"time"
 
+	"gitea.com/gitea/runner/act/container"
 	"gitea.com/gitea/runner/internal/pkg/config"
+	"gitea.com/gitea/runner/internal/pkg/labels"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -329,8 +331,20 @@ func TestRunnerOnIdleRemovesOrphanNetworks(t *testing.T) {
 		return nil
 	}
 	t.Cleanup(func() { removeOrphanJobVolumes = origRemoveOrphanJobVolumes })
+	var sweptPods []string
+	origRemoveOrphanKubernetesResources := removeOrphanKubernetesResources
+	removeOrphanKubernetesResources = func(ctx context.Context, options container.KubernetesOptions, createdBefore time.Time) error {
+		_, hasDeadline := ctx.Deadline()
+		assert.True(t, hasDeadline)
+		sweptPods = append(sweptPods, options.RunnerUUID)
+		assert.Equal(t, now.Add(-24*time.Hour), createdBefore)
+		return nil
+	}
+	t.Cleanup(func() { removeOrphanKubernetesResources = origRemoveOrphanKubernetesResources })
+	kubernetesLabel, err := labels.Parse("k8s:kubernetes://node")
+	require.NoError(t, err)
 
-	r := &Runner{uuid: "runner-1", cfg: cfg, now: func() time.Time { return now }}
+	r := &Runner{uuid: "runner-1", cfg: cfg, labels: labels.Labels{kubernetesLabel}, now: func() time.Time { return now }}
 	r.isolatedCacheContainer = func() string { return "a1b2c3d4e5f6" }
 	r.OnIdle(context.Background())
 	assert.Equal(t, []string{"runner-1"}, swept)
@@ -353,4 +367,5 @@ func TestRunnerOnIdleRemovesOrphanNetworks(t *testing.T) {
 	hostOnly.OnIdle(context.Background())
 	assert.Equal(t, []string{"runner-1", "runner-2"}, sweptVolumes)
 	assert.Equal(t, []string{"a1b2c3d4e5f6", ""}, detached)
+	assert.Equal(t, []string{"runner-1"}, sweptPods)
 }

internal/app/run/runner_test.go

@@ -245,6 +245,16 @@ func TestRunnerFallbackPlatform(t *testing.T) {
 	}
 }
 
+func TestRunnerKubernetesOptionsLayerTheLabelPodTemplatesInRunsOnOrder(t *testing.T) {
+	cfg := &config.Config{Kubernetes: config.Kubernetes{
+		PodTemplate:  map[string]any{"base": true},
+		PodTemplates: map[string]map[string]any{"gpu": {"gpu": true}, "large": {"large": true}, "other": {"other": true}},
+	}}
+	r := &Runner{cfg: cfg, labels: labels.Labels{{Name: "linux"}, {Name: "gpu"}, {Name: "large"}}}
+
+	require.Equal(t, []map[string]any{{"base": true}, {"large": true}, {"gpu": true}}, r.kubernetesOptions([]string{"linux", "large", "other", "gpu", "large"}).PodTemplates)
+}
+
 // Proxy variables are assembled per task, because a job's service containers have to be
 // reached directly and they are only known once the workflow is parsed.
 func TestNewRunnerLeavesProxyToTheTask(t *testing.T) {
@@ -350,7 +360,7 @@ func TestNewRunnerCacheServiceV2(t *testing.T) {
 	r.isolatedCacheContainer = func() string { return "a1b2c3d4e5f6" }
 	pickPlatform := func(runsOn []string) string { return map[string]string{"native": labels.SelfHostedPlatform}[runsOn[0]] }
 	for job, isolated := range map[string]bool{"native": false, "linux": true, "containerized": true, "empty": false} {
-		cacheURL, cacheContainer := r.cacheForJob(workflow.GetJob(job), pickPlatform)
+		cacheURL, cacheContainer := r.cacheForJob(workflow.GetJob(job), pickPlatform(workflow.GetJob(job).RunsOn()))
 		if isolated {
 			assert.Equal(t, "a1b2c3d4e5f6", cacheContainer, job)
 			assert.Equal(t, strings.Replace(r.cacheHandler.ExternalURL(), "127.0.0.1", cacheContainer, 1), cacheURL, job)