feat: add Kubernetes backend (#1260)

Adds a Kubernetes backend. A `name:kubernetes://image` label runs the job as one pod through the Kubernetes API, with services as containers of the same pod. Step env is exported by a script sent over exec stdin, keeping it out of the pod spec and process arguments, and service env goes into a per-job Secret. Docker actions and container options are rejected. See `docs/kubernetes.md`.

Operators shape job pods with `kubernetes.pod_template`, with per-label `pod_templates` merged in `runs-on` order and a `$services` entry for service containers. The idle cleanup removes pods and Secrets a runner left behind.

The client calls the Kubernetes API directly and runs step commands over `coder/websocket` (same dependency used in gitea) instead of pulling in the heavyweight `kubernetes/client-go`. Includes an e2e test against k3s.

Replaces: https://gitea.com/gitea/runner/pulls/1000
Fixes: https://gitea.com/gitea/runner/issues/31

Assisted-by: Claude Code:claude-opus-5-5
Co-authored-by: Zettat123 <[email protected]>
Co-authored-by: bircni <[email protected]>
Reviewed-on: https://gitea.com/gitea/runner/pulls/1260
Reviewed-by: Lunny Xiao <[email protected]>
Reviewed-by: Zettat123 <[email protected]>
Co-authored-by: silverwind <[email protected]>

internal/app/cmd/exec.go

@@ -618,7 +618,7 @@ func loadExecCmd(ctx context.Context) *cobra.Command {
 	execCmd.PersistentFlags().BoolVarP(&execArg.noSkipCheckout, "no-skip-checkout", "", false, "Do not skip actions/checkout")
 	execCmd.PersistentFlags().BoolVarP(&execArg.debug, "debug", "d", false, "enable debug log")
 	execCmd.PersistentFlags().BoolVarP(&execArg.dryrun, "dryrun", "n", false, "dryrun mode")
-	execCmd.PersistentFlags().StringVarP(&execArg.image, "image", "i", config.DefaultImage, "Docker image to use. Use \"-self-hosted\" to run directly on the host.")
+	execCmd.PersistentFlags().StringVarP(&execArg.image, "image", "i", config.DefaultImage, "Docker image to use. Use \"kubernetes://<image>\" to run it as a pod, \"-self-hosted\" to run directly on the host.")
 	execCmd.PersistentFlags().StringVarP(&execArg.toolCacheMode, "tool-cache-mode", "", config.ToolCacheModeNone, "What to mount at RUNNER_TOOL_CACHE: none, or shared to reuse one tool cache across runs")
 	execCmd.PersistentFlags().StringVarP(&execArg.network, "network", "", "", "Specify the network to which the container will connect")
 	execCmd.PersistentFlags().StringVarP(&execArg.githubInstance, "gitea-instance", "", "", "Gitea instance to use.")

internal/app/run/runner.go

@@ -153,6 +153,8 @@ var removeOrphanNetworks = container.RemoveOrphanNetworks
 
 var removeOrphanJobVolumes = container.RemoveOrphanJobVolumes
 
+var removeOrphanKubernetesResources = container.RemoveOrphanKubernetesResources
+
 // OnIdle performs lightweight maintenance during polling idle windows.
 // It runs synchronously on the poller goroutine; shouldRunIdleCleanup
 // throttles invocations to runner.idle_cleanup_interval so the impact on
@@ -175,6 +177,13 @@ func (r *Runner) OnIdle(ctx context.Context) {
 		r.cleanupStaleDirs(ctx, hostRoot, isHostScratchDir)
 	}
 	r.cleanupOrphanDockerResources(ctx)
+	if r.uuid != "" && r.usesKubernetes() {
+		ctx, cancel := context.WithTimeout(ctx, time.Minute)
+		defer cancel()
+		if err := removeOrphanKubernetesResources(ctx, r.kubernetesOptions(nil), r.now().Add(-r.cfg.Runner.WorkdirCleanupAge)); err != nil {
+			log.Warnf("failed to clean up job pods left behind by earlier jobs: %v", err)
+		}
+	}
 }
 
 func (r *Runner) cleanupOrphanDockerResources(ctx context.Context) {
@@ -402,10 +411,36 @@ func (r *Runner) requiresDocker() bool {
 	return r.labels.RequireDocker() || r.cfg.Container.RequireDocker
 }
 
+func (r *Runner) usesKubernetes() bool {
+	return slices.ContainsFunc(r.labels, func(label *labels.Label) bool { return label.Schema == labels.SchemeKubernetes })
+}
+
+// kubernetesOptions layers the pod templates of the runner labels a job runs on, in runs-on order, over the shared one.
+func (r *Runner) kubernetesOptions(runsOn []string) container.KubernetesOptions {
+	options := container.KubernetesOptions{
+		Kubeconfig:   r.cfg.Kubernetes.Kubeconfig,
+		Namespace:    r.cfg.Kubernetes.Namespace,
+		PodTemplates: []map[string]any{r.cfg.Kubernetes.PodTemplate},
+		RunnerUUID:   r.uuid,
+	}
+	names := r.labels.Names()
+	for index, name := range runsOn {
+		if template, ok := r.cfg.Kubernetes.PodTemplates[name]; ok && slices.Contains(names, name) && !slices.Contains(runsOn[:index], name) {
+			options.PodTemplates = append(options.PodTemplates, template)
+		}
+	}
+	return options
+}
+
 // fallbackPlatform is where a job runs whose runs-on matches no label, as any job without a
 // runs-on does, since Gitea sends those to every runner.
 func (r *Runner) fallbackPlatform(ctx context.Context) string {
-	if r.requiresDocker() || dockerReachable(ctx) {
+	switch {
+	case r.requiresDocker():
+		return r.cfg.Runner.DefaultImage
+	case r.usesKubernetes():
+		return labels.SchemeKubernetes + "://" + r.cfg.Runner.DefaultImage
+	case dockerReachable(ctx):
 		return r.cfg.Runner.DefaultImage
 	}
 	return labels.SelfHostedPlatform
@@ -449,7 +484,8 @@ func (r *Runner) run(ctx context.Context, task *runnerv1.Task, reporter *report.
 		}
 		return fallbackPlatform()
 	}
-	cacheURL, cacheContainer := r.cacheForJob(job, platformPicker)
+	platform := platformPicker(job.RunsOn())
+	cacheURL, cacheContainer := r.cacheForJob(job, platform)
 	if cacheURL != "" {
 		envs["ACTIONS_CACHE_URL"] = cacheURL + "/"
 	}
@@ -458,7 +494,13 @@ func (r *Runner) run(ctx context.Context, task *runnerv1.Task, reporter *report.
 	// act reaches them by their workflow key.
 	var services map[string]yaml.Node
 	_ = job.RawServices.Decode(&services) // a whole-value expression names no service before evaluation
-	proxyEnv := JobProxyEnv(envs, cacheURL, slices.Sorted(maps.Keys(services)))
+	serviceNames := slices.Sorted(maps.Keys(services))
+	if strings.HasPrefix(platform, labels.SchemeKubernetes+"://") {
+		for index, name := range serviceNames {
+			serviceNames[index] = container.KubernetesServiceName(name)
+		}
+	}
+	proxyEnv := JobProxyEnv(envs, cacheURL, serviceNames)
 	maps.Copy(envs, proxyEnv)
 
 	if r.capabilities != "" {
@@ -585,6 +627,7 @@ func (r *Runner) run(ctx context.Context, task *runnerv1.Task, reporter *report.
 		SharedToolCache:                   r.cfg.Runner.ToolCacheMode == config.ToolCacheModeShared,
 		InsecureSkipTLS:                   r.cfg.Runner.Insecure,
 		RunnerName:                        r.name,
+		KubernetesPicker:                  r.kubernetesOptions,
 	}
 
 	rr, err := runner.New(runnerConfig)
@@ -867,10 +910,10 @@ func warnIgnoredCacheSecret(cfg *config.Config) {
 }
 
 // cacheForJob returns the cache address for the job, and the container to attach to its network to reach it there.
-func (r *Runner) cacheForJob(job *model.Job, pickPlatform func([]string) string) (string, string) {
+func (r *Runner) cacheForJob(job *model.Job, platform string) (string, string) {
 	jobContainer := job.Container()
 	cacheContainer := ""
-	if jobContainer != nil && jobContainer.Image != "" || pickPlatform(job.RunsOn()) != labels.SelfHostedPlatform {
+	if !strings.HasPrefix(platform, labels.SchemeKubernetes+"://") && (jobContainer != nil && jobContainer.Image != "" || platform != labels.SelfHostedPlatform) {
 		cacheContainer = r.isolatedCacheContainer()
 	}
 	if cacheContainer == "" {

internal/app/run/runner_idle_cleanup_test.go

@@ -11,7 +11,9 @@ import (
 	"testing"
 	"time"
 
+	"gitea.com/gitea/runner/act/container"
 	"gitea.com/gitea/runner/internal/pkg/config"
+	"gitea.com/gitea/runner/internal/pkg/labels"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -329,8 +331,20 @@ func TestRunnerOnIdleRemovesOrphanNetworks(t *testing.T) {
 		return nil
 	}
 	t.Cleanup(func() { removeOrphanJobVolumes = origRemoveOrphanJobVolumes })
+	var sweptPods []string
+	origRemoveOrphanKubernetesResources := removeOrphanKubernetesResources
+	removeOrphanKubernetesResources = func(ctx context.Context, options container.KubernetesOptions, createdBefore time.Time) error {
+		_, hasDeadline := ctx.Deadline()
+		assert.True(t, hasDeadline)
+		sweptPods = append(sweptPods, options.RunnerUUID)
+		assert.Equal(t, now.Add(-24*time.Hour), createdBefore)
+		return nil
+	}
+	t.Cleanup(func() { removeOrphanKubernetesResources = origRemoveOrphanKubernetesResources })
+	kubernetesLabel, err := labels.Parse("k8s:kubernetes://node")
+	require.NoError(t, err)
 
-	r := &Runner{uuid: "runner-1", cfg: cfg, now: func() time.Time { return now }}
+	r := &Runner{uuid: "runner-1", cfg: cfg, labels: labels.Labels{kubernetesLabel}, now: func() time.Time { return now }}
 	r.isolatedCacheContainer = func() string { return "a1b2c3d4e5f6" }
 	r.OnIdle(context.Background())
 	assert.Equal(t, []string{"runner-1"}, swept)
@@ -353,4 +367,5 @@ func TestRunnerOnIdleRemovesOrphanNetworks(t *testing.T) {
 	hostOnly.OnIdle(context.Background())
 	assert.Equal(t, []string{"runner-1", "runner-2"}, sweptVolumes)
 	assert.Equal(t, []string{"a1b2c3d4e5f6", ""}, detached)
+	assert.Equal(t, []string{"runner-1"}, sweptPods)
 }

internal/app/run/runner_test.go

@@ -245,6 +245,16 @@ func TestRunnerFallbackPlatform(t *testing.T) {
 	}
 }
 
+func TestRunnerKubernetesOptionsLayerTheLabelPodTemplatesInRunsOnOrder(t *testing.T) {
+	cfg := &config.Config{Kubernetes: config.Kubernetes{
+		PodTemplate:  map[string]any{"base": true},
+		PodTemplates: map[string]map[string]any{"gpu": {"gpu": true}, "large": {"large": true}, "other": {"other": true}},
+	}}
+	r := &Runner{cfg: cfg, labels: labels.Labels{{Name: "linux"}, {Name: "gpu"}, {Name: "large"}}}
+
+	require.Equal(t, []map[string]any{{"base": true}, {"large": true}, {"gpu": true}}, r.kubernetesOptions([]string{"linux", "large", "other", "gpu", "large"}).PodTemplates)
+}
+
 // Proxy variables are assembled per task, because a job's service containers have to be
 // reached directly and they are only known once the workflow is parsed.
 func TestNewRunnerLeavesProxyToTheTask(t *testing.T) {
@@ -350,7 +360,7 @@ func TestNewRunnerCacheServiceV2(t *testing.T) {
 	r.isolatedCacheContainer = func() string { return "a1b2c3d4e5f6" }
 	pickPlatform := func(runsOn []string) string { return map[string]string{"native": labels.SelfHostedPlatform}[runsOn[0]] }
 	for job, isolated := range map[string]bool{"native": false, "linux": true, "containerized": true, "empty": false} {
-		cacheURL, cacheContainer := r.cacheForJob(workflow.GetJob(job), pickPlatform)
+		cacheURL, cacheContainer := r.cacheForJob(workflow.GetJob(job), pickPlatform(workflow.GetJob(job).RunsOn()))
 		if isolated {
 			assert.Equal(t, "a1b2c3d4e5f6", cacheContainer, job)
 			assert.Equal(t, strings.Replace(r.cacheHandler.ExternalURL(), "127.0.0.1", cacheContainer, 1), cacheURL, job)

internal/pkg/config/config.example.yaml

@@ -53,12 +53,13 @@ runner:
   # scratch directories (left behind when a host cleanup delete stalls) older than
   # this duration. Setting either workdir_cleanup_age or idle_cleanup_interval to 0
   # (or any non-positive value) disables stale-directory cleanup entirely, along with
-  # the docker network and volume cleanup below.
+  # the docker and Kubernetes cleanup below.
   #workdir_cleanup_age: 24h
   # Cadence for the idle cleanup pass. Besides the directories above, on runners that use
   # docker it removes the per-job networks and volumes of jobs this runner did not live to
   # tear down, which would otherwise hold a subnet of the daemon address pool and the job's
-  # workspace contents until the host is rebuilt.
+  # workspace contents until the host is rebuilt. On runners with kubernetes labels it
+  # removes such jobs' pods and Secrets.
   #idle_cleanup_interval: 10m
   # The base interval for periodic log flush to the Gitea instance.
   # Logs may be sent earlier if the buffer reaches log_report_batch_size
@@ -106,8 +107,7 @@ runner:
   # terminal; tools like `docker build` emit redrawing progress frames into the captured log
   # when a TTY is present.
   #allocate_pty: false
-  # Image for a job whose runs-on matches none of the labels above. A runner without a
-  # docker daemon runs such a job on the host instead.
+  # Image for a job whose runs-on matches none of the labels above, see the Labels section of the README for where it runs.
   #default_image: "docker.gitea.com/runner-images:ubuntu-latest"
   # What to mount at RUNNER_TOOL_CACHE (/opt/hostedtoolcache), where setup actions install tools:
   #   none:   nothing. A docker job sees what its image ships there, a host job an empty dir, and
@@ -280,6 +280,14 @@ host:
   # If it's empty, $HOME/.cache/act/ will be used.
   #workdir_parent:
 
+# Jobs that run as Kubernetes pods, see docs/kubernetes.md.
+kubernetes:
+  #kubeconfig: ""
+  #namespace: ""
+  #pod_template: {}
+  # Pod templates merged over pod_template, in runs-on order, for jobs whose runs-on has the label they are keyed by.
+  #pod_templates: {}
+
 # Optional local task-admission checks. Disabled by default. When enabled, low
 # disk space or a failing script pauses new task fetching; existing jobs continue.
 # No health checks run while any job is active; the last result is reused until idle.

internal/pkg/config/config.go

@@ -63,8 +63,8 @@ type Runner struct {
 	FetchTimeout          time.Duration     `yaml:"fetch_timeout"`            // FetchTimeout specifies the timeout duration for fetching resources.
 	FetchInterval         time.Duration     `yaml:"fetch_interval"`           // FetchInterval specifies the interval duration for fetching resources.
 	FetchIntervalMax      time.Duration     `yaml:"fetch_interval_max"`       // FetchIntervalMax specifies the maximum backoff interval when idle.
-	WorkdirCleanupAge     time.Duration     `yaml:"workdir_cleanup_age"`      // WorkdirCleanupAge removes stale bind-workdir task directories, orphaned host-mode scratch dirs and orphaned docker job resources older than this duration during idle cleanup.
-	IdleCleanupInterval   time.Duration     `yaml:"idle_cleanup_interval"`    // IdleCleanupInterval runs the idle cleanup (stale directories and orphaned docker networks and volumes) periodically while the runner is idle. Set to 0 to disable cleanup cadence.
+	WorkdirCleanupAge     time.Duration     `yaml:"workdir_cleanup_age"`      // WorkdirCleanupAge removes stale bind-workdir task directories, orphaned host-mode scratch dirs and orphaned docker and Kubernetes job resources older than this duration during idle cleanup.
+	IdleCleanupInterval   time.Duration     `yaml:"idle_cleanup_interval"`    // IdleCleanupInterval runs the idle cleanup (stale directories, orphaned docker networks and volumes, orphaned job pods) periodically while the runner is idle. Set to 0 to disable cleanup cadence.
 	LogReportInterval     time.Duration     `yaml:"log_report_interval"`      // LogReportInterval specifies the base interval for periodic log flush.
 	LogReportMaxLatency   time.Duration     `yaml:"log_report_max_latency"`   // LogReportMaxLatency specifies the max time a log row can wait before being sent.
 	LogReportBatchSize    int               `yaml:"log_report_batch_size"`    // LogReportBatchSize triggers immediate log flush when buffer reaches this size.
@@ -194,6 +194,14 @@ type Host struct {
 	WorkdirParent string `yaml:"workdir_parent"` // WorkdirParent specifies the parent directory for the host's working directory.
 }
 
+// Kubernetes configures the pods jobs run in.
+type Kubernetes struct {
+	Kubeconfig   string                    `yaml:"kubeconfig"`    // Kubeconfig to use instead of the in-cluster service account.
+	Namespace    string                    `yaml:"namespace"`     // Namespace of job pods, empty uses the service account's, else the kubeconfig context's, else default.
+	PodTemplate  map[string]any            `yaml:"pod_template"`  // PodTemplate is a pod manifest the runner merges its own onto.
+	PodTemplates map[string]map[string]any `yaml:"pod_templates"` // PodTemplates are merged over PodTemplate, in runs-on order, for jobs whose runs-on has the label they are keyed by.
+}
+
 // Metrics represents the configuration for the Prometheus metrics endpoint.
 type Metrics struct {
 	Enabled        bool          `yaml:"enabled"`         // Enabled indicates whether the metrics endpoint is exposed.
@@ -220,6 +228,7 @@ type Config struct {
 	Host        Host        `yaml:"host"`         // Host represents the configuration for the host.
 	Metrics     Metrics     `yaml:"metrics"`      // Metrics represents the configuration for the Prometheus metrics endpoint.
 	HealthCheck HealthCheck `yaml:"health_check"` // HealthCheck controls opt-in local task-admission checks.
+	Kubernetes  Kubernetes  `yaml:"kubernetes"`
 }
 
 // LoadDefault returns the default configuration.

internal/pkg/labels/labels.go

@@ -9,8 +9,9 @@ import (
 )
 
 const (
-	SchemeHost   = "host"
-	SchemeDocker = "docker"
+	SchemeHost       = "host"
+	SchemeDocker     = "docker"
+	SchemeKubernetes = "kubernetes"
 
 	// SelfHostedPlatform is the platform marker act treats as "run on the host".
 	SelfHostedPlatform = "-self-hosted"
@@ -42,8 +43,9 @@ func Parse(str string) (*Label, error) {
 	if len(splits) >= 3 {
 		label.Arg = splits[2]
 	}
-	if label.Schema != SchemeHost && label.Schema != SchemeDocker {
-		// Not a schema we know: the colon belongs to the label name itself.
+	kubernetes := label.Schema == SchemeKubernetes && strings.HasPrefix(label.Arg, "//") && len(label.Arg) > 2
+	if label.Schema != SchemeHost && label.Schema != SchemeDocker && !kubernetes {
+		// Not a schema we know, or kubernetes without an image: the colon belongs to the label name itself.
 		return &Label{
 			Name:   str,
 			Schema: SchemeHost,
@@ -72,10 +74,11 @@ func (l Labels) PickPlatform(runsOn []string) string {
 		case SchemeDocker:
 			// "//" will be ignored
 			platforms[label.Name] = strings.TrimPrefix(label.Arg, "//")
+		case SchemeKubernetes:
+			platforms[label.Name] = SchemeKubernetes + ":" + label.Arg
 		case SchemeHost:
 			platforms[label.Name] = SelfHostedPlatform
 		default:
-			// unreachable: Parse only produces host or docker schemas
 			continue
 		}
 	}

internal/pkg/labels/labels_test.go

@@ -115,6 +115,7 @@ func TestPickPlatform(t *testing.T) {
 	ls := mustParse(t,
 		"ubuntu:docker://node:18",
 		"self-hosted:host",
+		"k8s:kubernetes://node:24",
 	)
 
 	tests := []struct {
@@ -123,6 +124,7 @@ func TestPickPlatform(t *testing.T) {
 		want   string
 	}{
 		{"docker strips leading slashes", []string{"ubuntu"}, "node:18"},
+		{"kubernetes keeps its scheme", []string{"k8s"}, "kubernetes://node:24"},
 		{"host maps to self-hosted marker", []string{"self-hosted"}, SelfHostedPlatform},
 		{"first match wins", []string{"self-hosted", "ubuntu"}, SelfHostedPlatform},
 		{"unknown label picks nothing", []string{"windows"}, ""},