enhance(kubernetes): fetch job pod copies from the runner over the pod network (#1354)

The Kubernetes backend copies actions and workspaces into job containers with `tar` over `pods/exec`, so every byte goes through the API server. On managed clusters that often sits in other zones than the nodes, so each byte can cross zones twice, and the copies load the API server.

With `kubernetes.transfer.mode: auto` or `direct`, an init container copies `gitea-runner` into the job pod, and the job container fetches directories and copies of 64 KiB or more gzipped from the runner over the pod network. `auto` falls back to the API server for a job pod that cannot reach the runner, `direct` fails the job.

Copies can hold secrets, so each is served once, over TLS 1.3 pinned to a certificate the runner generates, only to the job pod's IPs.

Copying setup-go@v5, cache@v4, setup-java@v4 and build-push-action@v6 sends 11.6 MB through the API server on main and 532 B with this PR.

Co-authored-by: silverwind <[email protected]>
Co-authored-by: silverwind <[email protected]>
Reviewed-on: https://gitea.com/gitea/runner/pulls/1354
Reviewed-by: silverwind <[email protected]>
Co-authored-by: Lunny Xiao <[email protected]>

internal/pkg/config/config.example.yaml

@@ -306,6 +306,18 @@ kubernetes:
   #pod_template: {}
   # Pod templates merged over pod_template, in runs-on order, for jobs whose runs-on has the label they are keyed by.
   #pod_templates: {}
+  # Have job pods fetch directories, such as actions, and copies of 64 KiB or more gzipped from the runner over the pod network,
+  # instead of the runner streaming them through the API server. An init container copies gitea-runner from image into each job pod.
+  # api: through the API server (default). auto: over the pod network, through the API server for a job pod that
+  # cannot reach the runner. direct: over the pod network only, so a broken setup fails jobs instead of quietly
+  # costing API server traffic again.
+  #transfer:
+  #  mode: api
+  #  # Address and port the runner listens on and job pods reach it at, the outbound IP and a free port when empty.
+  #  host: ""
+  #  port: 0
+  #  # Image with gitea-runner on its PATH, the runner pod's image when it has a single container and runs in namespace.
+  #  image: ""
 
 # Optional local task-admission checks. Disabled by default. When enabled, low
 # disk space or a failing script pauses new task fetching; existing jobs continue.

internal/pkg/config/config.go

@@ -210,6 +210,25 @@ type Kubernetes struct {
 	Namespace    string                    `yaml:"namespace"`     // Namespace of job pods, empty uses the service account's, else the kubeconfig context's, else default.
 	PodTemplate  map[string]any            `yaml:"pod_template"`  // PodTemplate is a pod manifest the runner merges its own onto.
 	PodTemplates map[string]map[string]any `yaml:"pod_templates"` // PodTemplates are merged over PodTemplate, in runs-on order, for jobs whose runs-on has the label they are keyed by.
+	Transfer     KubernetesTransfer        `yaml:"transfer"`      // Transfer has job pods fetch copied files from the runner over the pod network instead of through the API server.
+}
+
+// Kubernetes transfer modes: how files the runner copies reach job pods.
+const (
+	KubernetesTransferModeAPI    = "api"    // through the API server
+	KubernetesTransferModeAuto   = "auto"   // over the pod network, through the API server for a job pod that cannot reach the runner
+	KubernetesTransferModeDirect = "direct" // over the pod network only, a copy that cannot reach the runner fails
+)
+
+// KubernetesTransferModes are the valid kubernetes.transfer.mode values.
+var KubernetesTransferModes = []string{KubernetesTransferModeAPI, KubernetesTransferModeAuto, KubernetesTransferModeDirect}
+
+// KubernetesTransfer configures copying files into job pods over the pod network.
+type KubernetesTransfer struct {
+	Mode  string `yaml:"mode"`  // Mode is one of KubernetesTransferModes, api when empty.
+	Host  string `yaml:"host"`  // Host is the address the runner listens on and job pods reach it at, empty uses the outbound IP.
+	Port  uint16 `yaml:"port"`  // Port the runner listens on for job pods, 0 picks a free one.
+	Image string `yaml:"image"` // Image has gitea-runner on its PATH, empty uses the image of a single-container runner pod.
 }
 
 // Metrics represents the configuration for the Prometheus metrics endpoint.
@@ -346,6 +365,12 @@ func LoadDefault(file string) (*Config, error) {
 	if !slices.Contains(ToolCacheModes, cfg.Runner.ToolCacheMode) {
 		return nil, fmt.Errorf("invalid runner.tool_cache_mode %q: must be one of %q", cfg.Runner.ToolCacheMode, ToolCacheModes)
 	}
+	if cfg.Kubernetes.Transfer.Mode == "" {
+		cfg.Kubernetes.Transfer.Mode = KubernetesTransferModeAPI
+	}
+	if !slices.Contains(KubernetesTransferModes, cfg.Kubernetes.Transfer.Mode) {
+		return nil, fmt.Errorf("invalid kubernetes.transfer.mode %q: must be one of %q", cfg.Kubernetes.Transfer.Mode, KubernetesTransferModes)
+	}
 	if cfg.Runner.LogReportLevel != "" {
 		if _, err := logging.ParseLevel(cfg.Runner.LogReportLevel); err != nil {
 			return nil, fmt.Errorf("invalid runner.log_report_level: %w", err)

internal/pkg/config/config_test.go

@@ -140,6 +140,19 @@ func TestLoadDefault_ToolCacheMode(t *testing.T) {
 	assert.Contains(t, err.Error(), "tool_cache_mode")
 }
 
+func TestLoadDefault_KubernetesTransferMode(t *testing.T) {
+	cfg, err := LoadDefault(write(t, "kubernetes:\n  transfer:\n    host: 10.0.0.5\n"))
+	require.NoError(t, err)
+	assert.Equal(t, KubernetesTransferModeAPI, cfg.Kubernetes.Transfer.Mode, "copies stay on the API server unless asked")
+
+	cfg, err = LoadDefault(write(t, "kubernetes:\n  transfer:\n    mode: direct\n"))
+	require.NoError(t, err)
+	assert.Equal(t, KubernetesTransferModeDirect, cfg.Kubernetes.Transfer.Mode)
+
+	_, err = LoadDefault(write(t, "kubernetes:\n  transfer:\n    mode: fast\n"))
+	require.ErrorContains(t, err, "kubernetes.transfer.mode")
+}
+
 func TestLoadDefault_RejectsInvalidLogReportLevel(t *testing.T) {
 	_, err := LoadDefault(write(t, "runner:\n  log_report_level: loud\n"))
 	require.ErrorContains(t, err, "runner.log_report_level")