enhance: support client certificates for mutual TLS (#1273)

Add `runner.client_cert_file` and `runner.client_key_file` for Gitea instances that require mutual TLS. The runner presents the certificate on its RPCs, job summary uploads, and the action, reusable workflow and `builtin:checkout` clones from the instance. The files are read on use, so renewed certificates apply without a restart. Job steps do not get them. An incomplete or mismatched pair fails startup.

Clones now decide the token, client certificate and `runner.insecure` with one host check, which also gives reusable workflow clones the TLS settings they lacked. `runner.insecure` still only covers the registered address.

Fixes: https://gitea.com/gitea/runner/issues/321
Co-authored-by: silverwind <[email protected]>
Co-authored-by: silverwind <[email protected]>
Reviewed-on: https://gitea.com/gitea/runner/pulls/1273
Reviewed-by: silverwind <[email protected]>
Co-authored-by: bircni <[email protected]>

internal/app/cmd/daemon.go

@@ -170,7 +170,7 @@ func runDaemon(ctx context.Context, daemArgs *daemonArgs, configFile *string) fu
 
 		cli := client.New(
 			reg.Address,
-			cfg.Runner.Insecure,
+			cfg.Runner.TLSConfig(),
 			reg.UUID,
 			reg.Token,
 			config.RequestTimeout,

internal/app/cmd/register.go

@@ -363,7 +363,7 @@ func doRegister(ctx context.Context, cfg *config.Config, inputs *registerInputs)
 	// initial http client
 	cli := client.New(
 		inputs.InstanceAddr,
-		cfg.Runner.Insecure,
+		cfg.Runner.TLSConfig(),
 		"",
 		"",
 		config.RequestTimeout,

internal/app/run/runner.go

@@ -74,6 +74,7 @@ type Runner struct {
 	capabilities string
 
 	isolatedCacheContainer func() string
+	instanceTransport      http.RoundTripper
 
 	runningTasks            sync.Map
 	runningCount            atomic.Int64
@@ -140,6 +141,7 @@ func NewRunner(cfg *config.Config, reg *config.Registration, cli client.Client)
 		runHealthCheck: executeHealthCheck,
 	}
 	runner.isolatedCacheContainer = sync.OnceValue(runner.detectIsolatedCacheContainer)
+	runner.instanceTransport = client.NewTransport(cli.Address(), cfg.Runner.TLSConfig())
 	return runner
 }
 
@@ -626,6 +628,9 @@ func (r *Runner) run(ctx context.Context, task *runnerv1.Task, reporter *report.
 		ValidVolumes:                      r.cfg.Container.ValidVolumes,
 		SharedToolCache:                   r.cfg.Runner.ToolCacheMode == config.ToolCacheModeShared,
 		InsecureSkipTLS:                   r.cfg.Runner.Insecure,
+		ClientCertFile:                    r.cfg.Runner.ClientCertFile,
+		ClientKeyFile:                     r.cfg.Runner.ClientKeyFile,
+		InstanceTransport:                 r.instanceTransport,
 		RunnerName:                        r.name,
 		KubernetesPicker:                  r.kubernetesOptions,
 	}

internal/pkg/action/action.go

@@ -20,5 +20,7 @@ type Context struct {
 	NoSkipCheckout  bool
 	UseGitIgnore    bool
 	InsecureSkipTLS bool
+	ClientCertFile  string
+	ClientKeyFile   string
 	Env             map[string]string
 }

internal/pkg/action/checkout/checkout.go

@@ -11,6 +11,7 @@ import (
 	"fmt"
 	"maps"
 	"net/url"
+	"os"
 	"path"
 	"path/filepath"
 	"runtime"
@@ -122,6 +123,26 @@ func Main(ctx context.Context, c *action.Context) (err error) {
 	if boolInput(c.Inputs, "set-safe-directory", true) {
 		remoteConfig = append(remoteConfig, "safe.directory=*") // a bound workdir belongs to the host user
 	}
+	if c.ClientCertFile != "" { // only for this step, later steps never get the runner's key
+		var certPEM, keyPEM []byte
+		if certPEM, err = os.ReadFile(c.ClientCertFile); err != nil {
+			return err
+		}
+		if keyPEM, err = os.ReadFile(c.ClientKeyFile); err != nil {
+			return err
+		}
+		cert, key := fmt.Sprintf("git-client-cert-%x.pem", digest[:8]), fmt.Sprintf("git-client-key-%x.pem", digest[:8])
+		defer func() {
+			cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), time.Minute)
+			defer cancel()
+			err = errors.Join(err, c.Container.Copy(runnerTemp, &container.FileEntry{Name: key, Mode: 0o600})(cleanupCtx))
+		}()
+		if err := c.Container.Copy(runnerTemp, &container.FileEntry{Name: cert, Mode: 0o600, Body: string(certPEM)}, &container.FileEntry{Name: key, Mode: 0o600, Body: string(keyPEM)})(ctx); err != nil {
+			return err
+		}
+		scope := "http." + strings.TrimSuffix(ghc.ServerURL, "/") + "/."
+		remoteConfig = append(remoteConfig, scope+"sslCert="+path.Join(runnerTemp, cert), scope+"sslKey="+path.Join(runnerTemp, key))
+	}
 	env := gitEnv(c, remoteConfig)
 	env["GIT_LFS_SKIP_SMUDGE"] = strconv.FormatBool(!co.lfs || len(co.sparse) == 0) // lfs pull fetches the objects of a full checkout in one go
 	if sshKey != "" {

internal/pkg/action/checkout/checkout_test.go

@@ -6,8 +6,13 @@ package checkout
 import (
 	"cmp"
 	"crypto/sha256"
+	"crypto/tls"
+	"crypto/x509"
+	"encoding/pem"
 	"fmt"
 	"io"
+	"net/http"
+	"net/http/httptest"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -286,6 +291,41 @@ exec sh -c "$(printf '%s' "$last" | sed "s#git-upload-pack '/\(.*\)\.git'#git up
 	}
 }
 
+func TestCheckoutOffersClientCertOnlyDuringTheStep(t *testing.T) {
+	var gotCert bool
+	server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		gotCert = len(r.TLS.PeerCertificates) > 0
+		w.WriteHeader(http.StatusNotFound)
+	}))
+	server.TLS = &tls.Config{ClientAuth: tls.RequireAnyClientCert}
+	server.StartTLS()
+	defer server.Close()
+	pair := server.TLS.Certificates[0]
+	keyDER, err := x509.MarshalPKCS8PrivateKey(pair.PrivateKey)
+	require.NoError(t, err)
+	dir, workspace, runnerTemp := t.TempDir(), t.TempDir(), t.TempDir()
+	certFile, keyFile := filepath.Join(dir, "client.crt"), filepath.Join(dir, "client.key")
+	require.NoError(t, os.WriteFile(certFile, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: pair.Certificate[0]}), 0o600))
+	require.NoError(t, os.WriteFile(keyFile, pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: keyDER}), 0o600))
+
+	require.ErrorContains(t, Main(t.Context(), &action.Context{
+		Container:       &container.HostEnvironment{Path: workspace, TmpDir: runnerTemp, StdOut: io.Discard},
+		Github:          &model.GithubContext{Repository: "org/repo", Ref: "refs/heads/main", ServerURL: server.URL},
+		Inputs:          map[string]string{"repository": "other/repo"},
+		Workspace:       workspace,
+		InsecureSkipTLS: true,
+		ClientCertFile:  certFile,
+		ClientKeyFile:   keyFile,
+		Env:             map[string]string{"PATH": os.Getenv("PATH"), "GIT_CONFIG_GLOBAL": os.DevNull, "GIT_CONFIG_NOSYSTEM": "1"},
+	}), "checkout: other/repo@")
+	require.True(t, gotCert)
+	keys, _ := filepath.Glob(filepath.Join(runnerTemp, "git-client-key-*"))
+	require.Len(t, keys, 1)
+	key, err := os.ReadFile(keys[0])
+	require.NoError(t, err)
+	require.Empty(t, key)
+}
+
 func publish(t *testing.T, server, name string, files, submodules map[string]string) {
 	work := t.TempDir()
 	runGit(t, "init", "--initial-branch=main", work)

internal/pkg/client/http.go

@@ -19,20 +19,23 @@ import (
 	"gitea.dev/actionslib/runner/v1/runnerv1connect"
 )
 
-func getHTTPClient(endpoint string, insecure bool, timeout time.Duration, extraHeaders map[string]string) *http.Client {
+func NewTransport(endpoint string, tlsConfig *tls.Config) *http.Transport {
 	transport := &http.Transport{
 		Proxy:               http.ProxyFromEnvironment,
 		MaxIdleConns:        10,
 		MaxIdleConnsPerHost: 10, // All requests go to one host; default is 2 which causes frequent reconnects.
 		IdleConnTimeout:     90 * time.Second,
+		ForceAttemptHTTP2:   true,
 	}
-	if strings.HasPrefix(endpoint, "https://") && insecure {
-		transport.TLSClientConfig = &tls.Config{
-			InsecureSkipVerify: true,
-		}
+	if strings.HasPrefix(endpoint, "https://") {
+		transport.TLSClientConfig = tlsConfig
 	}
+	return transport
+}
+
+func getHTTPClient(endpoint string, tlsConfig *tls.Config, timeout time.Duration, extraHeaders map[string]string) *http.Client {
 	return &http.Client{
-		Transport: transport,
+		Transport: NewTransport(endpoint, tlsConfig),
 		Timeout:   timeout,
 		CheckRedirect: func(req *http.Request, via []*http.Request) error {
 			if len(via) >= 10 { // a custom CheckRedirect replaces net/http's own limit
@@ -54,7 +57,7 @@ func getHTTPClient(endpoint string, insecure bool, timeout time.Duration, extraH
 // New returns a new runner client. timeout bounds every RPC: without it a
 // stalled connection parks the reporter for the whole job context, so logs and
 // heartbeats stop together and the task is reaped as a zombie.
-func New(endpoint string, insecure bool, uuid, token string, timeout time.Duration, extraHeaders map[string]string, opts ...connect.ClientOption) *HTTPClient {
+func New(endpoint string, tlsConfig *tls.Config, uuid, token string, timeout time.Duration, extraHeaders map[string]string, opts ...connect.ClientOption) *HTTPClient {
 	baseURL := strings.TrimRight(endpoint, "/") + "/api/actions"
 
 	opts = append(opts, connect.WithInterceptors(connect.UnaryInterceptorFunc(func(next connect.UnaryFunc) connect.UnaryFunc {
@@ -75,7 +78,7 @@ func New(endpoint string, insecure bool, uuid, token string, timeout time.Durati
 		}
 	})))
 
-	httpClient := getHTTPClient(endpoint, insecure, timeout, extraHeaders)
+	httpClient := getHTTPClient(endpoint, tlsConfig, timeout, extraHeaders)
 	return &HTTPClient{
 		PingServiceClient: pingv1connect.NewPingServiceClient(
 			httpClient,

internal/pkg/client/http_test.go

@@ -4,6 +4,7 @@
 package client
 
 import (
+	"crypto/tls"
 	"net"
 	"net/http"
 	"net/http/httptest"
@@ -20,7 +21,7 @@ import (
 func TestGetHTTPClientUsesProxyFromEnvironment(t *testing.T) {
 	t.Setenv("HTTP_PROXY", "http://proxy.example.com:8080")
 
-	client := getHTTPClient("http://gitea.example.com", false, time.Minute, nil)
+	client := getHTTPClient("http://gitea.example.com", nil, time.Minute, nil)
 	require.Equal(t, time.Minute, client.Timeout)
 	transport, ok := client.Transport.(*http.Transport)
 	require.True(t, ok)
@@ -34,29 +35,10 @@ func TestGetHTTPClientUsesProxyFromEnvironment(t *testing.T) {
 	require.Equal(t, "http://proxy.example.com:8080", proxyURL.String())
 }
 
-func TestGetHTTPClientInsecureTLS(t *testing.T) {
-	// insecure only takes effect for https endpoints
-	httpsInsecure := getHTTPClient("https://gitea.example.com", true, time.Minute, nil)
-	transport, ok := httpsInsecure.Transport.(*http.Transport)
-	require.True(t, ok)
-	require.NotNil(t, transport.TLSClientConfig)
-	require.True(t, transport.TLSClientConfig.InsecureSkipVerify)
-
-	for _, tc := range []struct {
-		name     string
-		endpoint string
-		insecure bool
-	}{
-		{"https secure", "https://gitea.example.com", false},
-		{"http insecure ignored", "http://gitea.example.com", true},
-	} {
-		t.Run(tc.name, func(t *testing.T) {
-			c := getHTTPClient(tc.endpoint, tc.insecure, time.Minute, nil)
-			tr, ok := c.Transport.(*http.Transport)
-			require.True(t, ok)
-			require.Nil(t, tr.TLSClientConfig)
-		})
-	}
+func TestGetHTTPClientAppliesTLSConfigOnlyToHTTPS(t *testing.T) {
+	tlsConfig := &tls.Config{}
+	require.Same(t, tlsConfig, getHTTPClient("https://gitea.example.com", tlsConfig, time.Minute, nil).Transport.(*http.Transport).TLSClientConfig)
+	require.Nil(t, getHTTPClient("http://gitea.example.com", tlsConfig, time.Minute, nil).Transport.(*http.Transport).TLSClientConfig)
 }
 
 func TestNewSetsBaseURLAndHeaders(t *testing.T) {
@@ -70,7 +52,7 @@ func TestNewSetsBaseURLAndHeaders(t *testing.T) {
 	defer server.Close()
 
 	// trailing slash must be trimmed before "/api/actions" is appended
-	c := New(server.URL+"/", false, "the-uuid", "the-token", time.Minute, map[string]string{
+	c := New(server.URL+"/", nil, "the-uuid", "the-token", time.Minute, map[string]string{
 		"X-Proxy-Token":      "proxy-token",
 		protocol.UUIDHeader:  "other-uuid",
 		protocol.TokenHeader: "other-token",
@@ -97,7 +79,7 @@ func TestNewOmitsEmptyHeaders(t *testing.T) {
 	}))
 	defer server.Close()
 
-	c := New(server.URL, false, "", "", time.Minute, nil)
+	c := New(server.URL, nil, "", "", time.Minute, nil)
 	_, _ = c.Ping(t.Context(), connect.NewRequest(&pingv1.PingRequest{Data: "hi"}))
 
 	require.Empty(t, gotHeaders.Get(protocol.UUIDHeader))
@@ -110,7 +92,7 @@ func TestNewStopsRedirectLoop(t *testing.T) {
 	}))
 	defer server.Close()
 
-	c := New(server.URL, false, "", "", time.Minute, nil)
+	c := New(server.URL, nil, "", "", time.Minute, nil)
 	_, err := c.Ping(t.Context(), connect.NewRequest(&pingv1.PingRequest{Data: "hi"}))
 	require.ErrorContains(t, err, "stopped after 10 redirects")
 }
@@ -136,7 +118,7 @@ func TestNewRedirectKeepsExtraHeadersOnlyOnSameHost(t *testing.T) {
 	} {
 		gotHeaders = nil
 		origin := tc.newOrigin(http.RedirectHandler("http://"+net.JoinHostPort(tc.host, port), http.StatusPermanentRedirect))
-		c := New(origin.URL, true, "the-uuid", "the-token", time.Minute, map[string]string{"X-Proxy-Token": "proxy-token", "Content-Type": "text/plain"})
+		c := New(origin.URL, &tls.Config{InsecureSkipVerify: true}, "the-uuid", "the-token", time.Minute, map[string]string{"X-Proxy-Token": "proxy-token", "Content-Type": "text/plain"})
 		_, _ = c.Ping(t.Context(), connect.NewRequest(&pingv1.PingRequest{Data: "hi"}))
 		origin.Close()
 

internal/pkg/config/config.example.yaml

@@ -41,6 +41,9 @@ runner:
   #shutdown_timeout: 0s
   # Whether skip verifying the TLS certificate of the Gitea instance.
   #insecure: false
+  # PEM client certificate and key for a Gitea instance that requires mutual TLS.
+  #client_cert_file: /etc/gitea-runner/client.crt
+  #client_key_file: /etc/gitea-runner/client.key
   # The timeout for fetching the job from the Gitea instance.
   # Values above the 60s RPC timeout are capped to it.
   #fetch_timeout: 5s

internal/pkg/config/config.go

@@ -4,6 +4,7 @@
 package config
 
 import (
+	"crypto/tls"
 	"errors"
 	"fmt"
 	"maps"
@@ -63,6 +64,8 @@ type Runner struct {
 	Timeout               time.Duration     `yaml:"timeout"`                  // Timeout specifies the duration for runner timeout.
 	ShutdownTimeout       time.Duration     `yaml:"shutdown_timeout"`         // ShutdownTimeout specifies the duration to wait for running jobs to complete during a shutdown of the runner.
 	Insecure              bool              `yaml:"insecure"`                 // Insecure indicates whether the runner operates in an insecure mode.
+	ClientCertFile        string            `yaml:"client_cert_file"`         // ClientCertFile is the PEM certificate presented to a Gitea instance that requires mutual TLS.
+	ClientKeyFile         string            `yaml:"client_key_file"`          // ClientKeyFile is the PEM private key paired with ClientCertFile.
 	ExtraHeaders          map[string]string `yaml:"extra_headers"`            // ExtraHeaders adds HTTP headers to runner API requests to Gitea.
 	FetchTimeout          time.Duration     `yaml:"fetch_timeout"`            // FetchTimeout specifies the timeout duration for fetching resources.
 	FetchInterval         time.Duration     `yaml:"fetch_interval"`           // FetchInterval specifies the interval duration for fetching resources.
@@ -272,6 +275,21 @@ func LoadDefault(file string) (*Config, error) {
 			maps.Copy(cfg.Runner.Envs, envs)
 		}
 	}
+	if (cfg.Runner.ClientCertFile == "") != (cfg.Runner.ClientKeyFile == "") {
+		return nil, errors.New("runner.client_cert_file and runner.client_key_file must be set together")
+	}
+	if cfg.Runner.ClientCertFile != "" {
+		if _, err := tls.LoadX509KeyPair(cfg.Runner.ClientCertFile, cfg.Runner.ClientKeyFile); err != nil {
+			return nil, fmt.Errorf("load runner client certificate: %w", err)
+		}
+		for _, file := range []*string{&cfg.Runner.ClientCertFile, &cfg.Runner.ClientKeyFile} {
+			abs, err := filepath.Abs(*file) // git resolves a relative path against the repository it runs in
+			if err != nil {
+				return nil, err
+			}
+			*file = abs
+		}
+	}
 
 	if cfg.Log.Level == "" {
 		cfg.Log.Level = "info"
@@ -422,6 +440,20 @@ func LoadDefault(file string) (*Config, error) {
 	return cfg, nil
 }
 
+func (r *Runner) TLSConfig() *tls.Config {
+	tlsConfig := &tls.Config{InsecureSkipVerify: r.Insecure}
+	if r.ClientCertFile != "" {
+		tlsConfig.GetClientCertificate = func(info *tls.CertificateRequestInfo) (*tls.Certificate, error) {
+			pair, err := tls.LoadX509KeyPair(r.ClientCertFile, r.ClientKeyFile) // per handshake, so renewed files apply
+			if err != nil || info.SupportsCertificate(&pair) != nil {
+				return &tls.Certificate{}, err
+			}
+			return &pair, nil
+		}
+	}
+	return tlsConfig
+}
+
 // expandEnvironment replaces ${NAME} in values and retypes them, so ${CAPACITY} can be an int and an empty value is unset.
 func expandEnvironment(node *yaml.Node) error {
 	var errs []error

internal/pkg/config/config_test.go

@@ -4,6 +4,13 @@
 package config
 
 import (
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	"crypto/rand"
+	"crypto/tls"
+	"crypto/x509"
+	"encoding/pem"
+	"math/big"
 	"os"
 	"path/filepath"
 	"testing"
@@ -42,6 +49,49 @@ cache:
 	require.NoError(t, err)
 }
 
+func TestClientCertIsValidatedAndAbsoluteOnLoadReadOnUseAndOfferedOnlyWhenAccepted(t *testing.T) {
+	require.True(t, (&Runner{Insecure: true}).TLSConfig().InsecureSkipVerify)
+
+	dir := t.TempDir()
+	certPath, keyPath := filepath.Join(dir, "client.crt"), filepath.Join(dir, "client.key")
+	writePair := func() []byte {
+		key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+		require.NoError(t, err)
+		template := &x509.Certificate{SerialNumber: big.NewInt(1)}
+		der, err := x509.CreateCertificate(rand.Reader, template, template, &key.PublicKey, key)
+		require.NoError(t, err)
+		keyDER, err := x509.MarshalPKCS8PrivateKey(key)
+		require.NoError(t, err)
+		require.NoError(t, os.WriteFile(certPath, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), 0o600))
+		require.NoError(t, os.WriteFile(keyPath, pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: keyDER}), 0o600))
+		return der
+	}
+	load := func(certFile, keyFile string) (*Config, error) {
+		return LoadDefault(write(t, "runner:\n  client_cert_file: '"+certFile+"'\n  client_key_file: '"+keyFile+"'\n"))
+	}
+	writePair()
+
+	_, err := load(certPath, "")
+	require.ErrorContains(t, err, "must be set together")
+	_, err = load(keyPath, keyPath)
+	require.ErrorContains(t, err, "load runner client certificate")
+
+	t.Chdir(dir)
+	cfg, err := load("client.crt", "client.key")
+	require.NoError(t, err)
+	require.True(t, filepath.IsAbs(cfg.Runner.ClientCertFile))
+	require.True(t, filepath.IsAbs(cfg.Runner.ClientKeyFile))
+	renewed := writePair()
+	info := &tls.CertificateRequestInfo{Version: tls.VersionTLS13, SignatureSchemes: []tls.SignatureScheme{tls.ECDSAWithP256AndSHA256}}
+	pair, err := cfg.Runner.TLSConfig().GetClientCertificate(info)
+	require.NoError(t, err)
+	require.Equal(t, renewed, pair.Certificate[0])
+	info.AcceptableCAs = [][]byte{[]byte("another CA")}
+	pair, err = cfg.Runner.TLSConfig().GetClientCertificate(info)
+	require.NoError(t, err)
+	require.Empty(t, pair.Certificate)
+}
+
 func TestLoadDefault_ExpandsEnvironmentValues(t *testing.T) {
 	t.Setenv("RUNNER_CAPACITY", "4")
 	t.Setenv("RUNNER_LABEL", "linux:host")