feat!: serve the actions cache from the runner (#1229)

Make cache requests go through the runner and remove outbound traffic from the cache server. The runner forwards them to `cache.external_server` when one is set, and attaches itself to each job network so jobs can reach it.

Benefits:

1. Jobs need no http proxy rules for the cache server, which `@actions/http-client` mishandles for CIDR ranges in `no_proxy` (https://github.com/actions/toolkit/pull/2495).
2. Only runners reach a shared cache server, which needs no route to Gitea.
3. A runner container sharing the host Docker socket now caches, where jobs previously could not reach it.

Breaks:

1. Jobs that reach `external_server` but not their runner, such as a remote `DOCKER_HOST`. The built-in cache has always needed that reachability, so both modes now agree. Set `cache.host` and `cache.port` to an address those jobs reach.
2. Jobs on `container.network: bridge`, where Docker rejects the alias, so the runner cannot attach and their cache fails as it does today. Leave `container.network` empty, or name a user-defined network.

*Written by Claude.*

---------

Co-authored-by: bircni <[email protected]>
Reviewed-on: https://gitea.com/gitea/runner/pulls/1229
Reviewed-by: bircni <[email protected]>
Co-authored-by: silverwind <[email protected]>

act/container/docker_network.go

@@ -16,6 +16,8 @@ import (
 
 	"gitea.com/gitea/runner/act/common"
 
+	cerrdefs "github.com/containerd/errdefs"
+	"github.com/moby/moby/api/types/network"
 	"github.com/moby/moby/client"
 )
 
@@ -33,17 +35,17 @@ const (
 // had an endpoint on it at the time. Each one holds a subnet of the daemon's address pool
 // until it is removed. Networks created after createdBefore are left alone, so a job starting
 // while this runs cannot lose the network it has created but not yet attached a container to.
-func RemoveOrphanNetworks(ctx context.Context, runnerUUID string, createdBefore time.Time) error {
+func RemoveOrphanNetworks(ctx context.Context, runnerUUID, detach string, createdBefore time.Time) error {
 	cli, err := GetDockerClient(ctx)
 	if err != nil {
 		return fmt.Errorf("failed to connect to the docker daemon: %w", err)
 	}
 	defer cli.Close()
 
-	return removeOrphanNetworks(ctx, cli, runnerUUID, createdBefore)
+	return removeOrphanNetworks(ctx, cli, runnerUUID, detach, createdBefore)
 }
 
-func removeOrphanNetworks(ctx context.Context, cli client.APIClient, runnerUUID string, createdBefore time.Time) error {
+func removeOrphanNetworks(ctx context.Context, cli client.APIClient, runnerUUID, detach string, createdBefore time.Time) error {
 	networks, err := cli.NetworkList(ctx, client.NetworkListOptions{
 		Filters: make(client.Filters).Add("label", runnerUUIDLabel+"="+runnerUUID),
 	})
@@ -52,6 +54,7 @@ func removeOrphanNetworks(ctx context.Context, cli client.APIClient, runnerUUID
 	}
 
 	var errs []error
+networks:
 	for _, n := range networks.Items {
 		result, err := cli.NetworkInspect(ctx, n.ID, client.NetworkInspectOptions{})
 		if err != nil {
@@ -60,9 +63,18 @@ func removeOrphanNetworks(ctx context.Context, cli client.APIClient, runnerUUID
 		}
 		// the emptiness check, not the label, is what keeps a live job of another process
 		// sharing this registration safe
-		if len(result.Network.Containers) != 0 || result.Network.Created.After(createdBefore) {
+		if len(result.Network.Containers) > 1 || result.Network.Created.After(createdBefore) {
 			continue
 		}
+		for containerID := range result.Network.Containers {
+			if detach == "" || !strings.HasPrefix(containerID, detach) {
+				continue networks
+			}
+			if err := disconnectNetwork(ctx, cli, n.ID, containerID); err != nil {
+				errs = append(errs, fmt.Errorf("failed to disconnect container from network %s: %w", n.Name, err))
+				continue networks
+			}
+		}
 		if _, err := cli.NetworkRemove(ctx, n.ID, client.NetworkRemoveOptions{}); err != nil {
 			errs = append(errs, fmt.Errorf("failed to remove network %s: %w", n.Name, err))
 			continue
@@ -135,6 +147,50 @@ func isAddressPoolExhausted(err error) bool {
 		strings.Contains(msg, "could not find an available, non-overlapping IPv4 address pool among the defaults") // docker 24 and older
 }
 
+func NewDockerNetworkConnectExecutor(networkName, container string) common.Executor {
+	return func(ctx context.Context) error {
+		cli, err := GetDockerClient(ctx)
+		if err != nil {
+			return err
+		}
+		defer cli.Close()
+		if err := connectNetwork(ctx, cli, networkName, container); err != nil {
+			common.Logger(ctx).Warnf("cannot connect %s to network %s, so jobs on it cannot reach the cache: %v", container, networkName, err)
+		}
+		return nil
+	}
+}
+
+func connectNetwork(ctx context.Context, cli client.APIClient, networkName, container string) error {
+	_, err := cli.NetworkConnect(ctx, networkName, client.NetworkConnectOptions{
+		Container:      container,
+		EndpointConfig: &network.EndpointSettings{Aliases: []string{container}},
+	})
+	if err != nil && (strings.Contains(err.Error(), "already exists in network") || strings.Contains(err.Error(), "already connected")) {
+		return nil
+	}
+	return err
+}
+
+func NewDockerNetworkDisconnectExecutor(networkName, container string) common.Executor {
+	return func(ctx context.Context) error {
+		cli, err := GetDockerClient(ctx)
+		if err != nil {
+			return err
+		}
+		defer cli.Close()
+		return disconnectNetwork(ctx, cli, networkName, container)
+	}
+}
+
+func disconnectNetwork(ctx context.Context, cli client.APIClient, networkName, container string) error {
+	_, err := cli.NetworkDisconnect(ctx, networkName, client.NetworkDisconnectOptions{Container: container, Force: true})
+	if cerrdefs.IsNotFound(err) || (err != nil && strings.Contains(err.Error(), "is not connected")) {
+		return nil
+	}
+	return err
+}
+
 func NewDockerNetworkRemoveExecutor(name string) common.Executor {
 	return func(ctx context.Context) error {
 		cli, err := GetDockerClient(ctx)
@@ -174,16 +230,17 @@ func NewDockerNetworkRemoveExecutor(name string) common.Executor {
 	}
 }
 
-func IsolatedNetwork(ctx context.Context, addr netip.Addr, jobNetwork string) (string, error) {
+// IsolatedCacheContainer returns the container holding addr, when jobs on jobNetwork cannot reach it there.
+func IsolatedCacheContainer(ctx context.Context, addr netip.Addr, jobNetwork string) (string, error) {
 	cli, err := GetDockerClient(ctx)
 	if err != nil {
 		return "", err
 	}
 	defer cli.Close()
-	return isolatedNetwork(ctx, cli, addr, jobNetwork)
+	return isolatedCacheContainer(ctx, cli, addr, jobNetwork)
 }
 
-func isolatedNetwork(ctx context.Context, cli client.APIClient, addr netip.Addr, jobNetwork string) (string, error) {
+func isolatedCacheContainer(ctx context.Context, cli client.APIClient, addr netip.Addr, jobNetwork string) (string, error) {
 	if jobNetwork == "host" || strings.HasPrefix(jobNetwork, "container:") {
 		return "", nil
 	}
@@ -191,7 +248,7 @@ func isolatedNetwork(ctx context.Context, cli client.APIClient, addr netip.Addr,
 	if err != nil {
 		return "", err
 	}
-	var holderID string
+	var holderID, holderContainer string
 	for _, summary := range containers.Items {
 		if summary.NetworkSettings == nil {
 			continue
@@ -199,6 +256,7 @@ func isolatedNetwork(ctx context.Context, cli client.APIClient, addr netip.Addr,
 		for _, endpoint := range summary.NetworkSettings.Networks {
 			if endpoint != nil && (endpoint.IPAddress == addr || endpoint.GlobalIPv6Address == addr) {
 				holderID = endpoint.NetworkID
+				holderContainer = summary.ID[:12]
 			}
 		}
 	}
@@ -215,5 +273,5 @@ func isolatedNetwork(ctx context.Context, cli client.APIClient, addr netip.Addr,
 			return "", err
 		}
 	}
-	return holder.Network.Name, nil
+	return holderContainer, nil
 }

act/container/docker_network_test.go

@@ -18,11 +18,12 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-func TestIsolatedNetwork(t *testing.T) {
+func TestIsolatedCacheContainer(t *testing.T) {
 	ctx := context.Background()
 	cache, lan := netip.MustParseAddr("172.18.0.3"), netip.MustParseAddr("192.168.1.20")
 	client := &mockDockerClient{}
 	client.On("ContainerList", ctx, mobyclient.ContainerListOptions{}).Return(mobyclient.ContainerListResult{Items: []container.Summary{{}, {
+		ID: "0123456789abcdef",
 		NetworkSettings: &container.NetworkSettingsSummary{Networks: map[string]*network.EndpointSettings{
 			"compose": {NetworkID: "c0ffee", IPAddress: cache},
 			"lan":     {NetworkID: "beef", IPAddress: lan},
@@ -38,12 +39,12 @@ func TestIsolatedNetwork(t *testing.T) {
 			Return(mobyclient.NetworkInspectResult{Network: network.Inspect{Network: inspected}}, nil)
 	}
 	check := func(addr netip.Addr, jobNetwork, want string) {
-		got, err := isolatedNetwork(ctx, client, addr, jobNetwork)
+		got, err := isolatedCacheContainer(ctx, client, addr, jobNetwork)
 		require.NoError(t, err)
 		assert.Equal(t, want, got, jobNetwork)
 	}
-	check(cache, "", "compose")
-	check(cache, "jobs", "compose")
+	check(cache, "", "0123456789ab")
+	check(cache, "jobs", "0123456789ab")
 	check(cache, "c0f", "")
 	check(cache, "host", "")
 	check(lan, "", "")
@@ -56,9 +57,44 @@ func TestIsAddressPoolExhausted(t *testing.T) {
 	assert.False(t, isAddressPoolExhausted(cerrdefs.ErrInvalidArgument.WithMessage("invalid subnet 10.0.0.0/8: it overlaps with an existing network")))
 }
 
-// Of this runner's networks, only the ones nothing is attached to and old enough to predate
-// any job now starting are the runner's to reclaim. An unexpected NetworkRemove fails the
-// test on its own, since testify has no expectation to match it against.
+func (m *mockDockerClient) NetworkConnect(ctx context.Context, id string, opts mobyclient.NetworkConnectOptions) (mobyclient.NetworkConnectResult, error) {
+	return mobyclient.NetworkConnectResult{}, m.Called(ctx, id, opts).Error(0)
+}
+
+func (m *mockDockerClient) NetworkDisconnect(ctx context.Context, id string, opts mobyclient.NetworkDisconnectOptions) (mobyclient.NetworkDisconnectResult, error) {
+	return mobyclient.NetworkDisconnectResult{}, m.Called(ctx, id, opts).Error(0)
+}
+
+func TestConnectAndDisconnectNetwork(t *testing.T) {
+	ctx := context.Background()
+	connect := func(answer error) error {
+		client := &mockDockerClient{}
+		client.On("NetworkConnect", ctx, "jobs", mobyclient.NetworkConnectOptions{
+			Container:      "0123456789ab",
+			EndpointConfig: &network.EndpointSettings{Aliases: []string{"0123456789ab"}},
+		}).Return(answer).Once()
+		return connectNetwork(ctx, client, "jobs", "0123456789ab")
+	}
+	require.NoError(t, connect(nil))
+	require.NoError(t, connect(errors.New("endpoint with name runner already exists in network jobs")))
+	require.NoError(t, connect(errors.New("container runner already connected to network jobs")))
+	require.ErrorIs(t, connect(context.Canceled), context.Canceled)
+
+	disconnect := func(answer error) error {
+		client := &mockDockerClient{}
+		client.On("NetworkDisconnect", ctx, "jobs", mobyclient.NetworkDisconnectOptions{
+			Container: "0123456789ab", Force: true,
+		}).Return(answer).Once()
+		return disconnectNetwork(ctx, client, "jobs", "0123456789ab")
+	}
+	require.NoError(t, disconnect(cerrdefs.ErrNotFound))
+	require.NoError(t, disconnect(errors.New("container runner is not connected to the network jobs")))
+	require.ErrorIs(t, disconnect(context.Canceled), context.Canceled)
+}
+
+// Of this runner's networks, only the ones nothing but its own cache container is attached to
+// and old enough to predate any job now starting are the runner's to reclaim. An unexpected
+// NetworkRemove fails the test on its own, since testify has no expectation to match it against.
 func TestRemoveOrphanNetworks(t *testing.T) {
 	ctx := context.Background()
 	cutoff := time.Date(2026, time.April, 29, 20, 0, 0, 0, time.UTC)
@@ -69,16 +105,29 @@ func TestRemoveOrphanNetworks(t *testing.T) {
 		{ID: "orphan"},
 		{ID: "busy"},
 		{ID: "starting"},
+		{ID: "cached"},
+		{ID: "shared"},
 	}}, nil)
 	client.On("NetworkInspect", ctx, "orphan", mobyclient.NetworkInspectOptions{}).
 		Return(mobyclient.NetworkInspectResult{}, nil)
 	client.On("NetworkInspect", ctx, "busy", mobyclient.NetworkInspectOptions{}).
 		Return(mobyclient.NetworkInspectResult{Network: network.Inspect{Containers: map[string]network.EndpointResource{"c": {}}}}, nil)
 	client.On("NetworkInspect", ctx, "starting", mobyclient.NetworkInspectOptions{}).
-		Return(mobyclient.NetworkInspectResult{Network: network.Inspect{Network: network.Network{Created: cutoff.Add(time.Second)}}}, nil)
+		Return(mobyclient.NetworkInspectResult{Network: network.Inspect{
+			Network:    network.Network{Created: cutoff.Add(time.Second)},
+			Containers: map[string]network.EndpointResource{"0123456789abcdef": {}},
+		}}, nil)
+	client.On("NetworkInspect", ctx, "cached", mobyclient.NetworkInspectOptions{}).
+		Return(mobyclient.NetworkInspectResult{Network: network.Inspect{Containers: map[string]network.EndpointResource{"0123456789abcdef": {}}}}, nil)
+	client.On("NetworkInspect", ctx, "shared", mobyclient.NetworkInspectOptions{}).
+		Return(mobyclient.NetworkInspectResult{Network: network.Inspect{Containers: map[string]network.EndpointResource{"0123456789abcdef": {}, "job": {}}}}, nil)
 	client.On("NetworkRemove", ctx, "orphan", mobyclient.NetworkRemoveOptions{}).
 		Return(mobyclient.NetworkRemoveResult{}, nil)
+	client.On("NetworkDisconnect", ctx, "cached", mobyclient.NetworkDisconnectOptions{Container: "0123456789abcdef", Force: true}).
+		Return(nil).Once()
+	client.On("NetworkRemove", ctx, "cached", mobyclient.NetworkRemoveOptions{}).
+		Return(mobyclient.NetworkRemoveResult{}, nil).Once()
 
-	require.NoError(t, removeOrphanNetworks(ctx, client, "runner-1", cutoff))
+	require.NoError(t, removeOrphanNetworks(ctx, client, "runner-1", "0123456789ab", cutoff))
 	client.AssertExpectations(t)
 }

act/container/docker_stub.go

@@ -18,7 +18,7 @@ import (
 	"github.com/moby/moby/api/types/system"
 )
 
-func IsolatedNetwork(context.Context, netip.Addr, string) (string, error) {
+func IsolatedCacheContainer(context.Context, netip.Addr, string) (string, error) {
 	return "", nil
 }
 
@@ -91,7 +91,19 @@ func NewDockerNetworkRemoveExecutor(name string) common.Executor {
 	}
 }
 
-func RemoveOrphanNetworks(ctx context.Context, runnerUUID string, createdBefore time.Time) error {
+func NewDockerNetworkConnectExecutor(_, _ string) common.Executor {
+	return func(_ context.Context) error {
+		return nil
+	}
+}
+
+func NewDockerNetworkDisconnectExecutor(_, _ string) common.Executor {
+	return func(_ context.Context) error {
+		return nil
+	}
+}
+
+func RemoveOrphanNetworks(_ context.Context, _, _ string, _ time.Time) error {
 	return nil
 }