nprasad2077/runner
1
0 Fork 0
mirror of https://gitea.com/gitea/runner.git synced 2026-11-07 15:50:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: remove the job network even when container removal overruns (#1318)

Browse Source 
Job teardown ran every step on one shared minute. When removing the job container took longer, the network removal ran on an expired context and the network stayed behind until the orphan reclaim, which only takes networks older than `workdir_cleanup_age`, 24h by default. All that time it held a subnet of the daemon's address pool.

The job and service containers are now removed in parallel, and the docker proxy sweep, the network and the volumes get a fresh budget after them. The idle reclaim takes unused networks and volumes once they are 10 minutes old, and network removal treats a network that is already gone as removed.

Reviewed-on: https://gitea.com/gitea/runner/pulls/1318
Reviewed-by: bircni <[email protected]>
Co-authored-by: silverwind <[email protected]>
This commit is contained in:
silverwind authored and bircni committed 2026-10-15 14:15:41 +00:00
1 parent b769a9765f
commit 0d84ec6dab
9 files changed
+127 -62
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

act/container/docker_network.go
+30 -27
View File
@@ -53,7 +53,9 @@ func removeOrphanNetworks(ctx context.Context, cli client.APIClient, runnerUUID,
networks:
for _, n := range networks.Items {
result, err := cli.NetworkInspect(ctx, n.ID, client.NetworkInspectOptions{})
if err != nil {
if cerrdefs.IsNotFound(err) {
continue
} else if err != nil {
errs = append(errs, fmt.Errorf("failed to inspect network %s: %w", n.Name, err))
continue
}
@@ -71,7 +73,9 @@ networks:
continue networks
}
}
if _, err := cli.NetworkRemove(ctx, n.ID, client.NetworkRemoveOptions{}); err != nil {
if _, err := cli.NetworkRemove(ctx, n.ID, client.NetworkRemoveOptions{}); cerrdefs.IsNotFound(err) {
continue
} else if err != nil {
errs = append(errs, fmt.Errorf("failed to remove network %s: %w", n.Name, err))
continue
}
@@ -203,36 +207,35 @@ func NewDockerNetworkRemoveExecutor(name string) common.Executor {
return err
}
defer cli.Close()
return removeNetwork(ctx, cli, name)
}
}
// Make sure that all network of the specified name are removed
// cli.NetworkRemove refuses to remove a network if there are duplicates
networks, err := cli.NetworkList(ctx, client.NetworkListOptions{})
if err != nil {
func removeNetwork(ctx context.Context, cli client.APIClient, name string) error {
networks, err := cli.NetworkList(ctx, client.NetworkListOptions{}) // NetworkRemove by name refuses duplicates, so remove each by ID
if err != nil {
return err
}
var errs []error
for _, n := range networks.Items {
if n.Name != name {
continue
}
result, err := cli.NetworkInspect(ctx, n.ID, client.NetworkInspectOptions{})
if cerrdefs.IsNotFound(err) {
continue
} else if err != nil {
return err
}
// For Gitea, reduce log noise
// common.Logger(ctx).Debugf("%v", networks)
var errs []error
for _, n := range networks.Items {
if n.Name == name {
result, err := cli.NetworkInspect(ctx, n.ID, client.NetworkInspectOptions{})
if err != nil {
return err
}
// it holds a subnet out of the daemon's pool until something reclaims it
if len(result.Network.Containers) != 0 {
common.Logger(ctx).Warnf("Refusing to remove network %s because it still has active endpoints, the idle cleanup reclaims it once they are gone", name)
continue
}
if _, err = cli.NetworkRemove(ctx, n.ID, client.NetworkRemoveOptions{}); err != nil {
errs = append(errs, fmt.Errorf("failed to remove network %s: %w", name, err))
}
}
if len(result.Network.Containers) != 0 {
common.Logger(ctx).Warnf("Refusing to remove network %s because it still has active endpoints, the orphan cleanup reclaims it once they are gone", name)
continue
}
if _, err = cli.NetworkRemove(ctx, n.ID, client.NetworkRemoveOptions{}); err != nil && !cerrdefs.IsNotFound(err) {
errs = append(errs, fmt.Errorf("failed to remove network %s: %w", name, err))
}
return errors.Join(errs...)
}
return errors.Join(errs...)
}
// IsolatedCacheContainer returns the container holding addr, when jobs on jobNetwork cannot reach it there.
act/container/docker_network_test.go
+21 -4
View File
@@ -116,7 +116,7 @@ func TestConnectAndDisconnectNetwork(t *testing.T) {
// Of this runner's networks, only the ones nothing but its own cache container is attached to
// and old enough to predate any job now starting are the runner's to reclaim. An unexpected
// NetworkRemove fails the test on its own, since testify has no expectation to match it against.
func TestRemoveOrphanNetworks(t *testing.T) {
func TestRemoveNetworks(t *testing.T) {
ctx := t.Context()
cutoff := time.Date(2026, time.April, 29, 20, 0, 0, 0, time.UTC)
client := &mockDockerClient{}
@@ -128,11 +128,13 @@ func TestRemoveOrphanNetworks(t *testing.T) {
{ID: "starting"},
{ID: "cached"},
{ID: "shared"},
{ID: "gone"},
{ID: "raced"},
}}, nil)
client.On("NetworkInspect", ctx, "orphan", mobyclient.NetworkInspectOptions{}).
Return(mobyclient.NetworkInspectResult{}, nil)
Return(mobyclient.NetworkInspectResult{}, nil).Twice()
client.On("NetworkInspect", ctx, "busy", mobyclient.NetworkInspectOptions{}).
Return(mobyclient.NetworkInspectResult{Network: network.Inspect{Containers: map[string]network.EndpointResource{"c": {}}}}, nil)
Return(mobyclient.NetworkInspectResult{Network: network.Inspect{Containers: map[string]network.EndpointResource{"c": {}}}}, nil).Twice()
client.On("NetworkInspect", ctx, "starting", mobyclient.NetworkInspectOptions{}).
Return(mobyclient.NetworkInspectResult{Network: network.Inspect{
Network: network.Network{Created: cutoff.Add(time.Second)},
@@ -142,13 +144,28 @@ func TestRemoveOrphanNetworks(t *testing.T) {
Return(mobyclient.NetworkInspectResult{Network: network.Inspect{Containers: map[string]network.EndpointResource{"0123456789abcdef": {}}}}, nil)
client.On("NetworkInspect", ctx, "shared", mobyclient.NetworkInspectOptions{}).
Return(mobyclient.NetworkInspectResult{Network: network.Inspect{Containers: map[string]network.EndpointResource{"0123456789abcdef": {}, "job": {}}}}, nil)
client.On("NetworkInspect", ctx, "gone", mobyclient.NetworkInspectOptions{}).
Return(mobyclient.NetworkInspectResult{}, cerrdefs.ErrNotFound).Twice()
client.On("NetworkInspect", ctx, "raced", mobyclient.NetworkInspectOptions{}).
Return(mobyclient.NetworkInspectResult{}, nil).Twice()
client.On("NetworkRemove", ctx, "raced", mobyclient.NetworkRemoveOptions{}).
Return(mobyclient.NetworkRemoveResult{}, cerrdefs.ErrNotFound).Twice()
client.On("NetworkRemove", ctx, "orphan", mobyclient.NetworkRemoveOptions{}).
Return(mobyclient.NetworkRemoveResult{}, nil)
Return(mobyclient.NetworkRemoveResult{}, nil).Twice()
client.On("NetworkDisconnect", ctx, "cached", mobyclient.NetworkDisconnectOptions{Container: "0123456789abcdef", Force: true}).
Return(nil).Once()
client.On("NetworkRemove", ctx, "cached", mobyclient.NetworkRemoveOptions{}).
Return(mobyclient.NetworkRemoveResult{}, nil).Once()
require.NoError(t, removeOrphanNetworks(ctx, client, "runner-1", "0123456789ab", cutoff))
client.On("NetworkList", ctx, mobyclient.NetworkListOptions{}).Return(mobyclient.NetworkListResult{Items: []network.Summary{
{ID: "orphan", Name: "job"},
{ID: "busy", Name: "job"},
{ID: "gone", Name: "job"},
{ID: "raced", Name: "job"},
{ID: "shared", Name: "other"},
}}, nil)
require.NoError(t, removeNetwork(ctx, client, "job"))
client.AssertExpectations(t)
}