mirror of
https://gitea.com/gitea/runner.git
synced 2026-11-06 15:20:29 +00:00
silverwind
0d84ec6dab
Job teardown ran every step on one shared minute. When removing the job container took longer, the network removal ran on an expired context and the network stayed behind until the orphan reclaim, which only takes networks older than `workdir_cleanup_age`, 24h by default. All that time it held a subnet of the daemon's address pool. The job and service containers are now removed in parallel, and the docker proxy sweep, the network and the volumes get a fresh budget after them. The idle reclaim takes unused networks and volumes once they are 10 minutes old, and network removal treats a network that is already gone as removed. Reviewed-on: https://gitea.com/gitea/runner/pulls/1318 Reviewed-by: bircni <[email protected]> Co-authored-by: silverwind <[email protected]>
374 lines
12 KiB
Go
374 lines
12 KiB
Go
// Copyright 2026 The Gitea Authors. All rights reserved.
|
|
// SPDX-License-Identifier: MIT
|
|
|
|
package run
|
|
|
|
import (
|
|
"context"
|
|
"os"
|
|
"path/filepath"
|
|
"strconv"
|
|
"testing"
|
|
"time"
|
|
|
|
"gitea.com/gitea/runner/act/container"
|
|
"gitea.com/gitea/runner/internal/pkg/config"
|
|
"gitea.com/gitea/runner/internal/pkg/labels"
|
|
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
)
|
|
|
|
func TestRunnerCleanupStaleTaskDirs(t *testing.T) {
|
|
now := time.Date(2026, time.April, 29, 20, 0, 0, 0, time.UTC)
|
|
workdirRoot := filepath.Join(t.TempDir(), "workspace")
|
|
require.NoError(t, os.MkdirAll(workdirRoot, 0o700))
|
|
|
|
oldTask := filepath.Join(workdirRoot, "1001")
|
|
freshTask := filepath.Join(workdirRoot, "1002")
|
|
nonTask := filepath.Join(workdirRoot, "shared")
|
|
alphaNumericTask := filepath.Join(workdirRoot, "123abc")
|
|
for _, path := range []string{oldTask, freshTask, nonTask, alphaNumericTask} {
|
|
require.NoError(t, os.MkdirAll(path, 0o700))
|
|
}
|
|
|
|
require.NoError(t, os.Chtimes(oldTask, now.Add(-3*time.Hour), now.Add(-3*time.Hour)))
|
|
require.NoError(t, os.Chtimes(freshTask, now.Add(-30*time.Minute), now.Add(-30*time.Minute)))
|
|
require.NoError(t, os.Chtimes(nonTask, now.Add(-5*time.Hour), now.Add(-5*time.Hour)))
|
|
require.NoError(t, os.Chtimes(alphaNumericTask, now.Add(-5*time.Hour), now.Add(-5*time.Hour)))
|
|
|
|
r := &Runner{
|
|
cfg: &config.Config{
|
|
Runner: config.Runner{
|
|
WorkdirCleanupAge: 2 * time.Hour,
|
|
},
|
|
},
|
|
now: func() time.Time { return now },
|
|
}
|
|
|
|
r.cleanupStaleDirs(t.Context(), workdirRoot, isTaskIDDir)
|
|
|
|
assert.NoDirExists(t, oldTask)
|
|
assert.DirExists(t, freshTask)
|
|
assert.DirExists(t, nonTask)
|
|
assert.DirExists(t, alphaNumericTask)
|
|
}
|
|
|
|
// TestRunnerOnIdleCleansStaleHostScratchDirs covers the host-mode leak path:
|
|
// a per-job scratch dir (16 hex chars) left behind by a timed-out cleanup must
|
|
// be reclaimed, while the shared tool_cache and operator data are preserved.
|
|
func TestRunnerOnIdleCleansStaleHostScratchDirs(t *testing.T) {
|
|
now := time.Date(2026, time.April, 29, 20, 0, 0, 0, time.UTC)
|
|
hostRoot := filepath.Join(t.TempDir(), "act")
|
|
require.NoError(t, os.MkdirAll(hostRoot, 0o700))
|
|
|
|
staleScratch := filepath.Join(hostRoot, "0123456789abcdef") // 16 hex
|
|
freshScratch := filepath.Join(hostRoot, "fedcba9876543210")
|
|
toolCache := filepath.Join(hostRoot, "tool_cache")
|
|
operatorData := filepath.Join(hostRoot, "keep-me")
|
|
for _, path := range []string{staleScratch, freshScratch, toolCache, operatorData} {
|
|
require.NoError(t, os.MkdirAll(path, 0o700))
|
|
}
|
|
require.NoError(t, os.Chtimes(staleScratch, now.Add(-48*time.Hour), now.Add(-48*time.Hour)))
|
|
require.NoError(t, os.Chtimes(freshScratch, now.Add(-10*time.Minute), now.Add(-10*time.Minute)))
|
|
require.NoError(t, os.Chtimes(toolCache, now.Add(-72*time.Hour), now.Add(-72*time.Hour)))
|
|
require.NoError(t, os.Chtimes(operatorData, now.Add(-72*time.Hour), now.Add(-72*time.Hour)))
|
|
|
|
r := &Runner{
|
|
cfg: &config.Config{
|
|
Host: config.Host{WorkdirParent: hostRoot},
|
|
Runner: config.Runner{
|
|
WorkdirCleanupAge: 24 * time.Hour,
|
|
IdleCleanupInterval: time.Minute,
|
|
},
|
|
},
|
|
now: func() time.Time { return now },
|
|
}
|
|
|
|
r.OnIdle(t.Context())
|
|
|
|
assert.NoDirExists(t, staleScratch) // stale scratch reclaimed
|
|
assert.DirExists(t, freshScratch) // within cleanup age, kept
|
|
assert.DirExists(t, toolCache) // shared cache, never a scratch match
|
|
assert.DirExists(t, operatorData) // non-hex name, untouched
|
|
}
|
|
|
|
func TestIsHostScratchDir(t *testing.T) {
|
|
assert.True(t, isHostScratchDir("0123456789abcdef"))
|
|
assert.True(t, isHostScratchDir("ffffffffffffffff"))
|
|
assert.False(t, isHostScratchDir("tool_cache"))
|
|
assert.False(t, isHostScratchDir("0123456789ABCDEF")) // hex.EncodeToString is lowercase
|
|
assert.False(t, isHostScratchDir("0123456789abcde")) // 15 chars
|
|
assert.False(t, isHostScratchDir("0123456789abcdef0")) // 17 chars
|
|
assert.False(t, isHostScratchDir("123"))
|
|
}
|
|
|
|
func TestRunnerCleanupStaleTaskDirsMissingRoot(t *testing.T) {
|
|
r := &Runner{
|
|
cfg: &config.Config{
|
|
Runner: config.Runner{WorkdirCleanupAge: time.Hour},
|
|
},
|
|
now: time.Now,
|
|
}
|
|
|
|
// Must be a silent no-op rather than a warning or panic when the root
|
|
// has not yet been created (e.g. the runner has never executed a task).
|
|
r.cleanupStaleDirs(t.Context(), filepath.Join(t.TempDir(), "missing"), isTaskIDDir)
|
|
}
|
|
|
|
func TestRunnerCleanupStaleTaskDirsHonorsContext(t *testing.T) {
|
|
now := time.Date(2026, time.April, 29, 20, 0, 0, 0, time.UTC)
|
|
workdirRoot := filepath.Join(t.TempDir(), "workspace")
|
|
require.NoError(t, os.MkdirAll(workdirRoot, 0o700))
|
|
|
|
for i := 1001; i <= 1003; i++ {
|
|
dir := filepath.Join(workdirRoot, strconv.Itoa(i))
|
|
require.NoError(t, os.MkdirAll(dir, 0o700))
|
|
require.NoError(t, os.Chtimes(dir, now.Add(-3*time.Hour), now.Add(-3*time.Hour)))
|
|
}
|
|
|
|
ctx, cancel := context.WithCancel(t.Context())
|
|
cancel()
|
|
|
|
r := &Runner{
|
|
cfg: &config.Config{
|
|
Runner: config.Runner{WorkdirCleanupAge: time.Hour},
|
|
},
|
|
now: func() time.Time { return now },
|
|
}
|
|
|
|
r.cleanupStaleDirs(ctx, workdirRoot, isTaskIDDir)
|
|
|
|
for i := 1001; i <= 1003; i++ {
|
|
assert.DirExists(t, filepath.Join(workdirRoot, strconv.Itoa(i)))
|
|
}
|
|
}
|
|
|
|
func TestRunnerShouldRunIdleCleanupThrottles(t *testing.T) {
|
|
now := time.Date(2026, time.April, 29, 20, 0, 0, 0, time.UTC)
|
|
r := &Runner{
|
|
cfg: &config.Config{
|
|
Container: config.Container{
|
|
BindWorkdir: true,
|
|
},
|
|
Runner: config.Runner{
|
|
WorkdirCleanupAge: 24 * time.Hour,
|
|
IdleCleanupInterval: time.Hour,
|
|
},
|
|
},
|
|
now: func() time.Time { return now },
|
|
}
|
|
|
|
assert.True(t, r.shouldRunIdleCleanup())
|
|
|
|
now = now.Add(30 * time.Minute)
|
|
assert.False(t, r.shouldRunIdleCleanup())
|
|
|
|
now = now.Add(31 * time.Minute)
|
|
assert.True(t, r.shouldRunIdleCleanup())
|
|
}
|
|
|
|
func TestRunnerShouldRunIdleCleanupSkipsWhenJobRunning(t *testing.T) {
|
|
r := &Runner{
|
|
cfg: &config.Config{
|
|
Container: config.Container{
|
|
BindWorkdir: true,
|
|
},
|
|
Runner: config.Runner{
|
|
WorkdirCleanupAge: 24 * time.Hour,
|
|
IdleCleanupInterval: time.Minute,
|
|
},
|
|
},
|
|
now: time.Now,
|
|
}
|
|
r.runningCount.Store(1)
|
|
|
|
assert.False(t, r.shouldRunIdleCleanup())
|
|
}
|
|
|
|
// Idle cleanup runs regardless of bind_workdir: host mode (bind_workdir off)
|
|
// still leaves per-job scratch dirs that the sweep must reclaim.
|
|
func TestRunnerShouldRunIdleCleanupRunsWithoutBindWorkdir(t *testing.T) {
|
|
now := time.Date(2026, time.April, 29, 20, 0, 0, 0, time.UTC)
|
|
r := &Runner{
|
|
cfg: &config.Config{
|
|
Runner: config.Runner{
|
|
WorkdirCleanupAge: 24 * time.Hour,
|
|
IdleCleanupInterval: time.Minute,
|
|
},
|
|
},
|
|
now: func() time.Time { return now },
|
|
}
|
|
|
|
assert.True(t, r.shouldRunIdleCleanup())
|
|
}
|
|
|
|
func TestRunnerShouldRunIdleCleanupSkipsWhenDisabled(t *testing.T) {
|
|
now := time.Date(2026, time.April, 29, 20, 0, 0, 0, time.UTC)
|
|
|
|
t.Run("cleanup age disabled", func(t *testing.T) {
|
|
r := &Runner{
|
|
cfg: &config.Config{
|
|
Container: config.Container{
|
|
BindWorkdir: true,
|
|
},
|
|
Runner: config.Runner{
|
|
WorkdirCleanupAge: -1,
|
|
IdleCleanupInterval: time.Minute,
|
|
},
|
|
},
|
|
now: func() time.Time { return now },
|
|
}
|
|
|
|
assert.False(t, r.shouldRunIdleCleanup())
|
|
})
|
|
|
|
t.Run("idle interval disabled", func(t *testing.T) {
|
|
r := &Runner{
|
|
cfg: &config.Config{
|
|
Container: config.Container{
|
|
BindWorkdir: true,
|
|
},
|
|
Runner: config.Runner{
|
|
WorkdirCleanupAge: 24 * time.Hour,
|
|
IdleCleanupInterval: -1,
|
|
},
|
|
},
|
|
now: func() time.Time { return now },
|
|
}
|
|
|
|
assert.False(t, r.shouldRunIdleCleanup())
|
|
})
|
|
}
|
|
|
|
// TestRunnerOnIdleIntegratesCleanup wires the full OnIdle entry point and
|
|
// confirms it walks workdir_parent (after the leading-slash trim that
|
|
// matches the production path construction) and removes stale numeric dirs.
|
|
func TestRunnerOnIdleIntegratesCleanup(t *testing.T) {
|
|
now := time.Date(2026, time.April, 29, 20, 0, 0, 0, time.UTC)
|
|
root := t.TempDir()
|
|
stale := filepath.Join(root, "1234")
|
|
require.NoError(t, os.MkdirAll(stale, 0o700))
|
|
require.NoError(t, os.Chtimes(stale, now.Add(-48*time.Hour), now.Add(-48*time.Hour)))
|
|
|
|
r := &Runner{
|
|
cfg: &config.Config{
|
|
Container: config.Container{
|
|
BindWorkdir: true,
|
|
WorkdirParent: root, // leading slash absent, OnIdle reattaches it
|
|
},
|
|
Runner: config.Runner{
|
|
WorkdirCleanupAge: 24 * time.Hour,
|
|
IdleCleanupInterval: time.Minute,
|
|
},
|
|
},
|
|
now: func() time.Time { return now },
|
|
}
|
|
|
|
r.OnIdle(t.Context())
|
|
|
|
assert.NoDirExists(t, stale)
|
|
}
|
|
|
|
// TestRunnerOnIdleSkipsWhenAlreadyCancelled verifies a pre-cancelled ctx
|
|
// short-circuits cleanup before any directory entry is touched.
|
|
func TestRunnerOnIdleSkipsWhenAlreadyCancelled(t *testing.T) {
|
|
now := time.Date(2026, time.April, 29, 20, 0, 0, 0, time.UTC)
|
|
root := t.TempDir()
|
|
stale := filepath.Join(root, "1234")
|
|
require.NoError(t, os.MkdirAll(stale, 0o700))
|
|
require.NoError(t, os.Chtimes(stale, now.Add(-48*time.Hour), now.Add(-48*time.Hour)))
|
|
|
|
r := &Runner{
|
|
cfg: &config.Config{
|
|
Container: config.Container{
|
|
BindWorkdir: true,
|
|
WorkdirParent: root,
|
|
},
|
|
Runner: config.Runner{
|
|
WorkdirCleanupAge: 24 * time.Hour,
|
|
IdleCleanupInterval: time.Minute,
|
|
},
|
|
},
|
|
now: func() time.Time { return now },
|
|
}
|
|
|
|
ctx, cancel := context.WithCancel(t.Context())
|
|
cancel()
|
|
r.OnIdle(ctx)
|
|
|
|
assert.DirExists(t, stale)
|
|
}
|
|
|
|
// The idle sweep reclaims the docker networks of jobs this runner did not live to tear down,
|
|
// and stays out of the way of runners that share the daemon but not the registration.
|
|
func TestRunnerOnIdleRemovesOrphanNetworks(t *testing.T) {
|
|
now := time.Date(2026, time.April, 29, 20, 0, 0, 0, time.UTC)
|
|
cfg := &config.Config{
|
|
Container: config.Container{RequireDocker: true},
|
|
Runner: config.Runner{
|
|
WorkdirCleanupAge: 24 * time.Hour,
|
|
IdleCleanupInterval: time.Minute,
|
|
},
|
|
}
|
|
|
|
var swept []string
|
|
var detached []string
|
|
var sweptVolumes []string
|
|
var sweptCutoff time.Time
|
|
origRemoveOrphanNetworks := removeOrphanNetworks
|
|
removeOrphanNetworks = func(ctx context.Context, runnerUUID, detach string, createdBefore time.Time) error {
|
|
_, hasDeadline := ctx.Deadline()
|
|
assert.True(t, hasDeadline)
|
|
swept = append(swept, runnerUUID)
|
|
detached = append(detached, detach)
|
|
sweptCutoff = createdBefore
|
|
return nil
|
|
}
|
|
t.Cleanup(func() { removeOrphanNetworks = origRemoveOrphanNetworks })
|
|
origRemoveOrphanJobVolumes := removeOrphanJobVolumes
|
|
removeOrphanJobVolumes = func(_ context.Context, runnerUUID string, createdBefore time.Time) error {
|
|
sweptVolumes = append(sweptVolumes, runnerUUID)
|
|
assert.Equal(t, now.Add(-orphanDockerResourceAge), createdBefore)
|
|
return nil
|
|
}
|
|
t.Cleanup(func() { removeOrphanJobVolumes = origRemoveOrphanJobVolumes })
|
|
var sweptPods []string
|
|
origRemoveOrphanKubernetesResources := removeOrphanKubernetesResources
|
|
removeOrphanKubernetesResources = func(ctx context.Context, options container.KubernetesOptions, createdBefore time.Time) error {
|
|
_, hasDeadline := ctx.Deadline()
|
|
assert.True(t, hasDeadline)
|
|
sweptPods = append(sweptPods, options.RunnerUUID)
|
|
assert.Equal(t, now.Add(-24*time.Hour), createdBefore)
|
|
return nil
|
|
}
|
|
t.Cleanup(func() { removeOrphanKubernetesResources = origRemoveOrphanKubernetesResources })
|
|
kubernetesLabel, err := labels.Parse("k8s:kubernetes://node")
|
|
require.NoError(t, err)
|
|
|
|
r := &Runner{uuid: "runner-1", cfg: cfg, labels: labels.Labels{kubernetesLabel}, now: func() time.Time { return now }}
|
|
r.isolatedCacheContainer = func() string { return "a1b2c3d4e5f6" }
|
|
r.OnIdle(t.Context())
|
|
assert.Equal(t, []string{"runner-1"}, swept)
|
|
assert.Equal(t, swept, sweptVolumes)
|
|
// a network of a job starting during the pass is younger than this and so out of scope
|
|
assert.Equal(t, now.Add(-orphanDockerResourceAge), sweptCutoff)
|
|
|
|
// a host-only runner has no daemon to sweep
|
|
origDockerReachable := dockerReachable
|
|
dockerReachable = func(context.Context) bool { return false }
|
|
t.Cleanup(func() { dockerReachable = origDockerReachable })
|
|
hostOnly := &Runner{uuid: "runner-2", cfg: &config.Config{Runner: cfg.Runner}, now: func() time.Time { return now }}
|
|
hostOnly.isolatedCacheContainer = func() string { return "" }
|
|
hostOnly.OnIdle(t.Context())
|
|
assert.Equal(t, []string{"runner-1"}, swept)
|
|
assert.Equal(t, swept, sweptVolumes)
|
|
|
|
dockerReachable = func(context.Context) bool { return true }
|
|
now = now.Add(time.Minute)
|
|
hostOnly.OnIdle(t.Context())
|
|
assert.Equal(t, []string{"runner-1", "runner-2"}, sweptVolumes)
|
|
assert.Equal(t, []string{"a1b2c3d4e5f6", ""}, detached)
|
|
assert.Equal(t, []string{"runner-1"}, sweptPods)
|
|
}
|