mirror of
https://github.com/GopeedLab/gopeed.git
synced 2026-11-11 19:04:08 +00:00
fix(windows): release FFmpeg memory and capture native crashes
This commit is contained in:
1 parent
89dfbe4c45
commit
89bd3f3af0
17 files changed
+486
-17
No files matched your search
@@ -0,0 +1,26 @@
|
||||
package download
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime/debug"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestLogPanicPreservesPreviousCrash(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "crash.log")
|
||||
const previous = "previous crash traceback\n"
|
||||
if err := os.WriteFile(path, []byte(previous), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
logPanic(dir)
|
||||
t.Cleanup(func() { _ = debug.SetCrashOutput(nil, debug.CrashOptions{}) })
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if string(data) != previous {
|
||||
t.Fatalf("previous crash was overwritten: %q", data)
|
||||
}
|
||||
}
|
||||
@@ -2659,11 +2659,12 @@ func logPanic(logDir string) {
|
||||
if err := util.CreateDirIfNotExist(logDir); err != nil {
|
||||
return
|
||||
}
|
||||
f, err := os.Create(filepath.Join(logDir, "crash.log"))
|
||||
f, err := os.OpenFile(filepath.Join(logDir, "crash.log"), os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0600)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
debug.SetCrashOutput(f, debug.CrashOptions{})
|
||||
defer f.Close() // SetCrashOutput duplicates the descriptor.
|
||||
_ = debug.SetCrashOutput(f, debug.CrashOptions{})
|
||||
}
|
||||
|
||||
func (d *Downloader) assignFetcherManager(task *Task) error {
|
||||
|
||||
@@ -174,3 +174,12 @@ compilation memory, host buffers and JS allocations are additional to the
|
||||
linear-memory limit. Playback support depends on the selected codecs, not just
|
||||
the `.mp4` suffix. Platform execution and performance must be tested on target
|
||||
devices, especially mobile/32-bit devices.
|
||||
|
||||
On 64-bit Windows, instance linear memory uses a reserved virtual address range with
|
||||
pages committed as it grows. Module completion, errors and cancellation release
|
||||
this range directly through `VirtualFree`, rather than waiting for Go's garbage
|
||||
collector to reclaim large buffers. Growing an instance preserves its address
|
||||
without copying the previous buffer. The 512 MiB limit is unchanged; reservation
|
||||
does not commit that entire amount. Shared compilation remains resident after
|
||||
the first merge, so process memory is not expected to return exactly to the
|
||||
pre-FFmpeg baseline. This does not diagnose unrelated idle memory growth.
|
||||
@@ -0,0 +1,32 @@
|
||||
package ffmpeg
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
|
||||
"github.com/tetratelabs/wazero/experimental"
|
||||
)
|
||||
|
||||
// The runtime does not always invoke Free when a module's start function traps.
|
||||
// Keep host ownership until execution returns, including failed instantiation.
|
||||
func withOwnedMemory(ctx context.Context, allocator experimental.MemoryAllocator) (context.Context, func()) {
|
||||
var memories []*ownedLinearMemory
|
||||
ctx = experimental.WithMemoryAllocator(ctx, experimental.MemoryAllocatorFunc(
|
||||
func(cap, max uint64) experimental.LinearMemory {
|
||||
memory := &ownedLinearMemory{LinearMemory: allocator.Allocate(cap, max)}
|
||||
memories = append(memories, memory)
|
||||
return memory
|
||||
}))
|
||||
return ctx, func() {
|
||||
for _, memory := range memories {
|
||||
memory.Free()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type ownedLinearMemory struct {
|
||||
experimental.LinearMemory
|
||||
once sync.Once
|
||||
}
|
||||
|
||||
func (m *ownedLinearMemory) Free() { m.once.Do(m.LinearMemory.Free) }
|
||||
@@ -0,0 +1,7 @@
|
||||
//go:build !windows || (!amd64 && !arm64)
|
||||
|
||||
package ffmpeg
|
||||
|
||||
import "context"
|
||||
|
||||
func withLinearMemory(ctx context.Context) (context.Context, func()) { return ctx, func() {} }
|
||||
@@ -0,0 +1,88 @@
|
||||
package ffmpeg
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/hex"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/tetratelabs/wazero"
|
||||
"github.com/tetratelabs/wazero/experimental"
|
||||
)
|
||||
|
||||
// Verify the pinned runtime's allocator lifecycle on every development host:
|
||||
// host-owned buffers must not be released before execution ends, or retained
|
||||
// after failed instantiation, traps or cancellation.
|
||||
func TestLinearMemoryModuleLifecycle(t *testing.T) {
|
||||
testLinearMemoryModuleLifecycle(t, func(_, _ uint64) experimental.LinearMemory {
|
||||
return &testLinearMemory{}
|
||||
})
|
||||
}
|
||||
|
||||
type testLinearMemory struct{ buffer []byte }
|
||||
|
||||
func (m *testLinearMemory) Reallocate(size uint64) []byte {
|
||||
m.buffer = append(m.buffer, make([]byte, int(size)-len(m.buffer))...)
|
||||
return m.buffer
|
||||
}
|
||||
func (m *testLinearMemory) Free() { m.buffer = nil }
|
||||
|
||||
type trackedLinearMemory struct {
|
||||
experimental.LinearMemory
|
||||
freeCount int
|
||||
}
|
||||
|
||||
func (m *trackedLinearMemory) Free() {
|
||||
m.freeCount++
|
||||
m.LinearMemory.Free()
|
||||
}
|
||||
|
||||
func testLinearMemoryModuleLifecycle(t *testing.T, allocate experimental.MemoryAllocatorFunc) {
|
||||
t.Helper()
|
||||
for _, tt := range []struct {
|
||||
name string
|
||||
// A module with one memory (min=1/max=2 pages), optionally a start
|
||||
// function that traps or loops until its context is cancelled.
|
||||
wasm string
|
||||
wantErr bool
|
||||
}{
|
||||
{"close", "0061736d01000000050401010102", false},
|
||||
{"trap", "0061736d01000000010401600000030201000504010101020801000a05010300000b", true},
|
||||
{"cancel", "0061736d01000000010401600000030201000504010101020801000a0901070003400c000b0b", true},
|
||||
} {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
|
||||
defer cancel()
|
||||
var memory *trackedLinearMemory
|
||||
ctx, release := withOwnedMemory(ctx, experimental.MemoryAllocatorFunc(
|
||||
func(_, max uint64) experimental.LinearMemory {
|
||||
memory = &trackedLinearMemory{LinearMemory: allocate(0, max)}
|
||||
return memory
|
||||
}))
|
||||
defer release()
|
||||
rt := wazero.NewRuntimeWithConfig(ctx, wazero.NewRuntimeConfig().WithCloseOnContextDone(true))
|
||||
defer rt.Close(context.Background())
|
||||
binary, err := hex.DecodeString(tt.wasm)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
module, err := rt.Instantiate(ctx, binary)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Fatalf("instantiate: %v", err)
|
||||
}
|
||||
if !tt.wantErr {
|
||||
if memory == nil || memory.freeCount != 0 {
|
||||
t.Fatal("module did not allocate native memory")
|
||||
}
|
||||
if err := module.Close(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
release()
|
||||
release()
|
||||
if memory == nil || memory.freeCount != 1 {
|
||||
t.Fatal("host did not release memory exactly once")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
//go:build windows && (amd64 || arm64)
|
||||
|
||||
package ffmpeg
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"unsafe"
|
||||
|
||||
"github.com/tetratelabs/wazero/experimental"
|
||||
"golang.org/x/sys/windows"
|
||||
)
|
||||
|
||||
// Keep large, short-lived WASM buffers out of the Go heap. Closing a module
|
||||
// releases its committed pages immediately, without waiting for GC/scavenging.
|
||||
func withLinearMemory(ctx context.Context) (context.Context, func()) {
|
||||
return withOwnedMemory(ctx, experimental.MemoryAllocatorFunc(
|
||||
func(_, max uint64) experimental.LinearMemory {
|
||||
return &windowsLinearMemory{max: max}
|
||||
}))
|
||||
}
|
||||
|
||||
// Reserve the address range once, committing zero-filled pages on demand. This
|
||||
// avoids both copying the previous buffer and temporarily retaining two large
|
||||
// Go allocations when memory.grow runs. The backing address never changes.
|
||||
type windowsLinearMemory struct {
|
||||
address uintptr
|
||||
max uint64
|
||||
size uint64
|
||||
freed bool
|
||||
}
|
||||
|
||||
func (m *windowsLinearMemory) Reallocate(size uint64) []byte {
|
||||
if m.freed || size > m.max || m.max > uint64(^uint(0)>>1) {
|
||||
return nil
|
||||
}
|
||||
if size == 0 {
|
||||
return []byte{}
|
||||
}
|
||||
if m.address == 0 {
|
||||
address, err := windows.VirtualAlloc(0, uintptr(m.max), windows.MEM_RESERVE, windows.PAGE_READWRITE)
|
||||
if err != nil {
|
||||
panic(fmt.Errorf("ffmpeg: reserve WASM memory: %w", err))
|
||||
}
|
||||
m.address = address
|
||||
}
|
||||
if size > m.size {
|
||||
_, err := windows.VirtualAlloc(m.address, uintptr(size), windows.MEM_COMMIT, windows.PAGE_READWRITE)
|
||||
if err != nil {
|
||||
if m.size == 0 {
|
||||
// Instantiation cannot proceed without the minimum memory. Wazero
|
||||
// has not taken ownership yet, so release the reservation here.
|
||||
m.Free()
|
||||
panic(fmt.Errorf("ffmpeg: commit WASM memory: %w", err))
|
||||
}
|
||||
return nil // memory.grow reports failure to the guest.
|
||||
}
|
||||
m.size = size
|
||||
}
|
||||
return unsafe.Slice((*byte)(unsafe.Pointer(m.address)), int(size))
|
||||
}
|
||||
|
||||
func (m *windowsLinearMemory) Free() {
|
||||
if m.address != 0 {
|
||||
_ = windows.VirtualFree(m.address, 0, windows.MEM_RELEASE)
|
||||
m.address = 0
|
||||
}
|
||||
m.size = 0
|
||||
m.freed = true
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
//go:build windows && (amd64 || arm64)
|
||||
|
||||
package ffmpeg
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"unsafe"
|
||||
|
||||
"github.com/tetratelabs/wazero/experimental"
|
||||
"golang.org/x/sys/windows"
|
||||
)
|
||||
|
||||
func TestWindowsLinearMemoryGrowthAndRelease(t *testing.T) {
|
||||
const page = 65536
|
||||
m := &windowsLinearMemory{max: 4 * page}
|
||||
defer m.Free()
|
||||
first := m.Reallocate(page)
|
||||
if len(first) != page {
|
||||
t.Fatalf("initial allocation: %d", len(first))
|
||||
}
|
||||
first[0], first[page-1] = 17, 29
|
||||
address := m.address
|
||||
grown := m.Reallocate(3 * page)
|
||||
if unsafe.SliceData(first) != unsafe.SliceData(grown) || grown[0] != 17 || grown[page-1] != 29 {
|
||||
t.Fatal("growing memory moved or lost existing bytes")
|
||||
}
|
||||
for _, b := range grown[page:] {
|
||||
if b != 0 {
|
||||
t.Fatal("newly committed memory is not zero-filled")
|
||||
}
|
||||
}
|
||||
if m.Reallocate(5*page) != nil || m.address != address || m.size != 3*page {
|
||||
t.Fatal("failed growth changed the existing allocation")
|
||||
}
|
||||
m.Free()
|
||||
m.Free()
|
||||
if m.address != 0 || m.Reallocate(page) != nil {
|
||||
t.Fatal("freed memory can still be used")
|
||||
}
|
||||
var info windows.MemoryBasicInformation
|
||||
if err := windows.VirtualQuery(address, &info, unsafe.Sizeof(info)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
const memFree = 0x10000 // MEMORY_BASIC_INFORMATION.State: MEM_FREE.
|
||||
if info.State != memFree {
|
||||
t.Fatalf("pages remain reserved after Free: state=%x", info.State)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWindowsLinearMemoryModuleLifecycle(t *testing.T) {
|
||||
testLinearMemoryModuleLifecycle(t, func(_, max uint64) experimental.LinearMemory {
|
||||
return &windowsLinearMemory{max: max}
|
||||
})
|
||||
}
|
||||
@@ -151,7 +151,9 @@ func RunAcquired(ctx context.Context, video, audio Input, out io.Writer, format
|
||||
files := &inputFS{files: map[string]Input{"video": video, "audio": audio}}
|
||||
fsconfig := wazero.NewFSConfig().(sysfs.FSConfig).WithSysFSMount(files, "/inputs")
|
||||
logs := &tailLog{}
|
||||
rc, err := wasm.Run(ctx, shared.runtime, shared.module, wasm.Args{
|
||||
runCtx, freeMemory := withLinearMemory(ctx)
|
||||
defer freeMemory()
|
||||
rc, err := wasm.Run(runCtx, shared.runtime, shared.module, wasm.Args{
|
||||
Name: "ffmpeg", Args: args, Stdout: out, Stderr: logs,
|
||||
Config: func(c wazero.ModuleConfig) wazero.ModuleConfig { return c.WithFSConfig(fsconfig) },
|
||||
})
|
||||
|
||||
Reference in new issue
Block a user