Merge latest PR #1512 into 2.0.0-beta.2

This commit is contained in:
Levi committed 2026-09-13 22:25:20 +08:00
commit a3f6ec5885
6 files changed
+144 -74

No files matched your search

+9 -8
View File
@@ -92,14 +92,15 @@ server validator, same-size remote changes cannot be reliably detected.
Content encoding is forced to identity. Authentication/status failures and Content encoding is forced to identity. Authentication/status failures and
malformed range responses are errors. There is no full-file seek fallback. malformed range responses are errors. There is no full-file seek fallback.
Sequential JS inputs are drained independently into temporary disk ring Sequential JS inputs are drained independently into memory ring buffers that
buffers (up to **64 MiB per input**, independent of total media size). This grow on demand up to **32 MiB per input**, independent of total media size.
allows shared upstream demultiplexers to make progress when FFmpeg reads one No temporary files are used. This does not enable seeking. Full buffers apply
track ahead of another. It does not enable seeking. Full buffers apply async async backpressure until space is available or the operation is cancelled.
backpressure; a full buffer stalled for 30 seconds fails explicitly. Extensions Extensions must propagate backpressure through their upstream queues and
must also bound their own upstream queues. Disk buffers are removed on normal network reads. Independent upstream track requests avoid blocking one track
completion, failure, cancellation and engine shutdown; abrupt process death behind another in a shared response. Buffers are released on close.
can leave temporary files in the OS temporary directory. Blob output writes also wait asynchronously for bounded downstream capacity,
so a slow consumer does not block the extension event loop.
At most two WASM merges execute concurrently per process, with a 512 MiB At most two WASM merges execute concurrently per process, with a 512 MiB
linear-memory limit per instance. Compilation is shared and lazy; the upstream linear-memory limit per instance. Compilation is shared and lazy; the upstream
+37 -14
View File
@@ -7,7 +7,6 @@ import (
"io" "io"
"net/http" "net/http"
"net/http/httptest" "net/http/httptest"
"os"
"strings" "strings"
"sync/atomic" "sync/atomic"
"testing" "testing"
@@ -120,7 +119,8 @@ func TestStreamRing(t *testing.T) {
s := NewStreamInput(context.Background()) s := NewStreamInput(context.Background())
defer s.Close() defer s.Close()
// Start near the boundary to exercise wrap-around without a huge fixture. // Start near the boundary to exercise wrap-around without a huge fixture.
s.read = StreamBufferLimit - 3 s.buffer = make([]byte, 256*1024)
s.read = int64(len(s.buffer)) - 3
s.written = s.read s.written = s.read
if _, err := s.Write([]byte("abcdefgh")); err != nil { if _, err := s.Write([]byte("abcdefgh")); err != nil {
t.Fatal(err) t.Fatal(err)
@@ -130,12 +130,11 @@ func TestStreamRing(t *testing.T) {
if err != nil || string(b) != "abcdefgh" { if err != nil || string(b) != "abcdefgh" {
t.Fatalf("%q %v", b, err) t.Fatalf("%q %v", b, err)
} }
name := s.file.Name()
if err = s.Close(); err != nil { if err = s.Close(); err != nil {
t.Fatal(err) t.Fatal(err)
} }
if _, err = os.Stat(name); err == nil { if s.buffer != nil {
t.Fatal("temporary file leaked") t.Fatal("buffer retained after close")
} }
} }
@@ -159,14 +158,7 @@ func TestStreamBackpressureAndCancel(t *testing.T) {
defer cancel() defer cancel()
s := NewStreamInput(ctx) s := NewStreamInput(ctx)
defer s.Close() defer s.Close()
f, err := os.CreateTemp(t.TempDir(), "ring") s.buffer = make([]byte, StreamBufferLimit)
if err != nil {
t.Fatal(err)
}
s.file = f
if err = f.Truncate(StreamBufferLimit); err != nil {
t.Fatal(err)
}
s.written = StreamBufferLimit s.written = StreamBufferLimit
finished := make(chan error, 1) finished := make(chan error, 1)
go func() { _, err := s.Write([]byte{42}); finished <- err }() go func() { _, err := s.Write([]byte{42}); finished <- err }()
@@ -175,7 +167,7 @@ func TestStreamBackpressureAndCancel(t *testing.T) {
t.Fatalf("did not apply backpressure: %v", err) t.Fatalf("did not apply backpressure: %v", err)
case <-time.After(10 * time.Millisecond): case <-time.After(10 * time.Millisecond):
} }
if _, err = s.Read(make([]byte, 1)); err != nil { if _, err := s.Read(make([]byte, 1)); err != nil {
t.Fatal(err) t.Fatal(err)
} }
select { select {
@@ -197,3 +189,34 @@ func TestStreamBackpressureAndCancel(t *testing.T) {
t.Fatal("writer did not cancel") t.Fatal("writer did not cancel")
} }
} }
func TestStreamMemoryGrowthAndWrap(t *testing.T) {
s := NewStreamInput(context.Background())
defer s.Close()
if len(s.buffer) != 0 {
t.Fatal("buffer allocated before first write")
}
a := bytes.Repeat([]byte{1}, 200*1024)
b := bytes.Repeat([]byte{2}, 180*1024)
s.Write(a)
consumed := make([]byte, 180*1024)
if _, err := io.ReadFull(s, consumed); err != nil {
t.Fatal(err)
}
s.Write(b) // Wrap in the initial 256 KiB allocation.
c := bytes.Repeat([]byte{3}, 300*1024)
s.Write(c) // Grow while unread data wraps around the old ring.
s.End(nil)
got, err := io.ReadAll(s)
want := append(append(append([]byte{}, a[len(consumed):]...), b...), c...)
if err != nil || !bytes.Equal(got, want) {
t.Fatalf("growth corrupted data: %v", err)
}
if len(s.buffer) > StreamBufferLimit {
t.Fatal("buffer exceeded limit")
}
s.Close()
if s.buffer != nil {
t.Fatal("buffer retained on close")
}
}
+27 -43
View File
@@ -4,21 +4,19 @@ import (
"context" "context"
"errors" "errors"
"io" "io"
"os"
"sync" "sync"
"time"
) )
// A bounded disk ring lets both JS inputs drain independently, including when // A bounded memory ring lets both JS inputs drain independently, including when
// they share a SABR producer. It does not make a sequential input seekable. // they share a SABR producer. It does not make a sequential input seekable.
// A full ring applies backpressure. A prolonged stall fails instead of // A full ring waits for consumption or cancellation. Producers must propagate
// deadlocking a shared upstream or growing forever. // backpressure and allow each track to make progress independently.
const StreamBufferLimit = 64 * 1024 * 1024 const StreamBufferLimit = 32 * 1024 * 1024
type StreamInput struct { type StreamInput struct {
ctx context.Context ctx context.Context
mu sync.Mutex mu sync.Mutex
file *os.File buffer []byte
read, written int64 read, written int64
ended, closed bool ended, closed bool
err error err error
@@ -36,24 +34,13 @@ func (s *StreamInput) Write(p []byte) (int, error) {
if len(p) > StreamBufferLimit { if len(p) > StreamBufferLimit {
return 0, errors.New("ffmpeg: input chunk exceeds buffer capacity") return 0, errors.New("ffmpeg: input chunk exceeds buffer capacity")
} }
var timer *time.Timer
defer func() {
if timer != nil {
timer.Stop()
}
}()
for int64(len(p)) > StreamBufferLimit-(s.written-s.read) && !s.closed && !s.ended { for int64(len(p)) > StreamBufferLimit-(s.written-s.read) && !s.closed && !s.ended {
if timer == nil {
timer = time.NewTimer(30 * time.Second)
}
s.mu.Unlock() s.mu.Unlock()
var err error var err error
select { select {
case <-s.space: case <-s.space:
case <-s.ctx.Done(): case <-s.ctx.Done():
err = s.ctx.Err() err = s.ctx.Err()
case <-timer.C:
err = errors.New("ffmpeg: input buffer stalled for 30 seconds; upstream tracks may be too far apart")
} }
s.mu.Lock() s.mu.Lock()
if err != nil { if err != nil {
@@ -69,23 +56,25 @@ func (s *StreamInput) Write(p []byte) (int, error) {
if len(p) == 0 { if len(p) == 0 {
return 0, nil return 0, nil
} }
if s.file == nil { // Grow only as needed. Rebase unread bytes when the ring grows.
f, err := os.CreateTemp("", "gopeed-ffmpeg-*") need := int(s.written-s.read) + len(p)
if err != nil { if need > len(s.buffer) {
return 0, err capacity := min(StreamBufferLimit, max(256*1024, max(need, len(s.buffer)*2)))
} buffer := make([]byte, capacity)
s.file = f unread := int(s.written - s.read)
} if unread > 0 {
off := s.written % StreamBufferLimit off := int(s.read % int64(len(s.buffer)))
first := min(len(p), StreamBufferLimit-int(off)) first := min(unread, len(s.buffer)-off)
if _, err := s.file.WriteAt(p[:first], off); err != nil { copy(buffer, s.buffer[off:off+first])
return 0, err copy(buffer[first:], s.buffer[:unread-first])
}
if first < len(p) {
if _, err := s.file.WriteAt(p[first:], 0); err != nil {
return 0, err
} }
s.buffer = buffer
s.read, s.written = 0, int64(unread)
} }
off := int(s.written % int64(len(s.buffer)))
first := min(len(p), len(s.buffer)-off)
copy(s.buffer[off:], p[:first])
copy(s.buffer, p[first:])
s.written += int64(len(p)) s.written += int64(len(p))
select { select {
case s.wake <- struct{}{}: case s.wake <- struct{}{}:
@@ -130,16 +119,16 @@ func (s *StreamInput) Read(p []byte) (int, error) {
return 0, err return 0, err
} }
if s.read < s.written { if s.read < s.written {
off := s.read % StreamBufferLimit off := s.read % int64(len(s.buffer))
n := min(int64(len(p)), s.written-s.read, StreamBufferLimit-off) n := min(int64(len(p)), s.written-s.read, int64(len(s.buffer))-off)
count, err := s.file.ReadAt(p[:n], off) count := copy(p, s.buffer[off:off+n])
s.read += int64(count) s.read += int64(count)
s.mu.Unlock() s.mu.Unlock()
select { select {
case s.space <- struct{}{}: case s.space <- struct{}{}:
default: default:
} }
return count, err return count, nil
} }
ended := s.ended ended := s.ended
s.mu.Unlock() s.mu.Unlock()
@@ -166,12 +155,7 @@ func (s *StreamInput) Close() error {
case s.wake <- struct{}{}: case s.wake <- struct{}{}:
default: default:
} }
if s.file != nil { s.buffer = nil
name := s.file.Name()
s.file.Close()
s.file = nil
return os.Remove(name)
}
return nil return nil
} }
func (*StreamInput) Seek(int64, int) (int64, error) { return 0, ErrSeek } func (*StreamInput) Seek(int64, int) (int64, error) { return 0, ErrSeek }
+9 -4
View File
@@ -86,18 +86,23 @@ func Enable(runtime *goja.Runtime, loop *eventloop.EventLoop, cfg *Config) error
} }
if err := runtime.Set("__gopeed_blob_pipe_chunk", func(call goja.FunctionCall) goja.Value { if err := runtime.Set("__gopeed_blob_pipe_chunk", func(call goja.FunctionCall) goja.Value {
id := call.Argument(0).String() id := call.Argument(0).String()
ok := blobPipes.Push(id, exportBytes(call.Argument(1))) chunk := append([]byte(nil), exportBytes(call.Argument(1))...)
return runtime.ToValue(ok) promise, resolve, _ := runtime.NewPromise()
go func() {
ok := blobPipes.Push(id, chunk)
loop.RunOnLoop(func(vm *goja.Runtime) { resolve(ok) })
}()
return runtime.ToValue(promise)
}); err != nil { }); err != nil {
return err return err
} }
if err := runtime.Set("__gopeed_blob_pipe_close", func(id string) { if err := runtime.Set("__gopeed_blob_pipe_close", func(id string) {
blobPipes.Close(id, nil) go blobPipes.Close(id, nil)
}); err != nil { }); err != nil {
return err return err
} }
if err := runtime.Set("__gopeed_blob_pipe_error", func(id string, message string) { if err := runtime.Set("__gopeed_blob_pipe_error", func(id string, message string) {
blobPipes.Close(id, errors.New(message)) go blobPipes.Close(id, errors.New(message))
}); err != nil { }); err != nil {
return err return err
} }
@@ -269,3 +269,57 @@ func newStreamTestEngine(t *testing.T, created chan<- createdObjectURL) (*engine
}) })
return e, e.Close return e, e.Close
} }
// A stalled disk/network consumer must pause the producer without blocking
// the JS event loop, so cancellation and other tracks can still run.
func TestBlobBackpressureLeavesEventLoopResponsive(t *testing.T) {
created := make(chan createdObjectURL, 1)
e, cleanup := newStreamTestEngine(t, created)
defer cleanup()
_, err := e.RunString(`
globalThis.produced = 0;
globalThis.sourceCancelled = false;
__gopeed_blob_create_object_url(() => new ReadableStream({
pull(c) { produced++; c.enqueue(new Uint8Array(64*1024)); },
cancel() { sourceCancelled = true; }
}));
`)
if err != nil {
t.Fatal(err)
}
object := <-created
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
defer cancel()
reader, err := object.open(ctx, stream.ObjectURLOpenRequest{End: -1})
if err != nil {
t.Fatal(err)
}
defer reader.Close()
time.Sleep(100 * time.Millisecond)
value, err := e.RunString("produced")
if err != nil {
t.Fatal(err)
}
before := value.(int64)
if before > 10 {
t.Fatalf("unbounded production: %d", before)
}
time.Sleep(50 * time.Millisecond)
after, err := e.RunString("produced")
if err != nil || after != value {
t.Fatalf("producer did not pause: %v -> %v (%v)", value, after, err)
}
if ctx.Err() != nil {
t.Fatal("event loop only resumed after cancellation")
}
if _, err = reader.Read(make([]byte, 64*1024)); err != nil {
t.Fatal(err)
}
time.Sleep(30 * time.Millisecond)
after, err = e.RunString("produced")
if err != nil || after.(int64) <= before {
t.Fatalf("producer did not resume: %v (%v)", after, err)
}
reader.Close()
waitForJSValue(t, e, "String(sourceCancelled)", "true")
}
+8 -5
View File
@@ -764,11 +764,14 @@
return; return;
} }
const chunk = value instanceof Uint8Array ? value : new Uint8Array(value); const chunk = value instanceof Uint8Array ? value : new Uint8Array(value);
const buffer = chunk.buffer.slice(chunk.byteOffset, chunk.byteOffset + chunk.byteLength); for (let offset = 0; offset < chunk.byteLength; offset += 256 * 1024) {
if (!globalThis.__gopeed_blob_pipe_chunk(pipeId, buffer)) { const part = chunk.subarray(offset, offset + 256 * 1024);
state.cancelRequested = true; const buffer = part.buffer.slice(part.byteOffset, part.byteOffset + part.byteLength);
state.cancelReason = "blob pipe closed"; if (!await globalThis.__gopeed_blob_pipe_chunk(pipeId, buffer)) {
return; state.cancelRequested = true;
state.cancelReason = "blob pipe closed";
return;
}
} }
await yieldBlobPipeTask(); await yieldBlobPipeTask();
} }