feat: spool media inputs to disk and report producer progress

This commit is contained in:
Levi committed 2026-09-16 23:16:19 +08:00
1 parent e99ef6c7f4
commit e0ced260e2
36 files changed
+1747 -366

No files matched your search

+14 -9
View File
@@ -1,7 +1,6 @@
(function () {
const create = __gopeed_ffmpeg_create;
const read = __gopeed_ffmpeg_read;
const acquire = __gopeed_ffmpeg_acquire;
const start = __gopeed_ffmpeg_start;
const push = __gopeed_ffmpeg_push;
const end = __gopeed_ffmpeg_end;
@@ -37,7 +36,8 @@
const format = options.format === undefined ? "mp4" : options.format;
if (typeof format !== "string") throw new TypeError("format must be a string");
const signal = options.signal;
let id, controller, stopped = false;
const id = create({ format, args });
let controller, initialized = false, stopped = false;
const readers = [];
const lifetime = new AbortController();
let ownedInputs;
@@ -91,7 +91,7 @@
}
}
}
return new ReadableStream({
const output = new ReadableStream({
start(c) {
controller = c;
active.add(shutdown);
@@ -103,9 +103,8 @@
async pull(c) {
if (stopped) return;
try {
if (!id) {
id = create({ format, args });
await acquire(id);
if (!initialized) {
initialized = true;
if (stopped) return;
if (factory) {
const inputs = await factory({ signal: lifetime.signal });
@@ -129,11 +128,17 @@
if (chunk === null) { stop(); c.close(); }
else c.enqueue(new Uint8Array(chunk));
} catch (error) {
if (!stopped) { c.error(error); stop(error); }
if (!stopped) {
const failure = typeof MessageError === "function" && !(error instanceof MessageError)
? new MessageError(String(error && error.message || error)) : error;
c.error(failure); stop(failure);
}
}
},
cancel(reason) { stop(reason); }
});
}, { highWaterMark: 0 });
if (globalThis.__gopeed_bind_producer) globalThis.__gopeed_bind_producer(output, id);
return output;
}
globalThis.__gopeed_ffmpeg = Object.freeze({ merge, supportsInputFactory: true });
globalThis.__gopeed_ffmpeg = Object.freeze({ merge, supportsInputFactory: true, supportsProducerProgress: true });
})();
+58 -45
View File
@@ -13,6 +13,8 @@ import (
"sync/atomic"
"github.com/GopeedLab/gopeed/internal/httpclient"
"github.com/GopeedLab/gopeed/internal/production"
"github.com/GopeedLab/gopeed/internal/tempfiles"
media "github.com/GopeedLab/gopeed/pkg/download/engine/ffmpeg"
"github.com/dop251/goja"
"github.com/dop251/goja_nodejs/eventloop"
@@ -22,6 +24,9 @@ import (
var script string
type Config struct {
TempFiles *tempfiles.Scope
Producers *production.Registry
TempDir string
DefaultUserAgent *string
ProxyHandler func(*http.Request) (*url.URL, error)
RegisterCleanup func(func())
@@ -33,16 +38,29 @@ type options struct {
Args []string `json:"args"`
}
type job struct {
inputs [2]media.Input
ctx context.Context
cancel context.CancelFunc
output *io.PipeReader
streams [2]*media.StreamInput
streams [2]*media.DiskInput
mu sync.Mutex
ready chan struct{}
start chan options
started bool
}
func (j *job) Progress() (downloaded, received int64) {
j.mu.Lock()
defer j.mu.Unlock()
for _, input := range j.inputs {
if p, ok := input.(production.Source); ok {
d, r := p.Progress()
downloaded += d
received += r
}
}
return
}
func (j *job) stop() {
j.cancel()
j.output.CloseWithError(context.Canceled)
@@ -59,6 +77,10 @@ func Enable(vm *goja.Runtime, loop *eventloop.EventLoop, cfg *Config) error {
if cfg == nil {
cfg = &Config{}
}
spoolDir := cfg.TempDir
if cfg.Producers == nil {
cfg.Producers = &production.Registry{}
}
client, err := httpclient.NewClient(httpclient.Options{Transport: httpclient.TransportOptions{Proxy: cfg.ProxyHandler}})
if err != nil {
return err
@@ -71,6 +93,7 @@ func Enable(vm *goja.Runtime, loop *eventloop.EventLoop, cfg *Config) error {
mu.Lock()
j := jobs[id]
delete(jobs, id)
cfg.Producers.Remove(id)
mu.Unlock()
if j != nil {
j.stop()
@@ -102,15 +125,14 @@ func Enable(vm *goja.Runtime, loop *eventloop.EventLoop, cfg *Config) error {
if err := vm.ExportTo(call.Argument(0), &opts); err != nil {
panic(vm.NewGoError(err))
}
if _, err := media.Arguments(opts.Format, opts.Args); err != nil {
panic(vm.NewGoError(err))
}
ctx, cancel := context.WithCancel(context.Background())
ctx, cancelCause := context.WithCancelCause(context.Background())
cancel := func() { cancelCause(context.Canceled) }
r, w := io.Pipe()
j := &job{ctx: ctx, cancel: cancel, output: r, ready: make(chan struct{}), start: make(chan options, 1)}
j := &job{ctx: ctx, cancel: cancel, output: r, start: make(chan options, 1)}
id := fmt.Sprintf("ffmpeg-%d", next.Add(1))
mu.Lock()
jobs[id] = j
cfg.Producers.Set(id, j)
mu.Unlock()
go func() {
var inputs [2]media.Input
@@ -119,6 +141,9 @@ func Enable(vm *goja.Runtime, loop *eventloop.EventLoop, cfg *Config) error {
if v := recover(); v != nil {
runErr = fmt.Errorf("ffmpeg panic: %v", v)
}
if errors.Is(runErr, context.Canceled) {
runErr = context.Cause(ctx)
}
cancel()
for _, in := range inputs {
if in != nil {
@@ -127,13 +152,6 @@ func Enable(vm *goja.Runtime, loop *eventloop.EventLoop, cfg *Config) error {
}
w.CloseWithError(runErr)
}()
release, err := media.Acquire(ctx)
if err != nil {
runErr = err
return
}
defer release()
close(j.ready)
var opts options
select {
case opts = <-j.start:
@@ -145,47 +163,36 @@ func Enable(vm *goja.Runtime, loop *eventloop.EventLoop, cfg *Config) error {
if source == nil {
inputs[i] = j.streams[i]
} else {
inputs[i], runErr = media.OpenHTTP(ctx, client, *source)
inputs[i], runErr = media.OpenSpoolingHTTP(ctx, client, *source, spoolDir, cfg.TempFiles)
if runErr != nil {
return
}
}
j.mu.Lock()
j.inputs[i] = inputs[i]
j.mu.Unlock()
if input, ok := inputs[i].(interface{ Failures() <-chan error }); ok {
go func() {
select {
case err := <-input.Failures():
cancelCause(err)
case <-ctx.Done():
}
}()
}
}
release, err := media.Acquire(ctx)
if err != nil {
runErr = err
return
}
defer release()
runErr = media.RunAcquired(ctx, inputs[0], inputs[1], w, opts.Format, opts.Args)
}()
return vm.ToValue(id)
}); err != nil {
return err
}
if err := vm.Set("__gopeed_ffmpeg_acquire", func(id string) goja.Value {
promise, resolve, reject := vm.NewPromise()
j := get(id)
go func() {
var err error
if j == nil {
err = io.ErrClosedPipe
} else {
select {
case <-j.ready:
case <-j.ctx.Done():
err = j.ctx.Err()
}
if j.ctx.Err() != nil {
err = j.ctx.Err()
}
}
loop.RunOnLoop(func(vm *goja.Runtime) {
if err != nil {
reject(vm.NewGoError(err))
} else {
resolve(true)
}
})
}()
return vm.ToValue(promise)
}); err != nil {
return err
}
if err := vm.Set("__gopeed_ffmpeg_start", func(call goja.FunctionCall) goja.Value {
id := call.Argument(0).String()
j := get(id)
@@ -226,7 +233,13 @@ func Enable(vm *goja.Runtime, loop *eventloop.EventLoop, cfg *Config) error {
j.started = true
for i, source := range []*media.HTTPSource{opts.Video, opts.Audio} {
if source == nil {
j.streams[i] = media.NewStreamInput(j.ctx)
var err error
j.streams[i], err = media.NewDiskInput(j.ctx, spoolDir, cfg.TempFiles)
if err != nil {
j.cancel()
panic(vm.NewGoError(err))
}
j.inputs[i] = j.streams[i]
}
}
j.start <- opts
+47 -6
View File
@@ -9,6 +9,7 @@ import (
"sync"
"sync/atomic"
"github.com/GopeedLab/gopeed/internal/production"
"github.com/dop251/goja"
"github.com/dop251/goja_nodejs/eventloop"
)
@@ -17,6 +18,7 @@ import (
var script string
type Config struct {
Producers *production.Registry
CreateObjectURL func(opts *ObjectURLOptions, open ObjectURLOpener) (string, error)
RevokeObjectURL func(url string) error
}
@@ -39,6 +41,23 @@ func Enable(runtime *goja.Runtime, loop *eventloop.EventLoop, cfg *Config) error
cfg = &Config{}
}
blobPipes := newBlobPipeRegistry()
if err := runtime.Set("__gopeed_blob_pipe_bind", func(pipeID, producerID string) {
var source production.Source
if cfg.Producers != nil {
source = cfg.Producers.Get(producerID)
}
blobPipes.mu.Lock()
pipe := blobPipes.pipes[pipeID]
blobPipes.mu.Unlock()
if pipe != nil {
pipe.mu.Lock()
pipe.producer = source
pipe.mu.Unlock()
pipe.readyOnce.Do(func() { close(pipe.ready) })
}
}); err != nil {
return err
}
if err := runtime.Set("__gopeed_create_blob_object_url", func(call goja.FunctionCall) goja.Value {
if cfg.CreateObjectURL == nil {
panic(runtime.NewGoError(fmt.Errorf("blob object url handler not configured")))
@@ -189,12 +208,15 @@ type blobPipeItem struct {
}
type blobPipe struct {
id string
loop *eventloop.EventLoop
registry *blobPipeRegistry
ctx context.Context
cancel context.CancelFunc
ch chan blobPipeItem
ready chan struct{}
readyOnce sync.Once
producer production.Source
id string
loop *eventloop.EventLoop
registry *blobPipeRegistry
ctx context.Context
cancel context.CancelFunc
ch chan blobPipeItem
mu sync.Mutex
buf []byte
@@ -205,6 +227,7 @@ type blobPipe struct {
func newBlobPipe(ctx context.Context, loop *eventloop.EventLoop, registry *blobPipeRegistry) *blobPipe {
pipeCtx, cancel := context.WithCancel(ctx)
return &blobPipe{
ready: make(chan struct{}),
id: registry.NewID(),
loop: loop,
registry: registry,
@@ -228,6 +251,7 @@ func (p *blobPipe) Push(chunk []byte) bool {
}
func (p *blobPipe) CloseWithError(err error) {
p.readyOnce.Do(func() { close(p.ready) })
p.once.Do(func() {
p.registry.Remove(p.id)
select {
@@ -480,3 +504,20 @@ func exportJSError(value goja.Value) error {
}
return errors.New(stack)
}
func (p *blobPipe) Production() production.Source {
p.mu.Lock()
defer p.mu.Unlock()
return p.producer
}
func (p *blobPipe) WaitProduction(ctx context.Context) error {
select {
case <-p.ready:
return nil
case <-p.ctx.Done():
return p.ctx.Err()
case <-ctx.Done():
return ctx.Err()
}
}
+5 -1
View File
@@ -734,6 +734,8 @@
finishBlobPipeReader(state, true, reason).catch(function () {});
};
const producers = new WeakMap();
globalThis.__gopeed_bind_producer = (stream, id) => producers.set(stream, id);
globalThis.__gopeed_blob_pipe_source = function (openReadable, request, pipeId) {
if (typeof openReadable !== "function") {
throw new TypeError("blob opener must be callable");
@@ -752,6 +754,8 @@
try {
const source = await openReadable(normalizeOpenRequest(request));
state.openSettled = true;
const producer = producers.get(source);
globalThis.__gopeed_blob_pipe_bind(pipeId, producer || "");
state.reader = toReadableStreamReader(source, "gopeed.runtime.blob.createObjectURL opener function");
if (state.cancelRequested) {
await finishBlobPipeReader(state, true, state.cancelReason);
@@ -777,7 +781,7 @@
}
} catch (error) {
if (!state.cancelRequested) {
globalThis.__gopeed_blob_pipe_error(pipeId, error && error.stack ? error.stack : String(error));
globalThis.__gopeed_blob_pipe_error(pipeId, error && (error.stack || error.message) ? (error.stack || error.message) : String(error));
}
} finally {
state.openSettled = true;