mirror of
https://github.com/GopeedLab/gopeed.git
synced 2026-11-17 05:43:41 +00:00
feat: spool media inputs to disk and report producer progress
This commit is contained in:
1 parent
e99ef6c7f4
commit
e0ced260e2
36 files changed
+1747
-366
No files matched your search
@@ -1,7 +1,6 @@
|
||||
(function () {
|
||||
const create = __gopeed_ffmpeg_create;
|
||||
const read = __gopeed_ffmpeg_read;
|
||||
const acquire = __gopeed_ffmpeg_acquire;
|
||||
const start = __gopeed_ffmpeg_start;
|
||||
const push = __gopeed_ffmpeg_push;
|
||||
const end = __gopeed_ffmpeg_end;
|
||||
@@ -37,7 +36,8 @@
|
||||
const format = options.format === undefined ? "mp4" : options.format;
|
||||
if (typeof format !== "string") throw new TypeError("format must be a string");
|
||||
const signal = options.signal;
|
||||
let id, controller, stopped = false;
|
||||
const id = create({ format, args });
|
||||
let controller, initialized = false, stopped = false;
|
||||
const readers = [];
|
||||
const lifetime = new AbortController();
|
||||
let ownedInputs;
|
||||
@@ -91,7 +91,7 @@
|
||||
}
|
||||
}
|
||||
}
|
||||
return new ReadableStream({
|
||||
const output = new ReadableStream({
|
||||
start(c) {
|
||||
controller = c;
|
||||
active.add(shutdown);
|
||||
@@ -103,9 +103,8 @@
|
||||
async pull(c) {
|
||||
if (stopped) return;
|
||||
try {
|
||||
if (!id) {
|
||||
id = create({ format, args });
|
||||
await acquire(id);
|
||||
if (!initialized) {
|
||||
initialized = true;
|
||||
if (stopped) return;
|
||||
if (factory) {
|
||||
const inputs = await factory({ signal: lifetime.signal });
|
||||
@@ -129,11 +128,17 @@
|
||||
if (chunk === null) { stop(); c.close(); }
|
||||
else c.enqueue(new Uint8Array(chunk));
|
||||
} catch (error) {
|
||||
if (!stopped) { c.error(error); stop(error); }
|
||||
if (!stopped) {
|
||||
const failure = typeof MessageError === "function" && !(error instanceof MessageError)
|
||||
? new MessageError(String(error && error.message || error)) : error;
|
||||
c.error(failure); stop(failure);
|
||||
}
|
||||
}
|
||||
},
|
||||
cancel(reason) { stop(reason); }
|
||||
});
|
||||
}, { highWaterMark: 0 });
|
||||
if (globalThis.__gopeed_bind_producer) globalThis.__gopeed_bind_producer(output, id);
|
||||
return output;
|
||||
}
|
||||
globalThis.__gopeed_ffmpeg = Object.freeze({ merge, supportsInputFactory: true });
|
||||
globalThis.__gopeed_ffmpeg = Object.freeze({ merge, supportsInputFactory: true, supportsProducerProgress: true });
|
||||
})();
|
||||
@@ -13,6 +13,8 @@ import (
|
||||
"sync/atomic"
|
||||
|
||||
"github.com/GopeedLab/gopeed/internal/httpclient"
|
||||
"github.com/GopeedLab/gopeed/internal/production"
|
||||
"github.com/GopeedLab/gopeed/internal/tempfiles"
|
||||
media "github.com/GopeedLab/gopeed/pkg/download/engine/ffmpeg"
|
||||
"github.com/dop251/goja"
|
||||
"github.com/dop251/goja_nodejs/eventloop"
|
||||
@@ -22,6 +24,9 @@ import (
|
||||
var script string
|
||||
|
||||
type Config struct {
|
||||
TempFiles *tempfiles.Scope
|
||||
Producers *production.Registry
|
||||
TempDir string
|
||||
DefaultUserAgent *string
|
||||
ProxyHandler func(*http.Request) (*url.URL, error)
|
||||
RegisterCleanup func(func())
|
||||
@@ -33,16 +38,29 @@ type options struct {
|
||||
Args []string `json:"args"`
|
||||
}
|
||||
type job struct {
|
||||
inputs [2]media.Input
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
output *io.PipeReader
|
||||
streams [2]*media.StreamInput
|
||||
streams [2]*media.DiskInput
|
||||
mu sync.Mutex
|
||||
ready chan struct{}
|
||||
start chan options
|
||||
started bool
|
||||
}
|
||||
|
||||
func (j *job) Progress() (downloaded, received int64) {
|
||||
j.mu.Lock()
|
||||
defer j.mu.Unlock()
|
||||
for _, input := range j.inputs {
|
||||
if p, ok := input.(production.Source); ok {
|
||||
d, r := p.Progress()
|
||||
downloaded += d
|
||||
received += r
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (j *job) stop() {
|
||||
j.cancel()
|
||||
j.output.CloseWithError(context.Canceled)
|
||||
@@ -59,6 +77,10 @@ func Enable(vm *goja.Runtime, loop *eventloop.EventLoop, cfg *Config) error {
|
||||
if cfg == nil {
|
||||
cfg = &Config{}
|
||||
}
|
||||
spoolDir := cfg.TempDir
|
||||
if cfg.Producers == nil {
|
||||
cfg.Producers = &production.Registry{}
|
||||
}
|
||||
client, err := httpclient.NewClient(httpclient.Options{Transport: httpclient.TransportOptions{Proxy: cfg.ProxyHandler}})
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -71,6 +93,7 @@ func Enable(vm *goja.Runtime, loop *eventloop.EventLoop, cfg *Config) error {
|
||||
mu.Lock()
|
||||
j := jobs[id]
|
||||
delete(jobs, id)
|
||||
cfg.Producers.Remove(id)
|
||||
mu.Unlock()
|
||||
if j != nil {
|
||||
j.stop()
|
||||
@@ -102,15 +125,14 @@ func Enable(vm *goja.Runtime, loop *eventloop.EventLoop, cfg *Config) error {
|
||||
if err := vm.ExportTo(call.Argument(0), &opts); err != nil {
|
||||
panic(vm.NewGoError(err))
|
||||
}
|
||||
if _, err := media.Arguments(opts.Format, opts.Args); err != nil {
|
||||
panic(vm.NewGoError(err))
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
ctx, cancelCause := context.WithCancelCause(context.Background())
|
||||
cancel := func() { cancelCause(context.Canceled) }
|
||||
r, w := io.Pipe()
|
||||
j := &job{ctx: ctx, cancel: cancel, output: r, ready: make(chan struct{}), start: make(chan options, 1)}
|
||||
j := &job{ctx: ctx, cancel: cancel, output: r, start: make(chan options, 1)}
|
||||
id := fmt.Sprintf("ffmpeg-%d", next.Add(1))
|
||||
mu.Lock()
|
||||
jobs[id] = j
|
||||
cfg.Producers.Set(id, j)
|
||||
mu.Unlock()
|
||||
go func() {
|
||||
var inputs [2]media.Input
|
||||
@@ -119,6 +141,9 @@ func Enable(vm *goja.Runtime, loop *eventloop.EventLoop, cfg *Config) error {
|
||||
if v := recover(); v != nil {
|
||||
runErr = fmt.Errorf("ffmpeg panic: %v", v)
|
||||
}
|
||||
if errors.Is(runErr, context.Canceled) {
|
||||
runErr = context.Cause(ctx)
|
||||
}
|
||||
cancel()
|
||||
for _, in := range inputs {
|
||||
if in != nil {
|
||||
@@ -127,13 +152,6 @@ func Enable(vm *goja.Runtime, loop *eventloop.EventLoop, cfg *Config) error {
|
||||
}
|
||||
w.CloseWithError(runErr)
|
||||
}()
|
||||
release, err := media.Acquire(ctx)
|
||||
if err != nil {
|
||||
runErr = err
|
||||
return
|
||||
}
|
||||
defer release()
|
||||
close(j.ready)
|
||||
var opts options
|
||||
select {
|
||||
case opts = <-j.start:
|
||||
@@ -145,47 +163,36 @@ func Enable(vm *goja.Runtime, loop *eventloop.EventLoop, cfg *Config) error {
|
||||
if source == nil {
|
||||
inputs[i] = j.streams[i]
|
||||
} else {
|
||||
inputs[i], runErr = media.OpenHTTP(ctx, client, *source)
|
||||
inputs[i], runErr = media.OpenSpoolingHTTP(ctx, client, *source, spoolDir, cfg.TempFiles)
|
||||
if runErr != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
j.mu.Lock()
|
||||
j.inputs[i] = inputs[i]
|
||||
j.mu.Unlock()
|
||||
if input, ok := inputs[i].(interface{ Failures() <-chan error }); ok {
|
||||
go func() {
|
||||
select {
|
||||
case err := <-input.Failures():
|
||||
cancelCause(err)
|
||||
case <-ctx.Done():
|
||||
}
|
||||
}()
|
||||
}
|
||||
}
|
||||
release, err := media.Acquire(ctx)
|
||||
if err != nil {
|
||||
runErr = err
|
||||
return
|
||||
}
|
||||
defer release()
|
||||
runErr = media.RunAcquired(ctx, inputs[0], inputs[1], w, opts.Format, opts.Args)
|
||||
}()
|
||||
return vm.ToValue(id)
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := vm.Set("__gopeed_ffmpeg_acquire", func(id string) goja.Value {
|
||||
promise, resolve, reject := vm.NewPromise()
|
||||
j := get(id)
|
||||
go func() {
|
||||
var err error
|
||||
if j == nil {
|
||||
err = io.ErrClosedPipe
|
||||
} else {
|
||||
select {
|
||||
case <-j.ready:
|
||||
case <-j.ctx.Done():
|
||||
err = j.ctx.Err()
|
||||
}
|
||||
if j.ctx.Err() != nil {
|
||||
err = j.ctx.Err()
|
||||
}
|
||||
}
|
||||
loop.RunOnLoop(func(vm *goja.Runtime) {
|
||||
if err != nil {
|
||||
reject(vm.NewGoError(err))
|
||||
} else {
|
||||
resolve(true)
|
||||
}
|
||||
})
|
||||
}()
|
||||
return vm.ToValue(promise)
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := vm.Set("__gopeed_ffmpeg_start", func(call goja.FunctionCall) goja.Value {
|
||||
id := call.Argument(0).String()
|
||||
j := get(id)
|
||||
@@ -226,7 +233,13 @@ func Enable(vm *goja.Runtime, loop *eventloop.EventLoop, cfg *Config) error {
|
||||
j.started = true
|
||||
for i, source := range []*media.HTTPSource{opts.Video, opts.Audio} {
|
||||
if source == nil {
|
||||
j.streams[i] = media.NewStreamInput(j.ctx)
|
||||
var err error
|
||||
j.streams[i], err = media.NewDiskInput(j.ctx, spoolDir, cfg.TempFiles)
|
||||
if err != nil {
|
||||
j.cancel()
|
||||
panic(vm.NewGoError(err))
|
||||
}
|
||||
j.inputs[i] = j.streams[i]
|
||||
}
|
||||
}
|
||||
j.start <- opts
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
|
||||
"github.com/GopeedLab/gopeed/internal/production"
|
||||
"github.com/dop251/goja"
|
||||
"github.com/dop251/goja_nodejs/eventloop"
|
||||
)
|
||||
@@ -17,6 +18,7 @@ import (
|
||||
var script string
|
||||
|
||||
type Config struct {
|
||||
Producers *production.Registry
|
||||
CreateObjectURL func(opts *ObjectURLOptions, open ObjectURLOpener) (string, error)
|
||||
RevokeObjectURL func(url string) error
|
||||
}
|
||||
@@ -39,6 +41,23 @@ func Enable(runtime *goja.Runtime, loop *eventloop.EventLoop, cfg *Config) error
|
||||
cfg = &Config{}
|
||||
}
|
||||
blobPipes := newBlobPipeRegistry()
|
||||
if err := runtime.Set("__gopeed_blob_pipe_bind", func(pipeID, producerID string) {
|
||||
var source production.Source
|
||||
if cfg.Producers != nil {
|
||||
source = cfg.Producers.Get(producerID)
|
||||
}
|
||||
blobPipes.mu.Lock()
|
||||
pipe := blobPipes.pipes[pipeID]
|
||||
blobPipes.mu.Unlock()
|
||||
if pipe != nil {
|
||||
pipe.mu.Lock()
|
||||
pipe.producer = source
|
||||
pipe.mu.Unlock()
|
||||
pipe.readyOnce.Do(func() { close(pipe.ready) })
|
||||
}
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := runtime.Set("__gopeed_create_blob_object_url", func(call goja.FunctionCall) goja.Value {
|
||||
if cfg.CreateObjectURL == nil {
|
||||
panic(runtime.NewGoError(fmt.Errorf("blob object url handler not configured")))
|
||||
@@ -189,12 +208,15 @@ type blobPipeItem struct {
|
||||
}
|
||||
|
||||
type blobPipe struct {
|
||||
id string
|
||||
loop *eventloop.EventLoop
|
||||
registry *blobPipeRegistry
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
ch chan blobPipeItem
|
||||
ready chan struct{}
|
||||
readyOnce sync.Once
|
||||
producer production.Source
|
||||
id string
|
||||
loop *eventloop.EventLoop
|
||||
registry *blobPipeRegistry
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
ch chan blobPipeItem
|
||||
|
||||
mu sync.Mutex
|
||||
buf []byte
|
||||
@@ -205,6 +227,7 @@ type blobPipe struct {
|
||||
func newBlobPipe(ctx context.Context, loop *eventloop.EventLoop, registry *blobPipeRegistry) *blobPipe {
|
||||
pipeCtx, cancel := context.WithCancel(ctx)
|
||||
return &blobPipe{
|
||||
ready: make(chan struct{}),
|
||||
id: registry.NewID(),
|
||||
loop: loop,
|
||||
registry: registry,
|
||||
@@ -228,6 +251,7 @@ func (p *blobPipe) Push(chunk []byte) bool {
|
||||
}
|
||||
|
||||
func (p *blobPipe) CloseWithError(err error) {
|
||||
p.readyOnce.Do(func() { close(p.ready) })
|
||||
p.once.Do(func() {
|
||||
p.registry.Remove(p.id)
|
||||
select {
|
||||
@@ -480,3 +504,20 @@ func exportJSError(value goja.Value) error {
|
||||
}
|
||||
return errors.New(stack)
|
||||
}
|
||||
|
||||
func (p *blobPipe) Production() production.Source {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
return p.producer
|
||||
}
|
||||
|
||||
func (p *blobPipe) WaitProduction(ctx context.Context) error {
|
||||
select {
|
||||
case <-p.ready:
|
||||
return nil
|
||||
case <-p.ctx.Done():
|
||||
return p.ctx.Err()
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
}
|
||||
@@ -734,6 +734,8 @@
|
||||
finishBlobPipeReader(state, true, reason).catch(function () {});
|
||||
};
|
||||
|
||||
const producers = new WeakMap();
|
||||
globalThis.__gopeed_bind_producer = (stream, id) => producers.set(stream, id);
|
||||
globalThis.__gopeed_blob_pipe_source = function (openReadable, request, pipeId) {
|
||||
if (typeof openReadable !== "function") {
|
||||
throw new TypeError("blob opener must be callable");
|
||||
@@ -752,6 +754,8 @@
|
||||
try {
|
||||
const source = await openReadable(normalizeOpenRequest(request));
|
||||
state.openSettled = true;
|
||||
const producer = producers.get(source);
|
||||
globalThis.__gopeed_blob_pipe_bind(pipeId, producer || "");
|
||||
state.reader = toReadableStreamReader(source, "gopeed.runtime.blob.createObjectURL opener function");
|
||||
if (state.cancelRequested) {
|
||||
await finishBlobPipeReader(state, true, state.cancelReason);
|
||||
@@ -777,7 +781,7 @@
|
||||
}
|
||||
} catch (error) {
|
||||
if (!state.cancelRequested) {
|
||||
globalThis.__gopeed_blob_pipe_error(pipeId, error && error.stack ? error.stack : String(error));
|
||||
globalThis.__gopeed_blob_pipe_error(pipeId, error && (error.stack || error.message) ? (error.stack || error.message) : String(error));
|
||||
}
|
||||
} finally {
|
||||
state.openSettled = true;
|
||||
|
||||
Reference in new issue
Block a user