mirror of
https://github.com/GopeedLab/gopeed.git
synced 2026-11-17 05:43:41 +00:00
feat: spool media inputs to disk and report producer progress
This commit is contained in:
1 parent
e99ef6c7f4
commit
e0ced260e2
36 files changed
+1747
-366
No files matched your search
@@ -0,0 +1,248 @@
|
||||
package ffmpeg
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"sync"
|
||||
|
||||
"github.com/GopeedLab/gopeed/internal/tempfiles"
|
||||
)
|
||||
|
||||
// HTTPSpool downloads ahead independently of the FFmpeg semaphore. Range
|
||||
// sources can fetch a missing block on demand, including blocks already evicted.
|
||||
type HTTPSpool struct {
|
||||
demand chan int64
|
||||
demandDone chan struct{}
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
source Input
|
||||
disk *DiskInput
|
||||
network sync.Mutex
|
||||
readMu sync.Mutex
|
||||
pos int64
|
||||
done chan struct{}
|
||||
}
|
||||
|
||||
func OpenSpoolingHTTP(ctx context.Context, client *http.Client, source HTTPSource, root string, owners ...*tempfiles.Scope) (Input, error) {
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
in, err := OpenHTTP(ctx, client, source)
|
||||
if err != nil {
|
||||
cancel()
|
||||
return nil, err
|
||||
}
|
||||
disk, err := NewDiskInput(ctx, root, owners...)
|
||||
if err != nil {
|
||||
cancel()
|
||||
in.Close()
|
||||
return nil, err
|
||||
}
|
||||
if ranged := in.(*httpInput); ranged.ranged {
|
||||
if _, err = disk.spool.WriteAt(ranged.block, 0); err != nil {
|
||||
cancel()
|
||||
in.Close()
|
||||
disk.Close()
|
||||
return nil, err
|
||||
}
|
||||
ranged.block = nil
|
||||
}
|
||||
h := &HTTPSpool{ctx: ctx, cancel: cancel, source: in, disk: disk, done: make(chan struct{}), demand: make(chan int64, 1), demandDone: make(chan struct{})}
|
||||
go func() {
|
||||
defer close(h.demandDone)
|
||||
for {
|
||||
select {
|
||||
case offset := <-h.demand:
|
||||
// A prefetch may have filled this position while demand was queued.
|
||||
var probe [1]byte
|
||||
if n, err, _ := h.disk.spool.readAt(probe[:], offset); n > 0 || err != nil {
|
||||
continue
|
||||
}
|
||||
if err := h.fetch(offset / SpoolBlockSize * SpoolBlockSize); err != nil {
|
||||
h.disk.End(err)
|
||||
return
|
||||
}
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
go func() {
|
||||
defer close(h.done)
|
||||
var err error
|
||||
if !in.Seekable() {
|
||||
_, err = io.CopyBuffer(disk, in, make([]byte, 64*1024))
|
||||
} else {
|
||||
for offset := int64(0); offset < in.Size(); offset += 3 * SpoolBlockSize {
|
||||
if err = h.fetch(offset); err != nil {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
disk.End(err)
|
||||
}()
|
||||
return h, nil
|
||||
}
|
||||
func (h *HTTPSpool) fetch(offset int64) error {
|
||||
h.network.Lock()
|
||||
defer h.network.Unlock()
|
||||
if err := h.ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
source := h.source.(*httpInput)
|
||||
end := offset + min(3*SpoolBlockSize, h.source.Size()-offset)
|
||||
buffer := make([]byte, 64*1024)
|
||||
for offset < end {
|
||||
n, err, _ := h.disk.spool.readAt(buffer[:min(int64(len(buffer)), end-offset)], offset)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if n > 0 {
|
||||
offset += int64(n)
|
||||
continue
|
||||
}
|
||||
resp, err := source.request(offset, end-1)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = func() error {
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusPartialContent {
|
||||
return fmt.Errorf("ffmpeg: Range request returned %d; source changed or stopped supporting Range", resp.StatusCode)
|
||||
}
|
||||
start, last, size, err := contentRange(resp.Header.Get("Content-Range"))
|
||||
if err != nil || start != offset || last >= end || size != source.size {
|
||||
return errors.New("ffmpeg: inconsistent Content-Range")
|
||||
}
|
||||
if source.etag != "" && resp.Header.Get("ETag") != source.etag {
|
||||
return errors.New("ffmpeg: HTTP input ETag changed")
|
||||
}
|
||||
if source.etag == "" && source.modified != "" && resp.Header.Get("Last-Modified") != source.modified {
|
||||
return errors.New("ffmpeg: HTTP input Last-Modified changed")
|
||||
}
|
||||
if err := identityEncoding(resp); err != nil {
|
||||
return err
|
||||
}
|
||||
remaining := last - start + 1
|
||||
if resp.ContentLength >= 0 && resp.ContentLength != remaining {
|
||||
return errors.New("ffmpeg: Range response length mismatch")
|
||||
}
|
||||
for remaining > 0 {
|
||||
n, err := resp.Body.Read(buffer[:min(int64(len(buffer)), remaining)])
|
||||
if n > 0 {
|
||||
if _, e := h.disk.spool.WriteAt(buffer[:n], offset); e != nil {
|
||||
return e
|
||||
}
|
||||
offset += int64(n)
|
||||
remaining -= int64(n)
|
||||
}
|
||||
if err != nil {
|
||||
if err == io.EOF && remaining == 0 {
|
||||
return nil
|
||||
}
|
||||
if err == io.EOF {
|
||||
return io.ErrUnexpectedEOF
|
||||
}
|
||||
return err
|
||||
}
|
||||
if n == 0 {
|
||||
return io.ErrNoProgress
|
||||
}
|
||||
}
|
||||
n, err := resp.Body.Read(buffer[:1])
|
||||
if n != 0 || err != io.EOF {
|
||||
if err != nil && err != io.EOF {
|
||||
return err
|
||||
}
|
||||
return errors.New("ffmpeg: oversized Range response")
|
||||
}
|
||||
return nil
|
||||
}()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
func (h *HTTPSpool) Read(p []byte) (int, error) {
|
||||
if !h.Seekable() {
|
||||
return h.disk.Read(p)
|
||||
}
|
||||
h.readMu.Lock()
|
||||
defer h.readMu.Unlock()
|
||||
if len(p) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
if h.pos >= h.Size() {
|
||||
if err := h.disk.spool.discard((h.Size() - 1) / SpoolBlockSize); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return 0, io.EOF
|
||||
}
|
||||
for {
|
||||
if err := h.ctx.Err(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
n, err, wake := h.disk.spool.readAt(p, h.pos)
|
||||
if n > 0 {
|
||||
block := h.pos / SpoolBlockSize
|
||||
h.pos += int64(n)
|
||||
if h.pos/SpoolBlockSize > block {
|
||||
if e := h.disk.spool.discard(block); err == nil {
|
||||
err = e
|
||||
}
|
||||
}
|
||||
return n, err
|
||||
}
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
select {
|
||||
case h.demand <- h.pos:
|
||||
default:
|
||||
}
|
||||
// Read as soon as a partial response supplies these bytes, rather than
|
||||
// waiting for its entire multi-block Range request to finish.
|
||||
select {
|
||||
case <-wake:
|
||||
case <-h.ctx.Done():
|
||||
return 0, h.ctx.Err()
|
||||
}
|
||||
}
|
||||
}
|
||||
func (h *HTTPSpool) Seek(offset int64, whence int) (int64, error) {
|
||||
if !h.Seekable() {
|
||||
return 0, ErrSeek
|
||||
}
|
||||
h.readMu.Lock()
|
||||
defer h.readMu.Unlock()
|
||||
base := int64(0)
|
||||
switch whence {
|
||||
case io.SeekStart:
|
||||
case io.SeekCurrent:
|
||||
base = h.pos
|
||||
case io.SeekEnd:
|
||||
base = h.Size()
|
||||
default:
|
||||
return 0, errors.New("invalid seek origin")
|
||||
}
|
||||
next := base + offset
|
||||
if next < 0 || (offset > 0 && next < base) {
|
||||
return 0, errors.New("invalid seek offset")
|
||||
}
|
||||
h.pos = next
|
||||
return next, nil
|
||||
}
|
||||
func (h *HTTPSpool) Size() int64 { return h.source.Size() }
|
||||
func (h *HTTPSpool) Seekable() bool { return h.source.Seekable() }
|
||||
func (h *HTTPSpool) Progress() (int64, int64) { return h.disk.Progress() }
|
||||
func (h *HTTPSpool) Close() error {
|
||||
h.cancel()
|
||||
h.source.Close()
|
||||
<-h.done
|
||||
<-h.demandDone
|
||||
return h.disk.Close()
|
||||
}
|
||||
|
||||
func (h *HTTPSpool) Failures() <-chan error { return h.disk.Failures() }
|
||||
Reference in new issue
Block a user