mirror of
https://github.com/GopeedLab/gopeed.git
synced 2026-11-16 21:33:41 +00:00
feat: spool media inputs to disk and report producer progress
This commit is contained in:
1 parent
e99ef6c7f4
commit
e0ced260e2
36 files changed
+1747
-366
No files matched your search
@@ -54,8 +54,8 @@ const output = gopeed.runtime.ffmpeg.merge({
|
||||
```
|
||||
|
||||
Creating the output does not start requests or acquire input readers until the
|
||||
output is pulled and a global execution slot is acquired. Queued jobs do not
|
||||
open HTTP inputs, invoke input factories, or allocate media ring buffers.
|
||||
output is pulled. Inputs then download to temporary disk chunks even while the
|
||||
job waits for a global FFmpeg execution slot. Only merging occupies a slot.
|
||||
The callback runs once per output; its signal aborts on cancellation, engine
|
||||
shutdown, failure, and completion. A callback must cooperate with cancellation;
|
||||
returned streams are cancelled even if the callback finishes after cancellation.
|
||||
@@ -63,8 +63,10 @@ A stream input is single-use. To retry a Blob-backed task,
|
||||
its opener must create a new SABR session and fresh input streams. Arbitrary
|
||||
output byte ranges / resumable merged output are not supported.
|
||||
|
||||
`gopeed.runtime.ffmpeg.supportsInputFactory === true` identifies this API.
|
||||
Older builds accept eager streams and must be upgraded before using callbacks.
|
||||
`gopeed.runtime.ffmpeg.supportsInputFactory === true` identifies the callback API.
|
||||
`gopeed.runtime.ffmpeg.supportsProducerProgress === true` identifies disk-backed
|
||||
inputs and automatic Blob production progress. Require the latter before using
|
||||
one shared SABR session for both tracks; older bounded buffers can deadlock.
|
||||
|
||||
## Options
|
||||
|
||||
@@ -109,15 +111,45 @@ server validator, same-size remote changes cannot be reliably detected.
|
||||
Content encoding is forced to identity. Authentication/status failures and
|
||||
malformed range responses are errors. There is no full-file seek fallback.
|
||||
|
||||
Sequential JS inputs are drained independently into memory ring buffers that
|
||||
grow on demand up to **32 MiB per input**, independent of total media size.
|
||||
No temporary files are used. This does not enable seeking. Full buffers apply
|
||||
async backpressure until space is available or the operation is cancelled.
|
||||
Extensions must propagate backpressure through their upstream queues and
|
||||
network reads. Independent upstream track requests avoid blocking one track
|
||||
behind another in a shared response. Buffers are released on close.
|
||||
Blob output writes also wait asynchronously for bounded downstream capacity,
|
||||
so a slow consumer does not block the extension event loop.
|
||||
HTTP and sequential JS inputs are drained into **8 MiB disk chunks** beneath
|
||||
`TempDir/gopeed-media-<random>`. Range downloads stream each response into as many chunks as
|
||||
needed, tracking valid byte intervals independently of sparse file length.
|
||||
Verified HTTP Range inputs can seek, skip missing blocks, and re-fetch deleted
|
||||
blocks. Sequential inputs cannot seek even though their bytes are on disk.
|
||||
Consumed chunks are deleted; cancellation, errors and completion remove the
|
||||
remaining input directories. Disk space is not capped; write failures stop the
|
||||
merge. Media size does not determine the size of in-memory input queues.
|
||||
|
||||
JS writes await disk writes, and Blob output writes await bounded downstream
|
||||
capacity. Extensions must still propagate backpressure through upstream queues
|
||||
and network reads. Both tracks can share a SABR response because disk spooling
|
||||
continues independently of which track FFmpeg currently reads.
|
||||
|
||||
When a Blob opener returns the merge stream directly, the runtime automatically
|
||||
associates its input progress with that Blob. No public progress callback or
|
||||
extra `createObjectURL` argument is required. Return the merge stream itself;
|
||||
wrapping it in another stream loses this association. Downloaded bytes count
|
||||
unique input offsets; speed counts received input bytes, including re-fetches.
|
||||
The task stays downloading with unknown total size while queued or merging;
|
||||
once input traffic stops the displayed speed settles to zero. Completion uses
|
||||
the actual merged output size. Generated output waits are governed by producer
|
||||
cancellation/errors rather than the HTTP body idle timeout; HTTP media inputs
|
||||
still enforce a 15-second network idle deadline.
|
||||
|
||||
`tempDir` defaults to Go's `os.TempDir()` when omitted. Flutter only supplies
|
||||
an explicit directory on Android/iOS, using `getTemporaryDirectory()` directly;
|
||||
no extra `gopeed` parent directory is added. Desktop/server callers can override
|
||||
it via startup JSON or `--temp-dir`.
|
||||
|
||||
Initialization never clears this directory. Media inputs, HTTP prefetch and
|
||||
extension installation staging create unique `gopeed-media-*`,
|
||||
`gopeed-prefetch-*`, and `gopeed-extension-*` paths directly beneath it. The
|
||||
downloader tracks only its own paths and removes remaining ones on normal
|
||||
shutdown, without scanning or deleting other files in a shared temporary
|
||||
directory. Resources still clean up on completion/cancellation, and consumed
|
||||
media chunks are deleted promptly. Forced process termination can leave files
|
||||
behind; the next startup does not delete them. Persistent HLS resume data and
|
||||
WebView profiles are unchanged.
|
||||
|
||||
At most two WASM merges execute concurrently per process, with a 512 MiB
|
||||
linear-memory limit per instance. Compilation is shared and lazy; the upstream
|
||||
|
||||
@@ -0,0 +1,248 @@
|
||||
package ffmpeg
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"sync"
|
||||
|
||||
"github.com/GopeedLab/gopeed/internal/tempfiles"
|
||||
)
|
||||
|
||||
// HTTPSpool downloads ahead independently of the FFmpeg semaphore. Range
|
||||
// sources can fetch a missing block on demand, including blocks already evicted.
|
||||
type HTTPSpool struct {
|
||||
demand chan int64
|
||||
demandDone chan struct{}
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
source Input
|
||||
disk *DiskInput
|
||||
network sync.Mutex
|
||||
readMu sync.Mutex
|
||||
pos int64
|
||||
done chan struct{}
|
||||
}
|
||||
|
||||
func OpenSpoolingHTTP(ctx context.Context, client *http.Client, source HTTPSource, root string, owners ...*tempfiles.Scope) (Input, error) {
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
in, err := OpenHTTP(ctx, client, source)
|
||||
if err != nil {
|
||||
cancel()
|
||||
return nil, err
|
||||
}
|
||||
disk, err := NewDiskInput(ctx, root, owners...)
|
||||
if err != nil {
|
||||
cancel()
|
||||
in.Close()
|
||||
return nil, err
|
||||
}
|
||||
if ranged := in.(*httpInput); ranged.ranged {
|
||||
if _, err = disk.spool.WriteAt(ranged.block, 0); err != nil {
|
||||
cancel()
|
||||
in.Close()
|
||||
disk.Close()
|
||||
return nil, err
|
||||
}
|
||||
ranged.block = nil
|
||||
}
|
||||
h := &HTTPSpool{ctx: ctx, cancel: cancel, source: in, disk: disk, done: make(chan struct{}), demand: make(chan int64, 1), demandDone: make(chan struct{})}
|
||||
go func() {
|
||||
defer close(h.demandDone)
|
||||
for {
|
||||
select {
|
||||
case offset := <-h.demand:
|
||||
// A prefetch may have filled this position while demand was queued.
|
||||
var probe [1]byte
|
||||
if n, err, _ := h.disk.spool.readAt(probe[:], offset); n > 0 || err != nil {
|
||||
continue
|
||||
}
|
||||
if err := h.fetch(offset / SpoolBlockSize * SpoolBlockSize); err != nil {
|
||||
h.disk.End(err)
|
||||
return
|
||||
}
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
go func() {
|
||||
defer close(h.done)
|
||||
var err error
|
||||
if !in.Seekable() {
|
||||
_, err = io.CopyBuffer(disk, in, make([]byte, 64*1024))
|
||||
} else {
|
||||
for offset := int64(0); offset < in.Size(); offset += 3 * SpoolBlockSize {
|
||||
if err = h.fetch(offset); err != nil {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
disk.End(err)
|
||||
}()
|
||||
return h, nil
|
||||
}
|
||||
func (h *HTTPSpool) fetch(offset int64) error {
|
||||
h.network.Lock()
|
||||
defer h.network.Unlock()
|
||||
if err := h.ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
source := h.source.(*httpInput)
|
||||
end := offset + min(3*SpoolBlockSize, h.source.Size()-offset)
|
||||
buffer := make([]byte, 64*1024)
|
||||
for offset < end {
|
||||
n, err, _ := h.disk.spool.readAt(buffer[:min(int64(len(buffer)), end-offset)], offset)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if n > 0 {
|
||||
offset += int64(n)
|
||||
continue
|
||||
}
|
||||
resp, err := source.request(offset, end-1)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = func() error {
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusPartialContent {
|
||||
return fmt.Errorf("ffmpeg: Range request returned %d; source changed or stopped supporting Range", resp.StatusCode)
|
||||
}
|
||||
start, last, size, err := contentRange(resp.Header.Get("Content-Range"))
|
||||
if err != nil || start != offset || last >= end || size != source.size {
|
||||
return errors.New("ffmpeg: inconsistent Content-Range")
|
||||
}
|
||||
if source.etag != "" && resp.Header.Get("ETag") != source.etag {
|
||||
return errors.New("ffmpeg: HTTP input ETag changed")
|
||||
}
|
||||
if source.etag == "" && source.modified != "" && resp.Header.Get("Last-Modified") != source.modified {
|
||||
return errors.New("ffmpeg: HTTP input Last-Modified changed")
|
||||
}
|
||||
if err := identityEncoding(resp); err != nil {
|
||||
return err
|
||||
}
|
||||
remaining := last - start + 1
|
||||
if resp.ContentLength >= 0 && resp.ContentLength != remaining {
|
||||
return errors.New("ffmpeg: Range response length mismatch")
|
||||
}
|
||||
for remaining > 0 {
|
||||
n, err := resp.Body.Read(buffer[:min(int64(len(buffer)), remaining)])
|
||||
if n > 0 {
|
||||
if _, e := h.disk.spool.WriteAt(buffer[:n], offset); e != nil {
|
||||
return e
|
||||
}
|
||||
offset += int64(n)
|
||||
remaining -= int64(n)
|
||||
}
|
||||
if err != nil {
|
||||
if err == io.EOF && remaining == 0 {
|
||||
return nil
|
||||
}
|
||||
if err == io.EOF {
|
||||
return io.ErrUnexpectedEOF
|
||||
}
|
||||
return err
|
||||
}
|
||||
if n == 0 {
|
||||
return io.ErrNoProgress
|
||||
}
|
||||
}
|
||||
n, err := resp.Body.Read(buffer[:1])
|
||||
if n != 0 || err != io.EOF {
|
||||
if err != nil && err != io.EOF {
|
||||
return err
|
||||
}
|
||||
return errors.New("ffmpeg: oversized Range response")
|
||||
}
|
||||
return nil
|
||||
}()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
func (h *HTTPSpool) Read(p []byte) (int, error) {
|
||||
if !h.Seekable() {
|
||||
return h.disk.Read(p)
|
||||
}
|
||||
h.readMu.Lock()
|
||||
defer h.readMu.Unlock()
|
||||
if len(p) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
if h.pos >= h.Size() {
|
||||
if err := h.disk.spool.discard((h.Size() - 1) / SpoolBlockSize); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return 0, io.EOF
|
||||
}
|
||||
for {
|
||||
if err := h.ctx.Err(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
n, err, wake := h.disk.spool.readAt(p, h.pos)
|
||||
if n > 0 {
|
||||
block := h.pos / SpoolBlockSize
|
||||
h.pos += int64(n)
|
||||
if h.pos/SpoolBlockSize > block {
|
||||
if e := h.disk.spool.discard(block); err == nil {
|
||||
err = e
|
||||
}
|
||||
}
|
||||
return n, err
|
||||
}
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
select {
|
||||
case h.demand <- h.pos:
|
||||
default:
|
||||
}
|
||||
// Read as soon as a partial response supplies these bytes, rather than
|
||||
// waiting for its entire multi-block Range request to finish.
|
||||
select {
|
||||
case <-wake:
|
||||
case <-h.ctx.Done():
|
||||
return 0, h.ctx.Err()
|
||||
}
|
||||
}
|
||||
}
|
||||
func (h *HTTPSpool) Seek(offset int64, whence int) (int64, error) {
|
||||
if !h.Seekable() {
|
||||
return 0, ErrSeek
|
||||
}
|
||||
h.readMu.Lock()
|
||||
defer h.readMu.Unlock()
|
||||
base := int64(0)
|
||||
switch whence {
|
||||
case io.SeekStart:
|
||||
case io.SeekCurrent:
|
||||
base = h.pos
|
||||
case io.SeekEnd:
|
||||
base = h.Size()
|
||||
default:
|
||||
return 0, errors.New("invalid seek origin")
|
||||
}
|
||||
next := base + offset
|
||||
if next < 0 || (offset > 0 && next < base) {
|
||||
return 0, errors.New("invalid seek offset")
|
||||
}
|
||||
h.pos = next
|
||||
return next, nil
|
||||
}
|
||||
func (h *HTTPSpool) Size() int64 { return h.source.Size() }
|
||||
func (h *HTTPSpool) Seekable() bool { return h.source.Seekable() }
|
||||
func (h *HTTPSpool) Progress() (int64, int64) { return h.disk.Progress() }
|
||||
func (h *HTTPSpool) Close() error {
|
||||
h.cancel()
|
||||
h.source.Close()
|
||||
<-h.done
|
||||
<-h.demandDone
|
||||
return h.disk.Close()
|
||||
}
|
||||
|
||||
func (h *HTTPSpool) Failures() <-chan error { return h.disk.Failures() }
|
||||
@@ -0,0 +1,198 @@
|
||||
package ffmpeg
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestHTTPSpoolRangeCrossBlocksAndRefetch(t *testing.T) {
|
||||
data := make([]byte, 4*SpoolBlockSize+57)
|
||||
for i := range data {
|
||||
data[i] = byte(i*31 + i/257)
|
||||
}
|
||||
var large atomic.Bool
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
var start, end int64
|
||||
fmt.Sscanf(r.Header.Get("Range"), "bytes=%d-%d", &start, &end)
|
||||
if end-start+1 > SpoolBlockSize {
|
||||
large.Store(true)
|
||||
}
|
||||
w.Header().Set("ETag", `"stable"`)
|
||||
http.ServeContent(w, r, "media", time.Time{}, bytes.NewReader(data))
|
||||
}))
|
||||
defer srv.Close()
|
||||
input, err := OpenSpoolingHTTP(context.Background(), srv.Client(), HTTPSource{URL: srv.URL}, t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer input.Close()
|
||||
h := input.(*HTTPSpool)
|
||||
select {
|
||||
case <-h.done:
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatal("prefetch did not finish")
|
||||
}
|
||||
if !large.Load() {
|
||||
t.Fatal("did not fetch multiple blocks in one response")
|
||||
}
|
||||
for _, offset := range []int64{3*SpoolBlockSize - 123, 0, 2*SpoolBlockSize + 41, SpoolBlockSize - 5} {
|
||||
if _, err := h.Seek(offset, io.SeekStart); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := make([]byte, min(int64(2*SpoolBlockSize), int64(len(data))-offset))
|
||||
if _, err := io.ReadFull(h, got); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !bytes.Equal(got, data[offset:offset+int64(len(got))]) {
|
||||
t.Fatal("incorrect random/cross-block read", offset)
|
||||
}
|
||||
}
|
||||
unique, received := h.Progress()
|
||||
if unique != int64(len(data)) || received <= unique {
|
||||
t.Fatal("refetch progress must not double-count stored bytes", unique, received)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHTTPSpoolSequentialAndCancellation(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
entered := make(chan struct{})
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Write([]byte("hello"))
|
||||
w.(http.Flusher).Flush()
|
||||
close(entered)
|
||||
<-r.Context().Done()
|
||||
}))
|
||||
defer srv.Close()
|
||||
in, err := OpenSpoolingHTTP(ctx, srv.Client(), HTTPSource{URL: srv.URL}, t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
<-entered
|
||||
if in.Seekable() {
|
||||
t.Fatal("200 response advertised seek")
|
||||
}
|
||||
got := make([]byte, 5)
|
||||
if _, err := io.ReadFull(in, got); err != nil || string(got) != "hello" {
|
||||
t.Fatal(string(got), err)
|
||||
}
|
||||
cancel()
|
||||
if err := in.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHTTPSpoolRejectsBadRanges(t *testing.T) {
|
||||
for _, mode := range []string{"etag", "length", "offset", "status", "truncated", "encoding"} {
|
||||
t.Run(mode, func(t *testing.T) {
|
||||
var calls atomic.Int32
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
first := calls.Add(1) == 1
|
||||
start, end := int64(0), int64(httpBlockSize-1)
|
||||
if !first {
|
||||
fmt.Sscanf(r.Header.Get("Range"), "bytes=%d-%d", &start, &end)
|
||||
}
|
||||
tag := `"stable"`
|
||||
if !first && mode == "etag" {
|
||||
tag = `"changed"`
|
||||
}
|
||||
w.Header().Set("ETag", tag)
|
||||
if !first && mode == "status" {
|
||||
w.WriteHeader(200)
|
||||
return
|
||||
}
|
||||
if !first && mode == "offset" {
|
||||
start++
|
||||
}
|
||||
if !first && mode == "encoding" {
|
||||
w.Header().Set("Content-Encoding", "gzip")
|
||||
}
|
||||
w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, 2*SpoolBlockSize))
|
||||
length := end - start + 1
|
||||
if !first && mode == "length" {
|
||||
length++
|
||||
}
|
||||
w.Header().Set("Content-Length", fmt.Sprint(length))
|
||||
w.WriteHeader(206)
|
||||
if !first && mode == "truncated" {
|
||||
length /= 2
|
||||
}
|
||||
chunk := make([]byte, 64*1024)
|
||||
for length > 0 {
|
||||
n := min(int64(len(chunk)), length)
|
||||
if _, err := w.Write(chunk[:n]); err != nil {
|
||||
return
|
||||
}
|
||||
length -= n
|
||||
}
|
||||
}))
|
||||
defer srv.Close()
|
||||
in, err := OpenSpoolingHTTP(context.Background(), srv.Client(), HTTPSource{URL: srv.URL}, t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer in.Close()
|
||||
h := in.(*HTTPSpool)
|
||||
select {
|
||||
case err := <-h.Failures():
|
||||
if err == nil {
|
||||
t.Fatal("nil failure")
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("bad range did not fail")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestHTTPSpoolReadsPartialRangeBeforeResponseFinishes(t *testing.T) {
|
||||
ready, finish := make(chan struct{}), make(chan struct{})
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
var start, end int64
|
||||
fmt.Sscanf(r.Header.Get("Range"), "bytes=%d-%d", &start, &end)
|
||||
end = min(end, 4*1024*1024-1)
|
||||
w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, 4*1024*1024))
|
||||
w.Header().Set("Content-Length", fmt.Sprint(end-start+1))
|
||||
w.WriteHeader(206)
|
||||
if start == 0 {
|
||||
w.Write(make([]byte, end+1))
|
||||
return
|
||||
}
|
||||
w.Write(bytes.Repeat([]byte{93}, 64*1024))
|
||||
w.(http.Flusher).Flush()
|
||||
close(ready)
|
||||
select {
|
||||
case <-finish:
|
||||
w.Write(make([]byte, end-start+1-64*1024))
|
||||
case <-r.Context().Done():
|
||||
}
|
||||
}))
|
||||
defer srv.Close()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
|
||||
defer cancel()
|
||||
input, err := OpenSpoolingHTTP(ctx, srv.Client(), HTTPSource{URL: srv.URL}, t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer input.Close()
|
||||
select {
|
||||
case <-ready:
|
||||
case <-ctx.Done():
|
||||
t.Fatal(ctx.Err())
|
||||
}
|
||||
if _, err := input.Seek(1024*1024, io.SeekStart); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b := make([]byte, 1)
|
||||
if _, err := io.ReadFull(input, b); err != nil || b[0] != 93 {
|
||||
t.Fatal("waited for complete response", b, err)
|
||||
}
|
||||
close(finish)
|
||||
}
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
const httpBlockSize = 1024 * 1024
|
||||
@@ -79,8 +80,10 @@ func OpenHTTP(ctx context.Context, client *http.Client, source HTTPSource) (Inpu
|
||||
}
|
||||
|
||||
func (h *httpInput) request(start, end int64) (*http.Response, error) {
|
||||
req, err := http.NewRequestWithContext(h.ctx, http.MethodGet, h.source.URL, nil)
|
||||
ctx, cancel := context.WithCancel(h.ctx)
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, h.source.URL, nil)
|
||||
if err != nil {
|
||||
cancel()
|
||||
return nil, err
|
||||
}
|
||||
for k, v := range h.source.Headers {
|
||||
@@ -94,7 +97,15 @@ func (h *httpInput) request(start, end int64) (*http.Response, error) {
|
||||
} else if h.modified != "" {
|
||||
req.Header.Set("If-Range", h.modified)
|
||||
}
|
||||
return h.client.Do(req)
|
||||
timer := time.AfterFunc(15*time.Second, cancel)
|
||||
resp, err := h.client.Do(req)
|
||||
timer.Stop()
|
||||
if err != nil {
|
||||
cancel()
|
||||
return nil, err
|
||||
}
|
||||
resp.Body = &idleMediaBody{ReadCloser: resp.Body, cancel: cancel}
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
func identityEncoding(resp *http.Response) error {
|
||||
@@ -225,3 +236,17 @@ func (h *httpInput) Close() error {
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Only time spent waiting on the network counts toward this idle deadline.
|
||||
// Pauses between reads (disk writes or FFmpeg scheduling) do not count.
|
||||
type idleMediaBody struct {
|
||||
io.ReadCloser
|
||||
cancel context.CancelFunc
|
||||
}
|
||||
|
||||
func (b *idleMediaBody) Read(p []byte) (int, error) {
|
||||
timer := time.AfterFunc(15*time.Second, b.cancel)
|
||||
defer timer.Stop()
|
||||
return b.ReadCloser.Read(p)
|
||||
}
|
||||
func (b *idleMediaBody) Close() error { b.cancel(); return b.ReadCloser.Close() }
|
||||
@@ -115,29 +115,6 @@ func TestHTTPSequentialAndValidation(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestStreamRing(t *testing.T) {
|
||||
s := NewStreamInput(context.Background())
|
||||
defer s.Close()
|
||||
// Start near the boundary to exercise wrap-around without a huge fixture.
|
||||
s.buffer = make([]byte, 256*1024)
|
||||
s.read = int64(len(s.buffer)) - 3
|
||||
s.written = s.read
|
||||
if _, err := s.Write([]byte("abcdefgh")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
s.End(nil)
|
||||
b, err := io.ReadAll(s)
|
||||
if err != nil || string(b) != "abcdefgh" {
|
||||
t.Fatalf("%q %v", b, err)
|
||||
}
|
||||
if err = s.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if s.buffer != nil {
|
||||
t.Fatal("buffer retained after close")
|
||||
}
|
||||
}
|
||||
|
||||
func TestOutputArguments(t *testing.T) {
|
||||
a, err := Arguments("", []string{"-shortest", "-metadata", "title=test"})
|
||||
if err != nil {
|
||||
@@ -152,71 +129,3 @@ func TestOutputArguments(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestStreamBackpressureAndCancel(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
s := NewStreamInput(ctx)
|
||||
defer s.Close()
|
||||
s.buffer = make([]byte, StreamBufferLimit)
|
||||
s.written = StreamBufferLimit
|
||||
finished := make(chan error, 1)
|
||||
go func() { _, err := s.Write([]byte{42}); finished <- err }()
|
||||
select {
|
||||
case err := <-finished:
|
||||
t.Fatalf("did not apply backpressure: %v", err)
|
||||
case <-time.After(10 * time.Millisecond):
|
||||
}
|
||||
if _, err := s.Read(make([]byte, 1)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
select {
|
||||
case err := <-finished:
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("writer did not resume")
|
||||
}
|
||||
go func() { _, err := s.Write([]byte{43}); finished <- err }()
|
||||
cancel()
|
||||
select {
|
||||
case err := <-finished:
|
||||
if err == nil {
|
||||
t.Fatal("cancelled write succeeded")
|
||||
}
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("writer did not cancel")
|
||||
}
|
||||
}
|
||||
|
||||
func TestStreamMemoryGrowthAndWrap(t *testing.T) {
|
||||
s := NewStreamInput(context.Background())
|
||||
defer s.Close()
|
||||
if len(s.buffer) != 0 {
|
||||
t.Fatal("buffer allocated before first write")
|
||||
}
|
||||
a := bytes.Repeat([]byte{1}, 200*1024)
|
||||
b := bytes.Repeat([]byte{2}, 180*1024)
|
||||
s.Write(a)
|
||||
consumed := make([]byte, 180*1024)
|
||||
if _, err := io.ReadFull(s, consumed); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
s.Write(b) // Wrap in the initial 256 KiB allocation.
|
||||
c := bytes.Repeat([]byte{3}, 300*1024)
|
||||
s.Write(c) // Grow while unread data wraps around the old ring.
|
||||
s.End(nil)
|
||||
got, err := io.ReadAll(s)
|
||||
want := append(append(append([]byte{}, a[len(consumed):]...), b...), c...)
|
||||
if err != nil || !bytes.Equal(got, want) {
|
||||
t.Fatalf("growth corrupted data: %v", err)
|
||||
}
|
||||
if len(s.buffer) > StreamBufferLimit {
|
||||
t.Fatal("buffer exceeded limit")
|
||||
}
|
||||
s.Close()
|
||||
if s.buffer != nil {
|
||||
t.Fatal("buffer retained on close")
|
||||
}
|
||||
}
|
||||
@@ -120,7 +120,8 @@ func Run(ctx context.Context, video, audio Input, out io.Writer, format string,
|
||||
return RunAcquired(ctx, video, audio, out, format, extra)
|
||||
}
|
||||
|
||||
// Acquire reserves execution capacity before input factories or HTTP requests start.
|
||||
// Acquire reserves WASM execution capacity. Disk-backed input downloads may
|
||||
// already be running while this call waits.
|
||||
func Acquire(ctx context.Context) (func(), error) {
|
||||
select {
|
||||
case slots <- struct{}{}:
|
||||
|
||||
@@ -122,13 +122,16 @@ func TestWebMToMP4(t *testing.T) {
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
inputs := make([]*StreamInput, 2)
|
||||
inputs := make([]*DiskInput, 2)
|
||||
for i, name := range []string{"video.webm", "audio.webm"} {
|
||||
data, err := os.ReadFile(filepath.Join("..", "testdata", "ffmpeg", name))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
inputs[i] = NewStreamInput(ctx)
|
||||
inputs[i], err = NewDiskInput(ctx, t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer inputs[i].Close()
|
||||
if _, err = inputs[i].Write(data); err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -205,7 +208,10 @@ func TestWASMMerge(t *testing.T) {
|
||||
})
|
||||
}
|
||||
// A malformed input must produce an error, not successful empty output.
|
||||
s := NewStreamInput(context.Background())
|
||||
s, err := NewDiskInput(context.Background(), t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer s.Close()
|
||||
s.Write([]byte("invalid"))
|
||||
s.End(nil)
|
||||
|
||||
@@ -0,0 +1,284 @@
|
||||
package ffmpeg
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
|
||||
"github.com/GopeedLab/gopeed/internal/tempfiles"
|
||||
)
|
||||
|
||||
const SpoolBlockSize int64 = 8 * 1024 * 1024
|
||||
|
||||
type byteSpan struct{ start, end int64 }
|
||||
|
||||
// blockSpool tracks valid byte intervals separately from file length: a sparse
|
||||
// file's unwritten holes are never valid media. No file handles survive a call.
|
||||
type blockSpool struct {
|
||||
owner *tempfiles.Scope
|
||||
failures chan error
|
||||
mu sync.Mutex
|
||||
dir string
|
||||
blocks map[int64][]byteSpan
|
||||
seen []byteSpan
|
||||
wake chan struct{}
|
||||
ended, closed bool
|
||||
err error
|
||||
downloaded, received int64
|
||||
}
|
||||
|
||||
func newBlockSpool(root string, owners ...*tempfiles.Scope) (*blockSpool, error) {
|
||||
if root != "" {
|
||||
if err := os.MkdirAll(root, 0700); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
dir, err := os.MkdirTemp(root, "gopeed-media-")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var owner *tempfiles.Scope
|
||||
if len(owners) > 0 {
|
||||
owner = owners[0]
|
||||
}
|
||||
if err := owner.Track(dir); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &blockSpool{owner: owner, failures: make(chan error, 1), dir: dir, blocks: make(map[int64][]byteSpan), wake: make(chan struct{})}, nil
|
||||
}
|
||||
func (s *blockSpool) notify() { close(s.wake); s.wake = make(chan struct{}) }
|
||||
func (s *blockSpool) path(index int64) string { return filepath.Join(s.dir, fmt.Sprintf("%d", index)) }
|
||||
func addSpan(spans []byteSpan, next byteSpan) ([]byteSpan, int64) {
|
||||
before := int64(0)
|
||||
for _, span := range spans {
|
||||
before += span.end - span.start
|
||||
}
|
||||
out := make([]byteSpan, 0, len(spans)+1)
|
||||
for _, span := range spans {
|
||||
if span.end < next.start {
|
||||
out = append(out, span)
|
||||
} else if next.end < span.start {
|
||||
out = append(out, next)
|
||||
next = span
|
||||
} else {
|
||||
next.start = min(next.start, span.start)
|
||||
next.end = max(next.end, span.end)
|
||||
}
|
||||
}
|
||||
out = append(out, next)
|
||||
after := int64(0)
|
||||
for _, span := range out {
|
||||
after += span.end - span.start
|
||||
}
|
||||
return out, after - before
|
||||
}
|
||||
func (s *blockSpool) WriteAt(p []byte, offset int64) (int, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if offset < 0 {
|
||||
return 0, errors.New("negative media offset")
|
||||
}
|
||||
if s.closed {
|
||||
return 0, io.ErrClosedPipe
|
||||
}
|
||||
if s.err != nil {
|
||||
return 0, s.err
|
||||
}
|
||||
written := 0
|
||||
for len(p) > 0 {
|
||||
index, local := offset/SpoolBlockSize, offset%SpoolBlockSize
|
||||
count := int(min(int64(len(p)), SpoolBlockSize-local))
|
||||
f, err := os.OpenFile(s.path(index), os.O_CREATE|os.O_WRONLY, 0600)
|
||||
if err != nil {
|
||||
s.fail(err)
|
||||
s.notify()
|
||||
return written, err
|
||||
}
|
||||
n, err := f.WriteAt(p[:count], local)
|
||||
closeErr := f.Close()
|
||||
if err == nil {
|
||||
err = closeErr
|
||||
}
|
||||
if n > 0 {
|
||||
s.blocks[index], _ = addSpan(s.blocks[index], byteSpan{local, local + int64(n)})
|
||||
var added int64
|
||||
s.seen, added = addSpan(s.seen, byteSpan{offset, offset + int64(n)})
|
||||
s.downloaded += added
|
||||
s.received += int64(n)
|
||||
written += n
|
||||
offset += int64(n)
|
||||
p = p[n:]
|
||||
}
|
||||
if err == nil && n != count {
|
||||
err = io.ErrShortWrite
|
||||
}
|
||||
if err != nil {
|
||||
s.fail(err)
|
||||
s.notify()
|
||||
return written, err
|
||||
}
|
||||
}
|
||||
s.notify()
|
||||
return written, nil
|
||||
}
|
||||
func (s *blockSpool) readAt(p []byte, offset int64) (int, error, <-chan struct{}) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.closed {
|
||||
return 0, io.ErrClosedPipe, nil
|
||||
}
|
||||
if s.err != nil {
|
||||
return 0, s.err, nil
|
||||
}
|
||||
index, local := offset/SpoolBlockSize, offset%SpoolBlockSize
|
||||
for _, span := range s.blocks[index] {
|
||||
if span.start <= local && local < span.end {
|
||||
f, err := os.Open(s.path(index))
|
||||
if err != nil {
|
||||
return 0, err, nil
|
||||
}
|
||||
n, err := f.ReadAt(p[:min(int64(len(p)), span.end-local)], local)
|
||||
f.Close()
|
||||
return n, err, nil
|
||||
}
|
||||
}
|
||||
return 0, nil, s.wake
|
||||
}
|
||||
func (s *blockSpool) discard(index int64) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if err := os.Remove(s.path(index)); err != nil && !os.IsNotExist(err) {
|
||||
return err
|
||||
}
|
||||
delete(s.blocks, index)
|
||||
return nil
|
||||
}
|
||||
func (s *blockSpool) end(err error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.ended = true
|
||||
if s.err == nil {
|
||||
s.fail(err)
|
||||
}
|
||||
s.notify()
|
||||
}
|
||||
func (s *blockSpool) Close() error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.closed = true
|
||||
clear(s.blocks)
|
||||
s.seen = nil
|
||||
s.notify()
|
||||
return s.owner.Remove(s.dir)
|
||||
}
|
||||
func (s *blockSpool) progress() (int64, int64) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return s.downloaded, s.received
|
||||
}
|
||||
|
||||
// DiskInput consumes a sequential producer without retaining its payload in RAM.
|
||||
// Writes acknowledge durable-to-the-OS chunks, allowing both SABR tracks to drain
|
||||
// even while FFmpeg is queued or currently reading only the other track.
|
||||
type DiskInput struct {
|
||||
ctx context.Context
|
||||
spool *blockSpool
|
||||
writeMu, readMu sync.Mutex
|
||||
written, read int64
|
||||
}
|
||||
|
||||
func NewDiskInput(ctx context.Context, root string, owners ...*tempfiles.Scope) (*DiskInput, error) {
|
||||
spool, err := newBlockSpool(root, owners...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &DiskInput{ctx: ctx, spool: spool}, nil
|
||||
}
|
||||
func (s *DiskInput) Write(p []byte) (int, error) {
|
||||
s.writeMu.Lock()
|
||||
defer s.writeMu.Unlock()
|
||||
if err := s.ctx.Err(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
s.spool.mu.Lock()
|
||||
ended := s.spool.ended
|
||||
s.spool.mu.Unlock()
|
||||
if ended {
|
||||
return 0, io.ErrClosedPipe
|
||||
}
|
||||
n, err := s.spool.WriteAt(p, s.written)
|
||||
s.written += int64(n)
|
||||
return n, err
|
||||
}
|
||||
func (s *DiskInput) End(err error) { s.writeMu.Lock(); defer s.writeMu.Unlock(); s.spool.end(err) }
|
||||
func (s *DiskInput) Read(p []byte) (int, error) {
|
||||
s.readMu.Lock()
|
||||
defer s.readMu.Unlock()
|
||||
if len(p) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
for {
|
||||
if err := s.ctx.Err(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
n, err, wake := s.spool.readAt(p, s.read)
|
||||
if n > 0 {
|
||||
previous := s.read / SpoolBlockSize
|
||||
s.read += int64(n)
|
||||
if s.read/SpoolBlockSize > previous {
|
||||
if e := s.spool.discard(previous); err == nil {
|
||||
err = e
|
||||
}
|
||||
}
|
||||
return n, err
|
||||
}
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
s.spool.mu.Lock()
|
||||
ended := s.spool.ended
|
||||
s.spool.mu.Unlock()
|
||||
// End can race the missing-data check; recheck after observing it.
|
||||
if ended {
|
||||
n, err, _ = s.spool.readAt(p, s.read)
|
||||
if n > 0 {
|
||||
// Use the regular consumption path, including block eviction.
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if err := s.spool.discard(s.read / SpoolBlockSize); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return 0, io.EOF
|
||||
}
|
||||
select {
|
||||
case <-wake:
|
||||
case <-s.ctx.Done():
|
||||
return 0, s.ctx.Err()
|
||||
}
|
||||
}
|
||||
}
|
||||
func (s *DiskInput) Close() error { return s.spool.Close() }
|
||||
func (*DiskInput) Seek(int64, int) (int64, error) { return 0, ErrSeek }
|
||||
func (*DiskInput) Size() int64 { return -1 }
|
||||
func (*DiskInput) Seekable() bool { return false }
|
||||
func (s *DiskInput) Progress() (int64, int64) { return s.spool.progress() }
|
||||
|
||||
// fail is called with mu held. The first error wakes a queued FFmpeg job.
|
||||
func (s *blockSpool) fail(err error) {
|
||||
if err == nil || s.err != nil {
|
||||
return
|
||||
}
|
||||
s.err = err
|
||||
select {
|
||||
case s.failures <- err:
|
||||
default:
|
||||
}
|
||||
}
|
||||
func (s *DiskInput) Failures() <-chan error { return s.spool.failures }
|
||||
@@ -0,0 +1,101 @@
|
||||
package ffmpeg
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"io"
|
||||
"os"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestSpoolSparseCrossBlock(t *testing.T) {
|
||||
s, err := newBlockSpool(t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer s.Close()
|
||||
offset := 7*SpoolBlockSize + 123
|
||||
data := bytes.Repeat([]byte{91}, int(3*SpoolBlockSize+37))
|
||||
if n, err := s.WriteAt(data, offset); err != nil || n != len(data) {
|
||||
t.Fatal(n, err)
|
||||
}
|
||||
if n, err, wake := s.readAt(make([]byte, 16), 7*SpoolBlockSize); n != 0 || err != nil || wake == nil {
|
||||
t.Fatal("sparse hole was exposed", n, err)
|
||||
}
|
||||
got := make([]byte, len(data))
|
||||
pos := 0
|
||||
for pos < len(got) {
|
||||
n, err, _ := s.readAt(got[pos:], offset+int64(pos))
|
||||
if err != nil || n == 0 {
|
||||
t.Fatal(n, err)
|
||||
}
|
||||
pos += n
|
||||
}
|
||||
if !bytes.Equal(data, got) {
|
||||
t.Fatal("cross-block contents differ")
|
||||
}
|
||||
if err := s.discard(8); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n, _, wake := s.readAt(got[:1], 8*SpoolBlockSize); n != 0 || wake == nil {
|
||||
t.Fatal("deleted block remained readable")
|
||||
}
|
||||
if _, err := s.WriteAt(data, offset); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
unique, received := s.progress()
|
||||
if unique != int64(len(data)) || received != 2*unique {
|
||||
t.Fatal(unique, received)
|
||||
}
|
||||
}
|
||||
func TestDiskInputDrainAndCleanup(t *testing.T) {
|
||||
s, err := NewDiskInput(context.Background(), t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
data := bytes.Repeat([]byte{33}, int(3*SpoolBlockSize+99))
|
||||
if _, err := s.Write(data); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
s.End(nil)
|
||||
got, err := io.ReadAll(s)
|
||||
if err != nil || !bytes.Equal(data, got) {
|
||||
t.Fatal("drain", err)
|
||||
}
|
||||
files, err := os.ReadDir(s.spool.dir)
|
||||
if err != nil || len(files) != 0 {
|
||||
t.Fatal("consumed blocks retained", len(files), err)
|
||||
}
|
||||
if err := s.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := os.Stat(s.spool.dir); !os.IsNotExist(err) {
|
||||
t.Fatal("spool not removed", err)
|
||||
}
|
||||
}
|
||||
func TestDiskInputCancelAndWriteFailure(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
s, err := NewDiskInput(ctx, t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer s.Close()
|
||||
cancel()
|
||||
if _, err := s.Read(make([]byte, 1)); err != context.Canceled {
|
||||
t.Fatal(err)
|
||||
}
|
||||
broken, err := NewDiskInput(context.Background(), t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer broken.Close()
|
||||
if err := os.RemoveAll(broken.spool.dir); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := broken.Write([]byte{1}); err == nil {
|
||||
t.Fatal("missing disk error")
|
||||
}
|
||||
if _, err := broken.Read(make([]byte, 1)); err == nil {
|
||||
t.Fatal("disk error not propagated")
|
||||
}
|
||||
}
|
||||
@@ -1,163 +0,0 @@
|
||||
package ffmpeg
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// A bounded memory ring lets both JS inputs drain independently, including when
|
||||
// they share a SABR producer. It does not make a sequential input seekable.
|
||||
// A full ring waits for consumption or cancellation. Producers must propagate
|
||||
// backpressure and allow each track to make progress independently.
|
||||
const StreamBufferLimit = 32 * 1024 * 1024
|
||||
|
||||
type StreamInput struct {
|
||||
ctx context.Context
|
||||
mu sync.Mutex
|
||||
buffer []byte
|
||||
read, written int64
|
||||
ended, closed bool
|
||||
err error
|
||||
wake chan struct{}
|
||||
space chan struct{}
|
||||
}
|
||||
|
||||
func NewStreamInput(ctx context.Context) *StreamInput {
|
||||
return &StreamInput{ctx: ctx, wake: make(chan struct{}, 1), space: make(chan struct{}, 1)}
|
||||
}
|
||||
|
||||
func (s *StreamInput) Write(p []byte) (int, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if len(p) > StreamBufferLimit {
|
||||
return 0, errors.New("ffmpeg: input chunk exceeds buffer capacity")
|
||||
}
|
||||
for int64(len(p)) > StreamBufferLimit-(s.written-s.read) && !s.closed && !s.ended {
|
||||
s.mu.Unlock()
|
||||
var err error
|
||||
select {
|
||||
case <-s.space:
|
||||
case <-s.ctx.Done():
|
||||
err = s.ctx.Err()
|
||||
}
|
||||
s.mu.Lock()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
if s.closed || s.ended {
|
||||
return 0, io.ErrClosedPipe
|
||||
}
|
||||
if err := s.ctx.Err(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if len(p) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
// Grow only as needed. Rebase unread bytes when the ring grows.
|
||||
need := int(s.written-s.read) + len(p)
|
||||
if need > len(s.buffer) {
|
||||
capacity := min(StreamBufferLimit, max(256*1024, max(need, len(s.buffer)*2)))
|
||||
buffer := make([]byte, capacity)
|
||||
unread := int(s.written - s.read)
|
||||
if unread > 0 {
|
||||
off := int(s.read % int64(len(s.buffer)))
|
||||
first := min(unread, len(s.buffer)-off)
|
||||
copy(buffer, s.buffer[off:off+first])
|
||||
copy(buffer[first:], s.buffer[:unread-first])
|
||||
}
|
||||
s.buffer = buffer
|
||||
s.read, s.written = 0, int64(unread)
|
||||
}
|
||||
off := int(s.written % int64(len(s.buffer)))
|
||||
first := min(len(p), len(s.buffer)-off)
|
||||
copy(s.buffer[off:], p[:first])
|
||||
copy(s.buffer, p[first:])
|
||||
s.written += int64(len(p))
|
||||
select {
|
||||
case s.wake <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
return len(p), nil
|
||||
}
|
||||
|
||||
func (s *StreamInput) End(err error) {
|
||||
s.mu.Lock()
|
||||
s.ended = true
|
||||
if s.err == nil {
|
||||
s.err = err
|
||||
}
|
||||
s.mu.Unlock()
|
||||
select {
|
||||
case s.space <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
select {
|
||||
case s.wake <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
func (s *StreamInput) Read(p []byte) (int, error) {
|
||||
if len(p) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
for {
|
||||
if err := s.ctx.Err(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
s.mu.Lock()
|
||||
if s.closed {
|
||||
s.mu.Unlock()
|
||||
return 0, io.ErrClosedPipe
|
||||
}
|
||||
if s.err != nil {
|
||||
err := s.err
|
||||
s.mu.Unlock()
|
||||
return 0, err
|
||||
}
|
||||
if s.read < s.written {
|
||||
off := s.read % int64(len(s.buffer))
|
||||
n := min(int64(len(p)), s.written-s.read, int64(len(s.buffer))-off)
|
||||
count := copy(p, s.buffer[off:off+n])
|
||||
s.read += int64(count)
|
||||
s.mu.Unlock()
|
||||
select {
|
||||
case s.space <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
return count, nil
|
||||
}
|
||||
ended := s.ended
|
||||
s.mu.Unlock()
|
||||
if ended {
|
||||
return 0, io.EOF
|
||||
}
|
||||
select {
|
||||
case <-s.wake:
|
||||
case <-s.ctx.Done():
|
||||
return 0, s.ctx.Err()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *StreamInput) Close() error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.closed = true
|
||||
select {
|
||||
case s.space <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
select {
|
||||
case s.wake <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
s.buffer = nil
|
||||
return nil
|
||||
}
|
||||
func (*StreamInput) Seek(int64, int) (int64, error) { return 0, ErrSeek }
|
||||
func (*StreamInput) Size() int64 { return -1 }
|
||||
func (*StreamInput) Seekable() bool { return false }
|
||||
Reference in new issue
Block a user