feat: spool media inputs to disk and report producer progress (#1520)

This commit is contained in:
Levi authored and GitHub committed 2026-09-18 22:17:22 +08:00
1 parent 38750d8505
commit b1dd46cc7c
37 files changed
+1831 -375

No files matched your search

+44
View File
@@ -7,6 +7,7 @@ import (
"encoding/hex"
"errors"
"fmt"
"github.com/GopeedLab/gopeed/internal/production"
"io"
"net"
"net/http"
@@ -61,6 +62,7 @@ type Metadata struct {
}
type Source struct {
producer production.Reader
ID string
URL string
ContentType string
@@ -359,11 +361,35 @@ func (r *Registry) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return
}
defer reader.Close()
if producer, ok := reader.(production.Reader); ok {
src.mu.Lock()
src.producer = producer
src.mu.Unlock()
if err := producer.WaitProduction(req.Context()); err != nil {
return
}
if producer.Production() != nil {
meta.Size = 0
meta.Range = false
start = 0
end = -1
ranged = false
src.mu.Lock()
src.size = 0
src.rangeEnabled = false
src.mu.Unlock()
}
}
writeHeaders(w, meta, start, end, ranged)
if ranged {
w.WriteHeader(http.StatusPartialContent)
}
if producer, ok := reader.(production.Reader); ok && producer.Production() != nil {
if flusher, ok := w.(http.Flusher); ok {
flusher.Flush()
}
}
limit := int64(-1)
if end >= start {
limit = end - start + 1
@@ -675,3 +701,21 @@ func randomID(size int) (string, error) {
}
return hex.EncodeToString(buf), nil
}
// Production returns the active opener's input counter, if it generates output.
func (r *Registry) Production(raw string) production.Source {
if r == nil {
return nil
}
src, err := r.get(raw)
if err != nil {
return nil
}
src.mu.Lock()
reader := src.producer
src.mu.Unlock()
if reader == nil {
return nil
}
return reader.Production()
}
+6 -2
View File
@@ -1,6 +1,7 @@
package controller
import (
"github.com/GopeedLab/gopeed/internal/tempfiles"
"github.com/GopeedLab/gopeed/pkg/base"
"net/http"
"net/url"
@@ -9,8 +10,11 @@ import (
)
type Controller struct {
GetConfig func(v any)
GetProxy func(requestProxy *base.RequestProxy) func(*http.Request) (*url.URL, error)
TempFiles *tempfiles.Scope
ManagedProduction func(string) bool
TempDir string
GetConfig func(v any)
GetProxy func(requestProxy *base.RequestProxy) func(*http.Request) (*url.URL, error)
FileController
//ContextDialer() (proxy.Dialer, error)
}
+35
View File
@@ -0,0 +1,35 @@
// Package production associates generated resources with their input progress.
package production
import (
"context"
"sync"
)
// Source counts unique input bytes and total received bytes (including refetches).
// Implementations must support concurrent snapshots.
type Source interface {
Progress() (downloaded, received int64)
}
// Reader resolves its optional producer before response headers are sent.
// A nil producer preserves ordinary Blob behavior.
type Reader interface {
Production() Source
WaitProduction(context.Context) error
}
type Registry struct {
mu sync.Mutex
sources map[string]Source
}
func (r *Registry) Set(id string, source Source) {
r.mu.Lock()
defer r.mu.Unlock()
if r.sources == nil {
r.sources = make(map[string]Source)
}
r.sources[id] = source
}
func (r *Registry) Get(id string) Source { r.mu.Lock(); defer r.mu.Unlock(); return r.sources[id] }
func (r *Registry) Remove(id string) { r.mu.Lock(); defer r.mu.Unlock(); delete(r.sources, id) }
+45 -15
View File
@@ -512,17 +512,9 @@ func (f *Fetcher) asyncPrefetch() {
return
}
// Create temporary file for prefetch data
tmpFile, err := os.CreateTemp("", "gopeed-prefetch-*")
if err != nil {
f.prefetchErr = err
return
}
f.prefetchFile = tmpFile
f.prefetchFilePath = tmpFile.Name()
defer func() {
// Close response body when prefetch stops
// Close the response even if temporary-file setup fails or shutdown
// rejects a late prefetch. Otherwise the server can remain blocked writing.
f.resolveRespLock.Lock()
if f.resolveResp != nil {
f.resolveResp.Body.Close()
@@ -531,8 +523,33 @@ func (f *Fetcher) asyncPrefetch() {
f.resolveRespLock.Unlock()
}()
// Create temporary file for prefetch data
tempDir := ""
if f.ctl != nil && f.ctl.TempDir != "" {
tempDir = f.ctl.TempDir
if err := os.MkdirAll(tempDir, 0700); err != nil {
f.prefetchErr = err
return
}
}
tmpFile, err := os.CreateTemp(tempDir, "gopeed-prefetch-*")
if err != nil {
f.prefetchErr = err
return
}
if f.ctl != nil {
if err := f.ctl.TempFiles.Track(tmpFile.Name()); err != nil {
tmpFile.Close()
os.Remove(tmpFile.Name())
f.prefetchErr = err
return
}
}
f.prefetchFile = tmpFile
f.prefetchFilePath = tmpFile.Name()
buf := make([]byte, 32*1024) // 32KB buffer
reader := NewTimeoutReader(resp.Body, readTimeout)
reader := f.responseReader(resp)
for {
select {
@@ -617,7 +634,11 @@ func (f *Fetcher) cleanupPrefetchFile() {
f.prefetchFile = nil
}
if f.prefetchFilePath != "" {
os.Remove(f.prefetchFilePath)
if f.ctl != nil {
f.ctl.TempFiles.Remove(f.prefetchFilePath)
} else {
os.Remove(f.prefetchFilePath)
}
f.prefetchFilePath = ""
}
// The byte count is only reusable while the backing prefetch file exists.
@@ -1329,7 +1350,7 @@ func (f *Fetcher) downloadChunkOnce(conn *connection, client *http.Client, buf [
f.slowStart.onConnectSuccess()
}
reader := NewTimeoutReader(resp.Body, readTimeout)
reader := f.responseReader(resp)
var responseBytesRead int64
for {
if conn.ctx.Err() != nil {
@@ -1711,7 +1732,7 @@ func (f *Fetcher) runConnectionWithResolveResp(conn *connection) {
}
// Download data from resolve response
reader := NewTimeoutReader(resp.Body, readTimeout)
reader := f.responseReader(resp)
for {
if conn.ctx.Err() != nil {
return
@@ -1828,7 +1849,7 @@ func (f *Fetcher) runConnectionFallback(conn *connection) {
f.slowStart.onConnectSuccess()
}
reader := NewTimeoutReader(resp.Body, readTimeout)
reader := f.responseReader(resp)
for {
if conn.ctx.Err() != nil {
return conn.ctx.Err()
@@ -2451,3 +2472,12 @@ func (f *Fetcher) Progress() fetcher.Progress {
func (f *Fetcher) Wait() error {
return <-f.doneCh
}
// Generated output can pause while its producer waits for processing capacity.
// Its producer owns upstream deadlines; cancellation still closes the response.
func (f *Fetcher) responseReader(resp *http.Response) io.Reader {
if f.ctl != nil && f.ctl.ManagedProduction != nil && resp.Request != nil && f.ctl.ManagedProduction(resp.Request.URL.String()) {
return resp.Body
}
return NewTimeoutReader(resp.Body, readTimeout)
}
@@ -0,0 +1,74 @@
package http
import (
"io"
"net/http"
"os"
"path/filepath"
"strings"
"testing"
"github.com/GopeedLab/gopeed/internal/controller"
"github.com/GopeedLab/gopeed/internal/tempfiles"
)
type prefetchSetupBody struct {
io.Reader
closed bool
}
func (b *prefetchSetupBody) Close() error {
b.closed = true
return nil
}
func TestPrefetchSetupFailureClosesResponse(t *testing.T) {
for _, closedScope := range []bool{false, true} {
name := "invalid_temp_directory"
if closedScope {
name = "downloader_already_closed"
}
t.Run(name, func(t *testing.T) {
root := t.TempDir()
scope := &tempfiles.Scope{}
dir := root
if closedScope {
if err := scope.Close(); err != nil {
t.Fatal(err)
}
} else {
dir = filepath.Join(root, "file")
if err := os.WriteFile(dir, nil, 0600); err != nil {
t.Fatal(err)
}
}
body := &prefetchSetupBody{Reader: strings.NewReader("unused response")}
f := &Fetcher{
ctl: &controller.Controller{TempDir: dir, TempFiles: scope},
resolveResp: &http.Response{Body: body},
}
f.asyncPrefetch()
if f.prefetchErr == nil {
t.Fatal("expected temporary-file initialization to fail")
}
if !body.closed {
t.Error("prefetch setup failure left the response body open")
}
if f.resolveResp != nil {
t.Error("prefetch retained its failed resolve response")
}
if !f.prefetchDone.Load() {
t.Error("prefetch did not report completion")
}
if closedScope {
entries, err := os.ReadDir(root)
if err != nil {
t.Fatal(err)
}
if len(entries) != 0 {
t.Errorf("late prefetch left temporary files: %v", entries)
}
}
})
}
}
+63
View File
@@ -0,0 +1,63 @@
// Package tempfiles tracks temporary paths owned by one downloader, without
// requiring a private parent directory or scanning the shared system temp root.
package tempfiles
import (
"errors"
"io"
"os"
"sync"
)
type Scope struct {
mu sync.Mutex
paths map[string]struct{}
closed bool
}
// Track adopts a newly created unique path. Late creations are removed when
// shutdown has already begun. A nil scope leaves cleanup to the resource owner.
func (s *Scope) Track(path string) error {
if s == nil {
return nil
}
s.mu.Lock()
defer s.mu.Unlock()
if s.closed {
return errors.Join(io.ErrClosedPipe, os.RemoveAll(path))
}
if s.paths == nil {
s.paths = make(map[string]struct{})
}
s.paths[path] = struct{}{}
return nil
}
func (s *Scope) Remove(path string) error {
if s == nil {
return os.RemoveAll(path)
}
s.mu.Lock()
defer s.mu.Unlock()
if err := os.RemoveAll(path); err != nil {
return err
}
delete(s.paths, path)
return nil
}
func (s *Scope) Close() error {
if s == nil {
return nil
}
s.mu.Lock()
defer s.mu.Unlock()
s.closed = true
var result error
for path := range s.paths {
if err := os.RemoveAll(path); err != nil {
result = errors.Join(result, err)
} else {
delete(s.paths, path)
}
}
return result
}
+76
View File
@@ -0,0 +1,76 @@
package tempfiles
import (
"errors"
"io"
"os"
"sync"
"testing"
)
func TestSharedDirectoryOwnersAndLateCreation(t *testing.T) {
root := t.TempDir()
a, b := &Scope{}, &Scope{}
own := func(scope *Scope) string {
t.Helper()
dir, err := os.MkdirTemp(root, "gopeed-media-")
if err != nil {
t.Fatal(err)
}
if err := scope.Track(dir); err != nil {
t.Fatal(err)
}
return dir
}
first, second := own(a), own(b)
if err := a.Close(); err != nil {
t.Fatal(err)
}
if _, err := os.Stat(first); !os.IsNotExist(err) {
t.Fatal(err)
}
if _, err := os.Stat(second); err != nil {
t.Fatal("removed other owner's path", err)
}
late, err := os.MkdirTemp(root, "gopeed-media-")
if err != nil {
t.Fatal(err)
}
if err := a.Track(late); !errors.Is(err, io.ErrClosedPipe) {
t.Fatal(err)
}
if _, err := os.Stat(late); !os.IsNotExist(err) {
t.Fatal("late creation leaked", err)
}
if err := b.Close(); err != nil {
t.Fatal(err)
}
}
func TestConcurrentCreationAndShutdown(t *testing.T) {
root := t.TempDir()
scope := &Scope{}
var wg sync.WaitGroup
for i := 0; i < 20; i++ {
wg.Add(1)
go func() {
defer wg.Done()
dir, err := os.MkdirTemp(root, "gopeed-media-")
if err != nil {
t.Error(err)
return
}
if err := scope.Track(dir); err != nil && !errors.Is(err, io.ErrClosedPipe) {
t.Error(err)
}
}()
}
if err := scope.Close(); err != nil {
t.Fatal(err)
}
wg.Wait()
entries, err := os.ReadDir(root)
if err != nil || len(entries) != 0 {
t.Fatal("shutdown leaked paths", entries, err)
}
}