feat: spool media inputs to disk and report producer progress

This commit is contained in:
Levi committed 2026-09-16 23:16:19 +08:00
1 parent e99ef6c7f4
commit e0ced260e2
36 files changed
+1747 -366

No files matched your search

+44
View File
@@ -7,6 +7,7 @@ import (
"encoding/hex"
"errors"
"fmt"
"github.com/GopeedLab/gopeed/internal/production"
"io"
"net"
"net/http"
@@ -61,6 +62,7 @@ type Metadata struct {
}
type Source struct {
producer production.Reader
ID string
URL string
ContentType string
@@ -359,11 +361,35 @@ func (r *Registry) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return
}
defer reader.Close()
if producer, ok := reader.(production.Reader); ok {
src.mu.Lock()
src.producer = producer
src.mu.Unlock()
if err := producer.WaitProduction(req.Context()); err != nil {
return
}
if producer.Production() != nil {
meta.Size = 0
meta.Range = false
start = 0
end = -1
ranged = false
src.mu.Lock()
src.size = 0
src.rangeEnabled = false
src.mu.Unlock()
}
}
writeHeaders(w, meta, start, end, ranged)
if ranged {
w.WriteHeader(http.StatusPartialContent)
}
if producer, ok := reader.(production.Reader); ok && producer.Production() != nil {
if flusher, ok := w.(http.Flusher); ok {
flusher.Flush()
}
}
limit := int64(-1)
if end >= start {
limit = end - start + 1
@@ -675,3 +701,21 @@ func randomID(size int) (string, error) {
}
return hex.EncodeToString(buf), nil
}
// Production returns the active opener's input counter, if it generates output.
func (r *Registry) Production(raw string) production.Source {
if r == nil {
return nil
}
src, err := r.get(raw)
if err != nil {
return nil
}
src.mu.Lock()
reader := src.producer
src.mu.Unlock()
if reader == nil {
return nil
}
return reader.Production()
}
+6 -2
View File
@@ -1,6 +1,7 @@
package controller
import (
"github.com/GopeedLab/gopeed/internal/tempfiles"
"github.com/GopeedLab/gopeed/pkg/base"
"net/http"
"net/url"
@@ -9,8 +10,11 @@ import (
)
type Controller struct {
GetConfig func(v any)
GetProxy func(requestProxy *base.RequestProxy) func(*http.Request) (*url.URL, error)
TempFiles *tempfiles.Scope
ManagedProduction func(string) bool
TempDir string
GetConfig func(v any)
GetProxy func(requestProxy *base.RequestProxy) func(*http.Request) (*url.URL, error)
FileController
//ContextDialer() (proxy.Dialer, error)
}
+35
View File
@@ -0,0 +1,35 @@
// Package production associates generated resources with their input progress.
package production
import (
"context"
"sync"
)
// Source counts unique input bytes and total received bytes (including refetches).
// Implementations must support concurrent snapshots.
type Source interface {
Progress() (downloaded, received int64)
}
// Reader resolves its optional producer before response headers are sent.
// A nil producer preserves ordinary Blob behavior.
type Reader interface {
Production() Source
WaitProduction(context.Context) error
}
type Registry struct {
mu sync.Mutex
sources map[string]Source
}
func (r *Registry) Set(id string, source Source) {
r.mu.Lock()
defer r.mu.Unlock()
if r.sources == nil {
r.sources = make(map[string]Source)
}
r.sources[id] = source
}
func (r *Registry) Get(id string) Source { r.mu.Lock(); defer r.mu.Unlock(); return r.sources[id] }
func (r *Registry) Remove(id string) { r.mu.Lock(); defer r.mu.Unlock(); delete(r.sources, id) }
+35 -6
View File
@@ -513,11 +513,27 @@ func (f *Fetcher) asyncPrefetch() {
}
// Create temporary file for prefetch data
tmpFile, err := os.CreateTemp("", "gopeed-prefetch-*")
tempDir := ""
if f.ctl != nil && f.ctl.TempDir != "" {
tempDir = f.ctl.TempDir
if err := os.MkdirAll(tempDir, 0700); err != nil {
f.prefetchErr = err
return
}
}
tmpFile, err := os.CreateTemp(tempDir, "gopeed-prefetch-*")
if err != nil {
f.prefetchErr = err
return
}
if f.ctl != nil {
if err := f.ctl.TempFiles.Track(tmpFile.Name()); err != nil {
tmpFile.Close()
os.Remove(tmpFile.Name())
f.prefetchErr = err
return
}
}
f.prefetchFile = tmpFile
f.prefetchFilePath = tmpFile.Name()
@@ -532,7 +548,7 @@ func (f *Fetcher) asyncPrefetch() {
}()
buf := make([]byte, 32*1024) // 32KB buffer
reader := NewTimeoutReader(resp.Body, readTimeout)
reader := f.responseReader(resp)
for {
select {
@@ -617,7 +633,11 @@ func (f *Fetcher) cleanupPrefetchFile() {
f.prefetchFile = nil
}
if f.prefetchFilePath != "" {
os.Remove(f.prefetchFilePath)
if f.ctl != nil {
f.ctl.TempFiles.Remove(f.prefetchFilePath)
} else {
os.Remove(f.prefetchFilePath)
}
f.prefetchFilePath = ""
}
// The byte count is only reusable while the backing prefetch file exists.
@@ -1329,7 +1349,7 @@ func (f *Fetcher) downloadChunkOnce(conn *connection, client *http.Client, buf [
f.slowStart.onConnectSuccess()
}
reader := NewTimeoutReader(resp.Body, readTimeout)
reader := f.responseReader(resp)
var responseBytesRead int64
for {
if conn.ctx.Err() != nil {
@@ -1711,7 +1731,7 @@ func (f *Fetcher) runConnectionWithResolveResp(conn *connection) {
}
// Download data from resolve response
reader := NewTimeoutReader(resp.Body, readTimeout)
reader := f.responseReader(resp)
for {
if conn.ctx.Err() != nil {
return
@@ -1828,7 +1848,7 @@ func (f *Fetcher) runConnectionFallback(conn *connection) {
f.slowStart.onConnectSuccess()
}
reader := NewTimeoutReader(resp.Body, readTimeout)
reader := f.responseReader(resp)
for {
if conn.ctx.Err() != nil {
return conn.ctx.Err()
@@ -2451,3 +2471,12 @@ func (f *Fetcher) Progress() fetcher.Progress {
func (f *Fetcher) Wait() error {
return <-f.doneCh
}
// Generated output can pause while its producer waits for processing capacity.
// Its producer owns upstream deadlines; cancellation still closes the response.
func (f *Fetcher) responseReader(resp *http.Response) io.Reader {
if f.ctl != nil && f.ctl.ManagedProduction != nil && resp.Request != nil && f.ctl.ManagedProduction(resp.Request.URL.String()) {
return resp.Body
}
return NewTimeoutReader(resp.Body, readTimeout)
}
+63
View File
@@ -0,0 +1,63 @@
// Package tempfiles tracks temporary paths owned by one downloader, without
// requiring a private parent directory or scanning the shared system temp root.
package tempfiles
import (
"errors"
"io"
"os"
"sync"
)
type Scope struct {
mu sync.Mutex
paths map[string]struct{}
closed bool
}
// Track adopts a newly created unique path. Late creations are removed when
// shutdown has already begun. A nil scope leaves cleanup to the resource owner.
func (s *Scope) Track(path string) error {
if s == nil {
return nil
}
s.mu.Lock()
defer s.mu.Unlock()
if s.closed {
return errors.Join(io.ErrClosedPipe, os.RemoveAll(path))
}
if s.paths == nil {
s.paths = make(map[string]struct{})
}
s.paths[path] = struct{}{}
return nil
}
func (s *Scope) Remove(path string) error {
if s == nil {
return os.RemoveAll(path)
}
s.mu.Lock()
defer s.mu.Unlock()
if err := os.RemoveAll(path); err != nil {
return err
}
delete(s.paths, path)
return nil
}
func (s *Scope) Close() error {
if s == nil {
return nil
}
s.mu.Lock()
defer s.mu.Unlock()
s.closed = true
var result error
for path := range s.paths {
if err := os.RemoveAll(path); err != nil {
result = errors.Join(result, err)
} else {
delete(s.paths, path)
}
}
return result
}
+76
View File
@@ -0,0 +1,76 @@
package tempfiles
import (
"errors"
"io"
"os"
"sync"
"testing"
)
func TestSharedDirectoryOwnersAndLateCreation(t *testing.T) {
root := t.TempDir()
a, b := &Scope{}, &Scope{}
own := func(scope *Scope) string {
t.Helper()
dir, err := os.MkdirTemp(root, "gopeed-media-")
if err != nil {
t.Fatal(err)
}
if err := scope.Track(dir); err != nil {
t.Fatal(err)
}
return dir
}
first, second := own(a), own(b)
if err := a.Close(); err != nil {
t.Fatal(err)
}
if _, err := os.Stat(first); !os.IsNotExist(err) {
t.Fatal(err)
}
if _, err := os.Stat(second); err != nil {
t.Fatal("removed other owner's path", err)
}
late, err := os.MkdirTemp(root, "gopeed-media-")
if err != nil {
t.Fatal(err)
}
if err := a.Track(late); !errors.Is(err, io.ErrClosedPipe) {
t.Fatal(err)
}
if _, err := os.Stat(late); !os.IsNotExist(err) {
t.Fatal("late creation leaked", err)
}
if err := b.Close(); err != nil {
t.Fatal(err)
}
}
func TestConcurrentCreationAndShutdown(t *testing.T) {
root := t.TempDir()
scope := &Scope{}
var wg sync.WaitGroup
for i := 0; i < 20; i++ {
wg.Add(1)
go func() {
defer wg.Done()
dir, err := os.MkdirTemp(root, "gopeed-media-")
if err != nil {
t.Error(err)
return
}
if err := scope.Track(dir); err != nil && !errors.Is(err, io.ErrClosedPipe) {
t.Error(err)
}
}()
}
if err := scope.Close(); err != nil {
t.Fatal(err)
}
wg.Wait()
entries, err := os.ReadDir(root)
if err != nil || len(entries) != 0 {
t.Fatal("shutdown leaked paths", entries, err)
}
}