mirror of
https://github.com/GopeedLab/gopeed.git
synced 2026-11-16 05:13:41 +00:00
feat: spool media inputs to disk and report producer progress
This commit is contained in:
1 parent
e99ef6c7f4
commit
e0ced260e2
36 files changed
+1747
-366
No files matched your search
@@ -7,6 +7,7 @@ import (
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"github.com/GopeedLab/gopeed/internal/production"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
@@ -61,6 +62,7 @@ type Metadata struct {
|
||||
}
|
||||
|
||||
type Source struct {
|
||||
producer production.Reader
|
||||
ID string
|
||||
URL string
|
||||
ContentType string
|
||||
@@ -359,11 +361,35 @@ func (r *Registry) ServeHTTP(w http.ResponseWriter, req *http.Request) {
|
||||
return
|
||||
}
|
||||
defer reader.Close()
|
||||
if producer, ok := reader.(production.Reader); ok {
|
||||
src.mu.Lock()
|
||||
src.producer = producer
|
||||
src.mu.Unlock()
|
||||
if err := producer.WaitProduction(req.Context()); err != nil {
|
||||
return
|
||||
}
|
||||
if producer.Production() != nil {
|
||||
meta.Size = 0
|
||||
meta.Range = false
|
||||
start = 0
|
||||
end = -1
|
||||
ranged = false
|
||||
src.mu.Lock()
|
||||
src.size = 0
|
||||
src.rangeEnabled = false
|
||||
src.mu.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
writeHeaders(w, meta, start, end, ranged)
|
||||
if ranged {
|
||||
w.WriteHeader(http.StatusPartialContent)
|
||||
}
|
||||
if producer, ok := reader.(production.Reader); ok && producer.Production() != nil {
|
||||
if flusher, ok := w.(http.Flusher); ok {
|
||||
flusher.Flush()
|
||||
}
|
||||
}
|
||||
limit := int64(-1)
|
||||
if end >= start {
|
||||
limit = end - start + 1
|
||||
@@ -675,3 +701,21 @@ func randomID(size int) (string, error) {
|
||||
}
|
||||
return hex.EncodeToString(buf), nil
|
||||
}
|
||||
|
||||
// Production returns the active opener's input counter, if it generates output.
|
||||
func (r *Registry) Production(raw string) production.Source {
|
||||
if r == nil {
|
||||
return nil
|
||||
}
|
||||
src, err := r.get(raw)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
src.mu.Lock()
|
||||
reader := src.producer
|
||||
src.mu.Unlock()
|
||||
if reader == nil {
|
||||
return nil
|
||||
}
|
||||
return reader.Production()
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
package controller
|
||||
|
||||
import (
|
||||
"github.com/GopeedLab/gopeed/internal/tempfiles"
|
||||
"github.com/GopeedLab/gopeed/pkg/base"
|
||||
"net/http"
|
||||
"net/url"
|
||||
@@ -9,8 +10,11 @@ import (
|
||||
)
|
||||
|
||||
type Controller struct {
|
||||
GetConfig func(v any)
|
||||
GetProxy func(requestProxy *base.RequestProxy) func(*http.Request) (*url.URL, error)
|
||||
TempFiles *tempfiles.Scope
|
||||
ManagedProduction func(string) bool
|
||||
TempDir string
|
||||
GetConfig func(v any)
|
||||
GetProxy func(requestProxy *base.RequestProxy) func(*http.Request) (*url.URL, error)
|
||||
FileController
|
||||
//ContextDialer() (proxy.Dialer, error)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,35 @@
|
||||
// Package production associates generated resources with their input progress.
|
||||
package production
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// Source counts unique input bytes and total received bytes (including refetches).
|
||||
// Implementations must support concurrent snapshots.
|
||||
type Source interface {
|
||||
Progress() (downloaded, received int64)
|
||||
}
|
||||
|
||||
// Reader resolves its optional producer before response headers are sent.
|
||||
// A nil producer preserves ordinary Blob behavior.
|
||||
type Reader interface {
|
||||
Production() Source
|
||||
WaitProduction(context.Context) error
|
||||
}
|
||||
type Registry struct {
|
||||
mu sync.Mutex
|
||||
sources map[string]Source
|
||||
}
|
||||
|
||||
func (r *Registry) Set(id string, source Source) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
if r.sources == nil {
|
||||
r.sources = make(map[string]Source)
|
||||
}
|
||||
r.sources[id] = source
|
||||
}
|
||||
func (r *Registry) Get(id string) Source { r.mu.Lock(); defer r.mu.Unlock(); return r.sources[id] }
|
||||
func (r *Registry) Remove(id string) { r.mu.Lock(); defer r.mu.Unlock(); delete(r.sources, id) }
|
||||
@@ -513,11 +513,27 @@ func (f *Fetcher) asyncPrefetch() {
|
||||
}
|
||||
|
||||
// Create temporary file for prefetch data
|
||||
tmpFile, err := os.CreateTemp("", "gopeed-prefetch-*")
|
||||
tempDir := ""
|
||||
if f.ctl != nil && f.ctl.TempDir != "" {
|
||||
tempDir = f.ctl.TempDir
|
||||
if err := os.MkdirAll(tempDir, 0700); err != nil {
|
||||
f.prefetchErr = err
|
||||
return
|
||||
}
|
||||
}
|
||||
tmpFile, err := os.CreateTemp(tempDir, "gopeed-prefetch-*")
|
||||
if err != nil {
|
||||
f.prefetchErr = err
|
||||
return
|
||||
}
|
||||
if f.ctl != nil {
|
||||
if err := f.ctl.TempFiles.Track(tmpFile.Name()); err != nil {
|
||||
tmpFile.Close()
|
||||
os.Remove(tmpFile.Name())
|
||||
f.prefetchErr = err
|
||||
return
|
||||
}
|
||||
}
|
||||
f.prefetchFile = tmpFile
|
||||
f.prefetchFilePath = tmpFile.Name()
|
||||
|
||||
@@ -532,7 +548,7 @@ func (f *Fetcher) asyncPrefetch() {
|
||||
}()
|
||||
|
||||
buf := make([]byte, 32*1024) // 32KB buffer
|
||||
reader := NewTimeoutReader(resp.Body, readTimeout)
|
||||
reader := f.responseReader(resp)
|
||||
|
||||
for {
|
||||
select {
|
||||
@@ -617,7 +633,11 @@ func (f *Fetcher) cleanupPrefetchFile() {
|
||||
f.prefetchFile = nil
|
||||
}
|
||||
if f.prefetchFilePath != "" {
|
||||
os.Remove(f.prefetchFilePath)
|
||||
if f.ctl != nil {
|
||||
f.ctl.TempFiles.Remove(f.prefetchFilePath)
|
||||
} else {
|
||||
os.Remove(f.prefetchFilePath)
|
||||
}
|
||||
f.prefetchFilePath = ""
|
||||
}
|
||||
// The byte count is only reusable while the backing prefetch file exists.
|
||||
@@ -1329,7 +1349,7 @@ func (f *Fetcher) downloadChunkOnce(conn *connection, client *http.Client, buf [
|
||||
f.slowStart.onConnectSuccess()
|
||||
}
|
||||
|
||||
reader := NewTimeoutReader(resp.Body, readTimeout)
|
||||
reader := f.responseReader(resp)
|
||||
var responseBytesRead int64
|
||||
for {
|
||||
if conn.ctx.Err() != nil {
|
||||
@@ -1711,7 +1731,7 @@ func (f *Fetcher) runConnectionWithResolveResp(conn *connection) {
|
||||
}
|
||||
|
||||
// Download data from resolve response
|
||||
reader := NewTimeoutReader(resp.Body, readTimeout)
|
||||
reader := f.responseReader(resp)
|
||||
for {
|
||||
if conn.ctx.Err() != nil {
|
||||
return
|
||||
@@ -1828,7 +1848,7 @@ func (f *Fetcher) runConnectionFallback(conn *connection) {
|
||||
f.slowStart.onConnectSuccess()
|
||||
}
|
||||
|
||||
reader := NewTimeoutReader(resp.Body, readTimeout)
|
||||
reader := f.responseReader(resp)
|
||||
for {
|
||||
if conn.ctx.Err() != nil {
|
||||
return conn.ctx.Err()
|
||||
@@ -2451,3 +2471,12 @@ func (f *Fetcher) Progress() fetcher.Progress {
|
||||
func (f *Fetcher) Wait() error {
|
||||
return <-f.doneCh
|
||||
}
|
||||
|
||||
// Generated output can pause while its producer waits for processing capacity.
|
||||
// Its producer owns upstream deadlines; cancellation still closes the response.
|
||||
func (f *Fetcher) responseReader(resp *http.Response) io.Reader {
|
||||
if f.ctl != nil && f.ctl.ManagedProduction != nil && resp.Request != nil && f.ctl.ManagedProduction(resp.Request.URL.String()) {
|
||||
return resp.Body
|
||||
}
|
||||
return NewTimeoutReader(resp.Body, readTimeout)
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
// Package tempfiles tracks temporary paths owned by one downloader, without
|
||||
// requiring a private parent directory or scanning the shared system temp root.
|
||||
package tempfiles
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"io"
|
||||
"os"
|
||||
"sync"
|
||||
)
|
||||
|
||||
type Scope struct {
|
||||
mu sync.Mutex
|
||||
paths map[string]struct{}
|
||||
closed bool
|
||||
}
|
||||
|
||||
// Track adopts a newly created unique path. Late creations are removed when
|
||||
// shutdown has already begun. A nil scope leaves cleanup to the resource owner.
|
||||
func (s *Scope) Track(path string) error {
|
||||
if s == nil {
|
||||
return nil
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.closed {
|
||||
return errors.Join(io.ErrClosedPipe, os.RemoveAll(path))
|
||||
}
|
||||
if s.paths == nil {
|
||||
s.paths = make(map[string]struct{})
|
||||
}
|
||||
s.paths[path] = struct{}{}
|
||||
return nil
|
||||
}
|
||||
func (s *Scope) Remove(path string) error {
|
||||
if s == nil {
|
||||
return os.RemoveAll(path)
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if err := os.RemoveAll(path); err != nil {
|
||||
return err
|
||||
}
|
||||
delete(s.paths, path)
|
||||
return nil
|
||||
}
|
||||
func (s *Scope) Close() error {
|
||||
if s == nil {
|
||||
return nil
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.closed = true
|
||||
var result error
|
||||
for path := range s.paths {
|
||||
if err := os.RemoveAll(path); err != nil {
|
||||
result = errors.Join(result, err)
|
||||
} else {
|
||||
delete(s.paths, path)
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
package tempfiles
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"io"
|
||||
"os"
|
||||
"sync"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestSharedDirectoryOwnersAndLateCreation(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
a, b := &Scope{}, &Scope{}
|
||||
own := func(scope *Scope) string {
|
||||
t.Helper()
|
||||
dir, err := os.MkdirTemp(root, "gopeed-media-")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := scope.Track(dir); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return dir
|
||||
}
|
||||
first, second := own(a), own(b)
|
||||
if err := a.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := os.Stat(first); !os.IsNotExist(err) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := os.Stat(second); err != nil {
|
||||
t.Fatal("removed other owner's path", err)
|
||||
}
|
||||
late, err := os.MkdirTemp(root, "gopeed-media-")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := a.Track(late); !errors.Is(err, io.ErrClosedPipe) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := os.Stat(late); !os.IsNotExist(err) {
|
||||
t.Fatal("late creation leaked", err)
|
||||
}
|
||||
if err := b.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConcurrentCreationAndShutdown(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
scope := &Scope{}
|
||||
var wg sync.WaitGroup
|
||||
for i := 0; i < 20; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
dir, err := os.MkdirTemp(root, "gopeed-media-")
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
return
|
||||
}
|
||||
if err := scope.Track(dir); err != nil && !errors.Is(err, io.ErrClosedPipe) {
|
||||
t.Error(err)
|
||||
}
|
||||
}()
|
||||
}
|
||||
if err := scope.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
wg.Wait()
|
||||
entries, err := os.ReadDir(root)
|
||||
if err != nil || len(entries) != 0 {
|
||||
t.Fatal("shutdown leaked paths", entries, err)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user