nprasad2077/gopeed
1
0 Fork 0
mirror of https://github.com/GopeedLab/gopeed.git synced 2026-11-08 09:30:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: replace gblob with blob http transport (#1407)

Browse Source
This commit is contained in:
Levi authored and GitHub committed 2026-07-14 20:05:33 +08:00
1 parent e2ac8ce64b
commit 9fb3758203
34 files changed
+5303 -3057
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

internal/blob/registry.go
+677
View File
@@ -0,0 +1,677 @@
package blob
import (
"bytes"
"context"
"crypto/rand"
"encoding/hex"
"errors"
"fmt"
"io"
"net"
"net/http"
"net/url"
"path"
"strconv"
"strings"
"sync"
"time"
)
const urlPathPrefix = "/__blob/"
const rangeSourceFailureLimit = 2
// unclaimedSourceTTL bounds how long a session-backed source may keep its
// engine alive without ever being claimed by a download task.
var unclaimedSourceTTL = 10 * time.Minute
var (
ErrInvalidURL = errors.New("invalid blob url")
ErrInvalidOptions = errors.New("invalid blob options")
ErrSourceNotFound = errors.New("blob source not found")
ErrSourceRevoked = errors.New("blob source revoked")
ErrSourceClosed = errors.New("blob source closed")
ErrRangeNotAllowed = errors.New("blob range not allowed")
)
type SessionRef interface {
Retain()
Release()
}
type OpenRequest struct {
Offset int64
End int64
}
type OpenFunc func(ctx context.Context, req OpenRequest) (io.ReadCloser, error)
type CreateOptions struct {
ContentType string
Size int64
Range bool
Session SessionRef
}
type Metadata struct {
ContentType string
Size int64
Range bool
}
type Source struct {
ID string
URL string
ContentType string
mu sync.Mutex
size int64
rangeEnabled bool
revoked bool
taskRefs int
readErr error
rangeFailures int
session SessionRef
open OpenFunc
unclaimed *time.Timer
}
type Registry struct {
dir string
mu sync.RWMutex
sources map[string]*Source
serverMu sync.Mutex
listener net.Listener
server *http.Server
baseURL string
}
func NewRegistry(dir string) *Registry {
return &Registry{
dir: dir,
sources: make(map[string]*Source),
}
}
func (r *Registry) Dir() string {
if r == nil {
return ""
}
return r.dir
}
func (r *Registry) IsURL(raw string) bool {
if r == nil {
return false
}
src, err := r.get(raw)
return err == nil && src != nil
}
func (r *Registry) CreateBlob(data []byte, contentType string) (string, error) {
buf := append([]byte(nil), data...)
return r.CreateOpener(func(ctx context.Context, req OpenRequest) (io.ReadCloser, error) {
if req.Offset < 0 || req.Offset > int64(len(buf)) {
return nil, ErrSourceNotFound
}
end := int64(len(buf))
if req.End >= req.Offset && req.End+1 < end {
end = req.End + 1
}
return io.NopCloser(bytes.NewReader(buf[req.Offset:end])), nil
}, &CreateOptions{
ContentType: contentType,
Size: int64(len(buf)),
Range: len(buf) > 0,
})
}
func (r *Registry) CreateOpener(open OpenFunc, opts *CreateOptions) (string, error) {
if r == nil {
return "", ErrSourceNotFound
}
if open == nil {
return "", ErrInvalidOptions
}
if opts == nil {
opts = &CreateOptions{}
}
if opts.Size < 0 {
opts.Size = 0
}
if opts.Range && opts.Size <= 0 {
return "", fmt.Errorf("%w: range requires positive size", ErrInvalidOptions)
}
// Keep server generation selection and source insertion atomic with Close.
r.serverMu.Lock()
baseURL, err := r.ensureServerLocked()
if err != nil {
r.serverMu.Unlock()
return "", err
}
id, err := randomID(18)
if err != nil {
r.serverMu.Unlock()
return "", err
}
if opts.Session != nil {
opts.Session.Retain()
}
srcURL := fmt.Sprintf("%s%s", baseURL, id)
src := &Source{
ID: id,
URL: srcURL,
ContentType: opts.ContentType,
size: opts.Size,
rangeEnabled: opts.Range,
session: opts.Session,
open: open,
}
r.mu.Lock()
r.sources[id] = src
r.mu.Unlock()
r.serverMu.Unlock()
if opts.Session != nil && unclaimedSourceTTL > 0 {
src.mu.Lock()
if !src.revoked {
src.unclaimed = time.AfterFunc(unclaimedSourceTTL, func() {
r.expireUnclaimed(src)
})
}
src.mu.Unlock()
}
return src.URL, nil
}
func (r *Registry) Metadata(raw string) (Metadata, error) {
src, err := r.get(raw)
if err != nil {
return Metadata{}, err
}
src.mu.Lock()
defer src.mu.Unlock()
return Metadata{
ContentType: src.ContentType,
Size: src.size,
Range: src.rangeEnabled,
}, nil
}
// Acquire retains a source for a task. A source remains available until every
// task that acquired it calls Release, unless it is explicitly revoked.
func (r *Registry) Acquire(raw string) error {
src, err := r.get(raw)
if err != nil {
return err
}
return src.acquireTask()
}
// Release releases a task's reference to a source. Releasing the final task
// reference revokes the source. Sources that have never been acquired remain
// registered until Revoke or Registry.Close is called; session-backed sources
// are also bounded by the unclaimed-source TTL.
func (r *Registry) Release(raw string) error {
src, err := r.get(raw)
if err != nil {
return err
}
session, last, err := src.releaseTask()
if err != nil {
return err
}
if !last {
return nil
}
r.deleteSource(src)
if session != nil {
session.Release()
}
return nil
}
// SourceError returns the first error encountered while opening or reading a
// source. Invalid and missing sources have no source error. HTTP response write
// errors are deliberately not recorded here.
func (r *Registry) SourceError(raw string) error {
src, err := r.get(raw)
if err != nil {
return nil
}
return src.sourceError()
}
func (r *Registry) Revoke(raw string) error {
src, err := r.get(raw)
if err != nil {
return err
}
r.removeSource(src)
return nil
}
func (r *Registry) Close() error {
if r == nil {
return nil
}
r.serverMu.Lock()
server := r.server
listener := r.listener
r.server = nil
r.listener = nil
r.baseURL = ""
r.mu.Lock()
sources := make([]*Source, 0, len(r.sources))
for _, src := range r.sources {
sources = append(sources, src)
}
r.sources = make(map[string]*Source)
r.mu.Unlock()
r.serverMu.Unlock()
if server != nil {
_ = server.Close()
}
if listener != nil {
_ = listener.Close()
}
for _, src := range sources {
src.close()
}
return nil
}
// ensureServerLocked returns the current server URL or starts a new server.
// The caller must hold serverMu.
func (r *Registry) ensureServerLocked() (string, error) {
if r.baseURL != "" {
return r.baseURL, nil
}
listener, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
return "", err
}
r.listener = listener
r.baseURL = "http://" + listener.Addr().String() + urlPathPrefix
server := &http.Server{Handler: r}
r.server = server
go func() {
if err := server.Serve(listener); err != nil && !errors.Is(err, http.ErrServerClosed) {
_ = r.Close()
}
}()
return r.baseURL, nil
}
func (r *Registry) ServeHTTP(w http.ResponseWriter, req *http.Request) {
if req.Method != http.MethodGet {
w.Header().Set("Allow", "GET")
http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
return
}
id := parseRequest(req)
if id == "" {
http.NotFound(w, req)
return
}
src, err := r.getByID(id)
if err != nil {
http.NotFound(w, req)
return
}
meta, open, session := src.acquireOpen()
if open == nil {
http.NotFound(w, req)
return
}
if session != nil {
defer session.Release()
}
if src.sourceError() != nil {
http.Error(w, "blob source unavailable", http.StatusGone)
return
}
start, end, ranged, err := parseRange(req.Header.Get("Range"), meta.Size, meta.Range)
if err != nil {
http.Error(w, err.Error(), http.StatusRequestedRangeNotSatisfiable)
return
}
if !meta.Range {
start, end, ranged = 0, -1, false
}
reader, err := open(req.Context(), OpenRequest{
Offset: start,
End: end,
})
if err != nil {
if req.Context().Err() == nil || !errors.Is(err, req.Context().Err()) {
src.recordSourceError(err, meta.Range)
}
http.Error(w, "blob source unavailable", http.StatusGone)
return
}
if reader == nil {
err = fmt.Errorf("%w: opener returned a nil reader", ErrSourceClosed)
src.recordSourceError(err, meta.Range)
http.Error(w, "blob source unavailable", http.StatusGone)
return
}
defer reader.Close()
writeHeaders(w, meta, start, end, ranged)
if ranged {
w.WriteHeader(http.StatusPartialContent)
}
limit := int64(-1)
if end >= start {
limit = end - start + 1
} else if meta.Size > 0 {
limit = meta.Size
}
_, readErr, writeErr := copyWithFlush(w, reader, limit)
if readErr != nil {
if ctxErr := req.Context().Err(); ctxErr == nil || !errors.Is(readErr, ctxErr) {
src.recordSourceError(readErr, meta.Range)
}
} else if writeErr == nil && req.Context().Err() == nil {
src.recordSourceSuccess(meta.Range)
}
}
func (s *Source) acquireOpen() (Metadata, OpenFunc, SessionRef) {
s.mu.Lock()
defer s.mu.Unlock()
if s.revoked {
return Metadata{}, nil, nil
}
if s.session != nil {
// The source's own retain guarantees that the session cannot become
// closed while this additional active-reader retain is acquired.
s.session.Retain()
}
return Metadata{
ContentType: s.ContentType,
Size: s.size,
Range: s.rangeEnabled,
}, s.open, s.session
}
func (s *Source) acquireTask() error {
s.mu.Lock()
if s.revoked {
s.mu.Unlock()
return ErrSourceRevoked
}
s.taskRefs++
timer := s.unclaimed
s.unclaimed = nil
s.mu.Unlock()
if timer != nil {
timer.Stop()
}
return nil
}
func (s *Source) releaseTask() (session SessionRef, last bool, err error) {
s.mu.Lock()
defer s.mu.Unlock()
if s.revoked {
return nil, false, ErrSourceRevoked
}
if s.taskRefs == 0 {
return nil, false, ErrSourceClosed
}
s.taskRefs--
if s.taskRefs > 0 {
return nil, false, nil
}
s.revoked = true
s.open = nil
session = s.session
s.session = nil
return session, true, nil
}
func (s *Source) sourceError() error {
s.mu.Lock()
defer s.mu.Unlock()
return s.readErr
}
func (s *Source) recordSourceError(err error, ranged bool) {
if err == nil {
return
}
s.mu.Lock()
if s.readErr == nil && ranged {
s.rangeFailures++
if s.rangeFailures < rangeSourceFailureLimit {
s.mu.Unlock()
return
}
}
if s.readErr == nil {
s.readErr = err
}
s.mu.Unlock()
}
func (s *Source) recordSourceSuccess(ranged bool) {
if !ranged {
return
}
s.mu.Lock()
if s.readErr == nil {
s.rangeFailures = 0
}
s.mu.Unlock()
}
func (s *Source) close() {
s.mu.Lock()
alreadyRevoked := s.revoked
s.revoked = true
s.open = nil
session := s.session
s.session = nil
timer := s.unclaimed
s.unclaimed = nil
s.mu.Unlock()
if timer != nil {
timer.Stop()
}
if !alreadyRevoked && session != nil {
session.Release()
}
}
func (s *Source) expireUnclaimed() (SessionRef, bool) {
s.mu.Lock()
defer s.mu.Unlock()
s.unclaimed = nil
if s.revoked || s.taskRefs > 0 {
return nil, false
}
s.revoked = true
s.open = nil
session := s.session
s.session = nil
return session, true
}
func (r *Registry) expireUnclaimed(src *Source) {
session, expired := src.expireUnclaimed()
if !expired {
return
}
r.deleteSource(src)
if session != nil {
session.Release()
}
}
func (r *Registry) removeSource(src *Source) {
src.close()
r.deleteSource(src)
}
func (r *Registry) deleteSource(src *Source) {
r.mu.Lock()
if current := r.sources[src.ID]; current == src {
delete(r.sources, src.ID)
}
r.mu.Unlock()
}
func writeHeaders(w http.ResponseWriter, meta Metadata, start, end int64, ranged bool) {
if meta.ContentType != "" {
w.Header().Set("Content-Type", meta.ContentType)
}
if meta.Range {
w.Header().Set("Accept-Ranges", "bytes")
}
if meta.Size > 0 {
if ranged {
w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, meta.Size))
w.Header().Set("Content-Length", strconv.FormatInt(end-start+1, 10))
return
}
w.Header().Set("Content-Length", strconv.FormatInt(meta.Size, 10))
}
}
func copyWithFlush(w http.ResponseWriter, reader io.Reader, limit int64) (written int64, readErr error, writeErr error) {
if limit >= 0 {
reader = io.LimitReader(reader, limit)
}
buf := make([]byte, 32*1024)
flusher, canFlush := w.(http.Flusher)
for {
nr, er := reader.Read(buf)
if er != nil && !errors.Is(er, io.EOF) {
readErr = er
}
if nr > 0 {
nw, ew := w.Write(buf[:nr])
if nw > 0 {
written += int64(nw)
if canFlush {
flusher.Flush()
}
}
if ew != nil {
return written, readErr, ew
}
if nr != nw {
return written, readErr, io.ErrShortWrite
}
}
if er != nil {
if errors.Is(er, io.EOF) {
if limit >= 0 && written < limit {
return written, io.ErrUnexpectedEOF, nil
}
return written, nil, nil
}
return written, readErr, nil
}
}
}
func parseRange(header string, size int64, rangeEnabled bool) (start int64, end int64, ranged bool, err error) {
if header == "" || !rangeEnabled {
return 0, -1, false, nil
}
if size <= 0 {
return 0, 0, false, ErrRangeNotAllowed
}
if !strings.HasPrefix(header, "bytes=") {
return 0, 0, false, fmt.Errorf("unsupported range")
}
parts := strings.SplitN(strings.TrimPrefix(header, "bytes="), "-", 2)
if len(parts) != 2 || parts[0] == "" {
return 0, 0, false, fmt.Errorf("unsupported range")
}
start, err = strconv.ParseInt(parts[0], 10, 64)
if err != nil || start < 0 {
return 0, 0, false, fmt.Errorf("invalid range")
}
if start >= size {
return 0, 0, false, fmt.Errorf("range out of bounds")
}
end = size - 1
if parts[1] != "" {
end, err = strconv.ParseInt(parts[1], 10, 64)
if err != nil || end < start {
return 0, 0, false, fmt.Errorf("invalid range")
}
if end >= size {
end = size - 1
}
}
return start, end, true, nil
}
func (r *Registry) get(raw string) (*Source, error) {
id, ok := r.parseURL(raw)
if !ok {
return nil, ErrInvalidURL
}
src, err := r.getByID(id)
if err != nil {
return nil, err
}
src.mu.Lock()
revoked := src.revoked
src.mu.Unlock()
if revoked {
return nil, ErrSourceRevoked
}
return src, nil
}
func (r *Registry) getByID(id string) (*Source, error) {
r.mu.RLock()
defer r.mu.RUnlock()
src := r.sources[id]
if src == nil {
return nil, ErrSourceNotFound
}
return src, nil
}
func (r *Registry) parseURL(raw string) (id string, ok bool) {
u, err := url.Parse(raw)
if err != nil || u.Scheme != "http" || !strings.HasPrefix(u.Path, urlPathPrefix) {
return "", false
}
r.serverMu.Lock()
baseURL := r.baseURL
r.serverMu.Unlock()
if baseURL == "" || !strings.HasPrefix(raw, baseURL) {
return "", false
}
id = strings.TrimPrefix(u.Path, urlPathPrefix)
if id == "" || path.Base(id) != id {
return "", false
}
return id, true
}
func parseRequest(req *http.Request) string {
id := strings.TrimPrefix(req.URL.Path, urlPathPrefix)
if id == "" || id == req.URL.Path || path.Base(id) != id {
return ""
}
return id
}
func randomID(size int) (string, error) {
buf := make([]byte, size)
if _, err := rand.Read(buf); err != nil {
return "", err
}
return hex.EncodeToString(buf), nil
}
internal/blob/registry_test.go
+898
View File
@@ -0,0 +1,898 @@
package blob
import (
"bytes"
"context"
"errors"
"io"
"net/http"
"strings"
"sync"
"testing"
"time"
)
type terminalErrorReader struct {
err error
}
func (r *terminalErrorReader) Read([]byte) (int, error) {
return 0, r.err
}
type contextErrorReader struct {
ctx context.Context
}
func (r *contextErrorReader) Read([]byte) (int, error) {
<-r.ctx.Done()
return 0, r.ctx.Err()
}
func (r *contextErrorReader) Close() error { return nil }
type failingResponseWriter struct {
header http.Header
err error
}
func (w *failingResponseWriter) Header() http.Header {
return w.header
}
func (w *failingResponseWriter) Write([]byte) (int, error) {
return 0, w.err
}
func (w *failingResponseWriter) WriteHeader(int) {}
type countingSession struct {
mu sync.Mutex
refs int
zero chan struct{}
zeroOnce sync.Once
}
func newCountingSession() *countingSession {
return &countingSession{zero: make(chan struct{})}
}
func (s *countingSession) Retain() {
s.mu.Lock()
s.refs++
s.mu.Unlock()
}
func (s *countingSession) Release() {
s.mu.Lock()
s.refs--
refs := s.refs
s.mu.Unlock()
if refs == 0 {
s.zeroOnce.Do(func() { close(s.zero) })
}
}
func (s *countingSession) RefCount() int {
s.mu.Lock()
defer s.mu.Unlock()
return s.refs
}
func setUnclaimedSourceTTLForTest(t *testing.T, ttl time.Duration) {
t.Helper()
previous := unclaimedSourceTTL
unclaimedSourceTTL = ttl
t.Cleanup(func() {
unclaimedSourceTTL = previous
})
}
func TestRegistryBlobHTTPGetAndRange(t *testing.T) {
registry := NewRegistry(t.TempDir())
defer registry.Close()
url, err := registry.CreateBlob([]byte("hello world"), "text/plain")
if err != nil {
t.Fatal(err)
}
if !registry.IsURL(url) {
t.Fatal("expected registry to recognize created url")
}
resp, err := http.Get(url)
if err != nil {
t.Fatal(err)
}
body, err := io.ReadAll(resp.Body)
_ = resp.Body.Close()
if err != nil {
t.Fatal(err)
}
if resp.StatusCode != http.StatusOK {
t.Fatalf("unexpected GET status: %d", resp.StatusCode)
}
if got := resp.Header.Get("Content-Length"); got != "11" {
t.Fatalf("unexpected content length: %q", got)
}
if got := resp.Header.Get("Accept-Ranges"); got != "bytes" {
t.Fatalf("unexpected accept ranges: %q", got)
}
if got := resp.Header.Get("Content-Type"); !strings.HasPrefix(got, "text/plain") {
t.Fatalf("unexpected content type: %q", got)
}
if string(body) != "hello world" {
t.Fatalf("unexpected GET body: %q", string(body))
}
req, err := http.NewRequest(http.MethodGet, url, nil)
if err != nil {
t.Fatal(err)
}
req.Header.Set("Range", "bytes=6-10")
resp, err = http.DefaultClient.Do(req)
if err != nil {
t.Fatal(err)
}
body, err = io.ReadAll(resp.Body)
_ = resp.Body.Close()
if err != nil {
t.Fatal(err)
}
if resp.StatusCode != http.StatusPartialContent {
t.Fatalf("unexpected range status: %d", resp.StatusCode)
}
if got := resp.Header.Get("Content-Range"); got != "bytes 6-10/11" {
t.Fatalf("unexpected content range: %q", got)
}
if string(body) != "world" {
t.Fatalf("unexpected range body: %q", string(body))
}
}
func TestRegistryEmptyBlob(t *testing.T) {
registry := NewRegistry(t.TempDir())
defer registry.Close()
url, err := registry.CreateBlob(nil, "application/octet-stream")
if err != nil {
t.Fatal(err)
}
meta, err := registry.Metadata(url)
if err != nil {
t.Fatal(err)
}
if meta.Size != 0 || meta.Range {
t.Fatalf("unexpected empty blob metadata: %#v", meta)
}
resp, err := http.Get(url)
if err != nil {
t.Fatal(err)
}
body, readErr := io.ReadAll(resp.Body)
_ = resp.Body.Close()
if readErr != nil {
t.Fatal(readErr)
}
if resp.StatusCode != http.StatusOK {
t.Fatalf("unexpected status: %d", resp.StatusCode)
}
if len(body) != 0 {
t.Fatalf("unexpected body: %q", body)
}
if got := resp.Header.Get("Content-Length"); got != "0" {
t.Fatalf("unexpected content length: %q", got)
}
}
func TestRegistryOpenerSizeWithoutRange(t *testing.T) {
registry := NewRegistry(t.TempDir())
defer registry.Close()
payload := []byte("hello opener")
url, err := registry.CreateOpener(func(ctx context.Context, req OpenRequest) (io.ReadCloser, error) {
if req.Offset != 0 || req.End != -1 {
t.Fatalf("unexpected non-range request: %#v", req)
}
return io.NopCloser(bytes.NewReader(payload)), nil
}, &CreateOptions{
ContentType: "text/plain",
Size: int64(len(payload)),
Range: false,
})
if err != nil {
t.Fatal(err)
}
req, err := http.NewRequest(http.MethodGet, url, nil)
if err != nil {
t.Fatal(err)
}
req.Header.Set("Range", "bytes=6-11")
resp, err := http.DefaultClient.Do(req)
if err != nil {
t.Fatal(err)
}
body, err := io.ReadAll(resp.Body)
_ = resp.Body.Close()
if err != nil {
t.Fatal(err)
}
if resp.StatusCode != http.StatusOK {
t.Fatalf("unexpected status: %d", resp.StatusCode)
}
if got := resp.Header.Get("Content-Length"); got != "12" {
t.Fatalf("unexpected content length: %q", got)
}
if got := resp.Header.Get("Accept-Ranges"); got != "" {
t.Fatalf("unexpected accept ranges: %q", got)
}
if string(body) != string(payload) {
t.Fatalf("unexpected body: %q", string(body))
}
}
func TestRegistryOpenerRangeConcurrentRequests(t *testing.T) {
registry := NewRegistry(t.TempDir())
defer registry.Close()
payload := []byte("abcdefghijklmnopqrstuvwxyz")
var mu sync.Mutex
var calls []OpenRequest
url, err := registry.CreateOpener(func(ctx context.Context, req OpenRequest) (io.ReadCloser, error) {
mu.Lock()
calls = append(calls, req)
mu.Unlock()
start := int(req.Offset)
end := len(payload)
if req.End >= 0 {
end = int(req.End) + 1
}
if start < 0 || start > end || end > len(payload) {
return nil, errors.New("bad test range")
}
return io.NopCloser(bytes.NewReader(payload[start:end])), nil
}, &CreateOptions{
Size: int64(len(payload)),
Range: true,
})
if err != nil {
t.Fatal(err)
}
ranges := []struct {
header string
want string
}{
{header: "bytes=0-4", want: "abcde"},
{header: "bytes=10-15", want: "klmnop"},
{header: "bytes=20-25", want: "uvwxyz"},
}
var wg sync.WaitGroup
for _, item := range ranges {
item := item
wg.Add(1)
go func() {
defer wg.Done()
req, err := http.NewRequest(http.MethodGet, url, nil)
if err != nil {
t.Error(err)
return
}
req.Header.Set("Range", item.header)
resp, err := http.DefaultClient.Do(req)
if err != nil {
t.Error(err)
return
}
body, err := io.ReadAll(resp.Body)
_ = resp.Body.Close()
if err != nil {
t.Error(err)
return
}
if resp.StatusCode != http.StatusPartialContent {
t.Errorf("unexpected status for %s: %d", item.header, resp.StatusCode)
}
if string(body) != item.want {
t.Errorf("unexpected body for %s: %q", item.header, string(body))
}
}()
}
wg.Wait()
mu.Lock()
defer mu.Unlock()
if len(calls) != len(ranges) {
t.Fatalf("unexpected opener call count: %d", len(calls))
}
seen := map[OpenRequest]bool{}
for _, call := range calls {
seen[call] = true
}
for _, want := range []OpenRequest{{Offset: 0, End: 4}, {Offset: 10, End: 15}, {Offset: 20, End: 25}} {
if !seen[want] {
t.Fatalf("missing opener request %#v, got %#v", want, calls)
}
}
}
func TestRegistryRejectsRangeWithoutSize(t *testing.T) {
registry := NewRegistry(t.TempDir())
defer registry.Close()
_, err := registry.CreateOpener(func(ctx context.Context, req OpenRequest) (io.ReadCloser, error) {
return io.NopCloser(strings.NewReader("")), nil
}, &CreateOptions{Range: true})
if !errors.Is(err, ErrInvalidOptions) {
t.Fatalf("expected invalid options error, got %v", err)
}
}
func TestRegistryTracksSourceErrors(t *testing.T) {
t.Run("reader error", func(t *testing.T) {
registry := NewRegistry(t.TempDir())
defer registry.Close()
boom := errors.New("reader failed")
url, err := registry.CreateOpener(func(context.Context, OpenRequest) (io.ReadCloser, error) {
return io.NopCloser(io.MultiReader(strings.NewReader("partial"), &terminalErrorReader{err: boom})), nil
}, &CreateOptions{})
if err != nil {
t.Fatal(err)
}
resp, err := http.Get(url)
if err != nil {
t.Fatal(err)
}
body, readErr := io.ReadAll(resp.Body)
_ = resp.Body.Close()
if readErr != nil {
t.Fatalf("the HTTP transport should end normally, got %v", readErr)
}
if string(body) != "partial" {
t.Fatalf("unexpected partial body: %q", body)
}
if got := registry.SourceError(url); !errors.Is(got, boom) {
t.Fatalf("expected source error %v, got %v", boom, got)
}
})
t.Run("short known-size reader", func(t *testing.T) {
registry := NewRegistry(t.TempDir())
defer registry.Close()
url, err := registry.CreateOpener(func(context.Context, OpenRequest) (io.ReadCloser, error) {
return io.NopCloser(strings.NewReader("short")), nil
}, &CreateOptions{Size: 10})
if err != nil {
t.Fatal(err)
}
resp, err := http.Get(url)
if err != nil {
t.Fatal(err)
}
_, _ = io.ReadAll(resp.Body)
_ = resp.Body.Close()
if got := registry.SourceError(url); !errors.Is(got, io.ErrUnexpectedEOF) {
t.Fatalf("expected unexpected EOF, got %v", got)
}
})
t.Run("range reader error becomes terminal", func(t *testing.T) {
registry := NewRegistry(t.TempDir())
defer registry.Close()
boom := errors.New("range reader failed")
url, err := registry.CreateOpener(func(context.Context, OpenRequest) (io.ReadCloser, error) {
return io.NopCloser(io.MultiReader(strings.NewReader("partial"), &terminalErrorReader{err: boom})), nil
}, &CreateOptions{Size: 10, Range: true})
if err != nil {
t.Fatal(err)
}
for attempt := 0; attempt < rangeSourceFailureLimit; attempt++ {
req, err := http.NewRequest(http.MethodGet, url, nil)
if err != nil {
t.Fatal(err)
}
req.Header.Set("Range", "bytes=0-9")
resp, err := http.DefaultClient.Do(req)
if err != nil {
t.Fatal(err)
}
_, _ = io.ReadAll(resp.Body)
_ = resp.Body.Close()
}
if got := registry.SourceError(url); !errors.Is(got, boom) {
t.Fatalf("expected range source error %v, got %v", boom, got)
}
resp, err := http.Get(url)
if err != nil {
t.Fatal(err)
}
_ = resp.Body.Close()
if resp.StatusCode != http.StatusGone {
t.Fatalf("expected terminal source status %d, got %d", http.StatusGone, resp.StatusCode)
}
})
t.Run("short range reader", func(t *testing.T) {
registry := NewRegistry(t.TempDir())
defer registry.Close()
url, err := registry.CreateOpener(func(context.Context, OpenRequest) (io.ReadCloser, error) {
return io.NopCloser(strings.NewReader("short")), nil
}, &CreateOptions{Size: 10, Range: true})
if err != nil {
t.Fatal(err)
}
for attempt := 0; attempt < rangeSourceFailureLimit; attempt++ {
req, err := http.NewRequest(http.MethodGet, url, nil)
if err != nil {
t.Fatal(err)
}
req.Header.Set("Range", "bytes=0-9")
resp, err := http.DefaultClient.Do(req)
if err != nil {
t.Fatal(err)
}
_, _ = io.ReadAll(resp.Body)
_ = resp.Body.Close()
}
if got := registry.SourceError(url); !errors.Is(got, io.ErrUnexpectedEOF) {
t.Fatalf("expected range unexpected EOF, got %v", got)
}
})
t.Run("successful range retry clears transient failure", func(t *testing.T) {
registry := NewRegistry(t.TempDir())
defer registry.Close()
attempt := 0
boom := errors.New("transient range failure")
url, err := registry.CreateOpener(func(context.Context, OpenRequest) (io.ReadCloser, error) {
attempt++
if attempt == 1 {
return io.NopCloser(&terminalErrorReader{err: boom}), nil
}
return io.NopCloser(strings.NewReader("0123456789")), nil
}, &CreateOptions{Size: 10, Range: true})
if err != nil {
t.Fatal(err)
}
for i := 0; i < 2; i++ {
req, err := http.NewRequest(http.MethodGet, url, nil)
if err != nil {
t.Fatal(err)
}
req.Header.Set("Range", "bytes=0-9")
resp, err := http.DefaultClient.Do(req)
if err != nil {
t.Fatal(err)
}
_, _ = io.ReadAll(resp.Body)
_ = resp.Body.Close()
}
if got := registry.SourceError(url); got != nil {
t.Fatalf("successful retry left transient source error: %v", got)
}
})
t.Run("request cancellation is not a source error", func(t *testing.T) {
registry := NewRegistry(t.TempDir())
defer registry.Close()
opened := make(chan struct{})
url, err := registry.CreateOpener(func(ctx context.Context, _ OpenRequest) (io.ReadCloser, error) {
close(opened)
return &contextErrorReader{ctx: ctx}, nil
}, &CreateOptions{})
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithCancel(context.Background())
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
t.Fatal(err)
}
done := make(chan struct{})
go func() {
resp, _ := http.DefaultClient.Do(req)
if resp != nil {
_ = resp.Body.Close()
}
close(done)
}()
<-opened
cancel()
select {
case <-done:
case <-time.After(2 * time.Second):
t.Fatal("cancelled request did not finish")
}
if got := registry.SourceError(url); got != nil {
t.Fatalf("request cancellation was recorded as source error: %v", got)
}
})
t.Run("nil reader", func(t *testing.T) {
registry := NewRegistry(t.TempDir())
defer registry.Close()
url, err := registry.CreateOpener(func(context.Context, OpenRequest) (io.ReadCloser, error) {
return nil, nil
}, &CreateOptions{})
if err != nil {
t.Fatal(err)
}
resp, err := http.Get(url)
if err != nil {
t.Fatal(err)
}
_ = resp.Body.Close()
if resp.StatusCode != http.StatusGone {
t.Fatalf("expected nil reader status %d, got %d", http.StatusGone, resp.StatusCode)
}
if got := registry.SourceError(url); !errors.Is(got, ErrSourceClosed) {
t.Fatalf("expected closed source error, got %v", got)
}
})
}
func TestRegistryDoesNotRecordResponseWriteError(t *testing.T) {
registry := NewRegistry(t.TempDir())
defer registry.Close()
url, err := registry.CreateOpener(func(context.Context, OpenRequest) (io.ReadCloser, error) {
return io.NopCloser(strings.NewReader("payload")), nil
}, &CreateOptions{Size: 7})
if err != nil {
t.Fatal(err)
}
req, err := http.NewRequest(http.MethodGet, url, nil)
if err != nil {
t.Fatal(err)
}
writerErr := errors.New("client connection closed")
registry.ServeHTTP(&failingResponseWriter{header: make(http.Header), err: writerErr}, req)
if got := registry.SourceError(url); got != nil {
t.Fatalf("response write error was recorded as a source error: %v", got)
}
}
func TestRegistryTaskReferencesShareSource(t *testing.T) {
registry := NewRegistry(t.TempDir())
defer registry.Close()
url, err := registry.CreateBlob([]byte("shared"), "text/plain")
if err != nil {
t.Fatal(err)
}
if err := registry.Acquire(url); err != nil {
t.Fatal(err)
}
if err := registry.Acquire(url); err != nil {
t.Fatal(err)
}
if err := registry.Release(url); err != nil {
t.Fatal(err)
}
if !registry.IsURL(url) {
t.Fatal("the first task release revoked a source still used by another task")
}
resp, err := http.Get(url)
if err != nil {
t.Fatal(err)
}
body, readErr := io.ReadAll(resp.Body)
_ = resp.Body.Close()
if readErr != nil || string(body) != "shared" {
t.Fatalf("unexpected shared response body %q, error %v", body, readErr)
}
if err := registry.Release(url); err != nil {
t.Fatal(err)
}
if registry.IsURL(url) {
t.Fatal("the final task release did not revoke the source")
}
}
func TestRegistryUnclaimedSessionSourceExpires(t *testing.T) {
setUnclaimedSourceTTLForTest(t, 25*time.Millisecond)
registry := NewRegistry(t.TempDir())
defer registry.Close()
session := newCountingSession()
url, err := registry.CreateOpener(func(context.Context, OpenRequest) (io.ReadCloser, error) {
return io.NopCloser(strings.NewReader("unused")), nil
}, &CreateOptions{Size: 6, Session: session})
if err != nil {
t.Fatal(err)
}
if got := session.RefCount(); got != 1 {
t.Fatalf("unexpected initial session ref count: %d", got)
}
select {
case <-session.zero:
case <-time.After(2 * time.Second):
t.Fatal("unclaimed source did not expire")
}
if registry.IsURL(url) {
t.Fatal("expired source is still registered")
}
if got := session.RefCount(); got != 0 {
t.Fatalf("expired source retained its session: %d", got)
}
}
func TestRegistryAcquireStopsUnclaimedSourceExpiration(t *testing.T) {
const ttl = 25 * time.Millisecond
setUnclaimedSourceTTLForTest(t, ttl)
registry := NewRegistry(t.TempDir())
defer registry.Close()
session := newCountingSession()
url, err := registry.CreateOpener(func(context.Context, OpenRequest) (io.ReadCloser, error) {
return io.NopCloser(strings.NewReader("claimed")), nil
}, &CreateOptions{Size: 7, Session: session})
if err != nil {
t.Fatal(err)
}
if err := registry.Acquire(url); err != nil {
t.Fatal(err)
}
select {
case <-session.zero:
t.Fatal("claimed source expired")
case <-time.After(4 * ttl):
}
if !registry.IsURL(url) {
t.Fatal("claimed source was removed")
}
if err := registry.Release(url); err != nil {
t.Fatal(err)
}
select {
case <-session.zero:
case <-time.After(2 * time.Second):
t.Fatal("final task release did not release the session")
}
}
func TestRegistryUnclaimedExpirationKeepsActiveReaderSession(t *testing.T) {
setUnclaimedSourceTTLForTest(t, 100*time.Millisecond)
registry := NewRegistry(t.TempDir())
defer registry.Close()
session := newCountingSession()
reader, writer := io.Pipe()
opened := make(chan struct{})
url, err := registry.CreateOpener(func(context.Context, OpenRequest) (io.ReadCloser, error) {
close(opened)
return reader, nil
}, &CreateOptions{Size: 7, Session: session})
if err != nil {
t.Fatal(err)
}
type result struct {
body []byte
err error
}
resultCh := make(chan result, 1)
go func() {
resp, err := http.Get(url)
if err != nil {
resultCh <- result{err: err}
return
}
body, err := io.ReadAll(resp.Body)
_ = resp.Body.Close()
resultCh <- result{body: body, err: err}
}()
select {
case <-opened:
case got := <-resultCh:
t.Fatalf("request ended before the opener ran: body %q, error %v", got.body, got.err)
case <-time.After(2 * time.Second):
t.Fatal("active request did not open")
}
deadline := time.NewTimer(2 * time.Second)
ticker := time.NewTicker(5 * time.Millisecond)
defer deadline.Stop()
defer ticker.Stop()
for registry.IsURL(url) {
select {
case <-ticker.C:
case <-deadline.C:
t.Fatal("unclaimed source did not expire while its reader was active")
}
}
if got := session.RefCount(); got != 1 {
t.Fatalf("expiration released the active-reader session ref: %d", got)
}
if _, err := writer.Write([]byte("payload")); err != nil {
t.Fatal(err)
}
_ = writer.Close()
select {
case got := <-resultCh:
if got.err != nil || string(got.body) != "payload" {
t.Fatalf("active response was interrupted: body %q, error %v", got.body, got.err)
}
case <-time.After(2 * time.Second):
t.Fatal("active response did not finish after source expiration")
}
select {
case <-session.zero:
if got := session.RefCount(); got != 0 {
t.Fatalf("unexpected final session ref count: %d", got)
}
case <-time.After(2 * time.Second):
t.Fatal("active-reader session ref was not released")
}
}
func TestRegistryCloseCancelsUnclaimedExpiration(t *testing.T) {
const ttl = 25 * time.Millisecond
setUnclaimedSourceTTLForTest(t, ttl)
registry := NewRegistry(t.TempDir())
session := newCountingSession()
_, err := registry.CreateOpener(func(context.Context, OpenRequest) (io.ReadCloser, error) {
return io.NopCloser(strings.NewReader("unused")), nil
}, &CreateOptions{Size: 6, Session: session})
if err != nil {
t.Fatal(err)
}
if err := registry.Close(); err != nil {
t.Fatal(err)
}
select {
case <-session.zero:
case <-time.After(2 * time.Second):
t.Fatal("registry close did not release the session")
}
<-time.After(4 * ttl)
if got := session.RefCount(); got != 0 {
t.Fatalf("expiration released the session again after close: %d", got)
}
}
func TestRegistryRevokeKeepsActiveReaderSession(t *testing.T) {
registry := NewRegistry(t.TempDir())
defer registry.Close()
session := newCountingSession()
reader, writer := io.Pipe()
opened := make(chan struct{})
url, err := registry.CreateOpener(func(context.Context, OpenRequest) (io.ReadCloser, error) {
close(opened)
return reader, nil
}, &CreateOptions{Size: 7, Session: session})
if err != nil {
t.Fatal(err)
}
type result struct {
body []byte
err error
}
resultCh := make(chan result, 1)
go func() {
resp, err := http.Get(url)
if err != nil {
resultCh <- result{err: err}
return
}
body, err := io.ReadAll(resp.Body)
_ = resp.Body.Close()
resultCh <- result{body: body, err: err}
}()
<-opened
if got := session.RefCount(); got != 2 {
t.Fatalf("expected source and active-reader session refs, got %d", got)
}
if err := registry.Revoke(url); err != nil {
t.Fatal(err)
}
if got := session.RefCount(); got != 1 {
t.Fatalf("revoke released the active-reader session ref: %d", got)
}
if _, err := writer.Write([]byte("payload")); err != nil {
t.Fatal(err)
}
_ = writer.Close()
select {
case got := <-resultCh:
if got.err != nil || string(got.body) != "payload" {
t.Fatalf("active response was interrupted: body %q, error %v", got.body, got.err)
}
case <-time.After(2 * time.Second):
t.Fatal("active response did not finish after revoke")
}
select {
case <-session.zero:
if got := session.RefCount(); got != 0 {
t.Fatalf("unexpected final session ref count: %d", got)
}
case <-time.After(2 * time.Second):
t.Fatal("active-reader session ref was not released")
}
}
func TestRegistryHTTPErrorStatuses(t *testing.T) {
registry := NewRegistry(t.TempDir())
defer registry.Close()
url, err := registry.CreateBlob([]byte("secret"), "text/plain")
if err != nil {
t.Fatal(err)
}
badURL := url + "-missing"
resp, err := http.Get(badURL)
if err != nil {
t.Fatal(err)
}
_ = resp.Body.Close()
if resp.StatusCode != http.StatusNotFound {
t.Fatalf("unexpected missing source status: %d", resp.StatusCode)
}
req, err := http.NewRequest(http.MethodHead, url, nil)
if err != nil {
t.Fatal(err)
}
resp, err = http.DefaultClient.Do(req)
if err != nil {
t.Fatal(err)
}
_ = resp.Body.Close()
if resp.StatusCode != http.StatusMethodNotAllowed {
t.Fatalf("unexpected method status: %d", resp.StatusCode)
}
req, err = http.NewRequest(http.MethodGet, url, nil)
if err != nil {
t.Fatal(err)
}
req.Header.Set("Range", "bytes=99-100")
resp, err = http.DefaultClient.Do(req)
if err != nil {
t.Fatal(err)
}
_ = resp.Body.Close()
if resp.StatusCode != http.StatusRequestedRangeNotSatisfiable {
t.Fatalf("unexpected invalid range status: %d", resp.StatusCode)
}
if err := registry.Revoke(url); err != nil {
t.Fatal(err)
}
resp, err = http.Get(url)
if err != nil {
t.Fatal(err)
}
_ = resp.Body.Close()
if resp.StatusCode != http.StatusNotFound {
t.Fatalf("unexpected revoked source status: %d", resp.StatusCode)
}
if registry.IsURL(url) {
t.Fatal("revoked source should not be usable")
}
}
internal/protocol/gblob/README.md
-285
View File
@@ -1,285 +0,0 @@
# gblob Protocol
## Background
Gopeed extensions can resolve a download target into something that is not a normal remote URL.
Typical examples are:
- text generated by an extension
- binary data assembled in JavaScript
- a stream produced incrementally inside the extension runtime
Returning a plain `http` or `https` URL is not enough for these cases.
The `gblob` protocol provides an internal object-URL style transport so extensions can hand generated content to the downloader in a browser-like way.
The extension-facing model is intentionally close to the browser:
- `URL.createObjectURL(new Blob(...))`
- `URL.createObjectURL(readableStream)`
- `URL.createObjectURL(openReadable)`
- `URL.revokeObjectURL(url)`
The returned URL looks like:
```text
gblob:<id>
```
`gblob` is a process-local protocol. The source only exists inside the current Gopeed process.
## Design Goals
- make extension-generated data download as a normal task
- support eager `Blob` data, one-shot `ReadableStream` data, and resumable opener functions
- avoid large in-memory buffering by spooling to disk
- keep pause/continue behavior intuitive
- allow an extension to recreate a new `gblob` URL after source expiration
## High-Level Architecture
The implementation has two layers in this directory:
- [`registry.go`](./registry.go): process-local source registry and spool-file manager
- [`fetcher.go`](./fetcher.go): downloader fetcher implementation for the `gblob` scheme
The end-to-end flow is:
1. JavaScript creates a `Blob`, `ReadableStream`, or opener-function object URL.
2. Go registers a `gblob` source and assigns it a unique `gblob:<id>` URL.
3. Source bytes are written into a local spool file.
4. The downloader resolves the `gblob` URL through the `gblob` fetcher.
5. The fetcher copies bytes from the spool file into the final download target.
## Registry
The registry owns all active `gblob` sources for the current process.
Each source tracks:
- source id and `gblob` URL
- local spool-file path
- source type: `blob` or `writable_stream`
- written byte count
- source state: `open`, `closed`, `aborted`, `failed`
- revoke state
- pin count
- optional extension session reference
Key behaviors:
- `Blob` sources are written once and immediately closed.
- `ReadableStream` sources are bridged into the internal writable-source pipeline.
- opener-function sources are opened lazily and can be reopened from an offset.
- readers wait on a per-source notification channel when no new bytes are available yet.
- revocation only prevents future use; cleanup happens when the source is no longer pinned.
## Fetcher
The fetcher treats `gblob` as a local file-backed download source.
### Resolve
`Resolve()` looks up the source in the registry and builds a single-file `Resource`.
Behavior differences by source type:
- `Blob`: resolved size is the current written size
- `ReadableStream` / opener function: resolved size is whatever the extension declared; if unknown, it stays `0` until download completion updates the final size
`Range` is `true` only when the extension provides a resumable opener and marks the resource as ranged.
### Start
`Start()` opens the final target file and reads from the source spool file.
There are two resume modes:
- same `gblob` id: continue from the current target-file size
- different `gblob` id: truncate the target file and restart from byte `0`
The fetcher persists the last started source id so it can distinguish these two cases across pause/continue and task restore.
### Pause / Continue
`Pause()` stops the current copy loop by canceling the reader side.
It does not abort the `gblob` source.
If the source id stays the same, `Continue()` resumes from the existing target-file offset.
This is how normal in-process `Pause -> Continue` works.
### Source Replacement
If an extension replaces `req.url` with a new `gblob:<new-id>` before the next start, the fetcher detects the id change and restarts from scratch.
This behavior is important for recovery flows such as:
- original `gblob` source expired
- extension recreates a new `gblob` URL in `onError`
- task continues with the new source
## Lifecycle Notes
### In-process pause and continue
For the same `gblob` id:
- the partially downloaded output file is kept
- the fetcher resumes from the current file size
- opener-function sources can be reopened from the current offset
For non-reopenable one-shot `ReadableStream` sources:
- a normal in-process pause/continue only works while the same in-memory source is still alive
- they are not suitable for cross-session or reconstructable range resume
### Process restart
`gblob` sources are not persisted across process restarts.
An unfinished `gblob` task is restored without a live source behind its old URL.
The intended recovery pattern is:
1. user clicks continue
2. the old `gblob` source fails
3. extension `onError` runs
4. extension recreates a new `gblob` URL from `req.rawUrl`
5. extension updates `ctx.task.meta.req.url`
6. extension calls `ctx.task.continue()`
Because the new URL uses a different `gblob` id, the fetcher restarts from byte `0`.
## Why `rawUrl` Exists
When an extension resolves a request into `gblob:<id>`, the original site URL would otherwise be lost.
`Request.RawURL` keeps the original creation URL so lifecycle events can still reason about the original target.
Extension event URL matching prefers `req.rawUrl` when present.
This is especially useful for `onStart`, `onError`, and `onDone` on `gblob` tasks.
## Extension Usage
### 1. Create a `Blob` object URL
```js
gopeed.events.onResolve(async function (ctx) {
const blob = new Blob(["hello world"], { type: "text/plain" });
const url = URL.createObjectURL(blob);
ctx.res = {
name: "example",
files: [
{
name: "hello.txt",
size: 11,
req: { url }
}
]
};
});
```
### 2. Create a one-shot `ReadableStream` object URL
```js
gopeed.events.onResolve(async function (ctx) {
const encoder = new TextEncoder();
const stream = new ReadableStream({
start(controller) {
controller.enqueue(encoder.encode("line 1\n"));
controller.enqueue(encoder.encode("line 2\n"));
controller.close();
}
});
const url = URL.createObjectURL(stream);
ctx.res = {
name: "stream-example",
files: [
{
name: "stream.txt",
req: { url }
}
]
};
});
```
### 3. Create a resumable opener object URL
```js
gopeed.events.onResolve(async function (ctx) {
const target = "https://example.com/file.bin";
const openReadable = async (offset) => {
const headers = {};
if (offset > 0) {
headers.Range = `bytes=${offset}-`;
}
const response = await fetch(target, { headers });
if (response.status !== 200 || response.status !== 206) {
throw new Error(`unexpected response status: ${response.status}, expected ${expectedStatus}`);
}
if (!response.body) {
throw new Error("empty response body");
}
return response.body;
};
ctx.res = {
name: "range-example",
range: true,
files: [
{
name: "file.bin",
req: { url: URL.createObjectURL(openReadable) }
}
]
};
});
```
Important:
- opener functions must return a `ReadableStream`
- do not return `response.body.getReader()`
- use opener functions for resumable / range-aware sources
- validate HTTP status codes explicitly:
first request usually expects `200`, resumed range requests usually expect `206`
### 4. Recover in `onError`
```js
gopeed.events.onError(async function (ctx) {
const req = ctx.task.meta.req;
req.url = URL.createObjectURL(new Blob(["recovered\n"]));
ctx.task.continue();
});
```
This creates a new `gblob` id, so the downloader restarts from byte `0`.
## Limitations
- `gblob` is not a network protocol. Range support only exists when the extension recreates bytes through a resumable opener.
- unfinished `gblob` sources do not survive process restarts.
- recovery after restart depends on extension logic recreating a new source.
- the current restart behavior is full restart on source-id change, not byte-range recovery across recreated sources.
- one-shot `ReadableStream` tasks with unknown total size report size `0` until completion updates the final size.
## Summary
`gblob` lets Gopeed extensions treat generated content as a first-class download source.
The registry handles source lifetime and disk spooling.
The fetcher turns those sources into normal download tasks with predictable pause/continue behavior.
The most important rule is simple:
- same `gblob` id means resume
- different `gblob` id means restart
And for extension authors:
- use `ReadableStream` for one-shot streaming content
- use opener functions for resumable content
- opener functions must return `ReadableStream`, not a reader
internal/protocol/gblob/fetcher.go
-415
View File
@@ -1,415 +0,0 @@
package gblob
import (
"context"
"errors"
"io"
"os"
"path/filepath"
"strings"
"sync"
"sync/atomic"
"github.com/GopeedLab/gopeed/internal/controller"
"github.com/GopeedLab/gopeed/internal/fetcher"
"github.com/GopeedLab/gopeed/pkg/base"
)
type config struct{}
type Fetcher struct {
ctl *controller.Controller
registry *Registry
meta *fetcher.FetcherMeta
doneCh chan error
mu sync.Mutex
cancel context.CancelFunc
runDone chan struct{}
paused bool
closed bool
downloaded atomic.Int64
lastSourceID string
}
type fetcherData struct {
LastSourceID string `json:"lastSourceId"`
}
func (f *Fetcher) Setup(ctl *controller.Controller) {
f.ctl = ctl
if f.meta == nil {
f.meta = &fetcher.FetcherMeta{}
}
f.doneCh = make(chan error, 1)
}
func (f *Fetcher) Resolve(req *base.Request, opts *base.Options) error {
if f.registry == nil {
return ErrSourceNotFound
}
src, err := f.registry.Get(req.URL)
if err != nil {
return err
}
f.meta.Req = req
f.meta.Opts = opts
if f.meta.Opts == nil {
f.meta.Opts = &base.Options{}
}
name := f.meta.Opts.Name
if name == "" {
name = src.ID
}
snapshot := src.Snapshot()
size := snapshot.Written
if snapshot.DeclaredSize > 0 {
size = snapshot.DeclaredSize
}
if src.Type == SourceTypeWritableStream {
// Writable streams may continue producing bytes after Resolve,
// so treat the total size as unknown until the writer closes.
if snapshot.DeclaredSize == 0 {
size = 0
}
}
f.meta.Res = &base.Resource{
Size: size,
Range: snapshot.Range,
Files: []*base.FileInfo{
{
Name: name,
Path: "",
Size: size,
Req: req,
},
},
}
return nil
}
func (f *Fetcher) Start() error {
if f.registry == nil {
return ErrSourceNotFound
}
currentSourceID, err := ParseURL(f.meta.Req.URL)
if err != nil {
return err
}
f.mu.Lock()
if f.runDone != nil {
f.mu.Unlock()
return nil
}
f.paused = false
runDone := make(chan struct{})
ctx, cancel := context.WithCancel(context.Background())
f.cancel = cancel
f.runDone = runDone
f.mu.Unlock()
cleanupStartErr := func(err error) error {
f.mu.Lock()
if f.runDone == runDone {
f.runDone = nil
f.cancel = nil
}
f.mu.Unlock()
return err
}
path := f.meta.SingleFilepath()
if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
return cleanupStartErr(err)
}
target, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY, 0644)
if err != nil {
return cleanupStartErr(err)
}
offset := int64(0)
allowResumeAcrossSource := f.meta.Res != nil && f.meta.Res.Range
if f.lastSourceID != "" && f.lastSourceID != currentSourceID && !allowResumeAcrossSource {
if err := target.Truncate(0); err != nil {
target.Close()
return cleanupStartErr(err)
}
if _, err := target.Seek(0, io.SeekStart); err != nil {
target.Close()
return cleanupStartErr(err)
}
} else {
stat, err := target.Stat()
if err != nil {
target.Close()
return cleanupStartErr(err)
}
offset = stat.Size()
if _, err := target.Seek(offset, io.SeekStart); err != nil {
target.Close()
return cleanupStartErr(err)
}
}
f.lastSourceID = currentSourceID
f.downloaded.Store(offset)
sourceOffset := offset
src, err := f.registry.Get(f.meta.Req.URL)
if err != nil {
target.Close()
return cleanupStartErr(err)
}
snapshot := src.Snapshot()
if snapshot.Written == 0 && !snapshot.Started {
if f.meta.Res != nil && f.meta.Res.Range && offset > 0 {
if err := f.registry.Reopen(f.meta.Req.URL, offset); err != nil {
target.Close()
return cleanupStartErr(err)
}
sourceOffset = 0
} else if err := f.registry.StartSource(f.meta.Req.URL); err != nil {
target.Close()
return cleanupStartErr(err)
}
src, err = f.registry.Get(f.meta.Req.URL)
if err != nil {
target.Close()
return cleanupStartErr(err)
}
snapshot = src.Snapshot()
}
if f.meta.Res != nil && f.meta.Res.Range && (snapshot.State == SourceStateAborted || snapshot.State == SourceStateFailed) {
if err := f.registry.Reopen(f.meta.Req.URL, offset); err != nil {
target.Close()
return cleanupStartErr(err)
}
sourceOffset = 0
}
go f.copyLoop(ctx, target, offset, sourceOffset)
return nil
}
func (f *Fetcher) copyLoop(ctx context.Context, target *os.File, targetOffset, sourceOffset int64) {
var finalErr error
defer target.Close()
defer func() {
f.mu.Lock()
runDone := f.runDone
f.runDone = nil
f.cancel = nil
paused := f.paused
closed := f.closed
f.mu.Unlock()
close(runDone)
if closed {
select {
case f.doneCh <- nil:
default:
}
return
}
if paused {
return
}
select {
case f.doneCh <- finalErr:
default:
}
}()
src, err := f.registry.Get(f.meta.Req.URL)
if err != nil {
finalErr = err
return
}
sourceFile, err := os.Open(src.Path)
if err != nil {
finalErr = err
return
}
defer sourceFile.Close()
buf := make([]byte, 32*1024)
for {
if err := f.registry.WaitForReadable(ctx, f.meta.Req.URL, sourceOffset); err != nil {
if errors.Is(err, context.Canceled) {
return
}
finalErr = err
return
}
snapshot, serr := f.registry.Get(f.meta.Req.URL)
if serr != nil {
finalErr = serr
return
}
current := snapshot.Snapshot()
for sourceOffset < current.Written {
remain := current.Written - sourceOffset
readSize := int64(len(buf))
if remain < readSize {
readSize = remain
}
n, err := sourceFile.ReadAt(buf[:readSize], sourceOffset)
if err != nil && !errors.Is(err, io.EOF) {
finalErr = err
return
}
if n == 0 {
break
}
if _, err := target.Write(buf[:n]); err != nil {
finalErr = err
return
}
sourceOffset += int64(n)
targetOffset += int64(n)
f.downloaded.Store(targetOffset)
}
if current.State == SourceStateClosed && sourceOffset >= current.Written {
if f.meta.Res != nil {
if f.meta.Res.Size > 0 && targetOffset < f.meta.Res.Size {
finalErr = io.ErrUnexpectedEOF
return
}
if f.meta.Res.Size == 0 {
f.meta.Res.Size = targetOffset
}
f.meta.Res.Files[0].Size = f.meta.Res.Size
}
finalErr = nil
return
}
if current.State == SourceStateAborted || current.State == SourceStateFailed {
err := current.Err
if err == nil {
err = ErrSourceAborted
}
finalErr = err
return
}
}
}
func (f *Fetcher) Patch(req *base.Request, opts *base.Options) error {
return nil
}
func (f *Fetcher) Pause() error {
f.mu.Lock()
f.paused = true
cancel := f.cancel
runDone := f.runDone
f.mu.Unlock()
if cancel != nil {
cancel()
}
if runDone != nil {
<-runDone
}
return nil
}
func (f *Fetcher) Close() error {
f.mu.Lock()
f.closed = true
cancel := f.cancel
runDone := f.runDone
f.mu.Unlock()
if cancel != nil {
cancel()
}
if runDone != nil {
<-runDone
}
select {
case f.doneCh <- nil:
default:
}
return nil
}
func (f *Fetcher) Stats() any {
return map[string]any{}
}
func (f *Fetcher) Meta() *fetcher.FetcherMeta {
return f.meta
}
func (f *Fetcher) Progress() fetcher.Progress {
return fetcher.Progress{f.downloaded.Load()}
}
func (f *Fetcher) Wait() error {
return <-f.doneCh
}
type FetcherManager struct {
registry *Registry
}
func (fm *FetcherManager) SetRegistry(registry *Registry) {
fm.registry = registry
}
func (fm *FetcherManager) Name() string {
return Scheme
}
func (fm *FetcherManager) Filters() []*fetcher.SchemeFilter {
return []*fetcher.SchemeFilter{
{
Type: fetcher.FilterTypeUrl,
Pattern: strings.ToUpper(Scheme),
},
}
}
func (fm *FetcherManager) Build() fetcher.Fetcher {
return &Fetcher{
registry: fm.registry,
}
}
func (fm *FetcherManager) ParseName(u string) string {
id, err := ParseURL(u)
if err != nil {
return ""
}
return id
}
func (fm *FetcherManager) AutoRename() bool {
return true
}
func (fm *FetcherManager) DefaultConfig() any {
return &config{}
}
func (fm *FetcherManager) Store(f fetcher.Fetcher) (data any, err error) {
_f := f.(*Fetcher)
return &fetcherData{
LastSourceID: _f.lastSourceID,
}, nil
}
func (fm *FetcherManager) Restore() (v any, f func(meta *fetcher.FetcherMeta, v any) fetcher.Fetcher) {
return &fetcherData{}, func(meta *fetcher.FetcherMeta, v any) fetcher.Fetcher {
fd := v.(*fetcherData)
return &Fetcher{
registry: fm.registry,
meta: meta,
doneCh: make(chan error, 1),
lastSourceID: fd.LastSourceID,
}
}
}
func (fm *FetcherManager) Close() error {
return nil
}
internal/protocol/gblob/fetcher_test.go
-122
View File
@@ -1,122 +0,0 @@
package gblob
import (
"errors"
"io"
"os"
"path/filepath"
"testing"
"time"
"github.com/GopeedLab/gopeed/internal/controller"
internalfetcher "github.com/GopeedLab/gopeed/internal/fetcher"
"github.com/GopeedLab/gopeed/pkg/base"
)
func TestFetcherClosedSourceBeforeExpectedSizeReturnsError(t *testing.T) {
registry := NewRegistry(t.TempDir())
url, err := registry.CreateWritableStream(&CreateWritableStreamOptions{})
if err != nil {
t.Fatal(err)
}
if err := registry.Write(url, []byte("abc")); err != nil {
t.Fatal(err)
}
if err := registry.CloseSource(url); err != nil {
t.Fatal(err)
}
downloadDir := t.TempDir()
filePath := filepath.Join(downloadDir, "partial.bin")
f := &Fetcher{
registry: registry,
meta: &internalfetcher.FetcherMeta{
Req: &base.Request{URL: url},
Res: &base.Resource{
Size: 10,
Range: true,
Files: []*base.FileInfo{{Name: "partial.bin", Req: &base.Request{URL: url}}},
},
Opts: &base.Options{Path: downloadDir, Name: "partial.bin"},
},
}
f.Setup(controller.NewController())
if err := f.Start(); err != nil {
t.Fatal(err)
}
select {
case err := <-f.doneCh:
if !errors.Is(err, io.ErrUnexpectedEOF) {
t.Fatalf("expected unexpected EOF, got %v", err)
}
case <-time.After(2 * time.Second):
t.Fatal("timeout waiting for fetcher completion")
}
data, err := os.ReadFile(filePath)
if err != nil {
t.Fatal(err)
}
if string(data) != "abc" {
t.Fatalf("unexpected partial file content: %q", string(data))
}
}
func TestFetcherClosedUnknownSizeSourceCompletesSuccessfully(t *testing.T) {
registry := NewRegistry(t.TempDir())
url, err := registry.CreateWritableStream(&CreateWritableStreamOptions{})
if err != nil {
t.Fatal(err)
}
if err := registry.Write(url, []byte("abcdef")); err != nil {
t.Fatal(err)
}
if err := registry.CloseSource(url); err != nil {
t.Fatal(err)
}
downloadDir := t.TempDir()
filePath := filepath.Join(downloadDir, "stream.bin")
f := &Fetcher{
registry: registry,
meta: &internalfetcher.FetcherMeta{
Req: &base.Request{URL: url},
Res: &base.Resource{
Size: 0,
Range: false,
Files: []*base.FileInfo{{Name: "stream.bin", Req: &base.Request{URL: url}}},
},
Opts: &base.Options{Path: downloadDir, Name: "stream.bin"},
},
}
f.Setup(controller.NewController())
if err := f.Start(); err != nil {
t.Fatal(err)
}
select {
case err := <-f.doneCh:
if err != nil {
t.Fatalf("expected nil completion error, got %v", err)
}
case <-time.After(2 * time.Second):
t.Fatal("timeout waiting for fetcher completion")
}
data, err := os.ReadFile(filePath)
if err != nil {
t.Fatal(err)
}
if string(data) != "abcdef" {
t.Fatalf("unexpected stream file content: %q", string(data))
}
if got := f.meta.Res.Size; got != int64(len(data)) {
t.Fatalf("unexpected resolved size: got %d want %d", got, len(data))
}
if got := f.meta.Res.Files[0].Size; got != int64(len(data)) {
t.Fatalf("unexpected resolved file size: got %d want %d", got, len(data))
}
}
internal/protocol/gblob/registry.go
-617
View File
@@ -1,617 +0,0 @@
package gblob
import (
"context"
"errors"
"os"
"path/filepath"
"strings"
"sync"
gonanoid "github.com/matoous/go-nanoid/v2"
)
const Scheme = "gblob"
var (
ErrInvalidURL = errors.New("invalid gblob url")
ErrSourceNotFound = errors.New("gblob source not found")
ErrSourceRevoked = errors.New("gblob source revoked")
ErrSourceClosed = errors.New("gblob source closed")
ErrSourceAborted = errors.New("gblob source aborted")
)
type SourceType string
const (
SourceTypeBlob SourceType = "blob"
SourceTypeWritableStream SourceType = "writable_stream"
)
type SourceState string
const (
SourceStateOpen SourceState = "open"
SourceStateClosed SourceState = "closed"
SourceStateAborted SourceState = "aborted"
SourceStateFailed SourceState = "failed"
)
type SessionRef interface {
Retain()
Release()
}
type ResumeFunc func(offset int64) error
type CreateWritableStreamOptions struct {
Session SessionRef
Reopenable bool
}
type Snapshot struct {
Written int64
DeclaredSize int64
Range bool
Started bool
State SourceState
Err error
WaitCh <-chan struct{}
}
type Source struct {
ID string
URL string
Path string
Type SourceType
ContentType string
mu sync.Mutex
file *os.File
written int64
state SourceState
err error
revoked bool
pins int
waitCh chan struct{}
session SessionRef
sessionReleased bool
reopenable bool
rangeEnabled bool
declaredSize int64
started bool
reopen ResumeFunc
}
func (s *Source) Snapshot() Snapshot {
s.mu.Lock()
defer s.mu.Unlock()
return Snapshot{
Written: s.written,
DeclaredSize: s.declaredSize,
Range: s.rangeEnabled,
Started: s.started,
State: s.state,
Err: s.err,
WaitCh: s.waitCh,
}
}
func (s *Source) State() SourceState {
s.mu.Lock()
defer s.mu.Unlock()
return s.state
}
func (s *Source) IsRevoked() bool {
s.mu.Lock()
defer s.mu.Unlock()
return s.revoked
}
func (s *Source) pin() {
s.mu.Lock()
defer s.mu.Unlock()
s.pins++
}
func (s *Source) unpin() int {
s.mu.Lock()
defer s.mu.Unlock()
if s.pins > 0 {
s.pins--
}
return s.pins
}
func (s *Source) shouldCleanupLocked() bool {
return s.revoked && s.pins == 0 && s.state != SourceStateOpen
}
func (s *Source) notifyLocked() {
close(s.waitCh)
s.waitCh = make(chan struct{})
}
func (s *Source) releaseSessionLocked() {
if s.session != nil && !s.sessionReleased {
s.session.Release()
s.sessionReleased = true
}
}
func (s *Source) retainSessionLocked() {
if s.session != nil && s.sessionReleased {
s.session.Retain()
s.sessionReleased = false
}
}
func (s *Source) write(data []byte) error {
s.mu.Lock()
defer s.mu.Unlock()
if s.revoked {
return ErrSourceRevoked
}
if s.state != SourceStateOpen {
if s.state == SourceStateAborted || s.state == SourceStateFailed {
if s.err != nil {
return s.err
}
return ErrSourceAborted
}
return ErrSourceClosed
}
if len(data) == 0 {
return nil
}
n, err := s.file.Write(data)
if err != nil {
s.state = SourceStateFailed
s.err = err
s.file.Close()
s.file = nil
if !s.rangeEnabled {
s.releaseSessionLocked()
}
s.notifyLocked()
return err
}
s.written += int64(n)
s.notifyLocked()
return nil
}
func (s *Source) close() error {
s.mu.Lock()
defer s.mu.Unlock()
if s.state != SourceStateOpen {
return nil
}
s.state = SourceStateClosed
if s.file != nil {
if err := s.file.Close(); err != nil {
s.err = err
}
s.file = nil
}
s.releaseSessionLocked()
s.notifyLocked()
return nil
}
func (s *Source) abort(err error) error {
s.mu.Lock()
defer s.mu.Unlock()
if s.state != SourceStateOpen {
return nil
}
s.state = SourceStateAborted
if err != nil {
s.err = err
}
if s.file != nil {
_ = s.file.Close()
s.file = nil
}
if !s.rangeEnabled {
s.releaseSessionLocked()
}
s.notifyLocked()
return nil
}
func (s *Source) revoke() bool {
s.mu.Lock()
defer s.mu.Unlock()
s.revoked = true
if s.state == SourceStateOpen {
s.state = SourceStateAborted
if s.err == nil {
s.err = ErrSourceRevoked
}
if s.file != nil {
_ = s.file.Close()
s.file = nil
}
s.releaseSessionLocked()
s.notifyLocked()
}
return s.shouldCleanupLocked()
}
func (s *Source) waitForReadable(ctx context.Context, offset int64) error {
for {
snapshot := s.Snapshot()
if offset < snapshot.Written {
return nil
}
switch snapshot.State {
case SourceStateClosed:
return nil
case SourceStateAborted, SourceStateFailed:
if snapshot.Err != nil {
return snapshot.Err
}
return ErrSourceAborted
}
select {
case <-ctx.Done():
return ctx.Err()
case <-snapshot.WaitCh:
}
}
}
type Registry struct {
dir string
mu sync.RWMutex
sources map[string]*Source
}
func (r *Registry) Dir() string {
if r == nil {
return ""
}
return r.dir
}
func NewRegistry(baseDir string) *Registry {
dir := baseDir
if dir == "" {
dir = filepath.Join(os.TempDir(), "gopeed-gblob")
}
return &Registry{
dir: dir,
sources: make(map[string]*Source),
}
}
func BuildURL(id string) string {
return Scheme + ":" + id
}
func ParseURL(raw string) (string, error) {
if !strings.HasPrefix(strings.ToLower(raw), Scheme+":") {
return "", ErrInvalidURL
}
id := raw[len(Scheme)+1:]
if idx := strings.IndexByte(id, '/'); idx >= 0 {
id = id[:idx]
}
if id == "" {
return "", ErrInvalidURL
}
return id, nil
}
func (r *Registry) createSource(sourceType SourceType, contentType string, session SessionRef, reopenable bool, declaredSize int64) (*Source, error) {
if err := os.MkdirAll(r.dir, 0755); err != nil {
return nil, err
}
id, err := gonanoid.New()
if err != nil {
return nil, err
}
filePath := filepath.Join(r.dir, id)
file, err := os.Create(filePath)
if err != nil {
return nil, err
}
if session != nil {
session.Retain()
}
src := &Source{
ID: id,
URL: BuildURL(id),
Path: filePath,
Type: sourceType,
ContentType: contentType,
file: file,
state: SourceStateOpen,
waitCh: make(chan struct{}),
session: session,
reopenable: reopenable,
rangeEnabled: reopenable,
declaredSize: declaredSize,
started: !reopenable,
}
r.mu.Lock()
r.sources[id] = src
r.mu.Unlock()
return src, nil
}
func (r *Registry) CreateBlob(data []byte, contentType string) (string, error) {
src, err := r.createSource(SourceTypeBlob, contentType, nil, false, int64(len(data)))
if err != nil {
return "", err
}
if err := src.write(data); err != nil {
_ = os.Remove(src.Path)
return "", err
}
if err := src.close(); err != nil {
_ = os.Remove(src.Path)
return "", err
}
return src.URL, nil
}
func (r *Registry) CreateWritableStream(opts *CreateWritableStreamOptions) (string, error) {
if opts == nil {
opts = &CreateWritableStreamOptions{}
}
src, err := r.createSource(SourceTypeWritableStream, "", opts.Session, opts.Reopenable, 0)
if err != nil {
return "", err
}
return src.URL, nil
}
func (r *Registry) SetResumeOpener(raw string, reopen ResumeFunc) error {
src, err := r.get(raw)
if err != nil {
return err
}
src.mu.Lock()
defer src.mu.Unlock()
if !src.reopenable {
return nil
}
src.reopen = reopen
return nil
}
func (r *Registry) SetRange(raw string, enabled bool) error {
src, err := r.get(raw)
if err != nil {
return err
}
src.mu.Lock()
src.rangeEnabled = enabled
if !enabled && (src.state == SourceStateAborted || src.state == SourceStateFailed) {
src.releaseSessionLocked()
}
src.mu.Unlock()
return nil
}
func (r *Registry) SetSize(raw string, size int64) error {
src, err := r.get(raw)
if err != nil {
return err
}
src.mu.Lock()
if size < 0 {
size = 0
}
src.declaredSize = size
src.mu.Unlock()
return nil
}
func (r *Registry) get(raw string) (*Source, error) {
id, err := ParseURL(raw)
if err != nil {
return nil, err
}
r.mu.RLock()
defer r.mu.RUnlock()
src := r.sources[id]
if src == nil {
return nil, ErrSourceNotFound
}
return src, nil
}
func (r *Registry) Get(raw string) (*Source, error) {
return r.get(raw)
}
func (r *Registry) Write(raw string, data []byte) error {
src, err := r.get(raw)
if err != nil {
return err
}
return src.write(data)
}
func (r *Registry) CloseSource(raw string) error {
src, err := r.get(raw)
if err != nil {
return err
}
if err := src.close(); err != nil {
return err
}
r.cleanupIfNeeded(src)
return nil
}
func (r *Registry) AbortSource(raw string, err error) error {
src, gerr := r.get(raw)
if gerr != nil {
return gerr
}
if err := src.abort(err); err != nil {
return err
}
r.cleanupIfNeeded(src)
return nil
}
func (r *Registry) Revoke(raw string) error {
src, err := r.get(raw)
if err != nil {
return err
}
if src.revoke() {
r.cleanupIfNeeded(src)
}
return nil
}
func (r *Registry) Pin(raw string) error {
src, err := r.get(raw)
if err != nil {
return err
}
if src.IsRevoked() {
return ErrSourceRevoked
}
src.pin()
return nil
}
func (r *Registry) Unpin(raw string) {
src, err := r.get(raw)
if err != nil {
return
}
src.unpin()
r.cleanupIfNeeded(src)
}
func (r *Registry) WaitForReadable(ctx context.Context, raw string, offset int64) error {
src, err := r.get(raw)
if err != nil {
return err
}
return src.waitForReadable(ctx, offset)
}
func (r *Registry) Reopen(raw string, offset int64) error {
src, err := r.get(raw)
if err != nil {
return err
}
src.mu.Lock()
if !src.reopenable || src.reopen == nil {
src.mu.Unlock()
return ErrSourceNotFound
}
if src.revoked {
src.mu.Unlock()
return ErrSourceRevoked
}
if src.file != nil {
_ = src.file.Close()
src.file = nil
}
if err := os.MkdirAll(filepath.Dir(src.Path), 0755); err != nil {
src.mu.Unlock()
return err
}
file, err := os.Create(src.Path)
if err != nil {
src.mu.Unlock()
return err
}
src.file = file
src.written = 0
src.state = SourceStateOpen
src.err = nil
src.started = true
src.retainSessionLocked()
reopen := src.reopen
src.mu.Unlock()
if err := reopen(offset); err != nil {
src.mu.Lock()
if src.file != nil {
_ = src.file.Close()
src.file = nil
}
src.state = SourceStateFailed
src.err = err
src.releaseSessionLocked()
src.notifyLocked()
src.mu.Unlock()
return err
}
return nil
}
func (r *Registry) StartSource(raw string) error {
src, err := r.get(raw)
if err != nil {
return err
}
src.mu.Lock()
if !src.reopenable || src.started {
src.mu.Unlock()
return nil
}
src.mu.Unlock()
return r.Reopen(raw, 0)
}
func (r *Registry) cleanupIfNeeded(src *Source) {
src.mu.Lock()
shouldCleanup := src.shouldCleanupLocked()
file := src.file
if shouldCleanup {
src.releaseSessionLocked()
}
src.mu.Unlock()
if !shouldCleanup {
return
}
r.mu.Lock()
delete(r.sources, src.ID)
r.mu.Unlock()
if file != nil {
_ = file.Close()
}
_ = os.Remove(src.Path)
}
func (r *Registry) Close() error {
r.mu.Lock()
sources := r.sources
r.sources = make(map[string]*Source)
r.mu.Unlock()
var lastErr error
for _, src := range sources {
src.mu.Lock()
if src.file != nil {
if err := src.file.Close(); err != nil {
lastErr = err
}
src.file = nil
}
src.releaseSessionLocked()
src.notifyLocked()
src.mu.Unlock()
if err := os.Remove(src.Path); err != nil && !errors.Is(err, os.ErrNotExist) {
lastErr = err
}
}
if err := os.RemoveAll(r.dir); err != nil && !errors.Is(err, os.ErrNotExist) {
lastErr = err
}
return lastErr
}
internal/protocol/gblob/registry_test.go
-46
View File
@@ -1,46 +0,0 @@
package gblob
import (
"errors"
"testing"
)
func TestRegistry_RevokeOpenWritableStreamAbortsAndCleansUpAfterUnpin(t *testing.T) {
registry := NewRegistry(t.TempDir())
url, err := registry.CreateWritableStream(&CreateWritableStreamOptions{})
if err != nil {
t.Fatal(err)
}
if err := registry.Pin(url); err != nil {
t.Fatal(err)
}
src, err := registry.Get(url)
if err != nil {
t.Fatal(err)
}
if state := src.Snapshot().State; state != SourceStateOpen {
t.Fatalf("expected initial open state, got %s", state)
}
if err := registry.Revoke(url); err != nil {
t.Fatal(err)
}
src, err = registry.Get(url)
if err != nil {
t.Fatal(err)
}
snapshot := src.Snapshot()
if snapshot.State != SourceStateAborted {
t.Fatalf("expected revoked open source to become aborted, got %s", snapshot.State)
}
if !errors.Is(snapshot.Err, ErrSourceRevoked) {
t.Fatalf("expected revoke error, got %v", snapshot.Err)
}
registry.Unpin(url)
if _, err := registry.Get(url); !errors.Is(err, ErrSourceNotFound) {
t.Fatalf("expected source cleanup after unpin, got %v", err)
}
}
pkg/download/downloader.go
+378 -123
View File
@@ -15,10 +15,10 @@ import (
"sync/atomic"
"time"
internalblob "github.com/GopeedLab/gopeed/internal/blob"
"github.com/GopeedLab/gopeed/internal/controller"
"github.com/GopeedLab/gopeed/internal/fetcher"
"github.com/GopeedLab/gopeed/internal/logger"
protogblob "github.com/GopeedLab/gopeed/internal/protocol/gblob"
"github.com/GopeedLab/gopeed/pkg/base"
"github.com/GopeedLab/gopeed/pkg/protocol/http"
"github.com/GopeedLab/gopeed/pkg/util"
@@ -112,7 +112,7 @@ type Downloader struct {
claimedExtractions sync.Map
extensions []*Extension
gblob *protogblob.Registry
blob *internalblob.Registry
}
func NewDownloader(cfg *DownloaderConfig) *Downloader {
@@ -144,11 +144,7 @@ func NewDownloader(cfg *DownloaderConfig) *Downloader {
}
func (d *Downloader) Setup() error {
gblobDir := filepath.Join(d.cfg.StorageDir, "gblob")
if err := os.RemoveAll(gblobDir); err != nil && !errors.Is(err, os.ErrNotExist) {
return err
}
d.gblob = protogblob.NewRegistry(gblobDir)
d.blob = internalblob.NewRegistry("")
// setup storage
if err := d.storage.Setup([]string{bucketTask, bucketSave, bucketProtocolState, bucketConfig, bucketExtension, bucketExtensionStorage}); err != nil {
@@ -175,9 +171,6 @@ func (d *Downloader) Setup() error {
if _, ok := d.cfg.DownloaderStoreConfig.ProtocolConfig[protocol]; !ok {
d.cfg.DownloaderStoreConfig.ProtocolConfig[protocol] = fm.DefaultConfig()
}
if gfm, ok := fm.(*protogblob.FetcherManager); ok {
gfm.SetRegistry(d.gblob)
}
if sfm, ok := fm.(fetcher.StatefulFetcherManager); ok {
sfm.SetStateStore(&protocolStateStore{
storage: d.storage,
@@ -203,10 +196,6 @@ func (d *Downloader) Setup() error {
}
d.assignFetcherManager(task)
initTask(task)
if task.Protocol == protogblob.Scheme && task.Status != base.DownloadStatusDone && task.Status != base.DownloadStatusError {
task.Status = base.DownloadStatusError
continue
}
if task.Status != base.DownloadStatusDone && task.Status != base.DownloadStatusError {
task.Status = base.DownloadStatusPause
}
@@ -253,13 +242,18 @@ func (d *Downloader) Setup() error {
if len(d.tasks) > 0 {
for _, task := range d.tasks {
func() {
task.statusLock.Lock()
defer task.statusLock.Unlock()
if task.Status != base.DownloadStatusRunning && !task.Uploading {
// Do not acquire d.lock (via GetTask) while holding
// statusLock; scheduling uses the opposite lock order.
if d.GetTask(task.ID) == nil {
return
}
// check if task is deleted
if d.GetTask(task.ID) == nil || task.fetcher == nil {
task.statusLock.Lock()
if task.Status != base.DownloadStatusRunning && !task.Uploading {
task.statusLock.Unlock()
return
}
if task.fetcher == nil {
task.statusLock.Unlock()
return
}
@@ -281,6 +275,8 @@ func (d *Downloader) Setup() error {
task.Progress.UploadSpeed = task.updateUploadSpeed(currentUploaded-task.Progress.Uploaded, tick)
task.Progress.Uploaded = currentUploaded
}
task.statusLock.Unlock()
// Listener callbacks may Pause/Continue and acquire statusLock.
d.emit(EventKeyProgress, task)
// store fetcher progress when download/upload data changed
@@ -360,6 +356,12 @@ func (d *Downloader) setupFetcher(fm fetcher.FetcherManager, fetcher fetcher.Fet
}
// Get proxy config, task request proxy config has higher priority, then use global proxy config
ctl.GetProxy = func(requestProxy *base.RequestProxy) func(*gohttp.Request) (*url.URL, error) {
// Registry-backed blob URLs are loopback capability URLs and must never be
// sent through a configured proxy. Only exact, currently registered URLs
// take this path; everything else keeps normal HTTP proxy behavior.
if d.blob != nil && fetcher != nil && fetcher.Meta() != nil && fetcher.Meta().Req != nil && d.blob.IsURL(fetcher.Meta().Req.URL) {
return nil
}
if requestProxy == nil {
return d.cfg.Proxy.ToHandler()
}
@@ -502,11 +504,6 @@ func (d *Downloader) CreateDirect(req *base.Request, opts *base.Options) (taskId
if err != nil {
return
}
if strings.HasPrefix(strings.ToLower(req.URL), protogblob.Scheme+":") {
if err = fetcher.Resolve(req, initOpt); err != nil {
return
}
}
return d.doCreate(fetcher, initOpt)
}
@@ -549,34 +546,40 @@ func (d *Downloader) Patch(id string, req *base.Request, opts *base.Options) err
if task == nil {
return ErrTaskNotFound
}
if err := func() error {
task.lock.Lock()
defer task.lock.Unlock()
// Restore fetcher if not loaded
if task.fetcher == nil {
err := func() error {
task.statusLock.Lock()
defer task.statusLock.Unlock()
// Restore fetcher if not loaded
if task.fetcher == nil {
err := func() error {
task.statusLock.Lock()
defer task.statusLock.Unlock()
return d.restoreFetcher(task)
}()
if err != nil {
return d.restoreFetcher(task)
}()
if err != nil {
return err
}
}
// Call the fetcher's Patch method
if err := task.fetcher.Patch(req, opts); err != nil {
return err
}
}
// Call the fetcher's Patch method
if err := task.fetcher.Patch(req, opts); err != nil {
// Update task meta from fetcher and transfer any live Blob lease.
task.Meta = task.fetcher.Meta()
if err := d.syncBlobTaskLease(task); err != nil {
return err
}
return d.saveTask(task)
}(); err != nil {
return err
}
// Update task meta from fetcher
task.Meta = task.fetcher.Meta()
// Save task to storage
if err := d.saveTask(task); err != nil {
return err
}
// Emit progress event to notify listeners
// Emit outside task.lock so listeners may safely call back into Patch/Pause.
d.emit(EventKeyProgress, task)
return nil
@@ -634,6 +637,7 @@ func (d *Downloader) Continue(filter *TaskFilter) (err error) {
}
realContinueTasks := make([]*Task, 0)
pausedTasks := make([]*Task, 0)
func() {
d.lock.Lock()
defer d.lock.Unlock()
@@ -646,12 +650,16 @@ func (d *Downloader) Continue(filter *TaskFilter) (err error) {
pausedCount := 0
for _, task := range d.tasks {
if task.Status == base.DownloadStatusRunning {
if err = d.doPause(task); err != nil {
var queued bool
queued, err = d.doPauseForScheduling(task)
if err != nil {
return
}
task.Status = base.DownloadStatusWait
d.waitTasks = append(d.waitTasks, task)
pausedCount++
if queued {
d.waitTasks = append(d.waitTasks, task)
pausedTasks = append(pausedTasks, task)
pausedCount++
}
}
if pausedCount == needPauseCount {
break
@@ -668,6 +676,14 @@ func (d *Downloader) Continue(filter *TaskFilter) (err error) {
}
}
}()
if err != nil {
return
}
// Scheduling pauses run while d.lock is held, but events must not: listeners
// are allowed to call back into Downloader methods that acquire d.lock.
for _, task := range pausedTasks {
d.emit(EventKeyPause, task)
}
for _, task := range realContinueTasks {
if err = d.doStart(task); err != nil {
@@ -811,7 +827,7 @@ func (d *Downloader) Stats(id string) (sr any, err error) {
}
func (d *Downloader) doDelete(task *Task, force bool) (err error) {
defer d.releaseGBlobTask(task)
defer d.releaseBlobTask(task)
err = func() error {
if err := d.storage.Delete(bucketTask, task.ID); err != nil {
return err
@@ -856,8 +872,8 @@ func (d *Downloader) Close() error {
for _, fm := range d.cfg.FetchManagers {
closeArr = append(closeArr, fm.Close)
}
if d.gblob != nil {
closeArr = append(closeArr, d.gblob.Close)
if d.blob != nil {
closeArr = append(closeArr, d.blob.Close)
}
closeArr = append(closeArr, d.storage.Close)
// Make sure all resources are released, if had error, return the last error
@@ -1024,7 +1040,6 @@ func (d *Downloader) watch(task *Task) {
if _, loaded := d.watchedTasks.LoadOrStore(task.ID, true); loaded {
return
}
defer func() {
d.watchedTasks.Delete(task.ID)
}()
@@ -1047,15 +1062,27 @@ func (d *Downloader) watch(task *Task) {
}
}
if task.Status == base.DownloadStatusDone {
if d.taskStatus(task) == base.DownloadStatusDone {
return
}
for {
err := task.fetcher.Wait()
if err != nil {
d.doOnError(task, err)
if task.Status == base.DownloadStatusRunning {
status := d.taskStatus(task)
if status == base.DownloadStatusPause || status == base.DownloadStatusWait {
return
}
var sourceErr error
if d.blob != nil && task.Meta != nil && task.Meta.Req != nil {
sourceErr = d.blob.SourceError(task.Meta.Req.URL)
}
if sourceErr != nil {
d.doOnBlobSourceError(task, sourceErr)
} else {
d.doOnError(task, err)
}
if d.taskStatus(task) == base.DownloadStatusRunning {
continue
}
return
@@ -1065,6 +1092,15 @@ func (d *Downloader) watch(task *Task) {
if task.Meta.Res == nil || d.GetTask(task.ID) == nil {
return
}
if d.blob != nil && task.Meta.Req != nil {
if err := d.blob.SourceError(task.Meta.Req.URL); err != nil {
d.doOnBlobSourceError(task, err)
if d.taskStatus(task) == base.DownloadStatusRunning {
continue
}
return
}
}
task.Progress.Used = task.timer.Used()
if task.Meta.Res.Size == 0 {
@@ -1077,12 +1113,14 @@ func (d *Downloader) watch(task *Task) {
totalSize := task.Meta.Res.Size
task.Progress.Speed = totalSize / used
task.Progress.Downloaded = totalSize
task.updateStatus(base.DownloadStatusDone)
if !d.markTaskDone(task) {
return
}
d.storage.Put(bucketTask, task.ID, task.clone())
d.emit(EventKeyDone, task)
d.emit(EventKeyFinally, task, err)
d.notifyRunning()
d.releaseGBlobTask(task)
d.releaseBlobTask(task)
d.triggerOnDone(task)
d.triggerWebhooks(WebhookEventDownloadDone, task, nil)
d.triggerScripts(ScriptEventDownloadDone, task, nil)
@@ -1148,13 +1186,51 @@ func (d *Downloader) watch(task *Task) {
}
func (d *Downloader) doOnError(task *Task, err error) {
d.handleOnError(task, err, false)
}
func (d *Downloader) doOnBlobSourceError(task *Task, err error) {
d.handleOnError(task, err, true)
}
func (d *Downloader) handleOnError(task *Task, err error, resetFetcher bool) {
if task == nil || task.lock == nil {
return
}
task.lock.Lock()
if !d.markTaskError(task) {
task.lock.Unlock()
return
}
d.Logger.Warn().Err(err).Msgf("task download failed, task id: %s", task.ID)
task.updateStatus(base.DownloadStatusError)
oldURL := ""
if task.Meta != nil && task.Meta.Req != nil {
oldURL = task.Meta.Req.URL
}
d.triggerOnError(task, err)
if task.Status == base.DownloadStatusError {
if !d.isResumableGBlobTask(task) {
d.releaseGBlobTask(task)
newURL := ""
if task.Meta != nil && task.Meta.Req != nil {
newURL = task.Meta.Req.URL
}
if oldURL != "" && newURL != oldURL {
// Extensions are allowed to replace an expired URL from onError. Patch the
// existing fetcher through its public API so protocol-owned redirect state
// is reset without teaching the HTTP fetcher about blob URLs.
if task.fetcher != nil {
if patchErr := task.fetcher.Patch(&base.Request{URL: newURL}, nil); patchErr != nil {
d.Logger.Warn().Err(patchErr).Msgf("patch recovered task url failed, task id: %s", task.ID)
}
}
d.releaseBlobTaskURL(task, oldURL)
}
if resetFetcher && d.taskStatus(task) == base.DownloadStatusRunning {
if resetErr := d.resetTaskFetcher(task); resetErr != nil {
d.Logger.Warn().Err(resetErr).Msgf("reset recovered task fetcher failed, task id: %s", task.ID)
}
}
task.lock.Unlock()
if d.taskStatus(task) == base.DownloadStatusError {
d.releaseBlobTask(task)
d.emit(EventKeyError, task, err)
d.emit(EventKeyFinally, task, err)
d.notifyRunning()
@@ -1162,13 +1238,43 @@ func (d *Downloader) doOnError(task *Task, err error) {
}
}
func (d *Downloader) resetTaskFetcher(task *Task) error {
if task == nil || task.Meta == nil || task.Meta.Req == nil {
return errors.New("task request is unavailable")
}
fm, err := d.parseFm(task.Meta.Req.URL)
if err != nil {
return err
}
opts := task.Meta.Opts
f := fm.Build()
d.setupFetcher(fm, f)
// A fresh source must never inherit bytes or preallocation from the failed
// source. HTTP opens existing targets without truncating them, so remove the
// old single-file target before installing a fresh fetcher.
if task.Meta.Res != nil && task.Meta.Opts != nil && len(task.Meta.Res.Files) > 0 {
if err := util.SafeRemove(task.Meta.SingleFilepath()); err != nil {
return err
}
}
f.Meta().Req = task.Meta.Req
f.Meta().Opts = opts
task.fetcherManager = fm
task.fetcher = f
task.Meta = f.Meta()
if task.Progress != nil {
task.Progress.Downloaded = 0
task.Progress.Speed = 0
}
return d.storage.Delete(bucketSave, task.ID)
}
func (d *Downloader) restoreTask(task *Task) error {
if task.fetcher == nil {
if err := d.restoreFetcher(task); err != nil {
return err
}
}
go d.watch(task)
return nil
}
@@ -1207,17 +1313,6 @@ func (d *Downloader) doCreate(f fetcher.Fetcher, opts *base.Options) (taskId str
if err != nil {
return
}
if fm.Name() == protogblob.Scheme && d.gblob != nil {
if err = d.gblob.Pin(f.Meta().Req.URL); err != nil {
return "", err
}
defer func() {
if err != nil {
d.gblob.Unpin(f.Meta().Req.URL)
}
}()
}
task := NewTask()
task.fetcherManager = fm
task.fetcher = f
@@ -1226,6 +1321,14 @@ func (d *Downloader) doCreate(f fetcher.Fetcher, opts *base.Options) (taskId str
task.Progress = &Progress{}
_, task.Uploading = f.(fetcher.Uploader)
initTask(task)
if err = d.syncBlobTaskLease(task); err != nil {
return "", err
}
defer func() {
if err != nil {
d.releaseBlobTask(task)
}
}()
if err = d.storage.Put(bucketTask, task.ID, task.clone()); err != nil {
return
}
@@ -1247,7 +1350,6 @@ func (d *Downloader) doCreate(f fetcher.Fetcher, opts *base.Options) (taskId str
err = d.doStart(task)
}()
go d.watch(task)
return
}
@@ -1293,6 +1395,7 @@ func (d *Downloader) statusMut(task *Task, fn func() (bool, error)) (bool, error
func (d *Downloader) doStart(task *Task) (err error) {
var isCreate bool
var generation uint64
isReturn, err := d.statusMut(task, func() (isReturn bool, err error) {
if task.Status == base.DownloadStatusRunning || task.Status == base.DownloadStatusDone {
isReturn = true
@@ -1306,6 +1409,8 @@ func (d *Downloader) doStart(task *Task) (err error) {
}
isCreate = task.Status == base.DownloadStatusReady
task.updateStatus(base.DownloadStatusRunning)
task.runGeneration++
generation = task.runGeneration
return
})
@@ -1317,16 +1422,31 @@ func (d *Downloader) doStart(task *Task) (err error) {
return
}
started := false
handler := func() error {
task.lock.Lock()
defer task.lock.Unlock()
if !d.taskIsRunningGeneration(task, generation) {
return nil
}
if task.fetcher == nil {
if err := d.restoreFetcher(task); err != nil {
return err
}
}
d.triggerOnStart(task)
if !d.taskIsRunningGeneration(task, generation) {
return nil
}
if task.fetcher != nil {
task.fetcher.Meta().Req = task.Meta.Req
task.fetcher.Meta().Res = task.Meta.Res
task.fetcher.Meta().Opts = task.Meta.Opts
}
if err := d.syncBlobTaskLease(task); err != nil {
return err
}
if task.Meta.Res == nil {
err := task.fetcher.Resolve(task.Meta.Req, task.Meta.Opts)
if err != nil {
@@ -1364,19 +1484,31 @@ func (d *Downloader) doStart(task *Task) (err error) {
}
task.Progress.Speed = 0
if !d.taskIsRunningGeneration(task, generation) {
return nil
}
task.timer.Start()
if err := task.fetcher.Start(); err != nil {
return err
}
started = true
if err := d.saveTask(task); err != nil {
return err
}
d.emit(EventKeyStart, task)
return nil
}
go func() {
if err := handler(); err != nil {
d.doOnError(task, err)
if d.taskIsRunningGeneration(task, generation) {
d.doOnError(task, err)
}
return
}
if started {
if d.taskIsRunningGeneration(task, generation) {
d.emit(EventKeyStart, task)
}
go d.watch(task)
}
}()
@@ -1384,51 +1516,137 @@ func (d *Downloader) doStart(task *Task) (err error) {
}
func (d *Downloader) doPause(task *Task) (err error) {
isReturn, err := d.statusMut(task, func() (isReturn bool, err error) {
generation, isReturn, err := d.preparePause(task)
if err != nil || isReturn {
return err
}
handled, err := d.runPauseHandler(task, generation, true)
if err != nil {
d.Logger.Error().Stack().Err(err).Msgf("pause task handle failed, task id: %s", task.ID)
return err
}
if handled {
d.emit(EventKeyPause, task)
}
return nil
}
// doPauseForScheduling physically pauses a running task without emitting while
// the caller holds d.lock, then atomically moves it to the wait queue state.
func (d *Downloader) doPauseForScheduling(task *Task) (bool, error) {
generation, isReturn, err := d.preparePause(task)
if err != nil || isReturn {
return false, err
}
handled, err := d.runPauseHandler(task, generation, true)
if err != nil || !handled {
return false, err
}
task.statusLock.Lock()
queued := task.Status == base.DownloadStatusPause && task.runGeneration == generation
if queued {
task.Status = base.DownloadStatusWait
}
task.statusLock.Unlock()
return queued, nil
}
func (d *Downloader) preparePause(task *Task) (generation uint64, isReturn bool, err error) {
isReturn, err = d.statusMut(task, func() (isReturn bool, err error) {
if task.Status == base.DownloadStatusPause || task.Status == base.DownloadStatusDone {
isReturn = true
return
}
task.updateStatus(base.DownloadStatusPause)
task.runGeneration++
generation = task.runGeneration
task.timer.Pause()
return
})
if err != nil {
d.Logger.Error().Stack().Err(err).Msgf("pause task failed, task id: %s", task.ID)
return
}
if isReturn {
return
return generation, isReturn, err
}
func (d *Downloader) runPauseHandler(task *Task, generation uint64, pauseFetcher bool) (bool, error) {
task.lock.Lock()
defer task.lock.Unlock()
if !d.taskIsPausedGeneration(task, generation) {
return false, nil
}
handler := func() error {
task.lock.Lock()
defer task.lock.Unlock()
if task.fetcher != nil {
if err := task.fetcher.Pause(); err != nil {
return err
}
if pauseFetcher && task.fetcher != nil {
if err := task.fetcher.Pause(); err != nil {
return false, err
}
if task.fetcherManager != nil && task.fetcher != nil {
if err := d.saveTask(task); err != nil {
return err
}
} else {
if err := d.storage.Put(bucketTask, task.ID, task.clone()); err != nil {
return err
}
}
d.emit(EventKeyPause, task)
return nil
}
go func() {
if err := handler(); err != nil {
d.Logger.Error().Stack().Err(err).Msgf("pause task handle failed, task id: %s", task.ID)
if task.fetcherManager != nil && task.fetcher != nil {
if err := d.saveTask(task); err != nil {
return false, err
}
}()
return
} else {
if err := d.storage.Put(bucketTask, task.ID, task.clone()); err != nil {
return false, err
}
}
return true, nil
}
func (d *Downloader) taskIsRunningGeneration(task *Task, generation uint64) bool {
if task == nil || task.statusLock == nil {
return false
}
task.statusLock.Lock()
defer task.statusLock.Unlock()
return task.Status == base.DownloadStatusRunning && task.runGeneration == generation
}
func (d *Downloader) taskIsPausedGeneration(task *Task, generation uint64) bool {
if task == nil || task.statusLock == nil {
return false
}
task.statusLock.Lock()
defer task.statusLock.Unlock()
return (task.Status == base.DownloadStatusPause || task.Status == base.DownloadStatusWait) && task.runGeneration == generation
}
func (d *Downloader) taskStatus(task *Task) base.Status {
if task == nil || task.statusLock == nil {
return ""
}
task.statusLock.Lock()
defer task.statusLock.Unlock()
return task.Status
}
func (d *Downloader) markTaskDone(task *Task) bool {
if task == nil || task.statusLock == nil {
return false
}
task.statusLock.Lock()
defer task.statusLock.Unlock()
if task.Status != base.DownloadStatusRunning {
return false
}
task.updateStatus(base.DownloadStatusDone)
task.runGeneration++
return true
}
func (d *Downloader) markTaskError(task *Task) bool {
if task == nil || task.statusLock == nil {
return false
}
task.statusLock.Lock()
defer task.statusLock.Unlock()
if task.Status != base.DownloadStatusRunning {
return false
}
task.updateStatus(base.DownloadStatusError)
task.runGeneration++
return true
}
// redirect stderr to log file, when panic happened log it
@@ -1845,40 +2063,77 @@ func initTask(task *Task) {
task.statusLock = &sync.Mutex{}
task.lock = &sync.Mutex{}
task.blobRefLock = &sync.Mutex{}
task.speedArr = make([]int64, 0)
task.uploadSpeedArr = make([]int64, 0)
}
func (d *Downloader) unpinGBlobTask(task *Task) {
if d.gblob == nil || task == nil || task.Meta == nil || task.Meta.Req == nil || task.Meta.Req.URL == "" {
func (d *Downloader) releaseBlobTask(task *Task) {
if d.blob == nil || task == nil {
return
}
if task.Protocol != protogblob.Scheme {
return
if task.blobRefLock != nil {
task.blobRefLock.Lock()
raw := task.blobURL
task.blobURL = ""
task.blobRefLock.Unlock()
if raw != "" {
_ = d.blob.Release(raw)
}
}
d.gblob.Unpin(task.Meta.Req.URL)
}
func (d *Downloader) isResumableGBlobTask(task *Task) bool {
if task == nil || task.Meta == nil || task.Meta.Res == nil {
return false
func (d *Downloader) syncBlobTaskLease(task *Task) error {
if d.blob == nil || task == nil || task.Meta == nil || task.Meta.Req == nil {
return nil
}
return task.Protocol == protogblob.Scheme && task.Meta.Res.Range
if task.blobRefLock == nil {
task.blobRefLock = &sync.Mutex{}
}
task.blobRefLock.Lock()
defer task.blobRefLock.Unlock()
terminal := false
if task.statusLock != nil {
task.statusLock.Lock()
terminal = task.Status == base.DownloadStatusDone || task.Status == base.DownloadStatusError
task.statusLock.Unlock()
}
newURL := ""
if !terminal && d.blob.IsURL(task.Meta.Req.URL) {
newURL = task.Meta.Req.URL
}
if task.blobURL == newURL {
return nil
}
if newURL != "" {
if err := d.blob.Acquire(newURL); err != nil {
return err
}
}
oldURL := task.blobURL
task.blobURL = newURL
if oldURL != "" {
_ = d.blob.Release(oldURL)
}
return nil
}
func (d *Downloader) revokeGBlobTask(task *Task) {
if d.gblob == nil || task == nil || task.Meta == nil || task.Meta.Req == nil || task.Meta.Req.URL == "" {
func (d *Downloader) releaseBlobTaskURL(task *Task, raw string) {
if d.blob == nil || task == nil || raw == "" {
return
}
if task.Protocol != protogblob.Scheme {
return
if task.blobRefLock == nil {
task.blobRefLock = &sync.Mutex{}
}
task.blobRefLock.Lock()
claimed := task.blobURL == raw
if claimed {
task.blobURL = ""
}
task.blobRefLock.Unlock()
if claimed {
_ = d.blob.Release(raw)
}
_ = d.gblob.Revoke(task.Meta.Req.URL)
}
func (d *Downloader) releaseGBlobTask(task *Task) {
d.revokeGBlobTask(task)
d.unpinGBlobTask(task)
}
var defaultDownloader = NewDownloader(nil)
pkg/download/downloader_test.go
+648 -51
View File
@@ -2,18 +2,26 @@ package download
import (
"archive/zip"
"bytes"
"context"
"errors"
"fmt"
"io"
"net"
gohttp "net/http"
"net/http/httptest"
"os"
"path/filepath"
"reflect"
"strconv"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
internalblob "github.com/GopeedLab/gopeed/internal/blob"
"github.com/GopeedLab/gopeed/internal/controller"
"github.com/GopeedLab/gopeed/internal/fetcher"
"github.com/GopeedLab/gopeed/internal/test"
"github.com/GopeedLab/gopeed/pkg/base"
@@ -21,9 +29,135 @@ import (
"github.com/GopeedLab/gopeed/pkg/util"
)
func newTestDownloadOpt() *base.Options {
type generationTestManager struct {
starts atomic.Int32
pauses atomic.Int32
holdOpen bool
resolveStarted chan struct{}
resolveRelease <-chan struct{}
resolveErr error
resolveOnce sync.Once
pauseStarted chan struct{}
pauseRelease <-chan struct{}
pauseOnce sync.Once
}
func TestExtensionTaskControlMethodsOnlyExistOnError(t *testing.T) {
taskType := reflect.TypeOf((*Task)(nil))
if _, ok := taskType.MethodByName("Continue"); ok {
t.Fatal("regular task unexpectedly exposes Continue")
}
if _, ok := taskType.MethodByName("Pause"); ok {
t.Fatal("regular task unexpectedly exposes Pause")
}
errorTaskType := reflect.TypeOf((*ExtensionTask)(nil))
if _, ok := errorTaskType.MethodByName("Continue"); !ok {
t.Fatal("onError task does not expose Continue")
}
if _, ok := errorTaskType.MethodByName("Pause"); ok {
t.Fatal("onError task unexpectedly exposes Pause")
}
for name, contextType := range map[string]reflect.Type{
"onStart": reflect.TypeOf(OnStartContext{}),
"onDone": reflect.TypeOf(OnDoneContext{}),
} {
field, _ := contextType.FieldByName("Task")
if field.Type != taskType {
t.Fatalf("%s unexpectedly injects a control wrapper: %v", name, field.Type)
}
}
errorTaskField, _ := reflect.TypeOf(OnErrorContext{}).FieldByName("Task")
if errorTaskField.Type != errorTaskType {
t.Fatalf("onError does not inject the Continue wrapper: %v", errorTaskField.Type)
}
}
func (m *generationTestManager) Name() string { return "generation" }
func (m *generationTestManager) Filters() []*fetcher.SchemeFilter {
return []*fetcher.SchemeFilter{{Type: fetcher.FilterTypeUrl, Pattern: "generation"}}
}
func (m *generationTestManager) Build() fetcher.Fetcher {
return &generationTestFetcher{manager: m, meta: &fetcher.FetcherMeta{}}
}
func (m *generationTestManager) ParseName(string) string { return "generation.bin" }
func (m *generationTestManager) AutoRename() bool { return false }
func (m *generationTestManager) DefaultConfig() any { return map[string]any{} }
func (m *generationTestManager) Store(fetcher.Fetcher) (any, error) { return nil, nil }
func (m *generationTestManager) Restore() (any, func(*fetcher.FetcherMeta, any) fetcher.Fetcher) {
return nil, func(meta *fetcher.FetcherMeta, _ any) fetcher.Fetcher {
return &generationTestFetcher{manager: m, meta: meta}
}
}
func (m *generationTestManager) Close() error { return nil }
type generationTestFetcher struct {
manager *generationTestManager
meta *fetcher.FetcherMeta
done chan error
}
func (f *generationTestFetcher) Setup(*controller.Controller) {
if f.done == nil {
f.done = make(chan error, 2)
}
}
func (f *generationTestFetcher) Resolve(req *base.Request, opts *base.Options) error {
if f.manager.resolveStarted != nil {
f.manager.resolveOnce.Do(func() { close(f.manager.resolveStarted) })
}
if f.manager.resolveRelease != nil {
<-f.manager.resolveRelease
}
if f.manager.resolveErr != nil {
return f.manager.resolveErr
}
f.meta.Req = req
f.meta.Opts = opts
f.meta.Res = &base.Resource{Files: []*base.FileInfo{{Name: "generation.bin", Size: 1}}}
return nil
}
func (f *generationTestFetcher) Start() error {
f.manager.starts.Add(1)
if !f.manager.holdOpen {
f.done <- nil
}
return nil
}
func (f *generationTestFetcher) Patch(req *base.Request, opts *base.Options) error {
if req != nil && req.URL != "" {
f.meta.Req.URL = req.URL
}
if opts != nil {
f.meta.Opts = opts
}
return nil
}
func (f *generationTestFetcher) Pause() error {
f.manager.pauses.Add(1)
if f.manager.pauseStarted != nil {
f.manager.pauseOnce.Do(func() { close(f.manager.pauseStarted) })
}
if f.manager.pauseRelease != nil {
<-f.manager.pauseRelease
}
return nil
}
func (f *generationTestFetcher) Close() error { return nil }
func (f *generationTestFetcher) Stats() any { return nil }
func (f *generationTestFetcher) Meta() *fetcher.FetcherMeta { return f.meta }
func (f *generationTestFetcher) Progress() fetcher.Progress { return fetcher.Progress{1} }
func (f *generationTestFetcher) Wait() error { return <-f.done }
func newTestDownloadOpt(t *testing.T) *base.Options {
t.Helper()
return newTestDownloadOptAt(t.TempDir())
}
func newTestDownloadOptAt(path string) *base.Options {
return &base.Options{
Path: test.Dir,
Path: path,
Name: test.DownloadName,
Extra: http.OptsExtra{
Connections: 4,
@@ -63,7 +197,7 @@ func TestDownloader_Resolve(t *testing.T) {
}
}
func TestDownloader_UsesStorageDirForGBlobRegistry(t *testing.T) {
func TestDownloader_BlobRegistryDoesNotUseStorageDirForSpooling(t *testing.T) {
storageDir := t.TempDir()
downloader := NewDownloader(&DownloaderConfig{
Storage: NewMemStorage(),
@@ -74,37 +208,486 @@ func TestDownloader_UsesStorageDirForGBlobRegistry(t *testing.T) {
}
defer downloader.Clear()
if downloader.gblob == nil {
t.Fatal("expected gblob registry to be initialized")
if downloader.blob == nil {
t.Fatal("expected blob registry to be initialized")
}
expectedDir := filepath.Join(storageDir, "gblob")
if filepath.Clean(downloader.gblob.Dir()) != filepath.Clean(expectedDir) {
t.Fatalf("expected gblob registry dir under storage dir, got %s want %s", downloader.gblob.Dir(), expectedDir)
if downloader.blob.Dir() != "" {
t.Fatalf("expected blob registry not to use a spool dir, got %s", downloader.blob.Dir())
}
if _, err := os.Stat(filepath.Join(storageDir, "blob")); !os.IsNotExist(err) {
t.Fatalf("expected blob spool dir not to be created, got err=%v", err)
}
}
func TestDownloader_SetupCleansStaleGBlobDir(t *testing.T) {
storageDir := t.TempDir()
gblobDir := filepath.Join(storageDir, "gblob")
if err := os.MkdirAll(gblobDir, 0o755); err != nil {
t.Fatal(err)
}
staleFile := filepath.Join(gblobDir, "stale")
if err := os.WriteFile(staleFile, []byte("stale"), 0o644); err != nil {
t.Fatal(err)
}
func TestDownloader_BlobTaskKeepsConfiguredHTTPConnections(t *testing.T) {
downloader := NewDownloader(&DownloaderConfig{
Storage: NewMemStorage(),
StorageDir: storageDir,
StorageDir: t.TempDir(),
})
if err := downloader.Setup(); err != nil {
t.Fatal(err)
}
defer downloader.Clear()
if _, err := os.Stat(staleFile); !os.IsNotExist(err) {
t.Fatalf("expected stale gblob file to be removed on setup, got err=%v", err)
url, err := downloader.blob.CreateBlob([]byte("hello"), "text/plain")
if err != nil {
t.Fatal(err)
}
id, err := downloader.CreateDirect(&base.Request{URL: url}, &base.Options{
Path: t.TempDir(),
Name: "blob-unmarked.txt",
Extra: &http.OptsExtra{Connections: 8},
})
if err != nil {
t.Fatal(err)
}
task := downloader.GetTask(id)
if task == nil {
t.Fatal("task not found")
}
if task.Protocol != "http" {
t.Fatalf("expected blob task protocol http, got %s", task.Protocol)
}
extra, ok := task.Meta.Opts.Extra.(*http.OptsExtra)
if !ok {
t.Fatalf("expected http extra, got %T", task.Meta.Opts.Extra)
}
if extra.Connections != 8 {
t.Fatalf("expected blob task connections to stay 8, got %d", extra.Connections)
}
waitForTaskStatus(t, downloader, id, base.DownloadStatusDone, 5*time.Second)
}
func TestDownloader_ExternalBlobLikePathUsesNormalHTTP(t *testing.T) {
payload := []byte("external blob-like path")
server := httptest.NewServer(gohttp.HandlerFunc(func(w gohttp.ResponseWriter, r *gohttp.Request) {
if r.URL.Path != "/__blob/file.bin" {
gohttp.NotFound(w, r)
return
}
w.Header().Set("Content-Length", strconv.Itoa(len(payload)))
_, _ = w.Write(payload)
}))
defer server.Close()
downloader := NewDownloader(nil)
if err := downloader.Setup(); err != nil {
t.Fatal(err)
}
defer downloader.Clear()
dir := t.TempDir()
id, err := downloader.CreateDirect(&base.Request{URL: server.URL + "/__blob/file.bin"}, &base.Options{
Path: dir,
Name: "external.bin",
})
if err != nil {
t.Fatal(err)
}
waitForTaskStatus(t, downloader, id, base.DownloadStatusDone, 5*time.Second)
got, err := os.ReadFile(filepath.Join(dir, "external.bin"))
if err != nil {
t.Fatal(err)
}
if string(got) != string(payload) {
t.Fatalf("unexpected external payload %q", string(got))
}
}
func TestDownloader_InternalBlobBypassesGlobalProxy(t *testing.T) {
downloader := NewDownloader(nil)
if err := downloader.Setup(); err != nil {
t.Fatal(err)
}
defer downloader.Clear()
downloader.cfg.DownloaderStoreConfig.Proxy = &base.DownloaderProxyConfig{
Enable: true,
Scheme: "http",
Host: "127.0.0.1:1",
}
payload := []byte("direct blob")
blobURL, err := downloader.blob.CreateBlob(payload, "application/octet-stream")
if err != nil {
t.Fatal(err)
}
dir := t.TempDir()
id, err := downloader.CreateDirect(&base.Request{URL: blobURL}, &base.Options{
Path: dir,
Name: "direct.bin",
})
if err != nil {
t.Fatal(err)
}
waitForTaskStatus(t, downloader, id, base.DownloadStatusDone, 5*time.Second)
got, err := os.ReadFile(filepath.Join(dir, "direct.bin"))
if err != nil {
t.Fatal(err)
}
if string(got) != string(payload) {
t.Fatalf("unexpected blob payload %q", string(got))
}
}
func TestDownloader_RegistryMissUsesOrdinaryHTTPError(t *testing.T) {
downloader := NewDownloader(nil)
if err := downloader.Setup(); err != nil {
t.Fatal(err)
}
defer downloader.Clear()
liveURL, err := downloader.blob.CreateBlob([]byte("unused"), "text/plain")
if err != nil {
t.Fatal(err)
}
missingURL := liveURL[:len(liveURL)-1] + "x"
if downloader.blob.IsURL(missingURL) {
t.Fatal("test URL unexpectedly matched the Registry")
}
errorCh := make(chan error, 1)
downloader.Listener(func(event *Event) {
if event.Key == EventKeyError {
select {
case errorCh <- event.Err:
default:
}
}
})
_, err = downloader.CreateDirect(&base.Request{URL: missingURL}, &base.Options{
Path: t.TempDir(),
Name: "missing.bin",
})
if err != nil {
t.Fatalf("Registry miss was rejected before ordinary HTTP handling: %v", err)
}
select {
case got := <-errorCh:
if errors.Is(got, internalblob.ErrSourceNotFound) {
t.Fatalf("Registry miss surfaced Blob error instead of HTTP failure: %v", got)
}
if got == nil || !strings.Contains(got.Error(), "404") {
t.Fatalf("expected ordinary HTTP 404 failure, got %v", got)
}
case <-time.After(10 * time.Second):
t.Fatal("Registry miss did not reach the ordinary HTTP error path")
}
}
func TestDownloader_TasksShareBlobLease(t *testing.T) {
downloader := NewDownloader(nil)
if err := downloader.Setup(); err != nil {
t.Fatal(err)
}
defer downloader.Clear()
downloader.cfg.MaxRunning = 1
payload := bytes.Repeat([]byte("shared-blob-"), 8192)
allowLaterOpens := make(chan struct{})
var opens atomic.Int32
blobURL, err := downloader.blob.CreateOpener(func(ctx context.Context, req internalblob.OpenRequest) (io.ReadCloser, error) {
if opens.Add(1) > 1 {
select {
case <-allowLaterOpens:
case <-ctx.Done():
return nil, ctx.Err()
}
}
start := req.Offset
end := int64(len(payload))
if req.End >= start && req.End+1 < end {
end = req.End + 1
}
return io.NopCloser(bytes.NewReader(payload[start:end])), nil
}, &internalblob.CreateOptions{Size: int64(len(payload))})
if err != nil {
t.Fatal(err)
}
dirA, dirB := t.TempDir(), t.TempDir()
idA, err := downloader.CreateDirect(&base.Request{URL: blobURL}, &base.Options{Path: dirA, Name: "a.bin"})
if err != nil {
t.Fatal(err)
}
idB, err := downloader.CreateDirect(&base.Request{URL: blobURL}, &base.Options{Path: dirB, Name: "b.bin"})
if err != nil {
t.Fatal(err)
}
waitForTaskStatus(t, downloader, idA, base.DownloadStatusDone, 5*time.Second)
if err := downloader.Delete(&TaskFilter{IDs: []string{idA}}, false); err != nil {
t.Fatal(err)
}
if !downloader.blob.IsURL(blobURL) {
t.Fatal("deleting the first task revoked a Blob still leased by the second task")
}
close(allowLaterOpens)
waitForTaskStatus(t, downloader, idB, base.DownloadStatusDone, 5*time.Second)
if downloader.blob.IsURL(blobURL) {
t.Fatal("final task completion did not release the shared Blob")
}
for _, file := range []string{filepath.Join(dirA, "a.bin"), filepath.Join(dirB, "b.bin")} {
got, err := os.ReadFile(file)
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(got, payload) {
t.Fatalf("unexpected shared Blob content in %s", file)
}
}
}
func TestDownloader_EmptyBlob(t *testing.T) {
downloader := NewDownloader(nil)
if err := downloader.Setup(); err != nil {
t.Fatal(err)
}
defer downloader.Clear()
blobURL, err := downloader.blob.CreateBlob(nil, "application/octet-stream")
if err != nil {
t.Fatal(err)
}
dir := t.TempDir()
id, err := downloader.CreateDirect(&base.Request{URL: blobURL}, &base.Options{Path: dir, Name: "empty.bin"})
if err != nil {
t.Fatal(err)
}
waitForTaskStatus(t, downloader, id, base.DownloadStatusDone, 5*time.Second)
info, err := os.Stat(filepath.Join(dir, "empty.bin"))
if err != nil {
t.Fatal(err)
}
if info.Size() != 0 {
t.Fatalf("unexpected empty Blob size: %d", info.Size())
}
}
func TestDownloader_PatchTransfersBlobLease(t *testing.T) {
downloader := NewDownloader(nil)
if err := downloader.Setup(); err != nil {
t.Fatal(err)
}
defer downloader.Clear()
// Keep the task waiting so Patch exercises only metadata and lease transfer.
downloader.cfg.MaxRunning = 0
urlA, err := downloader.blob.CreateBlob([]byte("a"), "text/plain")
if err != nil {
t.Fatal(err)
}
urlB, err := downloader.blob.CreateBlob([]byte("b"), "text/plain")
if err != nil {
t.Fatal(err)
}
id, err := downloader.CreateDirect(&base.Request{URL: urlA}, &base.Options{Path: t.TempDir(), Name: "patched.bin"})
if err != nil {
t.Fatal(err)
}
if err := downloader.Patch(id, &base.Request{URL: urlB}, nil); err != nil {
t.Fatal(err)
}
if downloader.blob.IsURL(urlA) {
t.Fatal("Patch retained the old Blob lease")
}
if !downloader.blob.IsURL(urlB) {
t.Fatal("Patch did not acquire the new Blob lease")
}
if err := downloader.Delete(&TaskFilter{IDs: []string{id}}, false); err != nil {
t.Fatal(err)
}
if downloader.blob.IsURL(urlB) {
t.Fatal("deleting patched task did not release the new Blob lease")
}
}
func TestDownloader_StaleStartGenerationCannotRestartTask(t *testing.T) {
manager := &generationTestManager{}
downloader := NewDownloader(&DownloaderConfig{FetchManagers: []fetcher.FetcherManager{manager}})
if err := downloader.Setup(); err != nil {
t.Fatal(err)
}
defer downloader.Clear()
downloader.cfg.MaxRunning = 0
id, err := downloader.CreateDirect(&base.Request{URL: "generation://test"}, &base.Options{
Path: t.TempDir(),
Name: "generation.bin",
})
if err != nil {
t.Fatal(err)
}
task := downloader.GetTask(id)
if task == nil || task.Status != base.DownloadStatusWait {
t.Fatalf("expected queued task, got %#v", task)
}
// Hold task.lock so the first Start, its Pause handler, and the second Start
// are all queued. Only the newest start generation may reach Fetcher.Start.
task.lock.Lock()
downloader.cfg.MaxRunning = 1
if err := downloader.Continue(&TaskFilter{IDs: []string{id}}); err != nil {
t.Fatal(err)
}
pauseDone := make(chan error, 1)
go func() {
pauseDone <- downloader.Pause(&TaskFilter{IDs: []string{id}})
}()
deadline := time.Now().Add(2 * time.Second)
for downloader.taskStatus(task) != base.DownloadStatusPause && time.Now().Before(deadline) {
time.Sleep(5 * time.Millisecond)
}
if downloader.taskStatus(task) != base.DownloadStatusPause {
t.Fatal("Pause did not update task status")
}
if err := downloader.Continue(&TaskFilter{IDs: []string{id}}); err != nil {
t.Fatal(err)
}
task.lock.Unlock()
select {
case err := <-pauseDone:
if err != nil {
t.Fatal(err)
}
case <-time.After(2 * time.Second):
t.Fatal("Pause did not finish")
}
waitForTaskStatus(t, downloader, id, base.DownloadStatusDone, 3*time.Second)
if got := manager.starts.Load(); got != 1 {
t.Fatalf("stale start generation reached Fetcher.Start: got %d starts, want 1", got)
}
}
func TestDownloader_PauseWinsAgainstStaleResolveError(t *testing.T) {
releaseResolve := make(chan struct{})
manager := &generationTestManager{
resolveStarted: make(chan struct{}),
resolveRelease: releaseResolve,
resolveErr: errors.New("stale resolve failed"),
}
downloader := NewDownloader(&DownloaderConfig{FetchManagers: []fetcher.FetcherManager{manager}})
if err := downloader.Setup(); err != nil {
t.Fatal(err)
}
defer downloader.Clear()
errorEvent := make(chan error, 1)
downloader.Listener(func(event *Event) {
if event.Key == EventKeyError {
errorEvent <- event.Err
}
})
id, err := downloader.CreateDirect(&base.Request{URL: "generation://stale-resolve"}, &base.Options{
Path: t.TempDir(),
Name: "stale.bin",
})
if err != nil {
t.Fatal(err)
}
select {
case <-manager.resolveStarted:
case <-time.After(2 * time.Second):
t.Fatal("Resolve did not start")
}
pauseDone := make(chan error, 1)
go func() {
pauseDone <- downloader.Pause(&TaskFilter{IDs: []string{id}})
}()
deadline := time.Now().Add(2 * time.Second)
for downloader.taskStatus(downloader.GetTask(id)) != base.DownloadStatusPause && time.Now().Before(deadline) {
time.Sleep(5 * time.Millisecond)
}
if got := downloader.taskStatus(downloader.GetTask(id)); got != base.DownloadStatusPause {
t.Fatalf("Pause did not win the status transition: %s", got)
}
close(releaseResolve)
select {
case err := <-pauseDone:
if err != nil {
t.Fatal(err)
}
case <-time.After(2 * time.Second):
t.Fatal("Pause did not finish after stale Resolve returned")
}
if got := downloader.taskStatus(downloader.GetTask(id)); got != base.DownloadStatusPause {
t.Fatalf("stale Resolve error overwrote Pause: %s", got)
}
select {
case err := <-errorEvent:
t.Fatalf("stale Resolve emitted onError after Pause: %v", err)
case <-time.After(100 * time.Millisecond):
}
}
func TestDownloader_SchedulingPauseCompletesBeforeReplacementStart(t *testing.T) {
releasePause := make(chan struct{})
manager := &generationTestManager{
holdOpen: true,
pauseStarted: make(chan struct{}),
pauseRelease: releasePause,
}
downloader := NewDownloader(&DownloaderConfig{FetchManagers: []fetcher.FetcherManager{manager}})
if err := downloader.Setup(); err != nil {
t.Fatal(err)
}
defer downloader.Clear()
downloader.cfg.MaxRunning = 1
_, err := downloader.CreateDirect(&base.Request{URL: "generation://a"}, &base.Options{
Path: t.TempDir(), Name: "a.bin",
})
if err != nil {
t.Fatal(err)
}
deadline := time.Now().Add(2 * time.Second)
for manager.starts.Load() != 1 && time.Now().Before(deadline) {
time.Sleep(5 * time.Millisecond)
}
if manager.starts.Load() != 1 {
t.Fatal("first task did not physically start")
}
idB, err := downloader.CreateDirect(&base.Request{URL: "generation://b"}, &base.Options{
Path: t.TempDir(), Name: "b.bin",
})
if err != nil {
t.Fatal(err)
}
if downloader.taskStatus(downloader.GetTask(idB)) != base.DownloadStatusWait {
t.Fatal("second task was not queued")
}
continueDone := make(chan error, 1)
go func() {
continueDone <- downloader.Continue(&TaskFilter{IDs: []string{idB}})
}()
select {
case <-manager.pauseStarted:
case <-time.After(2 * time.Second):
t.Fatal("running task was not physically paused")
}
if got := manager.starts.Load(); got != 1 {
t.Fatalf("replacement started before Pause completed: %d starts", got)
}
close(releasePause)
select {
case err := <-continueDone:
if err != nil {
t.Fatal(err)
}
case <-time.After(2 * time.Second):
t.Fatal("Continue did not return after Pause completed")
}
deadline = time.Now().Add(2 * time.Second)
for manager.starts.Load() != 2 && time.Now().Before(deadline) {
time.Sleep(5 * time.Millisecond)
}
if got := manager.starts.Load(); got != 2 {
t.Fatalf("replacement task did not start: %d starts", got)
}
if got := manager.pauses.Load(); got != 1 {
t.Fatalf("unexpected physical Pause count: %d", got)
}
}
@@ -120,7 +703,8 @@ func TestDownloader_Create(t *testing.T) {
req := &base.Request{
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
}
rr, err := downloader.Resolve(req, newTestDownloadOpt())
opts := newTestDownloadOpt(t)
rr, err := downloader.Resolve(req, opts)
if err != nil {
t.Fatal(err)
}
@@ -138,7 +722,7 @@ func TestDownloader_Create(t *testing.T) {
}
wg.Wait()
want := test.FileMd5(test.BuildFile)
got := test.FileMd5(test.DownloadFile)
got := test.FileMd5(filepath.Join(opts.Path, opts.Name))
if want != got {
t.Errorf("Downloader_Create() got = %v, want %v", got, want)
}
@@ -160,7 +744,7 @@ func TestDownloader_CreateNotInWhite(t *testing.T) {
}
// With new fetcher design, white list check happens during Resolve (not Create)
// because Resolve now requires Options which includes the download path
_, err := downloader.Resolve(req, newTestDownloadOpt())
_, err := downloader.Resolve(req, newTestDownloadOpt(t))
if err == nil {
t.Error("TestDownloader_CreateNotInWhite() expected error but got nil")
}
@@ -207,9 +791,10 @@ func TestDownloader_CreateDirectBatch(t *testing.T) {
}
})
opts := newTestDownloadOpt(t)
_, err := downloader.CreateDirectBatch(&base.CreateTaskBatch{
Reqs: reqs,
Opts: newTestDownloadOpt(),
Opts: opts,
})
if err != nil {
t.Fatal(err)
@@ -235,7 +820,7 @@ func TestDownloader_CreateDirectBatch(t *testing.T) {
// Check that all task files exist
for name := range taskNames {
if _, err := os.Stat(test.Dir + "/" + name); os.IsNotExist(err) {
if _, err := os.Stat(filepath.Join(opts.Path, name)); os.IsNotExist(err) {
t.Errorf("CreateDirectBatch() file not exist: %v", name)
}
}
@@ -388,6 +973,7 @@ func TestDownloader_CreateRename(t *testing.T) {
req := &base.Request{
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
}
downloadDir := t.TempDir()
var wg sync.WaitGroup
wg.Add(2)
downloader.Listener(func(event *Event) {
@@ -397,7 +983,7 @@ func TestDownloader_CreateRename(t *testing.T) {
})
for i := 0; i < 2; i++ {
_, err := downloader.CreateDirect(req, &base.Options{
Path: test.Dir,
Path: downloadDir,
Name: test.DownloadName,
Extra: http.OptsExtra{
Connections: 4,
@@ -410,11 +996,11 @@ func TestDownloader_CreateRename(t *testing.T) {
wg.Wait()
want := test.FileMd5(test.BuildFile)
got := test.FileMd5(test.DownloadFile)
got := test.FileMd5(filepath.Join(downloadDir, test.DownloadName))
if want != got {
t.Errorf("Downloader_CreateRename() got = %v, want %v", got, want)
}
got = test.FileMd5(test.DownloadRenameFile)
got = test.FileMd5(filepath.Join(downloadDir, test.DownloadRename))
if want != got {
t.Errorf("Downloader_CreateRename() got = %v, want %v", got, want)
}
@@ -424,8 +1010,9 @@ func TestDownloader_StoreAndRestore(t *testing.T) {
listener := test.StartTestSlowFileServer(time.Millisecond * 2000)
defer listener.Close()
storageDir := t.TempDir()
downloader := NewDownloader(&DownloaderConfig{
Storage: NewBoltStorage("./"),
Storage: NewBoltStorage(storageDir),
})
if err := downloader.Setup(); err != nil {
t.Fatal(err)
@@ -434,24 +1021,37 @@ func TestDownloader_StoreAndRestore(t *testing.T) {
req := &base.Request{
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
}
rr, err := downloader.Resolve(req, newTestDownloadOpt())
opts := newTestDownloadOpt(t)
rr, err := downloader.Resolve(req, opts)
if err != nil {
t.Fatal(err)
}
pauseResult := make(chan error, 1)
var pauseOnce sync.Once
downloader.Listener(func(event *Event) {
if event.Key == EventKeyStart {
pauseOnce.Do(func() {
pauseResult <- downloader.Pause(&TaskFilter{IDs: []string{event.Task.ID}})
})
}
})
id, err := downloader.Create(rr.ID)
if err != nil {
t.Fatal(err)
}
time.Sleep(time.Millisecond * 1001)
err = downloader.Pause(&TaskFilter{IDs: []string{id}})
if err != nil {
t.Fatal(err)
select {
case err = <-pauseResult:
if err != nil {
t.Fatal(err)
}
case <-time.After(10 * time.Second):
t.Fatal("timed out waiting for task to pause")
}
downloader.Close()
downloader = NewDownloader(&DownloaderConfig{
Storage: NewBoltStorage("./"),
Storage: NewBoltStorage(storageDir),
})
if err := downloader.Setup(); err != nil {
t.Fatal(err)
@@ -474,7 +1074,7 @@ func TestDownloader_StoreAndRestore(t *testing.T) {
t.Fatal(err)
}
want := test.FileMd5(test.BuildFile)
got := test.FileMd5(test.DownloadFile)
got := test.FileMd5(filepath.Join(opts.Path, opts.Name))
if want != got {
t.Errorf("StoreAndResume() got = %v, want %v", got, want)
}
@@ -568,15 +1168,10 @@ func TestDownloader_GetTasksByFilter(t *testing.T) {
}
})
batchOpts := newTestDownloadOpt(t)
taskIds, err := downloader.CreateDirectBatch(&base.CreateTaskBatch{
Reqs: reqs,
Opts: &base.Options{
Path: test.Dir,
Name: test.DownloadName,
Extra: http.OptsExtra{
Connections: 4,
},
},
Opts: batchOpts,
})
if err != nil {
t.Fatal(err)
@@ -703,7 +1298,7 @@ func TestDownloader_Stats(t *testing.T) {
req := &base.Request{
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
}
rr, err := downloader.Resolve(req, newTestDownloadOpt())
rr, err := downloader.Resolve(req, newTestDownloadOpt(t))
if err != nil {
t.Fatal(err)
}
@@ -746,6 +1341,7 @@ func TestDownloader_Delete(t *testing.T) {
// Create multiple tasks
var wg sync.WaitGroup
taskCount := 3
downloadDir := t.TempDir()
wg.Add(taskCount)
downloader.Listener(func(event *Event) {
if event.Key == EventKeyDone {
@@ -758,7 +1354,7 @@ func TestDownloader_Delete(t *testing.T) {
req := &base.Request{
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
}
taskId, err := downloader.CreateDirect(req, newTestDownloadOpt())
taskId, err := downloader.CreateDirect(req, newTestDownloadOptAt(downloadDir))
if err != nil {
t.Fatal(err)
}
@@ -822,7 +1418,7 @@ func TestDownloader_PauseAndContinue(t *testing.T) {
req := &base.Request{
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
}
rr, err := downloader.Resolve(req, newTestDownloadOpt())
rr, err := downloader.Resolve(req, newTestDownloadOpt(t))
if err != nil {
t.Fatal(err)
}
@@ -893,7 +1489,7 @@ func TestDownloader_PauseAllAndContinueAll(t *testing.T) {
req := &base.Request{
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
}
rr, err := downloader.Resolve(req, newTestDownloadOpt())
rr, err := downloader.Resolve(req, newTestDownloadOpt(t))
if err != nil {
t.Fatal(err)
}
@@ -1553,6 +2149,7 @@ func TestDownloader_DeleteAll(t *testing.T) {
// Create multiple tasks
var wg sync.WaitGroup
taskCount := 3
downloadDir := t.TempDir()
wg.Add(taskCount)
downloader.Listener(func(event *Event) {
if event.Key == EventKeyDone {
@@ -1565,7 +2162,7 @@ func TestDownloader_DeleteAll(t *testing.T) {
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
}
_, err := downloader.CreateDirect(req, &base.Options{
Path: test.Dir,
Path: downloadDir,
Name: test.DownloadName,
Extra: http.OptsExtra{
Connections: 4,
@@ -2719,7 +3316,7 @@ func TestDownloader_PatchTask_HTTP(t *testing.T) {
},
}
opts := &base.Options{
Path: test.Dir,
Path: t.TempDir(),
Name: test.DownloadName,
}
pkg/download/engine/engine_test.go
-15
View File
@@ -47,21 +47,6 @@ func TestError(t *testing.T) {
}
}
func TestURLCreateObjectURLRejectsUnsupportedType(t *testing.T) {
engine := NewEngine(nil)
defer engine.Close()
_, err := engine.RunString(`
URL.createObjectURL({});
`)
if err == nil {
t.Fatal("expected unsupported type error")
}
if !strings.Contains(err.Error(), "Unsupported object type for URL.createObjectURL: Object") {
t.Fatalf("unexpected unsupported type error: %v", err)
}
}
func TestCallFunction_DetachedAsyncWorkDoesNotBlockReturn(t *testing.T) {
engine := NewEngine(nil)
defer engine.Close()
pkg/download/engine/inject/stream/module.go
+645 -153
View File
@@ -11,9 +11,9 @@ import (
"net/url"
"strings"
"sync"
"sync/atomic"
"time"
protogblob "github.com/GopeedLab/gopeed/internal/protocol/gblob"
"github.com/GopeedLab/gopeed/pkg/download/engine/inject/file"
"github.com/GopeedLab/gopeed/pkg/download/engine/inject/formdata"
"github.com/GopeedLab/gopeed/pkg/download/engine/inject/xhr"
@@ -27,105 +27,68 @@ import (
var script string
type Config struct {
CreateBlobObjectURL func(data []byte, contentType string) (string, error)
CreateWritableStreamObjectURL func(opts *WritableStreamObjectURLOptions) (string, error)
RegisterWritableStreamResume func(url string, reopen func(offset int64) error) error
WriteWritableStreamObjectURL func(url string, data any) error
CloseWritableStreamObjectURL func(url string) error
AbortWritableStreamObjectURL func(url string, reason string) error
RevokeObjectURL func(url string) error
ProxyHandler func(r *http.Request) (*url.URL, error)
CreateObjectURL func(opts *ObjectURLOptions, open ObjectURLOpener) (string, error)
RevokeObjectURL func(url string) error
ProxyHandler func(r *http.Request) (*url.URL, error)
RegisterCleanup func(cleanup func())
}
type WritableStreamObjectURLOptions struct {
Reopenable bool
type ObjectURLOptions struct {
ContentType string
Size int64
Range bool
}
func isIgnorableGBlobError(err error) bool {
return errors.Is(err, protogblob.ErrSourceRevoked) ||
errors.Is(err, protogblob.ErrSourceNotFound) ||
errors.Is(err, protogblob.ErrSourceClosed) ||
errors.Is(err, protogblob.ErrSourceAborted)
type ObjectURLOpenRequest struct {
Offset int64
End int64
}
type ObjectURLOpener func(ctx context.Context, req ObjectURLOpenRequest) (io.ReadCloser, error)
func Enable(runtime *goja.Runtime, loop *eventloop.EventLoop, cfg *Config) error {
if cfg == nil {
cfg = &Config{}
}
if err := runtime.Set("__gopeed_create_blob_object_url", func(data []byte, contentType string) string {
if cfg.CreateBlobObjectURL == nil {
panic(runtime.NewGoError(fmt.Errorf("gblob blob object url handler not configured")))
blobPipes := newBlobPipeRegistry()
if err := runtime.Set("__gopeed_create_blob_object_url", func(call goja.FunctionCall) goja.Value {
if cfg.CreateObjectURL == nil {
panic(runtime.NewGoError(fmt.Errorf("blob object url handler not configured")))
}
url, err := cfg.CreateBlobObjectURL(data, contentType)
openValue := call.Argument(0)
if _, ok := goja.AssertFunction(openValue); !ok {
panic(runtime.NewGoError(fmt.Errorf("blob opener must be callable")))
}
optsObj := call.Argument(1).ToObject(runtime)
size := int64(0)
contentType := ""
rangeEnabled := false
if optsObj != nil {
size = optsObj.Get("size").ToInteger()
contentType = optsObj.Get("contentType").String()
rangeEnabled = optsObj.Get("range").ToBoolean()
}
if size < 0 {
size = 0
}
opts := &ObjectURLOptions{
ContentType: contentType,
Size: size,
Range: rangeEnabled,
}
url, err := cfg.CreateObjectURL(opts, func(ctx context.Context, req ObjectURLOpenRequest) (io.ReadCloser, error) {
return openBlobObjectURLReader(ctx, loop, blobPipes, openValue, req)
})
if err != nil {
panic(runtime.NewGoError(err))
}
return url
}); err != nil {
return err
}
if err := runtime.Set("__gopeed_create_writable_stream_object_url", func(reopenable bool) string {
if cfg.CreateWritableStreamObjectURL == nil {
panic(runtime.NewGoError(fmt.Errorf("gblob writable stream object url handler not configured")))
}
opts := &WritableStreamObjectURLOptions{Reopenable: reopenable}
url, err := cfg.CreateWritableStreamObjectURL(opts)
if err != nil {
panic(runtime.NewGoError(err))
}
if opts.Reopenable && cfg.RegisterWritableStreamResume != nil {
if err := cfg.RegisterWritableStreamResume(url, func(offset int64) error {
return openWritableStreamObjectURL(loop, url, offset)
}); err != nil {
panic(runtime.NewGoError(err))
}
}
return url
}); err != nil {
return err
}
if err := runtime.Set("__gopeed_write_writable_stream_object_url", func(url string, data any) {
if cfg.WriteWritableStreamObjectURL == nil {
panic(runtime.NewGoError(fmt.Errorf("gblob writable stream write handler not configured")))
}
if err := cfg.WriteWritableStreamObjectURL(url, data); err != nil {
if isIgnorableGBlobError(err) {
return
}
panic(runtime.NewGoError(err))
}
}); err != nil {
return err
}
if err := runtime.Set("__gopeed_close_writable_stream_object_url", func(url string) {
if cfg.CloseWritableStreamObjectURL == nil {
panic(runtime.NewGoError(fmt.Errorf("gblob writable stream close handler not configured")))
}
if err := cfg.CloseWritableStreamObjectURL(url); err != nil {
if isIgnorableGBlobError(err) {
return
}
panic(runtime.NewGoError(err))
}
}); err != nil {
return err
}
if err := runtime.Set("__gopeed_abort_writable_stream_object_url", func(url string, reason string) {
if cfg.AbortWritableStreamObjectURL == nil {
panic(runtime.NewGoError(fmt.Errorf("gblob writable stream abort handler not configured")))
}
if err := cfg.AbortWritableStreamObjectURL(url, reason); err != nil {
if isIgnorableGBlobError(err) {
return
}
panic(runtime.NewGoError(err))
}
return runtime.ToValue(url)
}); err != nil {
return err
}
if err := runtime.Set("__gopeed_revoke_object_url", func(url string) {
if cfg.RevokeObjectURL == nil {
panic(runtime.NewGoError(fmt.Errorf("gblob revoke handler not configured")))
panic(runtime.NewGoError(fmt.Errorf("blob revoke handler not configured")))
}
if err := cfg.RevokeObjectURL(url); err != nil {
panic(runtime.NewGoError(err))
@@ -134,30 +97,55 @@ func Enable(runtime *goja.Runtime, loop *eventloop.EventLoop, cfg *Config) error
return err
}
fetchRegistry := newFetchRegistry()
if cfg.RegisterCleanup != nil {
cfg.RegisterCleanup(fetchRegistry.CloseAll)
}
if err := runtime.Set("__gopeed_fetch_open", func(call goja.FunctionCall) goja.Value {
reqMeta, err := exportFetchRequest(runtime, call.Argument(0))
if err != nil {
panic(runtime.NewGoError(err))
}
meta, err := fetchRegistry.Open(runtime, cfg.ProxyHandler, reqMeta)
if err != nil {
panic(runtime.NewGoError(err))
}
return runtime.ToValue(meta)
fingerprint := util.SafeGet[string](runtime, xhr.FingerprintMagicKey)
promise, resolve, reject := runtime.NewPromise()
go func() {
meta, err := fetchRegistry.Open(fingerprint, cfg.ProxyHandler, reqMeta)
ok := loop.RunOnLoop(func(runtime *goja.Runtime) {
if err != nil {
reject(runtime.NewGoError(err))
return
}
resolve(runtime.ToValue(meta))
})
if !ok && meta != nil {
fetchRegistry.Close(meta.ID)
}
}()
return runtime.ToValue(promise)
}); err != nil {
return err
}
if err := runtime.Set("__gopeed_fetch_read", func(call goja.FunctionCall) goja.Value {
id := call.Argument(0).String()
chunkSize := int(call.Argument(1).ToInteger())
chunk, done, err := fetchRegistry.Read(id, chunkSize)
if err != nil {
panic(runtime.NewGoError(err))
}
if done {
return goja.Null()
}
return runtime.ToValue(runtime.NewArrayBuffer(chunk))
promise, resolve, reject := runtime.NewPromise()
go func() {
chunk, done, err := fetchRegistry.Read(id, chunkSize)
ok := loop.RunOnLoop(func(runtime *goja.Runtime) {
if err != nil {
reject(runtime.NewGoError(err))
return
}
if done {
resolve(goja.Null())
return
}
resolve(runtime.ToValue(runtime.NewArrayBuffer(chunk)))
})
if !ok {
fetchRegistry.Close(id)
}
}()
return runtime.ToValue(promise)
}); err != nil {
return err
}
@@ -171,19 +159,323 @@ func Enable(runtime *goja.Runtime, loop *eventloop.EventLoop, cfg *Config) error
}); err != nil {
return err
}
if err := runtime.Set("__gopeed_blob_pipe_chunk", func(call goja.FunctionCall) goja.Value {
id := call.Argument(0).String()
ok := blobPipes.Push(id, exportBytes(call.Argument(1)))
return runtime.ToValue(ok)
}); err != nil {
return err
}
if err := runtime.Set("__gopeed_blob_pipe_close", func(id string) {
blobPipes.Close(id, nil)
}); err != nil {
return err
}
if err := runtime.Set("__gopeed_blob_pipe_error", func(id string, message string) {
blobPipes.Close(id, errors.New(message))
}); err != nil {
return err
}
_, err := runtime.RunString(script)
return err
}
func openWritableStreamObjectURL(loop *eventloop.EventLoop, url string, offset int64) error {
func openBlobObjectURLReader(ctx context.Context, loop *eventloop.EventLoop, pipes *blobPipeRegistry, openValue goja.Value, req ObjectURLOpenRequest) (io.ReadCloser, error) {
if pipes == nil {
return nil, fmt.Errorf("blob pipe registry not configured")
}
pipe := newBlobPipe(ctx, loop, pipes)
pipes.Add(pipe)
_, err := runOnLoop(loop, func(runtime *goja.Runtime) (goja.Value, error) {
fnVal := runtime.Get("__gopeed_blob_pipe_source")
fn, ok := goja.AssertFunction(fnVal)
if !ok {
return nil, fmt.Errorf("blob pipe helper is not callable")
}
request := map[string]any{
"offset": req.Offset,
"end": req.End,
}
return fn(nil, openValue, runtime.ToValue(request), runtime.ToValue(pipe.id))
})
if err != nil {
pipe.Close()
return nil, err
}
return pipe, nil
}
type blobPipeRegistry struct {
mu sync.Mutex
pipes map[string]*blobPipe
next atomic.Uint64
}
func newBlobPipeRegistry() *blobPipeRegistry {
return &blobPipeRegistry{
pipes: map[string]*blobPipe{},
}
}
func (r *blobPipeRegistry) Add(pipe *blobPipe) {
r.mu.Lock()
r.pipes[pipe.id] = pipe
r.mu.Unlock()
}
func (r *blobPipeRegistry) Remove(id string) {
r.mu.Lock()
delete(r.pipes, id)
r.mu.Unlock()
}
func (r *blobPipeRegistry) Push(id string, chunk []byte) bool {
r.mu.Lock()
pipe := r.pipes[id]
r.mu.Unlock()
if pipe == nil {
return false
}
return pipe.Push(chunk)
}
func (r *blobPipeRegistry) Close(id string, err error) {
r.mu.Lock()
pipe := r.pipes[id]
r.mu.Unlock()
if pipe != nil {
pipe.CloseWithError(err)
}
}
func (r *blobPipeRegistry) NewID() string {
return fmt.Sprintf("pipe-%d", r.next.Add(1))
}
type blobPipeItem struct {
chunk []byte
err error
done bool
}
type blobPipe struct {
id string
loop *eventloop.EventLoop
registry *blobPipeRegistry
ctx context.Context
cancel context.CancelFunc
ch chan blobPipeItem
mu sync.Mutex
buf []byte
closed bool
once sync.Once
}
func newBlobPipe(ctx context.Context, loop *eventloop.EventLoop, registry *blobPipeRegistry) *blobPipe {
pipeCtx, cancel := context.WithCancel(ctx)
return &blobPipe{
id: registry.NewID(),
loop: loop,
registry: registry,
ctx: pipeCtx,
cancel: cancel,
ch: make(chan blobPipeItem, 8),
}
}
func (p *blobPipe) Push(chunk []byte) bool {
if len(chunk) == 0 {
return true
}
buf := append([]byte(nil), chunk...)
select {
case p.ch <- blobPipeItem{chunk: buf}:
return true
case <-p.ctx.Done():
return false
}
}
func (p *blobPipe) CloseWithError(err error) {
p.once.Do(func() {
p.registry.Remove(p.id)
select {
case p.ch <- blobPipeItem{done: true, err: err}:
case <-p.ctx.Done():
}
})
}
func (p *blobPipe) Read(dst []byte) (int, error) {
p.mu.Lock()
if len(p.buf) > 0 {
n := copy(dst, p.buf)
p.buf = p.buf[n:]
p.mu.Unlock()
return n, nil
}
p.mu.Unlock()
select {
case item := <-p.ch:
if item.done {
if item.err != nil {
return 0, item.err
}
return 0, io.EOF
}
n := copy(dst, item.chunk)
if n < len(item.chunk) {
p.mu.Lock()
p.buf = append(p.buf, item.chunk[n:]...)
p.mu.Unlock()
}
return n, nil
case <-p.ctx.Done():
return 0, p.ctx.Err()
}
}
func (p *blobPipe) Close() error {
p.mu.Lock()
if p.closed {
p.mu.Unlock()
return nil
}
p.closed = true
p.mu.Unlock()
p.registry.Remove(p.id)
p.cancel()
p.cancelSourceReader()
return nil
}
func (p *blobPipe) cancelSourceReader() {
if p.loop == nil {
return
}
// Invoke the JavaScript cancellation before returning from Close. The
// source session may be released immediately after the HTTP handler closes
// this reader, so merely queueing the callback could let the engine stop
// before reader.cancel() is called.
_, _ = runOnLoop(p.loop, func(runtime *goja.Runtime) (goja.Value, error) {
fnVal := runtime.Get("__gopeed_blob_cancel_pipe_source")
fn, ok := goja.AssertFunction(fnVal)
if !ok {
return goja.Undefined(), nil
}
_, err := fn(nil, runtime.ToValue(p.id), runtime.ToValue("blob request closed"))
return goja.Undefined(), err
})
}
func exportBytes(value goja.Value) []byte {
if value == nil || goja.IsNull(value) || goja.IsUndefined(value) {
return nil
}
if ab, ok := value.Export().(goja.ArrayBuffer); ok {
return ab.Bytes()
}
if b, ok := value.Export().([]byte); ok {
return b
}
return nil
}
type blobObjectURLReader struct {
ctx context.Context
loop *eventloop.EventLoop
id string
mu sync.Mutex
buf []byte
closed bool
}
func (r *blobObjectURLReader) Read(p []byte) (int, error) {
r.mu.Lock()
if len(r.buf) > 0 {
n := copy(p, r.buf)
r.buf = r.buf[n:]
r.mu.Unlock()
return n, nil
}
closed := r.closed
r.mu.Unlock()
if closed {
return 0, io.EOF
}
if err := r.ctx.Err(); err != nil {
return 0, err
}
chunkSize := len(p)
if chunkSize <= 0 {
chunkSize = 32 * 1024
}
value, err := runOnLoop(r.loop, func(runtime *goja.Runtime) (goja.Value, error) {
fnVal := runtime.Get("__gopeed_blob_read_source")
fn, ok := goja.AssertFunction(fnVal)
if !ok {
return nil, fmt.Errorf("blob read helper is not callable")
}
return fn(nil, runtime.ToValue(r.id), runtime.ToValue(chunkSize))
})
if err != nil {
return 0, err
}
if goja.IsNull(value) || goja.IsUndefined(value) {
return 0, io.EOF
}
var chunk []byte
if ab, ok := value.Export().(goja.ArrayBuffer); ok {
chunk = ab.Bytes()
} else if b, ok := value.Export().([]byte); ok {
chunk = b
} else {
return 0, fmt.Errorf("blob read helper returned %T", value.Export())
}
if len(chunk) == 0 {
return 0, nil
}
n := copy(p, chunk)
if n < len(chunk) {
r.mu.Lock()
r.buf = append(r.buf, chunk[n:]...)
r.mu.Unlock()
}
return n, nil
}
func (r *blobObjectURLReader) Close() error {
r.mu.Lock()
if r.closed {
r.mu.Unlock()
return nil
}
r.closed = true
r.mu.Unlock()
_, err := runOnLoop(r.loop, func(runtime *goja.Runtime) (goja.Value, error) {
fnVal := runtime.Get("__gopeed_blob_close_source")
fn, ok := goja.AssertFunction(fnVal)
if !ok {
return nil, nil
}
return fn(nil, runtime.ToValue(r.id))
})
return err
}
func runOnLoop(loop *eventloop.EventLoop, fn func(runtime *goja.Runtime) (goja.Value, error)) (goja.Value, error) {
type result struct {
err error
value goja.Value
err error
}
ch := make(chan result, 1)
ok := loop.RunOnLoop(func(runtime *goja.Runtime) {
send := func(err error) {
send := func(value goja.Value, err error) {
select {
case ch <- result{err: err}:
case ch <- result{value: value, err: err}:
default:
}
}
@@ -191,45 +483,55 @@ func openWritableStreamObjectURL(loop *eventloop.EventLoop, url string, offset i
if r := recover(); r != nil {
switch v := r.(type) {
case error:
send(v)
send(nil, v)
case goja.Value:
send(exportJSError(v))
send(nil, exportJSError(v))
default:
send(fmt.Errorf("panic: %v", r))
send(nil, fmt.Errorf("panic: %v", r))
}
}
}()
fnVal := runtime.Get("__gopeed_open_writable_stream_object_url")
fn, ok := goja.AssertFunction(fnVal)
if !ok {
send(fmt.Errorf("gblob writable stream open helper is not callable"))
return
}
value, err := fn(nil, runtime.ToValue(url), runtime.ToValue(offset))
value, err := fn(runtime)
if err != nil {
send(err)
send(nil, err)
return
}
if promise, ok := value.Export().(*goja.Promise); ok {
switch promise.State() {
case goja.PromiseStateFulfilled:
send(nil)
send(promise.Result(), nil)
return
case goja.PromiseStateRejected:
send(exportJSError(promise.Result()))
send(nil, exportJSError(promise.Result()))
return
default:
send(nil)
thenVal := value.ToObject(runtime).Get("then")
thenFn, ok := goja.AssertFunction(thenVal)
if !ok {
send(nil, errors.New("promise.then is not callable"))
return
}
onFulfilled := runtime.ToValue(func(call goja.FunctionCall) goja.Value {
send(call.Argument(0), nil)
return goja.Undefined()
})
onRejected := runtime.ToValue(func(call goja.FunctionCall) goja.Value {
send(nil, exportJSError(call.Argument(0)))
return goja.Undefined()
})
if _, err := thenFn(value, onFulfilled, onRejected); err != nil {
send(nil, err)
}
return
}
}
send(nil)
send(value, nil)
})
if !ok {
return errors.New("engine loop terminated")
return nil, errors.New("engine loop terminated")
}
return (<-ch).err
res := <-ch
return res.value, res.err
}
func exportJSError(value goja.Value) error {
@@ -252,12 +554,24 @@ func exportJSError(value goja.Value) error {
type fetchRegistry struct {
mu sync.Mutex
streams map[string]*fetchStream
ctx context.Context
cancel context.CancelFunc
closed bool
}
type fetchStream struct {
body io.ReadCloser
cancel context.CancelFunc
closeOnce sync.Once
ctx context.Context
ch chan fetchChunk
readMu sync.Mutex
pending []byte
}
type fetchChunk struct {
data []byte
err error
}
type fetchRequest struct {
@@ -269,6 +583,15 @@ type fetchRequest struct {
Credentials string
}
type fetchFormDataEntry struct {
name string
value any
}
type fetchFormDataSnapshot struct {
entries []fetchFormDataEntry
}
type fetchOpenMeta struct {
ID string `json:"id"`
Status int `json:"status"`
@@ -278,8 +601,11 @@ type fetchOpenMeta struct {
}
func newFetchRegistry() *fetchRegistry {
ctx, cancel := context.WithCancel(context.Background())
return &fetchRegistry{
streams: make(map[string]*fetchStream),
ctx: ctx,
cancel: cancel,
}
}
@@ -310,22 +636,98 @@ func exportFetchRequest(runtime *goja.Runtime, value goja.Value) (*fetchRequest,
}
bodyVal := obj.Get("body")
if bodyVal != nil && !goja.IsUndefined(bodyVal) && !goja.IsNull(bodyVal) {
meta.Body = bodyVal.Export()
body, err := snapshotFetchBody(bodyVal.Export())
if err != nil {
return nil, err
}
meta.Body = body
}
return meta, nil
}
func (r *fetchRegistry) Open(runtime *goja.Runtime, proxyHandler func(r *http.Request) (*url.URL, error), reqMeta *fetchRequest) (*fetchOpenMeta, error) {
// snapshotFetchBody runs on the JavaScript event loop before fetch work is
// handed to a background goroutine. Values exported by goja may still share
// mutable storage with JavaScript (notably ArrayBuffer and FormData), so the
// background request must only retain immutable Go-owned data.
func snapshotFetchBody(body any) (any, error) {
switch v := body.(type) {
case nil, string:
return v, nil
case []byte:
return append([]byte(nil), v...), nil
case goja.ArrayBuffer:
return append([]byte(nil), v.Bytes()...), nil
case *file.File:
return snapshotFetchFile(v), nil
case *formdata.FormData:
entries := v.Entries()
snapshot := &fetchFormDataSnapshot{
entries: make([]fetchFormDataEntry, 0, len(entries)),
}
for _, entry := range entries {
pair, ok := entry.([]any)
if !ok || len(pair) != 2 {
return nil, fmt.Errorf("invalid FormData entry %T", entry)
}
name, ok := pair[0].(string)
if !ok {
return nil, fmt.Errorf("invalid FormData field name %T", pair[0])
}
value := pair[1]
if formFile, ok := value.(*file.File); ok {
value = snapshotFetchFile(formFile)
}
snapshot.entries = append(snapshot.entries, fetchFormDataEntry{
name: name,
value: value,
})
}
return snapshot, nil
default:
if typed, ok := v.(interface{ Bytes() []byte }); ok {
return append([]byte(nil), typed.Bytes()...), nil
}
return v, nil
}
}
func snapshotFetchFile(src *file.File) *file.File {
if src == nil {
return nil
}
return &file.File{
Reader: src.Reader,
Closer: src.Closer,
Name: src.Name,
Size: src.Size,
}
}
func (r *fetchRegistry) Open(fingerprint string, proxyHandler func(r *http.Request) (*url.URL, error), reqMeta *fetchRequest) (*fetchOpenMeta, error) {
r.mu.Lock()
if r.closed {
r.mu.Unlock()
return nil, context.Canceled
}
registryCtx := r.ctx
r.mu.Unlock()
client := req.C()
if proxyHandler != nil {
client.SetProxy(proxyHandler)
}
setFetchFingerprint(client, util.SafeGet[string](runtime, xhr.FingerprintMagicKey))
setFetchFingerprint(client, fingerprint)
contentType, body, err := buildFetchBody(reqMeta.Body)
if err != nil {
return nil, err
}
ctx, cancel := context.WithCancel(context.Background())
ctx, cancel := context.WithCancel(registryCtx)
keepCancel := false
defer func() {
if !keepCancel {
cancel()
}
}()
reqBuilder := client.R()
reqBuilder.SetContext(ctx)
reqBuilder.DisableAutoReadResponse()
@@ -378,9 +780,22 @@ func (r *fetchRegistry) Open(runtime *goja.Runtime, proxyHandler func(r *http.Re
if resp.Response != nil && resp.Response.Body != nil {
bodyCloser = resp.Response.Body
}
stream := &fetchStream{
body: bodyCloser,
cancel: cancel,
ctx: ctx,
ch: make(chan fetchChunk, 8),
}
r.mu.Lock()
r.streams[id] = &fetchStream{body: bodyCloser, cancel: cancel}
if r.closed {
r.mu.Unlock()
closeFetchStream(stream)
return nil, context.Canceled
}
r.streams[id] = stream
r.mu.Unlock()
keepCancel = true
go stream.readLoop()
return meta, nil
}
@@ -389,23 +804,62 @@ func (r *fetchRegistry) Read(id string, chunkSize int) ([]byte, bool, error) {
if stream == nil {
return nil, true, nil
}
if chunkSize <= 0 {
chunkSize = 64 * 1024
stream.readMu.Lock()
defer stream.readMu.Unlock()
if len(stream.pending) > 0 {
n := len(stream.pending)
if chunkSize > 0 && n > chunkSize {
n = chunkSize
}
chunk := append([]byte(nil), stream.pending[:n]...)
stream.pending = stream.pending[n:]
return chunk, false, nil
}
buf := make([]byte, chunkSize)
n, err := stream.body.Read(buf)
if n > 0 {
return buf[:n], false, nil
select {
case item, ok := <-stream.ch:
if !ok {
r.Close(id)
return nil, true, nil
}
if item.err != nil {
r.Close(id)
return nil, false, item.err
}
if chunkSize > 0 && len(item.data) > chunkSize {
head := append([]byte(nil), item.data[:chunkSize]...)
stream.pending = append(stream.pending[:0], item.data[chunkSize:]...)
return head, false, nil
}
return item.data, false, nil
case <-stream.ctx.Done():
return nil, false, stream.ctx.Err()
}
if err == io.EOF {
r.Close(id)
return nil, true, nil
}
func (s *fetchStream) readLoop() {
defer close(s.ch)
buf := make([]byte, 64*1024)
for {
n, err := s.body.Read(buf)
if n > 0 {
chunk := append([]byte(nil), buf[:n]...)
select {
case s.ch <- fetchChunk{data: chunk}:
case <-s.ctx.Done():
return
}
}
if err == io.EOF {
return
}
if err != nil {
select {
case s.ch <- fetchChunk{err: err}:
case <-s.ctx.Done():
}
return
}
}
if err != nil {
r.Close(id)
return nil, false, err
}
return nil, false, nil
}
func (r *fetchRegistry) Close(id string) {
@@ -413,14 +867,43 @@ func (r *fetchRegistry) Close(id string) {
stream := r.streams[id]
delete(r.streams, id)
r.mu.Unlock()
if stream != nil {
stream.closeOnce.Do(func() {
if stream.cancel != nil {
stream.cancel()
}
_ = stream.body.Close()
})
closeFetchStream(stream)
}
// CloseAll permanently closes the registry and cancels both active and
// in-flight fetches. It is safe to call more than once.
func (r *fetchRegistry) CloseAll() {
if r == nil {
return
}
r.mu.Lock()
if r.closed {
r.mu.Unlock()
return
}
r.closed = true
if r.cancel != nil {
r.cancel()
}
streams := r.streams
r.streams = make(map[string]*fetchStream)
r.mu.Unlock()
for _, stream := range streams {
closeFetchStream(stream)
}
}
func closeFetchStream(stream *fetchStream) {
if stream == nil {
return
}
stream.closeOnce.Do(func() {
if stream.cancel != nil {
stream.cancel()
}
_ = stream.body.Close()
})
}
func (r *fetchRegistry) Abort(id string, _ string) {
@@ -445,24 +928,33 @@ func buildFetchBody(body any) (string, any, error) {
return "application/octet-stream", v.Bytes(), nil
case *file.File:
return "application/octet-stream", v.Reader, nil
case *formdata.FormData:
case *fetchFormDataSnapshot:
pr, pw := io.Pipe()
mw := xhr.NewMultipart(pw)
for _, e := range v.Entries() {
arr := e.([]any)
key := arr[0].(string)
val := arr[1]
switch vv := val.(type) {
for _, entry := range v.entries {
switch vv := entry.value.(type) {
case string:
mw.WriteField(key, vv)
if err := mw.WriteField(entry.name, vv); err != nil {
_ = pw.CloseWithError(err)
return "", nil, err
}
case *file.File:
mw.WriteFile(key, vv)
if err := mw.WriteFile(entry.name, vv); err != nil {
_ = pw.CloseWithError(err)
return "", nil, err
}
}
}
go func() {
defer pw.Close()
defer mw.Close()
mw.Send()
if err := mw.Send(); err != nil {
_ = pw.CloseWithError(err)
return
}
if err := mw.Close(); err != nil {
_ = pw.CloseWithError(err)
return
}
_ = pw.Close()
}()
return mw.FormDataContentType(), pr, nil
default:
pkg/download/engine/inject/stream/module_integration_test.go
+278
View File
@@ -0,0 +1,278 @@
package stream_test
import (
"context"
"errors"
"fmt"
"io"
"net/http"
"net/http/httptest"
"sync/atomic"
"testing"
"time"
"github.com/GopeedLab/gopeed/pkg/download/engine"
"github.com/GopeedLab/gopeed/pkg/download/engine/inject/stream"
)
type createdObjectURL struct {
opts *stream.ObjectURLOptions
open stream.ObjectURLOpener
}
func TestResponseBlobObjectURLReadsResponseStream(t *testing.T) {
const payload = "response blob payload"
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.Header().Set("Content-Type", "text/plain")
_, _ = io.WriteString(w, payload[:8])
w.(http.Flusher).Flush() // force an unknown-length/chunked response
_, _ = io.WriteString(w, payload[8:])
}))
defer server.Close()
created := make(chan createdObjectURL, 1)
engine, cleanup := newStreamTestEngine(t, created)
defer func() {
cleanup()
engine.Close()
}()
value, err := engine.RunString(fmt.Sprintf(`
(async () => {
const response = await fetch(%q);
const blob = await response.blob();
return __gopeed_blob_create_object_url(blob);
})()
`, server.URL))
if err != nil {
t.Fatal(err)
}
if value != "blob:test-1" {
t.Fatalf("unexpected object URL: %#v", value)
}
object := <-created
if object.opts.Size != int64(len(payload)) || object.opts.ContentType != "text/plain" {
t.Fatalf("unexpected object URL options: %#v", object.opts)
}
if !object.opts.Range {
t.Fatal("materialized response Blob should advertise range support")
}
for i := 0; i < 2; i++ {
reader, err := object.open(context.Background(), stream.ObjectURLOpenRequest{Offset: 0, End: -1})
if err != nil {
t.Fatal(err)
}
data, readErr := io.ReadAll(reader)
_ = reader.Close()
if readErr != nil {
t.Fatal(readErr)
}
if got := string(data); got != payload {
t.Fatalf("unexpected response Blob data on open %d: %q", i+1, got)
}
}
rangeReader, err := object.open(context.Background(), stream.ObjectURLOpenRequest{Offset: 9, End: 12})
if err != nil {
t.Fatal(err)
}
rangeData, readErr := io.ReadAll(rangeReader)
_ = rangeReader.Close()
if readErr != nil {
t.Fatal(readErr)
}
if got, want := string(rangeData), payload[9:13]; got != want {
t.Fatalf("unexpected materialized Blob range: got %q want %q", got, want)
}
}
func TestResponseBlobRejectsTruncatedResponse(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.Header().Set("Content-Length", "64")
_, _ = io.WriteString(w, "partial")
}))
defer server.Close()
created := make(chan createdObjectURL, 1)
engine, cleanup := newStreamTestEngine(t, created)
defer func() {
cleanup()
engine.Close()
}()
value, err := engine.RunString(fmt.Sprintf(`
(async () => {
const response = await fetch(%q);
try {
await response.blob();
return "resolved";
} catch (_) {
return "rejected";
}
})()
`, server.URL))
if err != nil {
t.Fatal(err)
}
if value != "rejected" {
t.Fatalf("truncated response unexpectedly materialized a Blob: %#v", value)
}
}
func TestEmptyBlobObjectURL(t *testing.T) {
created := make(chan createdObjectURL, 1)
engine, cleanup := newStreamTestEngine(t, created)
defer func() {
cleanup()
engine.Close()
}()
value, err := engine.RunString(`
__gopeed_blob_create_object_url(new Blob([], { type: "application/empty" }))
`)
if err != nil {
t.Fatal(err)
}
if value != "blob:test-1" {
t.Fatalf("unexpected object URL: %#v", value)
}
object := <-created
if object.opts.Size != 0 || object.opts.Range {
t.Fatalf("unexpected empty Blob options: %#v", object.opts)
}
reader, err := object.open(context.Background(), stream.ObjectURLOpenRequest{Offset: 0, End: -1})
if err != nil {
t.Fatal(err)
}
defer reader.Close()
data, err := io.ReadAll(reader)
if err != nil {
t.Fatal(err)
}
if len(data) != 0 {
t.Fatalf("expected empty Blob, got %q", data)
}
}
func TestBlobObjectURLReaderCloseCancelsPendingJSReader(t *testing.T) {
created := make(chan createdObjectURL, 1)
engine, cleanup := newStreamTestEngine(t, created)
defer func() {
cleanup()
engine.Close()
}()
value, err := engine.RunString(`
globalThis.__blobPipeCancelState = {
readCalls: 0,
cancelCalls: 0,
releaseCalls: 0,
cancelReason: "",
resolveRead: null,
};
const pendingSource = {
getReader() {
return {
read() {
__blobPipeCancelState.readCalls++;
return new Promise((resolve) => {
__blobPipeCancelState.resolveRead = resolve;
});
},
cancel(reason) {
__blobPipeCancelState.cancelCalls++;
__blobPipeCancelState.cancelReason = String(reason);
if (__blobPipeCancelState.resolveRead) {
__blobPipeCancelState.resolveRead({ done: true, value: undefined });
}
return Promise.resolve();
},
releaseLock() {
__blobPipeCancelState.releaseCalls++;
},
};
},
};
__gopeed_blob_create_object_url(async () => pendingSource);
`)
if err != nil {
t.Fatal(err)
}
if value != "blob:test-1" {
t.Fatalf("unexpected object URL: %#v", value)
}
object := <-created
reader, err := object.open(context.Background(), stream.ObjectURLOpenRequest{Offset: 0, End: -1})
if err != nil {
t.Fatal(err)
}
readDone := make(chan error, 1)
go func() {
_, readErr := reader.Read(make([]byte, 1))
readDone <- readErr
}()
waitForJSValue(t, engine, `String(__blobPipeCancelState.readCalls)`, "1")
if err := reader.Close(); err != nil {
t.Fatal(err)
}
if err := reader.Close(); err != nil {
t.Fatal(err)
}
select {
case readErr := <-readDone:
if !errors.Is(readErr, context.Canceled) {
t.Fatalf("unexpected pending read result: %v", readErr)
}
case <-time.After(2 * time.Second):
t.Fatal("pending Go read was not released by Close")
}
waitForJSValue(t, engine, `JSON.stringify({
cancelCalls: __blobPipeCancelState.cancelCalls,
releaseCalls: __blobPipeCancelState.releaseCalls,
cancelReason: __blobPipeCancelState.cancelReason,
})`, `{"cancelCalls":1,"releaseCalls":1,"cancelReason":"blob request closed"}`)
}
func waitForJSValue(t *testing.T, engine *engine.Engine, expression, want string) {
t.Helper()
deadline := time.Now().Add(2 * time.Second)
var got any
for time.Now().Before(deadline) {
var err error
got, err = engine.RunString(expression)
if err != nil {
t.Fatal(err)
}
if got == want {
return
}
time.Sleep(10 * time.Millisecond)
}
t.Fatalf("timed out waiting for JavaScript value %q, got %#v", want, got)
}
func newStreamTestEngine(t *testing.T, created chan<- createdObjectURL) (*engine.Engine, func()) {
t.Helper()
var next atomic.Int64
var cleanup func()
e := engine.NewEngine(&engine.Config{
StreamConfig: &stream.Config{
CreateObjectURL: func(opts *stream.ObjectURLOptions, open stream.ObjectURLOpener) (string, error) {
created <- createdObjectURL{opts: opts, open: open}
return fmt.Sprintf("blob:test-%d", next.Add(1)), nil
},
RevokeObjectURL: func(string) error { return nil },
RegisterCleanup: func(fn func()) {
cleanup = fn
},
},
})
if cleanup == nil {
t.Fatal("stream cleanup was not registered")
}
return e, cleanup
}
pkg/download/engine/inject/stream/module_test.go
+191
View File
@@ -0,0 +1,191 @@
package stream
import (
"context"
"io"
"strings"
"sync/atomic"
"testing"
"github.com/GopeedLab/gopeed/pkg/download/engine/inject/file"
"github.com/GopeedLab/gopeed/pkg/download/engine/inject/formdata"
"github.com/dop251/goja"
)
func TestExportFetchRequestSnapshotsArrayBuffer(t *testing.T) {
runtime := goja.New()
buffer := runtime.NewArrayBuffer([]byte("before"))
request := runtime.NewObject()
setFetchRequestDefaults(t, request)
if err := request.Set("body", buffer); err != nil {
t.Fatal(err)
}
meta, err := exportFetchRequest(runtime, request)
if err != nil {
t.Fatal(err)
}
buffer.Bytes()[0] = 'X'
body, ok := meta.Body.([]byte)
if !ok {
t.Fatalf("expected byte snapshot, got %T", meta.Body)
}
if got := string(body); got != "before" {
t.Fatalf("ArrayBuffer snapshot changed after export: %q", got)
}
}
func TestExportFetchRequestSnapshotsFormData(t *testing.T) {
runtime := goja.New()
runtime.SetFieldNameMapper(goja.TagFieldNameMapper("json", true))
if err := formdata.Enable(runtime); err != nil {
t.Fatal(err)
}
value, err := runtime.RunString(`
globalThis.form = new FormData();
form.append("field", "before");
form;
`)
if err != nil {
t.Fatal(err)
}
request := runtime.NewObject()
setFetchRequestDefaults(t, request)
if err := request.Set("body", value); err != nil {
t.Fatal(err)
}
meta, err := exportFetchRequest(runtime, request)
if err != nil {
t.Fatal(err)
}
if _, err := runtime.RunString(`form.set("field", "after")`); err != nil {
t.Fatal(err)
}
body, ok := meta.Body.(*fetchFormDataSnapshot)
if !ok {
t.Fatalf("expected FormData snapshot, got %T", meta.Body)
}
if len(body.entries) != 1 {
t.Fatalf("expected one FormData entry, got %d", len(body.entries))
}
if got := body.entries[0]; got.name != "field" || got.value != "before" {
t.Fatalf("unexpected FormData snapshot: %#v", got)
}
}
func setFetchRequestDefaults(t *testing.T, request *goja.Object) {
t.Helper()
for key, value := range map[string]any{
"url": "https://example.test",
"method": "POST",
"redirect": "follow",
"credentials": "same-origin",
} {
if err := request.Set(key, value); err != nil {
t.Fatal(err)
}
}
}
func TestSnapshotFetchBodyCopiesFileMetadata(t *testing.T) {
source := &file.File{
Reader: strings.NewReader("payload"),
Name: "before.txt",
Size: 7,
}
value, err := snapshotFetchBody(source)
if err != nil {
t.Fatal(err)
}
snapshot, ok := value.(*file.File)
if !ok {
t.Fatalf("expected file snapshot, got %T", value)
}
source.Name = "after.txt"
source.Size = 99
if snapshot.Name != "before.txt" || snapshot.Size != 7 {
t.Fatalf("file metadata was not snapshotted: %#v", snapshot)
}
}
func TestFetchRegistryCloseAll(t *testing.T) {
registry := newFetchRegistry()
streamCtx, cancel := context.WithCancel(context.Background())
body := &trackingReadCloser{}
registry.streams["active"] = &fetchStream{
body: body,
cancel: cancel,
ctx: streamCtx,
ch: make(chan fetchChunk),
}
registry.CloseAll()
registry.CloseAll()
select {
case <-streamCtx.Done():
default:
t.Fatal("active fetch context was not canceled")
}
if got := body.closeCalls.Load(); got != 1 {
t.Fatalf("expected body to close once, got %d", got)
}
registry.mu.Lock()
closed := registry.closed
remaining := len(registry.streams)
registry.mu.Unlock()
if !closed || remaining != 0 {
t.Fatalf("registry not fully closed: closed=%v remaining=%d", closed, remaining)
}
if _, err := registry.Open("", nil, &fetchRequest{URL: "https://example.test"}); err == nil {
t.Fatal("expected a closed registry to reject new fetches")
}
}
func TestFetchRegistryReadPreservesChunkOrder(t *testing.T) {
registry := newFetchRegistry()
ctx, cancel := context.WithCancel(context.Background())
stream := &fetchStream{
body: io.NopCloser(strings.NewReader("")),
cancel: cancel,
ctx: ctx,
ch: make(chan fetchChunk, 2),
}
stream.ch <- fetchChunk{data: []byte("abc")}
stream.ch <- fetchChunk{data: []byte("def")}
registry.streams["ordered"] = stream
defer registry.CloseAll()
var got strings.Builder
for range 4 {
chunk, done, err := registry.Read("ordered", 2)
if err != nil {
t.Fatal(err)
}
if done {
t.Fatal("stream ended before buffered chunks were read")
}
got.Write(chunk)
}
if got.String() != "abcdef" {
t.Fatalf("fetch chunks were reordered: %q", got.String())
}
}
type trackingReadCloser struct {
closeCalls atomic.Int32
}
func (r *trackingReadCloser) Read([]byte) (int, error) {
return 0, io.EOF
}
func (r *trackingReadCloser) Close() error {
r.closeCalls.Add(1)
return nil
}
pkg/download/engine/inject/stream/stream.js
+297 -207
View File
@@ -1,9 +1,5 @@
(function () {
const createBlobObjectURL = globalThis.__gopeed_create_blob_object_url;
const createWritableObjectURL = globalThis.__gopeed_create_writable_stream_object_url;
const writeWritableObjectURL = globalThis.__gopeed_write_writable_stream_object_url;
const closeWritableObjectURL = globalThis.__gopeed_close_writable_stream_object_url;
const abortWritableObjectURL = globalThis.__gopeed_abort_writable_stream_object_url;
const revokeObjectURL = globalThis.__gopeed_revoke_object_url;
const fetchOpen = globalThis.__gopeed_fetch_open;
const fetchRead = globalThis.__gopeed_fetch_read;
@@ -447,38 +443,6 @@
return merged;
}
function createStreamBackedBlob(stream, contentType, size) {
const blob = new Blob([], { type: contentType || "" });
if (Number.isFinite(size) && size >= 0) {
try {
Object.defineProperty(blob, "size", {
configurable: true,
enumerable: true,
value: size,
});
} catch (_) {
}
}
blob.__gopeedStreamBackedBlob = true;
blob.__gopeedBodyStream = stream;
blob.stream = function () {
const bodyStream = this.__gopeedBodyStream;
this.__gopeedBodyStream = null;
if (bodyStream instanceof ReadableStream) {
return bodyStream;
}
return new Blob([], { type: this.type || "" }).stream();
};
blob.arrayBuffer = async function () {
const bytes = await readAllFromStream(this.stream(), false);
return bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength);
};
blob.text = async function () {
return readAllFromStream(this.stream(), true);
};
return blob;
}
function attachResponseStreaming(response, stream) {
response.__gopeedBodyStream = stream;
response.__gopeedBodyConsumed = false;
@@ -515,8 +479,11 @@
ensureUnused();
markBodyUsed();
const contentType = this.headers && this.headers.get ? (this.headers.get("content-type") || "") : "";
const contentLength = this.headers && this.headers.get ? parseInt(this.headers.get("content-length") || "", 10) : NaN;
return createStreamBackedBlob(stream, contentType, contentLength);
// Response.blob() is a materializing body consumer. Keeping the live
// response stream behind an empty one-shot Blob makes retry, range and a
// second object-URL reader silently produce an empty file.
const bytes = await readAllFromStream(stream, false);
return new Blob([bytes], { type: contentType });
};
response.json = async function () {
const text = await this.text();
@@ -545,23 +512,7 @@
}
}
const originalCreateObjectURL = typeof URL.createObjectURL === "function"
? URL.createObjectURL.bind(URL)
: null;
const originalRevokeObjectURL = typeof URL.revokeObjectURL === "function"
? URL.revokeObjectURL.bind(URL)
: null;
const resumableReadableObjectURLs = new Map();
const activeReadableObjectURLs = new Map();
const readableObjectURLYieldBytes = 256 * 1024;
function isIgnorableGBlobObjectURLError(error) {
const message = error == null ? "" : String(error && error.message ? error.message : error);
return message.indexOf("gblob source revoked") >= 0 ||
message.indexOf("gblob source not found") >= 0 ||
message.indexOf("gblob source closed") >= 0 ||
message.indexOf("gblob source aborted") >= 0;
}
const blobReaders = new Map();
function getValueTypeName(value) {
if (value === null) {
@@ -577,45 +528,48 @@
}
function toReadableStreamReader(value, sourceLabel) {
if (value instanceof ReadableStream) {
if (value && typeof value.getReader === "function") {
return value.getReader();
}
throw new TypeError(sourceLabel + " must return a ReadableStream, got " + getValueTypeName(value));
}
function describeObjectURLValue(value) {
if (value && value.__gopeedStreamBackedBlob) {
const stream = value.__gopeedBodyStream;
value.__gopeedBodyStream = null;
if (stream instanceof ReadableStream) {
return {
kind: "readable",
initialReadable: stream,
openReadable: null,
sourceLabel: "URL.createObjectURL Response blob stream",
};
async function openBlobReadable(blob, request) {
const { offset, end } = request;
const sliced = blob.slice(offset, end >= offset ? end + 1 : undefined);
if (sliced && typeof sliced.stream === "function") {
try {
const stream = sliced.stream();
if (stream && typeof stream.getReader === "function") {
return stream;
}
} catch (_) {
}
}
if (sliced && typeof sliced.arrayBuffer === "function") {
const buffer = await sliced.arrayBuffer();
return new ReadableStream({
start(controller) {
controller.enqueue(new Uint8Array(buffer));
controller.close();
},
});
}
throw new TypeError("Blob source cannot be converted to a ReadableStream");
}
function describeObjectURLValue(value) {
if (value instanceof Blob) {
return {
kind: "blob",
value,
};
}
if (value instanceof ReadableStream) {
return {
kind: "readable",
initialReadable: value,
openReadable: null,
sourceLabel: "URL.createObjectURL ReadableStream",
};
}
if (typeof value === "function") {
return {
kind: "opener",
initialReadable: null,
openReadable: value,
sourceLabel: "URL.createObjectURL opener function",
sourceLabel: "gopeed.runtime.blob.createObjectURL opener function",
};
}
return {
@@ -624,13 +578,6 @@
};
}
function startReadableObjectURL(url, state) {
activeReadableObjectURLs.set(url, state);
setTimeout(() => {
void pumpReadableObjectURL(url, state);
}, 0);
}
function releaseReader(reader, reason) {
if (!reader) {
return;
@@ -652,144 +599,287 @@
}
}
function releaseActiveReadable(url, reason) {
const active = activeReadableObjectURLs.get(url);
if (!active) {
return;
function normalizeBlobObjectURLOptions(value, options, defaultRange) {
const normalized = {
contentType: "",
size: 0,
range: !!defaultRange,
};
if (options && typeof options === "object") {
if (typeof options.contentType === "string") {
normalized.contentType = options.contentType;
}
if (Number.isFinite(Number(options.size)) && Number(options.size) > 0) {
normalized.size = Number(options.size);
}
if (typeof options.range === "boolean") {
normalized.range = options.range;
}
}
activeReadableObjectURLs.delete(url);
active.cancelled = true;
releaseReader(active.reader, reason);
if (!normalized.contentType && value && typeof value.type === "string") {
normalized.contentType = value.type;
}
if (!normalized.size && value) {
if (Number.isFinite(Number(value.size)) && Number(value.size) > 0) {
normalized.size = Number(value.size);
} else if (value._buffer && Number.isFinite(Number(value._buffer.byteLength))) {
normalized.size = Number(value._buffer.byteLength);
}
}
return normalized;
}
async function pumpReadableObjectURL(url, state) {
let reader = null;
try {
let source;
if (state.initialReadable) {
source = state.initialReadable;
state.initialReadable = null;
function validateBlobObjectURLOptions(options) {
if (options.range && !(Number.isFinite(Number(options.size)) && Number(options.size) > 0)) {
throw new TypeError("gopeed.runtime.blob.createObjectURL options.range requires a positive options.size");
}
}
function normalizeOpenRequest(request) {
const offset = Math.max(0, Number(request && request.offset) || 0);
const endValue = Number(request && request.end);
const end = Number.isFinite(endValue) ? endValue : -1;
return { offset, end };
}
function yieldBlobPipeTask() {
return new Promise((resolve) => {
if (typeof setTimeout === "function") {
setTimeout(resolve, 0);
} else {
if (typeof state.openReadable !== "function") {
throw new Error("gblob readable stream is not reopenable");
}
source = await state.openReadable(state.offset);
Promise.resolve().then(resolve);
}
reader = toReadableStreamReader(source, state.sourceLabel);
if (activeReadableObjectURLs.get(url) !== state || state.cancelled) {
releaseReader(reader, "stale gblob producer");
reader = null;
return;
}
state.reader = reader;
let bytesSinceYield = 0;
while (!state.cancelled) {
const current = activeReadableObjectURLs.get(url);
if (current !== state) {
releaseReader(reader, "stale gblob producer");
reader = null;
return;
}
const { done, value } = await reader.read();
if (done) {
if (activeReadableObjectURLs.get(url) === state) {
activeReadableObjectURLs.delete(url);
await closeWritableObjectURL(url);
}
releaseReader(reader);
reader = null;
return;
}
const chunk = value instanceof Uint8Array ? value : new Uint8Array(value);
if (activeReadableObjectURLs.get(url) !== state || state.cancelled) {
releaseReader(reader, "stale gblob producer");
reader = null;
return;
}
await writeWritableObjectURL(url, chunk);
bytesSinceYield += chunk.byteLength;
if (bytesSinceYield >= readableObjectURLYieldBytes) {
bytesSinceYield = 0;
await new Promise((resolve) => setTimeout(resolve, 0));
}
}
releaseReader(reader, "gblob producer cancelled");
reader = null;
} catch (error) {
if (isIgnorableGBlobObjectURLError(error)) {
if (activeReadableObjectURLs.get(url) === state) {
activeReadableObjectURLs.delete(url);
}
if (reader) {
releaseReader(reader, "gblob source closed");
reader = null;
}
return;
}
if (activeReadableObjectURLs.get(url) === state) {
activeReadableObjectURLs.delete(url);
try {
await abortWritableObjectURL(url, error == null ? "" : String(error && error.message ? error.message : error));
} catch (_) {
}
}
if (reader) {
releaseReader(reader, error);
reader = null;
}
}
});
}
globalThis.__gopeed_open_writable_stream_object_url = function (url, offset) {
const openReadable = resumableReadableObjectURLs.get(url);
if (typeof openReadable !== "function") {
throw new Error("gblob resumable opener not found");
function createPendingBlobReader(openReadable, request) {
let activeReader;
let ready;
function ensureReader() {
if (ready) {
return ready;
}
ready = (async function () {
const source = await openReadable(normalizeOpenRequest(request));
activeReader = toReadableStreamReader(source, "gopeed.runtime.blob.createObjectURL opener function");
return activeReader;
})();
return ready;
}
releaseActiveReadable(url, "gblob source reopened");
startReadableObjectURL(url, {
initialReadable: null,
openReadable,
sourceLabel: "URL.createObjectURL opener function",
offset: Number(offset) || 0,
reader: null,
cancelled: false,
});
return {
async read() {
const reader = await ensureReader();
return reader.read();
},
cancel(reason) {
return ensureReader().then(function (reader) {
return reader.cancel(reason);
}, function () {});
},
releaseLock() {
if (!ready) {
return;
}
return ready.then(function (reader) {
return reader.releaseLock();
}, function () {});
}
};
}
globalThis.__gopeed_blob_open_source = function (openReadable, request) {
if (typeof openReadable !== "function") {
throw new TypeError("blob opener must be callable");
}
const id = crypto.randomUUID ? crypto.randomUUID() : String(Date.now()) + "-" + String(Math.random());
blobReaders.set(id, createPendingBlobReader(openReadable, request));
return id;
};
URL.createObjectURL = function (value) {
globalThis.__gopeed_blob_read_source = async function (id, chunkSize) {
const reader = blobReaders.get(id);
if (!reader) {
return null;
}
const { done, value } = await reader.read();
if (done) {
blobReaders.delete(id);
releaseReader(reader);
return null;
}
const chunk = value instanceof Uint8Array ? value : new Uint8Array(value);
const size = Math.max(1, Number(chunkSize) || chunk.byteLength);
if (chunk.byteLength <= size) {
return chunk.buffer.slice(chunk.byteOffset, chunk.byteOffset + chunk.byteLength);
}
const head = chunk.slice(0, size);
let offset = size;
blobReaders.set(id, {
read() {
if (offset >= chunk.byteLength) {
return reader.read();
}
const next = chunk.slice(offset, Math.min(offset + size, chunk.byteLength));
offset += next.byteLength;
return Promise.resolve({ done: false, value: next });
},
cancel(reason) {
return reader.cancel(reason);
},
releaseLock() {
return reader.releaseLock();
}
});
return head.buffer.slice(head.byteOffset, head.byteOffset + head.byteLength);
};
globalThis.__gopeed_blob_close_source = function (id) {
const reader = blobReaders.get(id);
if (!reader) {
return;
}
blobReaders.delete(id);
releaseReader(reader, "blob request closed");
};
const blobPipeReaderStates = new Map();
function releaseBlobPipeReader(state) {
if (!state || state.released) {
return;
}
state.released = true;
if (state.reader && typeof state.reader.releaseLock === "function") {
try {
state.reader.releaseLock();
} catch (_) {
}
}
if (blobPipeReaderStates.get(state.pipeId) === state) {
blobPipeReaderStates.delete(state.pipeId);
}
}
function finishBlobPipeReader(state, cancel, reason) {
if (!state) {
return Promise.resolve();
}
if (cancel) {
state.cancelRequested = true;
if (reason !== undefined) {
state.cancelReason = reason;
}
}
if (state.cleanupPromise) {
return state.cleanupPromise;
}
if (!state.reader) {
// A cancellation may arrive while the asynchronous opener is pending.
// Keep the state until the reader exists so it can still be canceled.
if (!state.cancelRequested || state.openSettled) {
releaseBlobPipeReader(state);
}
return Promise.resolve();
}
state.cleanupPromise = (async function () {
try {
if (state.cancelRequested && typeof state.reader.cancel === "function") {
await state.reader.cancel(state.cancelReason);
}
} catch (_) {
} finally {
releaseBlobPipeReader(state);
}
})();
return state.cleanupPromise;
}
globalThis.__gopeed_blob_cancel_pipe_source = function (pipeId, reason) {
const state = blobPipeReaderStates.get(pipeId);
if (!state) {
return;
}
// finishBlobPipeReader invokes reader.cancel() synchronously up to its
// first await. Do not return the cleanup Promise: Go only needs to ensure
// cancellation has started before releasing the source session.
finishBlobPipeReader(state, true, reason).catch(function () {});
};
globalThis.__gopeed_blob_pipe_source = function (openReadable, request, pipeId) {
if (typeof openReadable !== "function") {
throw new TypeError("blob opener must be callable");
}
const state = {
pipeId,
reader: null,
cancelRequested: false,
cancelReason: "blob pipe closed",
cleanupPromise: null,
openSettled: false,
released: false,
};
blobPipeReaderStates.set(pipeId, state);
const startPipe = async function () {
try {
const source = await openReadable(normalizeOpenRequest(request));
state.openSettled = true;
state.reader = toReadableStreamReader(source, "gopeed.runtime.blob.createObjectURL opener function");
if (state.cancelRequested) {
await finishBlobPipeReader(state, true, state.cancelReason);
return;
}
while (true) {
const { done, value } = await state.reader.read();
if (done) {
globalThis.__gopeed_blob_pipe_close(pipeId);
return;
}
const chunk = value instanceof Uint8Array ? value : new Uint8Array(value);
const buffer = chunk.buffer.slice(chunk.byteOffset, chunk.byteOffset + chunk.byteLength);
if (!globalThis.__gopeed_blob_pipe_chunk(pipeId, buffer)) {
state.cancelRequested = true;
state.cancelReason = "blob pipe closed";
return;
}
await yieldBlobPipeTask();
}
} catch (error) {
if (!state.cancelRequested) {
globalThis.__gopeed_blob_pipe_error(pipeId, error && error.stack ? error.stack : String(error));
}
} finally {
state.openSettled = true;
await finishBlobPipeReader(state, state.cancelRequested, state.cancelReason);
}
};
if (typeof setTimeout === "function") {
setTimeout(startPipe, 0);
} else {
Promise.resolve().then(startPipe);
}
};
globalThis.__gopeed_blob_create_object_url = function (value, options) {
const described = describeObjectURLValue(value);
if (described.kind === "blob") {
return createBlobObjectURL(described.value._buffer, described.value.type || "");
}
if (described.kind === "readable") {
const url = createWritableObjectURL(false);
startReadableObjectURL(url, {
initialReadable: described.initialReadable,
openReadable: null,
sourceLabel: described.sourceLabel,
offset: 0,
reader: null,
cancelled: false,
});
return url;
const blob = described.value;
const normalized = normalizeBlobObjectURLOptions(blob, options, Number(blob.size) > 0);
validateBlobObjectURLOptions(normalized);
const opener = async (request) => openBlobReadable(blob, request);
return createBlobObjectURL(opener, normalized);
}
if (described.kind === "opener") {
const url = createWritableObjectURL(true);
resumableReadableObjectURLs.set(url, described.openReadable);
return url;
const normalized = normalizeBlobObjectURLOptions(null, options, false);
validateBlobObjectURLOptions(normalized);
return createBlobObjectURL(described.openReadable, normalized);
}
throw new TypeError("Unsupported object type for URL.createObjectURL: " + getValueTypeName(described.value) + ". Expected Blob, ReadableStream, or opener function");
throw new TypeError("Unsupported object type for gopeed.runtime.blob.createObjectURL: " + getValueTypeName(described.value) + ". Expected Blob or opener function");
};
URL.revokeObjectURL = function (url) {
if (typeof url === "string" && url.indexOf("gblob:") === 0) {
resumableReadableObjectURLs.delete(url);
releaseActiveReadable(url, "gblob source revoked");
globalThis.__gopeed_blob_revoke_object_url = function (url) {
if (typeof url === "string") {
revokeObjectURL(url);
return;
}
if (originalRevokeObjectURL) {
originalRevokeObjectURL(url);
}
};
@@ -823,7 +913,7 @@
});
let meta;
try {
meta = fetchOpen({
meta = await fetchOpen({
url: request.url,
method: request.method,
headers,
@@ -835,10 +925,10 @@
throw error instanceof Error ? error : new TypeError(String(error));
}
const stream = new ReadableStream({
pull(controller) {
async pull(controller) {
let chunk;
try {
chunk = fetchRead(meta.id, 64 * 1024);
chunk = await fetchRead(meta.id, 64 * 1024);
} catch (error) {
fetchClose(meta.id);
controller.error(error instanceof Error ? error : new TypeError(String(error)));
pkg/download/extension.go
+19 -29
View File
@@ -271,7 +271,6 @@ func (d *Downloader) triggerOnResolve(req *base.Request) (res *base.Resource, er
}
ensureResourceRequestRawURLs(req, ctx.Res)
ctx.Res.CalcSize(nil)
d.applyGBlobResourceMetadata(ctx.Res)
}
res = ctx.Res
},
@@ -279,25 +278,12 @@ func (d *Downloader) triggerOnResolve(req *base.Request) (res *base.Resource, er
return
}
func (d *Downloader) applyGBlobResourceMetadata(res *base.Resource) {
if d.gblob == nil || res == nil {
return
}
for _, file := range res.Files {
if file == nil || file.Req == nil {
continue
}
_ = d.gblob.SetSize(file.Req.URL, file.Size)
_ = d.gblob.SetRange(file.Req.URL, res.Range)
}
}
func (d *Downloader) triggerOnStart(task *Task) {
doTrigger(d,
EventOnStart,
task.Meta.Req,
&OnStartContext{
Task: NewExtensionTask(d, task),
Task: newOnStartExtensionTask(task),
},
func(ext *Extension, gopeed *Instance, ctx *OnStartContext) {
// Validate request structure
@@ -309,7 +295,6 @@ func (d *Downloader) triggerOnStart(task *Task) {
}
},
)
return
}
func (d *Downloader) triggerOnError(task *Task, err error) {
@@ -317,7 +302,7 @@ func (d *Downloader) triggerOnError(task *Task, err error) {
EventOnError,
task.Meta.Req,
&OnErrorContext{
Task: NewExtensionTask(d, task),
Task: newOnErrorExtensionTask(d, task),
Error: err,
},
nil,
@@ -328,8 +313,8 @@ func (d *Downloader) triggerOnDone(task *Task) {
doTrigger(d,
EventOnDone,
task.Meta.Req,
&OnErrorContext{
Task: NewExtensionTask(d, task),
&OnDoneContext{
Task: newOnDoneExtensionTask(task),
},
nil,
)
@@ -720,7 +705,7 @@ type OnResolveContext struct {
}
type OnStartContext struct {
Task *ExtensionTask `json:"task"`
Task *Task `json:"task"`
}
type OnErrorContext struct {
@@ -741,28 +726,33 @@ type ExtensionTask struct {
*Task
}
func NewExtensionTask(download *Downloader, task *Task) *ExtensionTask {
// restricts extension scripts to only modify request info
func cloneExtensionTask(task *Task) *Task {
newTask := task.clone()
newTask.Meta.Req = task.Meta.Req
return newTask
}
func newOnStartExtensionTask(task *Task) *Task {
return cloneExtensionTask(task)
}
func newOnErrorExtensionTask(download *Downloader, task *Task) *ExtensionTask {
return &ExtensionTask{
download: download,
Task: newTask,
Task: cloneExtensionTask(task),
}
}
func newOnDoneExtensionTask(task *Task) *Task {
return cloneExtensionTask(task)
}
func (t *ExtensionTask) Continue() error {
return t.download.Continue(&TaskFilter{
IDs: []string{t.ID},
})
}
func (t *ExtensionTask) Pause() error {
return t.download.Pause(&TaskFilter{
IDs: []string{t.ID},
})
}
func parseSettings(settings []*Setting) map[string]any {
m := make(map[string]any)
for _, s := range settings {
pkg/download/extension_engine_session.go
+19 -46
View File
@@ -1,14 +1,14 @@
package download
import (
"fmt"
"context"
"io"
"sync"
"github.com/GopeedLab/gopeed/internal/protocol/gblob"
internalblob "github.com/GopeedLab/gopeed/internal/blob"
"github.com/GopeedLab/gopeed/pkg/download/engine"
"github.com/GopeedLab/gopeed/pkg/download/engine/inject/stream"
enginewebview "github.com/GopeedLab/gopeed/pkg/download/engine/webview"
"github.com/dop251/goja"
)
type engineSession struct {
@@ -96,57 +96,30 @@ func (s *engineSession) runClosers() {
}
}
func normalizeStreamChunk(data any) ([]byte, error) {
switch v := data.(type) {
case nil:
return nil, nil
case string:
return []byte(v), nil
case []byte:
return v, nil
case goja.ArrayBuffer:
return v.Bytes(), nil
default:
return nil, fmt.Errorf("unsupported stream chunk type: %T", data)
}
}
func (d *Downloader) newExtensionEngine() (*engine.Engine, *engineSession) {
session := newEngineSession(nil)
engineCfg := &stream.Config{
CreateBlobObjectURL: func(data []byte, contentType string) (string, error) {
return d.gblob.CreateBlob(data, contentType)
},
CreateWritableStreamObjectURL: func(opts *stream.WritableStreamObjectURLOptions) (string, error) {
reopenable := false
CreateObjectURL: func(opts *stream.ObjectURLOptions, open stream.ObjectURLOpener) (string, error) {
createOpts := &internalblob.CreateOptions{
Session: session,
}
if opts != nil {
reopenable = opts.Reopenable
createOpts.ContentType = opts.ContentType
createOpts.Size = opts.Size
createOpts.Range = opts.Range
}
return d.gblob.CreateWritableStream(&gblob.CreateWritableStreamOptions{
Session: session,
Reopenable: reopenable,
})
},
RegisterWritableStreamResume: func(url string, reopen func(offset int64) error) error {
return d.gblob.SetResumeOpener(url, reopen)
},
WriteWritableStreamObjectURL: func(url string, data any) error {
chunk, err := normalizeStreamChunk(data)
if err != nil {
return err
}
return d.gblob.Write(url, chunk)
},
CloseWritableStreamObjectURL: func(url string) error {
return d.gblob.CloseSource(url)
},
AbortWritableStreamObjectURL: func(url string, reason string) error {
return d.gblob.AbortSource(url, fmt.Errorf("%s", reason))
return d.blob.CreateOpener(func(ctx context.Context, req internalblob.OpenRequest) (io.ReadCloser, error) {
return open(ctx, stream.ObjectURLOpenRequest{
Offset: req.Offset,
End: req.End,
})
}, createOpts)
},
RevokeObjectURL: func(url string) error {
return d.gblob.Revoke(url)
return d.blob.Revoke(url)
},
ProxyHandler: d.cfg.Proxy.ToHandler(),
ProxyHandler: d.cfg.Proxy.ToHandler(),
RegisterCleanup: session.OnClose,
}
e := engine.NewEngine(&engine.Config{
ProxyConfig: d.cfg.Proxy,
pkg/download/extension_runtime_webview.go
+33 -2
View File
@@ -29,10 +29,15 @@ func injectGopeed(vm *goja.Runtime, gopeed *Instance) error {
return err
}
runtimeObject := vm.NewObject()
if gopeed.Runtime != nil && gopeed.Runtime.WebView != nil {
if err := runtimeObject.Set("webview", newJSWebViewRuntime(vm, gopeed.Runtime.WebView)); err != nil {
if gopeed.Runtime != nil {
if err := runtimeObject.Set("blob", newJSBlobRuntime(vm)); err != nil {
return err
}
if gopeed.Runtime.WebView != nil {
if err := runtimeObject.Set("webview", newJSWebViewRuntime(vm, gopeed.Runtime.WebView)); err != nil {
return err
}
}
}
if err := gopeedObject.Set("runtime", runtimeObject); err != nil {
return err
@@ -40,6 +45,32 @@ func injectGopeed(vm *goja.Runtime, gopeed *Instance) error {
return vm.Set("gopeed", gopeedObject)
}
func newJSBlobRuntime(vm *goja.Runtime) *goja.Object {
obj := vm.NewObject()
_ = obj.Set("createObjectURL", func(call goja.FunctionCall) goja.Value {
fn, ok := goja.AssertFunction(vm.Get("__gopeed_blob_create_object_url"))
if !ok {
panic(vm.ToValue(fmt.Errorf("blob runtime is not available")))
}
value, err := fn(goja.Undefined(), call.Argument(0), call.Argument(1))
if err != nil {
panic(vm.ToValue(err))
}
return value
})
_ = obj.Set("revokeObjectURL", func(call goja.FunctionCall) goja.Value {
fn, ok := goja.AssertFunction(vm.Get("__gopeed_blob_revoke_object_url"))
if !ok {
panic(vm.ToValue(fmt.Errorf("blob runtime is not available")))
}
if _, err := fn(goja.Undefined(), call.Argument(0)); err != nil {
panic(vm.ToValue(err))
}
return goja.Undefined()
})
return obj
}
func newJSEventsRuntime(vm *goja.Runtime, events InstanceEvents) *goja.Object {
obj := vm.NewObject()
register := func(event ActivationEvent, call goja.FunctionCall) goja.Value {
pkg/download/extension_test.go
+640 -700
View File
File diff suppressed because it is too large. Load diff
pkg/download/model.go
+3 -2
View File
@@ -9,7 +9,6 @@ import (
"github.com/GopeedLab/gopeed/internal/fetcher"
"github.com/GopeedLab/gopeed/internal/protocol/bt"
"github.com/GopeedLab/gopeed/internal/protocol/ed2k"
"github.com/GopeedLab/gopeed/internal/protocol/gblob"
"github.com/GopeedLab/gopeed/internal/protocol/http"
"github.com/GopeedLab/gopeed/pkg/base"
enginewebview "github.com/GopeedLab/gopeed/pkg/download/engine/webview"
@@ -37,6 +36,9 @@ type Task struct {
timer *util.Timer
statusLock *sync.Mutex
lock *sync.Mutex
blobRefLock *sync.Mutex
blobURL string
runGeneration uint64
speedArr []int64
uploadSpeedArr []int64
}
@@ -166,7 +168,6 @@ func (cfg *DownloaderConfig) Init() *DownloaderConfig {
}
if len(cfg.FetchManagers) == 0 {
cfg.FetchManagers = []fetcher.FetcherManager{
new(gblob.FetcherManager),
new(http.FetcherManager),
new(bt.FetcherManager),
new(ed2k.FetcherManager),
pkg/download/testdata/extensions/blob/index.js
Vendored
+111
View File
@@ -0,0 +1,111 @@
gopeed.events.onResolve(async function (ctx) {
const encoder = new TextEncoder();
if (ctx.req.url.endsWith("/blob")) {
const blob = new Blob(["hello world"], {type: "text/plain"});
const url = gopeed.runtime.blob.createObjectURL(blob);
ctx.res = {
name: "blob-blob",
files: [
{
name: "hello.txt",
size: 11,
req: {
url,
}
}
]
};
return;
}
if (ctx.req.url.endsWith("/opener-range")) {
const payload = encoder.encode("line 1\nline 2\nline 3\n");
const firstChunkLength = encoder.encode("line 1\n").byteLength;
let firstAttempt = true;
const open = async ({ offset = 0, end = -1 }) => new ReadableStream({
async start(controller) {
try {
if (offset === 0 && firstAttempt) {
firstAttempt = false;
controller.enqueue(payload.slice(0, firstChunkLength));
await new Promise((resolve) => setTimeout(resolve, 120));
controller.error(new Error("resume required"));
return;
}
controller.enqueue(payload.slice(offset, end >= 0 ? end + 1 : undefined));
controller.close();
} catch (err) {
controller.error(err);
}
}
});
const url = gopeed.runtime.blob.createObjectURL(open, { size: payload.byteLength, range: true });
ctx.res = {
name: "blob-opener-range",
range: true,
files: [
{
name: "opener-range.txt",
size: payload.byteLength,
req: {
url,
}
}
]
};
return;
}
if (ctx.req.url.endsWith("/opener-unknown")) {
const url = gopeed.runtime.blob.createObjectURL(async ({ offset = 0 }) => new ReadableStream({
async start(controller) {
if (offset > 0) {
controller.close();
return;
}
controller.enqueue(encoder.encode("line 1\n"));
await new Promise((resolve) => setTimeout(resolve, 1500));
controller.enqueue(encoder.encode("line 2\n"));
controller.close();
},
}));
ctx.res = {
name: "blob-opener-unknown",
files: [
{
name: "opener-unknown.txt",
req: {
url,
}
}
]
};
return;
}
const url = gopeed.runtime.blob.createObjectURL(async ({ offset = 0, end = -1 }) => new ReadableStream({
start(controller) {
const payload = encoder.encode("line 1\nline 2\n");
controller.enqueue(payload.slice(offset, end >= 0 ? end + 1 : undefined));
controller.close();
},
}), { size: 14 });
ctx.res = {
name: "blob-opener",
files: [
{
name: "opener.txt",
size: 14,
req: {
url,
}
}
]
};
});
pkg/download/testdata/extensions/gblob/manifest.json → pkg/download/testdata/extensions/blob/manifest.json
Vendored
+2 -2
View File
@@ -1,6 +1,6 @@
{
"name": "gblob",
"title": "gopeed extension gblob test",
"name": "blob",
"title": "gopeed extension blob test",
"version": "0.0.1",
"scripts": [
{
pkg/download/testdata/extensions/blob_pause_rebuild/index.js
Vendored
+140
View File
@@ -0,0 +1,140 @@
const payloadSize = 262144;
const chunkSize = 4096;
const payloadText = 'x'.repeat(payloadSize);
function delayedPayloadStream(offset = 0, end = -1) {
const limit = end >= 0 ? Math.min(end + 1, payloadSize) : payloadSize;
let pos = offset;
return new ReadableStream({
async pull(controller) {
if (pos >= limit) {
controller.close();
return;
}
const size = Math.min(chunkSize, limit - pos);
const chunk = new Uint8Array(size);
chunk.fill(120);
pos += size;
controller.enqueue(chunk);
await new Promise((resolve) => setTimeout(resolve, 20));
},
});
}
function fastPayloadStream(offset = 0, end = -1) {
const limit = end >= 0 ? Math.min(end + 1, payloadSize) : payloadSize;
let pos = offset;
return new ReadableStream({
pull(controller) {
if (pos >= limit) {
controller.close();
return;
}
const size = Math.min(chunkSize, limit - pos);
const chunk = new Uint8Array(size);
chunk.fill(120);
pos += size;
controller.enqueue(chunk);
},
});
}
function createBlobSourceURL(delayed) {
const blob = new Blob([payloadText], { type: 'application/octet-stream' });
const originalSlice = blob.slice.bind(blob);
blob.slice = function (offset = 0, end) {
if (!delayed) {
return originalSlice(offset, end);
}
const sliceEnd = Number.isFinite(Number(end)) ? Number(end) - 1 : -1;
return {
stream() {
return delayedPayloadStream(Number(offset) || 0, sliceEnd);
},
};
};
return gopeed.runtime.blob.createObjectURL(blob);
}
function createOpenerSourceURL(delayed) {
const open = async () => delayed ? delayedPayloadStream(0, -1) : fastPayloadStream(0, -1);
return gopeed.runtime.blob.createObjectURL(open, {
size: payloadSize,
contentType: 'application/octet-stream',
});
}
function createRangeSourceURL(delayed) {
const open = async ({ offset = 0, end = -1 }) => delayed ? delayedPayloadStream(offset, end) : fastPayloadStream(offset, end);
return gopeed.runtime.blob.createObjectURL(open, {
size: payloadSize,
range: true,
contentType: 'application/octet-stream',
});
}
function modeFromURL(url) {
if (url.includes('/pause-rebuild-blob')) {
return 'blob';
}
if (url.includes('/pause-rebuild-opener')) {
return 'opener';
}
if (url.includes('/pause-rebuild-range')) {
return 'range';
}
return '';
}
function createPayloadURL(mode, delayed) {
switch (mode) {
case 'blob':
return createBlobSourceURL(delayed);
case 'opener':
return createOpenerSourceURL(delayed);
case 'range':
return createRangeSourceURL(delayed);
default:
throw new Error('unsupported pause rebuild mode: ' + mode);
}
}
gopeed.events.onResolve(async function (ctx) {
const mode = modeFromURL(ctx.req.url);
if (!mode) {
return;
}
ctx.res = {
name: 'blob-pause-rebuild-' + mode,
range: mode !== 'opener',
files: [
{
name: 'pause-rebuild-' + mode + '.bin',
size: payloadSize,
req: {
url: createPayloadURL(mode, true),
rawUrl: ctx.req.rawUrl || ctx.req.url,
labels: {
mode: 'pause-rebuild',
source: mode,
},
},
},
],
};
});
gopeed.events.onError(async function (ctx) {
const req = ctx.task?.meta?.req;
const source = req?.labels?.source || modeFromURL(req?.rawUrl || '');
if (!req || req.labels?.mode !== 'pause-rebuild' || req.labels?.rebuilt === 'true' || !source) {
return;
}
req.url = createPayloadURL(source, false);
req.labels = req.labels || {};
req.labels.started = 'true';
req.labels.rebuilt = 'true';
ctx.task.continue();
});
pkg/download/testdata/extensions/blob_pause_rebuild/manifest.json
Vendored
+28
View File
@@ -0,0 +1,28 @@
{
"name": "blob-pause-rebuild",
"title": "gopeed extension blob pause rebuild test",
"version": "0.0.1",
"scripts": [
{
"event": "onResolve",
"match": {
"urls": [
"*://example.com/pause-rebuild*"
]
},
"entry": "index.js"
},
{
"event": "onError",
"match": {
"labels": [
"mode"
],
"urls": [
"*://example.com/pause-rebuild*"
]
},
"entry": "index.js"
}
]
}
pkg/download/testdata/extensions/blob_recover/index.js
Vendored
+46
View File
@@ -0,0 +1,46 @@
gopeed.events.onResolve(async function (ctx) {
const isRange = ctx.req.url.endsWith("/recover-range");
if (!isRange && !ctx.req.url.endsWith("/recover")) {
return;
}
const url = gopeed.runtime.blob.createObjectURL(async ({ offset = 0 }) => new ReadableStream({
async start(controller) {
if (offset > 0) {
controller.close();
return;
}
controller.enqueue(new TextEncoder().encode("stale-prefix-that-must-be-removed\n"));
await new Promise((resolve) => setTimeout(resolve, 50));
controller.error(new Error("expired"));
},
}), { size: 64, range: isRange });
ctx.res = {
name: "blob-recover",
files: [
{
name: "recover.txt",
size: 64,
req: {
url,
}
}
]
};
});
gopeed.events.onError(async function (ctx) {
const req = ctx.task.meta.req;
if (!req.rawUrl || (!req.rawUrl.endsWith("/recover") && !req.rawUrl.endsWith("/recover-range"))) {
return;
}
req.labels = req.labels || {};
if (req.labels.recovered === "true") {
return;
}
req.labels.recovered = "true";
req.url = gopeed.runtime.blob.createObjectURL(new Blob(["ok\n"], { type: "text/plain" }));
ctx.task.continue();
});
pkg/download/testdata/extensions/gblob_recover/manifest.json → pkg/download/testdata/extensions/blob_recover/manifest.json
Vendored
+2 -2
View File
@@ -1,6 +1,6 @@
{
"name": "gblob-recover",
"title": "gopeed extension gblob recover test",
"name": "blob-recover",
"title": "gopeed extension blob recover test",
"version": "0.0.1",
"scripts": [
{
pkg/download/testdata/extensions/gblob_restart/index.js → pkg/download/testdata/extensions/blob_restart/index.js
Vendored
+9 -9
View File
@@ -1,15 +1,15 @@
function createRangeGBlobUrl(target) {
return URL.createObjectURL(async (offset) => {
function createRangeBlobUrl(target) {
return gopeed.runtime.blob.createObjectURL(async ({ offset = 0, end = -1 }) => {
const headers = {};
if (offset > 0) {
headers.Range = `bytes=${offset}-`;
if (offset > 0 || end >= 0) {
headers.Range = `bytes=${offset}-${end >= 0 ? end : ''}`;
}
const response = await fetch(target, { headers });
if (!response.body) {
throw new Error('empty response body');
}
return response.body;
});
}, { size: 262144, range: true });
}
gopeed.events.onResolve(async function (ctx) {
@@ -18,14 +18,14 @@ gopeed.events.onResolve(async function (ctx) {
}
ctx.res = {
name: 'gblob-restart',
name: 'blob-restart',
range: true,
files: [
{
name: 'restart.bin',
size: 262144,
req: {
url: createRangeGBlobUrl(ctx.req.rawUrl || ctx.req.url),
url: createRangeBlobUrl(ctx.req.rawUrl || ctx.req.url),
rawUrl: ctx.req.rawUrl || ctx.req.url,
labels: {
mode: 'restart',
@@ -38,7 +38,7 @@ gopeed.events.onResolve(async function (ctx) {
gopeed.events.onError(async function (ctx) {
const req = ctx.task?.meta?.req;
if (!req || !req.rawUrl || !req.url || !req.url.startsWith('gblob:')) {
if (!req || !req.rawUrl || !req.url || !req.url.includes('/__blob/')) {
return;
}
req.labels = req.labels || {};
@@ -47,7 +47,7 @@ gopeed.events.onError(async function (ctx) {
}
try {
req.url = createRangeGBlobUrl(req.rawUrl);
req.url = createRangeBlobUrl(req.rawUrl);
req.labels.started = 'true';
req.labels.rebuilt = 'true';
ctx.task.continue();
pkg/download/testdata/extensions/gblob_restart/manifest.json → pkg/download/testdata/extensions/blob_restart/manifest.json
Vendored
+2 -2
View File
@@ -1,6 +1,6 @@
{
"name": "gblob-restart",
"title": "gopeed extension gblob restart resume test",
"name": "blob-restart",
"title": "gopeed extension blob restart resume test",
"version": "0.0.1",
"scripts": [
{
pkg/download/testdata/extensions/blob_stream_bridge/index.js
Vendored
+198
View File
@@ -0,0 +1,198 @@
const encoder = new TextEncoder();
const decoder = new TextDecoder();
function byteLength(value) {
return encoder.encode(value).byteLength;
}
function sourceURL(rawURL) {
const url = new URL(rawURL);
url.searchParams.set("source", "1");
return url.toString();
}
function createFetchDrainedStream(url) {
return new ReadableStream({
async start(controller) {
try {
const response = await fetch(url);
const reader = response.body.getReader();
while (true) {
const { done, value } = await reader.read();
if (done) {
controller.close();
return;
}
controller.enqueue(value);
}
} catch (error) {
controller.error(error);
}
},
});
}
function createFetchDrainedRangeStream(url, offset, end) {
return new ReadableStream({
async start(controller) {
try {
const headers = {};
if (offset > 0 || end >= 0) {
headers.Range = `bytes=${offset}-${end >= 0 ? end : ""}`;
}
const response = await fetch(url, { headers });
const reader = response.body.getReader();
while (true) {
const { done, value } = await reader.read();
if (done) {
controller.close();
return;
}
controller.enqueue(value);
}
} catch (error) {
controller.error(error);
}
},
});
}
function createMultiplexStreams(url) {
let videoController;
let audioController;
let started = false;
async function pump() {
if (started || !videoController || !audioController) {
return;
}
started = true;
try {
const response = await fetch(url);
const reader = response.body.getReader();
let pending = "";
while (true) {
const { done, value } = await reader.read();
if (done) {
break;
}
pending += decoder.decode(value, { stream: true });
let newlineIndex;
while ((newlineIndex = pending.indexOf("\n")) >= 0) {
const line = pending.slice(0, newlineIndex);
pending = pending.slice(newlineIndex + 1);
if (line.startsWith("v:")) {
videoController.enqueue(encoder.encode(line.slice(2)));
} else if (line.startsWith("a:")) {
audioController.enqueue(encoder.encode(line.slice(2)));
}
}
}
if (pending.startsWith("v:")) {
videoController.enqueue(encoder.encode(pending.slice(2)));
} else if (pending.startsWith("a:")) {
audioController.enqueue(encoder.encode(pending.slice(2)));
}
videoController.close();
audioController.close();
} catch (error) {
videoController.error(error);
audioController.error(error);
}
}
return {
video: new ReadableStream({
start(controller) {
videoController = controller;
pump();
},
}),
audio: new ReadableStream({
start(controller) {
audioController = controller;
pump();
},
}),
};
}
gopeed.events.onResolve(async function (ctx) {
if (ctx.req.url.includes("/single-fetch-stream")) {
const payloadSize = Number(new URL(ctx.req.url).searchParams.get("size")) || 0;
const url = gopeed.runtime.blob.createObjectURL(async () => createFetchDrainedStream(sourceURL(ctx.req.url)), {
size: payloadSize,
});
ctx.res = {
name: "single-fetch-stream",
files: [
{
name: "single.bin",
size: payloadSize,
req: { url },
},
],
};
return;
}
if (ctx.req.url.includes("/range-fetch-stream")) {
const payloadSize = Number(new URL(ctx.req.url).searchParams.get("size")) || 0;
const url = gopeed.runtime.blob.createObjectURL(
async ({ offset = 0, end = -1 }) => createFetchDrainedRangeStream(sourceURL(ctx.req.url), offset, end),
{
size: payloadSize,
range: true,
}
);
ctx.res = {
name: "range-fetch-stream",
range: true,
files: [
{
name: "range.bin",
size: payloadSize,
req: { url },
},
],
};
return;
}
if (ctx.req.url.includes("/multiplex-fetch-stream")) {
const videoChunks = ["video-0", "video-1", "video-2", "video-3"];
const audioChunks = ["audio-0", "audio-1", "audio-2", "audio-3"];
let streams;
const openStreams = () => {
if (!streams) {
streams = createMultiplexStreams(sourceURL(ctx.req.url));
}
return streams;
};
const videoUrl = gopeed.runtime.blob.createObjectURL(async () => openStreams().video, {
size: videoChunks.reduce((sum, chunk) => sum + byteLength(chunk), 0),
});
const audioUrl = gopeed.runtime.blob.createObjectURL(async () => openStreams().audio, {
size: audioChunks.reduce((sum, chunk) => sum + byteLength(chunk), 0),
});
ctx.res = {
name: "multiplex-fetch-stream",
files: [
{
name: "video.bin",
size: videoChunks.reduce((sum, chunk) => sum + byteLength(chunk), 0),
req: { url: videoUrl },
},
{
name: "audio.bin",
size: audioChunks.reduce((sum, chunk) => sum + byteLength(chunk), 0),
req: { url: audioUrl },
},
],
};
}
});
pkg/download/testdata/extensions/blob_stream_bridge/manifest.json
Vendored
+18
View File
@@ -0,0 +1,18 @@
{
"name": "blob-stream-bridge",
"title": "gopeed extension blob stream bridge test",
"version": "0.0.1",
"scripts": [
{
"event": "onResolve",
"match": {
"urls": [
"*://*/single-fetch-stream*",
"*://*/range-fetch-stream*",
"*://*/multiplex-fetch-stream*"
]
},
"entry": "index.js"
}
]
}
pkg/download/testdata/extensions/gblob/index.js
Vendored
-183
View File
@@ -1,183 +0,0 @@
gopeed.events.onResolve(async function (ctx) {
const encoder = new TextEncoder();
if (ctx.req.url.endsWith("/blob")) {
const blob = new Blob(["hello world"], {type: "text/plain"});
const url = URL.createObjectURL(blob);
ctx.res = {
name: "gblob-blob",
files: [
{
name: "hello.txt",
size: 11,
req: {
url,
}
}
]
};
return;
}
if (ctx.req.url.includes("/http-stream")) {
const source = new URL(ctx.req.url);
const target = source.searchParams.get("target");
const pair = source.searchParams.get("pair") === "1";
const resumable = ctx.req.url.includes("/http-stream-range");
const fileName = source.searchParams.get("name") || "http-stream.bin";
const headResponse = await fetch(target, {method: "HEAD"});
const headLength = parseInt(headResponse.headers.get("content-length") || "", 10);
const createOpen = () => async (offset) => {
const headers = {};
if (offset > 0) {
headers.Range = `bytes=${offset}-`;
}
const response = await fetch(target, {headers});
if (!response.body) {
throw new Error("empty response body");
}
return response.body;
};
ctx.res = {
name: "gblob-http-stream",
range: resumable,
files: pair ? [
{
name: "pair-video.bin",
size: Number.isFinite(headLength) ? headLength : 0,
req: {
url: URL.createObjectURL(createOpen()),
}
},
{
name: "pair-audio.bin",
size: Number.isFinite(headLength) ? headLength : 0,
req: {
url: URL.createObjectURL(createOpen()),
}
}
] : [
{
name: fileName,
size: Number.isFinite(headLength) ? headLength : 0,
req: {
url: URL.createObjectURL(createOpen()),
}
}
]
};
return;
}
if (ctx.req.url.includes("/http-blob")) {
const source = new URL(ctx.req.url);
const target = source.searchParams.get("target");
const fileName = source.searchParams.get("name") || "http-blob.bin";
const headResponse = await fetch(target, {method: "HEAD"});
const headLength = parseInt(headResponse.headers.get("content-length") || "", 10);
const response = await fetch(target);
const blob = await response.blob();
ctx.res = {
name: "gblob-http-blob",
files: [
{
name: fileName,
size: Number.isFinite(headLength) ? headLength : 0,
req: {
url: URL.createObjectURL(blob),
}
}
]
};
return;
}
if (ctx.req.url.endsWith("/stream-range")) {
const payload = encoder.encode("line 1\nline 2\nline 3\n");
const firstChunkLength = encoder.encode("line 1\n").byteLength;
let firstAttempt = true;
const open = async (offset) => new ReadableStream({
async start(controller) {
try {
if (offset === 0 && firstAttempt) {
firstAttempt = false;
controller.enqueue(payload.slice(0, firstChunkLength));
await new Promise((resolve) => setTimeout(resolve, 120));
controller.error(new Error("resume required"));
return;
}
controller.enqueue(payload.slice(offset));
controller.close();
} catch (err) {
controller.error(err);
}
}
});
const url = URL.createObjectURL(open);
ctx.res = {
name: "gblob-stream-range",
range: true,
files: [
{
name: "stream-range.txt",
size: payload.byteLength,
req: {
url,
}
}
]
};
return;
}
if (ctx.req.url.endsWith("/stream-unknown")) {
const url = URL.createObjectURL(new ReadableStream({
async start(controller) {
controller.enqueue(encoder.encode("line 1\n"));
await new Promise((resolve) => setTimeout(resolve, 350));
controller.enqueue(encoder.encode("line 2\n"));
controller.close();
},
}));
ctx.res = {
name: "gblob-stream-unknown",
files: [
{
name: "stream-unknown.txt",
req: {
url,
}
}
]
};
return;
}
const url = URL.createObjectURL(new ReadableStream({
start(controller) {
controller.enqueue(encoder.encode("line 1\n"));
controller.enqueue(encoder.encode("line 2\n"));
controller.close();
},
}));
ctx.res = {
name: "gblob-stream",
files: [
{
name: "stream.txt",
size: 14,
req: {
url,
}
}
]
};
});
pkg/download/testdata/extensions/gblob_recover/index.js
Vendored
-40
View File
@@ -1,40 +0,0 @@
gopeed.events.onResolve(async function (ctx) {
if (!ctx.req.url.endsWith("/recover")) {
return;
}
const url = URL.createObjectURL(new ReadableStream({
async start(controller) {
controller.enqueue(new TextEncoder().encode("stale\n"));
await new Promise((resolve) => setTimeout(resolve, 50));
controller.error(new Error("expired"));
},
}));
ctx.res = {
name: "gblob-recover",
files: [
{
name: "recover.txt",
req: {
url,
}
}
]
};
});
gopeed.events.onError(async function (ctx) {
const req = ctx.task.meta.req;
if (!req.rawUrl || !req.rawUrl.endsWith("/recover")) {
return;
}
req.labels = req.labels || {};
if (req.labels.recovered === "true") {
return;
}
req.labels.recovered = "true";
req.url = URL.createObjectURL(new Blob(["recovered\n"], { type: "text/plain" }));
ctx.task.continue();
});
pkg/rest/server_test.go
+20 -4
View File
@@ -67,6 +67,10 @@ var (
}
)
func currentTestDownloadFile() string {
return filepath.Join(createOpts.Path, createOpts.Name)
}
func TestInfo(t *testing.T) {
matchKeys := []string{"version", "runtime", "os", "arch", "inDocker"}
doTest(func() {
@@ -109,7 +113,7 @@ func TestCreateTask(t *testing.T) {
wg.Wait()
want := test.FileMd5(test.BuildFile)
got := test.FileMd5(test.DownloadFile)
got := test.FileMd5(currentTestDownloadFile())
if want != got {
t.Errorf("CreateTask() got = %v, want %v", got, want)
}
@@ -133,7 +137,7 @@ func TestCreateDirectTask(t *testing.T) {
wg.Wait()
want := test.FileMd5(test.BuildFile)
got := test.FileMd5(test.DownloadFile)
got := test.FileMd5(currentTestDownloadFile())
if want != got {
t.Errorf("CreateDirectTask() got = %v, want %v", got, want)
}
@@ -226,7 +230,7 @@ func TestPauseAndContinueTask(t *testing.T) {
wg.Wait()
want := test.FileMd5(test.BuildFile)
got := test.FileMd5(test.DownloadFile)
got := test.FileMd5(currentTestDownloadFile())
if !reflect.DeepEqual(got, want) {
t.Errorf("PauseAndContinueTask() got = %v, want %v", got, want)
}
@@ -344,7 +348,7 @@ func TestDeleteTaskForce(t *testing.T) {
httpRequestCheckOk[any](http.MethodDelete, "/api/v1/tasks/"+taskId+"?force=true", nil)
code, _ := httpRequest[*download.Task](http.MethodGet, "/api/v1/tasks/"+taskId, nil)
checkCode(code, model.CodeTaskNotFound)
if _, err := os.Stat(test.DownloadFile); !errors.Is(err, os.ErrNotExist) {
if _, err := os.Stat(currentTestDownloadFile()); !errors.Is(err, os.ErrNotExist) {
t.Errorf("DeleteTaskForce() got = %v, want %v", err, os.ErrNotExist)
}
})
@@ -935,10 +939,22 @@ func doTest(handler func()) {
func doTest0(onStart func(cfg *model.StartConfig), handler func()) {
testFunc := func(storage model.Storage) {
downloadDir, err := os.MkdirTemp("", "gopeed-rest-test-download-")
if err != nil {
panic(err)
}
defer os.RemoveAll(downloadDir)
oldCreatePath := createOpts.Path
createOpts.Path = downloadDir
defer func() {
createOpts.Path = oldCreatePath
}()
var cfg = &model.StartConfig{}
cfg.Init()
cfg.Storage = storage
cfg.StorageDir = ".test_storage"
cfg.DownloadConfig = (&base.DownloaderStoreConfig{DownloadDir: downloadDir}).Init()
cfg.WebEnable = true
if onStart != nil {
onStart(cfg)
ui/flutter/lib/api/model/task.dart
+1 -1
View File
@@ -6,7 +6,7 @@ part 'task.g.dart';
enum Status { ready, running, pause, wait, error, done }
enum Protocol { http, bt, ed2k, gblob }
enum Protocol { http, bt, ed2k }
// ExtractStatus enum matching Go backend
enum ExtractStatus {
ui/flutter/lib/api/model/task.g.dart
-1
View File
@@ -52,7 +52,6 @@ const _$ProtocolEnumMap = {
Protocol.http: 'http',
Protocol.bt: 'bt',
Protocol.ed2k: 'ed2k',
Protocol.gblob: 'gblob',
};
Progress _$ProgressFromJson(Map<String, dynamic> json) => Progress(