nprasad2077/gopeed
1
0 Fork 0
mirror of https://github.com/GopeedLab/gopeed.git synced 2026-11-08 17:40:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor(downloader): remove opts parameter from Resolve (#1595)

Browse Source
This commit is contained in:
Levi authored and GitHub committed 2026-10-19 21:40:57 +08:00
1 parent 5462b14705
commit de65149b29
45 files changed
+1589 -384
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

internal/protocol/bt/download_test.go
+448
View File
@@ -0,0 +1,448 @@
package bt
import (
"encoding/json"
"net/url"
"os"
"path/filepath"
"strings"
"sync/atomic"
"testing"
"time"
"github.com/GopeedLab/gopeed/internal/controller"
"github.com/GopeedLab/gopeed/internal/fetcher"
"github.com/GopeedLab/gopeed/pkg/base"
btprotocol "github.com/GopeedLab/gopeed/pkg/protocol/bt"
"github.com/anacrolix/torrent"
pp "github.com/anacrolix/torrent/peer_protocol"
"golang.org/x/time/rate"
)
type btDownloadHarness struct {
*Fetcher
pendingRequests atomic.Int64
}
// These tests exercise the BT fetcher, native storage and real peer protocol.
// All content is generated locally and served by a loopback seeder.
func btDownloadFetcher(t *testing.T, f fetcher.Fetcher) *btDownloadHarness {
t.Helper()
if f == nil {
f = new(FetcherManager).Build()
}
ctl := controller.NewController()
ctl.GetConfig = func(v any) {
encoded, err := json.Marshal(new(FetcherManager).DefaultConfig())
if err != nil {
t.Fatal(err)
}
if err := json.Unmarshal(encoded, v); err != nil {
t.Fatal(err)
}
}
f.Setup(ctl)
t.Cleanup(func() {
if err := f.Close(); err != nil {
t.Error(err)
}
if err := closeClient(); err != nil {
t.Error(err)
}
})
h := &btDownloadHarness{Fetcher: f.(*Fetcher)}
// Keep the test client on loopback and observe real cancellation replies.
// Waiting for them avoids the pinned library's immediate cancel/resume
// request scheduling bug, without relying on arbitrary sleep durations.
cfg = torrent.TestingConfig(t)
cfg.DisableTrackers = false
cfg.Seed = true
cfg.DisableUTP = true
cfg.DisableIPv6 = true
cfg.DisableWebtorrent = true
cfg.KeepAliveTimeout = time.Minute
cfg.MaxAllocPeerRequestDataPerConn = 1 << 20
cfg.Callbacks.SentRequest = []func(torrent.PeerRequestEvent){func(torrent.PeerRequestEvent) {
h.pendingRequests.Add(1)
}}
cfg.Callbacks.ReadMessage = func(_ *torrent.PeerConn, msg *pp.Message) {
if !msg.Keepalive && (msg.Type == pp.Piece || msg.Type == pp.Reject) {
h.pendingRequests.Add(-1)
}
}
var err error
client, err = torrent.NewClient(cfg)
if err != nil {
t.Fatal(err)
}
return h
}
func btResolve(t *testing.T, f *btDownloadHarness, source string) {
t.Helper()
if err := f.Resolve(&base.Request{URL: source}); err != nil {
t.Fatal(err)
}
}
func btStart(t *testing.T, f *btDownloadHarness) {
t.Helper()
if err := f.Start(); err != nil {
t.Fatal(err)
}
}
func btAwaitDone(t *testing.T, f *btDownloadHarness) {
t.Helper()
done := make(chan error, 1)
go func() { done <- f.Wait() }()
timeout := time.NewTimer(20 * time.Second)
defer timeout.Stop()
select {
case err := <-done:
if err != nil {
t.Fatalf("BT download failed: %v", err)
}
if !f.isDone() {
t.Fatal("Wait returned before selected files were verified")
}
case <-timeout.C:
var status strings.Builder
client.WriteStatus(&status)
t.Log(status.String())
t.Fatalf("BT download did not finish: progress=%v stats=%+v pieces=%+v peers=%+v", f.Progress(), f.torrent.Stats(), f.torrent.PieceStateRuns(), f.Stats().Runtime)
}
}
func btAwaitPartial(t *testing.T, f *btDownloadHarness) {
t.Helper()
timeout := time.NewTimer(15 * time.Second)
defer timeout.Stop()
ticker := time.NewTicker(20 * time.Millisecond)
defer ticker.Stop()
for {
select {
case <-ticker.C:
progress := f.Progress().TotalDownloaded()
if progress >= 32*1024 && progress < f.Meta().Res.Size {
return
}
case <-timeout.C:
t.Fatal("BT transfer did not reach partial progress")
}
}
}
func btPause(t *testing.T, f *btDownloadHarness) {
t.Helper()
if err := f.Pause(); err != nil {
t.Fatal(err)
}
timeout := time.NewTimer(5 * time.Second)
defer timeout.Stop()
ticker := time.NewTicker(10 * time.Millisecond)
defer ticker.Stop()
for {
if f.pendingRequests.Load() == 0 {
// The message callback runs under the client lock. Acquiring that
// lock ensures the final cancellation reply has finished processing.
client.Stats()
return
}
select {
case <-ticker.C:
case <-timeout.C:
t.Fatal("peer did not acknowledge cancelled requests")
}
}
}
func TestBTDownloadSources(t *testing.T) {
for _, kind := range []string{"magnet", "torrent", "file URI", "data URI"} {
t.Run(kind, func(t *testing.T) {
fixture := newBTFixture(t, map[string][]byte{"": btPayload(96*1024, kind)}, 0)
source := fixture.Magnet
switch kind {
case "torrent":
source = fixture.TorrentFile
case "file URI":
source = fixture.FileURI()
case "data URI":
source = fixture.DataURI
}
f := btDownloadFetcher(t, nil)
btResolve(t, f, source)
if f.Meta().Opts != nil {
t.Fatal("Resolve assigned download options")
}
f.storage.mu.RLock()
bound := f.storage.owner != nil
f.storage.mu.RUnlock()
if bound {
t.Fatal("Resolve bound download storage")
}
resolvedTorrent := f.torrent
dir := filepath.Join(t.TempDir(), "chosen-after-resolve")
f.Meta().Opts = &base.Options{Path: dir}
if _, err := os.Stat(dir); !os.IsNotExist(err) {
t.Fatalf("output directory exists before Start: %v", err)
}
btStart(t, f)
if f.torrent != resolvedTorrent {
t.Fatal("Start replaced the resolved torrent")
}
btAwaitDone(t, f)
fixture.AssertFiles(t, dir)
})
}
}
func TestBTDirectDownloadWithExtraTrackers(t *testing.T) {
for _, source := range []string{"request", "config"} {
t.Run(source, func(t *testing.T) {
fixture := makeBTFixture(t, map[string][]byte{
"a.bin": btPayload(20*1024, "direct-a"),
"nested/b.bin": btPayload(48*1024, "direct-b"),
}, 0, false, false)
magnet, err := url.Parse(fixture.Magnet)
if err != nil {
t.Fatal(err)
}
query := magnet.Query()
query.Del("tr")
query.Del("x.pe")
magnet.RawQuery = query.Encode()
req := &base.Request{URL: magnet.String()}
f := btDownloadFetcher(t, nil)
if source == "request" {
req.Extra = &btprotocol.ReqExtra{Trackers: []string{fixture.TrackerURL}}
} else {
f.config.Trackers = []string{fixture.TrackerURL}
}
// CreateDirect supplies options before Downloader resolves the task.
dir := filepath.Join(t.TempDir(), "direct-download")
opts := &base.Options{Path: dir, SelectFiles: []int{}}
f.Meta().Opts = opts
resolved := make(chan error, 1)
go func() { resolved <- f.Resolve(req) }()
select {
case err := <-resolved:
if err != nil {
t.Fatal(err)
}
case <-time.After(5 * time.Second):
f.torrentDropFunc()
<-resolved
t.Fatal("direct magnet download did not obtain metadata from its extra tracker")
}
if f.Meta().Opts != opts || len(opts.SelectFiles) != len(fixture.Files) {
t.Fatal("resolution lost direct download options or did not select all files")
}
btStart(t, f)
btAwaitDone(t, f)
fixture.AssertFiles(t, dir)
})
}
}
func TestBTDownloadSelectedFiles(t *testing.T) {
fixture := newBTFixture(t, map[string][]byte{
"a.bin": btPayload(20*1024, "a"), "nested/b.bin": btPayload(48*1024, "b"),
}, 0)
f := btDownloadFetcher(t, nil)
dir := t.TempDir()
btResolve(t, f, fixture.Magnet)
selected := -1
for i, file := range f.Meta().Res.Files {
if file.Name == "b.bin" {
selected = i
}
}
if selected < 0 {
t.Fatal("selected file absent from metadata")
}
f.Meta().Opts = &base.Options{Path: dir}
if err := f.Patch(nil, &base.Options{SelectFiles: []int{selected}}); err != nil {
t.Fatal(err)
}
btStart(t, f)
btAwaitDone(t, f)
fixture.AssertFiles(t, dir, filepath.Join("bundle", "nested", "b.bin"))
if _, err := os.Stat(filepath.Join(dir, "bundle", "a.bin")); !os.IsNotExist(err) {
t.Fatalf("unselected output remains: %v", err)
}
if f.Meta().Res.Size != 48*1024 {
t.Fatal("selected resource size is incorrect")
}
}
func TestBTPauseAndResume(t *testing.T) {
fixture := newBTFixture(t, map[string][]byte{"": btPayload(512*1024, "pause")}, 64*1024)
f := btDownloadFetcher(t, nil)
dir := t.TempDir()
btResolve(t, f, fixture.Magnet)
f.Meta().Opts = &base.Options{Path: dir}
btStart(t, f)
btAwaitPartial(t, f)
btPause(t, f)
resolvedTorrent := f.torrent
fixture.Limiter.SetLimit(rate.Inf)
// Once cancelled requests have settled, no more data should arrive.
stats := f.torrent.Stats()
readWhilePaused := stats.BytesReadData.Int64()
time.Sleep(200 * time.Millisecond)
stats = f.torrent.Stats()
if stats.BytesReadData.Int64() != readWhilePaused || f.isDone() {
t.Fatal("data download continued while paused")
}
btStart(t, f)
if f.torrent != resolvedTorrent {
t.Fatal("resume replaced the resolved torrent")
}
btAwaitDone(t, f)
fixture.AssertFiles(t, dir)
}
type btFetcherCheckpoint struct {
Meta *fetcher.FetcherMeta `json:"meta"`
Data json.RawMessage `json:"data"`
}
func btStore(t *testing.T, f *btDownloadHarness) []byte {
t.Helper()
data, err := new(FetcherManager).Store(f.Fetcher)
if err != nil {
t.Fatal(err)
}
encodedData, err := json.Marshal(data)
if err != nil {
t.Fatal(err)
}
encoded, err := json.Marshal(btFetcherCheckpoint{Meta: f.Meta(), Data: encodedData})
if err != nil {
t.Fatal(err)
}
return encoded
}
func btRestore(t *testing.T, encoded []byte) *btDownloadHarness {
t.Helper()
var checkpoint btFetcherCheckpoint
if err := json.Unmarshal(encoded, &checkpoint); err != nil {
t.Fatal(err)
}
data, restore := new(FetcherManager).Restore()
if err := json.Unmarshal(checkpoint.Data, data); err != nil {
t.Fatal(err)
}
return btDownloadFetcher(t, restore(checkpoint.Meta, data))
}
func btCloseForRestart(t *testing.T, f *btDownloadHarness) {
t.Helper()
if err := f.Close(); err != nil {
t.Fatal(err)
}
if err := new(FetcherManager).Close(); err != nil {
t.Fatal(err)
}
}
func TestBTColdStartResume(t *testing.T) {
for _, kind := range []string{"v1", "hybrid"} {
t.Run(kind, func(t *testing.T) {
var fixture *btFixture
if kind == "hybrid" {
fixture = newBTHybridFixture(t, btPayload(512*1024, "restart-hybrid"))
fixture.Limiter.SetLimit(64 * 1024)
} else {
fixture = newBTFixture(t, map[string][]byte{"": btPayload(512*1024, "restart")}, 64*1024)
}
f := btDownloadFetcher(t, nil)
dir := t.TempDir()
btResolve(t, f, fixture.Magnet)
f.Meta().Opts = &base.Options{Path: dir}
btStart(t, f)
btAwaitPartial(t, f)
btPause(t, f)
checkpoint := btStore(t, f)
hash := f.Meta().Res.Hash
btCloseForRestart(t, f)
seedStats := fixture.Torrent.Stats()
sentBeforeRestart := seedStats.BytesWrittenData.Int64()
restored := btRestore(t, checkpoint)
if restored.Meta().Opts.Path != dir || restored.Meta().Res.Hash != hash {
t.Fatal("download directory or hash was lost")
}
fixture.Limiter.SetLimit(rate.Inf)
btStart(t, restored)
btAwaitDone(t, restored)
fixture.AssertFiles(t, dir)
seedStats = fixture.Torrent.Stats()
if sent := seedStats.BytesWrittenData.Int64() - sentBeforeRestart; sent >= 512*1024 {
t.Fatalf("restart redownloaded the entire file: %d bytes", sent)
}
})
}
}
func TestBTDownloadHybrid(t *testing.T) {
fixture := newBTHybridFixture(t, btPayload(80*1024, "hybrid-v2"))
f := btDownloadFetcher(t, nil)
dir := t.TempDir()
btResolve(t, f, fixture.Magnet)
f.Meta().Opts = &base.Options{Path: dir}
btStart(t, f)
btAwaitDone(t, f)
fixture.AssertFiles(t, dir)
}
func TestBTRepairCorruptedExistingFile(t *testing.T) {
fixture := newBTFixture(t, map[string][]byte{"": btPayload(128*1024, "repair")}, 0)
dir := t.TempDir()
damaged := append([]byte(nil), fixture.Files["payload.bin"]...)
damaged[20000] ^= 0xff
if err := os.WriteFile(filepath.Join(dir, "payload.bin"), damaged, 0600); err != nil {
t.Fatal(err)
}
f := btDownloadFetcher(t, nil)
btResolve(t, f, fixture.Magnet)
f.Meta().Opts = &base.Options{Path: dir}
btStart(t, f)
btAwaitDone(t, f)
fixture.AssertFiles(t, dir)
}
func TestBTPatchSelectionWhilePaused(t *testing.T) {
fixture := newBTFixture(t, map[string][]byte{"a.bin": btPayload(256*1024, "patch-a"), "b.bin": btPayload(256*1024, "patch-b")}, 64*1024)
f := btDownloadFetcher(t, nil)
dir := t.TempDir()
btResolve(t, f, fixture.Magnet)
f.Meta().Opts = &base.Options{Path: dir, SelectFiles: []int{0}}
btStart(t, f)
btAwaitPartial(t, f)
btPause(t, f)
if err := f.Patch(nil, &base.Options{SelectFiles: []int{1}}); err != nil {
t.Fatal(err)
}
fixture.Limiter.SetLimit(rate.Inf)
btStart(t, f)
btAwaitDone(t, f)
fixture.AssertFiles(t, dir, filepath.Join("bundle", "b.bin"))
for _, suffix := range []string{"", ".part"} {
if _, err := os.Stat(filepath.Join(dir, "bundle", "a.bin") + suffix); !os.IsNotExist(err) {
t.Fatalf("unselected output remains: %v", err)
}
}
}
func TestBTDownloadMultiFileWithEmptyFile(t *testing.T) {
fixture := newBTFixture(t, map[string][]byte{"empty.txt": {}, "nested/data.bin": btPayload(48*1024, "empty-file")}, 0)
f := btDownloadFetcher(t, nil)
dir := t.TempDir()
btResolve(t, f, fixture.Magnet)
f.Meta().Opts = &base.Options{Path: dir}
btStart(t, f)
btAwaitDone(t, f)
fixture.AssertFiles(t, dir)
}
internal/protocol/bt/fetcher.go
+220 -105
View File
@@ -3,11 +3,13 @@ package bt
import (
"bytes"
"context"
"errors"
"fmt"
"io"
"net/url"
"os"
"path/filepath"
"runtime"
"strings"
"sync"
"sync/atomic"
@@ -33,6 +35,10 @@ var (
)
type Fetcher struct {
mu sync.Mutex
closeOnce sync.Once
storage *deferredFileStorage
ctl *controller.Controller
config *config
@@ -94,13 +100,17 @@ func (f *Fetcher) initClient() (err error) {
return
}
func (f *Fetcher) Resolve(req *base.Request, opts *base.Options) error {
f.meta.Req = req
f.meta.Opts = opts
if f.meta.Opts == nil {
f.meta.Opts = &base.Options{}
func (f *Fetcher) Resolve(req *base.Request) error {
f.mu.Lock()
defer f.mu.Unlock()
if f.torrent != nil {
f.safeDrop()
f.torrent = nil
f.storage = nil
f.torrentReady.Store(false)
}
if err := f.addTorrent(req, false); err != nil {
f.meta.Req = req
if err := f.addTorrent(req); err != nil {
return err
}
f.updateRes()
@@ -108,12 +118,72 @@ func (f *Fetcher) Resolve(req *base.Request, opts *base.Options) error {
}
func (f *Fetcher) Start() (err error) {
f.mu.Lock()
defer f.mu.Unlock()
if f.meta.Opts == nil || f.meta.Opts.Path == "" {
return errors.New("download directory is required when starting a torrent")
}
if err := f.torrentDropCtx.Err(); err != nil {
return err
}
if !f.torrentReady.Load() {
if err = f.addTorrent(f.meta.Req, false); err != nil {
if err = f.addTorrent(f.meta.Req); err != nil {
return
}
}
path := f.meta.Opts.Path
if err := os.MkdirAll(path, 0755); err != nil {
return err
}
hashes, err := f.storage.bind(storage.NewFileClientOpts{
ClientBaseDir: path,
FilePathMaker: func(opts storage.FilePathMakerOpts) string {
_, name := torrentFileLayout(opts.Info, *opts.File)
return name
},
})
if err != nil {
return err
}
if hashes != nil {
// File storage infers completion from final file names and sizes. Verify
// before publishing that state, or peers can see an unverified file as
// complete and disconnect before a damaged piece is discovered.
// Late v2 hashes schedule their own initial checks through the library.
changes := f.torrent.SubscribePieceStateChanges()
defer changes.Close()
for i, known := range hashes {
if !known {
continue
}
if err := f.torrentDropCtx.Err(); err != nil {
return err
}
if err := f.torrent.Piece(i).VerifyDataContext(f.torrentDropCtx); err != nil {
return err
}
// VerifyDataContext can return while the result is still being
// written to storage. Wait for it before refreshing completion.
for {
state := f.torrent.Piece(i).State()
if !state.Hashing && !state.QueuedForHash && !state.Marking {
break
}
select {
case <-changes.Values:
case <-f.torrentDropCtx.Done():
return f.torrentDropCtx.Err()
case <-f.torrent.Closed():
return errors.New("torrent closed during verification")
}
}
}
for i := range hashes {
f.torrent.Piece(i).UpdateCompletion()
}
}
files := f.torrent.Files()
// If the user does not specify the file to download, all files will be downloaded by default
if f.data.Progress == nil {
@@ -134,21 +204,29 @@ func (f *Fetcher) Start() (err error) {
}
}
f.torrent.AllowDataDownload()
go f.doUpload(false)
return
}
func (f *Fetcher) Pause() (err error) {
f.mu.Lock()
defer f.mu.Unlock()
if f.torrent == nil {
return nil
}
f.torrent.DisallowDataDownload()
return
}
func (f *Fetcher) Close() (err error) {
f.safeDrop()
f.torrentDropFunc()
f.uploadDoneCh <- nil
if len(client.Torrents()) == 0 {
err = closeClient()
}
f.closeOnce.Do(func() {
f.torrentDropFunc()
f.mu.Lock()
defer f.mu.Unlock()
f.safeDrop()
f.torrentReady.Store(false)
close(f.uploadDoneCh)
})
return nil
}
@@ -166,6 +244,8 @@ func (f *Fetcher) Meta() *fetcher.FetcherMeta {
}
func (f *Fetcher) Stats() *fetcher.Stats {
f.mu.Lock()
defer f.mu.Unlock()
var stats torrent.TorrentStats
var pieceMap *base.PieceMap
pieceMapReady := false
@@ -372,48 +452,65 @@ func normalizeBTTransport(network string) string {
}
func (f *Fetcher) Progress() fetcher.Progress {
f.mu.Lock()
defer f.mu.Unlock()
if !f.torrentReady.Load() {
return f.data.Progress
return append(fetcher.Progress(nil), f.data.Progress...)
}
for i := range f.data.Progress {
selectIndex := f.meta.Opts.SelectFiles[i]
file := f.torrent.Files()[selectIndex]
f.data.Progress[i] = file.BytesCompleted()
}
return f.data.Progress
return append(fetcher.Progress(nil), f.data.Progress...)
}
func (f *Fetcher) Wait() (err error) {
func (f *Fetcher) Wait() error {
f.mu.Lock()
var closed <-chan struct{}
if f.torrent != nil {
closed = f.torrent.Closed()
}
f.mu.Unlock()
ticker := time.NewTicker(time.Second)
defer ticker.Stop()
for {
select {
case <-f.torrentDropCtx.Done():
return
case <-time.After(time.Second):
if f.torrentReady.Load() && len(f.meta.Opts.SelectFiles) > 0 {
if f.isDone() {
// remove unselected files
for i, file := range f.torrent.Files() {
selected := false
for _, selectIndex := range f.meta.Opts.SelectFiles {
if i == selectIndex {
selected = true
break
}
}
if !selected {
root, name := torrentFileLayout(f.torrent.Info(), file.FileInfo())
if err := removeUnselectedFile(f.meta.Opts.Path, root, name); err != nil {
return err
}
}
}
return
}
return nil
case <-closed:
return nil
case <-ticker.C:
done, err := f.finishDownload()
if done || err != nil {
return err
}
}
}
}
func (f *Fetcher) finishDownload() (bool, error) {
f.mu.Lock()
defer f.mu.Unlock()
if !f.torrentReady.Load() || f.meta.Opts == nil || len(f.meta.Opts.SelectFiles) == 0 || !f.isDoneLocked() {
return false, nil
}
selected := make(map[int]bool, len(f.meta.Opts.SelectFiles))
for _, i := range f.meta.Opts.SelectFiles {
selected[i] = true
}
for i, file := range f.torrent.Files() {
if selected[i] {
continue
}
root, name := torrentFileLayout(f.torrent.Info(), file.FileInfo())
if err := removeUnselectedFile(f.meta.Opts.Path, root, name); err != nil {
return false, err
}
}
return true, nil
}
// Match the file storage layout, including its NoName convention for rootless
// torrents. File.Path() always prepends BestName, even when storage omits it.
func torrentFileLayout(info *metainfo.Info, file metainfo.FileInfo) (root, name string) {
@@ -471,13 +568,22 @@ func removeUnselectedFile(downloadDir, torrentRoot, torrentPath string) error {
}
func (f *Fetcher) isDone() bool {
if f.meta.Opts == nil {
f.mu.Lock()
defer f.mu.Unlock()
return f.isDoneLocked()
}
func (f *Fetcher) isDoneLocked() bool {
if f.meta.Opts == nil || f.torrent == nil {
return false
}
for _, selectIndex := range f.meta.Opts.SelectFiles {
file := f.torrent.Files()[selectIndex]
if file.BytesCompleted() < file.Length() {
return false
for _, index := range f.meta.Opts.SelectFiles {
file := f.torrent.Files()[index]
for i := file.BeginPieceIndex(); i < file.EndPieceIndex(); i++ {
state := f.torrent.Piece(i).State()
if !state.Ok || !state.Complete || state.Hashing || state.QueuedForHash {
return false
}
}
}
return true
@@ -486,6 +592,11 @@ func (f *Fetcher) isDone() bool {
// Patch modifies the BT task settings.
// Invalid file indices are silently ignored.
func (f *Fetcher) Patch(req *base.Request, opts *base.Options) error {
f.mu.Lock()
defer f.mu.Unlock()
if f.meta.Opts == nil {
f.meta.Opts = &base.Options{}
}
if opts == nil {
return nil
}
@@ -542,9 +653,9 @@ func (f *Fetcher) updateRes() {
Files: make([]*base.FileInfo, len(f.torrent.Files())),
Hash: f.torrent.InfoHash().String(),
}
// Directory torrent
if f.torrent.Info().Length == 0 {
res.Name = f.torrent.Name()
// Use the same root classification as storage (v2 single files have Length == 0).
if files := f.torrent.Files(); len(files) > 0 {
res.Name, _ = torrentFileLayout(f.torrent.Info(), files[0].FileInfo())
}
for i, file := range f.torrent.Files() {
res.Files[i] = &base.FileInfo{
@@ -561,61 +672,50 @@ func (f *Fetcher) updateRes() {
}
func (f *Fetcher) Upload() (err error) {
return f.addTorrent(f.meta.Req, true)
if err = f.Start(); err != nil {
return err
}
return nil
}
func (f *Fetcher) doUpload(fromUpload bool) {
if !f.torrentUpload.CompareAndSwap(false, true) {
return
}
// Check and update seed data
lastData := &fetcherData{
SeedBytes: f.data.SeedBytes,
SeedTime: f.data.SeedTime,
}
var doneTime int64 = 0
f.mu.Lock()
initialBytes, initialTime := f.data.SeedBytes, f.data.SeedTime
torrentClosed := f.torrent.Closed()
f.mu.Unlock()
var doneTime int64
ticker := time.NewTicker(time.Second)
defer ticker.Stop()
for {
select {
case <-f.torrentDropCtx.Done():
return
case <-time.After(time.Second):
case <-torrentClosed:
return
case <-ticker.C:
f.mu.Lock()
if !f.torrentReady.Load() {
f.mu.Unlock()
continue
}
stats := f.torrentStats()
f.data.SeedBytes = lastData.SeedBytes + stats.BytesWrittenData.Int64()
// Check is download complete, if not don't check and stop seeding
if !fromUpload && !f.isDone() {
f.data.SeedBytes = initialBytes + stats.BytesWrittenData.Int64()
if !fromUpload && !f.isDoneLocked() {
f.mu.Unlock()
continue
}
if doneTime == 0 {
doneTime = time.Now().Unix()
}
f.data.SeedTime = lastData.SeedTime + time.Now().Unix() - doneTime
// If the seed forever is true, keep seeding
if f.config.SeedKeep {
continue
}
// If the seed ratio is reached, stop seeding
if f.config.SeedRatio > 0 {
seedRadio := f.seedRadio()
if seedRadio >= f.config.SeedRatio {
f.Close()
break
}
}
// If the seed time is reached, stop seeding
if f.config.SeedTime > 0 {
if f.data.SeedTime >= f.config.SeedTime {
f.Close()
break
}
f.data.SeedTime = initialTime + time.Now().Unix() - doneTime
stop := !f.config.SeedKeep && ((f.config.SeedRatio > 0 && f.seedRadio() >= f.config.SeedRatio) || (f.config.SeedTime > 0 && f.data.SeedTime >= f.config.SeedTime))
f.mu.Unlock()
if stop {
f.Close()
return
}
}
}
@@ -633,6 +733,8 @@ func (f *Fetcher) torrentStats() torrent.TorrentStats {
}
func (f *Fetcher) UploadedBytes() int64 {
f.mu.Lock()
defer f.mu.Unlock()
return f.data.SeedBytes
}
@@ -641,7 +743,7 @@ func (f *Fetcher) WaitUpload() (err error) {
return nil
}
func (f *Fetcher) addTorrent(req *base.Request, fromUpload bool) (err error) {
func (f *Fetcher) addTorrent(req *base.Request) (err error) {
if err = base.ParseReqExtra[bt.ReqExtra](req); err != nil {
return
}
@@ -660,11 +762,16 @@ func (f *Fetcher) addTorrent(req *base.Request, fromUpload bool) (err error) {
var reader io.Reader
if schema == "FILE" {
fileUrl, _ := url.Parse(req.URL)
filePath := fileUrl.Path[1:]
filePath := fileUrl.Path
if runtime.GOOS == "windows" && len(filePath) > 2 && filePath[0] == '/' && filePath[2] == ':' {
filePath = filePath[1:]
}
filePath = filepath.FromSlash(filePath)
reader, err = os.Open(filePath)
if err != nil {
return
}
defer reader.(io.Closer).Close()
} else if schema == "DATA" {
_, data := util.ParseDataUri(req.URL)
reader = bytes.NewBuffer(data)
@@ -697,21 +804,6 @@ func (f *Fetcher) addTorrent(req *base.Request, fromUpload bool) (err error) {
return
}
}
spec.Storage = storage.NewFileOpts(storage.NewFileClientOpts{
ClientBaseDir: cfg.DataDir,
FilePathMaker: func(opts storage.FilePathMakerOpts) string {
_, name := torrentFileLayout(opts.Info, *opts.File)
return name
},
TorrentDirMaker: func(baseDir string, info *metainfo.Info, infoHash metainfo.Hash) string {
return f.meta.Opts.Path
},
})
f.torrent, _, err = client.AddTorrentSpec(spec)
if err != nil {
return
}
// Do not add external tracker to a private torrent.
if !privateTorrent {
// use map to deduplicate
@@ -730,17 +822,36 @@ func (f *Fetcher) addTorrent(req *base.Request, fromUpload bool) (err error) {
}
}
if len(trackers) > 0 {
announceList := make([][]string, 0)
for tracker := range trackers {
announceList = append(announceList, []string{tracker})
spec.Trackers = append(spec.Trackers, []string{tracker})
}
f.torrent.AddTrackers(announceList)
}
}
<-f.torrent.GotInfo()
s := &deferredFileStorage{}
spec.Storage = s
var added bool
f.torrent, added, err = client.AddTorrentSpec(spec)
if err != nil {
return err
}
if !added {
f.torrent = nil
return errors.New("torrent is already active")
}
f.storage = s
f.torrent.DisallowDataDownload()
// Extra trackers must be available while magnet metadata is still missing;
// they may be the only way to discover a peer that can provide it.
select {
case <-f.torrent.GotInfo():
case <-f.torrentDropCtx.Done():
f.safeDrop()
return f.torrentDropCtx.Err()
case <-f.torrent.Closed():
return errors.New("torrent closed during resolution")
}
f.torrentReady.Store(true)
go f.doUpload(fromUpload)
return
}
@@ -848,7 +959,11 @@ func (fm *FetcherManager) DefaultConfig() any {
func (fm *FetcherManager) Store(f fetcher.Fetcher) (data any, err error) {
_f := f.(*Fetcher)
return _f.data, nil
_f.mu.Lock()
defer _f.mu.Unlock()
snapshot := *_f.data
snapshot.Progress = append(fetcher.Progress(nil), snapshot.Progress...)
return &snapshot, nil
}
func (fm *FetcherManager) Restore() (v any, f func(meta *fetcher.FetcherMeta, v any) fetcher.Fetcher) {
internal/protocol/bt/fetcher_test.go
+19 -17
View File
@@ -18,11 +18,11 @@ import (
)
func TestFetcher_Resolve_Torrent(t *testing.T) {
doResolve(t, buildFetcher())
doResolve(t, buildFetcher(t))
}
func TestFetcher_Resolve_DataUri_Torrent(t *testing.T) {
fetcher := buildFetcher()
fetcher := buildFetcher(t)
buf, err := os.ReadFile("./testdata/ubuntu-22.04-live-server-amd64.iso.torrent")
if err != nil {
t.Fatal(err)
@@ -31,7 +31,7 @@ func TestFetcher_Resolve_DataUri_Torrent(t *testing.T) {
dataUri := "data:application/x-bittorrent;base64," + base64.StdEncoding.EncodeToString(buf)
err = fetcher.Resolve(&base.Request{
URL: dataUri,
}, nil)
})
if err != nil {
panic(err)
}
@@ -53,7 +53,7 @@ func TestFetcher_Resolve_DataUri_Torrent(t *testing.T) {
}
func TestFetcher_Config(t *testing.T) {
doResolve(t, buildConfigFetcher(nil))
doResolve(t, buildConfigFetcher(t, nil))
}
func TestFetcher_ResolveWithProxy(t *testing.T) {
@@ -61,7 +61,7 @@ func TestFetcher_ResolveWithProxy(t *testing.T) {
proxyListener := test.StartSocks5Server(usr, pwd)
defer proxyListener.Close()
doResolve(t, buildConfigFetcher(&base.DownloaderProxyConfig{
doResolve(t, buildConfigFetcher(t, &base.DownloaderProxyConfig{
Enable: true,
System: false,
Scheme: "socks5",
@@ -81,7 +81,7 @@ func doResolve(t *testing.T, fetcher fetcher.Fetcher) {
"udp://tracker.bitsearch.to:1337/announce",
},
},
}, nil)
})
if err != nil {
panic(err)
}
@@ -111,7 +111,7 @@ func doResolve(t *testing.T, fetcher fetcher.Fetcher) {
"udp://tracker.bitsearch.to:1337/announce",
},
},
}, nil)
})
if err != nil {
panic(err)
}
@@ -145,7 +145,7 @@ func doResolve(t *testing.T, fetcher fetcher.Fetcher) {
t.Run("Resolve Unclean Torrent", func(t *testing.T) {
err := fetcher.Resolve(&base.Request{
URL: "./testdata/test.unclean.torrent",
}, nil)
})
if err != nil {
t.Errorf("Resolve Unclean Torrent Resolve() got = %v, want nil", err)
}
@@ -156,7 +156,7 @@ func doResolve(t *testing.T, fetcher fetcher.Fetcher) {
uri := "file:///" + file
err := fetcher.Resolve(&base.Request{
URL: uri,
}, nil)
})
if err != nil {
t.Errorf("Resolve file scheme Torrent Resolve() got = %v, want nil", err)
}
@@ -211,7 +211,7 @@ func TestFetcherManager_ParseName(t *testing.T) {
}
}
func buildFetcher() fetcher.Fetcher {
func buildFetcher(t testing.TB) fetcher.Fetcher {
fb := new(FetcherManager)
fetcher := fb.Build()
newController := controller.NewController()
@@ -219,10 +219,11 @@ func buildFetcher() fetcher.Fetcher {
json.Unmarshal([]byte(test.ToJson(fb.DefaultConfig())), v)
}
fetcher.Setup(newController)
t.Cleanup(func() { fetcher.Close(); closeClient() })
return fetcher
}
func buildConfigFetcher(proxyConfig *base.DownloaderProxyConfig) fetcher.Fetcher {
func buildConfigFetcher(t testing.TB, proxyConfig *base.DownloaderProxyConfig) fetcher.Fetcher {
fetcher := new(FetcherManager).Build()
newController := controller.NewController()
mockCfg := config{
@@ -237,29 +238,30 @@ func buildConfigFetcher(proxyConfig *base.DownloaderProxyConfig) fetcher.Fetcher
return proxyConfig.ToHandler()
}
fetcher.Setup(newController)
t.Cleanup(func() { fetcher.Close(); closeClient() })
return fetcher
}
// TestFetcher_Patch tests the Patch functionality for BT fetcher.
// It tests modifying selected files after Resolve (without downloading).
func TestFetcher_Patch(t *testing.T) {
f := buildFetcher()
f := buildFetcher(t)
// Resolve a multi-file torrent
err := f.Resolve(&base.Request{
URL: "./testdata/test.torrent",
}, nil)
})
if err != nil {
t.Fatalf("Resolve failed: %v", err)
}
// Verify initial state: 3 files, all selected by default
// Resolution describes files without choosing download options.
meta := f.Meta()
if len(meta.Res.Files) != 3 {
t.Fatalf("Expected 3 files, got %d", len(meta.Res.Files))
}
if len(meta.Opts.SelectFiles) != 3 {
t.Fatalf("Expected 3 selected files, got %d", len(meta.Opts.SelectFiles))
if meta.Opts != nil {
t.Fatal("Resolve should not assign download options")
}
// Total size of all files: 107484864
@@ -395,7 +397,7 @@ func TestFetcher_PeerDiscoveryDisabledOnClient(t *testing.T) {
t.Fatal(err)
}
f := buildFetcherWithConfig(map[string]any{"disableDht": true, "disablePex": true})
if err := f.Resolve(&base.Request{URL: "./testdata/test.torrent"}, nil); err != nil {
if err := f.Resolve(&base.Request{URL: "./testdata/test.torrent"}); err != nil {
t.Fatal(err)
}
if !cfg.NoDHT || !cfg.DisablePEX {
internal/protocol/bt/fixture_test.go
+212
View File
@@ -0,0 +1,212 @@
package bt
import (
"bytes"
"context"
"encoding/base64"
"encoding/binary"
"net"
"net/http"
"net/http/httptest"
"net/url"
"os"
"path/filepath"
"strings"
"testing"
"time"
"github.com/anacrolix/torrent"
"github.com/anacrolix/torrent/bencode"
"github.com/anacrolix/torrent/merkle"
"github.com/anacrolix/torrent/metainfo"
"golang.org/x/time/rate"
)
// btFixture serves generated, trackerless content over a real loopback TCP peer.
// Its HTTP tracker exercises .torrent downloads; magnet links also carry the
// explicit x.pe address.
type btFixture struct {
MetaInfo *metainfo.MetaInfo
Magnet string
TorrentFile string
DataURI string
TrackerURL string
Files map[string][]byte // Paths relative to the selected download directory.
Seeder *torrent.Client
Torrent *torrent.Torrent
Limiter *rate.Limiter
}
func newBTFixture(t testing.TB, files map[string][]byte, bytesPerSecond int) *btFixture {
return makeBTFixture(t, files, bytesPerSecond, false, true)
}
func newBTHybridFixture(t testing.TB, payload []byte) *btFixture {
return makeBTFixture(t, map[string][]byte{"": payload}, 0, true, true)
}
func makeBTFixture(t testing.TB, files map[string][]byte, bytesPerSecond int, v2, private bool) *btFixture {
t.Helper()
source := t.TempDir()
name := "payload.bin"
if _, single := files[""]; !single {
name = "bundle"
}
root := filepath.Join(source, name)
expected := make(map[string][]byte, len(files))
for path, data := range files {
target := filepath.Join(root, filepath.FromSlash(path))
if err := os.MkdirAll(filepath.Dir(target), 0700); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(target, data, 0600); err != nil {
t.Fatal(err)
}
expected[filepath.Join(name, filepath.FromSlash(path))] = append([]byte(nil), data...)
}
info := metainfo.Info{PieceLength: 16 * 1024}
if err := info.BuildFromFilePath(root); err != nil {
t.Fatal(err)
}
var layers map[string]string
if v2 {
data := files[""]
h := merkle.NewHash()
h.Write(data)
root := h.Sum(nil)
var pieceHashes []byte
for off := 0; off < len(data); off += int(info.PieceLength) {
end := min(off+int(info.PieceLength), len(data))
ph := merkle.NewHash()
ph.Write(data[off:end])
pieceHashes = append(pieceHashes, ph.SumMinLength(nil, int(info.PieceLength))...)
}
info = metainfo.Info{Name: name, PieceLength: info.PieceLength, Length: int64(len(data)), Pieces: info.Pieces, MetaVersion: 2, FileTree: metainfo.FileTree{File: metainfo.FileTreeFile{Length: int64(len(data)), PiecesRoot: string(root)}}}
layers = map[string]string{string(root): string(pieceHashes)}
}
// Private torrents avoid public DHT discovery once metadata is available.
info.Private = &private
raw, err := bencode.Marshal(&info)
if err != nil {
t.Fatal(err)
}
mi := &metainfo.MetaInfo{InfoBytes: raw, PieceLayers: layers}
cfg := torrent.TestingConfig(t)
cfg.DataDir = source
cfg.Seed = true
cfg.DisableUTP = true
cfg.DisableIPv6 = true
cfg.DisableWebtorrent = true
cfg.KeepAliveTimeout = time.Minute
cfg.MaxAllocPeerRequestDataPerConn = 1 << 20
limiter := rate.NewLimiter(rate.Inf, 16*1024)
if bytesPerSecond > 0 {
limiter.SetLimit(rate.Limit(bytesPerSecond))
}
cfg.UploadRateLimiter = limiter
seed, err := torrent.NewClient(cfg)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { seed.Close() })
seeded, _, err := seed.AddTorrentSpec(torrent.TorrentSpecFromMetaInfo(mi))
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
if err := seeded.VerifyDataContext(ctx); err != nil {
t.Fatal(err)
}
select {
case <-seeded.Complete().On():
case <-ctx.Done():
t.Fatal("local seeder did not finish verification")
}
addr := seed.ListenAddrs()[0].String()
host, portString, err := net.SplitHostPort(addr)
if err != nil {
t.Fatal(err)
}
resolved, err := net.ResolveTCPAddr("tcp4", net.JoinHostPort(host, portString))
if err != nil {
t.Fatal(err)
}
compact := make([]byte, 6)
copy(compact, resolved.IP.To4())
binary.BigEndian.PutUint16(compact[4:], uint16(resolved.Port))
tracker := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
response, err := bencode.Marshal(map[string]any{"interval": 3600, "peers": string(compact)})
if err != nil {
http.Error(w, err.Error(), 500)
return
}
w.Write(response)
}))
t.Cleanup(tracker.Close)
// The pinned library has a hybrid tracker-dispatch assertion bug.
// Hybrid fixtures use their explicit loopback peer rather than a tracker.
if !v2 {
mi.Announce = tracker.URL
}
var encoded bytes.Buffer
if err := mi.Write(&encoded); err != nil {
t.Fatal(err)
}
torrentFile := filepath.Join(t.TempDir(), "fixture.torrent")
if err := os.WriteFile(torrentFile, encoded.Bytes(), 0600); err != nil {
t.Fatal(err)
}
m, err := mi.MagnetV2()
if err != nil {
t.Fatal(err)
}
magnet := m.String()
u, err := url.Parse(magnet)
if err != nil {
t.Fatal(err)
}
q := u.Query()
q.Add("x.pe", addr)
u.RawQuery = q.Encode()
return &btFixture{MetaInfo: mi, Magnet: u.String(), TorrentFile: torrentFile, DataURI: "data:application/x-bittorrent;base64," + base64.StdEncoding.EncodeToString(encoded.Bytes()), TrackerURL: tracker.URL, Files: expected, Seeder: seed, Torrent: seeded, Limiter: limiter}
}
func btPayload(size int, label string) []byte {
data := make([]byte, size)
for i := range data {
data[i] = byte((i*31 + i/251) % 251)
}
copy(data, label)
return data
}
func (f *btFixture) AssertFiles(t testing.TB, dir string, paths ...string) {
t.Helper()
if len(paths) == 0 {
for path := range f.Files {
paths = append(paths, path)
}
}
for _, path := range paths {
got, err := os.ReadFile(filepath.Join(dir, path))
if err != nil {
t.Fatal(err)
}
want, ok := f.Files[path]
if !ok {
t.Fatalf("unknown fixture file %q", path)
}
if !bytes.Equal(got, want) {
t.Fatalf("BT file %s differs: received %d bytes, want %d", path, len(got), len(want))
}
}
}
func (f *btFixture) FileURI() string {
path := filepath.ToSlash(f.TorrentFile)
if !strings.HasPrefix(path, "/") {
path = "/" + path
}
return (&url.URL{Scheme: "file", Path: path}).String()
}
internal/protocol/bt/selection_test.go
+70 -10
View File
@@ -17,25 +17,85 @@ import (
"github.com/anacrolix/torrent/storage"
)
func TestStartUsesSelectionSubmittedAfterResolve(t *testing.T) {
config := torrent.TestingConfig(t)
config.DataDir = t.TempDir()
config.DisableTCP = true
config.DisableUTP = true
c, err := torrent.NewClient(config)
func TestStartDoesNotPublishUnverifiedCompletion(t *testing.T) {
cfg := torrent.TestingConfig(t)
cfg.DisableTCP = true
cfg.DisableUTP = true
c, err := torrent.NewClient(cfg)
if err != nil {
t.Fatal(err)
}
defer c.Close()
tor, err := c.AddTorrentFromFile("testdata/test.torrent")
payload := btPayload(32*1024, "unverified")
info := metainfo.Info{Name: "payload.bin", Length: int64(len(payload)), PieceLength: 16 * 1024}
for offset := 0; offset < len(payload); offset += int(info.PieceLength) {
hash := sha1.Sum(payload[offset : offset+int(info.PieceLength)])
info.Pieces = append(info.Pieces, hash[:]...)
}
raw, err := bencode.Marshal(info)
if err != nil {
t.Fatal(err)
}
f := &Fetcher{torrent: tor, meta: &fetcher.FetcherMeta{Opts: &base.Options{}}, data: &fetcherData{}}
store := &deferredFileStorage{}
spec := torrent.TorrentSpecFromMetaInfo(&metainfo.MetaInfo{InfoBytes: raw})
spec.Storage = store
tor, _, err := c.AddTorrentSpec(spec)
if err != nil {
t.Fatal(err)
}
tor.DisallowDataDownload()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
f := &Fetcher{torrent: tor, storage: store, meta: &fetcher.FetcherMeta{}, data: &fetcherData{}, torrentDropCtx: ctx, torrentDropFunc: cancel, config: &config{SeedKeep: true}}
f.torrentReady.Store(true)
f.updateRes()
// Resolve initializes all files; creation replaces the selection before Start.
f.meta.Opts = &base.Options{SelectFiles: []int{2}}
dir := t.TempDir()
payload[20000] ^= 0xff
if err := os.WriteFile(filepath.Join(dir, info.Name), payload, 0600); err != nil {
t.Fatal(err)
}
f.meta.Opts = &base.Options{Path: dir}
// Retain the original event channel to detect even a transient completion.
// With no network transports, the damaged piece cannot have been repaired.
completed := tor.Complete().On()
if err := f.Start(); err != nil {
t.Fatal(err)
}
select {
case <-completed:
t.Fatal("Start published completion before verifying the existing file")
default:
}
}
func TestStartUsesSelectionSubmittedAfterResolve(t *testing.T) {
cfg := torrent.TestingConfig(t)
cfg.DataDir = t.TempDir()
cfg.DisableTCP = true
cfg.DisableUTP = true
c, err := torrent.NewClient(cfg)
if err != nil {
t.Fatal(err)
}
defer c.Close()
store := &deferredFileStorage{}
mi, err := metainfo.LoadFromFile("testdata/test.torrent")
if err != nil {
t.Fatal(err)
}
spec := torrent.TorrentSpecFromMetaInfo(mi)
spec.Storage = store
tor, _, err := c.AddTorrentSpec(spec)
if err != nil {
t.Fatal(err)
}
tor.DisallowDataDownload()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
f := &Fetcher{torrent: tor, storage: store, meta: &fetcher.FetcherMeta{}, data: &fetcherData{}, torrentDropCtx: ctx, torrentDropFunc: cancel, config: &config{SeedKeep: true}}
f.torrentReady.Store(true)
f.updateRes()
f.meta.Opts = &base.Options{Path: t.TempDir(), SelectFiles: []int{2}}
if err := f.Start(); err != nil {
t.Fatal(err)
}
internal/protocol/bt/storage.go
+116
View File
@@ -0,0 +1,116 @@
package bt
import (
"context"
"errors"
"os"
"sync"
g "github.com/anacrolix/generics"
"github.com/anacrolix/torrent/metainfo"
"github.com/anacrolix/torrent/storage"
)
// deferredFileStorage lets metadata resolve without opening the user's files.
// It binds once. After binding, pieces are supplied directly by file storage.
type deferredFileStorage struct {
mu sync.RWMutex
ctx context.Context
info *metainfo.Info
hash metainfo.Hash
knownHashes []bool
owner storage.ClientImplCloser
torrent storage.TorrentImpl
closed bool
}
func (s *deferredFileStorage) OpenTorrent(ctx context.Context, info *metainfo.Info, hash metainfo.Hash) (storage.TorrentImpl, error) {
s.mu.Lock()
defer s.mu.Unlock()
if s.closed || s.info != nil {
return storage.TorrentImpl{}, errors.New("torrent storage already opened or closed")
}
s.ctx, s.info, s.hash = ctx, info, hash
s.knownHashes = make([]bool, info.NumPieces())
return storage.TorrentImpl{
PieceWithHash: func(p metainfo.Piece, hash g.Option[[]byte]) storage.PieceImpl {
s.mu.Lock()
defer s.mu.Unlock()
if s.closed {
return metadataPiece{err: os.ErrClosed}
}
if s.owner == nil {
s.knownHashes[p.Index()] = hash.Ok
// A missing v2 hash must remain unknown until its arrival changes
// completion and updates the library's piece request priorities.
return metadataPiece{known: hash.Ok}
}
if s.torrent.PieceWithHash != nil {
return s.torrent.PieceWithHash(p, hash)
}
return s.torrent.Piece(p)
},
Close: s.Close,
}, nil
}
func (s *deferredFileStorage) bind(opts storage.NewFileClientOpts) ([]bool, error) {
s.mu.Lock()
defer s.mu.Unlock()
if s.closed {
return nil, os.ErrClosed
}
if s.info == nil {
return nil, errors.New("torrent metadata is not ready")
}
if s.owner != nil {
return nil, nil
}
if opts.FilePathMaker == nil {
opts.FilePathMaker = func(o storage.FilePathMakerOpts) string { _, name := torrentFileLayout(o.Info, *o.File); return name }
}
owner := storage.NewFileOpts(opts)
t, err := owner.OpenTorrent(s.ctx, s.info, s.hash)
if err != nil {
owner.Close()
return nil, err
}
s.owner, s.torrent = owner, t
return append([]bool(nil), s.knownHashes...), nil
}
func (s *deferredFileStorage) Close() error {
s.mu.Lock()
defer s.mu.Unlock()
if s.closed {
return nil
}
s.closed = true
if s.owner == nil {
return nil
}
var err error
if s.torrent.Close != nil {
err = s.torrent.Close()
}
return errors.Join(err, s.owner.Close())
}
type metadataPiece struct {
known bool
err error
}
func (p metadataPiece) Completion() storage.Completion {
return storage.Completion{Ok: p.known, Err: p.err}
}
func (p metadataPiece) ReadAt([]byte, int64) (int, error) {
return 0, errors.New("download storage is not initialized")
}
func (p metadataPiece) WriteAt([]byte, int64) (int, error) {
return 0, errors.New("download storage is not initialized")
}
func (p metadataPiece) MarkComplete() error { return errors.New("download storage is not initialized") }
func (p metadataPiece) MarkNotComplete() error {
return errors.New("download storage is not initialized")
}
internal/protocol/ed2k/fetcher.go
+2 -5
View File
@@ -67,17 +67,14 @@ func (f *Fetcher) Setup(ctl *controller.Controller) {
f.ctl.GetConfig(&f.config)
}
func (f *Fetcher) Resolve(req *base.Request, opts *base.Options) error {
func (f *Fetcher) Resolve(req *base.Request) error {
link, err := parseLink(req.URL)
if err != nil {
return err
}
f.meta.Req = req
f.meta.Opts = opts
if f.meta.Opts == nil {
f.meta.Opts = &base.Options{}
}
f.meta.Res = buildResource(link)
return nil
}
internal/protocol/ed2k/fetcher_test.go
+2 -1
View File
@@ -13,7 +13,8 @@ func TestFetcher_Resolve(t *testing.T) {
f := (&FetcherManager{}).Build()
f.Setup(controller.NewController())
err := f.Resolve(&base.Request{URL: testLink}, &base.Options{Path: t.TempDir()})
f.Meta().Opts = &base.Options{Path: t.TempDir()}
err := f.Resolve(&base.Request{URL: testLink})
if err != nil {
t.Fatalf("Resolve() error = %v", err)
}
internal/protocol/hls/fetcher.go
+2 -6
View File
@@ -172,11 +172,11 @@ func (f *Fetcher) Meta() *fetcher.FetcherMeta {
return f.meta
}
func (f *Fetcher) Resolve(req *base.Request, opts *base.Options) (err error) {
func (f *Fetcher) Resolve(req *base.Request) (err error) {
f.mu.Lock()
defer f.mu.Unlock()
f.meta.Req = req
f.meta.Opts = opts
f.extra = &ReqExtra{}
if err = base.ParseReqExtra[ReqExtra](req); err != nil {
return
@@ -194,10 +194,6 @@ func (f *Fetcher) Resolve(req *base.Request, opts *base.Options) (err error) {
}
state.StagingID = randomStagingID()
f.state = state
// Pin the staging folder now so the very first persisted state already
// carries it: a crash right after create still resumes into the same
// folder.
f.resolveStagingDirLocked()
size := planTotalSize(state.Segments)
f.meta.Res = &base.Resource{
Range: false,
internal/protocol/hls/fetcher_test.go
+16 -8
View File
@@ -80,7 +80,8 @@ func mustResolve(t *testing.T, f *Fetcher, playlistURL, referer string) {
if referer != "" {
req.Extra = &ReqExtra{Header: map[string]string{"Referer": referer}}
}
if err := f.Resolve(req, &base.Options{Path: t.TempDir()}); err != nil {
f.Meta().Opts = &base.Options{Path: t.TempDir()}
if err := f.Resolve(req); err != nil {
t.Fatalf("resolve: %v", err)
}
}
@@ -207,7 +208,8 @@ func TestFetcherE2E_RefererEnforced(t *testing.T) {
// Without Referer the resolve itself must fail (403 on the playlist).
blocked := newTestFetcher(t, testConfig())
req := &base.Request{URL: server.URL + "/video.m3u8"}
if err := blocked.Resolve(req, &base.Options{Path: t.TempDir()}); err == nil {
blocked.Meta().Opts = &base.Options{Path: t.TempDir()}
if err := blocked.Resolve(req); err == nil {
t.Fatal("resolve without Referer should fail")
}
@@ -217,7 +219,8 @@ func TestFetcherE2E_RefererEnforced(t *testing.T) {
URL: server.URL + "/video.m3u8",
Extra: &ReqExtra{Header: map[string]string{"Referer": page}},
}
if err := allowed.Resolve(allowedReq, &base.Options{Path: t.TempDir()}); err != nil {
allowed.Meta().Opts = &base.Options{Path: t.TempDir()}
if err := allowed.Resolve(allowedReq); err != nil {
t.Fatal(err)
}
mustRun(t, allowed)
@@ -560,7 +563,8 @@ func TestFetcherE2E_StoreRestoreAuthenticated(t *testing.T) {
URL: server.URL + "/video.m3u8",
Extra: &ReqExtra{Header: map[string]string{"Referer": page}},
}
if err := f.Resolve(req, &base.Options{Path: t.TempDir()}); err != nil {
f.Meta().Opts = &base.Options{Path: t.TempDir()}
if err := f.Resolve(req); err != nil {
t.Fatal(err)
}
if err := f.Start(); err != nil {
@@ -630,7 +634,8 @@ func TestFetcherE2E_PatchPartialUpdate(t *testing.T) {
Extra: &ReqExtra{Header: map[string]string{"Referer": page}},
}
opts := &base.Options{Path: t.TempDir()}
if err := f.Resolve(req, opts); err != nil {
f.Meta().Opts = opts
if err := f.Resolve(req); err != nil {
t.Fatal(err)
}
@@ -713,7 +718,8 @@ func TestFetcherE2E_PatchWhileRunningRejected(t *testing.T) {
defer server.Close()
f := newTestFetcher(t, testConfig())
if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: t.TempDir()}); err != nil {
f.Meta().Opts = &base.Options{Path: t.TempDir()}
if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}); err != nil {
t.Fatal(err)
}
if err := f.Start(); err != nil {
@@ -767,7 +773,8 @@ func TestFetcherE2E_RetryAfterFailure(t *testing.T) {
cfg.MaxRetries = 0 // fail the run on the first bad segment
cfg.PrefetchContentLength = false
f := newTestFetcher(t, cfg)
if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: t.TempDir()}); err != nil {
f.Meta().Opts = &base.Options{Path: t.TempDir()}
if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}); err != nil {
t.Fatal(err)
}
if err := f.Start(); err != nil {
@@ -828,7 +835,8 @@ func TestFetcherE2E_StateMissingSelfHeal(t *testing.T) {
defer server.Close()
f := newTestFetcher(t, testConfig())
if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: t.TempDir()}); err != nil {
f.Meta().Opts = &base.Options{Path: t.TempDir()}
if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}); err != nil {
t.Fatal(err)
}
// Simulate the crash window where the engine persisted the task with its
internal/protocol/hls/media_validate_test.go
+6 -3
View File
@@ -189,7 +189,8 @@ func TestHLSRealMedia_TS(t *testing.T) {
defer server.Close()
f := newTestFetcher(t, testConfig())
if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: t.TempDir()}); err != nil {
f.Meta().Opts = &base.Options{Path: t.TempDir()}
if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}); err != nil {
t.Fatal(err)
}
if err := f.Start(); err != nil {
@@ -225,7 +226,8 @@ func TestHLSRealMedia_FMP4(t *testing.T) {
defer server.Close()
f := newTestFetcher(t, testConfig())
if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: t.TempDir()}); err != nil {
f.Meta().Opts = &base.Options{Path: t.TempDir()}
if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}); err != nil {
t.Fatal(err)
}
if err := f.Start(); err != nil {
@@ -274,7 +276,8 @@ func TestHLSRealMedia_AES128(t *testing.T) {
defer server.Close()
f := newTestFetcher(t, testConfig())
if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: t.TempDir()}); err != nil {
f.Meta().Opts = &base.Options{Path: t.TempDir()}
if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}); err != nil {
t.Fatal(err)
}
if f.meta.Res.Size != 0 {
internal/protocol/hls/staging_test.go
+15 -7
View File
@@ -50,15 +50,20 @@ func TestFetcherE2E_SameURLTwoTasks(t *testing.T) {
// user duplicating a task. The engine would rename the second output
// (AutoRename); mirror that here.
f1 := newTestFetcher(t, cfg)
if err := f1.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: dir}); err != nil {
f1.Meta().Opts = &base.Options{Path: dir}
if err := f1.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}); err != nil {
t.Fatal(err)
}
f2 := newTestFetcher(t, cfg)
if err := f2.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: dir, Name: "video-2.ts"}); err != nil {
f2.Meta().Opts = &base.Options{Path: dir, Name: "video-2.ts"}
if err := f2.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}); err != nil {
t.Fatal(err)
}
if f1.state.StagingDir == "" || f1.state.StagingDir == f2.state.StagingDir {
t.Fatalf("staging dirs must be distinct per task: %q vs %q", f1.state.StagingDir, f2.state.StagingDir)
if f1.state.StagingID == "" || f1.state.StagingID == f2.state.StagingID {
t.Fatal("staging identities must be distinct per task")
}
if f1.state.StagingDir != "" || f2.state.StagingDir != "" {
t.Fatal("Resolve pinned an output directory")
}
// Start task 1, let it stage some segments, then pause it.
@@ -135,7 +140,8 @@ func TestFetcherE2E_ForeignJournalDiscarded(t *testing.T) {
cfg := testConfig()
cfg.PrefetchContentLength = false
f := newTestFetcher(t, cfg)
if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: t.TempDir()}); err != nil {
f.Meta().Opts = &base.Options{Path: t.TempDir()}
if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}); err != nil {
t.Fatal(err)
}
// Plant a journal that does not belong to this plan (wrong fingerprint).
@@ -176,7 +182,8 @@ func TestFetcherE2E_MergeFailurePreservesStaging(t *testing.T) {
defer server.Close()
f := newTestFetcher(t, testConfig())
if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: t.TempDir()}); err != nil {
f.Meta().Opts = &base.Options{Path: t.TempDir()}
if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}); err != nil {
t.Fatal(err)
}
run := &hlsRun{
@@ -250,7 +257,8 @@ func TestFetcherE2E_ReMergeOverExistingOutput(t *testing.T) {
defer server.Close()
f := newTestFetcher(t, testConfig())
if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: t.TempDir()}); err != nil {
f.Meta().Opts = &base.Options{Path: t.TempDir()}
if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}); err != nil {
t.Fatal(err)
}
tempDir := f.resolveStagingDirLocked()
internal/protocol/http/fetcher.go
+4 -9
View File
@@ -372,19 +372,11 @@ func (f *Fetcher) updateMaxConnTime(d time.Duration) {
}
}
func (f *Fetcher) Resolve(req *base.Request, opts *base.Options) error {
func (f *Fetcher) Resolve(req *base.Request) error {
if err := base.ParseReqExtra[fhttp.ReqExtra](req); err != nil {
return err
}
f.meta.Req = req
f.meta.Opts = opts
if f.meta.Opts == nil {
f.meta.Opts = &base.Options{}
}
if err := f.initOptions(); err != nil {
return err
}
f.setState(stateResolving)
@@ -693,6 +685,9 @@ func (f *Fetcher) prepareFullRedownload() {
// Options may be replaced between Resolve and Start when creating a resolved task.
func (f *Fetcher) initOptions() error {
if f.meta.Opts == nil {
f.meta.Opts = &base.Options{}
}
opts := f.meta.Opts
// Parse options
if err := base.ParseOptExtra[fhttp.OptsExtra](opts); err != nil {
internal/protocol/http/fetcher_checksum_retry_test.go
+6 -3
View File
@@ -95,9 +95,10 @@ func TestFetcher_ChecksumMismatch_Retry(t *testing.T) {
}
// 1. Initial attempt -> will download corruptContent, failing checksum verification
if err := f.Resolve(&base.Request{URL: server.URL + "/retry_test.txt"}, opts); err != nil {
if err := f.Resolve(&base.Request{URL: server.URL + "/retry_test.txt"}); err != nil {
t.Fatalf("Resolve error: %v", err)
}
f.Meta().Opts = opts
if err := f.Start(); err != nil {
t.Fatalf("Start error: %v", err)
}
@@ -215,9 +216,10 @@ func TestFetcher_ChecksumMismatch_Retry_PartialPrefetch(t *testing.T) {
}
// 1. Resolve phase
if err := f.Resolve(&base.Request{URL: server.URL + "/prefetch_retry.txt"}, opts); err != nil {
if err := f.Resolve(&base.Request{URL: server.URL + "/prefetch_retry.txt"}); err != nil {
t.Fatalf("Resolve error: %v", err)
}
f.Meta().Opts = opts
// Wait until prefetch has received exactly 7 bytes
for i := 0; i < 200; i++ {
@@ -338,9 +340,10 @@ func TestFetcher_ChecksumMismatch_Retry_NonRange(t *testing.T) {
}
// 1. Resolve and start initial attempt
if err := f.Resolve(&base.Request{URL: server.URL + "/nonrange_retry.txt"}, opts); err != nil {
if err := f.Resolve(&base.Request{URL: server.URL + "/nonrange_retry.txt"}); err != nil {
t.Fatalf("Resolve error: %v", err)
}
f.Meta().Opts = opts
if err := f.Start(); err != nil {
t.Fatalf("Start error: %v", err)
}
internal/protocol/http/fetcher_checksum_test.go
+38 -15
View File
@@ -52,9 +52,11 @@ func TestFetcher_ChecksumVerification(t *testing.T) {
},
}
if err := f.Resolve(&base.Request{URL: server.URL + "/test.txt"}, opts); err != nil {
if err := f.Resolve(&base.Request{URL: server.URL + "/test.txt"}); err != nil {
t.Fatalf("Resolve error: %v", err)
}
f.Meta().Opts = opts
if err := f.Start(); err != nil {
t.Fatalf("Start error: %v", err)
}
@@ -79,9 +81,11 @@ func TestFetcher_ChecksumVerification(t *testing.T) {
},
}
if err := f.Resolve(&base.Request{URL: server.URL + "/test.txt"}, opts); err != nil {
if err := f.Resolve(&base.Request{URL: server.URL + "/test.txt"}); err != nil {
t.Fatalf("Resolve error: %v", err)
}
f.Meta().Opts = opts
if err := f.Start(); err != nil {
t.Fatalf("Start error: %v", err)
}
@@ -106,9 +110,11 @@ func TestFetcher_ChecksumVerification(t *testing.T) {
},
}
if err := f.Resolve(&base.Request{URL: server.URL + "/test.txt"}, opts); err != nil {
if err := f.Resolve(&base.Request{URL: server.URL + "/test.txt"}); err != nil {
t.Fatalf("Resolve error: %v", err)
}
f.Meta().Opts = opts
if err := f.Start(); err != nil {
t.Fatalf("Start error: %v", err)
}
@@ -137,9 +143,13 @@ func TestFetcher_ChecksumVerification(t *testing.T) {
},
}
err := f.Resolve(&base.Request{URL: server.URL + "/test.txt"}, opts)
if err := f.Resolve(&base.Request{URL: server.URL + "/test.txt"}); err != nil {
t.Fatalf("Resolve error: %v", err)
}
f.Meta().Opts = opts
err := f.Start()
if err == nil {
t.Fatal("Resolve expected error for unsupported algorithm, got nil")
t.Fatal("Start expected error for unsupported algorithm, got nil")
}
if !strings.Contains(err.Error(), "unsupported checksum algorithm") {
t.Fatalf("expected 'unsupported checksum algorithm' in error, got %v", err)
@@ -165,9 +175,11 @@ func TestFetcher_ChecksumVerification(t *testing.T) {
},
}
if err := f.Resolve(&base.Request{URL: server.URL + "/test.txt"}, opts); err != nil {
if err := f.Resolve(&base.Request{URL: server.URL + "/test.txt"}); err != nil {
t.Fatalf("Resolve error: %v", err)
}
f.Meta().Opts = opts
if err := f.Start(); err != nil {
t.Fatalf("Start error: %v", err)
}
@@ -192,9 +204,11 @@ func TestFetcher_ChecksumVerification(t *testing.T) {
},
}
if err := f.Resolve(&base.Request{URL: server.URL + "/test.txt"}, opts); err != nil {
if err := f.Resolve(&base.Request{URL: server.URL + "/test.txt"}); err != nil {
t.Fatalf("Resolve error: %v", err)
}
f.Meta().Opts = opts
if err := f.Start(); err != nil {
t.Fatalf("Start error: %v", err)
}
@@ -235,9 +249,11 @@ func TestFetcher_ChecksumVerification(t *testing.T) {
},
}
if err := f.Resolve(&base.Request{URL: largeServer.URL + "/large.bin"}, opts); err != nil {
if err := f.Resolve(&base.Request{URL: largeServer.URL + "/large.bin"}); err != nil {
t.Fatalf("Resolve error: %v", err)
}
f.Meta().Opts = opts
if err := f.Start(); err != nil {
t.Fatalf("Start error: %v", err)
}
@@ -258,9 +274,11 @@ func TestFetcher_ChecksumVerification(t *testing.T) {
},
}
if err := f.Resolve(&base.Request{URL: server.URL + "/test.txt"}, opts); err != nil {
if err := f.Resolve(&base.Request{URL: server.URL + "/test.txt"}); err != nil {
t.Fatalf("Resolve error: %v", err)
}
f.Meta().Opts = opts
if err := f.Start(); err != nil {
t.Fatalf("Start error: %v", err)
}
@@ -269,7 +287,7 @@ func TestFetcher_ChecksumVerification(t *testing.T) {
}
})
t.Run("Invalid Checksum In Resolve Makes Zero Requests", func(t *testing.T) {
t.Run("Invalid Checksum In Start Makes No Additional Requests", func(t *testing.T) {
var reqReceived atomic.Int32
customServer := httptest.NewServer(gohttp.HandlerFunc(func(w gohttp.ResponseWriter, r *gohttp.Request) {
reqReceived.Add(1)
@@ -293,12 +311,17 @@ func TestFetcher_ChecksumVerification(t *testing.T) {
},
}
err := f.Resolve(&base.Request{URL: customServer.URL + "/test.txt"}, opts)
if err == nil {
t.Fatal("Resolve expected error for invalid checksum hash, got nil")
if err := f.Resolve(&base.Request{URL: customServer.URL + "/test.txt"}); err != nil {
t.Fatalf("Resolve error: %v", err)
}
if reqReceived.Load() != 0 {
t.Fatalf("expected 0 requests to server on invalid checksum in Resolve, got %d", reqReceived.Load())
requestsAfterResolve := reqReceived.Load()
f.Meta().Opts = opts
err := f.Start()
if err == nil {
t.Fatal("Start expected error for invalid checksum hash, got nil")
}
if got := reqReceived.Load(); got != requestsAfterResolve {
t.Fatalf("invalid checksum in Start issued additional requests: got %d, had %d after Resolve", got, requestsAfterResolve)
}
})
}
internal/protocol/http/fetcher_test.go
+103 -70
View File
@@ -233,10 +233,11 @@ func TestFetcher_ResolveFilenameFromContentType(t *testing.T) {
defer fetcher.Pause()
const opaqueName = "OIP-C.u5nEXTXND3PJF0gNTgw5YgHaFP"
err := fetcher.Resolve(&base.Request{URL: server.URL + "/" + opaqueName}, &base.Options{
fetcher.Meta().Opts = &base.Options{
Name: test.DownloadName,
Path: test.Dir,
})
}
err := fetcher.Resolve(&base.Request{URL: server.URL + "/" + opaqueName})
if err != nil {
t.Fatal(err)
}
@@ -251,6 +252,10 @@ func TestFetcher_ResolveWithHostHeader(t *testing.T) {
defer listener.Close()
fetcher := buildFetcher()
fetcher.Meta().Opts = &base.Options{
Name: test.DownloadName,
Path: test.Dir,
}
err := fetcher.Resolve(&base.Request{
URL: "http://" + listener.Addr().String() + "/",
Extra: &http.ReqExtra{
@@ -258,9 +263,6 @@ func TestFetcher_ResolveWithHostHeader(t *testing.T) {
"Host": "test",
},
},
}, &base.Options{
Name: test.DownloadName,
Path: test.Dir,
})
// The server should return 400 for invalid Host header
if err == nil || !strings.Contains(err.Error(), "400") {
@@ -274,6 +276,10 @@ func TestFetcher_ResolveWithInvalidHeader(t *testing.T) {
fetcher := buildFetcher()
defer fetcher.Pause() // Close the resolve response to allow server shutdown
fetcher.Meta().Opts = &base.Options{
Name: test.DownloadName,
Path: test.Dir,
}
err := fetcher.Resolve(&base.Request{
URL: "http://" + listener.Addr().String() + "/",
Extra: &http.ReqExtra{
@@ -281,9 +287,6 @@ func TestFetcher_ResolveWithInvalidHeader(t *testing.T) {
"Referer": "\rtest",
},
},
}, &base.Options{
Name: test.DownloadName,
Path: test.Dir,
})
// Invalid header with \r should be sanitized by Go's http client, allowing the request to succeed
if err != nil {
@@ -303,12 +306,13 @@ func TestFetcher_ResolveAutomaticallyFallsBackToBrowserFingerprint(t *testing.T)
fetcher := buildFetcher()
defer fetcher.Pause()
fetcher.Meta().Opts = &base.Options{
Name: test.DownloadName,
Path: test.Dir,
}
err := fetcher.Resolve(&base.Request{
URL: server.URL + "/file.bin",
SkipVerifyCert: true,
}, &base.Options{
Name: test.DownloadName,
Path: test.Dir,
})
if err != nil {
t.Fatalf("Resolve() error = %v", err)
@@ -369,11 +373,12 @@ func testResolve(startTestServer func() net.Listener, path string, t *testing.T,
defer listener.Close()
fetcher := buildFetcher()
defer fetcher.Pause() // Close the resolve response to allow server shutdown
err := fetcher.Resolve(&base.Request{
URL: "http://" + listener.Addr().String() + "/" + path,
}, &base.Options{
fetcher.Meta().Opts = &base.Options{
Name: test.DownloadName,
Path: test.Dir,
}
err := fetcher.Resolve(&base.Request{
URL: "http://" + listener.Addr().String() + "/" + path,
})
want, err := wantFn(err)
if err != nil {
@@ -425,13 +430,14 @@ func TestFetcher_PauseBeforeStartDiscardsPrefetchProgress(t *testing.T) {
defer server.Close()
fetcher := buildFetcher()
if err := fetcher.Resolve(&base.Request{URL: server.URL + "/" + test.BuildName}, &base.Options{
fetcher.Meta().Opts = &base.Options{
Path: t.TempDir(),
Name: test.DownloadName,
Extra: &http.OptsExtra{
Connections: 2,
},
}); err != nil {
}
if err := fetcher.Resolve(&base.Request{URL: server.URL + "/" + test.BuildName}); err != nil {
t.Fatal(err)
}
select {
@@ -772,13 +778,14 @@ func TestFetcher_SequentialRetryReprobesRange(t *testing.T) {
defer server.Close()
f := buildFetcher()
if err := f.Resolve(&base.Request{URL: server.URL + "/dynamic-range.data"}, &base.Options{
f.Meta().Opts = &base.Options{
Path: t.TempDir(),
Name: "dynamic-range.data",
Extra: &http.OptsExtra{
Connections: 4,
},
}); err != nil {
}
if err := f.Resolve(&base.Request{URL: server.URL + "/dynamic-range.data"}); err != nil {
t.Fatal(err)
}
if err := f.Start(); err != nil {
@@ -884,13 +891,14 @@ func TestFetcher_InitialIgnoredRangeUsesResponseSize(t *testing.T) {
defer releaseResolve()
f := buildFetcher()
if err := f.Resolve(&base.Request{URL: server.URL + "/ignored-range.data"}, &base.Options{
f.Meta().Opts = &base.Options{
Path: t.TempDir(),
Name: "ignored-range.data",
Extra: &http.OptsExtra{
Connections: 4,
},
}); err != nil {
}
if err := f.Resolve(&base.Request{URL: server.URL + "/ignored-range.data"}); err != nil {
t.Fatal(err)
}
if err := f.Start(); err != nil {
@@ -981,13 +989,14 @@ func TestFetcher_InterruptedChunkedReplacementRestartsFromZero(t *testing.T) {
defer server.Close()
f := buildFetcher()
if err := f.Resolve(&base.Request{URL: server.URL + "/chunked-replacement.data"}, &base.Options{
f.Meta().Opts = &base.Options{
Path: t.TempDir(),
Name: "chunked-replacement.data",
Extra: &http.OptsExtra{
Connections: 4,
},
}); err != nil {
}
if err := f.Resolve(&base.Request{URL: server.URL + "/chunked-replacement.data"}); err != nil {
t.Fatal(err)
}
if err := f.Start(); err != nil {
@@ -1167,13 +1176,14 @@ func TestFetcher_UsesResolveResponseWhenRangeTakeoverFails(t *testing.T) {
defer server.Close()
f := buildFetcher()
if err := f.Resolve(&base.Request{URL: server.URL + "/resolve-fallback.data"}, &base.Options{
f.Meta().Opts = &base.Options{
Path: t.TempDir(),
Name: "resolve-fallback.data",
Extra: &http.OptsExtra{
Connections: 1,
},
}); err != nil {
}
if err := f.Resolve(&base.Request{URL: server.URL + "/resolve-fallback.data"}); err != nil {
t.Fatal(err)
}
defer f.Close()
@@ -1246,13 +1256,14 @@ func TestFetcher_ValidRangeTakesOverResolveResponse(t *testing.T) {
defer server.Close()
f := buildFetcher()
if err := f.Resolve(&base.Request{URL: server.URL + "/resolve-takeover.data"}, &base.Options{
f.Meta().Opts = &base.Options{
Path: t.TempDir(),
Name: "resolve-takeover.data",
Extra: &http.OptsExtra{
Connections: 4,
},
}); err != nil {
}
if err := f.Resolve(&base.Request{URL: server.URL + "/resolve-takeover.data"}); err != nil {
t.Fatal(err)
}
defer f.Close()
@@ -1340,13 +1351,14 @@ func TestFetcher_UnknownSizeRangeUsesOpenEndedWorker(t *testing.T) {
defer server.Close()
f := buildFetcher()
if err := f.Resolve(&base.Request{URL: server.URL + "/unknown-size-range.data"}, &base.Options{
f.Meta().Opts = &base.Options{
Path: t.TempDir(),
Name: "unknown-size-range.data",
Extra: &http.OptsExtra{
Connections: 4,
},
}); err != nil {
}
if err := f.Resolve(&base.Request{URL: server.URL + "/unknown-size-range.data"}); err != nil {
t.Fatal(err)
}
defer f.Close()
@@ -1434,13 +1446,14 @@ func TestFetcher_UnknownSizeRangeRetriesShortPartialBody(t *testing.T) {
defer server.Close()
f := buildFetcher()
if err := f.Resolve(&base.Request{URL: server.URL + "/short-range.data"}, &base.Options{
f.Meta().Opts = &base.Options{
Path: t.TempDir(),
Name: "short-range.data",
Extra: &http.OptsExtra{
Connections: 1,
},
}); err != nil {
}
if err := f.Resolve(&base.Request{URL: server.URL + "/short-range.data"}); err != nil {
t.Fatal(err)
}
defer f.Close()
@@ -1493,13 +1506,14 @@ func TestFetcher_UnknownSizeRangeFallsBackToResolve(t *testing.T) {
defer server.Close()
f := buildFetcher()
if err := f.Resolve(&base.Request{URL: server.URL + "/unknown-size-fallback.data"}, &base.Options{
f.Meta().Opts = &base.Options{
Path: t.TempDir(),
Name: "unknown-size-fallback.data",
Extra: &http.OptsExtra{
Connections: 1,
},
}); err != nil {
}
if err := f.Resolve(&base.Request{URL: server.URL + "/unknown-size-fallback.data"}); err != nil {
t.Fatal(err)
}
defer f.Close()
@@ -1580,13 +1594,14 @@ func TestFetcher_UnknownSizeRangeRemainsOpenEndedAfterPause(t *testing.T) {
defer server.Close()
f := buildFetcher()
if err := f.Resolve(&base.Request{URL: server.URL + "/pause-open-range.data"}, &base.Options{
f.Meta().Opts = &base.Options{
Path: t.TempDir(),
Name: "pause-open-range.data",
Extra: &http.OptsExtra{
Connections: 1,
},
}); err != nil {
}
if err := f.Resolve(&base.Request{URL: server.URL + "/pause-open-range.data"}); err != nil {
t.Fatal(err)
}
defer f.Close()
@@ -1635,13 +1650,14 @@ func TestFetcher_UnknownSizeResolveReadErrorFailsTask(t *testing.T) {
defer server.Close()
f := buildFetcher()
if err := f.Resolve(&base.Request{URL: server.URL + "/broken-chunked.data"}, &base.Options{
f.Meta().Opts = &base.Options{
Path: t.TempDir(),
Name: "broken-chunked.data",
Extra: &http.OptsExtra{
Connections: 1,
},
}); err != nil {
}
if err := f.Resolve(&base.Request{URL: server.URL + "/broken-chunked.data"}); err != nil {
t.Fatal(err)
}
defer f.Close()
@@ -1667,13 +1683,14 @@ func TestFetcher_UnknownSizeResolveCleanEOFUpdatesSize(t *testing.T) {
defer server.Close()
f := buildFetcher()
if err := f.Resolve(&base.Request{URL: server.URL + "/unknown-size.data"}, &base.Options{
f.Meta().Opts = &base.Options{
Path: t.TempDir(),
Name: "unknown-size.data",
Extra: &http.OptsExtra{
Connections: 1,
},
}); err != nil {
}
if err := f.Resolve(&base.Request{URL: server.URL + "/unknown-size.data"}); err != nil {
t.Fatal(err)
}
defer f.Close()
@@ -2269,14 +2286,15 @@ func TestFetcher_DownloadOneTimeURL(t *testing.T) {
defer listener.Close()
fetcher := buildFetcher()
err := fetcher.Resolve(&base.Request{
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
}, &base.Options{
fetcher.Meta().Opts = &base.Options{
Name: test.DownloadName,
Path: test.Dir,
Extra: &http.OptsExtra{
Connections: 4, // Try to use multiple connections, but only first should work
},
}
err := fetcher.Resolve(&base.Request{
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
})
if err != nil {
t.Fatal(err)
@@ -2335,14 +2353,15 @@ func TestFetcher_SlowStartExpansion(t *testing.T) {
Connections: tc.maxConns,
})
err := fetcher.Resolve(&base.Request{
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
}, &base.Options{
fetcher.Meta().Opts = &base.Options{
Name: test.DownloadName,
Path: test.Dir,
Extra: &http.OptsExtra{
Connections: tc.maxConns,
},
}
err := fetcher.Resolve(&base.Request{
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
})
if err != nil {
cleanup()
@@ -2398,14 +2417,15 @@ func TestFetcher_AsyncPrefetch(t *testing.T) {
defer listener.Close()
fetcher := buildFetcher()
err := fetcher.Resolve(&base.Request{
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
}, &base.Options{
fetcher.Meta().Opts = &base.Options{
Name: test.DownloadName,
Path: test.Dir,
Extra: &http.OptsExtra{
Connections: 4,
},
}
err := fetcher.Resolve(&base.Request{
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
})
if err != nil {
t.Fatal(err)
@@ -2469,14 +2489,15 @@ func TestFetcher_AsyncPrefetch(t *testing.T) {
defer listener.Close()
fetcher := buildFetcher()
err := fetcher.Resolve(&base.Request{
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
}, &base.Options{
fetcher.Meta().Opts = &base.Options{
Name: test.DownloadName,
Path: test.Dir,
Extra: &http.OptsExtra{
Connections: 4,
},
}
err := fetcher.Resolve(&base.Request{
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
})
if err != nil {
t.Fatal(err)
@@ -2551,14 +2572,15 @@ func TestFetcher_DownloadExpiringRedirectURL(t *testing.T) {
defer listener.Close()
fetcher := buildFetcher()
err := fetcher.Resolve(&base.Request{
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
}, &base.Options{
fetcher.Meta().Opts = &base.Options{
Name: test.DownloadName,
Path: test.Dir,
Extra: &http.OptsExtra{
Connections: 4, // Use multiple connections to trigger redirect expiration
},
}
err := fetcher.Resolve(&base.Request{
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
})
if err != nil {
t.Fatal(err)
@@ -2592,14 +2614,15 @@ func TestFetcher_DownloadExpiringRedirectURL(t *testing.T) {
defer listener.Close()
fetcher := buildFetcher()
err := fetcher.Resolve(&base.Request{
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
}, &base.Options{
fetcher.Meta().Opts = &base.Options{
Name: test.DownloadName,
Path: test.Dir,
Extra: &http.OptsExtra{
Connections: 8,
},
}
err := fetcher.Resolve(&base.Request{
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
})
if err != nil {
t.Fatal(err)
@@ -2639,14 +2662,15 @@ func TestFetcher_RetryAfterError(t *testing.T) {
defer listener.Close()
fetcher := buildFetcher()
err := fetcher.Resolve(&base.Request{
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
}, &base.Options{
fetcher.Meta().Opts = &base.Options{
Name: test.DownloadName,
Path: test.Dir,
Extra: &http.OptsExtra{
Connections: 1, // Use single connection to simplify test
},
}
err := fetcher.Resolve(&base.Request{
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
})
if err != nil {
t.Fatal(err)
@@ -3067,9 +3091,10 @@ func doDownloadReady(f fetcher.Fetcher, listener net.Listener, connections int,
Path: test.Dir,
Extra: extra,
}
f.Meta().Opts = opts
err := f.Resolve(&base.Request{
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
}, opts)
})
if err != nil {
t.Fatal(err)
}
@@ -3118,7 +3143,8 @@ func downloadPost(listener net.Listener, connections int, t *testing.T) {
Body: fmt.Sprintf(`{"name":"%s"}`, test.BuildName),
},
}
err := f.Resolve(req, opts)
f.Meta().Opts = opts
err := f.Resolve(req)
if err != nil {
t.Fatal(err)
}
@@ -3164,11 +3190,12 @@ func downloadContinue(listener net.Listener, connections int, t *testing.T) {
func downloadError(listener net.Listener, connections int, t *testing.T) {
fetcher := buildFetcher()
err := fetcher.Resolve(&base.Request{
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
}, &base.Options{
fetcher.Meta().Opts = &base.Options{
Name: test.DownloadName,
Path: test.Dir,
}
err := fetcher.Resolve(&base.Request{
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
})
// With the new async design, Resolve may succeed (HTTP response received)
// but errors occur during async download or Start/Wait
@@ -3292,7 +3319,8 @@ func TestFetcher_Patch_URLChange(t *testing.T) {
}
// Step 1: Try to resolve with bad URL - should fail with error
err := f.Resolve(&base.Request{URL: badURL}, opts)
f.Meta().Opts = opts
err := f.Resolve(&base.Request{URL: badURL})
if err == nil {
t.Fatal("Expected error for bad URL, got nil")
}
@@ -3302,7 +3330,8 @@ func TestFetcher_Patch_URLChange(t *testing.T) {
f2 := buildFetcher()
// First resolve with good URL to create a valid fetcher state
err = f2.Resolve(&base.Request{URL: goodURL}, opts)
f2.Meta().Opts = opts
err = f2.Resolve(&base.Request{URL: goodURL})
if err != nil {
t.Fatal(err)
}
@@ -3354,13 +3383,14 @@ func TestFetcher_Patch_Labels(t *testing.T) {
Extra: &http.OptsExtra{Connections: 1},
}
f.Meta().Opts = opts
err := f.Resolve(&base.Request{
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
Labels: map[string]string{
"key1": "value1",
"key3": "value3",
},
}, opts)
})
if err != nil {
t.Fatal(err)
}
@@ -3411,6 +3441,7 @@ func TestFetcher_Patch_Extra(t *testing.T) {
}
// Resolve with initial Extra
f.Meta().Opts = opts
err := f.Resolve(&base.Request{
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
Extra: &http.ReqExtra{
@@ -3421,7 +3452,7 @@ func TestFetcher_Patch_Extra(t *testing.T) {
"X-Custom": "original",
},
},
}, opts)
})
if err != nil {
t.Fatal(err)
}
@@ -3499,9 +3530,10 @@ func TestFetcher_Patch_NilData(t *testing.T) {
Extra: &http.OptsExtra{Connections: 1},
}
f.Meta().Opts = opts
err := f.Resolve(&base.Request{
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
}, opts)
})
if err != nil {
t.Fatal(err)
}
@@ -3551,6 +3583,7 @@ func TestFetcher_Patch_CookieExpired(t *testing.T) {
// Step 1: Resolve with old_token - should succeed (first request accepts old_token)
f := buildFetcher()
f.Meta().Opts = opts
err := f.Resolve(&base.Request{
URL: downloadURL,
Extra: &http.ReqExtra{
@@ -3558,7 +3591,7 @@ func TestFetcher_Patch_CookieExpired(t *testing.T) {
"Cookie": "session=old_token",
},
},
}, opts)
})
if err != nil {
t.Fatalf("Resolve should succeed with old_token: %v", err)
}
internal/protocol/http/resolve_range_test.go
+4 -2
View File
@@ -49,7 +49,8 @@ func TestResolveResponseSize(t *testing.T) {
defer server.Close()
f := buildFetcher()
defer f.Close()
err := f.Resolve(&base.Request{URL: server.URL + "/file"}, &base.Options{Path: t.TempDir()})
f.Meta().Opts = &base.Options{Path: t.TempDir()}
err := f.Resolve(&base.Request{URL: server.URL + "/file"})
if tc.invalid {
if !errors.Is(err, errInvalidRangeResponse) {
t.Fatalf("Resolve error = %v, want invalid range", err)
@@ -97,7 +98,8 @@ func TestDownloadPartialResolve(t *testing.T) {
defer server.Close()
f := buildFetcher()
defer f.Close()
if err := f.Resolve(&base.Request{URL: server.URL + "/file"}, &base.Options{Path: t.TempDir(), Name: "file", Extra: &fhttp.OptsExtra{Connections: connections}}); err != nil {
f.Meta().Opts = &base.Options{Path: t.TempDir(), Name: "file", Extra: &fhttp.OptsExtra{Connections: connections}}
if err := f.Resolve(&base.Request{URL: server.URL + "/file"}); err != nil {
t.Fatal(err)
}
if err := f.Start(); err != nil {