refactor(downloader): remove opts parameter from Resolve (#1595)

This commit is contained in:
Levi authored and GitHub committed 2026-10-19 21:40:57 +08:00
1 parent 5462b14705
commit de65149b29
45 files changed
+1589 -384

No files matched your search

+2 -2
View File
@@ -166,7 +166,7 @@ func runResolve(args []string) error {
result, err := d.Resolve(&base.Request{ result, err := d.Resolve(&base.Request{
URL: url, URL: url,
Labels: map[string]string(labels), Labels: map[string]string(labels),
}, nil) })
if err != nil { if err != nil {
return err return err
} }
@@ -250,7 +250,7 @@ func runDownload(args []string) error {
result, err := d.Resolve(&base.Request{ result, err := d.Resolve(&base.Request{
URL: url, URL: url,
Labels: map[string]string(labels), Labels: map[string]string(labels),
}, nil) })
if err != nil { if err != nil {
return err return err
} }
+1 -1
View File
@@ -12,7 +12,7 @@ import (
// Each download task will have a corresponding Fetcher instance for the management of the download task // Each download task will have a corresponding Fetcher instance for the management of the download task
type Fetcher interface { type Fetcher interface {
Setup(ctl *controller.Controller) Setup(ctl *controller.Controller)
Resolve(req *base.Request, opts *base.Options) error Resolve(req *base.Request) error
Start() error Start() error
// Patch modifies task-specific data based on the protocol. // Patch modifies task-specific data based on the protocol.
// For HTTP: can modify Request info (URL, headers, etc.) // For HTTP: can modify Request info (URL, headers, etc.)
+448
View File
@@ -0,0 +1,448 @@
package bt
import (
"encoding/json"
"net/url"
"os"
"path/filepath"
"strings"
"sync/atomic"
"testing"
"time"
"github.com/GopeedLab/gopeed/internal/controller"
"github.com/GopeedLab/gopeed/internal/fetcher"
"github.com/GopeedLab/gopeed/pkg/base"
btprotocol "github.com/GopeedLab/gopeed/pkg/protocol/bt"
"github.com/anacrolix/torrent"
pp "github.com/anacrolix/torrent/peer_protocol"
"golang.org/x/time/rate"
)
type btDownloadHarness struct {
*Fetcher
pendingRequests atomic.Int64
}
// These tests exercise the BT fetcher, native storage and real peer protocol.
// All content is generated locally and served by a loopback seeder.
func btDownloadFetcher(t *testing.T, f fetcher.Fetcher) *btDownloadHarness {
t.Helper()
if f == nil {
f = new(FetcherManager).Build()
}
ctl := controller.NewController()
ctl.GetConfig = func(v any) {
encoded, err := json.Marshal(new(FetcherManager).DefaultConfig())
if err != nil {
t.Fatal(err)
}
if err := json.Unmarshal(encoded, v); err != nil {
t.Fatal(err)
}
}
f.Setup(ctl)
t.Cleanup(func() {
if err := f.Close(); err != nil {
t.Error(err)
}
if err := closeClient(); err != nil {
t.Error(err)
}
})
h := &btDownloadHarness{Fetcher: f.(*Fetcher)}
// Keep the test client on loopback and observe real cancellation replies.
// Waiting for them avoids the pinned library's immediate cancel/resume
// request scheduling bug, without relying on arbitrary sleep durations.
cfg = torrent.TestingConfig(t)
cfg.DisableTrackers = false
cfg.Seed = true
cfg.DisableUTP = true
cfg.DisableIPv6 = true
cfg.DisableWebtorrent = true
cfg.KeepAliveTimeout = time.Minute
cfg.MaxAllocPeerRequestDataPerConn = 1 << 20
cfg.Callbacks.SentRequest = []func(torrent.PeerRequestEvent){func(torrent.PeerRequestEvent) {
h.pendingRequests.Add(1)
}}
cfg.Callbacks.ReadMessage = func(_ *torrent.PeerConn, msg *pp.Message) {
if !msg.Keepalive && (msg.Type == pp.Piece || msg.Type == pp.Reject) {
h.pendingRequests.Add(-1)
}
}
var err error
client, err = torrent.NewClient(cfg)
if err != nil {
t.Fatal(err)
}
return h
}
func btResolve(t *testing.T, f *btDownloadHarness, source string) {
t.Helper()
if err := f.Resolve(&base.Request{URL: source}); err != nil {
t.Fatal(err)
}
}
func btStart(t *testing.T, f *btDownloadHarness) {
t.Helper()
if err := f.Start(); err != nil {
t.Fatal(err)
}
}
func btAwaitDone(t *testing.T, f *btDownloadHarness) {
t.Helper()
done := make(chan error, 1)
go func() { done <- f.Wait() }()
timeout := time.NewTimer(20 * time.Second)
defer timeout.Stop()
select {
case err := <-done:
if err != nil {
t.Fatalf("BT download failed: %v", err)
}
if !f.isDone() {
t.Fatal("Wait returned before selected files were verified")
}
case <-timeout.C:
var status strings.Builder
client.WriteStatus(&status)
t.Log(status.String())
t.Fatalf("BT download did not finish: progress=%v stats=%+v pieces=%+v peers=%+v", f.Progress(), f.torrent.Stats(), f.torrent.PieceStateRuns(), f.Stats().Runtime)
}
}
func btAwaitPartial(t *testing.T, f *btDownloadHarness) {
t.Helper()
timeout := time.NewTimer(15 * time.Second)
defer timeout.Stop()
ticker := time.NewTicker(20 * time.Millisecond)
defer ticker.Stop()
for {
select {
case <-ticker.C:
progress := f.Progress().TotalDownloaded()
if progress >= 32*1024 && progress < f.Meta().Res.Size {
return
}
case <-timeout.C:
t.Fatal("BT transfer did not reach partial progress")
}
}
}
func btPause(t *testing.T, f *btDownloadHarness) {
t.Helper()
if err := f.Pause(); err != nil {
t.Fatal(err)
}
timeout := time.NewTimer(5 * time.Second)
defer timeout.Stop()
ticker := time.NewTicker(10 * time.Millisecond)
defer ticker.Stop()
for {
if f.pendingRequests.Load() == 0 {
// The message callback runs under the client lock. Acquiring that
// lock ensures the final cancellation reply has finished processing.
client.Stats()
return
}
select {
case <-ticker.C:
case <-timeout.C:
t.Fatal("peer did not acknowledge cancelled requests")
}
}
}
func TestBTDownloadSources(t *testing.T) {
for _, kind := range []string{"magnet", "torrent", "file URI", "data URI"} {
t.Run(kind, func(t *testing.T) {
fixture := newBTFixture(t, map[string][]byte{"": btPayload(96*1024, kind)}, 0)
source := fixture.Magnet
switch kind {
case "torrent":
source = fixture.TorrentFile
case "file URI":
source = fixture.FileURI()
case "data URI":
source = fixture.DataURI
}
f := btDownloadFetcher(t, nil)
btResolve(t, f, source)
if f.Meta().Opts != nil {
t.Fatal("Resolve assigned download options")
}
f.storage.mu.RLock()
bound := f.storage.owner != nil
f.storage.mu.RUnlock()
if bound {
t.Fatal("Resolve bound download storage")
}
resolvedTorrent := f.torrent
dir := filepath.Join(t.TempDir(), "chosen-after-resolve")
f.Meta().Opts = &base.Options{Path: dir}
if _, err := os.Stat(dir); !os.IsNotExist(err) {
t.Fatalf("output directory exists before Start: %v", err)
}
btStart(t, f)
if f.torrent != resolvedTorrent {
t.Fatal("Start replaced the resolved torrent")
}
btAwaitDone(t, f)
fixture.AssertFiles(t, dir)
})
}
}
func TestBTDirectDownloadWithExtraTrackers(t *testing.T) {
for _, source := range []string{"request", "config"} {
t.Run(source, func(t *testing.T) {
fixture := makeBTFixture(t, map[string][]byte{
"a.bin": btPayload(20*1024, "direct-a"),
"nested/b.bin": btPayload(48*1024, "direct-b"),
}, 0, false, false)
magnet, err := url.Parse(fixture.Magnet)
if err != nil {
t.Fatal(err)
}
query := magnet.Query()
query.Del("tr")
query.Del("x.pe")
magnet.RawQuery = query.Encode()
req := &base.Request{URL: magnet.String()}
f := btDownloadFetcher(t, nil)
if source == "request" {
req.Extra = &btprotocol.ReqExtra{Trackers: []string{fixture.TrackerURL}}
} else {
f.config.Trackers = []string{fixture.TrackerURL}
}
// CreateDirect supplies options before Downloader resolves the task.
dir := filepath.Join(t.TempDir(), "direct-download")
opts := &base.Options{Path: dir, SelectFiles: []int{}}
f.Meta().Opts = opts
resolved := make(chan error, 1)
go func() { resolved <- f.Resolve(req) }()
select {
case err := <-resolved:
if err != nil {
t.Fatal(err)
}
case <-time.After(5 * time.Second):
f.torrentDropFunc()
<-resolved
t.Fatal("direct magnet download did not obtain metadata from its extra tracker")
}
if f.Meta().Opts != opts || len(opts.SelectFiles) != len(fixture.Files) {
t.Fatal("resolution lost direct download options or did not select all files")
}
btStart(t, f)
btAwaitDone(t, f)
fixture.AssertFiles(t, dir)
})
}
}
func TestBTDownloadSelectedFiles(t *testing.T) {
fixture := newBTFixture(t, map[string][]byte{
"a.bin": btPayload(20*1024, "a"), "nested/b.bin": btPayload(48*1024, "b"),
}, 0)
f := btDownloadFetcher(t, nil)
dir := t.TempDir()
btResolve(t, f, fixture.Magnet)
selected := -1
for i, file := range f.Meta().Res.Files {
if file.Name == "b.bin" {
selected = i
}
}
if selected < 0 {
t.Fatal("selected file absent from metadata")
}
f.Meta().Opts = &base.Options{Path: dir}
if err := f.Patch(nil, &base.Options{SelectFiles: []int{selected}}); err != nil {
t.Fatal(err)
}
btStart(t, f)
btAwaitDone(t, f)
fixture.AssertFiles(t, dir, filepath.Join("bundle", "nested", "b.bin"))
if _, err := os.Stat(filepath.Join(dir, "bundle", "a.bin")); !os.IsNotExist(err) {
t.Fatalf("unselected output remains: %v", err)
}
if f.Meta().Res.Size != 48*1024 {
t.Fatal("selected resource size is incorrect")
}
}
func TestBTPauseAndResume(t *testing.T) {
fixture := newBTFixture(t, map[string][]byte{"": btPayload(512*1024, "pause")}, 64*1024)
f := btDownloadFetcher(t, nil)
dir := t.TempDir()
btResolve(t, f, fixture.Magnet)
f.Meta().Opts = &base.Options{Path: dir}
btStart(t, f)
btAwaitPartial(t, f)
btPause(t, f)
resolvedTorrent := f.torrent
fixture.Limiter.SetLimit(rate.Inf)
// Once cancelled requests have settled, no more data should arrive.
stats := f.torrent.Stats()
readWhilePaused := stats.BytesReadData.Int64()
time.Sleep(200 * time.Millisecond)
stats = f.torrent.Stats()
if stats.BytesReadData.Int64() != readWhilePaused || f.isDone() {
t.Fatal("data download continued while paused")
}
btStart(t, f)
if f.torrent != resolvedTorrent {
t.Fatal("resume replaced the resolved torrent")
}
btAwaitDone(t, f)
fixture.AssertFiles(t, dir)
}
type btFetcherCheckpoint struct {
Meta *fetcher.FetcherMeta `json:"meta"`
Data json.RawMessage `json:"data"`
}
func btStore(t *testing.T, f *btDownloadHarness) []byte {
t.Helper()
data, err := new(FetcherManager).Store(f.Fetcher)
if err != nil {
t.Fatal(err)
}
encodedData, err := json.Marshal(data)
if err != nil {
t.Fatal(err)
}
encoded, err := json.Marshal(btFetcherCheckpoint{Meta: f.Meta(), Data: encodedData})
if err != nil {
t.Fatal(err)
}
return encoded
}
func btRestore(t *testing.T, encoded []byte) *btDownloadHarness {
t.Helper()
var checkpoint btFetcherCheckpoint
if err := json.Unmarshal(encoded, &checkpoint); err != nil {
t.Fatal(err)
}
data, restore := new(FetcherManager).Restore()
if err := json.Unmarshal(checkpoint.Data, data); err != nil {
t.Fatal(err)
}
return btDownloadFetcher(t, restore(checkpoint.Meta, data))
}
func btCloseForRestart(t *testing.T, f *btDownloadHarness) {
t.Helper()
if err := f.Close(); err != nil {
t.Fatal(err)
}
if err := new(FetcherManager).Close(); err != nil {
t.Fatal(err)
}
}
func TestBTColdStartResume(t *testing.T) {
for _, kind := range []string{"v1", "hybrid"} {
t.Run(kind, func(t *testing.T) {
var fixture *btFixture
if kind == "hybrid" {
fixture = newBTHybridFixture(t, btPayload(512*1024, "restart-hybrid"))
fixture.Limiter.SetLimit(64 * 1024)
} else {
fixture = newBTFixture(t, map[string][]byte{"": btPayload(512*1024, "restart")}, 64*1024)
}
f := btDownloadFetcher(t, nil)
dir := t.TempDir()
btResolve(t, f, fixture.Magnet)
f.Meta().Opts = &base.Options{Path: dir}
btStart(t, f)
btAwaitPartial(t, f)
btPause(t, f)
checkpoint := btStore(t, f)
hash := f.Meta().Res.Hash
btCloseForRestart(t, f)
seedStats := fixture.Torrent.Stats()
sentBeforeRestart := seedStats.BytesWrittenData.Int64()
restored := btRestore(t, checkpoint)
if restored.Meta().Opts.Path != dir || restored.Meta().Res.Hash != hash {
t.Fatal("download directory or hash was lost")
}
fixture.Limiter.SetLimit(rate.Inf)
btStart(t, restored)
btAwaitDone(t, restored)
fixture.AssertFiles(t, dir)
seedStats = fixture.Torrent.Stats()
if sent := seedStats.BytesWrittenData.Int64() - sentBeforeRestart; sent >= 512*1024 {
t.Fatalf("restart redownloaded the entire file: %d bytes", sent)
}
})
}
}
func TestBTDownloadHybrid(t *testing.T) {
fixture := newBTHybridFixture(t, btPayload(80*1024, "hybrid-v2"))
f := btDownloadFetcher(t, nil)
dir := t.TempDir()
btResolve(t, f, fixture.Magnet)
f.Meta().Opts = &base.Options{Path: dir}
btStart(t, f)
btAwaitDone(t, f)
fixture.AssertFiles(t, dir)
}
func TestBTRepairCorruptedExistingFile(t *testing.T) {
fixture := newBTFixture(t, map[string][]byte{"": btPayload(128*1024, "repair")}, 0)
dir := t.TempDir()
damaged := append([]byte(nil), fixture.Files["payload.bin"]...)
damaged[20000] ^= 0xff
if err := os.WriteFile(filepath.Join(dir, "payload.bin"), damaged, 0600); err != nil {
t.Fatal(err)
}
f := btDownloadFetcher(t, nil)
btResolve(t, f, fixture.Magnet)
f.Meta().Opts = &base.Options{Path: dir}
btStart(t, f)
btAwaitDone(t, f)
fixture.AssertFiles(t, dir)
}
func TestBTPatchSelectionWhilePaused(t *testing.T) {
fixture := newBTFixture(t, map[string][]byte{"a.bin": btPayload(256*1024, "patch-a"), "b.bin": btPayload(256*1024, "patch-b")}, 64*1024)
f := btDownloadFetcher(t, nil)
dir := t.TempDir()
btResolve(t, f, fixture.Magnet)
f.Meta().Opts = &base.Options{Path: dir, SelectFiles: []int{0}}
btStart(t, f)
btAwaitPartial(t, f)
btPause(t, f)
if err := f.Patch(nil, &base.Options{SelectFiles: []int{1}}); err != nil {
t.Fatal(err)
}
fixture.Limiter.SetLimit(rate.Inf)
btStart(t, f)
btAwaitDone(t, f)
fixture.AssertFiles(t, dir, filepath.Join("bundle", "b.bin"))
for _, suffix := range []string{"", ".part"} {
if _, err := os.Stat(filepath.Join(dir, "bundle", "a.bin") + suffix); !os.IsNotExist(err) {
t.Fatalf("unselected output remains: %v", err)
}
}
}
func TestBTDownloadMultiFileWithEmptyFile(t *testing.T) {
fixture := newBTFixture(t, map[string][]byte{"empty.txt": {}, "nested/data.bin": btPayload(48*1024, "empty-file")}, 0)
f := btDownloadFetcher(t, nil)
dir := t.TempDir()
btResolve(t, f, fixture.Magnet)
f.Meta().Opts = &base.Options{Path: dir}
btStart(t, f)
btAwaitDone(t, f)
fixture.AssertFiles(t, dir)
}
+220 -105
View File
@@ -3,11 +3,13 @@ package bt
import ( import (
"bytes" "bytes"
"context" "context"
"errors"
"fmt" "fmt"
"io" "io"
"net/url" "net/url"
"os" "os"
"path/filepath" "path/filepath"
"runtime"
"strings" "strings"
"sync" "sync"
"sync/atomic" "sync/atomic"
@@ -33,6 +35,10 @@ var (
) )
type Fetcher struct { type Fetcher struct {
mu sync.Mutex
closeOnce sync.Once
storage *deferredFileStorage
ctl *controller.Controller ctl *controller.Controller
config *config config *config
@@ -94,13 +100,17 @@ func (f *Fetcher) initClient() (err error) {
return return
} }
func (f *Fetcher) Resolve(req *base.Request, opts *base.Options) error { func (f *Fetcher) Resolve(req *base.Request) error {
f.meta.Req = req f.mu.Lock()
f.meta.Opts = opts defer f.mu.Unlock()
if f.meta.Opts == nil { if f.torrent != nil {
f.meta.Opts = &base.Options{} f.safeDrop()
f.torrent = nil
f.storage = nil
f.torrentReady.Store(false)
} }
if err := f.addTorrent(req, false); err != nil { f.meta.Req = req
if err := f.addTorrent(req); err != nil {
return err return err
} }
f.updateRes() f.updateRes()
@@ -108,12 +118,72 @@ func (f *Fetcher) Resolve(req *base.Request, opts *base.Options) error {
} }
func (f *Fetcher) Start() (err error) { func (f *Fetcher) Start() (err error) {
f.mu.Lock()
defer f.mu.Unlock()
if f.meta.Opts == nil || f.meta.Opts.Path == "" {
return errors.New("download directory is required when starting a torrent")
}
if err := f.torrentDropCtx.Err(); err != nil {
return err
}
if !f.torrentReady.Load() { if !f.torrentReady.Load() {
if err = f.addTorrent(f.meta.Req, false); err != nil { if err = f.addTorrent(f.meta.Req); err != nil {
return return
} }
} }
path := f.meta.Opts.Path
if err := os.MkdirAll(path, 0755); err != nil {
return err
}
hashes, err := f.storage.bind(storage.NewFileClientOpts{
ClientBaseDir: path,
FilePathMaker: func(opts storage.FilePathMakerOpts) string {
_, name := torrentFileLayout(opts.Info, *opts.File)
return name
},
})
if err != nil {
return err
}
if hashes != nil {
// File storage infers completion from final file names and sizes. Verify
// before publishing that state, or peers can see an unverified file as
// complete and disconnect before a damaged piece is discovered.
// Late v2 hashes schedule their own initial checks through the library.
changes := f.torrent.SubscribePieceStateChanges()
defer changes.Close()
for i, known := range hashes {
if !known {
continue
}
if err := f.torrentDropCtx.Err(); err != nil {
return err
}
if err := f.torrent.Piece(i).VerifyDataContext(f.torrentDropCtx); err != nil {
return err
}
// VerifyDataContext can return while the result is still being
// written to storage. Wait for it before refreshing completion.
for {
state := f.torrent.Piece(i).State()
if !state.Hashing && !state.QueuedForHash && !state.Marking {
break
}
select {
case <-changes.Values:
case <-f.torrentDropCtx.Done():
return f.torrentDropCtx.Err()
case <-f.torrent.Closed():
return errors.New("torrent closed during verification")
}
}
}
for i := range hashes {
f.torrent.Piece(i).UpdateCompletion()
}
}
files := f.torrent.Files() files := f.torrent.Files()
// If the user does not specify the file to download, all files will be downloaded by default // If the user does not specify the file to download, all files will be downloaded by default
if f.data.Progress == nil { if f.data.Progress == nil {
@@ -134,21 +204,29 @@ func (f *Fetcher) Start() (err error) {
} }
} }
f.torrent.AllowDataDownload() f.torrent.AllowDataDownload()
go f.doUpload(false)
return return
} }
func (f *Fetcher) Pause() (err error) { func (f *Fetcher) Pause() (err error) {
f.mu.Lock()
defer f.mu.Unlock()
if f.torrent == nil {
return nil
}
f.torrent.DisallowDataDownload() f.torrent.DisallowDataDownload()
return return
} }
func (f *Fetcher) Close() (err error) { func (f *Fetcher) Close() (err error) {
f.safeDrop() f.closeOnce.Do(func() {
f.torrentDropFunc() f.torrentDropFunc()
f.uploadDoneCh <- nil f.mu.Lock()
if len(client.Torrents()) == 0 { defer f.mu.Unlock()
err = closeClient() f.safeDrop()
} f.torrentReady.Store(false)
close(f.uploadDoneCh)
})
return nil return nil
} }
@@ -166,6 +244,8 @@ func (f *Fetcher) Meta() *fetcher.FetcherMeta {
} }
func (f *Fetcher) Stats() *fetcher.Stats { func (f *Fetcher) Stats() *fetcher.Stats {
f.mu.Lock()
defer f.mu.Unlock()
var stats torrent.TorrentStats var stats torrent.TorrentStats
var pieceMap *base.PieceMap var pieceMap *base.PieceMap
pieceMapReady := false pieceMapReady := false
@@ -372,48 +452,65 @@ func normalizeBTTransport(network string) string {
} }
func (f *Fetcher) Progress() fetcher.Progress { func (f *Fetcher) Progress() fetcher.Progress {
f.mu.Lock()
defer f.mu.Unlock()
if !f.torrentReady.Load() { if !f.torrentReady.Load() {
return f.data.Progress return append(fetcher.Progress(nil), f.data.Progress...)
} }
for i := range f.data.Progress { for i := range f.data.Progress {
selectIndex := f.meta.Opts.SelectFiles[i] selectIndex := f.meta.Opts.SelectFiles[i]
file := f.torrent.Files()[selectIndex] file := f.torrent.Files()[selectIndex]
f.data.Progress[i] = file.BytesCompleted() f.data.Progress[i] = file.BytesCompleted()
} }
return f.data.Progress return append(fetcher.Progress(nil), f.data.Progress...)
} }
func (f *Fetcher) Wait() (err error) { func (f *Fetcher) Wait() error {
f.mu.Lock()
var closed <-chan struct{}
if f.torrent != nil {
closed = f.torrent.Closed()
}
f.mu.Unlock()
ticker := time.NewTicker(time.Second)
defer ticker.Stop()
for { for {
select { select {
case <-f.torrentDropCtx.Done(): case <-f.torrentDropCtx.Done():
return return nil
case <-time.After(time.Second): case <-closed:
if f.torrentReady.Load() && len(f.meta.Opts.SelectFiles) > 0 { return nil
if f.isDone() { case <-ticker.C:
// remove unselected files done, err := f.finishDownload()
for i, file := range f.torrent.Files() { if done || err != nil {
selected := false return err
for _, selectIndex := range f.meta.Opts.SelectFiles {
if i == selectIndex {
selected = true
break
}
}
if !selected {
root, name := torrentFileLayout(f.torrent.Info(), file.FileInfo())
if err := removeUnselectedFile(f.meta.Opts.Path, root, name); err != nil {
return err
}
}
}
return
}
} }
} }
} }
} }
func (f *Fetcher) finishDownload() (bool, error) {
f.mu.Lock()
defer f.mu.Unlock()
if !f.torrentReady.Load() || f.meta.Opts == nil || len(f.meta.Opts.SelectFiles) == 0 || !f.isDoneLocked() {
return false, nil
}
selected := make(map[int]bool, len(f.meta.Opts.SelectFiles))
for _, i := range f.meta.Opts.SelectFiles {
selected[i] = true
}
for i, file := range f.torrent.Files() {
if selected[i] {
continue
}
root, name := torrentFileLayout(f.torrent.Info(), file.FileInfo())
if err := removeUnselectedFile(f.meta.Opts.Path, root, name); err != nil {
return false, err
}
}
return true, nil
}
// Match the file storage layout, including its NoName convention for rootless // Match the file storage layout, including its NoName convention for rootless
// torrents. File.Path() always prepends BestName, even when storage omits it. // torrents. File.Path() always prepends BestName, even when storage omits it.
func torrentFileLayout(info *metainfo.Info, file metainfo.FileInfo) (root, name string) { func torrentFileLayout(info *metainfo.Info, file metainfo.FileInfo) (root, name string) {
@@ -471,13 +568,22 @@ func removeUnselectedFile(downloadDir, torrentRoot, torrentPath string) error {
} }
func (f *Fetcher) isDone() bool { func (f *Fetcher) isDone() bool {
if f.meta.Opts == nil { f.mu.Lock()
defer f.mu.Unlock()
return f.isDoneLocked()
}
func (f *Fetcher) isDoneLocked() bool {
if f.meta.Opts == nil || f.torrent == nil {
return false return false
} }
for _, selectIndex := range f.meta.Opts.SelectFiles { for _, index := range f.meta.Opts.SelectFiles {
file := f.torrent.Files()[selectIndex] file := f.torrent.Files()[index]
if file.BytesCompleted() < file.Length() { for i := file.BeginPieceIndex(); i < file.EndPieceIndex(); i++ {
return false state := f.torrent.Piece(i).State()
if !state.Ok || !state.Complete || state.Hashing || state.QueuedForHash {
return false
}
} }
} }
return true return true
@@ -486,6 +592,11 @@ func (f *Fetcher) isDone() bool {
// Patch modifies the BT task settings. // Patch modifies the BT task settings.
// Invalid file indices are silently ignored. // Invalid file indices are silently ignored.
func (f *Fetcher) Patch(req *base.Request, opts *base.Options) error { func (f *Fetcher) Patch(req *base.Request, opts *base.Options) error {
f.mu.Lock()
defer f.mu.Unlock()
if f.meta.Opts == nil {
f.meta.Opts = &base.Options{}
}
if opts == nil { if opts == nil {
return nil return nil
} }
@@ -542,9 +653,9 @@ func (f *Fetcher) updateRes() {
Files: make([]*base.FileInfo, len(f.torrent.Files())), Files: make([]*base.FileInfo, len(f.torrent.Files())),
Hash: f.torrent.InfoHash().String(), Hash: f.torrent.InfoHash().String(),
} }
// Directory torrent // Use the same root classification as storage (v2 single files have Length == 0).
if f.torrent.Info().Length == 0 { if files := f.torrent.Files(); len(files) > 0 {
res.Name = f.torrent.Name() res.Name, _ = torrentFileLayout(f.torrent.Info(), files[0].FileInfo())
} }
for i, file := range f.torrent.Files() { for i, file := range f.torrent.Files() {
res.Files[i] = &base.FileInfo{ res.Files[i] = &base.FileInfo{
@@ -561,61 +672,50 @@ func (f *Fetcher) updateRes() {
} }
func (f *Fetcher) Upload() (err error) { func (f *Fetcher) Upload() (err error) {
return f.addTorrent(f.meta.Req, true) if err = f.Start(); err != nil {
return err
}
return nil
} }
func (f *Fetcher) doUpload(fromUpload bool) { func (f *Fetcher) doUpload(fromUpload bool) {
if !f.torrentUpload.CompareAndSwap(false, true) { if !f.torrentUpload.CompareAndSwap(false, true) {
return return
} }
f.mu.Lock()
// Check and update seed data initialBytes, initialTime := f.data.SeedBytes, f.data.SeedTime
lastData := &fetcherData{ torrentClosed := f.torrent.Closed()
SeedBytes: f.data.SeedBytes, f.mu.Unlock()
SeedTime: f.data.SeedTime, var doneTime int64
} ticker := time.NewTicker(time.Second)
var doneTime int64 = 0 defer ticker.Stop()
for { for {
select { select {
case <-f.torrentDropCtx.Done(): case <-f.torrentDropCtx.Done():
return return
case <-time.After(time.Second): case <-torrentClosed:
return
case <-ticker.C:
f.mu.Lock()
if !f.torrentReady.Load() { if !f.torrentReady.Load() {
f.mu.Unlock()
continue continue
} }
stats := f.torrentStats() stats := f.torrentStats()
f.data.SeedBytes = lastData.SeedBytes + stats.BytesWrittenData.Int64() f.data.SeedBytes = initialBytes + stats.BytesWrittenData.Int64()
if !fromUpload && !f.isDoneLocked() {
// Check is download complete, if not don't check and stop seeding f.mu.Unlock()
if !fromUpload && !f.isDone() {
continue continue
} }
if doneTime == 0 { if doneTime == 0 {
doneTime = time.Now().Unix() doneTime = time.Now().Unix()
} }
f.data.SeedTime = lastData.SeedTime + time.Now().Unix() - doneTime f.data.SeedTime = initialTime + time.Now().Unix() - doneTime
stop := !f.config.SeedKeep && ((f.config.SeedRatio > 0 && f.seedRadio() >= f.config.SeedRatio) || (f.config.SeedTime > 0 && f.data.SeedTime >= f.config.SeedTime))
// If the seed forever is true, keep seeding f.mu.Unlock()
if f.config.SeedKeep { if stop {
continue f.Close()
} return
// If the seed ratio is reached, stop seeding
if f.config.SeedRatio > 0 {
seedRadio := f.seedRadio()
if seedRadio >= f.config.SeedRatio {
f.Close()
break
}
}
// If the seed time is reached, stop seeding
if f.config.SeedTime > 0 {
if f.data.SeedTime >= f.config.SeedTime {
f.Close()
break
}
} }
} }
} }
@@ -633,6 +733,8 @@ func (f *Fetcher) torrentStats() torrent.TorrentStats {
} }
func (f *Fetcher) UploadedBytes() int64 { func (f *Fetcher) UploadedBytes() int64 {
f.mu.Lock()
defer f.mu.Unlock()
return f.data.SeedBytes return f.data.SeedBytes
} }
@@ -641,7 +743,7 @@ func (f *Fetcher) WaitUpload() (err error) {
return nil return nil
} }
func (f *Fetcher) addTorrent(req *base.Request, fromUpload bool) (err error) { func (f *Fetcher) addTorrent(req *base.Request) (err error) {
if err = base.ParseReqExtra[bt.ReqExtra](req); err != nil { if err = base.ParseReqExtra[bt.ReqExtra](req); err != nil {
return return
} }
@@ -660,11 +762,16 @@ func (f *Fetcher) addTorrent(req *base.Request, fromUpload bool) (err error) {
var reader io.Reader var reader io.Reader
if schema == "FILE" { if schema == "FILE" {
fileUrl, _ := url.Parse(req.URL) fileUrl, _ := url.Parse(req.URL)
filePath := fileUrl.Path[1:] filePath := fileUrl.Path
if runtime.GOOS == "windows" && len(filePath) > 2 && filePath[0] == '/' && filePath[2] == ':' {
filePath = filePath[1:]
}
filePath = filepath.FromSlash(filePath)
reader, err = os.Open(filePath) reader, err = os.Open(filePath)
if err != nil { if err != nil {
return return
} }
defer reader.(io.Closer).Close()
} else if schema == "DATA" { } else if schema == "DATA" {
_, data := util.ParseDataUri(req.URL) _, data := util.ParseDataUri(req.URL)
reader = bytes.NewBuffer(data) reader = bytes.NewBuffer(data)
@@ -697,21 +804,6 @@ func (f *Fetcher) addTorrent(req *base.Request, fromUpload bool) (err error) {
return return
} }
} }
spec.Storage = storage.NewFileOpts(storage.NewFileClientOpts{
ClientBaseDir: cfg.DataDir,
FilePathMaker: func(opts storage.FilePathMakerOpts) string {
_, name := torrentFileLayout(opts.Info, *opts.File)
return name
},
TorrentDirMaker: func(baseDir string, info *metainfo.Info, infoHash metainfo.Hash) string {
return f.meta.Opts.Path
},
})
f.torrent, _, err = client.AddTorrentSpec(spec)
if err != nil {
return
}
// Do not add external tracker to a private torrent. // Do not add external tracker to a private torrent.
if !privateTorrent { if !privateTorrent {
// use map to deduplicate // use map to deduplicate
@@ -730,17 +822,36 @@ func (f *Fetcher) addTorrent(req *base.Request, fromUpload bool) (err error) {
} }
} }
if len(trackers) > 0 { if len(trackers) > 0 {
announceList := make([][]string, 0)
for tracker := range trackers { for tracker := range trackers {
announceList = append(announceList, []string{tracker}) spec.Trackers = append(spec.Trackers, []string{tracker})
} }
f.torrent.AddTrackers(announceList)
} }
} }
<-f.torrent.GotInfo() s := &deferredFileStorage{}
spec.Storage = s
var added bool
f.torrent, added, err = client.AddTorrentSpec(spec)
if err != nil {
return err
}
if !added {
f.torrent = nil
return errors.New("torrent is already active")
}
f.storage = s
f.torrent.DisallowDataDownload()
// Extra trackers must be available while magnet metadata is still missing;
// they may be the only way to discover a peer that can provide it.
select {
case <-f.torrent.GotInfo():
case <-f.torrentDropCtx.Done():
f.safeDrop()
return f.torrentDropCtx.Err()
case <-f.torrent.Closed():
return errors.New("torrent closed during resolution")
}
f.torrentReady.Store(true) f.torrentReady.Store(true)
go f.doUpload(fromUpload)
return return
} }
@@ -848,7 +959,11 @@ func (fm *FetcherManager) DefaultConfig() any {
func (fm *FetcherManager) Store(f fetcher.Fetcher) (data any, err error) { func (fm *FetcherManager) Store(f fetcher.Fetcher) (data any, err error) {
_f := f.(*Fetcher) _f := f.(*Fetcher)
return _f.data, nil _f.mu.Lock()
defer _f.mu.Unlock()
snapshot := *_f.data
snapshot.Progress = append(fetcher.Progress(nil), snapshot.Progress...)
return &snapshot, nil
} }
func (fm *FetcherManager) Restore() (v any, f func(meta *fetcher.FetcherMeta, v any) fetcher.Fetcher) { func (fm *FetcherManager) Restore() (v any, f func(meta *fetcher.FetcherMeta, v any) fetcher.Fetcher) {
+19 -17
View File
@@ -18,11 +18,11 @@ import (
) )
func TestFetcher_Resolve_Torrent(t *testing.T) { func TestFetcher_Resolve_Torrent(t *testing.T) {
doResolve(t, buildFetcher()) doResolve(t, buildFetcher(t))
} }
func TestFetcher_Resolve_DataUri_Torrent(t *testing.T) { func TestFetcher_Resolve_DataUri_Torrent(t *testing.T) {
fetcher := buildFetcher() fetcher := buildFetcher(t)
buf, err := os.ReadFile("./testdata/ubuntu-22.04-live-server-amd64.iso.torrent") buf, err := os.ReadFile("./testdata/ubuntu-22.04-live-server-amd64.iso.torrent")
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
@@ -31,7 +31,7 @@ func TestFetcher_Resolve_DataUri_Torrent(t *testing.T) {
dataUri := "data:application/x-bittorrent;base64," + base64.StdEncoding.EncodeToString(buf) dataUri := "data:application/x-bittorrent;base64," + base64.StdEncoding.EncodeToString(buf)
err = fetcher.Resolve(&base.Request{ err = fetcher.Resolve(&base.Request{
URL: dataUri, URL: dataUri,
}, nil) })
if err != nil { if err != nil {
panic(err) panic(err)
} }
@@ -53,7 +53,7 @@ func TestFetcher_Resolve_DataUri_Torrent(t *testing.T) {
} }
func TestFetcher_Config(t *testing.T) { func TestFetcher_Config(t *testing.T) {
doResolve(t, buildConfigFetcher(nil)) doResolve(t, buildConfigFetcher(t, nil))
} }
func TestFetcher_ResolveWithProxy(t *testing.T) { func TestFetcher_ResolveWithProxy(t *testing.T) {
@@ -61,7 +61,7 @@ func TestFetcher_ResolveWithProxy(t *testing.T) {
proxyListener := test.StartSocks5Server(usr, pwd) proxyListener := test.StartSocks5Server(usr, pwd)
defer proxyListener.Close() defer proxyListener.Close()
doResolve(t, buildConfigFetcher(&base.DownloaderProxyConfig{ doResolve(t, buildConfigFetcher(t, &base.DownloaderProxyConfig{
Enable: true, Enable: true,
System: false, System: false,
Scheme: "socks5", Scheme: "socks5",
@@ -81,7 +81,7 @@ func doResolve(t *testing.T, fetcher fetcher.Fetcher) {
"udp://tracker.bitsearch.to:1337/announce", "udp://tracker.bitsearch.to:1337/announce",
}, },
}, },
}, nil) })
if err != nil { if err != nil {
panic(err) panic(err)
} }
@@ -111,7 +111,7 @@ func doResolve(t *testing.T, fetcher fetcher.Fetcher) {
"udp://tracker.bitsearch.to:1337/announce", "udp://tracker.bitsearch.to:1337/announce",
}, },
}, },
}, nil) })
if err != nil { if err != nil {
panic(err) panic(err)
} }
@@ -145,7 +145,7 @@ func doResolve(t *testing.T, fetcher fetcher.Fetcher) {
t.Run("Resolve Unclean Torrent", func(t *testing.T) { t.Run("Resolve Unclean Torrent", func(t *testing.T) {
err := fetcher.Resolve(&base.Request{ err := fetcher.Resolve(&base.Request{
URL: "./testdata/test.unclean.torrent", URL: "./testdata/test.unclean.torrent",
}, nil) })
if err != nil { if err != nil {
t.Errorf("Resolve Unclean Torrent Resolve() got = %v, want nil", err) t.Errorf("Resolve Unclean Torrent Resolve() got = %v, want nil", err)
} }
@@ -156,7 +156,7 @@ func doResolve(t *testing.T, fetcher fetcher.Fetcher) {
uri := "file:///" + file uri := "file:///" + file
err := fetcher.Resolve(&base.Request{ err := fetcher.Resolve(&base.Request{
URL: uri, URL: uri,
}, nil) })
if err != nil { if err != nil {
t.Errorf("Resolve file scheme Torrent Resolve() got = %v, want nil", err) t.Errorf("Resolve file scheme Torrent Resolve() got = %v, want nil", err)
} }
@@ -211,7 +211,7 @@ func TestFetcherManager_ParseName(t *testing.T) {
} }
} }
func buildFetcher() fetcher.Fetcher { func buildFetcher(t testing.TB) fetcher.Fetcher {
fb := new(FetcherManager) fb := new(FetcherManager)
fetcher := fb.Build() fetcher := fb.Build()
newController := controller.NewController() newController := controller.NewController()
@@ -219,10 +219,11 @@ func buildFetcher() fetcher.Fetcher {
json.Unmarshal([]byte(test.ToJson(fb.DefaultConfig())), v) json.Unmarshal([]byte(test.ToJson(fb.DefaultConfig())), v)
} }
fetcher.Setup(newController) fetcher.Setup(newController)
t.Cleanup(func() { fetcher.Close(); closeClient() })
return fetcher return fetcher
} }
func buildConfigFetcher(proxyConfig *base.DownloaderProxyConfig) fetcher.Fetcher { func buildConfigFetcher(t testing.TB, proxyConfig *base.DownloaderProxyConfig) fetcher.Fetcher {
fetcher := new(FetcherManager).Build() fetcher := new(FetcherManager).Build()
newController := controller.NewController() newController := controller.NewController()
mockCfg := config{ mockCfg := config{
@@ -237,29 +238,30 @@ func buildConfigFetcher(proxyConfig *base.DownloaderProxyConfig) fetcher.Fetcher
return proxyConfig.ToHandler() return proxyConfig.ToHandler()
} }
fetcher.Setup(newController) fetcher.Setup(newController)
t.Cleanup(func() { fetcher.Close(); closeClient() })
return fetcher return fetcher
} }
// TestFetcher_Patch tests the Patch functionality for BT fetcher. // TestFetcher_Patch tests the Patch functionality for BT fetcher.
// It tests modifying selected files after Resolve (without downloading). // It tests modifying selected files after Resolve (without downloading).
func TestFetcher_Patch(t *testing.T) { func TestFetcher_Patch(t *testing.T) {
f := buildFetcher() f := buildFetcher(t)
// Resolve a multi-file torrent // Resolve a multi-file torrent
err := f.Resolve(&base.Request{ err := f.Resolve(&base.Request{
URL: "./testdata/test.torrent", URL: "./testdata/test.torrent",
}, nil) })
if err != nil { if err != nil {
t.Fatalf("Resolve failed: %v", err) t.Fatalf("Resolve failed: %v", err)
} }
// Verify initial state: 3 files, all selected by default // Resolution describes files without choosing download options.
meta := f.Meta() meta := f.Meta()
if len(meta.Res.Files) != 3 { if len(meta.Res.Files) != 3 {
t.Fatalf("Expected 3 files, got %d", len(meta.Res.Files)) t.Fatalf("Expected 3 files, got %d", len(meta.Res.Files))
} }
if len(meta.Opts.SelectFiles) != 3 { if meta.Opts != nil {
t.Fatalf("Expected 3 selected files, got %d", len(meta.Opts.SelectFiles)) t.Fatal("Resolve should not assign download options")
} }
// Total size of all files: 107484864 // Total size of all files: 107484864
@@ -395,7 +397,7 @@ func TestFetcher_PeerDiscoveryDisabledOnClient(t *testing.T) {
t.Fatal(err) t.Fatal(err)
} }
f := buildFetcherWithConfig(map[string]any{"disableDht": true, "disablePex": true}) f := buildFetcherWithConfig(map[string]any{"disableDht": true, "disablePex": true})
if err := f.Resolve(&base.Request{URL: "./testdata/test.torrent"}, nil); err != nil { if err := f.Resolve(&base.Request{URL: "./testdata/test.torrent"}); err != nil {
t.Fatal(err) t.Fatal(err)
} }
if !cfg.NoDHT || !cfg.DisablePEX { if !cfg.NoDHT || !cfg.DisablePEX {
+212
View File
@@ -0,0 +1,212 @@
package bt
import (
"bytes"
"context"
"encoding/base64"
"encoding/binary"
"net"
"net/http"
"net/http/httptest"
"net/url"
"os"
"path/filepath"
"strings"
"testing"
"time"
"github.com/anacrolix/torrent"
"github.com/anacrolix/torrent/bencode"
"github.com/anacrolix/torrent/merkle"
"github.com/anacrolix/torrent/metainfo"
"golang.org/x/time/rate"
)
// btFixture serves generated, trackerless content over a real loopback TCP peer.
// Its HTTP tracker exercises .torrent downloads; magnet links also carry the
// explicit x.pe address.
type btFixture struct {
MetaInfo *metainfo.MetaInfo
Magnet string
TorrentFile string
DataURI string
TrackerURL string
Files map[string][]byte // Paths relative to the selected download directory.
Seeder *torrent.Client
Torrent *torrent.Torrent
Limiter *rate.Limiter
}
func newBTFixture(t testing.TB, files map[string][]byte, bytesPerSecond int) *btFixture {
return makeBTFixture(t, files, bytesPerSecond, false, true)
}
func newBTHybridFixture(t testing.TB, payload []byte) *btFixture {
return makeBTFixture(t, map[string][]byte{"": payload}, 0, true, true)
}
func makeBTFixture(t testing.TB, files map[string][]byte, bytesPerSecond int, v2, private bool) *btFixture {
t.Helper()
source := t.TempDir()
name := "payload.bin"
if _, single := files[""]; !single {
name = "bundle"
}
root := filepath.Join(source, name)
expected := make(map[string][]byte, len(files))
for path, data := range files {
target := filepath.Join(root, filepath.FromSlash(path))
if err := os.MkdirAll(filepath.Dir(target), 0700); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(target, data, 0600); err != nil {
t.Fatal(err)
}
expected[filepath.Join(name, filepath.FromSlash(path))] = append([]byte(nil), data...)
}
info := metainfo.Info{PieceLength: 16 * 1024}
if err := info.BuildFromFilePath(root); err != nil {
t.Fatal(err)
}
var layers map[string]string
if v2 {
data := files[""]
h := merkle.NewHash()
h.Write(data)
root := h.Sum(nil)
var pieceHashes []byte
for off := 0; off < len(data); off += int(info.PieceLength) {
end := min(off+int(info.PieceLength), len(data))
ph := merkle.NewHash()
ph.Write(data[off:end])
pieceHashes = append(pieceHashes, ph.SumMinLength(nil, int(info.PieceLength))...)
}
info = metainfo.Info{Name: name, PieceLength: info.PieceLength, Length: int64(len(data)), Pieces: info.Pieces, MetaVersion: 2, FileTree: metainfo.FileTree{File: metainfo.FileTreeFile{Length: int64(len(data)), PiecesRoot: string(root)}}}
layers = map[string]string{string(root): string(pieceHashes)}
}
// Private torrents avoid public DHT discovery once metadata is available.
info.Private = &private
raw, err := bencode.Marshal(&info)
if err != nil {
t.Fatal(err)
}
mi := &metainfo.MetaInfo{InfoBytes: raw, PieceLayers: layers}
cfg := torrent.TestingConfig(t)
cfg.DataDir = source
cfg.Seed = true
cfg.DisableUTP = true
cfg.DisableIPv6 = true
cfg.DisableWebtorrent = true
cfg.KeepAliveTimeout = time.Minute
cfg.MaxAllocPeerRequestDataPerConn = 1 << 20
limiter := rate.NewLimiter(rate.Inf, 16*1024)
if bytesPerSecond > 0 {
limiter.SetLimit(rate.Limit(bytesPerSecond))
}
cfg.UploadRateLimiter = limiter
seed, err := torrent.NewClient(cfg)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { seed.Close() })
seeded, _, err := seed.AddTorrentSpec(torrent.TorrentSpecFromMetaInfo(mi))
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
if err := seeded.VerifyDataContext(ctx); err != nil {
t.Fatal(err)
}
select {
case <-seeded.Complete().On():
case <-ctx.Done():
t.Fatal("local seeder did not finish verification")
}
addr := seed.ListenAddrs()[0].String()
host, portString, err := net.SplitHostPort(addr)
if err != nil {
t.Fatal(err)
}
resolved, err := net.ResolveTCPAddr("tcp4", net.JoinHostPort(host, portString))
if err != nil {
t.Fatal(err)
}
compact := make([]byte, 6)
copy(compact, resolved.IP.To4())
binary.BigEndian.PutUint16(compact[4:], uint16(resolved.Port))
tracker := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
response, err := bencode.Marshal(map[string]any{"interval": 3600, "peers": string(compact)})
if err != nil {
http.Error(w, err.Error(), 500)
return
}
w.Write(response)
}))
t.Cleanup(tracker.Close)
// The pinned library has a hybrid tracker-dispatch assertion bug.
// Hybrid fixtures use their explicit loopback peer rather than a tracker.
if !v2 {
mi.Announce = tracker.URL
}
var encoded bytes.Buffer
if err := mi.Write(&encoded); err != nil {
t.Fatal(err)
}
torrentFile := filepath.Join(t.TempDir(), "fixture.torrent")
if err := os.WriteFile(torrentFile, encoded.Bytes(), 0600); err != nil {
t.Fatal(err)
}
m, err := mi.MagnetV2()
if err != nil {
t.Fatal(err)
}
magnet := m.String()
u, err := url.Parse(magnet)
if err != nil {
t.Fatal(err)
}
q := u.Query()
q.Add("x.pe", addr)
u.RawQuery = q.Encode()
return &btFixture{MetaInfo: mi, Magnet: u.String(), TorrentFile: torrentFile, DataURI: "data:application/x-bittorrent;base64," + base64.StdEncoding.EncodeToString(encoded.Bytes()), TrackerURL: tracker.URL, Files: expected, Seeder: seed, Torrent: seeded, Limiter: limiter}
}
func btPayload(size int, label string) []byte {
data := make([]byte, size)
for i := range data {
data[i] = byte((i*31 + i/251) % 251)
}
copy(data, label)
return data
}
func (f *btFixture) AssertFiles(t testing.TB, dir string, paths ...string) {
t.Helper()
if len(paths) == 0 {
for path := range f.Files {
paths = append(paths, path)
}
}
for _, path := range paths {
got, err := os.ReadFile(filepath.Join(dir, path))
if err != nil {
t.Fatal(err)
}
want, ok := f.Files[path]
if !ok {
t.Fatalf("unknown fixture file %q", path)
}
if !bytes.Equal(got, want) {
t.Fatalf("BT file %s differs: received %d bytes, want %d", path, len(got), len(want))
}
}
}
func (f *btFixture) FileURI() string {
path := filepath.ToSlash(f.TorrentFile)
if !strings.HasPrefix(path, "/") {
path = "/" + path
}
return (&url.URL{Scheme: "file", Path: path}).String()
}
+70 -10
View File
@@ -17,25 +17,85 @@ import (
"github.com/anacrolix/torrent/storage" "github.com/anacrolix/torrent/storage"
) )
func TestStartUsesSelectionSubmittedAfterResolve(t *testing.T) { func TestStartDoesNotPublishUnverifiedCompletion(t *testing.T) {
config := torrent.TestingConfig(t) cfg := torrent.TestingConfig(t)
config.DataDir = t.TempDir() cfg.DisableTCP = true
config.DisableTCP = true cfg.DisableUTP = true
config.DisableUTP = true c, err := torrent.NewClient(cfg)
c, err := torrent.NewClient(config)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
defer c.Close() defer c.Close()
tor, err := c.AddTorrentFromFile("testdata/test.torrent") payload := btPayload(32*1024, "unverified")
info := metainfo.Info{Name: "payload.bin", Length: int64(len(payload)), PieceLength: 16 * 1024}
for offset := 0; offset < len(payload); offset += int(info.PieceLength) {
hash := sha1.Sum(payload[offset : offset+int(info.PieceLength)])
info.Pieces = append(info.Pieces, hash[:]...)
}
raw, err := bencode.Marshal(info)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
f := &Fetcher{torrent: tor, meta: &fetcher.FetcherMeta{Opts: &base.Options{}}, data: &fetcherData{}} store := &deferredFileStorage{}
spec := torrent.TorrentSpecFromMetaInfo(&metainfo.MetaInfo{InfoBytes: raw})
spec.Storage = store
tor, _, err := c.AddTorrentSpec(spec)
if err != nil {
t.Fatal(err)
}
tor.DisallowDataDownload()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
f := &Fetcher{torrent: tor, storage: store, meta: &fetcher.FetcherMeta{}, data: &fetcherData{}, torrentDropCtx: ctx, torrentDropFunc: cancel, config: &config{SeedKeep: true}}
f.torrentReady.Store(true) f.torrentReady.Store(true)
f.updateRes() f.updateRes()
// Resolve initializes all files; creation replaces the selection before Start. dir := t.TempDir()
f.meta.Opts = &base.Options{SelectFiles: []int{2}} payload[20000] ^= 0xff
if err := os.WriteFile(filepath.Join(dir, info.Name), payload, 0600); err != nil {
t.Fatal(err)
}
f.meta.Opts = &base.Options{Path: dir}
// Retain the original event channel to detect even a transient completion.
// With no network transports, the damaged piece cannot have been repaired.
completed := tor.Complete().On()
if err := f.Start(); err != nil {
t.Fatal(err)
}
select {
case <-completed:
t.Fatal("Start published completion before verifying the existing file")
default:
}
}
func TestStartUsesSelectionSubmittedAfterResolve(t *testing.T) {
cfg := torrent.TestingConfig(t)
cfg.DataDir = t.TempDir()
cfg.DisableTCP = true
cfg.DisableUTP = true
c, err := torrent.NewClient(cfg)
if err != nil {
t.Fatal(err)
}
defer c.Close()
store := &deferredFileStorage{}
mi, err := metainfo.LoadFromFile("testdata/test.torrent")
if err != nil {
t.Fatal(err)
}
spec := torrent.TorrentSpecFromMetaInfo(mi)
spec.Storage = store
tor, _, err := c.AddTorrentSpec(spec)
if err != nil {
t.Fatal(err)
}
tor.DisallowDataDownload()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
f := &Fetcher{torrent: tor, storage: store, meta: &fetcher.FetcherMeta{}, data: &fetcherData{}, torrentDropCtx: ctx, torrentDropFunc: cancel, config: &config{SeedKeep: true}}
f.torrentReady.Store(true)
f.updateRes()
f.meta.Opts = &base.Options{Path: t.TempDir(), SelectFiles: []int{2}}
if err := f.Start(); err != nil { if err := f.Start(); err != nil {
t.Fatal(err) t.Fatal(err)
} }
+116
View File
@@ -0,0 +1,116 @@
package bt
import (
"context"
"errors"
"os"
"sync"
g "github.com/anacrolix/generics"
"github.com/anacrolix/torrent/metainfo"
"github.com/anacrolix/torrent/storage"
)
// deferredFileStorage lets metadata resolve without opening the user's files.
// It binds once. After binding, pieces are supplied directly by file storage.
type deferredFileStorage struct {
mu sync.RWMutex
ctx context.Context
info *metainfo.Info
hash metainfo.Hash
knownHashes []bool
owner storage.ClientImplCloser
torrent storage.TorrentImpl
closed bool
}
func (s *deferredFileStorage) OpenTorrent(ctx context.Context, info *metainfo.Info, hash metainfo.Hash) (storage.TorrentImpl, error) {
s.mu.Lock()
defer s.mu.Unlock()
if s.closed || s.info != nil {
return storage.TorrentImpl{}, errors.New("torrent storage already opened or closed")
}
s.ctx, s.info, s.hash = ctx, info, hash
s.knownHashes = make([]bool, info.NumPieces())
return storage.TorrentImpl{
PieceWithHash: func(p metainfo.Piece, hash g.Option[[]byte]) storage.PieceImpl {
s.mu.Lock()
defer s.mu.Unlock()
if s.closed {
return metadataPiece{err: os.ErrClosed}
}
if s.owner == nil {
s.knownHashes[p.Index()] = hash.Ok
// A missing v2 hash must remain unknown until its arrival changes
// completion and updates the library's piece request priorities.
return metadataPiece{known: hash.Ok}
}
if s.torrent.PieceWithHash != nil {
return s.torrent.PieceWithHash(p, hash)
}
return s.torrent.Piece(p)
},
Close: s.Close,
}, nil
}
func (s *deferredFileStorage) bind(opts storage.NewFileClientOpts) ([]bool, error) {
s.mu.Lock()
defer s.mu.Unlock()
if s.closed {
return nil, os.ErrClosed
}
if s.info == nil {
return nil, errors.New("torrent metadata is not ready")
}
if s.owner != nil {
return nil, nil
}
if opts.FilePathMaker == nil {
opts.FilePathMaker = func(o storage.FilePathMakerOpts) string { _, name := torrentFileLayout(o.Info, *o.File); return name }
}
owner := storage.NewFileOpts(opts)
t, err := owner.OpenTorrent(s.ctx, s.info, s.hash)
if err != nil {
owner.Close()
return nil, err
}
s.owner, s.torrent = owner, t
return append([]bool(nil), s.knownHashes...), nil
}
func (s *deferredFileStorage) Close() error {
s.mu.Lock()
defer s.mu.Unlock()
if s.closed {
return nil
}
s.closed = true
if s.owner == nil {
return nil
}
var err error
if s.torrent.Close != nil {
err = s.torrent.Close()
}
return errors.Join(err, s.owner.Close())
}
type metadataPiece struct {
known bool
err error
}
func (p metadataPiece) Completion() storage.Completion {
return storage.Completion{Ok: p.known, Err: p.err}
}
func (p metadataPiece) ReadAt([]byte, int64) (int, error) {
return 0, errors.New("download storage is not initialized")
}
func (p metadataPiece) WriteAt([]byte, int64) (int, error) {
return 0, errors.New("download storage is not initialized")
}
func (p metadataPiece) MarkComplete() error { return errors.New("download storage is not initialized") }
func (p metadataPiece) MarkNotComplete() error {
return errors.New("download storage is not initialized")
}
+2 -5
View File
@@ -67,17 +67,14 @@ func (f *Fetcher) Setup(ctl *controller.Controller) {
f.ctl.GetConfig(&f.config) f.ctl.GetConfig(&f.config)
} }
func (f *Fetcher) Resolve(req *base.Request, opts *base.Options) error { func (f *Fetcher) Resolve(req *base.Request) error {
link, err := parseLink(req.URL) link, err := parseLink(req.URL)
if err != nil { if err != nil {
return err return err
} }
f.meta.Req = req f.meta.Req = req
f.meta.Opts = opts
if f.meta.Opts == nil {
f.meta.Opts = &base.Options{}
}
f.meta.Res = buildResource(link) f.meta.Res = buildResource(link)
return nil return nil
} }
+2 -1
View File
@@ -13,7 +13,8 @@ func TestFetcher_Resolve(t *testing.T) {
f := (&FetcherManager{}).Build() f := (&FetcherManager{}).Build()
f.Setup(controller.NewController()) f.Setup(controller.NewController())
err := f.Resolve(&base.Request{URL: testLink}, &base.Options{Path: t.TempDir()}) f.Meta().Opts = &base.Options{Path: t.TempDir()}
err := f.Resolve(&base.Request{URL: testLink})
if err != nil { if err != nil {
t.Fatalf("Resolve() error = %v", err) t.Fatalf("Resolve() error = %v", err)
} }
+2 -6
View File
@@ -172,11 +172,11 @@ func (f *Fetcher) Meta() *fetcher.FetcherMeta {
return f.meta return f.meta
} }
func (f *Fetcher) Resolve(req *base.Request, opts *base.Options) (err error) { func (f *Fetcher) Resolve(req *base.Request) (err error) {
f.mu.Lock() f.mu.Lock()
defer f.mu.Unlock() defer f.mu.Unlock()
f.meta.Req = req f.meta.Req = req
f.meta.Opts = opts
f.extra = &ReqExtra{} f.extra = &ReqExtra{}
if err = base.ParseReqExtra[ReqExtra](req); err != nil { if err = base.ParseReqExtra[ReqExtra](req); err != nil {
return return
@@ -194,10 +194,6 @@ func (f *Fetcher) Resolve(req *base.Request, opts *base.Options) (err error) {
} }
state.StagingID = randomStagingID() state.StagingID = randomStagingID()
f.state = state f.state = state
// Pin the staging folder now so the very first persisted state already
// carries it: a crash right after create still resumes into the same
// folder.
f.resolveStagingDirLocked()
size := planTotalSize(state.Segments) size := planTotalSize(state.Segments)
f.meta.Res = &base.Resource{ f.meta.Res = &base.Resource{
Range: false, Range: false,
+16 -8
View File
@@ -80,7 +80,8 @@ func mustResolve(t *testing.T, f *Fetcher, playlistURL, referer string) {
if referer != "" { if referer != "" {
req.Extra = &ReqExtra{Header: map[string]string{"Referer": referer}} req.Extra = &ReqExtra{Header: map[string]string{"Referer": referer}}
} }
if err := f.Resolve(req, &base.Options{Path: t.TempDir()}); err != nil { f.Meta().Opts = &base.Options{Path: t.TempDir()}
if err := f.Resolve(req); err != nil {
t.Fatalf("resolve: %v", err) t.Fatalf("resolve: %v", err)
} }
} }
@@ -207,7 +208,8 @@ func TestFetcherE2E_RefererEnforced(t *testing.T) {
// Without Referer the resolve itself must fail (403 on the playlist). // Without Referer the resolve itself must fail (403 on the playlist).
blocked := newTestFetcher(t, testConfig()) blocked := newTestFetcher(t, testConfig())
req := &base.Request{URL: server.URL + "/video.m3u8"} req := &base.Request{URL: server.URL + "/video.m3u8"}
if err := blocked.Resolve(req, &base.Options{Path: t.TempDir()}); err == nil { blocked.Meta().Opts = &base.Options{Path: t.TempDir()}
if err := blocked.Resolve(req); err == nil {
t.Fatal("resolve without Referer should fail") t.Fatal("resolve without Referer should fail")
} }
@@ -217,7 +219,8 @@ func TestFetcherE2E_RefererEnforced(t *testing.T) {
URL: server.URL + "/video.m3u8", URL: server.URL + "/video.m3u8",
Extra: &ReqExtra{Header: map[string]string{"Referer": page}}, Extra: &ReqExtra{Header: map[string]string{"Referer": page}},
} }
if err := allowed.Resolve(allowedReq, &base.Options{Path: t.TempDir()}); err != nil { allowed.Meta().Opts = &base.Options{Path: t.TempDir()}
if err := allowed.Resolve(allowedReq); err != nil {
t.Fatal(err) t.Fatal(err)
} }
mustRun(t, allowed) mustRun(t, allowed)
@@ -560,7 +563,8 @@ func TestFetcherE2E_StoreRestoreAuthenticated(t *testing.T) {
URL: server.URL + "/video.m3u8", URL: server.URL + "/video.m3u8",
Extra: &ReqExtra{Header: map[string]string{"Referer": page}}, Extra: &ReqExtra{Header: map[string]string{"Referer": page}},
} }
if err := f.Resolve(req, &base.Options{Path: t.TempDir()}); err != nil { f.Meta().Opts = &base.Options{Path: t.TempDir()}
if err := f.Resolve(req); err != nil {
t.Fatal(err) t.Fatal(err)
} }
if err := f.Start(); err != nil { if err := f.Start(); err != nil {
@@ -630,7 +634,8 @@ func TestFetcherE2E_PatchPartialUpdate(t *testing.T) {
Extra: &ReqExtra{Header: map[string]string{"Referer": page}}, Extra: &ReqExtra{Header: map[string]string{"Referer": page}},
} }
opts := &base.Options{Path: t.TempDir()} opts := &base.Options{Path: t.TempDir()}
if err := f.Resolve(req, opts); err != nil { f.Meta().Opts = opts
if err := f.Resolve(req); err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -713,7 +718,8 @@ func TestFetcherE2E_PatchWhileRunningRejected(t *testing.T) {
defer server.Close() defer server.Close()
f := newTestFetcher(t, testConfig()) f := newTestFetcher(t, testConfig())
if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: t.TempDir()}); err != nil { f.Meta().Opts = &base.Options{Path: t.TempDir()}
if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}); err != nil {
t.Fatal(err) t.Fatal(err)
} }
if err := f.Start(); err != nil { if err := f.Start(); err != nil {
@@ -767,7 +773,8 @@ func TestFetcherE2E_RetryAfterFailure(t *testing.T) {
cfg.MaxRetries = 0 // fail the run on the first bad segment cfg.MaxRetries = 0 // fail the run on the first bad segment
cfg.PrefetchContentLength = false cfg.PrefetchContentLength = false
f := newTestFetcher(t, cfg) f := newTestFetcher(t, cfg)
if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: t.TempDir()}); err != nil { f.Meta().Opts = &base.Options{Path: t.TempDir()}
if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}); err != nil {
t.Fatal(err) t.Fatal(err)
} }
if err := f.Start(); err != nil { if err := f.Start(); err != nil {
@@ -828,7 +835,8 @@ func TestFetcherE2E_StateMissingSelfHeal(t *testing.T) {
defer server.Close() defer server.Close()
f := newTestFetcher(t, testConfig()) f := newTestFetcher(t, testConfig())
if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: t.TempDir()}); err != nil { f.Meta().Opts = &base.Options{Path: t.TempDir()}
if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}); err != nil {
t.Fatal(err) t.Fatal(err)
} }
// Simulate the crash window where the engine persisted the task with its // Simulate the crash window where the engine persisted the task with its
+6 -3
View File
@@ -189,7 +189,8 @@ func TestHLSRealMedia_TS(t *testing.T) {
defer server.Close() defer server.Close()
f := newTestFetcher(t, testConfig()) f := newTestFetcher(t, testConfig())
if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: t.TempDir()}); err != nil { f.Meta().Opts = &base.Options{Path: t.TempDir()}
if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}); err != nil {
t.Fatal(err) t.Fatal(err)
} }
if err := f.Start(); err != nil { if err := f.Start(); err != nil {
@@ -225,7 +226,8 @@ func TestHLSRealMedia_FMP4(t *testing.T) {
defer server.Close() defer server.Close()
f := newTestFetcher(t, testConfig()) f := newTestFetcher(t, testConfig())
if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: t.TempDir()}); err != nil { f.Meta().Opts = &base.Options{Path: t.TempDir()}
if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}); err != nil {
t.Fatal(err) t.Fatal(err)
} }
if err := f.Start(); err != nil { if err := f.Start(); err != nil {
@@ -274,7 +276,8 @@ func TestHLSRealMedia_AES128(t *testing.T) {
defer server.Close() defer server.Close()
f := newTestFetcher(t, testConfig()) f := newTestFetcher(t, testConfig())
if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: t.TempDir()}); err != nil { f.Meta().Opts = &base.Options{Path: t.TempDir()}
if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}); err != nil {
t.Fatal(err) t.Fatal(err)
} }
if f.meta.Res.Size != 0 { if f.meta.Res.Size != 0 {
+15 -7
View File
@@ -50,15 +50,20 @@ func TestFetcherE2E_SameURLTwoTasks(t *testing.T) {
// user duplicating a task. The engine would rename the second output // user duplicating a task. The engine would rename the second output
// (AutoRename); mirror that here. // (AutoRename); mirror that here.
f1 := newTestFetcher(t, cfg) f1 := newTestFetcher(t, cfg)
if err := f1.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: dir}); err != nil { f1.Meta().Opts = &base.Options{Path: dir}
if err := f1.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}); err != nil {
t.Fatal(err) t.Fatal(err)
} }
f2 := newTestFetcher(t, cfg) f2 := newTestFetcher(t, cfg)
if err := f2.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: dir, Name: "video-2.ts"}); err != nil { f2.Meta().Opts = &base.Options{Path: dir, Name: "video-2.ts"}
if err := f2.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}); err != nil {
t.Fatal(err) t.Fatal(err)
} }
if f1.state.StagingDir == "" || f1.state.StagingDir == f2.state.StagingDir { if f1.state.StagingID == "" || f1.state.StagingID == f2.state.StagingID {
t.Fatalf("staging dirs must be distinct per task: %q vs %q", f1.state.StagingDir, f2.state.StagingDir) t.Fatal("staging identities must be distinct per task")
}
if f1.state.StagingDir != "" || f2.state.StagingDir != "" {
t.Fatal("Resolve pinned an output directory")
} }
// Start task 1, let it stage some segments, then pause it. // Start task 1, let it stage some segments, then pause it.
@@ -135,7 +140,8 @@ func TestFetcherE2E_ForeignJournalDiscarded(t *testing.T) {
cfg := testConfig() cfg := testConfig()
cfg.PrefetchContentLength = false cfg.PrefetchContentLength = false
f := newTestFetcher(t, cfg) f := newTestFetcher(t, cfg)
if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: t.TempDir()}); err != nil { f.Meta().Opts = &base.Options{Path: t.TempDir()}
if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}); err != nil {
t.Fatal(err) t.Fatal(err)
} }
// Plant a journal that does not belong to this plan (wrong fingerprint). // Plant a journal that does not belong to this plan (wrong fingerprint).
@@ -176,7 +182,8 @@ func TestFetcherE2E_MergeFailurePreservesStaging(t *testing.T) {
defer server.Close() defer server.Close()
f := newTestFetcher(t, testConfig()) f := newTestFetcher(t, testConfig())
if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: t.TempDir()}); err != nil { f.Meta().Opts = &base.Options{Path: t.TempDir()}
if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}); err != nil {
t.Fatal(err) t.Fatal(err)
} }
run := &hlsRun{ run := &hlsRun{
@@ -250,7 +257,8 @@ func TestFetcherE2E_ReMergeOverExistingOutput(t *testing.T) {
defer server.Close() defer server.Close()
f := newTestFetcher(t, testConfig()) f := newTestFetcher(t, testConfig())
if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: t.TempDir()}); err != nil { f.Meta().Opts = &base.Options{Path: t.TempDir()}
if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}); err != nil {
t.Fatal(err) t.Fatal(err)
} }
tempDir := f.resolveStagingDirLocked() tempDir := f.resolveStagingDirLocked()
+4 -9
View File
@@ -372,19 +372,11 @@ func (f *Fetcher) updateMaxConnTime(d time.Duration) {
} }
} }
func (f *Fetcher) Resolve(req *base.Request, opts *base.Options) error { func (f *Fetcher) Resolve(req *base.Request) error {
if err := base.ParseReqExtra[fhttp.ReqExtra](req); err != nil { if err := base.ParseReqExtra[fhttp.ReqExtra](req); err != nil {
return err return err
} }
f.meta.Req = req f.meta.Req = req
f.meta.Opts = opts
if f.meta.Opts == nil {
f.meta.Opts = &base.Options{}
}
if err := f.initOptions(); err != nil {
return err
}
f.setState(stateResolving) f.setState(stateResolving)
@@ -693,6 +685,9 @@ func (f *Fetcher) prepareFullRedownload() {
// Options may be replaced between Resolve and Start when creating a resolved task. // Options may be replaced between Resolve and Start when creating a resolved task.
func (f *Fetcher) initOptions() error { func (f *Fetcher) initOptions() error {
if f.meta.Opts == nil {
f.meta.Opts = &base.Options{}
}
opts := f.meta.Opts opts := f.meta.Opts
// Parse options // Parse options
if err := base.ParseOptExtra[fhttp.OptsExtra](opts); err != nil { if err := base.ParseOptExtra[fhttp.OptsExtra](opts); err != nil {
@@ -95,9 +95,10 @@ func TestFetcher_ChecksumMismatch_Retry(t *testing.T) {
} }
// 1. Initial attempt -> will download corruptContent, failing checksum verification // 1. Initial attempt -> will download corruptContent, failing checksum verification
if err := f.Resolve(&base.Request{URL: server.URL + "/retry_test.txt"}, opts); err != nil { if err := f.Resolve(&base.Request{URL: server.URL + "/retry_test.txt"}); err != nil {
t.Fatalf("Resolve error: %v", err) t.Fatalf("Resolve error: %v", err)
} }
f.Meta().Opts = opts
if err := f.Start(); err != nil { if err := f.Start(); err != nil {
t.Fatalf("Start error: %v", err) t.Fatalf("Start error: %v", err)
} }
@@ -215,9 +216,10 @@ func TestFetcher_ChecksumMismatch_Retry_PartialPrefetch(t *testing.T) {
} }
// 1. Resolve phase // 1. Resolve phase
if err := f.Resolve(&base.Request{URL: server.URL + "/prefetch_retry.txt"}, opts); err != nil { if err := f.Resolve(&base.Request{URL: server.URL + "/prefetch_retry.txt"}); err != nil {
t.Fatalf("Resolve error: %v", err) t.Fatalf("Resolve error: %v", err)
} }
f.Meta().Opts = opts
// Wait until prefetch has received exactly 7 bytes // Wait until prefetch has received exactly 7 bytes
for i := 0; i < 200; i++ { for i := 0; i < 200; i++ {
@@ -338,9 +340,10 @@ func TestFetcher_ChecksumMismatch_Retry_NonRange(t *testing.T) {
} }
// 1. Resolve and start initial attempt // 1. Resolve and start initial attempt
if err := f.Resolve(&base.Request{URL: server.URL + "/nonrange_retry.txt"}, opts); err != nil { if err := f.Resolve(&base.Request{URL: server.URL + "/nonrange_retry.txt"}); err != nil {
t.Fatalf("Resolve error: %v", err) t.Fatalf("Resolve error: %v", err)
} }
f.Meta().Opts = opts
if err := f.Start(); err != nil { if err := f.Start(); err != nil {
t.Fatalf("Start error: %v", err) t.Fatalf("Start error: %v", err)
} }
+38 -15
View File
@@ -52,9 +52,11 @@ func TestFetcher_ChecksumVerification(t *testing.T) {
}, },
} }
if err := f.Resolve(&base.Request{URL: server.URL + "/test.txt"}, opts); err != nil { if err := f.Resolve(&base.Request{URL: server.URL + "/test.txt"}); err != nil {
t.Fatalf("Resolve error: %v", err) t.Fatalf("Resolve error: %v", err)
} }
f.Meta().Opts = opts
if err := f.Start(); err != nil { if err := f.Start(); err != nil {
t.Fatalf("Start error: %v", err) t.Fatalf("Start error: %v", err)
} }
@@ -79,9 +81,11 @@ func TestFetcher_ChecksumVerification(t *testing.T) {
}, },
} }
if err := f.Resolve(&base.Request{URL: server.URL + "/test.txt"}, opts); err != nil { if err := f.Resolve(&base.Request{URL: server.URL + "/test.txt"}); err != nil {
t.Fatalf("Resolve error: %v", err) t.Fatalf("Resolve error: %v", err)
} }
f.Meta().Opts = opts
if err := f.Start(); err != nil { if err := f.Start(); err != nil {
t.Fatalf("Start error: %v", err) t.Fatalf("Start error: %v", err)
} }
@@ -106,9 +110,11 @@ func TestFetcher_ChecksumVerification(t *testing.T) {
}, },
} }
if err := f.Resolve(&base.Request{URL: server.URL + "/test.txt"}, opts); err != nil { if err := f.Resolve(&base.Request{URL: server.URL + "/test.txt"}); err != nil {
t.Fatalf("Resolve error: %v", err) t.Fatalf("Resolve error: %v", err)
} }
f.Meta().Opts = opts
if err := f.Start(); err != nil { if err := f.Start(); err != nil {
t.Fatalf("Start error: %v", err) t.Fatalf("Start error: %v", err)
} }
@@ -137,9 +143,13 @@ func TestFetcher_ChecksumVerification(t *testing.T) {
}, },
} }
err := f.Resolve(&base.Request{URL: server.URL + "/test.txt"}, opts) if err := f.Resolve(&base.Request{URL: server.URL + "/test.txt"}); err != nil {
t.Fatalf("Resolve error: %v", err)
}
f.Meta().Opts = opts
err := f.Start()
if err == nil { if err == nil {
t.Fatal("Resolve expected error for unsupported algorithm, got nil") t.Fatal("Start expected error for unsupported algorithm, got nil")
} }
if !strings.Contains(err.Error(), "unsupported checksum algorithm") { if !strings.Contains(err.Error(), "unsupported checksum algorithm") {
t.Fatalf("expected 'unsupported checksum algorithm' in error, got %v", err) t.Fatalf("expected 'unsupported checksum algorithm' in error, got %v", err)
@@ -165,9 +175,11 @@ func TestFetcher_ChecksumVerification(t *testing.T) {
}, },
} }
if err := f.Resolve(&base.Request{URL: server.URL + "/test.txt"}, opts); err != nil { if err := f.Resolve(&base.Request{URL: server.URL + "/test.txt"}); err != nil {
t.Fatalf("Resolve error: %v", err) t.Fatalf("Resolve error: %v", err)
} }
f.Meta().Opts = opts
if err := f.Start(); err != nil { if err := f.Start(); err != nil {
t.Fatalf("Start error: %v", err) t.Fatalf("Start error: %v", err)
} }
@@ -192,9 +204,11 @@ func TestFetcher_ChecksumVerification(t *testing.T) {
}, },
} }
if err := f.Resolve(&base.Request{URL: server.URL + "/test.txt"}, opts); err != nil { if err := f.Resolve(&base.Request{URL: server.URL + "/test.txt"}); err != nil {
t.Fatalf("Resolve error: %v", err) t.Fatalf("Resolve error: %v", err)
} }
f.Meta().Opts = opts
if err := f.Start(); err != nil { if err := f.Start(); err != nil {
t.Fatalf("Start error: %v", err) t.Fatalf("Start error: %v", err)
} }
@@ -235,9 +249,11 @@ func TestFetcher_ChecksumVerification(t *testing.T) {
}, },
} }
if err := f.Resolve(&base.Request{URL: largeServer.URL + "/large.bin"}, opts); err != nil { if err := f.Resolve(&base.Request{URL: largeServer.URL + "/large.bin"}); err != nil {
t.Fatalf("Resolve error: %v", err) t.Fatalf("Resolve error: %v", err)
} }
f.Meta().Opts = opts
if err := f.Start(); err != nil { if err := f.Start(); err != nil {
t.Fatalf("Start error: %v", err) t.Fatalf("Start error: %v", err)
} }
@@ -258,9 +274,11 @@ func TestFetcher_ChecksumVerification(t *testing.T) {
}, },
} }
if err := f.Resolve(&base.Request{URL: server.URL + "/test.txt"}, opts); err != nil { if err := f.Resolve(&base.Request{URL: server.URL + "/test.txt"}); err != nil {
t.Fatalf("Resolve error: %v", err) t.Fatalf("Resolve error: %v", err)
} }
f.Meta().Opts = opts
if err := f.Start(); err != nil { if err := f.Start(); err != nil {
t.Fatalf("Start error: %v", err) t.Fatalf("Start error: %v", err)
} }
@@ -269,7 +287,7 @@ func TestFetcher_ChecksumVerification(t *testing.T) {
} }
}) })
t.Run("Invalid Checksum In Resolve Makes Zero Requests", func(t *testing.T) { t.Run("Invalid Checksum In Start Makes No Additional Requests", func(t *testing.T) {
var reqReceived atomic.Int32 var reqReceived atomic.Int32
customServer := httptest.NewServer(gohttp.HandlerFunc(func(w gohttp.ResponseWriter, r *gohttp.Request) { customServer := httptest.NewServer(gohttp.HandlerFunc(func(w gohttp.ResponseWriter, r *gohttp.Request) {
reqReceived.Add(1) reqReceived.Add(1)
@@ -293,12 +311,17 @@ func TestFetcher_ChecksumVerification(t *testing.T) {
}, },
} }
err := f.Resolve(&base.Request{URL: customServer.URL + "/test.txt"}, opts) if err := f.Resolve(&base.Request{URL: customServer.URL + "/test.txt"}); err != nil {
if err == nil { t.Fatalf("Resolve error: %v", err)
t.Fatal("Resolve expected error for invalid checksum hash, got nil")
} }
if reqReceived.Load() != 0 { requestsAfterResolve := reqReceived.Load()
t.Fatalf("expected 0 requests to server on invalid checksum in Resolve, got %d", reqReceived.Load()) f.Meta().Opts = opts
err := f.Start()
if err == nil {
t.Fatal("Start expected error for invalid checksum hash, got nil")
}
if got := reqReceived.Load(); got != requestsAfterResolve {
t.Fatalf("invalid checksum in Start issued additional requests: got %d, had %d after Resolve", got, requestsAfterResolve)
} }
}) })
} }
File diff suppressed because it is too large. Load diff
+4 -2
View File
@@ -49,7 +49,8 @@ func TestResolveResponseSize(t *testing.T) {
defer server.Close() defer server.Close()
f := buildFetcher() f := buildFetcher()
defer f.Close() defer f.Close()
err := f.Resolve(&base.Request{URL: server.URL + "/file"}, &base.Options{Path: t.TempDir()}) f.Meta().Opts = &base.Options{Path: t.TempDir()}
err := f.Resolve(&base.Request{URL: server.URL + "/file"})
if tc.invalid { if tc.invalid {
if !errors.Is(err, errInvalidRangeResponse) { if !errors.Is(err, errInvalidRangeResponse) {
t.Fatalf("Resolve error = %v, want invalid range", err) t.Fatalf("Resolve error = %v, want invalid range", err)
@@ -97,7 +98,8 @@ func TestDownloadPartialResolve(t *testing.T) {
defer server.Close() defer server.Close()
f := buildFetcher() f := buildFetcher()
defer f.Close() defer f.Close()
if err := f.Resolve(&base.Request{URL: server.URL + "/file"}, &base.Options{Path: t.TempDir(), Name: "file", Extra: &fhttp.OptsExtra{Connections: connections}}); err != nil { f.Meta().Opts = &base.Options{Path: t.TempDir(), Name: "file", Extra: &fhttp.OptsExtra{Connections: connections}}
if err := f.Resolve(&base.Request{URL: server.URL + "/file"}); err != nil {
t.Fatal(err) t.Fatal(err)
} }
if err := f.Start(); err != nil { if err := f.Start(); err != nil {
+2 -2
View File
@@ -139,7 +139,7 @@ func (s *Service) resolve(ctx *Context) *Response {
if response := decodeRequest(ctx.Body, &request); response != nil { if response := decodeRequest(ctx.Body, &request); response != nil {
return response return response
} }
result, err := s.Downloader.Resolve(request.Req, request.Opts) result, err := s.Downloader.Resolve(request.Req)
if err != nil { if err != nil {
return newErrorResponse(err.Error()) return newErrorResponse(err.Error())
} }
@@ -188,7 +188,7 @@ func (s *Service) patchTask(ctx *Context) *Response {
if taskID == "" { if taskID == "" {
return invalidIDResponse() return invalidIDResponse()
} }
var request restmodel.ResolveTask var request restmodel.PatchTask
if response := decodeRequest(ctx.Body, &request); response != nil { if response := decodeRequest(ctx.Body, &request); response != nil {
return response return response
} }
+82
View File
@@ -0,0 +1,82 @@
package api
import (
"encoding/json"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"reflect"
"testing"
"github.com/GopeedLab/gopeed/pkg/base"
"github.com/GopeedLab/gopeed/pkg/download"
restmodel "github.com/GopeedLab/gopeed/pkg/rest/model"
)
func TestResolveThenCreateWithDownloadOptions(t *testing.T) {
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.Header().Set("Content-Disposition", `attachment; filename="payload.txt"`)
w.Header().Set("Content-Length", "4")
_, _ = w.Write([]byte("test"))
}))
t.Cleanup(upstream.Close)
d := download.NewDownloader(&download.DownloaderConfig{StorageDir: t.TempDir()})
if err := d.Setup(); err != nil {
t.Fatal(err)
}
t.Cleanup(func() {
if err := d.Close(); err != nil {
t.Error(err)
}
})
// Resolving must work even when the default download directory is unusable.
blocked := filepath.Join(t.TempDir(), "not-a-directory")
if err := os.WriteFile(blocked, nil, 0600); err != nil {
t.Fatal(err)
}
current, err := d.GetConfig()
if err != nil {
t.Fatal(err)
}
config := *current
config.DownloadDir = blocked
config.MaxRunning = 0 // Inspect the created task before it starts downloading.
if err := d.PutConfig(&config); err != nil {
t.Fatal(err)
}
service, err := NewService(d)
if err != nil {
t.Fatal(err)
}
resolved := serviceTestRequest[*download.ResolveResult](t, service, http.MethodPost, "/api/v1/resolve", &restmodel.ResolveTask{
Req: &base.Request{URL: upstream.URL},
})
if resolved.ID == "" || resolved.Res.Size != 4 || resolved.Res.Files[0].Name != "payload.txt" {
t.Fatalf("unexpected resolved resource: %+v", resolved)
}
dir := filepath.Join(t.TempDir(), "chosen-after-resolve")
opts := &base.Options{Path: dir, Name: "renamed.txt", SelectFiles: []int{0}}
id := serviceTestRequest[string](t, service, http.MethodPost, "/api/v1/tasks", &restmodel.CreateTask{Rid: resolved.ID, Opts: opts})
task := serviceTestRequest[*download.Task](t, service, http.MethodGet, "/api/v1/tasks/"+id, nil)
if task.Meta.Opts.Path != dir || task.Meta.Opts.Name != opts.Name || !reflect.DeepEqual(task.Meta.Opts.SelectFiles, opts.SelectFiles) {
t.Fatalf("creation lost submitted download options: %+v", task.Meta.Opts)
}
}
func serviceTestRequest[T any](t *testing.T, service *Service, method, path string, body any) T {
t.Helper()
encoded, err := json.Marshal(body)
if err != nil {
t.Fatal(err)
}
status, response := service.JSON(method, path, nil, encoded, nil)
var result restmodel.Result[T]
if err := json.Unmarshal(response, &result); err != nil {
t.Fatal(err)
}
if status != http.StatusOK || result.Code != restmodel.CodeOk {
t.Fatalf("%s %s failed: status=%d response=%s", method, path, status, response)
}
return result.Data
}
+41 -29
View File
@@ -244,16 +244,25 @@ func (d *Downloader) Setup() error {
tasks := append([]*Task(nil), d.tasks...) tasks := append([]*Task(nil), d.tasks...)
d.lock.Unlock() d.lock.Unlock()
for _, task := range tasks { for _, task := range tasks {
if task.Status == base.DownloadStatusDone && task.Uploading { func() {
task.lock.Lock()
defer task.lock.Unlock()
if d.closed.Load() {
return
}
if d.taskStatus(task) != base.DownloadStatusDone || !task.Uploading {
return
}
if err := d.restoreTask(task); err != nil { if err := d.restoreTask(task); err != nil {
d.Logger.Error().Stack().Err(err).Msgf("task upload restore fetcher failed, task id: %s", task.ID) d.Logger.Error().Err(err).Msgf("task upload restore fetcher failed, task id: %s", task.ID)
return
} }
if uploader, ok := task.fetcher.(fetcher.Uploader); ok { if uploader, ok := task.fetcher.(fetcher.Uploader); ok {
if err := uploader.Upload(); err != nil { if err := uploader.Upload(); err != nil {
d.Logger.Error().Stack().Err(err).Msgf("task upload failed, task id: %s", task.ID) d.Logger.Error().Err(err).Msgf("task upload failed, task id: %s", task.ID)
} }
} }
} }()
} }
}() }()
@@ -467,7 +476,10 @@ func (d *Downloader) saveTask(task *Task) error {
return err return err
} }
} }
if err := d.storage.Put(bucketTask, task.ID, task); err != nil { task.statusLock.Lock()
snapshot := task.clone()
task.statusLock.Unlock()
if err := d.storage.Put(bucketTask, task.ID, snapshot); err != nil {
d.Logger.Error().Stack().Err(err).Msgf("persist task failed: %s", task.ID) d.Logger.Error().Stack().Err(err).Msgf("persist task failed: %s", task.ID)
return err return err
} }
@@ -498,7 +510,7 @@ func ensureResourceRequestRawURLs(parentReq *base.Request, res *base.Resource) {
} }
} }
func (d *Downloader) Resolve(req *base.Request, opts *base.Options) (rr *ResolveResult, err error) { func (d *Downloader) Resolve(req *base.Request) (rr *ResolveResult, err error) {
rrId, err := gonanoid.New() rrId, err := gonanoid.New()
if err != nil { if err != nil {
return return
@@ -520,11 +532,7 @@ func (d *Downloader) Resolve(req *base.Request, opts *base.Options) (rr *Resolve
if err != nil { if err != nil {
return return
} }
initOpt, err := d.initOptions(opts) err = fetcher.Resolve(req)
if err != nil {
return
}
err = fetcher.Resolve(req, initOpt)
if err != nil { if err != nil {
return return
} }
@@ -558,7 +566,7 @@ func (d *Downloader) notifyRunning() {
func (d *Downloader) remainRunningCount() int { func (d *Downloader) remainRunningCount() int {
runningCount := 0 runningCount := 0
for _, t := range d.tasks { for _, t := range d.tasks {
if t.Status == base.DownloadStatusRunning { if d.taskStatus(t) == base.DownloadStatusRunning {
runningCount++ runningCount++
} }
} }
@@ -621,7 +629,7 @@ func (d *Downloader) Create(rrId string) (taskId string, err error) {
} }
// CreateWithOptions creates a resolved task using the submitted options, or the // CreateWithOptions creates a resolved task using the submitted options, or the
// resolve-time options when opts is nil. // default download options when opts is nil.
func (d *Downloader) CreateWithOptions(rrId string, opts *base.Options) (taskId string, err error) { func (d *Downloader) CreateWithOptions(rrId string, opts *base.Options) (taskId string, err error) {
d.fetcherMapLock.RLock() d.fetcherMapLock.RLock()
fetcher, ok := d.fetcherCache[rrId] fetcher, ok := d.fetcherCache[rrId]
@@ -629,21 +637,22 @@ func (d *Downloader) CreateWithOptions(rrId string, opts *base.Options) (taskId
if !ok { if !ok {
return "", errors.New("invalid resource id") return "", errors.New("invalid resource id")
} }
if opts != nil { if opts == nil {
opts, err = d.initOptions(opts.Clone()) opts = &base.Options{}
if err != nil {
return "", err
}
if res := fetcher.Meta().Res; res != nil {
for _, index := range opts.SelectFiles {
if index < 0 || index >= len(res.Files) {
return "", fmt.Errorf("invalid selected file index: %d", index)
}
}
opts.InitSelectFiles(len(res.Files))
}
fetcher.Meta().Opts = opts
} }
opts, err = d.initOptions(opts.Clone())
if err != nil {
return "", err
}
if res := fetcher.Meta().Res; res != nil {
for _, index := range opts.SelectFiles {
if index < 0 || index >= len(res.Files) {
return "", fmt.Errorf("invalid selected file index: %d", index)
}
}
opts.InitSelectFiles(len(res.Files))
}
fetcher.Meta().Opts = opts
if res := fetcher.Meta().Res; res != nil { if res := fetcher.Meta().Res; res != nil {
res.CalcSize(fetcher.Meta().Opts.SelectFiles) res.CalcSize(fetcher.Meta().Opts.SelectFiles)
} }
@@ -1469,7 +1478,10 @@ func (d *Downloader) watch(task *Task) {
if err := d.captureTaskStats(task, false, true); err != nil { if err := d.captureTaskStats(task, false, true); err != nil {
d.Logger.Error().Stack().Err(err).Msgf("persist completed task stats failed, task id: %s", task.ID) d.Logger.Error().Stack().Err(err).Msgf("persist completed task stats failed, task id: %s", task.ID)
} }
d.storage.Put(bucketTask, task.ID, task.clone()) task.statusLock.Lock()
snapshot := task.clone()
task.statusLock.Unlock()
d.storage.Put(bucketTask, task.ID, snapshot)
d.emit(EventKeyDone, task) d.emit(EventKeyDone, task)
d.emit(EventKeyFinally, task, err) d.emit(EventKeyFinally, task, err)
d.notifyRunning() d.notifyRunning()
@@ -1826,7 +1838,7 @@ func (d *Downloader) doStart(task *Task) (err error) {
return err return err
} }
if task.Meta.Res == nil { if task.Meta.Res == nil {
err := task.fetcher.Resolve(task.Meta.Req, task.Meta.Opts) err := task.fetcher.Resolve(task.Meta.Req)
if err != nil { if err != nil {
return err return err
} }
+23 -23
View File
@@ -132,7 +132,7 @@ func (f *generationTestFetcher) Setup(*controller.Controller) {
f.done = make(chan error, 2) f.done = make(chan error, 2)
} }
} }
func (f *generationTestFetcher) Resolve(req *base.Request, opts *base.Options) error { func (f *generationTestFetcher) Resolve(req *base.Request) error {
if f.manager.resolveStarted != nil { if f.manager.resolveStarted != nil {
f.manager.resolveOnce.Do(func() { close(f.manager.resolveStarted) }) f.manager.resolveOnce.Do(func() { close(f.manager.resolveStarted) })
} }
@@ -143,7 +143,7 @@ func (f *generationTestFetcher) Resolve(req *base.Request, opts *base.Options) e
return f.manager.resolveErr return f.manager.resolveErr
} }
f.meta.Req = req f.meta.Req = req
f.meta.Opts = opts
f.meta.Res = &base.Resource{Files: []*base.FileInfo{{Name: "generation.bin", Size: 1}}} f.meta.Res = &base.Resource{Files: []*base.FileInfo{{Name: "generation.bin", Size: 1}}}
return nil return nil
} }
@@ -245,7 +245,7 @@ func TestDownloader_Resolve(t *testing.T) {
req := &base.Request{ req := &base.Request{
URL: "http://" + listener.Addr().String() + "/" + test.BuildName, URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
} }
rr, err := downloader.Resolve(req, nil) rr, err := downloader.Resolve(req)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -772,7 +772,7 @@ func TestDownloader_Create(t *testing.T) {
URL: "http://" + listener.Addr().String() + "/" + test.BuildName, URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
} }
opts := newTestDownloadOpt(t) opts := newTestDownloadOpt(t)
rr, err := downloader.Resolve(req, opts) rr, err := downloader.Resolve(req)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -784,7 +784,7 @@ func TestDownloader_Create(t *testing.T) {
wg.Done() wg.Done()
} }
}) })
_, err = downloader.Create(rr.ID) _, err = downloader.CreateWithOptions(rr.ID, opts)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -878,15 +878,15 @@ func TestDownloader_CreateNotInWhite(t *testing.T) {
req := &base.Request{ req := &base.Request{
URL: "http://" + listener.Addr().String() + "/" + test.BuildName, URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
} }
// With new fetcher design, white list check happens during Resolve (not Create) rr, err := downloader.Resolve(req)
// because Resolve now requires Options which includes the download path if err != nil {
_, err := downloader.Resolve(req, newTestDownloadOpt(t)) t.Fatal(err)
}
_, err = downloader.CreateWithOptions(rr.ID, newTestDownloadOpt(t))
if err == nil { if err == nil {
t.Error("TestDownloader_CreateNotInWhite() expected error but got nil") t.Fatal("Create should reject a directory outside the allowlist")
}
if !strings.Contains(err.Error(), "white") {
t.Errorf("TestDownloader_CreateNotInWhite() got = %v, want error containing 'white'", err.Error())
} }
} }
func TestDownloader_CreateDirectBatch(t *testing.T) { func TestDownloader_CreateDirectBatch(t *testing.T) {
@@ -1072,7 +1072,7 @@ func doTestDownloaderCreateWithProxy(t *testing.T, auth bool, buildReqProxy func
Pwd: pwd, Pwd: pwd,
}) })
} }
rr, err := downloader.Resolve(req, nil) rr, err := downloader.Resolve(req)
if err != nil { if err != nil {
if errHandler == nil { if errHandler == nil {
t.Fatal(err) t.Fatal(err)
@@ -1158,7 +1158,7 @@ func TestDownloader_StoreAndRestore(t *testing.T) {
URL: "http://" + listener.Addr().String() + "/" + test.BuildName, URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
} }
opts := newTestDownloadOpt(t) opts := newTestDownloadOpt(t)
rr, err := downloader.Resolve(req, opts) rr, err := downloader.Resolve(req)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -1172,7 +1172,7 @@ func TestDownloader_StoreAndRestore(t *testing.T) {
}) })
} }
}) })
id, err := downloader.Create(rr.ID) id, err := downloader.CreateWithOptions(rr.ID, opts)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -1434,7 +1434,7 @@ func TestDownloader_Stats(t *testing.T) {
req := &base.Request{ req := &base.Request{
URL: "http://" + listener.Addr().String() + "/" + test.BuildName, URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
} }
rr, err := downloader.Resolve(req, newTestDownloadOpt(t)) rr, err := downloader.Resolve(req)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -1447,7 +1447,7 @@ func TestDownloader_Stats(t *testing.T) {
} }
}) })
taskId, err := downloader.Create(rr.ID) taskId, err := downloader.CreateWithOptions(rr.ID, newTestDownloadOpt(t))
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -1677,7 +1677,7 @@ func TestDownloader_CompletedHTTPStatsSurviveRestore(t *testing.T) {
t.Fatal(err) t.Fatal(err)
} }
req := &base.Request{URL: "http://" + listener.Addr().String() + "/" + test.BuildName} req := &base.Request{URL: "http://" + listener.Addr().String() + "/" + test.BuildName}
rr, err := downloader.Resolve(req, newTestDownloadOpt(t)) rr, err := downloader.Resolve(req)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -1688,7 +1688,7 @@ func TestDownloader_CompletedHTTPStatsSurviveRestore(t *testing.T) {
done <- struct{}{} done <- struct{}{}
} }
}) })
taskID, err := downloader.Create(rr.ID) taskID, err := downloader.CreateWithOptions(rr.ID, newTestDownloadOpt(t))
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -1861,11 +1861,11 @@ func TestDownloader_PauseAndContinue(t *testing.T) {
req := &base.Request{ req := &base.Request{
URL: "http://" + listener.Addr().String() + "/" + test.BuildName, URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
} }
rr, err := downloader.Resolve(req, newTestDownloadOpt(t)) rr, err := downloader.Resolve(req)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
taskId, err := downloader.Create(rr.ID) taskId, err := downloader.CreateWithOptions(rr.ID, newTestDownloadOpt(t))
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -2045,11 +2045,11 @@ func TestDownloader_PauseAllAndContinueAll(t *testing.T) {
req := &base.Request{ req := &base.Request{
URL: "http://" + listener.Addr().String() + "/" + test.BuildName, URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
} }
rr, err := downloader.Resolve(req, newTestDownloadOpt(t)) rr, err := downloader.Resolve(req)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
taskId, err := downloader.Create(rr.ID) taskId, err := downloader.CreateWithOptions(rr.ID, newTestDownloadOpt(t))
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
+4
View File
@@ -753,6 +753,10 @@ type OnErrorExtensionTask struct {
} }
func cloneExtensionTask(task *Task) *Task { func cloneExtensionTask(task *Task) *Task {
if task.statusLock != nil {
task.statusLock.Lock()
defer task.statusLock.Unlock()
}
newTask := task.clone() newTask := task.clone()
newTask.Meta.Req = task.Meta.Req newTask.Meta.Req = task.Meta.Req
return newTask return newTask
+2 -2
View File
@@ -65,11 +65,11 @@ func TestExtensionExtraMutationHTTP(t *testing.T) {
var id string var id string
var err error var err error
if resolved { if resolved {
rr, resolveErr := d.Resolve(req, opts) rr, resolveErr := d.Resolve(req)
if resolveErr != nil { if resolveErr != nil {
t.Fatal(resolveErr) t.Fatal(resolveErr)
} }
id, err = d.Create(rr.ID) id, err = d.CreateWithOptions(rr.ID, opts)
} else { } else {
id, err = d.CreateDirect(req, opts) id, err = d.CreateDirect(req, opts)
} }
+24 -24
View File
@@ -32,7 +32,7 @@ func TestDownloader_InstallExtensionByFolder(t *testing.T) {
URL: "https://github.com/test", URL: "https://github.com/test",
Labels: map[string]string{"original": "true"}, Labels: map[string]string{"original": "true"},
} }
rr, err := downloader.Resolve(req, nil) rr, err := downloader.Resolve(req)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -58,7 +58,7 @@ func TestDownloader_InstallExtensionByFolderDevMode(t *testing.T) {
} }
rr, err := downloader.Resolve(&base.Request{ rr, err := downloader.Resolve(&base.Request{
URL: "https://github.com/test", URL: "https://github.com/test",
}, nil) })
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -76,7 +76,7 @@ func TestDownloader_Extension_BlobBlob(t *testing.T) {
rr, err := downloader.Resolve(&base.Request{ rr, err := downloader.Resolve(&base.Request{
URL: "https://example.com/blob", URL: "https://example.com/blob",
}, nil) })
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -130,7 +130,7 @@ func TestDownloader_Extension_BlobFunctionOpener(t *testing.T) {
rr, err := downloader.Resolve(&base.Request{ rr, err := downloader.Resolve(&base.Request{
URL: "https://example.com/opener", URL: "https://example.com/opener",
}, nil) })
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -166,7 +166,7 @@ func TestDownloader_Extension_BlobFunctionOpenerUnknownSize(t *testing.T) {
rr, err := downloader.Resolve(&base.Request{ rr, err := downloader.Resolve(&base.Request{
URL: "https://example.com/opener-unknown", URL: "https://example.com/opener-unknown",
}, nil) })
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -222,7 +222,7 @@ func TestDownloader_Extension_BlobSourceSizePropagatesToCreatedTask(t *testing.T
rr, err := downloader.Resolve(&base.Request{ rr, err := downloader.Resolve(&base.Request{
URL: "https://example.com/opener", URL: "https://example.com/opener",
}, nil) })
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -303,7 +303,7 @@ func TestDownloader_Extension_BlobFetchDrainedStreamReportsEarlyProgress(t *test
rr, err := downloader.Resolve(&base.Request{ rr, err := downloader.Resolve(&base.Request{
URL: fmt.Sprintf("%s/single-fetch-stream?size=%d", server.URL, len(payload)), URL: fmt.Sprintf("%s/single-fetch-stream?size=%d", server.URL, len(payload)),
}, nil) })
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -419,7 +419,7 @@ func TestDownloader_Extension_BlobRangeFetchDrainedStreamReportsEarlyProgress(t
rr, err := downloader.Resolve(&base.Request{ rr, err := downloader.Resolve(&base.Request{
URL: fmt.Sprintf("%s/range-fetch-stream?size=%d", server.URL, len(payload)), URL: fmt.Sprintf("%s/range-fetch-stream?size=%d", server.URL, len(payload)),
}, nil) })
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -494,7 +494,7 @@ func TestDownloader_Extension_BlobMultiplexFetchStreamsReportEarlyProgress(t *te
rr, err := downloader.Resolve(&base.Request{ rr, err := downloader.Resolve(&base.Request{
URL: server.URL + "/multiplex-fetch-stream", URL: server.URL + "/multiplex-fetch-stream",
}, nil) })
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -550,7 +550,7 @@ func TestDownloader_Extension_BlobFunctionOpenerRangeResume(t *testing.T) {
rr, err := downloader.Resolve(&base.Request{ rr, err := downloader.Resolve(&base.Request{
URL: "https://example.com/opener-range", URL: "https://example.com/opener-range",
}, nil) })
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -683,7 +683,7 @@ func TestDownloader_Extension_BlobResumeAfterRestartViaOnError(t *testing.T) {
rr, err := downloader.Resolve(&base.Request{ rr, err := downloader.Resolve(&base.Request{
URL: server.URL + "/restart-range", URL: server.URL + "/restart-range",
}, nil) })
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -855,7 +855,7 @@ func TestDownloader_Extension_BlobPauseContinueKeepsSource(t *testing.T) {
rr, err := downloader.Resolve(&base.Request{ rr, err := downloader.Resolve(&base.Request{
URL: tc.url, URL: tc.url,
}, nil) })
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -932,7 +932,7 @@ func TestDownloader_Extension_BlobUnknownSizePauseContinueKeepsSource(t *testing
rr, err := downloader.Resolve(&base.Request{ rr, err := downloader.Resolve(&base.Request{
URL: "https://example.com/opener-unknown", URL: "https://example.com/opener-unknown",
}, nil) })
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -1009,7 +1009,7 @@ func TestDownloader_Extension_BlobRecoverOnError(t *testing.T) {
path := path path := path
t.Run(path, func(t *testing.T) { t.Run(path, func(t *testing.T) {
rawURL := "https://example.com/" + path rawURL := "https://example.com/" + path
rr, err := downloader.Resolve(&base.Request{URL: rawURL}, nil) rr, err := downloader.Resolve(&base.Request{URL: rawURL})
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -1076,7 +1076,7 @@ func TestDownloader_InstallExtensionByGit(t *testing.T) {
} }
rr, err := downloader.Resolve(&base.Request{ rr, err := downloader.Resolve(&base.Request{
URL: "https://github.com/GopeedLab/gopeed/releases", URL: "https://github.com/GopeedLab/gopeed/releases",
}, nil) })
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -1093,7 +1093,7 @@ func TestDownloader_InstallExtensionByGitSimple(t *testing.T) {
} }
rr, err := downloader.Resolve(&base.Request{ rr, err := downloader.Resolve(&base.Request{
URL: "https://github.com/GopeedLab/gopeed/releases", URL: "https://github.com/GopeedLab/gopeed/releases",
}, nil) })
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -1110,7 +1110,7 @@ func TestDownloader_InstallExtensionByGitFull(t *testing.T) {
} }
rr, err := downloader.Resolve(&base.Request{ rr, err := downloader.Resolve(&base.Request{
URL: "https://github.com/GopeedLab/gopeed/releases", URL: "https://github.com/GopeedLab/gopeed/releases",
}, nil) })
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -1180,7 +1180,7 @@ func TestDownloader_UpgradeExtension(t *testing.T) {
rr, err := downloader.Resolve(&base.Request{ rr, err := downloader.Resolve(&base.Request{
URL: "https://test.com", URL: "https://test.com",
}, nil) })
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -1355,7 +1355,7 @@ func TestDownloader_Extension_Errors(t *testing.T) {
} }
rr, err := downloader.Resolve(&base.Request{ rr, err := downloader.Resolve(&base.Request{
URL: "https://github.com/test", URL: "https://github.com/test",
}, nil) })
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -1370,7 +1370,7 @@ func TestDownloader_Extension_Errors(t *testing.T) {
} }
rr, err := downloader.Resolve(&base.Request{ rr, err := downloader.Resolve(&base.Request{
URL: "https://github.com/test", URL: "https://github.com/test",
}, nil) })
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -1385,7 +1385,7 @@ func TestDownloader_Extension_Errors(t *testing.T) {
} }
_, err := downloader.Resolve(&base.Request{ _, err := downloader.Resolve(&base.Request{
URL: "https://github.com/test", URL: "https://github.com/test",
}, nil) })
if err == nil { if err == nil {
t.Fatalf("except error, but got nil") t.Fatalf("except error, but got nil")
} }
@@ -1407,7 +1407,7 @@ func TestDownloader_Extension_Settings(t *testing.T) {
} }
rr, err := downloader.Resolve(&base.Request{ rr, err := downloader.Resolve(&base.Request{
URL: "https://github.com/test", URL: "https://github.com/test",
}, nil) })
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -1428,7 +1428,7 @@ func TestDownloader_Extension_Settings(t *testing.T) {
}) })
rr, err := downloader.Resolve(&base.Request{ rr, err := downloader.Resolve(&base.Request{
URL: "https://github.com/test", URL: "https://github.com/test",
}, nil) })
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -1445,7 +1445,7 @@ func TestDownloader_ExtensionStorage(t *testing.T) {
} }
rr, err := downloader.Resolve(&base.Request{ rr, err := downloader.Resolve(&base.Request{
URL: "https://github.com/test", URL: "https://github.com/test",
}, nil) })
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -24,7 +24,7 @@ func TestExtensionUnnamedResource(t *testing.T) {
if _, err = d.InstallExtensionByFolder(dir, true); err != nil { if _, err = d.InstallExtensionByFolder(dir, true); err != nil {
t.Fatal(err) t.Fatal(err)
} }
res, err := d.Resolve(&base.Request{URL: "https://example.com/unnamed"}, nil) res, err := d.Resolve(&base.Request{URL: "https://example.com/unnamed"})
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
+1 -1
View File
@@ -40,7 +40,7 @@ func TestDownloader_HlsLiveSite(t *testing.T) {
if referer != "" { if referer != "" {
req.Extra = map[string]any{"header": map[string]string{"Referer": referer}} req.Extra = map[string]any{"header": map[string]string{"Referer": referer}}
} }
rr, err := downloader.Resolve(req, nil) rr, err := downloader.Resolve(req)
if err != nil { if err != nil {
t.Fatalf("resolve: %v", err) t.Fatalf("resolve: %v", err)
} }
+1 -1
View File
@@ -89,7 +89,7 @@ func TestDownloader_HlsProtocolEndToEnd(t *testing.T) {
req := hlsResolveReferer() req := hlsResolveReferer()
req.URL = server.URL + "/master.m3u8" req.URL = server.URL + "/master.m3u8"
rr, err := downloader.Resolve(req, nil) rr, err := downloader.Resolve(req)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
+4 -4
View File
@@ -18,7 +18,7 @@ func TestCreateResolvedFileSelection(t *testing.T) {
}{ }{
{"partial", []int{2, 0}, true, []int{2, 0}, 40}, {"partial", []int{2, 0}, true, []int{2, 0}, 40},
{"default all", nil, true, []int{0, 1, 2}, 60}, {"default all", nil, true, []int{0, 1, 2}, 60},
{"legacy retains resolve options", nil, false, []int{1}, 20}, {"default creation options", nil, false, []int{0, 1, 2}, 60},
} { } {
t.Run(tc.name, func(t *testing.T) { t.Run(tc.name, func(t *testing.T) {
manager := &generationTestManager{} manager := &generationTestManager{}
@@ -28,8 +28,8 @@ func TestCreateResolvedFileSelection(t *testing.T) {
} }
defer d.Clear() defer d.Clear()
d.cfg.MaxRunning = 0 d.cfg.MaxRunning = 0
original := &base.Options{Path: t.TempDir(), SelectFiles: []int{1}} original := &base.Options{Path: t.TempDir()}
rr, err := d.Resolve(&base.Request{URL: "generation://selection"}, original) rr, err := d.Resolve(&base.Request{URL: "generation://selection"})
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -40,7 +40,7 @@ func TestCreateResolvedFileSelection(t *testing.T) {
if tc.submit { if tc.submit {
id, err = d.CreateWithOptions(rr.ID, &base.Options{Path: original.Path, SelectFiles: tc.selection}) id, err = d.CreateWithOptions(rr.ID, &base.Options{Path: original.Path, SelectFiles: tc.selection})
} else { } else {
id, err = d.Create(rr.ID) id, err = d.CreateWithOptions(rr.ID, original)
} }
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
+2 -3
View File
@@ -222,8 +222,7 @@ func (o *downloadOptions) baseOptions() *base.Options {
} }
type resolveTaskInput struct { type resolveTaskInput struct {
Req *downloadRequest `json:"req" jsonschema:"Download request to resolve"` Req *downloadRequest `json:"req" jsonschema:"Download request to resolve"`
Opts *downloadOptions `json:"opts,omitempty" jsonschema:"Optional download settings used while resolving"`
} }
type resolveTaskOutput struct { type resolveTaskOutput struct {
@@ -328,7 +327,7 @@ func registerTools(server *mcp.Server, downloader *download.Downloader) {
if err != nil { if err != nil {
return nil, nil, err return nil, nil, err
} }
result, err := downloader.Resolve(req, input.Opts.baseOptions()) result, err := downloader.Resolve(req)
if err != nil { if err != nil {
return nil, nil, err return nil, nil, err
} }
+7
View File
@@ -101,6 +101,13 @@ func TestStreamableHTTPTools(t *testing.T) {
if reqExtra["type"] != "object" || !ok || len(branches) != 2 { if reqExtra["type"] != "object" || !ok || len(branches) != 2 {
t.Fatalf("%s req.extra schema = %#v, want an object with two oneOf branches", tool.Name, reqExtra) t.Fatalf("%s req.extra schema = %#v, want an object with two oneOf branches", tool.Name, reqExtra)
} }
if tool.Name == "resolve_task" {
properties := schema["properties"].(map[string]any)
if _, exists := properties["opts"]; exists || len(properties) != 1 {
t.Fatalf("resolve_task must expose only req: %s", inputSchema)
}
continue
}
optsExtra := nestedSchemaProperty(t, schema, "opts", "extra") optsExtra := nestedSchemaProperty(t, schema, "opts", "extra")
if !schemaIncludesType(optsExtra, "object") || optsExtra["additionalProperties"] != false { if !schemaIncludesType(optsExtra, "object") || optsExtra["additionalProperties"] != false {
t.Fatalf("%s opts.extra schema = %#v, want concrete HTTP task settings", tool.Name, optsExtra) t.Fatalf("%s opts.extra schema = %#v, want concrete HTTP task settings", tool.Name, optsExtra)
+2 -2
View File
@@ -26,7 +26,7 @@ func Info(w http.ResponseWriter, r *http.Request) {
func Resolve(w http.ResponseWriter, r *http.Request) { func Resolve(w http.ResponseWriter, r *http.Request) {
var req model.ResolveTask var req model.ResolveTask
if ReadJson(r, w, &req) { if ReadJson(r, w, &req) {
rr, err := Downloader.Resolve(req.Req, req.Opts) rr, err := Downloader.Resolve(req.Req)
if err != nil { if err != nil {
WriteJson(w, model.NewErrorResult(err.Error())) WriteJson(w, model.NewErrorResult(err.Error()))
return return
@@ -82,7 +82,7 @@ func PatchTask(w http.ResponseWriter, r *http.Request) {
return return
} }
var req model.ResolveTask var req model.PatchTask
if ReadJson(r, w, &req) { if ReadJson(r, w, &req) {
if err := Downloader.Patch(taskId, req.Req, req.Opts); err != nil { if err := Downloader.Patch(taskId, req.Req, req.Opts); err != nil {
if err == download.ErrTaskNotFound { if err == download.ErrTaskNotFound {
+4
View File
@@ -3,6 +3,10 @@ package model
import "github.com/GopeedLab/gopeed/pkg/base" import "github.com/GopeedLab/gopeed/pkg/base"
type ResolveTask struct { type ResolveTask struct {
Req *base.Request `json:"req"`
}
type PatchTask struct {
Req *base.Request `json:"req"` Req *base.Request `json:"req"`
Opts *base.Options `json:"opts"` Opts *base.Options `json:"opts"`
} }
+5 -5
View File
@@ -55,8 +55,7 @@ var (
}, },
} }
resolveReq = &model.ResolveTask{ resolveReq = &model.ResolveTask{
Req: taskReq, Req: taskReq,
Opts: createOpts,
} }
createReq = &model.CreateTask{ createReq = &model.CreateTask{
Req: taskReq, Req: taskReq,
@@ -105,7 +104,8 @@ func TestCreateTask(t *testing.T) {
}) })
taskId := httpRequestCheckOk[string](http.MethodPost, "/api/v1/tasks", &model.CreateTask{ taskId := httpRequestCheckOk[string](http.MethodPost, "/api/v1/tasks", &model.CreateTask{
Rid: resp.ID, Rid: resp.ID,
Opts: createOpts,
}) })
if taskId == "" { if taskId == "" {
t.Fatal("create task failed") t.Fatal("create task failed")
@@ -295,7 +295,7 @@ func TestPatchTask(t *testing.T) {
time.Sleep(time.Millisecond * 100) time.Sleep(time.Millisecond * 100)
// Patch the task with new labels // Patch the task with new labels
patchReq := &model.ResolveTask{ patchReq := &model.PatchTask{
Req: &base.Request{ Req: &base.Request{
Labels: map[string]string{ Labels: map[string]string{
"patched": "true", "patched": "true",
@@ -322,7 +322,7 @@ func TestPatchTask(t *testing.T) {
func TestPatchTaskNotFound(t *testing.T) { func TestPatchTaskNotFound(t *testing.T) {
doTest(func() { doTest(func() {
// Try to patch a non-existent task // Try to patch a non-existent task
patchReq := &model.ResolveTask{ patchReq := &model.PatchTask{
Req: &base.Request{ Req: &base.Request{
Labels: map[string]string{ Labels: map[string]string{
"test": "value", "test": "value",
+2 -1
View File
@@ -10,6 +10,7 @@ import 'model/install_extension.dart';
import 'model/login.dart'; import 'model/login.dart';
import 'model/resolve_result.dart'; import 'model/resolve_result.dart';
import 'model/resolve_task.dart'; import 'model/resolve_task.dart';
import 'model/patch_task.dart';
import 'model/result.dart'; import 'model/result.dart';
import 'model/switch_extension.dart'; import 'model/switch_extension.dart';
import 'model/task.dart'; import 'model/task.dart';
@@ -91,7 +92,7 @@ Future<List<String>> createTaskBatch(CreateTaskBatch createTaskBatch) {
); );
} }
Future<void> patchTask(String id, ResolveTask patchTask) { Future<void> patchTask(String id, PatchTask patchTask) {
return _parse<void>(() => _request('api/v1/tasks/$id', method: 'PATCH', data: patchTask), null); return _parse<void>(() => _request('api/v1/tasks/$id', method: 'PATCH', data: patchTask), null);
} }
+16
View File
@@ -0,0 +1,16 @@
import 'options.dart';
import 'request.dart';
class PatchTask {
final Request? req;
final Options? opts;
PatchTask({this.req, this.opts});
factory PatchTask.fromJson(Map<String, dynamic> json) => PatchTask(
req: json['req'] == null ? null : Request.fromJson(json['req'] as Map<String, dynamic>),
opts: json['opts'] == null ? null : Options.fromJson(json['opts'] as Map<String, dynamic>),
);
Map<String, dynamic> toJson() => {'req': req?.toJson(), 'opts': opts?.toJson()};
}
+1 -3
View File
@@ -1,6 +1,5 @@
import 'package:json_annotation/json_annotation.dart'; import 'package:json_annotation/json_annotation.dart';
import 'options.dart';
import 'request.dart'; import 'request.dart';
part 'resolve_task.g.dart'; part 'resolve_task.g.dart';
@@ -8,9 +7,8 @@ part 'resolve_task.g.dart';
@JsonSerializable() @JsonSerializable()
class ResolveTask { class ResolveTask {
Request? req; Request? req;
Options? opts;
ResolveTask({this.req, this.opts}); ResolveTask({this.req});
factory ResolveTask.fromJson(Map<String, dynamic> json) => _$ResolveTaskFromJson(json); factory ResolveTask.fromJson(Map<String, dynamic> json) => _$ResolveTaskFromJson(json);
Map<String, dynamic> toJson() => _$ResolveTaskToJson(this); Map<String, dynamic> toJson() => _$ResolveTaskToJson(this);
+5 -6
View File
@@ -7,11 +7,10 @@ part of 'resolve_task.dart';
// ************************************************************************** // **************************************************************************
ResolveTask _$ResolveTaskFromJson(Map<String, dynamic> json) => ResolveTask( ResolveTask _$ResolveTaskFromJson(Map<String, dynamic> json) => ResolveTask(
req: json['req'] == null ? null : Request.fromJson(json['req'] as Map<String, dynamic>), req: json['req'] == null
opts: json['opts'] == null ? null : Options.fromJson(json['opts'] as Map<String, dynamic>), ? null
: Request.fromJson(json['req'] as Map<String, dynamic>),
); );
Map<String, dynamic> _$ResolveTaskToJson(ResolveTask instance) => <String, dynamic>{ Map<String, dynamic> _$ResolveTaskToJson(ResolveTask instance) =>
'req': ?instance.req, <String, dynamic>{'req': ?instance.req};
'opts': ?instance.opts,
};
@@ -8,6 +8,7 @@ import '../../api/model/extension.dart';
import '../../api/model/install_extension.dart'; import '../../api/model/install_extension.dart';
import '../../api/model/resolve_result.dart'; import '../../api/model/resolve_result.dart';
import '../../api/model/resolve_task.dart'; import '../../api/model/resolve_task.dart';
import '../../api/model/patch_task.dart';
import '../../api/model/switch_extension.dart'; import '../../api/model/switch_extension.dart';
import '../../api/model/task.dart'; import '../../api/model/task.dart';
import '../../api/model/update_check_extension_resp.dart'; import '../../api/model/update_check_extension_resp.dart';
@@ -85,7 +86,7 @@ void _bindGopeed(CapabilityRegistry registry) {
..bind(GopeedMethods.createTask, api.createTask) ..bind(GopeedMethods.createTask, api.createTask)
..bind(GopeedMethods.createTaskBatch, api.createTaskBatch) ..bind(GopeedMethods.createTaskBatch, api.createTaskBatch)
..bind(GopeedMethods.patchTask, (params) async { ..bind(GopeedMethods.patchTask, (params) async {
await api.patchTask(params['id'] as String, ResolveTask.fromJson(params['request'] as Map<String, dynamic>)); await api.patchTask(params['id'] as String, PatchTask.fromJson(params['request'] as Map<String, dynamic>));
return const RpcUnit(); return const RpcUnit();
}) })
..bind( ..bind(
@@ -5,6 +5,7 @@ import '../../api/model/extension.dart';
import '../../api/model/install_extension.dart'; import '../../api/model/install_extension.dart';
import '../../api/model/resolve_result.dart'; import '../../api/model/resolve_result.dart';
import '../../api/model/resolve_task.dart'; import '../../api/model/resolve_task.dart';
import '../../api/model/patch_task.dart';
import '../../api/model/switch_extension.dart'; import '../../api/model/switch_extension.dart';
import '../../api/model/task.dart'; import '../../api/model/task.dart';
import '../../api/model/update_check_extension_resp.dart'; import '../../api/model/update_check_extension_resp.dart';
@@ -49,7 +50,7 @@ class GopeedService {
Future<List<String>> createTaskBatch(CreateTaskBatch request) => Future<List<String>> createTaskBatch(CreateTaskBatch request) =>
_invoker.invoke(GopeedMethods.createTaskBatch, request); _invoker.invoke(GopeedMethods.createTaskBatch, request);
Future<void> patchTask(String id, ResolveTask request) async { Future<void> patchTask(String id, PatchTask request) async {
await _invoker.invoke(GopeedMethods.patchTask, {'id': id, 'request': request.toJson()}); await _invoker.invoke(GopeedMethods.patchTask, {'id': id, 'request': request.toJson()});
} }
@@ -3,7 +3,7 @@ import 'dart:async';
import 'package:flutter_riverpod/flutter_riverpod.dart'; import 'package:flutter_riverpod/flutter_riverpod.dart';
import '../../../api/model/request.dart'; import '../../../api/model/request.dart';
import '../../../api/model/resolve_task.dart'; import '../../../api/model/patch_task.dart';
import '../../../api/model/task.dart' as api_model; import '../../../api/model/task.dart' as api_model;
import '../../../core/capabilities/app_capabilities.dart'; import '../../../core/capabilities/app_capabilities.dart';
import '../domain/task_record.dart'; import '../domain/task_record.dart';
@@ -99,7 +99,7 @@ class TasksController extends AsyncNotifier<TasksState> {
} }
Future<void> updateRequest(String id, Request request) async { Future<void> updateRequest(String id, Request request) async {
await ref.read(gopeedServiceProvider).patchTask(id, ResolveTask(req: request)); await ref.read(gopeedServiceProvider).patchTask(id, PatchTask(req: request));
await ref.read(gopeedServiceProvider).continueTask(id); await ref.read(gopeedServiceProvider).continueTask(id);
await refresh(silent: true); await refresh(silent: true);
} }
@@ -927,8 +927,7 @@ class _CreateTaskWindowPageState extends ConsumerState<CreateTaskWindowPage> {
} }
final request = _buildRequest(urls.first, protocolUrl: _protocolUrlFor(urls.first)); final request = _buildRequest(urls.first, protocolUrl: _protocolUrlFor(urls.first));
final options = _buildOptions(); final result = await ref.read(gopeedServiceProvider).resolve(ResolveTask(req: request));
final result = await ref.read(gopeedServiceProvider).resolve(ResolveTask(req: request, opts: options));
if (!mounted) return; if (!mounted) return;
final created = await _showResolveDialog(request, result); final created = await _showResolveDialog(request, result);
if (!created) { if (!created) {
+32 -7
View File
@@ -3,30 +3,49 @@ import 'dart:convert';
import 'package:dio/dio.dart' as dio; import 'package:dio/dio.dart' as dio;
import 'package:flutter_test/flutter_test.dart'; import 'package:flutter_test/flutter_test.dart';
import 'package:gopeed/api/api.dart' as api; import 'package:gopeed/api/api.dart' as api;
import 'package:gopeed/api/model/create_task.dart';
import 'package:gopeed/api/model/options.dart'; import 'package:gopeed/api/model/options.dart';
import 'package:gopeed/api/model/patch_task.dart';
import 'package:gopeed/api/model/request.dart'; import 'package:gopeed/api/model/request.dart';
import 'package:gopeed/api/model/resolve_task.dart'; import 'package:gopeed/api/model/resolve_task.dart';
import 'package:gopeed/core/network/gopeed/gopeed_transport.dart'; import 'package:gopeed/core/network/gopeed/gopeed_transport.dart';
void main() { void main() {
test('resolve sends the complete resolve task for magnet links', () async { test('resolve omits download options and creation submits them separately', () async {
final transport = _RecordingTransport(); final transport = _RecordingTransport();
api.setTransportForTesting(transport); api.setTransportForTesting(transport);
final result = await api.resolve( final result = await api.resolve(
ResolveTask( ResolveTask(req: Request(url: 'magnet:?xt=urn:btih:0123456789abcdef0123456789abcdef01234567')),
req: Request(url: 'magnet:?xt=urn:btih:0123456789abcdef0123456789abcdef01234567'),
opts: Options(path: '/downloads', selectFiles: const []),
),
); );
expect(result.id, 'resolved-id'); expect(result.id, 'resolved-id');
final requestBody = transport.data as Map<String, dynamic>; final requestBody = transport.data as Map<String, dynamic>;
expect(requestBody.keys, containsAll(<String>['req', 'opts'])); expect(requestBody.keys, equals(<String>['req']));
expect( expect(
(requestBody['req'] as Map<String, dynamic>)['url'], (requestBody['req'] as Map<String, dynamic>)['url'],
'magnet:?xt=urn:btih:0123456789abcdef0123456789abcdef01234567', 'magnet:?xt=urn:btih:0123456789abcdef0123456789abcdef01234567',
); );
expect((requestBody['opts'] as Map<String, dynamic>)['path'], '/downloads'); expect(requestBody, isNot(contains('opts')));
final id = await api.createTask(
CreateTask(
rid: result.id,
opts: Options(path: '/chosen-after-resolve', selectFiles: [1]),
),
);
expect(id, 'created-id');
final createBody = transport.data as Map<String, dynamic>;
expect(createBody['rid'], 'resolved-id');
expect((createBody['opts'] as Map<String, dynamic>)['path'], '/chosen-after-resolve');
expect((createBody['opts'] as Map<String, dynamic>)['selectFiles'], [1]);
});
test('patch preserves options in its independent request model', () async {
final transport = _RecordingTransport();
api.setTransportForTesting(transport);
await api.patchTask('task-id', PatchTask(opts: Options(selectFiles: [1])));
final requestBody = transport.data as Map<String, dynamic>;
expect((requestBody['opts'] as Map<String, dynamic>)['selectFiles'], [1]);
}); });
} }
@@ -41,6 +60,12 @@ class _RecordingTransport implements GopeedTransport {
Map<String, dynamic>? queryParameters, Map<String, dynamic>? queryParameters,
}) async { }) async {
this.data = jsonDecode(jsonEncode(data)); this.data = jsonDecode(jsonEncode(data));
if (path == 'api/v1/tasks') {
return {'code': 0, 'data': 'created-id'};
}
if (method == 'PATCH') {
return {'code': 0, 'data': null};
}
return { return {
'code': 0, 'code': 0,
'data': { 'data': {
@@ -1,5 +1,13 @@
import 'package:flutter_test/flutter_test.dart'; import 'package:flutter_test/flutter_test.dart';
import 'package:gopeed/api/model/options.dart';
import 'package:gopeed/api/model/patch_task.dart';
import 'package:gopeed/api/model/request.dart';
import 'package:gopeed/api/model/resolve_result.dart';
import 'package:gopeed/api/model/resolve_task.dart';
import 'package:gopeed/api/model/resource.dart';
import 'package:gopeed/core/capabilities/app_capabilities.dart';
import 'package:gopeed/core/capabilities/capability_rpc.dart'; import 'package:gopeed/core/capabilities/capability_rpc.dart';
import 'package:gopeed/core/capabilities/gopeed_capability.dart';
import 'package:gopeed/core/window/app_window_appearance.dart'; import 'package:gopeed/core/window/app_window_appearance.dart';
class _Payload { class _Payload {
@@ -13,6 +21,33 @@ class _Payload {
} }
void main() { void main() {
test('serialized resolve and patch capabilities keep separate request contracts', () async {
final codecs = createAppCapabilityCodecs();
String? resolvedUrl;
Map<String, dynamic>? patch;
final registry = CapabilityRegistry(codecs)
..bind(GopeedMethods.resolve, (request) {
resolvedUrl = request.req?.url;
return ResolveResult(id: 'resolved-id', res: Resource(files: []));
})
..bind(GopeedMethods.patchTask, (request) {
patch = request;
return const RpcUnit();
});
final request = ResolveTask(req: Request(url: 'magnet:?xt=urn:btih:hash'));
final wireRequest = codecs.encode(request) as Map<String, dynamic>;
expect(wireRequest.keys, ['req']);
final result = codecs.decode<ResolveResult>(await registry.invoke(GopeedMethods.resolve.name, wireRequest));
expect(result.id, 'resolved-id');
expect(resolvedUrl, request.req?.url);
final service = GopeedService(LocalCapabilityInvoker(registry));
await service.patchTask('task-id', PatchTask(opts: Options(selectFiles: [1])));
expect(patch?['id'], 'task-id');
final patchBody = patch?['request'] as Map<String, dynamic>;
expect((patchBody['opts'] as Map<String, dynamic>)['selectFiles'], [1]);
});
test('one method descriptor supports direct and serialized dispatch', () async { test('one method descriptor supports direct and serialized dispatch', () async {
const method = RpcMethod<_Payload, _Payload>('test.increment'); const method = RpcMethod<_Payload, _Payload>('test.increment');
final codecs = RpcCodecRegistry()..register<_Payload>((json) => _Payload.fromJson(json! as Map<String, dynamic>)); final codecs = RpcCodecRegistry()..register<_Payload>((json) => _Payload.fromJson(json! as Map<String, dynamic>));