mirror of
https://github.com/GopeedLab/gopeed.git
synced 2026-11-16 13:23:41 +00:00
refactor(downloader): remove opts parameter from Resolve (#1595)
This commit is contained in:
45 files changed
+1589
-384
No files matched your search
+2
-2
@@ -166,7 +166,7 @@ func runResolve(args []string) error {
|
|||||||
result, err := d.Resolve(&base.Request{
|
result, err := d.Resolve(&base.Request{
|
||||||
URL: url,
|
URL: url,
|
||||||
Labels: map[string]string(labels),
|
Labels: map[string]string(labels),
|
||||||
}, nil)
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -250,7 +250,7 @@ func runDownload(args []string) error {
|
|||||||
result, err := d.Resolve(&base.Request{
|
result, err := d.Resolve(&base.Request{
|
||||||
URL: url,
|
URL: url,
|
||||||
Labels: map[string]string(labels),
|
Labels: map[string]string(labels),
|
||||||
}, nil)
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -12,7 +12,7 @@ import (
|
|||||||
// Each download task will have a corresponding Fetcher instance for the management of the download task
|
// Each download task will have a corresponding Fetcher instance for the management of the download task
|
||||||
type Fetcher interface {
|
type Fetcher interface {
|
||||||
Setup(ctl *controller.Controller)
|
Setup(ctl *controller.Controller)
|
||||||
Resolve(req *base.Request, opts *base.Options) error
|
Resolve(req *base.Request) error
|
||||||
Start() error
|
Start() error
|
||||||
// Patch modifies task-specific data based on the protocol.
|
// Patch modifies task-specific data based on the protocol.
|
||||||
// For HTTP: can modify Request info (URL, headers, etc.)
|
// For HTTP: can modify Request info (URL, headers, etc.)
|
||||||
|
|||||||
@@ -0,0 +1,448 @@
|
|||||||
|
package bt
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"net/url"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"sync/atomic"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/GopeedLab/gopeed/internal/controller"
|
||||||
|
"github.com/GopeedLab/gopeed/internal/fetcher"
|
||||||
|
"github.com/GopeedLab/gopeed/pkg/base"
|
||||||
|
btprotocol "github.com/GopeedLab/gopeed/pkg/protocol/bt"
|
||||||
|
"github.com/anacrolix/torrent"
|
||||||
|
pp "github.com/anacrolix/torrent/peer_protocol"
|
||||||
|
"golang.org/x/time/rate"
|
||||||
|
)
|
||||||
|
|
||||||
|
type btDownloadHarness struct {
|
||||||
|
*Fetcher
|
||||||
|
pendingRequests atomic.Int64
|
||||||
|
}
|
||||||
|
|
||||||
|
// These tests exercise the BT fetcher, native storage and real peer protocol.
|
||||||
|
// All content is generated locally and served by a loopback seeder.
|
||||||
|
func btDownloadFetcher(t *testing.T, f fetcher.Fetcher) *btDownloadHarness {
|
||||||
|
t.Helper()
|
||||||
|
if f == nil {
|
||||||
|
f = new(FetcherManager).Build()
|
||||||
|
}
|
||||||
|
ctl := controller.NewController()
|
||||||
|
ctl.GetConfig = func(v any) {
|
||||||
|
encoded, err := json.Marshal(new(FetcherManager).DefaultConfig())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(encoded, v); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
f.Setup(ctl)
|
||||||
|
t.Cleanup(func() {
|
||||||
|
if err := f.Close(); err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
|
if err := closeClient(); err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
h := &btDownloadHarness{Fetcher: f.(*Fetcher)}
|
||||||
|
// Keep the test client on loopback and observe real cancellation replies.
|
||||||
|
// Waiting for them avoids the pinned library's immediate cancel/resume
|
||||||
|
// request scheduling bug, without relying on arbitrary sleep durations.
|
||||||
|
cfg = torrent.TestingConfig(t)
|
||||||
|
cfg.DisableTrackers = false
|
||||||
|
cfg.Seed = true
|
||||||
|
cfg.DisableUTP = true
|
||||||
|
cfg.DisableIPv6 = true
|
||||||
|
cfg.DisableWebtorrent = true
|
||||||
|
cfg.KeepAliveTimeout = time.Minute
|
||||||
|
cfg.MaxAllocPeerRequestDataPerConn = 1 << 20
|
||||||
|
cfg.Callbacks.SentRequest = []func(torrent.PeerRequestEvent){func(torrent.PeerRequestEvent) {
|
||||||
|
h.pendingRequests.Add(1)
|
||||||
|
}}
|
||||||
|
cfg.Callbacks.ReadMessage = func(_ *torrent.PeerConn, msg *pp.Message) {
|
||||||
|
if !msg.Keepalive && (msg.Type == pp.Piece || msg.Type == pp.Reject) {
|
||||||
|
h.pendingRequests.Add(-1)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
var err error
|
||||||
|
client, err = torrent.NewClient(cfg)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return h
|
||||||
|
}
|
||||||
|
|
||||||
|
func btResolve(t *testing.T, f *btDownloadHarness, source string) {
|
||||||
|
t.Helper()
|
||||||
|
if err := f.Resolve(&base.Request{URL: source}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func btStart(t *testing.T, f *btDownloadHarness) {
|
||||||
|
t.Helper()
|
||||||
|
if err := f.Start(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func btAwaitDone(t *testing.T, f *btDownloadHarness) {
|
||||||
|
t.Helper()
|
||||||
|
done := make(chan error, 1)
|
||||||
|
go func() { done <- f.Wait() }()
|
||||||
|
timeout := time.NewTimer(20 * time.Second)
|
||||||
|
defer timeout.Stop()
|
||||||
|
select {
|
||||||
|
case err := <-done:
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("BT download failed: %v", err)
|
||||||
|
}
|
||||||
|
if !f.isDone() {
|
||||||
|
t.Fatal("Wait returned before selected files were verified")
|
||||||
|
}
|
||||||
|
case <-timeout.C:
|
||||||
|
var status strings.Builder
|
||||||
|
client.WriteStatus(&status)
|
||||||
|
t.Log(status.String())
|
||||||
|
t.Fatalf("BT download did not finish: progress=%v stats=%+v pieces=%+v peers=%+v", f.Progress(), f.torrent.Stats(), f.torrent.PieceStateRuns(), f.Stats().Runtime)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func btAwaitPartial(t *testing.T, f *btDownloadHarness) {
|
||||||
|
t.Helper()
|
||||||
|
timeout := time.NewTimer(15 * time.Second)
|
||||||
|
defer timeout.Stop()
|
||||||
|
ticker := time.NewTicker(20 * time.Millisecond)
|
||||||
|
defer ticker.Stop()
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-ticker.C:
|
||||||
|
progress := f.Progress().TotalDownloaded()
|
||||||
|
if progress >= 32*1024 && progress < f.Meta().Res.Size {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
case <-timeout.C:
|
||||||
|
t.Fatal("BT transfer did not reach partial progress")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func btPause(t *testing.T, f *btDownloadHarness) {
|
||||||
|
t.Helper()
|
||||||
|
if err := f.Pause(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
timeout := time.NewTimer(5 * time.Second)
|
||||||
|
defer timeout.Stop()
|
||||||
|
ticker := time.NewTicker(10 * time.Millisecond)
|
||||||
|
defer ticker.Stop()
|
||||||
|
for {
|
||||||
|
if f.pendingRequests.Load() == 0 {
|
||||||
|
// The message callback runs under the client lock. Acquiring that
|
||||||
|
// lock ensures the final cancellation reply has finished processing.
|
||||||
|
client.Stats()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-ticker.C:
|
||||||
|
case <-timeout.C:
|
||||||
|
t.Fatal("peer did not acknowledge cancelled requests")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBTDownloadSources(t *testing.T) {
|
||||||
|
for _, kind := range []string{"magnet", "torrent", "file URI", "data URI"} {
|
||||||
|
t.Run(kind, func(t *testing.T) {
|
||||||
|
fixture := newBTFixture(t, map[string][]byte{"": btPayload(96*1024, kind)}, 0)
|
||||||
|
source := fixture.Magnet
|
||||||
|
switch kind {
|
||||||
|
case "torrent":
|
||||||
|
source = fixture.TorrentFile
|
||||||
|
case "file URI":
|
||||||
|
source = fixture.FileURI()
|
||||||
|
case "data URI":
|
||||||
|
source = fixture.DataURI
|
||||||
|
}
|
||||||
|
f := btDownloadFetcher(t, nil)
|
||||||
|
btResolve(t, f, source)
|
||||||
|
if f.Meta().Opts != nil {
|
||||||
|
t.Fatal("Resolve assigned download options")
|
||||||
|
}
|
||||||
|
f.storage.mu.RLock()
|
||||||
|
bound := f.storage.owner != nil
|
||||||
|
f.storage.mu.RUnlock()
|
||||||
|
if bound {
|
||||||
|
t.Fatal("Resolve bound download storage")
|
||||||
|
}
|
||||||
|
resolvedTorrent := f.torrent
|
||||||
|
dir := filepath.Join(t.TempDir(), "chosen-after-resolve")
|
||||||
|
f.Meta().Opts = &base.Options{Path: dir}
|
||||||
|
if _, err := os.Stat(dir); !os.IsNotExist(err) {
|
||||||
|
t.Fatalf("output directory exists before Start: %v", err)
|
||||||
|
}
|
||||||
|
btStart(t, f)
|
||||||
|
if f.torrent != resolvedTorrent {
|
||||||
|
t.Fatal("Start replaced the resolved torrent")
|
||||||
|
}
|
||||||
|
btAwaitDone(t, f)
|
||||||
|
fixture.AssertFiles(t, dir)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBTDirectDownloadWithExtraTrackers(t *testing.T) {
|
||||||
|
for _, source := range []string{"request", "config"} {
|
||||||
|
t.Run(source, func(t *testing.T) {
|
||||||
|
fixture := makeBTFixture(t, map[string][]byte{
|
||||||
|
"a.bin": btPayload(20*1024, "direct-a"),
|
||||||
|
"nested/b.bin": btPayload(48*1024, "direct-b"),
|
||||||
|
}, 0, false, false)
|
||||||
|
magnet, err := url.Parse(fixture.Magnet)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
query := magnet.Query()
|
||||||
|
query.Del("tr")
|
||||||
|
query.Del("x.pe")
|
||||||
|
magnet.RawQuery = query.Encode()
|
||||||
|
req := &base.Request{URL: magnet.String()}
|
||||||
|
f := btDownloadFetcher(t, nil)
|
||||||
|
if source == "request" {
|
||||||
|
req.Extra = &btprotocol.ReqExtra{Trackers: []string{fixture.TrackerURL}}
|
||||||
|
} else {
|
||||||
|
f.config.Trackers = []string{fixture.TrackerURL}
|
||||||
|
}
|
||||||
|
// CreateDirect supplies options before Downloader resolves the task.
|
||||||
|
dir := filepath.Join(t.TempDir(), "direct-download")
|
||||||
|
opts := &base.Options{Path: dir, SelectFiles: []int{}}
|
||||||
|
f.Meta().Opts = opts
|
||||||
|
resolved := make(chan error, 1)
|
||||||
|
go func() { resolved <- f.Resolve(req) }()
|
||||||
|
select {
|
||||||
|
case err := <-resolved:
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
case <-time.After(5 * time.Second):
|
||||||
|
f.torrentDropFunc()
|
||||||
|
<-resolved
|
||||||
|
t.Fatal("direct magnet download did not obtain metadata from its extra tracker")
|
||||||
|
}
|
||||||
|
if f.Meta().Opts != opts || len(opts.SelectFiles) != len(fixture.Files) {
|
||||||
|
t.Fatal("resolution lost direct download options or did not select all files")
|
||||||
|
}
|
||||||
|
btStart(t, f)
|
||||||
|
btAwaitDone(t, f)
|
||||||
|
fixture.AssertFiles(t, dir)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBTDownloadSelectedFiles(t *testing.T) {
|
||||||
|
fixture := newBTFixture(t, map[string][]byte{
|
||||||
|
"a.bin": btPayload(20*1024, "a"), "nested/b.bin": btPayload(48*1024, "b"),
|
||||||
|
}, 0)
|
||||||
|
f := btDownloadFetcher(t, nil)
|
||||||
|
dir := t.TempDir()
|
||||||
|
btResolve(t, f, fixture.Magnet)
|
||||||
|
selected := -1
|
||||||
|
for i, file := range f.Meta().Res.Files {
|
||||||
|
if file.Name == "b.bin" {
|
||||||
|
selected = i
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if selected < 0 {
|
||||||
|
t.Fatal("selected file absent from metadata")
|
||||||
|
}
|
||||||
|
f.Meta().Opts = &base.Options{Path: dir}
|
||||||
|
if err := f.Patch(nil, &base.Options{SelectFiles: []int{selected}}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
btStart(t, f)
|
||||||
|
btAwaitDone(t, f)
|
||||||
|
fixture.AssertFiles(t, dir, filepath.Join("bundle", "nested", "b.bin"))
|
||||||
|
if _, err := os.Stat(filepath.Join(dir, "bundle", "a.bin")); !os.IsNotExist(err) {
|
||||||
|
t.Fatalf("unselected output remains: %v", err)
|
||||||
|
}
|
||||||
|
if f.Meta().Res.Size != 48*1024 {
|
||||||
|
t.Fatal("selected resource size is incorrect")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBTPauseAndResume(t *testing.T) {
|
||||||
|
fixture := newBTFixture(t, map[string][]byte{"": btPayload(512*1024, "pause")}, 64*1024)
|
||||||
|
f := btDownloadFetcher(t, nil)
|
||||||
|
dir := t.TempDir()
|
||||||
|
btResolve(t, f, fixture.Magnet)
|
||||||
|
f.Meta().Opts = &base.Options{Path: dir}
|
||||||
|
btStart(t, f)
|
||||||
|
btAwaitPartial(t, f)
|
||||||
|
btPause(t, f)
|
||||||
|
resolvedTorrent := f.torrent
|
||||||
|
fixture.Limiter.SetLimit(rate.Inf)
|
||||||
|
// Once cancelled requests have settled, no more data should arrive.
|
||||||
|
stats := f.torrent.Stats()
|
||||||
|
readWhilePaused := stats.BytesReadData.Int64()
|
||||||
|
time.Sleep(200 * time.Millisecond)
|
||||||
|
stats = f.torrent.Stats()
|
||||||
|
if stats.BytesReadData.Int64() != readWhilePaused || f.isDone() {
|
||||||
|
t.Fatal("data download continued while paused")
|
||||||
|
}
|
||||||
|
btStart(t, f)
|
||||||
|
if f.torrent != resolvedTorrent {
|
||||||
|
t.Fatal("resume replaced the resolved torrent")
|
||||||
|
}
|
||||||
|
btAwaitDone(t, f)
|
||||||
|
fixture.AssertFiles(t, dir)
|
||||||
|
}
|
||||||
|
|
||||||
|
type btFetcherCheckpoint struct {
|
||||||
|
Meta *fetcher.FetcherMeta `json:"meta"`
|
||||||
|
Data json.RawMessage `json:"data"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func btStore(t *testing.T, f *btDownloadHarness) []byte {
|
||||||
|
t.Helper()
|
||||||
|
data, err := new(FetcherManager).Store(f.Fetcher)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
encodedData, err := json.Marshal(data)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
encoded, err := json.Marshal(btFetcherCheckpoint{Meta: f.Meta(), Data: encodedData})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return encoded
|
||||||
|
}
|
||||||
|
|
||||||
|
func btRestore(t *testing.T, encoded []byte) *btDownloadHarness {
|
||||||
|
t.Helper()
|
||||||
|
var checkpoint btFetcherCheckpoint
|
||||||
|
if err := json.Unmarshal(encoded, &checkpoint); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
data, restore := new(FetcherManager).Restore()
|
||||||
|
if err := json.Unmarshal(checkpoint.Data, data); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return btDownloadFetcher(t, restore(checkpoint.Meta, data))
|
||||||
|
}
|
||||||
|
|
||||||
|
func btCloseForRestart(t *testing.T, f *btDownloadHarness) {
|
||||||
|
t.Helper()
|
||||||
|
if err := f.Close(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := new(FetcherManager).Close(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBTColdStartResume(t *testing.T) {
|
||||||
|
for _, kind := range []string{"v1", "hybrid"} {
|
||||||
|
t.Run(kind, func(t *testing.T) {
|
||||||
|
var fixture *btFixture
|
||||||
|
if kind == "hybrid" {
|
||||||
|
fixture = newBTHybridFixture(t, btPayload(512*1024, "restart-hybrid"))
|
||||||
|
fixture.Limiter.SetLimit(64 * 1024)
|
||||||
|
} else {
|
||||||
|
fixture = newBTFixture(t, map[string][]byte{"": btPayload(512*1024, "restart")}, 64*1024)
|
||||||
|
}
|
||||||
|
f := btDownloadFetcher(t, nil)
|
||||||
|
dir := t.TempDir()
|
||||||
|
btResolve(t, f, fixture.Magnet)
|
||||||
|
f.Meta().Opts = &base.Options{Path: dir}
|
||||||
|
btStart(t, f)
|
||||||
|
btAwaitPartial(t, f)
|
||||||
|
btPause(t, f)
|
||||||
|
checkpoint := btStore(t, f)
|
||||||
|
hash := f.Meta().Res.Hash
|
||||||
|
btCloseForRestart(t, f)
|
||||||
|
seedStats := fixture.Torrent.Stats()
|
||||||
|
sentBeforeRestart := seedStats.BytesWrittenData.Int64()
|
||||||
|
restored := btRestore(t, checkpoint)
|
||||||
|
if restored.Meta().Opts.Path != dir || restored.Meta().Res.Hash != hash {
|
||||||
|
t.Fatal("download directory or hash was lost")
|
||||||
|
}
|
||||||
|
fixture.Limiter.SetLimit(rate.Inf)
|
||||||
|
btStart(t, restored)
|
||||||
|
btAwaitDone(t, restored)
|
||||||
|
fixture.AssertFiles(t, dir)
|
||||||
|
seedStats = fixture.Torrent.Stats()
|
||||||
|
if sent := seedStats.BytesWrittenData.Int64() - sentBeforeRestart; sent >= 512*1024 {
|
||||||
|
t.Fatalf("restart redownloaded the entire file: %d bytes", sent)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBTDownloadHybrid(t *testing.T) {
|
||||||
|
fixture := newBTHybridFixture(t, btPayload(80*1024, "hybrid-v2"))
|
||||||
|
f := btDownloadFetcher(t, nil)
|
||||||
|
dir := t.TempDir()
|
||||||
|
btResolve(t, f, fixture.Magnet)
|
||||||
|
f.Meta().Opts = &base.Options{Path: dir}
|
||||||
|
btStart(t, f)
|
||||||
|
btAwaitDone(t, f)
|
||||||
|
fixture.AssertFiles(t, dir)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBTRepairCorruptedExistingFile(t *testing.T) {
|
||||||
|
fixture := newBTFixture(t, map[string][]byte{"": btPayload(128*1024, "repair")}, 0)
|
||||||
|
dir := t.TempDir()
|
||||||
|
damaged := append([]byte(nil), fixture.Files["payload.bin"]...)
|
||||||
|
damaged[20000] ^= 0xff
|
||||||
|
if err := os.WriteFile(filepath.Join(dir, "payload.bin"), damaged, 0600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
f := btDownloadFetcher(t, nil)
|
||||||
|
btResolve(t, f, fixture.Magnet)
|
||||||
|
f.Meta().Opts = &base.Options{Path: dir}
|
||||||
|
btStart(t, f)
|
||||||
|
btAwaitDone(t, f)
|
||||||
|
fixture.AssertFiles(t, dir)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBTPatchSelectionWhilePaused(t *testing.T) {
|
||||||
|
fixture := newBTFixture(t, map[string][]byte{"a.bin": btPayload(256*1024, "patch-a"), "b.bin": btPayload(256*1024, "patch-b")}, 64*1024)
|
||||||
|
f := btDownloadFetcher(t, nil)
|
||||||
|
dir := t.TempDir()
|
||||||
|
btResolve(t, f, fixture.Magnet)
|
||||||
|
f.Meta().Opts = &base.Options{Path: dir, SelectFiles: []int{0}}
|
||||||
|
btStart(t, f)
|
||||||
|
btAwaitPartial(t, f)
|
||||||
|
btPause(t, f)
|
||||||
|
if err := f.Patch(nil, &base.Options{SelectFiles: []int{1}}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
fixture.Limiter.SetLimit(rate.Inf)
|
||||||
|
btStart(t, f)
|
||||||
|
btAwaitDone(t, f)
|
||||||
|
fixture.AssertFiles(t, dir, filepath.Join("bundle", "b.bin"))
|
||||||
|
for _, suffix := range []string{"", ".part"} {
|
||||||
|
if _, err := os.Stat(filepath.Join(dir, "bundle", "a.bin") + suffix); !os.IsNotExist(err) {
|
||||||
|
t.Fatalf("unselected output remains: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBTDownloadMultiFileWithEmptyFile(t *testing.T) {
|
||||||
|
fixture := newBTFixture(t, map[string][]byte{"empty.txt": {}, "nested/data.bin": btPayload(48*1024, "empty-file")}, 0)
|
||||||
|
f := btDownloadFetcher(t, nil)
|
||||||
|
dir := t.TempDir()
|
||||||
|
btResolve(t, f, fixture.Magnet)
|
||||||
|
f.Meta().Opts = &base.Options{Path: dir}
|
||||||
|
btStart(t, f)
|
||||||
|
btAwaitDone(t, f)
|
||||||
|
fixture.AssertFiles(t, dir)
|
||||||
|
}
|
||||||
+220
-105
@@ -3,11 +3,13 @@ package bt
|
|||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"net/url"
|
"net/url"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"runtime"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
@@ -33,6 +35,10 @@ var (
|
|||||||
)
|
)
|
||||||
|
|
||||||
type Fetcher struct {
|
type Fetcher struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
closeOnce sync.Once
|
||||||
|
storage *deferredFileStorage
|
||||||
|
|
||||||
ctl *controller.Controller
|
ctl *controller.Controller
|
||||||
config *config
|
config *config
|
||||||
|
|
||||||
@@ -94,13 +100,17 @@ func (f *Fetcher) initClient() (err error) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
func (f *Fetcher) Resolve(req *base.Request, opts *base.Options) error {
|
func (f *Fetcher) Resolve(req *base.Request) error {
|
||||||
f.meta.Req = req
|
f.mu.Lock()
|
||||||
f.meta.Opts = opts
|
defer f.mu.Unlock()
|
||||||
if f.meta.Opts == nil {
|
if f.torrent != nil {
|
||||||
f.meta.Opts = &base.Options{}
|
f.safeDrop()
|
||||||
|
f.torrent = nil
|
||||||
|
f.storage = nil
|
||||||
|
f.torrentReady.Store(false)
|
||||||
}
|
}
|
||||||
if err := f.addTorrent(req, false); err != nil {
|
f.meta.Req = req
|
||||||
|
if err := f.addTorrent(req); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
f.updateRes()
|
f.updateRes()
|
||||||
@@ -108,12 +118,72 @@ func (f *Fetcher) Resolve(req *base.Request, opts *base.Options) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (f *Fetcher) Start() (err error) {
|
func (f *Fetcher) Start() (err error) {
|
||||||
|
f.mu.Lock()
|
||||||
|
defer f.mu.Unlock()
|
||||||
|
if f.meta.Opts == nil || f.meta.Opts.Path == "" {
|
||||||
|
return errors.New("download directory is required when starting a torrent")
|
||||||
|
}
|
||||||
|
if err := f.torrentDropCtx.Err(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
if !f.torrentReady.Load() {
|
if !f.torrentReady.Load() {
|
||||||
if err = f.addTorrent(f.meta.Req, false); err != nil {
|
if err = f.addTorrent(f.meta.Req); err != nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
path := f.meta.Opts.Path
|
||||||
|
if err := os.MkdirAll(path, 0755); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
hashes, err := f.storage.bind(storage.NewFileClientOpts{
|
||||||
|
ClientBaseDir: path,
|
||||||
|
FilePathMaker: func(opts storage.FilePathMakerOpts) string {
|
||||||
|
_, name := torrentFileLayout(opts.Info, *opts.File)
|
||||||
|
return name
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if hashes != nil {
|
||||||
|
// File storage infers completion from final file names and sizes. Verify
|
||||||
|
// before publishing that state, or peers can see an unverified file as
|
||||||
|
// complete and disconnect before a damaged piece is discovered.
|
||||||
|
// Late v2 hashes schedule their own initial checks through the library.
|
||||||
|
changes := f.torrent.SubscribePieceStateChanges()
|
||||||
|
defer changes.Close()
|
||||||
|
for i, known := range hashes {
|
||||||
|
if !known {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if err := f.torrentDropCtx.Err(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := f.torrent.Piece(i).VerifyDataContext(f.torrentDropCtx); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
// VerifyDataContext can return while the result is still being
|
||||||
|
// written to storage. Wait for it before refreshing completion.
|
||||||
|
for {
|
||||||
|
state := f.torrent.Piece(i).State()
|
||||||
|
if !state.Hashing && !state.QueuedForHash && !state.Marking {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-changes.Values:
|
||||||
|
case <-f.torrentDropCtx.Done():
|
||||||
|
return f.torrentDropCtx.Err()
|
||||||
|
case <-f.torrent.Closed():
|
||||||
|
return errors.New("torrent closed during verification")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for i := range hashes {
|
||||||
|
f.torrent.Piece(i).UpdateCompletion()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
files := f.torrent.Files()
|
files := f.torrent.Files()
|
||||||
// If the user does not specify the file to download, all files will be downloaded by default
|
// If the user does not specify the file to download, all files will be downloaded by default
|
||||||
if f.data.Progress == nil {
|
if f.data.Progress == nil {
|
||||||
@@ -134,21 +204,29 @@ func (f *Fetcher) Start() (err error) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
f.torrent.AllowDataDownload()
|
f.torrent.AllowDataDownload()
|
||||||
|
go f.doUpload(false)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
func (f *Fetcher) Pause() (err error) {
|
func (f *Fetcher) Pause() (err error) {
|
||||||
|
f.mu.Lock()
|
||||||
|
defer f.mu.Unlock()
|
||||||
|
if f.torrent == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
f.torrent.DisallowDataDownload()
|
f.torrent.DisallowDataDownload()
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
func (f *Fetcher) Close() (err error) {
|
func (f *Fetcher) Close() (err error) {
|
||||||
f.safeDrop()
|
f.closeOnce.Do(func() {
|
||||||
f.torrentDropFunc()
|
f.torrentDropFunc()
|
||||||
f.uploadDoneCh <- nil
|
f.mu.Lock()
|
||||||
if len(client.Torrents()) == 0 {
|
defer f.mu.Unlock()
|
||||||
err = closeClient()
|
f.safeDrop()
|
||||||
}
|
f.torrentReady.Store(false)
|
||||||
|
close(f.uploadDoneCh)
|
||||||
|
})
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -166,6 +244,8 @@ func (f *Fetcher) Meta() *fetcher.FetcherMeta {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (f *Fetcher) Stats() *fetcher.Stats {
|
func (f *Fetcher) Stats() *fetcher.Stats {
|
||||||
|
f.mu.Lock()
|
||||||
|
defer f.mu.Unlock()
|
||||||
var stats torrent.TorrentStats
|
var stats torrent.TorrentStats
|
||||||
var pieceMap *base.PieceMap
|
var pieceMap *base.PieceMap
|
||||||
pieceMapReady := false
|
pieceMapReady := false
|
||||||
@@ -372,48 +452,65 @@ func normalizeBTTransport(network string) string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (f *Fetcher) Progress() fetcher.Progress {
|
func (f *Fetcher) Progress() fetcher.Progress {
|
||||||
|
f.mu.Lock()
|
||||||
|
defer f.mu.Unlock()
|
||||||
if !f.torrentReady.Load() {
|
if !f.torrentReady.Load() {
|
||||||
return f.data.Progress
|
return append(fetcher.Progress(nil), f.data.Progress...)
|
||||||
}
|
}
|
||||||
for i := range f.data.Progress {
|
for i := range f.data.Progress {
|
||||||
selectIndex := f.meta.Opts.SelectFiles[i]
|
selectIndex := f.meta.Opts.SelectFiles[i]
|
||||||
file := f.torrent.Files()[selectIndex]
|
file := f.torrent.Files()[selectIndex]
|
||||||
f.data.Progress[i] = file.BytesCompleted()
|
f.data.Progress[i] = file.BytesCompleted()
|
||||||
}
|
}
|
||||||
return f.data.Progress
|
return append(fetcher.Progress(nil), f.data.Progress...)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (f *Fetcher) Wait() (err error) {
|
func (f *Fetcher) Wait() error {
|
||||||
|
f.mu.Lock()
|
||||||
|
var closed <-chan struct{}
|
||||||
|
if f.torrent != nil {
|
||||||
|
closed = f.torrent.Closed()
|
||||||
|
}
|
||||||
|
f.mu.Unlock()
|
||||||
|
ticker := time.NewTicker(time.Second)
|
||||||
|
defer ticker.Stop()
|
||||||
for {
|
for {
|
||||||
select {
|
select {
|
||||||
case <-f.torrentDropCtx.Done():
|
case <-f.torrentDropCtx.Done():
|
||||||
return
|
return nil
|
||||||
case <-time.After(time.Second):
|
case <-closed:
|
||||||
if f.torrentReady.Load() && len(f.meta.Opts.SelectFiles) > 0 {
|
return nil
|
||||||
if f.isDone() {
|
case <-ticker.C:
|
||||||
// remove unselected files
|
done, err := f.finishDownload()
|
||||||
for i, file := range f.torrent.Files() {
|
if done || err != nil {
|
||||||
selected := false
|
return err
|
||||||
for _, selectIndex := range f.meta.Opts.SelectFiles {
|
|
||||||
if i == selectIndex {
|
|
||||||
selected = true
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if !selected {
|
|
||||||
root, name := torrentFileLayout(f.torrent.Info(), file.FileInfo())
|
|
||||||
if err := removeUnselectedFile(f.meta.Opts.Path, root, name); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (f *Fetcher) finishDownload() (bool, error) {
|
||||||
|
f.mu.Lock()
|
||||||
|
defer f.mu.Unlock()
|
||||||
|
if !f.torrentReady.Load() || f.meta.Opts == nil || len(f.meta.Opts.SelectFiles) == 0 || !f.isDoneLocked() {
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
selected := make(map[int]bool, len(f.meta.Opts.SelectFiles))
|
||||||
|
for _, i := range f.meta.Opts.SelectFiles {
|
||||||
|
selected[i] = true
|
||||||
|
}
|
||||||
|
for i, file := range f.torrent.Files() {
|
||||||
|
if selected[i] {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
root, name := torrentFileLayout(f.torrent.Info(), file.FileInfo())
|
||||||
|
if err := removeUnselectedFile(f.meta.Opts.Path, root, name); err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true, nil
|
||||||
|
}
|
||||||
|
|
||||||
// Match the file storage layout, including its NoName convention for rootless
|
// Match the file storage layout, including its NoName convention for rootless
|
||||||
// torrents. File.Path() always prepends BestName, even when storage omits it.
|
// torrents. File.Path() always prepends BestName, even when storage omits it.
|
||||||
func torrentFileLayout(info *metainfo.Info, file metainfo.FileInfo) (root, name string) {
|
func torrentFileLayout(info *metainfo.Info, file metainfo.FileInfo) (root, name string) {
|
||||||
@@ -471,13 +568,22 @@ func removeUnselectedFile(downloadDir, torrentRoot, torrentPath string) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (f *Fetcher) isDone() bool {
|
func (f *Fetcher) isDone() bool {
|
||||||
if f.meta.Opts == nil {
|
f.mu.Lock()
|
||||||
|
defer f.mu.Unlock()
|
||||||
|
return f.isDoneLocked()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *Fetcher) isDoneLocked() bool {
|
||||||
|
if f.meta.Opts == nil || f.torrent == nil {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
for _, selectIndex := range f.meta.Opts.SelectFiles {
|
for _, index := range f.meta.Opts.SelectFiles {
|
||||||
file := f.torrent.Files()[selectIndex]
|
file := f.torrent.Files()[index]
|
||||||
if file.BytesCompleted() < file.Length() {
|
for i := file.BeginPieceIndex(); i < file.EndPieceIndex(); i++ {
|
||||||
return false
|
state := f.torrent.Piece(i).State()
|
||||||
|
if !state.Ok || !state.Complete || state.Hashing || state.QueuedForHash {
|
||||||
|
return false
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return true
|
return true
|
||||||
@@ -486,6 +592,11 @@ func (f *Fetcher) isDone() bool {
|
|||||||
// Patch modifies the BT task settings.
|
// Patch modifies the BT task settings.
|
||||||
// Invalid file indices are silently ignored.
|
// Invalid file indices are silently ignored.
|
||||||
func (f *Fetcher) Patch(req *base.Request, opts *base.Options) error {
|
func (f *Fetcher) Patch(req *base.Request, opts *base.Options) error {
|
||||||
|
f.mu.Lock()
|
||||||
|
defer f.mu.Unlock()
|
||||||
|
if f.meta.Opts == nil {
|
||||||
|
f.meta.Opts = &base.Options{}
|
||||||
|
}
|
||||||
if opts == nil {
|
if opts == nil {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
@@ -542,9 +653,9 @@ func (f *Fetcher) updateRes() {
|
|||||||
Files: make([]*base.FileInfo, len(f.torrent.Files())),
|
Files: make([]*base.FileInfo, len(f.torrent.Files())),
|
||||||
Hash: f.torrent.InfoHash().String(),
|
Hash: f.torrent.InfoHash().String(),
|
||||||
}
|
}
|
||||||
// Directory torrent
|
// Use the same root classification as storage (v2 single files have Length == 0).
|
||||||
if f.torrent.Info().Length == 0 {
|
if files := f.torrent.Files(); len(files) > 0 {
|
||||||
res.Name = f.torrent.Name()
|
res.Name, _ = torrentFileLayout(f.torrent.Info(), files[0].FileInfo())
|
||||||
}
|
}
|
||||||
for i, file := range f.torrent.Files() {
|
for i, file := range f.torrent.Files() {
|
||||||
res.Files[i] = &base.FileInfo{
|
res.Files[i] = &base.FileInfo{
|
||||||
@@ -561,61 +672,50 @@ func (f *Fetcher) updateRes() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (f *Fetcher) Upload() (err error) {
|
func (f *Fetcher) Upload() (err error) {
|
||||||
return f.addTorrent(f.meta.Req, true)
|
if err = f.Start(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (f *Fetcher) doUpload(fromUpload bool) {
|
func (f *Fetcher) doUpload(fromUpload bool) {
|
||||||
if !f.torrentUpload.CompareAndSwap(false, true) {
|
if !f.torrentUpload.CompareAndSwap(false, true) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
f.mu.Lock()
|
||||||
// Check and update seed data
|
initialBytes, initialTime := f.data.SeedBytes, f.data.SeedTime
|
||||||
lastData := &fetcherData{
|
torrentClosed := f.torrent.Closed()
|
||||||
SeedBytes: f.data.SeedBytes,
|
f.mu.Unlock()
|
||||||
SeedTime: f.data.SeedTime,
|
var doneTime int64
|
||||||
}
|
ticker := time.NewTicker(time.Second)
|
||||||
var doneTime int64 = 0
|
defer ticker.Stop()
|
||||||
for {
|
for {
|
||||||
select {
|
select {
|
||||||
case <-f.torrentDropCtx.Done():
|
case <-f.torrentDropCtx.Done():
|
||||||
return
|
return
|
||||||
case <-time.After(time.Second):
|
case <-torrentClosed:
|
||||||
|
return
|
||||||
|
case <-ticker.C:
|
||||||
|
f.mu.Lock()
|
||||||
if !f.torrentReady.Load() {
|
if !f.torrentReady.Load() {
|
||||||
|
f.mu.Unlock()
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
stats := f.torrentStats()
|
stats := f.torrentStats()
|
||||||
f.data.SeedBytes = lastData.SeedBytes + stats.BytesWrittenData.Int64()
|
f.data.SeedBytes = initialBytes + stats.BytesWrittenData.Int64()
|
||||||
|
if !fromUpload && !f.isDoneLocked() {
|
||||||
// Check is download complete, if not don't check and stop seeding
|
f.mu.Unlock()
|
||||||
if !fromUpload && !f.isDone() {
|
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if doneTime == 0 {
|
if doneTime == 0 {
|
||||||
doneTime = time.Now().Unix()
|
doneTime = time.Now().Unix()
|
||||||
}
|
}
|
||||||
f.data.SeedTime = lastData.SeedTime + time.Now().Unix() - doneTime
|
f.data.SeedTime = initialTime + time.Now().Unix() - doneTime
|
||||||
|
stop := !f.config.SeedKeep && ((f.config.SeedRatio > 0 && f.seedRadio() >= f.config.SeedRatio) || (f.config.SeedTime > 0 && f.data.SeedTime >= f.config.SeedTime))
|
||||||
// If the seed forever is true, keep seeding
|
f.mu.Unlock()
|
||||||
if f.config.SeedKeep {
|
if stop {
|
||||||
continue
|
f.Close()
|
||||||
}
|
return
|
||||||
|
|
||||||
// If the seed ratio is reached, stop seeding
|
|
||||||
if f.config.SeedRatio > 0 {
|
|
||||||
seedRadio := f.seedRadio()
|
|
||||||
if seedRadio >= f.config.SeedRatio {
|
|
||||||
f.Close()
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// If the seed time is reached, stop seeding
|
|
||||||
if f.config.SeedTime > 0 {
|
|
||||||
if f.data.SeedTime >= f.config.SeedTime {
|
|
||||||
f.Close()
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -633,6 +733,8 @@ func (f *Fetcher) torrentStats() torrent.TorrentStats {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (f *Fetcher) UploadedBytes() int64 {
|
func (f *Fetcher) UploadedBytes() int64 {
|
||||||
|
f.mu.Lock()
|
||||||
|
defer f.mu.Unlock()
|
||||||
return f.data.SeedBytes
|
return f.data.SeedBytes
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -641,7 +743,7 @@ func (f *Fetcher) WaitUpload() (err error) {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (f *Fetcher) addTorrent(req *base.Request, fromUpload bool) (err error) {
|
func (f *Fetcher) addTorrent(req *base.Request) (err error) {
|
||||||
if err = base.ParseReqExtra[bt.ReqExtra](req); err != nil {
|
if err = base.ParseReqExtra[bt.ReqExtra](req); err != nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -660,11 +762,16 @@ func (f *Fetcher) addTorrent(req *base.Request, fromUpload bool) (err error) {
|
|||||||
var reader io.Reader
|
var reader io.Reader
|
||||||
if schema == "FILE" {
|
if schema == "FILE" {
|
||||||
fileUrl, _ := url.Parse(req.URL)
|
fileUrl, _ := url.Parse(req.URL)
|
||||||
filePath := fileUrl.Path[1:]
|
filePath := fileUrl.Path
|
||||||
|
if runtime.GOOS == "windows" && len(filePath) > 2 && filePath[0] == '/' && filePath[2] == ':' {
|
||||||
|
filePath = filePath[1:]
|
||||||
|
}
|
||||||
|
filePath = filepath.FromSlash(filePath)
|
||||||
reader, err = os.Open(filePath)
|
reader, err = os.Open(filePath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
defer reader.(io.Closer).Close()
|
||||||
} else if schema == "DATA" {
|
} else if schema == "DATA" {
|
||||||
_, data := util.ParseDataUri(req.URL)
|
_, data := util.ParseDataUri(req.URL)
|
||||||
reader = bytes.NewBuffer(data)
|
reader = bytes.NewBuffer(data)
|
||||||
@@ -697,21 +804,6 @@ func (f *Fetcher) addTorrent(req *base.Request, fromUpload bool) (err error) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
spec.Storage = storage.NewFileOpts(storage.NewFileClientOpts{
|
|
||||||
ClientBaseDir: cfg.DataDir,
|
|
||||||
FilePathMaker: func(opts storage.FilePathMakerOpts) string {
|
|
||||||
_, name := torrentFileLayout(opts.Info, *opts.File)
|
|
||||||
return name
|
|
||||||
},
|
|
||||||
TorrentDirMaker: func(baseDir string, info *metainfo.Info, infoHash metainfo.Hash) string {
|
|
||||||
return f.meta.Opts.Path
|
|
||||||
},
|
|
||||||
})
|
|
||||||
f.torrent, _, err = client.AddTorrentSpec(spec)
|
|
||||||
if err != nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// Do not add external tracker to a private torrent.
|
// Do not add external tracker to a private torrent.
|
||||||
if !privateTorrent {
|
if !privateTorrent {
|
||||||
// use map to deduplicate
|
// use map to deduplicate
|
||||||
@@ -730,17 +822,36 @@ func (f *Fetcher) addTorrent(req *base.Request, fromUpload bool) (err error) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
if len(trackers) > 0 {
|
if len(trackers) > 0 {
|
||||||
announceList := make([][]string, 0)
|
|
||||||
for tracker := range trackers {
|
for tracker := range trackers {
|
||||||
announceList = append(announceList, []string{tracker})
|
spec.Trackers = append(spec.Trackers, []string{tracker})
|
||||||
}
|
}
|
||||||
f.torrent.AddTrackers(announceList)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
<-f.torrent.GotInfo()
|
s := &deferredFileStorage{}
|
||||||
|
spec.Storage = s
|
||||||
|
var added bool
|
||||||
|
f.torrent, added, err = client.AddTorrentSpec(spec)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if !added {
|
||||||
|
f.torrent = nil
|
||||||
|
return errors.New("torrent is already active")
|
||||||
|
}
|
||||||
|
f.storage = s
|
||||||
|
f.torrent.DisallowDataDownload()
|
||||||
|
// Extra trackers must be available while magnet metadata is still missing;
|
||||||
|
// they may be the only way to discover a peer that can provide it.
|
||||||
|
select {
|
||||||
|
case <-f.torrent.GotInfo():
|
||||||
|
case <-f.torrentDropCtx.Done():
|
||||||
|
f.safeDrop()
|
||||||
|
return f.torrentDropCtx.Err()
|
||||||
|
case <-f.torrent.Closed():
|
||||||
|
return errors.New("torrent closed during resolution")
|
||||||
|
}
|
||||||
f.torrentReady.Store(true)
|
f.torrentReady.Store(true)
|
||||||
|
|
||||||
go f.doUpload(fromUpload)
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -848,7 +959,11 @@ func (fm *FetcherManager) DefaultConfig() any {
|
|||||||
|
|
||||||
func (fm *FetcherManager) Store(f fetcher.Fetcher) (data any, err error) {
|
func (fm *FetcherManager) Store(f fetcher.Fetcher) (data any, err error) {
|
||||||
_f := f.(*Fetcher)
|
_f := f.(*Fetcher)
|
||||||
return _f.data, nil
|
_f.mu.Lock()
|
||||||
|
defer _f.mu.Unlock()
|
||||||
|
snapshot := *_f.data
|
||||||
|
snapshot.Progress = append(fetcher.Progress(nil), snapshot.Progress...)
|
||||||
|
return &snapshot, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (fm *FetcherManager) Restore() (v any, f func(meta *fetcher.FetcherMeta, v any) fetcher.Fetcher) {
|
func (fm *FetcherManager) Restore() (v any, f func(meta *fetcher.FetcherMeta, v any) fetcher.Fetcher) {
|
||||||
|
|||||||
@@ -18,11 +18,11 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
func TestFetcher_Resolve_Torrent(t *testing.T) {
|
func TestFetcher_Resolve_Torrent(t *testing.T) {
|
||||||
doResolve(t, buildFetcher())
|
doResolve(t, buildFetcher(t))
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestFetcher_Resolve_DataUri_Torrent(t *testing.T) {
|
func TestFetcher_Resolve_DataUri_Torrent(t *testing.T) {
|
||||||
fetcher := buildFetcher()
|
fetcher := buildFetcher(t)
|
||||||
buf, err := os.ReadFile("./testdata/ubuntu-22.04-live-server-amd64.iso.torrent")
|
buf, err := os.ReadFile("./testdata/ubuntu-22.04-live-server-amd64.iso.torrent")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
@@ -31,7 +31,7 @@ func TestFetcher_Resolve_DataUri_Torrent(t *testing.T) {
|
|||||||
dataUri := "data:application/x-bittorrent;base64," + base64.StdEncoding.EncodeToString(buf)
|
dataUri := "data:application/x-bittorrent;base64," + base64.StdEncoding.EncodeToString(buf)
|
||||||
err = fetcher.Resolve(&base.Request{
|
err = fetcher.Resolve(&base.Request{
|
||||||
URL: dataUri,
|
URL: dataUri,
|
||||||
}, nil)
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
panic(err)
|
panic(err)
|
||||||
}
|
}
|
||||||
@@ -53,7 +53,7 @@ func TestFetcher_Resolve_DataUri_Torrent(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestFetcher_Config(t *testing.T) {
|
func TestFetcher_Config(t *testing.T) {
|
||||||
doResolve(t, buildConfigFetcher(nil))
|
doResolve(t, buildConfigFetcher(t, nil))
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestFetcher_ResolveWithProxy(t *testing.T) {
|
func TestFetcher_ResolveWithProxy(t *testing.T) {
|
||||||
@@ -61,7 +61,7 @@ func TestFetcher_ResolveWithProxy(t *testing.T) {
|
|||||||
proxyListener := test.StartSocks5Server(usr, pwd)
|
proxyListener := test.StartSocks5Server(usr, pwd)
|
||||||
defer proxyListener.Close()
|
defer proxyListener.Close()
|
||||||
|
|
||||||
doResolve(t, buildConfigFetcher(&base.DownloaderProxyConfig{
|
doResolve(t, buildConfigFetcher(t, &base.DownloaderProxyConfig{
|
||||||
Enable: true,
|
Enable: true,
|
||||||
System: false,
|
System: false,
|
||||||
Scheme: "socks5",
|
Scheme: "socks5",
|
||||||
@@ -81,7 +81,7 @@ func doResolve(t *testing.T, fetcher fetcher.Fetcher) {
|
|||||||
"udp://tracker.bitsearch.to:1337/announce",
|
"udp://tracker.bitsearch.to:1337/announce",
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
}, nil)
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
panic(err)
|
panic(err)
|
||||||
}
|
}
|
||||||
@@ -111,7 +111,7 @@ func doResolve(t *testing.T, fetcher fetcher.Fetcher) {
|
|||||||
"udp://tracker.bitsearch.to:1337/announce",
|
"udp://tracker.bitsearch.to:1337/announce",
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
}, nil)
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
panic(err)
|
panic(err)
|
||||||
}
|
}
|
||||||
@@ -145,7 +145,7 @@ func doResolve(t *testing.T, fetcher fetcher.Fetcher) {
|
|||||||
t.Run("Resolve Unclean Torrent", func(t *testing.T) {
|
t.Run("Resolve Unclean Torrent", func(t *testing.T) {
|
||||||
err := fetcher.Resolve(&base.Request{
|
err := fetcher.Resolve(&base.Request{
|
||||||
URL: "./testdata/test.unclean.torrent",
|
URL: "./testdata/test.unclean.torrent",
|
||||||
}, nil)
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Errorf("Resolve Unclean Torrent Resolve() got = %v, want nil", err)
|
t.Errorf("Resolve Unclean Torrent Resolve() got = %v, want nil", err)
|
||||||
}
|
}
|
||||||
@@ -156,7 +156,7 @@ func doResolve(t *testing.T, fetcher fetcher.Fetcher) {
|
|||||||
uri := "file:///" + file
|
uri := "file:///" + file
|
||||||
err := fetcher.Resolve(&base.Request{
|
err := fetcher.Resolve(&base.Request{
|
||||||
URL: uri,
|
URL: uri,
|
||||||
}, nil)
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Errorf("Resolve file scheme Torrent Resolve() got = %v, want nil", err)
|
t.Errorf("Resolve file scheme Torrent Resolve() got = %v, want nil", err)
|
||||||
}
|
}
|
||||||
@@ -211,7 +211,7 @@ func TestFetcherManager_ParseName(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func buildFetcher() fetcher.Fetcher {
|
func buildFetcher(t testing.TB) fetcher.Fetcher {
|
||||||
fb := new(FetcherManager)
|
fb := new(FetcherManager)
|
||||||
fetcher := fb.Build()
|
fetcher := fb.Build()
|
||||||
newController := controller.NewController()
|
newController := controller.NewController()
|
||||||
@@ -219,10 +219,11 @@ func buildFetcher() fetcher.Fetcher {
|
|||||||
json.Unmarshal([]byte(test.ToJson(fb.DefaultConfig())), v)
|
json.Unmarshal([]byte(test.ToJson(fb.DefaultConfig())), v)
|
||||||
}
|
}
|
||||||
fetcher.Setup(newController)
|
fetcher.Setup(newController)
|
||||||
|
t.Cleanup(func() { fetcher.Close(); closeClient() })
|
||||||
return fetcher
|
return fetcher
|
||||||
}
|
}
|
||||||
|
|
||||||
func buildConfigFetcher(proxyConfig *base.DownloaderProxyConfig) fetcher.Fetcher {
|
func buildConfigFetcher(t testing.TB, proxyConfig *base.DownloaderProxyConfig) fetcher.Fetcher {
|
||||||
fetcher := new(FetcherManager).Build()
|
fetcher := new(FetcherManager).Build()
|
||||||
newController := controller.NewController()
|
newController := controller.NewController()
|
||||||
mockCfg := config{
|
mockCfg := config{
|
||||||
@@ -237,29 +238,30 @@ func buildConfigFetcher(proxyConfig *base.DownloaderProxyConfig) fetcher.Fetcher
|
|||||||
return proxyConfig.ToHandler()
|
return proxyConfig.ToHandler()
|
||||||
}
|
}
|
||||||
fetcher.Setup(newController)
|
fetcher.Setup(newController)
|
||||||
|
t.Cleanup(func() { fetcher.Close(); closeClient() })
|
||||||
return fetcher
|
return fetcher
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestFetcher_Patch tests the Patch functionality for BT fetcher.
|
// TestFetcher_Patch tests the Patch functionality for BT fetcher.
|
||||||
// It tests modifying selected files after Resolve (without downloading).
|
// It tests modifying selected files after Resolve (without downloading).
|
||||||
func TestFetcher_Patch(t *testing.T) {
|
func TestFetcher_Patch(t *testing.T) {
|
||||||
f := buildFetcher()
|
f := buildFetcher(t)
|
||||||
|
|
||||||
// Resolve a multi-file torrent
|
// Resolve a multi-file torrent
|
||||||
err := f.Resolve(&base.Request{
|
err := f.Resolve(&base.Request{
|
||||||
URL: "./testdata/test.torrent",
|
URL: "./testdata/test.torrent",
|
||||||
}, nil)
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Resolve failed: %v", err)
|
t.Fatalf("Resolve failed: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Verify initial state: 3 files, all selected by default
|
// Resolution describes files without choosing download options.
|
||||||
meta := f.Meta()
|
meta := f.Meta()
|
||||||
if len(meta.Res.Files) != 3 {
|
if len(meta.Res.Files) != 3 {
|
||||||
t.Fatalf("Expected 3 files, got %d", len(meta.Res.Files))
|
t.Fatalf("Expected 3 files, got %d", len(meta.Res.Files))
|
||||||
}
|
}
|
||||||
if len(meta.Opts.SelectFiles) != 3 {
|
if meta.Opts != nil {
|
||||||
t.Fatalf("Expected 3 selected files, got %d", len(meta.Opts.SelectFiles))
|
t.Fatal("Resolve should not assign download options")
|
||||||
}
|
}
|
||||||
|
|
||||||
// Total size of all files: 107484864
|
// Total size of all files: 107484864
|
||||||
@@ -395,7 +397,7 @@ func TestFetcher_PeerDiscoveryDisabledOnClient(t *testing.T) {
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
f := buildFetcherWithConfig(map[string]any{"disableDht": true, "disablePex": true})
|
f := buildFetcherWithConfig(map[string]any{"disableDht": true, "disablePex": true})
|
||||||
if err := f.Resolve(&base.Request{URL: "./testdata/test.torrent"}, nil); err != nil {
|
if err := f.Resolve(&base.Request{URL: "./testdata/test.torrent"}); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if !cfg.NoDHT || !cfg.DisablePEX {
|
if !cfg.NoDHT || !cfg.DisablePEX {
|
||||||
|
|||||||
@@ -0,0 +1,212 @@
|
|||||||
|
package bt
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"encoding/base64"
|
||||||
|
"encoding/binary"
|
||||||
|
"net"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"net/url"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/anacrolix/torrent"
|
||||||
|
"github.com/anacrolix/torrent/bencode"
|
||||||
|
"github.com/anacrolix/torrent/merkle"
|
||||||
|
"github.com/anacrolix/torrent/metainfo"
|
||||||
|
"golang.org/x/time/rate"
|
||||||
|
)
|
||||||
|
|
||||||
|
// btFixture serves generated, trackerless content over a real loopback TCP peer.
|
||||||
|
// Its HTTP tracker exercises .torrent downloads; magnet links also carry the
|
||||||
|
// explicit x.pe address.
|
||||||
|
type btFixture struct {
|
||||||
|
MetaInfo *metainfo.MetaInfo
|
||||||
|
Magnet string
|
||||||
|
TorrentFile string
|
||||||
|
DataURI string
|
||||||
|
TrackerURL string
|
||||||
|
Files map[string][]byte // Paths relative to the selected download directory.
|
||||||
|
Seeder *torrent.Client
|
||||||
|
Torrent *torrent.Torrent
|
||||||
|
Limiter *rate.Limiter
|
||||||
|
}
|
||||||
|
|
||||||
|
func newBTFixture(t testing.TB, files map[string][]byte, bytesPerSecond int) *btFixture {
|
||||||
|
return makeBTFixture(t, files, bytesPerSecond, false, true)
|
||||||
|
}
|
||||||
|
|
||||||
|
func newBTHybridFixture(t testing.TB, payload []byte) *btFixture {
|
||||||
|
return makeBTFixture(t, map[string][]byte{"": payload}, 0, true, true)
|
||||||
|
}
|
||||||
|
|
||||||
|
func makeBTFixture(t testing.TB, files map[string][]byte, bytesPerSecond int, v2, private bool) *btFixture {
|
||||||
|
t.Helper()
|
||||||
|
source := t.TempDir()
|
||||||
|
name := "payload.bin"
|
||||||
|
if _, single := files[""]; !single {
|
||||||
|
name = "bundle"
|
||||||
|
}
|
||||||
|
root := filepath.Join(source, name)
|
||||||
|
expected := make(map[string][]byte, len(files))
|
||||||
|
for path, data := range files {
|
||||||
|
target := filepath.Join(root, filepath.FromSlash(path))
|
||||||
|
if err := os.MkdirAll(filepath.Dir(target), 0700); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(target, data, 0600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
expected[filepath.Join(name, filepath.FromSlash(path))] = append([]byte(nil), data...)
|
||||||
|
}
|
||||||
|
info := metainfo.Info{PieceLength: 16 * 1024}
|
||||||
|
if err := info.BuildFromFilePath(root); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var layers map[string]string
|
||||||
|
if v2 {
|
||||||
|
data := files[""]
|
||||||
|
h := merkle.NewHash()
|
||||||
|
h.Write(data)
|
||||||
|
root := h.Sum(nil)
|
||||||
|
var pieceHashes []byte
|
||||||
|
for off := 0; off < len(data); off += int(info.PieceLength) {
|
||||||
|
end := min(off+int(info.PieceLength), len(data))
|
||||||
|
ph := merkle.NewHash()
|
||||||
|
ph.Write(data[off:end])
|
||||||
|
pieceHashes = append(pieceHashes, ph.SumMinLength(nil, int(info.PieceLength))...)
|
||||||
|
}
|
||||||
|
info = metainfo.Info{Name: name, PieceLength: info.PieceLength, Length: int64(len(data)), Pieces: info.Pieces, MetaVersion: 2, FileTree: metainfo.FileTree{File: metainfo.FileTreeFile{Length: int64(len(data)), PiecesRoot: string(root)}}}
|
||||||
|
layers = map[string]string{string(root): string(pieceHashes)}
|
||||||
|
}
|
||||||
|
// Private torrents avoid public DHT discovery once metadata is available.
|
||||||
|
info.Private = &private
|
||||||
|
raw, err := bencode.Marshal(&info)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
mi := &metainfo.MetaInfo{InfoBytes: raw, PieceLayers: layers}
|
||||||
|
cfg := torrent.TestingConfig(t)
|
||||||
|
cfg.DataDir = source
|
||||||
|
cfg.Seed = true
|
||||||
|
cfg.DisableUTP = true
|
||||||
|
cfg.DisableIPv6 = true
|
||||||
|
cfg.DisableWebtorrent = true
|
||||||
|
cfg.KeepAliveTimeout = time.Minute
|
||||||
|
cfg.MaxAllocPeerRequestDataPerConn = 1 << 20
|
||||||
|
limiter := rate.NewLimiter(rate.Inf, 16*1024)
|
||||||
|
if bytesPerSecond > 0 {
|
||||||
|
limiter.SetLimit(rate.Limit(bytesPerSecond))
|
||||||
|
}
|
||||||
|
cfg.UploadRateLimiter = limiter
|
||||||
|
seed, err := torrent.NewClient(cfg)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { seed.Close() })
|
||||||
|
seeded, _, err := seed.AddTorrentSpec(torrent.TorrentSpecFromMetaInfo(mi))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
if err := seeded.VerifyDataContext(ctx); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-seeded.Complete().On():
|
||||||
|
case <-ctx.Done():
|
||||||
|
t.Fatal("local seeder did not finish verification")
|
||||||
|
}
|
||||||
|
addr := seed.ListenAddrs()[0].String()
|
||||||
|
host, portString, err := net.SplitHostPort(addr)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
resolved, err := net.ResolveTCPAddr("tcp4", net.JoinHostPort(host, portString))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
compact := make([]byte, 6)
|
||||||
|
copy(compact, resolved.IP.To4())
|
||||||
|
binary.BigEndian.PutUint16(compact[4:], uint16(resolved.Port))
|
||||||
|
tracker := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
response, err := bencode.Marshal(map[string]any{"interval": 3600, "peers": string(compact)})
|
||||||
|
if err != nil {
|
||||||
|
http.Error(w, err.Error(), 500)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
w.Write(response)
|
||||||
|
}))
|
||||||
|
t.Cleanup(tracker.Close)
|
||||||
|
// The pinned library has a hybrid tracker-dispatch assertion bug.
|
||||||
|
// Hybrid fixtures use their explicit loopback peer rather than a tracker.
|
||||||
|
if !v2 {
|
||||||
|
mi.Announce = tracker.URL
|
||||||
|
}
|
||||||
|
var encoded bytes.Buffer
|
||||||
|
if err := mi.Write(&encoded); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
torrentFile := filepath.Join(t.TempDir(), "fixture.torrent")
|
||||||
|
if err := os.WriteFile(torrentFile, encoded.Bytes(), 0600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
m, err := mi.MagnetV2()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
magnet := m.String()
|
||||||
|
u, err := url.Parse(magnet)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
q := u.Query()
|
||||||
|
q.Add("x.pe", addr)
|
||||||
|
u.RawQuery = q.Encode()
|
||||||
|
return &btFixture{MetaInfo: mi, Magnet: u.String(), TorrentFile: torrentFile, DataURI: "data:application/x-bittorrent;base64," + base64.StdEncoding.EncodeToString(encoded.Bytes()), TrackerURL: tracker.URL, Files: expected, Seeder: seed, Torrent: seeded, Limiter: limiter}
|
||||||
|
}
|
||||||
|
|
||||||
|
func btPayload(size int, label string) []byte {
|
||||||
|
data := make([]byte, size)
|
||||||
|
for i := range data {
|
||||||
|
data[i] = byte((i*31 + i/251) % 251)
|
||||||
|
}
|
||||||
|
copy(data, label)
|
||||||
|
return data
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *btFixture) AssertFiles(t testing.TB, dir string, paths ...string) {
|
||||||
|
t.Helper()
|
||||||
|
if len(paths) == 0 {
|
||||||
|
for path := range f.Files {
|
||||||
|
paths = append(paths, path)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, path := range paths {
|
||||||
|
got, err := os.ReadFile(filepath.Join(dir, path))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
want, ok := f.Files[path]
|
||||||
|
if !ok {
|
||||||
|
t.Fatalf("unknown fixture file %q", path)
|
||||||
|
}
|
||||||
|
if !bytes.Equal(got, want) {
|
||||||
|
t.Fatalf("BT file %s differs: received %d bytes, want %d", path, len(got), len(want))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *btFixture) FileURI() string {
|
||||||
|
path := filepath.ToSlash(f.TorrentFile)
|
||||||
|
if !strings.HasPrefix(path, "/") {
|
||||||
|
path = "/" + path
|
||||||
|
}
|
||||||
|
return (&url.URL{Scheme: "file", Path: path}).String()
|
||||||
|
}
|
||||||
@@ -17,25 +17,85 @@ import (
|
|||||||
"github.com/anacrolix/torrent/storage"
|
"github.com/anacrolix/torrent/storage"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestStartUsesSelectionSubmittedAfterResolve(t *testing.T) {
|
func TestStartDoesNotPublishUnverifiedCompletion(t *testing.T) {
|
||||||
config := torrent.TestingConfig(t)
|
cfg := torrent.TestingConfig(t)
|
||||||
config.DataDir = t.TempDir()
|
cfg.DisableTCP = true
|
||||||
config.DisableTCP = true
|
cfg.DisableUTP = true
|
||||||
config.DisableUTP = true
|
c, err := torrent.NewClient(cfg)
|
||||||
c, err := torrent.NewClient(config)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
defer c.Close()
|
defer c.Close()
|
||||||
tor, err := c.AddTorrentFromFile("testdata/test.torrent")
|
payload := btPayload(32*1024, "unverified")
|
||||||
|
info := metainfo.Info{Name: "payload.bin", Length: int64(len(payload)), PieceLength: 16 * 1024}
|
||||||
|
for offset := 0; offset < len(payload); offset += int(info.PieceLength) {
|
||||||
|
hash := sha1.Sum(payload[offset : offset+int(info.PieceLength)])
|
||||||
|
info.Pieces = append(info.Pieces, hash[:]...)
|
||||||
|
}
|
||||||
|
raw, err := bencode.Marshal(info)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
f := &Fetcher{torrent: tor, meta: &fetcher.FetcherMeta{Opts: &base.Options{}}, data: &fetcherData{}}
|
store := &deferredFileStorage{}
|
||||||
|
spec := torrent.TorrentSpecFromMetaInfo(&metainfo.MetaInfo{InfoBytes: raw})
|
||||||
|
spec.Storage = store
|
||||||
|
tor, _, err := c.AddTorrentSpec(spec)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
tor.DisallowDataDownload()
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
defer cancel()
|
||||||
|
f := &Fetcher{torrent: tor, storage: store, meta: &fetcher.FetcherMeta{}, data: &fetcherData{}, torrentDropCtx: ctx, torrentDropFunc: cancel, config: &config{SeedKeep: true}}
|
||||||
f.torrentReady.Store(true)
|
f.torrentReady.Store(true)
|
||||||
f.updateRes()
|
f.updateRes()
|
||||||
// Resolve initializes all files; creation replaces the selection before Start.
|
dir := t.TempDir()
|
||||||
f.meta.Opts = &base.Options{SelectFiles: []int{2}}
|
payload[20000] ^= 0xff
|
||||||
|
if err := os.WriteFile(filepath.Join(dir, info.Name), payload, 0600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
f.meta.Opts = &base.Options{Path: dir}
|
||||||
|
// Retain the original event channel to detect even a transient completion.
|
||||||
|
// With no network transports, the damaged piece cannot have been repaired.
|
||||||
|
completed := tor.Complete().On()
|
||||||
|
if err := f.Start(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-completed:
|
||||||
|
t.Fatal("Start published completion before verifying the existing file")
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestStartUsesSelectionSubmittedAfterResolve(t *testing.T) {
|
||||||
|
cfg := torrent.TestingConfig(t)
|
||||||
|
cfg.DataDir = t.TempDir()
|
||||||
|
cfg.DisableTCP = true
|
||||||
|
cfg.DisableUTP = true
|
||||||
|
c, err := torrent.NewClient(cfg)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer c.Close()
|
||||||
|
store := &deferredFileStorage{}
|
||||||
|
mi, err := metainfo.LoadFromFile("testdata/test.torrent")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
spec := torrent.TorrentSpecFromMetaInfo(mi)
|
||||||
|
spec.Storage = store
|
||||||
|
tor, _, err := c.AddTorrentSpec(spec)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
tor.DisallowDataDownload()
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
defer cancel()
|
||||||
|
f := &Fetcher{torrent: tor, storage: store, meta: &fetcher.FetcherMeta{}, data: &fetcherData{}, torrentDropCtx: ctx, torrentDropFunc: cancel, config: &config{SeedKeep: true}}
|
||||||
|
f.torrentReady.Store(true)
|
||||||
|
f.updateRes()
|
||||||
|
f.meta.Opts = &base.Options{Path: t.TempDir(), SelectFiles: []int{2}}
|
||||||
if err := f.Start(); err != nil {
|
if err := f.Start(); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,116 @@
|
|||||||
|
package bt
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"os"
|
||||||
|
"sync"
|
||||||
|
|
||||||
|
g "github.com/anacrolix/generics"
|
||||||
|
"github.com/anacrolix/torrent/metainfo"
|
||||||
|
"github.com/anacrolix/torrent/storage"
|
||||||
|
)
|
||||||
|
|
||||||
|
// deferredFileStorage lets metadata resolve without opening the user's files.
|
||||||
|
// It binds once. After binding, pieces are supplied directly by file storage.
|
||||||
|
type deferredFileStorage struct {
|
||||||
|
mu sync.RWMutex
|
||||||
|
ctx context.Context
|
||||||
|
info *metainfo.Info
|
||||||
|
hash metainfo.Hash
|
||||||
|
knownHashes []bool
|
||||||
|
owner storage.ClientImplCloser
|
||||||
|
torrent storage.TorrentImpl
|
||||||
|
closed bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *deferredFileStorage) OpenTorrent(ctx context.Context, info *metainfo.Info, hash metainfo.Hash) (storage.TorrentImpl, error) {
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
if s.closed || s.info != nil {
|
||||||
|
return storage.TorrentImpl{}, errors.New("torrent storage already opened or closed")
|
||||||
|
}
|
||||||
|
s.ctx, s.info, s.hash = ctx, info, hash
|
||||||
|
s.knownHashes = make([]bool, info.NumPieces())
|
||||||
|
return storage.TorrentImpl{
|
||||||
|
PieceWithHash: func(p metainfo.Piece, hash g.Option[[]byte]) storage.PieceImpl {
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
if s.closed {
|
||||||
|
return metadataPiece{err: os.ErrClosed}
|
||||||
|
}
|
||||||
|
if s.owner == nil {
|
||||||
|
s.knownHashes[p.Index()] = hash.Ok
|
||||||
|
// A missing v2 hash must remain unknown until its arrival changes
|
||||||
|
// completion and updates the library's piece request priorities.
|
||||||
|
return metadataPiece{known: hash.Ok}
|
||||||
|
}
|
||||||
|
if s.torrent.PieceWithHash != nil {
|
||||||
|
return s.torrent.PieceWithHash(p, hash)
|
||||||
|
}
|
||||||
|
return s.torrent.Piece(p)
|
||||||
|
},
|
||||||
|
Close: s.Close,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *deferredFileStorage) bind(opts storage.NewFileClientOpts) ([]bool, error) {
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
if s.closed {
|
||||||
|
return nil, os.ErrClosed
|
||||||
|
}
|
||||||
|
if s.info == nil {
|
||||||
|
return nil, errors.New("torrent metadata is not ready")
|
||||||
|
}
|
||||||
|
if s.owner != nil {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
if opts.FilePathMaker == nil {
|
||||||
|
opts.FilePathMaker = func(o storage.FilePathMakerOpts) string { _, name := torrentFileLayout(o.Info, *o.File); return name }
|
||||||
|
}
|
||||||
|
owner := storage.NewFileOpts(opts)
|
||||||
|
t, err := owner.OpenTorrent(s.ctx, s.info, s.hash)
|
||||||
|
if err != nil {
|
||||||
|
owner.Close()
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
s.owner, s.torrent = owner, t
|
||||||
|
return append([]bool(nil), s.knownHashes...), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *deferredFileStorage) Close() error {
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
if s.closed {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
s.closed = true
|
||||||
|
if s.owner == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
var err error
|
||||||
|
if s.torrent.Close != nil {
|
||||||
|
err = s.torrent.Close()
|
||||||
|
}
|
||||||
|
return errors.Join(err, s.owner.Close())
|
||||||
|
}
|
||||||
|
|
||||||
|
type metadataPiece struct {
|
||||||
|
known bool
|
||||||
|
err error
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p metadataPiece) Completion() storage.Completion {
|
||||||
|
return storage.Completion{Ok: p.known, Err: p.err}
|
||||||
|
}
|
||||||
|
func (p metadataPiece) ReadAt([]byte, int64) (int, error) {
|
||||||
|
return 0, errors.New("download storage is not initialized")
|
||||||
|
}
|
||||||
|
func (p metadataPiece) WriteAt([]byte, int64) (int, error) {
|
||||||
|
return 0, errors.New("download storage is not initialized")
|
||||||
|
}
|
||||||
|
func (p metadataPiece) MarkComplete() error { return errors.New("download storage is not initialized") }
|
||||||
|
func (p metadataPiece) MarkNotComplete() error {
|
||||||
|
return errors.New("download storage is not initialized")
|
||||||
|
}
|
||||||
@@ -67,17 +67,14 @@ func (f *Fetcher) Setup(ctl *controller.Controller) {
|
|||||||
f.ctl.GetConfig(&f.config)
|
f.ctl.GetConfig(&f.config)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (f *Fetcher) Resolve(req *base.Request, opts *base.Options) error {
|
func (f *Fetcher) Resolve(req *base.Request) error {
|
||||||
link, err := parseLink(req.URL)
|
link, err := parseLink(req.URL)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
f.meta.Req = req
|
f.meta.Req = req
|
||||||
f.meta.Opts = opts
|
|
||||||
if f.meta.Opts == nil {
|
|
||||||
f.meta.Opts = &base.Options{}
|
|
||||||
}
|
|
||||||
f.meta.Res = buildResource(link)
|
f.meta.Res = buildResource(link)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -13,7 +13,8 @@ func TestFetcher_Resolve(t *testing.T) {
|
|||||||
f := (&FetcherManager{}).Build()
|
f := (&FetcherManager{}).Build()
|
||||||
f.Setup(controller.NewController())
|
f.Setup(controller.NewController())
|
||||||
|
|
||||||
err := f.Resolve(&base.Request{URL: testLink}, &base.Options{Path: t.TempDir()})
|
f.Meta().Opts = &base.Options{Path: t.TempDir()}
|
||||||
|
err := f.Resolve(&base.Request{URL: testLink})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Resolve() error = %v", err)
|
t.Fatalf("Resolve() error = %v", err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -172,11 +172,11 @@ func (f *Fetcher) Meta() *fetcher.FetcherMeta {
|
|||||||
return f.meta
|
return f.meta
|
||||||
}
|
}
|
||||||
|
|
||||||
func (f *Fetcher) Resolve(req *base.Request, opts *base.Options) (err error) {
|
func (f *Fetcher) Resolve(req *base.Request) (err error) {
|
||||||
f.mu.Lock()
|
f.mu.Lock()
|
||||||
defer f.mu.Unlock()
|
defer f.mu.Unlock()
|
||||||
f.meta.Req = req
|
f.meta.Req = req
|
||||||
f.meta.Opts = opts
|
|
||||||
f.extra = &ReqExtra{}
|
f.extra = &ReqExtra{}
|
||||||
if err = base.ParseReqExtra[ReqExtra](req); err != nil {
|
if err = base.ParseReqExtra[ReqExtra](req); err != nil {
|
||||||
return
|
return
|
||||||
@@ -194,10 +194,6 @@ func (f *Fetcher) Resolve(req *base.Request, opts *base.Options) (err error) {
|
|||||||
}
|
}
|
||||||
state.StagingID = randomStagingID()
|
state.StagingID = randomStagingID()
|
||||||
f.state = state
|
f.state = state
|
||||||
// Pin the staging folder now so the very first persisted state already
|
|
||||||
// carries it: a crash right after create still resumes into the same
|
|
||||||
// folder.
|
|
||||||
f.resolveStagingDirLocked()
|
|
||||||
size := planTotalSize(state.Segments)
|
size := planTotalSize(state.Segments)
|
||||||
f.meta.Res = &base.Resource{
|
f.meta.Res = &base.Resource{
|
||||||
Range: false,
|
Range: false,
|
||||||
|
|||||||
@@ -80,7 +80,8 @@ func mustResolve(t *testing.T, f *Fetcher, playlistURL, referer string) {
|
|||||||
if referer != "" {
|
if referer != "" {
|
||||||
req.Extra = &ReqExtra{Header: map[string]string{"Referer": referer}}
|
req.Extra = &ReqExtra{Header: map[string]string{"Referer": referer}}
|
||||||
}
|
}
|
||||||
if err := f.Resolve(req, &base.Options{Path: t.TempDir()}); err != nil {
|
f.Meta().Opts = &base.Options{Path: t.TempDir()}
|
||||||
|
if err := f.Resolve(req); err != nil {
|
||||||
t.Fatalf("resolve: %v", err)
|
t.Fatalf("resolve: %v", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -207,7 +208,8 @@ func TestFetcherE2E_RefererEnforced(t *testing.T) {
|
|||||||
// Without Referer the resolve itself must fail (403 on the playlist).
|
// Without Referer the resolve itself must fail (403 on the playlist).
|
||||||
blocked := newTestFetcher(t, testConfig())
|
blocked := newTestFetcher(t, testConfig())
|
||||||
req := &base.Request{URL: server.URL + "/video.m3u8"}
|
req := &base.Request{URL: server.URL + "/video.m3u8"}
|
||||||
if err := blocked.Resolve(req, &base.Options{Path: t.TempDir()}); err == nil {
|
blocked.Meta().Opts = &base.Options{Path: t.TempDir()}
|
||||||
|
if err := blocked.Resolve(req); err == nil {
|
||||||
t.Fatal("resolve without Referer should fail")
|
t.Fatal("resolve without Referer should fail")
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -217,7 +219,8 @@ func TestFetcherE2E_RefererEnforced(t *testing.T) {
|
|||||||
URL: server.URL + "/video.m3u8",
|
URL: server.URL + "/video.m3u8",
|
||||||
Extra: &ReqExtra{Header: map[string]string{"Referer": page}},
|
Extra: &ReqExtra{Header: map[string]string{"Referer": page}},
|
||||||
}
|
}
|
||||||
if err := allowed.Resolve(allowedReq, &base.Options{Path: t.TempDir()}); err != nil {
|
allowed.Meta().Opts = &base.Options{Path: t.TempDir()}
|
||||||
|
if err := allowed.Resolve(allowedReq); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
mustRun(t, allowed)
|
mustRun(t, allowed)
|
||||||
@@ -560,7 +563,8 @@ func TestFetcherE2E_StoreRestoreAuthenticated(t *testing.T) {
|
|||||||
URL: server.URL + "/video.m3u8",
|
URL: server.URL + "/video.m3u8",
|
||||||
Extra: &ReqExtra{Header: map[string]string{"Referer": page}},
|
Extra: &ReqExtra{Header: map[string]string{"Referer": page}},
|
||||||
}
|
}
|
||||||
if err := f.Resolve(req, &base.Options{Path: t.TempDir()}); err != nil {
|
f.Meta().Opts = &base.Options{Path: t.TempDir()}
|
||||||
|
if err := f.Resolve(req); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if err := f.Start(); err != nil {
|
if err := f.Start(); err != nil {
|
||||||
@@ -630,7 +634,8 @@ func TestFetcherE2E_PatchPartialUpdate(t *testing.T) {
|
|||||||
Extra: &ReqExtra{Header: map[string]string{"Referer": page}},
|
Extra: &ReqExtra{Header: map[string]string{"Referer": page}},
|
||||||
}
|
}
|
||||||
opts := &base.Options{Path: t.TempDir()}
|
opts := &base.Options{Path: t.TempDir()}
|
||||||
if err := f.Resolve(req, opts); err != nil {
|
f.Meta().Opts = opts
|
||||||
|
if err := f.Resolve(req); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -713,7 +718,8 @@ func TestFetcherE2E_PatchWhileRunningRejected(t *testing.T) {
|
|||||||
defer server.Close()
|
defer server.Close()
|
||||||
|
|
||||||
f := newTestFetcher(t, testConfig())
|
f := newTestFetcher(t, testConfig())
|
||||||
if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: t.TempDir()}); err != nil {
|
f.Meta().Opts = &base.Options{Path: t.TempDir()}
|
||||||
|
if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if err := f.Start(); err != nil {
|
if err := f.Start(); err != nil {
|
||||||
@@ -767,7 +773,8 @@ func TestFetcherE2E_RetryAfterFailure(t *testing.T) {
|
|||||||
cfg.MaxRetries = 0 // fail the run on the first bad segment
|
cfg.MaxRetries = 0 // fail the run on the first bad segment
|
||||||
cfg.PrefetchContentLength = false
|
cfg.PrefetchContentLength = false
|
||||||
f := newTestFetcher(t, cfg)
|
f := newTestFetcher(t, cfg)
|
||||||
if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: t.TempDir()}); err != nil {
|
f.Meta().Opts = &base.Options{Path: t.TempDir()}
|
||||||
|
if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if err := f.Start(); err != nil {
|
if err := f.Start(); err != nil {
|
||||||
@@ -828,7 +835,8 @@ func TestFetcherE2E_StateMissingSelfHeal(t *testing.T) {
|
|||||||
defer server.Close()
|
defer server.Close()
|
||||||
|
|
||||||
f := newTestFetcher(t, testConfig())
|
f := newTestFetcher(t, testConfig())
|
||||||
if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: t.TempDir()}); err != nil {
|
f.Meta().Opts = &base.Options{Path: t.TempDir()}
|
||||||
|
if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
// Simulate the crash window where the engine persisted the task with its
|
// Simulate the crash window where the engine persisted the task with its
|
||||||
|
|||||||
@@ -189,7 +189,8 @@ func TestHLSRealMedia_TS(t *testing.T) {
|
|||||||
defer server.Close()
|
defer server.Close()
|
||||||
|
|
||||||
f := newTestFetcher(t, testConfig())
|
f := newTestFetcher(t, testConfig())
|
||||||
if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: t.TempDir()}); err != nil {
|
f.Meta().Opts = &base.Options{Path: t.TempDir()}
|
||||||
|
if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if err := f.Start(); err != nil {
|
if err := f.Start(); err != nil {
|
||||||
@@ -225,7 +226,8 @@ func TestHLSRealMedia_FMP4(t *testing.T) {
|
|||||||
defer server.Close()
|
defer server.Close()
|
||||||
|
|
||||||
f := newTestFetcher(t, testConfig())
|
f := newTestFetcher(t, testConfig())
|
||||||
if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: t.TempDir()}); err != nil {
|
f.Meta().Opts = &base.Options{Path: t.TempDir()}
|
||||||
|
if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if err := f.Start(); err != nil {
|
if err := f.Start(); err != nil {
|
||||||
@@ -274,7 +276,8 @@ func TestHLSRealMedia_AES128(t *testing.T) {
|
|||||||
defer server.Close()
|
defer server.Close()
|
||||||
|
|
||||||
f := newTestFetcher(t, testConfig())
|
f := newTestFetcher(t, testConfig())
|
||||||
if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: t.TempDir()}); err != nil {
|
f.Meta().Opts = &base.Options{Path: t.TempDir()}
|
||||||
|
if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if f.meta.Res.Size != 0 {
|
if f.meta.Res.Size != 0 {
|
||||||
|
|||||||
@@ -50,15 +50,20 @@ func TestFetcherE2E_SameURLTwoTasks(t *testing.T) {
|
|||||||
// user duplicating a task. The engine would rename the second output
|
// user duplicating a task. The engine would rename the second output
|
||||||
// (AutoRename); mirror that here.
|
// (AutoRename); mirror that here.
|
||||||
f1 := newTestFetcher(t, cfg)
|
f1 := newTestFetcher(t, cfg)
|
||||||
if err := f1.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: dir}); err != nil {
|
f1.Meta().Opts = &base.Options{Path: dir}
|
||||||
|
if err := f1.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
f2 := newTestFetcher(t, cfg)
|
f2 := newTestFetcher(t, cfg)
|
||||||
if err := f2.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: dir, Name: "video-2.ts"}); err != nil {
|
f2.Meta().Opts = &base.Options{Path: dir, Name: "video-2.ts"}
|
||||||
|
if err := f2.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if f1.state.StagingDir == "" || f1.state.StagingDir == f2.state.StagingDir {
|
if f1.state.StagingID == "" || f1.state.StagingID == f2.state.StagingID {
|
||||||
t.Fatalf("staging dirs must be distinct per task: %q vs %q", f1.state.StagingDir, f2.state.StagingDir)
|
t.Fatal("staging identities must be distinct per task")
|
||||||
|
}
|
||||||
|
if f1.state.StagingDir != "" || f2.state.StagingDir != "" {
|
||||||
|
t.Fatal("Resolve pinned an output directory")
|
||||||
}
|
}
|
||||||
|
|
||||||
// Start task 1, let it stage some segments, then pause it.
|
// Start task 1, let it stage some segments, then pause it.
|
||||||
@@ -135,7 +140,8 @@ func TestFetcherE2E_ForeignJournalDiscarded(t *testing.T) {
|
|||||||
cfg := testConfig()
|
cfg := testConfig()
|
||||||
cfg.PrefetchContentLength = false
|
cfg.PrefetchContentLength = false
|
||||||
f := newTestFetcher(t, cfg)
|
f := newTestFetcher(t, cfg)
|
||||||
if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: t.TempDir()}); err != nil {
|
f.Meta().Opts = &base.Options{Path: t.TempDir()}
|
||||||
|
if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
// Plant a journal that does not belong to this plan (wrong fingerprint).
|
// Plant a journal that does not belong to this plan (wrong fingerprint).
|
||||||
@@ -176,7 +182,8 @@ func TestFetcherE2E_MergeFailurePreservesStaging(t *testing.T) {
|
|||||||
defer server.Close()
|
defer server.Close()
|
||||||
|
|
||||||
f := newTestFetcher(t, testConfig())
|
f := newTestFetcher(t, testConfig())
|
||||||
if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: t.TempDir()}); err != nil {
|
f.Meta().Opts = &base.Options{Path: t.TempDir()}
|
||||||
|
if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
run := &hlsRun{
|
run := &hlsRun{
|
||||||
@@ -250,7 +257,8 @@ func TestFetcherE2E_ReMergeOverExistingOutput(t *testing.T) {
|
|||||||
defer server.Close()
|
defer server.Close()
|
||||||
|
|
||||||
f := newTestFetcher(t, testConfig())
|
f := newTestFetcher(t, testConfig())
|
||||||
if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: t.TempDir()}); err != nil {
|
f.Meta().Opts = &base.Options{Path: t.TempDir()}
|
||||||
|
if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
tempDir := f.resolveStagingDirLocked()
|
tempDir := f.resolveStagingDirLocked()
|
||||||
|
|||||||
@@ -372,19 +372,11 @@ func (f *Fetcher) updateMaxConnTime(d time.Duration) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (f *Fetcher) Resolve(req *base.Request, opts *base.Options) error {
|
func (f *Fetcher) Resolve(req *base.Request) error {
|
||||||
if err := base.ParseReqExtra[fhttp.ReqExtra](req); err != nil {
|
if err := base.ParseReqExtra[fhttp.ReqExtra](req); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
f.meta.Req = req
|
f.meta.Req = req
|
||||||
f.meta.Opts = opts
|
|
||||||
if f.meta.Opts == nil {
|
|
||||||
f.meta.Opts = &base.Options{}
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := f.initOptions(); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
f.setState(stateResolving)
|
f.setState(stateResolving)
|
||||||
|
|
||||||
@@ -693,6 +685,9 @@ func (f *Fetcher) prepareFullRedownload() {
|
|||||||
|
|
||||||
// Options may be replaced between Resolve and Start when creating a resolved task.
|
// Options may be replaced between Resolve and Start when creating a resolved task.
|
||||||
func (f *Fetcher) initOptions() error {
|
func (f *Fetcher) initOptions() error {
|
||||||
|
if f.meta.Opts == nil {
|
||||||
|
f.meta.Opts = &base.Options{}
|
||||||
|
}
|
||||||
opts := f.meta.Opts
|
opts := f.meta.Opts
|
||||||
// Parse options
|
// Parse options
|
||||||
if err := base.ParseOptExtra[fhttp.OptsExtra](opts); err != nil {
|
if err := base.ParseOptExtra[fhttp.OptsExtra](opts); err != nil {
|
||||||
|
|||||||
@@ -95,9 +95,10 @@ func TestFetcher_ChecksumMismatch_Retry(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// 1. Initial attempt -> will download corruptContent, failing checksum verification
|
// 1. Initial attempt -> will download corruptContent, failing checksum verification
|
||||||
if err := f.Resolve(&base.Request{URL: server.URL + "/retry_test.txt"}, opts); err != nil {
|
if err := f.Resolve(&base.Request{URL: server.URL + "/retry_test.txt"}); err != nil {
|
||||||
t.Fatalf("Resolve error: %v", err)
|
t.Fatalf("Resolve error: %v", err)
|
||||||
}
|
}
|
||||||
|
f.Meta().Opts = opts
|
||||||
if err := f.Start(); err != nil {
|
if err := f.Start(); err != nil {
|
||||||
t.Fatalf("Start error: %v", err)
|
t.Fatalf("Start error: %v", err)
|
||||||
}
|
}
|
||||||
@@ -215,9 +216,10 @@ func TestFetcher_ChecksumMismatch_Retry_PartialPrefetch(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// 1. Resolve phase
|
// 1. Resolve phase
|
||||||
if err := f.Resolve(&base.Request{URL: server.URL + "/prefetch_retry.txt"}, opts); err != nil {
|
if err := f.Resolve(&base.Request{URL: server.URL + "/prefetch_retry.txt"}); err != nil {
|
||||||
t.Fatalf("Resolve error: %v", err)
|
t.Fatalf("Resolve error: %v", err)
|
||||||
}
|
}
|
||||||
|
f.Meta().Opts = opts
|
||||||
|
|
||||||
// Wait until prefetch has received exactly 7 bytes
|
// Wait until prefetch has received exactly 7 bytes
|
||||||
for i := 0; i < 200; i++ {
|
for i := 0; i < 200; i++ {
|
||||||
@@ -338,9 +340,10 @@ func TestFetcher_ChecksumMismatch_Retry_NonRange(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// 1. Resolve and start initial attempt
|
// 1. Resolve and start initial attempt
|
||||||
if err := f.Resolve(&base.Request{URL: server.URL + "/nonrange_retry.txt"}, opts); err != nil {
|
if err := f.Resolve(&base.Request{URL: server.URL + "/nonrange_retry.txt"}); err != nil {
|
||||||
t.Fatalf("Resolve error: %v", err)
|
t.Fatalf("Resolve error: %v", err)
|
||||||
}
|
}
|
||||||
|
f.Meta().Opts = opts
|
||||||
if err := f.Start(); err != nil {
|
if err := f.Start(); err != nil {
|
||||||
t.Fatalf("Start error: %v", err)
|
t.Fatalf("Start error: %v", err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -52,9 +52,11 @@ func TestFetcher_ChecksumVerification(t *testing.T) {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := f.Resolve(&base.Request{URL: server.URL + "/test.txt"}, opts); err != nil {
|
if err := f.Resolve(&base.Request{URL: server.URL + "/test.txt"}); err != nil {
|
||||||
t.Fatalf("Resolve error: %v", err)
|
t.Fatalf("Resolve error: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
f.Meta().Opts = opts
|
||||||
if err := f.Start(); err != nil {
|
if err := f.Start(); err != nil {
|
||||||
t.Fatalf("Start error: %v", err)
|
t.Fatalf("Start error: %v", err)
|
||||||
}
|
}
|
||||||
@@ -79,9 +81,11 @@ func TestFetcher_ChecksumVerification(t *testing.T) {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := f.Resolve(&base.Request{URL: server.URL + "/test.txt"}, opts); err != nil {
|
if err := f.Resolve(&base.Request{URL: server.URL + "/test.txt"}); err != nil {
|
||||||
t.Fatalf("Resolve error: %v", err)
|
t.Fatalf("Resolve error: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
f.Meta().Opts = opts
|
||||||
if err := f.Start(); err != nil {
|
if err := f.Start(); err != nil {
|
||||||
t.Fatalf("Start error: %v", err)
|
t.Fatalf("Start error: %v", err)
|
||||||
}
|
}
|
||||||
@@ -106,9 +110,11 @@ func TestFetcher_ChecksumVerification(t *testing.T) {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := f.Resolve(&base.Request{URL: server.URL + "/test.txt"}, opts); err != nil {
|
if err := f.Resolve(&base.Request{URL: server.URL + "/test.txt"}); err != nil {
|
||||||
t.Fatalf("Resolve error: %v", err)
|
t.Fatalf("Resolve error: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
f.Meta().Opts = opts
|
||||||
if err := f.Start(); err != nil {
|
if err := f.Start(); err != nil {
|
||||||
t.Fatalf("Start error: %v", err)
|
t.Fatalf("Start error: %v", err)
|
||||||
}
|
}
|
||||||
@@ -137,9 +143,13 @@ func TestFetcher_ChecksumVerification(t *testing.T) {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
err := f.Resolve(&base.Request{URL: server.URL + "/test.txt"}, opts)
|
if err := f.Resolve(&base.Request{URL: server.URL + "/test.txt"}); err != nil {
|
||||||
|
t.Fatalf("Resolve error: %v", err)
|
||||||
|
}
|
||||||
|
f.Meta().Opts = opts
|
||||||
|
err := f.Start()
|
||||||
if err == nil {
|
if err == nil {
|
||||||
t.Fatal("Resolve expected error for unsupported algorithm, got nil")
|
t.Fatal("Start expected error for unsupported algorithm, got nil")
|
||||||
}
|
}
|
||||||
if !strings.Contains(err.Error(), "unsupported checksum algorithm") {
|
if !strings.Contains(err.Error(), "unsupported checksum algorithm") {
|
||||||
t.Fatalf("expected 'unsupported checksum algorithm' in error, got %v", err)
|
t.Fatalf("expected 'unsupported checksum algorithm' in error, got %v", err)
|
||||||
@@ -165,9 +175,11 @@ func TestFetcher_ChecksumVerification(t *testing.T) {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := f.Resolve(&base.Request{URL: server.URL + "/test.txt"}, opts); err != nil {
|
if err := f.Resolve(&base.Request{URL: server.URL + "/test.txt"}); err != nil {
|
||||||
t.Fatalf("Resolve error: %v", err)
|
t.Fatalf("Resolve error: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
f.Meta().Opts = opts
|
||||||
if err := f.Start(); err != nil {
|
if err := f.Start(); err != nil {
|
||||||
t.Fatalf("Start error: %v", err)
|
t.Fatalf("Start error: %v", err)
|
||||||
}
|
}
|
||||||
@@ -192,9 +204,11 @@ func TestFetcher_ChecksumVerification(t *testing.T) {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := f.Resolve(&base.Request{URL: server.URL + "/test.txt"}, opts); err != nil {
|
if err := f.Resolve(&base.Request{URL: server.URL + "/test.txt"}); err != nil {
|
||||||
t.Fatalf("Resolve error: %v", err)
|
t.Fatalf("Resolve error: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
f.Meta().Opts = opts
|
||||||
if err := f.Start(); err != nil {
|
if err := f.Start(); err != nil {
|
||||||
t.Fatalf("Start error: %v", err)
|
t.Fatalf("Start error: %v", err)
|
||||||
}
|
}
|
||||||
@@ -235,9 +249,11 @@ func TestFetcher_ChecksumVerification(t *testing.T) {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := f.Resolve(&base.Request{URL: largeServer.URL + "/large.bin"}, opts); err != nil {
|
if err := f.Resolve(&base.Request{URL: largeServer.URL + "/large.bin"}); err != nil {
|
||||||
t.Fatalf("Resolve error: %v", err)
|
t.Fatalf("Resolve error: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
f.Meta().Opts = opts
|
||||||
if err := f.Start(); err != nil {
|
if err := f.Start(); err != nil {
|
||||||
t.Fatalf("Start error: %v", err)
|
t.Fatalf("Start error: %v", err)
|
||||||
}
|
}
|
||||||
@@ -258,9 +274,11 @@ func TestFetcher_ChecksumVerification(t *testing.T) {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := f.Resolve(&base.Request{URL: server.URL + "/test.txt"}, opts); err != nil {
|
if err := f.Resolve(&base.Request{URL: server.URL + "/test.txt"}); err != nil {
|
||||||
t.Fatalf("Resolve error: %v", err)
|
t.Fatalf("Resolve error: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
f.Meta().Opts = opts
|
||||||
if err := f.Start(); err != nil {
|
if err := f.Start(); err != nil {
|
||||||
t.Fatalf("Start error: %v", err)
|
t.Fatalf("Start error: %v", err)
|
||||||
}
|
}
|
||||||
@@ -269,7 +287,7 @@ func TestFetcher_ChecksumVerification(t *testing.T) {
|
|||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
t.Run("Invalid Checksum In Resolve Makes Zero Requests", func(t *testing.T) {
|
t.Run("Invalid Checksum In Start Makes No Additional Requests", func(t *testing.T) {
|
||||||
var reqReceived atomic.Int32
|
var reqReceived atomic.Int32
|
||||||
customServer := httptest.NewServer(gohttp.HandlerFunc(func(w gohttp.ResponseWriter, r *gohttp.Request) {
|
customServer := httptest.NewServer(gohttp.HandlerFunc(func(w gohttp.ResponseWriter, r *gohttp.Request) {
|
||||||
reqReceived.Add(1)
|
reqReceived.Add(1)
|
||||||
@@ -293,12 +311,17 @@ func TestFetcher_ChecksumVerification(t *testing.T) {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
err := f.Resolve(&base.Request{URL: customServer.URL + "/test.txt"}, opts)
|
if err := f.Resolve(&base.Request{URL: customServer.URL + "/test.txt"}); err != nil {
|
||||||
if err == nil {
|
t.Fatalf("Resolve error: %v", err)
|
||||||
t.Fatal("Resolve expected error for invalid checksum hash, got nil")
|
|
||||||
}
|
}
|
||||||
if reqReceived.Load() != 0 {
|
requestsAfterResolve := reqReceived.Load()
|
||||||
t.Fatalf("expected 0 requests to server on invalid checksum in Resolve, got %d", reqReceived.Load())
|
f.Meta().Opts = opts
|
||||||
|
err := f.Start()
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("Start expected error for invalid checksum hash, got nil")
|
||||||
|
}
|
||||||
|
if got := reqReceived.Load(); got != requestsAfterResolve {
|
||||||
|
t.Fatalf("invalid checksum in Start issued additional requests: got %d, had %d after Resolve", got, requestsAfterResolve)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
File diff suppressed because it is too large.
Load diff
@@ -49,7 +49,8 @@ func TestResolveResponseSize(t *testing.T) {
|
|||||||
defer server.Close()
|
defer server.Close()
|
||||||
f := buildFetcher()
|
f := buildFetcher()
|
||||||
defer f.Close()
|
defer f.Close()
|
||||||
err := f.Resolve(&base.Request{URL: server.URL + "/file"}, &base.Options{Path: t.TempDir()})
|
f.Meta().Opts = &base.Options{Path: t.TempDir()}
|
||||||
|
err := f.Resolve(&base.Request{URL: server.URL + "/file"})
|
||||||
if tc.invalid {
|
if tc.invalid {
|
||||||
if !errors.Is(err, errInvalidRangeResponse) {
|
if !errors.Is(err, errInvalidRangeResponse) {
|
||||||
t.Fatalf("Resolve error = %v, want invalid range", err)
|
t.Fatalf("Resolve error = %v, want invalid range", err)
|
||||||
@@ -97,7 +98,8 @@ func TestDownloadPartialResolve(t *testing.T) {
|
|||||||
defer server.Close()
|
defer server.Close()
|
||||||
f := buildFetcher()
|
f := buildFetcher()
|
||||||
defer f.Close()
|
defer f.Close()
|
||||||
if err := f.Resolve(&base.Request{URL: server.URL + "/file"}, &base.Options{Path: t.TempDir(), Name: "file", Extra: &fhttp.OptsExtra{Connections: connections}}); err != nil {
|
f.Meta().Opts = &base.Options{Path: t.TempDir(), Name: "file", Extra: &fhttp.OptsExtra{Connections: connections}}
|
||||||
|
if err := f.Resolve(&base.Request{URL: server.URL + "/file"}); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if err := f.Start(); err != nil {
|
if err := f.Start(); err != nil {
|
||||||
|
|||||||
+2
-2
@@ -139,7 +139,7 @@ func (s *Service) resolve(ctx *Context) *Response {
|
|||||||
if response := decodeRequest(ctx.Body, &request); response != nil {
|
if response := decodeRequest(ctx.Body, &request); response != nil {
|
||||||
return response
|
return response
|
||||||
}
|
}
|
||||||
result, err := s.Downloader.Resolve(request.Req, request.Opts)
|
result, err := s.Downloader.Resolve(request.Req)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return newErrorResponse(err.Error())
|
return newErrorResponse(err.Error())
|
||||||
}
|
}
|
||||||
@@ -188,7 +188,7 @@ func (s *Service) patchTask(ctx *Context) *Response {
|
|||||||
if taskID == "" {
|
if taskID == "" {
|
||||||
return invalidIDResponse()
|
return invalidIDResponse()
|
||||||
}
|
}
|
||||||
var request restmodel.ResolveTask
|
var request restmodel.PatchTask
|
||||||
if response := decodeRequest(ctx.Body, &request); response != nil {
|
if response := decodeRequest(ctx.Body, &request); response != nil {
|
||||||
return response
|
return response
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,82 @@
|
|||||||
|
package api
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"reflect"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/GopeedLab/gopeed/pkg/base"
|
||||||
|
"github.com/GopeedLab/gopeed/pkg/download"
|
||||||
|
restmodel "github.com/GopeedLab/gopeed/pkg/rest/model"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestResolveThenCreateWithDownloadOptions(t *testing.T) {
|
||||||
|
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||||
|
w.Header().Set("Content-Disposition", `attachment; filename="payload.txt"`)
|
||||||
|
w.Header().Set("Content-Length", "4")
|
||||||
|
_, _ = w.Write([]byte("test"))
|
||||||
|
}))
|
||||||
|
t.Cleanup(upstream.Close)
|
||||||
|
d := download.NewDownloader(&download.DownloaderConfig{StorageDir: t.TempDir()})
|
||||||
|
if err := d.Setup(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() {
|
||||||
|
if err := d.Close(); err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
// Resolving must work even when the default download directory is unusable.
|
||||||
|
blocked := filepath.Join(t.TempDir(), "not-a-directory")
|
||||||
|
if err := os.WriteFile(blocked, nil, 0600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
current, err := d.GetConfig()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
config := *current
|
||||||
|
config.DownloadDir = blocked
|
||||||
|
config.MaxRunning = 0 // Inspect the created task before it starts downloading.
|
||||||
|
if err := d.PutConfig(&config); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
service, err := NewService(d)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
resolved := serviceTestRequest[*download.ResolveResult](t, service, http.MethodPost, "/api/v1/resolve", &restmodel.ResolveTask{
|
||||||
|
Req: &base.Request{URL: upstream.URL},
|
||||||
|
})
|
||||||
|
if resolved.ID == "" || resolved.Res.Size != 4 || resolved.Res.Files[0].Name != "payload.txt" {
|
||||||
|
t.Fatalf("unexpected resolved resource: %+v", resolved)
|
||||||
|
}
|
||||||
|
dir := filepath.Join(t.TempDir(), "chosen-after-resolve")
|
||||||
|
opts := &base.Options{Path: dir, Name: "renamed.txt", SelectFiles: []int{0}}
|
||||||
|
id := serviceTestRequest[string](t, service, http.MethodPost, "/api/v1/tasks", &restmodel.CreateTask{Rid: resolved.ID, Opts: opts})
|
||||||
|
task := serviceTestRequest[*download.Task](t, service, http.MethodGet, "/api/v1/tasks/"+id, nil)
|
||||||
|
if task.Meta.Opts.Path != dir || task.Meta.Opts.Name != opts.Name || !reflect.DeepEqual(task.Meta.Opts.SelectFiles, opts.SelectFiles) {
|
||||||
|
t.Fatalf("creation lost submitted download options: %+v", task.Meta.Opts)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func serviceTestRequest[T any](t *testing.T, service *Service, method, path string, body any) T {
|
||||||
|
t.Helper()
|
||||||
|
encoded, err := json.Marshal(body)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
status, response := service.JSON(method, path, nil, encoded, nil)
|
||||||
|
var result restmodel.Result[T]
|
||||||
|
if err := json.Unmarshal(response, &result); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if status != http.StatusOK || result.Code != restmodel.CodeOk {
|
||||||
|
t.Fatalf("%s %s failed: status=%d response=%s", method, path, status, response)
|
||||||
|
}
|
||||||
|
return result.Data
|
||||||
|
}
|
||||||
+41
-29
@@ -244,16 +244,25 @@ func (d *Downloader) Setup() error {
|
|||||||
tasks := append([]*Task(nil), d.tasks...)
|
tasks := append([]*Task(nil), d.tasks...)
|
||||||
d.lock.Unlock()
|
d.lock.Unlock()
|
||||||
for _, task := range tasks {
|
for _, task := range tasks {
|
||||||
if task.Status == base.DownloadStatusDone && task.Uploading {
|
func() {
|
||||||
|
task.lock.Lock()
|
||||||
|
defer task.lock.Unlock()
|
||||||
|
if d.closed.Load() {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if d.taskStatus(task) != base.DownloadStatusDone || !task.Uploading {
|
||||||
|
return
|
||||||
|
}
|
||||||
if err := d.restoreTask(task); err != nil {
|
if err := d.restoreTask(task); err != nil {
|
||||||
d.Logger.Error().Stack().Err(err).Msgf("task upload restore fetcher failed, task id: %s", task.ID)
|
d.Logger.Error().Err(err).Msgf("task upload restore fetcher failed, task id: %s", task.ID)
|
||||||
|
return
|
||||||
}
|
}
|
||||||
if uploader, ok := task.fetcher.(fetcher.Uploader); ok {
|
if uploader, ok := task.fetcher.(fetcher.Uploader); ok {
|
||||||
if err := uploader.Upload(); err != nil {
|
if err := uploader.Upload(); err != nil {
|
||||||
d.Logger.Error().Stack().Err(err).Msgf("task upload failed, task id: %s", task.ID)
|
d.Logger.Error().Err(err).Msgf("task upload failed, task id: %s", task.ID)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}()
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
|
|
||||||
@@ -467,7 +476,10 @@ func (d *Downloader) saveTask(task *Task) error {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if err := d.storage.Put(bucketTask, task.ID, task); err != nil {
|
task.statusLock.Lock()
|
||||||
|
snapshot := task.clone()
|
||||||
|
task.statusLock.Unlock()
|
||||||
|
if err := d.storage.Put(bucketTask, task.ID, snapshot); err != nil {
|
||||||
d.Logger.Error().Stack().Err(err).Msgf("persist task failed: %s", task.ID)
|
d.Logger.Error().Stack().Err(err).Msgf("persist task failed: %s", task.ID)
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -498,7 +510,7 @@ func ensureResourceRequestRawURLs(parentReq *base.Request, res *base.Resource) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (d *Downloader) Resolve(req *base.Request, opts *base.Options) (rr *ResolveResult, err error) {
|
func (d *Downloader) Resolve(req *base.Request) (rr *ResolveResult, err error) {
|
||||||
rrId, err := gonanoid.New()
|
rrId, err := gonanoid.New()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return
|
return
|
||||||
@@ -520,11 +532,7 @@ func (d *Downloader) Resolve(req *base.Request, opts *base.Options) (rr *Resolve
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
initOpt, err := d.initOptions(opts)
|
err = fetcher.Resolve(req)
|
||||||
if err != nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
err = fetcher.Resolve(req, initOpt)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -558,7 +566,7 @@ func (d *Downloader) notifyRunning() {
|
|||||||
func (d *Downloader) remainRunningCount() int {
|
func (d *Downloader) remainRunningCount() int {
|
||||||
runningCount := 0
|
runningCount := 0
|
||||||
for _, t := range d.tasks {
|
for _, t := range d.tasks {
|
||||||
if t.Status == base.DownloadStatusRunning {
|
if d.taskStatus(t) == base.DownloadStatusRunning {
|
||||||
runningCount++
|
runningCount++
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -621,7 +629,7 @@ func (d *Downloader) Create(rrId string) (taskId string, err error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// CreateWithOptions creates a resolved task using the submitted options, or the
|
// CreateWithOptions creates a resolved task using the submitted options, or the
|
||||||
// resolve-time options when opts is nil.
|
// default download options when opts is nil.
|
||||||
func (d *Downloader) CreateWithOptions(rrId string, opts *base.Options) (taskId string, err error) {
|
func (d *Downloader) CreateWithOptions(rrId string, opts *base.Options) (taskId string, err error) {
|
||||||
d.fetcherMapLock.RLock()
|
d.fetcherMapLock.RLock()
|
||||||
fetcher, ok := d.fetcherCache[rrId]
|
fetcher, ok := d.fetcherCache[rrId]
|
||||||
@@ -629,21 +637,22 @@ func (d *Downloader) CreateWithOptions(rrId string, opts *base.Options) (taskId
|
|||||||
if !ok {
|
if !ok {
|
||||||
return "", errors.New("invalid resource id")
|
return "", errors.New("invalid resource id")
|
||||||
}
|
}
|
||||||
if opts != nil {
|
if opts == nil {
|
||||||
opts, err = d.initOptions(opts.Clone())
|
opts = &base.Options{}
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
if res := fetcher.Meta().Res; res != nil {
|
|
||||||
for _, index := range opts.SelectFiles {
|
|
||||||
if index < 0 || index >= len(res.Files) {
|
|
||||||
return "", fmt.Errorf("invalid selected file index: %d", index)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
opts.InitSelectFiles(len(res.Files))
|
|
||||||
}
|
|
||||||
fetcher.Meta().Opts = opts
|
|
||||||
}
|
}
|
||||||
|
opts, err = d.initOptions(opts.Clone())
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
if res := fetcher.Meta().Res; res != nil {
|
||||||
|
for _, index := range opts.SelectFiles {
|
||||||
|
if index < 0 || index >= len(res.Files) {
|
||||||
|
return "", fmt.Errorf("invalid selected file index: %d", index)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
opts.InitSelectFiles(len(res.Files))
|
||||||
|
}
|
||||||
|
fetcher.Meta().Opts = opts
|
||||||
if res := fetcher.Meta().Res; res != nil {
|
if res := fetcher.Meta().Res; res != nil {
|
||||||
res.CalcSize(fetcher.Meta().Opts.SelectFiles)
|
res.CalcSize(fetcher.Meta().Opts.SelectFiles)
|
||||||
}
|
}
|
||||||
@@ -1469,7 +1478,10 @@ func (d *Downloader) watch(task *Task) {
|
|||||||
if err := d.captureTaskStats(task, false, true); err != nil {
|
if err := d.captureTaskStats(task, false, true); err != nil {
|
||||||
d.Logger.Error().Stack().Err(err).Msgf("persist completed task stats failed, task id: %s", task.ID)
|
d.Logger.Error().Stack().Err(err).Msgf("persist completed task stats failed, task id: %s", task.ID)
|
||||||
}
|
}
|
||||||
d.storage.Put(bucketTask, task.ID, task.clone())
|
task.statusLock.Lock()
|
||||||
|
snapshot := task.clone()
|
||||||
|
task.statusLock.Unlock()
|
||||||
|
d.storage.Put(bucketTask, task.ID, snapshot)
|
||||||
d.emit(EventKeyDone, task)
|
d.emit(EventKeyDone, task)
|
||||||
d.emit(EventKeyFinally, task, err)
|
d.emit(EventKeyFinally, task, err)
|
||||||
d.notifyRunning()
|
d.notifyRunning()
|
||||||
@@ -1826,7 +1838,7 @@ func (d *Downloader) doStart(task *Task) (err error) {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if task.Meta.Res == nil {
|
if task.Meta.Res == nil {
|
||||||
err := task.fetcher.Resolve(task.Meta.Req, task.Meta.Opts)
|
err := task.fetcher.Resolve(task.Meta.Req)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -132,7 +132,7 @@ func (f *generationTestFetcher) Setup(*controller.Controller) {
|
|||||||
f.done = make(chan error, 2)
|
f.done = make(chan error, 2)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
func (f *generationTestFetcher) Resolve(req *base.Request, opts *base.Options) error {
|
func (f *generationTestFetcher) Resolve(req *base.Request) error {
|
||||||
if f.manager.resolveStarted != nil {
|
if f.manager.resolveStarted != nil {
|
||||||
f.manager.resolveOnce.Do(func() { close(f.manager.resolveStarted) })
|
f.manager.resolveOnce.Do(func() { close(f.manager.resolveStarted) })
|
||||||
}
|
}
|
||||||
@@ -143,7 +143,7 @@ func (f *generationTestFetcher) Resolve(req *base.Request, opts *base.Options) e
|
|||||||
return f.manager.resolveErr
|
return f.manager.resolveErr
|
||||||
}
|
}
|
||||||
f.meta.Req = req
|
f.meta.Req = req
|
||||||
f.meta.Opts = opts
|
|
||||||
f.meta.Res = &base.Resource{Files: []*base.FileInfo{{Name: "generation.bin", Size: 1}}}
|
f.meta.Res = &base.Resource{Files: []*base.FileInfo{{Name: "generation.bin", Size: 1}}}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
@@ -245,7 +245,7 @@ func TestDownloader_Resolve(t *testing.T) {
|
|||||||
req := &base.Request{
|
req := &base.Request{
|
||||||
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
|
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
|
||||||
}
|
}
|
||||||
rr, err := downloader.Resolve(req, nil)
|
rr, err := downloader.Resolve(req)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -772,7 +772,7 @@ func TestDownloader_Create(t *testing.T) {
|
|||||||
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
|
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
|
||||||
}
|
}
|
||||||
opts := newTestDownloadOpt(t)
|
opts := newTestDownloadOpt(t)
|
||||||
rr, err := downloader.Resolve(req, opts)
|
rr, err := downloader.Resolve(req)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -784,7 +784,7 @@ func TestDownloader_Create(t *testing.T) {
|
|||||||
wg.Done()
|
wg.Done()
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
_, err = downloader.Create(rr.ID)
|
_, err = downloader.CreateWithOptions(rr.ID, opts)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -878,15 +878,15 @@ func TestDownloader_CreateNotInWhite(t *testing.T) {
|
|||||||
req := &base.Request{
|
req := &base.Request{
|
||||||
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
|
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
|
||||||
}
|
}
|
||||||
// With new fetcher design, white list check happens during Resolve (not Create)
|
rr, err := downloader.Resolve(req)
|
||||||
// because Resolve now requires Options which includes the download path
|
if err != nil {
|
||||||
_, err := downloader.Resolve(req, newTestDownloadOpt(t))
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
_, err = downloader.CreateWithOptions(rr.ID, newTestDownloadOpt(t))
|
||||||
if err == nil {
|
if err == nil {
|
||||||
t.Error("TestDownloader_CreateNotInWhite() expected error but got nil")
|
t.Fatal("Create should reject a directory outside the allowlist")
|
||||||
}
|
|
||||||
if !strings.Contains(err.Error(), "white") {
|
|
||||||
t.Errorf("TestDownloader_CreateNotInWhite() got = %v, want error containing 'white'", err.Error())
|
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestDownloader_CreateDirectBatch(t *testing.T) {
|
func TestDownloader_CreateDirectBatch(t *testing.T) {
|
||||||
@@ -1072,7 +1072,7 @@ func doTestDownloaderCreateWithProxy(t *testing.T, auth bool, buildReqProxy func
|
|||||||
Pwd: pwd,
|
Pwd: pwd,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
rr, err := downloader.Resolve(req, nil)
|
rr, err := downloader.Resolve(req)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if errHandler == nil {
|
if errHandler == nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
@@ -1158,7 +1158,7 @@ func TestDownloader_StoreAndRestore(t *testing.T) {
|
|||||||
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
|
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
|
||||||
}
|
}
|
||||||
opts := newTestDownloadOpt(t)
|
opts := newTestDownloadOpt(t)
|
||||||
rr, err := downloader.Resolve(req, opts)
|
rr, err := downloader.Resolve(req)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -1172,7 +1172,7 @@ func TestDownloader_StoreAndRestore(t *testing.T) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
id, err := downloader.Create(rr.ID)
|
id, err := downloader.CreateWithOptions(rr.ID, opts)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -1434,7 +1434,7 @@ func TestDownloader_Stats(t *testing.T) {
|
|||||||
req := &base.Request{
|
req := &base.Request{
|
||||||
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
|
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
|
||||||
}
|
}
|
||||||
rr, err := downloader.Resolve(req, newTestDownloadOpt(t))
|
rr, err := downloader.Resolve(req)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -1447,7 +1447,7 @@ func TestDownloader_Stats(t *testing.T) {
|
|||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
taskId, err := downloader.Create(rr.ID)
|
taskId, err := downloader.CreateWithOptions(rr.ID, newTestDownloadOpt(t))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -1677,7 +1677,7 @@ func TestDownloader_CompletedHTTPStatsSurviveRestore(t *testing.T) {
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
req := &base.Request{URL: "http://" + listener.Addr().String() + "/" + test.BuildName}
|
req := &base.Request{URL: "http://" + listener.Addr().String() + "/" + test.BuildName}
|
||||||
rr, err := downloader.Resolve(req, newTestDownloadOpt(t))
|
rr, err := downloader.Resolve(req)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -1688,7 +1688,7 @@ func TestDownloader_CompletedHTTPStatsSurviveRestore(t *testing.T) {
|
|||||||
done <- struct{}{}
|
done <- struct{}{}
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
taskID, err := downloader.Create(rr.ID)
|
taskID, err := downloader.CreateWithOptions(rr.ID, newTestDownloadOpt(t))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -1861,11 +1861,11 @@ func TestDownloader_PauseAndContinue(t *testing.T) {
|
|||||||
req := &base.Request{
|
req := &base.Request{
|
||||||
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
|
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
|
||||||
}
|
}
|
||||||
rr, err := downloader.Resolve(req, newTestDownloadOpt(t))
|
rr, err := downloader.Resolve(req)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
taskId, err := downloader.Create(rr.ID)
|
taskId, err := downloader.CreateWithOptions(rr.ID, newTestDownloadOpt(t))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -2045,11 +2045,11 @@ func TestDownloader_PauseAllAndContinueAll(t *testing.T) {
|
|||||||
req := &base.Request{
|
req := &base.Request{
|
||||||
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
|
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
|
||||||
}
|
}
|
||||||
rr, err := downloader.Resolve(req, newTestDownloadOpt(t))
|
rr, err := downloader.Resolve(req)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
taskId, err := downloader.Create(rr.ID)
|
taskId, err := downloader.CreateWithOptions(rr.ID, newTestDownloadOpt(t))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -753,6 +753,10 @@ type OnErrorExtensionTask struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func cloneExtensionTask(task *Task) *Task {
|
func cloneExtensionTask(task *Task) *Task {
|
||||||
|
if task.statusLock != nil {
|
||||||
|
task.statusLock.Lock()
|
||||||
|
defer task.statusLock.Unlock()
|
||||||
|
}
|
||||||
newTask := task.clone()
|
newTask := task.clone()
|
||||||
newTask.Meta.Req = task.Meta.Req
|
newTask.Meta.Req = task.Meta.Req
|
||||||
return newTask
|
return newTask
|
||||||
|
|||||||
@@ -65,11 +65,11 @@ func TestExtensionExtraMutationHTTP(t *testing.T) {
|
|||||||
var id string
|
var id string
|
||||||
var err error
|
var err error
|
||||||
if resolved {
|
if resolved {
|
||||||
rr, resolveErr := d.Resolve(req, opts)
|
rr, resolveErr := d.Resolve(req)
|
||||||
if resolveErr != nil {
|
if resolveErr != nil {
|
||||||
t.Fatal(resolveErr)
|
t.Fatal(resolveErr)
|
||||||
}
|
}
|
||||||
id, err = d.Create(rr.ID)
|
id, err = d.CreateWithOptions(rr.ID, opts)
|
||||||
} else {
|
} else {
|
||||||
id, err = d.CreateDirect(req, opts)
|
id, err = d.CreateDirect(req, opts)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -32,7 +32,7 @@ func TestDownloader_InstallExtensionByFolder(t *testing.T) {
|
|||||||
URL: "https://github.com/test",
|
URL: "https://github.com/test",
|
||||||
Labels: map[string]string{"original": "true"},
|
Labels: map[string]string{"original": "true"},
|
||||||
}
|
}
|
||||||
rr, err := downloader.Resolve(req, nil)
|
rr, err := downloader.Resolve(req)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -58,7 +58,7 @@ func TestDownloader_InstallExtensionByFolderDevMode(t *testing.T) {
|
|||||||
}
|
}
|
||||||
rr, err := downloader.Resolve(&base.Request{
|
rr, err := downloader.Resolve(&base.Request{
|
||||||
URL: "https://github.com/test",
|
URL: "https://github.com/test",
|
||||||
}, nil)
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -76,7 +76,7 @@ func TestDownloader_Extension_BlobBlob(t *testing.T) {
|
|||||||
|
|
||||||
rr, err := downloader.Resolve(&base.Request{
|
rr, err := downloader.Resolve(&base.Request{
|
||||||
URL: "https://example.com/blob",
|
URL: "https://example.com/blob",
|
||||||
}, nil)
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -130,7 +130,7 @@ func TestDownloader_Extension_BlobFunctionOpener(t *testing.T) {
|
|||||||
|
|
||||||
rr, err := downloader.Resolve(&base.Request{
|
rr, err := downloader.Resolve(&base.Request{
|
||||||
URL: "https://example.com/opener",
|
URL: "https://example.com/opener",
|
||||||
}, nil)
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -166,7 +166,7 @@ func TestDownloader_Extension_BlobFunctionOpenerUnknownSize(t *testing.T) {
|
|||||||
|
|
||||||
rr, err := downloader.Resolve(&base.Request{
|
rr, err := downloader.Resolve(&base.Request{
|
||||||
URL: "https://example.com/opener-unknown",
|
URL: "https://example.com/opener-unknown",
|
||||||
}, nil)
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -222,7 +222,7 @@ func TestDownloader_Extension_BlobSourceSizePropagatesToCreatedTask(t *testing.T
|
|||||||
|
|
||||||
rr, err := downloader.Resolve(&base.Request{
|
rr, err := downloader.Resolve(&base.Request{
|
||||||
URL: "https://example.com/opener",
|
URL: "https://example.com/opener",
|
||||||
}, nil)
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -303,7 +303,7 @@ func TestDownloader_Extension_BlobFetchDrainedStreamReportsEarlyProgress(t *test
|
|||||||
|
|
||||||
rr, err := downloader.Resolve(&base.Request{
|
rr, err := downloader.Resolve(&base.Request{
|
||||||
URL: fmt.Sprintf("%s/single-fetch-stream?size=%d", server.URL, len(payload)),
|
URL: fmt.Sprintf("%s/single-fetch-stream?size=%d", server.URL, len(payload)),
|
||||||
}, nil)
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -419,7 +419,7 @@ func TestDownloader_Extension_BlobRangeFetchDrainedStreamReportsEarlyProgress(t
|
|||||||
|
|
||||||
rr, err := downloader.Resolve(&base.Request{
|
rr, err := downloader.Resolve(&base.Request{
|
||||||
URL: fmt.Sprintf("%s/range-fetch-stream?size=%d", server.URL, len(payload)),
|
URL: fmt.Sprintf("%s/range-fetch-stream?size=%d", server.URL, len(payload)),
|
||||||
}, nil)
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -494,7 +494,7 @@ func TestDownloader_Extension_BlobMultiplexFetchStreamsReportEarlyProgress(t *te
|
|||||||
|
|
||||||
rr, err := downloader.Resolve(&base.Request{
|
rr, err := downloader.Resolve(&base.Request{
|
||||||
URL: server.URL + "/multiplex-fetch-stream",
|
URL: server.URL + "/multiplex-fetch-stream",
|
||||||
}, nil)
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -550,7 +550,7 @@ func TestDownloader_Extension_BlobFunctionOpenerRangeResume(t *testing.T) {
|
|||||||
|
|
||||||
rr, err := downloader.Resolve(&base.Request{
|
rr, err := downloader.Resolve(&base.Request{
|
||||||
URL: "https://example.com/opener-range",
|
URL: "https://example.com/opener-range",
|
||||||
}, nil)
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -683,7 +683,7 @@ func TestDownloader_Extension_BlobResumeAfterRestartViaOnError(t *testing.T) {
|
|||||||
|
|
||||||
rr, err := downloader.Resolve(&base.Request{
|
rr, err := downloader.Resolve(&base.Request{
|
||||||
URL: server.URL + "/restart-range",
|
URL: server.URL + "/restart-range",
|
||||||
}, nil)
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -855,7 +855,7 @@ func TestDownloader_Extension_BlobPauseContinueKeepsSource(t *testing.T) {
|
|||||||
|
|
||||||
rr, err := downloader.Resolve(&base.Request{
|
rr, err := downloader.Resolve(&base.Request{
|
||||||
URL: tc.url,
|
URL: tc.url,
|
||||||
}, nil)
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -932,7 +932,7 @@ func TestDownloader_Extension_BlobUnknownSizePauseContinueKeepsSource(t *testing
|
|||||||
|
|
||||||
rr, err := downloader.Resolve(&base.Request{
|
rr, err := downloader.Resolve(&base.Request{
|
||||||
URL: "https://example.com/opener-unknown",
|
URL: "https://example.com/opener-unknown",
|
||||||
}, nil)
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -1009,7 +1009,7 @@ func TestDownloader_Extension_BlobRecoverOnError(t *testing.T) {
|
|||||||
path := path
|
path := path
|
||||||
t.Run(path, func(t *testing.T) {
|
t.Run(path, func(t *testing.T) {
|
||||||
rawURL := "https://example.com/" + path
|
rawURL := "https://example.com/" + path
|
||||||
rr, err := downloader.Resolve(&base.Request{URL: rawURL}, nil)
|
rr, err := downloader.Resolve(&base.Request{URL: rawURL})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -1076,7 +1076,7 @@ func TestDownloader_InstallExtensionByGit(t *testing.T) {
|
|||||||
}
|
}
|
||||||
rr, err := downloader.Resolve(&base.Request{
|
rr, err := downloader.Resolve(&base.Request{
|
||||||
URL: "https://github.com/GopeedLab/gopeed/releases",
|
URL: "https://github.com/GopeedLab/gopeed/releases",
|
||||||
}, nil)
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -1093,7 +1093,7 @@ func TestDownloader_InstallExtensionByGitSimple(t *testing.T) {
|
|||||||
}
|
}
|
||||||
rr, err := downloader.Resolve(&base.Request{
|
rr, err := downloader.Resolve(&base.Request{
|
||||||
URL: "https://github.com/GopeedLab/gopeed/releases",
|
URL: "https://github.com/GopeedLab/gopeed/releases",
|
||||||
}, nil)
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -1110,7 +1110,7 @@ func TestDownloader_InstallExtensionByGitFull(t *testing.T) {
|
|||||||
}
|
}
|
||||||
rr, err := downloader.Resolve(&base.Request{
|
rr, err := downloader.Resolve(&base.Request{
|
||||||
URL: "https://github.com/GopeedLab/gopeed/releases",
|
URL: "https://github.com/GopeedLab/gopeed/releases",
|
||||||
}, nil)
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -1180,7 +1180,7 @@ func TestDownloader_UpgradeExtension(t *testing.T) {
|
|||||||
|
|
||||||
rr, err := downloader.Resolve(&base.Request{
|
rr, err := downloader.Resolve(&base.Request{
|
||||||
URL: "https://test.com",
|
URL: "https://test.com",
|
||||||
}, nil)
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -1355,7 +1355,7 @@ func TestDownloader_Extension_Errors(t *testing.T) {
|
|||||||
}
|
}
|
||||||
rr, err := downloader.Resolve(&base.Request{
|
rr, err := downloader.Resolve(&base.Request{
|
||||||
URL: "https://github.com/test",
|
URL: "https://github.com/test",
|
||||||
}, nil)
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -1370,7 +1370,7 @@ func TestDownloader_Extension_Errors(t *testing.T) {
|
|||||||
}
|
}
|
||||||
rr, err := downloader.Resolve(&base.Request{
|
rr, err := downloader.Resolve(&base.Request{
|
||||||
URL: "https://github.com/test",
|
URL: "https://github.com/test",
|
||||||
}, nil)
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -1385,7 +1385,7 @@ func TestDownloader_Extension_Errors(t *testing.T) {
|
|||||||
}
|
}
|
||||||
_, err := downloader.Resolve(&base.Request{
|
_, err := downloader.Resolve(&base.Request{
|
||||||
URL: "https://github.com/test",
|
URL: "https://github.com/test",
|
||||||
}, nil)
|
})
|
||||||
if err == nil {
|
if err == nil {
|
||||||
t.Fatalf("except error, but got nil")
|
t.Fatalf("except error, but got nil")
|
||||||
}
|
}
|
||||||
@@ -1407,7 +1407,7 @@ func TestDownloader_Extension_Settings(t *testing.T) {
|
|||||||
}
|
}
|
||||||
rr, err := downloader.Resolve(&base.Request{
|
rr, err := downloader.Resolve(&base.Request{
|
||||||
URL: "https://github.com/test",
|
URL: "https://github.com/test",
|
||||||
}, nil)
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -1428,7 +1428,7 @@ func TestDownloader_Extension_Settings(t *testing.T) {
|
|||||||
})
|
})
|
||||||
rr, err := downloader.Resolve(&base.Request{
|
rr, err := downloader.Resolve(&base.Request{
|
||||||
URL: "https://github.com/test",
|
URL: "https://github.com/test",
|
||||||
}, nil)
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -1445,7 +1445,7 @@ func TestDownloader_ExtensionStorage(t *testing.T) {
|
|||||||
}
|
}
|
||||||
rr, err := downloader.Resolve(&base.Request{
|
rr, err := downloader.Resolve(&base.Request{
|
||||||
URL: "https://github.com/test",
|
URL: "https://github.com/test",
|
||||||
}, nil)
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -24,7 +24,7 @@ func TestExtensionUnnamedResource(t *testing.T) {
|
|||||||
if _, err = d.InstallExtensionByFolder(dir, true); err != nil {
|
if _, err = d.InstallExtensionByFolder(dir, true); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
res, err := d.Resolve(&base.Request{URL: "https://example.com/unnamed"}, nil)
|
res, err := d.Resolve(&base.Request{URL: "https://example.com/unnamed"})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -40,7 +40,7 @@ func TestDownloader_HlsLiveSite(t *testing.T) {
|
|||||||
if referer != "" {
|
if referer != "" {
|
||||||
req.Extra = map[string]any{"header": map[string]string{"Referer": referer}}
|
req.Extra = map[string]any{"header": map[string]string{"Referer": referer}}
|
||||||
}
|
}
|
||||||
rr, err := downloader.Resolve(req, nil)
|
rr, err := downloader.Resolve(req)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("resolve: %v", err)
|
t.Fatalf("resolve: %v", err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -89,7 +89,7 @@ func TestDownloader_HlsProtocolEndToEnd(t *testing.T) {
|
|||||||
|
|
||||||
req := hlsResolveReferer()
|
req := hlsResolveReferer()
|
||||||
req.URL = server.URL + "/master.m3u8"
|
req.URL = server.URL + "/master.m3u8"
|
||||||
rr, err := downloader.Resolve(req, nil)
|
rr, err := downloader.Resolve(req)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -18,7 +18,7 @@ func TestCreateResolvedFileSelection(t *testing.T) {
|
|||||||
}{
|
}{
|
||||||
{"partial", []int{2, 0}, true, []int{2, 0}, 40},
|
{"partial", []int{2, 0}, true, []int{2, 0}, 40},
|
||||||
{"default all", nil, true, []int{0, 1, 2}, 60},
|
{"default all", nil, true, []int{0, 1, 2}, 60},
|
||||||
{"legacy retains resolve options", nil, false, []int{1}, 20},
|
{"default creation options", nil, false, []int{0, 1, 2}, 60},
|
||||||
} {
|
} {
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
manager := &generationTestManager{}
|
manager := &generationTestManager{}
|
||||||
@@ -28,8 +28,8 @@ func TestCreateResolvedFileSelection(t *testing.T) {
|
|||||||
}
|
}
|
||||||
defer d.Clear()
|
defer d.Clear()
|
||||||
d.cfg.MaxRunning = 0
|
d.cfg.MaxRunning = 0
|
||||||
original := &base.Options{Path: t.TempDir(), SelectFiles: []int{1}}
|
original := &base.Options{Path: t.TempDir()}
|
||||||
rr, err := d.Resolve(&base.Request{URL: "generation://selection"}, original)
|
rr, err := d.Resolve(&base.Request{URL: "generation://selection"})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -40,7 +40,7 @@ func TestCreateResolvedFileSelection(t *testing.T) {
|
|||||||
if tc.submit {
|
if tc.submit {
|
||||||
id, err = d.CreateWithOptions(rr.ID, &base.Options{Path: original.Path, SelectFiles: tc.selection})
|
id, err = d.CreateWithOptions(rr.ID, &base.Options{Path: original.Path, SelectFiles: tc.selection})
|
||||||
} else {
|
} else {
|
||||||
id, err = d.Create(rr.ID)
|
id, err = d.CreateWithOptions(rr.ID, original)
|
||||||
}
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
|
|||||||
@@ -222,8 +222,7 @@ func (o *downloadOptions) baseOptions() *base.Options {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type resolveTaskInput struct {
|
type resolveTaskInput struct {
|
||||||
Req *downloadRequest `json:"req" jsonschema:"Download request to resolve"`
|
Req *downloadRequest `json:"req" jsonschema:"Download request to resolve"`
|
||||||
Opts *downloadOptions `json:"opts,omitempty" jsonschema:"Optional download settings used while resolving"`
|
|
||||||
}
|
}
|
||||||
|
|
||||||
type resolveTaskOutput struct {
|
type resolveTaskOutput struct {
|
||||||
@@ -328,7 +327,7 @@ func registerTools(server *mcp.Server, downloader *download.Downloader) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, nil, err
|
return nil, nil, err
|
||||||
}
|
}
|
||||||
result, err := downloader.Resolve(req, input.Opts.baseOptions())
|
result, err := downloader.Resolve(req)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, nil, err
|
return nil, nil, err
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -101,6 +101,13 @@ func TestStreamableHTTPTools(t *testing.T) {
|
|||||||
if reqExtra["type"] != "object" || !ok || len(branches) != 2 {
|
if reqExtra["type"] != "object" || !ok || len(branches) != 2 {
|
||||||
t.Fatalf("%s req.extra schema = %#v, want an object with two oneOf branches", tool.Name, reqExtra)
|
t.Fatalf("%s req.extra schema = %#v, want an object with two oneOf branches", tool.Name, reqExtra)
|
||||||
}
|
}
|
||||||
|
if tool.Name == "resolve_task" {
|
||||||
|
properties := schema["properties"].(map[string]any)
|
||||||
|
if _, exists := properties["opts"]; exists || len(properties) != 1 {
|
||||||
|
t.Fatalf("resolve_task must expose only req: %s", inputSchema)
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
optsExtra := nestedSchemaProperty(t, schema, "opts", "extra")
|
optsExtra := nestedSchemaProperty(t, schema, "opts", "extra")
|
||||||
if !schemaIncludesType(optsExtra, "object") || optsExtra["additionalProperties"] != false {
|
if !schemaIncludesType(optsExtra, "object") || optsExtra["additionalProperties"] != false {
|
||||||
t.Fatalf("%s opts.extra schema = %#v, want concrete HTTP task settings", tool.Name, optsExtra)
|
t.Fatalf("%s opts.extra schema = %#v, want concrete HTTP task settings", tool.Name, optsExtra)
|
||||||
|
|||||||
+2
-2
@@ -26,7 +26,7 @@ func Info(w http.ResponseWriter, r *http.Request) {
|
|||||||
func Resolve(w http.ResponseWriter, r *http.Request) {
|
func Resolve(w http.ResponseWriter, r *http.Request) {
|
||||||
var req model.ResolveTask
|
var req model.ResolveTask
|
||||||
if ReadJson(r, w, &req) {
|
if ReadJson(r, w, &req) {
|
||||||
rr, err := Downloader.Resolve(req.Req, req.Opts)
|
rr, err := Downloader.Resolve(req.Req)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
WriteJson(w, model.NewErrorResult(err.Error()))
|
WriteJson(w, model.NewErrorResult(err.Error()))
|
||||||
return
|
return
|
||||||
@@ -82,7 +82,7 @@ func PatchTask(w http.ResponseWriter, r *http.Request) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
var req model.ResolveTask
|
var req model.PatchTask
|
||||||
if ReadJson(r, w, &req) {
|
if ReadJson(r, w, &req) {
|
||||||
if err := Downloader.Patch(taskId, req.Req, req.Opts); err != nil {
|
if err := Downloader.Patch(taskId, req.Req, req.Opts); err != nil {
|
||||||
if err == download.ErrTaskNotFound {
|
if err == download.ErrTaskNotFound {
|
||||||
|
|||||||
@@ -3,6 +3,10 @@ package model
|
|||||||
import "github.com/GopeedLab/gopeed/pkg/base"
|
import "github.com/GopeedLab/gopeed/pkg/base"
|
||||||
|
|
||||||
type ResolveTask struct {
|
type ResolveTask struct {
|
||||||
|
Req *base.Request `json:"req"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type PatchTask struct {
|
||||||
Req *base.Request `json:"req"`
|
Req *base.Request `json:"req"`
|
||||||
Opts *base.Options `json:"opts"`
|
Opts *base.Options `json:"opts"`
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -55,8 +55,7 @@ var (
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
resolveReq = &model.ResolveTask{
|
resolveReq = &model.ResolveTask{
|
||||||
Req: taskReq,
|
Req: taskReq,
|
||||||
Opts: createOpts,
|
|
||||||
}
|
}
|
||||||
createReq = &model.CreateTask{
|
createReq = &model.CreateTask{
|
||||||
Req: taskReq,
|
Req: taskReq,
|
||||||
@@ -105,7 +104,8 @@ func TestCreateTask(t *testing.T) {
|
|||||||
})
|
})
|
||||||
|
|
||||||
taskId := httpRequestCheckOk[string](http.MethodPost, "/api/v1/tasks", &model.CreateTask{
|
taskId := httpRequestCheckOk[string](http.MethodPost, "/api/v1/tasks", &model.CreateTask{
|
||||||
Rid: resp.ID,
|
Rid: resp.ID,
|
||||||
|
Opts: createOpts,
|
||||||
})
|
})
|
||||||
if taskId == "" {
|
if taskId == "" {
|
||||||
t.Fatal("create task failed")
|
t.Fatal("create task failed")
|
||||||
@@ -295,7 +295,7 @@ func TestPatchTask(t *testing.T) {
|
|||||||
time.Sleep(time.Millisecond * 100)
|
time.Sleep(time.Millisecond * 100)
|
||||||
|
|
||||||
// Patch the task with new labels
|
// Patch the task with new labels
|
||||||
patchReq := &model.ResolveTask{
|
patchReq := &model.PatchTask{
|
||||||
Req: &base.Request{
|
Req: &base.Request{
|
||||||
Labels: map[string]string{
|
Labels: map[string]string{
|
||||||
"patched": "true",
|
"patched": "true",
|
||||||
@@ -322,7 +322,7 @@ func TestPatchTask(t *testing.T) {
|
|||||||
func TestPatchTaskNotFound(t *testing.T) {
|
func TestPatchTaskNotFound(t *testing.T) {
|
||||||
doTest(func() {
|
doTest(func() {
|
||||||
// Try to patch a non-existent task
|
// Try to patch a non-existent task
|
||||||
patchReq := &model.ResolveTask{
|
patchReq := &model.PatchTask{
|
||||||
Req: &base.Request{
|
Req: &base.Request{
|
||||||
Labels: map[string]string{
|
Labels: map[string]string{
|
||||||
"test": "value",
|
"test": "value",
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ import 'model/install_extension.dart';
|
|||||||
import 'model/login.dart';
|
import 'model/login.dart';
|
||||||
import 'model/resolve_result.dart';
|
import 'model/resolve_result.dart';
|
||||||
import 'model/resolve_task.dart';
|
import 'model/resolve_task.dart';
|
||||||
|
import 'model/patch_task.dart';
|
||||||
import 'model/result.dart';
|
import 'model/result.dart';
|
||||||
import 'model/switch_extension.dart';
|
import 'model/switch_extension.dart';
|
||||||
import 'model/task.dart';
|
import 'model/task.dart';
|
||||||
@@ -91,7 +92,7 @@ Future<List<String>> createTaskBatch(CreateTaskBatch createTaskBatch) {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
Future<void> patchTask(String id, ResolveTask patchTask) {
|
Future<void> patchTask(String id, PatchTask patchTask) {
|
||||||
return _parse<void>(() => _request('api/v1/tasks/$id', method: 'PATCH', data: patchTask), null);
|
return _parse<void>(() => _request('api/v1/tasks/$id', method: 'PATCH', data: patchTask), null);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,16 @@
|
|||||||
|
import 'options.dart';
|
||||||
|
import 'request.dart';
|
||||||
|
|
||||||
|
class PatchTask {
|
||||||
|
final Request? req;
|
||||||
|
final Options? opts;
|
||||||
|
|
||||||
|
PatchTask({this.req, this.opts});
|
||||||
|
|
||||||
|
factory PatchTask.fromJson(Map<String, dynamic> json) => PatchTask(
|
||||||
|
req: json['req'] == null ? null : Request.fromJson(json['req'] as Map<String, dynamic>),
|
||||||
|
opts: json['opts'] == null ? null : Options.fromJson(json['opts'] as Map<String, dynamic>),
|
||||||
|
);
|
||||||
|
|
||||||
|
Map<String, dynamic> toJson() => {'req': req?.toJson(), 'opts': opts?.toJson()};
|
||||||
|
}
|
||||||
@@ -1,6 +1,5 @@
|
|||||||
import 'package:json_annotation/json_annotation.dart';
|
import 'package:json_annotation/json_annotation.dart';
|
||||||
|
|
||||||
import 'options.dart';
|
|
||||||
import 'request.dart';
|
import 'request.dart';
|
||||||
|
|
||||||
part 'resolve_task.g.dart';
|
part 'resolve_task.g.dart';
|
||||||
@@ -8,9 +7,8 @@ part 'resolve_task.g.dart';
|
|||||||
@JsonSerializable()
|
@JsonSerializable()
|
||||||
class ResolveTask {
|
class ResolveTask {
|
||||||
Request? req;
|
Request? req;
|
||||||
Options? opts;
|
|
||||||
|
|
||||||
ResolveTask({this.req, this.opts});
|
ResolveTask({this.req});
|
||||||
|
|
||||||
factory ResolveTask.fromJson(Map<String, dynamic> json) => _$ResolveTaskFromJson(json);
|
factory ResolveTask.fromJson(Map<String, dynamic> json) => _$ResolveTaskFromJson(json);
|
||||||
Map<String, dynamic> toJson() => _$ResolveTaskToJson(this);
|
Map<String, dynamic> toJson() => _$ResolveTaskToJson(this);
|
||||||
|
|||||||
@@ -7,11 +7,10 @@ part of 'resolve_task.dart';
|
|||||||
// **************************************************************************
|
// **************************************************************************
|
||||||
|
|
||||||
ResolveTask _$ResolveTaskFromJson(Map<String, dynamic> json) => ResolveTask(
|
ResolveTask _$ResolveTaskFromJson(Map<String, dynamic> json) => ResolveTask(
|
||||||
req: json['req'] == null ? null : Request.fromJson(json['req'] as Map<String, dynamic>),
|
req: json['req'] == null
|
||||||
opts: json['opts'] == null ? null : Options.fromJson(json['opts'] as Map<String, dynamic>),
|
? null
|
||||||
|
: Request.fromJson(json['req'] as Map<String, dynamic>),
|
||||||
);
|
);
|
||||||
|
|
||||||
Map<String, dynamic> _$ResolveTaskToJson(ResolveTask instance) => <String, dynamic>{
|
Map<String, dynamic> _$ResolveTaskToJson(ResolveTask instance) =>
|
||||||
'req': ?instance.req,
|
<String, dynamic>{'req': ?instance.req};
|
||||||
'opts': ?instance.opts,
|
|
||||||
};
|
|
||||||
@@ -8,6 +8,7 @@ import '../../api/model/extension.dart';
|
|||||||
import '../../api/model/install_extension.dart';
|
import '../../api/model/install_extension.dart';
|
||||||
import '../../api/model/resolve_result.dart';
|
import '../../api/model/resolve_result.dart';
|
||||||
import '../../api/model/resolve_task.dart';
|
import '../../api/model/resolve_task.dart';
|
||||||
|
import '../../api/model/patch_task.dart';
|
||||||
import '../../api/model/switch_extension.dart';
|
import '../../api/model/switch_extension.dart';
|
||||||
import '../../api/model/task.dart';
|
import '../../api/model/task.dart';
|
||||||
import '../../api/model/update_check_extension_resp.dart';
|
import '../../api/model/update_check_extension_resp.dart';
|
||||||
@@ -85,7 +86,7 @@ void _bindGopeed(CapabilityRegistry registry) {
|
|||||||
..bind(GopeedMethods.createTask, api.createTask)
|
..bind(GopeedMethods.createTask, api.createTask)
|
||||||
..bind(GopeedMethods.createTaskBatch, api.createTaskBatch)
|
..bind(GopeedMethods.createTaskBatch, api.createTaskBatch)
|
||||||
..bind(GopeedMethods.patchTask, (params) async {
|
..bind(GopeedMethods.patchTask, (params) async {
|
||||||
await api.patchTask(params['id'] as String, ResolveTask.fromJson(params['request'] as Map<String, dynamic>));
|
await api.patchTask(params['id'] as String, PatchTask.fromJson(params['request'] as Map<String, dynamic>));
|
||||||
return const RpcUnit();
|
return const RpcUnit();
|
||||||
})
|
})
|
||||||
..bind(
|
..bind(
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ import '../../api/model/extension.dart';
|
|||||||
import '../../api/model/install_extension.dart';
|
import '../../api/model/install_extension.dart';
|
||||||
import '../../api/model/resolve_result.dart';
|
import '../../api/model/resolve_result.dart';
|
||||||
import '../../api/model/resolve_task.dart';
|
import '../../api/model/resolve_task.dart';
|
||||||
|
import '../../api/model/patch_task.dart';
|
||||||
import '../../api/model/switch_extension.dart';
|
import '../../api/model/switch_extension.dart';
|
||||||
import '../../api/model/task.dart';
|
import '../../api/model/task.dart';
|
||||||
import '../../api/model/update_check_extension_resp.dart';
|
import '../../api/model/update_check_extension_resp.dart';
|
||||||
@@ -49,7 +50,7 @@ class GopeedService {
|
|||||||
Future<List<String>> createTaskBatch(CreateTaskBatch request) =>
|
Future<List<String>> createTaskBatch(CreateTaskBatch request) =>
|
||||||
_invoker.invoke(GopeedMethods.createTaskBatch, request);
|
_invoker.invoke(GopeedMethods.createTaskBatch, request);
|
||||||
|
|
||||||
Future<void> patchTask(String id, ResolveTask request) async {
|
Future<void> patchTask(String id, PatchTask request) async {
|
||||||
await _invoker.invoke(GopeedMethods.patchTask, {'id': id, 'request': request.toJson()});
|
await _invoker.invoke(GopeedMethods.patchTask, {'id': id, 'request': request.toJson()});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -3,7 +3,7 @@ import 'dart:async';
|
|||||||
import 'package:flutter_riverpod/flutter_riverpod.dart';
|
import 'package:flutter_riverpod/flutter_riverpod.dart';
|
||||||
|
|
||||||
import '../../../api/model/request.dart';
|
import '../../../api/model/request.dart';
|
||||||
import '../../../api/model/resolve_task.dart';
|
import '../../../api/model/patch_task.dart';
|
||||||
import '../../../api/model/task.dart' as api_model;
|
import '../../../api/model/task.dart' as api_model;
|
||||||
import '../../../core/capabilities/app_capabilities.dart';
|
import '../../../core/capabilities/app_capabilities.dart';
|
||||||
import '../domain/task_record.dart';
|
import '../domain/task_record.dart';
|
||||||
@@ -99,7 +99,7 @@ class TasksController extends AsyncNotifier<TasksState> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
Future<void> updateRequest(String id, Request request) async {
|
Future<void> updateRequest(String id, Request request) async {
|
||||||
await ref.read(gopeedServiceProvider).patchTask(id, ResolveTask(req: request));
|
await ref.read(gopeedServiceProvider).patchTask(id, PatchTask(req: request));
|
||||||
await ref.read(gopeedServiceProvider).continueTask(id);
|
await ref.read(gopeedServiceProvider).continueTask(id);
|
||||||
await refresh(silent: true);
|
await refresh(silent: true);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -927,8 +927,7 @@ class _CreateTaskWindowPageState extends ConsumerState<CreateTaskWindowPage> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
final request = _buildRequest(urls.first, protocolUrl: _protocolUrlFor(urls.first));
|
final request = _buildRequest(urls.first, protocolUrl: _protocolUrlFor(urls.first));
|
||||||
final options = _buildOptions();
|
final result = await ref.read(gopeedServiceProvider).resolve(ResolveTask(req: request));
|
||||||
final result = await ref.read(gopeedServiceProvider).resolve(ResolveTask(req: request, opts: options));
|
|
||||||
if (!mounted) return;
|
if (!mounted) return;
|
||||||
final created = await _showResolveDialog(request, result);
|
final created = await _showResolveDialog(request, result);
|
||||||
if (!created) {
|
if (!created) {
|
||||||
|
|||||||
@@ -3,30 +3,49 @@ import 'dart:convert';
|
|||||||
import 'package:dio/dio.dart' as dio;
|
import 'package:dio/dio.dart' as dio;
|
||||||
import 'package:flutter_test/flutter_test.dart';
|
import 'package:flutter_test/flutter_test.dart';
|
||||||
import 'package:gopeed/api/api.dart' as api;
|
import 'package:gopeed/api/api.dart' as api;
|
||||||
|
import 'package:gopeed/api/model/create_task.dart';
|
||||||
import 'package:gopeed/api/model/options.dart';
|
import 'package:gopeed/api/model/options.dart';
|
||||||
|
import 'package:gopeed/api/model/patch_task.dart';
|
||||||
import 'package:gopeed/api/model/request.dart';
|
import 'package:gopeed/api/model/request.dart';
|
||||||
import 'package:gopeed/api/model/resolve_task.dart';
|
import 'package:gopeed/api/model/resolve_task.dart';
|
||||||
import 'package:gopeed/core/network/gopeed/gopeed_transport.dart';
|
import 'package:gopeed/core/network/gopeed/gopeed_transport.dart';
|
||||||
|
|
||||||
void main() {
|
void main() {
|
||||||
test('resolve sends the complete resolve task for magnet links', () async {
|
test('resolve omits download options and creation submits them separately', () async {
|
||||||
final transport = _RecordingTransport();
|
final transport = _RecordingTransport();
|
||||||
api.setTransportForTesting(transport);
|
api.setTransportForTesting(transport);
|
||||||
final result = await api.resolve(
|
final result = await api.resolve(
|
||||||
ResolveTask(
|
ResolveTask(req: Request(url: 'magnet:?xt=urn:btih:0123456789abcdef0123456789abcdef01234567')),
|
||||||
req: Request(url: 'magnet:?xt=urn:btih:0123456789abcdef0123456789abcdef01234567'),
|
|
||||||
opts: Options(path: '/downloads', selectFiles: const []),
|
|
||||||
),
|
|
||||||
);
|
);
|
||||||
|
|
||||||
expect(result.id, 'resolved-id');
|
expect(result.id, 'resolved-id');
|
||||||
final requestBody = transport.data as Map<String, dynamic>;
|
final requestBody = transport.data as Map<String, dynamic>;
|
||||||
expect(requestBody.keys, containsAll(<String>['req', 'opts']));
|
expect(requestBody.keys, equals(<String>['req']));
|
||||||
expect(
|
expect(
|
||||||
(requestBody['req'] as Map<String, dynamic>)['url'],
|
(requestBody['req'] as Map<String, dynamic>)['url'],
|
||||||
'magnet:?xt=urn:btih:0123456789abcdef0123456789abcdef01234567',
|
'magnet:?xt=urn:btih:0123456789abcdef0123456789abcdef01234567',
|
||||||
);
|
);
|
||||||
expect((requestBody['opts'] as Map<String, dynamic>)['path'], '/downloads');
|
expect(requestBody, isNot(contains('opts')));
|
||||||
|
|
||||||
|
final id = await api.createTask(
|
||||||
|
CreateTask(
|
||||||
|
rid: result.id,
|
||||||
|
opts: Options(path: '/chosen-after-resolve', selectFiles: [1]),
|
||||||
|
),
|
||||||
|
);
|
||||||
|
expect(id, 'created-id');
|
||||||
|
final createBody = transport.data as Map<String, dynamic>;
|
||||||
|
expect(createBody['rid'], 'resolved-id');
|
||||||
|
expect((createBody['opts'] as Map<String, dynamic>)['path'], '/chosen-after-resolve');
|
||||||
|
expect((createBody['opts'] as Map<String, dynamic>)['selectFiles'], [1]);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('patch preserves options in its independent request model', () async {
|
||||||
|
final transport = _RecordingTransport();
|
||||||
|
api.setTransportForTesting(transport);
|
||||||
|
await api.patchTask('task-id', PatchTask(opts: Options(selectFiles: [1])));
|
||||||
|
final requestBody = transport.data as Map<String, dynamic>;
|
||||||
|
expect((requestBody['opts'] as Map<String, dynamic>)['selectFiles'], [1]);
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -41,6 +60,12 @@ class _RecordingTransport implements GopeedTransport {
|
|||||||
Map<String, dynamic>? queryParameters,
|
Map<String, dynamic>? queryParameters,
|
||||||
}) async {
|
}) async {
|
||||||
this.data = jsonDecode(jsonEncode(data));
|
this.data = jsonDecode(jsonEncode(data));
|
||||||
|
if (path == 'api/v1/tasks') {
|
||||||
|
return {'code': 0, 'data': 'created-id'};
|
||||||
|
}
|
||||||
|
if (method == 'PATCH') {
|
||||||
|
return {'code': 0, 'data': null};
|
||||||
|
}
|
||||||
return {
|
return {
|
||||||
'code': 0,
|
'code': 0,
|
||||||
'data': {
|
'data': {
|
||||||
|
|||||||
@@ -1,5 +1,13 @@
|
|||||||
import 'package:flutter_test/flutter_test.dart';
|
import 'package:flutter_test/flutter_test.dart';
|
||||||
|
import 'package:gopeed/api/model/options.dart';
|
||||||
|
import 'package:gopeed/api/model/patch_task.dart';
|
||||||
|
import 'package:gopeed/api/model/request.dart';
|
||||||
|
import 'package:gopeed/api/model/resolve_result.dart';
|
||||||
|
import 'package:gopeed/api/model/resolve_task.dart';
|
||||||
|
import 'package:gopeed/api/model/resource.dart';
|
||||||
|
import 'package:gopeed/core/capabilities/app_capabilities.dart';
|
||||||
import 'package:gopeed/core/capabilities/capability_rpc.dart';
|
import 'package:gopeed/core/capabilities/capability_rpc.dart';
|
||||||
|
import 'package:gopeed/core/capabilities/gopeed_capability.dart';
|
||||||
import 'package:gopeed/core/window/app_window_appearance.dart';
|
import 'package:gopeed/core/window/app_window_appearance.dart';
|
||||||
|
|
||||||
class _Payload {
|
class _Payload {
|
||||||
@@ -13,6 +21,33 @@ class _Payload {
|
|||||||
}
|
}
|
||||||
|
|
||||||
void main() {
|
void main() {
|
||||||
|
test('serialized resolve and patch capabilities keep separate request contracts', () async {
|
||||||
|
final codecs = createAppCapabilityCodecs();
|
||||||
|
String? resolvedUrl;
|
||||||
|
Map<String, dynamic>? patch;
|
||||||
|
final registry = CapabilityRegistry(codecs)
|
||||||
|
..bind(GopeedMethods.resolve, (request) {
|
||||||
|
resolvedUrl = request.req?.url;
|
||||||
|
return ResolveResult(id: 'resolved-id', res: Resource(files: []));
|
||||||
|
})
|
||||||
|
..bind(GopeedMethods.patchTask, (request) {
|
||||||
|
patch = request;
|
||||||
|
return const RpcUnit();
|
||||||
|
});
|
||||||
|
final request = ResolveTask(req: Request(url: 'magnet:?xt=urn:btih:hash'));
|
||||||
|
final wireRequest = codecs.encode(request) as Map<String, dynamic>;
|
||||||
|
expect(wireRequest.keys, ['req']);
|
||||||
|
final result = codecs.decode<ResolveResult>(await registry.invoke(GopeedMethods.resolve.name, wireRequest));
|
||||||
|
expect(result.id, 'resolved-id');
|
||||||
|
expect(resolvedUrl, request.req?.url);
|
||||||
|
|
||||||
|
final service = GopeedService(LocalCapabilityInvoker(registry));
|
||||||
|
await service.patchTask('task-id', PatchTask(opts: Options(selectFiles: [1])));
|
||||||
|
expect(patch?['id'], 'task-id');
|
||||||
|
final patchBody = patch?['request'] as Map<String, dynamic>;
|
||||||
|
expect((patchBody['opts'] as Map<String, dynamic>)['selectFiles'], [1]);
|
||||||
|
});
|
||||||
|
|
||||||
test('one method descriptor supports direct and serialized dispatch', () async {
|
test('one method descriptor supports direct and serialized dispatch', () async {
|
||||||
const method = RpcMethod<_Payload, _Payload>('test.increment');
|
const method = RpcMethod<_Payload, _Payload>('test.increment');
|
||||||
final codecs = RpcCodecRegistry()..register<_Payload>((json) => _Payload.fromJson(json! as Map<String, dynamic>));
|
final codecs = RpcCodecRegistry()..register<_Payload>((json) => _Payload.fromJson(json! as Map<String, dynamic>));
|
||||||
|
|||||||
Reference in new issue
Block a user