nprasad2077/gopeed
1
0 Fork 0
mirror of https://github.com/GopeedLab/gopeed.git synced 2026-11-06 08:30:29 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor: fethcer interface & download flow (#1229)

Browse Source
This commit is contained in:
Levi authored and GitHub committed 2026-01-08 21:50:47 +08:00
1 parent abd16d5326
commit a9eee26be7
30 files changed
+2961 -1631
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

cmd/gopeed/main.go
+3 -2
View File
@@ -2,13 +2,14 @@ package main
import (
"fmt"
"strings"
"sync"
"github.com/GopeedLab/gopeed/internal/fetcher"
"github.com/GopeedLab/gopeed/pkg/base"
"github.com/GopeedLab/gopeed/pkg/download"
"github.com/GopeedLab/gopeed/pkg/protocol/http"
"github.com/GopeedLab/gopeed/pkg/util"
"strings"
"sync"
)
const progressWidth = 20
internal/fetcher/fetcher.go
+4 -6
View File
@@ -1,20 +1,18 @@
package fetcher
import (
"github.com/GopeedLab/gopeed/internal/controller"
"github.com/GopeedLab/gopeed/pkg/base"
"path"
"strings"
"github.com/GopeedLab/gopeed/internal/controller"
"github.com/GopeedLab/gopeed/pkg/base"
)
// Fetcher defines the interface for a download protocol.
// Each download task will have a corresponding Fetcher instance for the management of the download task
type Fetcher interface {
Setup(ctl *controller.Controller)
// Resolve resource info from request
Resolve(req *base.Request) error
// Create ready to download, but not started
Create(opts *base.Options) error
Resolve(req *base.Request, opts *base.Options) error
Start() error
Pause() error
Close() error
internal/protocol/bt/fetcher.go
+28 -30
View File
@@ -20,15 +20,15 @@ import (
"github.com/GopeedLab/gopeed/pkg/util"
"github.com/anacrolix/torrent"
"github.com/anacrolix/torrent/metainfo"
"github.com/anacrolix/torrent/storage"
)
var (
client *torrent.Client
lock sync.Mutex
torrentDirMap = make(map[string]string)
ftMap = make(map[string]*fileTorrentImpl)
closeCtx context.Context
closeFunc func()
cfg *torrent.ClientConfig
client *torrent.Client
lock sync.Mutex
closeCtx context.Context
closeFunc func()
)
type Fetcher struct {
@@ -71,21 +71,12 @@ func (f *Fetcher) initClient() (err error) {
closeCtx, closeFunc = context.WithCancel(context.Background())
}
cfg := torrent.NewDefaultClientConfig()
cfg = torrent.NewDefaultClientConfig()
cfg.Seed = true
cfg.Bep20 = fmt.Sprintf("-GP%s-", parseBep20())
cfg.ExtendedHandshakeClientVersion = fmt.Sprintf("Gopeed %s", base.Version)
cfg.ListenPort = f.config.ListenPort
cfg.HTTPProxy = f.ctl.GetProxy(f.meta.Req.Proxy)
cfg.DefaultStorage = newFileOpts(newFileClientOpts{
ClientBaseDir: cfg.DataDir,
HandleFileTorrent: func(infoHash metainfo.Hash, ft *fileTorrentImpl) {
if dir, ok := torrentDirMap[infoHash.String()]; ok {
ft.setTorrentDir(dir)
}
ftMap[infoHash.String()] = ft
},
})
dnsResolver := &DnsCacheResolver{RefreshTimeout: 5 * time.Minute}
cfg.TrackerDialContext = dnsResolver.DialContext
client, err = torrent.NewClient(cfg)
@@ -100,8 +91,12 @@ func (f *Fetcher) initClient() (err error) {
return
}
func (f *Fetcher) Resolve(req *base.Request) error {
func (f *Fetcher) Resolve(req *base.Request, opts *base.Options) error {
f.meta.Req = req
f.meta.Opts = opts
if f.meta.Opts == nil {
f.meta.Opts = &base.Options{}
}
if err := f.addTorrent(req, false); err != nil {
return err
}
@@ -109,23 +104,12 @@ func (f *Fetcher) Resolve(req *base.Request) error {
return nil
}
func (f *Fetcher) Create(opts *base.Options) (err error) {
f.meta.Opts = opts
if f.meta.Res != nil {
torrentDirMap[f.meta.Res.Hash] = opts.Path
}
return nil
}
func (f *Fetcher) Start() (err error) {
if !f.torrentReady.Load() {
if err = f.addTorrent(f.meta.Req, false); err != nil {
return
}
}
if ft, ok := ftMap[f.meta.Res.Hash]; ok {
ft.setTorrentDir(f.meta.Opts.Path)
}
files := f.torrent.Files()
// If the user does not specify the file to download, all files will be downloaded by default
if f.data.Progress == nil {
@@ -361,8 +345,12 @@ func (f *Fetcher) addTorrent(req *base.Request, fromUpload bool) (err error) {
}
schema := util.ParseSchema(req.URL)
privateTorrent := false
var spec *torrent.TorrentSpec
if schema == "MAGNET" {
f.torrent, err = client.AddMagnet(req.URL)
spec, err = torrent.TorrentSpecFromMagnetUri(req.URL)
if err != nil {
return
}
} else {
var reader io.Reader
if schema == "FILE" {
@@ -399,8 +387,18 @@ func (f *Fetcher) addTorrent(req *base.Request, fromUpload bool) (err error) {
if info.Private != nil && *info.Private {
privateTorrent = true
}
f.torrent, err = client.AddTorrent(metaInfo)
spec, err = torrent.TorrentSpecFromMetaInfoErr(metaInfo)
if err != nil {
return
}
}
spec.Storage = storage.NewFileOpts(storage.NewFileClientOpts{
ClientBaseDir: cfg.DataDir,
TorrentDirMaker: func(baseDir string, info *metainfo.Info, infoHash metainfo.Hash) string {
return f.meta.Opts.Path
},
})
f.torrent, _, err = client.AddTorrentSpec(spec)
if err != nil {
return
}
internal/protocol/bt/fetcher_test.go
+5 -5
View File
@@ -31,7 +31,7 @@ func TestFetcher_Resolve_DataUri_Torrent(t *testing.T) {
dataUri := "data:application/x-bittorrent;base64," + base64.StdEncoding.EncodeToString(buf)
err = fetcher.Resolve(&base.Request{
URL: dataUri,
})
}, nil)
if err != nil {
panic(err)
}
@@ -81,7 +81,7 @@ func doResolve(t *testing.T, fetcher fetcher.Fetcher) {
"udp://tracker.bitsearch.to:1337/announce",
},
},
})
}, nil)
if err != nil {
panic(err)
}
@@ -111,7 +111,7 @@ func doResolve(t *testing.T, fetcher fetcher.Fetcher) {
"udp://tracker.bitsearch.to:1337/announce",
},
},
})
}, nil)
if err != nil {
panic(err)
}
@@ -145,7 +145,7 @@ func doResolve(t *testing.T, fetcher fetcher.Fetcher) {
t.Run("Resolve Unclean Torrent", func(t *testing.T) {
err := fetcher.Resolve(&base.Request{
URL: "./testdata/test.unclean.torrent",
})
}, nil)
if err != nil {
t.Errorf("Resolve Unclean Torrent Resolve() got = %v, want nil", err)
}
@@ -156,7 +156,7 @@ func doResolve(t *testing.T, fetcher fetcher.Fetcher) {
uri := "file:///" + file
err := fetcher.Resolve(&base.Request{
URL: uri,
})
}, nil)
if err != nil {
t.Errorf("Resolve file scheme Torrent Resolve() got = %v, want nil", err)
}
internal/protocol/bt/file.go
-216
View File
@@ -1,216 +0,0 @@
// modify from github.com/anacrolix/torrent/storage/file.go
package bt
import (
"context"
"fmt"
"io"
"os"
"path/filepath"
"github.com/anacrolix/torrent/storage"
"github.com/anacrolix/missinggo/v2"
"github.com/anacrolix/torrent/common"
"github.com/anacrolix/torrent/metainfo"
"github.com/anacrolix/torrent/segments"
)
// File-based storage for torrents, that isn't yet bound to a particular torrent.
type fileClientImpl struct {
opts newFileClientOpts
}
type newFileClientOpts struct {
// The base directory for all downloads.
ClientBaseDir string
HandleFileTorrent func(infoHash metainfo.Hash, ft *fileTorrentImpl)
FilePathMaker storage.FilePathMaker
PieceCompletion storage.PieceCompletion
}
func newFileOpts(opts newFileClientOpts) storage.ClientImplCloser {
if opts.FilePathMaker == nil {
opts.FilePathMaker = func(opts storage.FilePathMakerOpts) string {
var parts []string
if opts.Info.Length == 0 || opts.File.Path == nil {
parts = append(parts, opts.Info.Name)
}
return filepath.Join(append(parts, opts.File.Path...)...)
}
}
if opts.PieceCompletion == nil {
ret, err := storage.NewDefaultPieceCompletionForDir(opts.ClientBaseDir)
if err != nil {
ret = storage.NewMapPieceCompletion()
}
opts.PieceCompletion = ret
}
return fileClientImpl{opts}
}
func (fs fileClientImpl) Close() error {
return fs.opts.PieceCompletion.Close()
}
func (fs fileClientImpl) OpenTorrent(ctx context.Context, info *metainfo.Info, infoHash metainfo.Hash) (_ storage.TorrentImpl, err error) {
upvertedFiles := info.UpvertedFiles()
files := make([]file, 0, len(upvertedFiles))
for _, fileInfo := range upvertedFiles {
filePath := filepath.Join("", fs.opts.FilePathMaker(storage.FilePathMakerOpts{
Info: info,
File: &fileInfo,
}))
f := file{
rawPath: filePath,
path: filePath,
length: fileInfo.Length,
}
if f.length == 0 {
err = CreateNativeZeroLengthFile(f.path)
if err != nil {
err = fmt.Errorf("creating zero length file: %w", err)
return
}
}
files = append(files, f)
}
t := &fileTorrentImpl{
files,
segments.NewIndex(common.LengthIterFromUpvertedFiles(upvertedFiles)),
infoHash,
fs.opts.PieceCompletion,
}
fs.opts.HandleFileTorrent(infoHash, t)
return storage.TorrentImpl{
Piece: t.Piece,
Close: t.Close,
}, nil
}
type file struct {
// The safe, OS-local file path.
rawPath string
path string
length int64
}
type fileTorrentImpl struct {
files []file
segmentLocater segments.Index
infoHash metainfo.Hash
completion storage.PieceCompletion
}
func (fts *fileTorrentImpl) Piece(p metainfo.Piece) storage.PieceImpl {
// Create a view onto the file-based torrent storage.
_io := fileTorrentImplIO{fts}
// Return the appropriate segments of this.
return &filePieceImpl{
fts,
p,
missinggo.NewSectionWriter(_io, p.Offset(), p.Length()),
io.NewSectionReader(_io, p.Offset(), p.Length()),
}
}
func (fts *fileTorrentImpl) Close() error {
return nil
}
func (fts *fileTorrentImpl) setTorrentDir(dir string) {
for i, f := range fts.files {
fts.files[i].path = filepath.Join(dir, f.rawPath)
}
}
// A helper to create zero-length files which won't appear for file-orientated storage since no
// writes will ever occur to them (no torrent data is associated with a zero-length file). The
// caller should make sure the file name provided is safe/sanitized.
func CreateNativeZeroLengthFile(name string) error {
os.MkdirAll(filepath.Dir(name), 0o777)
var f io.Closer
f, err := os.Create(name)
if err != nil {
return err
}
return f.Close()
}
// Exposes file-based storage of a torrent, as one big ReadWriterAt.
type fileTorrentImplIO struct {
fts *fileTorrentImpl
}
// Returns EOF on short or missing file.
func (fst *fileTorrentImplIO) readFileAt(file file, b []byte, off int64) (n int, err error) {
f, err := os.Open(file.path)
if os.IsNotExist(err) {
// File missing is treated the same as a short file.
err = io.EOF
return
}
if err != nil {
return
}
defer f.Close()
// Limit the read to within the expected bounds of this file.
if int64(len(b)) > file.length-off {
b = b[:file.length-off]
}
for off < file.length && len(b) != 0 {
n1, err1 := f.ReadAt(b, off)
b = b[n1:]
n += n1
off += int64(n1)
if n1 == 0 {
err = err1
break
}
}
return
}
// Only returns EOF at the end of the torrent. Premature EOF is ErrUnexpectedEOF.
func (fst fileTorrentImplIO) ReadAt(b []byte, off int64) (n int, err error) {
fst.fts.segmentLocater.Locate(segments.Extent{off, int64(len(b))}, func(i int, e segments.Extent) bool {
n1, err1 := fst.readFileAt(fst.fts.files[i], b[:e.Length], e.Start)
n += n1
b = b[n1:]
err = err1
return err == nil // && int64(n1) == e.Length
})
if len(b) != 0 && err == nil {
err = io.EOF
}
return
}
func (fst fileTorrentImplIO) WriteAt(p []byte, off int64) (n int, err error) {
// log.Printf("write at %v: %v bytes", off, len(p))
fst.fts.segmentLocater.Locate(segments.Extent{off, int64(len(p))}, func(i int, e segments.Extent) bool {
name := fst.fts.files[i].path
os.MkdirAll(filepath.Dir(name), 0o777)
var f *os.File
f, err = os.OpenFile(name, os.O_WRONLY|os.O_CREATE, 0o666)
if err != nil {
return false
}
var n1 int
n1, err = f.WriteAt(p[:e.Length], e.Start)
// log.Printf("%v %v wrote %v: %v", i, e, n1, err)
closeErr := f.Close()
n += n1
p = p[n1:]
if err == nil {
err = closeErr
}
if err == nil && int64(n1) != e.Length {
err = io.ErrShortWrite
}
return err == nil
})
return
}
internal/protocol/bt/file_misc.go
-36
View File
@@ -1,36 +0,0 @@
// modify from github.com/anacrolix/torrent/storage/file_misc.go
package bt
import "github.com/anacrolix/torrent/metainfo"
type requiredLength struct {
fileIndex int
length int64
}
func extentCompleteRequiredLengths(info *metainfo.Info, off, n int64) (ret []requiredLength) {
if n == 0 {
return
}
for i, fi := range info.UpvertedFiles() {
if off >= fi.Length {
off -= fi.Length
continue
}
n1 := n
if off+n1 > fi.Length {
n1 = fi.Length - off
}
ret = append(ret, requiredLength{
fileIndex: i,
length: off + n1,
})
n -= n1
if n == 0 {
return
}
off = 0
}
return
}
internal/protocol/bt/file_piece.go
-61
View File
@@ -1,61 +0,0 @@
// modify from github.com/anacrolix/torrent/storage/file_piece.go
package bt
import (
"github.com/anacrolix/torrent/metainfo"
"github.com/anacrolix/torrent/storage"
"io"
"log"
"os"
)
type filePieceImpl struct {
*fileTorrentImpl
p metainfo.Piece
io.WriterAt
io.ReaderAt
}
var _ storage.PieceImpl = (*filePieceImpl)(nil)
func (fs *filePieceImpl) pieceKey() metainfo.PieceKey {
return metainfo.PieceKey{InfoHash: fs.infoHash, Index: fs.p.Index()}
}
func (fs *filePieceImpl) Completion() storage.Completion {
c, err := fs.completion.Get(fs.pieceKey())
if err != nil {
log.Printf("error getting piece completion: %s", err)
c.Ok = false
return c
}
verified := true
if c.Complete {
// If it's allegedly complete, check that its constituent files have the necessary length.
for _, fi := range extentCompleteRequiredLengths(fs.p.Info, fs.p.Offset(), fs.p.Length()) {
s, err := os.Stat(fs.files[fi.fileIndex].path)
if err != nil || s.Size() < fi.length {
verified = false
break
}
}
}
if !verified {
// The completion was wrong, fix it.
c.Complete = false
fs.completion.Set(fs.pieceKey(), false)
}
return c
}
func (fs *filePieceImpl) MarkComplete() error {
return fs.completion.Set(fs.pieceKey(), true)
}
func (fs *filePieceImpl) MarkNotComplete() error {
return fs.completion.Set(fs.pieceKey(), false)
}
internal/protocol/http/fetcher.go
+1389 -807
View File
File diff suppressed because it is too large. Load diff
internal/protocol/http/fetcher_manager.go
+96
View File
@@ -0,0 +1,96 @@
package http
import (
"net/url"
"path"
"github.com/GopeedLab/gopeed/internal/fetcher"
"github.com/GopeedLab/gopeed/pkg/base"
fhttp "github.com/GopeedLab/gopeed/pkg/protocol/http"
)
// ============================================================================
// Fetcher Data (for Store/Restore)
// ============================================================================
type fetcherData struct {
Connections []*connection
}
// ============================================================================
// Fetcher Manager
// ============================================================================
type FetcherManager struct {
}
func (fm *FetcherManager) Name() string {
return "http"
}
func (fm *FetcherManager) Filters() []*fetcher.SchemeFilter {
return []*fetcher.SchemeFilter{
{
Type: fetcher.FilterTypeUrl,
Pattern: "HTTP",
},
{
Type: fetcher.FilterTypeUrl,
Pattern: "HTTPS",
},
}
}
func (fm *FetcherManager) Build() fetcher.Fetcher {
return &Fetcher{}
}
func (fm *FetcherManager) ParseName(u string) string {
var name string
url, err := url.Parse(u)
if err != nil {
return ""
}
name = path.Base(url.Path)
if name == "" || name == "/" || name == "." {
name = url.Hostname()
}
return name
}
func (fm *FetcherManager) AutoRename() bool {
return true
}
func (fm *FetcherManager) DefaultConfig() any {
return &config{
UserAgent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36",
Connections: 16,
}
}
func (fm *FetcherManager) Store(f fetcher.Fetcher) (data any, err error) {
_f := f.(*Fetcher)
return &fetcherData{
Connections: _f.connections,
}, nil
}
func (fm *FetcherManager) Restore() (v any, f func(meta *fetcher.FetcherMeta, v any) fetcher.Fetcher) {
return &fetcherData{}, func(meta *fetcher.FetcherMeta, v any) fetcher.Fetcher {
fd := v.(*fetcherData)
fb := &FetcherManager{}
fetcher := fb.Build().(*Fetcher)
fetcher.meta = meta
base.ParseReqExtra[fhttp.ReqExtra](fetcher.meta.Req)
base.ParseOptExtra[fhttp.OptsExtra](fetcher.meta.Opts)
if len(fd.Connections) > 0 {
fetcher.connections = fd.Connections
}
return fetcher
}
}
func (fm *FetcherManager) Close() error {
return nil
}
internal/protocol/http/fetcher_test.go
+583 -170
View File
@@ -3,188 +3,221 @@ package http
import (
"encoding/json"
"fmt"
"net"
gohttp "net/http"
"net/url"
"os"
"strings"
"testing"
"time"
"github.com/GopeedLab/gopeed/internal/controller"
"github.com/GopeedLab/gopeed/internal/fetcher"
"github.com/GopeedLab/gopeed/internal/test"
"github.com/GopeedLab/gopeed/pkg/base"
"github.com/GopeedLab/gopeed/pkg/protocol/http"
"net"
gohttp "net/http"
"net/url"
"strings"
"testing"
"time"
)
func TestFetcher_Resolve(t *testing.T) {
testResolve(test.StartTestFileServer, test.BuildName, &base.Resource{
Size: test.BuildSize,
Range: true,
Files: []*base.FileInfo{
{
Name: test.BuildName,
Size: test.BuildSize,
testResolve(test.StartTestFileServer, test.BuildName, t, func(err error) (*base.Resource, error) {
return &base.Resource{
Size: test.BuildSize,
Range: true,
Files: []*base.FileInfo{
{
Name: test.BuildName,
Size: test.BuildSize,
},
},
},
}, t)
testResolve(test.StartTestCustomServer, "disposition", &base.Resource{
Size: test.BuildSize,
Range: false,
Files: []*base.FileInfo{
{
Name: test.BuildName,
Size: test.BuildSize,
}, nil
})
testResolve(test.StartTestCustomServer, "disposition", t, func(err error) (*base.Resource, error) {
return &base.Resource{
Size: test.BuildSize,
Range: false,
Files: []*base.FileInfo{
{
Name: test.BuildName,
Size: test.BuildSize,
},
},
},
}, t)
testResolve(test.StartTestCustomServer, "encoded-word", &base.Resource{
Size: test.BuildSize,
Range: false,
Files: []*base.FileInfo{
{
Name: test.TestChineseFileName,
Size: test.BuildSize,
}, nil
})
testResolve(test.StartTestCustomServer, "encoded-word", t, func(err error) (*base.Resource, error) {
return &base.Resource{
Size: test.BuildSize,
Range: false,
Files: []*base.FileInfo{
{
Name: test.TestChineseFileName,
Size: test.BuildSize,
},
},
},
}, t)
testResolve(test.StartTestCustomServer, "no-encode", &base.Resource{
Size: test.BuildSize,
Range: false,
Files: []*base.FileInfo{
{
Name: test.TestChineseFileName,
Size: test.BuildSize,
}, nil
})
testResolve(test.StartTestCustomServer, "no-encode", t, func(err error) (*base.Resource, error) {
return &base.Resource{
Size: test.BuildSize,
Range: false,
Files: []*base.FileInfo{
{
Name: test.TestChineseFileName,
Size: test.BuildSize,
},
},
},
}, t)
testResolve(test.StartTestCustomServer, "%E6%B5%8B%E8%AF%95.zip", &base.Resource{
Size: 0,
Range: false,
Files: []*base.FileInfo{
{
Name: test.TestChineseFileName,
Size: 0,
}, nil
})
testResolve(test.StartTestCustomServer, "%E6%B5%8B%E8%AF%95.zip", t, func(err error) (*base.Resource, error) {
return &base.Resource{
Size: 0,
Range: false,
Files: []*base.FileInfo{
{
Name: test.TestChineseFileName,
Size: 0,
},
},
},
}, t)
testResolve(test.StartTestCustomServer, test.BuildName, &base.Resource{
Size: 0,
Range: false,
Files: []*base.FileInfo{
{
Name: test.BuildName,
Size: 0,
}, nil
})
testResolve(test.StartTestCustomServer, test.BuildName, t, func(err error) (*base.Resource, error) {
return &base.Resource{
Size: 0,
Range: false,
Files: []*base.FileInfo{
{
Name: test.BuildName,
Size: 0,
},
},
},
}, t)
}, nil
})
// Test mixed encoding Content-Disposition where mime.ParseMediaType fails
// due to invalid characters, but filename*= contains the correct UTF-8 encoded name
testResolve(test.StartTestCustomServer, "mixed-encoding", &base.Resource{
Size: test.BuildSize,
Range: false,
Files: []*base.FileInfo{
{
Name: test.TestChineseFileName,
Size: test.BuildSize,
testResolve(test.StartTestCustomServer, "mixed-encoding", t, func(err error) (*base.Resource, error) {
return &base.Resource{
Size: test.BuildSize,
Range: false,
Files: []*base.FileInfo{
{
Name: test.TestChineseFileName,
Size: test.BuildSize,
},
},
},
}, t)
}, nil
})
// Test filename*= only (RFC 5987 format)
testResolve(test.StartTestCustomServer, "filename-star", &base.Resource{
Size: test.BuildSize,
Range: false,
Files: []*base.FileInfo{
{
Name: test.TestChineseFileName,
Size: test.BuildSize,
testResolve(test.StartTestCustomServer, "filename-star", t, func(err error) (*base.Resource, error) {
return &base.Resource{
Size: test.BuildSize,
Range: false,
Files: []*base.FileInfo{
{
Name: test.TestChineseFileName,
Size: test.BuildSize,
},
},
},
}, t)
}, nil
})
// Test GBK-encoded filename (common on Chinese Windows servers)
// Before fix: "测试.zip" sent as GBK bytes -> parsed as "²âÊÔ.zip" (garbled)
// After fix: correctly decoded back to "测试.zip"
testResolve(test.StartTestCustomServer, "gbk-encoded", &base.Resource{
Size: test.BuildSize,
Range: false,
Files: []*base.FileInfo{
{
Name: test.TestChineseFileName,
Size: test.BuildSize,
testResolve(test.StartTestCustomServer, "gbk-encoded", t, func(err error) (*base.Resource, error) {
return &base.Resource{
Size: test.BuildSize,
Range: false,
Files: []*base.FileInfo{
{
Name: test.TestChineseFileName,
Size: test.BuildSize,
},
},
},
}, t)
}, nil
})
// Test filename with plus signs (e.g., C++ Primer)
// Before fix: %2B decoded to space -> "C++ Primer" became "C Primer"
// After fix: %2B correctly decoded to + -> "C++ Primer Plus.mobi"
testResolve(test.StartTestCustomServer, "plus-sign-encoded", &base.Resource{
Size: test.BuildSize,
Range: false,
Files: []*base.FileInfo{
{
Name: "C++ Primer Plus.mobi",
Size: test.BuildSize,
testResolve(test.StartTestCustomServer, "plus-sign-encoded", t, func(err error) (*base.Resource, error) {
return &base.Resource{
Size: test.BuildSize,
Range: false,
Files: []*base.FileInfo{
{
Name: "C++ Primer Plus.mobi",
Size: test.BuildSize,
},
},
},
}, t)
}, nil
})
// Test filename with plus sign in URL path
// Before fix: %2B decoded to space
// After fix: %2B correctly decoded to +
testResolve(test.StartTestCustomServer, "C%2B%2B%20Primer.txt", &base.Resource{
Size: 0,
Range: false,
Files: []*base.FileInfo{
{
Name: "C++ Primer.txt",
Size: 0,
testResolve(test.StartTestCustomServer, "C%2B%2B%20Primer.txt", t, func(err error) (*base.Resource, error) {
return &base.Resource{
Size: 0,
Range: false,
Files: []*base.FileInfo{
{
Name: "C++ Primer.txt",
Size: 0,
},
},
},
}, t)
}, nil
})
// Test filename with HTML-encoded ampersand (fixes issue with & being truncated)
// Before fix: "查询处理&amp;优化.pptx" -> "查询处理&amp" (truncated at semicolon)
// After fix: correctly decoded to "查询处理&优化.pptx"
testResolve(test.StartTestCustomServer, "ampersand-encoded", &base.Resource{
Size: test.BuildSize,
Range: false,
Files: []*base.FileInfo{
{
Name: "查询处理&优化.pptx",
Size: test.BuildSize,
testResolve(test.StartTestCustomServer, "ampersand-encoded", t, func(err error) (*base.Resource, error) {
return &base.Resource{
Size: test.BuildSize,
Range: false,
Files: []*base.FileInfo{
{
Name: "查询处理&优化.pptx",
Size: test.BuildSize,
},
},
},
}, t)
}, nil
})
// Test unquoted filename with HTML-encoded ampersand
testResolve(test.StartTestCustomServer, "ampersand-unquoted", &base.Resource{
Size: test.BuildSize,
Range: false,
Files: []*base.FileInfo{
{
Name: "test&file.txt",
Size: test.BuildSize,
testResolve(test.StartTestCustomServer, "ampersand-unquoted", t, func(err error) (*base.Resource, error) {
return &base.Resource{
Size: test.BuildSize,
Range: false,
Files: []*base.FileInfo{
{
Name: "test&file.txt",
Size: test.BuildSize,
},
},
},
}, t)
}, nil
})
// Test URL without file path - should use domain/host as filename
listener := test.StartTestRootServer()
defer listener.Close()
fetcher := buildFetcher()
err := fetcher.Resolve(&base.Request{
URL: "http://" + listener.Addr().String() + "/",
testResolve(test.StartTestCustomServer, "", t, func(err error) (*base.Resource, error) {
return &base.Resource{
Size: 0,
Range: false,
Files: []*base.FileInfo{
{
Name: "127.0.0.1",
Size: 0,
},
},
}, nil
})
// Test 403 Forbidden response handling
testResolve(test.StartTestCustomServer, "forbidden", t, func(err error) (*base.Resource, error) {
requestError := extractRequestError(err)
if requestError != nil && requestError.Code == 403 {
return nil, nil
}
return nil, err
})
if err != nil {
t.Fatal(err)
}
// When no filename is provided, it should use the hostname (without port) as the name
expectedName := "127.0.0.1"
if fetcher.Meta().Res.Files[0].Name != expectedName {
t.Errorf("Resolve() got name = %v, want %v", fetcher.Meta().Res.Files[0].Name, expectedName)
}
}
func TestFetcher_ResolveWithHostHeader(t *testing.T) {
listener := test.StartTestHostHeaderServer()
defer listener.Close()
fetcher := buildFetcher()
err := fetcher.Resolve(&base.Request{
URL: "http://" + listener.Addr().String() + "/",
@@ -193,6 +226,9 @@ func TestFetcher_ResolveWithHostHeader(t *testing.T) {
"Host": "test",
},
},
}, &base.Options{
Name: test.DownloadName,
Path: test.Dir,
})
// The server should return 400 for invalid Host header
if err == nil || !strings.Contains(err.Error(), "400") {
@@ -201,10 +237,11 @@ func TestFetcher_ResolveWithHostHeader(t *testing.T) {
}
func TestFetcher_ResolveWithInvalidHeader(t *testing.T) {
listener := test.StartTestRootServer()
listener := test.StartTestCustomServer()
defer listener.Close()
fetcher := buildFetcher()
defer fetcher.Pause() // Close the resolve response to allow server shutdown
err := fetcher.Resolve(&base.Request{
URL: "http://" + listener.Addr().String() + "/",
Extra: &http.ReqExtra{
@@ -212,6 +249,9 @@ func TestFetcher_ResolveWithInvalidHeader(t *testing.T) {
"Referer": "\rtest",
},
},
}, &base.Options{
Name: test.DownloadName,
Path: test.Dir,
})
// Invalid header with \r should be sanitized by Go's http client, allowing the request to succeed
if err != nil {
@@ -219,18 +259,23 @@ func TestFetcher_ResolveWithInvalidHeader(t *testing.T) {
}
}
func testResolve(startTestServer func() net.Listener, path string, want *base.Resource, t *testing.T) {
func testResolve(startTestServer func() net.Listener, path string, t *testing.T, wantFn func(error) (*base.Resource, error)) {
listener := startTestServer()
defer listener.Close()
fetcher := buildFetcher()
defer fetcher.Pause() // Close the resolve response to allow server shutdown
err := fetcher.Resolve(&base.Request{
URL: "http://" + listener.Addr().String() + "/" + path,
}, &base.Options{
Name: test.DownloadName,
Path: test.Dir,
})
want, err := wantFn(err)
if err != nil {
t.Fatal(err)
}
if !test.AssertResourceEqual(want, fetcher.meta.Res) {
t.Errorf("Resolve() got = %v, want %v", fetcher.meta.Res, want)
if want != nil && !test.AssertResourceEqual(want, fetcher.meta.Res) {
t.Errorf("Resolve() got = %+v, want %+v", fetcher.meta.Res, want)
}
}
@@ -293,11 +338,74 @@ func TestFetcher_DownloadLimit(t *testing.T) {
}
func TestFetcher_DownloadResponseBodyReadTimeout(t *testing.T) {
listener := test.StartTestLimitServer(16, readTimeout.Milliseconds()+5000)
// Server will timeout once (first request delays longer than readTimeout),
// then subsequent requests work normally
listener := test.StartTestTimeoutOnceServer(readTimeout.Milliseconds() + 5000)
defer listener.Close()
downloadError(listener, 1, t)
downloadError(listener, 4, t)
for _, connections := range []int{1, 4} {
os.Remove(test.DownloadFile)
fetcher := downloadReady(listener, connections, t)
if err := fetcher.Start(); err != nil {
t.Fatal(err)
}
if err := fetcher.Wait(); err != nil {
t.Fatal(err)
}
stats := fetcher.Stats().(*http.Stats)
if len(stats.Connections) == 0 {
t.Fatalf("expected connections stats for timeout test")
}
// Verify successful download after timeout recovery
want := test.FileMd5(test.BuildFile)
got := test.FileMd5(test.DownloadFile)
if want != got {
t.Errorf("Download() got = %v, want %v", got, want)
}
// Verify timeouts don't count as failures (retryTimes should be 0)
for _, conn := range stats.Connections {
if conn.Failed {
t.Fatalf("expected no counted failures after timeout recovery, got retries=%d", conn.RetryTimes)
}
if conn.RetryTimes != 0 {
t.Fatalf("expected retryTimes to stay zero for non-counted timeouts, got %d", conn.RetryTimes)
}
}
}
}
func TestFetcher_Download500Recovery(t *testing.T) {
// Server returns 500 for 15 seconds, then recovers
listener := test.StartTestTemporary500Server(15 * time.Second)
defer listener.Close()
os.Remove(test.DownloadFile)
fetcher := downloadReady(listener, 4, t)
if err := fetcher.Start(); err != nil {
t.Fatal(err)
}
if err := fetcher.Wait(); err != nil {
t.Fatal(err)
}
// Verify successful download after 500 errors
want := test.FileMd5(test.BuildFile)
got := test.FileMd5(test.DownloadFile)
if want != got {
t.Errorf("Download() got = %v, want %v", got, want)
}
// Verify 500 errors don't count as failures (retryTimes should be 0)
stats := fetcher.Stats().(*http.Stats)
for _, conn := range stats.Connections {
if conn.RetryTimes != 0 {
t.Errorf("Expected retryTimes to be 0 for 500 errors (exempt), got %d", conn.RetryTimes)
}
}
}
func TestFetcher_DownloadOnBugFileServer(t *testing.T) {
@@ -385,11 +493,14 @@ func TestFetcher_Stats(t *testing.T) {
t.Fatal(err)
}
stats := fetcher.Stats().(*http.Stats)
if len(stats.Connections) != 16 {
t.Errorf("Stats() got = %v, want %v", len(stats.Connections), 16)
// With slow-start strategy, connection count may be less than max if download is fast
// Just verify we have at least 1 connection and no more than max
if len(stats.Connections) < 1 || len(stats.Connections) > 16 {
t.Errorf("Stats() connection count got = %v, want between 1 and 16", len(stats.Connections))
}
totalDownloaded := int64(0)
for _, conn := range stats.Connections {
for i, conn := range stats.Connections {
t.Logf("Connection %d: Downloaded=%d, Completed=%v", i, conn.Downloaded, conn.Completed)
totalDownloaded += conn.Downloaded
}
if totalDownloaded != test.BuildSize {
@@ -397,6 +508,276 @@ func TestFetcher_Stats(t *testing.T) {
}
}
// TestFetcher_DownloadOneTimeURL tests downloading from a URL that can only be accessed once
// This simulates signed URLs or one-time download links that expire after first use
func TestFetcher_DownloadOneTimeURL(t *testing.T) {
listener := test.StartTestOneTimeServer()
defer listener.Close()
fetcher := buildFetcher()
err := fetcher.Resolve(&base.Request{
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
}, &base.Options{
Name: test.DownloadName,
Path: test.Dir,
Extra: &http.OptsExtra{
Connections: 4, // Try to use multiple connections, but only first should work
},
})
if err != nil {
t.Fatal(err)
}
err = fetcher.Start()
if err != nil {
t.Fatal(err)
}
err = fetcher.Wait()
if err != nil {
t.Fatal(err)
}
// Verify file content
want := test.FileMd5(test.BuildFile)
got := test.FileMd5(test.DownloadFile)
if want != got {
t.Errorf("Download() got = %v, want %v", got, want)
}
}
// TestFetcher_SlowStartExpansion tests slow-start connection expansion edge cases
// Tests that slow-start expansion reaches exactly maxConns
// Expansion pattern: 1 -> 2 -> 4 -> 8 -> 16...
// For max=5: 1 -> 2 -> 4 -> 5 (capped)
// For max=9: 1 -> 2 -> 4 -> 8 -> 9 (capped)
func TestFetcher_SlowStartExpansion(t *testing.T) {
testCases := []struct {
name string
maxConns int
}{
{"MaxConns5", 5}, // 1->2->4->5
{"MaxConns9", 9}, // 1->2->4->8->9
{"MaxConns8", 8}, // 1->2->4->8
}
for _, tc := range testCases {
tc := tc // capture range variable
t.Run(tc.name, func(t *testing.T) {
// Clean up any leftover files from previous tests
os.Remove(test.DownloadFile)
// Use 100ns delay per byte for faster test (~10MB/s theoretical)
listener := test.StartTestSlowStartServer(100 * time.Nanosecond)
// Ensure cleanup happens before next subtest
cleanup := func() {
listener.Close()
os.Remove(test.DownloadFile)
// Wait for server to fully stop
time.Sleep(50 * time.Millisecond)
}
fetcher := buildConfigFetcher(config{
Connections: tc.maxConns,
})
err := fetcher.Resolve(&base.Request{
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
}, &base.Options{
Name: test.DownloadName,
Path: test.Dir,
Extra: &http.OptsExtra{
Connections: tc.maxConns,
},
})
if err != nil {
cleanup()
t.Fatal(err)
}
err = fetcher.Start()
if err != nil {
cleanup()
t.Fatal(err)
}
err = fetcher.Wait()
if err != nil {
t.Logf("Wait() returned error: %v", err)
cleanup()
t.Fatal(err)
}
// Check final connection count equals maxConns exactly
stats := fetcher.Stats().(*http.Stats)
finalConns := len(stats.Connections)
// Debug: show connection details and metadata
httpFetcher := fetcher.(*Fetcher)
t.Logf("Resource: Size=%d, Range=%v", httpFetcher.Meta().Res.Size, httpFetcher.Meta().Res.Range)
for i, conn := range stats.Connections {
t.Logf("Connection %d: Downloaded=%d, Completed=%v", i, conn.Downloaded, conn.Completed)
}
if finalConns != tc.maxConns {
t.Errorf("Expected exactly %d connections, got %d", tc.maxConns, finalConns)
}
// Verify file content before cleanup
want := test.FileMd5(test.BuildFile)
got := test.FileMd5(test.DownloadFile)
if want != got {
t.Errorf("Download() got = %v, want %v", got, want)
}
cleanup()
})
}
}
// TestFetcher_AsyncPrefetch tests the async prefetch functionality
// where data is downloaded in background during resolve phase and reused in start
func TestFetcher_AsyncPrefetch(t *testing.T) {
// Test 1: Prefetch completes entire file before Start is called
t.Run("PrefetchComplete", func(t *testing.T) {
listener := test.StartTestFileServer()
defer listener.Close()
fetcher := buildFetcher()
err := fetcher.Resolve(&base.Request{
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
}, &base.Options{
Name: test.DownloadName,
Path: test.Dir,
Extra: &http.OptsExtra{
Connections: 4,
},
})
if err != nil {
t.Fatal(err)
}
// Poll until prefetch completes the entire file (with timeout)
timeout := time.After(30 * time.Second)
ticker := time.NewTicker(100 * time.Millisecond)
defer ticker.Stop()
pollLoop:
for {
select {
case <-timeout:
t.Fatal("Timeout waiting for prefetch to complete")
case <-ticker.C:
if fetcher.prefetchSize.Load() >= test.BuildSize {
break pollLoop
}
}
}
prefetchedBefore := fetcher.prefetchSize.Load()
t.Logf("Prefetched bytes before Start: %d (%.2f MB)", prefetchedBefore, float64(prefetchedBefore)/(1024*1024))
// Should have prefetched the entire file
if prefetchedBefore != test.BuildSize {
t.Errorf("Prefetch should complete entire file, got %d, want %d", prefetchedBefore, test.BuildSize)
}
// Now start the download
err = fetcher.Start()
if err != nil {
t.Fatal(err)
}
// Wait for download to complete
err = fetcher.Wait()
if err != nil {
t.Fatal(err)
}
// Check how much was utilized from prefetch
prefetchedUsed := fetcher.resolveDataPos.Load()
t.Logf("Prefetched bytes used: %d (%.2f MB)", prefetchedUsed, float64(prefetchedUsed)/(1024*1024))
// Verify file is correct
want := test.FileMd5(test.BuildFile)
got := test.FileMd5(test.DownloadFile)
if want != got {
t.Errorf("Download() got = %v, want %v", got, want)
}
os.Remove(test.DownloadFile)
})
// Test 2: Prefetch only downloads partial data before Start is called
t.Run("PrefetchPartial", func(t *testing.T) {
// Use slow server with 100 nanosecond delay per byte
// This means ~10MB/s speed, so 100ms should download ~1MB
listener := test.StartTestSlowStartServer(100 * time.Nanosecond)
defer listener.Close()
fetcher := buildFetcher()
err := fetcher.Resolve(&base.Request{
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
}, &base.Options{
Name: test.DownloadName,
Path: test.Dir,
Extra: &http.OptsExtra{
Connections: 4,
},
})
if err != nil {
t.Fatal(err)
}
// Wait only 100ms - should only prefetch a small portion
time.Sleep(100 * time.Millisecond)
prefetchedBefore := fetcher.prefetchSize.Load()
t.Logf("Prefetched bytes before Start: %d (%.2f KB)", prefetchedBefore, float64(prefetchedBefore)/1024)
// Verify we have partial data (not zero, but not complete)
if prefetchedBefore == 0 {
t.Log("Warning: No data prefetched, may be too slow")
}
if prefetchedBefore >= test.BuildSize {
t.Log("Warning: Prefetch completed entire file, test may not be valid")
}
// Now start the download
err = fetcher.Start()
if err != nil {
t.Fatal(err)
}
// Wait for download to complete
err = fetcher.Wait()
if err != nil {
t.Fatal(err)
}
// Check stats - should have connections that downloaded remaining data
stats := fetcher.Stats().(*http.Stats)
t.Logf("Final connections: %d", len(stats.Connections))
prefetchedUsed := fetcher.resolveDataPos.Load()
t.Logf("Prefetched bytes used: %d (%.2f KB)", prefetchedUsed, float64(prefetchedUsed)/1024)
// Verify connections picked up where prefetch left off
if len(stats.Connections) > 0 {
firstConn := stats.Connections[0]
t.Logf("First connection downloaded: %d bytes", firstConn.Downloaded)
}
// Verify file is correct
want := test.FileMd5(test.BuildFile)
got := test.FileMd5(test.DownloadFile)
if want != got {
t.Errorf("Download() got = %v, want %v", got, want)
}
os.Remove(test.DownloadFile)
})
}
func TestFetcherManager_ParseName(t *testing.T) {
type args struct {
u string
@@ -450,15 +831,9 @@ func downloadReady(listener net.Listener, connections int, t *testing.T) fetcher
}
func doDownloadReady(f fetcher.Fetcher, listener net.Listener, connections int, t *testing.T) fetcher.Fetcher {
err := f.Resolve(&base.Request{
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
})
if err != nil {
t.Fatal(err)
}
var extra any = nil
if connections > 0 {
extra = http.OptsExtra{
extra = &http.OptsExtra{
Connections: connections,
}
}
@@ -467,7 +842,9 @@ func doDownloadReady(f fetcher.Fetcher, listener net.Listener, connections int,
Path: test.Dir,
Extra: extra,
}
err = f.Create(opts)
err := f.Resolve(&base.Request{
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
}, opts)
if err != nil {
t.Fatal(err)
}
@@ -492,19 +869,39 @@ func downloadNormal(listener net.Listener, connections int, t *testing.T) {
}
func downloadPost(listener net.Listener, connections int, t *testing.T) {
fetcher := downloadReady(listener, connections, t)
fetcher.Meta().Req.Extra = &http.ReqExtra{
Method: "POST",
Header: map[string]string{
"Authorization": "Bearer 123456",
},
Body: fmt.Sprintf(`{"name":"%s"}`, test.BuildName),
// POST parameters must be set before Resolve since the new design
// starts downloading during Resolve phase
f := buildFetcher()
var extra any = nil
if connections > 0 {
extra = &http.OptsExtra{
Connections: connections,
}
}
err := fetcher.Start()
opts := &base.Options{
Name: test.DownloadName,
Path: test.Dir,
Extra: extra,
}
req := &base.Request{
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
Extra: &http.ReqExtra{
Method: "POST",
Header: map[string]string{
"Authorization": "Bearer 123456",
},
Body: fmt.Sprintf(`{"name":"%s"}`, test.BuildName),
},
}
err := f.Resolve(req, opts)
if err != nil {
t.Fatal(err)
}
err = fetcher.Wait()
err = f.Start()
if err != nil {
t.Fatal(err)
}
err = f.Wait()
if err != nil {
t.Fatal(err)
}
@@ -541,14 +938,30 @@ func downloadContinue(listener net.Listener, connections int, t *testing.T) {
}
func downloadError(listener net.Listener, connections int, t *testing.T) {
fetcher := downloadReady(listener, connections, t)
err := fetcher.Start()
fetcher := buildFetcher()
err := fetcher.Resolve(&base.Request{
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
}, &base.Options{
Name: test.DownloadName,
Path: test.Dir,
})
// With the new async design, Resolve may succeed (HTTP response received)
// but errors occur during async download or Start/Wait
if err != nil {
t.Fatal(err)
// Error detected in Resolve - this is fine
return
}
// Resolve succeeded, error should occur during Start/Wait
err = fetcher.Start()
if err != nil {
// Error detected in Start - this is fine
return
}
err = fetcher.Wait()
if err == nil {
t.Errorf("Download() got = %v, want %v", err, nil)
t.Errorf("Expected error during download, but got none")
}
}
internal/protocol/http/helper.go
+378
View File
@@ -0,0 +1,378 @@
package http
import (
"bytes"
"context"
"crypto/tls"
"errors"
"fmt"
"io"
"mime"
"net"
"net/http"
"net/http/cookiejar"
"net/url"
"strings"
"unicode/utf8"
"github.com/GopeedLab/gopeed/pkg/base"
fhttp "github.com/GopeedLab/gopeed/pkg/protocol/http"
"github.com/GopeedLab/gopeed/pkg/util"
"golang.org/x/text/encoding/simplifiedchinese"
)
type RequestError struct {
Code int
}
func NewRequestError(code int) *RequestError {
return &RequestError{Code: code}
}
func (re *RequestError) Error() string {
return fmt.Sprintf("http request fail, code:%d", re.Code)
}
func isFailureExemptHTTPCode(code int) bool {
if code >= 500 && code <= 599 {
return true
}
switch code {
case 429, 408, 440, 499:
return true
default:
return false
}
}
func shouldCountHTTPFailure(err error) bool {
var re *RequestError
if !errors.As(err, &re) {
return false
}
return !isFailureExemptHTTPCode(re.Code)
}
func extractRequestError(err error) *RequestError {
var re *RequestError
if errors.As(err, &re) {
return re
}
return nil
}
// ============================================================================
// HTTP Request/Client Building
// ============================================================================
func (f *Fetcher) buildRequest(ctx context.Context, req *base.Request) (httpReq *http.Request, err error) {
var reqUrl string
f.redirectLock.Lock()
if f.redirectURL != "" {
reqUrl = f.redirectURL
} else {
reqUrl = req.URL
}
f.redirectLock.Unlock()
var (
method string
body io.Reader
)
headers := http.Header{}
if req.Extra == nil {
method = http.MethodGet
} else {
extra := req.Extra.(*fhttp.ReqExtra)
if extra.Method != "" {
method = extra.Method
} else {
method = http.MethodGet
}
if len(extra.Header) > 0 {
for k, v := range extra.Header {
headers.Set(k, strings.TrimSpace(v))
}
}
if extra.Body != "" {
body = bytes.NewBufferString(extra.Body)
}
}
if _, ok := headers[base.HttpHeaderUserAgent]; !ok {
headers.Set(base.HttpHeaderUserAgent, strings.TrimSpace(f.config.UserAgent))
}
if ctx != nil {
httpReq, err = http.NewRequestWithContext(ctx, method, reqUrl, body)
} else {
httpReq, err = http.NewRequest(method, reqUrl, body)
}
if err != nil {
return
}
httpReq.Header = headers
if host := headers.Get(base.HttpHeaderHost); host != "" {
httpReq.Host = host
}
return httpReq, nil
}
func (f *Fetcher) buildClient() *http.Client {
transport := &http.Transport{
DialContext: (&net.Dialer{
Timeout: connectTimeout,
}).DialContext,
Proxy: f.ctl.GetProxy(f.meta.Req.Proxy),
TLSClientConfig: &tls.Config{
InsecureSkipVerify: f.meta.Req.SkipVerifyCert,
},
}
jar, _ := cookiejar.New(nil)
return &http.Client{
Transport: transport,
Jar: jar,
}
}
// ============================================================================
// Filename Parsing
// ============================================================================
// parseFilename extracts filename from Content-Disposition header
func parseFilename(contentDisposition string) string {
// Try RFC 5987 extended notation first (filename*=)
if filename := parseFilenameExtended(contentDisposition); filename != "" {
return filename
}
// Try standard MIME parsing
_, params, err := mime.ParseMediaType(contentDisposition)
if err == nil {
if filename := params["filename"]; filename != "" {
return decodeFilenameParam(filename)
}
}
// Fallback to manual parsing
return parseFilenameFallback(contentDisposition)
}
// parseFilenameExtended handles RFC 5987 extended notation (filename*=)
// Format: filename*=charset'language'value (e.g., UTF-8”%E6%B5%8B%E8%AF%95.zip)
func parseFilenameExtended(cd string) string {
lower := strings.ToLower(cd)
idx := strings.Index(lower, "filename*=")
if idx == -1 {
return ""
}
value := cd[idx+len("filename*="):]
// Find the end of the value using proper quote handling
endIdx := findParamValueEnd(value)
if endIdx != -1 {
value = value[:endIdx]
}
value = strings.TrimSpace(value)
// Try charset''encoded format (e.g., UTF-8''%E4%B8%AD%E6%96%87.txt)
parts := strings.SplitN(value, "''", 2)
if len(parts) == 2 {
// Use PathUnescape to handle %2B correctly (should decode to +, not space)
decoded, err := url.PathUnescape(parts[1])
if err == nil {
return decoded
}
}
// Try charset'language'encoded format
parts = strings.SplitN(value, "'", 3)
if len(parts) >= 3 {
// Use PathUnescape to handle %2B correctly (should decode to +, not space)
decoded, err := url.PathUnescape(parts[2])
if err == nil {
return decoded
}
}
return ""
}
// decodeFilenameParam decodes filename parameter value
// Handles HTML entities, MIME encoded-word, URL encoding, and GBK encoding fallback
func decodeFilenameParam(filename string) string {
// First, unescape HTML entities (e.g., &amp; -> &, &lt; -> <, &gt; -> >)
// This must be done before other decoding to handle cases where servers
// HTML-encode special characters in filenames
filename = unescapeHTMLEntities(filename)
// Handle RFC 2047 encoded word (=?charset?encoding?text?=)
if strings.HasPrefix(filename, "=?") {
decoder := new(mime.WordDecoder)
normalizedFilename := strings.Replace(filename, "UTF8", "UTF-8", 1)
if decoded, err := decoder.Decode(normalizedFilename); err == nil {
return decoded
}
}
// Try URL decoding - use PathUnescape for filenames to handle %2B correctly
decoded := util.TryUrlPathUnescape(filename)
// If not valid UTF-8, try GBK decoding (common for Chinese websites)
if !utf8.ValidString(decoded) {
if gbkDecoded := tryDecodeGBK(decoded); gbkDecoded != "" {
return gbkDecoded
}
}
return decoded
}
// unescapeHTMLEntities unescapes common HTML entities in filenames
// This handles cases where servers HTML-encode special characters like & to &amp;
func unescapeHTMLEntities(s string) string {
// Common HTML entities that might appear in filenames
replacements := map[string]string{
"&amp;": "&",
"&lt;": "<",
"&gt;": ">",
"&quot;": "\"",
"&#39;": "'",
"&apos;": "'",
}
result := s
for entity, char := range replacements {
result = strings.ReplaceAll(result, entity, char)
}
return result
}
// tryDecodeGBK attempts to decode string as GBK encoding
func tryDecodeGBK(s string) string {
if len(s) == 0 {
return ""
}
decoder := simplifiedchinese.GBK.NewDecoder()
decoded, err := decoder.Bytes([]byte(s))
if err != nil {
return ""
}
result := string(decoded)
if utf8.ValidString(result) {
return result
}
return ""
}
// parseFilenameFallback is a fallback parser for non-standard Content-Disposition
func parseFilenameFallback(cd string) string {
lower := strings.ToLower(cd)
idx := strings.Index(lower, "filename=")
if idx == -1 {
return ""
}
value := cd[idx+len("filename="):]
// Find the end of the value using proper quote handling
endIdx := findParamValueEnd(value)
if endIdx != -1 {
value = value[:endIdx]
}
value = strings.TrimSpace(value)
// Remove surrounding quotes
if len(value) >= 2 {
if (value[0] == '"' && value[len(value)-1] == '"') ||
(value[0] == '\'' && value[len(value)-1] == '\'') {
value = value[1 : len(value)-1]
}
}
return decodeFilenameParam(value)
}
// findParamValueEnd finds the end position of a parameter value in a Content-Disposition header.
// It correctly handles quoted values where semicolons inside quotes should not be treated as delimiters.
// It also handles HTML entities in unquoted values (e.g., &amp; should not be split at the semicolon).
// Returns the end index (exclusive) of the value, or -1 if it extends to the end of the string.
func findParamValueEnd(value string) int {
value = strings.TrimSpace(value)
if len(value) == 0 {
return 0
}
// If the value starts with a quote, find the matching closing quote
if value[0] == '"' || value[0] == '\'' {
quote := value[0]
// Find the closing quote, handling escaped quotes
for i := 1; i < len(value); i++ {
if value[i] == quote {
// Check if it's escaped
if i > 0 && value[i-1] == '\\' {
continue
}
// Found closing quote, now look for ; after it
remaining := value[i+1:]
if semiIdx := strings.Index(remaining, ";"); semiIdx != -1 {
return i + 1 + semiIdx
}
return -1 // No semicolon after closing quote
}
}
// No closing quote found, treat rest of string as value
return -1
}
// Unquoted value - find the next semicolon that's not part of an HTML entity
// HTML entities have the pattern &...; (e.g., &amp; &lt; &gt; &quot; &#39;)
for i := 0; i < len(value); i++ {
if value[i] == ';' {
// Check if this semicolon is part of an HTML entity
// Look backwards for an & character
isEntity := false
if i > 0 {
// Look for & before this semicolon (within reasonable distance, max 10 chars)
for j := i - 1; j >= 0 && j >= i-10; j-- {
if value[j] == '&' {
// Found &, this semicolon might be part of an HTML entity
// Check if there are only alphanumeric or # between & and ;
entityChars := value[j+1 : i]
if len(entityChars) > 0 && isValidHTMLEntityChars(entityChars) {
isEntity = true
}
break
}
// If we hit whitespace or another special char, stop looking
if value[j] == ' ' || value[j] == '"' || value[j] == '\'' {
break
}
}
}
if !isEntity {
return i
}
}
}
return -1 // No semicolon, extends to end
}
// isValidHTMLEntityChars checks if a string contains only valid HTML entity characters
// (alphanumeric and #, typically for entities like &amp; &lt; &#39; etc.)
func isValidHTMLEntityChars(s string) bool {
if len(s) == 0 {
return false
}
for _, c := range s {
if !((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '#') {
return false
}
}
return true
}
internal/test/httptest.go
+210 -28
View File
@@ -70,6 +70,9 @@ func StartTestSlowFileServer(delay time.Duration) net.Listener {
func StartTestCustomServer() net.Listener {
return startTestServer(func(sl *shutdownListener) http.Handler {
mux := http.NewServeMux()
mux.HandleFunc("/", func(writer http.ResponseWriter, request *http.Request) {
writer.WriteHeader(200)
})
mux.HandleFunc("/"+BuildName, func(writer http.ResponseWriter, request *http.Request) {
file, err := os.Open(BuildFile)
if err != nil {
@@ -221,6 +224,10 @@ func StartTestCustomServer() net.Listener {
defer file.Close()
io.Copy(writer, file)
})
// Test 403 Forbidden endpoint
mux.HandleFunc("/forbidden", func(writer http.ResponseWriter, request *http.Request) {
writer.WriteHeader(403)
})
return mux
})
}
@@ -244,20 +251,6 @@ func StartTestHostHeaderServer() net.Listener {
})
}
// StartTestRootServer starts a simple server at the root path
// Used to test URL resolution when no filename is provided
func StartTestRootServer() net.Listener {
return startTestServer(func(sl *shutdownListener) http.Handler {
mux := http.NewServeMux()
mux.HandleFunc("/", func(writer http.ResponseWriter, request *http.Request) {
writer.Header().Set("Content-Type", "text/html")
writer.WriteHeader(200)
writer.Write([]byte("<html><body>Test Page</body></html>"))
})
return mux
})
}
func StartTestRetryServer() net.Listener {
counter := 0
return startTestServer(func(sl *shutdownListener) http.Handler {
@@ -303,23 +296,100 @@ func StartTestPostServer() net.Listener {
}
func StartTestErrorServer() net.Listener {
counter := 0
return startTestServer(func(sl *shutdownListener) http.Handler {
mux := http.NewServeMux()
mux.HandleFunc("/"+BuildName, func(writer http.ResponseWriter, request *http.Request) {
counter++
if counter != 1 {
writer.WriteHeader(500)
return
}
// Always return 404 error to test error handling
writer.WriteHeader(404)
return
})
return mux
})
}
// StartTestOneTimeServer creates a server where the URL can only be downloaded once
// The first non-probe request succeeds, all subsequent requests return 404
// This simulates one-time download URLs (e.g., signed URLs that expire after first use)
func StartTestOneTimeServer() net.Listener {
var accessed atomic.Bool
return startTestServer(func(sl *shutdownListener) http.Handler {
mux := http.NewServeMux()
mux.HandleFunc("/"+BuildName, func(writer http.ResponseWriter, request *http.Request) {
// First full download request succeeds, subsequent requests fail
if accessed.Swap(true) {
writer.WriteHeader(404)
return
}
// First request - return full file without Range support
writer.Header().Set("Content-Length", fmt.Sprintf("%d", BuildSize))
file, err := os.Open(BuildFile)
if err != nil {
return
}
defer file.Close()
io.Copy(writer, file)
})
return mux
})
}
// StartTestSlowStartServer creates a server with configurable delay per request
// This allows testing slow-start connection expansion to reach max connections
func StartTestSlowStartServer(delayPerByte time.Duration) net.Listener {
return startTestServer(func(sl *shutdownListener) http.Handler {
mux := http.NewServeMux()
mux.HandleFunc("/"+BuildName, func(writer http.ResponseWriter, request *http.Request) {
rangeFileHandle(
writer,
request,
nil,
func(file *os.File, n int64) {
slowCopyNWithDelay(sl, writer, file, n, delayPerByte)
},
)
})
return mux
})
}
// slowCopyNWithDelay copies n bytes from src to dst with a delay per byte
func slowCopyNWithDelay(sl *shutdownListener, dst io.Writer, src io.Reader, n int64, delayPerByte time.Duration) {
buf := make([]byte, 32*1024)
remaining := n
for remaining > 0 {
if sl.isShutdown {
return
}
toRead := int64(len(buf))
if toRead > remaining {
toRead = remaining
}
nr, er := src.Read(buf[:toRead])
if nr > 0 {
nw, ew := dst.Write(buf[0:nr])
if nw > 0 {
remaining -= int64(nw)
// Add delay based on bytes written
if delayPerByte > 0 {
time.Sleep(delayPerByte * time.Duration(nw))
}
}
if ew != nil {
break
}
}
if er != nil {
break
}
}
}
// StartTestLimitServer connections limit server
func StartTestLimitServer(maxConnections int32, delay int64) net.Listener {
var connections atomic.Int32
var slowOnce atomic.Bool
return startTestServer(func(sl *shutdownListener) http.Handler {
mux := http.NewServeMux()
@@ -332,12 +402,93 @@ func StartTestLimitServer(maxConnections int32, delay int64) net.Listener {
writer.WriteHeader(403)
return
}
// First request intentionally delays the first write to trigger a single read timeout,
// subsequent requests respond at normal speed.
useInitialDelay := delay > 0 && !slowOnce.Swap(true)
rangeFileHandle(
writer,
request,
nil,
func(file *os.File, n int64) {
slowCopyN(sl, writer, file, n, delay)
if useInitialDelay {
slowCopyAfterDelay(sl, writer, file, n, delay)
return
}
slowCopyN(sl, writer, file, n, 0)
},
)
})
return mux
})
}
// StartTestTimeoutOnceServer creates a server that times out on first request, then works normally
func StartTestTimeoutOnceServer(delay int64) net.Listener {
var timeoutOnce atomic.Bool
return startTestServer(func(sl *shutdownListener) http.Handler {
mux := http.NewServeMux()
mux.HandleFunc("/"+BuildName, func(writer http.ResponseWriter, request *http.Request) {
// First request delays to trigger timeout, subsequent requests respond normally
useInitialDelay := delay > 0 && !timeoutOnce.Swap(true)
rangeFileHandle(
writer,
request,
nil,
func(file *os.File, n int64) {
if useInitialDelay {
slowCopyAfterDelay(sl, writer, file, n, delay)
return
}
slowCopyN(sl, writer, file, n, 0)
},
)
})
return mux
})
}
// StartTestTemporary500Server creates a server that returns 500 for a duration, then recovers
// Uses slow transfer to ensure the file isn't fully downloaded during resolve phase
func StartTestTemporary500Server(errorDuration time.Duration) net.Listener {
startTime := time.Now()
var requestCount atomic.Int32
return startTestServer(func(sl *shutdownListener) http.Handler {
mux := http.NewServeMux()
mux.HandleFunc("/"+BuildName, func(writer http.ResponseWriter, request *http.Request) {
reqNum := requestCount.Add(1)
// First request (Resolve) succeeds with slow transfer to prevent full download
// Subsequent requests return 500 for the specified duration, then recover
if reqNum == 1 {
// Slow transfer for resolve: 50 microsecond per byte (~20MB/s)
// This ensures resolve doesn't complete the full 200MB file
rangeFileHandle(
writer,
request,
nil,
func(file *os.File, n int64) {
slowCopyNWithDelay(sl, writer, file, n, 50*time.Microsecond)
},
)
return
}
// Subsequent requests: return 500 for errorDuration, then normal
if time.Since(startTime) < errorDuration {
writer.WriteHeader(500)
writer.Write([]byte("Internal Server Error"))
return
}
rangeFileHandle(
writer,
request,
nil,
func(file *os.File, n int64) {
slowCopyN(sl, writer, file, n, 0)
},
)
})
@@ -375,6 +526,26 @@ func StartTestRangeBugServer() net.Listener {
func rangeFileHandle(writer http.ResponseWriter, request *http.Request, modifyEnd func(end int64) int64, iocpN func(file *os.File, n int64)) {
r := request.Header.Get("Range")
// If no Range header, return full file with Accept-Ranges header
if r == "" {
// Open file first to ensure it exists
file, err := os.Open(BuildFile)
if err != nil {
writer.WriteHeader(500)
return
}
defer file.Close()
writer.Header().Set("Content-Length", fmt.Sprintf("%d", BuildSize))
writer.Header().Set("Accept-Ranges", "bytes")
writer.WriteHeader(200)
(writer.(http.Flusher)).Flush()
iocpN(file, BuildSize)
return
}
// split range
s := strings.Split(r, "=")
if len(s) != 2 {
@@ -408,12 +579,7 @@ func rangeFileHandle(writer http.ResponseWriter, request *http.Request, modifyEn
end = modifyEnd(end)
}
writer.Header().Set("Content-Length", fmt.Sprintf("%d", end-start+1))
writer.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, BuildSize))
writer.Header().Set("Accept-Ranges", "bytes")
writer.WriteHeader(206)
(writer.(http.Flusher)).Flush()
// Open file before sending headers to ensure it exists
file, err := os.Open(BuildFile)
if err != nil {
writer.WriteHeader(500)
@@ -421,6 +587,13 @@ func rangeFileHandle(writer http.ResponseWriter, request *http.Request, modifyEn
}
defer file.Close()
file.Seek(start, 0)
writer.Header().Set("Content-Length", fmt.Sprintf("%d", end-start+1))
writer.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, BuildSize))
writer.Header().Set("Accept-Ranges", "bytes")
writer.WriteHeader(206)
(writer.(http.Flusher)).Flush()
iocpN(file, end-start+1)
}
@@ -471,6 +644,14 @@ func slowCopyN(sl *shutdownListener, dst io.Writer, src io.Reader, n int64, dela
return
}
// slowCopyAfterDelay sleeps once before performing a normal copy to simulate a single timeout.
func slowCopyAfterDelay(sl *shutdownListener, dst io.Writer, src io.Reader, n int64, delay int64) (written int64, err error) {
if delay > 0 {
time.Sleep(time.Millisecond * time.Duration(delay))
}
return slowCopyN(sl, dst, src, n, 0)
}
func startTestServer(serverHandle func(sl *shutdownListener) http.Handler) net.Listener {
listener, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
@@ -515,8 +696,9 @@ type shutdownListener struct {
}
func (c *shutdownListener) Close() error {
c.isShutdown = true
// Shutdown server first (waits for in-flight requests), then set isShutdown
closeErr := c.server.Shutdown(context.Background())
c.isShutdown = true
if err := ifExistAndRemove(BuildFile); err != nil {
fmt.Println(err)
}
pkg/base/model.go
+1 -1
View File
@@ -149,7 +149,7 @@ func ParseReqExtra[E any](req *Request) error {
return nil
}
func ParseOptsExtra[E any](opts *Options) error {
func ParseOptExtra[E any](opts *Options) error {
if opts.Extra == nil {
return nil
}
pkg/download/downloader.go
+52 -51
View File
@@ -321,7 +321,7 @@ func (d *Downloader) saveTask(task *Task) error {
return nil
}
func (d *Downloader) Resolve(req *base.Request) (rr *ResolveResult, err error) {
func (d *Downloader) Resolve(req *base.Request, opts *base.Options) (rr *ResolveResult, err error) {
rrId, err := gonanoid.New()
if err != nil {
return
@@ -342,7 +342,11 @@ func (d *Downloader) Resolve(req *base.Request) (rr *ResolveResult, err error) {
if err != nil {
return
}
err = fetcher.Resolve(req)
initOpt, err := d.initOptions(opts)
if err != nil {
return
}
err = fetcher.Resolve(req, initOpt)
if err != nil {
return
}
@@ -390,7 +394,11 @@ func (d *Downloader) CreateDirect(req *base.Request, opts *base.Options) (taskId
return
}
fetcher.Meta().Req = req
return d.doCreate(fetcher, opts)
initOpt, err := d.initOptions(opts)
if err != nil {
return
}
return d.doCreate(fetcher, initOpt)
}
func (d *Downloader) CreateDirectBatch(req *base.CreateTaskBatch) (taskId []string, err error) {
@@ -409,7 +417,7 @@ func (d *Downloader) CreateDirectBatch(req *base.CreateTaskBatch) (taskId []stri
return taskIds, nil
}
func (d *Downloader) Create(rrId string, opts *base.Options) (taskId string, err error) {
func (d *Downloader) Create(rrId string) (taskId string, err error) {
d.fetcherMapLock.RLock()
fetcher, ok := d.fetcherCache[rrId]
d.fetcherMapLock.RUnlock()
@@ -421,7 +429,7 @@ func (d *Downloader) Create(rrId string, opts *base.Options) (taskId string, err
delete(d.fetcherCache, rrId)
d.fetcherMapLock.Unlock()
}()
return d.doCreate(fetcher, opts)
return d.doCreate(fetcher, nil)
}
func (d *Downloader) Pause(filter *TaskFilter) (err error) {
@@ -855,8 +863,6 @@ func (d *Downloader) watch(task *Task) {
if err != nil {
d.Logger.Warn().Err(err).Msgf("task wait upload failed, task id: %s", task.ID)
}
d.lock.Lock()
defer d.lock.Unlock()
// Check if the task is deleted
if d.GetTask(task.ID) != nil {
@@ -947,7 +953,6 @@ func (d *Downloader) restoreTask(task *Task) error {
}
}
go d.watch(task)
task.fetcher.Create(task.Meta.Opts)
return nil
}
@@ -970,42 +975,15 @@ func (d *Downloader) restoreFetcher(task *Task) error {
if task.fetcher.Meta().Res == nil {
task.fetcher.Meta().Res = task.Meta.Res
}
if task.fetcher.Meta().Opts == nil {
task.fetcher.Meta().Opts = task.Meta.Opts
}
return nil
}
func (d *Downloader) doCreate(f fetcher.Fetcher, opts *base.Options) (taskId string, err error) {
if opts == nil {
opts = &base.Options{}
}
if opts.SelectFiles == nil {
opts.SelectFiles = make([]int, 0)
}
meta := f.Meta()
meta.Opts = opts
if opts.Path == "" {
storeConfig, err := d.GetConfig()
if err != nil {
return "", err
}
opts.Path = storeConfig.DownloadDir
}
// Replace placeholders in download path (e.g., %year%, %month%, %day%, %date%)
opts.Path = util.ReplacePathPlaceholders(opts.Path)
// if enable white download directory, check if the download directory is in the white list
if len(d.cfg.WhiteDownloadDirs) > 0 {
inWhiteList := false
for _, dir := range d.cfg.WhiteDownloadDirs {
if match, err := filepath.Match(dir, opts.Path); match && err == nil {
inWhiteList = true
break
}
}
if !inWhiteList {
return "", errors.New("download directory is not in white list")
}
if f.Meta().Opts == nil {
f.Meta().Opts = opts
}
fm, err := d.parseFm(f.Meta().Req.URL)
@@ -1021,9 +999,6 @@ func (d *Downloader) doCreate(f fetcher.Fetcher, opts *base.Options) (taskId str
task.Progress = &Progress{}
_, task.Uploading = f.(fetcher.Uploader)
initTask(task)
if err = f.Create(opts); err != nil {
return
}
if err = d.storage.Put(bucketTask, task.ID, task.clone()); err != nil {
return
}
@@ -1049,6 +1024,39 @@ func (d *Downloader) doCreate(f fetcher.Fetcher, opts *base.Options) (taskId str
return
}
func (d *Downloader) initOptions(opts *base.Options) (*base.Options, error) {
if opts == nil {
opts = &base.Options{}
}
if opts.SelectFiles == nil {
opts.SelectFiles = make([]int, 0)
}
if opts.Path == "" {
storeConfig, err := d.GetConfig()
if err != nil {
return nil, err
}
opts.Path = storeConfig.DownloadDir
}
// Replace placeholders in download path (e.g., %year%, %month%, %day%, %date%)
opts.Path = util.ReplacePathPlaceholders(opts.Path)
// if enable white download directory, check if the download directory is in the white list
if len(d.cfg.WhiteDownloadDirs) > 0 {
inWhiteList := false
for _, dir := range d.cfg.WhiteDownloadDirs {
if match, err := filepath.Match(dir, opts.Path); match && err == nil {
inWhiteList = true
break
}
}
if !inWhiteList {
return nil, errors.New("download directory is not in white list")
}
}
return opts, nil
}
func (d *Downloader) statusMut(task *Task, fn func() (bool, error)) (bool, error) {
task.statusLock.Lock()
defer task.statusLock.Unlock()
@@ -1088,7 +1096,7 @@ func (d *Downloader) doStart(task *Task) (err error) {
d.triggerOnStart(task)
if task.Meta.Res == nil {
err := task.fetcher.Resolve(task.Meta.Req)
err := task.fetcher.Resolve(task.Meta.Req, task.Meta.Opts)
if err != nil {
return err
}
@@ -1621,13 +1629,6 @@ func (b *boot) Extra(extra interface{}) *boot {
return b
}
func (b *boot) Resolve() (*ResolveResult, error) {
return defaultDownloader.Resolve(&base.Request{
URL: b.url,
Extra: b.extra,
})
}
func (b *boot) Listener(listener Listener) *boot {
b.listener = listener
return b
pkg/download/downloader_test.go
+28 -75
View File
@@ -20,6 +20,14 @@ import (
"github.com/GopeedLab/gopeed/pkg/protocol/http"
)
var testDownloadOpt = &base.Options{
Path: test.Dir,
Name: test.DownloadName,
Extra: http.OptsExtra{
Connections: 4,
},
}
func TestDownloader_Resolve(t *testing.T) {
listener := test.StartTestFileServer()
defer listener.Close()
@@ -32,7 +40,7 @@ func TestDownloader_Resolve(t *testing.T) {
req := &base.Request{
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
}
rr, err := downloader.Resolve(req)
rr, err := downloader.Resolve(req, nil)
if err != nil {
t.Fatal(err)
}
@@ -64,7 +72,7 @@ func TestDownloader_Create(t *testing.T) {
req := &base.Request{
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
}
rr, err := downloader.Resolve(req)
rr, err := downloader.Resolve(req, testDownloadOpt)
if err != nil {
t.Fatal(err)
}
@@ -76,13 +84,7 @@ func TestDownloader_Create(t *testing.T) {
wg.Done()
}
})
_, err = downloader.Create(rr.ID, &base.Options{
Path: test.Dir,
Name: test.DownloadName,
Extra: http.OptsExtra{
Connections: 4,
},
})
_, err = downloader.Create(rr.ID)
if err != nil {
t.Fatal(err)
}
@@ -108,27 +110,14 @@ func TestDownloader_CreateNotInWhite(t *testing.T) {
req := &base.Request{
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
}
rr, err := downloader.Resolve(req)
if err != nil {
t.Fatal(err)
// With new fetcher design, white list check happens during Resolve (not Create)
// because Resolve now requires Options which includes the download path
_, err := downloader.Resolve(req, testDownloadOpt)
if err == nil {
t.Error("TestDownloader_CreateNotInWhite() expected error but got nil")
}
var wg sync.WaitGroup
wg.Add(1)
downloader.Listener(func(event *Event) {
if event.Key == EventKeyDone {
wg.Done()
}
})
_, err = downloader.Create(rr.ID, &base.Options{
Path: test.Dir,
Name: test.DownloadName,
Extra: http.OptsExtra{
Connections: 4,
},
})
if !strings.Contains(err.Error(), "white") {
t.Errorf("TestDownloader_CreateNotInWhite() got = %v, want %v", err.Error(), "not in white list")
t.Errorf("TestDownloader_CreateNotInWhite() got = %v, want error containing 'white'", err.Error())
}
}
@@ -172,13 +161,7 @@ func TestDownloader_CreateDirectBatch(t *testing.T) {
_, err := downloader.CreateDirectBatch(&base.CreateTaskBatch{
Reqs: reqs,
Opts: &base.Options{
Path: test.Dir,
Name: test.DownloadName,
Extra: http.OptsExtra{
Connections: 4,
},
},
Opts: testDownloadOpt,
})
if err != nil {
t.Fatal(err)
@@ -317,7 +300,7 @@ func doTestDownloaderCreateWithProxy(t *testing.T, auth bool, buildReqProxy func
Pwd: pwd,
})
}
rr, err := downloader.Resolve(req)
rr, err := downloader.Resolve(req, nil)
if err != nil {
if errHandler == nil {
t.Fatal(err)
@@ -400,18 +383,12 @@ func TestDownloader_StoreAndRestore(t *testing.T) {
req := &base.Request{
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
}
rr, err := downloader.Resolve(req)
rr, err := downloader.Resolve(req, testDownloadOpt)
if err != nil {
t.Fatal(err)
}
id, err := downloader.Create(rr.ID, &base.Options{
Path: test.Dir,
Name: test.DownloadName,
Extra: http.OptsExtra{
Connections: 4,
},
})
id, err := downloader.Create(rr.ID)
if err != nil {
t.Fatal(err)
}
@@ -675,7 +652,7 @@ func TestDownloader_Stats(t *testing.T) {
req := &base.Request{
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
}
rr, err := downloader.Resolve(req)
rr, err := downloader.Resolve(req, testDownloadOpt)
if err != nil {
t.Fatal(err)
}
@@ -688,13 +665,7 @@ func TestDownloader_Stats(t *testing.T) {
}
})
taskId, err := downloader.Create(rr.ID, &base.Options{
Path: test.Dir,
Name: test.DownloadName,
Extra: http.OptsExtra{
Connections: 4,
},
})
taskId, err := downloader.Create(rr.ID)
if err != nil {
t.Fatal(err)
}
@@ -736,13 +707,7 @@ func TestDownloader_Delete(t *testing.T) {
req := &base.Request{
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
}
taskId, err := downloader.CreateDirect(req, &base.Options{
Path: test.Dir,
Name: test.DownloadName,
Extra: http.OptsExtra{
Connections: 4,
},
})
taskId, err := downloader.CreateDirect(req, testDownloadOpt)
if err != nil {
t.Fatal(err)
}
@@ -810,17 +775,11 @@ func TestDownloader_ContinueBatch(t *testing.T) {
req := &base.Request{
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
}
rr, err := downloader.Resolve(req)
rr, err := downloader.Resolve(req, testDownloadOpt)
if err != nil {
t.Fatal(err)
}
taskId, err := downloader.Create(rr.ID, &base.Options{
Path: test.Dir,
Name: test.DownloadName,
Extra: http.OptsExtra{
Connections: 4,
},
})
taskId, err := downloader.Create(rr.ID)
if err != nil {
t.Fatal(err)
}
@@ -905,18 +864,12 @@ func TestDownloader_PauseAndContinue(t *testing.T) {
req := &base.Request{
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
}
rr, err := downloader.Resolve(req)
rr, err := downloader.Resolve(req, testDownloadOpt)
if err != nil {
t.Fatal(err)
}
taskId, err := downloader.Create(rr.ID, &base.Options{
Path: test.Dir,
Name: test.DownloadName,
Extra: http.OptsExtra{
Connections: 4,
},
})
taskId, err := downloader.Create(rr.ID)
if err != nil {
t.Fatal(err)
}
pkg/download/engine/polyfill/package-lock.json
Generated
+6 -6
View File
@@ -217,7 +217,7 @@
"@webassemblyjs/helper-wasm-bytecode": "1.11.1",
"@webassemblyjs/helper-wasm-section": "1.11.1",
"@webassemblyjs/wasm-gen": "1.11.1",
"@webassemblyjs/wasm-opt": "1.11.1",
"@webassemblyjs/wasm-opts": "1.11.1",
"@webassemblyjs/wasm-parser": "1.11.1",
"@webassemblyjs/wast-printer": "1.11.1"
}
@@ -235,9 +235,9 @@
"@webassemblyjs/utf8": "1.11.1"
}
},
"node_modules/@webassemblyjs/wasm-opt": {
"node_modules/@webassemblyjs/wasm-opts": {
"version": "1.11.1",
"resolved": "https://registry.npmmirror.com/@webassemblyjs/wasm-opt/-/wasm-opt-1.11.1.tgz",
"resolved": "https://registry.npmmirror.com/@webassemblyjs/wasm-opts/-/wasm-opts-1.11.1.tgz",
"integrity": "sha512-VqnkNqnZlU5EB64pp1l7hdm3hmQw7Vgqa0KF/KCNO9sIpI6Fk6brDEiX+iCOYrvMuBWDws0NkTOxYEb85XQHHw==",
"dev": true,
"dependencies": {
@@ -2114,7 +2114,7 @@
"@webassemblyjs/helper-wasm-bytecode": "1.11.1",
"@webassemblyjs/helper-wasm-section": "1.11.1",
"@webassemblyjs/wasm-gen": "1.11.1",
"@webassemblyjs/wasm-opt": "1.11.1",
"@webassemblyjs/wasm-opts": "1.11.1",
"@webassemblyjs/wasm-parser": "1.11.1",
"@webassemblyjs/wast-printer": "1.11.1"
}
@@ -2132,9 +2132,9 @@
"@webassemblyjs/utf8": "1.11.1"
}
},
"@webassemblyjs/wasm-opt": {
"@webassemblyjs/wasm-opts": {
"version": "1.11.1",
"resolved": "https://registry.npmmirror.com/@webassemblyjs/wasm-opt/-/wasm-opt-1.11.1.tgz",
"resolved": "https://registry.npmmirror.com/@webassemblyjs/wasm-opts/-/wasm-opts-1.11.1.tgz",
"integrity": "sha512-VqnkNqnZlU5EB64pp1l7hdm3hmQw7Vgqa0KF/KCNO9sIpI6Fk6brDEiX+iCOYrvMuBWDws0NkTOxYEb85XQHHw==",
"dev": true,
"requires": {
pkg/download/extension_test.go
+19 -18
View File
@@ -2,13 +2,14 @@ package download
import (
"errors"
"os"
"testing"
"time"
"github.com/GopeedLab/gopeed/internal/logger"
"github.com/GopeedLab/gopeed/pkg/base"
gojaerror "github.com/GopeedLab/gopeed/pkg/download/engine/inject/error"
"github.com/dop251/goja"
"os"
"testing"
"time"
)
func TestDownloader_InstallExtensionByFolder(t *testing.T) {
@@ -18,7 +19,7 @@ func TestDownloader_InstallExtensionByFolder(t *testing.T) {
}
rr, err := downloader.Resolve(&base.Request{
URL: "https://github.com/test",
})
}, nil)
if err != nil {
t.Fatal(err)
}
@@ -35,7 +36,7 @@ func TestDownloader_InstallExtensionByFolderDevMode(t *testing.T) {
}
rr, err := downloader.Resolve(&base.Request{
URL: "https://github.com/test",
})
}, nil)
if err != nil {
t.Fatal(err)
}
@@ -52,7 +53,7 @@ func TestDownloader_InstallExtensionByGit(t *testing.T) {
}
rr, err := downloader.Resolve(&base.Request{
URL: "https://github.com/GopeedLab/gopeed/releases",
})
}, nil)
if err != nil {
t.Fatal(err)
}
@@ -69,7 +70,7 @@ func TestDownloader_InstallExtensionByGitSimple(t *testing.T) {
}
rr, err := downloader.Resolve(&base.Request{
URL: "https://github.com/GopeedLab/gopeed/releases",
})
}, nil)
if err != nil {
t.Fatal(err)
}
@@ -86,7 +87,7 @@ func TestDownloader_InstallExtensionByGitFull(t *testing.T) {
}
rr, err := downloader.Resolve(&base.Request{
URL: "https://github.com/GopeedLab/gopeed/releases",
})
}, nil)
if err != nil {
t.Fatal(err)
}
@@ -156,7 +157,7 @@ func TestDownloader_UpgradeExtension(t *testing.T) {
rr, err := downloader.Resolve(&base.Request{
URL: "https://test.com",
})
}, nil)
if err != nil {
t.Fatal(err)
}
@@ -185,7 +186,7 @@ func TestDownloader_Extension_OnStart(t *testing.T) {
select {
case err = <-errCh:
break
case <-time.After(time.Second * 10):
case <-time.After(time.Second * 30): // Increased timeout for real network requests
err = errors.New("timeout")
}
if err != nil {
@@ -238,7 +239,7 @@ func TestDownloader_Extension_OnError(t *testing.T) {
select {
case err = <-errCh:
break
case <-time.After(time.Second * 10):
case <-time.After(time.Second * 30): // Increased timeout for real network requests
err = errors.New("timeout")
}
@@ -273,7 +274,7 @@ func TestDownloader_Extension_OnDone(t *testing.T) {
select {
case err = <-errCh:
break
case <-time.After(time.Second * 10):
case <-time.After(time.Second * 30): // Increased timeout for real network requests
err = errors.New("timeout")
}
// wait for script execution
@@ -300,7 +301,7 @@ func TestDownloader_Extension_Errors(t *testing.T) {
}
rr, err := downloader.Resolve(&base.Request{
URL: "https://github.com/test",
})
}, nil)
if err != nil {
t.Fatal(err)
}
@@ -315,7 +316,7 @@ func TestDownloader_Extension_Errors(t *testing.T) {
}
rr, err := downloader.Resolve(&base.Request{
URL: "https://github.com/test",
})
}, nil)
if err != nil {
t.Fatal(err)
}
@@ -330,7 +331,7 @@ func TestDownloader_Extension_Errors(t *testing.T) {
}
_, err := downloader.Resolve(&base.Request{
URL: "https://github.com/test",
})
}, nil)
if err == nil {
t.Fatalf("except error, but got nil")
}
@@ -352,7 +353,7 @@ func TestDownloader_Extension_Settings(t *testing.T) {
}
rr, err := downloader.Resolve(&base.Request{
URL: "https://github.com/test",
})
}, nil)
if err != nil {
t.Fatal(err)
}
@@ -373,7 +374,7 @@ func TestDownloader_Extension_Settings(t *testing.T) {
})
rr, err := downloader.Resolve(&base.Request{
URL: "https://github.com/test",
})
}, nil)
if err != nil {
t.Fatal(err)
}
@@ -390,7 +391,7 @@ func TestDownloader_ExtensionStorage(t *testing.T) {
}
rr, err := downloader.Resolve(&base.Request{
URL: "https://github.com/test",
})
}, nil)
if err != nil {
t.Fatal(err)
}
pkg/download/model.go
+3 -2
View File
@@ -2,6 +2,9 @@ package download
import (
"encoding/json"
"sync"
"time"
"github.com/GopeedLab/gopeed/internal/controller"
"github.com/GopeedLab/gopeed/internal/fetcher"
"github.com/GopeedLab/gopeed/internal/protocol/bt"
@@ -9,8 +12,6 @@ import (
"github.com/GopeedLab/gopeed/pkg/base"
"github.com/GopeedLab/gopeed/pkg/util"
gonanoid "github.com/matoous/go-nanoid/v2"
"sync"
"time"
)
type ResolveResult struct {
pkg/download/webhook_test.go
+38 -38
View File
@@ -50,9 +50,9 @@ func TestWebhook_TriggerOnDone(t *testing.T) {
// Configure webhook URLs
cfg, _ := downloader.GetConfig()
cfg.Webhook = &base.WebhookConfig{
Enable: true,
URLs: []string{server.URL},
}
Enable: true,
URLs: []string{server.URL},
}
downloader.PutConfig(cfg)
// Create a mock task
@@ -99,9 +99,9 @@ func TestWebhook_TriggerOnError(t *testing.T) {
// Configure webhook URLs
cfg, _ := downloader.GetConfig()
cfg.Webhook = &base.WebhookConfig{
Enable: true,
URLs: []string{server.URL},
}
Enable: true,
URLs: []string{server.URL},
}
downloader.PutConfig(cfg)
// Create a mock task
@@ -144,9 +144,9 @@ func TestWebhook_SendTestWebhook(t *testing.T) {
// Configure webhook URLs
cfg, _ := downloader.GetConfig()
cfg.Webhook = &base.WebhookConfig{
Enable: true,
URLs: []string{server.URL},
}
Enable: true,
URLs: []string{server.URL},
}
downloader.PutConfig(cfg)
// Send test webhook
@@ -207,9 +207,9 @@ func TestWebhook_MultipleUrls(t *testing.T) {
// Configure multiple webhook URLs
cfg, _ := downloader.GetConfig()
cfg.Webhook = &base.WebhookConfig{
Enable: true,
URLs: []string{server1.URL, server2.URL},
}
Enable: true,
URLs: []string{server1.URL, server2.URL},
}
downloader.PutConfig(cfg)
// Create a mock task
@@ -240,9 +240,9 @@ func TestWebhook_TestWebhookFailsOnNon200(t *testing.T) {
// Configure webhook URLs
cfg, _ := downloader.GetConfig()
cfg.Webhook = &base.WebhookConfig{
Enable: true,
URLs: []string{server.URL},
}
Enable: true,
URLs: []string{server.URL},
}
downloader.PutConfig(cfg)
// Send test webhook - should fail with non-200 status
@@ -264,9 +264,9 @@ func TestWebhook_TestWebhookFailsOn201(t *testing.T) {
// Configure webhook URLs
cfg, _ := downloader.GetConfig()
cfg.Webhook = &base.WebhookConfig{
Enable: true,
URLs: []string{server.URL},
}
Enable: true,
URLs: []string{server.URL},
}
downloader.PutConfig(cfg)
// Send test webhook - should fail with 201 status (only 200 is success)
@@ -346,7 +346,7 @@ func TestWebhook_GetWebhookUrls_NoExtraField(t *testing.T) {
func TestWebhook_GetWebhookUrls_NoWebhookUrlsKey(t *testing.T) {
setupWebhookTest(t, func(downloader *Downloader) {
cfg, _ := downloader.GetConfig()
cfg.Webhook = &base.WebhookConfig{Enable: true} // No URLs set
cfg.Webhook = &base.WebhookConfig{Enable: true} // No URLs set
downloader.PutConfig(cfg)
urls := downloader.getWebhookUrls()
@@ -360,9 +360,9 @@ func TestWebhook_GetWebhookUrls_StringSlice(t *testing.T) {
setupWebhookTest(t, func(downloader *Downloader) {
cfg, _ := downloader.GetConfig()
cfg.Webhook = &base.WebhookConfig{
Enable: true,
URLs: []string{"http://example.com", "http://example2.com"},
}
Enable: true,
URLs: []string{"http://example.com", "http://example2.com"},
}
downloader.PutConfig(cfg)
urls := downloader.getWebhookUrls()
@@ -379,9 +379,9 @@ func TestWebhook_GetWebhookUrls_InterfaceSlice(t *testing.T) {
setupWebhookTest(t, func(downloader *Downloader) {
cfg, _ := downloader.GetConfig()
cfg.Webhook = &base.WebhookConfig{
Enable: true,
URLs: []string{"http://example.com", "http://example2.com"},
}
Enable: true,
URLs: []string{"http://example.com", "http://example2.com"},
}
downloader.PutConfig(cfg)
urls := downloader.getWebhookUrls()
@@ -412,9 +412,9 @@ func TestWebhook_GetWebhookUrls_InterfaceSliceWithEmptyStrings(t *testing.T) {
setupWebhookTest(t, func(downloader *Downloader) {
cfg, _ := downloader.GetConfig()
cfg.Webhook = &base.WebhookConfig{
Enable: true,
URLs: []string{"", "", ""},
}
Enable: true,
URLs: []string{"", "", ""},
}
downloader.PutConfig(cfg)
urls := downloader.getWebhookUrls()
@@ -446,7 +446,7 @@ func TestWebhook_GetWebhookUrls_InterfaceSliceMixedTypes(t *testing.T) {
func TestWebhook_GetWebhookUrls_InvalidType(t *testing.T) {
setupWebhookTest(t, func(downloader *Downloader) {
cfg, _ := downloader.GetConfig()
cfg.Webhook = &base.WebhookConfig{Enable: false} // Disabled webhook
cfg.Webhook = &base.WebhookConfig{Enable: false} // Disabled webhook
downloader.PutConfig(cfg)
urls := downloader.getWebhookUrls()
@@ -640,9 +640,9 @@ func TestWebhook_TriggerWebhooks_EmptyUrlSkipped(t *testing.T) {
setupWebhookTest(t, func(downloader *Downloader) {
cfg, _ := downloader.GetConfig()
cfg.Webhook = &base.WebhookConfig{
Enable: true,
URLs: []string{server.URL, "", server.URL},
}
Enable: true,
URLs: []string{server.URL, "", server.URL},
}
downloader.PutConfig(cfg)
task := NewTask()
@@ -675,9 +675,9 @@ func TestWebhook_WebhookDataStructure(t *testing.T) {
setupWebhookTest(t, func(downloader *Downloader) {
cfg, _ := downloader.GetConfig()
cfg.Webhook = &base.WebhookConfig{
Enable: true,
URLs: []string{server.URL},
}
Enable: true,
URLs: []string{server.URL},
}
downloader.PutConfig(cfg)
task := NewTask()
@@ -744,9 +744,9 @@ func TestWebhook_SendTestWebhook_MixedResults(t *testing.T) {
setupWebhookTest(t, func(downloader *Downloader) {
cfg, _ := downloader.GetConfig()
cfg.Webhook = &base.WebhookConfig{
Enable: true,
URLs: []string{server1.URL, server2.URL},
}
Enable: true,
URLs: []string{server1.URL, server2.URL},
}
downloader.PutConfig(cfg)
// Should fail because server2 returns 500
pkg/rest/api.go
+9 -8
View File
@@ -1,14 +1,15 @@
package rest
import (
"github.com/GopeedLab/gopeed/pkg/base"
"github.com/GopeedLab/gopeed/pkg/download"
"github.com/GopeedLab/gopeed/pkg/rest/model"
"github.com/gorilla/mux"
"io"
"net/http"
"net/url"
"runtime"
"github.com/GopeedLab/gopeed/pkg/base"
"github.com/GopeedLab/gopeed/pkg/download"
"github.com/GopeedLab/gopeed/pkg/rest/model"
"github.com/gorilla/mux"
)
func Info(w http.ResponseWriter, r *http.Request) {
@@ -23,9 +24,9 @@ func Info(w http.ResponseWriter, r *http.Request) {
}
func Resolve(w http.ResponseWriter, r *http.Request) {
var req base.Request
var req model.ResolveTask
if ReadJson(r, w, &req) {
rr, err := Downloader.Resolve(&req)
rr, err := Downloader.Resolve(req.Req, req.Opts)
if err != nil {
WriteJson(w, model.NewErrorResult(err.Error()))
return
@@ -42,9 +43,9 @@ func CreateTask(w http.ResponseWriter, r *http.Request) {
err error
)
if req.Rid != "" {
taskId, err = Downloader.Create(req.Rid, req.Opt)
taskId, err = Downloader.Create(req.Rid)
} else if req.Req != nil {
taskId, err = Downloader.CreateDirect(req.Req, req.Opt)
taskId, err = Downloader.CreateDirect(req.Req, req.Opts)
} else {
WriteJson(w, model.NewErrorResult("param invalid: rid or req", model.CodeInvalidParam))
return
pkg/rest/model/task.go
+8 -3
View File
@@ -2,8 +2,13 @@ package model
import "github.com/GopeedLab/gopeed/pkg/base"
type ResolveTask struct {
Req *base.Request `json:"req"`
Opts *base.Options `json:"opts"`
}
type CreateTask struct {
Rid string `json:"rid"`
Req *base.Request `json:"req"`
Opt *base.Options `json:"opt"`
Rid string `json:"rid"`
Req *base.Request `json:"req"`
Opts *base.Options `json:"opts"`
}
pkg/rest/server_test.go
+8 -33
View File
@@ -53,13 +53,13 @@ var (
"connections": 2,
},
}
createReq = &model.CreateTask{
Req: taskReq,
Opt: createOpts,
resolveReq = &model.ResolveTask{
Req: taskReq,
Opts: createOpts,
}
createResoledReq = &model.CreateTask{
Req: taskReq,
Opt: createOpts,
createReq = &model.CreateTask{
Req: taskReq,
Opts: createOpts,
}
installExtensionReq = &model.InstallExtension{
URL: "https://github.com/GopeedLab/gopeed-extension-samples#github-contributor-avatars-sample",
@@ -80,7 +80,7 @@ func TestInfo(t *testing.T) {
func TestResolve(t *testing.T) {
doTest(func() {
resp := httpRequestCheckOk[*download.ResolveResult](http.MethodPost, "/api/v1/resolve", taskReq)
resp := httpRequestCheckOk[*download.ResolveResult](http.MethodPost, "/api/v1/resolve", resolveReq)
if !test.AssertResourceEqual(taskRes, resp.Res) {
t.Errorf("Resolve() got = %v, want %v", test.ToJson(resp.Res), test.ToJson(taskRes))
}
@@ -89,7 +89,7 @@ func TestResolve(t *testing.T) {
func TestCreateTask(t *testing.T) {
doTest(func() {
resp := httpRequestCheckOk[*download.ResolveResult](http.MethodPost, "/api/v1/resolve", taskReq)
resp := httpRequestCheckOk[*download.ResolveResult](http.MethodPost, "/api/v1/resolve", resolveReq)
var wg sync.WaitGroup
wg.Add(1)
@@ -101,7 +101,6 @@ func TestCreateTask(t *testing.T) {
taskId := httpRequestCheckOk[string](http.MethodPost, "/api/v1/tasks", &model.CreateTask{
Rid: resp.ID,
Opt: createOpts,
})
if taskId == "" {
t.Fatal("create task failed")
@@ -196,30 +195,6 @@ func TestCreateDirectTaskBatchWithOpt(t *testing.T) {
})
}
func TestCreateDirectTaskWithResoled(t *testing.T) {
doTest(func() {
var wg sync.WaitGroup
wg.Add(1)
Downloader.Listener(func(event *download.Event) {
if event.Key == download.EventKeyFinally {
wg.Done()
}
})
taskId := httpRequestCheckOk[string](http.MethodPost, "/api/v1/tasks", createResoledReq)
if taskId == "" {
t.Fatal("create task failed")
}
wg.Wait()
want := test.FileMd5(test.BuildFile)
got := test.FileMd5(test.DownloadFile)
if want != got {
t.Errorf("CreateDirectTaskWithResoled() got = %v, want %v", got, want)
}
})
}
func TestPauseAndContinueTask(t *testing.T) {
doTest(func() {
var wg sync.WaitGroup
ui/flutter/lib/api/api.dart
+3 -2
View File
@@ -16,6 +16,7 @@ import 'model/install_extension.dart';
import 'model/login.dart';
import 'model/request.dart';
import 'model/resolve_result.dart';
import 'model/resolve_task.dart';
import 'model/result.dart';
import 'model/switch_extension.dart';
import 'model/task.dart';
@@ -123,9 +124,9 @@ Future<T> _parse<T>(
}
}
Future<ResolveResult> resolve(Request request) async {
Future<ResolveResult> resolve(ResolveTask resolveTask) async {
return _parse<ResolveResult>(
() => _client.dio.post("api/v1/resolve", data: request),
() => _client.dio.post("api/v1/resolve", data: resolveTask),
(data) => ResolveResult.fromJson(data));
}
ui/flutter/lib/api/model/create_task.dart
+2 -2
View File
@@ -9,12 +9,12 @@ part 'create_task.g.dart';
class CreateTask {
String? rid;
Request? req;
Options? opt;
Options? opts;
CreateTask({
this.rid,
this.req,
this.opt,
this.opts,
});
factory CreateTask.fromJson(
ui/flutter/lib/api/model/create_task.g.dart
+3 -3
View File
@@ -11,9 +11,9 @@ CreateTask _$CreateTaskFromJson(Map<String, dynamic> json) => CreateTask(
req: json['req'] == null
? null
: Request.fromJson(json['req'] as Map<String, dynamic>),
opt: json['opt'] == null
opts: json['opts'] == null
? null
: Options.fromJson(json['opt'] as Map<String, dynamic>),
: Options.fromJson(json['opts'] as Map<String, dynamic>),
);
Map<String, dynamic> _$CreateTaskToJson(CreateTask instance) {
@@ -27,6 +27,6 @@ Map<String, dynamic> _$CreateTaskToJson(CreateTask instance) {
writeNotNull('rid', instance.rid);
writeNotNull('req', instance.req?.toJson());
writeNotNull('opt', instance.opt?.toJson());
writeNotNull('opts', instance.opts?.toJson());
return val;
}
ui/flutter/lib/api/model/create_task_batch.dart
+2 -2
View File
@@ -8,11 +8,11 @@ part 'create_task_batch.g.dart';
@JsonSerializable(explicitToJson: true)
class CreateTaskBatch {
List<CreateTaskBatchItem>? reqs;
Options? opt;
Options? opts;
CreateTaskBatch({
this.reqs,
this.opt,
this.opts,
});
factory CreateTaskBatch.fromJson(
ui/flutter/lib/api/model/create_task_batch.g.dart
+3 -3
View File
@@ -11,9 +11,9 @@ CreateTaskBatch _$CreateTaskBatchFromJson(Map<String, dynamic> json) =>
reqs: (json['reqs'] as List<dynamic>?)
?.map((e) => CreateTaskBatchItem.fromJson(e as Map<String, dynamic>))
.toList(),
opt: json['opt'] == null
opts: json['opts'] == null
? null
: Options.fromJson(json['opt'] as Map<String, dynamic>),
: Options.fromJson(json['opts'] as Map<String, dynamic>),
);
Map<String, dynamic> _$CreateTaskBatchToJson(CreateTaskBatch instance) {
@@ -26,7 +26,7 @@ Map<String, dynamic> _$CreateTaskBatchToJson(CreateTaskBatch instance) {
}
writeNotNull('reqs', instance.reqs?.map((e) => e.toJson()).toList());
writeNotNull('opt', instance.opt?.toJson());
writeNotNull('opts', instance.opts?.toJson());
return val;
}
ui/flutter/lib/api/model/resolve_task.dart
+21
View File
@@ -0,0 +1,21 @@
import 'package:json_annotation/json_annotation.dart';
import 'options.dart';
import 'request.dart';
part 'resolve_task.g.dart';
@JsonSerializable()
class ResolveTask {
Request? req;
Options? opts;
ResolveTask({
this.req,
this.opts,
});
factory ResolveTask.fromJson(Map<String, dynamic> json) =>
_$ResolveTaskFromJson(json);
Map<String, dynamic> toJson() => _$ResolveTaskToJson(this);
}
ui/flutter/lib/api/model/resolve_task.g.dart
+30
View File
@@ -0,0 +1,30 @@
// GENERATED CODE - DO NOT MODIFY BY HAND
part of 'resolve_task.dart';
// **************************************************************************
// JsonSerializableGenerator
// **************************************************************************
ResolveTask _$ResolveTaskFromJson(Map<String, dynamic> json) => ResolveTask(
req: json['req'] == null
? null
: Request.fromJson(json['req'] as Map<String, dynamic>),
opts: json['opts'] == null
? null
: Options.fromJson(json['opts'] as Map<String, dynamic>),
);
Map<String, dynamic> _$ResolveTaskToJson(ResolveTask instance) {
final val = <String, dynamic>{};
void writeNotNull(String key, dynamic value) {
if (value != null) {
val[key] = value;
}
}
writeNotNull('req', instance.req);
writeNotNull('opts', instance.opts);
return val;
}
ui/flutter/lib/app/modules/create/views/create_view.dart
+29 -23
View File
@@ -16,6 +16,7 @@ import '../../../../api/model/downloader_config.dart';
import '../../../../api/model/options.dart';
import '../../../../api/model/request.dart';
import '../../../../api/model/resolve_result.dart';
import '../../../../api/model/resolve_task.dart';
import '../../../../api/model/task.dart';
import '../../../../database/database.dart';
import '../../../../util/input_formatter.dart';
@@ -125,13 +126,13 @@ class CreateView extends GetView<CreateController> {
}
// handle options
if (routerParams.opt != null) {
_renameController.text = routerParams.opt!.name;
_pathController.text = routerParams.opt!.path;
if (routerParams.opts != null) {
_renameController.text = routerParams.opts!.name;
_pathController.text = routerParams.opts!.path;
final optionsHandlers = {
Protocol.http: () {
final opt = routerParams.opt!;
final opt = routerParams.opts!;
_renameController.text = opt.name;
_pathController.text = opt.path;
if (opt.extra != null) {
@@ -143,7 +144,7 @@ class CreateView extends GetView<CreateController> {
},
Protocol.bt: null,
};
if (routerParams.opt?.extra != null) {
if (routerParams.opts?.extra != null) {
optionsHandlers[protocol]?.call();
}
}
@@ -814,29 +815,34 @@ class CreateView extends GetView<CreateController> {
*/
final isMultiLine = urls.length > 1;
final isDirect = controller.directDownload.value || isMultiLine;
final opt = Options(
name: isMultiLine ? "" : _renameController.text,
path: _pathController.text,
selectFiles: [],
extra: parseReqOptsExtra(),
);
if (isDirect) {
await Future.wait(urls.map((url) {
return createTask(CreateTask(
req: Request(
url: url,
extra: parseReqExtra(url),
proxy: parseProxy(),
skipVerifyCert: _skipVerifyCertController.value,
),
opt: Options(
name: isMultiLine ? "" : _renameController.text,
path: _pathController.text,
selectFiles: [],
extra: parseReqOptsExtra(),
)));
req: Request(
url: url,
extra: parseReqExtra(url),
proxy: parseProxy(),
skipVerifyCert: _skipVerifyCertController.value,
),
opts: opt,
));
}));
Get.rootDelegate.offNamed(Routes.TASK);
} else {
final rr = await resolve(Request(
url: submitUrl,
extra: parseReqExtra(_urlController.text),
proxy: parseProxy(),
skipVerifyCert: _skipVerifyCertController.value,
final rr = await resolve(ResolveTask(
req: Request(
url: submitUrl,
extra: parseReqExtra(_urlController.text),
proxy: parseProxy(),
skipVerifyCert: _skipVerifyCertController.value,
),
opts: opt,
));
await _showResolveDialog(rr);
}
@@ -983,7 +989,7 @@ class CreateView extends GetView<CreateController> {
} else {
await createTask(CreateTask(
rid: rr.id,
opt: Options(
opts: Options(
name: _renameController.text,
path: _pathController.text,
selectFiles: controller.selectedIndexes,