refactor: fethcer interface & download flow (#1229)

This commit is contained in:
Levi authored and GitHub committed 2026-01-08 21:50:47 +08:00
1 parent abd16d5326
commit a9eee26be7
30 files changed
+2961 -1631

No files matched your search

+3 -2
View File
@@ -2,13 +2,14 @@ package main
import ( import (
"fmt" "fmt"
"strings"
"sync"
"github.com/GopeedLab/gopeed/internal/fetcher" "github.com/GopeedLab/gopeed/internal/fetcher"
"github.com/GopeedLab/gopeed/pkg/base" "github.com/GopeedLab/gopeed/pkg/base"
"github.com/GopeedLab/gopeed/pkg/download" "github.com/GopeedLab/gopeed/pkg/download"
"github.com/GopeedLab/gopeed/pkg/protocol/http" "github.com/GopeedLab/gopeed/pkg/protocol/http"
"github.com/GopeedLab/gopeed/pkg/util" "github.com/GopeedLab/gopeed/pkg/util"
"strings"
"sync"
) )
const progressWidth = 20 const progressWidth = 20
+4 -6
View File
@@ -1,20 +1,18 @@
package fetcher package fetcher
import ( import (
"github.com/GopeedLab/gopeed/internal/controller"
"github.com/GopeedLab/gopeed/pkg/base"
"path" "path"
"strings" "strings"
"github.com/GopeedLab/gopeed/internal/controller"
"github.com/GopeedLab/gopeed/pkg/base"
) )
// Fetcher defines the interface for a download protocol. // Fetcher defines the interface for a download protocol.
// Each download task will have a corresponding Fetcher instance for the management of the download task // Each download task will have a corresponding Fetcher instance for the management of the download task
type Fetcher interface { type Fetcher interface {
Setup(ctl *controller.Controller) Setup(ctl *controller.Controller)
// Resolve resource info from request Resolve(req *base.Request, opts *base.Options) error
Resolve(req *base.Request) error
// Create ready to download, but not started
Create(opts *base.Options) error
Start() error Start() error
Pause() error Pause() error
Close() error Close() error
+28 -30
View File
@@ -20,15 +20,15 @@ import (
"github.com/GopeedLab/gopeed/pkg/util" "github.com/GopeedLab/gopeed/pkg/util"
"github.com/anacrolix/torrent" "github.com/anacrolix/torrent"
"github.com/anacrolix/torrent/metainfo" "github.com/anacrolix/torrent/metainfo"
"github.com/anacrolix/torrent/storage"
) )
var ( var (
client *torrent.Client cfg *torrent.ClientConfig
lock sync.Mutex client *torrent.Client
torrentDirMap = make(map[string]string) lock sync.Mutex
ftMap = make(map[string]*fileTorrentImpl) closeCtx context.Context
closeCtx context.Context closeFunc func()
closeFunc func()
) )
type Fetcher struct { type Fetcher struct {
@@ -71,21 +71,12 @@ func (f *Fetcher) initClient() (err error) {
closeCtx, closeFunc = context.WithCancel(context.Background()) closeCtx, closeFunc = context.WithCancel(context.Background())
} }
cfg := torrent.NewDefaultClientConfig() cfg = torrent.NewDefaultClientConfig()
cfg.Seed = true cfg.Seed = true
cfg.Bep20 = fmt.Sprintf("-GP%s-", parseBep20()) cfg.Bep20 = fmt.Sprintf("-GP%s-", parseBep20())
cfg.ExtendedHandshakeClientVersion = fmt.Sprintf("Gopeed %s", base.Version) cfg.ExtendedHandshakeClientVersion = fmt.Sprintf("Gopeed %s", base.Version)
cfg.ListenPort = f.config.ListenPort cfg.ListenPort = f.config.ListenPort
cfg.HTTPProxy = f.ctl.GetProxy(f.meta.Req.Proxy) cfg.HTTPProxy = f.ctl.GetProxy(f.meta.Req.Proxy)
cfg.DefaultStorage = newFileOpts(newFileClientOpts{
ClientBaseDir: cfg.DataDir,
HandleFileTorrent: func(infoHash metainfo.Hash, ft *fileTorrentImpl) {
if dir, ok := torrentDirMap[infoHash.String()]; ok {
ft.setTorrentDir(dir)
}
ftMap[infoHash.String()] = ft
},
})
dnsResolver := &DnsCacheResolver{RefreshTimeout: 5 * time.Minute} dnsResolver := &DnsCacheResolver{RefreshTimeout: 5 * time.Minute}
cfg.TrackerDialContext = dnsResolver.DialContext cfg.TrackerDialContext = dnsResolver.DialContext
client, err = torrent.NewClient(cfg) client, err = torrent.NewClient(cfg)
@@ -100,8 +91,12 @@ func (f *Fetcher) initClient() (err error) {
return return
} }
func (f *Fetcher) Resolve(req *base.Request) error { func (f *Fetcher) Resolve(req *base.Request, opts *base.Options) error {
f.meta.Req = req f.meta.Req = req
f.meta.Opts = opts
if f.meta.Opts == nil {
f.meta.Opts = &base.Options{}
}
if err := f.addTorrent(req, false); err != nil { if err := f.addTorrent(req, false); err != nil {
return err return err
} }
@@ -109,23 +104,12 @@ func (f *Fetcher) Resolve(req *base.Request) error {
return nil return nil
} }
func (f *Fetcher) Create(opts *base.Options) (err error) {
f.meta.Opts = opts
if f.meta.Res != nil {
torrentDirMap[f.meta.Res.Hash] = opts.Path
}
return nil
}
func (f *Fetcher) Start() (err error) { func (f *Fetcher) Start() (err error) {
if !f.torrentReady.Load() { if !f.torrentReady.Load() {
if err = f.addTorrent(f.meta.Req, false); err != nil { if err = f.addTorrent(f.meta.Req, false); err != nil {
return return
} }
} }
if ft, ok := ftMap[f.meta.Res.Hash]; ok {
ft.setTorrentDir(f.meta.Opts.Path)
}
files := f.torrent.Files() files := f.torrent.Files()
// If the user does not specify the file to download, all files will be downloaded by default // If the user does not specify the file to download, all files will be downloaded by default
if f.data.Progress == nil { if f.data.Progress == nil {
@@ -361,8 +345,12 @@ func (f *Fetcher) addTorrent(req *base.Request, fromUpload bool) (err error) {
} }
schema := util.ParseSchema(req.URL) schema := util.ParseSchema(req.URL)
privateTorrent := false privateTorrent := false
var spec *torrent.TorrentSpec
if schema == "MAGNET" { if schema == "MAGNET" {
f.torrent, err = client.AddMagnet(req.URL) spec, err = torrent.TorrentSpecFromMagnetUri(req.URL)
if err != nil {
return
}
} else { } else {
var reader io.Reader var reader io.Reader
if schema == "FILE" { if schema == "FILE" {
@@ -399,8 +387,18 @@ func (f *Fetcher) addTorrent(req *base.Request, fromUpload bool) (err error) {
if info.Private != nil && *info.Private { if info.Private != nil && *info.Private {
privateTorrent = true privateTorrent = true
} }
f.torrent, err = client.AddTorrent(metaInfo) spec, err = torrent.TorrentSpecFromMetaInfoErr(metaInfo)
if err != nil {
return
}
} }
spec.Storage = storage.NewFileOpts(storage.NewFileClientOpts{
ClientBaseDir: cfg.DataDir,
TorrentDirMaker: func(baseDir string, info *metainfo.Info, infoHash metainfo.Hash) string {
return f.meta.Opts.Path
},
})
f.torrent, _, err = client.AddTorrentSpec(spec)
if err != nil { if err != nil {
return return
} }
+5 -5
View File
@@ -31,7 +31,7 @@ func TestFetcher_Resolve_DataUri_Torrent(t *testing.T) {
dataUri := "data:application/x-bittorrent;base64," + base64.StdEncoding.EncodeToString(buf) dataUri := "data:application/x-bittorrent;base64," + base64.StdEncoding.EncodeToString(buf)
err = fetcher.Resolve(&base.Request{ err = fetcher.Resolve(&base.Request{
URL: dataUri, URL: dataUri,
}) }, nil)
if err != nil { if err != nil {
panic(err) panic(err)
} }
@@ -81,7 +81,7 @@ func doResolve(t *testing.T, fetcher fetcher.Fetcher) {
"udp://tracker.bitsearch.to:1337/announce", "udp://tracker.bitsearch.to:1337/announce",
}, },
}, },
}) }, nil)
if err != nil { if err != nil {
panic(err) panic(err)
} }
@@ -111,7 +111,7 @@ func doResolve(t *testing.T, fetcher fetcher.Fetcher) {
"udp://tracker.bitsearch.to:1337/announce", "udp://tracker.bitsearch.to:1337/announce",
}, },
}, },
}) }, nil)
if err != nil { if err != nil {
panic(err) panic(err)
} }
@@ -145,7 +145,7 @@ func doResolve(t *testing.T, fetcher fetcher.Fetcher) {
t.Run("Resolve Unclean Torrent", func(t *testing.T) { t.Run("Resolve Unclean Torrent", func(t *testing.T) {
err := fetcher.Resolve(&base.Request{ err := fetcher.Resolve(&base.Request{
URL: "./testdata/test.unclean.torrent", URL: "./testdata/test.unclean.torrent",
}) }, nil)
if err != nil { if err != nil {
t.Errorf("Resolve Unclean Torrent Resolve() got = %v, want nil", err) t.Errorf("Resolve Unclean Torrent Resolve() got = %v, want nil", err)
} }
@@ -156,7 +156,7 @@ func doResolve(t *testing.T, fetcher fetcher.Fetcher) {
uri := "file:///" + file uri := "file:///" + file
err := fetcher.Resolve(&base.Request{ err := fetcher.Resolve(&base.Request{
URL: uri, URL: uri,
}) }, nil)
if err != nil { if err != nil {
t.Errorf("Resolve file scheme Torrent Resolve() got = %v, want nil", err) t.Errorf("Resolve file scheme Torrent Resolve() got = %v, want nil", err)
} }
-216
View File
@@ -1,216 +0,0 @@
// modify from github.com/anacrolix/torrent/storage/file.go
package bt
import (
"context"
"fmt"
"io"
"os"
"path/filepath"
"github.com/anacrolix/torrent/storage"
"github.com/anacrolix/missinggo/v2"
"github.com/anacrolix/torrent/common"
"github.com/anacrolix/torrent/metainfo"
"github.com/anacrolix/torrent/segments"
)
// File-based storage for torrents, that isn't yet bound to a particular torrent.
type fileClientImpl struct {
opts newFileClientOpts
}
type newFileClientOpts struct {
// The base directory for all downloads.
ClientBaseDir string
HandleFileTorrent func(infoHash metainfo.Hash, ft *fileTorrentImpl)
FilePathMaker storage.FilePathMaker
PieceCompletion storage.PieceCompletion
}
func newFileOpts(opts newFileClientOpts) storage.ClientImplCloser {
if opts.FilePathMaker == nil {
opts.FilePathMaker = func(opts storage.FilePathMakerOpts) string {
var parts []string
if opts.Info.Length == 0 || opts.File.Path == nil {
parts = append(parts, opts.Info.Name)
}
return filepath.Join(append(parts, opts.File.Path...)...)
}
}
if opts.PieceCompletion == nil {
ret, err := storage.NewDefaultPieceCompletionForDir(opts.ClientBaseDir)
if err != nil {
ret = storage.NewMapPieceCompletion()
}
opts.PieceCompletion = ret
}
return fileClientImpl{opts}
}
func (fs fileClientImpl) Close() error {
return fs.opts.PieceCompletion.Close()
}
func (fs fileClientImpl) OpenTorrent(ctx context.Context, info *metainfo.Info, infoHash metainfo.Hash) (_ storage.TorrentImpl, err error) {
upvertedFiles := info.UpvertedFiles()
files := make([]file, 0, len(upvertedFiles))
for _, fileInfo := range upvertedFiles {
filePath := filepath.Join("", fs.opts.FilePathMaker(storage.FilePathMakerOpts{
Info: info,
File: &fileInfo,
}))
f := file{
rawPath: filePath,
path: filePath,
length: fileInfo.Length,
}
if f.length == 0 {
err = CreateNativeZeroLengthFile(f.path)
if err != nil {
err = fmt.Errorf("creating zero length file: %w", err)
return
}
}
files = append(files, f)
}
t := &fileTorrentImpl{
files,
segments.NewIndex(common.LengthIterFromUpvertedFiles(upvertedFiles)),
infoHash,
fs.opts.PieceCompletion,
}
fs.opts.HandleFileTorrent(infoHash, t)
return storage.TorrentImpl{
Piece: t.Piece,
Close: t.Close,
}, nil
}
type file struct {
// The safe, OS-local file path.
rawPath string
path string
length int64
}
type fileTorrentImpl struct {
files []file
segmentLocater segments.Index
infoHash metainfo.Hash
completion storage.PieceCompletion
}
func (fts *fileTorrentImpl) Piece(p metainfo.Piece) storage.PieceImpl {
// Create a view onto the file-based torrent storage.
_io := fileTorrentImplIO{fts}
// Return the appropriate segments of this.
return &filePieceImpl{
fts,
p,
missinggo.NewSectionWriter(_io, p.Offset(), p.Length()),
io.NewSectionReader(_io, p.Offset(), p.Length()),
}
}
func (fts *fileTorrentImpl) Close() error {
return nil
}
func (fts *fileTorrentImpl) setTorrentDir(dir string) {
for i, f := range fts.files {
fts.files[i].path = filepath.Join(dir, f.rawPath)
}
}
// A helper to create zero-length files which won't appear for file-orientated storage since no
// writes will ever occur to them (no torrent data is associated with a zero-length file). The
// caller should make sure the file name provided is safe/sanitized.
func CreateNativeZeroLengthFile(name string) error {
os.MkdirAll(filepath.Dir(name), 0o777)
var f io.Closer
f, err := os.Create(name)
if err != nil {
return err
}
return f.Close()
}
// Exposes file-based storage of a torrent, as one big ReadWriterAt.
type fileTorrentImplIO struct {
fts *fileTorrentImpl
}
// Returns EOF on short or missing file.
func (fst *fileTorrentImplIO) readFileAt(file file, b []byte, off int64) (n int, err error) {
f, err := os.Open(file.path)
if os.IsNotExist(err) {
// File missing is treated the same as a short file.
err = io.EOF
return
}
if err != nil {
return
}
defer f.Close()
// Limit the read to within the expected bounds of this file.
if int64(len(b)) > file.length-off {
b = b[:file.length-off]
}
for off < file.length && len(b) != 0 {
n1, err1 := f.ReadAt(b, off)
b = b[n1:]
n += n1
off += int64(n1)
if n1 == 0 {
err = err1
break
}
}
return
}
// Only returns EOF at the end of the torrent. Premature EOF is ErrUnexpectedEOF.
func (fst fileTorrentImplIO) ReadAt(b []byte, off int64) (n int, err error) {
fst.fts.segmentLocater.Locate(segments.Extent{off, int64(len(b))}, func(i int, e segments.Extent) bool {
n1, err1 := fst.readFileAt(fst.fts.files[i], b[:e.Length], e.Start)
n += n1
b = b[n1:]
err = err1
return err == nil // && int64(n1) == e.Length
})
if len(b) != 0 && err == nil {
err = io.EOF
}
return
}
func (fst fileTorrentImplIO) WriteAt(p []byte, off int64) (n int, err error) {
// log.Printf("write at %v: %v bytes", off, len(p))
fst.fts.segmentLocater.Locate(segments.Extent{off, int64(len(p))}, func(i int, e segments.Extent) bool {
name := fst.fts.files[i].path
os.MkdirAll(filepath.Dir(name), 0o777)
var f *os.File
f, err = os.OpenFile(name, os.O_WRONLY|os.O_CREATE, 0o666)
if err != nil {
return false
}
var n1 int
n1, err = f.WriteAt(p[:e.Length], e.Start)
// log.Printf("%v %v wrote %v: %v", i, e, n1, err)
closeErr := f.Close()
n += n1
p = p[n1:]
if err == nil {
err = closeErr
}
if err == nil && int64(n1) != e.Length {
err = io.ErrShortWrite
}
return err == nil
})
return
}
-36
View File
@@ -1,36 +0,0 @@
// modify from github.com/anacrolix/torrent/storage/file_misc.go
package bt
import "github.com/anacrolix/torrent/metainfo"
type requiredLength struct {
fileIndex int
length int64
}
func extentCompleteRequiredLengths(info *metainfo.Info, off, n int64) (ret []requiredLength) {
if n == 0 {
return
}
for i, fi := range info.UpvertedFiles() {
if off >= fi.Length {
off -= fi.Length
continue
}
n1 := n
if off+n1 > fi.Length {
n1 = fi.Length - off
}
ret = append(ret, requiredLength{
fileIndex: i,
length: off + n1,
})
n -= n1
if n == 0 {
return
}
off = 0
}
return
}
-61
View File
@@ -1,61 +0,0 @@
// modify from github.com/anacrolix/torrent/storage/file_piece.go
package bt
import (
"github.com/anacrolix/torrent/metainfo"
"github.com/anacrolix/torrent/storage"
"io"
"log"
"os"
)
type filePieceImpl struct {
*fileTorrentImpl
p metainfo.Piece
io.WriterAt
io.ReaderAt
}
var _ storage.PieceImpl = (*filePieceImpl)(nil)
func (fs *filePieceImpl) pieceKey() metainfo.PieceKey {
return metainfo.PieceKey{InfoHash: fs.infoHash, Index: fs.p.Index()}
}
func (fs *filePieceImpl) Completion() storage.Completion {
c, err := fs.completion.Get(fs.pieceKey())
if err != nil {
log.Printf("error getting piece completion: %s", err)
c.Ok = false
return c
}
verified := true
if c.Complete {
// If it's allegedly complete, check that its constituent files have the necessary length.
for _, fi := range extentCompleteRequiredLengths(fs.p.Info, fs.p.Offset(), fs.p.Length()) {
s, err := os.Stat(fs.files[fi.fileIndex].path)
if err != nil || s.Size() < fi.length {
verified = false
break
}
}
}
if !verified {
// The completion was wrong, fix it.
c.Complete = false
fs.completion.Set(fs.pieceKey(), false)
}
return c
}
func (fs *filePieceImpl) MarkComplete() error {
return fs.completion.Set(fs.pieceKey(), true)
}
func (fs *filePieceImpl) MarkNotComplete() error {
return fs.completion.Set(fs.pieceKey(), false)
}
File diff suppressed because it is too large. Load diff
+96
View File
@@ -0,0 +1,96 @@
package http
import (
"net/url"
"path"
"github.com/GopeedLab/gopeed/internal/fetcher"
"github.com/GopeedLab/gopeed/pkg/base"
fhttp "github.com/GopeedLab/gopeed/pkg/protocol/http"
)
// ============================================================================
// Fetcher Data (for Store/Restore)
// ============================================================================
type fetcherData struct {
Connections []*connection
}
// ============================================================================
// Fetcher Manager
// ============================================================================
type FetcherManager struct {
}
func (fm *FetcherManager) Name() string {
return "http"
}
func (fm *FetcherManager) Filters() []*fetcher.SchemeFilter {
return []*fetcher.SchemeFilter{
{
Type: fetcher.FilterTypeUrl,
Pattern: "HTTP",
},
{
Type: fetcher.FilterTypeUrl,
Pattern: "HTTPS",
},
}
}
func (fm *FetcherManager) Build() fetcher.Fetcher {
return &Fetcher{}
}
func (fm *FetcherManager) ParseName(u string) string {
var name string
url, err := url.Parse(u)
if err != nil {
return ""
}
name = path.Base(url.Path)
if name == "" || name == "/" || name == "." {
name = url.Hostname()
}
return name
}
func (fm *FetcherManager) AutoRename() bool {
return true
}
func (fm *FetcherManager) DefaultConfig() any {
return &config{
UserAgent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36",
Connections: 16,
}
}
func (fm *FetcherManager) Store(f fetcher.Fetcher) (data any, err error) {
_f := f.(*Fetcher)
return &fetcherData{
Connections: _f.connections,
}, nil
}
func (fm *FetcherManager) Restore() (v any, f func(meta *fetcher.FetcherMeta, v any) fetcher.Fetcher) {
return &fetcherData{}, func(meta *fetcher.FetcherMeta, v any) fetcher.Fetcher {
fd := v.(*fetcherData)
fb := &FetcherManager{}
fetcher := fb.Build().(*Fetcher)
fetcher.meta = meta
base.ParseReqExtra[fhttp.ReqExtra](fetcher.meta.Req)
base.ParseOptExtra[fhttp.OptsExtra](fetcher.meta.Opts)
if len(fd.Connections) > 0 {
fetcher.connections = fd.Connections
}
return fetcher
}
}
func (fm *FetcherManager) Close() error {
return nil
}
File diff suppressed because it is too large. Load diff
+378
View File
@@ -0,0 +1,378 @@
package http
import (
"bytes"
"context"
"crypto/tls"
"errors"
"fmt"
"io"
"mime"
"net"
"net/http"
"net/http/cookiejar"
"net/url"
"strings"
"unicode/utf8"
"github.com/GopeedLab/gopeed/pkg/base"
fhttp "github.com/GopeedLab/gopeed/pkg/protocol/http"
"github.com/GopeedLab/gopeed/pkg/util"
"golang.org/x/text/encoding/simplifiedchinese"
)
type RequestError struct {
Code int
}
func NewRequestError(code int) *RequestError {
return &RequestError{Code: code}
}
func (re *RequestError) Error() string {
return fmt.Sprintf("http request fail, code:%d", re.Code)
}
func isFailureExemptHTTPCode(code int) bool {
if code >= 500 && code <= 599 {
return true
}
switch code {
case 429, 408, 440, 499:
return true
default:
return false
}
}
func shouldCountHTTPFailure(err error) bool {
var re *RequestError
if !errors.As(err, &re) {
return false
}
return !isFailureExemptHTTPCode(re.Code)
}
func extractRequestError(err error) *RequestError {
var re *RequestError
if errors.As(err, &re) {
return re
}
return nil
}
// ============================================================================
// HTTP Request/Client Building
// ============================================================================
func (f *Fetcher) buildRequest(ctx context.Context, req *base.Request) (httpReq *http.Request, err error) {
var reqUrl string
f.redirectLock.Lock()
if f.redirectURL != "" {
reqUrl = f.redirectURL
} else {
reqUrl = req.URL
}
f.redirectLock.Unlock()
var (
method string
body io.Reader
)
headers := http.Header{}
if req.Extra == nil {
method = http.MethodGet
} else {
extra := req.Extra.(*fhttp.ReqExtra)
if extra.Method != "" {
method = extra.Method
} else {
method = http.MethodGet
}
if len(extra.Header) > 0 {
for k, v := range extra.Header {
headers.Set(k, strings.TrimSpace(v))
}
}
if extra.Body != "" {
body = bytes.NewBufferString(extra.Body)
}
}
if _, ok := headers[base.HttpHeaderUserAgent]; !ok {
headers.Set(base.HttpHeaderUserAgent, strings.TrimSpace(f.config.UserAgent))
}
if ctx != nil {
httpReq, err = http.NewRequestWithContext(ctx, method, reqUrl, body)
} else {
httpReq, err = http.NewRequest(method, reqUrl, body)
}
if err != nil {
return
}
httpReq.Header = headers
if host := headers.Get(base.HttpHeaderHost); host != "" {
httpReq.Host = host
}
return httpReq, nil
}
func (f *Fetcher) buildClient() *http.Client {
transport := &http.Transport{
DialContext: (&net.Dialer{
Timeout: connectTimeout,
}).DialContext,
Proxy: f.ctl.GetProxy(f.meta.Req.Proxy),
TLSClientConfig: &tls.Config{
InsecureSkipVerify: f.meta.Req.SkipVerifyCert,
},
}
jar, _ := cookiejar.New(nil)
return &http.Client{
Transport: transport,
Jar: jar,
}
}
// ============================================================================
// Filename Parsing
// ============================================================================
// parseFilename extracts filename from Content-Disposition header
func parseFilename(contentDisposition string) string {
// Try RFC 5987 extended notation first (filename*=)
if filename := parseFilenameExtended(contentDisposition); filename != "" {
return filename
}
// Try standard MIME parsing
_, params, err := mime.ParseMediaType(contentDisposition)
if err == nil {
if filename := params["filename"]; filename != "" {
return decodeFilenameParam(filename)
}
}
// Fallback to manual parsing
return parseFilenameFallback(contentDisposition)
}
// parseFilenameExtended handles RFC 5987 extended notation (filename*=)
// Format: filename*=charset'language'value (e.g., UTF-8”%E6%B5%8B%E8%AF%95.zip)
func parseFilenameExtended(cd string) string {
lower := strings.ToLower(cd)
idx := strings.Index(lower, "filename*=")
if idx == -1 {
return ""
}
value := cd[idx+len("filename*="):]
// Find the end of the value using proper quote handling
endIdx := findParamValueEnd(value)
if endIdx != -1 {
value = value[:endIdx]
}
value = strings.TrimSpace(value)
// Try charset''encoded format (e.g., UTF-8''%E4%B8%AD%E6%96%87.txt)
parts := strings.SplitN(value, "''", 2)
if len(parts) == 2 {
// Use PathUnescape to handle %2B correctly (should decode to +, not space)
decoded, err := url.PathUnescape(parts[1])
if err == nil {
return decoded
}
}
// Try charset'language'encoded format
parts = strings.SplitN(value, "'", 3)
if len(parts) >= 3 {
// Use PathUnescape to handle %2B correctly (should decode to +, not space)
decoded, err := url.PathUnescape(parts[2])
if err == nil {
return decoded
}
}
return ""
}
// decodeFilenameParam decodes filename parameter value
// Handles HTML entities, MIME encoded-word, URL encoding, and GBK encoding fallback
func decodeFilenameParam(filename string) string {
// First, unescape HTML entities (e.g., &amp; -> &, &lt; -> <, &gt; -> >)
// This must be done before other decoding to handle cases where servers
// HTML-encode special characters in filenames
filename = unescapeHTMLEntities(filename)
// Handle RFC 2047 encoded word (=?charset?encoding?text?=)
if strings.HasPrefix(filename, "=?") {
decoder := new(mime.WordDecoder)
normalizedFilename := strings.Replace(filename, "UTF8", "UTF-8", 1)
if decoded, err := decoder.Decode(normalizedFilename); err == nil {
return decoded
}
}
// Try URL decoding - use PathUnescape for filenames to handle %2B correctly
decoded := util.TryUrlPathUnescape(filename)
// If not valid UTF-8, try GBK decoding (common for Chinese websites)
if !utf8.ValidString(decoded) {
if gbkDecoded := tryDecodeGBK(decoded); gbkDecoded != "" {
return gbkDecoded
}
}
return decoded
}
// unescapeHTMLEntities unescapes common HTML entities in filenames
// This handles cases where servers HTML-encode special characters like & to &amp;
func unescapeHTMLEntities(s string) string {
// Common HTML entities that might appear in filenames
replacements := map[string]string{
"&amp;": "&",
"&lt;": "<",
"&gt;": ">",
"&quot;": "\"",
"&#39;": "'",
"&apos;": "'",
}
result := s
for entity, char := range replacements {
result = strings.ReplaceAll(result, entity, char)
}
return result
}
// tryDecodeGBK attempts to decode string as GBK encoding
func tryDecodeGBK(s string) string {
if len(s) == 0 {
return ""
}
decoder := simplifiedchinese.GBK.NewDecoder()
decoded, err := decoder.Bytes([]byte(s))
if err != nil {
return ""
}
result := string(decoded)
if utf8.ValidString(result) {
return result
}
return ""
}
// parseFilenameFallback is a fallback parser for non-standard Content-Disposition
func parseFilenameFallback(cd string) string {
lower := strings.ToLower(cd)
idx := strings.Index(lower, "filename=")
if idx == -1 {
return ""
}
value := cd[idx+len("filename="):]
// Find the end of the value using proper quote handling
endIdx := findParamValueEnd(value)
if endIdx != -1 {
value = value[:endIdx]
}
value = strings.TrimSpace(value)
// Remove surrounding quotes
if len(value) >= 2 {
if (value[0] == '"' && value[len(value)-1] == '"') ||
(value[0] == '\'' && value[len(value)-1] == '\'') {
value = value[1 : len(value)-1]
}
}
return decodeFilenameParam(value)
}
// findParamValueEnd finds the end position of a parameter value in a Content-Disposition header.
// It correctly handles quoted values where semicolons inside quotes should not be treated as delimiters.
// It also handles HTML entities in unquoted values (e.g., &amp; should not be split at the semicolon).
// Returns the end index (exclusive) of the value, or -1 if it extends to the end of the string.
func findParamValueEnd(value string) int {
value = strings.TrimSpace(value)
if len(value) == 0 {
return 0
}
// If the value starts with a quote, find the matching closing quote
if value[0] == '"' || value[0] == '\'' {
quote := value[0]
// Find the closing quote, handling escaped quotes
for i := 1; i < len(value); i++ {
if value[i] == quote {
// Check if it's escaped
if i > 0 && value[i-1] == '\\' {
continue
}
// Found closing quote, now look for ; after it
remaining := value[i+1:]
if semiIdx := strings.Index(remaining, ";"); semiIdx != -1 {
return i + 1 + semiIdx
}
return -1 // No semicolon after closing quote
}
}
// No closing quote found, treat rest of string as value
return -1
}
// Unquoted value - find the next semicolon that's not part of an HTML entity
// HTML entities have the pattern &...; (e.g., &amp; &lt; &gt; &quot; &#39;)
for i := 0; i < len(value); i++ {
if value[i] == ';' {
// Check if this semicolon is part of an HTML entity
// Look backwards for an & character
isEntity := false
if i > 0 {
// Look for & before this semicolon (within reasonable distance, max 10 chars)
for j := i - 1; j >= 0 && j >= i-10; j-- {
if value[j] == '&' {
// Found &, this semicolon might be part of an HTML entity
// Check if there are only alphanumeric or # between & and ;
entityChars := value[j+1 : i]
if len(entityChars) > 0 && isValidHTMLEntityChars(entityChars) {
isEntity = true
}
break
}
// If we hit whitespace or another special char, stop looking
if value[j] == ' ' || value[j] == '"' || value[j] == '\'' {
break
}
}
}
if !isEntity {
return i
}
}
}
return -1 // No semicolon, extends to end
}
// isValidHTMLEntityChars checks if a string contains only valid HTML entity characters
// (alphanumeric and #, typically for entities like &amp; &lt; &#39; etc.)
func isValidHTMLEntityChars(s string) bool {
if len(s) == 0 {
return false
}
for _, c := range s {
if !((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '#') {
return false
}
}
return true
}
+210 -28
View File
@@ -70,6 +70,9 @@ func StartTestSlowFileServer(delay time.Duration) net.Listener {
func StartTestCustomServer() net.Listener { func StartTestCustomServer() net.Listener {
return startTestServer(func(sl *shutdownListener) http.Handler { return startTestServer(func(sl *shutdownListener) http.Handler {
mux := http.NewServeMux() mux := http.NewServeMux()
mux.HandleFunc("/", func(writer http.ResponseWriter, request *http.Request) {
writer.WriteHeader(200)
})
mux.HandleFunc("/"+BuildName, func(writer http.ResponseWriter, request *http.Request) { mux.HandleFunc("/"+BuildName, func(writer http.ResponseWriter, request *http.Request) {
file, err := os.Open(BuildFile) file, err := os.Open(BuildFile)
if err != nil { if err != nil {
@@ -221,6 +224,10 @@ func StartTestCustomServer() net.Listener {
defer file.Close() defer file.Close()
io.Copy(writer, file) io.Copy(writer, file)
}) })
// Test 403 Forbidden endpoint
mux.HandleFunc("/forbidden", func(writer http.ResponseWriter, request *http.Request) {
writer.WriteHeader(403)
})
return mux return mux
}) })
} }
@@ -244,20 +251,6 @@ func StartTestHostHeaderServer() net.Listener {
}) })
} }
// StartTestRootServer starts a simple server at the root path
// Used to test URL resolution when no filename is provided
func StartTestRootServer() net.Listener {
return startTestServer(func(sl *shutdownListener) http.Handler {
mux := http.NewServeMux()
mux.HandleFunc("/", func(writer http.ResponseWriter, request *http.Request) {
writer.Header().Set("Content-Type", "text/html")
writer.WriteHeader(200)
writer.Write([]byte("<html><body>Test Page</body></html>"))
})
return mux
})
}
func StartTestRetryServer() net.Listener { func StartTestRetryServer() net.Listener {
counter := 0 counter := 0
return startTestServer(func(sl *shutdownListener) http.Handler { return startTestServer(func(sl *shutdownListener) http.Handler {
@@ -303,23 +296,100 @@ func StartTestPostServer() net.Listener {
} }
func StartTestErrorServer() net.Listener { func StartTestErrorServer() net.Listener {
counter := 0
return startTestServer(func(sl *shutdownListener) http.Handler { return startTestServer(func(sl *shutdownListener) http.Handler {
mux := http.NewServeMux() mux := http.NewServeMux()
mux.HandleFunc("/"+BuildName, func(writer http.ResponseWriter, request *http.Request) { mux.HandleFunc("/"+BuildName, func(writer http.ResponseWriter, request *http.Request) {
counter++ // Always return 404 error to test error handling
if counter != 1 { writer.WriteHeader(404)
writer.WriteHeader(500) return
return
}
}) })
return mux return mux
}) })
} }
// StartTestOneTimeServer creates a server where the URL can only be downloaded once
// The first non-probe request succeeds, all subsequent requests return 404
// This simulates one-time download URLs (e.g., signed URLs that expire after first use)
func StartTestOneTimeServer() net.Listener {
var accessed atomic.Bool
return startTestServer(func(sl *shutdownListener) http.Handler {
mux := http.NewServeMux()
mux.HandleFunc("/"+BuildName, func(writer http.ResponseWriter, request *http.Request) {
// First full download request succeeds, subsequent requests fail
if accessed.Swap(true) {
writer.WriteHeader(404)
return
}
// First request - return full file without Range support
writer.Header().Set("Content-Length", fmt.Sprintf("%d", BuildSize))
file, err := os.Open(BuildFile)
if err != nil {
return
}
defer file.Close()
io.Copy(writer, file)
})
return mux
})
}
// StartTestSlowStartServer creates a server with configurable delay per request
// This allows testing slow-start connection expansion to reach max connections
func StartTestSlowStartServer(delayPerByte time.Duration) net.Listener {
return startTestServer(func(sl *shutdownListener) http.Handler {
mux := http.NewServeMux()
mux.HandleFunc("/"+BuildName, func(writer http.ResponseWriter, request *http.Request) {
rangeFileHandle(
writer,
request,
nil,
func(file *os.File, n int64) {
slowCopyNWithDelay(sl, writer, file, n, delayPerByte)
},
)
})
return mux
})
}
// slowCopyNWithDelay copies n bytes from src to dst with a delay per byte
func slowCopyNWithDelay(sl *shutdownListener, dst io.Writer, src io.Reader, n int64, delayPerByte time.Duration) {
buf := make([]byte, 32*1024)
remaining := n
for remaining > 0 {
if sl.isShutdown {
return
}
toRead := int64(len(buf))
if toRead > remaining {
toRead = remaining
}
nr, er := src.Read(buf[:toRead])
if nr > 0 {
nw, ew := dst.Write(buf[0:nr])
if nw > 0 {
remaining -= int64(nw)
// Add delay based on bytes written
if delayPerByte > 0 {
time.Sleep(delayPerByte * time.Duration(nw))
}
}
if ew != nil {
break
}
}
if er != nil {
break
}
}
}
// StartTestLimitServer connections limit server // StartTestLimitServer connections limit server
func StartTestLimitServer(maxConnections int32, delay int64) net.Listener { func StartTestLimitServer(maxConnections int32, delay int64) net.Listener {
var connections atomic.Int32 var connections atomic.Int32
var slowOnce atomic.Bool
return startTestServer(func(sl *shutdownListener) http.Handler { return startTestServer(func(sl *shutdownListener) http.Handler {
mux := http.NewServeMux() mux := http.NewServeMux()
@@ -332,12 +402,93 @@ func StartTestLimitServer(maxConnections int32, delay int64) net.Listener {
writer.WriteHeader(403) writer.WriteHeader(403)
return return
} }
// First request intentionally delays the first write to trigger a single read timeout,
// subsequent requests respond at normal speed.
useInitialDelay := delay > 0 && !slowOnce.Swap(true)
rangeFileHandle( rangeFileHandle(
writer, writer,
request, request,
nil, nil,
func(file *os.File, n int64) { func(file *os.File, n int64) {
slowCopyN(sl, writer, file, n, delay) if useInitialDelay {
slowCopyAfterDelay(sl, writer, file, n, delay)
return
}
slowCopyN(sl, writer, file, n, 0)
},
)
})
return mux
})
}
// StartTestTimeoutOnceServer creates a server that times out on first request, then works normally
func StartTestTimeoutOnceServer(delay int64) net.Listener {
var timeoutOnce atomic.Bool
return startTestServer(func(sl *shutdownListener) http.Handler {
mux := http.NewServeMux()
mux.HandleFunc("/"+BuildName, func(writer http.ResponseWriter, request *http.Request) {
// First request delays to trigger timeout, subsequent requests respond normally
useInitialDelay := delay > 0 && !timeoutOnce.Swap(true)
rangeFileHandle(
writer,
request,
nil,
func(file *os.File, n int64) {
if useInitialDelay {
slowCopyAfterDelay(sl, writer, file, n, delay)
return
}
slowCopyN(sl, writer, file, n, 0)
},
)
})
return mux
})
}
// StartTestTemporary500Server creates a server that returns 500 for a duration, then recovers
// Uses slow transfer to ensure the file isn't fully downloaded during resolve phase
func StartTestTemporary500Server(errorDuration time.Duration) net.Listener {
startTime := time.Now()
var requestCount atomic.Int32
return startTestServer(func(sl *shutdownListener) http.Handler {
mux := http.NewServeMux()
mux.HandleFunc("/"+BuildName, func(writer http.ResponseWriter, request *http.Request) {
reqNum := requestCount.Add(1)
// First request (Resolve) succeeds with slow transfer to prevent full download
// Subsequent requests return 500 for the specified duration, then recover
if reqNum == 1 {
// Slow transfer for resolve: 50 microsecond per byte (~20MB/s)
// This ensures resolve doesn't complete the full 200MB file
rangeFileHandle(
writer,
request,
nil,
func(file *os.File, n int64) {
slowCopyNWithDelay(sl, writer, file, n, 50*time.Microsecond)
},
)
return
}
// Subsequent requests: return 500 for errorDuration, then normal
if time.Since(startTime) < errorDuration {
writer.WriteHeader(500)
writer.Write([]byte("Internal Server Error"))
return
}
rangeFileHandle(
writer,
request,
nil,
func(file *os.File, n int64) {
slowCopyN(sl, writer, file, n, 0)
}, },
) )
}) })
@@ -375,6 +526,26 @@ func StartTestRangeBugServer() net.Listener {
func rangeFileHandle(writer http.ResponseWriter, request *http.Request, modifyEnd func(end int64) int64, iocpN func(file *os.File, n int64)) { func rangeFileHandle(writer http.ResponseWriter, request *http.Request, modifyEnd func(end int64) int64, iocpN func(file *os.File, n int64)) {
r := request.Header.Get("Range") r := request.Header.Get("Range")
// If no Range header, return full file with Accept-Ranges header
if r == "" {
// Open file first to ensure it exists
file, err := os.Open(BuildFile)
if err != nil {
writer.WriteHeader(500)
return
}
defer file.Close()
writer.Header().Set("Content-Length", fmt.Sprintf("%d", BuildSize))
writer.Header().Set("Accept-Ranges", "bytes")
writer.WriteHeader(200)
(writer.(http.Flusher)).Flush()
iocpN(file, BuildSize)
return
}
// split range // split range
s := strings.Split(r, "=") s := strings.Split(r, "=")
if len(s) != 2 { if len(s) != 2 {
@@ -408,12 +579,7 @@ func rangeFileHandle(writer http.ResponseWriter, request *http.Request, modifyEn
end = modifyEnd(end) end = modifyEnd(end)
} }
writer.Header().Set("Content-Length", fmt.Sprintf("%d", end-start+1)) // Open file before sending headers to ensure it exists
writer.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, BuildSize))
writer.Header().Set("Accept-Ranges", "bytes")
writer.WriteHeader(206)
(writer.(http.Flusher)).Flush()
file, err := os.Open(BuildFile) file, err := os.Open(BuildFile)
if err != nil { if err != nil {
writer.WriteHeader(500) writer.WriteHeader(500)
@@ -421,6 +587,13 @@ func rangeFileHandle(writer http.ResponseWriter, request *http.Request, modifyEn
} }
defer file.Close() defer file.Close()
file.Seek(start, 0) file.Seek(start, 0)
writer.Header().Set("Content-Length", fmt.Sprintf("%d", end-start+1))
writer.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, BuildSize))
writer.Header().Set("Accept-Ranges", "bytes")
writer.WriteHeader(206)
(writer.(http.Flusher)).Flush()
iocpN(file, end-start+1) iocpN(file, end-start+1)
} }
@@ -471,6 +644,14 @@ func slowCopyN(sl *shutdownListener, dst io.Writer, src io.Reader, n int64, dela
return return
} }
// slowCopyAfterDelay sleeps once before performing a normal copy to simulate a single timeout.
func slowCopyAfterDelay(sl *shutdownListener, dst io.Writer, src io.Reader, n int64, delay int64) (written int64, err error) {
if delay > 0 {
time.Sleep(time.Millisecond * time.Duration(delay))
}
return slowCopyN(sl, dst, src, n, 0)
}
func startTestServer(serverHandle func(sl *shutdownListener) http.Handler) net.Listener { func startTestServer(serverHandle func(sl *shutdownListener) http.Handler) net.Listener {
listener, err := net.Listen("tcp", "127.0.0.1:0") listener, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil { if err != nil {
@@ -515,8 +696,9 @@ type shutdownListener struct {
} }
func (c *shutdownListener) Close() error { func (c *shutdownListener) Close() error {
c.isShutdown = true // Shutdown server first (waits for in-flight requests), then set isShutdown
closeErr := c.server.Shutdown(context.Background()) closeErr := c.server.Shutdown(context.Background())
c.isShutdown = true
if err := ifExistAndRemove(BuildFile); err != nil { if err := ifExistAndRemove(BuildFile); err != nil {
fmt.Println(err) fmt.Println(err)
} }
+1 -1
View File
@@ -149,7 +149,7 @@ func ParseReqExtra[E any](req *Request) error {
return nil return nil
} }
func ParseOptsExtra[E any](opts *Options) error { func ParseOptExtra[E any](opts *Options) error {
if opts.Extra == nil { if opts.Extra == nil {
return nil return nil
} }
+52 -51
View File
@@ -321,7 +321,7 @@ func (d *Downloader) saveTask(task *Task) error {
return nil return nil
} }
func (d *Downloader) Resolve(req *base.Request) (rr *ResolveResult, err error) { func (d *Downloader) Resolve(req *base.Request, opts *base.Options) (rr *ResolveResult, err error) {
rrId, err := gonanoid.New() rrId, err := gonanoid.New()
if err != nil { if err != nil {
return return
@@ -342,7 +342,11 @@ func (d *Downloader) Resolve(req *base.Request) (rr *ResolveResult, err error) {
if err != nil { if err != nil {
return return
} }
err = fetcher.Resolve(req) initOpt, err := d.initOptions(opts)
if err != nil {
return
}
err = fetcher.Resolve(req, initOpt)
if err != nil { if err != nil {
return return
} }
@@ -390,7 +394,11 @@ func (d *Downloader) CreateDirect(req *base.Request, opts *base.Options) (taskId
return return
} }
fetcher.Meta().Req = req fetcher.Meta().Req = req
return d.doCreate(fetcher, opts) initOpt, err := d.initOptions(opts)
if err != nil {
return
}
return d.doCreate(fetcher, initOpt)
} }
func (d *Downloader) CreateDirectBatch(req *base.CreateTaskBatch) (taskId []string, err error) { func (d *Downloader) CreateDirectBatch(req *base.CreateTaskBatch) (taskId []string, err error) {
@@ -409,7 +417,7 @@ func (d *Downloader) CreateDirectBatch(req *base.CreateTaskBatch) (taskId []stri
return taskIds, nil return taskIds, nil
} }
func (d *Downloader) Create(rrId string, opts *base.Options) (taskId string, err error) { func (d *Downloader) Create(rrId string) (taskId string, err error) {
d.fetcherMapLock.RLock() d.fetcherMapLock.RLock()
fetcher, ok := d.fetcherCache[rrId] fetcher, ok := d.fetcherCache[rrId]
d.fetcherMapLock.RUnlock() d.fetcherMapLock.RUnlock()
@@ -421,7 +429,7 @@ func (d *Downloader) Create(rrId string, opts *base.Options) (taskId string, err
delete(d.fetcherCache, rrId) delete(d.fetcherCache, rrId)
d.fetcherMapLock.Unlock() d.fetcherMapLock.Unlock()
}() }()
return d.doCreate(fetcher, opts) return d.doCreate(fetcher, nil)
} }
func (d *Downloader) Pause(filter *TaskFilter) (err error) { func (d *Downloader) Pause(filter *TaskFilter) (err error) {
@@ -855,8 +863,6 @@ func (d *Downloader) watch(task *Task) {
if err != nil { if err != nil {
d.Logger.Warn().Err(err).Msgf("task wait upload failed, task id: %s", task.ID) d.Logger.Warn().Err(err).Msgf("task wait upload failed, task id: %s", task.ID)
} }
d.lock.Lock()
defer d.lock.Unlock()
// Check if the task is deleted // Check if the task is deleted
if d.GetTask(task.ID) != nil { if d.GetTask(task.ID) != nil {
@@ -947,7 +953,6 @@ func (d *Downloader) restoreTask(task *Task) error {
} }
} }
go d.watch(task) go d.watch(task)
task.fetcher.Create(task.Meta.Opts)
return nil return nil
} }
@@ -970,42 +975,15 @@ func (d *Downloader) restoreFetcher(task *Task) error {
if task.fetcher.Meta().Res == nil { if task.fetcher.Meta().Res == nil {
task.fetcher.Meta().Res = task.Meta.Res task.fetcher.Meta().Res = task.Meta.Res
} }
if task.fetcher.Meta().Opts == nil {
task.fetcher.Meta().Opts = task.Meta.Opts
}
return nil return nil
} }
func (d *Downloader) doCreate(f fetcher.Fetcher, opts *base.Options) (taskId string, err error) { func (d *Downloader) doCreate(f fetcher.Fetcher, opts *base.Options) (taskId string, err error) {
if opts == nil { if f.Meta().Opts == nil {
opts = &base.Options{} f.Meta().Opts = opts
}
if opts.SelectFiles == nil {
opts.SelectFiles = make([]int, 0)
}
meta := f.Meta()
meta.Opts = opts
if opts.Path == "" {
storeConfig, err := d.GetConfig()
if err != nil {
return "", err
}
opts.Path = storeConfig.DownloadDir
}
// Replace placeholders in download path (e.g., %year%, %month%, %day%, %date%)
opts.Path = util.ReplacePathPlaceholders(opts.Path)
// if enable white download directory, check if the download directory is in the white list
if len(d.cfg.WhiteDownloadDirs) > 0 {
inWhiteList := false
for _, dir := range d.cfg.WhiteDownloadDirs {
if match, err := filepath.Match(dir, opts.Path); match && err == nil {
inWhiteList = true
break
}
}
if !inWhiteList {
return "", errors.New("download directory is not in white list")
}
} }
fm, err := d.parseFm(f.Meta().Req.URL) fm, err := d.parseFm(f.Meta().Req.URL)
@@ -1021,9 +999,6 @@ func (d *Downloader) doCreate(f fetcher.Fetcher, opts *base.Options) (taskId str
task.Progress = &Progress{} task.Progress = &Progress{}
_, task.Uploading = f.(fetcher.Uploader) _, task.Uploading = f.(fetcher.Uploader)
initTask(task) initTask(task)
if err = f.Create(opts); err != nil {
return
}
if err = d.storage.Put(bucketTask, task.ID, task.clone()); err != nil { if err = d.storage.Put(bucketTask, task.ID, task.clone()); err != nil {
return return
} }
@@ -1049,6 +1024,39 @@ func (d *Downloader) doCreate(f fetcher.Fetcher, opts *base.Options) (taskId str
return return
} }
func (d *Downloader) initOptions(opts *base.Options) (*base.Options, error) {
if opts == nil {
opts = &base.Options{}
}
if opts.SelectFiles == nil {
opts.SelectFiles = make([]int, 0)
}
if opts.Path == "" {
storeConfig, err := d.GetConfig()
if err != nil {
return nil, err
}
opts.Path = storeConfig.DownloadDir
}
// Replace placeholders in download path (e.g., %year%, %month%, %day%, %date%)
opts.Path = util.ReplacePathPlaceholders(opts.Path)
// if enable white download directory, check if the download directory is in the white list
if len(d.cfg.WhiteDownloadDirs) > 0 {
inWhiteList := false
for _, dir := range d.cfg.WhiteDownloadDirs {
if match, err := filepath.Match(dir, opts.Path); match && err == nil {
inWhiteList = true
break
}
}
if !inWhiteList {
return nil, errors.New("download directory is not in white list")
}
}
return opts, nil
}
func (d *Downloader) statusMut(task *Task, fn func() (bool, error)) (bool, error) { func (d *Downloader) statusMut(task *Task, fn func() (bool, error)) (bool, error) {
task.statusLock.Lock() task.statusLock.Lock()
defer task.statusLock.Unlock() defer task.statusLock.Unlock()
@@ -1088,7 +1096,7 @@ func (d *Downloader) doStart(task *Task) (err error) {
d.triggerOnStart(task) d.triggerOnStart(task)
if task.Meta.Res == nil { if task.Meta.Res == nil {
err := task.fetcher.Resolve(task.Meta.Req) err := task.fetcher.Resolve(task.Meta.Req, task.Meta.Opts)
if err != nil { if err != nil {
return err return err
} }
@@ -1621,13 +1629,6 @@ func (b *boot) Extra(extra interface{}) *boot {
return b return b
} }
func (b *boot) Resolve() (*ResolveResult, error) {
return defaultDownloader.Resolve(&base.Request{
URL: b.url,
Extra: b.extra,
})
}
func (b *boot) Listener(listener Listener) *boot { func (b *boot) Listener(listener Listener) *boot {
b.listener = listener b.listener = listener
return b return b
+28 -75
View File
@@ -20,6 +20,14 @@ import (
"github.com/GopeedLab/gopeed/pkg/protocol/http" "github.com/GopeedLab/gopeed/pkg/protocol/http"
) )
var testDownloadOpt = &base.Options{
Path: test.Dir,
Name: test.DownloadName,
Extra: http.OptsExtra{
Connections: 4,
},
}
func TestDownloader_Resolve(t *testing.T) { func TestDownloader_Resolve(t *testing.T) {
listener := test.StartTestFileServer() listener := test.StartTestFileServer()
defer listener.Close() defer listener.Close()
@@ -32,7 +40,7 @@ func TestDownloader_Resolve(t *testing.T) {
req := &base.Request{ req := &base.Request{
URL: "http://" + listener.Addr().String() + "/" + test.BuildName, URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
} }
rr, err := downloader.Resolve(req) rr, err := downloader.Resolve(req, nil)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -64,7 +72,7 @@ func TestDownloader_Create(t *testing.T) {
req := &base.Request{ req := &base.Request{
URL: "http://" + listener.Addr().String() + "/" + test.BuildName, URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
} }
rr, err := downloader.Resolve(req) rr, err := downloader.Resolve(req, testDownloadOpt)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -76,13 +84,7 @@ func TestDownloader_Create(t *testing.T) {
wg.Done() wg.Done()
} }
}) })
_, err = downloader.Create(rr.ID, &base.Options{ _, err = downloader.Create(rr.ID)
Path: test.Dir,
Name: test.DownloadName,
Extra: http.OptsExtra{
Connections: 4,
},
})
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -108,27 +110,14 @@ func TestDownloader_CreateNotInWhite(t *testing.T) {
req := &base.Request{ req := &base.Request{
URL: "http://" + listener.Addr().String() + "/" + test.BuildName, URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
} }
rr, err := downloader.Resolve(req) // With new fetcher design, white list check happens during Resolve (not Create)
if err != nil { // because Resolve now requires Options which includes the download path
t.Fatal(err) _, err := downloader.Resolve(req, testDownloadOpt)
if err == nil {
t.Error("TestDownloader_CreateNotInWhite() expected error but got nil")
} }
var wg sync.WaitGroup
wg.Add(1)
downloader.Listener(func(event *Event) {
if event.Key == EventKeyDone {
wg.Done()
}
})
_, err = downloader.Create(rr.ID, &base.Options{
Path: test.Dir,
Name: test.DownloadName,
Extra: http.OptsExtra{
Connections: 4,
},
})
if !strings.Contains(err.Error(), "white") { if !strings.Contains(err.Error(), "white") {
t.Errorf("TestDownloader_CreateNotInWhite() got = %v, want %v", err.Error(), "not in white list") t.Errorf("TestDownloader_CreateNotInWhite() got = %v, want error containing 'white'", err.Error())
} }
} }
@@ -172,13 +161,7 @@ func TestDownloader_CreateDirectBatch(t *testing.T) {
_, err := downloader.CreateDirectBatch(&base.CreateTaskBatch{ _, err := downloader.CreateDirectBatch(&base.CreateTaskBatch{
Reqs: reqs, Reqs: reqs,
Opts: &base.Options{ Opts: testDownloadOpt,
Path: test.Dir,
Name: test.DownloadName,
Extra: http.OptsExtra{
Connections: 4,
},
},
}) })
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
@@ -317,7 +300,7 @@ func doTestDownloaderCreateWithProxy(t *testing.T, auth bool, buildReqProxy func
Pwd: pwd, Pwd: pwd,
}) })
} }
rr, err := downloader.Resolve(req) rr, err := downloader.Resolve(req, nil)
if err != nil { if err != nil {
if errHandler == nil { if errHandler == nil {
t.Fatal(err) t.Fatal(err)
@@ -400,18 +383,12 @@ func TestDownloader_StoreAndRestore(t *testing.T) {
req := &base.Request{ req := &base.Request{
URL: "http://" + listener.Addr().String() + "/" + test.BuildName, URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
} }
rr, err := downloader.Resolve(req) rr, err := downloader.Resolve(req, testDownloadOpt)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
id, err := downloader.Create(rr.ID, &base.Options{ id, err := downloader.Create(rr.ID)
Path: test.Dir,
Name: test.DownloadName,
Extra: http.OptsExtra{
Connections: 4,
},
})
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -675,7 +652,7 @@ func TestDownloader_Stats(t *testing.T) {
req := &base.Request{ req := &base.Request{
URL: "http://" + listener.Addr().String() + "/" + test.BuildName, URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
} }
rr, err := downloader.Resolve(req) rr, err := downloader.Resolve(req, testDownloadOpt)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -688,13 +665,7 @@ func TestDownloader_Stats(t *testing.T) {
} }
}) })
taskId, err := downloader.Create(rr.ID, &base.Options{ taskId, err := downloader.Create(rr.ID)
Path: test.Dir,
Name: test.DownloadName,
Extra: http.OptsExtra{
Connections: 4,
},
})
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -736,13 +707,7 @@ func TestDownloader_Delete(t *testing.T) {
req := &base.Request{ req := &base.Request{
URL: "http://" + listener.Addr().String() + "/" + test.BuildName, URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
} }
taskId, err := downloader.CreateDirect(req, &base.Options{ taskId, err := downloader.CreateDirect(req, testDownloadOpt)
Path: test.Dir,
Name: test.DownloadName,
Extra: http.OptsExtra{
Connections: 4,
},
})
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -810,17 +775,11 @@ func TestDownloader_ContinueBatch(t *testing.T) {
req := &base.Request{ req := &base.Request{
URL: "http://" + listener.Addr().String() + "/" + test.BuildName, URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
} }
rr, err := downloader.Resolve(req) rr, err := downloader.Resolve(req, testDownloadOpt)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
taskId, err := downloader.Create(rr.ID, &base.Options{ taskId, err := downloader.Create(rr.ID)
Path: test.Dir,
Name: test.DownloadName,
Extra: http.OptsExtra{
Connections: 4,
},
})
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -905,18 +864,12 @@ func TestDownloader_PauseAndContinue(t *testing.T) {
req := &base.Request{ req := &base.Request{
URL: "http://" + listener.Addr().String() + "/" + test.BuildName, URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
} }
rr, err := downloader.Resolve(req) rr, err := downloader.Resolve(req, testDownloadOpt)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
taskId, err := downloader.Create(rr.ID, &base.Options{ taskId, err := downloader.Create(rr.ID)
Path: test.Dir,
Name: test.DownloadName,
Extra: http.OptsExtra{
Connections: 4,
},
})
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
+6 -6
View File
@@ -217,7 +217,7 @@
"@webassemblyjs/helper-wasm-bytecode": "1.11.1", "@webassemblyjs/helper-wasm-bytecode": "1.11.1",
"@webassemblyjs/helper-wasm-section": "1.11.1", "@webassemblyjs/helper-wasm-section": "1.11.1",
"@webassemblyjs/wasm-gen": "1.11.1", "@webassemblyjs/wasm-gen": "1.11.1",
"@webassemblyjs/wasm-opt": "1.11.1", "@webassemblyjs/wasm-opts": "1.11.1",
"@webassemblyjs/wasm-parser": "1.11.1", "@webassemblyjs/wasm-parser": "1.11.1",
"@webassemblyjs/wast-printer": "1.11.1" "@webassemblyjs/wast-printer": "1.11.1"
} }
@@ -235,9 +235,9 @@
"@webassemblyjs/utf8": "1.11.1" "@webassemblyjs/utf8": "1.11.1"
} }
}, },
"node_modules/@webassemblyjs/wasm-opt": { "node_modules/@webassemblyjs/wasm-opts": {
"version": "1.11.1", "version": "1.11.1",
"resolved": "https://registry.npmmirror.com/@webassemblyjs/wasm-opt/-/wasm-opt-1.11.1.tgz", "resolved": "https://registry.npmmirror.com/@webassemblyjs/wasm-opts/-/wasm-opts-1.11.1.tgz",
"integrity": "sha512-VqnkNqnZlU5EB64pp1l7hdm3hmQw7Vgqa0KF/KCNO9sIpI6Fk6brDEiX+iCOYrvMuBWDws0NkTOxYEb85XQHHw==", "integrity": "sha512-VqnkNqnZlU5EB64pp1l7hdm3hmQw7Vgqa0KF/KCNO9sIpI6Fk6brDEiX+iCOYrvMuBWDws0NkTOxYEb85XQHHw==",
"dev": true, "dev": true,
"dependencies": { "dependencies": {
@@ -2114,7 +2114,7 @@
"@webassemblyjs/helper-wasm-bytecode": "1.11.1", "@webassemblyjs/helper-wasm-bytecode": "1.11.1",
"@webassemblyjs/helper-wasm-section": "1.11.1", "@webassemblyjs/helper-wasm-section": "1.11.1",
"@webassemblyjs/wasm-gen": "1.11.1", "@webassemblyjs/wasm-gen": "1.11.1",
"@webassemblyjs/wasm-opt": "1.11.1", "@webassemblyjs/wasm-opts": "1.11.1",
"@webassemblyjs/wasm-parser": "1.11.1", "@webassemblyjs/wasm-parser": "1.11.1",
"@webassemblyjs/wast-printer": "1.11.1" "@webassemblyjs/wast-printer": "1.11.1"
} }
@@ -2132,9 +2132,9 @@
"@webassemblyjs/utf8": "1.11.1" "@webassemblyjs/utf8": "1.11.1"
} }
}, },
"@webassemblyjs/wasm-opt": { "@webassemblyjs/wasm-opts": {
"version": "1.11.1", "version": "1.11.1",
"resolved": "https://registry.npmmirror.com/@webassemblyjs/wasm-opt/-/wasm-opt-1.11.1.tgz", "resolved": "https://registry.npmmirror.com/@webassemblyjs/wasm-opts/-/wasm-opts-1.11.1.tgz",
"integrity": "sha512-VqnkNqnZlU5EB64pp1l7hdm3hmQw7Vgqa0KF/KCNO9sIpI6Fk6brDEiX+iCOYrvMuBWDws0NkTOxYEb85XQHHw==", "integrity": "sha512-VqnkNqnZlU5EB64pp1l7hdm3hmQw7Vgqa0KF/KCNO9sIpI6Fk6brDEiX+iCOYrvMuBWDws0NkTOxYEb85XQHHw==",
"dev": true, "dev": true,
"requires": { "requires": {
+19 -18
View File
@@ -2,13 +2,14 @@ package download
import ( import (
"errors" "errors"
"os"
"testing"
"time"
"github.com/GopeedLab/gopeed/internal/logger" "github.com/GopeedLab/gopeed/internal/logger"
"github.com/GopeedLab/gopeed/pkg/base" "github.com/GopeedLab/gopeed/pkg/base"
gojaerror "github.com/GopeedLab/gopeed/pkg/download/engine/inject/error" gojaerror "github.com/GopeedLab/gopeed/pkg/download/engine/inject/error"
"github.com/dop251/goja" "github.com/dop251/goja"
"os"
"testing"
"time"
) )
func TestDownloader_InstallExtensionByFolder(t *testing.T) { func TestDownloader_InstallExtensionByFolder(t *testing.T) {
@@ -18,7 +19,7 @@ func TestDownloader_InstallExtensionByFolder(t *testing.T) {
} }
rr, err := downloader.Resolve(&base.Request{ rr, err := downloader.Resolve(&base.Request{
URL: "https://github.com/test", URL: "https://github.com/test",
}) }, nil)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -35,7 +36,7 @@ func TestDownloader_InstallExtensionByFolderDevMode(t *testing.T) {
} }
rr, err := downloader.Resolve(&base.Request{ rr, err := downloader.Resolve(&base.Request{
URL: "https://github.com/test", URL: "https://github.com/test",
}) }, nil)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -52,7 +53,7 @@ func TestDownloader_InstallExtensionByGit(t *testing.T) {
} }
rr, err := downloader.Resolve(&base.Request{ rr, err := downloader.Resolve(&base.Request{
URL: "https://github.com/GopeedLab/gopeed/releases", URL: "https://github.com/GopeedLab/gopeed/releases",
}) }, nil)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -69,7 +70,7 @@ func TestDownloader_InstallExtensionByGitSimple(t *testing.T) {
} }
rr, err := downloader.Resolve(&base.Request{ rr, err := downloader.Resolve(&base.Request{
URL: "https://github.com/GopeedLab/gopeed/releases", URL: "https://github.com/GopeedLab/gopeed/releases",
}) }, nil)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -86,7 +87,7 @@ func TestDownloader_InstallExtensionByGitFull(t *testing.T) {
} }
rr, err := downloader.Resolve(&base.Request{ rr, err := downloader.Resolve(&base.Request{
URL: "https://github.com/GopeedLab/gopeed/releases", URL: "https://github.com/GopeedLab/gopeed/releases",
}) }, nil)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -156,7 +157,7 @@ func TestDownloader_UpgradeExtension(t *testing.T) {
rr, err := downloader.Resolve(&base.Request{ rr, err := downloader.Resolve(&base.Request{
URL: "https://test.com", URL: "https://test.com",
}) }, nil)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -185,7 +186,7 @@ func TestDownloader_Extension_OnStart(t *testing.T) {
select { select {
case err = <-errCh: case err = <-errCh:
break break
case <-time.After(time.Second * 10): case <-time.After(time.Second * 30): // Increased timeout for real network requests
err = errors.New("timeout") err = errors.New("timeout")
} }
if err != nil { if err != nil {
@@ -238,7 +239,7 @@ func TestDownloader_Extension_OnError(t *testing.T) {
select { select {
case err = <-errCh: case err = <-errCh:
break break
case <-time.After(time.Second * 10): case <-time.After(time.Second * 30): // Increased timeout for real network requests
err = errors.New("timeout") err = errors.New("timeout")
} }
@@ -273,7 +274,7 @@ func TestDownloader_Extension_OnDone(t *testing.T) {
select { select {
case err = <-errCh: case err = <-errCh:
break break
case <-time.After(time.Second * 10): case <-time.After(time.Second * 30): // Increased timeout for real network requests
err = errors.New("timeout") err = errors.New("timeout")
} }
// wait for script execution // wait for script execution
@@ -300,7 +301,7 @@ func TestDownloader_Extension_Errors(t *testing.T) {
} }
rr, err := downloader.Resolve(&base.Request{ rr, err := downloader.Resolve(&base.Request{
URL: "https://github.com/test", URL: "https://github.com/test",
}) }, nil)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -315,7 +316,7 @@ func TestDownloader_Extension_Errors(t *testing.T) {
} }
rr, err := downloader.Resolve(&base.Request{ rr, err := downloader.Resolve(&base.Request{
URL: "https://github.com/test", URL: "https://github.com/test",
}) }, nil)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -330,7 +331,7 @@ func TestDownloader_Extension_Errors(t *testing.T) {
} }
_, err := downloader.Resolve(&base.Request{ _, err := downloader.Resolve(&base.Request{
URL: "https://github.com/test", URL: "https://github.com/test",
}) }, nil)
if err == nil { if err == nil {
t.Fatalf("except error, but got nil") t.Fatalf("except error, but got nil")
} }
@@ -352,7 +353,7 @@ func TestDownloader_Extension_Settings(t *testing.T) {
} }
rr, err := downloader.Resolve(&base.Request{ rr, err := downloader.Resolve(&base.Request{
URL: "https://github.com/test", URL: "https://github.com/test",
}) }, nil)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -373,7 +374,7 @@ func TestDownloader_Extension_Settings(t *testing.T) {
}) })
rr, err := downloader.Resolve(&base.Request{ rr, err := downloader.Resolve(&base.Request{
URL: "https://github.com/test", URL: "https://github.com/test",
}) }, nil)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -390,7 +391,7 @@ func TestDownloader_ExtensionStorage(t *testing.T) {
} }
rr, err := downloader.Resolve(&base.Request{ rr, err := downloader.Resolve(&base.Request{
URL: "https://github.com/test", URL: "https://github.com/test",
}) }, nil)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
+3 -2
View File
@@ -2,6 +2,9 @@ package download
import ( import (
"encoding/json" "encoding/json"
"sync"
"time"
"github.com/GopeedLab/gopeed/internal/controller" "github.com/GopeedLab/gopeed/internal/controller"
"github.com/GopeedLab/gopeed/internal/fetcher" "github.com/GopeedLab/gopeed/internal/fetcher"
"github.com/GopeedLab/gopeed/internal/protocol/bt" "github.com/GopeedLab/gopeed/internal/protocol/bt"
@@ -9,8 +12,6 @@ import (
"github.com/GopeedLab/gopeed/pkg/base" "github.com/GopeedLab/gopeed/pkg/base"
"github.com/GopeedLab/gopeed/pkg/util" "github.com/GopeedLab/gopeed/pkg/util"
gonanoid "github.com/matoous/go-nanoid/v2" gonanoid "github.com/matoous/go-nanoid/v2"
"sync"
"time"
) )
type ResolveResult struct { type ResolveResult struct {
+38 -38
View File
@@ -50,9 +50,9 @@ func TestWebhook_TriggerOnDone(t *testing.T) {
// Configure webhook URLs // Configure webhook URLs
cfg, _ := downloader.GetConfig() cfg, _ := downloader.GetConfig()
cfg.Webhook = &base.WebhookConfig{ cfg.Webhook = &base.WebhookConfig{
Enable: true, Enable: true,
URLs: []string{server.URL}, URLs: []string{server.URL},
} }
downloader.PutConfig(cfg) downloader.PutConfig(cfg)
// Create a mock task // Create a mock task
@@ -99,9 +99,9 @@ func TestWebhook_TriggerOnError(t *testing.T) {
// Configure webhook URLs // Configure webhook URLs
cfg, _ := downloader.GetConfig() cfg, _ := downloader.GetConfig()
cfg.Webhook = &base.WebhookConfig{ cfg.Webhook = &base.WebhookConfig{
Enable: true, Enable: true,
URLs: []string{server.URL}, URLs: []string{server.URL},
} }
downloader.PutConfig(cfg) downloader.PutConfig(cfg)
// Create a mock task // Create a mock task
@@ -144,9 +144,9 @@ func TestWebhook_SendTestWebhook(t *testing.T) {
// Configure webhook URLs // Configure webhook URLs
cfg, _ := downloader.GetConfig() cfg, _ := downloader.GetConfig()
cfg.Webhook = &base.WebhookConfig{ cfg.Webhook = &base.WebhookConfig{
Enable: true, Enable: true,
URLs: []string{server.URL}, URLs: []string{server.URL},
} }
downloader.PutConfig(cfg) downloader.PutConfig(cfg)
// Send test webhook // Send test webhook
@@ -207,9 +207,9 @@ func TestWebhook_MultipleUrls(t *testing.T) {
// Configure multiple webhook URLs // Configure multiple webhook URLs
cfg, _ := downloader.GetConfig() cfg, _ := downloader.GetConfig()
cfg.Webhook = &base.WebhookConfig{ cfg.Webhook = &base.WebhookConfig{
Enable: true, Enable: true,
URLs: []string{server1.URL, server2.URL}, URLs: []string{server1.URL, server2.URL},
} }
downloader.PutConfig(cfg) downloader.PutConfig(cfg)
// Create a mock task // Create a mock task
@@ -240,9 +240,9 @@ func TestWebhook_TestWebhookFailsOnNon200(t *testing.T) {
// Configure webhook URLs // Configure webhook URLs
cfg, _ := downloader.GetConfig() cfg, _ := downloader.GetConfig()
cfg.Webhook = &base.WebhookConfig{ cfg.Webhook = &base.WebhookConfig{
Enable: true, Enable: true,
URLs: []string{server.URL}, URLs: []string{server.URL},
} }
downloader.PutConfig(cfg) downloader.PutConfig(cfg)
// Send test webhook - should fail with non-200 status // Send test webhook - should fail with non-200 status
@@ -264,9 +264,9 @@ func TestWebhook_TestWebhookFailsOn201(t *testing.T) {
// Configure webhook URLs // Configure webhook URLs
cfg, _ := downloader.GetConfig() cfg, _ := downloader.GetConfig()
cfg.Webhook = &base.WebhookConfig{ cfg.Webhook = &base.WebhookConfig{
Enable: true, Enable: true,
URLs: []string{server.URL}, URLs: []string{server.URL},
} }
downloader.PutConfig(cfg) downloader.PutConfig(cfg)
// Send test webhook - should fail with 201 status (only 200 is success) // Send test webhook - should fail with 201 status (only 200 is success)
@@ -346,7 +346,7 @@ func TestWebhook_GetWebhookUrls_NoExtraField(t *testing.T) {
func TestWebhook_GetWebhookUrls_NoWebhookUrlsKey(t *testing.T) { func TestWebhook_GetWebhookUrls_NoWebhookUrlsKey(t *testing.T) {
setupWebhookTest(t, func(downloader *Downloader) { setupWebhookTest(t, func(downloader *Downloader) {
cfg, _ := downloader.GetConfig() cfg, _ := downloader.GetConfig()
cfg.Webhook = &base.WebhookConfig{Enable: true} // No URLs set cfg.Webhook = &base.WebhookConfig{Enable: true} // No URLs set
downloader.PutConfig(cfg) downloader.PutConfig(cfg)
urls := downloader.getWebhookUrls() urls := downloader.getWebhookUrls()
@@ -360,9 +360,9 @@ func TestWebhook_GetWebhookUrls_StringSlice(t *testing.T) {
setupWebhookTest(t, func(downloader *Downloader) { setupWebhookTest(t, func(downloader *Downloader) {
cfg, _ := downloader.GetConfig() cfg, _ := downloader.GetConfig()
cfg.Webhook = &base.WebhookConfig{ cfg.Webhook = &base.WebhookConfig{
Enable: true, Enable: true,
URLs: []string{"http://example.com", "http://example2.com"}, URLs: []string{"http://example.com", "http://example2.com"},
} }
downloader.PutConfig(cfg) downloader.PutConfig(cfg)
urls := downloader.getWebhookUrls() urls := downloader.getWebhookUrls()
@@ -379,9 +379,9 @@ func TestWebhook_GetWebhookUrls_InterfaceSlice(t *testing.T) {
setupWebhookTest(t, func(downloader *Downloader) { setupWebhookTest(t, func(downloader *Downloader) {
cfg, _ := downloader.GetConfig() cfg, _ := downloader.GetConfig()
cfg.Webhook = &base.WebhookConfig{ cfg.Webhook = &base.WebhookConfig{
Enable: true, Enable: true,
URLs: []string{"http://example.com", "http://example2.com"}, URLs: []string{"http://example.com", "http://example2.com"},
} }
downloader.PutConfig(cfg) downloader.PutConfig(cfg)
urls := downloader.getWebhookUrls() urls := downloader.getWebhookUrls()
@@ -412,9 +412,9 @@ func TestWebhook_GetWebhookUrls_InterfaceSliceWithEmptyStrings(t *testing.T) {
setupWebhookTest(t, func(downloader *Downloader) { setupWebhookTest(t, func(downloader *Downloader) {
cfg, _ := downloader.GetConfig() cfg, _ := downloader.GetConfig()
cfg.Webhook = &base.WebhookConfig{ cfg.Webhook = &base.WebhookConfig{
Enable: true, Enable: true,
URLs: []string{"", "", ""}, URLs: []string{"", "", ""},
} }
downloader.PutConfig(cfg) downloader.PutConfig(cfg)
urls := downloader.getWebhookUrls() urls := downloader.getWebhookUrls()
@@ -446,7 +446,7 @@ func TestWebhook_GetWebhookUrls_InterfaceSliceMixedTypes(t *testing.T) {
func TestWebhook_GetWebhookUrls_InvalidType(t *testing.T) { func TestWebhook_GetWebhookUrls_InvalidType(t *testing.T) {
setupWebhookTest(t, func(downloader *Downloader) { setupWebhookTest(t, func(downloader *Downloader) {
cfg, _ := downloader.GetConfig() cfg, _ := downloader.GetConfig()
cfg.Webhook = &base.WebhookConfig{Enable: false} // Disabled webhook cfg.Webhook = &base.WebhookConfig{Enable: false} // Disabled webhook
downloader.PutConfig(cfg) downloader.PutConfig(cfg)
urls := downloader.getWebhookUrls() urls := downloader.getWebhookUrls()
@@ -640,9 +640,9 @@ func TestWebhook_TriggerWebhooks_EmptyUrlSkipped(t *testing.T) {
setupWebhookTest(t, func(downloader *Downloader) { setupWebhookTest(t, func(downloader *Downloader) {
cfg, _ := downloader.GetConfig() cfg, _ := downloader.GetConfig()
cfg.Webhook = &base.WebhookConfig{ cfg.Webhook = &base.WebhookConfig{
Enable: true, Enable: true,
URLs: []string{server.URL, "", server.URL}, URLs: []string{server.URL, "", server.URL},
} }
downloader.PutConfig(cfg) downloader.PutConfig(cfg)
task := NewTask() task := NewTask()
@@ -675,9 +675,9 @@ func TestWebhook_WebhookDataStructure(t *testing.T) {
setupWebhookTest(t, func(downloader *Downloader) { setupWebhookTest(t, func(downloader *Downloader) {
cfg, _ := downloader.GetConfig() cfg, _ := downloader.GetConfig()
cfg.Webhook = &base.WebhookConfig{ cfg.Webhook = &base.WebhookConfig{
Enable: true, Enable: true,
URLs: []string{server.URL}, URLs: []string{server.URL},
} }
downloader.PutConfig(cfg) downloader.PutConfig(cfg)
task := NewTask() task := NewTask()
@@ -744,9 +744,9 @@ func TestWebhook_SendTestWebhook_MixedResults(t *testing.T) {
setupWebhookTest(t, func(downloader *Downloader) { setupWebhookTest(t, func(downloader *Downloader) {
cfg, _ := downloader.GetConfig() cfg, _ := downloader.GetConfig()
cfg.Webhook = &base.WebhookConfig{ cfg.Webhook = &base.WebhookConfig{
Enable: true, Enable: true,
URLs: []string{server1.URL, server2.URL}, URLs: []string{server1.URL, server2.URL},
} }
downloader.PutConfig(cfg) downloader.PutConfig(cfg)
// Should fail because server2 returns 500 // Should fail because server2 returns 500
+9 -8
View File
@@ -1,14 +1,15 @@
package rest package rest
import ( import (
"github.com/GopeedLab/gopeed/pkg/base"
"github.com/GopeedLab/gopeed/pkg/download"
"github.com/GopeedLab/gopeed/pkg/rest/model"
"github.com/gorilla/mux"
"io" "io"
"net/http" "net/http"
"net/url" "net/url"
"runtime" "runtime"
"github.com/GopeedLab/gopeed/pkg/base"
"github.com/GopeedLab/gopeed/pkg/download"
"github.com/GopeedLab/gopeed/pkg/rest/model"
"github.com/gorilla/mux"
) )
func Info(w http.ResponseWriter, r *http.Request) { func Info(w http.ResponseWriter, r *http.Request) {
@@ -23,9 +24,9 @@ func Info(w http.ResponseWriter, r *http.Request) {
} }
func Resolve(w http.ResponseWriter, r *http.Request) { func Resolve(w http.ResponseWriter, r *http.Request) {
var req base.Request var req model.ResolveTask
if ReadJson(r, w, &req) { if ReadJson(r, w, &req) {
rr, err := Downloader.Resolve(&req) rr, err := Downloader.Resolve(req.Req, req.Opts)
if err != nil { if err != nil {
WriteJson(w, model.NewErrorResult(err.Error())) WriteJson(w, model.NewErrorResult(err.Error()))
return return
@@ -42,9 +43,9 @@ func CreateTask(w http.ResponseWriter, r *http.Request) {
err error err error
) )
if req.Rid != "" { if req.Rid != "" {
taskId, err = Downloader.Create(req.Rid, req.Opt) taskId, err = Downloader.Create(req.Rid)
} else if req.Req != nil { } else if req.Req != nil {
taskId, err = Downloader.CreateDirect(req.Req, req.Opt) taskId, err = Downloader.CreateDirect(req.Req, req.Opts)
} else { } else {
WriteJson(w, model.NewErrorResult("param invalid: rid or req", model.CodeInvalidParam)) WriteJson(w, model.NewErrorResult("param invalid: rid or req", model.CodeInvalidParam))
return return
+8 -3
View File
@@ -2,8 +2,13 @@ package model
import "github.com/GopeedLab/gopeed/pkg/base" import "github.com/GopeedLab/gopeed/pkg/base"
type ResolveTask struct {
Req *base.Request `json:"req"`
Opts *base.Options `json:"opts"`
}
type CreateTask struct { type CreateTask struct {
Rid string `json:"rid"` Rid string `json:"rid"`
Req *base.Request `json:"req"`
Opt *base.Options `json:"opt"` Req *base.Request `json:"req"`
Opts *base.Options `json:"opts"`
} }
+8 -33
View File
@@ -53,13 +53,13 @@ var (
"connections": 2, "connections": 2,
}, },
} }
createReq = &model.CreateTask{ resolveReq = &model.ResolveTask{
Req: taskReq, Req: taskReq,
Opt: createOpts, Opts: createOpts,
} }
createResoledReq = &model.CreateTask{ createReq = &model.CreateTask{
Req: taskReq, Req: taskReq,
Opt: createOpts, Opts: createOpts,
} }
installExtensionReq = &model.InstallExtension{ installExtensionReq = &model.InstallExtension{
URL: "https://github.com/GopeedLab/gopeed-extension-samples#github-contributor-avatars-sample", URL: "https://github.com/GopeedLab/gopeed-extension-samples#github-contributor-avatars-sample",
@@ -80,7 +80,7 @@ func TestInfo(t *testing.T) {
func TestResolve(t *testing.T) { func TestResolve(t *testing.T) {
doTest(func() { doTest(func() {
resp := httpRequestCheckOk[*download.ResolveResult](http.MethodPost, "/api/v1/resolve", taskReq) resp := httpRequestCheckOk[*download.ResolveResult](http.MethodPost, "/api/v1/resolve", resolveReq)
if !test.AssertResourceEqual(taskRes, resp.Res) { if !test.AssertResourceEqual(taskRes, resp.Res) {
t.Errorf("Resolve() got = %v, want %v", test.ToJson(resp.Res), test.ToJson(taskRes)) t.Errorf("Resolve() got = %v, want %v", test.ToJson(resp.Res), test.ToJson(taskRes))
} }
@@ -89,7 +89,7 @@ func TestResolve(t *testing.T) {
func TestCreateTask(t *testing.T) { func TestCreateTask(t *testing.T) {
doTest(func() { doTest(func() {
resp := httpRequestCheckOk[*download.ResolveResult](http.MethodPost, "/api/v1/resolve", taskReq) resp := httpRequestCheckOk[*download.ResolveResult](http.MethodPost, "/api/v1/resolve", resolveReq)
var wg sync.WaitGroup var wg sync.WaitGroup
wg.Add(1) wg.Add(1)
@@ -101,7 +101,6 @@ func TestCreateTask(t *testing.T) {
taskId := httpRequestCheckOk[string](http.MethodPost, "/api/v1/tasks", &model.CreateTask{ taskId := httpRequestCheckOk[string](http.MethodPost, "/api/v1/tasks", &model.CreateTask{
Rid: resp.ID, Rid: resp.ID,
Opt: createOpts,
}) })
if taskId == "" { if taskId == "" {
t.Fatal("create task failed") t.Fatal("create task failed")
@@ -196,30 +195,6 @@ func TestCreateDirectTaskBatchWithOpt(t *testing.T) {
}) })
} }
func TestCreateDirectTaskWithResoled(t *testing.T) {
doTest(func() {
var wg sync.WaitGroup
wg.Add(1)
Downloader.Listener(func(event *download.Event) {
if event.Key == download.EventKeyFinally {
wg.Done()
}
})
taskId := httpRequestCheckOk[string](http.MethodPost, "/api/v1/tasks", createResoledReq)
if taskId == "" {
t.Fatal("create task failed")
}
wg.Wait()
want := test.FileMd5(test.BuildFile)
got := test.FileMd5(test.DownloadFile)
if want != got {
t.Errorf("CreateDirectTaskWithResoled() got = %v, want %v", got, want)
}
})
}
func TestPauseAndContinueTask(t *testing.T) { func TestPauseAndContinueTask(t *testing.T) {
doTest(func() { doTest(func() {
var wg sync.WaitGroup var wg sync.WaitGroup
+3 -2
View File
@@ -16,6 +16,7 @@ import 'model/install_extension.dart';
import 'model/login.dart'; import 'model/login.dart';
import 'model/request.dart'; import 'model/request.dart';
import 'model/resolve_result.dart'; import 'model/resolve_result.dart';
import 'model/resolve_task.dart';
import 'model/result.dart'; import 'model/result.dart';
import 'model/switch_extension.dart'; import 'model/switch_extension.dart';
import 'model/task.dart'; import 'model/task.dart';
@@ -123,9 +124,9 @@ Future<T> _parse<T>(
} }
} }
Future<ResolveResult> resolve(Request request) async { Future<ResolveResult> resolve(ResolveTask resolveTask) async {
return _parse<ResolveResult>( return _parse<ResolveResult>(
() => _client.dio.post("api/v1/resolve", data: request), () => _client.dio.post("api/v1/resolve", data: resolveTask),
(data) => ResolveResult.fromJson(data)); (data) => ResolveResult.fromJson(data));
} }
+2 -2
View File
@@ -9,12 +9,12 @@ part 'create_task.g.dart';
class CreateTask { class CreateTask {
String? rid; String? rid;
Request? req; Request? req;
Options? opt; Options? opts;
CreateTask({ CreateTask({
this.rid, this.rid,
this.req, this.req,
this.opt, this.opts,
}); });
factory CreateTask.fromJson( factory CreateTask.fromJson(
+3 -3
View File
@@ -11,9 +11,9 @@ CreateTask _$CreateTaskFromJson(Map<String, dynamic> json) => CreateTask(
req: json['req'] == null req: json['req'] == null
? null ? null
: Request.fromJson(json['req'] as Map<String, dynamic>), : Request.fromJson(json['req'] as Map<String, dynamic>),
opt: json['opt'] == null opts: json['opts'] == null
? null ? null
: Options.fromJson(json['opt'] as Map<String, dynamic>), : Options.fromJson(json['opts'] as Map<String, dynamic>),
); );
Map<String, dynamic> _$CreateTaskToJson(CreateTask instance) { Map<String, dynamic> _$CreateTaskToJson(CreateTask instance) {
@@ -27,6 +27,6 @@ Map<String, dynamic> _$CreateTaskToJson(CreateTask instance) {
writeNotNull('rid', instance.rid); writeNotNull('rid', instance.rid);
writeNotNull('req', instance.req?.toJson()); writeNotNull('req', instance.req?.toJson());
writeNotNull('opt', instance.opt?.toJson()); writeNotNull('opts', instance.opts?.toJson());
return val; return val;
} }
@@ -8,11 +8,11 @@ part 'create_task_batch.g.dart';
@JsonSerializable(explicitToJson: true) @JsonSerializable(explicitToJson: true)
class CreateTaskBatch { class CreateTaskBatch {
List<CreateTaskBatchItem>? reqs; List<CreateTaskBatchItem>? reqs;
Options? opt; Options? opts;
CreateTaskBatch({ CreateTaskBatch({
this.reqs, this.reqs,
this.opt, this.opts,
}); });
factory CreateTaskBatch.fromJson( factory CreateTaskBatch.fromJson(
@@ -11,9 +11,9 @@ CreateTaskBatch _$CreateTaskBatchFromJson(Map<String, dynamic> json) =>
reqs: (json['reqs'] as List<dynamic>?) reqs: (json['reqs'] as List<dynamic>?)
?.map((e) => CreateTaskBatchItem.fromJson(e as Map<String, dynamic>)) ?.map((e) => CreateTaskBatchItem.fromJson(e as Map<String, dynamic>))
.toList(), .toList(),
opt: json['opt'] == null opts: json['opts'] == null
? null ? null
: Options.fromJson(json['opt'] as Map<String, dynamic>), : Options.fromJson(json['opts'] as Map<String, dynamic>),
); );
Map<String, dynamic> _$CreateTaskBatchToJson(CreateTaskBatch instance) { Map<String, dynamic> _$CreateTaskBatchToJson(CreateTaskBatch instance) {
@@ -26,7 +26,7 @@ Map<String, dynamic> _$CreateTaskBatchToJson(CreateTaskBatch instance) {
} }
writeNotNull('reqs', instance.reqs?.map((e) => e.toJson()).toList()); writeNotNull('reqs', instance.reqs?.map((e) => e.toJson()).toList());
writeNotNull('opt', instance.opt?.toJson()); writeNotNull('opts', instance.opts?.toJson());
return val; return val;
} }
@@ -0,0 +1,21 @@
import 'package:json_annotation/json_annotation.dart';
import 'options.dart';
import 'request.dart';
part 'resolve_task.g.dart';
@JsonSerializable()
class ResolveTask {
Request? req;
Options? opts;
ResolveTask({
this.req,
this.opts,
});
factory ResolveTask.fromJson(Map<String, dynamic> json) =>
_$ResolveTaskFromJson(json);
Map<String, dynamic> toJson() => _$ResolveTaskToJson(this);
}
@@ -0,0 +1,30 @@
// GENERATED CODE - DO NOT MODIFY BY HAND
part of 'resolve_task.dart';
// **************************************************************************
// JsonSerializableGenerator
// **************************************************************************
ResolveTask _$ResolveTaskFromJson(Map<String, dynamic> json) => ResolveTask(
req: json['req'] == null
? null
: Request.fromJson(json['req'] as Map<String, dynamic>),
opts: json['opts'] == null
? null
: Options.fromJson(json['opts'] as Map<String, dynamic>),
);
Map<String, dynamic> _$ResolveTaskToJson(ResolveTask instance) {
final val = <String, dynamic>{};
void writeNotNull(String key, dynamic value) {
if (value != null) {
val[key] = value;
}
}
writeNotNull('req', instance.req);
writeNotNull('opts', instance.opts);
return val;
}
@@ -16,6 +16,7 @@ import '../../../../api/model/downloader_config.dart';
import '../../../../api/model/options.dart'; import '../../../../api/model/options.dart';
import '../../../../api/model/request.dart'; import '../../../../api/model/request.dart';
import '../../../../api/model/resolve_result.dart'; import '../../../../api/model/resolve_result.dart';
import '../../../../api/model/resolve_task.dart';
import '../../../../api/model/task.dart'; import '../../../../api/model/task.dart';
import '../../../../database/database.dart'; import '../../../../database/database.dart';
import '../../../../util/input_formatter.dart'; import '../../../../util/input_formatter.dart';
@@ -125,13 +126,13 @@ class CreateView extends GetView<CreateController> {
} }
// handle options // handle options
if (routerParams.opt != null) { if (routerParams.opts != null) {
_renameController.text = routerParams.opt!.name; _renameController.text = routerParams.opts!.name;
_pathController.text = routerParams.opt!.path; _pathController.text = routerParams.opts!.path;
final optionsHandlers = { final optionsHandlers = {
Protocol.http: () { Protocol.http: () {
final opt = routerParams.opt!; final opt = routerParams.opts!;
_renameController.text = opt.name; _renameController.text = opt.name;
_pathController.text = opt.path; _pathController.text = opt.path;
if (opt.extra != null) { if (opt.extra != null) {
@@ -143,7 +144,7 @@ class CreateView extends GetView<CreateController> {
}, },
Protocol.bt: null, Protocol.bt: null,
}; };
if (routerParams.opt?.extra != null) { if (routerParams.opts?.extra != null) {
optionsHandlers[protocol]?.call(); optionsHandlers[protocol]?.call();
} }
} }
@@ -814,29 +815,34 @@ class CreateView extends GetView<CreateController> {
*/ */
final isMultiLine = urls.length > 1; final isMultiLine = urls.length > 1;
final isDirect = controller.directDownload.value || isMultiLine; final isDirect = controller.directDownload.value || isMultiLine;
final opt = Options(
name: isMultiLine ? "" : _renameController.text,
path: _pathController.text,
selectFiles: [],
extra: parseReqOptsExtra(),
);
if (isDirect) { if (isDirect) {
await Future.wait(urls.map((url) { await Future.wait(urls.map((url) {
return createTask(CreateTask( return createTask(CreateTask(
req: Request( req: Request(
url: url, url: url,
extra: parseReqExtra(url), extra: parseReqExtra(url),
proxy: parseProxy(), proxy: parseProxy(),
skipVerifyCert: _skipVerifyCertController.value, skipVerifyCert: _skipVerifyCertController.value,
), ),
opt: Options( opts: opt,
name: isMultiLine ? "" : _renameController.text, ));
path: _pathController.text,
selectFiles: [],
extra: parseReqOptsExtra(),
)));
})); }));
Get.rootDelegate.offNamed(Routes.TASK); Get.rootDelegate.offNamed(Routes.TASK);
} else { } else {
final rr = await resolve(Request( final rr = await resolve(ResolveTask(
url: submitUrl, req: Request(
extra: parseReqExtra(_urlController.text), url: submitUrl,
proxy: parseProxy(), extra: parseReqExtra(_urlController.text),
skipVerifyCert: _skipVerifyCertController.value, proxy: parseProxy(),
skipVerifyCert: _skipVerifyCertController.value,
),
opts: opt,
)); ));
await _showResolveDialog(rr); await _showResolveDialog(rr);
} }
@@ -983,7 +989,7 @@ class CreateView extends GetView<CreateController> {
} else { } else {
await createTask(CreateTask( await createTask(CreateTask(
rid: rr.id, rid: rr.id,
opt: Options( opts: Options(
name: _renameController.text, name: _renameController.text,
path: _pathController.text, path: _pathController.text,
selectFiles: controller.selectedIndexes, selectFiles: controller.selectedIndexes,