diff --git a/cmd/gopeed/main.go b/cmd/gopeed/main.go index 431ca538..cd4e0587 100644 --- a/cmd/gopeed/main.go +++ b/cmd/gopeed/main.go @@ -2,13 +2,14 @@ package main import ( "fmt" + "strings" + "sync" + "github.com/GopeedLab/gopeed/internal/fetcher" "github.com/GopeedLab/gopeed/pkg/base" "github.com/GopeedLab/gopeed/pkg/download" "github.com/GopeedLab/gopeed/pkg/protocol/http" "github.com/GopeedLab/gopeed/pkg/util" - "strings" - "sync" ) const progressWidth = 20 diff --git a/internal/fetcher/fetcher.go b/internal/fetcher/fetcher.go index 891e79dd..24cb8adc 100644 --- a/internal/fetcher/fetcher.go +++ b/internal/fetcher/fetcher.go @@ -1,20 +1,18 @@ package fetcher import ( - "github.com/GopeedLab/gopeed/internal/controller" - "github.com/GopeedLab/gopeed/pkg/base" "path" "strings" + + "github.com/GopeedLab/gopeed/internal/controller" + "github.com/GopeedLab/gopeed/pkg/base" ) // Fetcher defines the interface for a download protocol. // Each download task will have a corresponding Fetcher instance for the management of the download task type Fetcher interface { Setup(ctl *controller.Controller) - // Resolve resource info from request - Resolve(req *base.Request) error - // Create ready to download, but not started - Create(opts *base.Options) error + Resolve(req *base.Request, opts *base.Options) error Start() error Pause() error Close() error diff --git a/internal/protocol/bt/fetcher.go b/internal/protocol/bt/fetcher.go index 017631a9..79a94812 100644 --- a/internal/protocol/bt/fetcher.go +++ b/internal/protocol/bt/fetcher.go @@ -20,15 +20,15 @@ import ( "github.com/GopeedLab/gopeed/pkg/util" "github.com/anacrolix/torrent" "github.com/anacrolix/torrent/metainfo" + "github.com/anacrolix/torrent/storage" ) var ( - client *torrent.Client - lock sync.Mutex - torrentDirMap = make(map[string]string) - ftMap = make(map[string]*fileTorrentImpl) - closeCtx context.Context - closeFunc func() + cfg *torrent.ClientConfig + client *torrent.Client + lock sync.Mutex + closeCtx context.Context + closeFunc func() ) type Fetcher struct { @@ -71,21 +71,12 @@ func (f *Fetcher) initClient() (err error) { closeCtx, closeFunc = context.WithCancel(context.Background()) } - cfg := torrent.NewDefaultClientConfig() + cfg = torrent.NewDefaultClientConfig() cfg.Seed = true cfg.Bep20 = fmt.Sprintf("-GP%s-", parseBep20()) cfg.ExtendedHandshakeClientVersion = fmt.Sprintf("Gopeed %s", base.Version) cfg.ListenPort = f.config.ListenPort cfg.HTTPProxy = f.ctl.GetProxy(f.meta.Req.Proxy) - cfg.DefaultStorage = newFileOpts(newFileClientOpts{ - ClientBaseDir: cfg.DataDir, - HandleFileTorrent: func(infoHash metainfo.Hash, ft *fileTorrentImpl) { - if dir, ok := torrentDirMap[infoHash.String()]; ok { - ft.setTorrentDir(dir) - } - ftMap[infoHash.String()] = ft - }, - }) dnsResolver := &DnsCacheResolver{RefreshTimeout: 5 * time.Minute} cfg.TrackerDialContext = dnsResolver.DialContext client, err = torrent.NewClient(cfg) @@ -100,8 +91,12 @@ func (f *Fetcher) initClient() (err error) { return } -func (f *Fetcher) Resolve(req *base.Request) error { +func (f *Fetcher) Resolve(req *base.Request, opts *base.Options) error { f.meta.Req = req + f.meta.Opts = opts + if f.meta.Opts == nil { + f.meta.Opts = &base.Options{} + } if err := f.addTorrent(req, false); err != nil { return err } @@ -109,23 +104,12 @@ func (f *Fetcher) Resolve(req *base.Request) error { return nil } -func (f *Fetcher) Create(opts *base.Options) (err error) { - f.meta.Opts = opts - if f.meta.Res != nil { - torrentDirMap[f.meta.Res.Hash] = opts.Path - } - return nil -} - func (f *Fetcher) Start() (err error) { if !f.torrentReady.Load() { if err = f.addTorrent(f.meta.Req, false); err != nil { return } } - if ft, ok := ftMap[f.meta.Res.Hash]; ok { - ft.setTorrentDir(f.meta.Opts.Path) - } files := f.torrent.Files() // If the user does not specify the file to download, all files will be downloaded by default if f.data.Progress == nil { @@ -361,8 +345,12 @@ func (f *Fetcher) addTorrent(req *base.Request, fromUpload bool) (err error) { } schema := util.ParseSchema(req.URL) privateTorrent := false + var spec *torrent.TorrentSpec if schema == "MAGNET" { - f.torrent, err = client.AddMagnet(req.URL) + spec, err = torrent.TorrentSpecFromMagnetUri(req.URL) + if err != nil { + return + } } else { var reader io.Reader if schema == "FILE" { @@ -399,8 +387,18 @@ func (f *Fetcher) addTorrent(req *base.Request, fromUpload bool) (err error) { if info.Private != nil && *info.Private { privateTorrent = true } - f.torrent, err = client.AddTorrent(metaInfo) + spec, err = torrent.TorrentSpecFromMetaInfoErr(metaInfo) + if err != nil { + return + } } + spec.Storage = storage.NewFileOpts(storage.NewFileClientOpts{ + ClientBaseDir: cfg.DataDir, + TorrentDirMaker: func(baseDir string, info *metainfo.Info, infoHash metainfo.Hash) string { + return f.meta.Opts.Path + }, + }) + f.torrent, _, err = client.AddTorrentSpec(spec) if err != nil { return } diff --git a/internal/protocol/bt/fetcher_test.go b/internal/protocol/bt/fetcher_test.go index 5975681f..c445d345 100644 --- a/internal/protocol/bt/fetcher_test.go +++ b/internal/protocol/bt/fetcher_test.go @@ -31,7 +31,7 @@ func TestFetcher_Resolve_DataUri_Torrent(t *testing.T) { dataUri := "data:application/x-bittorrent;base64," + base64.StdEncoding.EncodeToString(buf) err = fetcher.Resolve(&base.Request{ URL: dataUri, - }) + }, nil) if err != nil { panic(err) } @@ -81,7 +81,7 @@ func doResolve(t *testing.T, fetcher fetcher.Fetcher) { "udp://tracker.bitsearch.to:1337/announce", }, }, - }) + }, nil) if err != nil { panic(err) } @@ -111,7 +111,7 @@ func doResolve(t *testing.T, fetcher fetcher.Fetcher) { "udp://tracker.bitsearch.to:1337/announce", }, }, - }) + }, nil) if err != nil { panic(err) } @@ -145,7 +145,7 @@ func doResolve(t *testing.T, fetcher fetcher.Fetcher) { t.Run("Resolve Unclean Torrent", func(t *testing.T) { err := fetcher.Resolve(&base.Request{ URL: "./testdata/test.unclean.torrent", - }) + }, nil) if err != nil { t.Errorf("Resolve Unclean Torrent Resolve() got = %v, want nil", err) } @@ -156,7 +156,7 @@ func doResolve(t *testing.T, fetcher fetcher.Fetcher) { uri := "file:///" + file err := fetcher.Resolve(&base.Request{ URL: uri, - }) + }, nil) if err != nil { t.Errorf("Resolve file scheme Torrent Resolve() got = %v, want nil", err) } diff --git a/internal/protocol/bt/file.go b/internal/protocol/bt/file.go deleted file mode 100644 index 8d63864c..00000000 --- a/internal/protocol/bt/file.go +++ /dev/null @@ -1,216 +0,0 @@ -// modify from github.com/anacrolix/torrent/storage/file.go - -package bt - -import ( - "context" - "fmt" - "io" - "os" - "path/filepath" - - "github.com/anacrolix/torrent/storage" - - "github.com/anacrolix/missinggo/v2" - - "github.com/anacrolix/torrent/common" - "github.com/anacrolix/torrent/metainfo" - "github.com/anacrolix/torrent/segments" -) - -// File-based storage for torrents, that isn't yet bound to a particular torrent. -type fileClientImpl struct { - opts newFileClientOpts -} - -type newFileClientOpts struct { - // The base directory for all downloads. - ClientBaseDir string - HandleFileTorrent func(infoHash metainfo.Hash, ft *fileTorrentImpl) - FilePathMaker storage.FilePathMaker - PieceCompletion storage.PieceCompletion -} - -func newFileOpts(opts newFileClientOpts) storage.ClientImplCloser { - if opts.FilePathMaker == nil { - opts.FilePathMaker = func(opts storage.FilePathMakerOpts) string { - var parts []string - if opts.Info.Length == 0 || opts.File.Path == nil { - parts = append(parts, opts.Info.Name) - } - return filepath.Join(append(parts, opts.File.Path...)...) - } - } - if opts.PieceCompletion == nil { - ret, err := storage.NewDefaultPieceCompletionForDir(opts.ClientBaseDir) - if err != nil { - ret = storage.NewMapPieceCompletion() - } - opts.PieceCompletion = ret - } - return fileClientImpl{opts} -} - -func (fs fileClientImpl) Close() error { - return fs.opts.PieceCompletion.Close() -} - -func (fs fileClientImpl) OpenTorrent(ctx context.Context, info *metainfo.Info, infoHash metainfo.Hash) (_ storage.TorrentImpl, err error) { - upvertedFiles := info.UpvertedFiles() - files := make([]file, 0, len(upvertedFiles)) - for _, fileInfo := range upvertedFiles { - filePath := filepath.Join("", fs.opts.FilePathMaker(storage.FilePathMakerOpts{ - Info: info, - File: &fileInfo, - })) - f := file{ - rawPath: filePath, - path: filePath, - length: fileInfo.Length, - } - if f.length == 0 { - err = CreateNativeZeroLengthFile(f.path) - if err != nil { - err = fmt.Errorf("creating zero length file: %w", err) - return - } - } - files = append(files, f) - } - t := &fileTorrentImpl{ - files, - segments.NewIndex(common.LengthIterFromUpvertedFiles(upvertedFiles)), - infoHash, - fs.opts.PieceCompletion, - } - fs.opts.HandleFileTorrent(infoHash, t) - return storage.TorrentImpl{ - Piece: t.Piece, - Close: t.Close, - }, nil -} - -type file struct { - // The safe, OS-local file path. - rawPath string - path string - length int64 -} - -type fileTorrentImpl struct { - files []file - segmentLocater segments.Index - infoHash metainfo.Hash - completion storage.PieceCompletion -} - -func (fts *fileTorrentImpl) Piece(p metainfo.Piece) storage.PieceImpl { - // Create a view onto the file-based torrent storage. - _io := fileTorrentImplIO{fts} - // Return the appropriate segments of this. - return &filePieceImpl{ - fts, - p, - missinggo.NewSectionWriter(_io, p.Offset(), p.Length()), - io.NewSectionReader(_io, p.Offset(), p.Length()), - } -} - -func (fts *fileTorrentImpl) Close() error { - return nil -} - -func (fts *fileTorrentImpl) setTorrentDir(dir string) { - for i, f := range fts.files { - fts.files[i].path = filepath.Join(dir, f.rawPath) - } -} - -// A helper to create zero-length files which won't appear for file-orientated storage since no -// writes will ever occur to them (no torrent data is associated with a zero-length file). The -// caller should make sure the file name provided is safe/sanitized. -func CreateNativeZeroLengthFile(name string) error { - os.MkdirAll(filepath.Dir(name), 0o777) - var f io.Closer - f, err := os.Create(name) - if err != nil { - return err - } - return f.Close() -} - -// Exposes file-based storage of a torrent, as one big ReadWriterAt. -type fileTorrentImplIO struct { - fts *fileTorrentImpl -} - -// Returns EOF on short or missing file. -func (fst *fileTorrentImplIO) readFileAt(file file, b []byte, off int64) (n int, err error) { - f, err := os.Open(file.path) - if os.IsNotExist(err) { - // File missing is treated the same as a short file. - err = io.EOF - return - } - if err != nil { - return - } - defer f.Close() - // Limit the read to within the expected bounds of this file. - if int64(len(b)) > file.length-off { - b = b[:file.length-off] - } - for off < file.length && len(b) != 0 { - n1, err1 := f.ReadAt(b, off) - b = b[n1:] - n += n1 - off += int64(n1) - if n1 == 0 { - err = err1 - break - } - } - return -} - -// Only returns EOF at the end of the torrent. Premature EOF is ErrUnexpectedEOF. -func (fst fileTorrentImplIO) ReadAt(b []byte, off int64) (n int, err error) { - fst.fts.segmentLocater.Locate(segments.Extent{off, int64(len(b))}, func(i int, e segments.Extent) bool { - n1, err1 := fst.readFileAt(fst.fts.files[i], b[:e.Length], e.Start) - n += n1 - b = b[n1:] - err = err1 - return err == nil // && int64(n1) == e.Length - }) - if len(b) != 0 && err == nil { - err = io.EOF - } - return -} - -func (fst fileTorrentImplIO) WriteAt(p []byte, off int64) (n int, err error) { - // log.Printf("write at %v: %v bytes", off, len(p)) - fst.fts.segmentLocater.Locate(segments.Extent{off, int64(len(p))}, func(i int, e segments.Extent) bool { - name := fst.fts.files[i].path - os.MkdirAll(filepath.Dir(name), 0o777) - var f *os.File - f, err = os.OpenFile(name, os.O_WRONLY|os.O_CREATE, 0o666) - if err != nil { - return false - } - var n1 int - n1, err = f.WriteAt(p[:e.Length], e.Start) - // log.Printf("%v %v wrote %v: %v", i, e, n1, err) - closeErr := f.Close() - n += n1 - p = p[n1:] - if err == nil { - err = closeErr - } - if err == nil && int64(n1) != e.Length { - err = io.ErrShortWrite - } - return err == nil - }) - return -} diff --git a/internal/protocol/bt/file_misc.go b/internal/protocol/bt/file_misc.go deleted file mode 100644 index 5f2f4186..00000000 --- a/internal/protocol/bt/file_misc.go +++ /dev/null @@ -1,36 +0,0 @@ -// modify from github.com/anacrolix/torrent/storage/file_misc.go - -package bt - -import "github.com/anacrolix/torrent/metainfo" - -type requiredLength struct { - fileIndex int - length int64 -} - -func extentCompleteRequiredLengths(info *metainfo.Info, off, n int64) (ret []requiredLength) { - if n == 0 { - return - } - for i, fi := range info.UpvertedFiles() { - if off >= fi.Length { - off -= fi.Length - continue - } - n1 := n - if off+n1 > fi.Length { - n1 = fi.Length - off - } - ret = append(ret, requiredLength{ - fileIndex: i, - length: off + n1, - }) - n -= n1 - if n == 0 { - return - } - off = 0 - } - return -} diff --git a/internal/protocol/bt/file_piece.go b/internal/protocol/bt/file_piece.go deleted file mode 100644 index 9a1de1b7..00000000 --- a/internal/protocol/bt/file_piece.go +++ /dev/null @@ -1,61 +0,0 @@ -// modify from github.com/anacrolix/torrent/storage/file_piece.go - -package bt - -import ( - "github.com/anacrolix/torrent/metainfo" - "github.com/anacrolix/torrent/storage" - "io" - "log" - "os" -) - -type filePieceImpl struct { - *fileTorrentImpl - p metainfo.Piece - io.WriterAt - io.ReaderAt -} - -var _ storage.PieceImpl = (*filePieceImpl)(nil) - -func (fs *filePieceImpl) pieceKey() metainfo.PieceKey { - return metainfo.PieceKey{InfoHash: fs.infoHash, Index: fs.p.Index()} -} - -func (fs *filePieceImpl) Completion() storage.Completion { - c, err := fs.completion.Get(fs.pieceKey()) - if err != nil { - log.Printf("error getting piece completion: %s", err) - c.Ok = false - return c - } - - verified := true - if c.Complete { - // If it's allegedly complete, check that its constituent files have the necessary length. - for _, fi := range extentCompleteRequiredLengths(fs.p.Info, fs.p.Offset(), fs.p.Length()) { - s, err := os.Stat(fs.files[fi.fileIndex].path) - if err != nil || s.Size() < fi.length { - verified = false - break - } - } - } - - if !verified { - // The completion was wrong, fix it. - c.Complete = false - fs.completion.Set(fs.pieceKey(), false) - } - - return c -} - -func (fs *filePieceImpl) MarkComplete() error { - return fs.completion.Set(fs.pieceKey(), true) -} - -func (fs *filePieceImpl) MarkNotComplete() error { - return fs.completion.Set(fs.pieceKey(), false) -} diff --git a/internal/protocol/http/fetcher.go b/internal/protocol/http/fetcher.go index 909eb30e..4be4cd23 100644 --- a/internal/protocol/http/fetcher.go +++ b/internal/protocol/http/fetcher.go @@ -1,53 +1,71 @@ package http import ( - "bytes" "context" - "crypto/tls" "errors" "fmt" "io" - "mime" - "net" "net/http" - "net/http/cookiejar" "net/url" "os" "path" "strconv" "strings" "sync" + "sync/atomic" "time" - "unicode/utf8" "github.com/GopeedLab/gopeed/internal/controller" "github.com/GopeedLab/gopeed/internal/fetcher" "github.com/GopeedLab/gopeed/pkg/base" fhttp "github.com/GopeedLab/gopeed/pkg/protocol/http" - "github.com/GopeedLab/gopeed/pkg/util" "github.com/xiaoqidun/setft" - "golang.org/x/sync/errgroup" - "golang.org/x/text/encoding/simplifiedchinese" ) const ( connectTimeout = 15 * time.Second readTimeout = 15 * time.Second - helpMinSize = 1 * 1024 * 1024 + helpMinSize = 1 * 1024 * 1024 // Minimum chunk size for helper connections ) -type RequestError struct { - Code int - Msg string -} +// ============================================================================ +// State Machine +// ============================================================================ -func NewRequestError(code int, msg string) *RequestError { - return &RequestError{Code: code, Msg: msg} -} +type fetcherState int32 -func (re *RequestError) Error() string { - return fmt.Sprintf("http request fail,code:%d", re.Code) -} +const ( + stateIdle fetcherState = iota // Initial state + stateResolving // Resolving resource info + stateResolved // Resolved, waiting for Start or downloading + stateSlowStart // Slow-start phase: exponential connection growth + stateSteady // Steady state: max connections reached + statePaused // Paused + stateDone // Completed + stateError // Error occurred +) + +// ============================================================================ +// Connection +// ============================================================================ + +type connectionState int32 + +const ( + connNotStarted connectionState = iota // Not yet started + connConnecting // Sending HTTP request + connDownloading // HTTP response OK, downloading + connCompleted // Completed + connFailed // Failed +) + +type connectionRole int + +const ( + roleResolve connectionRole = iota // Resolve connection: initial probe + temp download + rolePrimary // Primary connection: first successful takeover from Resolve + roleWorker // Worker connection: subsequent connections +) type chunk struct { Begin int64 @@ -55,16 +73,6 @@ type chunk struct { Downloaded int64 } -type connection struct { - Chunk *chunk - Downloaded int64 - Completed bool - - failed bool - retryTimes int -} - -// get remain to download bytes func (c *chunk) remain() int64 { return c.End - c.Begin + 1 - c.Downloaded } @@ -76,20 +84,189 @@ func newChunk(begin int64, end int64) *chunk { } } +type connection struct { + ID int + Role connectionRole + State connectionState + Chunk *chunk + Downloaded int64 + Completed bool + + failed bool + retryTimes int + lastErr error + + ctx context.Context + cancel context.CancelFunc +} + +// ============================================================================ +// Slow Start Controller +// ============================================================================ + +type slowStartController struct { + mu sync.Mutex + maxConnections int + totalLaunched int + batchPending int // Connections in current batch waiting for HTTP response + batchReady int // Connections in current batch that succeeded + nextBatchSize int // Next batch size: 1, 2, 4, 8... + expansionCh chan struct{} // Signal to trigger next expansion + paused bool // Pause expansion (e.g., on 429) +} + +func newSlowStartController(maxConnections int) *slowStartController { + return &slowStartController{ + maxConnections: maxConnections, + nextBatchSize: 1, + expansionCh: make(chan struct{}, 1), + } +} + +// startBatch marks the start of a new batch with count connections +func (s *slowStartController) startBatch(count int) { + s.mu.Lock() + defer s.mu.Unlock() + s.batchPending = count + s.batchReady = 0 +} + +// onConnectSuccess is called when a connection successfully gets HTTP response +// Returns true if this completes the current batch +func (s *slowStartController) onConnectSuccess() bool { + s.mu.Lock() + defer s.mu.Unlock() + + s.batchReady++ + if s.batchReady >= s.batchPending { + // Batch complete, signal expansion + select { + case s.expansionCh <- struct{}{}: + default: + } + return true + } + return false +} + +// onConnectFailed is called when a connection fails +func (s *slowStartController) onConnectFailed() { + s.mu.Lock() + defer s.mu.Unlock() + + // Reduce pending count + if s.batchPending > 0 { + s.batchPending-- + } + // If all pending resolved (success or fail), trigger expansion + // This handles both successful completion and all-failures case + if s.batchPending == 0 { + select { + case s.expansionCh <- struct{}{}: + default: + } + } +} + +// getNextBatchSize returns how many connections to start in next batch +// Returns 0 if max reached +func (s *slowStartController) getNextBatchSize() int { + s.mu.Lock() + defer s.mu.Unlock() + + if s.paused { + return 0 + } + + remaining := s.maxConnections - s.totalLaunched + if remaining <= 0 { + return 0 + } + + batchSize := s.nextBatchSize + if batchSize > remaining { + batchSize = remaining + } + + return batchSize +} + +// commitBatch confirms that a batch of connections is being launched +func (s *slowStartController) commitBatch(count int) { + s.mu.Lock() + defer s.mu.Unlock() + + s.totalLaunched += count + s.nextBatchSize = s.nextBatchSize * 2 // Exponential growth: 1, 2, 4, 8... + s.batchPending = count + s.batchReady = 0 +} + +// isMaxReached returns true if max connections have been launched +func (s *slowStartController) isMaxReached() bool { + s.mu.Lock() + defer s.mu.Unlock() + return s.totalLaunched >= s.maxConnections +} + +// ============================================================================ +// Fetcher +// ============================================================================ + type Fetcher struct { ctl *controller.Controller config *config doneCh chan error - meta *fetcher.FetcherMeta - connections []*connection - helpLock sync.Mutex + meta *fetcher.FetcherMeta + + // State machine + state atomic.Int32 // fetcherState + + // Connections + connMu sync.Mutex + connections []*connection + resolveConn *connection // The special resolve connection + + // Slow start controller + slowStart *slowStartController + + // First primary connection success signal + primaryReadyOnce sync.Once + primaryReadyCh chan struct{} + + // Start pending mechanism + startPending atomic.Bool + resolvedCh chan struct{} // Signal when resolve completes + resolvedOnce sync.Once + resolveDataPos atomic.Int64 // How many bytes downloaded during resolve + + // Resolve response - kept open for one-time URLs + resolveResp *http.Response + resolveRespLock sync.Mutex + + // Async prefetch during resolve phase + prefetchFile *os.File // Temporary file for prefetch data + prefetchFilePath string // Path to temporary file + prefetchSize atomic.Int64 // Bytes prefetched so far + prefetchDone atomic.Bool // Prefetch completed or stopped + prefetchErr error // Error during prefetch (if any) + prefetchStopCh chan struct{} // Signal to stop prefetch + + // Target file + file *os.File + fileMu sync.Mutex redirectURL string redirectLock sync.Mutex - file *os.File + // Lifecycle control + ctx context.Context cancel context.CancelFunc - eg *errgroup.Group + wg sync.WaitGroup + + // Resolve connection control + resolveCtx context.Context + resolveCancel context.CancelFunc } func (f *Fetcher) Setup(ctl *controller.Controller) { @@ -99,96 +276,39 @@ func (f *Fetcher) Setup(ctl *controller.Controller) { f.meta = &fetcher.FetcherMeta{} } f.ctl.GetConfig(&f.config) - return + f.resolvedCh = make(chan struct{}) + f.primaryReadyCh = make(chan struct{}) + + // Check if this is a restore scenario (has existing connections or meta) + if f.meta.Res != nil { + // Already resolved, close the channel immediately + close(f.resolvedCh) + f.state.Store(int32(stateResolved)) + } else { + f.state.Store(int32(stateIdle)) + } } -func (f *Fetcher) Resolve(req *base.Request) error { +func (f *Fetcher) getState() fetcherState { + return fetcherState(f.state.Load()) +} + +func (f *Fetcher) setState(s fetcherState) { + f.state.Store(int32(s)) +} + +func (f *Fetcher) Resolve(req *base.Request, opts *base.Options) error { if err := base.ParseReqExtra[fhttp.ReqExtra](req); err != nil { return err } f.meta.Req = req - httpReq, err := f.buildRequest(nil, req) - if err != nil { - return err - } - client := f.buildClient() - // send Range request to check whether the server supports breakpoint continuation - // just test one byte, Range: bytes=0-0 - httpReq.Header.Set(base.HttpHeaderRange, fmt.Sprintf(base.HttpHeaderRangeFormat, 0, 0)) - httpResp, err := client.Do(httpReq) - if err != nil { - return err - } - // close response body immediately - httpResp.Body.Close() - res := &base.Resource{ - Range: false, - Files: []*base.FileInfo{}, - } - - if base.HttpCodePartialContent == httpResp.StatusCode || (base.HttpCodeOK == httpResp.StatusCode && httpResp.Header.Get(base.HttpHeaderAcceptRanges) == base.HttpHeaderBytes && strings.HasPrefix(httpResp.Header.Get(base.HttpHeaderContentRange), base.HttpHeaderBytes)) { - // response 206 status code, support breakpoint continuation - res.Range = true - // parse content length from Content-Range header, eg: bytes 0-1000/1001 or bytes 0-0/* - contentTotal := path.Base(httpResp.Header.Get(base.HttpHeaderContentRange)) - if contentTotal != "" && contentTotal != "*" { - parse, err := strconv.ParseInt(contentTotal, 10, 64) - if err != nil { - return err - } - res.Size = parse - } - } else if base.HttpCodeOK == httpResp.StatusCode { - // response 200 status code, not support breakpoint continuation, get file size by Content-Length header - // if not found, maybe chunked encoding - contentLength := httpResp.Header.Get(base.HttpHeaderContentLength) - if contentLength != "" { - parse, err := strconv.ParseInt(contentLength, 10, 64) - if err != nil { - return err - } - res.Size = parse - } - } else { - return NewRequestError(httpResp.StatusCode, httpResp.Status) - } - // Parse last modified time - var lastModifiedTime *time.Time - lastModified := httpResp.Header.Get(base.HttpHeaderLastModified) - if lastModified != "" { - // ignore parse error - t, _ := time.Parse(time.RFC1123, lastModified) - lastModifiedTime = &t - } - file := &base.FileInfo{ - Size: res.Size, - Ctime: lastModifiedTime, - } - contentDisposition := httpResp.Header.Get(base.HttpHeaderContentDisposition) - if contentDisposition != "" { - file.Name = parseFilename(contentDisposition) - } - // get file filePath by URL - if file.Name == "" { - file.Name = path.Base(httpReq.URL.Path) - // Url decode - use PathUnescape for paths to handle %2B correctly - if file.Name != "" { - file.Name, _ = url.PathUnescape(file.Name) - } - } - // unknown file filePath - if file.Name == "" || file.Name == "/" || file.Name == "." { - file.Name = httpReq.URL.Hostname() - } - res.Files = append(res.Files, file) - f.meta.Res = res - return nil -} - -func (f *Fetcher) Create(opts *base.Options) error { f.meta.Opts = opts + if f.meta.Opts == nil { + f.meta.Opts = &base.Options{} + } - if err := base.ParseOptsExtra[fhttp.OptsExtra](f.meta.Opts); err != nil { + // Parse options + if err := base.ParseOptExtra[fhttp.OptsExtra](opts); err != nil { return err } if opts.Extra == nil { @@ -197,59 +317,1171 @@ func (f *Fetcher) Create(opts *base.Options) error { extra := opts.Extra.(*fhttp.OptsExtra) if extra.Connections <= 0 { extra.Connections = f.config.Connections - // Avoid zero connections configuration if extra.Connections <= 0 { extra.Connections = 1 } } + + f.setState(stateResolving) + + // Build HTTP request WITHOUT Range header (normal request) + // This allows the response to be reused for downloading (important for one-time URLs) + httpReq, err := f.buildRequest(nil, req) + if err != nil { + f.setState(stateError) + return err + } + + client := f.buildClient() + + // Send normal HTTP request (no Range header) + resp, err := client.Do(httpReq) + if err != nil { + f.setState(stateError) + return err + } + + // Parse response to get resource info + res := &base.Resource{ + Range: false, + Files: []*base.FileInfo{}, + } + + if resp.StatusCode != base.HttpCodeOK && resp.StatusCode != base.HttpCodePartialContent { + resp.Body.Close() + f.setState(stateError) + return NewRequestError(resp.StatusCode) + } + + // Check if server supports range requests + acceptRanges := resp.Header.Get(base.HttpHeaderAcceptRanges) + contentRange := resp.Header.Get(base.HttpHeaderContentRange) + if acceptRanges == base.HttpHeaderBytes || strings.HasPrefix(contentRange, base.HttpHeaderBytes) { + res.Range = true + } + + // Get content length from Content-Length header + contentLength := resp.Header.Get(base.HttpHeaderContentLength) + if contentLength != "" { + parse, err := strconv.ParseInt(contentLength, 10, 64) + if err == nil { + res.Size = parse + } + } + + // Parse last modified time + var lastModifiedTime *time.Time + lastModified := resp.Header.Get(base.HttpHeaderLastModified) + if lastModified != "" { + t, _ := time.Parse(time.RFC1123, lastModified) + lastModifiedTime = &t + } + + file := &base.FileInfo{ + Size: res.Size, + Ctime: lastModifiedTime, + } + + // Parse filename + contentDisposition := resp.Header.Get(base.HttpHeaderContentDisposition) + if contentDisposition != "" { + file.Name = parseFilename(contentDisposition) + } + if file.Name == "" { + file.Name = path.Base(httpReq.URL.Path) + if file.Name != "" { + // Use PathUnescape instead of QueryUnescape to correctly handle %2B (should decode to +, not space) + file.Name, _ = url.PathUnescape(file.Name) + } + } + if file.Name == "" || file.Name == "/" || file.Name == "." { + file.Name = httpReq.URL.Hostname() + } + + res.Files = append(res.Files, file) + f.meta.Res = res + + // Save redirect URL for later connections + f.redirectURL = resp.Request.URL.String() + + // IMPORTANT: Keep the response body open for downloading in Start phase + // This is crucial for one-time URLs that can only be accessed once + f.resolveRespLock.Lock() + f.resolveResp = resp + f.resolveRespLock.Unlock() + + f.setState(stateResolved) + + // Signal that resolve is complete + f.resolvedOnce.Do(func() { + close(f.resolvedCh) + }) + + // Start async prefetch in background (only for range-supported resources) + // For non-range resources, the response will be used directly in Start + if res.Range && res.Size > 0 { + f.prefetchStopCh = make(chan struct{}) + go f.asyncPrefetch() + } + + // If start was called before resolve completed, auto-start + if f.startPending.Load() { + go f.doStart() + } + return nil } -func (f *Fetcher) Start() (err error) { +// asyncPrefetch downloads data in background during resolve phase +// This data can be reused when Start is called to save time +func (f *Fetcher) asyncPrefetch() { + defer func() { + f.prefetchDone.Store(true) + }() + + // Get the resolve response + f.resolveRespLock.Lock() + resp := f.resolveResp + f.resolveRespLock.Unlock() + + if resp == nil { + return + } + + // Create temporary file for prefetch data + tmpFile, err := os.CreateTemp("", "gopeed-prefetch-*") + if err != nil { + f.prefetchErr = err + return + } + f.prefetchFile = tmpFile + f.prefetchFilePath = tmpFile.Name() + + defer func() { + // Close response body when prefetch stops + f.resolveRespLock.Lock() + if f.resolveResp != nil { + f.resolveResp.Body.Close() + f.resolveResp = nil + } + f.resolveRespLock.Unlock() + }() + + buf := make([]byte, 32*1024) // 32KB buffer + reader := NewTimeoutReader(resp.Body, readTimeout) + + for { + select { + case <-f.prefetchStopCh: + // Stop signal received (Start was called) + return + default: + } + + n, err := reader.Read(buf) + if n > 0 { + _, writeErr := tmpFile.Write(buf[:n]) + if writeErr != nil { + f.prefetchErr = writeErr + return + } + f.prefetchSize.Add(int64(n)) + } + if err != nil { + if err == io.EOF { + // Prefetch completed + return + } + f.prefetchErr = err + return + } + } +} + +// stopPrefetchAndGetData stops the async prefetch and returns prefetched bytes +// It also copies prefetched data to the target file +func (f *Fetcher) stopPrefetchAndCopyData() int64 { + // Signal prefetch to stop (safely) + if f.prefetchStopCh != nil { + select { + case <-f.prefetchStopCh: + // Already closed + default: + close(f.prefetchStopCh) + } + } + + // Wait for prefetch to finish (with timeout) + for i := 0; i < 1000 && !f.prefetchDone.Load(); i++ { + time.Sleep(10 * time.Millisecond) + } + + prefetched := f.prefetchSize.Load() + if prefetched == 0 { + f.cleanupPrefetchFile() + return 0 + } + + // Copy prefetch data to target file + if f.prefetchFile != nil && f.file != nil { + // Seek to beginning of prefetch file + f.prefetchFile.Seek(0, io.SeekStart) + + // Copy to target file at position 0 + buf := make([]byte, 32*1024) + var copied int64 + for copied < prefetched { + n, err := f.prefetchFile.Read(buf) + if n > 0 { + f.file.WriteAt(buf[:n], copied) + copied += int64(n) + } + if err != nil { + break + } + } + } + + f.cleanupPrefetchFile() + return prefetched +} + +// cleanupPrefetchFile closes and removes the prefetch temporary file +func (f *Fetcher) cleanupPrefetchFile() { + if f.prefetchFile != nil { + f.prefetchFile.Close() + f.prefetchFile = nil + } + if f.prefetchFilePath != "" { + os.Remove(f.prefetchFilePath) + f.prefetchFilePath = "" + } +} + +func (f *Fetcher) Start() error { + state := f.getState() + + switch state { + case stateResolved, statePaused: + // Normal case: resolved or resuming from pause + return f.doStart() + + case stateResolving: + // Early start: mark pending and return immediately + f.startPending.Store(true) + return nil + + case stateSlowStart, stateSteady: + // Already downloading, this is a resume from pause + return f.doStart() + + default: + return fmt.Errorf("cannot start in current state: %v", state) + } +} + +func (f *Fetcher) doStart() error { + // Wait for resolve to complete + <-f.resolvedCh + + state := f.getState() + if state == stateDone || state == stateError { + return nil + } + + // Open or create target file first (needed for prefetch copy) name := f.meta.SingleFilepath() - // if file not exist, create it, else open it + var err error + var file *os.File _, err = os.Stat(name) if err != nil { if os.IsNotExist(err) { - f.file, err = f.ctl.Touch(name, f.meta.Res.Size) + file, err = f.ctl.Touch(name, f.meta.Res.Size) } else { - return + return err } } else { - f.file, err = os.OpenFile(name, os.O_RDWR, os.ModeAppend) + file, err = os.OpenFile(name, os.O_RDWR, os.ModeAppend) } if err != nil { return err } + f.fileMu.Lock() + f.file = file + f.fileMu.Unlock() + + // For range-supported resources, stop prefetch and copy data + // For non-range resources, the response will be used directly + var prefetchedBytes int64 + if f.meta.Res.Range { + // Stop async prefetch and copy data to target file + prefetchedBytes = f.stopPrefetchAndCopyData() + f.resolveDataPos.Store(prefetchedBytes) + + // Also close resolve response if still open + f.resolveRespLock.Lock() + if f.resolveResp != nil { + f.resolveResp.Body.Close() + f.resolveResp = nil + } + f.resolveRespLock.Unlock() + } // Avoid request extra modified by extension if err = base.ParseReqExtra[fhttp.ReqExtra](f.meta.Req); err != nil { return err } - if f.connections == nil { - f.connections = f.splitConnection() - } - f.redirectURL = "" - f.fetch() - return + // Initialize slow start controller + maxConns := f.meta.Opts.Extra.(*fhttp.OptsExtra).Connections + f.slowStart = newSlowStartController(maxConns) + + // Create main context + f.ctx, f.cancel = context.WithCancel(context.Background()) + + // Start download + f.setState(stateSlowStart) + go f.downloadLoop() + + return nil } -func (f *Fetcher) Pause() (err error) { - if f.cancel != nil { - f.cancel() - // wait for pause handle complete - f.eg.Wait() - f.file.Close() - } - return -} +func (f *Fetcher) downloadLoop() { + defer func() { + f.fileMu.Lock() + if f.file != nil { + f.file.Close() + } + f.fileMu.Unlock() -func (f *Fetcher) Close() (err error) { - if err = f.Pause(); err != nil { + // Update file last modified time + if f.config.UseServerCtime && f.meta.Res.Files[0].Ctime != nil { + setft.SetFileTime(f.meta.SingleFilepath(), time.Now(), *f.meta.Res.Files[0].Ctime, *f.meta.Res.Files[0].Ctime) + } + }() + + // Check if this is a resume or fresh start + isResume := len(f.connections) > 0 + + if !isResume { + // Fresh start: begin with resolve connection + f.startResolveDownload() + } else { + // Resume: restart existing connections + f.resumeConnections() + f.waitForCompletion() return } - return + + // Slow start loop + for { + select { + case <-f.ctx.Done(): + // Paused or cancelled + return + case <-f.slowStart.expansionCh: + // Batch completed, try to expand + if f.checkCompletion() { + return + } + f.expandConnections() + } + } +} + +func (f *Fetcher) startResolveDownload() { + // If no range support or size unknown, just use single connection with resolve response + if !f.meta.Res.Range || f.meta.Res.Size == 0 { + // Create a single connection for the entire file + conn := &connection{ + ID: 0, + Role: rolePrimary, + State: connNotStarted, + Chunk: newChunk(0, 0), // For non-range, end doesn't matter + } + conn.ctx, conn.cancel = context.WithCancel(f.ctx) + f.connections = append(f.connections, conn) + + f.wg.Add(1) + // Use the resolve response directly + go f.runConnectionWithResolveResp(conn) + + // For non-range downloads, wait for completion in the main downloadLoop + // by triggering immediate completion check + go func() { + f.wg.Wait() + // Signal expansion channel to trigger completion check in downloadLoop + select { + case f.slowStart.expansionCh <- struct{}{}: + default: + } + }() + return + } + + // Range supported: use slow start to launch connections + // Start first batch of connections + f.expandConnections() +} + +func (f *Fetcher) expandConnections() { + batchSize := f.slowStart.getNextBatchSize() + if batchSize <= 0 { + // Max reached, transition to steady state + f.setState(stateSteady) + go f.waitForCompletion() + return + } + + totalSize := f.meta.Res.Size + + f.connMu.Lock() + + // For first batch (no existing connections), allocate the remaining file to first connection + if len(f.connections) == 0 { + // Check if we have prefetched data + prefetched := f.resolveDataPos.Load() + + // If prefetched all data, mark as done + if prefetched >= totalSize { + f.connMu.Unlock() + f.setState(stateDone) + f.doneCh <- nil + return + } + + // First connection starts from prefetched position + conn := &connection{ + ID: 0, + Role: rolePrimary, + State: connNotStarted, + Chunk: newChunk(prefetched, totalSize-1), + } + // Mark prefetched bytes as already downloaded + conn.Chunk.Downloaded = 0 // Start fresh from prefetched position + conn.Downloaded = prefetched // Track total downloaded including prefetch + + conn.ctx, conn.cancel = context.WithCancel(f.ctx) + f.connections = append(f.connections, conn) + f.connMu.Unlock() + + f.slowStart.commitBatch(1) + f.wg.Add(1) + go f.runConnection(conn) + return + } + + // For subsequent batches, use "help other connection" strategy + // Find connections with enough remaining work to split + newConns := make([]*connection, 0, batchSize) + for i := 0; i < batchSize; i++ { + // Find the connection with most remaining work + var maxRemainConn *connection + var maxRemain int64 + + for _, conn := range f.connections { + if conn.Completed || conn.State == connFailed { + continue + } + remain := conn.Chunk.remain() + if remain > maxRemain && remain > helpMinSize*2 { + maxRemainConn = conn + maxRemain = remain + } + } + + if maxRemainConn == nil { + // No connection has enough work to split + break + } + + // Split the work: new connection takes the latter half + splitPoint := maxRemainConn.Chunk.End - maxRemainConn.Chunk.remain()/2 + newChunk := newChunk(splitPoint+1, maxRemainConn.Chunk.End) + maxRemainConn.Chunk.End = splitPoint + + connID := len(f.connections) + conn := &connection{ + ID: connID, + Role: roleWorker, + State: connNotStarted, + Chunk: newChunk, + } + conn.ctx, conn.cancel = context.WithCancel(f.ctx) + + newConns = append(newConns, conn) + f.connections = append(f.connections, conn) + } + + f.connMu.Unlock() + + if len(newConns) == 0 { + // No new connections could be created, stop expansion + f.setState(stateSteady) + go f.waitForCompletion() + return + } + + // Commit batch to slow start controller + f.slowStart.commitBatch(len(newConns)) + + // Launch connections + for _, conn := range newConns { + f.wg.Add(1) + go f.runConnection(conn) + } +} + +func (f *Fetcher) runConnection(conn *connection) { + defer f.wg.Done() + + f.connMu.Lock() + conn.State = connConnecting + f.connMu.Unlock() + + client := f.buildClient() + buf := make([]byte, 8192) + + retries := 0 + conn.retryTimes = 0 + + for { + err := f.downloadChunkOnce(conn, client, buf) + if err == nil { + if !f.meta.Res.Range || !f.helpOtherConnection(conn) { + f.connMu.Lock() + conn.Completed = true + conn.State = connCompleted + f.connMu.Unlock() + return + } + + // Reset counters after a successful help switch + retries = 0 + conn.retryTimes = 0 + continue + } + + if errors.Is(err, context.Canceled) { + return + } + + if re := extractRequestError(err); re != nil { + conn.lastErr = re + } else { + conn.lastErr = err + } + + if shouldCountHTTPFailure(err) { + if re := extractRequestError(err); re != nil && re.Code == 403 { + f.connMu.Lock() + conn.State = connFailed + conn.failed = true + f.connMu.Unlock() + if f.slowStart != nil { + f.slowStart.onConnectFailed() + } + return + } + conn.retryTimes++ + f.connMu.Lock() + conn.failed = true + f.connMu.Unlock() + if f.slowStart != nil { + f.slowStart.onConnectFailed() + } + if conn.retryTimes >= 3 { + f.connMu.Lock() + conn.State = connFailed + f.connMu.Unlock() + return + } + } + + f.connMu.Lock() + conn.State = connFailed + f.connMu.Unlock() + retryDelay := time.Second * time.Duration(retries+1) + if retryDelay > 5*time.Second { + retryDelay = 5 * time.Second + } + retries++ + time.Sleep(retryDelay) + } +} + +// downloadChunkOnce performs a single HTTP request for the current chunk without retrying. +func (f *Fetcher) downloadChunkOnce(conn *connection, client *http.Client, buf []byte) error { + if conn.ctx.Err() != nil { + return conn.ctx.Err() + } + + // Read chunk boundaries under lock to get a consistent snapshot + // This protects against concurrent modification by helpOtherConnection + f.connMu.Lock() + if f.meta.Res.Range && conn.Chunk.remain() <= 0 { + f.connMu.Unlock() + return nil + } + rangeStart := conn.Chunk.Begin + conn.Chunk.Downloaded + rangeEnd := conn.Chunk.End + f.connMu.Unlock() + + httpReq, err := f.buildRequest(conn.ctx, f.meta.Req) + if err != nil { + return err + } + + if f.meta.Res.Range { + httpReq.Header.Set(base.HttpHeaderRange, + fmt.Sprintf(base.HttpHeaderRangeFormat, rangeStart, rangeEnd)) + } + + resp, err := client.Do(httpReq) + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode != base.HttpCodeOK && resp.StatusCode != base.HttpCodePartialContent { + return NewRequestError(resp.StatusCode) + } + + f.connMu.Lock() + conn.State = connDownloading + conn.failed = false + f.connMu.Unlock() + + if conn.Role == rolePrimary || conn.ID == 0 { + f.primaryReadyOnce.Do(func() { + close(f.primaryReadyCh) + }) + } + if f.slowStart != nil { + f.slowStart.onConnectSuccess() + } + + reader := NewTimeoutReader(resp.Body, readTimeout) + for { + if conn.ctx.Err() != nil { + return conn.ctx.Err() + } + + n, err := reader.Read(buf) + if n > 0 { + finished := false + var writeOffset int64 + + // Lock to safely read chunk state and calculate write parameters + // This protects against concurrent chunk splitting by helpOtherConnection + f.connMu.Lock() + if f.meta.Res.Range { + // Check current chunk boundaries - this respects any concurrent chunk splitting + remain := conn.Chunk.remain() + if remain <= 0 { + // Chunk has been fully downloaded (possibly split and reduced) + f.connMu.Unlock() + return nil + } + if remain < int64(n) { + n = int(remain) + finished = true + } + } + writeOffset = conn.Chunk.Begin + conn.Chunk.Downloaded + f.connMu.Unlock() + + f.fileMu.Lock() + if f.file != nil { + _, writeErr := f.file.WriteAt(buf[:n], writeOffset) + if writeErr != nil { + f.fileMu.Unlock() + return writeErr + } + } + f.fileMu.Unlock() + + // Lock again to update Downloaded atomically with the read above + f.connMu.Lock() + conn.Chunk.Downloaded += int64(n) + conn.Downloaded += int64(n) + f.connMu.Unlock() + + if finished { + return nil + } + } + + if err != nil { + if err == io.EOF { + return nil + } + return err + } + } +} + +// runConnectionWithResolveResp uses the response body from Resolve phase +// This is crucial for one-time URLs that can only be accessed once +func (f *Fetcher) runConnectionWithResolveResp(conn *connection) { + defer f.wg.Done() + + f.connMu.Lock() + conn.State = connConnecting + f.connMu.Unlock() + + buf := make([]byte, 8192) + + // Get the resolve response + f.resolveRespLock.Lock() + resp := f.resolveResp + f.resolveResp = nil // Take ownership + f.resolveRespLock.Unlock() + + if resp == nil { + // No resolve response available, fall back to normal connection + f.runConnectionFallback(conn) + return + } + + defer resp.Body.Close() + + f.connMu.Lock() + conn.State = connDownloading + conn.failed = false + f.connMu.Unlock() + + // Signal primary ready + f.primaryReadyOnce.Do(func() { + close(f.primaryReadyCh) + }) + if f.slowStart != nil { + f.slowStart.onConnectSuccess() + } + + // Download data from resolve response + reader := NewTimeoutReader(resp.Body, readTimeout) + for { + if conn.ctx.Err() != nil { + return + } + + n, err := reader.Read(buf) + if n > 0 { + f.fileMu.Lock() + if f.file != nil { + _, writeErr := f.file.WriteAt(buf[:n], conn.Chunk.Downloaded) + if writeErr != nil { + f.fileMu.Unlock() + f.connMu.Lock() + conn.State = connFailed + conn.failed = true + f.connMu.Unlock() + if f.slowStart != nil { + f.slowStart.onConnectFailed() + } + return + } + } + f.fileMu.Unlock() + + conn.Chunk.Downloaded += int64(n) + conn.Downloaded += int64(n) + } + if err != nil { + if err == io.EOF { + f.connMu.Lock() + conn.Completed = true + conn.State = connCompleted + f.connMu.Unlock() + return + } + // Reading from resolve response failed: treat as transient (do not count as fail) + f.connMu.Lock() + conn.State = connFailed + f.connMu.Unlock() + return + } + } +} + +// runConnectionFallback is used when resolve response is not available +func (f *Fetcher) runConnectionFallback(conn *connection) { + client := f.buildClient() + buf := make([]byte, 8192) + + retries := 0 + countedRetries := 0 + + for { + if conn.ctx.Err() != nil { + return + } + + f.connMu.Lock() + conn.State = connConnecting + f.connMu.Unlock() + + err := func() error { + httpReq, err := f.buildRequest(conn.ctx, f.meta.Req) + if err != nil { + return err + } + + resp, err := client.Do(httpReq) + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode != base.HttpCodeOK && resp.StatusCode != base.HttpCodePartialContent { + return NewRequestError(resp.StatusCode) + } + + f.connMu.Lock() + conn.State = connDownloading + conn.failed = false + f.connMu.Unlock() + + f.primaryReadyOnce.Do(func() { + close(f.primaryReadyCh) + }) + if f.slowStart != nil { + f.slowStart.onConnectSuccess() + } + + reader := NewTimeoutReader(resp.Body, readTimeout) + for { + if conn.ctx.Err() != nil { + return conn.ctx.Err() + } + + n, err := reader.Read(buf) + if n > 0 { + f.fileMu.Lock() + if f.file != nil { + _, writeErr := f.file.WriteAt(buf[:n], conn.Chunk.Downloaded) + if writeErr != nil { + f.fileMu.Unlock() + return writeErr + } + } + f.fileMu.Unlock() + + conn.Chunk.Downloaded += int64(n) + conn.Downloaded += int64(n) + } + if err != nil { + if err == io.EOF { + return nil + } + return err + } + } + }() + + if err == nil { + f.connMu.Lock() + conn.Completed = true + conn.State = connCompleted + f.connMu.Unlock() + return + } + + if errors.Is(err, context.Canceled) { + return + } + + if re := extractRequestError(err); re != nil { + conn.lastErr = re + } else { + conn.lastErr = err + } + + if shouldCountHTTPFailure(err) { + // Immediate fail for server connection limit (403) + if re := extractRequestError(err); re != nil && re.Code == 403 { + f.connMu.Lock() + conn.State = connFailed + conn.failed = true + f.connMu.Unlock() + if f.slowStart != nil { + f.slowStart.onConnectFailed() + } + return + } + conn.retryTimes++ + countedRetries++ + if countedRetries >= 3 { + f.connMu.Lock() + conn.State = connFailed + conn.failed = true + f.connMu.Unlock() + if f.slowStart != nil { + f.slowStart.onConnectFailed() + } + return + } + // Retry again for counted failures below the cap + f.connMu.Lock() + conn.State = connFailed + f.connMu.Unlock() + retryDelay := time.Second * time.Duration(retries+1) + if retryDelay > 5*time.Second { + retryDelay = 5 * time.Second + } + retries++ + time.Sleep(retryDelay) + continue + } + + // Retry indefinitely for non-counted errors + f.connMu.Lock() + conn.State = connFailed + f.connMu.Unlock() + retryDelay := time.Second * time.Duration(retries+1) + if retryDelay > 5*time.Second { + retryDelay = 5 * time.Second + } + retries++ + time.Sleep(retryDelay) + } +} + +func (f *Fetcher) helpOtherConnection(helper *connection) bool { + f.connMu.Lock() + defer f.connMu.Unlock() + + // Find the slowest connection + var maxRemainConnection *connection + var maxRemain int64 + for _, r := range f.connections { + if r == helper || r.Completed || r.State == connFailed { + continue + } + + remain := r.Chunk.remain() + if remain > maxRemain && remain > helpMinSize { + maxRemainConnection = r + maxRemain = remain + } + } + + if maxRemainConnection == nil { + return false + } + + // Re-calculate the chunk range + helper.Chunk.Begin = maxRemainConnection.Chunk.End - maxRemainConnection.Chunk.remain()/2 + helper.Chunk.End = maxRemainConnection.Chunk.End + helper.Chunk.Downloaded = 0 + maxRemainConnection.Chunk.End = helper.Chunk.Begin - 1 + return true +} + +func (f *Fetcher) resumeConnections() { + // Collect connections to resume while holding the lock + var toResume []*connection + + f.connMu.Lock() + for _, conn := range f.connections { + // Only skip connections that have truly completed successfully + if conn.Completed || conn.State == connCompleted { + continue + } + // For failed connections, skip if: + // 1. They have exhausted retries (retryTimes >= 3), OR + // 2. They failed with a permanent error like 403 + if conn.State == connFailed && conn.failed { + // Check if it's a permanent error (like 403) + if re := extractRequestError(conn.lastErr); re != nil && re.Code == 403 { + continue + } + // Check if retries exhausted + if conn.retryTimes >= 3 { + continue + } + } + // Reset the connection state for resume + conn.ctx, conn.cancel = context.WithCancel(f.ctx) + conn.State = connNotStarted + conn.failed = false // Clear failed flag for resumed connection + toResume = append(toResume, conn) + } + f.connMu.Unlock() + + // Start connections outside the lock + for _, conn := range toResume { + f.wg.Add(1) + go f.runConnection(conn) + } +} + +func (f *Fetcher) waitForCompletion() { + f.wg.Wait() + // Only trigger completion if not cancelled/paused + if f.ctx != nil && f.ctx.Err() == nil { + f.onDownloadComplete() + } +} + +func (f *Fetcher) onDownloadComplete() { + f.connMu.Lock() + + // First, check if download actually completed successfully + // Calculate total downloaded from all connections + totalDownloaded := int64(0) + if f.resolveConn != nil { + totalDownloaded += f.resolveConn.Downloaded + } + for _, conn := range f.connections { + totalDownloaded += conn.Downloaded + } + + // Check if all chunks are complete (no remaining bytes) + allChunksComplete := true + for _, conn := range f.connections { + if conn.Chunk != nil && conn.Chunk.remain() > 0 && !conn.Completed && conn.State != connCompleted { + // This connection has remaining work and isn't done + // Check if it failed with 403 (server limit) - these can be ignored if other connections completed the work + if conn.State == connFailed && conn.failed { + if re := extractRequestError(conn.lastErr); re != nil && re.Code == 403 { + // 403 is server connection limit, check if other connections will complete this chunk + continue + } + } + allChunksComplete = false + break + } + } + + // If total downloaded matches file size, consider it a success regardless of connection failures + downloadComplete := f.meta.Res.Size > 0 && totalDownloaded >= f.meta.Res.Size + + // Check for any errors, but ignore 403 (server connection limit) errors if download completed + var finalErr error + if !downloadComplete && !allChunksComplete { + for _, conn := range f.connections { + if conn.State == connFailed && conn.failed { + // Skip 403 errors (server connection limit) - these are expected when exceeding server's limit + if re := extractRequestError(conn.lastErr); re != nil && re.Code == 403 { + continue + } + if re := extractRequestError(conn.lastErr); re != nil { + finalErr = fmt.Errorf("connection %d failed: retries=%d, status=%d", conn.ID, conn.retryTimes, re.Code) + } else if conn.lastErr != nil { + finalErr = fmt.Errorf("connection %d failed: retries=%d, err=%v", conn.ID, conn.retryTimes, conn.lastErr) + } else { + finalErr = fmt.Errorf("connection %d failed: retries=%d", conn.ID, conn.retryTimes) + } + break + } + } + } + f.connMu.Unlock() + + if finalErr != nil { + f.setState(stateError) + } else { + f.setState(stateDone) + } + + select { + case f.doneCh <- finalErr: + default: + } +} + +func (f *Fetcher) checkCompletion() bool { + // Check if all data has been downloaded + f.connMu.Lock() + defer f.connMu.Unlock() + + totalDownloaded := int64(0) + if f.resolveConn != nil { + totalDownloaded += f.resolveConn.Downloaded + } + for _, conn := range f.connections { + totalDownloaded += conn.Downloaded + } + + if f.meta.Res.Size > 0 && totalDownloaded >= f.meta.Res.Size { + f.setState(stateSteady) + go f.waitForCompletion() + return true + } + + // Check if all connections completed + allCompleted := true + if f.resolveConn != nil && !f.resolveConn.Completed && f.resolveConn.State != connCompleted { + allCompleted = false + } + for _, conn := range f.connections { + if !conn.Completed && conn.State != connCompleted && conn.State != connFailed { + allCompleted = false + break + } + } + + if allCompleted { + f.setState(stateSteady) + go f.waitForCompletion() + return true + } + + return false +} + +func (f *Fetcher) Pause() error { + if f.cancel != nil { + f.cancel() + } + if f.resolveCancel != nil { + f.resolveCancel() + } + + // Stop prefetch if running + if f.prefetchStopCh != nil { + select { + case <-f.prefetchStopCh: + // Already closed + default: + close(f.prefetchStopCh) + } + } + + // Wait for all goroutines to stop + f.wg.Wait() + + // Wait for prefetch to finish + for f.prefetchStopCh != nil && !f.prefetchDone.Load() { + time.Sleep(10 * time.Millisecond) + } + + // Clean up prefetch file + f.cleanupPrefetchFile() + + // Clean up resolve response if still held + f.resolveRespLock.Lock() + if f.resolveResp != nil { + f.resolveResp.Body.Close() + f.resolveResp = nil + } + f.resolveRespLock.Unlock() + + f.fileMu.Lock() + if f.file != nil { + f.file.Close() + f.file = nil + } + f.fileMu.Unlock() + + f.setState(statePaused) + return nil +} + +func (f *Fetcher) Close() error { + return f.Pause() } func (f *Fetcher) Meta() *fetcher.FetcherMeta { @@ -257,6 +1489,9 @@ func (f *Fetcher) Meta() *fetcher.FetcherMeta { } func (f *Fetcher) Stats() any { + f.connMu.Lock() + defer f.connMu.Unlock() + statsConnections := make([]*fhttp.StatsConnection, 0) for _, connection := range f.connections { statsConnections = append(statsConnections, &fhttp.StatsConnection{ @@ -273,675 +1508,22 @@ func (f *Fetcher) Stats() any { func (f *Fetcher) Progress() fetcher.Progress { p := make(fetcher.Progress, 0) - if len(f.connections) > 0 { - total := int64(0) - for _, connection := range f.connections { - total += connection.Downloaded - } - p = append(p, total) + + total := int64(0) + if f.resolveConn != nil { + total += f.resolveConn.Downloaded } + + f.connMu.Lock() + for _, conn := range f.connections { + total += conn.Downloaded + } + f.connMu.Unlock() + + p = append(p, total) return p } -func (f *Fetcher) Wait() (err error) { +func (f *Fetcher) Wait() error { return <-f.doneCh } - -func (f *Fetcher) fetch() { - var ctx context.Context - ctx, f.cancel = context.WithCancel(context.Background()) - f.eg, _ = errgroup.WithContext(ctx) - connectionErrs := make([]error, len(f.connections)) - for i := 0; i < len(f.connections); i++ { - i := i - f.eg.Go(func() error { - err := f.run(i, ctx) - // if canceled, fail fast - if errors.Is(err, context.Canceled) { - return err - } - connectionErrs[i] = err - return nil - }) - } - - go func() { - err := f.eg.Wait() - // error returned only if canceled, just return - if err != nil { - return - } - // check all fetch results, if any error, return - for _, chunkErr := range connectionErrs { - if chunkErr != nil { - err = chunkErr - break - } - } - - f.file.Close() - // Update file last modified time - if f.config.UseServerCtime && f.meta.Res.Files[0].Ctime != nil { - setft.SetFileTime(f.file.Name(), time.Now(), *f.meta.Res.Files[0].Ctime, *f.meta.Res.Files[0].Ctime) - } - f.doneCh <- err - }() -} - -func (f *Fetcher) run(index int, ctx context.Context) (err error) { - connection := f.connections[index] - connection.failed = false - connection.retryTimes = 0 - var ( - client = f.buildClient() - buf = make([]byte, 8192) - ) - - downloadChunk := func(chunk *chunk) (err error) { - // retry until all remain chunks failed - for { - // if chunk is completed, return - if f.meta.Res.Range && chunk.remain() <= 0 { - return nil - } - // if all chunks failed, return - if connection.failed { - allFailed := true - for _, c := range f.connections { - if c.Completed { - continue - } - if !c.failed { - allFailed = false - break - } - } - if allFailed { - if connection.retryTimes >= 3 { - return - } else { - connection.retryTimes++ - } - } - } - - err = func() error { - var ( - httpReq *http.Request - resp *http.Response - ) - f.redirectLock.Lock() - redirectURL := f.redirectURL - if redirectURL != "" { - f.redirectLock.Unlock() - } else { - // Only hold the lock when we need to potentially set redirectURL - defer func() { - if redirectURL == "" && err == nil { - f.redirectURL = resp.Request.URL.String() - } - f.redirectLock.Unlock() - }() - } - - err = func() (err error) { - httpReq, err = f.buildRequest(ctx, f.meta.Req) - if err != nil { - return - } - if f.meta.Res.Range { - httpReq.Header.Set(base.HttpHeaderRange, - fmt.Sprintf(base.HttpHeaderRangeFormat, chunk.Begin+chunk.Downloaded, chunk.End)) - } else { - chunk.Downloaded = 0 - } - resp, err = client.Do(httpReq) - if err != nil { - return - } - return - }() - if err != nil { - return err - } - - defer resp.Body.Close() - if resp.StatusCode != base.HttpCodeOK && resp.StatusCode != base.HttpCodePartialContent { - err = NewRequestError(resp.StatusCode, resp.Status) - return err - } - connection.failed = false - reader := NewTimeoutReader(resp.Body, readTimeout) - for { - n, err := reader.Read(buf) - if n > 0 { - finished := false - if f.meta.Res.Range { - remain := chunk.remain() - // If downloaded bytes exceed the remain bytes, only write remain bytes - if remain < int64(n) { - n = int(remain) - finished = true - } - } - - _, err := f.file.WriteAt(buf[:n], chunk.Begin+chunk.Downloaded) - if err != nil { - return err - } - chunk.Downloaded += int64(n) - connection.Downloaded += int64(n) - - if finished { - return nil - } - } - if err != nil { - if err == io.EOF { - return nil - } - return err - } - } - }() - if err != nil { - // If canceled, do not retry - if errors.Is(err, context.Canceled) { - return - } - // retry request after 1 second - connection.failed = true - time.Sleep(time.Second) - continue - } - break - } - return - } - - for { - if err = downloadChunk(connection.Chunk); err != nil { - return - } - - // check this connection is completed - if !f.meta.Res.Range || !f.helpOtherConnection(connection) { - connection.Completed = true - return - } - } -} - -func (f *Fetcher) helpOtherConnection(helper *connection) bool { - f.helpLock.Lock() - defer f.helpLock.Unlock() - - // find the slowest connection - var maxRemainConnection *connection - var maxRemain int64 - for _, r := range f.connections { - if r == helper || r.Completed { - continue - } - - remain := r.Chunk.remain() - if remain > maxRemain && remain > helpMinSize { - maxRemainConnection = r - maxRemain = remain - } - } - - if maxRemainConnection == nil { - return false - } - - // re-calculate the chunk range - helper.Chunk.Begin = maxRemainConnection.Chunk.End - maxRemainConnection.Chunk.remain()/2 - helper.Chunk.End = maxRemainConnection.Chunk.End - helper.Chunk.Downloaded = 0 - maxRemainConnection.Chunk.End = helper.Chunk.Begin - 1 - return true -} - -func (f *Fetcher) buildRequest(ctx context.Context, req *base.Request) (httpReq *http.Request, err error) { - var reqUrl string - if f.redirectURL != "" { - reqUrl = f.redirectURL - } else { - reqUrl = req.URL - } - - var ( - method string - body io.Reader - ) - headers := http.Header{} - if req.Extra == nil { - method = http.MethodGet - } else { - extra := req.Extra.(*fhttp.ReqExtra) - if extra.Method != "" { - method = extra.Method - } else { - method = http.MethodGet - } - if len(extra.Header) > 0 { - for k, v := range extra.Header { - headers.Set(k, strings.TrimSpace(v)) - } - } - if extra.Body != "" { - body = bytes.NewBufferString(extra.Body) - } - } - if _, ok := headers[base.HttpHeaderUserAgent]; !ok { - headers.Set(base.HttpHeaderUserAgent, strings.TrimSpace(f.config.UserAgent)) - } - - if ctx != nil { - httpReq, err = http.NewRequestWithContext(ctx, method, reqUrl, body) - } else { - httpReq, err = http.NewRequest(method, reqUrl, body) - } - if err != nil { - return - } - httpReq.Header = headers - // Override Host header - if host := headers.Get(base.HttpHeaderHost); host != "" { - httpReq.Host = host - } - return httpReq, nil -} - -func (f *Fetcher) splitConnection() (connections []*connection) { - if f.meta.Res.Range { - optConnections := f.meta.Opts.Extra.(*fhttp.OptsExtra).Connections - // 每个连接平均需要下载的分块大小 - chunkSize := f.meta.Res.Size / int64(optConnections) - connections = make([]*connection, optConnections) - for i := 0; i < optConnections; i++ { - var ( - begin = chunkSize * int64(i) - end int64 - ) - if i == optConnections-1 { - // 最后一个分块需要保证把文件下载完 - end = f.meta.Res.Size - 1 - } else { - end = begin + chunkSize - 1 - } - connections[i] = &connection{ - Chunk: newChunk(begin, end), - } - } - } else { - // 只支持单连接下载 - connections = make([]*connection, 1) - connections[0] = &connection{ - Chunk: newChunk(0, 0), - } - } - return -} - -func (f *Fetcher) buildClient() *http.Client { - transport := &http.Transport{ - DialContext: (&net.Dialer{ - Timeout: connectTimeout, - }).DialContext, - Proxy: f.ctl.GetProxy(f.meta.Req.Proxy), - TLSClientConfig: &tls.Config{ - InsecureSkipVerify: f.meta.Req.SkipVerifyCert, - }, - } - // Cookie handle - jar, _ := cookiejar.New(nil) - return &http.Client{ - Transport: transport, - Jar: jar, - } -} - -// parseFilename extracts filename from Content-Disposition header -// It handles multiple encoding scenarios: -// 1. RFC 5987/RFC 2231 format: filename*=UTF-8''%E6%B5%8B%E8%AF%95.zip (preferred, checked first) -// 2. MIME encoded-word: filename="=?UTF-8?B?5rWL6K+VLnppcA==?=" -// 3. URL-encoded: filename="%E6%B5%8B%E8%AF%95.zip" -// 4. Plain ASCII filename -// -// The key fix is checking filename*= first before mime.ParseMediaType, because -// some servers send Content-Disposition headers with invalid characters that cause -// mime.ParseMediaType to fail, but the filename*= parameter is still valid. -func parseFilename(contentDisposition string) string { - // First, try to find filename*= (RFC 5987 format, most reliable for non-ASCII) - if filename := parseFilenameExtended(contentDisposition); filename != "" { - return filename - } - - // Try standard MIME parsing for regular filename= parameter - _, params, err := mime.ParseMediaType(contentDisposition) - if err == nil { - if filename := params["filename"]; filename != "" { - return decodeFilenameParam(filename) - } - } - - // Fallback: manual parsing if mime.ParseMediaType fails - return parseFilenameFallback(contentDisposition) -} - -// findParamValueEnd finds the end position of a parameter value in a Content-Disposition header. -// It correctly handles quoted values where semicolons inside quotes should not be treated as delimiters. -// It also handles HTML entities in unquoted values (e.g., & should not be split at the semicolon). -// Returns the end index (exclusive) of the value, or -1 if it extends to the end of the string. -func findParamValueEnd(value string) int { - value = strings.TrimSpace(value) - if len(value) == 0 { - return 0 - } - - // If the value starts with a quote, find the matching closing quote - if value[0] == '"' || value[0] == '\'' { - quote := value[0] - // Find the closing quote, handling escaped quotes - for i := 1; i < len(value); i++ { - if value[i] == quote { - // Check if it's escaped - if i > 0 && value[i-1] == '\\' { - continue - } - // Found closing quote, now look for ; after it - remaining := value[i+1:] - if semiIdx := strings.Index(remaining, ";"); semiIdx != -1 { - return i + 1 + semiIdx - } - return -1 // No semicolon after closing quote - } - } - // No closing quote found, treat rest of string as value - return -1 - } - - // Unquoted value - find the next semicolon that's not part of an HTML entity - // HTML entities have the pattern &...; (e.g., & < > " ') - for i := 0; i < len(value); i++ { - if value[i] == ';' { - // Check if this semicolon is part of an HTML entity - // Look backwards for an & character - isEntity := false - if i > 0 { - // Look for & before this semicolon (within reasonable distance, max 10 chars) - for j := i - 1; j >= 0 && j >= i-10; j-- { - if value[j] == '&' { - // Found &, this semicolon might be part of an HTML entity - // Check if there are only alphanumeric or # between & and ; - entityChars := value[j+1 : i] - if len(entityChars) > 0 && isValidHTMLEntityChars(entityChars) { - isEntity = true - } - break - } - // If we hit whitespace or another special char, stop looking - if value[j] == ' ' || value[j] == '"' || value[j] == '\'' { - break - } - } - } - - if !isEntity { - return i - } - } - } - return -1 // No semicolon, extends to end -} - -// isValidHTMLEntityChars checks if a string contains only valid HTML entity characters -// (alphanumeric and #, typically for entities like & < ' etc.) -func isValidHTMLEntityChars(s string) bool { - if len(s) == 0 { - return false - } - for _, c := range s { - if !((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '#') { - return false - } - } - return true -} - -// parseFilenameExtended parses RFC 5987/RFC 2231 extended parameter format -// Format: filename*=charset'language'value (e.g., UTF-8''%E6%B5%8B%E8%AF%95.zip) -func parseFilenameExtended(cd string) string { - // Look for filename*= (case-insensitive) - lower := strings.ToLower(cd) - idx := strings.Index(lower, "filename*=") - if idx == -1 { - return "" - } - - // Extract the value after filename*= - value := cd[idx+len("filename*="):] - - // Find the end of the value using proper quote handling - endIdx := findParamValueEnd(value) - if endIdx != -1 { - value = value[:endIdx] - } - value = strings.TrimSpace(value) - - // Parse charset'language'encoded-value format - // Common format: UTF-8''%E6%B5%8B%E8%AF%95.zip - parts := strings.SplitN(value, "''", 2) - if len(parts) == 2 { - // parts[0] is charset (e.g., "UTF-8") - // parts[1] is percent-encoded value - // Use PathUnescape to handle %2B correctly (should decode to +, not space) - decoded, err := url.PathUnescape(parts[1]) - if err == nil { - return decoded - } - } - - // Try with single quote delimiter as well (some servers use this) - parts = strings.SplitN(value, "'", 3) - if len(parts) >= 3 { - // Use PathUnescape to handle %2B correctly (should decode to +, not space) - decoded, err := url.PathUnescape(parts[2]) - if err == nil { - return decoded - } - } - - return "" -} - -// decodeFilenameParam decodes a filename parameter value -// Handles HTML entities, MIME encoded-word, URL encoding, and GBK encoding fallback -func decodeFilenameParam(filename string) string { - // First, unescape HTML entities (e.g., & -> &, < -> <, > -> >) - // This must be done before other decoding to handle cases where servers - // HTML-encode special characters in filenames - filename = unescapeHTMLEntities(filename) - - // Check if the filename is MIME encoded-word (e.g., =?UTF-8?B?...?=) - if strings.HasPrefix(filename, "=?") { - decoder := new(mime.WordDecoder) - // Some servers use "UTF8" instead of "UTF-8", create a normalized copy - normalizedFilename := strings.Replace(filename, "UTF8", "UTF-8", 1) - if decoded, err := decoder.Decode(normalizedFilename); err == nil { - return decoded - } - } - - // Try URL decoding - use PathUnescape for filenames to handle %2B correctly - decoded := util.TryUrlPathUnescape(filename) - - // Check if the result is valid UTF-8. If not, try GBK decoding. - // This handles the case where Chinese Windows servers send GBK-encoded filenames - // which appear as garbled characters (e.g., "下载地址.zip" -> "���ص�ַ.zip") - if !utf8.ValidString(decoded) { - if gbkDecoded := tryDecodeGBK(decoded); gbkDecoded != "" { - return gbkDecoded - } - } - - return decoded -} - -// unescapeHTMLEntities unescapes common HTML entities in filenames -// This handles cases where servers HTML-encode special characters like & to & -func unescapeHTMLEntities(s string) string { - // Common HTML entities that might appear in filenames - replacements := map[string]string{ - "&": "&", - "<": "<", - ">": ">", - """: "\"", - "'": "'", - "'": "'", - } - - result := s - for entity, char := range replacements { - result = strings.ReplaceAll(result, entity, char) - } - return result -} - -// gbkDecoder is a reusable GBK decoder for better performance -var gbkDecoder = simplifiedchinese.GBK.NewDecoder() - -// tryDecodeGBK attempts to decode a string as GBK/GB2312/GB18030 encoding -// Returns empty string if decoding fails or result is not valid UTF-8 -func tryDecodeGBK(s string) string { - // GBK uses 1-2 bytes per character. Single-byte chars are 0x00-0x7F (ASCII compatible). - // Double-byte chars have first byte 0x81-0xFE and second byte 0x40-0xFE. - // Skip if string is empty or all ASCII (valid UTF-8) - if len(s) == 0 { - return "" - } - - // Create a fresh decoder since the transform state may be corrupted - decoder := simplifiedchinese.GBK.NewDecoder() - decoded, err := decoder.Bytes([]byte(s)) - if err != nil { - return "" - } - result := string(decoded) - if utf8.ValidString(result) { - return result - } - return "" -} - -// parseFilenameFallback manually parses filename= when mime.ParseMediaType fails -func parseFilenameFallback(cd string) string { - // Look for filename= (case-insensitive) - lower := strings.ToLower(cd) - idx := strings.Index(lower, "filename=") - if idx == -1 { - return "" - } - - // Skip "filename=" prefix - value := cd[idx+len("filename="):] - - // Find the end of the value using proper quote handling - endIdx := findParamValueEnd(value) - if endIdx != -1 { - value = value[:endIdx] - } - value = strings.TrimSpace(value) - - // Remove quotes if present - if len(value) >= 2 { - if (value[0] == '"' && value[len(value)-1] == '"') || - (value[0] == '\'' && value[len(value)-1] == '\'') { - value = value[1 : len(value)-1] - } - } - - return decodeFilenameParam(value) -} - -type fetcherData struct { - Connections []*connection -} - -type FetcherManager struct { -} - -func (fm *FetcherManager) Name() string { - return "http" -} - -func (fm *FetcherManager) Filters() []*fetcher.SchemeFilter { - return []*fetcher.SchemeFilter{ - { - Type: fetcher.FilterTypeUrl, - Pattern: "HTTP", - }, - { - Type: fetcher.FilterTypeUrl, - Pattern: "HTTPS", - }, - } -} - -func (fm *FetcherManager) Build() fetcher.Fetcher { - return &Fetcher{} -} - -func (fm *FetcherManager) ParseName(u string) string { - var name string - url, err := url.Parse(u) - if err != nil { - return "" - } - // Get filePath by URL - name = path.Base(url.Path) - // If file name is empty, use host name - if name == "" || name == "/" || name == "." { - name = url.Hostname() - } - return name -} - -func (fm *FetcherManager) AutoRename() bool { - return true -} - -func (fm *FetcherManager) DefaultConfig() any { - return &config{ - UserAgent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36", - Connections: 16, - } -} - -func (fm *FetcherManager) Store(f fetcher.Fetcher) (data any, err error) { - _f := f.(*Fetcher) - return &fetcherData{ - Connections: _f.connections, - }, nil -} - -func (fm *FetcherManager) Restore() (v any, f func(meta *fetcher.FetcherMeta, v any) fetcher.Fetcher) { - return &fetcherData{}, func(meta *fetcher.FetcherMeta, v any) fetcher.Fetcher { - fd := v.(*fetcherData) - fb := &FetcherManager{} - fetcher := fb.Build().(*Fetcher) - fetcher.meta = meta - base.ParseReqExtra[fhttp.ReqExtra](fetcher.meta.Req) - base.ParseOptsExtra[fhttp.OptsExtra](fetcher.meta.Opts) - if len(fd.Connections) > 0 { - fetcher.connections = fd.Connections - } - return fetcher - } -} - -func (fm *FetcherManager) Close() error { - return nil -} diff --git a/internal/protocol/http/fetcher_manager.go b/internal/protocol/http/fetcher_manager.go new file mode 100644 index 00000000..11f01b37 --- /dev/null +++ b/internal/protocol/http/fetcher_manager.go @@ -0,0 +1,96 @@ +package http + +import ( + "net/url" + "path" + + "github.com/GopeedLab/gopeed/internal/fetcher" + "github.com/GopeedLab/gopeed/pkg/base" + fhttp "github.com/GopeedLab/gopeed/pkg/protocol/http" +) + +// ============================================================================ +// Fetcher Data (for Store/Restore) +// ============================================================================ + +type fetcherData struct { + Connections []*connection +} + +// ============================================================================ +// Fetcher Manager +// ============================================================================ + +type FetcherManager struct { +} + +func (fm *FetcherManager) Name() string { + return "http" +} + +func (fm *FetcherManager) Filters() []*fetcher.SchemeFilter { + return []*fetcher.SchemeFilter{ + { + Type: fetcher.FilterTypeUrl, + Pattern: "HTTP", + }, + { + Type: fetcher.FilterTypeUrl, + Pattern: "HTTPS", + }, + } +} + +func (fm *FetcherManager) Build() fetcher.Fetcher { + return &Fetcher{} +} + +func (fm *FetcherManager) ParseName(u string) string { + var name string + url, err := url.Parse(u) + if err != nil { + return "" + } + name = path.Base(url.Path) + if name == "" || name == "/" || name == "." { + name = url.Hostname() + } + return name +} + +func (fm *FetcherManager) AutoRename() bool { + return true +} + +func (fm *FetcherManager) DefaultConfig() any { + return &config{ + UserAgent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36", + Connections: 16, + } +} + +func (fm *FetcherManager) Store(f fetcher.Fetcher) (data any, err error) { + _f := f.(*Fetcher) + return &fetcherData{ + Connections: _f.connections, + }, nil +} + +func (fm *FetcherManager) Restore() (v any, f func(meta *fetcher.FetcherMeta, v any) fetcher.Fetcher) { + return &fetcherData{}, func(meta *fetcher.FetcherMeta, v any) fetcher.Fetcher { + fd := v.(*fetcherData) + fb := &FetcherManager{} + fetcher := fb.Build().(*Fetcher) + fetcher.meta = meta + base.ParseReqExtra[fhttp.ReqExtra](fetcher.meta.Req) + base.ParseOptExtra[fhttp.OptsExtra](fetcher.meta.Opts) + if len(fd.Connections) > 0 { + fetcher.connections = fd.Connections + } + return fetcher + } +} + +func (fm *FetcherManager) Close() error { + return nil +} diff --git a/internal/protocol/http/fetcher_test.go b/internal/protocol/http/fetcher_test.go index c4a57d83..c6e41278 100644 --- a/internal/protocol/http/fetcher_test.go +++ b/internal/protocol/http/fetcher_test.go @@ -3,188 +3,221 @@ package http import ( "encoding/json" "fmt" + "net" + gohttp "net/http" + "net/url" + "os" + "strings" + "testing" + "time" + "github.com/GopeedLab/gopeed/internal/controller" "github.com/GopeedLab/gopeed/internal/fetcher" "github.com/GopeedLab/gopeed/internal/test" "github.com/GopeedLab/gopeed/pkg/base" "github.com/GopeedLab/gopeed/pkg/protocol/http" - "net" - gohttp "net/http" - "net/url" - "strings" - "testing" - "time" ) func TestFetcher_Resolve(t *testing.T) { - testResolve(test.StartTestFileServer, test.BuildName, &base.Resource{ - Size: test.BuildSize, - Range: true, - Files: []*base.FileInfo{ - { - Name: test.BuildName, - Size: test.BuildSize, + testResolve(test.StartTestFileServer, test.BuildName, t, func(err error) (*base.Resource, error) { + return &base.Resource{ + Size: test.BuildSize, + Range: true, + Files: []*base.FileInfo{ + { + Name: test.BuildName, + Size: test.BuildSize, + }, }, - }, - }, t) - testResolve(test.StartTestCustomServer, "disposition", &base.Resource{ - Size: test.BuildSize, - Range: false, - Files: []*base.FileInfo{ - { - Name: test.BuildName, - Size: test.BuildSize, + }, nil + }) + testResolve(test.StartTestCustomServer, "disposition", t, func(err error) (*base.Resource, error) { + return &base.Resource{ + Size: test.BuildSize, + Range: false, + Files: []*base.FileInfo{ + { + Name: test.BuildName, + Size: test.BuildSize, + }, }, - }, - }, t) - testResolve(test.StartTestCustomServer, "encoded-word", &base.Resource{ - Size: test.BuildSize, - Range: false, - Files: []*base.FileInfo{ - { - Name: test.TestChineseFileName, - Size: test.BuildSize, + }, nil + }) + testResolve(test.StartTestCustomServer, "encoded-word", t, func(err error) (*base.Resource, error) { + return &base.Resource{ + Size: test.BuildSize, + Range: false, + Files: []*base.FileInfo{ + { + Name: test.TestChineseFileName, + Size: test.BuildSize, + }, }, - }, - }, t) - testResolve(test.StartTestCustomServer, "no-encode", &base.Resource{ - Size: test.BuildSize, - Range: false, - Files: []*base.FileInfo{ - { - Name: test.TestChineseFileName, - Size: test.BuildSize, + }, nil + }) + testResolve(test.StartTestCustomServer, "no-encode", t, func(err error) (*base.Resource, error) { + return &base.Resource{ + Size: test.BuildSize, + Range: false, + Files: []*base.FileInfo{ + { + Name: test.TestChineseFileName, + Size: test.BuildSize, + }, }, - }, - }, t) - testResolve(test.StartTestCustomServer, "%E6%B5%8B%E8%AF%95.zip", &base.Resource{ - Size: 0, - Range: false, - Files: []*base.FileInfo{ - { - Name: test.TestChineseFileName, - Size: 0, + }, nil + }) + testResolve(test.StartTestCustomServer, "%E6%B5%8B%E8%AF%95.zip", t, func(err error) (*base.Resource, error) { + return &base.Resource{ + Size: 0, + Range: false, + Files: []*base.FileInfo{ + { + Name: test.TestChineseFileName, + Size: 0, + }, }, - }, - }, t) - testResolve(test.StartTestCustomServer, test.BuildName, &base.Resource{ - Size: 0, - Range: false, - Files: []*base.FileInfo{ - { - Name: test.BuildName, - Size: 0, + }, nil + }) + testResolve(test.StartTestCustomServer, test.BuildName, t, func(err error) (*base.Resource, error) { + return &base.Resource{ + Size: 0, + Range: false, + Files: []*base.FileInfo{ + { + Name: test.BuildName, + Size: 0, + }, }, - }, - }, t) + }, nil + }) // Test mixed encoding Content-Disposition where mime.ParseMediaType fails // due to invalid characters, but filename*= contains the correct UTF-8 encoded name - testResolve(test.StartTestCustomServer, "mixed-encoding", &base.Resource{ - Size: test.BuildSize, - Range: false, - Files: []*base.FileInfo{ - { - Name: test.TestChineseFileName, - Size: test.BuildSize, + testResolve(test.StartTestCustomServer, "mixed-encoding", t, func(err error) (*base.Resource, error) { + return &base.Resource{ + Size: test.BuildSize, + Range: false, + Files: []*base.FileInfo{ + { + Name: test.TestChineseFileName, + Size: test.BuildSize, + }, }, - }, - }, t) + }, nil + }) // Test filename*= only (RFC 5987 format) - testResolve(test.StartTestCustomServer, "filename-star", &base.Resource{ - Size: test.BuildSize, - Range: false, - Files: []*base.FileInfo{ - { - Name: test.TestChineseFileName, - Size: test.BuildSize, + testResolve(test.StartTestCustomServer, "filename-star", t, func(err error) (*base.Resource, error) { + return &base.Resource{ + Size: test.BuildSize, + Range: false, + Files: []*base.FileInfo{ + { + Name: test.TestChineseFileName, + Size: test.BuildSize, + }, }, - }, - }, t) + }, nil + }) // Test GBK-encoded filename (common on Chinese Windows servers) // Before fix: "测试.zip" sent as GBK bytes -> parsed as "²âÊÔ.zip" (garbled) // After fix: correctly decoded back to "测试.zip" - testResolve(test.StartTestCustomServer, "gbk-encoded", &base.Resource{ - Size: test.BuildSize, - Range: false, - Files: []*base.FileInfo{ - { - Name: test.TestChineseFileName, - Size: test.BuildSize, + testResolve(test.StartTestCustomServer, "gbk-encoded", t, func(err error) (*base.Resource, error) { + return &base.Resource{ + Size: test.BuildSize, + Range: false, + Files: []*base.FileInfo{ + { + Name: test.TestChineseFileName, + Size: test.BuildSize, + }, }, - }, - }, t) + }, nil + }) // Test filename with plus signs (e.g., C++ Primer) // Before fix: %2B decoded to space -> "C++ Primer" became "C Primer" // After fix: %2B correctly decoded to + -> "C++ Primer Plus.mobi" - testResolve(test.StartTestCustomServer, "plus-sign-encoded", &base.Resource{ - Size: test.BuildSize, - Range: false, - Files: []*base.FileInfo{ - { - Name: "C++ Primer Plus.mobi", - Size: test.BuildSize, + testResolve(test.StartTestCustomServer, "plus-sign-encoded", t, func(err error) (*base.Resource, error) { + return &base.Resource{ + Size: test.BuildSize, + Range: false, + Files: []*base.FileInfo{ + { + Name: "C++ Primer Plus.mobi", + Size: test.BuildSize, + }, }, - }, - }, t) + }, nil + }) // Test filename with plus sign in URL path // Before fix: %2B decoded to space // After fix: %2B correctly decoded to + - testResolve(test.StartTestCustomServer, "C%2B%2B%20Primer.txt", &base.Resource{ - Size: 0, - Range: false, - Files: []*base.FileInfo{ - { - Name: "C++ Primer.txt", - Size: 0, + testResolve(test.StartTestCustomServer, "C%2B%2B%20Primer.txt", t, func(err error) (*base.Resource, error) { + return &base.Resource{ + Size: 0, + Range: false, + Files: []*base.FileInfo{ + { + Name: "C++ Primer.txt", + Size: 0, + }, }, - }, - }, t) + }, nil + }) // Test filename with HTML-encoded ampersand (fixes issue with & being truncated) // Before fix: "查询处理&优化.pptx" -> "查询处理&" (truncated at semicolon) // After fix: correctly decoded to "查询处理&优化.pptx" - testResolve(test.StartTestCustomServer, "ampersand-encoded", &base.Resource{ - Size: test.BuildSize, - Range: false, - Files: []*base.FileInfo{ - { - Name: "查询处理&优化.pptx", - Size: test.BuildSize, + testResolve(test.StartTestCustomServer, "ampersand-encoded", t, func(err error) (*base.Resource, error) { + return &base.Resource{ + Size: test.BuildSize, + Range: false, + Files: []*base.FileInfo{ + { + Name: "查询处理&优化.pptx", + Size: test.BuildSize, + }, }, - }, - }, t) + }, nil + }) // Test unquoted filename with HTML-encoded ampersand - testResolve(test.StartTestCustomServer, "ampersand-unquoted", &base.Resource{ - Size: test.BuildSize, - Range: false, - Files: []*base.FileInfo{ - { - Name: "test&file.txt", - Size: test.BuildSize, + testResolve(test.StartTestCustomServer, "ampersand-unquoted", t, func(err error) (*base.Resource, error) { + return &base.Resource{ + Size: test.BuildSize, + Range: false, + Files: []*base.FileInfo{ + { + Name: "test&file.txt", + Size: test.BuildSize, + }, }, - }, - }, t) - + }, nil + }) // Test URL without file path - should use domain/host as filename - listener := test.StartTestRootServer() - defer listener.Close() - fetcher := buildFetcher() - err := fetcher.Resolve(&base.Request{ - URL: "http://" + listener.Addr().String() + "/", + testResolve(test.StartTestCustomServer, "", t, func(err error) (*base.Resource, error) { + return &base.Resource{ + Size: 0, + Range: false, + Files: []*base.FileInfo{ + { + Name: "127.0.0.1", + Size: 0, + }, + }, + }, nil + }) + // Test 403 Forbidden response handling + testResolve(test.StartTestCustomServer, "forbidden", t, func(err error) (*base.Resource, error) { + requestError := extractRequestError(err) + if requestError != nil && requestError.Code == 403 { + return nil, nil + } + return nil, err }) - if err != nil { - t.Fatal(err) - } - // When no filename is provided, it should use the hostname (without port) as the name - expectedName := "127.0.0.1" - if fetcher.Meta().Res.Files[0].Name != expectedName { - t.Errorf("Resolve() got name = %v, want %v", fetcher.Meta().Res.Files[0].Name, expectedName) - } } func TestFetcher_ResolveWithHostHeader(t *testing.T) { listener := test.StartTestHostHeaderServer() defer listener.Close() - + fetcher := buildFetcher() err := fetcher.Resolve(&base.Request{ URL: "http://" + listener.Addr().String() + "/", @@ -193,6 +226,9 @@ func TestFetcher_ResolveWithHostHeader(t *testing.T) { "Host": "test", }, }, + }, &base.Options{ + Name: test.DownloadName, + Path: test.Dir, }) // The server should return 400 for invalid Host header if err == nil || !strings.Contains(err.Error(), "400") { @@ -201,10 +237,11 @@ func TestFetcher_ResolveWithHostHeader(t *testing.T) { } func TestFetcher_ResolveWithInvalidHeader(t *testing.T) { - listener := test.StartTestRootServer() + listener := test.StartTestCustomServer() defer listener.Close() - + fetcher := buildFetcher() + defer fetcher.Pause() // Close the resolve response to allow server shutdown err := fetcher.Resolve(&base.Request{ URL: "http://" + listener.Addr().String() + "/", Extra: &http.ReqExtra{ @@ -212,6 +249,9 @@ func TestFetcher_ResolveWithInvalidHeader(t *testing.T) { "Referer": "\rtest", }, }, + }, &base.Options{ + Name: test.DownloadName, + Path: test.Dir, }) // Invalid header with \r should be sanitized by Go's http client, allowing the request to succeed if err != nil { @@ -219,18 +259,23 @@ func TestFetcher_ResolveWithInvalidHeader(t *testing.T) { } } -func testResolve(startTestServer func() net.Listener, path string, want *base.Resource, t *testing.T) { +func testResolve(startTestServer func() net.Listener, path string, t *testing.T, wantFn func(error) (*base.Resource, error)) { listener := startTestServer() defer listener.Close() fetcher := buildFetcher() + defer fetcher.Pause() // Close the resolve response to allow server shutdown err := fetcher.Resolve(&base.Request{ URL: "http://" + listener.Addr().String() + "/" + path, + }, &base.Options{ + Name: test.DownloadName, + Path: test.Dir, }) + want, err := wantFn(err) if err != nil { t.Fatal(err) } - if !test.AssertResourceEqual(want, fetcher.meta.Res) { - t.Errorf("Resolve() got = %v, want %v", fetcher.meta.Res, want) + if want != nil && !test.AssertResourceEqual(want, fetcher.meta.Res) { + t.Errorf("Resolve() got = %+v, want %+v", fetcher.meta.Res, want) } } @@ -293,11 +338,74 @@ func TestFetcher_DownloadLimit(t *testing.T) { } func TestFetcher_DownloadResponseBodyReadTimeout(t *testing.T) { - listener := test.StartTestLimitServer(16, readTimeout.Milliseconds()+5000) + // Server will timeout once (first request delays longer than readTimeout), + // then subsequent requests work normally + listener := test.StartTestTimeoutOnceServer(readTimeout.Milliseconds() + 5000) defer listener.Close() - downloadError(listener, 1, t) - downloadError(listener, 4, t) + for _, connections := range []int{1, 4} { + os.Remove(test.DownloadFile) + + fetcher := downloadReady(listener, connections, t) + if err := fetcher.Start(); err != nil { + t.Fatal(err) + } + if err := fetcher.Wait(); err != nil { + t.Fatal(err) + } + + stats := fetcher.Stats().(*http.Stats) + if len(stats.Connections) == 0 { + t.Fatalf("expected connections stats for timeout test") + } + + // Verify successful download after timeout recovery + want := test.FileMd5(test.BuildFile) + got := test.FileMd5(test.DownloadFile) + if want != got { + t.Errorf("Download() got = %v, want %v", got, want) + } + + // Verify timeouts don't count as failures (retryTimes should be 0) + for _, conn := range stats.Connections { + if conn.Failed { + t.Fatalf("expected no counted failures after timeout recovery, got retries=%d", conn.RetryTimes) + } + if conn.RetryTimes != 0 { + t.Fatalf("expected retryTimes to stay zero for non-counted timeouts, got %d", conn.RetryTimes) + } + } + } +} + +func TestFetcher_Download500Recovery(t *testing.T) { + // Server returns 500 for 15 seconds, then recovers + listener := test.StartTestTemporary500Server(15 * time.Second) + defer listener.Close() + + os.Remove(test.DownloadFile) + fetcher := downloadReady(listener, 4, t) + if err := fetcher.Start(); err != nil { + t.Fatal(err) + } + if err := fetcher.Wait(); err != nil { + t.Fatal(err) + } + + // Verify successful download after 500 errors + want := test.FileMd5(test.BuildFile) + got := test.FileMd5(test.DownloadFile) + if want != got { + t.Errorf("Download() got = %v, want %v", got, want) + } + + // Verify 500 errors don't count as failures (retryTimes should be 0) + stats := fetcher.Stats().(*http.Stats) + for _, conn := range stats.Connections { + if conn.RetryTimes != 0 { + t.Errorf("Expected retryTimes to be 0 for 500 errors (exempt), got %d", conn.RetryTimes) + } + } } func TestFetcher_DownloadOnBugFileServer(t *testing.T) { @@ -385,11 +493,14 @@ func TestFetcher_Stats(t *testing.T) { t.Fatal(err) } stats := fetcher.Stats().(*http.Stats) - if len(stats.Connections) != 16 { - t.Errorf("Stats() got = %v, want %v", len(stats.Connections), 16) + // With slow-start strategy, connection count may be less than max if download is fast + // Just verify we have at least 1 connection and no more than max + if len(stats.Connections) < 1 || len(stats.Connections) > 16 { + t.Errorf("Stats() connection count got = %v, want between 1 and 16", len(stats.Connections)) } totalDownloaded := int64(0) - for _, conn := range stats.Connections { + for i, conn := range stats.Connections { + t.Logf("Connection %d: Downloaded=%d, Completed=%v", i, conn.Downloaded, conn.Completed) totalDownloaded += conn.Downloaded } if totalDownloaded != test.BuildSize { @@ -397,6 +508,276 @@ func TestFetcher_Stats(t *testing.T) { } } +// TestFetcher_DownloadOneTimeURL tests downloading from a URL that can only be accessed once +// This simulates signed URLs or one-time download links that expire after first use +func TestFetcher_DownloadOneTimeURL(t *testing.T) { + listener := test.StartTestOneTimeServer() + defer listener.Close() + + fetcher := buildFetcher() + err := fetcher.Resolve(&base.Request{ + URL: "http://" + listener.Addr().String() + "/" + test.BuildName, + }, &base.Options{ + Name: test.DownloadName, + Path: test.Dir, + Extra: &http.OptsExtra{ + Connections: 4, // Try to use multiple connections, but only first should work + }, + }) + if err != nil { + t.Fatal(err) + } + + err = fetcher.Start() + if err != nil { + t.Fatal(err) + } + err = fetcher.Wait() + if err != nil { + t.Fatal(err) + } + + // Verify file content + want := test.FileMd5(test.BuildFile) + got := test.FileMd5(test.DownloadFile) + if want != got { + t.Errorf("Download() got = %v, want %v", got, want) + } +} + +// TestFetcher_SlowStartExpansion tests slow-start connection expansion edge cases +// Tests that slow-start expansion reaches exactly maxConns +// Expansion pattern: 1 -> 2 -> 4 -> 8 -> 16... +// For max=5: 1 -> 2 -> 4 -> 5 (capped) +// For max=9: 1 -> 2 -> 4 -> 8 -> 9 (capped) +func TestFetcher_SlowStartExpansion(t *testing.T) { + testCases := []struct { + name string + maxConns int + }{ + {"MaxConns5", 5}, // 1->2->4->5 + {"MaxConns9", 9}, // 1->2->4->8->9 + {"MaxConns8", 8}, // 1->2->4->8 + } + + for _, tc := range testCases { + tc := tc // capture range variable + t.Run(tc.name, func(t *testing.T) { + // Clean up any leftover files from previous tests + os.Remove(test.DownloadFile) + + // Use 100ns delay per byte for faster test (~10MB/s theoretical) + listener := test.StartTestSlowStartServer(100 * time.Nanosecond) + + // Ensure cleanup happens before next subtest + cleanup := func() { + listener.Close() + os.Remove(test.DownloadFile) + // Wait for server to fully stop + time.Sleep(50 * time.Millisecond) + } + + fetcher := buildConfigFetcher(config{ + Connections: tc.maxConns, + }) + + err := fetcher.Resolve(&base.Request{ + URL: "http://" + listener.Addr().String() + "/" + test.BuildName, + }, &base.Options{ + Name: test.DownloadName, + Path: test.Dir, + Extra: &http.OptsExtra{ + Connections: tc.maxConns, + }, + }) + if err != nil { + cleanup() + t.Fatal(err) + } + + err = fetcher.Start() + if err != nil { + cleanup() + t.Fatal(err) + } + + err = fetcher.Wait() + if err != nil { + t.Logf("Wait() returned error: %v", err) + cleanup() + t.Fatal(err) + } + + // Check final connection count equals maxConns exactly + stats := fetcher.Stats().(*http.Stats) + finalConns := len(stats.Connections) + + // Debug: show connection details and metadata + httpFetcher := fetcher.(*Fetcher) + t.Logf("Resource: Size=%d, Range=%v", httpFetcher.Meta().Res.Size, httpFetcher.Meta().Res.Range) + for i, conn := range stats.Connections { + t.Logf("Connection %d: Downloaded=%d, Completed=%v", i, conn.Downloaded, conn.Completed) + } + + if finalConns != tc.maxConns { + t.Errorf("Expected exactly %d connections, got %d", tc.maxConns, finalConns) + } + + // Verify file content before cleanup + want := test.FileMd5(test.BuildFile) + got := test.FileMd5(test.DownloadFile) + if want != got { + t.Errorf("Download() got = %v, want %v", got, want) + } + + cleanup() + }) + } +} + +// TestFetcher_AsyncPrefetch tests the async prefetch functionality +// where data is downloaded in background during resolve phase and reused in start +func TestFetcher_AsyncPrefetch(t *testing.T) { + // Test 1: Prefetch completes entire file before Start is called + t.Run("PrefetchComplete", func(t *testing.T) { + listener := test.StartTestFileServer() + defer listener.Close() + + fetcher := buildFetcher() + err := fetcher.Resolve(&base.Request{ + URL: "http://" + listener.Addr().String() + "/" + test.BuildName, + }, &base.Options{ + Name: test.DownloadName, + Path: test.Dir, + Extra: &http.OptsExtra{ + Connections: 4, + }, + }) + if err != nil { + t.Fatal(err) + } + + // Poll until prefetch completes the entire file (with timeout) + timeout := time.After(30 * time.Second) + ticker := time.NewTicker(100 * time.Millisecond) + defer ticker.Stop() + pollLoop: + for { + select { + case <-timeout: + t.Fatal("Timeout waiting for prefetch to complete") + case <-ticker.C: + if fetcher.prefetchSize.Load() >= test.BuildSize { + break pollLoop + } + } + } + + prefetchedBefore := fetcher.prefetchSize.Load() + t.Logf("Prefetched bytes before Start: %d (%.2f MB)", prefetchedBefore, float64(prefetchedBefore)/(1024*1024)) + + // Should have prefetched the entire file + if prefetchedBefore != test.BuildSize { + t.Errorf("Prefetch should complete entire file, got %d, want %d", prefetchedBefore, test.BuildSize) + } + + // Now start the download + err = fetcher.Start() + if err != nil { + t.Fatal(err) + } + + // Wait for download to complete + err = fetcher.Wait() + if err != nil { + t.Fatal(err) + } + + // Check how much was utilized from prefetch + prefetchedUsed := fetcher.resolveDataPos.Load() + t.Logf("Prefetched bytes used: %d (%.2f MB)", prefetchedUsed, float64(prefetchedUsed)/(1024*1024)) + + // Verify file is correct + want := test.FileMd5(test.BuildFile) + got := test.FileMd5(test.DownloadFile) + if want != got { + t.Errorf("Download() got = %v, want %v", got, want) + } + + os.Remove(test.DownloadFile) + }) + + // Test 2: Prefetch only downloads partial data before Start is called + t.Run("PrefetchPartial", func(t *testing.T) { + // Use slow server with 100 nanosecond delay per byte + // This means ~10MB/s speed, so 100ms should download ~1MB + listener := test.StartTestSlowStartServer(100 * time.Nanosecond) + defer listener.Close() + + fetcher := buildFetcher() + err := fetcher.Resolve(&base.Request{ + URL: "http://" + listener.Addr().String() + "/" + test.BuildName, + }, &base.Options{ + Name: test.DownloadName, + Path: test.Dir, + Extra: &http.OptsExtra{ + Connections: 4, + }, + }) + if err != nil { + t.Fatal(err) + } + + // Wait only 100ms - should only prefetch a small portion + time.Sleep(100 * time.Millisecond) + + prefetchedBefore := fetcher.prefetchSize.Load() + t.Logf("Prefetched bytes before Start: %d (%.2f KB)", prefetchedBefore, float64(prefetchedBefore)/1024) + + // Verify we have partial data (not zero, but not complete) + if prefetchedBefore == 0 { + t.Log("Warning: No data prefetched, may be too slow") + } + if prefetchedBefore >= test.BuildSize { + t.Log("Warning: Prefetch completed entire file, test may not be valid") + } + + // Now start the download + err = fetcher.Start() + if err != nil { + t.Fatal(err) + } + + // Wait for download to complete + err = fetcher.Wait() + if err != nil { + t.Fatal(err) + } + + // Check stats - should have connections that downloaded remaining data + stats := fetcher.Stats().(*http.Stats) + t.Logf("Final connections: %d", len(stats.Connections)) + + prefetchedUsed := fetcher.resolveDataPos.Load() + t.Logf("Prefetched bytes used: %d (%.2f KB)", prefetchedUsed, float64(prefetchedUsed)/1024) + + // Verify connections picked up where prefetch left off + if len(stats.Connections) > 0 { + firstConn := stats.Connections[0] + t.Logf("First connection downloaded: %d bytes", firstConn.Downloaded) + } + + // Verify file is correct + want := test.FileMd5(test.BuildFile) + got := test.FileMd5(test.DownloadFile) + if want != got { + t.Errorf("Download() got = %v, want %v", got, want) + } + + os.Remove(test.DownloadFile) + }) +} + func TestFetcherManager_ParseName(t *testing.T) { type args struct { u string @@ -450,15 +831,9 @@ func downloadReady(listener net.Listener, connections int, t *testing.T) fetcher } func doDownloadReady(f fetcher.Fetcher, listener net.Listener, connections int, t *testing.T) fetcher.Fetcher { - err := f.Resolve(&base.Request{ - URL: "http://" + listener.Addr().String() + "/" + test.BuildName, - }) - if err != nil { - t.Fatal(err) - } var extra any = nil if connections > 0 { - extra = http.OptsExtra{ + extra = &http.OptsExtra{ Connections: connections, } } @@ -467,7 +842,9 @@ func doDownloadReady(f fetcher.Fetcher, listener net.Listener, connections int, Path: test.Dir, Extra: extra, } - err = f.Create(opts) + err := f.Resolve(&base.Request{ + URL: "http://" + listener.Addr().String() + "/" + test.BuildName, + }, opts) if err != nil { t.Fatal(err) } @@ -492,19 +869,39 @@ func downloadNormal(listener net.Listener, connections int, t *testing.T) { } func downloadPost(listener net.Listener, connections int, t *testing.T) { - fetcher := downloadReady(listener, connections, t) - fetcher.Meta().Req.Extra = &http.ReqExtra{ - Method: "POST", - Header: map[string]string{ - "Authorization": "Bearer 123456", - }, - Body: fmt.Sprintf(`{"name":"%s"}`, test.BuildName), + // POST parameters must be set before Resolve since the new design + // starts downloading during Resolve phase + f := buildFetcher() + var extra any = nil + if connections > 0 { + extra = &http.OptsExtra{ + Connections: connections, + } } - err := fetcher.Start() + opts := &base.Options{ + Name: test.DownloadName, + Path: test.Dir, + Extra: extra, + } + req := &base.Request{ + URL: "http://" + listener.Addr().String() + "/" + test.BuildName, + Extra: &http.ReqExtra{ + Method: "POST", + Header: map[string]string{ + "Authorization": "Bearer 123456", + }, + Body: fmt.Sprintf(`{"name":"%s"}`, test.BuildName), + }, + } + err := f.Resolve(req, opts) if err != nil { t.Fatal(err) } - err = fetcher.Wait() + err = f.Start() + if err != nil { + t.Fatal(err) + } + err = f.Wait() if err != nil { t.Fatal(err) } @@ -541,14 +938,30 @@ func downloadContinue(listener net.Listener, connections int, t *testing.T) { } func downloadError(listener net.Listener, connections int, t *testing.T) { - fetcher := downloadReady(listener, connections, t) - err := fetcher.Start() + fetcher := buildFetcher() + err := fetcher.Resolve(&base.Request{ + URL: "http://" + listener.Addr().String() + "/" + test.BuildName, + }, &base.Options{ + Name: test.DownloadName, + Path: test.Dir, + }) + // With the new async design, Resolve may succeed (HTTP response received) + // but errors occur during async download or Start/Wait if err != nil { - t.Fatal(err) + // Error detected in Resolve - this is fine + return } + + // Resolve succeeded, error should occur during Start/Wait + err = fetcher.Start() + if err != nil { + // Error detected in Start - this is fine + return + } + err = fetcher.Wait() if err == nil { - t.Errorf("Download() got = %v, want %v", err, nil) + t.Errorf("Expected error during download, but got none") } } diff --git a/internal/protocol/http/helper.go b/internal/protocol/http/helper.go new file mode 100644 index 00000000..51060891 --- /dev/null +++ b/internal/protocol/http/helper.go @@ -0,0 +1,378 @@ +package http + +import ( + "bytes" + "context" + "crypto/tls" + "errors" + "fmt" + "io" + "mime" + "net" + "net/http" + "net/http/cookiejar" + "net/url" + "strings" + "unicode/utf8" + + "github.com/GopeedLab/gopeed/pkg/base" + fhttp "github.com/GopeedLab/gopeed/pkg/protocol/http" + "github.com/GopeedLab/gopeed/pkg/util" + "golang.org/x/text/encoding/simplifiedchinese" +) + +type RequestError struct { + Code int +} + +func NewRequestError(code int) *RequestError { + return &RequestError{Code: code} +} + +func (re *RequestError) Error() string { + return fmt.Sprintf("http request fail, code:%d", re.Code) +} + +func isFailureExemptHTTPCode(code int) bool { + if code >= 500 && code <= 599 { + return true + } + + switch code { + case 429, 408, 440, 499: + return true + default: + return false + } +} + +func shouldCountHTTPFailure(err error) bool { + var re *RequestError + if !errors.As(err, &re) { + return false + } + + return !isFailureExemptHTTPCode(re.Code) +} + +func extractRequestError(err error) *RequestError { + var re *RequestError + if errors.As(err, &re) { + return re + } + + return nil +} + +// ============================================================================ +// HTTP Request/Client Building +// ============================================================================ + +func (f *Fetcher) buildRequest(ctx context.Context, req *base.Request) (httpReq *http.Request, err error) { + var reqUrl string + f.redirectLock.Lock() + if f.redirectURL != "" { + reqUrl = f.redirectURL + } else { + reqUrl = req.URL + } + f.redirectLock.Unlock() + + var ( + method string + body io.Reader + ) + headers := http.Header{} + if req.Extra == nil { + method = http.MethodGet + } else { + extra := req.Extra.(*fhttp.ReqExtra) + if extra.Method != "" { + method = extra.Method + } else { + method = http.MethodGet + } + if len(extra.Header) > 0 { + for k, v := range extra.Header { + headers.Set(k, strings.TrimSpace(v)) + } + } + if extra.Body != "" { + body = bytes.NewBufferString(extra.Body) + } + } + if _, ok := headers[base.HttpHeaderUserAgent]; !ok { + headers.Set(base.HttpHeaderUserAgent, strings.TrimSpace(f.config.UserAgent)) + } + + if ctx != nil { + httpReq, err = http.NewRequestWithContext(ctx, method, reqUrl, body) + } else { + httpReq, err = http.NewRequest(method, reqUrl, body) + } + if err != nil { + return + } + httpReq.Header = headers + if host := headers.Get(base.HttpHeaderHost); host != "" { + httpReq.Host = host + } + return httpReq, nil +} + +func (f *Fetcher) buildClient() *http.Client { + transport := &http.Transport{ + DialContext: (&net.Dialer{ + Timeout: connectTimeout, + }).DialContext, + Proxy: f.ctl.GetProxy(f.meta.Req.Proxy), + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: f.meta.Req.SkipVerifyCert, + }, + } + jar, _ := cookiejar.New(nil) + return &http.Client{ + Transport: transport, + Jar: jar, + } +} + +// ============================================================================ +// Filename Parsing +// ============================================================================ + +// parseFilename extracts filename from Content-Disposition header +func parseFilename(contentDisposition string) string { + // Try RFC 5987 extended notation first (filename*=) + if filename := parseFilenameExtended(contentDisposition); filename != "" { + return filename + } + + // Try standard MIME parsing + _, params, err := mime.ParseMediaType(contentDisposition) + if err == nil { + if filename := params["filename"]; filename != "" { + return decodeFilenameParam(filename) + } + } + + // Fallback to manual parsing + return parseFilenameFallback(contentDisposition) +} + +// parseFilenameExtended handles RFC 5987 extended notation (filename*=) +// Format: filename*=charset'language'value (e.g., UTF-8”%E6%B5%8B%E8%AF%95.zip) +func parseFilenameExtended(cd string) string { + lower := strings.ToLower(cd) + idx := strings.Index(lower, "filename*=") + if idx == -1 { + return "" + } + + value := cd[idx+len("filename*="):] + + // Find the end of the value using proper quote handling + endIdx := findParamValueEnd(value) + if endIdx != -1 { + value = value[:endIdx] + } + value = strings.TrimSpace(value) + + // Try charset''encoded format (e.g., UTF-8''%E4%B8%AD%E6%96%87.txt) + parts := strings.SplitN(value, "''", 2) + if len(parts) == 2 { + // Use PathUnescape to handle %2B correctly (should decode to +, not space) + decoded, err := url.PathUnescape(parts[1]) + if err == nil { + return decoded + } + } + + // Try charset'language'encoded format + parts = strings.SplitN(value, "'", 3) + if len(parts) >= 3 { + // Use PathUnescape to handle %2B correctly (should decode to +, not space) + decoded, err := url.PathUnescape(parts[2]) + if err == nil { + return decoded + } + } + + return "" +} + +// decodeFilenameParam decodes filename parameter value +// Handles HTML entities, MIME encoded-word, URL encoding, and GBK encoding fallback +func decodeFilenameParam(filename string) string { + // First, unescape HTML entities (e.g., & -> &, < -> <, > -> >) + // This must be done before other decoding to handle cases where servers + // HTML-encode special characters in filenames + filename = unescapeHTMLEntities(filename) + + // Handle RFC 2047 encoded word (=?charset?encoding?text?=) + if strings.HasPrefix(filename, "=?") { + decoder := new(mime.WordDecoder) + normalizedFilename := strings.Replace(filename, "UTF8", "UTF-8", 1) + if decoded, err := decoder.Decode(normalizedFilename); err == nil { + return decoded + } + } + + // Try URL decoding - use PathUnescape for filenames to handle %2B correctly + decoded := util.TryUrlPathUnescape(filename) + + // If not valid UTF-8, try GBK decoding (common for Chinese websites) + if !utf8.ValidString(decoded) { + if gbkDecoded := tryDecodeGBK(decoded); gbkDecoded != "" { + return gbkDecoded + } + } + + return decoded +} + +// unescapeHTMLEntities unescapes common HTML entities in filenames +// This handles cases where servers HTML-encode special characters like & to & +func unescapeHTMLEntities(s string) string { + // Common HTML entities that might appear in filenames + replacements := map[string]string{ + "&": "&", + "<": "<", + ">": ">", + """: "\"", + "'": "'", + "'": "'", + } + + result := s + for entity, char := range replacements { + result = strings.ReplaceAll(result, entity, char) + } + return result +} + +// tryDecodeGBK attempts to decode string as GBK encoding +func tryDecodeGBK(s string) string { + if len(s) == 0 { + return "" + } + + decoder := simplifiedchinese.GBK.NewDecoder() + decoded, err := decoder.Bytes([]byte(s)) + if err != nil { + return "" + } + result := string(decoded) + if utf8.ValidString(result) { + return result + } + return "" +} + +// parseFilenameFallback is a fallback parser for non-standard Content-Disposition +func parseFilenameFallback(cd string) string { + lower := strings.ToLower(cd) + idx := strings.Index(lower, "filename=") + if idx == -1 { + return "" + } + + value := cd[idx+len("filename="):] + + // Find the end of the value using proper quote handling + endIdx := findParamValueEnd(value) + if endIdx != -1 { + value = value[:endIdx] + } + value = strings.TrimSpace(value) + + // Remove surrounding quotes + if len(value) >= 2 { + if (value[0] == '"' && value[len(value)-1] == '"') || + (value[0] == '\'' && value[len(value)-1] == '\'') { + value = value[1 : len(value)-1] + } + } + + return decodeFilenameParam(value) +} + +// findParamValueEnd finds the end position of a parameter value in a Content-Disposition header. +// It correctly handles quoted values where semicolons inside quotes should not be treated as delimiters. +// It also handles HTML entities in unquoted values (e.g., & should not be split at the semicolon). +// Returns the end index (exclusive) of the value, or -1 if it extends to the end of the string. +func findParamValueEnd(value string) int { + value = strings.TrimSpace(value) + if len(value) == 0 { + return 0 + } + + // If the value starts with a quote, find the matching closing quote + if value[0] == '"' || value[0] == '\'' { + quote := value[0] + // Find the closing quote, handling escaped quotes + for i := 1; i < len(value); i++ { + if value[i] == quote { + // Check if it's escaped + if i > 0 && value[i-1] == '\\' { + continue + } + // Found closing quote, now look for ; after it + remaining := value[i+1:] + if semiIdx := strings.Index(remaining, ";"); semiIdx != -1 { + return i + 1 + semiIdx + } + return -1 // No semicolon after closing quote + } + } + // No closing quote found, treat rest of string as value + return -1 + } + + // Unquoted value - find the next semicolon that's not part of an HTML entity + // HTML entities have the pattern &...; (e.g., & < > " ') + for i := 0; i < len(value); i++ { + if value[i] == ';' { + // Check if this semicolon is part of an HTML entity + // Look backwards for an & character + isEntity := false + if i > 0 { + // Look for & before this semicolon (within reasonable distance, max 10 chars) + for j := i - 1; j >= 0 && j >= i-10; j-- { + if value[j] == '&' { + // Found &, this semicolon might be part of an HTML entity + // Check if there are only alphanumeric or # between & and ; + entityChars := value[j+1 : i] + if len(entityChars) > 0 && isValidHTMLEntityChars(entityChars) { + isEntity = true + } + break + } + // If we hit whitespace or another special char, stop looking + if value[j] == ' ' || value[j] == '"' || value[j] == '\'' { + break + } + } + } + + if !isEntity { + return i + } + } + } + return -1 // No semicolon, extends to end +} + +// isValidHTMLEntityChars checks if a string contains only valid HTML entity characters +// (alphanumeric and #, typically for entities like & < ' etc.) +func isValidHTMLEntityChars(s string) bool { + if len(s) == 0 { + return false + } + for _, c := range s { + if !((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '#') { + return false + } + } + return true +} diff --git a/internal/test/httptest.go b/internal/test/httptest.go index 7be34e63..72c08c62 100644 --- a/internal/test/httptest.go +++ b/internal/test/httptest.go @@ -70,6 +70,9 @@ func StartTestSlowFileServer(delay time.Duration) net.Listener { func StartTestCustomServer() net.Listener { return startTestServer(func(sl *shutdownListener) http.Handler { mux := http.NewServeMux() + mux.HandleFunc("/", func(writer http.ResponseWriter, request *http.Request) { + writer.WriteHeader(200) + }) mux.HandleFunc("/"+BuildName, func(writer http.ResponseWriter, request *http.Request) { file, err := os.Open(BuildFile) if err != nil { @@ -221,6 +224,10 @@ func StartTestCustomServer() net.Listener { defer file.Close() io.Copy(writer, file) }) + // Test 403 Forbidden endpoint + mux.HandleFunc("/forbidden", func(writer http.ResponseWriter, request *http.Request) { + writer.WriteHeader(403) + }) return mux }) } @@ -244,20 +251,6 @@ func StartTestHostHeaderServer() net.Listener { }) } -// StartTestRootServer starts a simple server at the root path -// Used to test URL resolution when no filename is provided -func StartTestRootServer() net.Listener { - return startTestServer(func(sl *shutdownListener) http.Handler { - mux := http.NewServeMux() - mux.HandleFunc("/", func(writer http.ResponseWriter, request *http.Request) { - writer.Header().Set("Content-Type", "text/html") - writer.WriteHeader(200) - writer.Write([]byte("Test Page")) - }) - return mux - }) -} - func StartTestRetryServer() net.Listener { counter := 0 return startTestServer(func(sl *shutdownListener) http.Handler { @@ -303,23 +296,100 @@ func StartTestPostServer() net.Listener { } func StartTestErrorServer() net.Listener { - counter := 0 return startTestServer(func(sl *shutdownListener) http.Handler { mux := http.NewServeMux() mux.HandleFunc("/"+BuildName, func(writer http.ResponseWriter, request *http.Request) { - counter++ - if counter != 1 { - writer.WriteHeader(500) - return - } + // Always return 404 error to test error handling + writer.WriteHeader(404) + return }) return mux }) } +// StartTestOneTimeServer creates a server where the URL can only be downloaded once +// The first non-probe request succeeds, all subsequent requests return 404 +// This simulates one-time download URLs (e.g., signed URLs that expire after first use) +func StartTestOneTimeServer() net.Listener { + var accessed atomic.Bool + + return startTestServer(func(sl *shutdownListener) http.Handler { + mux := http.NewServeMux() + mux.HandleFunc("/"+BuildName, func(writer http.ResponseWriter, request *http.Request) { + // First full download request succeeds, subsequent requests fail + if accessed.Swap(true) { + writer.WriteHeader(404) + return + } + + // First request - return full file without Range support + writer.Header().Set("Content-Length", fmt.Sprintf("%d", BuildSize)) + file, err := os.Open(BuildFile) + if err != nil { + return + } + defer file.Close() + io.Copy(writer, file) + }) + return mux + }) +} + +// StartTestSlowStartServer creates a server with configurable delay per request +// This allows testing slow-start connection expansion to reach max connections +func StartTestSlowStartServer(delayPerByte time.Duration) net.Listener { + return startTestServer(func(sl *shutdownListener) http.Handler { + mux := http.NewServeMux() + mux.HandleFunc("/"+BuildName, func(writer http.ResponseWriter, request *http.Request) { + rangeFileHandle( + writer, + request, + nil, + func(file *os.File, n int64) { + slowCopyNWithDelay(sl, writer, file, n, delayPerByte) + }, + ) + }) + return mux + }) +} + +// slowCopyNWithDelay copies n bytes from src to dst with a delay per byte +func slowCopyNWithDelay(sl *shutdownListener, dst io.Writer, src io.Reader, n int64, delayPerByte time.Duration) { + buf := make([]byte, 32*1024) + remaining := n + for remaining > 0 { + if sl.isShutdown { + return + } + toRead := int64(len(buf)) + if toRead > remaining { + toRead = remaining + } + nr, er := src.Read(buf[:toRead]) + if nr > 0 { + nw, ew := dst.Write(buf[0:nr]) + if nw > 0 { + remaining -= int64(nw) + // Add delay based on bytes written + if delayPerByte > 0 { + time.Sleep(delayPerByte * time.Duration(nw)) + } + } + if ew != nil { + break + } + } + if er != nil { + break + } + } +} + // StartTestLimitServer connections limit server func StartTestLimitServer(maxConnections int32, delay int64) net.Listener { var connections atomic.Int32 + var slowOnce atomic.Bool return startTestServer(func(sl *shutdownListener) http.Handler { mux := http.NewServeMux() @@ -332,12 +402,93 @@ func StartTestLimitServer(maxConnections int32, delay int64) net.Listener { writer.WriteHeader(403) return } + + // First request intentionally delays the first write to trigger a single read timeout, + // subsequent requests respond at normal speed. + useInitialDelay := delay > 0 && !slowOnce.Swap(true) rangeFileHandle( writer, request, nil, func(file *os.File, n int64) { - slowCopyN(sl, writer, file, n, delay) + if useInitialDelay { + slowCopyAfterDelay(sl, writer, file, n, delay) + return + } + slowCopyN(sl, writer, file, n, 0) + }, + ) + }) + return mux + }) +} + +// StartTestTimeoutOnceServer creates a server that times out on first request, then works normally +func StartTestTimeoutOnceServer(delay int64) net.Listener { + var timeoutOnce atomic.Bool + + return startTestServer(func(sl *shutdownListener) http.Handler { + mux := http.NewServeMux() + mux.HandleFunc("/"+BuildName, func(writer http.ResponseWriter, request *http.Request) { + // First request delays to trigger timeout, subsequent requests respond normally + useInitialDelay := delay > 0 && !timeoutOnce.Swap(true) + rangeFileHandle( + writer, + request, + nil, + func(file *os.File, n int64) { + if useInitialDelay { + slowCopyAfterDelay(sl, writer, file, n, delay) + return + } + slowCopyN(sl, writer, file, n, 0) + }, + ) + }) + return mux + }) +} + +// StartTestTemporary500Server creates a server that returns 500 for a duration, then recovers +// Uses slow transfer to ensure the file isn't fully downloaded during resolve phase +func StartTestTemporary500Server(errorDuration time.Duration) net.Listener { + startTime := time.Now() + var requestCount atomic.Int32 + + return startTestServer(func(sl *shutdownListener) http.Handler { + mux := http.NewServeMux() + mux.HandleFunc("/"+BuildName, func(writer http.ResponseWriter, request *http.Request) { + reqNum := requestCount.Add(1) + + // First request (Resolve) succeeds with slow transfer to prevent full download + // Subsequent requests return 500 for the specified duration, then recover + if reqNum == 1 { + // Slow transfer for resolve: 50 microsecond per byte (~20MB/s) + // This ensures resolve doesn't complete the full 200MB file + rangeFileHandle( + writer, + request, + nil, + func(file *os.File, n int64) { + slowCopyNWithDelay(sl, writer, file, n, 50*time.Microsecond) + }, + ) + return + } + + // Subsequent requests: return 500 for errorDuration, then normal + if time.Since(startTime) < errorDuration { + writer.WriteHeader(500) + writer.Write([]byte("Internal Server Error")) + return + } + + rangeFileHandle( + writer, + request, + nil, + func(file *os.File, n int64) { + slowCopyN(sl, writer, file, n, 0) }, ) }) @@ -375,6 +526,26 @@ func StartTestRangeBugServer() net.Listener { func rangeFileHandle(writer http.ResponseWriter, request *http.Request, modifyEnd func(end int64) int64, iocpN func(file *os.File, n int64)) { r := request.Header.Get("Range") + + // If no Range header, return full file with Accept-Ranges header + if r == "" { + // Open file first to ensure it exists + file, err := os.Open(BuildFile) + if err != nil { + writer.WriteHeader(500) + return + } + defer file.Close() + + writer.Header().Set("Content-Length", fmt.Sprintf("%d", BuildSize)) + writer.Header().Set("Accept-Ranges", "bytes") + writer.WriteHeader(200) + (writer.(http.Flusher)).Flush() + + iocpN(file, BuildSize) + return + } + // split range s := strings.Split(r, "=") if len(s) != 2 { @@ -408,12 +579,7 @@ func rangeFileHandle(writer http.ResponseWriter, request *http.Request, modifyEn end = modifyEnd(end) } - writer.Header().Set("Content-Length", fmt.Sprintf("%d", end-start+1)) - writer.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, BuildSize)) - writer.Header().Set("Accept-Ranges", "bytes") - writer.WriteHeader(206) - (writer.(http.Flusher)).Flush() - + // Open file before sending headers to ensure it exists file, err := os.Open(BuildFile) if err != nil { writer.WriteHeader(500) @@ -421,6 +587,13 @@ func rangeFileHandle(writer http.ResponseWriter, request *http.Request, modifyEn } defer file.Close() file.Seek(start, 0) + + writer.Header().Set("Content-Length", fmt.Sprintf("%d", end-start+1)) + writer.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, BuildSize)) + writer.Header().Set("Accept-Ranges", "bytes") + writer.WriteHeader(206) + (writer.(http.Flusher)).Flush() + iocpN(file, end-start+1) } @@ -471,6 +644,14 @@ func slowCopyN(sl *shutdownListener, dst io.Writer, src io.Reader, n int64, dela return } +// slowCopyAfterDelay sleeps once before performing a normal copy to simulate a single timeout. +func slowCopyAfterDelay(sl *shutdownListener, dst io.Writer, src io.Reader, n int64, delay int64) (written int64, err error) { + if delay > 0 { + time.Sleep(time.Millisecond * time.Duration(delay)) + } + return slowCopyN(sl, dst, src, n, 0) +} + func startTestServer(serverHandle func(sl *shutdownListener) http.Handler) net.Listener { listener, err := net.Listen("tcp", "127.0.0.1:0") if err != nil { @@ -515,8 +696,9 @@ type shutdownListener struct { } func (c *shutdownListener) Close() error { - c.isShutdown = true + // Shutdown server first (waits for in-flight requests), then set isShutdown closeErr := c.server.Shutdown(context.Background()) + c.isShutdown = true if err := ifExistAndRemove(BuildFile); err != nil { fmt.Println(err) } diff --git a/pkg/base/model.go b/pkg/base/model.go index a58ca28c..6b36652a 100644 --- a/pkg/base/model.go +++ b/pkg/base/model.go @@ -149,7 +149,7 @@ func ParseReqExtra[E any](req *Request) error { return nil } -func ParseOptsExtra[E any](opts *Options) error { +func ParseOptExtra[E any](opts *Options) error { if opts.Extra == nil { return nil } diff --git a/pkg/download/downloader.go b/pkg/download/downloader.go index cd5b5870..7b3b4b43 100644 --- a/pkg/download/downloader.go +++ b/pkg/download/downloader.go @@ -321,7 +321,7 @@ func (d *Downloader) saveTask(task *Task) error { return nil } -func (d *Downloader) Resolve(req *base.Request) (rr *ResolveResult, err error) { +func (d *Downloader) Resolve(req *base.Request, opts *base.Options) (rr *ResolveResult, err error) { rrId, err := gonanoid.New() if err != nil { return @@ -342,7 +342,11 @@ func (d *Downloader) Resolve(req *base.Request) (rr *ResolveResult, err error) { if err != nil { return } - err = fetcher.Resolve(req) + initOpt, err := d.initOptions(opts) + if err != nil { + return + } + err = fetcher.Resolve(req, initOpt) if err != nil { return } @@ -390,7 +394,11 @@ func (d *Downloader) CreateDirect(req *base.Request, opts *base.Options) (taskId return } fetcher.Meta().Req = req - return d.doCreate(fetcher, opts) + initOpt, err := d.initOptions(opts) + if err != nil { + return + } + return d.doCreate(fetcher, initOpt) } func (d *Downloader) CreateDirectBatch(req *base.CreateTaskBatch) (taskId []string, err error) { @@ -409,7 +417,7 @@ func (d *Downloader) CreateDirectBatch(req *base.CreateTaskBatch) (taskId []stri return taskIds, nil } -func (d *Downloader) Create(rrId string, opts *base.Options) (taskId string, err error) { +func (d *Downloader) Create(rrId string) (taskId string, err error) { d.fetcherMapLock.RLock() fetcher, ok := d.fetcherCache[rrId] d.fetcherMapLock.RUnlock() @@ -421,7 +429,7 @@ func (d *Downloader) Create(rrId string, opts *base.Options) (taskId string, err delete(d.fetcherCache, rrId) d.fetcherMapLock.Unlock() }() - return d.doCreate(fetcher, opts) + return d.doCreate(fetcher, nil) } func (d *Downloader) Pause(filter *TaskFilter) (err error) { @@ -855,8 +863,6 @@ func (d *Downloader) watch(task *Task) { if err != nil { d.Logger.Warn().Err(err).Msgf("task wait upload failed, task id: %s", task.ID) } - d.lock.Lock() - defer d.lock.Unlock() // Check if the task is deleted if d.GetTask(task.ID) != nil { @@ -947,7 +953,6 @@ func (d *Downloader) restoreTask(task *Task) error { } } go d.watch(task) - task.fetcher.Create(task.Meta.Opts) return nil } @@ -970,42 +975,15 @@ func (d *Downloader) restoreFetcher(task *Task) error { if task.fetcher.Meta().Res == nil { task.fetcher.Meta().Res = task.Meta.Res } + if task.fetcher.Meta().Opts == nil { + task.fetcher.Meta().Opts = task.Meta.Opts + } return nil } func (d *Downloader) doCreate(f fetcher.Fetcher, opts *base.Options) (taskId string, err error) { - if opts == nil { - opts = &base.Options{} - } - if opts.SelectFiles == nil { - opts.SelectFiles = make([]int, 0) - } - - meta := f.Meta() - meta.Opts = opts - if opts.Path == "" { - storeConfig, err := d.GetConfig() - if err != nil { - return "", err - } - opts.Path = storeConfig.DownloadDir - } - - // Replace placeholders in download path (e.g., %year%, %month%, %day%, %date%) - opts.Path = util.ReplacePathPlaceholders(opts.Path) - - // if enable white download directory, check if the download directory is in the white list - if len(d.cfg.WhiteDownloadDirs) > 0 { - inWhiteList := false - for _, dir := range d.cfg.WhiteDownloadDirs { - if match, err := filepath.Match(dir, opts.Path); match && err == nil { - inWhiteList = true - break - } - } - if !inWhiteList { - return "", errors.New("download directory is not in white list") - } + if f.Meta().Opts == nil { + f.Meta().Opts = opts } fm, err := d.parseFm(f.Meta().Req.URL) @@ -1021,9 +999,6 @@ func (d *Downloader) doCreate(f fetcher.Fetcher, opts *base.Options) (taskId str task.Progress = &Progress{} _, task.Uploading = f.(fetcher.Uploader) initTask(task) - if err = f.Create(opts); err != nil { - return - } if err = d.storage.Put(bucketTask, task.ID, task.clone()); err != nil { return } @@ -1049,6 +1024,39 @@ func (d *Downloader) doCreate(f fetcher.Fetcher, opts *base.Options) (taskId str return } +func (d *Downloader) initOptions(opts *base.Options) (*base.Options, error) { + if opts == nil { + opts = &base.Options{} + } + if opts.SelectFiles == nil { + opts.SelectFiles = make([]int, 0) + } + if opts.Path == "" { + storeConfig, err := d.GetConfig() + if err != nil { + return nil, err + } + opts.Path = storeConfig.DownloadDir + } + // Replace placeholders in download path (e.g., %year%, %month%, %day%, %date%) + opts.Path = util.ReplacePathPlaceholders(opts.Path) + + // if enable white download directory, check if the download directory is in the white list + if len(d.cfg.WhiteDownloadDirs) > 0 { + inWhiteList := false + for _, dir := range d.cfg.WhiteDownloadDirs { + if match, err := filepath.Match(dir, opts.Path); match && err == nil { + inWhiteList = true + break + } + } + if !inWhiteList { + return nil, errors.New("download directory is not in white list") + } + } + return opts, nil +} + func (d *Downloader) statusMut(task *Task, fn func() (bool, error)) (bool, error) { task.statusLock.Lock() defer task.statusLock.Unlock() @@ -1088,7 +1096,7 @@ func (d *Downloader) doStart(task *Task) (err error) { d.triggerOnStart(task) if task.Meta.Res == nil { - err := task.fetcher.Resolve(task.Meta.Req) + err := task.fetcher.Resolve(task.Meta.Req, task.Meta.Opts) if err != nil { return err } @@ -1621,13 +1629,6 @@ func (b *boot) Extra(extra interface{}) *boot { return b } -func (b *boot) Resolve() (*ResolveResult, error) { - return defaultDownloader.Resolve(&base.Request{ - URL: b.url, - Extra: b.extra, - }) -} - func (b *boot) Listener(listener Listener) *boot { b.listener = listener return b diff --git a/pkg/download/downloader_test.go b/pkg/download/downloader_test.go index 22bec4b4..cef94713 100644 --- a/pkg/download/downloader_test.go +++ b/pkg/download/downloader_test.go @@ -20,6 +20,14 @@ import ( "github.com/GopeedLab/gopeed/pkg/protocol/http" ) +var testDownloadOpt = &base.Options{ + Path: test.Dir, + Name: test.DownloadName, + Extra: http.OptsExtra{ + Connections: 4, + }, +} + func TestDownloader_Resolve(t *testing.T) { listener := test.StartTestFileServer() defer listener.Close() @@ -32,7 +40,7 @@ func TestDownloader_Resolve(t *testing.T) { req := &base.Request{ URL: "http://" + listener.Addr().String() + "/" + test.BuildName, } - rr, err := downloader.Resolve(req) + rr, err := downloader.Resolve(req, nil) if err != nil { t.Fatal(err) } @@ -64,7 +72,7 @@ func TestDownloader_Create(t *testing.T) { req := &base.Request{ URL: "http://" + listener.Addr().String() + "/" + test.BuildName, } - rr, err := downloader.Resolve(req) + rr, err := downloader.Resolve(req, testDownloadOpt) if err != nil { t.Fatal(err) } @@ -76,13 +84,7 @@ func TestDownloader_Create(t *testing.T) { wg.Done() } }) - _, err = downloader.Create(rr.ID, &base.Options{ - Path: test.Dir, - Name: test.DownloadName, - Extra: http.OptsExtra{ - Connections: 4, - }, - }) + _, err = downloader.Create(rr.ID) if err != nil { t.Fatal(err) } @@ -108,27 +110,14 @@ func TestDownloader_CreateNotInWhite(t *testing.T) { req := &base.Request{ URL: "http://" + listener.Addr().String() + "/" + test.BuildName, } - rr, err := downloader.Resolve(req) - if err != nil { - t.Fatal(err) + // With new fetcher design, white list check happens during Resolve (not Create) + // because Resolve now requires Options which includes the download path + _, err := downloader.Resolve(req, testDownloadOpt) + if err == nil { + t.Error("TestDownloader_CreateNotInWhite() expected error but got nil") } - - var wg sync.WaitGroup - wg.Add(1) - downloader.Listener(func(event *Event) { - if event.Key == EventKeyDone { - wg.Done() - } - }) - _, err = downloader.Create(rr.ID, &base.Options{ - Path: test.Dir, - Name: test.DownloadName, - Extra: http.OptsExtra{ - Connections: 4, - }, - }) if !strings.Contains(err.Error(), "white") { - t.Errorf("TestDownloader_CreateNotInWhite() got = %v, want %v", err.Error(), "not in white list") + t.Errorf("TestDownloader_CreateNotInWhite() got = %v, want error containing 'white'", err.Error()) } } @@ -172,13 +161,7 @@ func TestDownloader_CreateDirectBatch(t *testing.T) { _, err := downloader.CreateDirectBatch(&base.CreateTaskBatch{ Reqs: reqs, - Opts: &base.Options{ - Path: test.Dir, - Name: test.DownloadName, - Extra: http.OptsExtra{ - Connections: 4, - }, - }, + Opts: testDownloadOpt, }) if err != nil { t.Fatal(err) @@ -317,7 +300,7 @@ func doTestDownloaderCreateWithProxy(t *testing.T, auth bool, buildReqProxy func Pwd: pwd, }) } - rr, err := downloader.Resolve(req) + rr, err := downloader.Resolve(req, nil) if err != nil { if errHandler == nil { t.Fatal(err) @@ -400,18 +383,12 @@ func TestDownloader_StoreAndRestore(t *testing.T) { req := &base.Request{ URL: "http://" + listener.Addr().String() + "/" + test.BuildName, } - rr, err := downloader.Resolve(req) + rr, err := downloader.Resolve(req, testDownloadOpt) if err != nil { t.Fatal(err) } - id, err := downloader.Create(rr.ID, &base.Options{ - Path: test.Dir, - Name: test.DownloadName, - Extra: http.OptsExtra{ - Connections: 4, - }, - }) + id, err := downloader.Create(rr.ID) if err != nil { t.Fatal(err) } @@ -675,7 +652,7 @@ func TestDownloader_Stats(t *testing.T) { req := &base.Request{ URL: "http://" + listener.Addr().String() + "/" + test.BuildName, } - rr, err := downloader.Resolve(req) + rr, err := downloader.Resolve(req, testDownloadOpt) if err != nil { t.Fatal(err) } @@ -688,13 +665,7 @@ func TestDownloader_Stats(t *testing.T) { } }) - taskId, err := downloader.Create(rr.ID, &base.Options{ - Path: test.Dir, - Name: test.DownloadName, - Extra: http.OptsExtra{ - Connections: 4, - }, - }) + taskId, err := downloader.Create(rr.ID) if err != nil { t.Fatal(err) } @@ -736,13 +707,7 @@ func TestDownloader_Delete(t *testing.T) { req := &base.Request{ URL: "http://" + listener.Addr().String() + "/" + test.BuildName, } - taskId, err := downloader.CreateDirect(req, &base.Options{ - Path: test.Dir, - Name: test.DownloadName, - Extra: http.OptsExtra{ - Connections: 4, - }, - }) + taskId, err := downloader.CreateDirect(req, testDownloadOpt) if err != nil { t.Fatal(err) } @@ -810,17 +775,11 @@ func TestDownloader_ContinueBatch(t *testing.T) { req := &base.Request{ URL: "http://" + listener.Addr().String() + "/" + test.BuildName, } - rr, err := downloader.Resolve(req) + rr, err := downloader.Resolve(req, testDownloadOpt) if err != nil { t.Fatal(err) } - taskId, err := downloader.Create(rr.ID, &base.Options{ - Path: test.Dir, - Name: test.DownloadName, - Extra: http.OptsExtra{ - Connections: 4, - }, - }) + taskId, err := downloader.Create(rr.ID) if err != nil { t.Fatal(err) } @@ -905,18 +864,12 @@ func TestDownloader_PauseAndContinue(t *testing.T) { req := &base.Request{ URL: "http://" + listener.Addr().String() + "/" + test.BuildName, } - rr, err := downloader.Resolve(req) + rr, err := downloader.Resolve(req, testDownloadOpt) if err != nil { t.Fatal(err) } - taskId, err := downloader.Create(rr.ID, &base.Options{ - Path: test.Dir, - Name: test.DownloadName, - Extra: http.OptsExtra{ - Connections: 4, - }, - }) + taskId, err := downloader.Create(rr.ID) if err != nil { t.Fatal(err) } diff --git a/pkg/download/engine/polyfill/package-lock.json b/pkg/download/engine/polyfill/package-lock.json index 8a2d4d85..e594889d 100644 --- a/pkg/download/engine/polyfill/package-lock.json +++ b/pkg/download/engine/polyfill/package-lock.json @@ -217,7 +217,7 @@ "@webassemblyjs/helper-wasm-bytecode": "1.11.1", "@webassemblyjs/helper-wasm-section": "1.11.1", "@webassemblyjs/wasm-gen": "1.11.1", - "@webassemblyjs/wasm-opt": "1.11.1", + "@webassemblyjs/wasm-opts": "1.11.1", "@webassemblyjs/wasm-parser": "1.11.1", "@webassemblyjs/wast-printer": "1.11.1" } @@ -235,9 +235,9 @@ "@webassemblyjs/utf8": "1.11.1" } }, - "node_modules/@webassemblyjs/wasm-opt": { + "node_modules/@webassemblyjs/wasm-opts": { "version": "1.11.1", - "resolved": "https://registry.npmmirror.com/@webassemblyjs/wasm-opt/-/wasm-opt-1.11.1.tgz", + "resolved": "https://registry.npmmirror.com/@webassemblyjs/wasm-opts/-/wasm-opts-1.11.1.tgz", "integrity": "sha512-VqnkNqnZlU5EB64pp1l7hdm3hmQw7Vgqa0KF/KCNO9sIpI6Fk6brDEiX+iCOYrvMuBWDws0NkTOxYEb85XQHHw==", "dev": true, "dependencies": { @@ -2114,7 +2114,7 @@ "@webassemblyjs/helper-wasm-bytecode": "1.11.1", "@webassemblyjs/helper-wasm-section": "1.11.1", "@webassemblyjs/wasm-gen": "1.11.1", - "@webassemblyjs/wasm-opt": "1.11.1", + "@webassemblyjs/wasm-opts": "1.11.1", "@webassemblyjs/wasm-parser": "1.11.1", "@webassemblyjs/wast-printer": "1.11.1" } @@ -2132,9 +2132,9 @@ "@webassemblyjs/utf8": "1.11.1" } }, - "@webassemblyjs/wasm-opt": { + "@webassemblyjs/wasm-opts": { "version": "1.11.1", - "resolved": "https://registry.npmmirror.com/@webassemblyjs/wasm-opt/-/wasm-opt-1.11.1.tgz", + "resolved": "https://registry.npmmirror.com/@webassemblyjs/wasm-opts/-/wasm-opts-1.11.1.tgz", "integrity": "sha512-VqnkNqnZlU5EB64pp1l7hdm3hmQw7Vgqa0KF/KCNO9sIpI6Fk6brDEiX+iCOYrvMuBWDws0NkTOxYEb85XQHHw==", "dev": true, "requires": { diff --git a/pkg/download/extension_test.go b/pkg/download/extension_test.go index 50160e70..5290b5f5 100644 --- a/pkg/download/extension_test.go +++ b/pkg/download/extension_test.go @@ -2,13 +2,14 @@ package download import ( "errors" + "os" + "testing" + "time" + "github.com/GopeedLab/gopeed/internal/logger" "github.com/GopeedLab/gopeed/pkg/base" gojaerror "github.com/GopeedLab/gopeed/pkg/download/engine/inject/error" "github.com/dop251/goja" - "os" - "testing" - "time" ) func TestDownloader_InstallExtensionByFolder(t *testing.T) { @@ -18,7 +19,7 @@ func TestDownloader_InstallExtensionByFolder(t *testing.T) { } rr, err := downloader.Resolve(&base.Request{ URL: "https://github.com/test", - }) + }, nil) if err != nil { t.Fatal(err) } @@ -35,7 +36,7 @@ func TestDownloader_InstallExtensionByFolderDevMode(t *testing.T) { } rr, err := downloader.Resolve(&base.Request{ URL: "https://github.com/test", - }) + }, nil) if err != nil { t.Fatal(err) } @@ -52,7 +53,7 @@ func TestDownloader_InstallExtensionByGit(t *testing.T) { } rr, err := downloader.Resolve(&base.Request{ URL: "https://github.com/GopeedLab/gopeed/releases", - }) + }, nil) if err != nil { t.Fatal(err) } @@ -69,7 +70,7 @@ func TestDownloader_InstallExtensionByGitSimple(t *testing.T) { } rr, err := downloader.Resolve(&base.Request{ URL: "https://github.com/GopeedLab/gopeed/releases", - }) + }, nil) if err != nil { t.Fatal(err) } @@ -86,7 +87,7 @@ func TestDownloader_InstallExtensionByGitFull(t *testing.T) { } rr, err := downloader.Resolve(&base.Request{ URL: "https://github.com/GopeedLab/gopeed/releases", - }) + }, nil) if err != nil { t.Fatal(err) } @@ -156,7 +157,7 @@ func TestDownloader_UpgradeExtension(t *testing.T) { rr, err := downloader.Resolve(&base.Request{ URL: "https://test.com", - }) + }, nil) if err != nil { t.Fatal(err) } @@ -185,7 +186,7 @@ func TestDownloader_Extension_OnStart(t *testing.T) { select { case err = <-errCh: break - case <-time.After(time.Second * 10): + case <-time.After(time.Second * 30): // Increased timeout for real network requests err = errors.New("timeout") } if err != nil { @@ -238,7 +239,7 @@ func TestDownloader_Extension_OnError(t *testing.T) { select { case err = <-errCh: break - case <-time.After(time.Second * 10): + case <-time.After(time.Second * 30): // Increased timeout for real network requests err = errors.New("timeout") } @@ -273,7 +274,7 @@ func TestDownloader_Extension_OnDone(t *testing.T) { select { case err = <-errCh: break - case <-time.After(time.Second * 10): + case <-time.After(time.Second * 30): // Increased timeout for real network requests err = errors.New("timeout") } // wait for script execution @@ -300,7 +301,7 @@ func TestDownloader_Extension_Errors(t *testing.T) { } rr, err := downloader.Resolve(&base.Request{ URL: "https://github.com/test", - }) + }, nil) if err != nil { t.Fatal(err) } @@ -315,7 +316,7 @@ func TestDownloader_Extension_Errors(t *testing.T) { } rr, err := downloader.Resolve(&base.Request{ URL: "https://github.com/test", - }) + }, nil) if err != nil { t.Fatal(err) } @@ -330,7 +331,7 @@ func TestDownloader_Extension_Errors(t *testing.T) { } _, err := downloader.Resolve(&base.Request{ URL: "https://github.com/test", - }) + }, nil) if err == nil { t.Fatalf("except error, but got nil") } @@ -352,7 +353,7 @@ func TestDownloader_Extension_Settings(t *testing.T) { } rr, err := downloader.Resolve(&base.Request{ URL: "https://github.com/test", - }) + }, nil) if err != nil { t.Fatal(err) } @@ -373,7 +374,7 @@ func TestDownloader_Extension_Settings(t *testing.T) { }) rr, err := downloader.Resolve(&base.Request{ URL: "https://github.com/test", - }) + }, nil) if err != nil { t.Fatal(err) } @@ -390,7 +391,7 @@ func TestDownloader_ExtensionStorage(t *testing.T) { } rr, err := downloader.Resolve(&base.Request{ URL: "https://github.com/test", - }) + }, nil) if err != nil { t.Fatal(err) } diff --git a/pkg/download/model.go b/pkg/download/model.go index 6a7b0246..c3126c1c 100644 --- a/pkg/download/model.go +++ b/pkg/download/model.go @@ -2,6 +2,9 @@ package download import ( "encoding/json" + "sync" + "time" + "github.com/GopeedLab/gopeed/internal/controller" "github.com/GopeedLab/gopeed/internal/fetcher" "github.com/GopeedLab/gopeed/internal/protocol/bt" @@ -9,8 +12,6 @@ import ( "github.com/GopeedLab/gopeed/pkg/base" "github.com/GopeedLab/gopeed/pkg/util" gonanoid "github.com/matoous/go-nanoid/v2" - "sync" - "time" ) type ResolveResult struct { diff --git a/pkg/download/webhook_test.go b/pkg/download/webhook_test.go index dcfcbb7b..b2b28093 100644 --- a/pkg/download/webhook_test.go +++ b/pkg/download/webhook_test.go @@ -50,9 +50,9 @@ func TestWebhook_TriggerOnDone(t *testing.T) { // Configure webhook URLs cfg, _ := downloader.GetConfig() cfg.Webhook = &base.WebhookConfig{ - Enable: true, - URLs: []string{server.URL}, - } + Enable: true, + URLs: []string{server.URL}, + } downloader.PutConfig(cfg) // Create a mock task @@ -99,9 +99,9 @@ func TestWebhook_TriggerOnError(t *testing.T) { // Configure webhook URLs cfg, _ := downloader.GetConfig() cfg.Webhook = &base.WebhookConfig{ - Enable: true, - URLs: []string{server.URL}, - } + Enable: true, + URLs: []string{server.URL}, + } downloader.PutConfig(cfg) // Create a mock task @@ -144,9 +144,9 @@ func TestWebhook_SendTestWebhook(t *testing.T) { // Configure webhook URLs cfg, _ := downloader.GetConfig() cfg.Webhook = &base.WebhookConfig{ - Enable: true, - URLs: []string{server.URL}, - } + Enable: true, + URLs: []string{server.URL}, + } downloader.PutConfig(cfg) // Send test webhook @@ -207,9 +207,9 @@ func TestWebhook_MultipleUrls(t *testing.T) { // Configure multiple webhook URLs cfg, _ := downloader.GetConfig() cfg.Webhook = &base.WebhookConfig{ - Enable: true, - URLs: []string{server1.URL, server2.URL}, - } + Enable: true, + URLs: []string{server1.URL, server2.URL}, + } downloader.PutConfig(cfg) // Create a mock task @@ -240,9 +240,9 @@ func TestWebhook_TestWebhookFailsOnNon200(t *testing.T) { // Configure webhook URLs cfg, _ := downloader.GetConfig() cfg.Webhook = &base.WebhookConfig{ - Enable: true, - URLs: []string{server.URL}, - } + Enable: true, + URLs: []string{server.URL}, + } downloader.PutConfig(cfg) // Send test webhook - should fail with non-200 status @@ -264,9 +264,9 @@ func TestWebhook_TestWebhookFailsOn201(t *testing.T) { // Configure webhook URLs cfg, _ := downloader.GetConfig() cfg.Webhook = &base.WebhookConfig{ - Enable: true, - URLs: []string{server.URL}, - } + Enable: true, + URLs: []string{server.URL}, + } downloader.PutConfig(cfg) // Send test webhook - should fail with 201 status (only 200 is success) @@ -346,7 +346,7 @@ func TestWebhook_GetWebhookUrls_NoExtraField(t *testing.T) { func TestWebhook_GetWebhookUrls_NoWebhookUrlsKey(t *testing.T) { setupWebhookTest(t, func(downloader *Downloader) { cfg, _ := downloader.GetConfig() - cfg.Webhook = &base.WebhookConfig{Enable: true} // No URLs set + cfg.Webhook = &base.WebhookConfig{Enable: true} // No URLs set downloader.PutConfig(cfg) urls := downloader.getWebhookUrls() @@ -360,9 +360,9 @@ func TestWebhook_GetWebhookUrls_StringSlice(t *testing.T) { setupWebhookTest(t, func(downloader *Downloader) { cfg, _ := downloader.GetConfig() cfg.Webhook = &base.WebhookConfig{ - Enable: true, - URLs: []string{"http://example.com", "http://example2.com"}, - } + Enable: true, + URLs: []string{"http://example.com", "http://example2.com"}, + } downloader.PutConfig(cfg) urls := downloader.getWebhookUrls() @@ -379,9 +379,9 @@ func TestWebhook_GetWebhookUrls_InterfaceSlice(t *testing.T) { setupWebhookTest(t, func(downloader *Downloader) { cfg, _ := downloader.GetConfig() cfg.Webhook = &base.WebhookConfig{ - Enable: true, - URLs: []string{"http://example.com", "http://example2.com"}, - } + Enable: true, + URLs: []string{"http://example.com", "http://example2.com"}, + } downloader.PutConfig(cfg) urls := downloader.getWebhookUrls() @@ -412,9 +412,9 @@ func TestWebhook_GetWebhookUrls_InterfaceSliceWithEmptyStrings(t *testing.T) { setupWebhookTest(t, func(downloader *Downloader) { cfg, _ := downloader.GetConfig() cfg.Webhook = &base.WebhookConfig{ - Enable: true, - URLs: []string{"", "", ""}, - } + Enable: true, + URLs: []string{"", "", ""}, + } downloader.PutConfig(cfg) urls := downloader.getWebhookUrls() @@ -446,7 +446,7 @@ func TestWebhook_GetWebhookUrls_InterfaceSliceMixedTypes(t *testing.T) { func TestWebhook_GetWebhookUrls_InvalidType(t *testing.T) { setupWebhookTest(t, func(downloader *Downloader) { cfg, _ := downloader.GetConfig() - cfg.Webhook = &base.WebhookConfig{Enable: false} // Disabled webhook + cfg.Webhook = &base.WebhookConfig{Enable: false} // Disabled webhook downloader.PutConfig(cfg) urls := downloader.getWebhookUrls() @@ -640,9 +640,9 @@ func TestWebhook_TriggerWebhooks_EmptyUrlSkipped(t *testing.T) { setupWebhookTest(t, func(downloader *Downloader) { cfg, _ := downloader.GetConfig() cfg.Webhook = &base.WebhookConfig{ - Enable: true, - URLs: []string{server.URL, "", server.URL}, - } + Enable: true, + URLs: []string{server.URL, "", server.URL}, + } downloader.PutConfig(cfg) task := NewTask() @@ -675,9 +675,9 @@ func TestWebhook_WebhookDataStructure(t *testing.T) { setupWebhookTest(t, func(downloader *Downloader) { cfg, _ := downloader.GetConfig() cfg.Webhook = &base.WebhookConfig{ - Enable: true, - URLs: []string{server.URL}, - } + Enable: true, + URLs: []string{server.URL}, + } downloader.PutConfig(cfg) task := NewTask() @@ -744,9 +744,9 @@ func TestWebhook_SendTestWebhook_MixedResults(t *testing.T) { setupWebhookTest(t, func(downloader *Downloader) { cfg, _ := downloader.GetConfig() cfg.Webhook = &base.WebhookConfig{ - Enable: true, - URLs: []string{server1.URL, server2.URL}, - } + Enable: true, + URLs: []string{server1.URL, server2.URL}, + } downloader.PutConfig(cfg) // Should fail because server2 returns 500 diff --git a/pkg/rest/api.go b/pkg/rest/api.go index 6c388fcf..c180b991 100644 --- a/pkg/rest/api.go +++ b/pkg/rest/api.go @@ -1,14 +1,15 @@ package rest import ( - "github.com/GopeedLab/gopeed/pkg/base" - "github.com/GopeedLab/gopeed/pkg/download" - "github.com/GopeedLab/gopeed/pkg/rest/model" - "github.com/gorilla/mux" "io" "net/http" "net/url" "runtime" + + "github.com/GopeedLab/gopeed/pkg/base" + "github.com/GopeedLab/gopeed/pkg/download" + "github.com/GopeedLab/gopeed/pkg/rest/model" + "github.com/gorilla/mux" ) func Info(w http.ResponseWriter, r *http.Request) { @@ -23,9 +24,9 @@ func Info(w http.ResponseWriter, r *http.Request) { } func Resolve(w http.ResponseWriter, r *http.Request) { - var req base.Request + var req model.ResolveTask if ReadJson(r, w, &req) { - rr, err := Downloader.Resolve(&req) + rr, err := Downloader.Resolve(req.Req, req.Opts) if err != nil { WriteJson(w, model.NewErrorResult(err.Error())) return @@ -42,9 +43,9 @@ func CreateTask(w http.ResponseWriter, r *http.Request) { err error ) if req.Rid != "" { - taskId, err = Downloader.Create(req.Rid, req.Opt) + taskId, err = Downloader.Create(req.Rid) } else if req.Req != nil { - taskId, err = Downloader.CreateDirect(req.Req, req.Opt) + taskId, err = Downloader.CreateDirect(req.Req, req.Opts) } else { WriteJson(w, model.NewErrorResult("param invalid: rid or req", model.CodeInvalidParam)) return diff --git a/pkg/rest/model/task.go b/pkg/rest/model/task.go index 4cd9e656..67b7331c 100644 --- a/pkg/rest/model/task.go +++ b/pkg/rest/model/task.go @@ -2,8 +2,13 @@ package model import "github.com/GopeedLab/gopeed/pkg/base" +type ResolveTask struct { + Req *base.Request `json:"req"` + Opts *base.Options `json:"opts"` +} type CreateTask struct { - Rid string `json:"rid"` - Req *base.Request `json:"req"` - Opt *base.Options `json:"opt"` + Rid string `json:"rid"` + + Req *base.Request `json:"req"` + Opts *base.Options `json:"opts"` } diff --git a/pkg/rest/server_test.go b/pkg/rest/server_test.go index dc57a946..47fcec4d 100644 --- a/pkg/rest/server_test.go +++ b/pkg/rest/server_test.go @@ -53,13 +53,13 @@ var ( "connections": 2, }, } - createReq = &model.CreateTask{ - Req: taskReq, - Opt: createOpts, + resolveReq = &model.ResolveTask{ + Req: taskReq, + Opts: createOpts, } - createResoledReq = &model.CreateTask{ - Req: taskReq, - Opt: createOpts, + createReq = &model.CreateTask{ + Req: taskReq, + Opts: createOpts, } installExtensionReq = &model.InstallExtension{ URL: "https://github.com/GopeedLab/gopeed-extension-samples#github-contributor-avatars-sample", @@ -80,7 +80,7 @@ func TestInfo(t *testing.T) { func TestResolve(t *testing.T) { doTest(func() { - resp := httpRequestCheckOk[*download.ResolveResult](http.MethodPost, "/api/v1/resolve", taskReq) + resp := httpRequestCheckOk[*download.ResolveResult](http.MethodPost, "/api/v1/resolve", resolveReq) if !test.AssertResourceEqual(taskRes, resp.Res) { t.Errorf("Resolve() got = %v, want %v", test.ToJson(resp.Res), test.ToJson(taskRes)) } @@ -89,7 +89,7 @@ func TestResolve(t *testing.T) { func TestCreateTask(t *testing.T) { doTest(func() { - resp := httpRequestCheckOk[*download.ResolveResult](http.MethodPost, "/api/v1/resolve", taskReq) + resp := httpRequestCheckOk[*download.ResolveResult](http.MethodPost, "/api/v1/resolve", resolveReq) var wg sync.WaitGroup wg.Add(1) @@ -101,7 +101,6 @@ func TestCreateTask(t *testing.T) { taskId := httpRequestCheckOk[string](http.MethodPost, "/api/v1/tasks", &model.CreateTask{ Rid: resp.ID, - Opt: createOpts, }) if taskId == "" { t.Fatal("create task failed") @@ -196,30 +195,6 @@ func TestCreateDirectTaskBatchWithOpt(t *testing.T) { }) } -func TestCreateDirectTaskWithResoled(t *testing.T) { - doTest(func() { - var wg sync.WaitGroup - wg.Add(1) - Downloader.Listener(func(event *download.Event) { - if event.Key == download.EventKeyFinally { - wg.Done() - } - }) - - taskId := httpRequestCheckOk[string](http.MethodPost, "/api/v1/tasks", createResoledReq) - if taskId == "" { - t.Fatal("create task failed") - } - - wg.Wait() - want := test.FileMd5(test.BuildFile) - got := test.FileMd5(test.DownloadFile) - if want != got { - t.Errorf("CreateDirectTaskWithResoled() got = %v, want %v", got, want) - } - }) -} - func TestPauseAndContinueTask(t *testing.T) { doTest(func() { var wg sync.WaitGroup diff --git a/ui/flutter/lib/api/api.dart b/ui/flutter/lib/api/api.dart index e7056b04..296a3699 100644 --- a/ui/flutter/lib/api/api.dart +++ b/ui/flutter/lib/api/api.dart @@ -16,6 +16,7 @@ import 'model/install_extension.dart'; import 'model/login.dart'; import 'model/request.dart'; import 'model/resolve_result.dart'; +import 'model/resolve_task.dart'; import 'model/result.dart'; import 'model/switch_extension.dart'; import 'model/task.dart'; @@ -123,9 +124,9 @@ Future _parse( } } -Future resolve(Request request) async { +Future resolve(ResolveTask resolveTask) async { return _parse( - () => _client.dio.post("api/v1/resolve", data: request), + () => _client.dio.post("api/v1/resolve", data: resolveTask), (data) => ResolveResult.fromJson(data)); } diff --git a/ui/flutter/lib/api/model/create_task.dart b/ui/flutter/lib/api/model/create_task.dart index 92bcbcdf..5ddbe999 100644 --- a/ui/flutter/lib/api/model/create_task.dart +++ b/ui/flutter/lib/api/model/create_task.dart @@ -9,12 +9,12 @@ part 'create_task.g.dart'; class CreateTask { String? rid; Request? req; - Options? opt; + Options? opts; CreateTask({ this.rid, this.req, - this.opt, + this.opts, }); factory CreateTask.fromJson( diff --git a/ui/flutter/lib/api/model/create_task.g.dart b/ui/flutter/lib/api/model/create_task.g.dart index 3e147035..f36e9432 100644 --- a/ui/flutter/lib/api/model/create_task.g.dart +++ b/ui/flutter/lib/api/model/create_task.g.dart @@ -11,9 +11,9 @@ CreateTask _$CreateTaskFromJson(Map json) => CreateTask( req: json['req'] == null ? null : Request.fromJson(json['req'] as Map), - opt: json['opt'] == null + opts: json['opts'] == null ? null - : Options.fromJson(json['opt'] as Map), + : Options.fromJson(json['opts'] as Map), ); Map _$CreateTaskToJson(CreateTask instance) { @@ -27,6 +27,6 @@ Map _$CreateTaskToJson(CreateTask instance) { writeNotNull('rid', instance.rid); writeNotNull('req', instance.req?.toJson()); - writeNotNull('opt', instance.opt?.toJson()); + writeNotNull('opts', instance.opts?.toJson()); return val; } diff --git a/ui/flutter/lib/api/model/create_task_batch.dart b/ui/flutter/lib/api/model/create_task_batch.dart index 8b2f7c98..470c5465 100644 --- a/ui/flutter/lib/api/model/create_task_batch.dart +++ b/ui/flutter/lib/api/model/create_task_batch.dart @@ -8,11 +8,11 @@ part 'create_task_batch.g.dart'; @JsonSerializable(explicitToJson: true) class CreateTaskBatch { List? reqs; - Options? opt; + Options? opts; CreateTaskBatch({ this.reqs, - this.opt, + this.opts, }); factory CreateTaskBatch.fromJson( diff --git a/ui/flutter/lib/api/model/create_task_batch.g.dart b/ui/flutter/lib/api/model/create_task_batch.g.dart index 5bc113e5..c9e24b0f 100644 --- a/ui/flutter/lib/api/model/create_task_batch.g.dart +++ b/ui/flutter/lib/api/model/create_task_batch.g.dart @@ -11,9 +11,9 @@ CreateTaskBatch _$CreateTaskBatchFromJson(Map json) => reqs: (json['reqs'] as List?) ?.map((e) => CreateTaskBatchItem.fromJson(e as Map)) .toList(), - opt: json['opt'] == null + opts: json['opts'] == null ? null - : Options.fromJson(json['opt'] as Map), + : Options.fromJson(json['opts'] as Map), ); Map _$CreateTaskBatchToJson(CreateTaskBatch instance) { @@ -26,7 +26,7 @@ Map _$CreateTaskBatchToJson(CreateTaskBatch instance) { } writeNotNull('reqs', instance.reqs?.map((e) => e.toJson()).toList()); - writeNotNull('opt', instance.opt?.toJson()); + writeNotNull('opts', instance.opts?.toJson()); return val; } diff --git a/ui/flutter/lib/api/model/resolve_task.dart b/ui/flutter/lib/api/model/resolve_task.dart new file mode 100644 index 00000000..6ec5eba7 --- /dev/null +++ b/ui/flutter/lib/api/model/resolve_task.dart @@ -0,0 +1,21 @@ +import 'package:json_annotation/json_annotation.dart'; + +import 'options.dart'; +import 'request.dart'; + +part 'resolve_task.g.dart'; + +@JsonSerializable() +class ResolveTask { + Request? req; + Options? opts; + + ResolveTask({ + this.req, + this.opts, + }); + + factory ResolveTask.fromJson(Map json) => + _$ResolveTaskFromJson(json); + Map toJson() => _$ResolveTaskToJson(this); +} diff --git a/ui/flutter/lib/api/model/resolve_task.g.dart b/ui/flutter/lib/api/model/resolve_task.g.dart new file mode 100644 index 00000000..df0ca4cd --- /dev/null +++ b/ui/flutter/lib/api/model/resolve_task.g.dart @@ -0,0 +1,30 @@ +// GENERATED CODE - DO NOT MODIFY BY HAND + +part of 'resolve_task.dart'; + +// ************************************************************************** +// JsonSerializableGenerator +// ************************************************************************** + +ResolveTask _$ResolveTaskFromJson(Map json) => ResolveTask( + req: json['req'] == null + ? null + : Request.fromJson(json['req'] as Map), + opts: json['opts'] == null + ? null + : Options.fromJson(json['opts'] as Map), + ); + +Map _$ResolveTaskToJson(ResolveTask instance) { + final val = {}; + + void writeNotNull(String key, dynamic value) { + if (value != null) { + val[key] = value; + } + } + + writeNotNull('req', instance.req); + writeNotNull('opts', instance.opts); + return val; +} diff --git a/ui/flutter/lib/app/modules/create/views/create_view.dart b/ui/flutter/lib/app/modules/create/views/create_view.dart index f067bbc6..0171448f 100644 --- a/ui/flutter/lib/app/modules/create/views/create_view.dart +++ b/ui/flutter/lib/app/modules/create/views/create_view.dart @@ -16,6 +16,7 @@ import '../../../../api/model/downloader_config.dart'; import '../../../../api/model/options.dart'; import '../../../../api/model/request.dart'; import '../../../../api/model/resolve_result.dart'; +import '../../../../api/model/resolve_task.dart'; import '../../../../api/model/task.dart'; import '../../../../database/database.dart'; import '../../../../util/input_formatter.dart'; @@ -125,13 +126,13 @@ class CreateView extends GetView { } // handle options - if (routerParams.opt != null) { - _renameController.text = routerParams.opt!.name; - _pathController.text = routerParams.opt!.path; + if (routerParams.opts != null) { + _renameController.text = routerParams.opts!.name; + _pathController.text = routerParams.opts!.path; final optionsHandlers = { Protocol.http: () { - final opt = routerParams.opt!; + final opt = routerParams.opts!; _renameController.text = opt.name; _pathController.text = opt.path; if (opt.extra != null) { @@ -143,7 +144,7 @@ class CreateView extends GetView { }, Protocol.bt: null, }; - if (routerParams.opt?.extra != null) { + if (routerParams.opts?.extra != null) { optionsHandlers[protocol]?.call(); } } @@ -814,29 +815,34 @@ class CreateView extends GetView { */ final isMultiLine = urls.length > 1; final isDirect = controller.directDownload.value || isMultiLine; + final opt = Options( + name: isMultiLine ? "" : _renameController.text, + path: _pathController.text, + selectFiles: [], + extra: parseReqOptsExtra(), + ); if (isDirect) { await Future.wait(urls.map((url) { return createTask(CreateTask( - req: Request( - url: url, - extra: parseReqExtra(url), - proxy: parseProxy(), - skipVerifyCert: _skipVerifyCertController.value, - ), - opt: Options( - name: isMultiLine ? "" : _renameController.text, - path: _pathController.text, - selectFiles: [], - extra: parseReqOptsExtra(), - ))); + req: Request( + url: url, + extra: parseReqExtra(url), + proxy: parseProxy(), + skipVerifyCert: _skipVerifyCertController.value, + ), + opts: opt, + )); })); Get.rootDelegate.offNamed(Routes.TASK); } else { - final rr = await resolve(Request( - url: submitUrl, - extra: parseReqExtra(_urlController.text), - proxy: parseProxy(), - skipVerifyCert: _skipVerifyCertController.value, + final rr = await resolve(ResolveTask( + req: Request( + url: submitUrl, + extra: parseReqExtra(_urlController.text), + proxy: parseProxy(), + skipVerifyCert: _skipVerifyCertController.value, + ), + opts: opt, )); await _showResolveDialog(rr); } @@ -983,7 +989,7 @@ class CreateView extends GetView { } else { await createTask(CreateTask( rid: rr.id, - opt: Options( + opts: Options( name: _renameController.text, path: _pathController.text, selectFiles: controller.selectedIndexes,