perf: slow down DHT announcing on errors (#1186)

This commit is contained in:
Levi authored and GitHub committed 2026-01-14 20:04:21 +08:00
1 parent f86a379f93
commit c9626cb389
17 files changed
+573 -563

No files matched your search

+1 -1
View File
@@ -1,4 +1,4 @@
FROM golang:1.24.0 AS go FROM golang:1.24.11 AS go
WORKDIR /app WORKDIR /app
COPY ./go.mod ./go.sum ./ COPY ./go.mod ./go.sum ./
RUN go mod download RUN go mod download
+1 -1
View File
@@ -201,7 +201,7 @@ This project is divided into two parts, the front end uses `flutter`, the back e
### 🌍 Environment ### 🌍 Environment
1. Golang 1.23+ 1. Golang 1.24+
2. Flutter 3.24+ 2. Flutter 3.24+
### 📋 Clone ### 📋 Clone
+1 -1
View File
@@ -182,7 +182,7 @@ Gopeed (正式名 Go Speed) は `Golang` + `Flutter` によって開発された
### 🌍 環境 ### 🌍 環境
1. Go 言語 1.23+ 1. Go 言語 1.24+
2. Flutter 3.24+ 2. Flutter 3.24+
### 📋 クローン ### 📋 クローン
+1 -1
View File
@@ -188,7 +188,7 @@ Dự án này được chia thành hai phần, phần giao diện sử dụng `f
### 🌍 Environment ### 🌍 Environment
1. Golang 1.23+ 1. Golang 1.24+
2. Flutter 3.24+ 2. Flutter 3.24+
### 📋 Clone ### 📋 Clone
+1 -1
View File
@@ -192,7 +192,7 @@ Gopeed 还提供了浏览器扩展用于接管浏览器下载,支持 Chrome、
### 🌍 环境要求 ### 🌍 环境要求
1. Golang 1.23+ 1. Golang 1.24+
2. Flutter 3.24+ 2. Flutter 3.24+
### 📋 克隆项目 ### 📋 克隆项目
+1 -1
View File
@@ -189,7 +189,7 @@ go install github.com/GopeedLab/gopeed/cmd/gopeed@latest
### 🌍 開發環境 ### 🌍 開發環境
1. Golang 1.23+ 1. Golang 1.24+
2. Flutter 3.24+ 2. Flutter 3.24+
### 📋 克隆項目 ### 📋 克隆項目
+59 -55
View File
@@ -1,13 +1,13 @@
module github.com/GopeedLab/gopeed module github.com/GopeedLab/gopeed
go 1.24.0 go 1.24.9
toolchain go1.24.11 toolchain go1.24.11
require ( require (
github.com/anacrolix/missinggo/v2 v2.8.0 github.com/anacrolix/torrent v1.60.1-0.20251217073903-486bcbe758e0
github.com/anacrolix/torrent v1.57.2-0.20241017235801-4d8437a05621
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5
github.com/bodgit/sevenzip v1.6.1
github.com/dop251/goja v0.0.0-20240919115326-6c7d1df7ff05 github.com/dop251/goja v0.0.0-20240919115326-6c7d1df7ff05
github.com/dop251/goja_nodejs v0.0.0-20240728170619-29b559befffc github.com/dop251/goja_nodejs v0.0.0-20240728170619-29b559befffc
github.com/go-git/go-git/v5 v5.8.1 github.com/go-git/go-git/v5 v5.8.1
@@ -21,19 +21,22 @@ require (
github.com/rs/dnscache v0.0.0-20230804202142-fc85eb664529 github.com/rs/dnscache v0.0.0-20230804202142-fc85eb664529
github.com/rs/zerolog v1.31.0 github.com/rs/zerolog v1.31.0
github.com/xiaoqidun/setft v0.0.0-20220310121541-be86327699ad github.com/xiaoqidun/setft v0.0.0-20220310121541-be86327699ad
go.etcd.io/bbolt v1.3.11 go.etcd.io/bbolt v1.4.3
golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f golang.org/x/exp v0.0.0-20251219203646-944ab1f22d93
golang.org/x/sync v0.17.0
) )
require ( require (
github.com/STARRY-S/zip v0.2.3 // indirect github.com/STARRY-S/zip v0.2.3 // indirect
github.com/anacrolix/btree v0.1.1 // indirect
github.com/anacrolix/missinggo/v2 v2.10.0 // indirect
github.com/andybalholm/brotli v1.2.0 // indirect github.com/andybalholm/brotli v1.2.0 // indirect
github.com/bodgit/plumbing v1.3.0 // indirect github.com/bodgit/plumbing v1.3.0 // indirect
github.com/bodgit/sevenzip v1.6.1 // indirect
github.com/bodgit/windows v1.0.1 // indirect github.com/bodgit/windows v1.0.1 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707 // indirect github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707 // indirect
github.com/felixge/fgprof v0.9.5 // indirect
github.com/go-task/slim-sprig/v3 v3.0.0 // indirect github.com/go-task/slim-sprig/v3 v3.0.0 // indirect
github.com/google/go-cmp v0.7.0 // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect
github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
@@ -45,15 +48,18 @@ require (
github.com/nwaples/rardecode/v2 v2.2.0 // indirect github.com/nwaples/rardecode/v2 v2.2.0 // indirect
github.com/onsi/ginkgo/v2 v2.23.4 // indirect github.com/onsi/ginkgo/v2 v2.23.4 // indirect
github.com/pierrec/lz4/v4 v4.1.22 // indirect github.com/pierrec/lz4/v4 v4.1.22 // indirect
github.com/pion/transport/v4 v4.0.1 // indirect
github.com/quic-go/qpack v0.5.1 // indirect github.com/quic-go/qpack v0.5.1 // indirect
github.com/quic-go/quic-go v0.51.0 // indirect github.com/quic-go/quic-go v0.51.0 // indirect
github.com/refraction-networking/utls v1.6.7 // indirect github.com/refraction-networking/utls v1.6.7 // indirect
github.com/sorairolake/lzip-go v0.3.8 // indirect github.com/sorairolake/lzip-go v0.3.8 // indirect
github.com/spf13/afero v1.15.0 // indirect github.com/spf13/afero v1.15.0 // indirect
github.com/ulikunitz/xz v0.5.15 // indirect github.com/ulikunitz/xz v0.5.15 // indirect
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
go.uber.org/automaxprocs v1.6.0 // indirect go.uber.org/automaxprocs v1.6.0 // indirect
go.uber.org/mock v0.5.1 // indirect go.uber.org/mock v0.5.1 // indirect
go4.org v0.0.0-20230225012048-214862532bf5 // indirect go4.org v0.0.0-20230225012048-214862532bf5 // indirect
golang.org/x/sync v0.19.0 // indirect
) )
require ( require (
@@ -62,25 +68,24 @@ require (
github.com/ProtonMail/go-crypto v0.0.0-20230717121422-5aa5874ade95 // indirect github.com/ProtonMail/go-crypto v0.0.0-20230717121422-5aa5874ade95 // indirect
github.com/RoaringBitmap/roaring v1.9.4 // indirect github.com/RoaringBitmap/roaring v1.9.4 // indirect
github.com/acomagu/bufpipe v1.0.4 // indirect github.com/acomagu/bufpipe v1.0.4 // indirect
github.com/ajwerner/btree v0.0.0-20211221152037-f427b3e689c0 // indirect
github.com/alecthomas/atomic v0.1.0-alpha2 // indirect github.com/alecthomas/atomic v0.1.0-alpha2 // indirect
github.com/anacrolix/chansync v0.6.0 // indirect github.com/anacrolix/chansync v0.7.0 // indirect
github.com/anacrolix/dht/v2 v2.22.0 // indirect github.com/anacrolix/dht/v2 v2.23.0 // indirect
github.com/anacrolix/envpprof v1.4.0 // indirect github.com/anacrolix/envpprof v1.5.0 // indirect
github.com/anacrolix/generics v0.0.3-0.20240902042256-7fb2702ef0ca // indirect github.com/anacrolix/generics v0.2.0 // indirect
github.com/anacrolix/go-libutp v1.3.1 // indirect github.com/anacrolix/go-libutp v1.3.2 // indirect
github.com/anacrolix/log v0.16.0 // indirect github.com/anacrolix/log v0.17.1-0.20251118025802-918f1157b7bb // indirect
github.com/anacrolix/missinggo v1.3.0 // indirect github.com/anacrolix/missinggo v1.3.0 // indirect
github.com/anacrolix/missinggo/perf v1.0.0 // indirect github.com/anacrolix/missinggo/perf v1.0.0 // indirect
github.com/anacrolix/mmsg v1.1.1 // indirect github.com/anacrolix/mmsg v1.1.1 // indirect
github.com/anacrolix/multiless v0.4.0 // indirect github.com/anacrolix/multiless v0.4.0 // indirect
github.com/anacrolix/stm v0.5.0 // indirect github.com/anacrolix/stm v0.5.0 // indirect
github.com/anacrolix/sync v0.5.3 // indirect github.com/anacrolix/sync v0.6.0 // indirect
github.com/anacrolix/upnp v0.1.4 // indirect github.com/anacrolix/upnp v0.1.4 // indirect
github.com/anacrolix/utp v0.2.0 // indirect github.com/anacrolix/utp v0.2.0 // indirect
github.com/bahlo/generic-list-go v0.2.0 // indirect github.com/bahlo/generic-list-go v0.2.0 // indirect
github.com/benbjohnson/immutable v0.4.3 // indirect github.com/benbjohnson/immutable v0.4.3 // indirect
github.com/bits-and-blooms/bitset v1.15.0 // indirect github.com/bits-and-blooms/bitset v1.24.4 // indirect
github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8 // indirect github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8 // indirect
github.com/cespare/xxhash v1.1.0 // indirect github.com/cespare/xxhash v1.1.0 // indirect
github.com/cloudflare/circl v1.6.1 // indirect github.com/cloudflare/circl v1.6.1 // indirect
@@ -92,44 +97,43 @@ require (
github.com/felixge/httpsnoop v1.0.1 // indirect github.com/felixge/httpsnoop v1.0.1 // indirect
github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
github.com/go-git/go-billy/v5 v5.4.1 // indirect github.com/go-git/go-billy/v5 v5.4.1 // indirect
github.com/go-llsqlite/adapter v0.1.0 // indirect github.com/go-llsqlite/adapter v0.2.0 // indirect
github.com/go-llsqlite/crawshaw v0.5.5 // indirect github.com/go-llsqlite/crawshaw v0.6.0 // indirect
github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/logr v1.4.3 // indirect
github.com/go-logr/stdr v1.2.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-sourcemap/sourcemap v2.1.4+incompatible // indirect github.com/go-sourcemap/sourcemap v2.1.4+incompatible // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/google/btree v1.1.3 // indirect github.com/google/btree v1.1.3 // indirect
github.com/google/pprof v0.0.0-20250423184734-337e5dd93bb4 // indirect github.com/google/pprof v0.0.0-20260106004452-d7df1bf2cac7 // indirect
github.com/google/uuid v1.6.0 // indirect github.com/google/uuid v1.6.0 // indirect
github.com/gorilla/websocket v1.5.3 // indirect github.com/gorilla/websocket v1.5.3 // indirect
github.com/huandu/xstrings v1.5.0 // indirect github.com/huandu/xstrings v1.5.0 // indirect
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
github.com/kevinburke/ssh_config v1.2.0 // indirect github.com/kevinburke/ssh_config v1.2.0 // indirect
github.com/klauspost/cpuid/v2 v2.2.9 // indirect github.com/klauspost/cpuid/v2 v2.3.0 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-isatty v0.0.20 // indirect
github.com/minio/sha256-simd v1.0.1 // indirect github.com/minio/sha256-simd v1.0.1 // indirect
github.com/mr-tron/base58 v1.2.0 // indirect github.com/mr-tron/base58 v1.2.0 // indirect
github.com/mschoch/smat v0.2.0 // indirect github.com/mschoch/smat v0.2.0 // indirect
github.com/multiformats/go-multihash v0.2.3 // indirect github.com/multiformats/go-multihash v0.2.3 // indirect
github.com/multiformats/go-varint v0.0.7 // indirect github.com/multiformats/go-varint v0.1.0 // indirect
github.com/ncruces/go-strftime v0.1.9 // indirect github.com/ncruces/go-strftime v1.0.0 // indirect
github.com/pion/datachannel v1.5.9 // indirect github.com/pion/datachannel v1.6.0 // indirect
github.com/pion/dtls/v3 v3.0.4 // indirect github.com/pion/dtls/v3 v3.0.10 // indirect
github.com/pion/ice/v4 v4.0.2 // indirect github.com/pion/ice/v4 v4.2.0 // indirect
github.com/pion/interceptor v0.1.37 // indirect github.com/pion/interceptor v0.1.42 // indirect
github.com/pion/logging v0.2.2 // indirect github.com/pion/logging v0.2.4 // indirect
github.com/pion/mdns/v2 v2.0.7 // indirect github.com/pion/mdns/v2 v2.1.0 // indirect
github.com/pion/randutil v0.1.0 // indirect github.com/pion/randutil v0.1.0 // indirect
github.com/pion/rtcp v1.2.14 // indirect github.com/pion/rtcp v1.2.16 // indirect
github.com/pion/rtp v1.8.9 // indirect github.com/pion/rtp v1.10.0 // indirect
github.com/pion/sctp v1.8.33 // indirect github.com/pion/sctp v1.9.1 // indirect
github.com/pion/sdp/v3 v3.0.9 // indirect github.com/pion/sdp/v3 v3.0.17 // indirect
github.com/pion/srtp/v3 v3.0.4 // indirect github.com/pion/srtp/v3 v3.0.10 // indirect
github.com/pion/stun/v3 v3.0.0 // indirect github.com/pion/stun/v3 v3.1.1 // indirect
github.com/pion/transport/v3 v3.0.7 // indirect github.com/pion/turn/v4 v4.1.4 // indirect
github.com/pion/turn/v4 v4.0.0 // indirect github.com/pion/webrtc/v4 v4.2.2-0.20260109001657-a5962f314db7 // indirect
github.com/pion/webrtc/v4 v4.0.2 // indirect
github.com/pjbgf/sha1cd v0.3.0 // indirect github.com/pjbgf/sha1cd v0.3.0 // indirect
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c
github.com/protolambda/ctxlock v0.1.0 // indirect github.com/protolambda/ctxlock v0.1.0 // indirect
@@ -137,24 +141,24 @@ require (
github.com/sergi/go-diff v1.1.0 // indirect github.com/sergi/go-diff v1.1.0 // indirect
github.com/skeema/knownhosts v1.2.0 // indirect github.com/skeema/knownhosts v1.2.0 // indirect
github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect
github.com/tidwall/btree v1.7.0 // indirect github.com/tidwall/btree v1.8.1 // indirect
github.com/wlynxg/anet v0.0.5 // indirect github.com/wlynxg/anet v0.0.5 // indirect
github.com/xanzy/ssh-agent v0.3.3 // indirect github.com/xanzy/ssh-agent v0.3.3 // indirect
go.opentelemetry.io/otel v1.32.0 // indirect go.opentelemetry.io/otel v1.39.0 // indirect
go.opentelemetry.io/otel/metric v1.32.0 // indirect go.opentelemetry.io/otel/metric v1.39.0 // indirect
go.opentelemetry.io/otel/trace v1.32.0 // indirect go.opentelemetry.io/otel/trace v1.39.0 // indirect
golang.org/x/crypto v0.41.0 // indirect golang.org/x/crypto v0.46.0 // indirect
golang.org/x/mod v0.27.0 // indirect golang.org/x/mod v0.31.0 // indirect
golang.org/x/net v0.43.0 // indirect golang.org/x/net v0.48.0 // indirect
golang.org/x/sys v0.35.0 // indirect golang.org/x/sys v0.40.0 // indirect
golang.org/x/text v0.29.0 golang.org/x/text v0.32.0
golang.org/x/time v0.8.0 // indirect golang.org/x/time v0.14.0 // indirect
golang.org/x/tools v0.36.0 // indirect golang.org/x/tools v0.40.0 // indirect
gopkg.in/warnings.v0 v0.1.2 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect
lukechampine.com/blake3 v1.3.0 // indirect lukechampine.com/blake3 v1.4.1 // indirect
modernc.org/libc v1.61.0 // indirect modernc.org/libc v1.67.4 // indirect
modernc.org/mathutil v1.6.0 // indirect modernc.org/mathutil v1.7.1 // indirect
modernc.org/memory v1.8.0 // indirect modernc.org/memory v1.11.0 // indirect
modernc.org/sqlite v1.33.1 // indirect modernc.org/sqlite v1.43.0 // indirect
zombiezen.com/go/sqlite v1.4.0 // indirect zombiezen.com/go/sqlite v1.4.2 // indirect
) )
+167 -206
View File
File diff suppressed because it is too large. Load diff
+3 -2
View File
@@ -110,6 +110,7 @@ func (f *Fetcher) Start() (err error) {
return return
} }
} }
files := f.torrent.Files() files := f.torrent.Files()
// If the user does not specify the file to download, all files will be downloaded by default // If the user does not specify the file to download, all files will be downloaded by default
if f.data.Progress == nil { if f.data.Progress == nil {
@@ -129,12 +130,12 @@ func (f *Fetcher) Start() (err error) {
file.Download() file.Download()
} }
} }
f.torrent.AllowDataDownload()
return return
} }
func (f *Fetcher) Pause() (err error) { func (f *Fetcher) Pause() (err error) {
f.torrentReady.Store(false) f.torrent.DisallowDataDownload()
f.safeDrop()
return return
} }
+152 -56
View File
@@ -23,9 +23,14 @@ import (
) )
const ( const (
connectTimeout = 15 * time.Second connectTimeout = 15 * time.Second
readTimeout = 15 * time.Second readTimeout = 15 * time.Second
helpMinSize = 1 * 1024 * 1024 // Minimum chunk size for helper connections minFastFailTimeout = int64(3 * time.Second) // Minimum timeout for fast-fail retry
// Work stealing parameters
// When a connection finishes its chunk, it can "steal" work from slow connections.
stealThresholdSeconds = 3 // Only steal if victim needs > 3 seconds to finish
stealMinChunkSize = 512 * 1024 // Min steal size: 512KB (avoid tiny chunks)
) )
// ============================================================================ // ============================================================================
@@ -96,6 +101,11 @@ type connection struct {
retryTimes int retryTimes int
lastErr error lastErr error
// Speed tracking for work stealing decisions
speed int64 // bytes per second
lastSpeedCheck int64 // timestamp in nanoseconds
lastSpeedDownload int64 // bytes downloaded at last check
ctx context.Context ctx context.Context
cancel context.CancelFunc cancel context.CancelFunc
} }
@@ -123,14 +133,6 @@ func newSlowStartController(maxConnections int) *slowStartController {
} }
} }
// startBatch marks the start of a new batch with count connections
func (s *slowStartController) startBatch(count int) {
s.mu.Lock()
defer s.mu.Unlock()
s.batchPending = count
s.batchReady = 0
}
// onConnectSuccess is called when a connection successfully gets HTTP response // onConnectSuccess is called when a connection successfully gets HTTP response
// Returns true if this completes the current batch // Returns true if this completes the current batch
func (s *slowStartController) onConnectSuccess() bool { func (s *slowStartController) onConnectSuccess() bool {
@@ -202,13 +204,6 @@ func (s *slowStartController) commitBatch(count int) {
s.batchReady = 0 s.batchReady = 0
} }
// isMaxReached returns true if max connections have been launched
func (s *slowStartController) isMaxReached() bool {
s.mu.Lock()
defer s.mu.Unlock()
return s.totalLaunched >= s.maxConnections
}
// ============================================================================ // ============================================================================
// Fetcher // Fetcher
// ============================================================================ // ============================================================================
@@ -231,6 +226,9 @@ type Fetcher struct {
// Slow start controller // Slow start controller
slowStart *slowStartController slowStart *slowStartController
// Max connection time for adaptive timeout (stored as int64 nanoseconds for atomic ops)
maxConnTime atomic.Int64
// First primary connection success signal // First primary connection success signal
primaryReadyOnce sync.Once primaryReadyOnce sync.Once
primaryReadyCh chan struct{} primaryReadyCh chan struct{}
@@ -264,6 +262,9 @@ type Fetcher struct {
cancel context.CancelFunc cancel context.CancelFunc
wg sync.WaitGroup wg sync.WaitGroup
// downloadLoop lifecycle tracking
downloadLoopDone chan struct{} // Closed when downloadLoop exits
// Resolve connection control // Resolve connection control
resolveCtx context.Context resolveCtx context.Context
resolveCancel context.CancelFunc resolveCancel context.CancelFunc
@@ -297,6 +298,14 @@ func (f *Fetcher) setState(s fetcherState) {
f.state.Store(int32(s)) f.state.Store(int32(s))
} }
// updateMaxConnTime updates maxConnTime if the new duration is larger
func (f *Fetcher) updateMaxConnTime(d time.Duration) {
newVal := int64(d)
if newVal > f.maxConnTime.Load() {
f.maxConnTime.Store(newVal)
}
}
func (f *Fetcher) Resolve(req *base.Request, opts *base.Options) error { func (f *Fetcher) Resolve(req *base.Request, opts *base.Options) error {
if err := base.ParseReqExtra[fhttp.ReqExtra](req); err != nil { if err := base.ParseReqExtra[fhttp.ReqExtra](req); err != nil {
return err return err
@@ -326,7 +335,7 @@ func (f *Fetcher) Resolve(req *base.Request, opts *base.Options) error {
// Build HTTP request WITHOUT Range header (normal request) // Build HTTP request WITHOUT Range header (normal request)
// This allows the response to be reused for downloading (important for one-time URLs) // This allows the response to be reused for downloading (important for one-time URLs)
httpReq, err := f.buildRequest(nil, req) httpReq, err := f.buildRequest(context.TODO(), req)
if err != nil { if err != nil {
f.setState(stateError) f.setState(stateError)
return err return err
@@ -335,11 +344,15 @@ func (f *Fetcher) Resolve(req *base.Request, opts *base.Options) error {
client := f.buildClient() client := f.buildClient()
// Send normal HTTP request (no Range header) // Send normal HTTP request (no Range header)
// Track connection time for adaptive timeout in download phase
connStartTime := time.Now()
resp, err := client.Do(httpReq) resp, err := client.Do(httpReq)
if err != nil { if err != nil {
f.setState(stateError) f.setState(stateError)
return err return err
} }
// Record connection time as baseline for fast-fail timeout
f.updateMaxConnTime(time.Since(connStartTime))
// Parse response to get resource info // Parse response to get resource info
res := &base.Resource{ res := &base.Resource{
@@ -639,6 +652,9 @@ func (f *Fetcher) doStart() error {
// Create main context // Create main context
f.ctx, f.cancel = context.WithCancel(context.Background()) f.ctx, f.cancel = context.WithCancel(context.Background())
// Create downloadLoop lifecycle channel
f.downloadLoopDone = make(chan struct{})
// Start download // Start download
f.setState(stateSlowStart) f.setState(stateSlowStart)
go f.downloadLoop() go f.downloadLoop()
@@ -648,16 +664,15 @@ func (f *Fetcher) doStart() error {
func (f *Fetcher) downloadLoop() { func (f *Fetcher) downloadLoop() {
defer func() { defer func() {
f.fileMu.Lock() // Update file last modified time before closing
if f.file != nil {
f.file.Close()
}
f.fileMu.Unlock()
// Update file last modified time
if f.config.UseServerCtime && f.meta.Res.Files[0].Ctime != nil { if f.config.UseServerCtime && f.meta.Res.Files[0].Ctime != nil {
setft.SetFileTime(f.meta.SingleFilepath(), time.Now(), *f.meta.Res.Files[0].Ctime, *f.meta.Res.Files[0].Ctime) setft.SetFileTime(f.meta.SingleFilepath(), time.Now(), *f.meta.Res.Files[0].Ctime, *f.meta.Res.Files[0].Ctime)
} }
// Signal that downloadLoop has exited
if f.downloadLoopDone != nil {
close(f.downloadLoopDone)
}
}() }()
// Check if this is a resume or fresh start // Check if this is a resume or fresh start
@@ -682,9 +697,18 @@ func (f *Fetcher) downloadLoop() {
case <-f.slowStart.expansionCh: case <-f.slowStart.expansionCh:
// Batch completed, try to expand // Batch completed, try to expand
if f.checkCompletion() { if f.checkCompletion() {
// All work is done, wait for connections to finish
f.waitForCompletion()
return return
} }
f.expandConnections() f.expandConnections()
// Check if we've reached steady state (max connections)
if f.getState() == stateSteady {
// Wait for all connections to complete
f.waitForCompletion()
return
}
} }
} }
} }
@@ -706,16 +730,9 @@ func (f *Fetcher) startResolveDownload() {
// Use the resolve response directly // Use the resolve response directly
go f.runConnectionWithResolveResp(conn) go f.runConnectionWithResolveResp(conn)
// For non-range downloads, wait for completion in the main downloadLoop // For non-range downloads, wait for completion directly in this goroutine
// by triggering immediate completion check // Don't create another goroutine to avoid WaitGroup reuse issues
go func() { f.waitForCompletion()
f.wg.Wait()
// Signal expansion channel to trigger completion check in downloadLoop
select {
case f.slowStart.expansionCh <- struct{}{}:
default:
}
}()
return return
} }
@@ -729,7 +746,8 @@ func (f *Fetcher) expandConnections() {
if batchSize <= 0 { if batchSize <= 0 {
// Max reached, transition to steady state // Max reached, transition to steady state
f.setState(stateSteady) f.setState(stateSteady)
go f.waitForCompletion() // Don't start a new goroutine - let the downloadLoop handle completion
// This avoids multiple goroutines calling wg.Wait() simultaneously
return return
} }
@@ -745,6 +763,15 @@ func (f *Fetcher) expandConnections() {
// If prefetched all data, mark as done // If prefetched all data, mark as done
if prefetched >= totalSize { if prefetched >= totalSize {
f.connMu.Unlock() f.connMu.Unlock()
// Close the file before signaling completion
f.fileMu.Lock()
if f.file != nil {
f.file.Close()
f.file = nil
}
f.fileMu.Unlock()
f.setState(stateDone) f.setState(stateDone)
f.doneCh <- nil f.doneCh <- nil
return return
@@ -773,6 +800,9 @@ func (f *Fetcher) expandConnections() {
// For subsequent batches, use "help other connection" strategy // For subsequent batches, use "help other connection" strategy
// Find connections with enough remaining work to split // Find connections with enough remaining work to split
// During slow start, use fixed minimum size since speed is not yet stable
minSplitSize := int64(stealMinChunkSize)
newConns := make([]*connection, 0, batchSize) newConns := make([]*connection, 0, batchSize)
for i := 0; i < batchSize; i++ { for i := 0; i < batchSize; i++ {
// Find the connection with most remaining work // Find the connection with most remaining work
@@ -784,7 +814,8 @@ func (f *Fetcher) expandConnections() {
continue continue
} }
remain := conn.Chunk.remain() remain := conn.Chunk.remain()
if remain > maxRemain && remain > helpMinSize*2 { // Only split if remaining work is at least 2x the minimum split size
if remain > maxRemain && remain > minSplitSize*2 {
maxRemainConn = conn maxRemainConn = conn
maxRemain = remain maxRemain = remain
} }
@@ -839,13 +870,19 @@ func (f *Fetcher) runConnection(conn *connection) {
conn.State = connConnecting conn.State = connConnecting
f.connMu.Unlock() f.connMu.Unlock()
client := f.buildClient() // Use fast-fail client for quick retry during download phase
client := f.buildFastFailClient()
buf := make([]byte, 8192) buf := make([]byte, 8192)
retries := 0 retries := 0
conn.retryTimes = 0 conn.retryTimes = 0
for { for {
// Rebuild client with updated fast-fail timeout on retries
if retries > 0 {
client = f.buildFastFailClient()
}
err := f.downloadChunkOnce(conn, client, buf) err := f.downloadChunkOnce(conn, client, buf)
if err == nil { if err == nil {
if !f.meta.Res.Range || !f.helpOtherConnection(conn) { if !f.meta.Res.Range || !f.helpOtherConnection(conn) {
@@ -937,6 +974,9 @@ func (f *Fetcher) downloadChunkOnce(conn *connection, client *http.Client, buf [
fmt.Sprintf(base.HttpHeaderRangeFormat, rangeStart, rangeEnd)) fmt.Sprintf(base.HttpHeaderRangeFormat, rangeStart, rangeEnd))
} }
// Record connection start time for adaptive timeout tracking
connStartTime := time.Now()
resp, err := client.Do(httpReq) resp, err := client.Do(httpReq)
if err != nil { if err != nil {
return err return err
@@ -947,6 +987,9 @@ func (f *Fetcher) downloadChunkOnce(conn *connection, client *http.Client, buf [
return NewRequestError(resp.StatusCode) return NewRequestError(resp.StatusCode)
} }
// Record successful connection time for adaptive timeout
f.updateMaxConnTime(time.Since(connStartTime))
f.connMu.Lock() f.connMu.Lock()
conn.State = connDownloading conn.State = connDownloading
conn.failed = false conn.failed = false
@@ -1005,6 +1048,19 @@ func (f *Fetcher) downloadChunkOnce(conn *connection, client *http.Client, buf [
f.connMu.Lock() f.connMu.Lock()
conn.Chunk.Downloaded += int64(n) conn.Chunk.Downloaded += int64(n)
conn.Downloaded += int64(n) conn.Downloaded += int64(n)
// Update connection speed periodically
now := time.Now().UnixNano()
if conn.lastSpeedCheck == 0 {
conn.lastSpeedCheck = now
conn.lastSpeedDownload = conn.Downloaded
} else if now-conn.lastSpeedCheck >= int64(500*time.Millisecond) {
elapsed := float64(now-conn.lastSpeedCheck) / float64(time.Second)
if elapsed > 0 {
conn.speed = int64(float64(conn.Downloaded-conn.lastSpeedDownload) / elapsed)
}
conn.lastSpeedCheck = now
conn.lastSpeedDownload = conn.Downloaded
}
f.connMu.Unlock() f.connMu.Unlock()
if finished { if finished {
@@ -1107,7 +1163,8 @@ func (f *Fetcher) runConnectionWithResolveResp(conn *connection) {
// runConnectionFallback is used when resolve response is not available // runConnectionFallback is used when resolve response is not available
func (f *Fetcher) runConnectionFallback(conn *connection) { func (f *Fetcher) runConnectionFallback(conn *connection) {
client := f.buildClient() // Use fast-fail client for quick retry during download phase
client := f.buildFastFailClient()
buf := make([]byte, 8192) buf := make([]byte, 8192)
retries := 0 retries := 0
@@ -1118,6 +1175,11 @@ func (f *Fetcher) runConnectionFallback(conn *connection) {
return return
} }
// Rebuild client with updated fast-fail timeout on retries
if retries > 0 {
client = f.buildFastFailClient()
}
f.connMu.Lock() f.connMu.Lock()
conn.State = connConnecting conn.State = connConnecting
f.connMu.Unlock() f.connMu.Unlock()
@@ -1128,6 +1190,9 @@ func (f *Fetcher) runConnectionFallback(conn *connection) {
return err return err
} }
// Record connection start time for adaptive timeout tracking
connStartTime := time.Now()
resp, err := client.Do(httpReq) resp, err := client.Do(httpReq)
if err != nil { if err != nil {
return err return err
@@ -1138,6 +1203,9 @@ func (f *Fetcher) runConnectionFallback(conn *connection) {
return NewRequestError(resp.StatusCode) return NewRequestError(resp.StatusCode)
} }
// Record successful connection time for adaptive timeout
f.updateMaxConnTime(time.Since(connStartTime))
f.connMu.Lock() f.connMu.Lock()
conn.State = connDownloading conn.State = connDownloading
conn.failed = false conn.failed = false
@@ -1248,34 +1316,50 @@ func (f *Fetcher) runConnectionFallback(conn *connection) {
} }
} }
// helpOtherConnection implements work stealing: when a connection finishes its chunk,
// it looks for connections that need more than stealThresholdSeconds to finish and steals half of its work.
func (f *Fetcher) helpOtherConnection(helper *connection) bool { func (f *Fetcher) helpOtherConnection(helper *connection) bool {
f.connMu.Lock() f.connMu.Lock()
defer f.connMu.Unlock() defer f.connMu.Unlock()
// Find the slowest connection // Find the connection with longest remaining time
var maxRemainConnection *connection var slowestConn *connection
var maxRemain int64 var maxRemainSeconds int64
for _, r := range f.connections { for _, r := range f.connections {
if r == helper || r.Completed || r.State == connFailed { if r == helper || r.Completed || r.State == connFailed {
continue continue
} }
remain := r.Chunk.remain() remain := r.Chunk.remain()
if remain > maxRemain && remain > helpMinSize { if remain < stealMinChunkSize {
maxRemainConnection = r continue
maxRemain = remain }
// Calculate remaining time in seconds for this connection
var remainSeconds int64
if r.speed > 0 {
remainSeconds = remain / r.speed
} else {
// Speed unknown, assume it needs help if chunk is large enough
remainSeconds = stealThresholdSeconds + 1
}
// Only consider if it needs more than threshold seconds to finish
if remainSeconds > stealThresholdSeconds && remainSeconds > maxRemainSeconds {
slowestConn = r
maxRemainSeconds = remainSeconds
} }
} }
if maxRemainConnection == nil { if slowestConn == nil {
return false return false
} }
// Re-calculate the chunk range // Re-calculate the chunk range: steal half of the remaining work
helper.Chunk.Begin = maxRemainConnection.Chunk.End - maxRemainConnection.Chunk.remain()/2 helper.Chunk.Begin = slowestConn.Chunk.End - slowestConn.Chunk.remain()/2
helper.Chunk.End = maxRemainConnection.Chunk.End helper.Chunk.End = slowestConn.Chunk.End
helper.Chunk.Downloaded = 0 helper.Chunk.Downloaded = 0
maxRemainConnection.Chunk.End = helper.Chunk.Begin - 1 slowestConn.Chunk.End = helper.Chunk.Begin - 1
return true return true
} }
@@ -1380,6 +1464,15 @@ func (f *Fetcher) onDownloadComplete() {
} }
f.connMu.Unlock() f.connMu.Unlock()
// Close the file before signaling completion
// This ensures the file handle is released before Wait() returns
f.fileMu.Lock()
if f.file != nil {
f.file.Close()
f.file = nil
}
f.fileMu.Unlock()
if finalErr != nil { if finalErr != nil {
f.setState(stateError) f.setState(stateError)
} else { } else {
@@ -1406,8 +1499,7 @@ func (f *Fetcher) checkCompletion() bool {
} }
if f.meta.Res.Size > 0 && totalDownloaded >= f.meta.Res.Size { if f.meta.Res.Size > 0 && totalDownloaded >= f.meta.Res.Size {
f.setState(stateSteady) // Don't start a new goroutine - let the caller handle completion
go f.waitForCompletion()
return true return true
} }
@@ -1424,8 +1516,7 @@ func (f *Fetcher) checkCompletion() bool {
} }
if allCompleted { if allCompleted {
f.setState(stateSteady) // Don't start a new goroutine - let the caller handle completion
go f.waitForCompletion()
return true return true
} }
@@ -1450,7 +1541,12 @@ func (f *Fetcher) Pause() error {
} }
} }
// Wait for all goroutines to stop // Wait for downloadLoop to exit first (it will call wg.Wait internally)
if f.downloadLoopDone != nil {
<-f.downloadLoopDone
}
// Wait for all connection goroutines to stop
f.wg.Wait() f.wg.Wait()
// Wait for prefetch to finish // Wait for prefetch to finish
+29 -1
View File
@@ -13,6 +13,7 @@ import (
"net/http/cookiejar" "net/http/cookiejar"
"net/url" "net/url"
"strings" "strings"
"time"
"unicode/utf8" "unicode/utf8"
"github.com/GopeedLab/gopeed/pkg/base" "github.com/GopeedLab/gopeed/pkg/base"
@@ -120,15 +121,42 @@ func (f *Fetcher) buildRequest(ctx context.Context, req *base.Request) (httpReq
return httpReq, nil return httpReq, nil
} }
// buildClient creates an HTTP client with the default connection timeout.
// Used for resolve phase where we don't have connection time data yet.
func (f *Fetcher) buildClient() *http.Client { func (f *Fetcher) buildClient() *http.Client {
return f.buildClientWithTimeout(connectTimeout)
}
// buildFastFailClient creates an HTTP client with fast-fail timeout.
// Uses max(minFastFailTimeout, maxConnTime) for fast-fail retry during download phase.
func (f *Fetcher) buildFastFailClient() *http.Client {
maxConn := f.maxConnTime.Load()
if maxConn == 0 {
// No successful connection yet, use default timeout
return f.buildClientWithTimeout(connectTimeout)
}
timeout := maxConn
if timeout >= minFastFailTimeout {
// If greater than minFastFailTimeout, increase by 50% for safety margin
timeout = int64(float64(timeout) * 1.5)
} else {
timeout = minFastFailTimeout
}
return f.buildClientWithTimeout(time.Duration(timeout))
}
// buildClientWithTimeout creates an HTTP client with the specified connection timeout.
func (f *Fetcher) buildClientWithTimeout(timeout time.Duration) *http.Client {
transport := &http.Transport{ transport := &http.Transport{
DialContext: (&net.Dialer{ DialContext: (&net.Dialer{
Timeout: connectTimeout, Timeout: timeout,
}).DialContext, }).DialContext,
Proxy: f.ctl.GetProxy(f.meta.Req.Proxy), Proxy: f.ctl.GetProxy(f.meta.Req.Proxy),
TLSClientConfig: &tls.Config{ TLSClientConfig: &tls.Config{
InsecureSkipVerify: f.meta.Req.SkipVerifyCert, InsecureSkipVerify: f.meta.Req.SkipVerifyCert,
}, },
TLSHandshakeTimeout: timeout,
} }
jar, _ := cookiejar.New(nil) jar, _ := cookiejar.New(nil)
return &http.Client{ return &http.Client{
+101 -215
View File
@@ -757,7 +757,7 @@ func TestDownloader_Delete(t *testing.T) {
}) })
} }
func TestDownloader_ContinueBatch(t *testing.T) { func TestDownloader_PauseAndContinue(t *testing.T) {
listener := test.StartTestSlowFileServer(time.Millisecond * 2000) listener := test.StartTestSlowFileServer(time.Millisecond * 2000)
defer listener.Close() defer listener.Close()
@@ -767,7 +767,74 @@ func TestDownloader_ContinueBatch(t *testing.T) {
} }
defer downloader.Clear() defer downloader.Clear()
// Create multiple tasks and pause them // Create a single task
req := &base.Request{
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
}
rr, err := downloader.Resolve(req, testDownloadOpt)
if err != nil {
t.Fatal(err)
}
taskId, err := downloader.Create(rr.ID)
if err != nil {
t.Fatal(err)
}
// Wait for task to start
time.Sleep(time.Millisecond * 100)
// Pause with specific taskId
err = downloader.Pause(&TaskFilter{IDs: []string{taskId}})
if err != nil {
t.Fatal(err)
}
time.Sleep(time.Millisecond * 100)
// Verify task is paused
task := downloader.GetTask(taskId)
if task.Status != base.DownloadStatusPause {
t.Errorf("Task should be paused, got %s", task.Status)
}
// Continue with specific taskId
var wg sync.WaitGroup
wg.Add(1)
downloader.Listener(func(event *Event) {
if event.Key == EventKeyDone {
wg.Done()
}
})
err = downloader.Continue(&TaskFilter{IDs: []string{taskId}})
if err != nil {
t.Fatal(err)
}
// Wait for task to complete
wg.Wait()
// Verify task is done
task = downloader.GetTask(taskId)
if task.Status != base.DownloadStatusDone {
t.Errorf("Task should be done, got %s", task.Status)
}
// Clean up
downloader.Delete(nil, true)
}
func TestDownloader_PauseAllAndContinueAll(t *testing.T) {
listener := test.StartTestSlowFileServer(time.Millisecond * 2000)
defer listener.Close()
downloader := NewDownloader(nil)
if err := downloader.Setup(); err != nil {
t.Fatal(err)
}
defer downloader.Clear()
// Create multiple tasks
taskCount := 3 taskCount := 3
taskIds := make([]string, 0) taskIds := make([]string, 0)
@@ -786,183 +853,57 @@ func TestDownloader_ContinueBatch(t *testing.T) {
taskIds = append(taskIds, taskId) taskIds = append(taskIds, taskId)
} }
// Wait a moment for tasks to start // Wait for tasks to start
time.Sleep(time.Millisecond * 500) time.Sleep(time.Millisecond * 100)
// Pause all tasks // Pause all tasks with nil filter
err := downloader.Pause(nil) err := downloader.Pause(nil)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
// Verify tasks are paused time.Sleep(time.Millisecond * 100)
// Verify all tasks are paused
pausedCount := 0
for _, taskId := range taskIds { for _, taskId := range taskIds {
task := downloader.GetTask(taskId) task := downloader.GetTask(taskId)
if task.Status != base.DownloadStatusPause && task.Status != base.DownloadStatusDone { if task.Status == base.DownloadStatusPause {
t.Errorf("Task %s should be paused or done, got %s", taskId, task.Status) pausedCount++
} }
} }
if pausedCount != taskCount {
t.Errorf("Expected %d paused tasks, got %d", taskCount, pausedCount)
}
// Test ContinueBatch with specific IDs // Continue all tasks with nil filter
t.Run("ContinueBatch with filter", func(t *testing.T) { var wg sync.WaitGroup
var wg sync.WaitGroup wg.Add(taskCount)
wg.Add(1) downloader.Listener(func(event *Event) {
downloader.Listener(func(event *Event) { if event.Key == EventKeyDone {
if event.Key == EventKeyDone { wg.Done()
wg.Done()
}
})
err := downloader.ContinueBatch(&TaskFilter{IDs: []string{taskIds[0]}})
if err != nil {
t.Errorf("ContinueBatch() unexpected error: %v", err)
}
wg.Wait()
})
// Test ContinueBatch with nil filter (continues all)
t.Run("ContinueBatch with nil filter", func(t *testing.T) {
var wg sync.WaitGroup
// Count remaining paused tasks
pausedCount := 0
for _, taskId := range taskIds {
task := downloader.GetTask(taskId)
if task.Status == base.DownloadStatusPause {
pausedCount++
}
}
wg.Add(pausedCount)
downloader.Listener(func(event *Event) {
if event.Key == EventKeyDone {
wg.Done()
}
})
err := downloader.ContinueBatch(nil)
if err != nil {
t.Errorf("ContinueBatch() unexpected error: %v", err)
}
if pausedCount > 0 {
wg.Wait()
} }
}) })
// Clean up err = downloader.Continue(nil)
downloader.Delete(nil, true)
}
func TestDownloader_PauseAndContinue(t *testing.T) {
listener := test.StartTestSlowFileServer(time.Millisecond * 5000)
defer listener.Close()
downloader := NewDownloader(nil)
if err := downloader.Setup(); err != nil {
t.Fatal(err)
}
defer downloader.Clear()
req := &base.Request{
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
}
rr, err := downloader.Resolve(req, testDownloadOpt)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
taskId, err := downloader.Create(rr.ID) // Wait for all tasks to complete
if err != nil { wg.Wait()
t.Fatal(err)
// Verify all tasks are done
doneCount := 0
for _, taskId := range taskIds {
task := downloader.GetTask(taskId)
if task.Status == base.DownloadStatusDone {
doneCount++
}
}
if doneCount != taskCount {
t.Errorf("Expected %d done tasks, got %d", taskCount, doneCount)
} }
// Wait for task to start
time.Sleep(time.Millisecond * 1000)
// Test Pause with empty filter (returns nil, not error - empty filter means no op)
t.Run("Pause with empty filter", func(t *testing.T) {
err := downloader.Pause(&TaskFilter{})
// Empty filter returns error from Pause function
if err == nil {
// This is fine - different implementations might handle this differently
}
})
// Test Pause with filter
t.Run("Pause with filter", func(t *testing.T) {
task := downloader.GetTask(taskId)
if task == nil {
t.Skip("Task already completed")
}
if task.Status == base.DownloadStatusDone {
t.Skip("Task already completed")
}
err := downloader.Pause(&TaskFilter{IDs: []string{taskId}})
if err != nil {
// Task might have finished between check and pause
t.Logf("Pause() error (task may have finished): %v", err)
} else {
task = downloader.GetTask(taskId)
if task.Status != base.DownloadStatusPause && task.Status != base.DownloadStatusDone {
t.Errorf("Task should be paused or done, got %s", task.Status)
}
}
})
// Test Pause with non-matching filter
t.Run("Pause with non-matching filter", func(t *testing.T) {
err := downloader.Pause(&TaskFilter{IDs: []string{"non-existent-id"}})
if err == nil {
t.Error("Pause() with non-matching filter should return error")
}
})
// Test Continue with empty filter (returns error - empty filter means no op)
t.Run("Continue with empty filter", func(t *testing.T) {
err := downloader.Continue(&TaskFilter{})
// Empty filter returns error from Continue function
if err == nil {
// This is fine - different implementations might handle this differently
}
})
// Test Continue with non-matching filter
t.Run("Continue with non-matching filter", func(t *testing.T) {
err := downloader.Continue(&TaskFilter{IDs: []string{"non-existent-id"}})
if err == nil {
t.Error("Continue() with non-matching filter should return error")
}
})
// Test Continue with valid filter
t.Run("Continue with valid filter", func(t *testing.T) {
task := downloader.GetTask(taskId)
if task == nil {
t.Skip("Task not found")
}
if task.Status == base.DownloadStatusDone {
t.Skip("Task already completed")
}
var wg sync.WaitGroup
wg.Add(1)
downloader.Listener(func(event *Event) {
if event.Key == EventKeyDone && event.Task != nil && event.Task.ID == taskId {
wg.Done()
}
})
err := downloader.Continue(&TaskFilter{IDs: []string{taskId}})
if err != nil {
// Task might have finished or not be in pausable state
t.Logf("Continue() note: %v", err)
}
wg.Wait()
task = downloader.GetTask(taskId)
if task != nil && task.Status != base.DownloadStatusDone {
t.Errorf("Task should be done, got %s", task.Status)
}
})
// Clean up // Clean up
downloader.Delete(nil, true) downloader.Delete(nil, true)
@@ -1603,61 +1544,6 @@ func TestDownloader_DeleteAll(t *testing.T) {
} }
} }
func TestDownloader_ContinueAll(t *testing.T) {
listener := test.StartTestFileServer()
defer listener.Close()
downloader := NewDownloader(nil)
if err := downloader.Setup(); err != nil {
t.Fatal(err)
}
defer downloader.Clear()
// Create a task and wait for completion
req := &base.Request{
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
}
var wg sync.WaitGroup
wg.Add(1)
downloader.Listener(func(event *Event) {
if event.Key == EventKeyDone {
wg.Done()
}
})
taskId, err := downloader.CreateDirect(req, &base.Options{
Path: test.Dir,
Name: test.DownloadName,
Extra: http.OptsExtra{
Connections: 4,
},
})
if err != nil {
t.Fatal(err)
}
wg.Wait()
// Verify task completed
task := downloader.GetTask(taskId)
if task == nil {
t.Fatal("Task should exist")
}
if task.Status != base.DownloadStatusDone {
t.Errorf("Task should be done, got %s", task.Status)
}
// Test ContinueBatch with nil on an already done task (should be a no-op)
err = downloader.ContinueBatch(nil)
if err != nil {
t.Logf("ContinueBatch(nil) on done tasks returned: %v", err)
}
// Clean up
downloader.Delete(nil, true)
}
func TestDownloader_ProtocolConfigNotExist(t *testing.T) { func TestDownloader_ProtocolConfigNotExist(t *testing.T) {
downloader := NewDownloader(nil) downloader := NewDownloader(nil)
if err := downloader.Setup(); err != nil { if err := downloader.Setup(); err != nil {
+1 -1
View File
@@ -229,7 +229,7 @@ func extractArchive(archivePath string, destDir string, password string, progres
func createCountingHandler(count *int) func(ctx context.Context, fileInfo archives.FileInfo) error { func createCountingHandler(count *int) func(ctx context.Context, fileInfo archives.FileInfo) error {
return func(ctx context.Context, fileInfo archives.FileInfo) error { return func(ctx context.Context, fileInfo archives.FileInfo) error {
if !fileInfo.IsDir() { if !fileInfo.IsDir() {
*count++ (*count)++
} }
return nil return nil
} }
+31 -9
View File
@@ -227,11 +227,14 @@ func createTestZipWithMultipleFiles(path string, numFiles int) error {
w := zip.NewWriter(zipFile) w := zip.NewWriter(zipFile)
for i := 0; i < numFiles; i++ { for i := 0; i < numFiles; i++ {
f, err := w.Create(filepath.Join("dir", "file"+string(rune('A'+i))+".txt")) // Use Windows-safe names. The previous rune-based approach could generate
// invalid Windows filename characters (e.g. '\\', ':', '|') when numFiles is large.
nameInZip := fmt.Sprintf("dir/file%03d.txt", i+1)
f, err := w.Create(nameInZip)
if err != nil { if err != nil {
return err return err
} }
_, err = f.Write([]byte("Content of file " + string(rune('A'+i)))) _, err = f.Write([]byte(fmt.Sprintf("Content of file %03d", i+1)))
if err != nil { if err != nil {
return err return err
} }
@@ -1055,7 +1058,15 @@ func TestExtractArchive_ReadOnlyDestDir(t *testing.T) {
t.Fatal(err) t.Fatal(err)
} }
// Make it writable for cleanup // Make it writable for cleanup
defer os.Chmod(destDir, 0755) defer func() { _ = os.Chmod(destDir, 0755) }()
// Windows ACL semantics mean chmod often does not prevent writes.
// If the filesystem doesn't enforce the permission change, skip.
probe := filepath.Join(destDir, "__write_probe.tmp")
if err := os.WriteFile(probe, []byte("x"), 0644); err == nil {
_ = os.Remove(probe)
t.Skip("filesystem does not enforce read-only directory permissions")
}
// Extraction should fail due to read-only destination // Extraction should fail due to read-only destination
err = extractArchive(zipPath, destDir, "", nil) err = extractArchive(zipPath, destDir, "", nil)
@@ -1589,9 +1600,14 @@ func TestGetMultiPartArchiveBaseName(t *testing.T) {
for _, tt := range tests { for _, tt := range tests {
t.Run(tt.filename, func(t *testing.T) { t.Run(tt.filename, func(t *testing.T) {
result := GetMultiPartArchiveBaseName(tt.filename) filename := filepath.FromSlash(tt.filename)
if result != tt.expected { expected := tt.expected
t.Errorf("GetMultiPartArchiveBaseName(%q) = %q, expected %q", tt.filename, result, tt.expected) if expected != "" {
expected = filepath.FromSlash(expected)
}
result := GetMultiPartArchiveBaseName(filename)
if result != expected {
t.Errorf("GetMultiPartArchiveBaseName(%q) = %q, expected %q", filename, result, expected)
} }
}) })
} }
@@ -2067,10 +2083,16 @@ func TestDetermineFirstPartPath(t *testing.T) {
for _, tt := range tests { for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) { t.Run(tt.name, func(t *testing.T) {
result := determineFirstPartPath(tt.dir, tt.baseName, tt.pattern) dir := filepath.FromSlash(tt.dir)
if result != tt.expected { expected := tt.expected
if expected != "" {
expected = filepath.FromSlash(expected)
}
result := determineFirstPartPath(dir, tt.baseName, tt.pattern)
if result != expected {
t.Errorf("determineFirstPartPath(%q, %q, %q) = %q, expected %q", t.Errorf("determineFirstPartPath(%q, %q, %q) = %q, expected %q",
tt.dir, tt.baseName, tt.pattern, result, tt.expected) dir, tt.baseName, tt.pattern, result, expected)
} }
}) })
} }
+14 -11
View File
@@ -2,11 +2,12 @@ package download
import ( import (
"encoding/json" "encoding/json"
"go.etcd.io/bbolt"
"os" "os"
"path/filepath" "path/filepath"
"reflect" "reflect"
"sync" "sync"
"go.etcd.io/bbolt"
) )
type Storage interface { type Storage interface {
@@ -45,22 +46,24 @@ func changeValue(p any, v any) {
} }
} }
var memData = make(map[string]map[string]any)
type MemStorage struct { type MemStorage struct {
lock *sync.RWMutex lock *sync.RWMutex
data map[string]map[string]any
} }
func NewMemStorage() *MemStorage { func NewMemStorage() *MemStorage {
return &MemStorage{ return &MemStorage{
lock: &sync.RWMutex{}, lock: &sync.RWMutex{},
data: make(map[string]map[string]any),
} }
} }
func (n *MemStorage) Setup(buckets []string) error { func (n *MemStorage) Setup(buckets []string) error {
n.lock.Lock()
defer n.lock.Unlock()
for _, bucket := range buckets { for _, bucket := range buckets {
if _, ok := memData[bucket]; !ok { if _, ok := n.data[bucket]; !ok {
memData[bucket] = make(map[string]any) n.data[bucket] = make(map[string]any)
} }
} }
return nil return nil
@@ -69,7 +72,7 @@ func (n *MemStorage) Setup(buckets []string) error {
func (n *MemStorage) Put(bucket string, key string, v any) error { func (n *MemStorage) Put(bucket string, key string, v any) error {
n.lock.Lock() n.lock.Lock()
defer n.lock.Unlock() defer n.lock.Unlock()
if bucketData, ok := memData[bucket]; ok { if bucketData, ok := n.data[bucket]; ok {
bucketData[key] = v bucketData[key] = v
} }
return nil return nil
@@ -78,7 +81,7 @@ func (n *MemStorage) Put(bucket string, key string, v any) error {
func (n *MemStorage) Get(bucket string, key string, v any) (bool, error) { func (n *MemStorage) Get(bucket string, key string, v any) (bool, error) {
n.lock.RLock() n.lock.RLock()
defer n.lock.RUnlock() defer n.lock.RUnlock()
if dv, ok := memData[bucket][key]; ok { if dv, ok := n.data[bucket][key]; ok {
changeValue(v, dv) changeValue(v, dv)
return true, nil return true, nil
} }
@@ -88,7 +91,7 @@ func (n *MemStorage) Get(bucket string, key string, v any) (bool, error) {
func (n *MemStorage) List(bucket string, v any) error { func (n *MemStorage) List(bucket string, v any) error {
n.lock.RLock() n.lock.RLock()
defer n.lock.RUnlock() defer n.lock.RUnlock()
data := memData[bucket] data := n.data[bucket]
list := make([]any, 0) list := make([]any, 0)
for _, v := range data { for _, v := range data {
list = append(list, v) list = append(list, v)
@@ -100,7 +103,7 @@ func (n *MemStorage) List(bucket string, v any) error {
func (n *MemStorage) Pop(bucket string, key string, v any) error { func (n *MemStorage) Pop(bucket string, key string, v any) error {
n.lock.Lock() n.lock.Lock()
defer n.lock.Unlock() defer n.lock.Unlock()
data := memData[bucket] data := n.data[bucket]
changeValue(v, data[key]) changeValue(v, data[key])
delete(data, key) delete(data, key)
return nil return nil
@@ -109,7 +112,7 @@ func (n *MemStorage) Pop(bucket string, key string, v any) error {
func (n *MemStorage) Delete(bucket string, key string) error { func (n *MemStorage) Delete(bucket string, key string) error {
n.lock.Lock() n.lock.Lock()
defer n.lock.Unlock() defer n.lock.Unlock()
delete(memData[bucket], key) delete(n.data[bucket], key)
return nil return nil
} }
@@ -120,7 +123,7 @@ func (n *MemStorage) Close() error {
func (n *MemStorage) Clear() error { func (n *MemStorage) Clear() error {
n.lock.Lock() n.lock.Lock()
defer n.lock.Unlock() defer n.lock.Unlock()
memData = make(map[string]map[string]any) n.data = make(map[string]map[string]any)
return nil return nil
} }
+3
View File
@@ -216,6 +216,7 @@ func TestPauseAndContinueTask(t *testing.T) {
if t2.Status != base.DownloadStatusPause { if t2.Status != base.DownloadStatusPause {
t.Errorf("PauseTask() got = %v, want %v", t2.Status, base.DownloadStatusPause) t.Errorf("PauseTask() got = %v, want %v", t2.Status, base.DownloadStatusPause)
} }
time.Sleep(time.Millisecond * 100)
httpRequestCheckOk[any](http.MethodPut, "/api/v1/tasks/"+taskId+"/continue", nil) httpRequestCheckOk[any](http.MethodPut, "/api/v1/tasks/"+taskId+"/continue", nil)
t3 := httpRequestCheckOk[*download.Task](http.MethodGet, "/api/v1/tasks/"+taskId, nil) t3 := httpRequestCheckOk[*download.Task](http.MethodGet, "/api/v1/tasks/"+taskId, nil)
if t3.Status != base.DownloadStatusRunning { if t3.Status != base.DownloadStatusRunning {
@@ -240,6 +241,7 @@ func TestPauseAllAndContinueALLTasks(t *testing.T) {
createAndPause := func() { createAndPause := func() {
taskId := httpRequestCheckOk[string](http.MethodPost, "/api/v1/tasks", createReq) taskId := httpRequestCheckOk[string](http.MethodPost, "/api/v1/tasks", createReq)
time.Sleep(time.Millisecond * 5)
httpRequestCheckOk[*download.Task](http.MethodPut, "/api/v1/tasks/"+taskId+"/pause", nil) httpRequestCheckOk[*download.Task](http.MethodPut, "/api/v1/tasks/"+taskId+"/pause", nil)
} }
@@ -247,6 +249,7 @@ func TestPauseAllAndContinueALLTasks(t *testing.T) {
for i := 0; i < total; i++ { for i := 0; i < total; i++ {
createAndPause() createAndPause()
} }
time.Sleep(time.Millisecond * 50)
// continue all // continue all
httpRequestCheckOk[any](http.MethodPut, "/api/v1/tasks/continue", nil) httpRequestCheckOk[any](http.MethodPut, "/api/v1/tasks/continue", nil)
+7 -1
View File
@@ -8,7 +8,13 @@ class FileExplorer {
if (await FileSystemEntity.isFile(filePath)) { if (await FileSystemEntity.isFile(filePath)) {
_openFile(filePath); _openFile(filePath);
} else if (await FileSystemEntity.isDirectory(filePath)) { } else if (await FileSystemEntity.isDirectory(filePath)) {
_openDirectory(filePath); await _openDirectory(filePath);
} else {
// If file does not exist, open its parent directory
final parentPath = path.dirname(filePath);
if (await FileSystemEntity.isDirectory(parentPath)) {
await _openDirectory(parentPath);
}
} }
} }