mirror of
https://github.com/GopeedLab/gopeed.git
synced 2026-11-17 22:03:41 +00:00
perf: slow down DHT announcing on errors (#1186)
This commit is contained in:
17 files changed
+573
-563
No files matched your search
+1
-1
@@ -1,4 +1,4 @@
|
|||||||
FROM golang:1.24.0 AS go
|
FROM golang:1.24.11 AS go
|
||||||
WORKDIR /app
|
WORKDIR /app
|
||||||
COPY ./go.mod ./go.sum ./
|
COPY ./go.mod ./go.sum ./
|
||||||
RUN go mod download
|
RUN go mod download
|
||||||
|
|||||||
@@ -201,7 +201,7 @@ This project is divided into two parts, the front end uses `flutter`, the back e
|
|||||||
|
|
||||||
### 🌍 Environment
|
### 🌍 Environment
|
||||||
|
|
||||||
1. Golang 1.23+
|
1. Golang 1.24+
|
||||||
2. Flutter 3.24+
|
2. Flutter 3.24+
|
||||||
|
|
||||||
### 📋 Clone
|
### 📋 Clone
|
||||||
|
|||||||
+1
-1
@@ -182,7 +182,7 @@ Gopeed (正式名 Go Speed) は `Golang` + `Flutter` によって開発された
|
|||||||
|
|
||||||
### 🌍 環境
|
### 🌍 環境
|
||||||
|
|
||||||
1. Go 言語 1.23+
|
1. Go 言語 1.24+
|
||||||
2. Flutter 3.24+
|
2. Flutter 3.24+
|
||||||
|
|
||||||
### 📋 クローン
|
### 📋 クローン
|
||||||
|
|||||||
+1
-1
@@ -188,7 +188,7 @@ Dự án này được chia thành hai phần, phần giao diện sử dụng `f
|
|||||||
|
|
||||||
### 🌍 Environment
|
### 🌍 Environment
|
||||||
|
|
||||||
1. Golang 1.23+
|
1. Golang 1.24+
|
||||||
2. Flutter 3.24+
|
2. Flutter 3.24+
|
||||||
|
|
||||||
### 📋 Clone
|
### 📋 Clone
|
||||||
|
|||||||
+1
-1
@@ -192,7 +192,7 @@ Gopeed 还提供了浏览器扩展用于接管浏览器下载,支持 Chrome、
|
|||||||
|
|
||||||
### 🌍 环境要求
|
### 🌍 环境要求
|
||||||
|
|
||||||
1. Golang 1.23+
|
1. Golang 1.24+
|
||||||
2. Flutter 3.24+
|
2. Flutter 3.24+
|
||||||
|
|
||||||
### 📋 克隆项目
|
### 📋 克隆项目
|
||||||
|
|||||||
+1
-1
@@ -189,7 +189,7 @@ go install github.com/GopeedLab/gopeed/cmd/gopeed@latest
|
|||||||
|
|
||||||
### 🌍 開發環境
|
### 🌍 開發環境
|
||||||
|
|
||||||
1. Golang 1.23+
|
1. Golang 1.24+
|
||||||
2. Flutter 3.24+
|
2. Flutter 3.24+
|
||||||
|
|
||||||
### 📋 克隆項目
|
### 📋 克隆項目
|
||||||
|
|||||||
@@ -1,13 +1,13 @@
|
|||||||
module github.com/GopeedLab/gopeed
|
module github.com/GopeedLab/gopeed
|
||||||
|
|
||||||
go 1.24.0
|
go 1.24.9
|
||||||
|
|
||||||
toolchain go1.24.11
|
toolchain go1.24.11
|
||||||
|
|
||||||
require (
|
require (
|
||||||
github.com/anacrolix/missinggo/v2 v2.8.0
|
github.com/anacrolix/torrent v1.60.1-0.20251217073903-486bcbe758e0
|
||||||
github.com/anacrolix/torrent v1.57.2-0.20241017235801-4d8437a05621
|
|
||||||
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5
|
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5
|
||||||
|
github.com/bodgit/sevenzip v1.6.1
|
||||||
github.com/dop251/goja v0.0.0-20240919115326-6c7d1df7ff05
|
github.com/dop251/goja v0.0.0-20240919115326-6c7d1df7ff05
|
||||||
github.com/dop251/goja_nodejs v0.0.0-20240728170619-29b559befffc
|
github.com/dop251/goja_nodejs v0.0.0-20240728170619-29b559befffc
|
||||||
github.com/go-git/go-git/v5 v5.8.1
|
github.com/go-git/go-git/v5 v5.8.1
|
||||||
@@ -21,19 +21,22 @@ require (
|
|||||||
github.com/rs/dnscache v0.0.0-20230804202142-fc85eb664529
|
github.com/rs/dnscache v0.0.0-20230804202142-fc85eb664529
|
||||||
github.com/rs/zerolog v1.31.0
|
github.com/rs/zerolog v1.31.0
|
||||||
github.com/xiaoqidun/setft v0.0.0-20220310121541-be86327699ad
|
github.com/xiaoqidun/setft v0.0.0-20220310121541-be86327699ad
|
||||||
go.etcd.io/bbolt v1.3.11
|
go.etcd.io/bbolt v1.4.3
|
||||||
golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f
|
golang.org/x/exp v0.0.0-20251219203646-944ab1f22d93
|
||||||
golang.org/x/sync v0.17.0
|
|
||||||
)
|
)
|
||||||
|
|
||||||
require (
|
require (
|
||||||
github.com/STARRY-S/zip v0.2.3 // indirect
|
github.com/STARRY-S/zip v0.2.3 // indirect
|
||||||
|
github.com/anacrolix/btree v0.1.1 // indirect
|
||||||
|
github.com/anacrolix/missinggo/v2 v2.10.0 // indirect
|
||||||
github.com/andybalholm/brotli v1.2.0 // indirect
|
github.com/andybalholm/brotli v1.2.0 // indirect
|
||||||
github.com/bodgit/plumbing v1.3.0 // indirect
|
github.com/bodgit/plumbing v1.3.0 // indirect
|
||||||
github.com/bodgit/sevenzip v1.6.1 // indirect
|
|
||||||
github.com/bodgit/windows v1.0.1 // indirect
|
github.com/bodgit/windows v1.0.1 // indirect
|
||||||
|
github.com/cespare/xxhash/v2 v2.3.0 // indirect
|
||||||
github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707 // indirect
|
github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707 // indirect
|
||||||
|
github.com/felixge/fgprof v0.9.5 // indirect
|
||||||
github.com/go-task/slim-sprig/v3 v3.0.0 // indirect
|
github.com/go-task/slim-sprig/v3 v3.0.0 // indirect
|
||||||
|
github.com/google/go-cmp v0.7.0 // indirect
|
||||||
github.com/hashicorp/errwrap v1.1.0 // indirect
|
github.com/hashicorp/errwrap v1.1.0 // indirect
|
||||||
github.com/hashicorp/go-multierror v1.1.1 // indirect
|
github.com/hashicorp/go-multierror v1.1.1 // indirect
|
||||||
github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
|
github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
|
||||||
@@ -45,15 +48,18 @@ require (
|
|||||||
github.com/nwaples/rardecode/v2 v2.2.0 // indirect
|
github.com/nwaples/rardecode/v2 v2.2.0 // indirect
|
||||||
github.com/onsi/ginkgo/v2 v2.23.4 // indirect
|
github.com/onsi/ginkgo/v2 v2.23.4 // indirect
|
||||||
github.com/pierrec/lz4/v4 v4.1.22 // indirect
|
github.com/pierrec/lz4/v4 v4.1.22 // indirect
|
||||||
|
github.com/pion/transport/v4 v4.0.1 // indirect
|
||||||
github.com/quic-go/qpack v0.5.1 // indirect
|
github.com/quic-go/qpack v0.5.1 // indirect
|
||||||
github.com/quic-go/quic-go v0.51.0 // indirect
|
github.com/quic-go/quic-go v0.51.0 // indirect
|
||||||
github.com/refraction-networking/utls v1.6.7 // indirect
|
github.com/refraction-networking/utls v1.6.7 // indirect
|
||||||
github.com/sorairolake/lzip-go v0.3.8 // indirect
|
github.com/sorairolake/lzip-go v0.3.8 // indirect
|
||||||
github.com/spf13/afero v1.15.0 // indirect
|
github.com/spf13/afero v1.15.0 // indirect
|
||||||
github.com/ulikunitz/xz v0.5.15 // indirect
|
github.com/ulikunitz/xz v0.5.15 // indirect
|
||||||
|
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
|
||||||
go.uber.org/automaxprocs v1.6.0 // indirect
|
go.uber.org/automaxprocs v1.6.0 // indirect
|
||||||
go.uber.org/mock v0.5.1 // indirect
|
go.uber.org/mock v0.5.1 // indirect
|
||||||
go4.org v0.0.0-20230225012048-214862532bf5 // indirect
|
go4.org v0.0.0-20230225012048-214862532bf5 // indirect
|
||||||
|
golang.org/x/sync v0.19.0 // indirect
|
||||||
)
|
)
|
||||||
|
|
||||||
require (
|
require (
|
||||||
@@ -62,25 +68,24 @@ require (
|
|||||||
github.com/ProtonMail/go-crypto v0.0.0-20230717121422-5aa5874ade95 // indirect
|
github.com/ProtonMail/go-crypto v0.0.0-20230717121422-5aa5874ade95 // indirect
|
||||||
github.com/RoaringBitmap/roaring v1.9.4 // indirect
|
github.com/RoaringBitmap/roaring v1.9.4 // indirect
|
||||||
github.com/acomagu/bufpipe v1.0.4 // indirect
|
github.com/acomagu/bufpipe v1.0.4 // indirect
|
||||||
github.com/ajwerner/btree v0.0.0-20211221152037-f427b3e689c0 // indirect
|
|
||||||
github.com/alecthomas/atomic v0.1.0-alpha2 // indirect
|
github.com/alecthomas/atomic v0.1.0-alpha2 // indirect
|
||||||
github.com/anacrolix/chansync v0.6.0 // indirect
|
github.com/anacrolix/chansync v0.7.0 // indirect
|
||||||
github.com/anacrolix/dht/v2 v2.22.0 // indirect
|
github.com/anacrolix/dht/v2 v2.23.0 // indirect
|
||||||
github.com/anacrolix/envpprof v1.4.0 // indirect
|
github.com/anacrolix/envpprof v1.5.0 // indirect
|
||||||
github.com/anacrolix/generics v0.0.3-0.20240902042256-7fb2702ef0ca // indirect
|
github.com/anacrolix/generics v0.2.0 // indirect
|
||||||
github.com/anacrolix/go-libutp v1.3.1 // indirect
|
github.com/anacrolix/go-libutp v1.3.2 // indirect
|
||||||
github.com/anacrolix/log v0.16.0 // indirect
|
github.com/anacrolix/log v0.17.1-0.20251118025802-918f1157b7bb // indirect
|
||||||
github.com/anacrolix/missinggo v1.3.0 // indirect
|
github.com/anacrolix/missinggo v1.3.0 // indirect
|
||||||
github.com/anacrolix/missinggo/perf v1.0.0 // indirect
|
github.com/anacrolix/missinggo/perf v1.0.0 // indirect
|
||||||
github.com/anacrolix/mmsg v1.1.1 // indirect
|
github.com/anacrolix/mmsg v1.1.1 // indirect
|
||||||
github.com/anacrolix/multiless v0.4.0 // indirect
|
github.com/anacrolix/multiless v0.4.0 // indirect
|
||||||
github.com/anacrolix/stm v0.5.0 // indirect
|
github.com/anacrolix/stm v0.5.0 // indirect
|
||||||
github.com/anacrolix/sync v0.5.3 // indirect
|
github.com/anacrolix/sync v0.6.0 // indirect
|
||||||
github.com/anacrolix/upnp v0.1.4 // indirect
|
github.com/anacrolix/upnp v0.1.4 // indirect
|
||||||
github.com/anacrolix/utp v0.2.0 // indirect
|
github.com/anacrolix/utp v0.2.0 // indirect
|
||||||
github.com/bahlo/generic-list-go v0.2.0 // indirect
|
github.com/bahlo/generic-list-go v0.2.0 // indirect
|
||||||
github.com/benbjohnson/immutable v0.4.3 // indirect
|
github.com/benbjohnson/immutable v0.4.3 // indirect
|
||||||
github.com/bits-and-blooms/bitset v1.15.0 // indirect
|
github.com/bits-and-blooms/bitset v1.24.4 // indirect
|
||||||
github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8 // indirect
|
github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8 // indirect
|
||||||
github.com/cespare/xxhash v1.1.0 // indirect
|
github.com/cespare/xxhash v1.1.0 // indirect
|
||||||
github.com/cloudflare/circl v1.6.1 // indirect
|
github.com/cloudflare/circl v1.6.1 // indirect
|
||||||
@@ -92,44 +97,43 @@ require (
|
|||||||
github.com/felixge/httpsnoop v1.0.1 // indirect
|
github.com/felixge/httpsnoop v1.0.1 // indirect
|
||||||
github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
|
github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
|
||||||
github.com/go-git/go-billy/v5 v5.4.1 // indirect
|
github.com/go-git/go-billy/v5 v5.4.1 // indirect
|
||||||
github.com/go-llsqlite/adapter v0.1.0 // indirect
|
github.com/go-llsqlite/adapter v0.2.0 // indirect
|
||||||
github.com/go-llsqlite/crawshaw v0.5.5 // indirect
|
github.com/go-llsqlite/crawshaw v0.6.0 // indirect
|
||||||
github.com/go-logr/logr v1.4.2 // indirect
|
github.com/go-logr/logr v1.4.3 // indirect
|
||||||
github.com/go-logr/stdr v1.2.2 // indirect
|
github.com/go-logr/stdr v1.2.2 // indirect
|
||||||
github.com/go-sourcemap/sourcemap v2.1.4+incompatible // indirect
|
github.com/go-sourcemap/sourcemap v2.1.4+incompatible // indirect
|
||||||
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
|
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
|
||||||
github.com/google/btree v1.1.3 // indirect
|
github.com/google/btree v1.1.3 // indirect
|
||||||
github.com/google/pprof v0.0.0-20250423184734-337e5dd93bb4 // indirect
|
github.com/google/pprof v0.0.0-20260106004452-d7df1bf2cac7 // indirect
|
||||||
github.com/google/uuid v1.6.0 // indirect
|
github.com/google/uuid v1.6.0 // indirect
|
||||||
github.com/gorilla/websocket v1.5.3 // indirect
|
github.com/gorilla/websocket v1.5.3 // indirect
|
||||||
github.com/huandu/xstrings v1.5.0 // indirect
|
github.com/huandu/xstrings v1.5.0 // indirect
|
||||||
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
|
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
|
||||||
github.com/kevinburke/ssh_config v1.2.0 // indirect
|
github.com/kevinburke/ssh_config v1.2.0 // indirect
|
||||||
github.com/klauspost/cpuid/v2 v2.2.9 // indirect
|
github.com/klauspost/cpuid/v2 v2.3.0 // indirect
|
||||||
github.com/mattn/go-colorable v0.1.13 // indirect
|
github.com/mattn/go-colorable v0.1.13 // indirect
|
||||||
github.com/mattn/go-isatty v0.0.20 // indirect
|
github.com/mattn/go-isatty v0.0.20 // indirect
|
||||||
github.com/minio/sha256-simd v1.0.1 // indirect
|
github.com/minio/sha256-simd v1.0.1 // indirect
|
||||||
github.com/mr-tron/base58 v1.2.0 // indirect
|
github.com/mr-tron/base58 v1.2.0 // indirect
|
||||||
github.com/mschoch/smat v0.2.0 // indirect
|
github.com/mschoch/smat v0.2.0 // indirect
|
||||||
github.com/multiformats/go-multihash v0.2.3 // indirect
|
github.com/multiformats/go-multihash v0.2.3 // indirect
|
||||||
github.com/multiformats/go-varint v0.0.7 // indirect
|
github.com/multiformats/go-varint v0.1.0 // indirect
|
||||||
github.com/ncruces/go-strftime v0.1.9 // indirect
|
github.com/ncruces/go-strftime v1.0.0 // indirect
|
||||||
github.com/pion/datachannel v1.5.9 // indirect
|
github.com/pion/datachannel v1.6.0 // indirect
|
||||||
github.com/pion/dtls/v3 v3.0.4 // indirect
|
github.com/pion/dtls/v3 v3.0.10 // indirect
|
||||||
github.com/pion/ice/v4 v4.0.2 // indirect
|
github.com/pion/ice/v4 v4.2.0 // indirect
|
||||||
github.com/pion/interceptor v0.1.37 // indirect
|
github.com/pion/interceptor v0.1.42 // indirect
|
||||||
github.com/pion/logging v0.2.2 // indirect
|
github.com/pion/logging v0.2.4 // indirect
|
||||||
github.com/pion/mdns/v2 v2.0.7 // indirect
|
github.com/pion/mdns/v2 v2.1.0 // indirect
|
||||||
github.com/pion/randutil v0.1.0 // indirect
|
github.com/pion/randutil v0.1.0 // indirect
|
||||||
github.com/pion/rtcp v1.2.14 // indirect
|
github.com/pion/rtcp v1.2.16 // indirect
|
||||||
github.com/pion/rtp v1.8.9 // indirect
|
github.com/pion/rtp v1.10.0 // indirect
|
||||||
github.com/pion/sctp v1.8.33 // indirect
|
github.com/pion/sctp v1.9.1 // indirect
|
||||||
github.com/pion/sdp/v3 v3.0.9 // indirect
|
github.com/pion/sdp/v3 v3.0.17 // indirect
|
||||||
github.com/pion/srtp/v3 v3.0.4 // indirect
|
github.com/pion/srtp/v3 v3.0.10 // indirect
|
||||||
github.com/pion/stun/v3 v3.0.0 // indirect
|
github.com/pion/stun/v3 v3.1.1 // indirect
|
||||||
github.com/pion/transport/v3 v3.0.7 // indirect
|
github.com/pion/turn/v4 v4.1.4 // indirect
|
||||||
github.com/pion/turn/v4 v4.0.0 // indirect
|
github.com/pion/webrtc/v4 v4.2.2-0.20260109001657-a5962f314db7 // indirect
|
||||||
github.com/pion/webrtc/v4 v4.0.2 // indirect
|
|
||||||
github.com/pjbgf/sha1cd v0.3.0 // indirect
|
github.com/pjbgf/sha1cd v0.3.0 // indirect
|
||||||
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c
|
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c
|
||||||
github.com/protolambda/ctxlock v0.1.0 // indirect
|
github.com/protolambda/ctxlock v0.1.0 // indirect
|
||||||
@@ -137,24 +141,24 @@ require (
|
|||||||
github.com/sergi/go-diff v1.1.0 // indirect
|
github.com/sergi/go-diff v1.1.0 // indirect
|
||||||
github.com/skeema/knownhosts v1.2.0 // indirect
|
github.com/skeema/knownhosts v1.2.0 // indirect
|
||||||
github.com/spaolacci/murmur3 v1.1.0 // indirect
|
github.com/spaolacci/murmur3 v1.1.0 // indirect
|
||||||
github.com/tidwall/btree v1.7.0 // indirect
|
github.com/tidwall/btree v1.8.1 // indirect
|
||||||
github.com/wlynxg/anet v0.0.5 // indirect
|
github.com/wlynxg/anet v0.0.5 // indirect
|
||||||
github.com/xanzy/ssh-agent v0.3.3 // indirect
|
github.com/xanzy/ssh-agent v0.3.3 // indirect
|
||||||
go.opentelemetry.io/otel v1.32.0 // indirect
|
go.opentelemetry.io/otel v1.39.0 // indirect
|
||||||
go.opentelemetry.io/otel/metric v1.32.0 // indirect
|
go.opentelemetry.io/otel/metric v1.39.0 // indirect
|
||||||
go.opentelemetry.io/otel/trace v1.32.0 // indirect
|
go.opentelemetry.io/otel/trace v1.39.0 // indirect
|
||||||
golang.org/x/crypto v0.41.0 // indirect
|
golang.org/x/crypto v0.46.0 // indirect
|
||||||
golang.org/x/mod v0.27.0 // indirect
|
golang.org/x/mod v0.31.0 // indirect
|
||||||
golang.org/x/net v0.43.0 // indirect
|
golang.org/x/net v0.48.0 // indirect
|
||||||
golang.org/x/sys v0.35.0 // indirect
|
golang.org/x/sys v0.40.0 // indirect
|
||||||
golang.org/x/text v0.29.0
|
golang.org/x/text v0.32.0
|
||||||
golang.org/x/time v0.8.0 // indirect
|
golang.org/x/time v0.14.0 // indirect
|
||||||
golang.org/x/tools v0.36.0 // indirect
|
golang.org/x/tools v0.40.0 // indirect
|
||||||
gopkg.in/warnings.v0 v0.1.2 // indirect
|
gopkg.in/warnings.v0 v0.1.2 // indirect
|
||||||
lukechampine.com/blake3 v1.3.0 // indirect
|
lukechampine.com/blake3 v1.4.1 // indirect
|
||||||
modernc.org/libc v1.61.0 // indirect
|
modernc.org/libc v1.67.4 // indirect
|
||||||
modernc.org/mathutil v1.6.0 // indirect
|
modernc.org/mathutil v1.7.1 // indirect
|
||||||
modernc.org/memory v1.8.0 // indirect
|
modernc.org/memory v1.11.0 // indirect
|
||||||
modernc.org/sqlite v1.33.1 // indirect
|
modernc.org/sqlite v1.43.0 // indirect
|
||||||
zombiezen.com/go/sqlite v1.4.0 // indirect
|
zombiezen.com/go/sqlite v1.4.2 // indirect
|
||||||
)
|
)
|
||||||
@@ -110,6 +110,7 @@ func (f *Fetcher) Start() (err error) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
files := f.torrent.Files()
|
files := f.torrent.Files()
|
||||||
// If the user does not specify the file to download, all files will be downloaded by default
|
// If the user does not specify the file to download, all files will be downloaded by default
|
||||||
if f.data.Progress == nil {
|
if f.data.Progress == nil {
|
||||||
@@ -129,12 +130,12 @@ func (f *Fetcher) Start() (err error) {
|
|||||||
file.Download()
|
file.Download()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
f.torrent.AllowDataDownload()
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
func (f *Fetcher) Pause() (err error) {
|
func (f *Fetcher) Pause() (err error) {
|
||||||
f.torrentReady.Store(false)
|
f.torrent.DisallowDataDownload()
|
||||||
f.safeDrop()
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -23,9 +23,14 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
connectTimeout = 15 * time.Second
|
connectTimeout = 15 * time.Second
|
||||||
readTimeout = 15 * time.Second
|
readTimeout = 15 * time.Second
|
||||||
helpMinSize = 1 * 1024 * 1024 // Minimum chunk size for helper connections
|
minFastFailTimeout = int64(3 * time.Second) // Minimum timeout for fast-fail retry
|
||||||
|
|
||||||
|
// Work stealing parameters
|
||||||
|
// When a connection finishes its chunk, it can "steal" work from slow connections.
|
||||||
|
stealThresholdSeconds = 3 // Only steal if victim needs > 3 seconds to finish
|
||||||
|
stealMinChunkSize = 512 * 1024 // Min steal size: 512KB (avoid tiny chunks)
|
||||||
)
|
)
|
||||||
|
|
||||||
// ============================================================================
|
// ============================================================================
|
||||||
@@ -96,6 +101,11 @@ type connection struct {
|
|||||||
retryTimes int
|
retryTimes int
|
||||||
lastErr error
|
lastErr error
|
||||||
|
|
||||||
|
// Speed tracking for work stealing decisions
|
||||||
|
speed int64 // bytes per second
|
||||||
|
lastSpeedCheck int64 // timestamp in nanoseconds
|
||||||
|
lastSpeedDownload int64 // bytes downloaded at last check
|
||||||
|
|
||||||
ctx context.Context
|
ctx context.Context
|
||||||
cancel context.CancelFunc
|
cancel context.CancelFunc
|
||||||
}
|
}
|
||||||
@@ -123,14 +133,6 @@ func newSlowStartController(maxConnections int) *slowStartController {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// startBatch marks the start of a new batch with count connections
|
|
||||||
func (s *slowStartController) startBatch(count int) {
|
|
||||||
s.mu.Lock()
|
|
||||||
defer s.mu.Unlock()
|
|
||||||
s.batchPending = count
|
|
||||||
s.batchReady = 0
|
|
||||||
}
|
|
||||||
|
|
||||||
// onConnectSuccess is called when a connection successfully gets HTTP response
|
// onConnectSuccess is called when a connection successfully gets HTTP response
|
||||||
// Returns true if this completes the current batch
|
// Returns true if this completes the current batch
|
||||||
func (s *slowStartController) onConnectSuccess() bool {
|
func (s *slowStartController) onConnectSuccess() bool {
|
||||||
@@ -202,13 +204,6 @@ func (s *slowStartController) commitBatch(count int) {
|
|||||||
s.batchReady = 0
|
s.batchReady = 0
|
||||||
}
|
}
|
||||||
|
|
||||||
// isMaxReached returns true if max connections have been launched
|
|
||||||
func (s *slowStartController) isMaxReached() bool {
|
|
||||||
s.mu.Lock()
|
|
||||||
defer s.mu.Unlock()
|
|
||||||
return s.totalLaunched >= s.maxConnections
|
|
||||||
}
|
|
||||||
|
|
||||||
// ============================================================================
|
// ============================================================================
|
||||||
// Fetcher
|
// Fetcher
|
||||||
// ============================================================================
|
// ============================================================================
|
||||||
@@ -231,6 +226,9 @@ type Fetcher struct {
|
|||||||
// Slow start controller
|
// Slow start controller
|
||||||
slowStart *slowStartController
|
slowStart *slowStartController
|
||||||
|
|
||||||
|
// Max connection time for adaptive timeout (stored as int64 nanoseconds for atomic ops)
|
||||||
|
maxConnTime atomic.Int64
|
||||||
|
|
||||||
// First primary connection success signal
|
// First primary connection success signal
|
||||||
primaryReadyOnce sync.Once
|
primaryReadyOnce sync.Once
|
||||||
primaryReadyCh chan struct{}
|
primaryReadyCh chan struct{}
|
||||||
@@ -264,6 +262,9 @@ type Fetcher struct {
|
|||||||
cancel context.CancelFunc
|
cancel context.CancelFunc
|
||||||
wg sync.WaitGroup
|
wg sync.WaitGroup
|
||||||
|
|
||||||
|
// downloadLoop lifecycle tracking
|
||||||
|
downloadLoopDone chan struct{} // Closed when downloadLoop exits
|
||||||
|
|
||||||
// Resolve connection control
|
// Resolve connection control
|
||||||
resolveCtx context.Context
|
resolveCtx context.Context
|
||||||
resolveCancel context.CancelFunc
|
resolveCancel context.CancelFunc
|
||||||
@@ -297,6 +298,14 @@ func (f *Fetcher) setState(s fetcherState) {
|
|||||||
f.state.Store(int32(s))
|
f.state.Store(int32(s))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// updateMaxConnTime updates maxConnTime if the new duration is larger
|
||||||
|
func (f *Fetcher) updateMaxConnTime(d time.Duration) {
|
||||||
|
newVal := int64(d)
|
||||||
|
if newVal > f.maxConnTime.Load() {
|
||||||
|
f.maxConnTime.Store(newVal)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func (f *Fetcher) Resolve(req *base.Request, opts *base.Options) error {
|
func (f *Fetcher) Resolve(req *base.Request, opts *base.Options) error {
|
||||||
if err := base.ParseReqExtra[fhttp.ReqExtra](req); err != nil {
|
if err := base.ParseReqExtra[fhttp.ReqExtra](req); err != nil {
|
||||||
return err
|
return err
|
||||||
@@ -326,7 +335,7 @@ func (f *Fetcher) Resolve(req *base.Request, opts *base.Options) error {
|
|||||||
|
|
||||||
// Build HTTP request WITHOUT Range header (normal request)
|
// Build HTTP request WITHOUT Range header (normal request)
|
||||||
// This allows the response to be reused for downloading (important for one-time URLs)
|
// This allows the response to be reused for downloading (important for one-time URLs)
|
||||||
httpReq, err := f.buildRequest(nil, req)
|
httpReq, err := f.buildRequest(context.TODO(), req)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
f.setState(stateError)
|
f.setState(stateError)
|
||||||
return err
|
return err
|
||||||
@@ -335,11 +344,15 @@ func (f *Fetcher) Resolve(req *base.Request, opts *base.Options) error {
|
|||||||
client := f.buildClient()
|
client := f.buildClient()
|
||||||
|
|
||||||
// Send normal HTTP request (no Range header)
|
// Send normal HTTP request (no Range header)
|
||||||
|
// Track connection time for adaptive timeout in download phase
|
||||||
|
connStartTime := time.Now()
|
||||||
resp, err := client.Do(httpReq)
|
resp, err := client.Do(httpReq)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
f.setState(stateError)
|
f.setState(stateError)
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
// Record connection time as baseline for fast-fail timeout
|
||||||
|
f.updateMaxConnTime(time.Since(connStartTime))
|
||||||
|
|
||||||
// Parse response to get resource info
|
// Parse response to get resource info
|
||||||
res := &base.Resource{
|
res := &base.Resource{
|
||||||
@@ -639,6 +652,9 @@ func (f *Fetcher) doStart() error {
|
|||||||
// Create main context
|
// Create main context
|
||||||
f.ctx, f.cancel = context.WithCancel(context.Background())
|
f.ctx, f.cancel = context.WithCancel(context.Background())
|
||||||
|
|
||||||
|
// Create downloadLoop lifecycle channel
|
||||||
|
f.downloadLoopDone = make(chan struct{})
|
||||||
|
|
||||||
// Start download
|
// Start download
|
||||||
f.setState(stateSlowStart)
|
f.setState(stateSlowStart)
|
||||||
go f.downloadLoop()
|
go f.downloadLoop()
|
||||||
@@ -648,16 +664,15 @@ func (f *Fetcher) doStart() error {
|
|||||||
|
|
||||||
func (f *Fetcher) downloadLoop() {
|
func (f *Fetcher) downloadLoop() {
|
||||||
defer func() {
|
defer func() {
|
||||||
f.fileMu.Lock()
|
// Update file last modified time before closing
|
||||||
if f.file != nil {
|
|
||||||
f.file.Close()
|
|
||||||
}
|
|
||||||
f.fileMu.Unlock()
|
|
||||||
|
|
||||||
// Update file last modified time
|
|
||||||
if f.config.UseServerCtime && f.meta.Res.Files[0].Ctime != nil {
|
if f.config.UseServerCtime && f.meta.Res.Files[0].Ctime != nil {
|
||||||
setft.SetFileTime(f.meta.SingleFilepath(), time.Now(), *f.meta.Res.Files[0].Ctime, *f.meta.Res.Files[0].Ctime)
|
setft.SetFileTime(f.meta.SingleFilepath(), time.Now(), *f.meta.Res.Files[0].Ctime, *f.meta.Res.Files[0].Ctime)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Signal that downloadLoop has exited
|
||||||
|
if f.downloadLoopDone != nil {
|
||||||
|
close(f.downloadLoopDone)
|
||||||
|
}
|
||||||
}()
|
}()
|
||||||
|
|
||||||
// Check if this is a resume or fresh start
|
// Check if this is a resume or fresh start
|
||||||
@@ -682,9 +697,18 @@ func (f *Fetcher) downloadLoop() {
|
|||||||
case <-f.slowStart.expansionCh:
|
case <-f.slowStart.expansionCh:
|
||||||
// Batch completed, try to expand
|
// Batch completed, try to expand
|
||||||
if f.checkCompletion() {
|
if f.checkCompletion() {
|
||||||
|
// All work is done, wait for connections to finish
|
||||||
|
f.waitForCompletion()
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
f.expandConnections()
|
f.expandConnections()
|
||||||
|
|
||||||
|
// Check if we've reached steady state (max connections)
|
||||||
|
if f.getState() == stateSteady {
|
||||||
|
// Wait for all connections to complete
|
||||||
|
f.waitForCompletion()
|
||||||
|
return
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -706,16 +730,9 @@ func (f *Fetcher) startResolveDownload() {
|
|||||||
// Use the resolve response directly
|
// Use the resolve response directly
|
||||||
go f.runConnectionWithResolveResp(conn)
|
go f.runConnectionWithResolveResp(conn)
|
||||||
|
|
||||||
// For non-range downloads, wait for completion in the main downloadLoop
|
// For non-range downloads, wait for completion directly in this goroutine
|
||||||
// by triggering immediate completion check
|
// Don't create another goroutine to avoid WaitGroup reuse issues
|
||||||
go func() {
|
f.waitForCompletion()
|
||||||
f.wg.Wait()
|
|
||||||
// Signal expansion channel to trigger completion check in downloadLoop
|
|
||||||
select {
|
|
||||||
case f.slowStart.expansionCh <- struct{}{}:
|
|
||||||
default:
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -729,7 +746,8 @@ func (f *Fetcher) expandConnections() {
|
|||||||
if batchSize <= 0 {
|
if batchSize <= 0 {
|
||||||
// Max reached, transition to steady state
|
// Max reached, transition to steady state
|
||||||
f.setState(stateSteady)
|
f.setState(stateSteady)
|
||||||
go f.waitForCompletion()
|
// Don't start a new goroutine - let the downloadLoop handle completion
|
||||||
|
// This avoids multiple goroutines calling wg.Wait() simultaneously
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -745,6 +763,15 @@ func (f *Fetcher) expandConnections() {
|
|||||||
// If prefetched all data, mark as done
|
// If prefetched all data, mark as done
|
||||||
if prefetched >= totalSize {
|
if prefetched >= totalSize {
|
||||||
f.connMu.Unlock()
|
f.connMu.Unlock()
|
||||||
|
|
||||||
|
// Close the file before signaling completion
|
||||||
|
f.fileMu.Lock()
|
||||||
|
if f.file != nil {
|
||||||
|
f.file.Close()
|
||||||
|
f.file = nil
|
||||||
|
}
|
||||||
|
f.fileMu.Unlock()
|
||||||
|
|
||||||
f.setState(stateDone)
|
f.setState(stateDone)
|
||||||
f.doneCh <- nil
|
f.doneCh <- nil
|
||||||
return
|
return
|
||||||
@@ -773,6 +800,9 @@ func (f *Fetcher) expandConnections() {
|
|||||||
|
|
||||||
// For subsequent batches, use "help other connection" strategy
|
// For subsequent batches, use "help other connection" strategy
|
||||||
// Find connections with enough remaining work to split
|
// Find connections with enough remaining work to split
|
||||||
|
// During slow start, use fixed minimum size since speed is not yet stable
|
||||||
|
minSplitSize := int64(stealMinChunkSize)
|
||||||
|
|
||||||
newConns := make([]*connection, 0, batchSize)
|
newConns := make([]*connection, 0, batchSize)
|
||||||
for i := 0; i < batchSize; i++ {
|
for i := 0; i < batchSize; i++ {
|
||||||
// Find the connection with most remaining work
|
// Find the connection with most remaining work
|
||||||
@@ -784,7 +814,8 @@ func (f *Fetcher) expandConnections() {
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
remain := conn.Chunk.remain()
|
remain := conn.Chunk.remain()
|
||||||
if remain > maxRemain && remain > helpMinSize*2 {
|
// Only split if remaining work is at least 2x the minimum split size
|
||||||
|
if remain > maxRemain && remain > minSplitSize*2 {
|
||||||
maxRemainConn = conn
|
maxRemainConn = conn
|
||||||
maxRemain = remain
|
maxRemain = remain
|
||||||
}
|
}
|
||||||
@@ -839,13 +870,19 @@ func (f *Fetcher) runConnection(conn *connection) {
|
|||||||
conn.State = connConnecting
|
conn.State = connConnecting
|
||||||
f.connMu.Unlock()
|
f.connMu.Unlock()
|
||||||
|
|
||||||
client := f.buildClient()
|
// Use fast-fail client for quick retry during download phase
|
||||||
|
client := f.buildFastFailClient()
|
||||||
buf := make([]byte, 8192)
|
buf := make([]byte, 8192)
|
||||||
|
|
||||||
retries := 0
|
retries := 0
|
||||||
conn.retryTimes = 0
|
conn.retryTimes = 0
|
||||||
|
|
||||||
for {
|
for {
|
||||||
|
// Rebuild client with updated fast-fail timeout on retries
|
||||||
|
if retries > 0 {
|
||||||
|
client = f.buildFastFailClient()
|
||||||
|
}
|
||||||
|
|
||||||
err := f.downloadChunkOnce(conn, client, buf)
|
err := f.downloadChunkOnce(conn, client, buf)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
if !f.meta.Res.Range || !f.helpOtherConnection(conn) {
|
if !f.meta.Res.Range || !f.helpOtherConnection(conn) {
|
||||||
@@ -937,6 +974,9 @@ func (f *Fetcher) downloadChunkOnce(conn *connection, client *http.Client, buf [
|
|||||||
fmt.Sprintf(base.HttpHeaderRangeFormat, rangeStart, rangeEnd))
|
fmt.Sprintf(base.HttpHeaderRangeFormat, rangeStart, rangeEnd))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Record connection start time for adaptive timeout tracking
|
||||||
|
connStartTime := time.Now()
|
||||||
|
|
||||||
resp, err := client.Do(httpReq)
|
resp, err := client.Do(httpReq)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
@@ -947,6 +987,9 @@ func (f *Fetcher) downloadChunkOnce(conn *connection, client *http.Client, buf [
|
|||||||
return NewRequestError(resp.StatusCode)
|
return NewRequestError(resp.StatusCode)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Record successful connection time for adaptive timeout
|
||||||
|
f.updateMaxConnTime(time.Since(connStartTime))
|
||||||
|
|
||||||
f.connMu.Lock()
|
f.connMu.Lock()
|
||||||
conn.State = connDownloading
|
conn.State = connDownloading
|
||||||
conn.failed = false
|
conn.failed = false
|
||||||
@@ -1005,6 +1048,19 @@ func (f *Fetcher) downloadChunkOnce(conn *connection, client *http.Client, buf [
|
|||||||
f.connMu.Lock()
|
f.connMu.Lock()
|
||||||
conn.Chunk.Downloaded += int64(n)
|
conn.Chunk.Downloaded += int64(n)
|
||||||
conn.Downloaded += int64(n)
|
conn.Downloaded += int64(n)
|
||||||
|
// Update connection speed periodically
|
||||||
|
now := time.Now().UnixNano()
|
||||||
|
if conn.lastSpeedCheck == 0 {
|
||||||
|
conn.lastSpeedCheck = now
|
||||||
|
conn.lastSpeedDownload = conn.Downloaded
|
||||||
|
} else if now-conn.lastSpeedCheck >= int64(500*time.Millisecond) {
|
||||||
|
elapsed := float64(now-conn.lastSpeedCheck) / float64(time.Second)
|
||||||
|
if elapsed > 0 {
|
||||||
|
conn.speed = int64(float64(conn.Downloaded-conn.lastSpeedDownload) / elapsed)
|
||||||
|
}
|
||||||
|
conn.lastSpeedCheck = now
|
||||||
|
conn.lastSpeedDownload = conn.Downloaded
|
||||||
|
}
|
||||||
f.connMu.Unlock()
|
f.connMu.Unlock()
|
||||||
|
|
||||||
if finished {
|
if finished {
|
||||||
@@ -1107,7 +1163,8 @@ func (f *Fetcher) runConnectionWithResolveResp(conn *connection) {
|
|||||||
|
|
||||||
// runConnectionFallback is used when resolve response is not available
|
// runConnectionFallback is used when resolve response is not available
|
||||||
func (f *Fetcher) runConnectionFallback(conn *connection) {
|
func (f *Fetcher) runConnectionFallback(conn *connection) {
|
||||||
client := f.buildClient()
|
// Use fast-fail client for quick retry during download phase
|
||||||
|
client := f.buildFastFailClient()
|
||||||
buf := make([]byte, 8192)
|
buf := make([]byte, 8192)
|
||||||
|
|
||||||
retries := 0
|
retries := 0
|
||||||
@@ -1118,6 +1175,11 @@ func (f *Fetcher) runConnectionFallback(conn *connection) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Rebuild client with updated fast-fail timeout on retries
|
||||||
|
if retries > 0 {
|
||||||
|
client = f.buildFastFailClient()
|
||||||
|
}
|
||||||
|
|
||||||
f.connMu.Lock()
|
f.connMu.Lock()
|
||||||
conn.State = connConnecting
|
conn.State = connConnecting
|
||||||
f.connMu.Unlock()
|
f.connMu.Unlock()
|
||||||
@@ -1128,6 +1190,9 @@ func (f *Fetcher) runConnectionFallback(conn *connection) {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Record connection start time for adaptive timeout tracking
|
||||||
|
connStartTime := time.Now()
|
||||||
|
|
||||||
resp, err := client.Do(httpReq)
|
resp, err := client.Do(httpReq)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
@@ -1138,6 +1203,9 @@ func (f *Fetcher) runConnectionFallback(conn *connection) {
|
|||||||
return NewRequestError(resp.StatusCode)
|
return NewRequestError(resp.StatusCode)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Record successful connection time for adaptive timeout
|
||||||
|
f.updateMaxConnTime(time.Since(connStartTime))
|
||||||
|
|
||||||
f.connMu.Lock()
|
f.connMu.Lock()
|
||||||
conn.State = connDownloading
|
conn.State = connDownloading
|
||||||
conn.failed = false
|
conn.failed = false
|
||||||
@@ -1248,34 +1316,50 @@ func (f *Fetcher) runConnectionFallback(conn *connection) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// helpOtherConnection implements work stealing: when a connection finishes its chunk,
|
||||||
|
// it looks for connections that need more than stealThresholdSeconds to finish and steals half of its work.
|
||||||
func (f *Fetcher) helpOtherConnection(helper *connection) bool {
|
func (f *Fetcher) helpOtherConnection(helper *connection) bool {
|
||||||
f.connMu.Lock()
|
f.connMu.Lock()
|
||||||
defer f.connMu.Unlock()
|
defer f.connMu.Unlock()
|
||||||
|
|
||||||
// Find the slowest connection
|
// Find the connection with longest remaining time
|
||||||
var maxRemainConnection *connection
|
var slowestConn *connection
|
||||||
var maxRemain int64
|
var maxRemainSeconds int64
|
||||||
for _, r := range f.connections {
|
for _, r := range f.connections {
|
||||||
if r == helper || r.Completed || r.State == connFailed {
|
if r == helper || r.Completed || r.State == connFailed {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
remain := r.Chunk.remain()
|
remain := r.Chunk.remain()
|
||||||
if remain > maxRemain && remain > helpMinSize {
|
if remain < stealMinChunkSize {
|
||||||
maxRemainConnection = r
|
continue
|
||||||
maxRemain = remain
|
}
|
||||||
|
|
||||||
|
// Calculate remaining time in seconds for this connection
|
||||||
|
var remainSeconds int64
|
||||||
|
if r.speed > 0 {
|
||||||
|
remainSeconds = remain / r.speed
|
||||||
|
} else {
|
||||||
|
// Speed unknown, assume it needs help if chunk is large enough
|
||||||
|
remainSeconds = stealThresholdSeconds + 1
|
||||||
|
}
|
||||||
|
|
||||||
|
// Only consider if it needs more than threshold seconds to finish
|
||||||
|
if remainSeconds > stealThresholdSeconds && remainSeconds > maxRemainSeconds {
|
||||||
|
slowestConn = r
|
||||||
|
maxRemainSeconds = remainSeconds
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if maxRemainConnection == nil {
|
if slowestConn == nil {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
// Re-calculate the chunk range
|
// Re-calculate the chunk range: steal half of the remaining work
|
||||||
helper.Chunk.Begin = maxRemainConnection.Chunk.End - maxRemainConnection.Chunk.remain()/2
|
helper.Chunk.Begin = slowestConn.Chunk.End - slowestConn.Chunk.remain()/2
|
||||||
helper.Chunk.End = maxRemainConnection.Chunk.End
|
helper.Chunk.End = slowestConn.Chunk.End
|
||||||
helper.Chunk.Downloaded = 0
|
helper.Chunk.Downloaded = 0
|
||||||
maxRemainConnection.Chunk.End = helper.Chunk.Begin - 1
|
slowestConn.Chunk.End = helper.Chunk.Begin - 1
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1380,6 +1464,15 @@ func (f *Fetcher) onDownloadComplete() {
|
|||||||
}
|
}
|
||||||
f.connMu.Unlock()
|
f.connMu.Unlock()
|
||||||
|
|
||||||
|
// Close the file before signaling completion
|
||||||
|
// This ensures the file handle is released before Wait() returns
|
||||||
|
f.fileMu.Lock()
|
||||||
|
if f.file != nil {
|
||||||
|
f.file.Close()
|
||||||
|
f.file = nil
|
||||||
|
}
|
||||||
|
f.fileMu.Unlock()
|
||||||
|
|
||||||
if finalErr != nil {
|
if finalErr != nil {
|
||||||
f.setState(stateError)
|
f.setState(stateError)
|
||||||
} else {
|
} else {
|
||||||
@@ -1406,8 +1499,7 @@ func (f *Fetcher) checkCompletion() bool {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if f.meta.Res.Size > 0 && totalDownloaded >= f.meta.Res.Size {
|
if f.meta.Res.Size > 0 && totalDownloaded >= f.meta.Res.Size {
|
||||||
f.setState(stateSteady)
|
// Don't start a new goroutine - let the caller handle completion
|
||||||
go f.waitForCompletion()
|
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1424,8 +1516,7 @@ func (f *Fetcher) checkCompletion() bool {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if allCompleted {
|
if allCompleted {
|
||||||
f.setState(stateSteady)
|
// Don't start a new goroutine - let the caller handle completion
|
||||||
go f.waitForCompletion()
|
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1450,7 +1541,12 @@ func (f *Fetcher) Pause() error {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Wait for all goroutines to stop
|
// Wait for downloadLoop to exit first (it will call wg.Wait internally)
|
||||||
|
if f.downloadLoopDone != nil {
|
||||||
|
<-f.downloadLoopDone
|
||||||
|
}
|
||||||
|
|
||||||
|
// Wait for all connection goroutines to stop
|
||||||
f.wg.Wait()
|
f.wg.Wait()
|
||||||
|
|
||||||
// Wait for prefetch to finish
|
// Wait for prefetch to finish
|
||||||
|
|||||||
@@ -13,6 +13,7 @@ import (
|
|||||||
"net/http/cookiejar"
|
"net/http/cookiejar"
|
||||||
"net/url"
|
"net/url"
|
||||||
"strings"
|
"strings"
|
||||||
|
"time"
|
||||||
"unicode/utf8"
|
"unicode/utf8"
|
||||||
|
|
||||||
"github.com/GopeedLab/gopeed/pkg/base"
|
"github.com/GopeedLab/gopeed/pkg/base"
|
||||||
@@ -120,15 +121,42 @@ func (f *Fetcher) buildRequest(ctx context.Context, req *base.Request) (httpReq
|
|||||||
return httpReq, nil
|
return httpReq, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// buildClient creates an HTTP client with the default connection timeout.
|
||||||
|
// Used for resolve phase where we don't have connection time data yet.
|
||||||
func (f *Fetcher) buildClient() *http.Client {
|
func (f *Fetcher) buildClient() *http.Client {
|
||||||
|
return f.buildClientWithTimeout(connectTimeout)
|
||||||
|
}
|
||||||
|
|
||||||
|
// buildFastFailClient creates an HTTP client with fast-fail timeout.
|
||||||
|
// Uses max(minFastFailTimeout, maxConnTime) for fast-fail retry during download phase.
|
||||||
|
func (f *Fetcher) buildFastFailClient() *http.Client {
|
||||||
|
maxConn := f.maxConnTime.Load()
|
||||||
|
if maxConn == 0 {
|
||||||
|
// No successful connection yet, use default timeout
|
||||||
|
return f.buildClientWithTimeout(connectTimeout)
|
||||||
|
}
|
||||||
|
|
||||||
|
timeout := maxConn
|
||||||
|
if timeout >= minFastFailTimeout {
|
||||||
|
// If greater than minFastFailTimeout, increase by 50% for safety margin
|
||||||
|
timeout = int64(float64(timeout) * 1.5)
|
||||||
|
} else {
|
||||||
|
timeout = minFastFailTimeout
|
||||||
|
}
|
||||||
|
return f.buildClientWithTimeout(time.Duration(timeout))
|
||||||
|
}
|
||||||
|
|
||||||
|
// buildClientWithTimeout creates an HTTP client with the specified connection timeout.
|
||||||
|
func (f *Fetcher) buildClientWithTimeout(timeout time.Duration) *http.Client {
|
||||||
transport := &http.Transport{
|
transport := &http.Transport{
|
||||||
DialContext: (&net.Dialer{
|
DialContext: (&net.Dialer{
|
||||||
Timeout: connectTimeout,
|
Timeout: timeout,
|
||||||
}).DialContext,
|
}).DialContext,
|
||||||
Proxy: f.ctl.GetProxy(f.meta.Req.Proxy),
|
Proxy: f.ctl.GetProxy(f.meta.Req.Proxy),
|
||||||
TLSClientConfig: &tls.Config{
|
TLSClientConfig: &tls.Config{
|
||||||
InsecureSkipVerify: f.meta.Req.SkipVerifyCert,
|
InsecureSkipVerify: f.meta.Req.SkipVerifyCert,
|
||||||
},
|
},
|
||||||
|
TLSHandshakeTimeout: timeout,
|
||||||
}
|
}
|
||||||
jar, _ := cookiejar.New(nil)
|
jar, _ := cookiejar.New(nil)
|
||||||
return &http.Client{
|
return &http.Client{
|
||||||
|
|||||||
+101
-215
@@ -757,7 +757,7 @@ func TestDownloader_Delete(t *testing.T) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestDownloader_ContinueBatch(t *testing.T) {
|
func TestDownloader_PauseAndContinue(t *testing.T) {
|
||||||
listener := test.StartTestSlowFileServer(time.Millisecond * 2000)
|
listener := test.StartTestSlowFileServer(time.Millisecond * 2000)
|
||||||
defer listener.Close()
|
defer listener.Close()
|
||||||
|
|
||||||
@@ -767,7 +767,74 @@ func TestDownloader_ContinueBatch(t *testing.T) {
|
|||||||
}
|
}
|
||||||
defer downloader.Clear()
|
defer downloader.Clear()
|
||||||
|
|
||||||
// Create multiple tasks and pause them
|
// Create a single task
|
||||||
|
req := &base.Request{
|
||||||
|
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
|
||||||
|
}
|
||||||
|
rr, err := downloader.Resolve(req, testDownloadOpt)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
taskId, err := downloader.Create(rr.ID)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Wait for task to start
|
||||||
|
time.Sleep(time.Millisecond * 100)
|
||||||
|
|
||||||
|
// Pause with specific taskId
|
||||||
|
err = downloader.Pause(&TaskFilter{IDs: []string{taskId}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
time.Sleep(time.Millisecond * 100)
|
||||||
|
|
||||||
|
// Verify task is paused
|
||||||
|
task := downloader.GetTask(taskId)
|
||||||
|
if task.Status != base.DownloadStatusPause {
|
||||||
|
t.Errorf("Task should be paused, got %s", task.Status)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Continue with specific taskId
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
wg.Add(1)
|
||||||
|
downloader.Listener(func(event *Event) {
|
||||||
|
if event.Key == EventKeyDone {
|
||||||
|
wg.Done()
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
err = downloader.Continue(&TaskFilter{IDs: []string{taskId}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Wait for task to complete
|
||||||
|
wg.Wait()
|
||||||
|
|
||||||
|
// Verify task is done
|
||||||
|
task = downloader.GetTask(taskId)
|
||||||
|
if task.Status != base.DownloadStatusDone {
|
||||||
|
t.Errorf("Task should be done, got %s", task.Status)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Clean up
|
||||||
|
downloader.Delete(nil, true)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDownloader_PauseAllAndContinueAll(t *testing.T) {
|
||||||
|
listener := test.StartTestSlowFileServer(time.Millisecond * 2000)
|
||||||
|
defer listener.Close()
|
||||||
|
|
||||||
|
downloader := NewDownloader(nil)
|
||||||
|
if err := downloader.Setup(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer downloader.Clear()
|
||||||
|
|
||||||
|
// Create multiple tasks
|
||||||
taskCount := 3
|
taskCount := 3
|
||||||
taskIds := make([]string, 0)
|
taskIds := make([]string, 0)
|
||||||
|
|
||||||
@@ -786,183 +853,57 @@ func TestDownloader_ContinueBatch(t *testing.T) {
|
|||||||
taskIds = append(taskIds, taskId)
|
taskIds = append(taskIds, taskId)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Wait a moment for tasks to start
|
// Wait for tasks to start
|
||||||
time.Sleep(time.Millisecond * 500)
|
time.Sleep(time.Millisecond * 100)
|
||||||
|
|
||||||
// Pause all tasks
|
// Pause all tasks with nil filter
|
||||||
err := downloader.Pause(nil)
|
err := downloader.Pause(nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Verify tasks are paused
|
time.Sleep(time.Millisecond * 100)
|
||||||
|
|
||||||
|
// Verify all tasks are paused
|
||||||
|
pausedCount := 0
|
||||||
for _, taskId := range taskIds {
|
for _, taskId := range taskIds {
|
||||||
task := downloader.GetTask(taskId)
|
task := downloader.GetTask(taskId)
|
||||||
if task.Status != base.DownloadStatusPause && task.Status != base.DownloadStatusDone {
|
if task.Status == base.DownloadStatusPause {
|
||||||
t.Errorf("Task %s should be paused or done, got %s", taskId, task.Status)
|
pausedCount++
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if pausedCount != taskCount {
|
||||||
|
t.Errorf("Expected %d paused tasks, got %d", taskCount, pausedCount)
|
||||||
|
}
|
||||||
|
|
||||||
// Test ContinueBatch with specific IDs
|
// Continue all tasks with nil filter
|
||||||
t.Run("ContinueBatch with filter", func(t *testing.T) {
|
var wg sync.WaitGroup
|
||||||
var wg sync.WaitGroup
|
wg.Add(taskCount)
|
||||||
wg.Add(1)
|
downloader.Listener(func(event *Event) {
|
||||||
downloader.Listener(func(event *Event) {
|
if event.Key == EventKeyDone {
|
||||||
if event.Key == EventKeyDone {
|
wg.Done()
|
||||||
wg.Done()
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
err := downloader.ContinueBatch(&TaskFilter{IDs: []string{taskIds[0]}})
|
|
||||||
if err != nil {
|
|
||||||
t.Errorf("ContinueBatch() unexpected error: %v", err)
|
|
||||||
}
|
|
||||||
wg.Wait()
|
|
||||||
})
|
|
||||||
|
|
||||||
// Test ContinueBatch with nil filter (continues all)
|
|
||||||
t.Run("ContinueBatch with nil filter", func(t *testing.T) {
|
|
||||||
var wg sync.WaitGroup
|
|
||||||
// Count remaining paused tasks
|
|
||||||
pausedCount := 0
|
|
||||||
for _, taskId := range taskIds {
|
|
||||||
task := downloader.GetTask(taskId)
|
|
||||||
if task.Status == base.DownloadStatusPause {
|
|
||||||
pausedCount++
|
|
||||||
}
|
|
||||||
}
|
|
||||||
wg.Add(pausedCount)
|
|
||||||
downloader.Listener(func(event *Event) {
|
|
||||||
if event.Key == EventKeyDone {
|
|
||||||
wg.Done()
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
err := downloader.ContinueBatch(nil)
|
|
||||||
if err != nil {
|
|
||||||
t.Errorf("ContinueBatch() unexpected error: %v", err)
|
|
||||||
}
|
|
||||||
if pausedCount > 0 {
|
|
||||||
wg.Wait()
|
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
// Clean up
|
err = downloader.Continue(nil)
|
||||||
downloader.Delete(nil, true)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestDownloader_PauseAndContinue(t *testing.T) {
|
|
||||||
listener := test.StartTestSlowFileServer(time.Millisecond * 5000)
|
|
||||||
defer listener.Close()
|
|
||||||
|
|
||||||
downloader := NewDownloader(nil)
|
|
||||||
if err := downloader.Setup(); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
defer downloader.Clear()
|
|
||||||
|
|
||||||
req := &base.Request{
|
|
||||||
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
|
|
||||||
}
|
|
||||||
rr, err := downloader.Resolve(req, testDownloadOpt)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
taskId, err := downloader.Create(rr.ID)
|
// Wait for all tasks to complete
|
||||||
if err != nil {
|
wg.Wait()
|
||||||
t.Fatal(err)
|
|
||||||
|
// Verify all tasks are done
|
||||||
|
doneCount := 0
|
||||||
|
for _, taskId := range taskIds {
|
||||||
|
task := downloader.GetTask(taskId)
|
||||||
|
if task.Status == base.DownloadStatusDone {
|
||||||
|
doneCount++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if doneCount != taskCount {
|
||||||
|
t.Errorf("Expected %d done tasks, got %d", taskCount, doneCount)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Wait for task to start
|
|
||||||
time.Sleep(time.Millisecond * 1000)
|
|
||||||
|
|
||||||
// Test Pause with empty filter (returns nil, not error - empty filter means no op)
|
|
||||||
t.Run("Pause with empty filter", func(t *testing.T) {
|
|
||||||
err := downloader.Pause(&TaskFilter{})
|
|
||||||
// Empty filter returns error from Pause function
|
|
||||||
if err == nil {
|
|
||||||
// This is fine - different implementations might handle this differently
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
// Test Pause with filter
|
|
||||||
t.Run("Pause with filter", func(t *testing.T) {
|
|
||||||
task := downloader.GetTask(taskId)
|
|
||||||
if task == nil {
|
|
||||||
t.Skip("Task already completed")
|
|
||||||
}
|
|
||||||
if task.Status == base.DownloadStatusDone {
|
|
||||||
t.Skip("Task already completed")
|
|
||||||
}
|
|
||||||
|
|
||||||
err := downloader.Pause(&TaskFilter{IDs: []string{taskId}})
|
|
||||||
if err != nil {
|
|
||||||
// Task might have finished between check and pause
|
|
||||||
t.Logf("Pause() error (task may have finished): %v", err)
|
|
||||||
} else {
|
|
||||||
task = downloader.GetTask(taskId)
|
|
||||||
if task.Status != base.DownloadStatusPause && task.Status != base.DownloadStatusDone {
|
|
||||||
t.Errorf("Task should be paused or done, got %s", task.Status)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
// Test Pause with non-matching filter
|
|
||||||
t.Run("Pause with non-matching filter", func(t *testing.T) {
|
|
||||||
err := downloader.Pause(&TaskFilter{IDs: []string{"non-existent-id"}})
|
|
||||||
if err == nil {
|
|
||||||
t.Error("Pause() with non-matching filter should return error")
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
// Test Continue with empty filter (returns error - empty filter means no op)
|
|
||||||
t.Run("Continue with empty filter", func(t *testing.T) {
|
|
||||||
err := downloader.Continue(&TaskFilter{})
|
|
||||||
// Empty filter returns error from Continue function
|
|
||||||
if err == nil {
|
|
||||||
// This is fine - different implementations might handle this differently
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
// Test Continue with non-matching filter
|
|
||||||
t.Run("Continue with non-matching filter", func(t *testing.T) {
|
|
||||||
err := downloader.Continue(&TaskFilter{IDs: []string{"non-existent-id"}})
|
|
||||||
if err == nil {
|
|
||||||
t.Error("Continue() with non-matching filter should return error")
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
// Test Continue with valid filter
|
|
||||||
t.Run("Continue with valid filter", func(t *testing.T) {
|
|
||||||
task := downloader.GetTask(taskId)
|
|
||||||
if task == nil {
|
|
||||||
t.Skip("Task not found")
|
|
||||||
}
|
|
||||||
if task.Status == base.DownloadStatusDone {
|
|
||||||
t.Skip("Task already completed")
|
|
||||||
}
|
|
||||||
|
|
||||||
var wg sync.WaitGroup
|
|
||||||
wg.Add(1)
|
|
||||||
downloader.Listener(func(event *Event) {
|
|
||||||
if event.Key == EventKeyDone && event.Task != nil && event.Task.ID == taskId {
|
|
||||||
wg.Done()
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
err := downloader.Continue(&TaskFilter{IDs: []string{taskId}})
|
|
||||||
if err != nil {
|
|
||||||
// Task might have finished or not be in pausable state
|
|
||||||
t.Logf("Continue() note: %v", err)
|
|
||||||
}
|
|
||||||
wg.Wait()
|
|
||||||
|
|
||||||
task = downloader.GetTask(taskId)
|
|
||||||
if task != nil && task.Status != base.DownloadStatusDone {
|
|
||||||
t.Errorf("Task should be done, got %s", task.Status)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
// Clean up
|
// Clean up
|
||||||
downloader.Delete(nil, true)
|
downloader.Delete(nil, true)
|
||||||
@@ -1603,61 +1544,6 @@ func TestDownloader_DeleteAll(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestDownloader_ContinueAll(t *testing.T) {
|
|
||||||
listener := test.StartTestFileServer()
|
|
||||||
defer listener.Close()
|
|
||||||
|
|
||||||
downloader := NewDownloader(nil)
|
|
||||||
if err := downloader.Setup(); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
defer downloader.Clear()
|
|
||||||
|
|
||||||
// Create a task and wait for completion
|
|
||||||
req := &base.Request{
|
|
||||||
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
|
|
||||||
}
|
|
||||||
|
|
||||||
var wg sync.WaitGroup
|
|
||||||
wg.Add(1)
|
|
||||||
downloader.Listener(func(event *Event) {
|
|
||||||
if event.Key == EventKeyDone {
|
|
||||||
wg.Done()
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
taskId, err := downloader.CreateDirect(req, &base.Options{
|
|
||||||
Path: test.Dir,
|
|
||||||
Name: test.DownloadName,
|
|
||||||
Extra: http.OptsExtra{
|
|
||||||
Connections: 4,
|
|
||||||
},
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
wg.Wait()
|
|
||||||
|
|
||||||
// Verify task completed
|
|
||||||
task := downloader.GetTask(taskId)
|
|
||||||
if task == nil {
|
|
||||||
t.Fatal("Task should exist")
|
|
||||||
}
|
|
||||||
if task.Status != base.DownloadStatusDone {
|
|
||||||
t.Errorf("Task should be done, got %s", task.Status)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Test ContinueBatch with nil on an already done task (should be a no-op)
|
|
||||||
err = downloader.ContinueBatch(nil)
|
|
||||||
if err != nil {
|
|
||||||
t.Logf("ContinueBatch(nil) on done tasks returned: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Clean up
|
|
||||||
downloader.Delete(nil, true)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestDownloader_ProtocolConfigNotExist(t *testing.T) {
|
func TestDownloader_ProtocolConfigNotExist(t *testing.T) {
|
||||||
downloader := NewDownloader(nil)
|
downloader := NewDownloader(nil)
|
||||||
if err := downloader.Setup(); err != nil {
|
if err := downloader.Setup(); err != nil {
|
||||||
|
|||||||
@@ -229,7 +229,7 @@ func extractArchive(archivePath string, destDir string, password string, progres
|
|||||||
func createCountingHandler(count *int) func(ctx context.Context, fileInfo archives.FileInfo) error {
|
func createCountingHandler(count *int) func(ctx context.Context, fileInfo archives.FileInfo) error {
|
||||||
return func(ctx context.Context, fileInfo archives.FileInfo) error {
|
return func(ctx context.Context, fileInfo archives.FileInfo) error {
|
||||||
if !fileInfo.IsDir() {
|
if !fileInfo.IsDir() {
|
||||||
*count++
|
(*count)++
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -227,11 +227,14 @@ func createTestZipWithMultipleFiles(path string, numFiles int) error {
|
|||||||
w := zip.NewWriter(zipFile)
|
w := zip.NewWriter(zipFile)
|
||||||
|
|
||||||
for i := 0; i < numFiles; i++ {
|
for i := 0; i < numFiles; i++ {
|
||||||
f, err := w.Create(filepath.Join("dir", "file"+string(rune('A'+i))+".txt"))
|
// Use Windows-safe names. The previous rune-based approach could generate
|
||||||
|
// invalid Windows filename characters (e.g. '\\', ':', '|') when numFiles is large.
|
||||||
|
nameInZip := fmt.Sprintf("dir/file%03d.txt", i+1)
|
||||||
|
f, err := w.Create(nameInZip)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
_, err = f.Write([]byte("Content of file " + string(rune('A'+i))))
|
_, err = f.Write([]byte(fmt.Sprintf("Content of file %03d", i+1)))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -1055,7 +1058,15 @@ func TestExtractArchive_ReadOnlyDestDir(t *testing.T) {
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
// Make it writable for cleanup
|
// Make it writable for cleanup
|
||||||
defer os.Chmod(destDir, 0755)
|
defer func() { _ = os.Chmod(destDir, 0755) }()
|
||||||
|
|
||||||
|
// Windows ACL semantics mean chmod often does not prevent writes.
|
||||||
|
// If the filesystem doesn't enforce the permission change, skip.
|
||||||
|
probe := filepath.Join(destDir, "__write_probe.tmp")
|
||||||
|
if err := os.WriteFile(probe, []byte("x"), 0644); err == nil {
|
||||||
|
_ = os.Remove(probe)
|
||||||
|
t.Skip("filesystem does not enforce read-only directory permissions")
|
||||||
|
}
|
||||||
|
|
||||||
// Extraction should fail due to read-only destination
|
// Extraction should fail due to read-only destination
|
||||||
err = extractArchive(zipPath, destDir, "", nil)
|
err = extractArchive(zipPath, destDir, "", nil)
|
||||||
@@ -1589,9 +1600,14 @@ func TestGetMultiPartArchiveBaseName(t *testing.T) {
|
|||||||
|
|
||||||
for _, tt := range tests {
|
for _, tt := range tests {
|
||||||
t.Run(tt.filename, func(t *testing.T) {
|
t.Run(tt.filename, func(t *testing.T) {
|
||||||
result := GetMultiPartArchiveBaseName(tt.filename)
|
filename := filepath.FromSlash(tt.filename)
|
||||||
if result != tt.expected {
|
expected := tt.expected
|
||||||
t.Errorf("GetMultiPartArchiveBaseName(%q) = %q, expected %q", tt.filename, result, tt.expected)
|
if expected != "" {
|
||||||
|
expected = filepath.FromSlash(expected)
|
||||||
|
}
|
||||||
|
result := GetMultiPartArchiveBaseName(filename)
|
||||||
|
if result != expected {
|
||||||
|
t.Errorf("GetMultiPartArchiveBaseName(%q) = %q, expected %q", filename, result, expected)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
@@ -2067,10 +2083,16 @@ func TestDetermineFirstPartPath(t *testing.T) {
|
|||||||
|
|
||||||
for _, tt := range tests {
|
for _, tt := range tests {
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
result := determineFirstPartPath(tt.dir, tt.baseName, tt.pattern)
|
dir := filepath.FromSlash(tt.dir)
|
||||||
if result != tt.expected {
|
expected := tt.expected
|
||||||
|
if expected != "" {
|
||||||
|
expected = filepath.FromSlash(expected)
|
||||||
|
}
|
||||||
|
|
||||||
|
result := determineFirstPartPath(dir, tt.baseName, tt.pattern)
|
||||||
|
if result != expected {
|
||||||
t.Errorf("determineFirstPartPath(%q, %q, %q) = %q, expected %q",
|
t.Errorf("determineFirstPartPath(%q, %q, %q) = %q, expected %q",
|
||||||
tt.dir, tt.baseName, tt.pattern, result, tt.expected)
|
dir, tt.baseName, tt.pattern, result, expected)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|||||||
+14
-11
@@ -2,11 +2,12 @@ package download
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"go.etcd.io/bbolt"
|
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"reflect"
|
"reflect"
|
||||||
"sync"
|
"sync"
|
||||||
|
|
||||||
|
"go.etcd.io/bbolt"
|
||||||
)
|
)
|
||||||
|
|
||||||
type Storage interface {
|
type Storage interface {
|
||||||
@@ -45,22 +46,24 @@ func changeValue(p any, v any) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
var memData = make(map[string]map[string]any)
|
|
||||||
|
|
||||||
type MemStorage struct {
|
type MemStorage struct {
|
||||||
lock *sync.RWMutex
|
lock *sync.RWMutex
|
||||||
|
data map[string]map[string]any
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewMemStorage() *MemStorage {
|
func NewMemStorage() *MemStorage {
|
||||||
return &MemStorage{
|
return &MemStorage{
|
||||||
lock: &sync.RWMutex{},
|
lock: &sync.RWMutex{},
|
||||||
|
data: make(map[string]map[string]any),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *MemStorage) Setup(buckets []string) error {
|
func (n *MemStorage) Setup(buckets []string) error {
|
||||||
|
n.lock.Lock()
|
||||||
|
defer n.lock.Unlock()
|
||||||
for _, bucket := range buckets {
|
for _, bucket := range buckets {
|
||||||
if _, ok := memData[bucket]; !ok {
|
if _, ok := n.data[bucket]; !ok {
|
||||||
memData[bucket] = make(map[string]any)
|
n.data[bucket] = make(map[string]any)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
@@ -69,7 +72,7 @@ func (n *MemStorage) Setup(buckets []string) error {
|
|||||||
func (n *MemStorage) Put(bucket string, key string, v any) error {
|
func (n *MemStorage) Put(bucket string, key string, v any) error {
|
||||||
n.lock.Lock()
|
n.lock.Lock()
|
||||||
defer n.lock.Unlock()
|
defer n.lock.Unlock()
|
||||||
if bucketData, ok := memData[bucket]; ok {
|
if bucketData, ok := n.data[bucket]; ok {
|
||||||
bucketData[key] = v
|
bucketData[key] = v
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
@@ -78,7 +81,7 @@ func (n *MemStorage) Put(bucket string, key string, v any) error {
|
|||||||
func (n *MemStorage) Get(bucket string, key string, v any) (bool, error) {
|
func (n *MemStorage) Get(bucket string, key string, v any) (bool, error) {
|
||||||
n.lock.RLock()
|
n.lock.RLock()
|
||||||
defer n.lock.RUnlock()
|
defer n.lock.RUnlock()
|
||||||
if dv, ok := memData[bucket][key]; ok {
|
if dv, ok := n.data[bucket][key]; ok {
|
||||||
changeValue(v, dv)
|
changeValue(v, dv)
|
||||||
return true, nil
|
return true, nil
|
||||||
}
|
}
|
||||||
@@ -88,7 +91,7 @@ func (n *MemStorage) Get(bucket string, key string, v any) (bool, error) {
|
|||||||
func (n *MemStorage) List(bucket string, v any) error {
|
func (n *MemStorage) List(bucket string, v any) error {
|
||||||
n.lock.RLock()
|
n.lock.RLock()
|
||||||
defer n.lock.RUnlock()
|
defer n.lock.RUnlock()
|
||||||
data := memData[bucket]
|
data := n.data[bucket]
|
||||||
list := make([]any, 0)
|
list := make([]any, 0)
|
||||||
for _, v := range data {
|
for _, v := range data {
|
||||||
list = append(list, v)
|
list = append(list, v)
|
||||||
@@ -100,7 +103,7 @@ func (n *MemStorage) List(bucket string, v any) error {
|
|||||||
func (n *MemStorage) Pop(bucket string, key string, v any) error {
|
func (n *MemStorage) Pop(bucket string, key string, v any) error {
|
||||||
n.lock.Lock()
|
n.lock.Lock()
|
||||||
defer n.lock.Unlock()
|
defer n.lock.Unlock()
|
||||||
data := memData[bucket]
|
data := n.data[bucket]
|
||||||
changeValue(v, data[key])
|
changeValue(v, data[key])
|
||||||
delete(data, key)
|
delete(data, key)
|
||||||
return nil
|
return nil
|
||||||
@@ -109,7 +112,7 @@ func (n *MemStorage) Pop(bucket string, key string, v any) error {
|
|||||||
func (n *MemStorage) Delete(bucket string, key string) error {
|
func (n *MemStorage) Delete(bucket string, key string) error {
|
||||||
n.lock.Lock()
|
n.lock.Lock()
|
||||||
defer n.lock.Unlock()
|
defer n.lock.Unlock()
|
||||||
delete(memData[bucket], key)
|
delete(n.data[bucket], key)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -120,7 +123,7 @@ func (n *MemStorage) Close() error {
|
|||||||
func (n *MemStorage) Clear() error {
|
func (n *MemStorage) Clear() error {
|
||||||
n.lock.Lock()
|
n.lock.Lock()
|
||||||
defer n.lock.Unlock()
|
defer n.lock.Unlock()
|
||||||
memData = make(map[string]map[string]any)
|
n.data = make(map[string]map[string]any)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -216,6 +216,7 @@ func TestPauseAndContinueTask(t *testing.T) {
|
|||||||
if t2.Status != base.DownloadStatusPause {
|
if t2.Status != base.DownloadStatusPause {
|
||||||
t.Errorf("PauseTask() got = %v, want %v", t2.Status, base.DownloadStatusPause)
|
t.Errorf("PauseTask() got = %v, want %v", t2.Status, base.DownloadStatusPause)
|
||||||
}
|
}
|
||||||
|
time.Sleep(time.Millisecond * 100)
|
||||||
httpRequestCheckOk[any](http.MethodPut, "/api/v1/tasks/"+taskId+"/continue", nil)
|
httpRequestCheckOk[any](http.MethodPut, "/api/v1/tasks/"+taskId+"/continue", nil)
|
||||||
t3 := httpRequestCheckOk[*download.Task](http.MethodGet, "/api/v1/tasks/"+taskId, nil)
|
t3 := httpRequestCheckOk[*download.Task](http.MethodGet, "/api/v1/tasks/"+taskId, nil)
|
||||||
if t3.Status != base.DownloadStatusRunning {
|
if t3.Status != base.DownloadStatusRunning {
|
||||||
@@ -240,6 +241,7 @@ func TestPauseAllAndContinueALLTasks(t *testing.T) {
|
|||||||
|
|
||||||
createAndPause := func() {
|
createAndPause := func() {
|
||||||
taskId := httpRequestCheckOk[string](http.MethodPost, "/api/v1/tasks", createReq)
|
taskId := httpRequestCheckOk[string](http.MethodPost, "/api/v1/tasks", createReq)
|
||||||
|
time.Sleep(time.Millisecond * 5)
|
||||||
httpRequestCheckOk[*download.Task](http.MethodPut, "/api/v1/tasks/"+taskId+"/pause", nil)
|
httpRequestCheckOk[*download.Task](http.MethodPut, "/api/v1/tasks/"+taskId+"/pause", nil)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -247,6 +249,7 @@ func TestPauseAllAndContinueALLTasks(t *testing.T) {
|
|||||||
for i := 0; i < total; i++ {
|
for i := 0; i < total; i++ {
|
||||||
createAndPause()
|
createAndPause()
|
||||||
}
|
}
|
||||||
|
time.Sleep(time.Millisecond * 50)
|
||||||
|
|
||||||
// continue all
|
// continue all
|
||||||
httpRequestCheckOk[any](http.MethodPut, "/api/v1/tasks/continue", nil)
|
httpRequestCheckOk[any](http.MethodPut, "/api/v1/tasks/continue", nil)
|
||||||
|
|||||||
@@ -8,7 +8,13 @@ class FileExplorer {
|
|||||||
if (await FileSystemEntity.isFile(filePath)) {
|
if (await FileSystemEntity.isFile(filePath)) {
|
||||||
_openFile(filePath);
|
_openFile(filePath);
|
||||||
} else if (await FileSystemEntity.isDirectory(filePath)) {
|
} else if (await FileSystemEntity.isDirectory(filePath)) {
|
||||||
_openDirectory(filePath);
|
await _openDirectory(filePath);
|
||||||
|
} else {
|
||||||
|
// If file does not exist, open its parent directory
|
||||||
|
final parentPath = path.dirname(filePath);
|
||||||
|
if (await FileSystemEntity.isDirectory(parentPath)) {
|
||||||
|
await _openDirectory(parentPath);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Reference in new issue
Block a user