mirror of
https://github.com/GopeedLab/gopeed.git
synced 2026-11-10 10:30:28 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c03132d423 | ||
|
|
128fada585 | ||
|
|
7dd704fc71 |
No files matched your search
@@ -106,32 +106,17 @@ jobs:
|
||||
if ("${{ matrix.arch }}" -eq "amd64") {
|
||||
# for amd64
|
||||
$release = "build\windows\x64\runner\Release\"
|
||||
# Use the same toolchain as cgo instead of Git's bundled DLLs.
|
||||
$cc = (Get-Command -Name (go env CC) -CommandType Application -ErrorAction Stop | Select-Object -First 1).Source
|
||||
foreach ($dll in @("libstdc++-6.dll", "libgcc_s_seh-1.dll", "libwinpthread-1.dll")) {
|
||||
$runtimePath = [string](& $cc "-print-file-name=$dll")
|
||||
if ($LASTEXITCODE -ne 0 -or [string]::IsNullOrWhiteSpace($runtimePath)) {
|
||||
throw "Failed to locate $dll using $cc"
|
||||
}
|
||||
$runtimePath = $runtimePath.Trim()
|
||||
# Some MinGW distributions keep runtime DLLs beside gcc.exe.
|
||||
if ($runtimePath -eq $dll) {
|
||||
$runtimePath = Join-Path (Split-Path -Parent $cc) $dll
|
||||
}
|
||||
if (!(Test-Path -LiteralPath $runtimePath -PathType Leaf)) {
|
||||
throw "MinGW runtime $dll was not found by $cc (reported path: $runtimePath)"
|
||||
}
|
||||
Write-Host "Copying MinGW runtime: $runtimePath"
|
||||
Copy-Item -LiteralPath $runtimePath -Destination $release
|
||||
}
|
||||
$mingw = "C:\Program Files\Git\mingw64\bin"
|
||||
cp $mingw\libstdc++-6.dll $release
|
||||
cp $mingw\libgcc_s_seh-1.dll $release
|
||||
} else {
|
||||
# for ARM64
|
||||
$release = "build\windows\arm64\runner\Release\"
|
||||
$mingw = "$env:CLANGARM64_ROOT\aarch64-w64-mingw32\bin"
|
||||
cp $mingw\libc++.dll $release
|
||||
cp $mingw\libunwind.dll $release
|
||||
cp $mingw\libwinpthread-1.dll $release
|
||||
}
|
||||
cp $mingw\libwinpthread-1.dll $release
|
||||
cp $system\msvcp140.dll $release
|
||||
cp $system\vcruntime140.dll $release
|
||||
cp $system\vcruntime140_1.dll $release
|
||||
@@ -798,5 +783,5 @@ jobs:
|
||||
linux/arm64
|
||||
linux/arm/v7
|
||||
tags: |
|
||||
${{ !contains(env.VERSION, 'beta') && 'liwei2633/gopeed:latest' || '' }}
|
||||
liwei2633/gopeed:latest
|
||||
liwei2633/gopeed:v${{ env.VERSION }}
|
||||
@@ -69,49 +69,6 @@ jobs:
|
||||
with:
|
||||
files: ./coverage.txt
|
||||
token: ${{ secrets.CODECOV_UPLOAD_TOKEN }}
|
||||
test-flutter-category:
|
||||
# The settings editor and the backend have to read an extension list the
|
||||
# same way, so both sides run the shared rule table. The Go side is covered
|
||||
# by the test job above, this runs the ui side of it.
|
||||
name: Test shared category rules
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- uses: subosito/flutter-action@v2
|
||||
with:
|
||||
flutter-version: ${{ env.FLUTTER_VERSION }}
|
||||
- name: Run the shared category rule table on the ui side
|
||||
working-directory: ui/flutter
|
||||
run: |
|
||||
flutter pub get
|
||||
flutter gen-l10n
|
||||
flutter test test/api/downloader_config_test.dart
|
||||
test-flutter-task-context-menu:
|
||||
name: Test task context menus and text inputs
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 20
|
||||
env:
|
||||
CHROME_EXECUTABLE: /usr/bin/google-chrome
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- uses: subosito/flutter-action@v2
|
||||
with:
|
||||
flutter-version: ${{ env.FLUTTER_VERSION }}
|
||||
- name: Prepare Flutter tests
|
||||
working-directory: ui/flutter
|
||||
run: |
|
||||
flutter pub get
|
||||
flutter gen-l10n
|
||||
- name: Test task menu actions and native input policy
|
||||
working-directory: ui/flutter
|
||||
run: |
|
||||
flutter test --no-pub test/features/tasks/presentation/widgets/task_context_menu_test.dart test/shared/widgets/app_text_field_test.dart
|
||||
flutter test --no-pub test/widget_test.dart --plain-name 'task context menu exposes'
|
||||
- name: Test browser menu cancellation and input policy
|
||||
working-directory: ui/flutter
|
||||
run: |
|
||||
"$CHROME_EXECUTABLE" --version
|
||||
flutter test --no-pub --platform chrome test/features/tasks/presentation/widgets/task_context_menu_test.dart test/shared/widgets/app_text_field_test.dart
|
||||
test-webview-integration:
|
||||
name: Test webview integration (${{ matrix.name }})
|
||||
strategy:
|
||||
|
||||
@@ -20,7 +20,7 @@ Beyond core download management, Gopeed offers browser integration, JavaScript e
|
||||
|
||||
Visit ✈ [Official Website](https://gopeed.com)
|
||||
|
||||

|
||||

|
||||
|
||||
## ✨ Features
|
||||
|
||||
|
||||
+1
-1
@@ -20,7 +20,7 @@ Gopeed(**Go Speed** の略)は、Go と Flutter で開発された高速で
|
||||
|
||||
✈ [公式サイト](https://gopeed.com)
|
||||
|
||||

|
||||

|
||||
|
||||
## ✨ 主な機能
|
||||
|
||||
|
||||
+1
-1
@@ -20,7 +20,7 @@ Ngoài việc quản lý tải xuống hằng ngày, Gopeed còn hỗ trợ tíc
|
||||
|
||||
Truy cập ✈ [Trang web chính thức](https://gopeed.com)
|
||||
|
||||

|
||||

|
||||
|
||||
## ✨ Tính năng chính
|
||||
|
||||
|
||||
+1
-1
@@ -20,7 +20,7 @@ Gopeed(全称 **Go Speed**,也叫“够快下载器”)是一款使用 Go
|
||||
|
||||
访问 ✈ [官方网站](https://gopeed.com)
|
||||
|
||||

|
||||

|
||||
|
||||
## ✨ 主要特性
|
||||
|
||||
|
||||
+1
-1
@@ -20,7 +20,7 @@ Gopeed(**Go Speed** 的縮寫)是一款以 Go 與 Flutter 開發的高速、
|
||||
|
||||
瀏覽 ✈ [官方網站](https://gopeed.com)
|
||||
|
||||

|
||||

|
||||
|
||||
## ✨ 主要功能
|
||||
|
||||
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 74 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 77 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 1.3 MiB |
Binary file not shown.
|
Before Width: | Height: | Size: 792 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 1.3 MiB |
Binary file not shown.
|
Before Width: | Height: | Size: 787 KiB |
@@ -1,52 +0,0 @@
|
||||
# Batch task creation
|
||||
|
||||
`POST /api/v1/tasks/batch` accepts ordinary and grouped batches:
|
||||
|
||||
```json
|
||||
{
|
||||
"group": true,
|
||||
"groupName": "album",
|
||||
"opts": {"path": "/downloads"},
|
||||
"reqs": [
|
||||
{"rid": "resolved-resource-id", "opts": {"selectFiles": [1]}},
|
||||
{"req": {"url": "https://example.com/cover.jpg"}}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
Each member accepts `rid` or `req`. A nonempty `rid` takes priority and creates a
|
||||
task from the cached resolve result. A member's `opts`, when present, replaces
|
||||
batch `opts` as a whole; omitted fields do not inherit individual batch values.
|
||||
Members without `opts` use batch `opts`. BT file selections use the indexes from
|
||||
the resolve result. Requests without `rid` continue to work.
|
||||
|
||||
For groups, batch `opts.path` selects the parent directory. Members inheriting
|
||||
batch options, or with an empty path, use the reserved group directory. Explicit
|
||||
relative member paths are resolved under that directory; do not prefix them with
|
||||
the batch parent path. For example, parent `downloads` and member path `nested`
|
||||
place the member in `downloads/<group name>/nested`. Absolute paths under the
|
||||
parent directory preserve their relative structure within the group. In ordinary
|
||||
batches, an empty member path uses the configured default download directory.
|
||||
|
||||
All members are attempted. If at least one task is persisted, the response is
|
||||
successful and contains the created task IDs. A task that fails to start after
|
||||
being persisted is included; use the existing task controls to retry it.
|
||||
Successful tasks are kept when other members fail. If every member fails, the
|
||||
API returns an error and removes any newly reserved empty group and directory.
|
||||
|
||||
Both ordinary and grouped batches now return an object in `data`:
|
||||
|
||||
```json
|
||||
{"code": 0, "msg": "", "data": {"taskIds": ["task-1", "task-2"], "parentId": "group-id"}}
|
||||
```
|
||||
|
||||
Ordinary batches omit `parentId`. **This replaces the former array response for
|
||||
ordinary batches.** The outer `code`/`msg`/`data` envelope is unchanged. Clients
|
||||
must read `data.taskIds`; there are no per-member results. A successful response
|
||||
completes the batch, including partial success, so clients must not automatically
|
||||
resubmit missing members.
|
||||
|
||||
With `group: true`, the server generates the group ID and reserves its directory.
|
||||
Creation does not accept a client group ID or append to an existing group. Group
|
||||
metadata remains available through the existing task list and detail responses;
|
||||
`parentId` identifies membership for group operations.
|
||||
+1
-7
@@ -1,9 +1,6 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"github.com/GopeedLab/gopeed/cmd"
|
||||
"github.com/GopeedLab/gopeed/pkg/rest/model"
|
||||
)
|
||||
@@ -16,8 +13,5 @@ func main() {
|
||||
Storage: model.StorageBolt,
|
||||
WebEnable: true,
|
||||
}
|
||||
if err := cmd.Start(cfg); err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
cmd.Start(cfg)
|
||||
}
|
||||
+2
-2
@@ -166,7 +166,7 @@ func runResolve(args []string) error {
|
||||
result, err := d.Resolve(&base.Request{
|
||||
URL: url,
|
||||
Labels: map[string]string(labels),
|
||||
})
|
||||
}, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -250,7 +250,7 @@ func runDownload(args []string) error {
|
||||
result, err := d.Resolve(&base.Request{
|
||||
URL: url,
|
||||
Labels: map[string]string(labels),
|
||||
})
|
||||
}, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
+6
-14
@@ -16,18 +16,17 @@ import (
|
||||
//go:embed banner.txt
|
||||
var banner string
|
||||
|
||||
func Start(cfg *model.StartConfig) error {
|
||||
func Start(cfg *model.StartConfig) {
|
||||
fmt.Println(banner)
|
||||
srv, listener, err := rest.BuildServer(cfg)
|
||||
if err != nil {
|
||||
return err
|
||||
panic(err)
|
||||
}
|
||||
fmt.Println(banner)
|
||||
downloadCfg, err := rest.Downloader.GetConfig()
|
||||
if err != nil {
|
||||
listener.Close()
|
||||
return err
|
||||
panic(err)
|
||||
}
|
||||
if rest.Downloader.IsFirstLoad() {
|
||||
if downloadCfg.FirstLoad {
|
||||
// Set default download config
|
||||
if cfg.DownloadConfig != nil {
|
||||
cfg.DownloadConfig.Merge(downloadCfg)
|
||||
@@ -52,20 +51,13 @@ func Start(cfg *model.StartConfig) error {
|
||||
rest.Downloader.PutConfig(downloadCfg)
|
||||
}
|
||||
}
|
||||
|
||||
// The default download dir is resolved only after the core starts, so
|
||||
// seed the built-in categories now that the dir is known. Init is
|
||||
// idempotent and leaves an existing category list untouched.
|
||||
downloadCfg.Init()
|
||||
rest.Downloader.PutConfig(downloadCfg)
|
||||
}
|
||||
watchExit()
|
||||
|
||||
fmt.Printf("Server start success on http://%s\n", listener.Addr().String())
|
||||
if err := srv.Serve(listener); err != nil && err != http.ErrServerClosed {
|
||||
return err
|
||||
panic(err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func watchExit() {
|
||||
|
||||
@@ -1,35 +0,0 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/GopeedLab/gopeed/pkg/rest/model"
|
||||
)
|
||||
|
||||
func TestStartInvalidWebAPITokenReturnsError(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
webAuth *model.WebAuth
|
||||
}{
|
||||
{name: "no credentials"},
|
||||
{name: "missing username", webAuth: &model.WebAuth{Password: "secret"}},
|
||||
{name: "missing password", webAuth: &model.WebAuth{Username: "gopeed"}},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
err := Start(&model.StartConfig{
|
||||
WebEnable: true,
|
||||
ApiToken: "test-token",
|
||||
WebAuth: tt.webAuth,
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("Start() error = nil, want web authentication validation error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "web authentication username and password are required") {
|
||||
t.Fatalf("Start() error = %q, want missing web authentication credentials", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
+2
-2
@@ -52,8 +52,8 @@ func loadCliArgs() *args {
|
||||
flag.StringVar(cfg.Username, "username", "gopeed", "Web Authentication Username")
|
||||
cfg.Password = flag.String("p", "", "Web Authentication Password, if no password is set, web authentication will not be enabled")
|
||||
flag.StringVar(cfg.Password, "password", "", "Web Authentication Password, if no password is set, web authentication will not be enabled")
|
||||
cfg.ApiToken = flag.String("T", "", "API token (requires web authentication with username and password)")
|
||||
flag.StringVar(cfg.ApiToken, "api-token", "", "API token (requires web authentication with username and password)")
|
||||
cfg.ApiToken = flag.String("T", "", "API token, it must be configured when using HTTP API in the case of enabling web authentication")
|
||||
flag.StringVar(cfg.ApiToken, "api-token", "", "API token, it must be configured when using HTTP API in the case of enabling web authentication")
|
||||
flag.BoolVar(&cfg.MCPEnable, "mcp-enable", false, "Enable MCP endpoint")
|
||||
cfg.TempDir = flag.String("temp-dir", "", "Temporary directory (defaults to the OS temporary directory)")
|
||||
cfg.StorageDir = flag.String("d", "", "Storage directory")
|
||||
|
||||
+1
-4
@@ -62,10 +62,7 @@ func main() {
|
||||
WebFS: sub,
|
||||
WebAuth: webAuth,
|
||||
}
|
||||
if err := cmd.Start(cfg); err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
cmd.Start(cfg)
|
||||
}
|
||||
|
||||
func isNotBlank(str *string) bool {
|
||||
|
||||
@@ -10,13 +10,11 @@ require (
|
||||
github.com/bodgit/sevenzip v1.6.1
|
||||
github.com/dop251/goja v0.0.0-20260106131823-651366fbe6e3
|
||||
github.com/dop251/goja_nodejs v0.0.0-20240728170619-29b559befffc
|
||||
github.com/fclairamb/ftpserverlib v0.31.0
|
||||
github.com/go-git/go-git/v5 v5.19.2
|
||||
github.com/go-git/go-git/v5 v5.8.1
|
||||
github.com/google/jsonschema-go v0.4.3
|
||||
github.com/gorilla/handlers v1.5.1
|
||||
github.com/gorilla/mux v1.8.0
|
||||
github.com/imroc/req/v3 v3.58.0
|
||||
github.com/jlaffaye/ftp v0.2.4
|
||||
github.com/imroc/req/v3 v3.52.2
|
||||
github.com/matoous/go-nanoid/v2 v2.0.0
|
||||
github.com/mattn/go-ieproxy v0.0.12
|
||||
github.com/mholt/archives v0.1.5
|
||||
@@ -25,17 +23,10 @@ require (
|
||||
github.com/pkg/errors v0.9.1
|
||||
github.com/rs/dnscache v0.0.0-20230804202142-fc85eb664529
|
||||
github.com/rs/zerolog v1.31.0
|
||||
github.com/spf13/afero v1.15.0
|
||||
github.com/tetratelabs/wazero v1.12.0
|
||||
github.com/xiaoqidun/setft v0.0.0-20220310121541-be86327699ad
|
||||
go.etcd.io/bbolt v1.4.3
|
||||
golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/cyphar/filepath-securejoin v0.6.1 // indirect
|
||||
github.com/google/go-querystring v1.1.0 // indirect
|
||||
github.com/onsi/gomega v1.36.3 // indirect
|
||||
golang.org/x/exp v0.0.0-20251219203646-944ab1f22d93
|
||||
)
|
||||
|
||||
require (
|
||||
@@ -48,40 +39,48 @@ require (
|
||||
github.com/cespare/xxhash/v2 v2.3.0 // indirect
|
||||
github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707 // indirect
|
||||
github.com/felixge/fgprof v0.9.5 // indirect
|
||||
github.com/go-task/slim-sprig/v3 v3.0.0 // indirect
|
||||
github.com/google/go-cmp v0.7.0 // indirect
|
||||
github.com/hashicorp/errwrap v1.1.0 // indirect
|
||||
github.com/hashicorp/go-multierror v1.1.1 // indirect
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
|
||||
github.com/icholy/digest v1.1.0 // indirect
|
||||
github.com/klauspost/compress v1.18.2 // indirect
|
||||
github.com/klauspost/compress v1.18.0 // indirect
|
||||
github.com/klauspost/pgzip v1.2.6 // indirect
|
||||
github.com/mikelolasagasti/xz v1.0.1 // indirect
|
||||
github.com/minio/minlz v1.0.1 // indirect
|
||||
github.com/nwaples/rardecode/v2 v2.2.0 // indirect
|
||||
github.com/onsi/ginkgo/v2 v2.23.4 // indirect
|
||||
github.com/pierrec/lz4/v4 v4.1.22 // indirect
|
||||
github.com/pion/transport/v4 v4.0.2 // indirect
|
||||
github.com/quic-go/qpack v0.6.0 // indirect
|
||||
github.com/quic-go/quic-go v0.59.0 // indirect
|
||||
github.com/refraction-networking/utls v1.8.2 // indirect
|
||||
github.com/pion/transport/v4 v4.0.1 // indirect
|
||||
github.com/quic-go/qpack v0.5.1 // indirect
|
||||
github.com/quic-go/quic-go v0.51.0 // indirect
|
||||
github.com/refraction-networking/utls v1.6.7 // indirect
|
||||
github.com/segmentio/asm v1.1.3 // indirect
|
||||
github.com/segmentio/encoding v0.5.4 // indirect
|
||||
github.com/sorairolake/lzip-go v0.3.8 // indirect
|
||||
github.com/spf13/afero v1.15.0 // indirect
|
||||
github.com/ulikunitz/xz v0.5.15 // indirect
|
||||
github.com/yosida95/uritemplate/v3 v3.0.2 // indirect
|
||||
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
|
||||
go.uber.org/automaxprocs v1.6.0 // indirect
|
||||
go.uber.org/mock v0.5.1 // indirect
|
||||
go4.org v0.0.0-20230225012048-214862532bf5 // indirect
|
||||
golang.org/x/oauth2 v0.35.0 // indirect
|
||||
golang.org/x/sync v0.22.0 // indirect
|
||||
golang.org/x/sync v0.20.0 // indirect
|
||||
)
|
||||
|
||||
require (
|
||||
dario.cat/mergo v1.0.0 // indirect
|
||||
github.com/Microsoft/go-winio v0.6.2
|
||||
github.com/ProtonMail/go-crypto v1.1.6 // indirect
|
||||
github.com/Microsoft/go-winio v0.6.1
|
||||
github.com/ProtonMail/go-crypto v0.0.0-20230717121422-5aa5874ade95 // indirect
|
||||
github.com/RoaringBitmap/roaring v1.9.4
|
||||
github.com/acomagu/bufpipe v1.0.4 // indirect
|
||||
github.com/alecthomas/atomic v0.1.0-alpha2 // indirect
|
||||
github.com/anacrolix/chansync v0.7.0 // indirect
|
||||
github.com/anacrolix/dht/v2 v2.23.0 // indirect
|
||||
github.com/anacrolix/envpprof v1.5.0 // indirect
|
||||
github.com/anacrolix/generics v0.2.0
|
||||
github.com/anacrolix/generics v0.2.0 // indirect
|
||||
github.com/anacrolix/go-libutp v1.3.2 // indirect
|
||||
github.com/anacrolix/log v0.17.1-0.20251118025802-918f1157b7bb // indirect
|
||||
github.com/anacrolix/missinggo v1.3.0 // indirect
|
||||
@@ -97,7 +96,7 @@ require (
|
||||
github.com/bits-and-blooms/bitset v1.24.4 // indirect
|
||||
github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8 // indirect
|
||||
github.com/cespare/xxhash v1.1.0 // indirect
|
||||
github.com/cloudflare/circl v1.6.3 // indirect
|
||||
github.com/cloudflare/circl v1.6.1 // indirect
|
||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||
github.com/dlclark/regexp2 v1.11.5 // indirect
|
||||
github.com/dustin/go-humanize v1.0.1 // indirect
|
||||
@@ -105,13 +104,13 @@ require (
|
||||
github.com/emirpasic/gods v1.18.1 // indirect
|
||||
github.com/felixge/httpsnoop v1.0.1 // indirect
|
||||
github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
|
||||
github.com/go-git/go-billy/v5 v5.9.2 // indirect
|
||||
github.com/go-git/go-billy/v5 v5.4.1 // indirect
|
||||
github.com/go-llsqlite/adapter v0.2.0 // indirect
|
||||
github.com/go-llsqlite/crawshaw v0.6.0 // indirect
|
||||
github.com/go-logr/logr v1.4.3 // indirect
|
||||
github.com/go-logr/stdr v1.2.2 // indirect
|
||||
github.com/go-sourcemap/sourcemap v2.1.4+incompatible // indirect
|
||||
github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect
|
||||
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
|
||||
github.com/google/btree v1.1.3 // indirect
|
||||
github.com/google/pprof v0.0.0-20260115054156-294ebfa9ad83 // indirect
|
||||
github.com/google/uuid v1.6.0 // indirect
|
||||
@@ -129,7 +128,7 @@ require (
|
||||
github.com/multiformats/go-varint v0.1.0 // indirect
|
||||
github.com/ncruces/go-strftime v1.0.0 // indirect
|
||||
github.com/pion/datachannel v1.6.0 // indirect
|
||||
github.com/pion/dtls/v3 v3.1.4 // indirect
|
||||
github.com/pion/dtls/v3 v3.0.10 // indirect
|
||||
github.com/pion/ice/v4 v4.2.0 // indirect
|
||||
github.com/pion/interceptor v0.1.42 // indirect
|
||||
github.com/pion/logging v0.2.4 // indirect
|
||||
@@ -140,15 +139,15 @@ require (
|
||||
github.com/pion/sctp v1.9.1 // indirect
|
||||
github.com/pion/sdp/v3 v3.0.17 // indirect
|
||||
github.com/pion/srtp/v3 v3.0.10 // indirect
|
||||
github.com/pion/stun/v3 v3.1.5 // indirect
|
||||
github.com/pion/stun/v3 v3.1.1 // indirect
|
||||
github.com/pion/turn/v4 v4.1.4 // indirect
|
||||
github.com/pion/webrtc/v4 v4.2.2-0.20260109001657-a5962f314db7 // indirect
|
||||
github.com/pjbgf/sha1cd v0.6.0 // indirect
|
||||
github.com/pjbgf/sha1cd v0.3.0 // indirect
|
||||
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c
|
||||
github.com/protolambda/ctxlock v0.1.0 // indirect
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
|
||||
github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect
|
||||
github.com/skeema/knownhosts v1.3.1 // indirect
|
||||
github.com/sergi/go-diff v1.1.0 // indirect
|
||||
github.com/skeema/knownhosts v1.2.0 // indirect
|
||||
github.com/spaolacci/murmur3 v1.1.0 // indirect
|
||||
github.com/tidwall/btree v1.8.1 // indirect
|
||||
github.com/wlynxg/anet v0.0.5 // indirect
|
||||
@@ -156,11 +155,13 @@ require (
|
||||
go.opentelemetry.io/otel v1.39.0 // indirect
|
||||
go.opentelemetry.io/otel/metric v1.39.0 // indirect
|
||||
go.opentelemetry.io/otel/trace v1.39.0 // indirect
|
||||
golang.org/x/crypto v0.55.0 // indirect
|
||||
golang.org/x/net v0.58.0
|
||||
golang.org/x/sys v0.47.0 // indirect
|
||||
golang.org/x/text v0.41.0
|
||||
golang.org/x/time v0.15.0
|
||||
golang.org/x/crypto v0.48.0 // indirect
|
||||
golang.org/x/mod v0.33.0 // indirect
|
||||
golang.org/x/net v0.50.0
|
||||
golang.org/x/sys v0.44.0 // indirect
|
||||
golang.org/x/text v0.34.0
|
||||
golang.org/x/time v0.15.0 // indirect
|
||||
golang.org/x/tools v0.42.0 // indirect
|
||||
gopkg.in/warnings.v0 v0.1.2 // indirect
|
||||
lukechampine.com/blake3 v1.4.1 // indirect
|
||||
modernc.org/libc v1.67.4 // indirect
|
||||
|
||||
@@ -25,17 +25,29 @@ filippo.io/edwards25519 v1.0.0-rc.1 h1:m0VOOB23frXZvAOK44usCgLWvtsxIoMCTBGJZlpmG
|
||||
filippo.io/edwards25519 v1.0.0-rc.1/go.mod h1:N1IkdkCkiLB6tki+MYJoSx2JTY9NUlxZE7eHn5EwJns=
|
||||
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
|
||||
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
|
||||
github.com/GopeedLab/webview_go v0.0.0-20260915010614-2567e492369a h1:Vzz+8aMVX9ZPPpkxgxa7mCorrAgdXCZL5EunOpOBvao=
|
||||
github.com/GopeedLab/webview_go v0.0.0-20260915010614-2567e492369a/go.mod h1:FvV19lu9rQ9NcbczN5y8pbWecURRRbWa8tbfFARBDa0=
|
||||
github.com/GopeedLab/webview_go v0.0.0-20260917153919-dcf00737f32a h1:nz1bC6TGwQ1kv/OVLahNZ0EEtb7v5LHfceAkW4MvS6I=
|
||||
github.com/GopeedLab/webview_go v0.0.0-20260917153919-dcf00737f32a/go.mod h1:FvV19lu9rQ9NcbczN5y8pbWecURRRbWa8tbfFARBDa0=
|
||||
github.com/GopeedLab/webview_go v0.0.0-20260917155039-1ee7ffb4169a h1:yudcfXpipKLnMaaw1losn6YeOBjvj2rAFicfjqYJexY=
|
||||
github.com/GopeedLab/webview_go v0.0.0-20260917155039-1ee7ffb4169a/go.mod h1:FvV19lu9rQ9NcbczN5y8pbWecURRRbWa8tbfFARBDa0=
|
||||
github.com/GopeedLab/webview_go v0.0.0-20260917155218-bd16da0d6f53 h1:nMSHKhbrEaWPE/x3mYl/270GPjswbUmS+E7yYjly4xk=
|
||||
github.com/GopeedLab/webview_go v0.0.0-20260917155218-bd16da0d6f53/go.mod h1:oNO07SCX8xgoHLds2OBwML8TU4xils88vMQ4gadKUHQ=
|
||||
github.com/GopeedLab/webview_go v0.0.0-20260917161756-2a4f989dcb1d h1:4fjlhJpBWa6IcdKVPkEMMIDd6Xo9JM09dJHdKU+1V7k=
|
||||
github.com/GopeedLab/webview_go v0.0.0-20260917161756-2a4f989dcb1d/go.mod h1:oNO07SCX8xgoHLds2OBwML8TU4xils88vMQ4gadKUHQ=
|
||||
github.com/GopeedLab/webview_go v0.0.0-20260917163929-ca26c3e6bcfa h1:VkdBMcNHYTqCMwOOCimA7b3xZKV/qFIGrhh3+vm9n3s=
|
||||
github.com/GopeedLab/webview_go v0.0.0-20260917163929-ca26c3e6bcfa/go.mod h1:oNO07SCX8xgoHLds2OBwML8TU4xils88vMQ4gadKUHQ=
|
||||
github.com/GopeedLab/webview_go v0.0.0-20260918010912-0d4c27bb7914 h1:9VJQZIZ1R6T4m+4vlNQYNXSPNUv2gf2PiTfIXiQ3MTw=
|
||||
github.com/GopeedLab/webview_go v0.0.0-20260918010912-0d4c27bb7914/go.mod h1:oNO07SCX8xgoHLds2OBwML8TU4xils88vMQ4gadKUHQ=
|
||||
github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0rYXWg0=
|
||||
github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ=
|
||||
github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY=
|
||||
github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=
|
||||
github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
|
||||
github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow=
|
||||
github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM=
|
||||
github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE=
|
||||
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
|
||||
github.com/ProtonMail/go-crypto v1.1.6 h1:ZcV+Ropw6Qn0AX9brlQLAUXfqLBc7Bl+f/DmNxpLfdw=
|
||||
github.com/ProtonMail/go-crypto v1.1.6/go.mod h1:rA3QumHc/FZ8pAHreoekgiAbzpNsfQAosU5td4SnOrE=
|
||||
github.com/ProtonMail/go-crypto v0.0.0-20230717121422-5aa5874ade95 h1:KLq8BE0KwCL+mmXnjLWEAOYO+2l2AE4YMmqG1ZpZHBs=
|
||||
github.com/ProtonMail/go-crypto v0.0.0-20230717121422-5aa5874ade95/go.mod h1:EjAoLdwvbIOoOQr3ihjnSoLZRtE8azugULFRteWMNc0=
|
||||
github.com/RoaringBitmap/roaring v0.4.7/go.mod h1:8khRDP4HmeXns4xIj9oGrKSz7XTQiJx2zgh7AcNke4w=
|
||||
github.com/RoaringBitmap/roaring v0.4.17/go.mod h1:D3qVegWTmfCaX4Bl5CrBE9hfrSrrXIr8KVNvRsDi1NI=
|
||||
github.com/RoaringBitmap/roaring v0.4.23/go.mod h1:D0gp8kJQgE1A4LQ5wFLggQEyvDi06Mq5mKs52e1TwOo=
|
||||
@@ -45,6 +57,8 @@ github.com/STARRY-S/zip v0.2.3 h1:luE4dMvRPDOWQdeDdUxUoZkzUIpTccdKdhHHsQJ1fm4=
|
||||
github.com/STARRY-S/zip v0.2.3/go.mod h1:lqJ9JdeRipyOQJrYSOtpNAiaesFO6zVDsE8GIGFaoSk=
|
||||
github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo=
|
||||
github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI=
|
||||
github.com/acomagu/bufpipe v1.0.4 h1:e3H4WUzM3npvo5uv95QuJM3cQspFNtFBzvJ2oNjKIDQ=
|
||||
github.com/acomagu/bufpipe v1.0.4/go.mod h1:mxdxdup/WdsKVreO5GpW4+M/1CE2sMG4jeGJ2sYmHc4=
|
||||
github.com/alecthomas/assert/v2 v2.0.0-alpha3 h1:pcHeMvQ3OMstAWgaeaXIAL8uzB9xMm2zlxt+/4ml8lk=
|
||||
github.com/alecthomas/assert/v2 v2.0.0-alpha3/go.mod h1:+zD0lmDXTeQj7TgDgCt0ePWxb0hMC1G+PGTsTCv1B9o=
|
||||
github.com/alecthomas/atomic v0.1.0-alpha2 h1:dqwXmax66gXvHhsOS4pGPZKqYOlTkapELkLb3MNdlH8=
|
||||
@@ -141,6 +155,7 @@ github.com/bradfitz/iter v0.0.0-20140124041915-454541ec3da2/go.mod h1:PyRFw1Lt2w
|
||||
github.com/bradfitz/iter v0.0.0-20190303215204-33e6a9893b0c/go.mod h1:PyRFw1Lt2wKX4ZVSQ2mk+PeDa1rxyObEDlApuIsUKuo=
|
||||
github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8 h1:GKTyiRCL6zVf5wWaqKnf+7Qs6GbEPfd4iMOitWzXJx8=
|
||||
github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8/go.mod h1:spo1JLcs67NmW1aVLEgtA8Yy1elc+X8y5SRW1sFW4Og=
|
||||
github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0=
|
||||
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
|
||||
github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
|
||||
github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
|
||||
@@ -157,12 +172,11 @@ github.com/chzyer/readline v1.5.1/go.mod h1:Eh+b79XXUwfKfcPLepksvw2tcLE/Ct21YObk
|
||||
github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
|
||||
github.com/chzyer/test v1.0.0/go.mod h1:2JlltgoNkt4TW/z9V/IzDdFaMTM2JPIi26O1pF38GC8=
|
||||
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
|
||||
github.com/cloudflare/circl v1.6.3 h1:9GPOhQGF9MCYUeXyMYlqTR6a5gTrgR/fBLXvUgtVcg8=
|
||||
github.com/cloudflare/circl v1.6.3/go.mod h1:2eXP6Qfat4O/Yhh8BznvKnJ+uzEoTQ6jVKJRn81BiS4=
|
||||
github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA=
|
||||
github.com/cloudflare/circl v1.6.1 h1:zqIqSPIndyBh1bjLVVDHMPpVKqp8Su/V+6MeDzzQBQ0=
|
||||
github.com/cloudflare/circl v1.6.1/go.mod h1:uddAzsPgqdMAYatqJ0lsjX1oECcQLIlRpzZh3pJrofs=
|
||||
github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
|
||||
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
|
||||
github.com/cyphar/filepath-securejoin v0.6.1 h1:5CeZ1jPXEiYt3+Z6zqprSAgSWiggmpVyciv8syjIpVE=
|
||||
github.com/cyphar/filepath-securejoin v0.6.1/go.mod h1:A8hd4EnAeyujCJRrICiOWqjS1AX0a9kM5XL+NwKoYSc=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
@@ -185,14 +199,12 @@ github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1
|
||||
github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I=
|
||||
github.com/edsrzf/mmap-go v1.2.0 h1:hXLYlkbaPzt1SaQk+anYwKSRNhufIDCchSPkUD6dD84=
|
||||
github.com/edsrzf/mmap-go v1.2.0/go.mod h1:19H/e8pUPLicwkyNgOykDXkJ9F0MHE+Z52B8EIth78Q=
|
||||
github.com/elazarl/goproxy v1.7.2 h1:Y2o6urb7Eule09PjlhQRGNsqRfPmYI3KKQLFpCAV3+o=
|
||||
github.com/elazarl/goproxy v1.7.2/go.mod h1:82vkLNir0ALaW14Rc399OTTjyNREgmdL2cVoIbS6XaE=
|
||||
github.com/elazarl/goproxy v0.0.0-20221015165544-a0805db90819 h1:RIB4cRk+lBqKK3Oy0r2gRX4ui7tuhiZq2SuTtTCi0/0=
|
||||
github.com/elazarl/goproxy v0.0.0-20221015165544-a0805db90819/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM=
|
||||
github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc=
|
||||
github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ=
|
||||
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
|
||||
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
|
||||
github.com/fclairamb/ftpserverlib v0.31.0 h1:ws4fLJvuU/mgg1uIgRjCJfc6CrFSgIBAOxVsnawXegE=
|
||||
github.com/fclairamb/ftpserverlib v0.31.0/go.mod h1:C5hUT0sd9P5XsHMxsmpcrT6wdZexu3qtUu1z0Ih6aG0=
|
||||
github.com/felixge/fgprof v0.9.5 h1:8+vR6yu2vvSKn08urWyEuxx75NWPEvybbkBirEpsbVY=
|
||||
github.com/felixge/fgprof v0.9.5/go.mod h1:yKl+ERSa++RYOs32d8K6WEXCB4uXdLls4ZaZPpayhMM=
|
||||
github.com/felixge/httpsnoop v1.0.1 h1:lvB5Jl89CsZtGIWuTcDM1E/vkVs49/Ml7JJe07l8SPQ=
|
||||
@@ -202,8 +214,8 @@ github.com/frankban/quicktest v1.14.4/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7z
|
||||
github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
|
||||
github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
|
||||
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
|
||||
github.com/gliderlabs/ssh v0.3.8 h1:a4YXD1V7xMF9g5nTkdfnja3Sxy1PVDCj1Zg4Wb8vY6c=
|
||||
github.com/gliderlabs/ssh v0.3.8/go.mod h1:xYoytBv1sV0aL3CavoDuJIQNURXkkfPA/wxQ1pL1fAU=
|
||||
github.com/gliderlabs/ssh v0.3.5 h1:OcaySEmAQJgyYcArR+gGGTHCyE7nvhEMTlYY+Dp8CpY=
|
||||
github.com/gliderlabs/ssh v0.3.5/go.mod h1:8XB4KraRrX39qHhT6yxPsHedjA08I/uBVwj4xC+/+z4=
|
||||
github.com/glycerine/go-unsnap-stream v0.0.0-20180323001048-9f0cb55181dd/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE=
|
||||
github.com/glycerine/go-unsnap-stream v0.0.0-20181221182339-f9677308dec2/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE=
|
||||
github.com/glycerine/go-unsnap-stream v0.0.0-20190901134440-81cf024a9e0a/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE=
|
||||
@@ -212,12 +224,12 @@ github.com/glycerine/goconvey v0.0.0-20190315024820-982ee783a72e/go.mod h1:Ogl1T
|
||||
github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24=
|
||||
github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI=
|
||||
github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmSxCcxctByoQdvwPiA7DTK7jaaFDBTtu0ic=
|
||||
github.com/go-git/go-billy/v5 v5.9.2 h1:OXFSRyz4g20upsGDJgQG9Bak1l/ZEv8GHVYB52O71sE=
|
||||
github.com/go-git/go-billy/v5 v5.9.2/go.mod h1:ExsU+jcGwXTBOnyilvAnEM1wug1IxHr4yP2ZXsNRtV0=
|
||||
github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399 h1:eMje31YglSBqCdIqdhKBW8lokaMrL3uTkpGYlE2OOT4=
|
||||
github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399/go.mod h1:1OCfN199q1Jm3HZlxleg+Dw/mwps2Wbk9frAWm+4FII=
|
||||
github.com/go-git/go-git/v5 v5.19.2 h1:wkfn7vOlUBu8ivAWKBWisTiwJK4jYHzTF8Ndv1LyGqY=
|
||||
github.com/go-git/go-git/v5 v5.19.2/go.mod h1:QqCBE1EFN5ddFmrliLQ3/ntRCUjZU3EJuwuB/jWEHjk=
|
||||
github.com/go-git/go-billy/v5 v5.4.1 h1:Uwp5tDRkPr+l/TnbHOQzp+tmJfLceOlbVucgpTz8ix4=
|
||||
github.com/go-git/go-billy/v5 v5.4.1/go.mod h1:vjbugF6Fz7JIflbVpl1hJsGjSHNltrSw45YK/ukIvQg=
|
||||
github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20230305113008-0c11038e723f h1:Pz0DHeFij3XFhoBRGUDPzSJ+w2UcK5/0JvF8DRI58r8=
|
||||
github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20230305113008-0c11038e723f/go.mod h1:8LHG1a3SRW71ettAD/jW13h8c6AqjVSeL11RAdgaqpo=
|
||||
github.com/go-git/go-git/v5 v5.8.1 h1:Zo79E4p7TRk0xoRgMq0RShiTHGKcKI4+DI6BfJc/Q+A=
|
||||
github.com/go-git/go-git/v5 v5.8.1/go.mod h1:FHFuoD6yGz5OSKEBK+aWN9Oah0q54Jxl0abmj6GnqAo=
|
||||
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
|
||||
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
|
||||
github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
|
||||
@@ -238,6 +250,8 @@ github.com/go-quicktest/qt v1.101.0/go.mod h1:14Bz/f7NwaXPtdYEgzsx46kqSxVwTbzVZs
|
||||
github.com/go-sourcemap/sourcemap v2.1.4+incompatible h1:a+iTbH5auLKxaNwQFg0B+TCYl6lbukKPc7b5x0n1s6Q=
|
||||
github.com/go-sourcemap/sourcemap v2.1.4+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg=
|
||||
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
|
||||
github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI=
|
||||
github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8=
|
||||
github.com/gobwas/httphead v0.1.0/go.mod h1:O/RXo79gxV8G+RqlR/otEwx4Q36zl9rqC5u12GKvMCM=
|
||||
github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw=
|
||||
github.com/gobwas/ws v1.2.1/go.mod h1:hRKAFb8wOxFROYNsT1bqfWnhX+b5MFeJM9r2ZSwg/KY=
|
||||
@@ -250,8 +264,8 @@ github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfU
|
||||
github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
|
||||
github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
|
||||
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
|
||||
github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 h1:f+oWsMOmNPc8JmEHVZIycC7hBoQxHH9pNKQORJNozsQ=
|
||||
github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8/go.mod h1:wcDNUvekVysuuOpQKo3191zZyTpiI6se1N1ULghS0sw=
|
||||
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE=
|
||||
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
|
||||
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
|
||||
github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
|
||||
github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
|
||||
@@ -276,13 +290,10 @@ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5a
|
||||
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
|
||||
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
|
||||
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
|
||||
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
|
||||
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
|
||||
github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=
|
||||
github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=
|
||||
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
|
||||
github.com/google/jsonschema-go v0.4.3 h1:/DBOLZTfDow7pe2GmaJNhltueGTtDKICi8V8p+DQPd0=
|
||||
github.com/google/jsonschema-go v0.4.3/go.mod h1:r5quNTdLOYEz95Ru18zA0ydNbBuYoo9tgaYcxEYhJVE=
|
||||
@@ -310,6 +321,11 @@ github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
|
||||
github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
|
||||
github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg=
|
||||
github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
|
||||
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
|
||||
github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
|
||||
github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
|
||||
github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
|
||||
github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
|
||||
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
|
||||
github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
|
||||
@@ -327,12 +343,10 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:
|
||||
github.com/ianlancetaylor/demangle v0.0.0-20230524184225-eabc099b10ab/go.mod h1:gx7rwoVhcfuVKG5uya9Hs3Sxj7EIvldVofAWIUtGouw=
|
||||
github.com/icholy/digest v1.1.0 h1:HfGg9Irj7i+IX1o1QAmPfIBNu/Q5A5Tu3n/MED9k9H4=
|
||||
github.com/icholy/digest v1.1.0/go.mod h1:QNrsSGQ5v7v9cReDI0+eyjsXGUoRSUZQHeQ5C4XLa0Y=
|
||||
github.com/imroc/req/v3 v3.58.0 h1:rBqJhv2gZjMymGot4vKmNSUs/mAITMpGASqElhuMvS4=
|
||||
github.com/imroc/req/v3 v3.58.0/go.mod h1:rC7NTGgfMSkVWR4GO46rMeJsUfaeOpa6pLsgpq9q3zI=
|
||||
github.com/imroc/req/v3 v3.52.2 h1:xJocr1aIv0a2K9knfBQ4JnZHk+kWTITdjf0mgDg229I=
|
||||
github.com/imroc/req/v3 v3.52.2/go.mod h1:dBGsDloOSZJcFs6PnTjZXYBJK70OXbZpizHBLNqcH2k=
|
||||
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A=
|
||||
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo=
|
||||
github.com/jlaffaye/ftp v0.2.4 h1:JqI85DdkfZj8ntaHk8W9U2SC3jNfiPUU70+wtIWmlfE=
|
||||
github.com/jlaffaye/ftp v0.2.4/go.mod h1:Y1ZnkzxownGIuX7xQ1mQzzkZ21+DbjVIyeKL/V+IIz4=
|
||||
github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
|
||||
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
|
||||
github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
|
||||
@@ -345,8 +359,8 @@ github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4
|
||||
github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM=
|
||||
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
|
||||
github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
|
||||
github.com/klauspost/compress v1.18.2 h1:iiPHWW0YrcFgpBYhsA6D1+fqHssJscY/Tm/y2Uqnapk=
|
||||
github.com/klauspost/compress v1.18.2/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4=
|
||||
github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
|
||||
github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
|
||||
github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
|
||||
github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y=
|
||||
github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
|
||||
@@ -356,6 +370,7 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxv
|
||||
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
|
||||
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
|
||||
github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
|
||||
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
|
||||
github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
|
||||
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
|
||||
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
|
||||
@@ -368,6 +383,8 @@ github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJ
|
||||
github.com/matoous/go-nanoid v1.5.0/go.mod h1:zyD2a71IubI24efhpvkJz+ZwfwagzgSO6UNiFsZKN7U=
|
||||
github.com/matoous/go-nanoid/v2 v2.0.0 h1:d19kur2QuLeHmJBkvYkFdhFBzLoo1XVm2GgTpL+9Tj0=
|
||||
github.com/matoous/go-nanoid/v2 v2.0.0/go.mod h1:FtS4aGPVfEkxKxhdWPAspZpZSh1cOjtM7Ej/So3hR0g=
|
||||
github.com/matryer/is v1.2.0 h1:92UTHpy8CDwaJ08GqLDzhhuixiBUUD1p3AU6PHddz4A=
|
||||
github.com/matryer/is v1.2.0/go.mod h1:2fLPjFQM9rhQ15aVEtbuwhJinnOqrmgXPNdZsdwlWXA=
|
||||
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
|
||||
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
|
||||
github.com/mattn/go-ieproxy v0.0.12 h1:OZkUFJC3ESNZPQ+6LzC3VJIFSnreeFLQyqvBWtvfL2M=
|
||||
@@ -405,10 +422,13 @@ github.com/multiformats/go-varint v0.1.0/go.mod h1:5KVAVXegtfmNQQm/lCY+ATvDzvJJh
|
||||
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
|
||||
github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w=
|
||||
github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
|
||||
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
|
||||
github.com/nwaples/rardecode/v2 v2.2.0 h1:4ufPGHiNe1rYJxYfehALLjup4Ls3ck42CWwjKiOqu0A=
|
||||
github.com/nwaples/rardecode/v2 v2.2.0/go.mod h1:7uz379lSxPe6j9nvzxUZ+n7mnJNgjsRNb6IbvGVHRmw=
|
||||
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
||||
github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
||||
github.com/onsi/ginkgo/v2 v2.23.4 h1:ktYTpKJAVZnDT4VjxSbiBenUjmlL/5QkBEocaWXiQus=
|
||||
github.com/onsi/ginkgo/v2 v2.23.4/go.mod h1:Bt66ApGPBFzHyR+JO10Zbt0Gsp4uWxu5mIOTusL46e8=
|
||||
github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
|
||||
github.com/onsi/gomega v1.36.3 h1:hID7cr8t3Wp26+cYnfcjR6HpJ00fdogN6dqZ1t6IylU=
|
||||
github.com/onsi/gomega v1.36.3/go.mod h1:8D9+Txp43QWKhM24yyOBEdpkzN8FvJyAwecBgsU4KU0=
|
||||
@@ -420,8 +440,8 @@ github.com/pierrec/lz4/v4 v4.1.22 h1:cKFw6uJDK+/gfw5BcDL0JL5aBsAFdsIT18eRtLj7VIU
|
||||
github.com/pierrec/lz4/v4 v4.1.22/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
|
||||
github.com/pion/datachannel v1.6.0 h1:XecBlj+cvsxhAMZWFfFcPyUaDZtd7IJvrXqlXD/53i0=
|
||||
github.com/pion/datachannel v1.6.0/go.mod h1:ur+wzYF8mWdC+Mkis5Thosk+u/VOL287apDNEbFpsIk=
|
||||
github.com/pion/dtls/v3 v3.1.4 h1:QhvtMflMfu9Kf0RcDC5BJBle4caPskByrKQR6uuYqpY=
|
||||
github.com/pion/dtls/v3 v3.1.4/go.mod h1:cr/qotLISUw/9C1m83ZPNZtj9WnXkYLpfCptPqbkInc=
|
||||
github.com/pion/dtls/v3 v3.0.10 h1:k9ekkq1kaZoxnNEbyLKI8DI37j/Nbk1HWmMuywpQJgg=
|
||||
github.com/pion/dtls/v3 v3.0.10/go.mod h1:YEmmBYIoBsY3jmG56dsziTv/Lca9y4Om83370CXfqJ8=
|
||||
github.com/pion/ice/v4 v4.2.0 h1:jJC8S+CvXCCvIQUgx+oNZnoUpt6zwc34FhjWwCU4nlw=
|
||||
github.com/pion/ice/v4 v4.2.0/go.mod h1:EgjBGxDgmd8xB0OkYEVFlzQuEI7kWSCFu+mULqaisy4=
|
||||
github.com/pion/interceptor v0.1.42 h1:0/4tvNtruXflBxLfApMVoMubUMik57VZ+94U0J7cmkQ=
|
||||
@@ -442,18 +462,18 @@ github.com/pion/sdp/v3 v3.0.17 h1:9SfLAW/fF1XC8yRqQ3iWGzxkySxup4k4V7yN8Fs8nuo=
|
||||
github.com/pion/sdp/v3 v3.0.17/go.mod h1:9tyKzznud3qiweZcD86kS0ff1pGYB3VX+Bcsmkx6IXo=
|
||||
github.com/pion/srtp/v3 v3.0.10 h1:tFirkpBb3XccP5VEXLi50GqXhv5SKPxqrdlhDCJlZrQ=
|
||||
github.com/pion/srtp/v3 v3.0.10/go.mod h1:3mOTIB0cq9qlbn59V4ozvv9ClW/BSEbRp4cY0VtaR7M=
|
||||
github.com/pion/stun/v3 v3.1.5 h1:Y1FHlhaI6+4UoC5i/zQf4F7JvdZtB24/05oyy/GF1x8=
|
||||
github.com/pion/stun/v3 v3.1.5/go.mod h1:zRUghXSQU32Lx5orJsz3uYMkIihweXb3mu5gIns02fs=
|
||||
github.com/pion/stun/v3 v3.1.1 h1:CkQxveJ4xGQjulGSROXbXq94TAWu8gIX2dT+ePhUkqw=
|
||||
github.com/pion/stun/v3 v3.1.1/go.mod h1:qC1DfmcCTQjl9PBaMa5wSn3x9IPmKxSdcCsxBcDBndM=
|
||||
github.com/pion/transport/v3 v3.1.1 h1:Tr684+fnnKlhPceU+ICdrw6KKkTms+5qHMgw6bIkYOM=
|
||||
github.com/pion/transport/v3 v3.1.1/go.mod h1:+c2eewC5WJQHiAA46fkMMzoYZSuGzA/7E2FPrOYHctQ=
|
||||
github.com/pion/transport/v4 v4.0.2 h1:ifYlPqNwsy6aKQ9y8yzxXlHae5431ZrH2avkD/Rn6Tk=
|
||||
github.com/pion/transport/v4 v4.0.2/go.mod h1:06hFI+jCFcok2X2MekVufNZ/uzNZXivGBPfviSVcjgM=
|
||||
github.com/pion/transport/v4 v4.0.1 h1:sdROELU6BZ63Ab7FrOLn13M6YdJLY20wldXW2Cu2k8o=
|
||||
github.com/pion/transport/v4 v4.0.1/go.mod h1:nEuEA4AD5lPdcIegQDpVLgNoDGreqM/YqmEx3ovP4jM=
|
||||
github.com/pion/turn/v4 v4.1.4 h1:EU11yMXKIsK43FhcUnjLlrhE4nboHZq+TXBIi3QpcxQ=
|
||||
github.com/pion/turn/v4 v4.1.4/go.mod h1:ES1DXVFKnOhuDkqn9hn5VJlSWmZPaRJLyBXoOeO/BmQ=
|
||||
github.com/pion/webrtc/v4 v4.2.2-0.20260109001657-a5962f314db7 h1:gnkvpxCSzeTKovDsxeFFl+W/i6WydzSHhu5lH/RpFfM=
|
||||
github.com/pion/webrtc/v4 v4.2.2-0.20260109001657-a5962f314db7/go.mod h1:/td+qaV3q20qTbRjYV9ABWjmCkvt+Pog/S+NqLMoUKk=
|
||||
github.com/pjbgf/sha1cd v0.6.0 h1:3WJ8Wz8gvDz29quX1OcEmkAlUg9diU4GxJHqs0/XiwU=
|
||||
github.com/pjbgf/sha1cd v0.6.0/go.mod h1:lhpGlyHLpQZoxMv8HcgXvZEhcGs0PG/vsZnEJ7H0iCM=
|
||||
github.com/pjbgf/sha1cd v0.3.0 h1:4D5XXmUUBUl/xQ6IjCkEAbqXskkq/4O7LmGn0AqMDs4=
|
||||
github.com/pjbgf/sha1cd v0.3.0/go.mod h1:nZ1rrWOcGJ5uZgEEVL1VUM9iRQiZvWdbZjkKyFzPPsI=
|
||||
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ=
|
||||
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU=
|
||||
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
|
||||
@@ -461,7 +481,10 @@ github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE
|
||||
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
|
||||
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g=
|
||||
github.com/prashantv/gostub v1.1.0/go.mod h1:A5zLQHz7ieHGG7is6LLXLz7I8+3LZzsrV0P1IAHhP5U=
|
||||
github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
|
||||
github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs=
|
||||
github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
|
||||
@@ -481,13 +504,13 @@ github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+Gx
|
||||
github.com/prometheus/procfs v0.0.11/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
|
||||
github.com/protolambda/ctxlock v0.1.0 h1:rCUY3+vRdcdZXqT07iXgyr744J2DU2LCBIXowYAjBCE=
|
||||
github.com/protolambda/ctxlock v0.1.0/go.mod h1:vefhX6rIZH8rsg5ZpOJfEDYQOppZi19SfPiGOFrNnwM=
|
||||
github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8=
|
||||
github.com/quic-go/qpack v0.6.0/go.mod h1:lUpLKChi8njB4ty2bFLX2x4gzDqXwUpaO1DP9qMDZII=
|
||||
github.com/quic-go/quic-go v0.59.0 h1:OLJkp1Mlm/aS7dpKgTc6cnpynnD2Xg7C1pwL6vy/SAw=
|
||||
github.com/quic-go/quic-go v0.59.0/go.mod h1:upnsH4Ju1YkqpLXC305eW3yDZ4NfnNbmQRCMWS58IKU=
|
||||
github.com/quic-go/qpack v0.5.1 h1:giqksBPnT/HDtZ6VhtFKgoLOWmlyo9Ei6u9PqzIMbhI=
|
||||
github.com/quic-go/qpack v0.5.1/go.mod h1:+PC4XFrEskIVkcLzpEkbLqq1uCoxPhQuvK5rH1ZgaEg=
|
||||
github.com/quic-go/quic-go v0.51.0 h1:K8exxe9zXxeRKxaXxi/GpUqYiTrtdiWP8bo1KFya6Wc=
|
||||
github.com/quic-go/quic-go v0.51.0/go.mod h1:MFlGGpcpJqRAfmYi6NC2cptDPSxRWTOGNuP4wqrWmzQ=
|
||||
github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
|
||||
github.com/refraction-networking/utls v1.8.2 h1:j4Q1gJj0xngdeH+Ox/qND11aEfhpgoEvV+S9iJ2IdQo=
|
||||
github.com/refraction-networking/utls v1.8.2/go.mod h1:jkSOEkLqn+S/jtpEHPOsVv/4V4EVnelwbMQl4vCWXAM=
|
||||
github.com/refraction-networking/utls v1.6.7 h1:zVJ7sP1dJx/WtVuITug3qYUq034cDq9B2MR1K67ULZM=
|
||||
github.com/refraction-networking/utls v1.6.7/go.mod h1:BC3O4vQzye5hqpmDTWUqi4P5DDhzJfkV1tdqtawQIH0=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
|
||||
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
|
||||
@@ -504,19 +527,17 @@ github.com/rs/zerolog v1.31.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWR
|
||||
github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk=
|
||||
github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 h1:GHRpF1pTW19a8tTFrMLUcfWwyC0pnifVo2ClaLq+hP8=
|
||||
github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46/go.mod h1:uAQ5PCi+MFsC7HjREoAz1BU+Mq60+05gifQSsHSDG/8=
|
||||
github.com/secsy/goftp v0.0.0-20200609142545-aa2de14babf4 h1:PT+ElG/UUFMfqy5HrxJxNzj3QBOf7dZwupeVC+mG1Lo=
|
||||
github.com/secsy/goftp v0.0.0-20200609142545-aa2de14babf4/go.mod h1:MnkX001NG75g3p8bhFycnyIjeQoOjGL6CEIsdE/nKSY=
|
||||
github.com/segmentio/asm v1.1.3 h1:WM03sfUOENvvKexOLp+pCqgb/WDjsi7EK8gIsICtzhc=
|
||||
github.com/segmentio/asm v1.1.3/go.mod h1:Ld3L4ZXGNcSLRg4JBsZ3//1+f/TjYl0Mzen/DQy1EJg=
|
||||
github.com/segmentio/encoding v0.5.4 h1:OW1VRern8Nw6ITAtwSZ7Idrl3MXCFwXHPgqESYfvNt0=
|
||||
github.com/segmentio/encoding v0.5.4/go.mod h1:HS1ZKa3kSN32ZHVZ7ZLPLXWvOVIiZtyJnO1gPH1sKt0=
|
||||
github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8=
|
||||
github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4=
|
||||
github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0=
|
||||
github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
|
||||
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
|
||||
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
|
||||
github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
|
||||
github.com/skeema/knownhosts v1.3.1 h1:X2osQ+RAjK76shCbvhHHHVl3ZlgDm8apHEHFqRjnBY8=
|
||||
github.com/skeema/knownhosts v1.3.1/go.mod h1:r7KTdC8l4uxWRyK2TpQZ/1o5HaSzh06ePQNxPwTcfiY=
|
||||
github.com/skeema/knownhosts v1.2.0 h1:h9r9cf0+u7wSE+M183ZtMGgOJKiL96brpaz5ekfJCpM=
|
||||
github.com/skeema/knownhosts v1.2.0/go.mod h1:g4fPeYpque7P0xefxtGzV81ihjC8sX2IqpAoNkjxbMo=
|
||||
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
|
||||
github.com/smartystreets/assertions v0.0.0-20190215210624-980c5ac6f3ac/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
|
||||
github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s=
|
||||
@@ -543,8 +564,8 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
|
||||
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
|
||||
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
|
||||
github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE=
|
||||
github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg=
|
||||
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
|
||||
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
||||
github.com/tetratelabs/wazero v1.12.0 h1:DuWcpNu/FzgEXgGBDp8J1Spc+CWOvvtvVyjKlaZopYU=
|
||||
github.com/tetratelabs/wazero v1.12.0/go.mod h1:LvKtzl2RqO4gyF27BiXU+nKAjcV8f38U+kP/q2vgxh0=
|
||||
github.com/tidwall/btree v1.8.1 h1:27ehoXvm5AG/g+1VxLS1SD3vRhp/H7LuEfwNvddEdmA=
|
||||
@@ -585,12 +606,12 @@ go.opentelemetry.io/otel/metric v1.39.0 h1:d1UzonvEZriVfpNKEVmHXbdf909uGTOQjA0HF
|
||||
go.opentelemetry.io/otel/metric v1.39.0/go.mod h1:jrZSWL33sD7bBxg1xjrqyDjnuzTUB0x1nBERXd7Ftcs=
|
||||
go.opentelemetry.io/otel/trace v1.39.0 h1:2d2vfpEDmCJ5zVYz7ijaJdOF59xLomrvj7bjt6/qCJI=
|
||||
go.opentelemetry.io/otel/trace v1.39.0/go.mod h1:88w4/PnZSazkGzz/w84VHpQafiU4EtqqlVdxWy+rNOA=
|
||||
go.uber.org/automaxprocs v1.6.0 h1:O3y2/QNTOdbF+e/dpXNNW7Rx2hZ4sTIPyybbxyNqTUs=
|
||||
go.uber.org/automaxprocs v1.6.0/go.mod h1:ifeIMSnPZuznNm6jmdzmU3/bfk01Fe2fotchwEFJ8r8=
|
||||
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
|
||||
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
|
||||
go.uber.org/mock v0.6.0 h1:hyF9dfmbgIX5EfOdasqLsWD6xqpNZlXblLB/Dbnwv3Y=
|
||||
go.uber.org/mock v0.6.0/go.mod h1:KiVJ4BqZJaMj4svdfmHM0AUx4NJYO8ZNpPnZn1Z+BBU=
|
||||
go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw=
|
||||
go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg=
|
||||
go.uber.org/mock v0.5.1 h1:ASgazW/qBmR+A32MYFDB6E2POoTgOwT509VP0CT/fjs=
|
||||
go.uber.org/mock v0.5.1/go.mod h1:ge71pBPLYDk7QIi1LupWxdAykm7KIEFchiOqd6z7qMM=
|
||||
go4.org v0.0.0-20230225012048-214862532bf5 h1:nifaUDeh+rPaBCMPMQHZmvJf+QdpLFnuQPwx+LxVmtc=
|
||||
go4.org v0.0.0-20230225012048-214862532bf5/go.mod h1:F57wTi5Lrj6WLyswp5EYV1ncrEbFGHD4hhz6S1ZYeaU=
|
||||
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
@@ -600,9 +621,11 @@ golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8U
|
||||
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
||||
golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
|
||||
golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4=
|
||||
golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU=
|
||||
golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
|
||||
golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M=
|
||||
golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis=
|
||||
golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts=
|
||||
golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos=
|
||||
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
|
||||
@@ -612,8 +635,8 @@ golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u0
|
||||
golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
|
||||
golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
|
||||
golang.org/x/exp v0.0.0-20220428152302-39d4317da171/go.mod h1:lgLbSvA5ygNOMpwM/9anMpWVlVJ7Z+cHWq/eFuinpGE=
|
||||
golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f h1:W3F4c+6OLc6H2lb//N1q4WpJkhzJCK5J6kUi1NTVXfM=
|
||||
golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f/go.mod h1:J1xhfL/vlindoeF/aINzNzt2Bket5bjo9sdOYzOsU80=
|
||||
golang.org/x/exp v0.0.0-20251219203646-944ab1f22d93 h1:fQsdNF2N+/YewlRZiricy4P1iimyPKZ/xwniHj8Q2a0=
|
||||
golang.org/x/exp v0.0.0-20251219203646-944ab1f22d93/go.mod h1:EPRbTFwzwjXj9NpYyyrvenVh9Y+GFeEvMNh7Xuz7xgU=
|
||||
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
|
||||
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
|
||||
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
@@ -635,8 +658,8 @@ golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro=
|
||||
golang.org/x/mod v0.6.0-dev.0.20211013180041-c96bc1413d57/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY=
|
||||
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
|
||||
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||
golang.org/x/mod v0.38.0 h1:MECBjubtXD7yj4HrhIUcywNaGeNVUdfVnxmPajOk4yk=
|
||||
golang.org/x/mod v0.38.0/go.mod h1:V6Xz0pq8TQ3dGqVQ1FVHuelZpAL0uNhSkk9ogYP3c40=
|
||||
golang.org/x/mod v0.33.0 h1:tHFzIWbBifEmbwtGz65eaWyGiGZatSrT9prnU8DbVL8=
|
||||
golang.org/x/mod v0.33.0/go.mod h1:swjeQEj+6r7fODbD2cqrnje9PnziFuw4bmLbBZFrQ5w=
|
||||
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
@@ -658,12 +681,14 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v
|
||||
golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
|
||||
golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY=
|
||||
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
|
||||
golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
|
||||
golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
|
||||
golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
|
||||
golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
|
||||
golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To=
|
||||
golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU=
|
||||
golang.org/x/net v0.50.0 h1:ucWh9eiCGyDR3vtzso0WMQinm2Dnt8cFMuQa9K33J60=
|
||||
golang.org/x/net v0.50.0/go.mod h1:UgoSli3F/pBgdJBHCTc+tp3gmrU4XswgGRgtnwWTfyM=
|
||||
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
@@ -680,8 +705,8 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ
|
||||
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek=
|
||||
golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
|
||||
golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
|
||||
golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
|
||||
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
@@ -714,20 +739,26 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc
|
||||
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
|
||||
golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k=
|
||||
golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
|
||||
golang.org/x/sys v0.44.0 h1:ildZl3J4uzeKP07r2F++Op7E9B29JRUy+a27EibtBTQ=
|
||||
golang.org/x/sys v0.44.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc=
|
||||
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
|
||||
golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U=
|
||||
golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
|
||||
golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U=
|
||||
golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0=
|
||||
golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w=
|
||||
golang.org/x/term v0.40.0 h1:36e4zGLqU4yhjlmxEaagx2KuYbJq3EwY8K943ZsHcvg=
|
||||
golang.org/x/term v0.40.0/go.mod h1:w2P8uVp06p2iyKKuvXIm7N/y0UCRt3UfJTfZ7oOpglM=
|
||||
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
@@ -735,11 +766,13 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
|
||||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
|
||||
golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
|
||||
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
|
||||
golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
|
||||
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
|
||||
golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
|
||||
golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8=
|
||||
golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M=
|
||||
golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk=
|
||||
golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA=
|
||||
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
|
||||
@@ -772,8 +805,8 @@ golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapK
|
||||
golang.org/x/tools v0.1.8-0.20211029000441-d6a9af8af023/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU=
|
||||
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
|
||||
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
|
||||
golang.org/x/tools v0.48.0 h1:3+hClM1aLL5mjMKm5ovokw9epgRXPuu2tILgismM6RE=
|
||||
golang.org/x/tools v0.48.0/go.mod h1:08xX0orndb/F7jJxGDicx061tyd5pcMto75YMAXr6lk=
|
||||
golang.org/x/tools v0.42.0 h1:uNgphsn75Tdz5Ji2q36v/nsFSfR/9BRFvqhGBaJGd5k=
|
||||
golang.org/x/tools v0.42.0/go.mod h1:Ma6lCIwGZvHK6XtgbswSoWroEkhugApmsXyrUmBhfr0=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
@@ -818,6 +851,8 @@ google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ
|
||||
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
|
||||
google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
|
||||
google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
|
||||
google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE=
|
||||
google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
|
||||
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
|
||||
@@ -3,7 +3,6 @@ package controller
|
||||
import (
|
||||
"github.com/GopeedLab/gopeed/internal/tempfiles"
|
||||
"github.com/GopeedLab/gopeed/pkg/base"
|
||||
"github.com/rs/zerolog"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
@@ -16,9 +15,6 @@ type Controller struct {
|
||||
TempDir string
|
||||
GetConfig func(v any)
|
||||
GetProxy func(requestProxy *base.RequestProxy) func(*http.Request) (*url.URL, error)
|
||||
// Logger is the downloader's core logger. It may be nil, in which case
|
||||
// fetchers log nothing.
|
||||
Logger *zerolog.Logger
|
||||
FileController
|
||||
//ContextDialer() (proxy.Dialer, error)
|
||||
}
|
||||
|
||||
@@ -12,7 +12,7 @@ import (
|
||||
// Each download task will have a corresponding Fetcher instance for the management of the download task
|
||||
type Fetcher interface {
|
||||
Setup(ctl *controller.Controller)
|
||||
Resolve(req *base.Request) error
|
||||
Resolve(req *base.Request, opts *base.Options) error
|
||||
Start() error
|
||||
// Patch modifies task-specific data based on the protocol.
|
||||
// For HTTP: can modify Request info (URL, headers, etc.)
|
||||
|
||||
@@ -1,72 +0,0 @@
|
||||
package fetcher
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"time"
|
||||
)
|
||||
|
||||
type TriggerSource string
|
||||
|
||||
const (
|
||||
TriggerScheduled TriggerSource = "scheduled"
|
||||
TriggerManual TriggerSource = "manual"
|
||||
)
|
||||
|
||||
var ErrJobSkipped = errors.New("protocol job skipped")
|
||||
var ErrConfigChanged = errors.New("protocol configuration changed during execution")
|
||||
|
||||
// ConfigState embeds the common backend-owned state section in a protocol config.
|
||||
// Public config PUTs always retain this section from the latest stored config.
|
||||
type ConfigState[T any] struct {
|
||||
State T `json:"state"`
|
||||
}
|
||||
|
||||
// ProtocolHost is scoped to one manager. UpdateState runs against the latest
|
||||
// config under Downloader's lock; callbacks must not perform network I/O.
|
||||
type ProtocolHost struct {
|
||||
Read func(any) error
|
||||
UpdateState func(func(config, state json.RawMessage) (any, error)) error
|
||||
Fetch func(context.Context, string, int64, func([]byte) error) ([]byte, error)
|
||||
}
|
||||
|
||||
func ReadConfig[T any](host ProtocolHost) (*T, error) {
|
||||
var cfg T
|
||||
if err := host.Read(&cfg); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &cfg, nil
|
||||
}
|
||||
|
||||
// UpdateState gives the callback a detached config snapshot and mutable state.
|
||||
// Only the state is committed; edits to the config snapshot are discarded.
|
||||
func UpdateState[C, S any](host ProtocolHost, update func(*C, *S) error) error {
|
||||
return host.UpdateState(func(rawConfig, rawState json.RawMessage) (any, error) {
|
||||
var cfg C
|
||||
var state S
|
||||
if err := json.Unmarshal(rawConfig, &cfg); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := json.Unmarshal(rawState, &state); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := update(&cfg, &state); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &state, nil
|
||||
})
|
||||
}
|
||||
|
||||
type CronJob struct {
|
||||
Name string
|
||||
Every time.Duration
|
||||
Timeout time.Duration
|
||||
Run func(context.Context, TriggerSource) error
|
||||
}
|
||||
type CronProvider interface{ CronJobs(ProtocolHost) []CronJob }
|
||||
|
||||
// ConfigNormalizer is optional. It owns protocol validation and derived fields.
|
||||
type ConfigNormalizer interface {
|
||||
NormalizeConfig(json.RawMessage) (any, error)
|
||||
}
|
||||
@@ -1,73 +0,0 @@
|
||||
# GitHub mirror execution
|
||||
|
||||
This package owns GitHub URL rewriting and execution strategies. It depends only
|
||||
on the standard library and does not depend on Downloader, HTTP clients, Git, or
|
||||
storage. Downloader persists `Config` at its top-level `githubMirror` field;
|
||||
`pkg/base` preserves the existing configuration names through aliases.
|
||||
|
||||
## Race: resource reads
|
||||
|
||||
```go
|
||||
body, err := githubmirror.Race(ctx, cfg.GithubMirror, rawURL,
|
||||
githubmirror.Options[[]byte]{
|
||||
Kind: githubmirror.Resource,
|
||||
Timeout: 30 * time.Second,
|
||||
},
|
||||
func(ctx context.Context, candidate string) ([]byte, error) {
|
||||
return fetchAndValidate(ctx, client, candidate)
|
||||
},
|
||||
)
|
||||
```
|
||||
|
||||
The callback owns HTTP requests, size limits, and content validation. The first
|
||||
completed valid result wins. Other operations are canceled and drained. There is
|
||||
no HEAD-probe assumption or client-specific HTTP behavior inside the executor.
|
||||
|
||||
## Fallback: repository operations
|
||||
|
||||
```go
|
||||
repo, err := githubmirror.Fallback(ctx, cfg.GithubMirror, officialURL,
|
||||
githubmirror.Options[*git.Repository]{
|
||||
Kind: githubmirror.Repository,
|
||||
},
|
||||
func(ctx context.Context, candidate string) (*git.Repository, error) {
|
||||
// Use a clean temporary checkout for each attempt.
|
||||
return clone(ctx, candidate)
|
||||
},
|
||||
)
|
||||
```
|
||||
|
||||
The official URL is attempted first; mirrors follow their configured order.
|
||||
jsDelivr is excluded for repositories. Stop on the first success. Perform
|
||||
persistent installation or other business state changes after success, rather
|
||||
than inside a potentially repeated attempt.
|
||||
|
||||
## Results with resources
|
||||
|
||||
For temporary files, set `Options[*Download].Discard` to remove the file. Return
|
||||
the partial `*Download` alongside errors, so cleanup also covers failed reads or
|
||||
validation. The executor discards every failure and every race loser exactly
|
||||
once. The caller owns and must track/release the winning result.
|
||||
|
||||
- `Timeout == 0` uses 30 seconds per resource attempt and no overall deadline
|
||||
for repository attempts; a negative timeout is invalid. Extension Git HTTP
|
||||
transport limits connections/TLS handshakes to 10 seconds and resets a 60-second
|
||||
read deadline as data arrives. Slow continuous transfers can finish, while
|
||||
stalled sources fail and allow ordered fallback. Parent cancellation still applies.
|
||||
- Attempts must honor the supplied context; draining waits for them to return.
|
||||
- Attempt contexts are canceled after return, so callbacks must finish reading
|
||||
HTTP bodies before returning. Returning a still-streaming response is unsupported.
|
||||
- Discard callbacks must accept zero values, must not panic, and run serially.
|
||||
- Parent cancellation wins over a result selected while the parent is canceled;
|
||||
the selected result is then discarded as well.
|
||||
- Errors preserve their causes through wrapping/joining for `errors.Is`.
|
||||
- `github.com` and `raw.githubusercontent.com` expand into eligible mirrors.
|
||||
jsDelivr is only eligible for raw source resources. `api.github.com` is never
|
||||
mirrored and is rejected by the public mirror candidates API.
|
||||
- Client application update checks request the official GitHub API directly,
|
||||
falling back to the independent Gopeed website API on request or validation
|
||||
failure. Only package downloads obtain URLs from
|
||||
`POST /api/v1/mirrors/github/candidates`. Candidate generation never performs
|
||||
HTTP requests.
|
||||
- Disabled mirrors and other URLs execute only the original URL.
|
||||
- Config is a caller-owned immutable snapshot for the duration of a call.
|
||||
@@ -1,90 +0,0 @@
|
||||
// Package githubmirror rewrites GitHub URLs and executes mirrored operations.
|
||||
package githubmirror
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/url"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type Mirror struct {
|
||||
Type string `json:"type"`
|
||||
URL string `json:"url"`
|
||||
IsBuiltIn bool `json:"isBuiltIn"`
|
||||
IsDeleted bool `json:"isDeleted"`
|
||||
}
|
||||
type Config struct {
|
||||
Enabled bool `json:"enabled"`
|
||||
Mirrors []Mirror `json:"mirrors"`
|
||||
}
|
||||
|
||||
func DefaultConfig() *Config {
|
||||
return &Config{Enabled: true, Mirrors: []Mirror{
|
||||
{Type: "jsdelivr", URL: "https://fastly.jsdelivr.net/gh", IsBuiltIn: true},
|
||||
{Type: "ghProxy", URL: "https://fastgit.cc", IsBuiltIn: true},
|
||||
{Type: "ghProxy", URL: "https://gh-proxy.com", IsBuiltIn: true},
|
||||
{Type: "ghProxy", URL: "https://ghfast.top", IsBuiltIn: true},
|
||||
}}
|
||||
}
|
||||
func (c *Config) Validate() error {
|
||||
if len(c.Mirrors) > 32 {
|
||||
return fmt.Errorf("too many GitHub mirrors")
|
||||
}
|
||||
for _, mirror := range c.Mirrors {
|
||||
u, err := url.Parse(mirror.URL)
|
||||
if err != nil || u == nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" || u.User != nil || u.RawQuery != "" || u.Fragment != "" {
|
||||
return fmt.Errorf("invalid GitHub mirror URL")
|
||||
}
|
||||
if mirror.Type != "jsdelivr" && mirror.Type != "ghProxy" {
|
||||
return fmt.Errorf("invalid GitHub mirror type: %s", mirror.Type)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Kind describes what a candidate endpoint must support.
|
||||
type Kind uint8
|
||||
|
||||
const (
|
||||
Resource Kind = iota
|
||||
Repository
|
||||
)
|
||||
|
||||
// Candidates preserves the official URL first. jsDelivr is only eligible for
|
||||
// raw source resources; Git operations use mirrors supporting Git Smart HTTP.
|
||||
// GitHub API URLs are never mirrored.
|
||||
func Candidates(c *Config, raw string, kind Kind) []string {
|
||||
result := []string{raw}
|
||||
u, err := url.Parse(raw)
|
||||
if err != nil || c == nil || !c.Enabled || u.User != nil || (u.Scheme != "https" && u.Scheme != "http") {
|
||||
return result
|
||||
}
|
||||
if u.Host != "github.com" && u.Host != "raw.githubusercontent.com" {
|
||||
return result
|
||||
}
|
||||
seen := map[string]bool{raw: true}
|
||||
for _, mirror := range c.Mirrors {
|
||||
if mirror.IsDeleted {
|
||||
continue
|
||||
}
|
||||
var candidate string
|
||||
switch mirror.Type {
|
||||
case "ghProxy":
|
||||
candidate = strings.TrimRight(mirror.URL, "/") + "/" + raw
|
||||
case "jsdelivr":
|
||||
if kind == Repository || u.Host != "raw.githubusercontent.com" {
|
||||
continue
|
||||
}
|
||||
parts := strings.SplitN(strings.TrimPrefix(u.EscapedPath(), "/"), "/", 4)
|
||||
if len(parts) != 4 {
|
||||
continue
|
||||
}
|
||||
candidate = strings.TrimRight(mirror.URL, "/") + "/" + parts[0] + "/" + parts[1] + "@" + parts[2] + "/" + parts[3]
|
||||
}
|
||||
if candidate != "" && !seen[candidate] {
|
||||
result = append(result, candidate)
|
||||
seen[candidate] = true
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
@@ -1,38 +0,0 @@
|
||||
package githubmirror
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestMirrorCandidates(t *testing.T) {
|
||||
cfg := &Config{Enabled: true, Mirrors: []Mirror{
|
||||
{Type: "jsdelivr", URL: "https://cdn.example/gh/"},
|
||||
{Type: "ghProxy", URL: "https://mirror.example/"},
|
||||
{Type: "ghProxy", URL: "https://deleted.example", IsDeleted: true},
|
||||
}}
|
||||
source := "https://raw.githubusercontent.com/owner/repo/main/a.txt"
|
||||
want := []string{source, "https://cdn.example/gh/owner/repo@main/a.txt", "https://mirror.example/" + source}
|
||||
if got := Candidates(cfg, source, Resource); !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("got %v", got)
|
||||
}
|
||||
repo := "https://github.com/owner/repo.git"
|
||||
if got := Candidates(cfg, repo, Repository); !reflect.DeepEqual(got, []string{repo, "https://mirror.example/" + repo}) {
|
||||
t.Fatalf("git candidates %v", got)
|
||||
}
|
||||
for _, raw := range []string{"https://github.com.evil.example/a", "https://example.com/a", "https://user:[email protected]/a"} {
|
||||
if got := Candidates(cfg, raw, Resource); !reflect.DeepEqual(got, []string{raw}) {
|
||||
t.Fatalf("unexpected mirror: %v", got)
|
||||
}
|
||||
}
|
||||
apiURL := "https://api.github.com/repos/GopeedLab/gopeed/releases?per_page=10"
|
||||
for _, kind := range []Kind{Resource, Repository} {
|
||||
if got := Candidates(cfg, apiURL, kind); !reflect.DeepEqual(got, []string{apiURL}) {
|
||||
t.Fatalf("GitHub API must never use mirrors: %v", got)
|
||||
}
|
||||
}
|
||||
cfg.Enabled = false
|
||||
if got := Candidates(cfg, source, Resource); !reflect.DeepEqual(got, []string{source}) {
|
||||
t.Fatalf("disabled mirror: %v", got)
|
||||
}
|
||||
}
|
||||
@@ -1,160 +0,0 @@
|
||||
package githubmirror
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Options configures execution, not URL persistence. Discard releases results
|
||||
// from failed attempts and successful race losers, including partial results
|
||||
// returned together with an error. The caller owns the winning result.
|
||||
type Options[T any] struct {
|
||||
Kind Kind
|
||||
// Timeout limits each attempt. Zero uses 30 seconds for resources and no
|
||||
// overall deadline for repositories, whose transport owns idle timeouts.
|
||||
Timeout time.Duration
|
||||
Discard func(T)
|
||||
}
|
||||
|
||||
// Attempt must honor ctx and return a completed, validated result. Its context
|
||||
// is canceled after it returns, so live response streams must not be returned.
|
||||
// Return any partially acquired resources along with errors so Discard can
|
||||
// release them. Failed attempts must not mutate persistent business state.
|
||||
type Attempt[T any] func(context.Context, string) (T, error)
|
||||
|
||||
const defaultTimeout = 30 * time.Second
|
||||
|
||||
func prepare[T any](ctx context.Context, config *Config, raw string, options Options[T], attempt Attempt[T]) ([]string, time.Duration, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
if raw == "" {
|
||||
return nil, 0, errors.New("empty GitHub source URL")
|
||||
}
|
||||
if options.Kind != Resource && options.Kind != Repository {
|
||||
return nil, 0, errors.New("invalid GitHub resource kind")
|
||||
}
|
||||
if options.Timeout < 0 {
|
||||
return nil, 0, errors.New("invalid GitHub attempt timeout")
|
||||
}
|
||||
if attempt == nil {
|
||||
return nil, 0, errors.New("GitHub mirror attempt is required")
|
||||
}
|
||||
if config != nil {
|
||||
if err := config.Validate(); err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
}
|
||||
timeout := options.Timeout
|
||||
if timeout == 0 && options.Kind == Resource {
|
||||
timeout = defaultTimeout
|
||||
}
|
||||
return Candidates(config, raw, options.Kind), timeout, nil
|
||||
}
|
||||
func invoke[T any](ctx context.Context, timeout time.Duration, candidate string, attempt Attempt[T]) (value T, err error) {
|
||||
var cancel context.CancelFunc
|
||||
if timeout > 0 {
|
||||
ctx, cancel = context.WithTimeout(ctx, timeout)
|
||||
} else {
|
||||
ctx, cancel = context.WithCancel(ctx)
|
||||
}
|
||||
defer cancel()
|
||||
defer func() {
|
||||
if recovered := recover(); recovered != nil {
|
||||
err = fmt.Errorf("GitHub mirror attempt panicked: %v", recovered)
|
||||
}
|
||||
}()
|
||||
value, err = attempt(ctx, candidate)
|
||||
if err == nil {
|
||||
err = ctx.Err()
|
||||
}
|
||||
return
|
||||
}
|
||||
func discard[T any](options Options[T], value T) {
|
||||
if options.Discard != nil {
|
||||
options.Discard(value)
|
||||
}
|
||||
}
|
||||
|
||||
// Race executes the official URL and eligible mirrors concurrently. The first
|
||||
// completed successful result wins; losing attempts are canceled and drained
|
||||
// before return so no temporary files or other resources outlive the call.
|
||||
// HTTP status/content validation is the attempt's responsibility.
|
||||
func Race[T any](ctx context.Context, config *Config, raw string, options Options[T], attempt Attempt[T]) (T, error) {
|
||||
var zero T
|
||||
candidates, timeout, err := prepare(ctx, config, raw, options, attempt)
|
||||
if err != nil {
|
||||
return zero, err
|
||||
}
|
||||
raceCtx, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
type outcome struct {
|
||||
index int
|
||||
value T
|
||||
err error
|
||||
}
|
||||
outcomes := make(chan outcome, len(candidates))
|
||||
for index, candidate := range candidates {
|
||||
go func(index int, candidate string) {
|
||||
value, err := invoke(raceCtx, timeout, candidate, attempt)
|
||||
outcomes <- outcome{index, value, err}
|
||||
}(index, candidate)
|
||||
}
|
||||
var winner T
|
||||
won := false
|
||||
failures := make([]error, len(candidates))
|
||||
for range candidates {
|
||||
outcome := <-outcomes
|
||||
if !won && outcome.err == nil && ctx.Err() == nil {
|
||||
winner = outcome.value
|
||||
won = true
|
||||
cancel()
|
||||
} else {
|
||||
discard(options, outcome.value)
|
||||
if outcome.err != nil {
|
||||
failures[outcome.index] = fmt.Errorf("GitHub source %d: %w", outcome.index+1, outcome.err)
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
cancel()
|
||||
}
|
||||
}
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
if won {
|
||||
discard(options, winner)
|
||||
}
|
||||
return zero, err
|
||||
}
|
||||
if won {
|
||||
return winner, nil
|
||||
}
|
||||
return zero, fmt.Errorf("all GitHub sources failed: %w", errors.Join(failures...))
|
||||
}
|
||||
|
||||
// Fallback executes the official URL first, then eligible mirrors in their
|
||||
// configured order. It stops immediately on success and cleans every failure.
|
||||
func Fallback[T any](ctx context.Context, config *Config, raw string, options Options[T], attempt Attempt[T]) (T, error) {
|
||||
var zero T
|
||||
candidates, timeout, err := prepare(ctx, config, raw, options, attempt)
|
||||
if err != nil {
|
||||
return zero, err
|
||||
}
|
||||
failures := make([]error, 0, len(candidates))
|
||||
for index, candidate := range candidates {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return zero, err
|
||||
}
|
||||
value, err := invoke(ctx, timeout, candidate, attempt)
|
||||
if err == nil && ctx.Err() == nil {
|
||||
return value, nil
|
||||
}
|
||||
discard(options, value)
|
||||
if parentErr := ctx.Err(); parentErr != nil {
|
||||
return zero, parentErr
|
||||
}
|
||||
failures = append(failures, fmt.Errorf("GitHub source %d: %w", index+1, err))
|
||||
}
|
||||
return zero, fmt.Errorf("all GitHub sources failed: %w", errors.Join(failures...))
|
||||
}
|
||||
@@ -1,214 +0,0 @@
|
||||
package githubmirror
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"reflect"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
const testSource = "https://github.com/owner/repo.git"
|
||||
|
||||
func testConfig() *Config {
|
||||
return &Config{Enabled: true, Mirrors: []Mirror{
|
||||
{Type: "ghProxy", URL: "https://first.example"},
|
||||
{Type: "ghProxy", URL: "https://second.example"},
|
||||
}}
|
||||
}
|
||||
func TestRaceWinsAndDrainsOwnedResults(t *testing.T) {
|
||||
config := testConfig()
|
||||
candidates := Candidates(config, testSource, Resource)
|
||||
var started sync.WaitGroup
|
||||
started.Add(len(candidates))
|
||||
discarded := []string{}
|
||||
winner, err := Race(context.Background(), config, testSource, Options[string]{
|
||||
Discard: func(value string) { discarded = append(discarded, value) },
|
||||
}, func(ctx context.Context, address string) (string, error) {
|
||||
started.Done()
|
||||
started.Wait()
|
||||
switch address {
|
||||
case candidates[0]:
|
||||
return address, errors.New("official offline")
|
||||
case candidates[1]:
|
||||
return address, nil
|
||||
default:
|
||||
<-ctx.Done()
|
||||
return address, ctx.Err()
|
||||
}
|
||||
})
|
||||
if err != nil || winner != candidates[1] {
|
||||
t.Fatalf("winner=%s err=%v", winner, err)
|
||||
}
|
||||
if len(discarded) != 2 || !contains(discarded, candidates[0]) || !contains(discarded, candidates[2]) {
|
||||
t.Fatalf("discarded %v", discarded)
|
||||
}
|
||||
}
|
||||
func contains(values []string, value string) bool {
|
||||
for _, item := range values {
|
||||
if item == value {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
func TestRaceCleansSimultaneousSuccessfulLosers(t *testing.T) {
|
||||
config := testConfig()
|
||||
var started sync.WaitGroup
|
||||
started.Add(3)
|
||||
cleaned := []string{}
|
||||
winner, err := Race(context.Background(), config, testSource, Options[string]{Discard: func(value string) { cleaned = append(cleaned, value) }}, func(_ context.Context, address string) (string, error) {
|
||||
started.Done()
|
||||
started.Wait()
|
||||
return address, nil
|
||||
})
|
||||
if err != nil || winner == "" || len(cleaned) != 2 || contains(cleaned, winner) {
|
||||
t.Fatalf("winner=%s discarded=%v err=%v", winner, cleaned, err)
|
||||
}
|
||||
}
|
||||
func TestRaceParentCancellationCleansWinnerToo(t *testing.T) {
|
||||
config := testConfig()
|
||||
config.Mirrors = config.Mirrors[:1]
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
cleaned := []string{}
|
||||
winner, err := Race(ctx, config, testSource, Options[string]{Discard: func(value string) { cleaned = append(cleaned, value) }}, func(ctx context.Context, address string) (string, error) {
|
||||
if address == testSource {
|
||||
return address, nil
|
||||
}
|
||||
<-ctx.Done()
|
||||
cancel()
|
||||
return address, ctx.Err()
|
||||
})
|
||||
if !errors.Is(err, context.Canceled) || winner != "" || len(cleaned) != 2 || !contains(cleaned, testSource) {
|
||||
t.Fatalf("winner=%s cleanup=%v err=%v", winner, cleaned, err)
|
||||
}
|
||||
}
|
||||
func TestRaceTimeoutAndErrorAggregation(t *testing.T) {
|
||||
expected := errors.New("offline")
|
||||
config := testConfig()
|
||||
config.Mirrors = config.Mirrors[:1]
|
||||
discarded := 0
|
||||
_, err := Race(context.Background(), config, testSource, Options[string]{Timeout: 10 * time.Millisecond, Discard: func(string) { discarded++ }}, func(ctx context.Context, address string) (string, error) {
|
||||
if address == testSource {
|
||||
return address, expected
|
||||
}
|
||||
<-ctx.Done()
|
||||
return address, ctx.Err()
|
||||
})
|
||||
if !errors.Is(err, expected) || !errors.Is(err, context.DeadlineExceeded) || discarded != 2 {
|
||||
t.Fatalf("discarded=%d err=%v", discarded, err)
|
||||
}
|
||||
}
|
||||
func TestFallbackPreservesOrderAndStopsOnSuccess(t *testing.T) {
|
||||
config := testConfig()
|
||||
candidates := Candidates(config, testSource, Repository)
|
||||
tried := []string{}
|
||||
cleaned := []string{}
|
||||
winner, err := Fallback(context.Background(), config, testSource, Options[string]{Kind: Repository, Discard: func(value string) { cleaned = append(cleaned, value) }}, func(_ context.Context, address string) (string, error) {
|
||||
tried = append(tried, address)
|
||||
if address == candidates[1] {
|
||||
return address, nil
|
||||
}
|
||||
return address, errors.New("offline")
|
||||
})
|
||||
if err != nil || winner != candidates[1] || !reflect.DeepEqual(tried, candidates[:2]) || !reflect.DeepEqual(cleaned, candidates[:1]) {
|
||||
t.Fatalf("winner=%s tried=%v discarded=%v err=%v", winner, tried, cleaned, err)
|
||||
}
|
||||
tried = nil
|
||||
_, err = Fallback(context.Background(), config, testSource, Options[string]{Kind: Repository}, func(_ context.Context, address string) (string, error) {
|
||||
tried = append(tried, address)
|
||||
return address, nil
|
||||
})
|
||||
if err != nil || !reflect.DeepEqual(tried, candidates[:1]) {
|
||||
t.Fatalf("tried=%v err=%v", tried, err)
|
||||
}
|
||||
}
|
||||
func TestFallbackTimeoutContinuesWithFreshContext(t *testing.T) {
|
||||
discarded := []string{}
|
||||
winner, err := Fallback(context.Background(), testConfig(), testSource, Options[string]{Kind: Repository, Timeout: 10 * time.Millisecond, Discard: func(value string) { discarded = append(discarded, value) }}, func(ctx context.Context, address string) (string, error) {
|
||||
if address == testSource {
|
||||
<-ctx.Done()
|
||||
return address, ctx.Err()
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return address, err
|
||||
}
|
||||
return address, nil
|
||||
})
|
||||
if err != nil || !strings.HasPrefix(winner, "https://first.example/") || !reflect.DeepEqual(discarded, []string{testSource}) {
|
||||
t.Fatalf("winner=%s cleanup=%v err=%v", winner, discarded, err)
|
||||
}
|
||||
}
|
||||
func TestFallbackCancellationStopsBeforeNextMirror(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
calls := 0
|
||||
cleaned := 0
|
||||
_, err := Fallback(ctx, testConfig(), testSource, Options[string]{Kind: Repository, Discard: func(string) { cleaned++ }}, func(ctx context.Context, address string) (string, error) {
|
||||
calls++
|
||||
cancel()
|
||||
return address, ctx.Err()
|
||||
})
|
||||
if !errors.Is(err, context.Canceled) || calls != 1 || cleaned != 1 {
|
||||
t.Fatalf("calls=%d discarded=%d err=%v", calls, cleaned, err)
|
||||
}
|
||||
}
|
||||
func TestExecutorsDoNotStartInvalidOrCanceledRequests(t *testing.T) {
|
||||
canceled, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
for _, test := range []struct {
|
||||
ctx context.Context
|
||||
config *Config
|
||||
options Options[string]
|
||||
attempt Attempt[string]
|
||||
}{
|
||||
{ctx: canceled, options: Options[string]{}, attempt: func(context.Context, string) (string, error) { t.Fatal("called after cancellation"); return "", nil }},
|
||||
{ctx: context.Background(), options: Options[string]{Kind: 99}, attempt: func(context.Context, string) (string, error) { t.Fatal("invalid kind"); return "", nil }},
|
||||
{ctx: context.Background(), options: Options[string]{Timeout: -time.Second}, attempt: func(context.Context, string) (string, error) { t.Fatal("invalid timeout"); return "", nil }},
|
||||
{ctx: context.Background(), options: Options[string]{}, attempt: nil},
|
||||
{ctx: context.Background(), config: &Config{Mirrors: []Mirror{{Type: "invalid", URL: "https://mirror.example"}}}, attempt: func(context.Context, string) (string, error) { t.Fatal("invalid config"); return "", nil }},
|
||||
} {
|
||||
if _, err := Race(test.ctx, test.config, testSource, test.options, test.attempt); err == nil {
|
||||
t.Fatal("invalid race accepted")
|
||||
}
|
||||
if _, err := Fallback(test.ctx, test.config, testSource, test.options, test.attempt); err == nil {
|
||||
t.Fatal("invalid fallback accepted")
|
||||
}
|
||||
}
|
||||
}
|
||||
func TestAttemptPanicBecomesError(t *testing.T) {
|
||||
attempt := func(context.Context, string) (string, error) { panic("broken operation") }
|
||||
if _, err := Race(context.Background(), nil, testSource, Options[string]{}, attempt); err == nil || !strings.Contains(err.Error(), "broken operation") {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := Fallback(context.Background(), nil, testSource, Options[string]{}, attempt); err == nil || !strings.Contains(err.Error(), "broken operation") {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
func TestNonGithubResourcesAndDisabledMirrorsUseOnlyOriginal(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
config *Config
|
||||
raw string
|
||||
}{{testConfig(), "https://example.com/resource"}, {&Config{Enabled: false, Mirrors: testConfig().Mirrors}, testSource}} {
|
||||
calls := 0
|
||||
got, err := Race(context.Background(), test.config, test.raw, Options[string]{}, func(_ context.Context, address string) (string, error) { calls++; return address, nil })
|
||||
if err != nil || got != test.raw || calls != 1 {
|
||||
t.Fatalf("got=%s calls=%d err=%v", got, calls, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRepositoryAttemptsHaveNoDefaultDeadline(t *testing.T) {
|
||||
value, err := Fallback(context.Background(), nil, testSource, Options[string]{Kind: Repository}, func(ctx context.Context, raw string) (string, error) {
|
||||
if _, ok := ctx.Deadline(); ok {
|
||||
t.Fatal("repository attempt has a total deadline")
|
||||
}
|
||||
return raw, nil
|
||||
})
|
||||
if err != nil || value != testSource {
|
||||
t.Fatalf("value=%s error=%v", value, err)
|
||||
}
|
||||
}
|
||||
@@ -1,37 +1,12 @@
|
||||
package bt
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/GopeedLab/gopeed/internal/fetcher"
|
||||
)
|
||||
|
||||
type trackerSubscriptionStatus struct {
|
||||
Success bool `json:"success"`
|
||||
TrackerCount int `json:"trackerCount"`
|
||||
LastFetchTime time.Time `json:"lastFetchTime"`
|
||||
}
|
||||
|
||||
type state struct {
|
||||
TrackerSubscriptionStatuses map[string]trackerSubscriptionStatus `json:"trackerSubscriptionStatuses"`
|
||||
TrackerSources map[string][]string `json:"trackerSources"`
|
||||
LastTrackerUpdateTime time.Time `json:"lastTrackerUpdateTime"`
|
||||
}
|
||||
|
||||
type config struct {
|
||||
fetcher.ConfigState[state]
|
||||
TrackerSubscribeURLs []string `json:"trackerSubscribeUrls"`
|
||||
AutoUpdateTrackers bool `json:"autoUpdateTrackers"`
|
||||
CustomTrackers []string `json:"customTrackers"`
|
||||
ListenPort int `json:"listenPort"`
|
||||
ListenPort int `json:"listenPort"`
|
||||
Trackers []string `json:"trackers"`
|
||||
// SeedKeep is always keep seeding after downloading is complete, unless manually stopped.
|
||||
SeedKeep bool `json:"seedKeep"`
|
||||
// SeedRatio is the ratio of uploaded data to downloaded data to seed.
|
||||
SeedRatio float64 `json:"seedRatio"`
|
||||
// SeedTime is the time in seconds to seed after downloading is complete.
|
||||
SeedTime int64 `json:"seedTime"`
|
||||
// DisableDHT and DisablePEX turn off DHT and PEX peer discovery. Like ListenPort,
|
||||
// they are read when the shared client is built.
|
||||
DisableDHT bool `json:"disableDht"`
|
||||
DisablePEX bool `json:"disablePex"`
|
||||
}
|
||||
@@ -1,464 +0,0 @@
|
||||
package bt
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/GopeedLab/gopeed/internal/controller"
|
||||
"github.com/GopeedLab/gopeed/internal/fetcher"
|
||||
"github.com/GopeedLab/gopeed/pkg/base"
|
||||
btprotocol "github.com/GopeedLab/gopeed/pkg/protocol/bt"
|
||||
"github.com/anacrolix/torrent"
|
||||
pp "github.com/anacrolix/torrent/peer_protocol"
|
||||
"golang.org/x/time/rate"
|
||||
)
|
||||
|
||||
type btDownloadHarness struct {
|
||||
*Fetcher
|
||||
pendingRequests atomic.Int64
|
||||
}
|
||||
|
||||
// These tests exercise the BT fetcher, native storage and real peer protocol.
|
||||
// All content is generated locally and served by a loopback seeder.
|
||||
func btDownloadFetcher(t *testing.T, f fetcher.Fetcher) *btDownloadHarness {
|
||||
t.Helper()
|
||||
if f == nil {
|
||||
f = new(FetcherManager).Build()
|
||||
}
|
||||
ctl := controller.NewController()
|
||||
ctl.GetConfig = func(v any) {
|
||||
encoded, err := json.Marshal(new(FetcherManager).DefaultConfig())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := json.Unmarshal(encoded, v); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
f.Setup(ctl)
|
||||
t.Cleanup(func() {
|
||||
if err := f.Close(); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
if err := closeClient(); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
})
|
||||
h := &btDownloadHarness{Fetcher: f.(*Fetcher)}
|
||||
// Keep the test client on loopback and observe real cancellation replies.
|
||||
// Waiting for them avoids the pinned library's immediate cancel/resume
|
||||
// request scheduling bug, without relying on arbitrary sleep durations.
|
||||
cfg = torrent.TestingConfig(t)
|
||||
cfg.DisableTrackers = false
|
||||
cfg.Seed = true
|
||||
cfg.DisableUTP = true
|
||||
cfg.DisableIPv6 = true
|
||||
cfg.DisableWebtorrent = true
|
||||
cfg.KeepAliveTimeout = time.Minute
|
||||
cfg.MaxAllocPeerRequestDataPerConn = 1 << 20
|
||||
cfg.Callbacks.SentRequest = []func(torrent.PeerRequestEvent){func(torrent.PeerRequestEvent) {
|
||||
h.pendingRequests.Add(1)
|
||||
}}
|
||||
cfg.Callbacks.ReadMessage = func(_ *torrent.PeerConn, msg *pp.Message) {
|
||||
if !msg.Keepalive && (msg.Type == pp.Piece || msg.Type == pp.Reject) {
|
||||
h.pendingRequests.Add(-1)
|
||||
}
|
||||
}
|
||||
var err error
|
||||
client, err = torrent.NewClient(cfg)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return h
|
||||
}
|
||||
|
||||
func btResolve(t *testing.T, f *btDownloadHarness, source string) {
|
||||
t.Helper()
|
||||
if err := f.Resolve(&base.Request{URL: source}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func btStart(t *testing.T, f *btDownloadHarness) {
|
||||
t.Helper()
|
||||
if err := f.Start(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func btAwaitDone(t *testing.T, f *btDownloadHarness) {
|
||||
t.Helper()
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- f.Wait() }()
|
||||
timeout := time.NewTimer(20 * time.Second)
|
||||
defer timeout.Stop()
|
||||
select {
|
||||
case err := <-done:
|
||||
if err != nil {
|
||||
t.Fatalf("BT download failed: %v", err)
|
||||
}
|
||||
if !f.isDone() {
|
||||
t.Fatal("Wait returned before selected files were verified")
|
||||
}
|
||||
case <-timeout.C:
|
||||
var status strings.Builder
|
||||
client.WriteStatus(&status)
|
||||
t.Log(status.String())
|
||||
t.Fatalf("BT download did not finish: progress=%v stats=%+v pieces=%+v peers=%+v", f.Progress(), f.torrent.Stats(), f.torrent.PieceStateRuns(), f.Stats().Runtime)
|
||||
}
|
||||
}
|
||||
|
||||
func btAwaitPartial(t *testing.T, f *btDownloadHarness) {
|
||||
t.Helper()
|
||||
timeout := time.NewTimer(15 * time.Second)
|
||||
defer timeout.Stop()
|
||||
ticker := time.NewTicker(20 * time.Millisecond)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ticker.C:
|
||||
progress := f.Progress().TotalDownloaded()
|
||||
if progress >= 32*1024 && progress < f.Meta().Res.Size {
|
||||
return
|
||||
}
|
||||
case <-timeout.C:
|
||||
t.Fatal("BT transfer did not reach partial progress")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func btPause(t *testing.T, f *btDownloadHarness) {
|
||||
t.Helper()
|
||||
if err := f.Pause(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
timeout := time.NewTimer(5 * time.Second)
|
||||
defer timeout.Stop()
|
||||
ticker := time.NewTicker(10 * time.Millisecond)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
if f.pendingRequests.Load() == 0 {
|
||||
// The message callback runs under the client lock. Acquiring that
|
||||
// lock ensures the final cancellation reply has finished processing.
|
||||
client.Stats()
|
||||
return
|
||||
}
|
||||
select {
|
||||
case <-ticker.C:
|
||||
case <-timeout.C:
|
||||
t.Fatal("peer did not acknowledge cancelled requests")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBTDownloadSources(t *testing.T) {
|
||||
for _, kind := range []string{"magnet", "torrent", "file URI", "data URI"} {
|
||||
t.Run(kind, func(t *testing.T) {
|
||||
fixture := newBTFixture(t, map[string][]byte{"": btPayload(96*1024, kind)}, 0)
|
||||
source := fixture.Magnet
|
||||
switch kind {
|
||||
case "torrent":
|
||||
source = fixture.TorrentFile
|
||||
case "file URI":
|
||||
source = fixture.FileURI()
|
||||
case "data URI":
|
||||
source = fixture.DataURI
|
||||
}
|
||||
f := btDownloadFetcher(t, nil)
|
||||
btResolve(t, f, source)
|
||||
if f.Meta().Opts != nil {
|
||||
t.Fatal("Resolve assigned download options")
|
||||
}
|
||||
f.storage.mu.RLock()
|
||||
bound := f.storage.owner != nil
|
||||
f.storage.mu.RUnlock()
|
||||
if bound {
|
||||
t.Fatal("Resolve bound download storage")
|
||||
}
|
||||
resolvedTorrent := f.torrent
|
||||
dir := filepath.Join(t.TempDir(), "chosen-after-resolve")
|
||||
f.Meta().Opts = &base.Options{Path: dir}
|
||||
if _, err := os.Stat(dir); !os.IsNotExist(err) {
|
||||
t.Fatalf("output directory exists before Start: %v", err)
|
||||
}
|
||||
btStart(t, f)
|
||||
if f.torrent != resolvedTorrent {
|
||||
t.Fatal("Start replaced the resolved torrent")
|
||||
}
|
||||
btAwaitDone(t, f)
|
||||
fixture.AssertFiles(t, dir)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestBTDirectDownloadWithExtraTrackers(t *testing.T) {
|
||||
for _, source := range []string{"request", "config", "subscription"} {
|
||||
t.Run(source, func(t *testing.T) {
|
||||
fixture := makeBTFixture(t, map[string][]byte{
|
||||
"a.bin": btPayload(20*1024, "direct-a"),
|
||||
"nested/b.bin": btPayload(48*1024, "direct-b"),
|
||||
}, 0, false, false)
|
||||
magnet, err := url.Parse(fixture.Magnet)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
query := magnet.Query()
|
||||
query.Del("tr")
|
||||
query.Del("x.pe")
|
||||
magnet.RawQuery = query.Encode()
|
||||
req := &base.Request{URL: magnet.String()}
|
||||
f := btDownloadFetcher(t, nil)
|
||||
if source == "request" {
|
||||
req.Extra = &btprotocol.ReqExtra{Trackers: []string{fixture.TrackerURL}}
|
||||
} else {
|
||||
backendConfig := new(FetcherManager).DefaultConfig().(*config)
|
||||
if source == "config" {
|
||||
backendConfig.CustomTrackers = []string{fixture.TrackerURL}
|
||||
} else {
|
||||
backendConfig.TrackerSubscribeURLs = []string{"https://subscription.example/list"}
|
||||
backendConfig.State.TrackerSources[backendConfig.TrackerSubscribeURLs[0]] = []string{fixture.TrackerURL}
|
||||
}
|
||||
f.ctl.GetConfig = func(dst any) {
|
||||
raw, err := json.Marshal(backendConfig)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
return
|
||||
}
|
||||
if err := json.Unmarshal(raw, dst); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
// CreateDirect supplies options before Downloader resolves the task.
|
||||
dir := filepath.Join(t.TempDir(), "direct-download")
|
||||
opts := &base.Options{Path: dir, SelectFiles: []int{}}
|
||||
f.Meta().Opts = opts
|
||||
resolved := make(chan error, 1)
|
||||
go func() { resolved <- f.Resolve(req) }()
|
||||
select {
|
||||
case err := <-resolved:
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
f.torrentDropFunc()
|
||||
<-resolved
|
||||
t.Fatal("direct magnet download did not obtain metadata from its extra tracker")
|
||||
}
|
||||
if f.Meta().Opts != opts || len(opts.SelectFiles) != len(fixture.Files) {
|
||||
t.Fatal("resolution lost direct download options or did not select all files")
|
||||
}
|
||||
btStart(t, f)
|
||||
btAwaitDone(t, f)
|
||||
fixture.AssertFiles(t, dir)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestBTDownloadSelectedFiles(t *testing.T) {
|
||||
fixture := newBTFixture(t, map[string][]byte{
|
||||
"a.bin": btPayload(20*1024, "a"), "nested/b.bin": btPayload(48*1024, "b"),
|
||||
}, 0)
|
||||
f := btDownloadFetcher(t, nil)
|
||||
dir := t.TempDir()
|
||||
btResolve(t, f, fixture.Magnet)
|
||||
selected := -1
|
||||
for i, file := range f.Meta().Res.Files {
|
||||
if file.Name == "b.bin" {
|
||||
selected = i
|
||||
}
|
||||
}
|
||||
if selected < 0 {
|
||||
t.Fatal("selected file absent from metadata")
|
||||
}
|
||||
f.Meta().Opts = &base.Options{Path: dir}
|
||||
if err := f.Patch(nil, &base.Options{SelectFiles: []int{selected}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
btStart(t, f)
|
||||
btAwaitDone(t, f)
|
||||
fixture.AssertFiles(t, dir, filepath.Join("bundle", "nested", "b.bin"))
|
||||
if _, err := os.Stat(filepath.Join(dir, "bundle", "a.bin")); !os.IsNotExist(err) {
|
||||
t.Fatalf("unselected output remains: %v", err)
|
||||
}
|
||||
if f.Meta().Res.Size != 48*1024 {
|
||||
t.Fatal("selected resource size is incorrect")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBTPauseAndResume(t *testing.T) {
|
||||
fixture := newBTFixture(t, map[string][]byte{"": btPayload(512*1024, "pause")}, 64*1024)
|
||||
f := btDownloadFetcher(t, nil)
|
||||
dir := t.TempDir()
|
||||
btResolve(t, f, fixture.Magnet)
|
||||
f.Meta().Opts = &base.Options{Path: dir}
|
||||
btStart(t, f)
|
||||
btAwaitPartial(t, f)
|
||||
btPause(t, f)
|
||||
resolvedTorrent := f.torrent
|
||||
fixture.Limiter.SetLimit(rate.Inf)
|
||||
// Once cancelled requests have settled, no more data should arrive.
|
||||
stats := f.torrent.Stats()
|
||||
readWhilePaused := stats.BytesReadData.Int64()
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
stats = f.torrent.Stats()
|
||||
if stats.BytesReadData.Int64() != readWhilePaused || f.isDone() {
|
||||
t.Fatal("data download continued while paused")
|
||||
}
|
||||
btStart(t, f)
|
||||
if f.torrent != resolvedTorrent {
|
||||
t.Fatal("resume replaced the resolved torrent")
|
||||
}
|
||||
btAwaitDone(t, f)
|
||||
fixture.AssertFiles(t, dir)
|
||||
}
|
||||
|
||||
type btFetcherCheckpoint struct {
|
||||
Meta *fetcher.FetcherMeta `json:"meta"`
|
||||
Data json.RawMessage `json:"data"`
|
||||
}
|
||||
|
||||
func btStore(t *testing.T, f *btDownloadHarness) []byte {
|
||||
t.Helper()
|
||||
data, err := new(FetcherManager).Store(f.Fetcher)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
encodedData, err := json.Marshal(data)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
encoded, err := json.Marshal(btFetcherCheckpoint{Meta: f.Meta(), Data: encodedData})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return encoded
|
||||
}
|
||||
|
||||
func btRestore(t *testing.T, encoded []byte) *btDownloadHarness {
|
||||
t.Helper()
|
||||
var checkpoint btFetcherCheckpoint
|
||||
if err := json.Unmarshal(encoded, &checkpoint); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
data, restore := new(FetcherManager).Restore()
|
||||
if err := json.Unmarshal(checkpoint.Data, data); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return btDownloadFetcher(t, restore(checkpoint.Meta, data))
|
||||
}
|
||||
|
||||
func btCloseForRestart(t *testing.T, f *btDownloadHarness) {
|
||||
t.Helper()
|
||||
if err := f.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := new(FetcherManager).Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBTColdStartResume(t *testing.T) {
|
||||
for _, kind := range []string{"v1", "hybrid"} {
|
||||
t.Run(kind, func(t *testing.T) {
|
||||
var fixture *btFixture
|
||||
if kind == "hybrid" {
|
||||
fixture = newBTHybridFixture(t, btPayload(512*1024, "restart-hybrid"))
|
||||
fixture.Limiter.SetLimit(64 * 1024)
|
||||
} else {
|
||||
fixture = newBTFixture(t, map[string][]byte{"": btPayload(512*1024, "restart")}, 64*1024)
|
||||
}
|
||||
f := btDownloadFetcher(t, nil)
|
||||
dir := t.TempDir()
|
||||
btResolve(t, f, fixture.Magnet)
|
||||
f.Meta().Opts = &base.Options{Path: dir}
|
||||
btStart(t, f)
|
||||
btAwaitPartial(t, f)
|
||||
btPause(t, f)
|
||||
checkpoint := btStore(t, f)
|
||||
hash := f.Meta().Res.Hash
|
||||
btCloseForRestart(t, f)
|
||||
seedStats := fixture.Torrent.Stats()
|
||||
sentBeforeRestart := seedStats.BytesWrittenData.Int64()
|
||||
restored := btRestore(t, checkpoint)
|
||||
if restored.Meta().Opts.Path != dir || restored.Meta().Res.Hash != hash {
|
||||
t.Fatal("download directory or hash was lost")
|
||||
}
|
||||
fixture.Limiter.SetLimit(rate.Inf)
|
||||
btStart(t, restored)
|
||||
btAwaitDone(t, restored)
|
||||
fixture.AssertFiles(t, dir)
|
||||
seedStats = fixture.Torrent.Stats()
|
||||
if sent := seedStats.BytesWrittenData.Int64() - sentBeforeRestart; sent >= 512*1024 {
|
||||
t.Fatalf("restart redownloaded the entire file: %d bytes", sent)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestBTDownloadHybrid(t *testing.T) {
|
||||
fixture := newBTHybridFixture(t, btPayload(80*1024, "hybrid-v2"))
|
||||
f := btDownloadFetcher(t, nil)
|
||||
dir := t.TempDir()
|
||||
btResolve(t, f, fixture.Magnet)
|
||||
f.Meta().Opts = &base.Options{Path: dir}
|
||||
btStart(t, f)
|
||||
btAwaitDone(t, f)
|
||||
fixture.AssertFiles(t, dir)
|
||||
}
|
||||
|
||||
func TestBTRepairCorruptedExistingFile(t *testing.T) {
|
||||
fixture := newBTFixture(t, map[string][]byte{"": btPayload(128*1024, "repair")}, 0)
|
||||
dir := t.TempDir()
|
||||
damaged := append([]byte(nil), fixture.Files["payload.bin"]...)
|
||||
damaged[20000] ^= 0xff
|
||||
if err := os.WriteFile(filepath.Join(dir, "payload.bin"), damaged, 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f := btDownloadFetcher(t, nil)
|
||||
btResolve(t, f, fixture.Magnet)
|
||||
f.Meta().Opts = &base.Options{Path: dir}
|
||||
btStart(t, f)
|
||||
btAwaitDone(t, f)
|
||||
fixture.AssertFiles(t, dir)
|
||||
}
|
||||
|
||||
func TestBTPatchSelectionWhilePaused(t *testing.T) {
|
||||
fixture := newBTFixture(t, map[string][]byte{"a.bin": btPayload(256*1024, "patch-a"), "b.bin": btPayload(256*1024, "patch-b")}, 64*1024)
|
||||
f := btDownloadFetcher(t, nil)
|
||||
dir := t.TempDir()
|
||||
btResolve(t, f, fixture.Magnet)
|
||||
f.Meta().Opts = &base.Options{Path: dir, SelectFiles: []int{0}}
|
||||
btStart(t, f)
|
||||
btAwaitPartial(t, f)
|
||||
btPause(t, f)
|
||||
if err := f.Patch(nil, &base.Options{SelectFiles: []int{1}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fixture.Limiter.SetLimit(rate.Inf)
|
||||
btStart(t, f)
|
||||
btAwaitDone(t, f)
|
||||
fixture.AssertFiles(t, dir, filepath.Join("bundle", "b.bin"))
|
||||
for _, suffix := range []string{"", ".part"} {
|
||||
if _, err := os.Stat(filepath.Join(dir, "bundle", "a.bin") + suffix); !os.IsNotExist(err) {
|
||||
t.Fatalf("unselected output remains: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBTDownloadMultiFileWithEmptyFile(t *testing.T) {
|
||||
fixture := newBTFixture(t, map[string][]byte{"empty.txt": {}, "nested/data.bin": btPayload(48*1024, "empty-file")}, 0)
|
||||
f := btDownloadFetcher(t, nil)
|
||||
dir := t.TempDir()
|
||||
btResolve(t, f, fixture.Magnet)
|
||||
f.Meta().Opts = &base.Options{Path: dir}
|
||||
btStart(t, f)
|
||||
btAwaitDone(t, f)
|
||||
fixture.AssertFiles(t, dir)
|
||||
}
|
||||
+125
-252
@@ -3,13 +3,11 @@ package bt
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
@@ -35,10 +33,6 @@ var (
|
||||
)
|
||||
|
||||
type Fetcher struct {
|
||||
mu sync.Mutex
|
||||
closeOnce sync.Once
|
||||
storage *deferredFileStorage
|
||||
|
||||
ctl *controller.Controller
|
||||
config *config
|
||||
|
||||
@@ -67,19 +61,22 @@ func (f *Fetcher) Setup(ctl *controller.Controller) {
|
||||
return
|
||||
}
|
||||
|
||||
// initClient must be called with lock held.
|
||||
func (f *Fetcher) initClient() (err error) {
|
||||
lock.Lock()
|
||||
defer lock.Unlock()
|
||||
|
||||
if client != nil {
|
||||
return
|
||||
}
|
||||
if closeCtx == nil {
|
||||
closeCtx, closeFunc = context.WithCancel(context.Background())
|
||||
}
|
||||
|
||||
cfg = torrent.NewDefaultClientConfig()
|
||||
cfg.Seed = true
|
||||
cfg.Bep20 = fmt.Sprintf("-GP%s-", parseBep20())
|
||||
cfg.ExtendedHandshakeClientVersion = fmt.Sprintf("Gopeed %s", base.Version)
|
||||
cfg.ListenPort = f.config.ListenPort
|
||||
cfg.NoDHT = f.config.DisableDHT
|
||||
cfg.DisablePEX = f.config.DisablePEX
|
||||
cfg.HTTPProxy = f.ctl.GetProxy(f.meta.Req.Proxy)
|
||||
dnsResolver := &DnsCacheResolver{RefreshTimeout: 5 * time.Minute}
|
||||
cfg.TrackerDialContext = dnsResolver.DialContext
|
||||
@@ -89,34 +86,19 @@ func (f *Fetcher) initClient() (err error) {
|
||||
}
|
||||
|
||||
closeCtx, closeFunc = context.WithCancel(context.Background())
|
||||
// closeClient may reset closeCtx before the goroutine runs, so pass it as an argument.
|
||||
go dnsResolver.Run(closeCtx)
|
||||
go func() {
|
||||
dnsResolver.Run(closeCtx)
|
||||
}()
|
||||
return
|
||||
}
|
||||
|
||||
// addTorrentSpec holds lock from client initialization until the torrent is added,
|
||||
// so closeClient cannot close or reset the client in between.
|
||||
func (f *Fetcher) addTorrentSpec(spec *torrent.TorrentSpec) (*torrent.Torrent, bool, error) {
|
||||
lock.Lock()
|
||||
defer lock.Unlock()
|
||||
|
||||
if err := f.initClient(); err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
return client.AddTorrentSpec(spec)
|
||||
}
|
||||
|
||||
func (f *Fetcher) Resolve(req *base.Request) error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if f.torrent != nil {
|
||||
f.safeDrop()
|
||||
f.torrent = nil
|
||||
f.storage = nil
|
||||
f.torrentReady.Store(false)
|
||||
}
|
||||
func (f *Fetcher) Resolve(req *base.Request, opts *base.Options) error {
|
||||
f.meta.Req = req
|
||||
if err := f.addTorrent(req); err != nil {
|
||||
f.meta.Opts = opts
|
||||
if f.meta.Opts == nil {
|
||||
f.meta.Opts = &base.Options{}
|
||||
}
|
||||
if err := f.addTorrent(req, false); err != nil {
|
||||
return err
|
||||
}
|
||||
f.updateRes()
|
||||
@@ -124,72 +106,12 @@ func (f *Fetcher) Resolve(req *base.Request) error {
|
||||
}
|
||||
|
||||
func (f *Fetcher) Start() (err error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if f.meta.Opts == nil || f.meta.Opts.Path == "" {
|
||||
return errors.New("download directory is required when starting a torrent")
|
||||
}
|
||||
if err := f.torrentDropCtx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if !f.torrentReady.Load() {
|
||||
if err = f.addTorrent(f.meta.Req); err != nil {
|
||||
if err = f.addTorrent(f.meta.Req, false); err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
path := f.meta.Opts.Path
|
||||
if err := os.MkdirAll(path, 0755); err != nil {
|
||||
return err
|
||||
}
|
||||
hashes, err := f.storage.bind(storage.NewFileClientOpts{
|
||||
ClientBaseDir: path,
|
||||
FilePathMaker: func(opts storage.FilePathMakerOpts) string {
|
||||
_, name := torrentFileLayout(opts.Info, *opts.File)
|
||||
return name
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if hashes != nil {
|
||||
// File storage infers completion from final file names and sizes. Verify
|
||||
// before publishing that state, or peers can see an unverified file as
|
||||
// complete and disconnect before a damaged piece is discovered.
|
||||
// Late v2 hashes schedule their own initial checks through the library.
|
||||
changes := f.torrent.SubscribePieceStateChanges()
|
||||
defer changes.Close()
|
||||
for i, known := range hashes {
|
||||
if !known {
|
||||
continue
|
||||
}
|
||||
if err := f.torrentDropCtx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := f.torrent.Piece(i).VerifyDataContext(f.torrentDropCtx); err != nil {
|
||||
return err
|
||||
}
|
||||
// VerifyDataContext can return while the result is still being
|
||||
// written to storage. Wait for it before refreshing completion.
|
||||
for {
|
||||
state := f.torrent.Piece(i).State()
|
||||
if !state.Hashing && !state.QueuedForHash && !state.Marking {
|
||||
break
|
||||
}
|
||||
select {
|
||||
case <-changes.Values:
|
||||
case <-f.torrentDropCtx.Done():
|
||||
return f.torrentDropCtx.Err()
|
||||
case <-f.torrent.Closed():
|
||||
return errors.New("torrent closed during verification")
|
||||
}
|
||||
}
|
||||
}
|
||||
for i := range hashes {
|
||||
f.torrent.Piece(i).UpdateCompletion()
|
||||
}
|
||||
}
|
||||
|
||||
files := f.torrent.Files()
|
||||
// If the user does not specify the file to download, all files will be downloaded by default
|
||||
if f.data.Progress == nil {
|
||||
@@ -210,29 +132,21 @@ func (f *Fetcher) Start() (err error) {
|
||||
}
|
||||
}
|
||||
f.torrent.AllowDataDownload()
|
||||
go f.doUpload(false)
|
||||
return
|
||||
}
|
||||
|
||||
func (f *Fetcher) Pause() (err error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if f.torrent == nil {
|
||||
return nil
|
||||
}
|
||||
f.torrent.DisallowDataDownload()
|
||||
return
|
||||
}
|
||||
|
||||
func (f *Fetcher) Close() (err error) {
|
||||
f.closeOnce.Do(func() {
|
||||
f.torrentDropFunc()
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.safeDrop()
|
||||
f.torrentReady.Store(false)
|
||||
close(f.uploadDoneCh)
|
||||
})
|
||||
f.safeDrop()
|
||||
f.torrentDropFunc()
|
||||
f.uploadDoneCh <- nil
|
||||
if len(client.Torrents()) == 0 {
|
||||
err = closeClient()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -250,8 +164,6 @@ func (f *Fetcher) Meta() *fetcher.FetcherMeta {
|
||||
}
|
||||
|
||||
func (f *Fetcher) Stats() *fetcher.Stats {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
var stats torrent.TorrentStats
|
||||
var pieceMap *base.PieceMap
|
||||
pieceMapReady := false
|
||||
@@ -458,65 +370,48 @@ func normalizeBTTransport(network string) string {
|
||||
}
|
||||
|
||||
func (f *Fetcher) Progress() fetcher.Progress {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if !f.torrentReady.Load() {
|
||||
return append(fetcher.Progress(nil), f.data.Progress...)
|
||||
return f.data.Progress
|
||||
}
|
||||
for i := range f.data.Progress {
|
||||
selectIndex := f.meta.Opts.SelectFiles[i]
|
||||
file := f.torrent.Files()[selectIndex]
|
||||
f.data.Progress[i] = file.BytesCompleted()
|
||||
}
|
||||
return append(fetcher.Progress(nil), f.data.Progress...)
|
||||
return f.data.Progress
|
||||
}
|
||||
|
||||
func (f *Fetcher) Wait() error {
|
||||
f.mu.Lock()
|
||||
var closed <-chan struct{}
|
||||
if f.torrent != nil {
|
||||
closed = f.torrent.Closed()
|
||||
}
|
||||
f.mu.Unlock()
|
||||
ticker := time.NewTicker(time.Second)
|
||||
defer ticker.Stop()
|
||||
func (f *Fetcher) Wait() (err error) {
|
||||
for {
|
||||
select {
|
||||
case <-f.torrentDropCtx.Done():
|
||||
return nil
|
||||
case <-closed:
|
||||
return nil
|
||||
case <-ticker.C:
|
||||
done, err := f.finishDownload()
|
||||
if done || err != nil {
|
||||
return err
|
||||
return
|
||||
case <-time.After(time.Second):
|
||||
if f.torrentReady.Load() && len(f.meta.Opts.SelectFiles) > 0 {
|
||||
if f.isDone() {
|
||||
// remove unselected files
|
||||
for i, file := range f.torrent.Files() {
|
||||
selected := false
|
||||
for _, selectIndex := range f.meta.Opts.SelectFiles {
|
||||
if i == selectIndex {
|
||||
selected = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !selected {
|
||||
root, name := torrentFileLayout(f.torrent.Info(), file.FileInfo())
|
||||
if err := removeUnselectedFile(f.meta.Opts.Path, root, name); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (f *Fetcher) finishDownload() (bool, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if !f.torrentReady.Load() || f.meta.Opts == nil || len(f.meta.Opts.SelectFiles) == 0 || !f.isDoneLocked() {
|
||||
return false, nil
|
||||
}
|
||||
selected := make(map[int]bool, len(f.meta.Opts.SelectFiles))
|
||||
for _, i := range f.meta.Opts.SelectFiles {
|
||||
selected[i] = true
|
||||
}
|
||||
for i, file := range f.torrent.Files() {
|
||||
if selected[i] {
|
||||
continue
|
||||
}
|
||||
root, name := torrentFileLayout(f.torrent.Info(), file.FileInfo())
|
||||
if err := removeUnselectedFile(f.meta.Opts.Path, root, name); err != nil {
|
||||
return false, err
|
||||
}
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// Match the file storage layout, including its NoName convention for rootless
|
||||
// torrents. File.Path() always prepends BestName, even when storage omits it.
|
||||
func torrentFileLayout(info *metainfo.Info, file metainfo.FileInfo) (root, name string) {
|
||||
@@ -574,22 +469,13 @@ func removeUnselectedFile(downloadDir, torrentRoot, torrentPath string) error {
|
||||
}
|
||||
|
||||
func (f *Fetcher) isDone() bool {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
return f.isDoneLocked()
|
||||
}
|
||||
|
||||
func (f *Fetcher) isDoneLocked() bool {
|
||||
if f.meta.Opts == nil || f.torrent == nil {
|
||||
if f.meta.Opts == nil {
|
||||
return false
|
||||
}
|
||||
for _, index := range f.meta.Opts.SelectFiles {
|
||||
file := f.torrent.Files()[index]
|
||||
for i := file.BeginPieceIndex(); i < file.EndPieceIndex(); i++ {
|
||||
state := f.torrent.Piece(i).State()
|
||||
if !state.Ok || !state.Complete || state.Hashing || state.QueuedForHash {
|
||||
return false
|
||||
}
|
||||
for _, selectIndex := range f.meta.Opts.SelectFiles {
|
||||
file := f.torrent.Files()[selectIndex]
|
||||
if file.BytesCompleted() < file.Length() {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
@@ -598,11 +484,6 @@ func (f *Fetcher) isDoneLocked() bool {
|
||||
// Patch modifies the BT task settings.
|
||||
// Invalid file indices are silently ignored.
|
||||
func (f *Fetcher) Patch(req *base.Request, opts *base.Options) error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if f.meta.Opts == nil {
|
||||
f.meta.Opts = &base.Options{}
|
||||
}
|
||||
if opts == nil {
|
||||
return nil
|
||||
}
|
||||
@@ -659,9 +540,9 @@ func (f *Fetcher) updateRes() {
|
||||
Files: make([]*base.FileInfo, len(f.torrent.Files())),
|
||||
Hash: f.torrent.InfoHash().String(),
|
||||
}
|
||||
// Use the same root classification as storage (v2 single files have Length == 0).
|
||||
if files := f.torrent.Files(); len(files) > 0 {
|
||||
res.Name, _ = torrentFileLayout(f.torrent.Info(), files[0].FileInfo())
|
||||
// Directory torrent
|
||||
if f.torrent.Info().Length == 0 {
|
||||
res.Name = f.torrent.Name()
|
||||
}
|
||||
for i, file := range f.torrent.Files() {
|
||||
res.Files[i] = &base.FileInfo{
|
||||
@@ -678,50 +559,61 @@ func (f *Fetcher) updateRes() {
|
||||
}
|
||||
|
||||
func (f *Fetcher) Upload() (err error) {
|
||||
if err = f.Start(); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
return f.addTorrent(f.meta.Req, true)
|
||||
}
|
||||
|
||||
func (f *Fetcher) doUpload(fromUpload bool) {
|
||||
if !f.torrentUpload.CompareAndSwap(false, true) {
|
||||
return
|
||||
}
|
||||
f.mu.Lock()
|
||||
initialBytes, initialTime := f.data.SeedBytes, f.data.SeedTime
|
||||
torrentClosed := f.torrent.Closed()
|
||||
f.mu.Unlock()
|
||||
var doneTime int64
|
||||
ticker := time.NewTicker(time.Second)
|
||||
defer ticker.Stop()
|
||||
|
||||
// Check and update seed data
|
||||
lastData := &fetcherData{
|
||||
SeedBytes: f.data.SeedBytes,
|
||||
SeedTime: f.data.SeedTime,
|
||||
}
|
||||
var doneTime int64 = 0
|
||||
for {
|
||||
select {
|
||||
case <-f.torrentDropCtx.Done():
|
||||
return
|
||||
case <-torrentClosed:
|
||||
return
|
||||
case <-ticker.C:
|
||||
f.mu.Lock()
|
||||
case <-time.After(time.Second):
|
||||
if !f.torrentReady.Load() {
|
||||
f.mu.Unlock()
|
||||
continue
|
||||
}
|
||||
|
||||
stats := f.torrentStats()
|
||||
f.data.SeedBytes = initialBytes + stats.BytesWrittenData.Int64()
|
||||
if !fromUpload && !f.isDoneLocked() {
|
||||
f.mu.Unlock()
|
||||
f.data.SeedBytes = lastData.SeedBytes + stats.BytesWrittenData.Int64()
|
||||
|
||||
// Check is download complete, if not don't check and stop seeding
|
||||
if !fromUpload && !f.isDone() {
|
||||
continue
|
||||
}
|
||||
if doneTime == 0 {
|
||||
doneTime = time.Now().Unix()
|
||||
}
|
||||
f.data.SeedTime = initialTime + time.Now().Unix() - doneTime
|
||||
stop := !f.config.SeedKeep && ((f.config.SeedRatio > 0 && f.seedRadio() >= f.config.SeedRatio) || (f.config.SeedTime > 0 && f.data.SeedTime >= f.config.SeedTime))
|
||||
f.mu.Unlock()
|
||||
if stop {
|
||||
f.Close()
|
||||
return
|
||||
f.data.SeedTime = lastData.SeedTime + time.Now().Unix() - doneTime
|
||||
|
||||
// If the seed forever is true, keep seeding
|
||||
if f.config.SeedKeep {
|
||||
continue
|
||||
}
|
||||
|
||||
// If the seed ratio is reached, stop seeding
|
||||
if f.config.SeedRatio > 0 {
|
||||
seedRadio := f.seedRadio()
|
||||
if seedRadio >= f.config.SeedRatio {
|
||||
f.Close()
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
// If the seed time is reached, stop seeding
|
||||
if f.config.SeedTime > 0 {
|
||||
if f.data.SeedTime >= f.config.SeedTime {
|
||||
f.Close()
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -739,8 +631,6 @@ func (f *Fetcher) torrentStats() torrent.TorrentStats {
|
||||
}
|
||||
|
||||
func (f *Fetcher) UploadedBytes() int64 {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
return f.data.SeedBytes
|
||||
}
|
||||
|
||||
@@ -749,11 +639,13 @@ func (f *Fetcher) WaitUpload() (err error) {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *Fetcher) addTorrent(req *base.Request) (err error) {
|
||||
f.ctl.GetConfig(&f.config)
|
||||
func (f *Fetcher) addTorrent(req *base.Request, fromUpload bool) (err error) {
|
||||
if err = base.ParseReqExtra[bt.ReqExtra](req); err != nil {
|
||||
return
|
||||
}
|
||||
if err = f.initClient(); err != nil {
|
||||
return
|
||||
}
|
||||
schema := util.ParseSchema(req.URL)
|
||||
privateTorrent := false
|
||||
var spec *torrent.TorrentSpec
|
||||
@@ -766,16 +658,11 @@ func (f *Fetcher) addTorrent(req *base.Request) (err error) {
|
||||
var reader io.Reader
|
||||
if schema == "FILE" {
|
||||
fileUrl, _ := url.Parse(req.URL)
|
||||
filePath := fileUrl.Path
|
||||
if runtime.GOOS == "windows" && len(filePath) > 2 && filePath[0] == '/' && filePath[2] == ':' {
|
||||
filePath = filePath[1:]
|
||||
}
|
||||
filePath = filepath.FromSlash(filePath)
|
||||
filePath := fileUrl.Path[1:]
|
||||
reader, err = os.Open(filePath)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer reader.(io.Closer).Close()
|
||||
} else if schema == "DATA" {
|
||||
_, data := util.ParseDataUri(req.URL)
|
||||
reader = bytes.NewBuffer(data)
|
||||
@@ -808,6 +695,21 @@ func (f *Fetcher) addTorrent(req *base.Request) (err error) {
|
||||
return
|
||||
}
|
||||
}
|
||||
spec.Storage = storage.NewFileOpts(storage.NewFileClientOpts{
|
||||
ClientBaseDir: cfg.DataDir,
|
||||
FilePathMaker: func(opts storage.FilePathMakerOpts) string {
|
||||
_, name := torrentFileLayout(opts.Info, *opts.File)
|
||||
return name
|
||||
},
|
||||
TorrentDirMaker: func(baseDir string, info *metainfo.Info, infoHash metainfo.Hash) string {
|
||||
return f.meta.Opts.Path
|
||||
},
|
||||
})
|
||||
f.torrent, _, err = client.AddTorrentSpec(spec)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
// Do not add external tracker to a private torrent.
|
||||
if !privateTorrent {
|
||||
// use map to deduplicate
|
||||
@@ -820,40 +722,23 @@ func (f *Fetcher) addTorrent(req *base.Request) (err error) {
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, tracker := range f.config.effectiveTrackers() {
|
||||
trackers[tracker] = true
|
||||
}
|
||||
if len(trackers) > 0 {
|
||||
for tracker := range trackers {
|
||||
spec.Trackers = append(spec.Trackers, []string{tracker})
|
||||
if len(f.config.Trackers) > 0 {
|
||||
for _, tracker := range f.config.Trackers {
|
||||
trackers[tracker] = true
|
||||
}
|
||||
}
|
||||
if len(trackers) > 0 {
|
||||
announceList := make([][]string, 0)
|
||||
for tracker := range trackers {
|
||||
announceList = append(announceList, []string{tracker})
|
||||
}
|
||||
f.torrent.AddTrackers(announceList)
|
||||
}
|
||||
}
|
||||
s := &deferredFileStorage{}
|
||||
spec.Storage = s
|
||||
var added bool
|
||||
f.torrent, added, err = f.addTorrentSpec(spec)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !added {
|
||||
f.torrent = nil
|
||||
return errors.New("torrent is already active")
|
||||
}
|
||||
f.storage = s
|
||||
f.torrent.DisallowDataDownload()
|
||||
// Extra trackers must be available while magnet metadata is still missing;
|
||||
// they may be the only way to discover a peer that can provide it.
|
||||
select {
|
||||
case <-f.torrent.GotInfo():
|
||||
case <-f.torrentDropCtx.Done():
|
||||
f.safeDrop()
|
||||
return f.torrentDropCtx.Err()
|
||||
case <-f.torrent.Closed():
|
||||
return errors.New("torrent closed during resolution")
|
||||
}
|
||||
<-f.torrent.GotInfo()
|
||||
f.torrentReady.Store(true)
|
||||
|
||||
go f.doUpload(fromUpload)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -949,29 +834,17 @@ func (fm *FetcherManager) AutoRename() bool {
|
||||
|
||||
func (fm *FetcherManager) DefaultConfig() any {
|
||||
return &config{
|
||||
TrackerSubscribeURLs: append([]string{}, defaultTrackerSubscribeURLs...),
|
||||
AutoUpdateTrackers: true,
|
||||
CustomTrackers: []string{},
|
||||
ConfigState: fetcher.ConfigState[state]{State: state{
|
||||
TrackerSources: map[string][]string{},
|
||||
TrackerSubscriptionStatuses: map[string]trackerSubscriptionStatus{},
|
||||
}},
|
||||
ListenPort: 0,
|
||||
Trackers: []string{},
|
||||
SeedKeep: false,
|
||||
SeedRatio: 1.0,
|
||||
SeedTime: 120 * 60,
|
||||
DisableDHT: false,
|
||||
DisablePEX: false,
|
||||
}
|
||||
}
|
||||
|
||||
func (fm *FetcherManager) Store(f fetcher.Fetcher) (data any, err error) {
|
||||
_f := f.(*Fetcher)
|
||||
_f.mu.Lock()
|
||||
defer _f.mu.Unlock()
|
||||
snapshot := *_f.data
|
||||
snapshot.Progress = append(fetcher.Progress(nil), snapshot.Progress...)
|
||||
return &snapshot, nil
|
||||
return _f.data, nil
|
||||
}
|
||||
|
||||
func (fm *FetcherManager) Restore() (v any, f func(meta *fetcher.FetcherMeta, v any) fetcher.Fetcher) {
|
||||
|
||||
@@ -8,7 +8,6 @@ import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/GopeedLab/gopeed/internal/controller"
|
||||
@@ -19,11 +18,11 @@ import (
|
||||
)
|
||||
|
||||
func TestFetcher_Resolve_Torrent(t *testing.T) {
|
||||
doResolve(t, buildFetcher(t))
|
||||
doResolve(t, buildFetcher())
|
||||
}
|
||||
|
||||
func TestFetcher_Resolve_DataUri_Torrent(t *testing.T) {
|
||||
fetcher := buildFetcher(t)
|
||||
fetcher := buildFetcher()
|
||||
buf, err := os.ReadFile("./testdata/ubuntu-22.04-live-server-amd64.iso.torrent")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -32,7 +31,7 @@ func TestFetcher_Resolve_DataUri_Torrent(t *testing.T) {
|
||||
dataUri := "data:application/x-bittorrent;base64," + base64.StdEncoding.EncodeToString(buf)
|
||||
err = fetcher.Resolve(&base.Request{
|
||||
URL: dataUri,
|
||||
})
|
||||
}, nil)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
@@ -54,7 +53,7 @@ func TestFetcher_Resolve_DataUri_Torrent(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestFetcher_Config(t *testing.T) {
|
||||
doResolve(t, buildConfigFetcher(t, nil))
|
||||
doResolve(t, buildConfigFetcher(nil))
|
||||
}
|
||||
|
||||
func TestFetcher_ResolveWithProxy(t *testing.T) {
|
||||
@@ -62,7 +61,7 @@ func TestFetcher_ResolveWithProxy(t *testing.T) {
|
||||
proxyListener := test.StartSocks5Server(usr, pwd)
|
||||
defer proxyListener.Close()
|
||||
|
||||
doResolve(t, buildConfigFetcher(t, &base.DownloaderProxyConfig{
|
||||
doResolve(t, buildConfigFetcher(&base.DownloaderProxyConfig{
|
||||
Enable: true,
|
||||
System: false,
|
||||
Scheme: "socks5",
|
||||
@@ -72,82 +71,6 @@ func TestFetcher_ResolveWithProxy(t *testing.T) {
|
||||
}))
|
||||
}
|
||||
|
||||
func TestFetcher_CloseAfterInit(t *testing.T) {
|
||||
if err := closeClient(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for i := 0; i < 100; i++ {
|
||||
f := buildFetcher(t).(*Fetcher)
|
||||
f.meta.Req = &base.Request{}
|
||||
lock.Lock()
|
||||
err := f.initClient()
|
||||
lock.Unlock()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := closeClient(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetcher_CloseWithoutClient(t *testing.T) {
|
||||
if err := closeClient(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
t.Run("restored task", func(t *testing.T) {
|
||||
_, build := new(FetcherManager).Restore()
|
||||
f := build(&fetcher.FetcherMeta{
|
||||
Req: &base.Request{URL: "./testdata/test.torrent"},
|
||||
Opts: &base.Options{},
|
||||
}, &fetcherData{})
|
||||
f.Setup(controller.NewController())
|
||||
if err := f.Close(); err != nil {
|
||||
t.Errorf("Close() got = %v, want nil", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("failed resolve", func(t *testing.T) {
|
||||
f := buildFetcher(t)
|
||||
if err := f.Resolve(&base.Request{URL: "./testdata/not-exist.torrent"}); err == nil {
|
||||
t.Fatal("Resolve() got = nil, want error")
|
||||
}
|
||||
if err := f.Close(); err != nil {
|
||||
t.Errorf("Close() got = %v, want nil", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestFetcher_ResolveWhileClientCloses(t *testing.T) {
|
||||
if err := closeClient(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { closeClient() })
|
||||
|
||||
var wg sync.WaitGroup
|
||||
for _, u := range []string{"./testdata/test.torrent", "./testdata/ubuntu-22.04-live-server-amd64.iso.torrent"} {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for i := 0; i < 10; i++ {
|
||||
f := buildFetcher(t)
|
||||
// The other goroutine may close the client after the torrent was added.
|
||||
if err := f.Resolve(&base.Request{URL: u}); err != nil && err.Error() != "torrent closed during resolution" {
|
||||
t.Errorf("Resolve(%s) got = %v", u, err)
|
||||
return
|
||||
}
|
||||
f.Close()
|
||||
if err := closeClient(); err != nil {
|
||||
t.Errorf("closeClient() got = %v", err)
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
func doResolve(t *testing.T, fetcher fetcher.Fetcher) {
|
||||
t.Run("Resolve Single File", func(t *testing.T) {
|
||||
err := fetcher.Resolve(&base.Request{
|
||||
@@ -158,7 +81,7 @@ func doResolve(t *testing.T, fetcher fetcher.Fetcher) {
|
||||
"udp://tracker.bitsearch.to:1337/announce",
|
||||
},
|
||||
},
|
||||
})
|
||||
}, nil)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
@@ -188,7 +111,7 @@ func doResolve(t *testing.T, fetcher fetcher.Fetcher) {
|
||||
"udp://tracker.bitsearch.to:1337/announce",
|
||||
},
|
||||
},
|
||||
})
|
||||
}, nil)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
@@ -222,7 +145,7 @@ func doResolve(t *testing.T, fetcher fetcher.Fetcher) {
|
||||
t.Run("Resolve Unclean Torrent", func(t *testing.T) {
|
||||
err := fetcher.Resolve(&base.Request{
|
||||
URL: "./testdata/test.unclean.torrent",
|
||||
})
|
||||
}, nil)
|
||||
if err != nil {
|
||||
t.Errorf("Resolve Unclean Torrent Resolve() got = %v, want nil", err)
|
||||
}
|
||||
@@ -233,7 +156,7 @@ func doResolve(t *testing.T, fetcher fetcher.Fetcher) {
|
||||
uri := "file:///" + file
|
||||
err := fetcher.Resolve(&base.Request{
|
||||
URL: uri,
|
||||
})
|
||||
}, nil)
|
||||
if err != nil {
|
||||
t.Errorf("Resolve file scheme Torrent Resolve() got = %v, want nil", err)
|
||||
}
|
||||
@@ -288,7 +211,7 @@ func TestFetcherManager_ParseName(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func buildFetcher(t testing.TB) fetcher.Fetcher {
|
||||
func buildFetcher() fetcher.Fetcher {
|
||||
fb := new(FetcherManager)
|
||||
fetcher := fb.Build()
|
||||
newController := controller.NewController()
|
||||
@@ -296,15 +219,14 @@ func buildFetcher(t testing.TB) fetcher.Fetcher {
|
||||
json.Unmarshal([]byte(test.ToJson(fb.DefaultConfig())), v)
|
||||
}
|
||||
fetcher.Setup(newController)
|
||||
t.Cleanup(func() { fetcher.Close(); closeClient() })
|
||||
return fetcher
|
||||
}
|
||||
|
||||
func buildConfigFetcher(t testing.TB, proxyConfig *base.DownloaderProxyConfig) fetcher.Fetcher {
|
||||
func buildConfigFetcher(proxyConfig *base.DownloaderProxyConfig) fetcher.Fetcher {
|
||||
fetcher := new(FetcherManager).Build()
|
||||
newController := controller.NewController()
|
||||
mockCfg := config{
|
||||
CustomTrackers: []string{
|
||||
Trackers: []string{
|
||||
"udp://tracker.birkenwald.de:6969/announce",
|
||||
"udp://tracker.bitsearch.to:1337/announce",
|
||||
}}
|
||||
@@ -315,30 +237,29 @@ func buildConfigFetcher(t testing.TB, proxyConfig *base.DownloaderProxyConfig) f
|
||||
return proxyConfig.ToHandler()
|
||||
}
|
||||
fetcher.Setup(newController)
|
||||
t.Cleanup(func() { fetcher.Close(); closeClient() })
|
||||
return fetcher
|
||||
}
|
||||
|
||||
// TestFetcher_Patch tests the Patch functionality for BT fetcher.
|
||||
// It tests modifying selected files after Resolve (without downloading).
|
||||
func TestFetcher_Patch(t *testing.T) {
|
||||
f := buildFetcher(t)
|
||||
f := buildFetcher()
|
||||
|
||||
// Resolve a multi-file torrent
|
||||
err := f.Resolve(&base.Request{
|
||||
URL: "./testdata/test.torrent",
|
||||
})
|
||||
}, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("Resolve failed: %v", err)
|
||||
}
|
||||
|
||||
// Resolution describes files without choosing download options.
|
||||
// Verify initial state: 3 files, all selected by default
|
||||
meta := f.Meta()
|
||||
if len(meta.Res.Files) != 3 {
|
||||
t.Fatalf("Expected 3 files, got %d", len(meta.Res.Files))
|
||||
}
|
||||
if meta.Opts != nil {
|
||||
t.Fatal("Resolve should not assign download options")
|
||||
if len(meta.Opts.SelectFiles) != 3 {
|
||||
t.Fatalf("Expected 3 selected files, got %d", len(meta.Opts.SelectFiles))
|
||||
}
|
||||
|
||||
// Total size of all files: 107484864
|
||||
@@ -445,52 +366,3 @@ func TestFetcher_Patch(t *testing.T) {
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestFetcher_PeerDiscoveryConfig(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
stored any
|
||||
wantDisableDHT bool
|
||||
wantDisablePEX bool
|
||||
}{
|
||||
{"default config", new(FetcherManager).DefaultConfig(), false, false},
|
||||
{"stored config without the fields", map[string]any{"listenPort": 0, "seedRatio": 1.0}, false, false},
|
||||
{"both disabled", map[string]any{"disableDht": true, "disablePex": true}, true, true},
|
||||
{"only DHT disabled", map[string]any{"disableDht": true}, true, false},
|
||||
{"only PEX disabled", map[string]any{"disablePex": true}, false, true},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
f := buildFetcherWithConfig(tt.stored).(*Fetcher)
|
||||
if f.config.DisableDHT != tt.wantDisableDHT || f.config.DisablePEX != tt.wantDisablePEX {
|
||||
t.Errorf("config got DisableDHT = %v, DisablePEX = %v, want %v, %v", f.config.DisableDHT, f.config.DisablePEX, tt.wantDisableDHT, tt.wantDisablePEX)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetcher_PeerDiscoveryDisabledOnClient(t *testing.T) {
|
||||
if err := closeClient(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f := buildFetcherWithConfig(map[string]any{"disableDht": true, "disablePex": true})
|
||||
if err := f.Resolve(&base.Request{URL: "./testdata/test.torrent"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !cfg.NoDHT || !cfg.DisablePEX {
|
||||
t.Errorf("client config got NoDHT = %v, DisablePEX = %v, want true", cfg.NoDHT, cfg.DisablePEX)
|
||||
}
|
||||
if n := len(client.DhtServers()); n != 0 {
|
||||
t.Errorf("client got %d DHT servers, want 0", n)
|
||||
}
|
||||
}
|
||||
|
||||
func buildFetcherWithConfig(stored any) fetcher.Fetcher {
|
||||
fetcher := new(FetcherManager).Build()
|
||||
newController := controller.NewController()
|
||||
newController.GetConfig = func(v any) {
|
||||
json.Unmarshal([]byte(test.ToJson(stored)), v)
|
||||
}
|
||||
fetcher.Setup(newController)
|
||||
return fetcher
|
||||
}
|
||||
@@ -1,212 +0,0 @@
|
||||
package bt
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/binary"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/anacrolix/torrent"
|
||||
"github.com/anacrolix/torrent/bencode"
|
||||
"github.com/anacrolix/torrent/merkle"
|
||||
"github.com/anacrolix/torrent/metainfo"
|
||||
"golang.org/x/time/rate"
|
||||
)
|
||||
|
||||
// btFixture serves generated, trackerless content over a real loopback TCP peer.
|
||||
// Its HTTP tracker exercises .torrent downloads; magnet links also carry the
|
||||
// explicit x.pe address.
|
||||
type btFixture struct {
|
||||
MetaInfo *metainfo.MetaInfo
|
||||
Magnet string
|
||||
TorrentFile string
|
||||
DataURI string
|
||||
TrackerURL string
|
||||
Files map[string][]byte // Paths relative to the selected download directory.
|
||||
Seeder *torrent.Client
|
||||
Torrent *torrent.Torrent
|
||||
Limiter *rate.Limiter
|
||||
}
|
||||
|
||||
func newBTFixture(t testing.TB, files map[string][]byte, bytesPerSecond int) *btFixture {
|
||||
return makeBTFixture(t, files, bytesPerSecond, false, true)
|
||||
}
|
||||
|
||||
func newBTHybridFixture(t testing.TB, payload []byte) *btFixture {
|
||||
return makeBTFixture(t, map[string][]byte{"": payload}, 0, true, true)
|
||||
}
|
||||
|
||||
func makeBTFixture(t testing.TB, files map[string][]byte, bytesPerSecond int, v2, private bool) *btFixture {
|
||||
t.Helper()
|
||||
source := t.TempDir()
|
||||
name := "payload.bin"
|
||||
if _, single := files[""]; !single {
|
||||
name = "bundle"
|
||||
}
|
||||
root := filepath.Join(source, name)
|
||||
expected := make(map[string][]byte, len(files))
|
||||
for path, data := range files {
|
||||
target := filepath.Join(root, filepath.FromSlash(path))
|
||||
if err := os.MkdirAll(filepath.Dir(target), 0700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(target, data, 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
expected[filepath.Join(name, filepath.FromSlash(path))] = append([]byte(nil), data...)
|
||||
}
|
||||
info := metainfo.Info{PieceLength: 16 * 1024}
|
||||
if err := info.BuildFromFilePath(root); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var layers map[string]string
|
||||
if v2 {
|
||||
data := files[""]
|
||||
h := merkle.NewHash()
|
||||
h.Write(data)
|
||||
root := h.Sum(nil)
|
||||
var pieceHashes []byte
|
||||
for off := 0; off < len(data); off += int(info.PieceLength) {
|
||||
end := min(off+int(info.PieceLength), len(data))
|
||||
ph := merkle.NewHash()
|
||||
ph.Write(data[off:end])
|
||||
pieceHashes = append(pieceHashes, ph.SumMinLength(nil, int(info.PieceLength))...)
|
||||
}
|
||||
info = metainfo.Info{Name: name, PieceLength: info.PieceLength, Length: int64(len(data)), Pieces: info.Pieces, MetaVersion: 2, FileTree: metainfo.FileTree{File: metainfo.FileTreeFile{Length: int64(len(data)), PiecesRoot: string(root)}}}
|
||||
layers = map[string]string{string(root): string(pieceHashes)}
|
||||
}
|
||||
// Private torrents avoid public DHT discovery once metadata is available.
|
||||
info.Private = &private
|
||||
raw, err := bencode.Marshal(&info)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
mi := &metainfo.MetaInfo{InfoBytes: raw, PieceLayers: layers}
|
||||
cfg := torrent.TestingConfig(t)
|
||||
cfg.DataDir = source
|
||||
cfg.Seed = true
|
||||
cfg.DisableUTP = true
|
||||
cfg.DisableIPv6 = true
|
||||
cfg.DisableWebtorrent = true
|
||||
cfg.KeepAliveTimeout = time.Minute
|
||||
cfg.MaxAllocPeerRequestDataPerConn = 1 << 20
|
||||
limiter := rate.NewLimiter(rate.Inf, 16*1024)
|
||||
if bytesPerSecond > 0 {
|
||||
limiter.SetLimit(rate.Limit(bytesPerSecond))
|
||||
}
|
||||
cfg.UploadRateLimiter = limiter
|
||||
seed, err := torrent.NewClient(cfg)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { seed.Close() })
|
||||
seeded, _, err := seed.AddTorrentSpec(torrent.TorrentSpecFromMetaInfo(mi))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
if err := seeded.VerifyDataContext(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
select {
|
||||
case <-seeded.Complete().On():
|
||||
case <-ctx.Done():
|
||||
t.Fatal("local seeder did not finish verification")
|
||||
}
|
||||
addr := seed.ListenAddrs()[0].String()
|
||||
host, portString, err := net.SplitHostPort(addr)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
resolved, err := net.ResolveTCPAddr("tcp4", net.JoinHostPort(host, portString))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
compact := make([]byte, 6)
|
||||
copy(compact, resolved.IP.To4())
|
||||
binary.BigEndian.PutUint16(compact[4:], uint16(resolved.Port))
|
||||
tracker := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
response, err := bencode.Marshal(map[string]any{"interval": 3600, "peers": string(compact)})
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), 500)
|
||||
return
|
||||
}
|
||||
w.Write(response)
|
||||
}))
|
||||
t.Cleanup(tracker.Close)
|
||||
// The pinned library has a hybrid tracker-dispatch assertion bug.
|
||||
// Hybrid fixtures use their explicit loopback peer rather than a tracker.
|
||||
if !v2 {
|
||||
mi.Announce = tracker.URL
|
||||
}
|
||||
var encoded bytes.Buffer
|
||||
if err := mi.Write(&encoded); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
torrentFile := filepath.Join(t.TempDir(), "fixture.torrent")
|
||||
if err := os.WriteFile(torrentFile, encoded.Bytes(), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m, err := mi.MagnetV2()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
magnet := m.String()
|
||||
u, err := url.Parse(magnet)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
q := u.Query()
|
||||
q.Add("x.pe", addr)
|
||||
u.RawQuery = q.Encode()
|
||||
return &btFixture{MetaInfo: mi, Magnet: u.String(), TorrentFile: torrentFile, DataURI: "data:application/x-bittorrent;base64," + base64.StdEncoding.EncodeToString(encoded.Bytes()), TrackerURL: tracker.URL, Files: expected, Seeder: seed, Torrent: seeded, Limiter: limiter}
|
||||
}
|
||||
|
||||
func btPayload(size int, label string) []byte {
|
||||
data := make([]byte, size)
|
||||
for i := range data {
|
||||
data[i] = byte((i*31 + i/251) % 251)
|
||||
}
|
||||
copy(data, label)
|
||||
return data
|
||||
}
|
||||
|
||||
func (f *btFixture) AssertFiles(t testing.TB, dir string, paths ...string) {
|
||||
t.Helper()
|
||||
if len(paths) == 0 {
|
||||
for path := range f.Files {
|
||||
paths = append(paths, path)
|
||||
}
|
||||
}
|
||||
for _, path := range paths {
|
||||
got, err := os.ReadFile(filepath.Join(dir, path))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want, ok := f.Files[path]
|
||||
if !ok {
|
||||
t.Fatalf("unknown fixture file %q", path)
|
||||
}
|
||||
if !bytes.Equal(got, want) {
|
||||
t.Fatalf("BT file %s differs: received %d bytes, want %d", path, len(got), len(want))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (f *btFixture) FileURI() string {
|
||||
path := filepath.ToSlash(f.TorrentFile)
|
||||
if !strings.HasPrefix(path, "/") {
|
||||
path = "/" + path
|
||||
}
|
||||
return (&url.URL{Scheme: "file", Path: path}).String()
|
||||
}
|
||||
@@ -17,85 +17,25 @@ import (
|
||||
"github.com/anacrolix/torrent/storage"
|
||||
)
|
||||
|
||||
func TestStartDoesNotPublishUnverifiedCompletion(t *testing.T) {
|
||||
cfg := torrent.TestingConfig(t)
|
||||
cfg.DisableTCP = true
|
||||
cfg.DisableUTP = true
|
||||
c, err := torrent.NewClient(cfg)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer c.Close()
|
||||
payload := btPayload(32*1024, "unverified")
|
||||
info := metainfo.Info{Name: "payload.bin", Length: int64(len(payload)), PieceLength: 16 * 1024}
|
||||
for offset := 0; offset < len(payload); offset += int(info.PieceLength) {
|
||||
hash := sha1.Sum(payload[offset : offset+int(info.PieceLength)])
|
||||
info.Pieces = append(info.Pieces, hash[:]...)
|
||||
}
|
||||
raw, err := bencode.Marshal(info)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
store := &deferredFileStorage{}
|
||||
spec := torrent.TorrentSpecFromMetaInfo(&metainfo.MetaInfo{InfoBytes: raw})
|
||||
spec.Storage = store
|
||||
tor, _, err := c.AddTorrentSpec(spec)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
tor.DisallowDataDownload()
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
f := &Fetcher{torrent: tor, storage: store, meta: &fetcher.FetcherMeta{}, data: &fetcherData{}, torrentDropCtx: ctx, torrentDropFunc: cancel, config: &config{SeedKeep: true}}
|
||||
f.torrentReady.Store(true)
|
||||
f.updateRes()
|
||||
dir := t.TempDir()
|
||||
payload[20000] ^= 0xff
|
||||
if err := os.WriteFile(filepath.Join(dir, info.Name), payload, 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f.meta.Opts = &base.Options{Path: dir}
|
||||
// Retain the original event channel to detect even a transient completion.
|
||||
// With no network transports, the damaged piece cannot have been repaired.
|
||||
completed := tor.Complete().On()
|
||||
if err := f.Start(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
select {
|
||||
case <-completed:
|
||||
t.Fatal("Start published completion before verifying the existing file")
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
func TestStartUsesSelectionSubmittedAfterResolve(t *testing.T) {
|
||||
cfg := torrent.TestingConfig(t)
|
||||
cfg.DataDir = t.TempDir()
|
||||
cfg.DisableTCP = true
|
||||
cfg.DisableUTP = true
|
||||
c, err := torrent.NewClient(cfg)
|
||||
config := torrent.TestingConfig(t)
|
||||
config.DataDir = t.TempDir()
|
||||
config.DisableTCP = true
|
||||
config.DisableUTP = true
|
||||
c, err := torrent.NewClient(config)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer c.Close()
|
||||
store := &deferredFileStorage{}
|
||||
mi, err := metainfo.LoadFromFile("testdata/test.torrent")
|
||||
tor, err := c.AddTorrentFromFile("testdata/test.torrent")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
spec := torrent.TorrentSpecFromMetaInfo(mi)
|
||||
spec.Storage = store
|
||||
tor, _, err := c.AddTorrentSpec(spec)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
tor.DisallowDataDownload()
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
f := &Fetcher{torrent: tor, storage: store, meta: &fetcher.FetcherMeta{}, data: &fetcherData{}, torrentDropCtx: ctx, torrentDropFunc: cancel, config: &config{SeedKeep: true}}
|
||||
f := &Fetcher{torrent: tor, meta: &fetcher.FetcherMeta{Opts: &base.Options{}}, data: &fetcherData{}}
|
||||
f.torrentReady.Store(true)
|
||||
f.updateRes()
|
||||
f.meta.Opts = &base.Options{Path: t.TempDir(), SelectFiles: []int{2}}
|
||||
// Resolve initializes all files; creation replaces the selection before Start.
|
||||
f.meta.Opts = &base.Options{SelectFiles: []int{2}}
|
||||
if err := f.Start(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -1,116 +0,0 @@
|
||||
package bt
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"sync"
|
||||
|
||||
g "github.com/anacrolix/generics"
|
||||
"github.com/anacrolix/torrent/metainfo"
|
||||
"github.com/anacrolix/torrent/storage"
|
||||
)
|
||||
|
||||
// deferredFileStorage lets metadata resolve without opening the user's files.
|
||||
// It binds once. After binding, pieces are supplied directly by file storage.
|
||||
type deferredFileStorage struct {
|
||||
mu sync.RWMutex
|
||||
ctx context.Context
|
||||
info *metainfo.Info
|
||||
hash metainfo.Hash
|
||||
knownHashes []bool
|
||||
owner storage.ClientImplCloser
|
||||
torrent storage.TorrentImpl
|
||||
closed bool
|
||||
}
|
||||
|
||||
func (s *deferredFileStorage) OpenTorrent(ctx context.Context, info *metainfo.Info, hash metainfo.Hash) (storage.TorrentImpl, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.closed || s.info != nil {
|
||||
return storage.TorrentImpl{}, errors.New("torrent storage already opened or closed")
|
||||
}
|
||||
s.ctx, s.info, s.hash = ctx, info, hash
|
||||
s.knownHashes = make([]bool, info.NumPieces())
|
||||
return storage.TorrentImpl{
|
||||
PieceWithHash: func(p metainfo.Piece, hash g.Option[[]byte]) storage.PieceImpl {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.closed {
|
||||
return metadataPiece{err: os.ErrClosed}
|
||||
}
|
||||
if s.owner == nil {
|
||||
s.knownHashes[p.Index()] = hash.Ok
|
||||
// A missing v2 hash must remain unknown until its arrival changes
|
||||
// completion and updates the library's piece request priorities.
|
||||
return metadataPiece{known: hash.Ok}
|
||||
}
|
||||
if s.torrent.PieceWithHash != nil {
|
||||
return s.torrent.PieceWithHash(p, hash)
|
||||
}
|
||||
return s.torrent.Piece(p)
|
||||
},
|
||||
Close: s.Close,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *deferredFileStorage) bind(opts storage.NewFileClientOpts) ([]bool, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.closed {
|
||||
return nil, os.ErrClosed
|
||||
}
|
||||
if s.info == nil {
|
||||
return nil, errors.New("torrent metadata is not ready")
|
||||
}
|
||||
if s.owner != nil {
|
||||
return nil, nil
|
||||
}
|
||||
if opts.FilePathMaker == nil {
|
||||
opts.FilePathMaker = func(o storage.FilePathMakerOpts) string { _, name := torrentFileLayout(o.Info, *o.File); return name }
|
||||
}
|
||||
owner := storage.NewFileOpts(opts)
|
||||
t, err := owner.OpenTorrent(s.ctx, s.info, s.hash)
|
||||
if err != nil {
|
||||
owner.Close()
|
||||
return nil, err
|
||||
}
|
||||
s.owner, s.torrent = owner, t
|
||||
return append([]bool(nil), s.knownHashes...), nil
|
||||
}
|
||||
|
||||
func (s *deferredFileStorage) Close() error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.closed {
|
||||
return nil
|
||||
}
|
||||
s.closed = true
|
||||
if s.owner == nil {
|
||||
return nil
|
||||
}
|
||||
var err error
|
||||
if s.torrent.Close != nil {
|
||||
err = s.torrent.Close()
|
||||
}
|
||||
return errors.Join(err, s.owner.Close())
|
||||
}
|
||||
|
||||
type metadataPiece struct {
|
||||
known bool
|
||||
err error
|
||||
}
|
||||
|
||||
func (p metadataPiece) Completion() storage.Completion {
|
||||
return storage.Completion{Ok: p.known, Err: p.err}
|
||||
}
|
||||
func (p metadataPiece) ReadAt([]byte, int64) (int, error) {
|
||||
return 0, errors.New("download storage is not initialized")
|
||||
}
|
||||
func (p metadataPiece) WriteAt([]byte, int64) (int, error) {
|
||||
return 0, errors.New("download storage is not initialized")
|
||||
}
|
||||
func (p metadataPiece) MarkComplete() error { return errors.New("download storage is not initialized") }
|
||||
func (p metadataPiece) MarkNotComplete() error {
|
||||
return errors.New("download storage is not initialized")
|
||||
}
|
||||
@@ -1,234 +0,0 @@
|
||||
package bt
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/GopeedLab/gopeed/internal/fetcher"
|
||||
)
|
||||
|
||||
var defaultTrackerSubscribeURLs = []string{
|
||||
"https://raw.githubusercontent.com/ngosang/trackerslist/master/trackers_all.txt",
|
||||
"https://raw.githubusercontent.com/ngosang/trackerslist/master/trackers_all_http.txt",
|
||||
"https://raw.githubusercontent.com/ngosang/trackerslist/master/trackers_all_https.txt",
|
||||
"https://raw.githubusercontent.com/ngosang/trackerslist/master/trackers_all_ip.txt",
|
||||
"https://raw.githubusercontent.com/ngosang/trackerslist/master/trackers_all_udp.txt",
|
||||
"https://raw.githubusercontent.com/ngosang/trackerslist/master/trackers_all_ws.txt",
|
||||
"https://raw.githubusercontent.com/ngosang/trackerslist/master/trackers_best.txt",
|
||||
"https://raw.githubusercontent.com/ngosang/trackerslist/master/trackers_best_ip.txt",
|
||||
"https://raw.githubusercontent.com/XIU2/TrackersListCollection/master/all.txt",
|
||||
"https://raw.githubusercontent.com/XIU2/TrackersListCollection/master/best.txt",
|
||||
"https://raw.githubusercontent.com/XIU2/TrackersListCollection/master/http.txt",
|
||||
}
|
||||
|
||||
func validTracker(raw string) bool {
|
||||
u, err := url.Parse(raw)
|
||||
if err != nil || u.Hostname() == "" || u.User != nil {
|
||||
return false
|
||||
}
|
||||
switch u.Scheme {
|
||||
case "udp", "http", "https", "ws", "wss":
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
func uniqueTrackers(groups ...[]string) []string {
|
||||
result := []string{}
|
||||
seen := map[string]bool{}
|
||||
for _, group := range groups {
|
||||
for _, raw := range group {
|
||||
value := strings.TrimSpace(raw)
|
||||
if value != "" && !seen[value] {
|
||||
seen[value] = true
|
||||
result = append(result, value)
|
||||
}
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// effectiveTrackers derives the client list without persisting duplicate lists.
|
||||
func (c *config) effectiveTrackers() []string {
|
||||
groups := [][]string{c.CustomTrackers}
|
||||
for _, source := range c.TrackerSubscribeURLs {
|
||||
groups = append(groups, c.State.TrackerSources[source])
|
||||
}
|
||||
return uniqueTrackers(groups...)
|
||||
}
|
||||
func (fm *FetcherManager) NormalizeConfig(raw json.RawMessage) (any, error) {
|
||||
c := fm.DefaultConfig().(*config)
|
||||
if err := json.Unmarshal(raw, c); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if c.ListenPort < 0 || c.ListenPort > 65535 || c.SeedRatio < 0 || c.SeedTime < 0 {
|
||||
return nil, errors.New("invalid BT configuration")
|
||||
}
|
||||
if len(c.TrackerSubscribeURLs) > 64 {
|
||||
return nil, errors.New("too many tracker subscriptions")
|
||||
}
|
||||
c.TrackerSubscribeURLs = uniqueTrackers(c.TrackerSubscribeURLs)
|
||||
for _, source := range c.TrackerSubscribeURLs {
|
||||
u, err := url.Parse(source)
|
||||
if err != nil || u == nil || (u.Scheme != "https" && u.Scheme != "http") || u.Hostname() == "" || u.User != nil {
|
||||
return nil, errors.New("invalid tracker subscription URL")
|
||||
}
|
||||
}
|
||||
c.CustomTrackers = uniqueTrackers(c.CustomTrackers)
|
||||
for _, tracker := range c.CustomTrackers {
|
||||
if !validTracker(tracker) {
|
||||
return nil, fmt.Errorf("invalid tracker URL: %s", tracker)
|
||||
}
|
||||
}
|
||||
if c.State.TrackerSources == nil {
|
||||
c.State.TrackerSources = map[string][]string{}
|
||||
}
|
||||
for source := range c.State.TrackerSources {
|
||||
if !slices.Contains(c.TrackerSubscribeURLs, source) {
|
||||
delete(c.State.TrackerSources, source)
|
||||
}
|
||||
}
|
||||
if c.State.TrackerSubscriptionStatuses == nil {
|
||||
c.State.TrackerSubscriptionStatuses = map[string]trackerSubscriptionStatus{}
|
||||
}
|
||||
for source := range c.State.TrackerSubscriptionStatuses {
|
||||
if !slices.Contains(c.TrackerSubscribeURLs, source) {
|
||||
delete(c.State.TrackerSubscriptionStatuses, source)
|
||||
}
|
||||
}
|
||||
return c, nil
|
||||
}
|
||||
func (fm *FetcherManager) CronJobs(host fetcher.ProtocolHost) []fetcher.CronJob {
|
||||
return []fetcher.CronJob{{
|
||||
Name: "update-trackers",
|
||||
Every: 24 * time.Hour,
|
||||
Timeout: 90 * time.Second,
|
||||
Run: func(ctx context.Context, source fetcher.TriggerSource) error {
|
||||
return fm.updateTrackers(ctx, host, source)
|
||||
},
|
||||
}}
|
||||
}
|
||||
func parseTrackers(data []byte) ([]string, error) {
|
||||
scanner := bufio.NewScanner(bytes.NewReader(data))
|
||||
scanner.Buffer(make([]byte, 4096), 64*1024)
|
||||
result := []string{}
|
||||
for scanner.Scan() {
|
||||
line := strings.TrimSpace(strings.TrimPrefix(scanner.Text(), "\ufeff"))
|
||||
if validTracker(line) {
|
||||
result = append(result, line)
|
||||
}
|
||||
if len(result) > 20000 {
|
||||
return nil, errors.New("too many trackers")
|
||||
}
|
||||
}
|
||||
if err := scanner.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(result) == 0 {
|
||||
return nil, errors.New("subscription contains no valid trackers")
|
||||
}
|
||||
return uniqueTrackers(result), nil
|
||||
}
|
||||
func (fm *FetcherManager) updateTrackers(ctx context.Context, host fetcher.ProtocolHost, source fetcher.TriggerSource) error {
|
||||
snapshot, err := fetcher.ReadConfig[config](host)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if (source == fetcher.TriggerScheduled && !snapshot.AutoUpdateTrackers) || len(snapshot.TrackerSubscribeURLs) == 0 {
|
||||
return fetcher.ErrJobSkipped
|
||||
}
|
||||
fetched := map[string][]string{}
|
||||
statuses := map[string]trackerSubscriptionStatus{}
|
||||
var failures []error
|
||||
var mu sync.Mutex
|
||||
var wg sync.WaitGroup
|
||||
slots := make(chan struct{}, 4)
|
||||
for _, address := range snapshot.TrackerSubscribeURLs {
|
||||
wg.Add(1)
|
||||
go func(address string) {
|
||||
defer wg.Done()
|
||||
select {
|
||||
case slots <- struct{}{}:
|
||||
defer func() { <-slots }()
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
data, err := host.Fetch(ctx, address, 2<<20, func(body []byte) error { _, err := parseTrackers(body); return err })
|
||||
var trackers []string
|
||||
if err == nil {
|
||||
trackers, err = parseTrackers(data)
|
||||
}
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
statuses[address] = trackerSubscriptionStatus{Success: err == nil, TrackerCount: len(trackers), LastFetchTime: time.Now().UTC()}
|
||||
if err != nil {
|
||||
failures = append(failures, fmt.Errorf("subscription %s: %w", address, err))
|
||||
} else {
|
||||
fetched[address] = trackers
|
||||
}
|
||||
}(address)
|
||||
}
|
||||
wg.Wait()
|
||||
if err := ctx.Err(); err != nil {
|
||||
failures = append(failures, err)
|
||||
}
|
||||
if len(statuses) == 0 {
|
||||
return errors.Join(failures...)
|
||||
}
|
||||
err = fetcher.UpdateState[config, state](host, func(current *config, state *state) error {
|
||||
if !slices.Equal(current.TrackerSubscribeURLs, snapshot.TrackerSubscribeURLs) {
|
||||
return fetcher.ErrConfigChanged
|
||||
}
|
||||
if state.TrackerSources == nil {
|
||||
state.TrackerSources = map[string][]string{}
|
||||
}
|
||||
if state.TrackerSubscriptionStatuses == nil {
|
||||
state.TrackerSubscriptionStatuses = map[string]trackerSubscriptionStatus{}
|
||||
}
|
||||
for address, status := range statuses {
|
||||
state.TrackerSubscriptionStatuses[address] = status
|
||||
}
|
||||
for address, trackers := range fetched {
|
||||
state.TrackerSources[address] = trackers
|
||||
}
|
||||
if len(failures) == 0 {
|
||||
state.LastTrackerUpdateTime = time.Now().UTC()
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
current, err := fetcher.ReadConfig[config](host)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fm.applyTrackers(current.effectiveTrackers())
|
||||
return errors.Join(failures...)
|
||||
}
|
||||
func (fm *FetcherManager) applyTrackers(trackers []string) {
|
||||
lock.Lock()
|
||||
defer lock.Unlock()
|
||||
if client == nil {
|
||||
return
|
||||
}
|
||||
announce := make([][]string, 0, len(trackers))
|
||||
for _, tracker := range trackers {
|
||||
announce = append(announce, []string{tracker})
|
||||
}
|
||||
for _, t := range client.Torrents() {
|
||||
info := t.Info()
|
||||
if info == nil || (info.Private != nil && *info.Private) {
|
||||
continue
|
||||
}
|
||||
t.AddTrackers(announce)
|
||||
}
|
||||
}
|
||||
@@ -1,217 +0,0 @@
|
||||
package bt
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"reflect"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/GopeedLab/gopeed/internal/fetcher"
|
||||
)
|
||||
|
||||
func trackerTestHost(t *testing.T, c *config, fetch func(context.Context, string, int64, func([]byte) error) ([]byte, error)) (fetcher.ProtocolHost, *sync.Mutex) {
|
||||
t.Helper()
|
||||
mu := &sync.Mutex{}
|
||||
return fetcher.ProtocolHost{
|
||||
Read: func(dst any) error {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
raw, err := json.Marshal(c)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return json.Unmarshal(raw, dst)
|
||||
},
|
||||
UpdateState: func(change func(config, state json.RawMessage) (any, error)) error {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
raw, _ := json.Marshal(c)
|
||||
currentState, _ := json.Marshal(c.State)
|
||||
next, err := change(raw, currentState)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
raw, err = json.Marshal(next)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return json.Unmarshal(raw, &c.State)
|
||||
},
|
||||
Fetch: fetch,
|
||||
}, mu
|
||||
}
|
||||
func TestTrackerRefreshPartialSuccessUsesLatestConfig(t *testing.T) {
|
||||
fm := &FetcherManager{}
|
||||
cfg := fm.DefaultConfig().(*config)
|
||||
cfg.TrackerSubscribeURLs = []string{"https://a.example/list", "https://b.example/list"}
|
||||
cfg.CustomTrackers = []string{"udp://custom.example:80/announce"}
|
||||
cfg.State.TrackerSources = map[string][]string{cfg.TrackerSubscribeURLs[0]: {"udp://old.example:80/announce"}, cfg.TrackerSubscribeURLs[1]: {"udp://cached.example:80/announce"}}
|
||||
host, mu := trackerTestHost(t, cfg, nil)
|
||||
host.Fetch = func(ctx context.Context, address string, _ int64, validate func([]byte) error) ([]byte, error) {
|
||||
if address == "https://b.example/list" {
|
||||
return nil, errors.New("offline")
|
||||
}
|
||||
// Simulate a frontend update while the network request is in flight.
|
||||
mu.Lock()
|
||||
cfg.CustomTrackers = []string{"udp://new-custom.example:80/announce"}
|
||||
mu.Unlock()
|
||||
data := []byte("udp://fresh.example:80/announce\n# ignored\nudp://fresh.example:80/announce\n")
|
||||
return data, validate(data)
|
||||
}
|
||||
err := fm.updateTrackers(context.Background(), host, fetcher.TriggerManual)
|
||||
if err == nil {
|
||||
t.Fatal("partial failure must be reported")
|
||||
}
|
||||
want := []string{"udp://new-custom.example:80/announce", "udp://fresh.example:80/announce", "udp://cached.example:80/announce"}
|
||||
if !reflect.DeepEqual(cfg.effectiveTrackers(), want) {
|
||||
t.Fatalf("got %v", cfg.effectiveTrackers())
|
||||
}
|
||||
success := cfg.State.TrackerSubscriptionStatuses["https://a.example/list"]
|
||||
failure := cfg.State.TrackerSubscriptionStatuses["https://b.example/list"]
|
||||
if !success.Success || success.TrackerCount != 1 || success.LastFetchTime.IsZero() || failure.Success || failure.TrackerCount != 0 || failure.LastFetchTime.IsZero() {
|
||||
t.Fatalf("unexpected subscription statuses: %+v", cfg.State.TrackerSubscriptionStatuses)
|
||||
}
|
||||
if !cfg.State.LastTrackerUpdateTime.IsZero() {
|
||||
t.Fatal("partial refresh marked fully successful")
|
||||
}
|
||||
}
|
||||
func TestTrackerRefreshDiscardsChangedSubscriptions(t *testing.T) {
|
||||
fm := &FetcherManager{}
|
||||
cfg := fm.DefaultConfig().(*config)
|
||||
cfg.TrackerSubscribeURLs = []string{"https://a.example/list"}
|
||||
host, mu := trackerTestHost(t, cfg, nil)
|
||||
host.Fetch = func(context.Context, string, int64, func([]byte) error) ([]byte, error) {
|
||||
mu.Lock()
|
||||
cfg.TrackerSubscribeURLs = []string{}
|
||||
mu.Unlock()
|
||||
return []byte("udp://fresh.example:80/announce"), nil
|
||||
}
|
||||
if err := fm.updateTrackers(context.Background(), host, fetcher.TriggerManual); !errors.Is(err, fetcher.ErrConfigChanged) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(cfg.State.TrackerSources) != 0 {
|
||||
t.Fatal("outdated source was committed")
|
||||
}
|
||||
}
|
||||
func TestTrackerRefreshManualWorksWhenAutomaticDisabled(t *testing.T) {
|
||||
fm := &FetcherManager{}
|
||||
cfg := fm.DefaultConfig().(*config)
|
||||
cfg.AutoUpdateTrackers = false
|
||||
cfg.TrackerSubscribeURLs = []string{"https://a.example/list"}
|
||||
calls := 0
|
||||
host, _ := trackerTestHost(t, cfg, func(context.Context, string, int64, func([]byte) error) ([]byte, error) {
|
||||
calls++
|
||||
return []byte("udp://fresh.example:80/announce"), nil
|
||||
})
|
||||
if err := fm.updateTrackers(context.Background(), host, fetcher.TriggerScheduled); !errors.Is(err, fetcher.ErrJobSkipped) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if calls != 0 {
|
||||
t.Fatal("scheduled job fetched while disabled")
|
||||
}
|
||||
if err := fm.updateTrackers(context.Background(), host, fetcher.TriggerManual); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if calls != 1 || cfg.State.LastTrackerUpdateTime.IsZero() {
|
||||
t.Fatal("manual refresh did not succeed")
|
||||
}
|
||||
}
|
||||
func TestTrackerConfigPreservesExplicitSettings(t *testing.T) {
|
||||
value, err := (&FetcherManager{}).NormalizeConfig(json.RawMessage(`{
|
||||
"listenPort":6881, "trackerSubscribeUrls":[], "autoUpdateTrackers":false,
|
||||
"customTrackers":["udp://custom.example:80/announce"]
|
||||
}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cfg := value.(*config)
|
||||
if cfg.AutoUpdateTrackers || len(cfg.TrackerSubscribeURLs) != 0 || cfg.ListenPort != 6881 {
|
||||
t.Fatalf("%+v", cfg)
|
||||
}
|
||||
if !reflect.DeepEqual(cfg.effectiveTrackers(), cfg.CustomTrackers) {
|
||||
t.Fatalf("derived trackers: %v", cfg.effectiveTrackers())
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseTrackerList(t *testing.T) {
|
||||
got, err := parseTrackers([]byte("\ufeffudp://tracker.example:80/announce\r\n\n<html>error</html>\nhttps://tracker.example/announce\n"))
|
||||
if err != nil || len(got) != 2 {
|
||||
t.Fatalf("%v %v", got, err)
|
||||
}
|
||||
if _, err := parseTrackers([]byte("<html>mirror error</html>")); err == nil {
|
||||
t.Fatal("invalid mirror body accepted")
|
||||
}
|
||||
}
|
||||
func TestTrackerDefaultsRegisterDailyJob(t *testing.T) {
|
||||
jobs := (&FetcherManager{}).CronJobs(fetcher.ProtocolHost{})
|
||||
if len(jobs) != 1 || jobs[0].Name != "update-trackers" || jobs[0].Every != 24*time.Hour {
|
||||
t.Fatal(jobs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTrackerRefreshAllFailuresPersistStatusesAndKeepCache(t *testing.T) {
|
||||
fm := &FetcherManager{}
|
||||
cfg := fm.DefaultConfig().(*config)
|
||||
address := "https://custom.example/trackers.txt"
|
||||
cfg.TrackerSubscribeURLs = []string{address}
|
||||
cfg.State.TrackerSources[address] = []string{"udp://cached.example:80/announce"}
|
||||
cfg.State.TrackerSubscriptionStatuses[address] = trackerSubscriptionStatus{Success: true, TrackerCount: 1, LastFetchTime: time.Now().Add(-time.Hour)}
|
||||
host, _ := trackerTestHost(t, cfg, func(context.Context, string, int64, func([]byte) error) ([]byte, error) {
|
||||
return nil, errors.New("offline")
|
||||
})
|
||||
before := time.Now()
|
||||
if err := fm.updateTrackers(context.Background(), host, fetcher.TriggerManual); err == nil {
|
||||
t.Fatal("failure was not reported")
|
||||
}
|
||||
status, ok := cfg.State.TrackerSubscriptionStatuses[address]
|
||||
if !ok || status.Success || status.TrackerCount != 0 || status.LastFetchTime.Before(before) {
|
||||
t.Fatalf("failed fetch status not stored: %+v", status)
|
||||
}
|
||||
if !reflect.DeepEqual(cfg.State.TrackerSources[address], []string{"udp://cached.example:80/announce"}) {
|
||||
t.Fatal("last good cache lost")
|
||||
}
|
||||
if !reflect.DeepEqual(cfg.effectiveTrackers(), cfg.State.TrackerSources[address]) {
|
||||
t.Fatal("cached trackers not retained")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTrackerConfigPrunesRemovedSubscriptions(t *testing.T) {
|
||||
fm := &FetcherManager{}
|
||||
cfg := fm.DefaultConfig().(*config)
|
||||
address := "https://custom.example/trackers.txt"
|
||||
cfg.TrackerSubscribeURLs = []string{}
|
||||
cfg.State.TrackerSources[address] = []string{"udp://cached.example:80/announce"}
|
||||
cfg.State.TrackerSubscriptionStatuses[address] = trackerSubscriptionStatus{Success: true, TrackerCount: 1, LastFetchTime: time.Now()}
|
||||
raw, _ := json.Marshal(cfg)
|
||||
value, err := fm.NormalizeConfig(raw)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
next := value.(*config)
|
||||
if len(next.State.TrackerSources) != 0 || len(next.State.TrackerSubscriptionStatuses) != 0 || len(next.effectiveTrackers()) != 0 {
|
||||
t.Fatal("removed subscription still owns cached results")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTrackerRefreshTimeoutPersistsCompletedAttemptStatus(t *testing.T) {
|
||||
fm := &FetcherManager{}
|
||||
cfg := fm.DefaultConfig().(*config)
|
||||
address := "https://custom.example/trackers.txt"
|
||||
cfg.TrackerSubscribeURLs = []string{address}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
host, _ := trackerTestHost(t, cfg, func(context.Context, string, int64, func([]byte) error) ([]byte, error) {
|
||||
cancel()
|
||||
return nil, context.DeadlineExceeded
|
||||
})
|
||||
if err := fm.updateTrackers(ctx, host, fetcher.TriggerManual); !errors.Is(err, context.DeadlineExceeded) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
status, ok := cfg.State.TrackerSubscriptionStatuses[address]
|
||||
if !ok || status.Success || status.TrackerCount != 0 || status.LastFetchTime.IsZero() {
|
||||
t.Fatalf("timeout status lost: %+v", status)
|
||||
}
|
||||
}
|
||||
@@ -67,14 +67,17 @@ func (f *Fetcher) Setup(ctl *controller.Controller) {
|
||||
f.ctl.GetConfig(&f.config)
|
||||
}
|
||||
|
||||
func (f *Fetcher) Resolve(req *base.Request) error {
|
||||
func (f *Fetcher) Resolve(req *base.Request, opts *base.Options) error {
|
||||
link, err := parseLink(req.URL)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
f.meta.Req = req
|
||||
|
||||
f.meta.Opts = opts
|
||||
if f.meta.Opts == nil {
|
||||
f.meta.Opts = &base.Options{}
|
||||
}
|
||||
f.meta.Res = buildResource(link)
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -13,8 +13,7 @@ func TestFetcher_Resolve(t *testing.T) {
|
||||
f := (&FetcherManager{}).Build()
|
||||
f.Setup(controller.NewController())
|
||||
|
||||
f.Meta().Opts = &base.Options{Path: t.TempDir()}
|
||||
err := f.Resolve(&base.Request{URL: testLink})
|
||||
err := f.Resolve(&base.Request{URL: testLink}, &base.Options{Path: t.TempDir()})
|
||||
if err != nil {
|
||||
t.Fatalf("Resolve() error = %v", err)
|
||||
}
|
||||
|
||||
@@ -1,404 +0,0 @@
|
||||
package ftp
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/textproto"
|
||||
"net/url"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/jlaffaye/ftp"
|
||||
"golang.org/x/net/proxy"
|
||||
|
||||
pftp "github.com/GopeedLab/gopeed/pkg/protocol/ftp"
|
||||
)
|
||||
|
||||
const (
|
||||
connectTimeout = 15 * time.Second
|
||||
readTimeout = 15 * time.Second
|
||||
quitTimeout = 2 * time.Second
|
||||
)
|
||||
|
||||
// target is a parsed FTP URL. It holds the password, so it must never be
|
||||
// formatted into an error or a log line.
|
||||
type target struct {
|
||||
host string // host name or IP, the TLS server name
|
||||
addr string // host:port
|
||||
user string
|
||||
pass string
|
||||
path string // path relative to the login directory, "" for the directory itself
|
||||
tlsMode string
|
||||
}
|
||||
|
||||
// parseTarget reads an ftp, ftps or ftpes URL. The path follows RFC 1738: it
|
||||
// is relative to the login directory, and an encoded leading slash
|
||||
// (ftp://host/%2Fpub) makes it absolute.
|
||||
func parseTarget(rawURL, optTLS string) (*target, error) {
|
||||
u, err := url.Parse(rawURL)
|
||||
if err != nil {
|
||||
// url.Error repeats the whole URL, password included.
|
||||
return nil, errors.New("ftp: invalid URL")
|
||||
}
|
||||
t := &target{}
|
||||
switch strings.ToLower(u.Scheme) {
|
||||
case "ftps":
|
||||
t.tlsMode = pftp.TLSImplicit
|
||||
case "ftpes":
|
||||
t.tlsMode = pftp.TLSExplicit
|
||||
case "ftp":
|
||||
switch strings.ToLower(optTLS) {
|
||||
case "", pftp.TLSNone:
|
||||
t.tlsMode = pftp.TLSNone
|
||||
case pftp.TLSExplicit:
|
||||
t.tlsMode = pftp.TLSExplicit
|
||||
case pftp.TLSImplicit:
|
||||
t.tlsMode = pftp.TLSImplicit
|
||||
default:
|
||||
return nil, fmt.Errorf("ftp: unknown tls mode %q", optTLS)
|
||||
}
|
||||
default:
|
||||
return nil, fmt.Errorf("ftp: unsupported scheme %q", u.Scheme)
|
||||
}
|
||||
t.host = u.Hostname()
|
||||
if t.host == "" {
|
||||
return nil, errors.New("ftp: URL has no host")
|
||||
}
|
||||
port := u.Port()
|
||||
if port == "" {
|
||||
port = "21"
|
||||
if t.tlsMode == pftp.TLSImplicit {
|
||||
port = "990"
|
||||
}
|
||||
}
|
||||
t.addr = net.JoinHostPort(t.host, port)
|
||||
if u.User != nil {
|
||||
// Username and Password are already percent-decoded.
|
||||
t.user = u.User.Username()
|
||||
t.pass, _ = u.User.Password()
|
||||
}
|
||||
if t.user == "" {
|
||||
t.user, t.pass = "anonymous", "anonymous@"
|
||||
}
|
||||
t.path = strings.TrimPrefix(u.Path, "/")
|
||||
return t, nil
|
||||
}
|
||||
|
||||
type dialContextFunc func(ctx context.Context, network, addr string) (net.Conn, error)
|
||||
|
||||
// session is one logged-in control connection. It owns the network
|
||||
// connections it opens, so abort can unblock any read or write in progress.
|
||||
type session struct {
|
||||
*ftp.ServerConn
|
||||
|
||||
ctrl net.Conn // control connection as dialed, under any TLS layer
|
||||
dataHost string // where data connections go
|
||||
|
||||
mu sync.Mutex
|
||||
data net.Conn // current data connection as dialed
|
||||
closed bool
|
||||
stop func() bool
|
||||
}
|
||||
|
||||
// deadline bounds the next control-connection exchange. The control
|
||||
// connection is idle during a transfer, so every exchange sets a new one.
|
||||
func (s *session) deadline() {
|
||||
_ = s.ctrl.SetDeadline(time.Now().Add(readTimeout))
|
||||
}
|
||||
|
||||
// clearDeadline lets the control connection stay idle through a transfer.
|
||||
func (s *session) clearDeadline() {
|
||||
_ = s.ctrl.SetDeadline(time.Time{})
|
||||
}
|
||||
|
||||
// abort closes every connection at once. Pending calls fail immediately.
|
||||
func (s *session) abort() {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.closed = true
|
||||
_ = s.ctrl.Close()
|
||||
if s.data != nil {
|
||||
_ = s.data.Close()
|
||||
}
|
||||
}
|
||||
|
||||
// quit logs out and waits, briefly, for the server to hang up. Servers that
|
||||
// cap logins count a session until they have seen it end, so the next login
|
||||
// would otherwise be refused.
|
||||
func (s *session) quit() {
|
||||
if s.stop != nil {
|
||||
s.stop()
|
||||
}
|
||||
_ = s.ctrl.SetDeadline(time.Now().Add(quitTimeout))
|
||||
if s.ServerConn != nil {
|
||||
_ = s.ServerConn.Quit()
|
||||
}
|
||||
s.abort()
|
||||
}
|
||||
|
||||
func (s *session) setData(c net.Conn) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.closed {
|
||||
_ = c.Close()
|
||||
return net.ErrClosed
|
||||
}
|
||||
s.data = c
|
||||
return nil
|
||||
}
|
||||
|
||||
// ctrlConn is the control connection handed to the ftp library.
|
||||
type ctrlConn struct {
|
||||
net.Conn
|
||||
}
|
||||
|
||||
// RemoteAddr must be a *net.TCPAddr: the library asserts it. A connection
|
||||
// through a proxy may report something else, and its address is not used
|
||||
// for data connections anyway (see dialData).
|
||||
func (c ctrlConn) RemoteAddr() net.Addr {
|
||||
if a, ok := c.Conn.RemoteAddr().(*net.TCPAddr); ok {
|
||||
return a
|
||||
}
|
||||
return &net.TCPAddr{}
|
||||
}
|
||||
|
||||
// Close is called by the library's Quit right after it sends QUIT. Reading
|
||||
// until the server hangs up (bounded by the deadline quit set) tells us the
|
||||
// server has ended the session.
|
||||
func (c ctrlConn) Close() error {
|
||||
_, _ = io.Copy(io.Discard, c.Conn)
|
||||
return c.Conn.Close()
|
||||
}
|
||||
|
||||
// dial opens a control connection, upgrades it to TLS when asked, and logs in.
|
||||
//
|
||||
// The library is given a dial function for every connection. That is how
|
||||
// data connections go through the same proxy as the control connection, but
|
||||
// it also means the library does no TLS on them: with a dial function it
|
||||
// neither wraps an implicit-TLS control connection nor any data connection.
|
||||
// We do both here. Explicit TLS (AUTH TLS) is still done by the library, and
|
||||
// so are PBSZ and PROT P after login. For the same reason the library's
|
||||
// DialWithTimeout and DialWithContext would have no effect: connectTimeout
|
||||
// and ctx are applied to our own dials instead.
|
||||
func (f *Fetcher) dial(ctx context.Context, t *target) (*session, error) {
|
||||
netDial, proxied, err := f.netDialer(t)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var tlsConf *tls.Config
|
||||
if t.tlsMode != pftp.TLSNone {
|
||||
tlsConf = f.tlsConfig(t.host)
|
||||
}
|
||||
|
||||
dialCtx, cancel := context.WithTimeout(ctx, connectTimeout)
|
||||
defer cancel()
|
||||
raw, err := netDial(dialCtx, "tcp", t.addr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ctrl := raw
|
||||
if t.tlsMode == pftp.TLSImplicit {
|
||||
tc := tls.Client(raw, tlsConf)
|
||||
if err := tc.HandshakeContext(dialCtx); err != nil {
|
||||
_ = raw.Close()
|
||||
return nil, err
|
||||
}
|
||||
ctrl = tc
|
||||
}
|
||||
|
||||
s := &session{ctrl: raw, dataHost: t.host}
|
||||
if a, ok := raw.RemoteAddr().(*net.TCPAddr); ok && !proxied {
|
||||
// Same IP as the control connection, as the library itself does,
|
||||
// so a name with several addresses cannot send us elsewhere.
|
||||
s.dataHost = a.IP.String()
|
||||
}
|
||||
s.stop = context.AfterFunc(ctx, s.abort)
|
||||
s.deadline()
|
||||
|
||||
handedOut := false
|
||||
opts := []ftp.DialOption{
|
||||
ftp.DialWithShutTimeout(readTimeout),
|
||||
ftp.DialWithDialFunc(func(network, addr string) (net.Conn, error) {
|
||||
if !handedOut {
|
||||
// Dial asks for the control connection first, once.
|
||||
handedOut = true
|
||||
return ctrlConn{ctrl}, nil
|
||||
}
|
||||
return s.dialData(ctx, netDial, tlsConf, addr)
|
||||
}),
|
||||
}
|
||||
switch t.tlsMode {
|
||||
case pftp.TLSImplicit:
|
||||
opts = append(opts, ftp.DialWithTLS(tlsConf))
|
||||
case pftp.TLSExplicit:
|
||||
opts = append(opts, ftp.DialWithExplicitTLS(tlsConf))
|
||||
}
|
||||
c, err := ftp.Dial(t.addr, opts...)
|
||||
if err != nil {
|
||||
s.stop()
|
||||
s.abort()
|
||||
return nil, err
|
||||
}
|
||||
s.ServerConn = c
|
||||
s.deadline()
|
||||
if err := c.Login(t.user, t.pass); err != nil {
|
||||
s.stop()
|
||||
s.abort()
|
||||
return nil, err
|
||||
}
|
||||
return s, nil
|
||||
}
|
||||
|
||||
// dialData opens a passive data connection. The library takes the host from
|
||||
// the control connection's peer, which through a proxy is the proxy, so only
|
||||
// the port is used here.
|
||||
func (s *session) dialData(ctx context.Context, netDial dialContextFunc, tlsConf *tls.Config, addr string) (net.Conn, error) {
|
||||
_, port, err := net.SplitHostPort(addr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
dialCtx, cancel := context.WithTimeout(ctx, connectTimeout)
|
||||
defer cancel()
|
||||
raw, err := netDial(dialCtx, "tcp", net.JoinHostPort(s.dataHost, port))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := s.setData(raw); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if tlsConf != nil {
|
||||
// The handshake runs on the first read, as in the library, because
|
||||
// some servers only start it once the transfer command is answered.
|
||||
return tls.Client(raw, tlsConf), nil
|
||||
}
|
||||
return raw, nil
|
||||
}
|
||||
|
||||
// tlsConfig returns the TLS settings for one control connection and its data
|
||||
// connections. Certificates are verified unless the request sets
|
||||
// skipVerifyCert, as for HTTP.
|
||||
//
|
||||
// Each call gets its own one-entry session cache, so a data connection
|
||||
// resumes the TLS session of its own control connection. vsftpd
|
||||
// (require_ssl_reuse) only checks that some session was resumed, but ProFTPD's
|
||||
// mod_tls also checks that it is the control connection's; a cache shared by
|
||||
// several workers would hand a data connection another worker's session.
|
||||
// A cache in the manager's base config is kept as given.
|
||||
func (f *Fetcher) tlsConfig(host string) *tls.Config {
|
||||
cfg := &tls.Config{}
|
||||
if f.manager != nil && f.manager.baseTLS != nil {
|
||||
cfg = f.manager.baseTLS.Clone()
|
||||
}
|
||||
cfg.ServerName = host
|
||||
if f.meta.Req != nil && f.meta.Req.SkipVerifyCert {
|
||||
cfg.InsecureSkipVerify = true
|
||||
}
|
||||
if cfg.ClientSessionCache == nil {
|
||||
cfg.ClientSessionCache = tls.NewLRUClientSessionCache(1)
|
||||
}
|
||||
return cfg
|
||||
}
|
||||
|
||||
// netDialer picks how to reach the server, from the same proxy settings the
|
||||
// HTTP protocol reads (the task's proxy, otherwise the global one).
|
||||
func (f *Fetcher) netDialer(t *target) (dialContextFunc, bool, error) {
|
||||
direct := (&net.Dialer{Timeout: connectTimeout}).DialContext
|
||||
pu, err := f.proxyURL(t)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
if pu == nil {
|
||||
return direct, false, nil
|
||||
}
|
||||
switch strings.ToLower(pu.Scheme) {
|
||||
case "socks5", "socks5h":
|
||||
d, err := proxy.FromURL(pu, &net.Dialer{Timeout: connectTimeout})
|
||||
if err != nil {
|
||||
return nil, false, fmt.Errorf("ftp: socks5 proxy: %w", err)
|
||||
}
|
||||
if cd, ok := d.(proxy.ContextDialer); ok {
|
||||
return cd.DialContext, true, nil
|
||||
}
|
||||
return func(_ context.Context, network, addr string) (net.Conn, error) {
|
||||
return d.Dial(network, addr)
|
||||
}, true, nil
|
||||
default:
|
||||
// An HTTP proxy is not used. Every passive data connection goes to
|
||||
// a port the server picks, and HTTP proxies usually allow CONNECT to
|
||||
// port 443 only, so the transfer would fail after the login worked.
|
||||
f.warnOnce.Do(func() {
|
||||
if l := f.logger(); l != nil {
|
||||
l.Warn().Msgf("ftp: the %s proxy at %s cannot carry FTP data connections (HTTP proxy), connecting directly; use a SOCKS5 proxy for FTP", strings.ToLower(pu.Scheme), pu.Host)
|
||||
}
|
||||
})
|
||||
return direct, false, nil
|
||||
}
|
||||
}
|
||||
|
||||
func (f *Fetcher) proxyURL(t *target) (*url.URL, error) {
|
||||
if f.ctl == nil || f.ctl.GetProxy == nil {
|
||||
return nil, nil
|
||||
}
|
||||
handler := f.ctl.GetProxy(f.meta.Req.Proxy)
|
||||
if handler == nil {
|
||||
return nil, nil
|
||||
}
|
||||
// The proxy chooser reads only the URL; the credentials stay out.
|
||||
req := &http.Request{URL: &url.URL{Scheme: "ftp", Host: t.addr, Path: "/" + t.path}, Header: http.Header{}}
|
||||
pu, err := handler(req)
|
||||
if err != nil {
|
||||
return nil, errors.New("ftp: cannot read the proxy settings")
|
||||
}
|
||||
return pu, nil
|
||||
}
|
||||
|
||||
// isLoginRefusal tells a server that is full apart from bad credentials.
|
||||
// 421 always means "not now". 530 means a refused login; it is only a
|
||||
// capacity limit when these credentials have worked before. The library
|
||||
// drops the code when USER itself is refused, so a "too many" message counts
|
||||
// as well.
|
||||
func isLoginRefusal(err error, credentialsWorked bool) bool {
|
||||
var te *textproto.Error
|
||||
if errors.As(err, &te) {
|
||||
switch te.Code {
|
||||
case ftp.StatusNotAvailable:
|
||||
return true
|
||||
case ftp.StatusNotLoggedIn:
|
||||
return credentialsWorked || strings.Contains(strings.ToLower(te.Msg), "too many")
|
||||
}
|
||||
return false
|
||||
}
|
||||
return strings.Contains(strings.ToLower(err.Error()), "too many")
|
||||
}
|
||||
|
||||
// isPermanent reports errors a retry cannot fix: a 5xx reply and a
|
||||
// certificate that does not verify.
|
||||
func isPermanent(err error) bool {
|
||||
var te *textproto.Error
|
||||
if errors.As(err, &te) {
|
||||
return te.Code >= 500
|
||||
}
|
||||
var verifyErr *tls.CertificateVerificationError
|
||||
if errors.As(err, &verifyErr) {
|
||||
return true
|
||||
}
|
||||
var recordErr tls.RecordHeaderError
|
||||
if errors.As(err, &recordErr) {
|
||||
return true
|
||||
}
|
||||
return errors.Is(err, errSizeChanged)
|
||||
}
|
||||
|
||||
func replyCode(err error) int {
|
||||
var te *textproto.Error
|
||||
if errors.As(err, &te) {
|
||||
return te.Code
|
||||
}
|
||||
return 0
|
||||
}
|
||||
@@ -1,146 +0,0 @@
|
||||
package ftp_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/rs/zerolog"
|
||||
|
||||
"github.com/GopeedLab/gopeed/internal/logger"
|
||||
iftp "github.com/GopeedLab/gopeed/internal/protocol/ftp"
|
||||
"github.com/GopeedLab/gopeed/pkg/base"
|
||||
"github.com/GopeedLab/gopeed/pkg/download"
|
||||
pftp "github.com/GopeedLab/gopeed/pkg/protocol/ftp"
|
||||
)
|
||||
|
||||
// syncBuffer is a log sink that the downloader's goroutines can share.
|
||||
type syncBuffer struct {
|
||||
mu sync.Mutex
|
||||
buf bytes.Buffer
|
||||
}
|
||||
|
||||
func (b *syncBuffer) Write(p []byte) (int, error) {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
return b.buf.Write(p)
|
||||
}
|
||||
|
||||
func (b *syncBuffer) String() string {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
return b.buf.String()
|
||||
}
|
||||
|
||||
// taskEvents records the first done or error event of every task, so the
|
||||
// test never reads a task's status while the downloader writes it.
|
||||
type taskEvents struct {
|
||||
mu sync.Mutex
|
||||
final map[string]download.EventKey
|
||||
}
|
||||
|
||||
func (e *taskEvents) listen(ev *download.Event) {
|
||||
if ev.Key != download.EventKeyDone && ev.Key != download.EventKeyError {
|
||||
return
|
||||
}
|
||||
e.mu.Lock()
|
||||
defer e.mu.Unlock()
|
||||
if _, seen := e.final[ev.Task.ID]; !seen {
|
||||
e.final[ev.Task.ID] = ev.Key
|
||||
}
|
||||
}
|
||||
|
||||
func (e *taskEvents) wait(t *testing.T, id string) download.EventKey {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(60 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
e.mu.Lock()
|
||||
key, ok := e.final[id]
|
||||
e.mu.Unlock()
|
||||
if ok {
|
||||
return key
|
||||
}
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
}
|
||||
t.Fatalf("task %s did not finish", id)
|
||||
return ""
|
||||
}
|
||||
|
||||
// TestPasswordNeverLogged runs FTP tasks through the downloader with its core
|
||||
// logger captured: a success, a missing file and a wrong password, all behind
|
||||
// an HTTP proxy that FTP must skip with a warning.
|
||||
func TestPasswordNeverLogged(t *testing.T) {
|
||||
srv := iftp.StartFTPServer(t, iftp.ServerOptions{User: "u", Pass: "p@ss"})
|
||||
want := srv.WriteFile(t, "secret.bin", iftp.RandomBytes(300000, 21))
|
||||
|
||||
d := download.NewDownloader(&download.DownloaderConfig{Storage: download.NewMemStorage(), StorageDir: t.TempDir()})
|
||||
logs := &syncBuffer{}
|
||||
d.Logger = &logger.Logger{Logger: zerolog.New(logs)}
|
||||
events := &taskEvents{final: map[string]download.EventKey{}}
|
||||
d.Listener(events.listen)
|
||||
if err := d.Setup(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Close, not Clear: Clear resets the extension list without a lock while
|
||||
// the last task's done hooks may still read it.
|
||||
defer d.Close()
|
||||
cfg, err := d.GetConfig()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cfg.Proxy = &base.DownloaderProxyConfig{Enable: true, Scheme: "http", Host: "127.0.0.1:1", Usr: "proxyuser", Pwd: "proxysecret"}
|
||||
if err := d.PutConfig(cfg); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
dir := t.TempDir()
|
||||
create := func(rawURL string) string {
|
||||
id, err := d.CreateDirect(&base.Request{URL: rawURL}, &base.Options{Path: dir, Extra: &pftp.OptsExtra{Connections: 2}})
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "p@ss") || strings.Contains(err.Error(), "p%40ss") {
|
||||
t.Fatalf("the password leaked into an error: %v", err)
|
||||
}
|
||||
t.Fatal(err)
|
||||
}
|
||||
return id
|
||||
}
|
||||
root := "ftp://u:p%40ss@" + srv.Addr + "/"
|
||||
|
||||
okID := create(root + "secret.bin")
|
||||
if key := events.wait(t, okID); key != download.EventKeyDone {
|
||||
t.Fatalf("download ended with %q, logs:\n%s", key, logs.String())
|
||||
}
|
||||
ok := d.GetTask(okID)
|
||||
if ok.Protocol != "ftp" {
|
||||
t.Fatalf("protocol %q, want ftp", ok.Protocol)
|
||||
}
|
||||
if got := iftp.FileSHA256(t, ok.Meta.SingleFilepath()); got != want {
|
||||
t.Fatalf("sha256 %s, want %s", got, want)
|
||||
}
|
||||
missingID := create(root + "missing.bin")
|
||||
if key := events.wait(t, missingID); key != download.EventKeyError {
|
||||
t.Fatalf("a missing file must fail, got %q", key)
|
||||
}
|
||||
wrongID := create("ftp://u:wr0ng%40pw@" + srv.Addr + "/secret.bin")
|
||||
if key := events.wait(t, wrongID); key != download.EventKeyError {
|
||||
t.Fatalf("a wrong password must fail, got %q", key)
|
||||
}
|
||||
missing, wrong := d.GetTask(missingID), d.GetTask(wrongID)
|
||||
for _, task := range []*download.Task{ok, missing, wrong} {
|
||||
if name := task.Name(); strings.Contains(name, "p@ss") || strings.Contains(name, "p%40ss") || strings.Contains(name, "wr0ng") {
|
||||
t.Fatalf("task name %q shows the password", name)
|
||||
}
|
||||
}
|
||||
|
||||
out := logs.String()
|
||||
if !strings.Contains(out, "task id") || !strings.Contains(out, "HTTP proxy") {
|
||||
t.Fatalf("the captured log misses the expected lines, so it proves nothing:\n%s", out)
|
||||
}
|
||||
for _, secret := range []string{"p@ss", "p%40ss", "wr0ng", "proxysecret"} {
|
||||
if strings.Contains(out, secret) {
|
||||
t.Fatalf("%q appears in the log:\n%s", secret, out)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,810 +0,0 @@
|
||||
// Package ftp downloads files and directory trees over FTP, FTPS (implicit
|
||||
// TLS) and FTPES (explicit TLS, AUTH TLS).
|
||||
//
|
||||
// A file is split into segments of at least 1 MiB. Each segment is fetched on
|
||||
// its own login with REST and RETR, and its data connection is closed once
|
||||
// the segment's last byte has arrived. A server that refuses a login lowers
|
||||
// the task's connection ceiling to the logins that worked, and a server that
|
||||
// refuses REST gets one sequential stream per file.
|
||||
package ftp
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/jlaffaye/ftp"
|
||||
"github.com/rs/zerolog"
|
||||
|
||||
"github.com/GopeedLab/gopeed/internal/controller"
|
||||
"github.com/GopeedLab/gopeed/internal/fetcher"
|
||||
"github.com/GopeedLab/gopeed/pkg/base"
|
||||
pftp "github.com/GopeedLab/gopeed/pkg/protocol/ftp"
|
||||
)
|
||||
|
||||
const (
|
||||
// Many servers cap logins per user, so FTP starts lower than HTTP.
|
||||
defaultConnections = 4
|
||||
maxConnections = 256
|
||||
minSegmentSize = 1 << 20
|
||||
maxRetries = 5
|
||||
copyBufferSize = 256 << 10
|
||||
)
|
||||
|
||||
// retryDelay is the first wait before a retry. Later retries wait longer.
|
||||
var retryDelay = 500 * time.Millisecond
|
||||
|
||||
var (
|
||||
errSizeChanged = errors.New("ftp: the remote file changed size since the task was resolved")
|
||||
errShortTransfer = errors.New("ftp: the transfer ended before the segment was complete")
|
||||
errRestUnsupported = errors.New("ftp: the server does not support REST")
|
||||
errDiskWrite = errors.New("ftp: cannot write the downloaded data")
|
||||
)
|
||||
|
||||
type config struct {
|
||||
Connections int `json:"connections"`
|
||||
}
|
||||
|
||||
// segment is a byte range of one file. End is inclusive, so an empty file
|
||||
// has End == Begin-1 and nothing to fetch.
|
||||
type segment struct {
|
||||
File int `json:",omitempty"` // index in Res.Files
|
||||
Begin int64
|
||||
End int64
|
||||
Downloaded int64
|
||||
}
|
||||
|
||||
func (s *segment) remain() int64 {
|
||||
return s.End - s.Begin + 1 - s.Downloaded
|
||||
}
|
||||
|
||||
// lane is one worker that logged in, shown as one connection in the stats.
|
||||
type lane struct {
|
||||
downloaded int64
|
||||
retryTimes int
|
||||
failed bool
|
||||
completed bool
|
||||
}
|
||||
|
||||
// loginError marks a failure to connect or log in, as opposed to a failure
|
||||
// during a transfer.
|
||||
type loginError struct{ err error }
|
||||
|
||||
func (e *loginError) Error() string { return e.err.Error() }
|
||||
func (e *loginError) Unwrap() error { return e.err }
|
||||
|
||||
type Fetcher struct {
|
||||
manager *FetcherManager
|
||||
ctl *controller.Controller
|
||||
config *config
|
||||
meta *fetcher.FetcherMeta
|
||||
doneCh chan error
|
||||
|
||||
warnOnce sync.Once
|
||||
// loginOK is set once these credentials have logged in, after which a
|
||||
// 530 on another login means the server is full.
|
||||
loginOK atomic.Bool
|
||||
|
||||
// mu guards the download state below.
|
||||
mu sync.Mutex
|
||||
segments []*segment
|
||||
maxConns int
|
||||
noRest bool
|
||||
lanes []*lane
|
||||
done bool
|
||||
|
||||
// runMu serializes Start and Pause.
|
||||
runMu sync.Mutex
|
||||
cancel context.CancelFunc
|
||||
runDone chan struct{}
|
||||
}
|
||||
|
||||
func (f *Fetcher) Setup(ctl *controller.Controller) {
|
||||
f.ctl = ctl
|
||||
f.doneCh = make(chan error, 1)
|
||||
if f.meta == nil {
|
||||
f.meta = &fetcher.FetcherMeta{}
|
||||
}
|
||||
if ctl != nil && ctl.GetConfig != nil {
|
||||
ctl.GetConfig(&f.config)
|
||||
}
|
||||
if f.config == nil {
|
||||
f.config = &config{Connections: defaultConnections}
|
||||
}
|
||||
}
|
||||
|
||||
func (f *Fetcher) logger() *zerolog.Logger {
|
||||
if f.ctl == nil {
|
||||
return nil
|
||||
}
|
||||
return f.ctl.Logger
|
||||
}
|
||||
|
||||
func (f *Fetcher) warn(format string, args ...any) {
|
||||
if l := f.logger(); l != nil {
|
||||
l.Warn().Msgf(format, args...)
|
||||
}
|
||||
}
|
||||
|
||||
func (f *Fetcher) initOptions() error {
|
||||
if f.meta.Opts == nil {
|
||||
f.meta.Opts = &base.Options{}
|
||||
}
|
||||
opts := f.meta.Opts
|
||||
if err := base.ParseOptExtra[pftp.OptsExtra](opts); err != nil {
|
||||
return err
|
||||
}
|
||||
if opts.Extra == nil {
|
||||
opts.Extra = &pftp.OptsExtra{}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *Fetcher) extra() *pftp.OptsExtra {
|
||||
if f.meta.Opts != nil {
|
||||
if e, ok := f.meta.Opts.Extra.(*pftp.OptsExtra); ok && e != nil {
|
||||
return e
|
||||
}
|
||||
}
|
||||
return &pftp.OptsExtra{}
|
||||
}
|
||||
|
||||
// tlsMode is the request's "tls" field, empty when the request has none.
|
||||
func (f *Fetcher) tlsMode() string {
|
||||
if f.meta.Req != nil {
|
||||
if e, ok := f.meta.Req.Extra.(*pftp.ReqExtra); ok && e != nil {
|
||||
return e.TLS
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// connections is the number of parallel logins to use. The caller holds mu.
|
||||
func (f *Fetcher) connections() int {
|
||||
if f.noRest {
|
||||
return 1
|
||||
}
|
||||
n := f.extra().Connections
|
||||
if n <= 0 && f.config != nil {
|
||||
n = f.config.Connections
|
||||
}
|
||||
if n <= 0 {
|
||||
n = defaultConnections
|
||||
}
|
||||
n = min(n, maxConnections)
|
||||
if f.maxConns > 0 {
|
||||
n = min(n, f.maxConns)
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
func (f *Fetcher) Resolve(req *base.Request) error {
|
||||
if err := base.ParseReqExtra[pftp.ReqExtra](req); err != nil {
|
||||
return err
|
||||
}
|
||||
f.meta.Req = req
|
||||
t, err := parseTarget(req.URL, f.tlsMode())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
s, err := f.dial(context.Background(), t)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// quit, not abort: a server that caps logins must see this session end
|
||||
// before the download's own logins arrive.
|
||||
defer s.quit()
|
||||
f.loginOK.Store(true)
|
||||
res, err := f.resolveTarget(s, t)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
f.meta.Res = res
|
||||
f.mu.Lock()
|
||||
f.segments = nil
|
||||
f.done = false
|
||||
f.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
// selected returns the indexes of the files to download, in SelectFiles
|
||||
// order. The caller holds mu or owns the fetcher.
|
||||
func (f *Fetcher) selected() []int {
|
||||
if f.meta == nil || f.meta.Res == nil {
|
||||
return nil
|
||||
}
|
||||
if f.meta.Opts != nil && len(f.meta.Opts.SelectFiles) > 0 {
|
||||
return f.meta.Opts.SelectFiles
|
||||
}
|
||||
all := make([]int, len(f.meta.Res.Files))
|
||||
for i := range all {
|
||||
all[i] = i
|
||||
}
|
||||
return all
|
||||
}
|
||||
|
||||
// localPath is where a file lands: under the task path, with the file's
|
||||
// directories below the URL kept.
|
||||
func (f *Fetcher) localPath(index int) string {
|
||||
if f.meta.Res.Name == "" {
|
||||
return f.meta.SingleFilepath()
|
||||
}
|
||||
file := f.meta.Res.Files[index]
|
||||
return path.Join(f.meta.FolderPath(), file.Path, file.Name)
|
||||
}
|
||||
|
||||
// remotePath is the server path of a file. A single file uses the URL's own
|
||||
// path, which a local rename never touches.
|
||||
func (f *Fetcher) remotePath(t *target, index int) string {
|
||||
if f.meta.Res.Name == "" {
|
||||
return t.path
|
||||
}
|
||||
file := f.meta.Res.Files[index]
|
||||
return path.Join(t.path, file.Path, file.Name)
|
||||
}
|
||||
|
||||
// plan splits every selected file into segments of at least minSegmentSize,
|
||||
// one per connection at most. The caller holds mu.
|
||||
func (f *Fetcher) plan() {
|
||||
conns := f.connections()
|
||||
f.segments = nil
|
||||
for _, index := range f.selected() {
|
||||
size := f.meta.Res.Files[index].Size
|
||||
n := int64(1)
|
||||
if !f.noRest {
|
||||
n = max(1, min(int64(conns), size/minSegmentSize))
|
||||
}
|
||||
for i := int64(0); i < n; i++ {
|
||||
f.segments = append(f.segments, &segment{
|
||||
File: index,
|
||||
Begin: i * size / n,
|
||||
End: (i+1)*size/n - 1,
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// planSequential gives every unfinished file one segment from byte 0,
|
||||
// because without REST a transfer cannot start in the middle. The caller
|
||||
// holds mu.
|
||||
func (f *Fetcher) planSequential() {
|
||||
complete := map[int]bool{}
|
||||
for _, s := range f.segments {
|
||||
if _, seen := complete[s.File]; !seen {
|
||||
complete[s.File] = true
|
||||
}
|
||||
if s.remain() > 0 {
|
||||
complete[s.File] = false
|
||||
}
|
||||
}
|
||||
var out []*segment
|
||||
for _, index := range f.selected() {
|
||||
size := f.meta.Res.Files[index].Size
|
||||
if complete[index] {
|
||||
out = append(out, &segment{File: index, Begin: 0, End: size - 1, Downloaded: size})
|
||||
continue
|
||||
}
|
||||
out = append(out, &segment{File: index, Begin: 0, End: size - 1})
|
||||
}
|
||||
f.segments = out
|
||||
// Bytes counted by the old segments are fetched again.
|
||||
f.lanes = nil
|
||||
}
|
||||
|
||||
// openFiles opens every selected file for writing. A fresh plan creates and
|
||||
// sizes them; a resumed one keeps their content, and a file that has gone
|
||||
// missing starts over.
|
||||
func (f *Fetcher) openFiles(fresh bool) (map[int]*os.File, error) {
|
||||
files := map[int]*os.File{}
|
||||
closeAll := func() {
|
||||
for _, file := range files {
|
||||
_ = file.Close()
|
||||
}
|
||||
}
|
||||
for _, index := range f.selected() {
|
||||
name := f.localPath(index)
|
||||
var (
|
||||
file *os.File
|
||||
err error
|
||||
)
|
||||
_, statErr := os.Stat(name)
|
||||
switch {
|
||||
case fresh || os.IsNotExist(statErr):
|
||||
file, err = f.ctl.Touch(name, f.meta.Res.Files[index].Size)
|
||||
if err == nil && !fresh {
|
||||
f.mu.Lock()
|
||||
for _, s := range f.segments {
|
||||
if s.File == index {
|
||||
s.Downloaded = 0
|
||||
}
|
||||
}
|
||||
f.mu.Unlock()
|
||||
}
|
||||
case statErr != nil:
|
||||
err = statErr
|
||||
default:
|
||||
file, err = os.OpenFile(name, os.O_RDWR, 0)
|
||||
}
|
||||
if err != nil {
|
||||
closeAll()
|
||||
return nil, err
|
||||
}
|
||||
files[index] = file
|
||||
}
|
||||
return files, nil
|
||||
}
|
||||
|
||||
func (f *Fetcher) Start() error {
|
||||
f.runMu.Lock()
|
||||
defer f.runMu.Unlock()
|
||||
if f.runDone != nil {
|
||||
select {
|
||||
case <-f.runDone:
|
||||
default:
|
||||
return nil // already running
|
||||
}
|
||||
}
|
||||
if err := f.initOptions(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := base.ParseReqExtra[pftp.ReqExtra](f.meta.Req); err != nil {
|
||||
return err
|
||||
}
|
||||
t, err := parseTarget(f.meta.Req.URL, f.tlsMode())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// A paused or failed run may have left a result nobody read.
|
||||
select {
|
||||
case <-f.doneCh:
|
||||
default:
|
||||
}
|
||||
|
||||
f.mu.Lock()
|
||||
if f.done {
|
||||
f.mu.Unlock()
|
||||
f.doneCh <- nil
|
||||
return nil
|
||||
}
|
||||
fresh := len(f.segments) == 0
|
||||
if fresh {
|
||||
f.plan()
|
||||
} else if f.noRest {
|
||||
f.planSequential()
|
||||
}
|
||||
f.mu.Unlock()
|
||||
|
||||
files, err := f.openFiles(fresh)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
f.cancel = cancel
|
||||
f.runDone = make(chan struct{})
|
||||
go f.run(ctx, t, files, f.runDone)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *Fetcher) run(ctx context.Context, t *target, files map[int]*os.File, done chan struct{}) {
|
||||
defer close(done)
|
||||
err := f.download(ctx, t, files)
|
||||
for _, file := range files {
|
||||
if cerr := file.Close(); cerr != nil && err == nil {
|
||||
err = cerr
|
||||
}
|
||||
}
|
||||
f.mu.Lock()
|
||||
if err == nil {
|
||||
f.done = true
|
||||
for _, l := range f.lanes {
|
||||
l.completed = true
|
||||
}
|
||||
}
|
||||
f.mu.Unlock()
|
||||
if err != nil && ctx.Err() != nil {
|
||||
return // paused
|
||||
}
|
||||
select {
|
||||
case f.doneCh <- err:
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
func (f *Fetcher) download(ctx context.Context, t *target, files map[int]*os.File) error {
|
||||
for {
|
||||
err := f.runWorkers(ctx, t, files)
|
||||
if !errors.Is(err, errRestUnsupported) {
|
||||
return err
|
||||
}
|
||||
f.mu.Lock()
|
||||
f.noRest = true
|
||||
f.planSequential()
|
||||
f.mu.Unlock()
|
||||
f.warn("ftp: the server refused REST, so each file is downloaded in one stream")
|
||||
}
|
||||
}
|
||||
|
||||
// pool hands out the segments of one run to its workers. Every decision to
|
||||
// leave the run is taken under mu together with the queue it depends on, so
|
||||
// a segment put back can never be left without a worker.
|
||||
type pool struct {
|
||||
mu sync.Mutex
|
||||
queue []*segment
|
||||
running int // workers still in the run
|
||||
}
|
||||
|
||||
// take returns the next segment. A worker that finds the queue empty leaves
|
||||
// the run.
|
||||
func (p *pool) take() *segment {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
if len(p.queue) == 0 {
|
||||
p.running--
|
||||
return nil
|
||||
}
|
||||
s := p.queue[0]
|
||||
p.queue = p.queue[1:]
|
||||
return s
|
||||
}
|
||||
|
||||
func (p *pool) putBack(s *segment) {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
p.queue = append([]*segment{s}, p.queue...)
|
||||
}
|
||||
|
||||
// refused puts back the segment of a worker whose login the server turned
|
||||
// away. The worker leaves when another worker is still in the run to take
|
||||
// the segment, and left is the number of workers that remain; the last
|
||||
// worker stays and retries.
|
||||
func (p *pool) refused(s *segment) (leave bool, left int) {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
p.queue = append([]*segment{s}, p.queue...)
|
||||
if p.running > 1 {
|
||||
p.running--
|
||||
return true, p.running
|
||||
}
|
||||
return false, p.running
|
||||
}
|
||||
|
||||
// leave takes a worker out of the run when it stops for any other reason.
|
||||
func (p *pool) leave() {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
p.running--
|
||||
}
|
||||
|
||||
func (f *Fetcher) runWorkers(ctx context.Context, t *target, files map[int]*os.File) error {
|
||||
runCtx, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
|
||||
f.mu.Lock()
|
||||
p := &pool{}
|
||||
for _, s := range f.segments {
|
||||
if s.remain() > 0 {
|
||||
p.queue = append(p.queue, s)
|
||||
}
|
||||
}
|
||||
workers := min(f.connections(), len(p.queue))
|
||||
f.mu.Unlock()
|
||||
if workers == 0 {
|
||||
return nil
|
||||
}
|
||||
p.running = workers
|
||||
|
||||
var (
|
||||
wg sync.WaitGroup
|
||||
errMu sync.Mutex
|
||||
firstErr error
|
||||
)
|
||||
for id := 0; id < workers; id++ {
|
||||
wg.Add(1)
|
||||
go func(id int) {
|
||||
defer wg.Done()
|
||||
if err := f.worker(runCtx, t, files, p, id); err != nil {
|
||||
errMu.Lock()
|
||||
if firstErr == nil {
|
||||
firstErr = err
|
||||
}
|
||||
errMu.Unlock()
|
||||
cancel()
|
||||
}
|
||||
}(id)
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if firstErr != nil {
|
||||
return firstErr
|
||||
}
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
for _, s := range f.segments {
|
||||
if s.remain() > 0 {
|
||||
return errors.New("ftp: the download stopped with data missing")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// worker fetches segments from the pool until it is empty. A worker whose
|
||||
// login the server refuses (421, or 530 once these credentials have worked)
|
||||
// hands its segment back and leaves while other workers remain, lowering the
|
||||
// task's ceiling to the workers left. This holds whether or not the worker
|
||||
// had logged in before: a slot can also go to another client mid-download.
|
||||
func (f *Fetcher) worker(ctx context.Context, t *target, files map[int]*os.File, p *pool, id int) error {
|
||||
var ln *lane
|
||||
f.mu.Lock()
|
||||
if id < len(f.lanes) {
|
||||
ln = f.lanes[id]
|
||||
}
|
||||
f.mu.Unlock()
|
||||
onLogin := func() {
|
||||
f.mu.Lock()
|
||||
if ln == nil {
|
||||
ln = &lane{}
|
||||
f.lanes = append(f.lanes, ln)
|
||||
}
|
||||
f.mu.Unlock()
|
||||
}
|
||||
|
||||
failures := 0
|
||||
pendingRetries := 0
|
||||
for {
|
||||
seg := p.take()
|
||||
if seg == nil {
|
||||
return nil
|
||||
}
|
||||
err := f.fetchSegment(ctx, t, files, seg, &ln, onLogin)
|
||||
if err == nil {
|
||||
failures = 0
|
||||
continue
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
p.putBack(seg)
|
||||
p.leave()
|
||||
return nil
|
||||
}
|
||||
|
||||
var le *loginError
|
||||
if errors.As(err, &le) && isLoginRefusal(le.err, f.loginOK.Load()) {
|
||||
if leave, left := p.refused(seg); leave {
|
||||
f.lowerCeiling(left)
|
||||
return nil
|
||||
}
|
||||
// The last worker: wait for a slot.
|
||||
} else {
|
||||
p.putBack(seg)
|
||||
if errors.Is(err, errRestUnsupported) {
|
||||
p.leave()
|
||||
return err
|
||||
}
|
||||
if isPermanent(err) || errors.Is(err, errDiskWrite) {
|
||||
f.markFailed(ln)
|
||||
p.leave()
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
failures++
|
||||
f.mu.Lock()
|
||||
if ln != nil {
|
||||
ln.retryTimes += pendingRetries + 1
|
||||
pendingRetries = 0
|
||||
} else {
|
||||
pendingRetries++
|
||||
}
|
||||
f.mu.Unlock()
|
||||
if failures > maxRetries {
|
||||
f.markFailed(ln)
|
||||
p.leave()
|
||||
return fmt.Errorf("ftp: giving up after %d retries: %w", maxRetries, err)
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
p.leave()
|
||||
return nil
|
||||
case <-time.After(retryDelay * time.Duration(failures)):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (f *Fetcher) lowerCeiling(n int) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if f.maxConns == 0 || n < f.maxConns {
|
||||
f.maxConns = n
|
||||
}
|
||||
}
|
||||
|
||||
func (f *Fetcher) markFailed(ln *lane) {
|
||||
if ln == nil {
|
||||
return
|
||||
}
|
||||
f.mu.Lock()
|
||||
ln.failed = true
|
||||
f.mu.Unlock()
|
||||
}
|
||||
|
||||
// isRestRefused tells a REST the server does not implement. RETR itself
|
||||
// fails with 550 and the like, never with these codes.
|
||||
func isRestRefused(err error) bool {
|
||||
switch replyCode(err) {
|
||||
case ftp.StatusBadCommand, ftp.StatusBadArguments, ftp.StatusNotImplemented, ftp.StatusNotImplementedParameter:
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// fetchSegment logs in, retrieves the segment from its first missing byte
|
||||
// and writes it in place. It stops reading at the segment's end and closes
|
||||
// the data connection; the server then reports the transfer aborted, which
|
||||
// is expected.
|
||||
func (f *Fetcher) fetchSegment(ctx context.Context, t *target, files map[int]*os.File, seg *segment, ln **lane, onLogin func()) error {
|
||||
s, err := f.dial(ctx, t)
|
||||
if err != nil {
|
||||
return &loginError{err}
|
||||
}
|
||||
defer s.quit()
|
||||
f.loginOK.Store(true)
|
||||
onLogin()
|
||||
|
||||
remote := f.remotePath(t, seg.File)
|
||||
f.mu.Lock()
|
||||
noRest := f.noRest
|
||||
wantSize := f.meta.Res.Files[seg.File].Size
|
||||
if noRest {
|
||||
seg.Downloaded = 0
|
||||
}
|
||||
pos := seg.Begin + seg.Downloaded
|
||||
end := seg.End
|
||||
f.mu.Unlock()
|
||||
|
||||
// A file that changed on the server would mix old and new bytes.
|
||||
s.deadline()
|
||||
if size, err := s.FileSize(remote); err == nil && size != wantSize {
|
||||
return errSizeChanged
|
||||
} else if err != nil && replyCode(err) == 0 {
|
||||
return err
|
||||
}
|
||||
|
||||
s.deadline()
|
||||
var resp *ftp.Response
|
||||
if noRest {
|
||||
resp, err = s.Retr(remote)
|
||||
} else {
|
||||
resp, err = s.RetrFrom(remote, uint64(pos))
|
||||
}
|
||||
if err != nil {
|
||||
if !noRest && pos > 0 && isRestRefused(err) {
|
||||
return errRestUnsupported
|
||||
}
|
||||
return err
|
||||
}
|
||||
defer resp.Close()
|
||||
s.clearDeadline()
|
||||
|
||||
file := files[seg.File]
|
||||
buf := make([]byte, copyBufferSize)
|
||||
for pos <= end {
|
||||
n := int(min(int64(len(buf)), end-pos+1))
|
||||
_ = resp.SetDeadline(time.Now().Add(readTimeout))
|
||||
nr, rerr := resp.Read(buf[:n])
|
||||
if nr > 0 {
|
||||
if _, werr := file.WriteAt(buf[:nr], pos); werr != nil {
|
||||
return fmt.Errorf("%w: %w", errDiskWrite, werr)
|
||||
}
|
||||
pos += int64(nr)
|
||||
f.mu.Lock()
|
||||
seg.Downloaded += int64(nr)
|
||||
if *ln != nil {
|
||||
(*ln).downloaded += int64(nr)
|
||||
}
|
||||
f.mu.Unlock()
|
||||
}
|
||||
if rerr != nil {
|
||||
if pos > end {
|
||||
break
|
||||
}
|
||||
if errors.Is(rerr, io.EOF) {
|
||||
return errShortTransfer
|
||||
}
|
||||
return rerr
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *Fetcher) Patch(req *base.Request, opts *base.Options) error {
|
||||
if req == nil {
|
||||
return nil
|
||||
}
|
||||
if req.URL != "" {
|
||||
f.meta.Req.URL = req.URL
|
||||
}
|
||||
if req.Labels != nil {
|
||||
if f.meta.Req.Labels == nil {
|
||||
f.meta.Req.Labels = make(map[string]string)
|
||||
}
|
||||
for k, v := range req.Labels {
|
||||
f.meta.Req.Labels[k] = v
|
||||
}
|
||||
}
|
||||
if req.Proxy != nil {
|
||||
f.meta.Req.Proxy = req.Proxy
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *Fetcher) Pause() error {
|
||||
f.runMu.Lock()
|
||||
defer f.runMu.Unlock()
|
||||
if f.cancel != nil {
|
||||
f.cancel()
|
||||
}
|
||||
if f.runDone != nil {
|
||||
<-f.runDone
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *Fetcher) Close() error {
|
||||
return f.Pause()
|
||||
}
|
||||
|
||||
func (f *Fetcher) Meta() *fetcher.FetcherMeta {
|
||||
return f.meta
|
||||
}
|
||||
|
||||
// Stats reports one connection per worker that logged in, in the HTTP
|
||||
// protocol's shape.
|
||||
func (f *Fetcher) Stats() *fetcher.Stats {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
var total int64
|
||||
if n := int64(len(f.lanes)); n > 0 && f.meta != nil && f.meta.Res != nil && f.meta.Res.Size > 0 {
|
||||
total = (f.meta.Res.Size + n - 1) / n
|
||||
}
|
||||
conns := make([]*pftp.StatsConnection, 0, len(f.lanes))
|
||||
for _, l := range f.lanes {
|
||||
conns = append(conns, &pftp.StatsConnection{
|
||||
Downloaded: l.downloaded,
|
||||
Total: total,
|
||||
Completed: l.completed || (f.done && !l.failed),
|
||||
Failed: l.failed,
|
||||
RetryTimes: l.retryTimes,
|
||||
})
|
||||
}
|
||||
return &fetcher.Stats{Snapshot: &pftp.Stats{Connections: conns}}
|
||||
}
|
||||
|
||||
// Progress returns the bytes downloaded of each selected file, in
|
||||
// SelectFiles order.
|
||||
func (f *Fetcher) Progress() fetcher.Progress {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
perFile := map[int]int64{}
|
||||
for _, s := range f.segments {
|
||||
perFile[s.File] += s.Downloaded
|
||||
}
|
||||
sel := f.selected()
|
||||
p := make(fetcher.Progress, len(sel))
|
||||
for i, index := range sel {
|
||||
p[i] = perFile[index]
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
func (f *Fetcher) Wait() error {
|
||||
return <-f.doneCh
|
||||
}
|
||||
@@ -1,109 +0,0 @@
|
||||
package ftp
|
||||
|
||||
import (
|
||||
"crypto/tls"
|
||||
"net/url"
|
||||
"path"
|
||||
"strings"
|
||||
|
||||
"github.com/GopeedLab/gopeed/internal/fetcher"
|
||||
"github.com/GopeedLab/gopeed/pkg/base"
|
||||
pftp "github.com/GopeedLab/gopeed/pkg/protocol/ftp"
|
||||
)
|
||||
|
||||
// fetcherData is what a task saves to resume later. Segments has the same
|
||||
// Begin, End and Downloaded fields every version writes; the other fields are
|
||||
// optional, so a save without them still restores.
|
||||
type fetcherData struct {
|
||||
Segments []*segment
|
||||
// MaxConns is the login ceiling the server enforced, 0 when none was hit.
|
||||
MaxConns int `json:",omitempty"`
|
||||
// NoRest records that the server refused REST, so the task downloads each
|
||||
// file in one sequential stream.
|
||||
NoRest bool `json:",omitempty"`
|
||||
}
|
||||
|
||||
// FetcherManager handles ftp://, ftps:// (implicit TLS) and ftpes:// (explicit
|
||||
// TLS) URLs.
|
||||
type FetcherManager struct {
|
||||
// baseTLS is the base for every TLS connection. Nil means the system
|
||||
// roots; the package's tests set it to trust their own certificate. The
|
||||
// task's skipVerifyCert still turns verification off.
|
||||
baseTLS *tls.Config
|
||||
}
|
||||
|
||||
func (fm *FetcherManager) Name() string {
|
||||
return "ftp"
|
||||
}
|
||||
|
||||
func (fm *FetcherManager) Filters() []*fetcher.SchemeFilter {
|
||||
return []*fetcher.SchemeFilter{
|
||||
{Type: fetcher.FilterTypeUrl, Pattern: "FTP"},
|
||||
{Type: fetcher.FilterTypeUrl, Pattern: "FTPS"},
|
||||
{Type: fetcher.FilterTypeUrl, Pattern: "FTPES"},
|
||||
}
|
||||
}
|
||||
|
||||
func (fm *FetcherManager) Build() fetcher.Fetcher {
|
||||
return &Fetcher{manager: fm}
|
||||
}
|
||||
|
||||
// ParseName names a task before it is resolved. It never includes the
|
||||
// userinfo of the URL.
|
||||
func (fm *FetcherManager) ParseName(u string) string {
|
||||
parsed, err := url.Parse(u)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
name := path.Base(strings.TrimSuffix(parsed.Path, "/"))
|
||||
if name == "" || name == "/" || name == "." {
|
||||
name = parsed.Hostname()
|
||||
}
|
||||
return name
|
||||
}
|
||||
|
||||
func (fm *FetcherManager) AutoRename() bool {
|
||||
return true
|
||||
}
|
||||
|
||||
func (fm *FetcherManager) DefaultConfig() any {
|
||||
return &config{Connections: defaultConnections}
|
||||
}
|
||||
|
||||
func (fm *FetcherManager) Store(f fetcher.Fetcher) (any, error) {
|
||||
ff := f.(*Fetcher)
|
||||
ff.mu.Lock()
|
||||
defer ff.mu.Unlock()
|
||||
segments := make([]*segment, len(ff.segments))
|
||||
for i, s := range ff.segments {
|
||||
c := *s
|
||||
segments[i] = &c
|
||||
}
|
||||
return &fetcherData{
|
||||
Segments: segments,
|
||||
MaxConns: ff.maxConns,
|
||||
NoRest: ff.noRest,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (fm *FetcherManager) Restore() (v any, f func(meta *fetcher.FetcherMeta, v any) fetcher.Fetcher) {
|
||||
return &fetcherData{}, func(meta *fetcher.FetcherMeta, v any) fetcher.Fetcher {
|
||||
fd := v.(*fetcherData)
|
||||
ff := fm.Build().(*Fetcher)
|
||||
ff.meta = meta
|
||||
if meta != nil && meta.Req != nil {
|
||||
_ = base.ParseReqExtra[pftp.ReqExtra](meta.Req)
|
||||
}
|
||||
if meta != nil && meta.Opts != nil {
|
||||
_ = base.ParseOptExtra[pftp.OptsExtra](meta.Opts)
|
||||
}
|
||||
ff.segments = fd.Segments
|
||||
ff.maxConns = fd.MaxConns
|
||||
ff.noRest = fd.NoRest
|
||||
return ff
|
||||
}
|
||||
}
|
||||
|
||||
func (fm *FetcherManager) Close() error {
|
||||
return nil
|
||||
}
|
||||
@@ -1,818 +0,0 @@
|
||||
package ftp
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/armon/go-socks5"
|
||||
"github.com/rs/zerolog"
|
||||
|
||||
"github.com/GopeedLab/gopeed/internal/controller"
|
||||
"github.com/GopeedLab/gopeed/internal/fetcher"
|
||||
"github.com/GopeedLab/gopeed/pkg/base"
|
||||
pftp "github.com/GopeedLab/gopeed/pkg/protocol/ftp"
|
||||
)
|
||||
|
||||
const mib = 1 << 20
|
||||
|
||||
// testManager trusts the test server's certificate, the way an embedding
|
||||
// application could add private roots.
|
||||
func testManager(srv *FTPServer) *FetcherManager {
|
||||
fm := &FetcherManager{}
|
||||
if srv != nil {
|
||||
fm.baseTLS = &tls.Config{RootCAs: srv.CA}
|
||||
}
|
||||
return fm
|
||||
}
|
||||
|
||||
func testController(fm *FetcherManager) *controller.Controller {
|
||||
ctl := controller.NewController()
|
||||
ctl.GetConfig = func(v any) {
|
||||
data, _ := json.Marshal(fm.DefaultConfig())
|
||||
_ = json.Unmarshal(data, v)
|
||||
}
|
||||
return ctl
|
||||
}
|
||||
|
||||
func buildFetcher(fm *FetcherManager) *Fetcher {
|
||||
f := fm.Build().(*Fetcher)
|
||||
f.Setup(testController(fm))
|
||||
return f
|
||||
}
|
||||
|
||||
func ftpURL(srv *FTPServer, scheme, userinfo, p string) string {
|
||||
if userinfo != "" {
|
||||
userinfo += "@"
|
||||
}
|
||||
return scheme + "://" + userinfo + srv.Addr + "/" + strings.TrimPrefix(p, "/")
|
||||
}
|
||||
|
||||
// resolveOpts resolves req and then gives the fetcher its options, in the
|
||||
// order the downloader uses.
|
||||
func resolveOpts(f *Fetcher, req *base.Request, opts *base.Options) error {
|
||||
if err := f.Resolve(req); err != nil {
|
||||
return err
|
||||
}
|
||||
f.Meta().Opts = opts
|
||||
return nil
|
||||
}
|
||||
|
||||
func resolveTask(t *testing.T, f *Fetcher, rawURL string, extra *pftp.OptsExtra) {
|
||||
t.Helper()
|
||||
resolveRequest(t, f, &base.Request{URL: rawURL}, extra)
|
||||
}
|
||||
|
||||
func resolveRequest(t *testing.T, f *Fetcher, req *base.Request, extra *pftp.OptsExtra) {
|
||||
t.Helper()
|
||||
opts := &base.Options{Path: t.TempDir()}
|
||||
if extra != nil {
|
||||
opts.Extra = extra
|
||||
}
|
||||
if err := resolveOpts(f, req, opts); err != nil {
|
||||
t.Fatalf("resolve %s: %v", req.URL, err)
|
||||
}
|
||||
}
|
||||
|
||||
func waitFetcher(t *testing.T, f fetcher.Fetcher) error {
|
||||
t.Helper()
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- f.Wait() }()
|
||||
select {
|
||||
case err := <-done:
|
||||
return err
|
||||
case <-time.After(60 * time.Second):
|
||||
t.Fatal("download did not finish in 60s")
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func startAndWait(t *testing.T, f *Fetcher) {
|
||||
t.Helper()
|
||||
if err := f.Start(); err != nil {
|
||||
t.Fatalf("start: %v", err)
|
||||
}
|
||||
if err := waitFetcher(t, f); err != nil {
|
||||
t.Fatalf("download: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func checkFile(t *testing.T, name, want string) {
|
||||
t.Helper()
|
||||
if got := FileSHA256(t, name); got != want {
|
||||
t.Fatalf("%s: sha256 %s, want %s", name, got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func ftpStats(t *testing.T, f *Fetcher) *pftp.Stats {
|
||||
t.Helper()
|
||||
stats, ok := f.Stats().Snapshot.(*pftp.Stats)
|
||||
if !ok {
|
||||
t.Fatalf("stats snapshot is %T, want *ftp.Stats", f.Stats().Snapshot)
|
||||
}
|
||||
return stats
|
||||
}
|
||||
|
||||
func storeData(t *testing.T, fm *FetcherManager, f *Fetcher) []byte {
|
||||
t.Helper()
|
||||
data, err := fm.Store(f)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
blob, err := json.Marshal(data)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return blob
|
||||
}
|
||||
|
||||
// restoreFetcher rebuilds a fetcher the way the downloader does after a
|
||||
// restart: the meta and the save blob both come back from JSON.
|
||||
func restoreFetcher(t *testing.T, fm *FetcherManager, meta *fetcher.FetcherMeta, blob []byte) *Fetcher {
|
||||
t.Helper()
|
||||
metaJSON, err := json.Marshal(meta)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var restoredMeta fetcher.FetcherMeta
|
||||
if err := json.Unmarshal(metaJSON, &restoredMeta); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
v, build := fm.Restore()
|
||||
if err := json.Unmarshal(blob, v); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f := build(&restoredMeta, v).(*Fetcher)
|
||||
f.Setup(testController(fm))
|
||||
return f
|
||||
}
|
||||
|
||||
func TestResolveFile(t *testing.T) {
|
||||
for _, noMLST := range []bool{false, true} {
|
||||
t.Run(fmt.Sprintf("noMLST=%v", noMLST), func(t *testing.T) {
|
||||
srv := StartFTPServer(t, ServerOptions{NoMLST: noMLST})
|
||||
data := RandomBytes(100*1024, 1)
|
||||
srv.WriteFile(t, "pub/file name.bin", data)
|
||||
|
||||
f := buildFetcher(testManager(srv))
|
||||
resolveTask(t, f, ftpURL(srv, "ftp", "", "pub/file%20name.bin"), nil)
|
||||
res := f.Meta().Res
|
||||
if res.Name != "" {
|
||||
t.Fatalf("a file must not be a folder resource, got name %q", res.Name)
|
||||
}
|
||||
if len(res.Files) != 1 {
|
||||
t.Fatalf("got %d files, want 1", len(res.Files))
|
||||
}
|
||||
file := res.Files[0]
|
||||
if file.Name != "file name.bin" || file.Path != "" || file.Size != int64(len(data)) {
|
||||
t.Fatalf("got file %+v, want name %q, empty path, size %d", file, "file name.bin", len(data))
|
||||
}
|
||||
if res.Size != int64(len(data)) {
|
||||
t.Fatalf("resource size %d, want %d", res.Size, len(data))
|
||||
}
|
||||
if file.Ctime == nil || time.Since(*file.Ctime) > time.Hour {
|
||||
t.Fatalf("modification time %v is missing or wrong", file.Ctime)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func writeTree(t *testing.T, srv *FTPServer) map[string]string {
|
||||
t.Helper()
|
||||
return map[string]string{
|
||||
"a.txt": srv.WriteFile(t, "pub/tree/a.txt", []byte("hello")),
|
||||
"sub/b.bin": srv.WriteFile(t, "pub/tree/sub/b.bin", RandomBytes(3*mib+17, 2)),
|
||||
"sub/deeper/c.dat": srv.WriteFile(t, "pub/tree/sub/deeper/c.dat", RandomBytes(2000, 3)),
|
||||
"sub/empty.txt": srv.WriteFile(t, "pub/tree/sub/empty.txt", nil),
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveDir(t *testing.T) {
|
||||
for _, noMLST := range []bool{false, true} {
|
||||
for _, p := range []string{"pub/tree", "pub/tree/"} {
|
||||
t.Run(fmt.Sprintf("noMLST=%v/%s", noMLST, p), func(t *testing.T) {
|
||||
srv := StartFTPServer(t, ServerOptions{NoMLST: noMLST})
|
||||
writeTree(t, srv)
|
||||
if err := os.MkdirAll(filepath.Join(srv.Root, "pub/tree/void"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
f := buildFetcher(testManager(srv))
|
||||
resolveTask(t, f, ftpURL(srv, "ftp", "", p), nil)
|
||||
res := f.Meta().Res
|
||||
if res.Name != "tree" {
|
||||
t.Fatalf("folder name %q, want tree", res.Name)
|
||||
}
|
||||
var got []string
|
||||
for _, file := range res.Files {
|
||||
got = append(got, fmt.Sprintf("%s|%s|%d", file.Path, file.Name, file.Size))
|
||||
}
|
||||
want := []string{
|
||||
"|a.txt|5",
|
||||
"sub|b.bin|3145745",
|
||||
"sub|empty.txt|0",
|
||||
"sub/deeper|c.dat|2000",
|
||||
}
|
||||
if !slices.Equal(got, want) {
|
||||
t.Fatalf("files\n got %q\nwant %q", got, want)
|
||||
}
|
||||
if res.Size != 5+3145745+2000 {
|
||||
t.Fatalf("resource size %d", res.Size)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveDirLimits(t *testing.T) {
|
||||
srv := StartFTPServer(t, ServerOptions{})
|
||||
srv.WriteFile(t, "deep/1/2/3/4/5/6/7/8/f.txt", []byte("ok"))
|
||||
srv.WriteFile(t, "deep9/1/2/3/4/5/6/7/8/9/f.txt", []byte("too deep"))
|
||||
for i := 0; i < 4; i++ {
|
||||
srv.WriteFile(t, fmt.Sprintf("many/f%d", i), []byte("x"))
|
||||
}
|
||||
|
||||
f := buildFetcher(testManager(srv))
|
||||
resolveTask(t, f, ftpURL(srv, "ftp", "", "deep"), nil)
|
||||
if got := f.Meta().Res.Files[0].Path; got != "1/2/3/4/5/6/7/8" {
|
||||
t.Fatalf("depth 8 path %q", got)
|
||||
}
|
||||
|
||||
f = buildFetcher(testManager(srv))
|
||||
err := resolveOpts(f, &base.Request{URL: ftpURL(srv, "ftp", "", "deep9")}, &base.Options{Path: t.TempDir()})
|
||||
if err == nil || !strings.Contains(err.Error(), "8 levels") {
|
||||
t.Fatalf("a tree deeper than 8 levels must fail and name the limit, got %v", err)
|
||||
}
|
||||
|
||||
saved := maxWalkFiles
|
||||
maxWalkFiles = 3
|
||||
defer func() { maxWalkFiles = saved }()
|
||||
f = buildFetcher(testManager(srv))
|
||||
err = resolveOpts(f, &base.Request{URL: ftpURL(srv, "ftp", "", "many")}, &base.Options{Path: t.TempDir()})
|
||||
if err == nil || !strings.Contains(err.Error(), "3 files") {
|
||||
t.Fatalf("a tree over the file cap must fail and name the limit, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloadPlain(t *testing.T) {
|
||||
srv := StartFTPServer(t, ServerOptions{})
|
||||
data := RandomBytes(3*mib+12345, 4)
|
||||
want := srv.WriteFile(t, "pub/plain.bin", data)
|
||||
|
||||
f := buildFetcher(testManager(srv))
|
||||
resolveTask(t, f, ftpURL(srv, "ftp", "", "pub/plain.bin"), &pftp.OptsExtra{Connections: 4})
|
||||
startAndWait(t, f)
|
||||
checkFile(t, f.Meta().SingleFilepath(), want)
|
||||
if got := f.Progress().TotalDownloaded(); got != int64(len(data)) {
|
||||
t.Fatalf("progress %d, want %d", got, len(data))
|
||||
}
|
||||
stats := ftpStats(t, f)
|
||||
if len(stats.Connections) == 0 || len(stats.Connections) > 4 {
|
||||
t.Fatalf("got %d connections in stats", len(stats.Connections))
|
||||
}
|
||||
var sum int64
|
||||
for _, c := range stats.Connections {
|
||||
sum += c.Downloaded
|
||||
if !c.Completed || c.Failed {
|
||||
t.Fatalf("connection stats %+v after success", c)
|
||||
}
|
||||
}
|
||||
if sum != int64(len(data)) {
|
||||
t.Fatalf("connections downloaded %d bytes, want %d", sum, len(data))
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloadDir(t *testing.T) {
|
||||
srv := StartFTPServer(t, ServerOptions{})
|
||||
want := writeTree(t, srv)
|
||||
|
||||
f := buildFetcher(testManager(srv))
|
||||
resolveTask(t, f, ftpURL(srv, "ftp", "", "pub/tree"), &pftp.OptsExtra{Connections: 2})
|
||||
startAndWait(t, f)
|
||||
root := f.Meta().FolderPath()
|
||||
if root != filepath.Join(f.Meta().Opts.Path, "tree") {
|
||||
t.Fatalf("folder path %q", root)
|
||||
}
|
||||
for rel, sum := range want {
|
||||
checkFile(t, filepath.Join(root, filepath.FromSlash(rel)), sum)
|
||||
}
|
||||
}
|
||||
|
||||
func testDownloadTLS(t *testing.T, mode string, schemes map[string]string) {
|
||||
srv := StartFTPServer(t, ServerOptions{TLS: mode})
|
||||
data := RandomBytes(2*mib+999, 5)
|
||||
want := srv.WriteFile(t, "secure.bin", data)
|
||||
for scheme, reqTLS := range schemes {
|
||||
t.Run(scheme+"+"+reqTLS, func(t *testing.T) {
|
||||
f := buildFetcher(testManager(srv))
|
||||
req := &base.Request{URL: ftpURL(srv, scheme, "", "secure.bin"), Extra: &pftp.ReqExtra{TLS: reqTLS}}
|
||||
resolveRequest(t, f, req, &pftp.OptsExtra{Connections: 2})
|
||||
startAndWait(t, f)
|
||||
checkFile(t, f.Meta().SingleFilepath(), want)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloadExplicitTLS(t *testing.T) {
|
||||
testDownloadTLS(t, pftp.TLSExplicit, map[string]string{
|
||||
"ftpes": "",
|
||||
"ftp": pftp.TLSExplicit,
|
||||
})
|
||||
}
|
||||
|
||||
func TestDownloadImplicitTLS(t *testing.T) {
|
||||
testDownloadTLS(t, pftp.TLSImplicit, map[string]string{
|
||||
"ftps": "",
|
||||
"ftp": pftp.TLSImplicit,
|
||||
})
|
||||
}
|
||||
|
||||
// The TLS mode of a plain ftp:// link lives on the request, so a task that is
|
||||
// saved and restored from JSON still logs in over TLS.
|
||||
func TestRestoredRequestKeepsTLSMode(t *testing.T) {
|
||||
srv := StartFTPServer(t, ServerOptions{TLS: pftp.TLSExplicit})
|
||||
want := srv.WriteFile(t, "keep.bin", RandomBytes(2*mib+5, 12))
|
||||
fm := testManager(srv)
|
||||
f := buildFetcher(fm)
|
||||
req := &base.Request{URL: ftpURL(srv, "ftp", "", "keep.bin"), Extra: &pftp.ReqExtra{TLS: pftp.TLSExplicit}}
|
||||
resolveRequest(t, f, req, &pftp.OptsExtra{Connections: 2})
|
||||
|
||||
restored := restoreFetcher(t, fm, f.Meta(), storeData(t, fm, f))
|
||||
if _, ok := restored.Meta().Req.Extra.(*pftp.ReqExtra); !ok {
|
||||
t.Fatalf("restored request extra is %T, want *ftp.ReqExtra", restored.Meta().Req.Extra)
|
||||
}
|
||||
// Explicit mode refuses plain logins, so the download only finishes over TLS.
|
||||
startAndWait(t, restored)
|
||||
checkFile(t, restored.Meta().SingleFilepath(), want)
|
||||
}
|
||||
|
||||
func TestBadCertRejectedByDefault(t *testing.T) {
|
||||
for mode, scheme := range map[string]string{pftp.TLSExplicit: "ftpes", pftp.TLSImplicit: "ftps"} {
|
||||
t.Run(mode, func(t *testing.T) {
|
||||
srv := StartFTPServer(t, ServerOptions{TLS: mode})
|
||||
want := srv.WriteFile(t, "f.bin", RandomBytes(4096, 6))
|
||||
rawURL := ftpURL(srv, scheme, "", "f.bin")
|
||||
|
||||
// No injected roots: the self-signed certificate is unknown.
|
||||
f := buildFetcher(testManager(nil))
|
||||
err := resolveOpts(f, &base.Request{URL: rawURL}, &base.Options{Path: t.TempDir()})
|
||||
var verifyErr *tls.CertificateVerificationError
|
||||
if !errors.As(err, &verifyErr) {
|
||||
t.Fatalf("want a certificate verification error, got %v", err)
|
||||
}
|
||||
|
||||
// The request's existing skipVerifyCert switch accepts it.
|
||||
f = buildFetcher(testManager(nil))
|
||||
if err := resolveOpts(f, &base.Request{URL: rawURL, SkipVerifyCert: true}, &base.Options{Path: t.TempDir()}); err != nil {
|
||||
t.Fatalf("skipVerifyCert resolve: %v", err)
|
||||
}
|
||||
startAndWait(t, f)
|
||||
checkFile(t, f.Meta().SingleFilepath(), want)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSegmentsUseRest(t *testing.T) {
|
||||
srv := StartFTPServer(t, ServerOptions{})
|
||||
srv.ReadDelay.Store(int64(time.Millisecond))
|
||||
size := 8 * mib
|
||||
want := srv.WriteFile(t, "big.bin", RandomBytes(size, 7))
|
||||
|
||||
f := buildFetcher(testManager(srv))
|
||||
resolveTask(t, f, ftpURL(srv, "ftp", "", "big.bin"), &pftp.OptsExtra{Connections: 4})
|
||||
startAndWait(t, f)
|
||||
checkFile(t, f.Meta().SingleFilepath(), want)
|
||||
|
||||
rests := srv.RestOffsets()
|
||||
for _, off := range []int64{2 * mib, 4 * mib, 6 * mib} {
|
||||
if !slices.Contains(rests, off) {
|
||||
t.Fatalf("REST offsets %v miss the segment start %d", rests, off)
|
||||
}
|
||||
}
|
||||
for _, off := range rests {
|
||||
if off <= 0 || off >= int64(size) {
|
||||
t.Fatalf("REST offset %d out of range", off)
|
||||
}
|
||||
}
|
||||
if got := len(ftpStats(t, f).Connections); got != 4 {
|
||||
t.Fatalf("stats report %d connections, want 4", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoginCapFallsBack(t *testing.T) {
|
||||
srv := StartFTPServer(t, ServerOptions{MaxLogins: 1})
|
||||
want := srv.WriteFile(t, "capped.bin", RandomBytes(4*mib+5, 8))
|
||||
|
||||
fm := testManager(srv)
|
||||
f := buildFetcher(fm)
|
||||
resolveTask(t, f, ftpURL(srv, "ftp", "", "capped.bin"), &pftp.OptsExtra{Connections: 4})
|
||||
startAndWait(t, f)
|
||||
checkFile(t, f.Meta().SingleFilepath(), want)
|
||||
|
||||
if srv.Refused() == 0 {
|
||||
t.Fatal("the server never refused a login, so the cap was not exercised")
|
||||
}
|
||||
if got := len(ftpStats(t, f).Connections); got != 1 {
|
||||
t.Fatalf("stats report %d connections, want 1", got)
|
||||
}
|
||||
var saved fetcherData
|
||||
if err := json.Unmarshal(storeData(t, fm, f), &saved); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if saved.MaxConns != 1 {
|
||||
t.Fatalf("saved login ceiling %d, want 1", saved.MaxConns)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNoRestSequential(t *testing.T) {
|
||||
srv := StartFTPServer(t, ServerOptions{NoRest: true})
|
||||
srv.ReadDelay.Store(int64(time.Millisecond))
|
||||
want := srv.WriteFile(t, "norest.bin", RandomBytes(4*mib+3, 9))
|
||||
|
||||
fm := testManager(srv)
|
||||
f := buildFetcher(fm)
|
||||
resolveTask(t, f, ftpURL(srv, "ftp", "", "norest.bin"), &pftp.OptsExtra{Connections: 4})
|
||||
startAndWait(t, f)
|
||||
checkFile(t, f.Meta().SingleFilepath(), want)
|
||||
|
||||
if rests := srv.RestOffsets(); len(rests) != 0 {
|
||||
t.Fatalf("the server cannot have seen a REST, got %v", rests)
|
||||
}
|
||||
var saved fetcherData
|
||||
if err := json.Unmarshal(storeData(t, fm, f), &saved); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !saved.NoRest {
|
||||
t.Fatal("the missing REST support was not recorded")
|
||||
}
|
||||
}
|
||||
|
||||
// TestResumeAfterKill stops the server's transfers at a fixed byte count,
|
||||
// waits until the client has written exactly those bytes, then pauses, saves,
|
||||
// restores and finishes. No timing decides where the pause lands.
|
||||
func TestResumeAfterKill(t *testing.T) {
|
||||
srv := StartFTPServer(t, ServerOptions{})
|
||||
size := int64(4*mib + 777)
|
||||
want := srv.WriteFile(t, "resume.bin", RandomBytes(int(size), 10))
|
||||
stallAt := size / 2
|
||||
srv.StallAfter(stallAt)
|
||||
|
||||
fm := testManager(srv)
|
||||
f := buildFetcher(fm)
|
||||
resolveTask(t, f, ftpURL(srv, "ftp", "", "resume.bin"), &pftp.OptsExtra{Connections: 2})
|
||||
if err := f.Start(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
select {
|
||||
case <-srv.Stalled():
|
||||
case <-time.After(30 * time.Second):
|
||||
t.Fatal("the server never reached the stall point")
|
||||
}
|
||||
deadline := time.Now().Add(30 * time.Second)
|
||||
for f.Progress().TotalDownloaded() != stallAt {
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("progress %d, want the %d bytes the server sent", f.Progress().TotalDownloaded(), stallAt)
|
||||
}
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
}
|
||||
if err := f.Pause(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
srv.Release()
|
||||
blob := storeData(t, fm, f)
|
||||
|
||||
var saved fetcherData
|
||||
if err := json.Unmarshal(blob, &saved); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var savedTotal int64
|
||||
resumeAt := map[int64]bool{}
|
||||
for _, seg := range saved.Segments {
|
||||
savedTotal += seg.Downloaded
|
||||
if seg.Begin+seg.Downloaded <= seg.End {
|
||||
resumeAt[seg.Begin+seg.Downloaded] = true
|
||||
}
|
||||
}
|
||||
if savedTotal != stallAt {
|
||||
t.Fatalf("saved progress %d, want %d", savedTotal, stallAt)
|
||||
}
|
||||
|
||||
before := len(srv.RetrOffsets())
|
||||
restored := restoreFetcher(t, fm, f.Meta(), blob)
|
||||
if got := restored.Progress().TotalDownloaded(); got != savedTotal {
|
||||
t.Fatalf("restored progress %d, want %d", got, savedTotal)
|
||||
}
|
||||
startAndWait(t, restored)
|
||||
checkFile(t, restored.Meta().SingleFilepath(), want)
|
||||
|
||||
resumed := srv.RetrOffsets()[before:]
|
||||
if len(resumed) == 0 {
|
||||
t.Fatal("no transfer after resume")
|
||||
}
|
||||
for _, off := range resumed {
|
||||
if !resumeAt[off] {
|
||||
t.Fatalf("resumed RETR at %d, want one of the saved offsets %v", off, resumeAt)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestLateRefusalAfterQueueDrained: worker B holds the second file's segment
|
||||
// while the server takes its time to refuse B's login. Worker A finishes the
|
||||
// first file meanwhile and finds the queue empty. B must not leave too, or
|
||||
// nobody fetches the second file.
|
||||
func TestLateRefusalAfterQueueDrained(t *testing.T) {
|
||||
srv := StartFTPServer(t, ServerOptions{MaxLogins: 1, RefuseDelay: 500 * time.Millisecond})
|
||||
want := map[string]string{
|
||||
"a.bin": srv.WriteFile(t, "dir/a.bin", RandomBytes(1000, 16)),
|
||||
"b.bin": srv.WriteFile(t, "dir/b.bin", RandomBytes(1000, 17)),
|
||||
}
|
||||
|
||||
f := buildFetcher(testManager(srv))
|
||||
resolveTask(t, f, ftpURL(srv, "ftp", "", "dir"), &pftp.OptsExtra{Connections: 2})
|
||||
startAndWait(t, f)
|
||||
for name, sum := range want {
|
||||
checkFile(t, filepath.Join(f.Meta().FolderPath(), name), sum)
|
||||
}
|
||||
if srv.Refused() == 0 {
|
||||
t.Fatal("the server never refused a login, so the late refusal was not exercised")
|
||||
}
|
||||
}
|
||||
|
||||
// TestLoginCapLoweredMidway: both workers have logged in when the server's
|
||||
// cap drops to one, as when another client takes a slot. The refused worker
|
||||
// leaves, the ceiling drops to one, and the task finishes.
|
||||
func TestLoginCapLoweredMidway(t *testing.T) {
|
||||
srv := StartFTPServer(t, ServerOptions{})
|
||||
srv.ReadDelay.Store(int64(time.Millisecond))
|
||||
want := map[string]string{}
|
||||
for i := 0; i < 8; i++ {
|
||||
name := fmt.Sprintf("f%d.bin", i)
|
||||
want[name] = srv.WriteFile(t, "dir/"+name, RandomBytes(512<<10, byte(30+i)))
|
||||
}
|
||||
|
||||
fm := testManager(srv)
|
||||
f := buildFetcher(fm)
|
||||
resolveTask(t, f, ftpURL(srv, "ftp", "", "dir"), &pftp.OptsExtra{Connections: 2})
|
||||
if err := f.Start(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
deadline := time.Now().Add(30 * time.Second)
|
||||
for len(ftpStats(t, f).Connections) < 2 || f.Progress().TotalDownloaded() < 512<<10 {
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatal("the two workers never got going")
|
||||
}
|
||||
time.Sleep(2 * time.Millisecond)
|
||||
}
|
||||
srv.SetMaxLogins(1)
|
||||
if err := waitFetcher(t, f); err != nil {
|
||||
t.Fatalf("download: %v", err)
|
||||
}
|
||||
for name, sum := range want {
|
||||
checkFile(t, filepath.Join(f.Meta().FolderPath(), name), sum)
|
||||
}
|
||||
if srv.Refused() == 0 {
|
||||
t.Fatal("the server never refused a login, so the lowered cap was not exercised")
|
||||
}
|
||||
for _, c := range ftpStats(t, f).Connections {
|
||||
if c.Failed {
|
||||
t.Fatalf("a refused worker failed: %+v", c)
|
||||
}
|
||||
}
|
||||
var saved fetcherData
|
||||
if err := json.Unmarshal(storeData(t, fm, f), &saved); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if saved.MaxConns != 1 {
|
||||
t.Fatalf("saved login ceiling %d, want 1", saved.MaxConns)
|
||||
}
|
||||
}
|
||||
|
||||
// TestDataConnectionsResumeOwnSession runs four FTPS workers against a server
|
||||
// that, like ProFTPD, accepts a data connection only when it resumes the TLS
|
||||
// session of its own control connection.
|
||||
func TestDataConnectionsResumeOwnSession(t *testing.T) {
|
||||
srv := StartFTPServer(t, ServerOptions{TLS: pftp.TLSImplicit, RequireSessionReuse: true})
|
||||
srv.ReadDelay.Store(int64(200 * time.Microsecond))
|
||||
want := srv.WriteFile(t, "reuse.bin", RandomBytes(8*mib, 18))
|
||||
|
||||
f := buildFetcher(testManager(srv))
|
||||
resolveTask(t, f, ftpURL(srv, "ftps", "", "reuse.bin"), &pftp.OptsExtra{Connections: 4})
|
||||
startAndWait(t, f)
|
||||
checkFile(t, f.Meta().SingleFilepath(), want)
|
||||
ok, bad := srv.SessionReuse()
|
||||
if ok < 4 || bad != 0 {
|
||||
t.Fatalf("data connections: %d resumed their own session, %d did not", ok, bad)
|
||||
}
|
||||
for _, c := range ftpStats(t, f).Connections {
|
||||
if c.RetryTimes != 0 {
|
||||
t.Fatalf("a worker needed retries: %+v", c)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRetryAfterDroppedConnection(t *testing.T) {
|
||||
srv := StartFTPServer(t, ServerOptions{})
|
||||
want := srv.WriteFile(t, "drop.bin", RandomBytes(3*mib, 11))
|
||||
srv.DropAfter.Store(mib + 100)
|
||||
|
||||
f := buildFetcher(testManager(srv))
|
||||
resolveTask(t, f, ftpURL(srv, "ftp", "", "drop.bin"), &pftp.OptsExtra{Connections: 1})
|
||||
startAndWait(t, f)
|
||||
checkFile(t, f.Meta().SingleFilepath(), want)
|
||||
|
||||
offsets := srv.RetrOffsets()
|
||||
if len(offsets) != 2 || offsets[0] != 0 || offsets[1] <= 0 || offsets[1] > mib+100 {
|
||||
t.Fatalf("RETR offsets %v, want a restart at the dropped offset", offsets)
|
||||
}
|
||||
if got := ftpStats(t, f).Connections[0].RetryTimes; got != 1 {
|
||||
t.Fatalf("retry count %d, want 1", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPercentEncodedPassword(t *testing.T) {
|
||||
srv := StartFTPServer(t, ServerOptions{User: "u", Pass: "p@ss"})
|
||||
want := srv.WriteFile(t, "secret.bin", RandomBytes(5000, 12))
|
||||
|
||||
f := buildFetcher(testManager(srv))
|
||||
resolveTask(t, f, ftpURL(srv, "ftp", "u:p%40ss", "secret.bin"), nil)
|
||||
startAndWait(t, f)
|
||||
checkFile(t, f.Meta().SingleFilepath(), want)
|
||||
|
||||
f = buildFetcher(testManager(srv))
|
||||
err := resolveOpts(f, &base.Request{URL: ftpURL(srv, "ftp", "u:wrong", "secret.bin")}, &base.Options{Path: t.TempDir()})
|
||||
if err == nil || !strings.Contains(err.Error(), "530") {
|
||||
t.Fatalf("a wrong password must fail with 530, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOldSaveFormatResumes(t *testing.T) {
|
||||
srv := StartFTPServer(t, ServerOptions{})
|
||||
data := RandomBytes(2*mib+1, 13)
|
||||
want := srv.WriteFile(t, "old.bin", data)
|
||||
|
||||
fm := testManager(srv)
|
||||
f := buildFetcher(fm)
|
||||
resolveTask(t, f, ftpURL(srv, "ftp", "", "old.bin"), &pftp.OptsExtra{Connections: 1})
|
||||
|
||||
// A save blob that has only the segment fields every version writes.
|
||||
const done = 700000
|
||||
blob := []byte(fmt.Sprintf(`{"Segments":[{"Begin":0,"End":%d,"Downloaded":%d}]}`, len(data)-1, done))
|
||||
name := f.Meta().SingleFilepath()
|
||||
if err := os.WriteFile(name, append(data[:done:done], make([]byte, len(data)-done)...), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
restored := restoreFetcher(t, fm, f.Meta(), blob)
|
||||
if got := restored.Progress().TotalDownloaded(); got != done {
|
||||
t.Fatalf("restored progress %d, want %d", got, done)
|
||||
}
|
||||
startAndWait(t, restored)
|
||||
checkFile(t, name, want)
|
||||
if got := srv.RetrOffsets(); !slices.Equal(got, []int64{done}) {
|
||||
t.Fatalf("RETR offsets %v, want [%d]", got, done)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloadThroughSOCKS5(t *testing.T) {
|
||||
srv := StartFTPServer(t, ServerOptions{TLS: pftp.TLSExplicit})
|
||||
want := srv.WriteFile(t, "proxied.bin", RandomBytes(2*mib+1, 14))
|
||||
|
||||
var dials atomic.Int32
|
||||
proxySrv, err := socks5.New(&socks5.Config{
|
||||
Dial: func(ctx context.Context, network, addr string) (net.Conn, error) {
|
||||
dials.Add(1)
|
||||
var d net.Dialer
|
||||
return d.DialContext(ctx, network, addr)
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ln, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer ln.Close()
|
||||
go proxySrv.Serve(ln)
|
||||
|
||||
f := buildFetcher(testManager(srv))
|
||||
req := &base.Request{
|
||||
URL: ftpURL(srv, "ftpes", "", "proxied.bin"),
|
||||
Proxy: &base.RequestProxy{
|
||||
Mode: base.RequestProxyModeCustom,
|
||||
Scheme: "socks5",
|
||||
Host: ln.Addr().String(),
|
||||
},
|
||||
}
|
||||
if err := resolveOpts(f, req, &base.Options{Path: t.TempDir(), Extra: &pftp.OptsExtra{Connections: 2}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
startAndWait(t, f)
|
||||
checkFile(t, f.Meta().SingleFilepath(), want)
|
||||
// Resolve's control and listing, then per worker one control and one
|
||||
// data connection at least: every one of them went through the proxy.
|
||||
if got := dials.Load(); got < 4 {
|
||||
t.Fatalf("the proxy saw %d connections, want every control and data connection", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHTTPProxyFallsBackToDirect(t *testing.T) {
|
||||
srv := StartFTPServer(t, ServerOptions{})
|
||||
want := srv.WriteFile(t, "direct.bin", RandomBytes(100000, 15))
|
||||
|
||||
var logs bytes.Buffer
|
||||
logger := zerolog.New(&logs)
|
||||
fm := testManager(srv)
|
||||
f := fm.Build().(*Fetcher)
|
||||
ctl := testController(fm)
|
||||
ctl.Logger = &logger
|
||||
f.Setup(ctl)
|
||||
req := &base.Request{
|
||||
URL: ftpURL(srv, "ftp", "", "direct.bin"),
|
||||
Proxy: &base.RequestProxy{
|
||||
Mode: base.RequestProxyModeCustom,
|
||||
Scheme: "http",
|
||||
Host: "127.0.0.1:1",
|
||||
Usr: "proxyuser",
|
||||
Pwd: "proxysecret",
|
||||
},
|
||||
}
|
||||
if err := resolveOpts(f, req, &base.Options{Path: t.TempDir(), Extra: &pftp.OptsExtra{Connections: 2}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
startAndWait(t, f)
|
||||
checkFile(t, f.Meta().SingleFilepath(), want)
|
||||
|
||||
out := logs.String()
|
||||
if n := strings.Count(out, "HTTP proxy"); n != 1 {
|
||||
t.Fatalf("want exactly one HTTP proxy warning, got %d in %q", n, out)
|
||||
}
|
||||
if strings.Contains(out, "proxysecret") {
|
||||
t.Fatalf("the proxy password leaked into the log: %q", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTorrentNameIsFile(t *testing.T) {
|
||||
fm := testManager(nil)
|
||||
for _, u := range []string{"ftp://h/x.torrent", "ftps://h/x.torrent", "ftpes://h/x.torrent", "FTP://h/v.m3u8"} {
|
||||
matched := false
|
||||
for _, filter := range fm.Filters() {
|
||||
if filter.Match(u) {
|
||||
matched = true
|
||||
}
|
||||
}
|
||||
if !matched {
|
||||
t.Fatalf("%s is not matched by the FTP filters", u)
|
||||
}
|
||||
}
|
||||
if fm.Name() != "ftp" {
|
||||
t.Fatalf("manager name %q", fm.Name())
|
||||
}
|
||||
|
||||
srv := StartFTPServer(t, ServerOptions{})
|
||||
want := srv.WriteFile(t, "x.torrent", []byte("d8:announce0:e"))
|
||||
f := buildFetcher(testManager(srv))
|
||||
resolveTask(t, f, ftpURL(srv, "ftp", "", "x.torrent"), nil)
|
||||
res := f.Meta().Res
|
||||
if res.Name != "" || len(res.Files) != 1 || res.Files[0].Name != "x.torrent" {
|
||||
t.Fatalf("x.torrent must resolve to one FTP file, got name %q and %d files", res.Name, len(res.Files))
|
||||
}
|
||||
startAndWait(t, f)
|
||||
checkFile(t, f.Meta().SingleFilepath(), want)
|
||||
}
|
||||
|
||||
func TestParseNameHidesCredentials(t *testing.T) {
|
||||
fm := testManager(nil)
|
||||
for u, want := range map[string]string{
|
||||
"ftp://u:p%[email protected]/pub/a.iso": "a.iso",
|
||||
"ftp://u:p%[email protected]/": "example.com",
|
||||
"ftps://u:p%[email protected]": "example.com",
|
||||
"ftp://u:p%[email protected]/pub/dir/": "dir",
|
||||
} {
|
||||
if got := fm.ParseName(u); got != want {
|
||||
t.Fatalf("ParseName(%q) = %q, want %q", u, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestUnsafeEntryNames(t *testing.T) {
|
||||
for name, ok := range map[string]bool{
|
||||
"a.txt": true, "..a": true, "a b": true,
|
||||
"": false, ".": false, "..": false, "a/b": false, `a\b`: false, "../x": false, "x\x00y": false,
|
||||
} {
|
||||
if got := safeEntryName(name); got != ok {
|
||||
t.Fatalf("safeEntryName(%q) = %v, want %v", name, got, ok)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,587 +0,0 @@
|
||||
package ftp
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"crypto/ecdsa"
|
||||
"crypto/elliptic"
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"crypto/x509/pkix"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"io"
|
||||
"math/big"
|
||||
"net"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
ftpserver "github.com/fclairamb/ftpserverlib"
|
||||
"github.com/spf13/afero"
|
||||
|
||||
pftp "github.com/GopeedLab/gopeed/pkg/protocol/ftp"
|
||||
)
|
||||
|
||||
// ServerOptions are the knobs of the in-process FTP test server.
|
||||
type ServerOptions struct {
|
||||
// TLS is pftp.TLSNone (or empty), pftp.TLSExplicit or pftp.TLSImplicit.
|
||||
// Explicit mode refuses plain logins, so a passing test proves TLS was used.
|
||||
TLS string
|
||||
// MaxLogins caps concurrent control connections. Beyond it the server
|
||||
// greets with "421 Too many connections" and hangs up. Zero means no cap.
|
||||
MaxLogins int
|
||||
// NoRest answers "502" to REST. It filters the plain control stream, so it
|
||||
// works with TLSNone and TLSImplicit only.
|
||||
NoRest bool
|
||||
// NoMLST disables MLST and MLSD, so clients fall back to SIZE and LIST.
|
||||
NoMLST bool
|
||||
// User and Pass are the only accepted credentials. An empty User accepts
|
||||
// any login, anonymous included.
|
||||
User string
|
||||
Pass string
|
||||
// RefuseDelay holds every 421 refusal back this long, like a server
|
||||
// that answers slowly when it is full.
|
||||
RefuseDelay time.Duration
|
||||
// RequireSessionReuse makes every data connection resume the TLS
|
||||
// session of the control connection that opened it, as ProFTPD's
|
||||
// mod_tls does by default. A data connection that does not is refused.
|
||||
// It needs TLSImplicit, where this test's wrapper sees the plain
|
||||
// control stream and so learns which passive port belongs to whom.
|
||||
RequireSessionReuse bool
|
||||
}
|
||||
|
||||
// FTPServer is an ftpserverlib server over a temp directory.
|
||||
type FTPServer struct {
|
||||
Addr string // host:port
|
||||
Root string // directory served as "/"
|
||||
// CA trusts the server's self-signed certificate.
|
||||
CA *x509.CertPool
|
||||
|
||||
opts ServerOptions
|
||||
tlsConf *tls.Config
|
||||
server *ftpserver.FtpServer
|
||||
|
||||
// ReadDelay is slept before every read of a transferred file, in
|
||||
// nanoseconds, to slow downloads down. Tests may change it at any time.
|
||||
ReadDelay atomic.Int64
|
||||
|
||||
// DropAfter, when positive, makes the next transfer fail once after it
|
||||
// has sent that many bytes, as if the data connection were killed.
|
||||
DropAfter atomic.Int64
|
||||
|
||||
mu sync.Mutex
|
||||
active int
|
||||
retrs []int64 // offset of every RETR, 0 when no REST came first
|
||||
refused int
|
||||
logins int
|
||||
|
||||
// Passive port to the control connection that announced it, and the
|
||||
// outcome of every data connection's session check.
|
||||
pasvOwner map[int]string
|
||||
reuseOK int
|
||||
reuseBad int
|
||||
|
||||
// Stall state, see StallAfter. stallMu is held across file reads.
|
||||
stallMu sync.Mutex
|
||||
stallAt int64
|
||||
sent int64
|
||||
stallCh chan struct{}
|
||||
stalled chan struct{}
|
||||
stalledOnce *sync.Once
|
||||
}
|
||||
|
||||
// StartFTPServer starts a server and stops it when the test ends.
|
||||
func StartFTPServer(t *testing.T, opts ServerOptions) *FTPServer {
|
||||
t.Helper()
|
||||
s := &FTPServer{Root: t.TempDir(), opts: opts}
|
||||
s.tlsConf, s.CA = selfSignedTLS(t)
|
||||
|
||||
base, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var ln net.Listener = base
|
||||
if opts.RequireSessionReuse {
|
||||
if opts.TLS != pftp.TLSImplicit {
|
||||
t.Fatal("RequireSessionReuse needs TLSImplicit")
|
||||
}
|
||||
s.requireSessionReuse(t, base.Addr().(*net.TCPAddr).Port)
|
||||
}
|
||||
if opts.TLS == pftp.TLSImplicit {
|
||||
ln = tls.NewListener(ln, s.tlsConf)
|
||||
}
|
||||
ln = &limitListener{Listener: ln, srv: s}
|
||||
|
||||
s.Addr = base.Addr().String()
|
||||
s.server = ftpserver.NewFtpServer(&mainDriver{srv: s, listener: ln})
|
||||
if err := s.server.Listen(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
go s.server.Serve()
|
||||
t.Cleanup(func() { _ = s.server.Stop() })
|
||||
return s
|
||||
}
|
||||
|
||||
// WriteFile creates a file under the served root and returns its SHA-256.
|
||||
func (s *FTPServer) WriteFile(t *testing.T, rel string, data []byte) string {
|
||||
t.Helper()
|
||||
name := filepath.Join(s.Root, filepath.FromSlash(rel))
|
||||
if err := os.MkdirAll(filepath.Dir(name), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(name, data, 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return SHA256Hex(data)
|
||||
}
|
||||
|
||||
// RetrOffsets returns the start offset of every RETR the server received.
|
||||
func (s *FTPServer) RetrOffsets() []int64 {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return append([]int64(nil), s.retrs...)
|
||||
}
|
||||
|
||||
// RestOffsets returns the non-zero RETR offsets, which only a REST sets.
|
||||
func (s *FTPServer) RestOffsets() []int64 {
|
||||
var out []int64
|
||||
for _, off := range s.RetrOffsets() {
|
||||
if off != 0 {
|
||||
out = append(out, off)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Refused returns how many connections were turned away with 421.
|
||||
func (s *FTPServer) Refused() int {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return s.refused
|
||||
}
|
||||
|
||||
// Logins returns the number of successful logins.
|
||||
func (s *FTPServer) Logins() int {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return s.logins
|
||||
}
|
||||
|
||||
// SetMaxLogins changes the login cap while the server runs, as when other
|
||||
// clients take slots.
|
||||
func (s *FTPServer) SetMaxLogins(n int) {
|
||||
s.mu.Lock()
|
||||
s.opts.MaxLogins = n
|
||||
s.mu.Unlock()
|
||||
}
|
||||
|
||||
// SessionReuse returns how many data connections resumed their own control
|
||||
// connection's TLS session, and how many did not.
|
||||
func (s *FTPServer) SessionReuse() (ok, bad int) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return s.reuseOK, s.reuseBad
|
||||
}
|
||||
|
||||
// StallAfter makes every transfer stop sending once the server has sent n
|
||||
// bytes in total, until Release. Stalled is closed when that happens.
|
||||
func (s *FTPServer) StallAfter(n int64) {
|
||||
s.stallMu.Lock()
|
||||
defer s.stallMu.Unlock()
|
||||
s.stallAt, s.sent = n, 0
|
||||
s.stallCh = make(chan struct{})
|
||||
s.stalled = make(chan struct{})
|
||||
s.stalledOnce = &sync.Once{}
|
||||
}
|
||||
|
||||
// Stalled is closed once a transfer has stopped at the StallAfter limit.
|
||||
func (s *FTPServer) Stalled() <-chan struct{} {
|
||||
s.stallMu.Lock()
|
||||
defer s.stallMu.Unlock()
|
||||
return s.stalled
|
||||
}
|
||||
|
||||
// Sent returns the bytes sent since StallAfter.
|
||||
func (s *FTPServer) Sent() int64 {
|
||||
s.stallMu.Lock()
|
||||
defer s.stallMu.Unlock()
|
||||
return s.sent
|
||||
}
|
||||
|
||||
// Release ends the stall. The stalled transfers fail.
|
||||
func (s *FTPServer) Release() {
|
||||
s.stallMu.Lock()
|
||||
defer s.stallMu.Unlock()
|
||||
if s.stallAt > 0 {
|
||||
s.stallAt = 0
|
||||
close(s.stallCh)
|
||||
}
|
||||
}
|
||||
|
||||
var errStalled = errors.New("test server released a stalled transfer")
|
||||
|
||||
// stall limits a read to the StallAfter budget, and blocks once the budget
|
||||
// is spent.
|
||||
func (s *FTPServer) stall(read func([]byte) (int, error), p []byte) (int, error) {
|
||||
s.stallMu.Lock()
|
||||
if s.stallAt <= 0 {
|
||||
s.stallMu.Unlock()
|
||||
return read(p)
|
||||
}
|
||||
allowed := s.stallAt - s.sent
|
||||
if allowed <= 0 {
|
||||
ch := s.stallCh
|
||||
s.stalledOnce.Do(func() { close(s.stalled) })
|
||||
s.stallMu.Unlock()
|
||||
<-ch
|
||||
return 0, errStalled
|
||||
}
|
||||
if int64(len(p)) > allowed {
|
||||
p = p[:allowed]
|
||||
}
|
||||
n, err := read(p)
|
||||
s.sent += int64(n)
|
||||
s.stallMu.Unlock()
|
||||
return n, err
|
||||
}
|
||||
|
||||
var pasvReply = regexp.MustCompile(`^(?:229 [^(]*\(\|\|\|(\d+)\|\)|227 [^(]*\(\d+,\d+,\d+,\d+,(\d+),(\d+)\))`)
|
||||
|
||||
// notePassive records the passive port a control connection announced.
|
||||
func (s *FTPServer) notePassive(owner string, reply []byte) {
|
||||
m := pasvReply.FindSubmatch(reply)
|
||||
if m == nil {
|
||||
return
|
||||
}
|
||||
var port int
|
||||
if len(m[1]) > 0 {
|
||||
port, _ = strconv.Atoi(string(m[1]))
|
||||
} else {
|
||||
hi, _ := strconv.Atoi(string(m[2]))
|
||||
lo, _ := strconv.Atoi(string(m[3]))
|
||||
port = hi<<8 | lo
|
||||
}
|
||||
s.mu.Lock()
|
||||
s.pasvOwner[port] = owner
|
||||
s.mu.Unlock()
|
||||
}
|
||||
|
||||
// requireSessionReuse stamps each control connection's session tickets with
|
||||
// its address, and checks that each data connection resumed a ticket of the
|
||||
// control connection that announced its passive port.
|
||||
func (s *FTPServer) requireSessionReuse(t *testing.T, controlPort int) {
|
||||
base := s.tlsConf
|
||||
var key [32]byte
|
||||
if _, err := rand.Read(key[:]); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
base.SetSessionTicketKeys([][32]byte{key})
|
||||
s.pasvOwner = map[int]string{}
|
||||
stamp := func(owner string) func(tls.ConnectionState, *tls.SessionState) ([]byte, error) {
|
||||
return func(cs tls.ConnectionState, ss *tls.SessionState) ([]byte, error) {
|
||||
ss.Extra = [][]byte{[]byte(owner)}
|
||||
return base.EncryptTicket(cs, ss)
|
||||
}
|
||||
}
|
||||
base.GetConfigForClient = func(hello *tls.ClientHelloInfo) (*tls.Config, error) {
|
||||
cfg := base.Clone()
|
||||
cfg.GetConfigForClient = nil
|
||||
local, _ := hello.Conn.LocalAddr().(*net.TCPAddr)
|
||||
if local == nil || local.Port == controlPort {
|
||||
cfg.WrapSession = stamp(hello.Conn.RemoteAddr().String())
|
||||
return cfg, nil
|
||||
}
|
||||
s.mu.Lock()
|
||||
owner := s.pasvOwner[local.Port]
|
||||
s.mu.Unlock()
|
||||
var resumedFrom string
|
||||
cfg.UnwrapSession = func(identity []byte, cs tls.ConnectionState) (*tls.SessionState, error) {
|
||||
ss, err := base.DecryptTicket(identity, cs)
|
||||
if err != nil || ss == nil {
|
||||
return nil, nil
|
||||
}
|
||||
if len(ss.Extra) > 0 {
|
||||
resumedFrom = string(ss.Extra[0])
|
||||
}
|
||||
return ss, nil
|
||||
}
|
||||
cfg.WrapSession = stamp(owner)
|
||||
cfg.VerifyConnection = func(cs tls.ConnectionState) error {
|
||||
ok := owner != "" && cs.DidResume && resumedFrom == owner
|
||||
s.mu.Lock()
|
||||
if ok {
|
||||
s.reuseOK++
|
||||
} else {
|
||||
s.reuseBad++
|
||||
}
|
||||
s.mu.Unlock()
|
||||
if !ok {
|
||||
return errors.New("data connection did not resume its control connection's TLS session")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
return cfg, nil
|
||||
}
|
||||
}
|
||||
|
||||
func (s *FTPServer) acquire() bool {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.opts.MaxLogins > 0 && s.active >= s.opts.MaxLogins {
|
||||
s.refused++
|
||||
return false
|
||||
}
|
||||
s.active++
|
||||
return true
|
||||
}
|
||||
|
||||
func (s *FTPServer) release() {
|
||||
s.mu.Lock()
|
||||
s.active--
|
||||
s.mu.Unlock()
|
||||
}
|
||||
|
||||
// limitListener counts control connections and refuses those beyond
|
||||
// MaxLogins with a 421 greeting, like vsftpd's max_clients and max_per_ip.
|
||||
type limitListener struct {
|
||||
net.Listener
|
||||
srv *FTPServer
|
||||
}
|
||||
|
||||
func (l *limitListener) Accept() (net.Conn, error) {
|
||||
for {
|
||||
c, err := l.Listener.Accept()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !l.srv.acquire() {
|
||||
go func() {
|
||||
time.Sleep(l.srv.opts.RefuseDelay)
|
||||
_ = c.SetDeadline(time.Now().Add(5 * time.Second))
|
||||
_, _ = io.WriteString(c, "421 Too many connections\r\n")
|
||||
_ = c.Close()
|
||||
}()
|
||||
continue
|
||||
}
|
||||
cc := &countedConn{Conn: c, release: l.srv.release}
|
||||
if l.srv.opts.RequireSessionReuse {
|
||||
owner := c.RemoteAddr().String()
|
||||
cc.sniff = func(b []byte) { l.srv.notePassive(owner, b) }
|
||||
}
|
||||
var conn net.Conn = cc
|
||||
if l.srv.opts.NoRest {
|
||||
conn = &noRestConn{countedConn: conn.(*countedConn), in: bufio.NewReader(c)}
|
||||
}
|
||||
return conn, nil
|
||||
}
|
||||
}
|
||||
|
||||
type countedConn struct {
|
||||
net.Conn
|
||||
once sync.Once
|
||||
release func()
|
||||
sniff func([]byte) // sees every reply the server writes
|
||||
}
|
||||
|
||||
func (c *countedConn) Write(b []byte) (int, error) {
|
||||
if c.sniff != nil {
|
||||
c.sniff(b)
|
||||
}
|
||||
return c.Conn.Write(b)
|
||||
}
|
||||
|
||||
func (c *countedConn) Close() error {
|
||||
c.once.Do(c.release)
|
||||
return c.Conn.Close()
|
||||
}
|
||||
|
||||
// noRestConn answers REST itself with 502 and hides it from the server.
|
||||
type noRestConn struct {
|
||||
*countedConn
|
||||
in *bufio.Reader
|
||||
pending []byte
|
||||
}
|
||||
|
||||
func (c *noRestConn) Read(p []byte) (int, error) {
|
||||
for len(c.pending) == 0 {
|
||||
line, err := c.in.ReadBytes('\n')
|
||||
if len(line) > 0 {
|
||||
if strings.HasPrefix(strings.ToUpper(string(line)), "REST ") {
|
||||
if _, werr := io.WriteString(c.Conn, "502 REST not implemented\r\n"); werr != nil {
|
||||
return 0, werr
|
||||
}
|
||||
} else {
|
||||
c.pending = line
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
if len(c.pending) > 0 {
|
||||
break
|
||||
}
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
n := copy(p, c.pending)
|
||||
c.pending = c.pending[n:]
|
||||
return n, nil
|
||||
}
|
||||
|
||||
type mainDriver struct {
|
||||
srv *FTPServer
|
||||
listener net.Listener
|
||||
}
|
||||
|
||||
func (d *mainDriver) GetSettings() (*ftpserver.Settings, error) {
|
||||
settings := &ftpserver.Settings{
|
||||
Listener: d.listener,
|
||||
DisableMLSD: d.srv.opts.NoMLST,
|
||||
DisableMLST: d.srv.opts.NoMLST,
|
||||
}
|
||||
switch d.srv.opts.TLS {
|
||||
case pftp.TLSExplicit:
|
||||
settings.TLSRequired = ftpserver.MandatoryEncryption
|
||||
case pftp.TLSImplicit:
|
||||
settings.TLSRequired = ftpserver.ImplicitEncryption
|
||||
}
|
||||
return settings, nil
|
||||
}
|
||||
|
||||
func (d *mainDriver) ClientConnected(ftpserver.ClientContext) (string, error) {
|
||||
return "gopeed test server", nil
|
||||
}
|
||||
|
||||
func (d *mainDriver) ClientDisconnected(ftpserver.ClientContext) {}
|
||||
|
||||
func (d *mainDriver) AuthUser(_ ftpserver.ClientContext, user, pass string) (ftpserver.ClientDriver, error) {
|
||||
if d.srv.opts.User != "" && (user != d.srv.opts.User || pass != d.srv.opts.Pass) {
|
||||
return nil, errors.New("bad credentials")
|
||||
}
|
||||
d.srv.mu.Lock()
|
||||
d.srv.logins++
|
||||
d.srv.mu.Unlock()
|
||||
return &clientDriver{Fs: afero.NewBasePathFs(afero.NewOsFs(), d.srv.Root), srv: d.srv}, nil
|
||||
}
|
||||
|
||||
func (d *mainDriver) GetTLSConfig() (*tls.Config, error) {
|
||||
return d.srv.tlsConf, nil
|
||||
}
|
||||
|
||||
type clientDriver struct {
|
||||
afero.Fs
|
||||
srv *FTPServer
|
||||
}
|
||||
|
||||
// GetHandle sees the REST offset of every transfer. The server seeks the
|
||||
// returned file to that offset itself.
|
||||
func (d *clientDriver) GetHandle(name string, flags int, offset int64) (ftpserver.FileTransfer, error) {
|
||||
f, err := d.Fs.OpenFile(name, flags, 0o644)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if flags == os.O_RDONLY {
|
||||
d.srv.mu.Lock()
|
||||
d.srv.retrs = append(d.srv.retrs, offset)
|
||||
d.srv.mu.Unlock()
|
||||
}
|
||||
return &slowFile{File: f, srv: d.srv, dropAfter: d.srv.DropAfter.Swap(0)}, nil
|
||||
}
|
||||
|
||||
var errDropped = errors.New("test server dropped the transfer")
|
||||
|
||||
type slowFile struct {
|
||||
afero.File
|
||||
srv *FTPServer
|
||||
sent int64
|
||||
dropAfter int64
|
||||
}
|
||||
|
||||
func (f *slowFile) Read(p []byte) (int, error) {
|
||||
if delay := time.Duration(f.srv.ReadDelay.Load()); delay > 0 {
|
||||
time.Sleep(delay)
|
||||
if len(p) > 16<<10 {
|
||||
p = p[:16<<10]
|
||||
}
|
||||
}
|
||||
if f.dropAfter > 0 {
|
||||
if f.sent >= f.dropAfter {
|
||||
return 0, errDropped
|
||||
}
|
||||
if rest := f.dropAfter - f.sent; int64(len(p)) > rest {
|
||||
p = p[:rest]
|
||||
}
|
||||
}
|
||||
n, err := f.srv.stall(f.File.Read, p)
|
||||
f.sent += int64(n)
|
||||
return n, err
|
||||
}
|
||||
|
||||
// selfSignedTLS returns a server config with a fresh certificate for
|
||||
// 127.0.0.1 and localhost, and a pool that trusts it.
|
||||
func selfSignedTLS(t *testing.T) (*tls.Config, *x509.CertPool) {
|
||||
t.Helper()
|
||||
key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
tmpl := &x509.Certificate{
|
||||
SerialNumber: big.NewInt(time.Now().UnixNano()),
|
||||
Subject: pkix.Name{CommonName: "gopeed ftp test"},
|
||||
NotBefore: time.Now().Add(-time.Hour),
|
||||
NotAfter: time.Now().Add(time.Hour),
|
||||
IPAddresses: []net.IP{net.ParseIP("127.0.0.1")},
|
||||
DNSNames: []string{"localhost"},
|
||||
KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign,
|
||||
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
|
||||
IsCA: true,
|
||||
BasicConstraintsValid: true,
|
||||
}
|
||||
der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cert, err := x509.ParseCertificate(der)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pool := x509.NewCertPool()
|
||||
pool.AddCert(cert)
|
||||
return &tls.Config{
|
||||
Certificates: []tls.Certificate{{Certificate: [][]byte{der}, PrivateKey: key, Leaf: cert}},
|
||||
MinVersion: tls.VersionTLS12,
|
||||
}, pool
|
||||
}
|
||||
|
||||
func SHA256Hex(data []byte) string {
|
||||
sum := sha256.Sum256(data)
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
func FileSHA256(t *testing.T, name string) string {
|
||||
t.Helper()
|
||||
data, err := os.ReadFile(name)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return SHA256Hex(data)
|
||||
}
|
||||
|
||||
// RandomBytes returns n reproducible pseudo-random bytes.
|
||||
func RandomBytes(n int, seed byte) []byte {
|
||||
out := make([]byte, n)
|
||||
x := uint32(seed) + 1
|
||||
for i := range out {
|
||||
x ^= x << 13
|
||||
x ^= x >> 17
|
||||
x ^= x << 5
|
||||
out[i] = byte(x)
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -1,187 +0,0 @@
|
||||
package ftp
|
||||
|
||||
import (
|
||||
"cmp"
|
||||
"fmt"
|
||||
"path"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/jlaffaye/ftp"
|
||||
|
||||
"github.com/GopeedLab/gopeed/pkg/base"
|
||||
)
|
||||
|
||||
// Limits of a directory download. A tree past either one is refused with an
|
||||
// error that names the limit; it is never cut short.
|
||||
const maxWalkDepth = 8
|
||||
|
||||
// maxWalkFiles is a variable so tests can lower it.
|
||||
var maxWalkFiles = 10000
|
||||
|
||||
// resolveTarget describes the URL's path: one file, or a directory walked
|
||||
// into a multi-file resource whose paths are relative to it.
|
||||
func (f *Fetcher) resolveTarget(s *session, t *target) (*base.Resource, error) {
|
||||
isDir, size, mtime, err := f.stat(s, t.path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !isDir {
|
||||
return &base.Resource{
|
||||
Size: size,
|
||||
Range: true,
|
||||
Files: []*base.FileInfo{{
|
||||
Name: path.Base(t.path),
|
||||
Size: size,
|
||||
Ctime: mtime,
|
||||
}},
|
||||
}, nil
|
||||
}
|
||||
|
||||
files, err := f.walk(s, t.path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
name := path.Base(strings.TrimSuffix(t.path, "/"))
|
||||
if name == "" || name == "." || name == "/" {
|
||||
name = t.host
|
||||
}
|
||||
res := &base.Resource{Name: name, Range: true, Files: files}
|
||||
res.CalcSize(nil)
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// stat tells a file from a directory. MLST answers both at once. Without
|
||||
// it, SIZE answers for a file, and a directory is one we can change into.
|
||||
func (f *Fetcher) stat(s *session, p string) (isDir bool, size int64, mtime *time.Time, err error) {
|
||||
if p == "" || strings.HasSuffix(p, "/") {
|
||||
return true, 0, nil, nil
|
||||
}
|
||||
s.deadline()
|
||||
if entry, mlstErr := s.GetEntry(p); mlstErr == nil {
|
||||
switch entry.Type {
|
||||
case ftp.EntryTypeFolder:
|
||||
return true, 0, nil, nil
|
||||
case ftp.EntryTypeFile:
|
||||
if !entry.Time.IsZero() {
|
||||
t := entry.Time
|
||||
mtime = &t
|
||||
}
|
||||
return false, int64(entry.Size), mtime, nil
|
||||
}
|
||||
// A link: SIZE below follows it on the server.
|
||||
}
|
||||
// MLST is missing or refused the path: the older commands decide. A
|
||||
// broken connection fails them too, with its own error.
|
||||
|
||||
s.deadline()
|
||||
size, sizeErr := s.FileSize(p)
|
||||
if sizeErr == nil {
|
||||
if s.IsGetTimeSupported() {
|
||||
s.deadline()
|
||||
if t, err := s.GetTime(p); err == nil {
|
||||
mtime = &t
|
||||
}
|
||||
}
|
||||
return false, size, mtime, nil
|
||||
}
|
||||
if replyCode(sizeErr) == 0 {
|
||||
return false, 0, nil, sizeErr
|
||||
}
|
||||
|
||||
// SIZE failed: a directory, or a missing path.
|
||||
s.deadline()
|
||||
cwd, err := s.CurrentDir()
|
||||
if err != nil {
|
||||
return false, 0, nil, err
|
||||
}
|
||||
s.deadline()
|
||||
if err := s.ChangeDir(p); err != nil {
|
||||
return false, 0, nil, sizeErr
|
||||
}
|
||||
s.deadline()
|
||||
if err := s.ChangeDir(cwd); err != nil {
|
||||
return false, 0, nil, err
|
||||
}
|
||||
return true, 0, nil, nil
|
||||
}
|
||||
|
||||
// safeEntryName reports whether a name from a listing can become a local
|
||||
// path element. A hostile server could otherwise list "../x" and write
|
||||
// outside the task directory.
|
||||
func safeEntryName(name string) bool {
|
||||
if name == "" || name == "." || name == ".." {
|
||||
return false
|
||||
}
|
||||
return !strings.ContainsAny(name, "/\\\x00")
|
||||
}
|
||||
|
||||
// walk lists a directory tree, at most maxWalkDepth levels below root and
|
||||
// maxWalkFiles files. Files come back sorted by path, then name.
|
||||
func (f *Fetcher) walk(s *session, root string) ([]*base.FileInfo, error) {
|
||||
type dir struct {
|
||||
remote string // path on the server
|
||||
rel string // path below root, "" for root itself
|
||||
depth int
|
||||
}
|
||||
var files []*base.FileInfo
|
||||
stack := []dir{{remote: strings.TrimSuffix(root, "/"), rel: "", depth: 0}}
|
||||
for len(stack) > 0 {
|
||||
d := stack[len(stack)-1]
|
||||
stack = stack[:len(stack)-1]
|
||||
|
||||
s.deadline()
|
||||
entries, err := s.List(d.remote)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, e := range entries {
|
||||
if e.Name == "." || e.Name == ".." {
|
||||
continue
|
||||
}
|
||||
if !safeEntryName(e.Name) {
|
||||
f.warn("ftp: skipping a listed name that is not a safe file name: %q", e.Name)
|
||||
continue
|
||||
}
|
||||
remote := path.Join(d.remote, e.Name)
|
||||
rel := path.Join(d.rel, e.Name)
|
||||
if d.remote == "" {
|
||||
remote = e.Name
|
||||
}
|
||||
entryType, size := e.Type, int64(e.Size)
|
||||
if entryType == ftp.EntryTypeLink {
|
||||
// A link to a file has a size; a link to a directory is
|
||||
// not followed, so a loop cannot make the walk endless.
|
||||
s.deadline()
|
||||
linkSize, err := s.FileSize(remote)
|
||||
if err != nil {
|
||||
f.warn("ftp: skipping link %q: %v", rel, err)
|
||||
continue
|
||||
}
|
||||
entryType, size = ftp.EntryTypeFile, linkSize
|
||||
}
|
||||
switch entryType {
|
||||
case ftp.EntryTypeFolder:
|
||||
if d.depth+1 > maxWalkDepth {
|
||||
return nil, fmt.Errorf("ftp: the folder is deeper than %d levels, the limit for a folder download", maxWalkDepth)
|
||||
}
|
||||
stack = append(stack, dir{remote: remote, rel: rel, depth: d.depth + 1})
|
||||
case ftp.EntryTypeFile:
|
||||
if len(files) >= maxWalkFiles {
|
||||
return nil, fmt.Errorf("ftp: the folder has more than %d files, the limit for a folder download", maxWalkFiles)
|
||||
}
|
||||
fi := &base.FileInfo{Name: e.Name, Path: d.rel, Size: size}
|
||||
if !e.Time.IsZero() {
|
||||
t := e.Time
|
||||
fi.Ctime = &t
|
||||
}
|
||||
files = append(files, fi)
|
||||
}
|
||||
}
|
||||
}
|
||||
slices.SortFunc(files, func(a, b *base.FileInfo) int {
|
||||
return cmp.Or(strings.Compare(a.Path, b.Path), strings.Compare(a.Name, b.Name))
|
||||
})
|
||||
return files, nil
|
||||
}
|
||||
@@ -172,11 +172,11 @@ func (f *Fetcher) Meta() *fetcher.FetcherMeta {
|
||||
return f.meta
|
||||
}
|
||||
|
||||
func (f *Fetcher) Resolve(req *base.Request) (err error) {
|
||||
func (f *Fetcher) Resolve(req *base.Request, opts *base.Options) (err error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.meta.Req = req
|
||||
|
||||
f.meta.Opts = opts
|
||||
f.extra = &ReqExtra{}
|
||||
if err = base.ParseReqExtra[ReqExtra](req); err != nil {
|
||||
return
|
||||
@@ -194,6 +194,10 @@ func (f *Fetcher) Resolve(req *base.Request) (err error) {
|
||||
}
|
||||
state.StagingID = randomStagingID()
|
||||
f.state = state
|
||||
// Pin the staging folder now so the very first persisted state already
|
||||
// carries it: a crash right after create still resumes into the same
|
||||
// folder.
|
||||
f.resolveStagingDirLocked()
|
||||
size := planTotalSize(state.Segments)
|
||||
f.meta.Res = &base.Resource{
|
||||
Range: false,
|
||||
|
||||
@@ -80,8 +80,7 @@ func mustResolve(t *testing.T, f *Fetcher, playlistURL, referer string) {
|
||||
if referer != "" {
|
||||
req.Extra = &ReqExtra{Header: map[string]string{"Referer": referer}}
|
||||
}
|
||||
f.Meta().Opts = &base.Options{Path: t.TempDir()}
|
||||
if err := f.Resolve(req); err != nil {
|
||||
if err := f.Resolve(req, &base.Options{Path: t.TempDir()}); err != nil {
|
||||
t.Fatalf("resolve: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -208,8 +207,7 @@ func TestFetcherE2E_RefererEnforced(t *testing.T) {
|
||||
// Without Referer the resolve itself must fail (403 on the playlist).
|
||||
blocked := newTestFetcher(t, testConfig())
|
||||
req := &base.Request{URL: server.URL + "/video.m3u8"}
|
||||
blocked.Meta().Opts = &base.Options{Path: t.TempDir()}
|
||||
if err := blocked.Resolve(req); err == nil {
|
||||
if err := blocked.Resolve(req, &base.Options{Path: t.TempDir()}); err == nil {
|
||||
t.Fatal("resolve without Referer should fail")
|
||||
}
|
||||
|
||||
@@ -219,8 +217,7 @@ func TestFetcherE2E_RefererEnforced(t *testing.T) {
|
||||
URL: server.URL + "/video.m3u8",
|
||||
Extra: &ReqExtra{Header: map[string]string{"Referer": page}},
|
||||
}
|
||||
allowed.Meta().Opts = &base.Options{Path: t.TempDir()}
|
||||
if err := allowed.Resolve(allowedReq); err != nil {
|
||||
if err := allowed.Resolve(allowedReq, &base.Options{Path: t.TempDir()}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
mustRun(t, allowed)
|
||||
@@ -563,8 +560,7 @@ func TestFetcherE2E_StoreRestoreAuthenticated(t *testing.T) {
|
||||
URL: server.URL + "/video.m3u8",
|
||||
Extra: &ReqExtra{Header: map[string]string{"Referer": page}},
|
||||
}
|
||||
f.Meta().Opts = &base.Options{Path: t.TempDir()}
|
||||
if err := f.Resolve(req); err != nil {
|
||||
if err := f.Resolve(req, &base.Options{Path: t.TempDir()}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := f.Start(); err != nil {
|
||||
@@ -634,8 +630,7 @@ func TestFetcherE2E_PatchPartialUpdate(t *testing.T) {
|
||||
Extra: &ReqExtra{Header: map[string]string{"Referer": page}},
|
||||
}
|
||||
opts := &base.Options{Path: t.TempDir()}
|
||||
f.Meta().Opts = opts
|
||||
if err := f.Resolve(req); err != nil {
|
||||
if err := f.Resolve(req, opts); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -718,8 +713,7 @@ func TestFetcherE2E_PatchWhileRunningRejected(t *testing.T) {
|
||||
defer server.Close()
|
||||
|
||||
f := newTestFetcher(t, testConfig())
|
||||
f.Meta().Opts = &base.Options{Path: t.TempDir()}
|
||||
if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}); err != nil {
|
||||
if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: t.TempDir()}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := f.Start(); err != nil {
|
||||
@@ -773,8 +767,7 @@ func TestFetcherE2E_RetryAfterFailure(t *testing.T) {
|
||||
cfg.MaxRetries = 0 // fail the run on the first bad segment
|
||||
cfg.PrefetchContentLength = false
|
||||
f := newTestFetcher(t, cfg)
|
||||
f.Meta().Opts = &base.Options{Path: t.TempDir()}
|
||||
if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}); err != nil {
|
||||
if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: t.TempDir()}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := f.Start(); err != nil {
|
||||
@@ -835,8 +828,7 @@ func TestFetcherE2E_StateMissingSelfHeal(t *testing.T) {
|
||||
defer server.Close()
|
||||
|
||||
f := newTestFetcher(t, testConfig())
|
||||
f.Meta().Opts = &base.Options{Path: t.TempDir()}
|
||||
if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}); err != nil {
|
||||
if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: t.TempDir()}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Simulate the crash window where the engine persisted the task with its
|
||||
|
||||
@@ -189,8 +189,7 @@ func TestHLSRealMedia_TS(t *testing.T) {
|
||||
defer server.Close()
|
||||
|
||||
f := newTestFetcher(t, testConfig())
|
||||
f.Meta().Opts = &base.Options{Path: t.TempDir()}
|
||||
if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}); err != nil {
|
||||
if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: t.TempDir()}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := f.Start(); err != nil {
|
||||
@@ -226,8 +225,7 @@ func TestHLSRealMedia_FMP4(t *testing.T) {
|
||||
defer server.Close()
|
||||
|
||||
f := newTestFetcher(t, testConfig())
|
||||
f.Meta().Opts = &base.Options{Path: t.TempDir()}
|
||||
if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}); err != nil {
|
||||
if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: t.TempDir()}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := f.Start(); err != nil {
|
||||
@@ -276,8 +274,7 @@ func TestHLSRealMedia_AES128(t *testing.T) {
|
||||
defer server.Close()
|
||||
|
||||
f := newTestFetcher(t, testConfig())
|
||||
f.Meta().Opts = &base.Options{Path: t.TempDir()}
|
||||
if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}); err != nil {
|
||||
if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: t.TempDir()}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if f.meta.Res.Size != 0 {
|
||||
|
||||
@@ -50,20 +50,15 @@ func TestFetcherE2E_SameURLTwoTasks(t *testing.T) {
|
||||
// user duplicating a task. The engine would rename the second output
|
||||
// (AutoRename); mirror that here.
|
||||
f1 := newTestFetcher(t, cfg)
|
||||
f1.Meta().Opts = &base.Options{Path: dir}
|
||||
if err := f1.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}); err != nil {
|
||||
if err := f1.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: dir}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f2 := newTestFetcher(t, cfg)
|
||||
f2.Meta().Opts = &base.Options{Path: dir, Name: "video-2.ts"}
|
||||
if err := f2.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}); err != nil {
|
||||
if err := f2.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: dir, Name: "video-2.ts"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if f1.state.StagingID == "" || f1.state.StagingID == f2.state.StagingID {
|
||||
t.Fatal("staging identities must be distinct per task")
|
||||
}
|
||||
if f1.state.StagingDir != "" || f2.state.StagingDir != "" {
|
||||
t.Fatal("Resolve pinned an output directory")
|
||||
if f1.state.StagingDir == "" || f1.state.StagingDir == f2.state.StagingDir {
|
||||
t.Fatalf("staging dirs must be distinct per task: %q vs %q", f1.state.StagingDir, f2.state.StagingDir)
|
||||
}
|
||||
|
||||
// Start task 1, let it stage some segments, then pause it.
|
||||
@@ -140,8 +135,7 @@ func TestFetcherE2E_ForeignJournalDiscarded(t *testing.T) {
|
||||
cfg := testConfig()
|
||||
cfg.PrefetchContentLength = false
|
||||
f := newTestFetcher(t, cfg)
|
||||
f.Meta().Opts = &base.Options{Path: t.TempDir()}
|
||||
if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}); err != nil {
|
||||
if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: t.TempDir()}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Plant a journal that does not belong to this plan (wrong fingerprint).
|
||||
@@ -182,8 +176,7 @@ func TestFetcherE2E_MergeFailurePreservesStaging(t *testing.T) {
|
||||
defer server.Close()
|
||||
|
||||
f := newTestFetcher(t, testConfig())
|
||||
f.Meta().Opts = &base.Options{Path: t.TempDir()}
|
||||
if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}); err != nil {
|
||||
if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: t.TempDir()}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
run := &hlsRun{
|
||||
@@ -257,8 +250,7 @@ func TestFetcherE2E_ReMergeOverExistingOutput(t *testing.T) {
|
||||
defer server.Close()
|
||||
|
||||
f := newTestFetcher(t, testConfig())
|
||||
f.Meta().Opts = &base.Options{Path: t.TempDir()}
|
||||
if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}); err != nil {
|
||||
if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: t.TempDir()}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
tempDir := f.resolveStagingDirLocked()
|
||||
|
||||
+185
-195
@@ -39,11 +39,6 @@ const (
|
||||
stealMinChunkSize = 512 * 1024 // Min steal size: 512KB (avoid tiny chunks)
|
||||
)
|
||||
|
||||
// connReadBufferSize is the read buffer each connection allocates once and
|
||||
// reuses for its lifetime. 64 KiB keeps per-read overhead low on fast links;
|
||||
// at 256 connections it is 16 MiB per task.
|
||||
const connReadBufferSize = 64 * 1024
|
||||
|
||||
var (
|
||||
errRangeRequestIgnored = errors.New("server ignored HTTP range request")
|
||||
errInvalidRangeResponse = errors.New("invalid HTTP range response")
|
||||
@@ -303,9 +298,8 @@ type Fetcher struct {
|
||||
resolveDataPos atomic.Int64 // How many bytes downloaded during resolve
|
||||
|
||||
// Resolve response - kept open for one-time URLs
|
||||
resolveResp *http.Response
|
||||
resolveRespCancel context.CancelFunc // Cancels the Resolve request
|
||||
resolveRespLock sync.Mutex
|
||||
resolveResp *http.Response
|
||||
resolveRespLock sync.Mutex
|
||||
|
||||
// Async prefetch during resolve phase
|
||||
prefetchFile *os.File // Temporary file for prefetch data
|
||||
@@ -314,21 +308,16 @@ type Fetcher struct {
|
||||
prefetchDone atomic.Bool // Prefetch completed or stopped
|
||||
prefetchErr error // Error during prefetch (if any)
|
||||
prefetchStopCh chan struct{} // Signal to stop prefetch
|
||||
prefetchExitCh chan struct{} // Closed when the prefetch goroutine exits
|
||||
resolveFallback atomic.Bool // Resolve prefetch is still available until the first Range request takes over
|
||||
|
||||
// Target file
|
||||
file *os.File
|
||||
fileMu sync.Mutex
|
||||
redirectURL string
|
||||
redirectLock sync.Mutex
|
||||
ifRange string
|
||||
rangeReprobeEligible bool
|
||||
rangeValidatorPinned bool
|
||||
// sizeKnown distinguishes a display estimate from an HTTP-confirmed length,
|
||||
// including zero. Nil preserves the interpretation of legacy snapshots.
|
||||
sizeKnown *bool
|
||||
// An interrupted chunked sequential response must restart from byte zero.
|
||||
file *os.File
|
||||
fileMu sync.Mutex
|
||||
redirectURL string
|
||||
redirectLock sync.Mutex
|
||||
ifRange string
|
||||
rangeReprobeEligible bool
|
||||
rangeValidatorPinned bool
|
||||
sequentialSizeUnknown bool
|
||||
|
||||
// Lifecycle control
|
||||
@@ -375,40 +364,6 @@ func (f *Fetcher) setState(s fetcherState) {
|
||||
f.state.Store(int32(s))
|
||||
}
|
||||
|
||||
// Size state is read under connMu or before download workers are started.
|
||||
func (f *Fetcher) hasKnownSize() bool {
|
||||
if f.sizeKnown != nil {
|
||||
return *f.sizeKnown
|
||||
}
|
||||
return f.meta != nil && f.meta.Res != nil && f.meta.Res.Size > 0 && !f.sequentialSizeUnknown
|
||||
}
|
||||
|
||||
func (f *Fetcher) knownSize() int64 {
|
||||
if f.hasKnownSize() {
|
||||
return f.meta.Res.Size
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func (f *Fetcher) sizeHint() int64 {
|
||||
if f.meta.Opts != nil && f.meta.Opts.Size > 0 {
|
||||
return f.meta.Opts.Size
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// setResourceSizeLocked publishes the value and its authority in one snapshot.
|
||||
func (f *Fetcher) setResourceSizeLocked(size int64, known bool) {
|
||||
if !known {
|
||||
size = f.sizeHint()
|
||||
}
|
||||
f.sizeKnown = &known
|
||||
f.meta.Res.Size = size
|
||||
if len(f.meta.Res.Files) > 0 && f.meta.Res.Files[0] != nil {
|
||||
f.meta.Res.Files[0].Size = size
|
||||
}
|
||||
}
|
||||
|
||||
// updateMaxConnTime updates maxConnTime if the new duration is larger
|
||||
func (f *Fetcher) updateMaxConnTime(d time.Duration) {
|
||||
newVal := int64(d)
|
||||
@@ -417,21 +372,26 @@ func (f *Fetcher) updateMaxConnTime(d time.Duration) {
|
||||
}
|
||||
}
|
||||
|
||||
func (f *Fetcher) Resolve(req *base.Request) error {
|
||||
func (f *Fetcher) Resolve(req *base.Request, opts *base.Options) error {
|
||||
if err := base.ParseReqExtra[fhttp.ReqExtra](req); err != nil {
|
||||
return err
|
||||
}
|
||||
f.meta.Req = req
|
||||
f.meta.Opts = opts
|
||||
if f.meta.Opts == nil {
|
||||
f.meta.Opts = &base.Options{}
|
||||
}
|
||||
|
||||
if err := f.initOptions(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
f.setState(stateResolving)
|
||||
|
||||
// Build HTTP request WITHOUT Range header (normal request)
|
||||
// This allows the response to be reused for downloading (important for one-time URLs)
|
||||
// The body outlives Resolve, so the read timeout gets its own cancel func.
|
||||
reqCtx, reqCancel := context.WithCancel(context.Background())
|
||||
httpReq, err := f.buildRequest(reqCtx, req)
|
||||
httpReq, err := f.buildRequest(context.TODO(), req)
|
||||
if err != nil {
|
||||
reqCancel()
|
||||
f.setState(stateError)
|
||||
return err
|
||||
}
|
||||
@@ -443,7 +403,6 @@ func (f *Fetcher) Resolve(req *base.Request) error {
|
||||
connStartTime := time.Now()
|
||||
resp, err := client.Do(httpReq)
|
||||
if err != nil {
|
||||
reqCancel()
|
||||
f.setState(stateError)
|
||||
return err
|
||||
}
|
||||
@@ -458,7 +417,6 @@ func (f *Fetcher) Resolve(req *base.Request) error {
|
||||
|
||||
if resp.StatusCode != base.HttpCodeOK && resp.StatusCode != base.HttpCodePartialContent {
|
||||
resp.Body.Close()
|
||||
reqCancel()
|
||||
f.setState(stateError)
|
||||
return NewRequestError(resp.StatusCode)
|
||||
}
|
||||
@@ -476,7 +434,6 @@ func (f *Fetcher) Resolve(req *base.Request) error {
|
||||
start, _, total, err := parseRangeResponse(resp)
|
||||
if err != nil {
|
||||
resp.Body.Close()
|
||||
reqCancel()
|
||||
f.setState(stateError)
|
||||
return err
|
||||
}
|
||||
@@ -493,13 +450,6 @@ func (f *Fetcher) Resolve(req *base.Request) error {
|
||||
}
|
||||
}
|
||||
}
|
||||
// A hint is only a display value, even when the origin advertises Range.
|
||||
known := resp.StatusCode == base.HttpCodePartialContent || resp.ContentLength >= 0
|
||||
f.sizeKnown = &known
|
||||
f.sequentialSizeUnknown = !res.Range && !known
|
||||
if !known {
|
||||
res.Size = f.sizeHint()
|
||||
}
|
||||
|
||||
// Parse last modified time
|
||||
var lastModifiedTime *time.Time
|
||||
@@ -544,10 +494,8 @@ func (f *Fetcher) Resolve(req *base.Request) error {
|
||||
f.resolveRespLock.Lock()
|
||||
if reuseResolveBody {
|
||||
f.resolveResp = resp
|
||||
f.resolveRespCancel = reqCancel
|
||||
} else {
|
||||
resp.Body.Close()
|
||||
reqCancel()
|
||||
f.prefetchDone.Store(true)
|
||||
}
|
||||
f.resolveRespLock.Unlock()
|
||||
@@ -563,7 +511,6 @@ func (f *Fetcher) Resolve(req *base.Request) error {
|
||||
// For non-range resources, the response will be used directly in Start
|
||||
if res.Range && reuseResolveBody {
|
||||
f.prefetchStopCh = make(chan struct{})
|
||||
f.prefetchExitCh = make(chan struct{})
|
||||
f.resolveFallback.Store(true)
|
||||
go f.asyncPrefetch()
|
||||
}
|
||||
@@ -581,14 +528,11 @@ func (f *Fetcher) Resolve(req *base.Request) error {
|
||||
func (f *Fetcher) asyncPrefetch() {
|
||||
defer func() {
|
||||
f.prefetchDone.Store(true)
|
||||
if f.prefetchExitCh != nil {
|
||||
close(f.prefetchExitCh)
|
||||
}
|
||||
}()
|
||||
|
||||
// Get the resolve response
|
||||
f.resolveRespLock.Lock()
|
||||
resp, cancel := f.resolveResp, f.resolveRespCancel
|
||||
resp := f.resolveResp
|
||||
f.resolveRespLock.Unlock()
|
||||
|
||||
if resp == nil {
|
||||
@@ -599,7 +543,10 @@ func (f *Fetcher) asyncPrefetch() {
|
||||
// Close the response even if temporary-file setup fails or shutdown
|
||||
// rejects a late prefetch. Otherwise the server can remain blocked writing.
|
||||
f.resolveRespLock.Lock()
|
||||
f.closeResolveRespLocked()
|
||||
if f.resolveResp != nil {
|
||||
f.resolveResp.Body.Close()
|
||||
f.resolveResp = nil
|
||||
}
|
||||
f.resolveRespLock.Unlock()
|
||||
}()
|
||||
|
||||
@@ -629,7 +576,7 @@ func (f *Fetcher) asyncPrefetch() {
|
||||
f.prefetchFilePath = tmpFile.Name()
|
||||
|
||||
buf := make([]byte, 32*1024) // 32KB buffer
|
||||
reader := f.responseReader(resp, cancel)
|
||||
reader := f.responseReader(resp)
|
||||
|
||||
for {
|
||||
select {
|
||||
@@ -659,38 +606,23 @@ func (f *Fetcher) asyncPrefetch() {
|
||||
}
|
||||
}
|
||||
|
||||
// stopPrefetch signals the async prefetch to stop and waits until it exits.
|
||||
// Closing the resolve body unblocks a pending Read, so a stalled server cannot
|
||||
// keep the prefetch goroutine writing to the temporary file after we reuse it.
|
||||
func (f *Fetcher) stopPrefetch() {
|
||||
if f.prefetchStopCh == nil {
|
||||
return
|
||||
}
|
||||
|
||||
// Pause and the first connection's takeover can stop concurrently, so the
|
||||
// check-and-close must be under the lock to avoid closing the channel twice.
|
||||
f.resolveRespLock.Lock()
|
||||
select {
|
||||
case <-f.prefetchStopCh:
|
||||
// Already closed
|
||||
default:
|
||||
close(f.prefetchStopCh)
|
||||
}
|
||||
if f.resolveResp != nil {
|
||||
f.resolveResp.Body.Close()
|
||||
f.resolveResp = nil
|
||||
}
|
||||
f.resolveRespLock.Unlock()
|
||||
|
||||
if f.prefetchExitCh != nil {
|
||||
<-f.prefetchExitCh
|
||||
}
|
||||
}
|
||||
|
||||
// stopPrefetchAndGetData stops the async prefetch and returns prefetched bytes
|
||||
// It also copies prefetched data to the target file
|
||||
func (f *Fetcher) stopPrefetchAndCopyData() int64 {
|
||||
f.stopPrefetch()
|
||||
// Signal prefetch to stop (safely)
|
||||
if f.prefetchStopCh != nil {
|
||||
select {
|
||||
case <-f.prefetchStopCh:
|
||||
// Already closed
|
||||
default:
|
||||
close(f.prefetchStopCh)
|
||||
}
|
||||
}
|
||||
|
||||
// Wait for prefetch to finish (with timeout)
|
||||
for i := 0; f.prefetchStopCh != nil && i < 1000 && !f.prefetchDone.Load(); i++ {
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
|
||||
prefetched := f.prefetchSize.Load()
|
||||
if prefetched == 0 {
|
||||
@@ -761,9 +693,6 @@ func (f *Fetcher) prepareFullRedownload() {
|
||||
|
||||
// Options may be replaced between Resolve and Start when creating a resolved task.
|
||||
func (f *Fetcher) initOptions() error {
|
||||
if f.meta.Opts == nil {
|
||||
f.meta.Opts = &base.Options{}
|
||||
}
|
||||
opts := f.meta.Opts
|
||||
// Parse options
|
||||
if err := base.ParseOptExtra[fhttp.OptsExtra](opts); err != nil {
|
||||
@@ -874,7 +803,7 @@ func (f *Fetcher) doStart() error {
|
||||
_, err = os.Stat(name)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
file, err = f.ctl.Touch(name, f.knownSize())
|
||||
file, err = f.ctl.Touch(name, f.meta.Res.Size)
|
||||
} else {
|
||||
return err
|
||||
}
|
||||
@@ -892,16 +821,12 @@ func (f *Fetcher) doStart() error {
|
||||
// For non-range resources, the response will be used directly
|
||||
var prefetchedBytes int64
|
||||
if f.meta.Res.Range {
|
||||
// Read the size once and judge completeness by it alone: Resolve may have
|
||||
// written every byte before it sets prefetchDone, and a snapshot that
|
||||
// covers the whole file would otherwise finish without copying it.
|
||||
prefetched := f.prefetchSize.Load()
|
||||
resolveComplete := f.knownSize() > 0 && prefetched >= f.knownSize()
|
||||
resolveComplete := f.meta.Res.Size > 0 && f.prefetchDone.Load() && f.prefetchErr == nil && f.prefetchSize.Load() >= f.meta.Res.Size
|
||||
if f.resolveFallback.Load() && !resolveComplete {
|
||||
// Keep Resolve downloading until the first Range response has been
|
||||
// validated. The Range request starts at a stable snapshot; a valid
|
||||
// takeover discards any overlap downloaded by Resolve in the meantime.
|
||||
prefetchedBytes = prefetched
|
||||
prefetchedBytes = f.prefetchSize.Load()
|
||||
} else {
|
||||
// Resolve already completed, or no live Resolve fallback remains.
|
||||
prefetchedBytes = f.stopPrefetchAndCopyData()
|
||||
@@ -911,7 +836,10 @@ func (f *Fetcher) doStart() error {
|
||||
if !f.resolveFallback.Load() {
|
||||
// Also close resolve response if still open.
|
||||
f.resolveRespLock.Lock()
|
||||
f.closeResolveRespLocked()
|
||||
if f.resolveResp != nil {
|
||||
f.resolveResp.Body.Close()
|
||||
f.resolveResp = nil
|
||||
}
|
||||
f.resolveRespLock.Unlock()
|
||||
}
|
||||
}
|
||||
@@ -1065,7 +993,7 @@ func (f *Fetcher) expandConnections() {
|
||||
return
|
||||
}
|
||||
|
||||
totalSize := f.knownSize()
|
||||
totalSize := f.meta.Res.Size
|
||||
|
||||
f.connMu.Lock()
|
||||
|
||||
@@ -1213,7 +1141,7 @@ func (f *Fetcher) runConnection(conn *connection) {
|
||||
|
||||
// Use fast-fail client for quick retry during download phase
|
||||
client := f.buildFastFailClient()
|
||||
buf := make([]byte, connReadBufferSize)
|
||||
buf := make([]byte, 8192)
|
||||
|
||||
retries := 0
|
||||
f.connMu.Lock()
|
||||
@@ -1356,7 +1284,7 @@ func (f *Fetcher) downloadChunkOnce(conn *connection, client *http.Client, buf [
|
||||
rangeMode := f.meta.Res.Range
|
||||
ifRange := ""
|
||||
if resumeProbe {
|
||||
end := f.knownSize() - 1
|
||||
end := f.meta.Res.Size - 1
|
||||
rangeEnd = &end
|
||||
ifRange = f.ifRange
|
||||
} else if rangeMode && (f.rangeValidatorPinned || f.resolveFallback.Load()) {
|
||||
@@ -1364,11 +1292,7 @@ func (f *Fetcher) downloadChunkOnce(conn *connection, client *http.Client, buf [
|
||||
}
|
||||
f.connMu.Unlock()
|
||||
|
||||
// The read timeout cancels this request through its own context.
|
||||
reqCtx, reqCancel := context.WithCancel(conn.ctx)
|
||||
defer reqCancel()
|
||||
|
||||
httpReq, err := f.buildRequest(reqCtx, f.meta.Req)
|
||||
httpReq, err := f.buildRequest(conn.ctx, f.meta.Req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -1396,7 +1320,7 @@ func (f *Fetcher) downloadChunkOnce(conn *connection, client *http.Client, buf [
|
||||
// Check if this might be a redirect URL expiration error
|
||||
// If so, try falling back to the original URL
|
||||
if f.hasRedirectURL() && isRedirectExpiredError(originalErr) {
|
||||
fallbackResp, fallbackErr := f.tryFallbackToOriginalURL(reqCtx, client, rangeStart, rangeEnd, rangeRequested, ifRange)
|
||||
fallbackResp, fallbackErr := f.tryFallbackToOriginalURL(conn.ctx, client, rangeStart, rangeEnd, rangeRequested, ifRange)
|
||||
if fallbackErr == nil && fallbackResp != nil {
|
||||
// Fallback succeeded, use this response instead
|
||||
resp = fallbackResp
|
||||
@@ -1440,7 +1364,7 @@ func (f *Fetcher) downloadChunkOnce(conn *connection, client *http.Client, buf [
|
||||
}
|
||||
if rangeRequested && resp.StatusCode == base.HttpCodePartialContent {
|
||||
var responseTotal int64
|
||||
expectedResponseLength, responseTotal, err = validateRangeResponse(resp, rangeStart, rangeEnd, f.knownSize())
|
||||
expectedResponseLength, responseTotal, err = validateRangeResponse(resp, rangeStart, rangeEnd, f.meta.Res.Size)
|
||||
if err != nil {
|
||||
resp.Body.Close()
|
||||
return err
|
||||
@@ -1465,7 +1389,7 @@ func (f *Fetcher) downloadChunkOnce(conn *connection, client *http.Client, buf [
|
||||
f.rangeValidatorPinned = true
|
||||
f.sequentialSizeUnknown = false
|
||||
conn.Chunk.Begin = 0
|
||||
conn.Chunk.setEnd(f.knownSize() - 1)
|
||||
conn.Chunk.setEnd(f.meta.Res.Size - 1)
|
||||
f.connMu.Unlock()
|
||||
}
|
||||
}
|
||||
@@ -1490,7 +1414,7 @@ func (f *Fetcher) downloadChunkOnce(conn *connection, client *http.Client, buf [
|
||||
resp.Body.Close()
|
||||
return err
|
||||
}
|
||||
} else if f.hasKnownSize() && f.meta.Res.Size > 0 {
|
||||
} else if f.meta.Res.Size > 0 {
|
||||
expectedResponseLength = f.meta.Res.Size
|
||||
if resp.ContentLength >= 0 && resp.ContentLength != expectedResponseLength {
|
||||
resp.Body.Close()
|
||||
@@ -1517,7 +1441,7 @@ func (f *Fetcher) downloadChunkOnce(conn *connection, client *http.Client, buf [
|
||||
f.slowStart.onConnectSuccess()
|
||||
}
|
||||
|
||||
reader := f.responseReader(resp, reqCancel)
|
||||
reader := f.responseReader(resp)
|
||||
var responseBytesRead int64
|
||||
for {
|
||||
if conn.ctx.Err() != nil {
|
||||
@@ -1626,7 +1550,7 @@ func (f *Fetcher) commitResolveTakeover(conn *connection, requestedStart int64)
|
||||
}
|
||||
|
||||
prefetched := f.stopPrefetchAndCopyData()
|
||||
if prefetched < requestedStart || prefetched > f.knownSize() {
|
||||
if prefetched < requestedStart || prefetched > f.meta.Res.Size {
|
||||
return 0, fmt.Errorf("resolve prefetch advanced to invalid offset %d for range start %d", prefetched, requestedStart)
|
||||
}
|
||||
f.resolveDataPos.Store(prefetched)
|
||||
@@ -1651,12 +1575,13 @@ func (f *Fetcher) completeFromResolveFallback(conn *connection) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
select {
|
||||
case <-f.prefetchExitCh:
|
||||
case <-conn.ctx.Done():
|
||||
return false
|
||||
for !f.prefetchDone.Load() {
|
||||
if conn.ctx.Err() != nil {
|
||||
return false
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
knownSize := f.hasKnownSize()
|
||||
knownSize := f.meta.Res.Size > 0
|
||||
if f.prefetchErr != nil || (knownSize && f.prefetchSize.Load() != f.meta.Res.Size) {
|
||||
f.cleanupPrefetchFile()
|
||||
return false
|
||||
@@ -1670,7 +1595,10 @@ func (f *Fetcher) completeFromResolveFallback(conn *connection) bool {
|
||||
|
||||
f.connMu.Lock()
|
||||
if !knownSize {
|
||||
f.setResourceSizeLocked(prefetched, true)
|
||||
f.meta.Res.Size = prefetched
|
||||
if len(f.meta.Res.Files) > 0 && f.meta.Res.Files[0] != nil {
|
||||
f.meta.Res.Files[0].Size = prefetched
|
||||
}
|
||||
}
|
||||
f.meta.Res.Range = false
|
||||
f.rangeReprobeEligible = false
|
||||
@@ -1752,7 +1680,10 @@ func (f *Fetcher) resolveOpenEndedRange(conn *connection, totalSize int64) error
|
||||
f.connMu.Unlock()
|
||||
return err
|
||||
}
|
||||
f.setResourceSizeLocked(totalSize, true)
|
||||
f.meta.Res.Size = totalSize
|
||||
if len(f.meta.Res.Files) > 0 && f.meta.Res.Files[0] != nil {
|
||||
f.meta.Res.Files[0].Size = totalSize
|
||||
}
|
||||
conn.Chunk.setEnd(totalSize - 1)
|
||||
f.connMu.Unlock()
|
||||
return nil
|
||||
@@ -1824,7 +1755,10 @@ func (f *Fetcher) resizeSequentialResource(size int64) error {
|
||||
return err
|
||||
}
|
||||
|
||||
f.setResourceSizeLocked(size, true)
|
||||
f.meta.Res.Size = size
|
||||
if len(f.meta.Res.Files) > 0 {
|
||||
f.meta.Res.Files[0].Size = size
|
||||
}
|
||||
f.sequentialSizeUnknown = false
|
||||
f.connMu.Unlock()
|
||||
return nil
|
||||
@@ -1849,7 +1783,10 @@ func (f *Fetcher) beginUnknownSizeSequentialResource() error {
|
||||
return err
|
||||
}
|
||||
|
||||
f.setResourceSizeLocked(0, false)
|
||||
f.meta.Res.Size = 0
|
||||
if len(f.meta.Res.Files) > 0 {
|
||||
f.meta.Res.Files[0].Size = 0
|
||||
}
|
||||
f.sequentialSizeUnknown = true
|
||||
f.ifRange = ""
|
||||
f.rangeValidatorPinned = false
|
||||
@@ -1866,12 +1803,12 @@ func (f *Fetcher) runConnectionWithResolveResp(conn *connection) {
|
||||
conn.State = connConnecting
|
||||
f.connMu.Unlock()
|
||||
|
||||
buf := make([]byte, connReadBufferSize)
|
||||
buf := make([]byte, 8192)
|
||||
|
||||
// Get the resolve response
|
||||
f.resolveRespLock.Lock()
|
||||
resp, cancel := f.resolveResp, f.resolveRespCancel
|
||||
f.resolveResp, f.resolveRespCancel = nil, nil // Take ownership
|
||||
resp := f.resolveResp
|
||||
f.resolveResp = nil // Take ownership
|
||||
f.resolveRespLock.Unlock()
|
||||
|
||||
if resp == nil {
|
||||
@@ -1880,9 +1817,6 @@ func (f *Fetcher) runConnectionWithResolveResp(conn *connection) {
|
||||
return
|
||||
}
|
||||
|
||||
if cancel != nil {
|
||||
defer cancel()
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
f.connMu.Lock()
|
||||
@@ -1899,7 +1833,7 @@ func (f *Fetcher) runConnectionWithResolveResp(conn *connection) {
|
||||
}
|
||||
|
||||
// Download data from resolve response
|
||||
reader := f.responseReader(resp, cancel)
|
||||
reader := f.responseReader(resp)
|
||||
for {
|
||||
if conn.ctx.Err() != nil {
|
||||
return
|
||||
@@ -1936,9 +1870,11 @@ func (f *Fetcher) runConnectionWithResolveResp(conn *connection) {
|
||||
if err != nil {
|
||||
if err == io.EOF {
|
||||
f.connMu.Lock()
|
||||
if !f.hasKnownSize() {
|
||||
f.setResourceSizeLocked(conn.Downloaded, true)
|
||||
f.sequentialSizeUnknown = false
|
||||
if f.meta.Res.Size == 0 {
|
||||
f.meta.Res.Size = conn.Downloaded
|
||||
if len(f.meta.Res.Files) > 0 && f.meta.Res.Files[0] != nil {
|
||||
f.meta.Res.Files[0].Size = conn.Downloaded
|
||||
}
|
||||
}
|
||||
conn.Completed = true
|
||||
conn.State = connCompleted
|
||||
@@ -1961,7 +1897,7 @@ func (f *Fetcher) runConnectionWithResolveResp(conn *connection) {
|
||||
func (f *Fetcher) runConnectionFallback(conn *connection) {
|
||||
// Use fast-fail client for quick retry during download phase
|
||||
client := f.buildFastFailClient()
|
||||
buf := make([]byte, connReadBufferSize)
|
||||
buf := make([]byte, 8192)
|
||||
|
||||
retries := 0
|
||||
countedRetries := 0
|
||||
@@ -1977,15 +1913,78 @@ func (f *Fetcher) runConnectionFallback(conn *connection) {
|
||||
}
|
||||
|
||||
f.connMu.Lock()
|
||||
if retries > 0 {
|
||||
f.resetConnectionForRestart(conn)
|
||||
}
|
||||
conn.State = connConnecting
|
||||
f.connMu.Unlock()
|
||||
|
||||
// Restored tasks without a reusable Resolve response need the same
|
||||
// length handling and restart semantics as other sequential requests.
|
||||
err := f.downloadChunkOnce(conn, client, buf)
|
||||
err := func() error {
|
||||
httpReq, err := f.buildRequest(conn.ctx, f.meta.Req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Record connection start time for adaptive timeout tracking
|
||||
connStartTime := time.Now()
|
||||
|
||||
resp, err := client.Do(httpReq)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != base.HttpCodeOK && resp.StatusCode != base.HttpCodePartialContent {
|
||||
return NewRequestError(resp.StatusCode)
|
||||
}
|
||||
|
||||
// Record successful connection time for adaptive timeout
|
||||
f.updateMaxConnTime(time.Since(connStartTime))
|
||||
|
||||
f.connMu.Lock()
|
||||
conn.State = connDownloading
|
||||
conn.failed = false
|
||||
f.connMu.Unlock()
|
||||
|
||||
f.primaryReadyOnce.Do(func() {
|
||||
close(f.primaryReadyCh)
|
||||
})
|
||||
if f.slowStart != nil {
|
||||
f.slowStart.onConnectSuccess()
|
||||
}
|
||||
|
||||
reader := f.responseReader(resp)
|
||||
for {
|
||||
if conn.ctx.Err() != nil {
|
||||
return conn.ctx.Err()
|
||||
}
|
||||
|
||||
n, err := reader.Read(buf)
|
||||
if n > 0 {
|
||||
f.connMu.Lock()
|
||||
writeOffset := conn.Chunk.Downloaded
|
||||
f.connMu.Unlock()
|
||||
|
||||
f.fileMu.Lock()
|
||||
if f.file != nil {
|
||||
_, writeErr := f.file.WriteAt(buf[:n], writeOffset)
|
||||
if writeErr != nil {
|
||||
f.fileMu.Unlock()
|
||||
return writeErr
|
||||
}
|
||||
}
|
||||
f.fileMu.Unlock()
|
||||
|
||||
f.connMu.Lock()
|
||||
conn.Chunk.Downloaded += int64(n)
|
||||
conn.Downloaded += int64(n)
|
||||
f.connMu.Unlock()
|
||||
}
|
||||
if err != nil {
|
||||
if err == io.EOF {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
if err == nil {
|
||||
f.connMu.Lock()
|
||||
@@ -2134,14 +2133,14 @@ func (f *Fetcher) resetConnectionForRestart(conn *connection) {
|
||||
// hasSequentialPrefixLocked reports whether a single sequential connection has
|
||||
// a useful contiguous prefix. The caller must hold connMu.
|
||||
func (f *Fetcher) hasSequentialPrefixLocked(conn *connection) bool {
|
||||
if f.meta == nil || f.meta.Res == nil || f.meta.Res.Range || !f.rangeReprobeEligible || f.knownSize() <= 0 {
|
||||
if f.meta == nil || f.meta.Res == nil || f.meta.Res.Range || !f.rangeReprobeEligible || f.meta.Res.Size <= 0 {
|
||||
return false
|
||||
}
|
||||
if conn == nil || conn.ID != 0 || len(f.connections) != 1 || conn.Chunk == nil {
|
||||
return false
|
||||
}
|
||||
downloaded := conn.Chunk.Downloaded
|
||||
return downloaded > 0 && downloaded < f.knownSize()
|
||||
return downloaded > 0 && downloaded < f.meta.Res.Size
|
||||
}
|
||||
|
||||
// canProbeSequentialResumeLocked reports whether the prefix can be resumed
|
||||
@@ -2227,7 +2226,7 @@ func (f *Fetcher) onDownloadComplete() {
|
||||
needsMoreData := false
|
||||
if f.meta.Res.Range {
|
||||
needsMoreData = conn.Chunk != nil && (conn.Chunk.openEnded() || conn.Chunk.remain() > 0)
|
||||
} else if f.hasKnownSize() && f.meta.Res.Size > 0 {
|
||||
} else if f.meta.Res.Size > 0 {
|
||||
needsMoreData = conn.Downloaded < f.meta.Res.Size
|
||||
} else {
|
||||
needsMoreData = !conn.Completed && conn.State != connCompleted
|
||||
@@ -2237,7 +2236,7 @@ func (f *Fetcher) onDownloadComplete() {
|
||||
// This connection has remaining work and isn't done
|
||||
// Check if it failed with 403 (server limit) - these can be ignored if other connections completed the work
|
||||
if conn.State == connFailed && conn.failed {
|
||||
if re := extractRequestError(conn.lastErr); re != nil && re.Code == 403 && f.hasKnownSize() {
|
||||
if re := extractRequestError(conn.lastErr); re != nil && re.Code == 403 {
|
||||
// 403 is server connection limit, check if other connections will complete this chunk
|
||||
continue
|
||||
}
|
||||
@@ -2248,7 +2247,7 @@ func (f *Fetcher) onDownloadComplete() {
|
||||
}
|
||||
|
||||
// If total downloaded matches file size, consider it a success regardless of connection failures
|
||||
downloadComplete := f.hasKnownSize() && f.meta.Res.Size > 0 && totalDownloaded >= f.meta.Res.Size
|
||||
downloadComplete := f.meta.Res.Size > 0 && totalDownloaded >= f.meta.Res.Size
|
||||
|
||||
// Integrity errors must win over byte-count completion. A malformed response
|
||||
// may be detected only after the expected bytes have already been written.
|
||||
@@ -2265,7 +2264,7 @@ func (f *Fetcher) onDownloadComplete() {
|
||||
for _, conn := range f.connections {
|
||||
if conn.State == connFailed && conn.failed {
|
||||
// Skip 403 errors (server connection limit) - these are expected when exceeding server's limit
|
||||
if re := extractRequestError(conn.lastErr); re != nil && re.Code == 403 && f.hasKnownSize() {
|
||||
if re := extractRequestError(conn.lastErr); re != nil && re.Code == 403 {
|
||||
continue
|
||||
}
|
||||
if re := extractRequestError(conn.lastErr); re != nil {
|
||||
@@ -2365,7 +2364,7 @@ func (f *Fetcher) checkCompletion() bool {
|
||||
totalDownloaded += conn.Downloaded
|
||||
}
|
||||
|
||||
if f.hasKnownSize() && f.meta.Res.Size > 0 && totalDownloaded >= f.meta.Res.Size {
|
||||
if f.meta.Res.Size > 0 && totalDownloaded >= f.meta.Res.Size {
|
||||
// Don't start a new goroutine - let the caller handle completion
|
||||
return true
|
||||
}
|
||||
@@ -2392,14 +2391,6 @@ func (f *Fetcher) checkCompletion() bool {
|
||||
|
||||
// Patch modifies the HTTP request information.
|
||||
func (f *Fetcher) Patch(req *base.Request, opts *base.Options) error {
|
||||
if opts != nil && opts.Size > 0 {
|
||||
f.connMu.Lock()
|
||||
f.meta.Opts.Size = opts.Size
|
||||
if f.meta.Res != nil && !f.hasKnownSize() {
|
||||
f.setResourceSizeLocked(0, false)
|
||||
}
|
||||
f.connMu.Unlock()
|
||||
}
|
||||
// Patch request info
|
||||
if req != nil {
|
||||
if req.URL != "" {
|
||||
@@ -2461,7 +2452,14 @@ func (f *Fetcher) Pause() error {
|
||||
}
|
||||
|
||||
// Stop prefetch if running
|
||||
f.stopPrefetch()
|
||||
if f.prefetchStopCh != nil {
|
||||
select {
|
||||
case <-f.prefetchStopCh:
|
||||
// Already closed
|
||||
default:
|
||||
close(f.prefetchStopCh)
|
||||
}
|
||||
}
|
||||
|
||||
// Wait for downloadLoop to exit first (it will call wg.Wait internally)
|
||||
if f.downloadLoopDone != nil {
|
||||
@@ -2471,6 +2469,11 @@ func (f *Fetcher) Pause() error {
|
||||
// Wait for all connection goroutines to stop
|
||||
f.wg.Wait()
|
||||
|
||||
// Wait for prefetch to finish
|
||||
for f.prefetchStopCh != nil && !f.prefetchDone.Load() {
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
|
||||
// If Pause wins before the first Range response is validated, materialize
|
||||
// the Resolve prefix before discarding its temporary file. The first
|
||||
// connection was allocated after the earlier prefetch snapshot, so advance
|
||||
@@ -2485,8 +2488,8 @@ func (f *Fetcher) Pause() error {
|
||||
conn := f.connections[0]
|
||||
if conn != nil && conn.ID == 0 && conn.Chunk != nil && !conn.Completed {
|
||||
conn.Chunk.Begin = prefetched
|
||||
if f.knownSize() > 0 {
|
||||
conn.Chunk.setEnd(f.knownSize() - 1)
|
||||
if f.meta.Res.Size > 0 {
|
||||
conn.Chunk.setEnd(f.meta.Res.Size - 1)
|
||||
} else {
|
||||
conn.Chunk.End = nil
|
||||
}
|
||||
@@ -2502,7 +2505,10 @@ func (f *Fetcher) Pause() error {
|
||||
|
||||
// Clean up resolve response if still held
|
||||
f.resolveRespLock.Lock()
|
||||
f.closeResolveRespLocked()
|
||||
if f.resolveResp != nil {
|
||||
f.resolveResp.Body.Close()
|
||||
f.resolveResp = nil
|
||||
}
|
||||
f.resolveRespLock.Unlock()
|
||||
|
||||
f.fileMu.Lock()
|
||||
@@ -2620,25 +2626,9 @@ func (f *Fetcher) Wait() error {
|
||||
|
||||
// Generated output can pause while its producer waits for processing capacity.
|
||||
// Its producer owns upstream deadlines; cancellation still closes the response.
|
||||
// Otherwise a stalled read cancels the request via cancel, which must belong
|
||||
// to the request that produced resp.
|
||||
func (f *Fetcher) responseReader(resp *http.Response, cancel context.CancelFunc) io.Reader {
|
||||
func (f *Fetcher) responseReader(resp *http.Response) io.Reader {
|
||||
if f.ctl != nil && f.ctl.ManagedProduction != nil && resp.Request != nil && f.ctl.ManagedProduction(resp.Request.URL.String()) {
|
||||
return resp.Body
|
||||
}
|
||||
return NewTimeoutReader(resp.Body, readTimeout, cancel)
|
||||
}
|
||||
|
||||
// closeResolveRespLocked closes the kept Resolve response, if any, and
|
||||
// releases its request context. The caller must hold resolveRespLock.
|
||||
func (f *Fetcher) closeResolveRespLocked() {
|
||||
if f.resolveResp == nil {
|
||||
return
|
||||
}
|
||||
f.resolveResp.Body.Close()
|
||||
f.resolveResp = nil
|
||||
if f.resolveRespCancel != nil {
|
||||
f.resolveRespCancel()
|
||||
f.resolveRespCancel = nil
|
||||
}
|
||||
return NewTimeoutReader(resp.Body, readTimeout)
|
||||
}
|
||||
@@ -95,10 +95,9 @@ func TestFetcher_ChecksumMismatch_Retry(t *testing.T) {
|
||||
}
|
||||
|
||||
// 1. Initial attempt -> will download corruptContent, failing checksum verification
|
||||
if err := f.Resolve(&base.Request{URL: server.URL + "/retry_test.txt"}); err != nil {
|
||||
if err := f.Resolve(&base.Request{URL: server.URL + "/retry_test.txt"}, opts); err != nil {
|
||||
t.Fatalf("Resolve error: %v", err)
|
||||
}
|
||||
f.Meta().Opts = opts
|
||||
if err := f.Start(); err != nil {
|
||||
t.Fatalf("Start error: %v", err)
|
||||
}
|
||||
@@ -216,10 +215,9 @@ func TestFetcher_ChecksumMismatch_Retry_PartialPrefetch(t *testing.T) {
|
||||
}
|
||||
|
||||
// 1. Resolve phase
|
||||
if err := f.Resolve(&base.Request{URL: server.URL + "/prefetch_retry.txt"}); err != nil {
|
||||
if err := f.Resolve(&base.Request{URL: server.URL + "/prefetch_retry.txt"}, opts); err != nil {
|
||||
t.Fatalf("Resolve error: %v", err)
|
||||
}
|
||||
f.Meta().Opts = opts
|
||||
|
||||
// Wait until prefetch has received exactly 7 bytes
|
||||
for i := 0; i < 200; i++ {
|
||||
@@ -340,10 +338,9 @@ func TestFetcher_ChecksumMismatch_Retry_NonRange(t *testing.T) {
|
||||
}
|
||||
|
||||
// 1. Resolve and start initial attempt
|
||||
if err := f.Resolve(&base.Request{URL: server.URL + "/nonrange_retry.txt"}); err != nil {
|
||||
if err := f.Resolve(&base.Request{URL: server.URL + "/nonrange_retry.txt"}, opts); err != nil {
|
||||
t.Fatalf("Resolve error: %v", err)
|
||||
}
|
||||
f.Meta().Opts = opts
|
||||
if err := f.Start(); err != nil {
|
||||
t.Fatalf("Start error: %v", err)
|
||||
}
|
||||
|
||||
@@ -52,11 +52,9 @@ func TestFetcher_ChecksumVerification(t *testing.T) {
|
||||
},
|
||||
}
|
||||
|
||||
if err := f.Resolve(&base.Request{URL: server.URL + "/test.txt"}); err != nil {
|
||||
if err := f.Resolve(&base.Request{URL: server.URL + "/test.txt"}, opts); err != nil {
|
||||
t.Fatalf("Resolve error: %v", err)
|
||||
}
|
||||
|
||||
f.Meta().Opts = opts
|
||||
if err := f.Start(); err != nil {
|
||||
t.Fatalf("Start error: %v", err)
|
||||
}
|
||||
@@ -81,11 +79,9 @@ func TestFetcher_ChecksumVerification(t *testing.T) {
|
||||
},
|
||||
}
|
||||
|
||||
if err := f.Resolve(&base.Request{URL: server.URL + "/test.txt"}); err != nil {
|
||||
if err := f.Resolve(&base.Request{URL: server.URL + "/test.txt"}, opts); err != nil {
|
||||
t.Fatalf("Resolve error: %v", err)
|
||||
}
|
||||
|
||||
f.Meta().Opts = opts
|
||||
if err := f.Start(); err != nil {
|
||||
t.Fatalf("Start error: %v", err)
|
||||
}
|
||||
@@ -110,11 +106,9 @@ func TestFetcher_ChecksumVerification(t *testing.T) {
|
||||
},
|
||||
}
|
||||
|
||||
if err := f.Resolve(&base.Request{URL: server.URL + "/test.txt"}); err != nil {
|
||||
if err := f.Resolve(&base.Request{URL: server.URL + "/test.txt"}, opts); err != nil {
|
||||
t.Fatalf("Resolve error: %v", err)
|
||||
}
|
||||
|
||||
f.Meta().Opts = opts
|
||||
if err := f.Start(); err != nil {
|
||||
t.Fatalf("Start error: %v", err)
|
||||
}
|
||||
@@ -143,13 +137,9 @@ func TestFetcher_ChecksumVerification(t *testing.T) {
|
||||
},
|
||||
}
|
||||
|
||||
if err := f.Resolve(&base.Request{URL: server.URL + "/test.txt"}); err != nil {
|
||||
t.Fatalf("Resolve error: %v", err)
|
||||
}
|
||||
f.Meta().Opts = opts
|
||||
err := f.Start()
|
||||
err := f.Resolve(&base.Request{URL: server.URL + "/test.txt"}, opts)
|
||||
if err == nil {
|
||||
t.Fatal("Start expected error for unsupported algorithm, got nil")
|
||||
t.Fatal("Resolve expected error for unsupported algorithm, got nil")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "unsupported checksum algorithm") {
|
||||
t.Fatalf("expected 'unsupported checksum algorithm' in error, got %v", err)
|
||||
@@ -175,11 +165,9 @@ func TestFetcher_ChecksumVerification(t *testing.T) {
|
||||
},
|
||||
}
|
||||
|
||||
if err := f.Resolve(&base.Request{URL: server.URL + "/test.txt"}); err != nil {
|
||||
if err := f.Resolve(&base.Request{URL: server.URL + "/test.txt"}, opts); err != nil {
|
||||
t.Fatalf("Resolve error: %v", err)
|
||||
}
|
||||
|
||||
f.Meta().Opts = opts
|
||||
if err := f.Start(); err != nil {
|
||||
t.Fatalf("Start error: %v", err)
|
||||
}
|
||||
@@ -204,11 +192,9 @@ func TestFetcher_ChecksumVerification(t *testing.T) {
|
||||
},
|
||||
}
|
||||
|
||||
if err := f.Resolve(&base.Request{URL: server.URL + "/test.txt"}); err != nil {
|
||||
if err := f.Resolve(&base.Request{URL: server.URL + "/test.txt"}, opts); err != nil {
|
||||
t.Fatalf("Resolve error: %v", err)
|
||||
}
|
||||
|
||||
f.Meta().Opts = opts
|
||||
if err := f.Start(); err != nil {
|
||||
t.Fatalf("Start error: %v", err)
|
||||
}
|
||||
@@ -249,11 +235,9 @@ func TestFetcher_ChecksumVerification(t *testing.T) {
|
||||
},
|
||||
}
|
||||
|
||||
if err := f.Resolve(&base.Request{URL: largeServer.URL + "/large.bin"}); err != nil {
|
||||
if err := f.Resolve(&base.Request{URL: largeServer.URL + "/large.bin"}, opts); err != nil {
|
||||
t.Fatalf("Resolve error: %v", err)
|
||||
}
|
||||
|
||||
f.Meta().Opts = opts
|
||||
if err := f.Start(); err != nil {
|
||||
t.Fatalf("Start error: %v", err)
|
||||
}
|
||||
@@ -274,11 +258,9 @@ func TestFetcher_ChecksumVerification(t *testing.T) {
|
||||
},
|
||||
}
|
||||
|
||||
if err := f.Resolve(&base.Request{URL: server.URL + "/test.txt"}); err != nil {
|
||||
if err := f.Resolve(&base.Request{URL: server.URL + "/test.txt"}, opts); err != nil {
|
||||
t.Fatalf("Resolve error: %v", err)
|
||||
}
|
||||
|
||||
f.Meta().Opts = opts
|
||||
if err := f.Start(); err != nil {
|
||||
t.Fatalf("Start error: %v", err)
|
||||
}
|
||||
@@ -287,7 +269,7 @@ func TestFetcher_ChecksumVerification(t *testing.T) {
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("Invalid Checksum In Start Makes No Additional Requests", func(t *testing.T) {
|
||||
t.Run("Invalid Checksum In Resolve Makes Zero Requests", func(t *testing.T) {
|
||||
var reqReceived atomic.Int32
|
||||
customServer := httptest.NewServer(gohttp.HandlerFunc(func(w gohttp.ResponseWriter, r *gohttp.Request) {
|
||||
reqReceived.Add(1)
|
||||
@@ -311,17 +293,12 @@ func TestFetcher_ChecksumVerification(t *testing.T) {
|
||||
},
|
||||
}
|
||||
|
||||
if err := f.Resolve(&base.Request{URL: customServer.URL + "/test.txt"}); err != nil {
|
||||
t.Fatalf("Resolve error: %v", err)
|
||||
}
|
||||
requestsAfterResolve := reqReceived.Load()
|
||||
f.Meta().Opts = opts
|
||||
err := f.Start()
|
||||
err := f.Resolve(&base.Request{URL: customServer.URL + "/test.txt"}, opts)
|
||||
if err == nil {
|
||||
t.Fatal("Start expected error for invalid checksum hash, got nil")
|
||||
t.Fatal("Resolve expected error for invalid checksum hash, got nil")
|
||||
}
|
||||
if got := reqReceived.Load(); got != requestsAfterResolve {
|
||||
t.Fatalf("invalid checksum in Start issued additional requests: got %d, had %d after Resolve", got, requestsAfterResolve)
|
||||
if reqReceived.Load() != 0 {
|
||||
t.Fatalf("expected 0 requests to server on invalid checksum in Resolve, got %d", reqReceived.Load())
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -19,11 +19,10 @@ type fetcherData struct {
|
||||
IfRange string // Strong ETag or Last-Modified validator for safe resume
|
||||
RangeReprobeEligible bool // Origin advertised Range before falling back to sequential mode
|
||||
RangeValidatorPinned bool // Recovered Range mode must keep the same If-Range validator
|
||||
SequentialSizeUnknown bool // Interrupted chunked sequential streams restart from byte zero
|
||||
SizeKnown *bool // Nil for legacy snapshots; false means Size is only a display estimate
|
||||
SequentialSizeUnknown bool // Interrupted chunked restart must retry from byte zero
|
||||
Range *bool // Authoritative Range mode; nil for records saved by older versions
|
||||
ResourceSize *int64 // Display size from the same snapshot; authority is recorded by SizeKnown
|
||||
FileSize *int64 // Single-file size from the same snapshot; nil for older records
|
||||
ResourceSize *int64 // Authoritative resource size from the same snapshot as Connections
|
||||
FileSize *int64 // Authoritative single-file size; nil for older records
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
@@ -94,7 +93,6 @@ func (fm *FetcherManager) Store(f fetcher.Fetcher) (data any, err error) {
|
||||
rangeReprobeEligible := _f.rangeReprobeEligible
|
||||
rangeValidatorPinned := _f.rangeValidatorPinned
|
||||
sequentialSizeUnknown := _f.sequentialSizeUnknown
|
||||
sizeKnown := _f.hasKnownSize()
|
||||
var rangeMode *bool
|
||||
var resourceSize *int64
|
||||
var fileSize *int64
|
||||
@@ -117,7 +115,6 @@ func (fm *FetcherManager) Store(f fetcher.Fetcher) (data any, err error) {
|
||||
RangeReprobeEligible: rangeReprobeEligible,
|
||||
RangeValidatorPinned: rangeValidatorPinned,
|
||||
SequentialSizeUnknown: sequentialSizeUnknown,
|
||||
SizeKnown: &sizeKnown,
|
||||
Range: rangeMode,
|
||||
ResourceSize: resourceSize,
|
||||
FileSize: fileSize,
|
||||
@@ -171,10 +168,6 @@ func (fm *FetcherManager) Restore() (v any, f func(meta *fetcher.FetcherMeta, v
|
||||
fetcher.rangeReprobeEligible = fd.RangeReprobeEligible
|
||||
fetcher.rangeValidatorPinned = fd.RangeValidatorPinned
|
||||
fetcher.sequentialSizeUnknown = fd.SequentialSizeUnknown
|
||||
if fd.SizeKnown != nil {
|
||||
known := *fd.SizeKnown
|
||||
fetcher.sizeKnown = &known
|
||||
}
|
||||
if fd.Range != nil && fetcher.meta.Res != nil {
|
||||
// Recovery-critical resource fields live in the same persisted snapshot
|
||||
// as Connections, making them authoritative over task metadata that may
|
||||
|
||||
@@ -233,11 +233,10 @@ func TestFetcher_ResolveFilenameFromContentType(t *testing.T) {
|
||||
defer fetcher.Pause()
|
||||
|
||||
const opaqueName = "OIP-C.u5nEXTXND3PJF0gNTgw5YgHaFP"
|
||||
fetcher.Meta().Opts = &base.Options{
|
||||
err := fetcher.Resolve(&base.Request{URL: server.URL + "/" + opaqueName}, &base.Options{
|
||||
Name: test.DownloadName,
|
||||
Path: test.Dir,
|
||||
}
|
||||
err := fetcher.Resolve(&base.Request{URL: server.URL + "/" + opaqueName})
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -252,10 +251,6 @@ func TestFetcher_ResolveWithHostHeader(t *testing.T) {
|
||||
defer listener.Close()
|
||||
|
||||
fetcher := buildFetcher()
|
||||
fetcher.Meta().Opts = &base.Options{
|
||||
Name: test.DownloadName,
|
||||
Path: test.Dir,
|
||||
}
|
||||
err := fetcher.Resolve(&base.Request{
|
||||
URL: "http://" + listener.Addr().String() + "/",
|
||||
Extra: &http.ReqExtra{
|
||||
@@ -263,6 +258,9 @@ func TestFetcher_ResolveWithHostHeader(t *testing.T) {
|
||||
"Host": "test",
|
||||
},
|
||||
},
|
||||
}, &base.Options{
|
||||
Name: test.DownloadName,
|
||||
Path: test.Dir,
|
||||
})
|
||||
// The server should return 400 for invalid Host header
|
||||
if err == nil || !strings.Contains(err.Error(), "400") {
|
||||
@@ -276,10 +274,6 @@ func TestFetcher_ResolveWithInvalidHeader(t *testing.T) {
|
||||
|
||||
fetcher := buildFetcher()
|
||||
defer fetcher.Pause() // Close the resolve response to allow server shutdown
|
||||
fetcher.Meta().Opts = &base.Options{
|
||||
Name: test.DownloadName,
|
||||
Path: test.Dir,
|
||||
}
|
||||
err := fetcher.Resolve(&base.Request{
|
||||
URL: "http://" + listener.Addr().String() + "/",
|
||||
Extra: &http.ReqExtra{
|
||||
@@ -287,6 +281,9 @@ func TestFetcher_ResolveWithInvalidHeader(t *testing.T) {
|
||||
"Referer": "\rtest",
|
||||
},
|
||||
},
|
||||
}, &base.Options{
|
||||
Name: test.DownloadName,
|
||||
Path: test.Dir,
|
||||
})
|
||||
// Invalid header with \r should be sanitized by Go's http client, allowing the request to succeed
|
||||
if err != nil {
|
||||
@@ -306,13 +303,12 @@ func TestFetcher_ResolveAutomaticallyFallsBackToBrowserFingerprint(t *testing.T)
|
||||
|
||||
fetcher := buildFetcher()
|
||||
defer fetcher.Pause()
|
||||
fetcher.Meta().Opts = &base.Options{
|
||||
Name: test.DownloadName,
|
||||
Path: test.Dir,
|
||||
}
|
||||
err := fetcher.Resolve(&base.Request{
|
||||
URL: server.URL + "/file.bin",
|
||||
SkipVerifyCert: true,
|
||||
}, &base.Options{
|
||||
Name: test.DownloadName,
|
||||
Path: test.Dir,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Resolve() error = %v", err)
|
||||
@@ -373,12 +369,11 @@ func testResolve(startTestServer func() net.Listener, path string, t *testing.T,
|
||||
defer listener.Close()
|
||||
fetcher := buildFetcher()
|
||||
defer fetcher.Pause() // Close the resolve response to allow server shutdown
|
||||
fetcher.Meta().Opts = &base.Options{
|
||||
Name: test.DownloadName,
|
||||
Path: test.Dir,
|
||||
}
|
||||
err := fetcher.Resolve(&base.Request{
|
||||
URL: "http://" + listener.Addr().String() + "/" + path,
|
||||
}, &base.Options{
|
||||
Name: test.DownloadName,
|
||||
Path: test.Dir,
|
||||
})
|
||||
want, err := wantFn(err)
|
||||
if err != nil {
|
||||
@@ -430,14 +425,13 @@ func TestFetcher_PauseBeforeStartDiscardsPrefetchProgress(t *testing.T) {
|
||||
defer server.Close()
|
||||
|
||||
fetcher := buildFetcher()
|
||||
fetcher.Meta().Opts = &base.Options{
|
||||
if err := fetcher.Resolve(&base.Request{URL: server.URL + "/" + test.BuildName}, &base.Options{
|
||||
Path: t.TempDir(),
|
||||
Name: test.DownloadName,
|
||||
Extra: &http.OptsExtra{
|
||||
Connections: 2,
|
||||
},
|
||||
}
|
||||
if err := fetcher.Resolve(&base.Request{URL: server.URL + "/" + test.BuildName}); err != nil {
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
select {
|
||||
@@ -778,14 +772,13 @@ func TestFetcher_SequentialRetryReprobesRange(t *testing.T) {
|
||||
defer server.Close()
|
||||
|
||||
f := buildFetcher()
|
||||
f.Meta().Opts = &base.Options{
|
||||
if err := f.Resolve(&base.Request{URL: server.URL + "/dynamic-range.data"}, &base.Options{
|
||||
Path: t.TempDir(),
|
||||
Name: "dynamic-range.data",
|
||||
Extra: &http.OptsExtra{
|
||||
Connections: 4,
|
||||
},
|
||||
}
|
||||
if err := f.Resolve(&base.Request{URL: server.URL + "/dynamic-range.data"}); err != nil {
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := f.Start(); err != nil {
|
||||
@@ -891,14 +884,13 @@ func TestFetcher_InitialIgnoredRangeUsesResponseSize(t *testing.T) {
|
||||
defer releaseResolve()
|
||||
|
||||
f := buildFetcher()
|
||||
f.Meta().Opts = &base.Options{
|
||||
if err := f.Resolve(&base.Request{URL: server.URL + "/ignored-range.data"}, &base.Options{
|
||||
Path: t.TempDir(),
|
||||
Name: "ignored-range.data",
|
||||
Extra: &http.OptsExtra{
|
||||
Connections: 4,
|
||||
},
|
||||
}
|
||||
if err := f.Resolve(&base.Request{URL: server.URL + "/ignored-range.data"}); err != nil {
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := f.Start(); err != nil {
|
||||
@@ -989,14 +981,13 @@ func TestFetcher_InterruptedChunkedReplacementRestartsFromZero(t *testing.T) {
|
||||
defer server.Close()
|
||||
|
||||
f := buildFetcher()
|
||||
f.Meta().Opts = &base.Options{
|
||||
if err := f.Resolve(&base.Request{URL: server.URL + "/chunked-replacement.data"}, &base.Options{
|
||||
Path: t.TempDir(),
|
||||
Name: "chunked-replacement.data",
|
||||
Extra: &http.OptsExtra{
|
||||
Connections: 4,
|
||||
},
|
||||
}
|
||||
if err := f.Resolve(&base.Request{URL: server.URL + "/chunked-replacement.data"}); err != nil {
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := f.Start(); err != nil {
|
||||
@@ -1176,14 +1167,13 @@ func TestFetcher_UsesResolveResponseWhenRangeTakeoverFails(t *testing.T) {
|
||||
defer server.Close()
|
||||
|
||||
f := buildFetcher()
|
||||
f.Meta().Opts = &base.Options{
|
||||
if err := f.Resolve(&base.Request{URL: server.URL + "/resolve-fallback.data"}, &base.Options{
|
||||
Path: t.TempDir(),
|
||||
Name: "resolve-fallback.data",
|
||||
Extra: &http.OptsExtra{
|
||||
Connections: 1,
|
||||
},
|
||||
}
|
||||
if err := f.Resolve(&base.Request{URL: server.URL + "/resolve-fallback.data"}); err != nil {
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer f.Close()
|
||||
@@ -1256,14 +1246,13 @@ func TestFetcher_ValidRangeTakesOverResolveResponse(t *testing.T) {
|
||||
defer server.Close()
|
||||
|
||||
f := buildFetcher()
|
||||
f.Meta().Opts = &base.Options{
|
||||
if err := f.Resolve(&base.Request{URL: server.URL + "/resolve-takeover.data"}, &base.Options{
|
||||
Path: t.TempDir(),
|
||||
Name: "resolve-takeover.data",
|
||||
Extra: &http.OptsExtra{
|
||||
Connections: 4,
|
||||
},
|
||||
}
|
||||
if err := f.Resolve(&base.Request{URL: server.URL + "/resolve-takeover.data"}); err != nil {
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer f.Close()
|
||||
@@ -1287,12 +1276,6 @@ func TestFetcher_ValidRangeTakesOverResolveResponse(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestFetcher_UnknownSizeRangeUsesOpenEndedWorker(t *testing.T) {
|
||||
for _, hint := range []int64{0, 3, 8 * 1024 * 1024} {
|
||||
t.Run(fmt.Sprint(hint), func(t *testing.T) { testFetcherUnknownSizeRange(t, hint) })
|
||||
}
|
||||
}
|
||||
|
||||
func testFetcherUnknownSizeRange(t *testing.T, hint int64) {
|
||||
const resolveETag = `"unknown-size-v1"`
|
||||
payload := make([]byte, 4*1024*1024)
|
||||
for i := range payload {
|
||||
@@ -1357,20 +1340,18 @@ func testFetcherUnknownSizeRange(t *testing.T, hint int64) {
|
||||
defer server.Close()
|
||||
|
||||
f := buildFetcher()
|
||||
f.Meta().Opts = &base.Options{
|
||||
Size: hint,
|
||||
if err := f.Resolve(&base.Request{URL: server.URL + "/unknown-size-range.data"}, &base.Options{
|
||||
Path: t.TempDir(),
|
||||
Name: "unknown-size-range.data",
|
||||
Extra: &http.OptsExtra{
|
||||
Connections: 4,
|
||||
},
|
||||
}
|
||||
if err := f.Resolve(&base.Request{URL: server.URL + "/unknown-size-range.data"}); err != nil {
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
if f.meta.Res.Size != hint || f.hasKnownSize() || !f.meta.Res.Range {
|
||||
if f.meta.Res.Size != 0 || !f.meta.Res.Range {
|
||||
t.Fatalf("resolved resource = %+v, want unknown size with Range support", f.meta.Res)
|
||||
}
|
||||
if err := f.Start(); err != nil {
|
||||
@@ -1453,14 +1434,13 @@ func TestFetcher_UnknownSizeRangeRetriesShortPartialBody(t *testing.T) {
|
||||
defer server.Close()
|
||||
|
||||
f := buildFetcher()
|
||||
f.Meta().Opts = &base.Options{
|
||||
if err := f.Resolve(&base.Request{URL: server.URL + "/short-range.data"}, &base.Options{
|
||||
Path: t.TempDir(),
|
||||
Name: "short-range.data",
|
||||
Extra: &http.OptsExtra{
|
||||
Connections: 1,
|
||||
},
|
||||
}
|
||||
if err := f.Resolve(&base.Request{URL: server.URL + "/short-range.data"}); err != nil {
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer f.Close()
|
||||
@@ -1513,14 +1493,13 @@ func TestFetcher_UnknownSizeRangeFallsBackToResolve(t *testing.T) {
|
||||
defer server.Close()
|
||||
|
||||
f := buildFetcher()
|
||||
f.Meta().Opts = &base.Options{
|
||||
if err := f.Resolve(&base.Request{URL: server.URL + "/unknown-size-fallback.data"}, &base.Options{
|
||||
Path: t.TempDir(),
|
||||
Name: "unknown-size-fallback.data",
|
||||
Extra: &http.OptsExtra{
|
||||
Connections: 1,
|
||||
},
|
||||
}
|
||||
if err := f.Resolve(&base.Request{URL: server.URL + "/unknown-size-fallback.data"}); err != nil {
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer f.Close()
|
||||
@@ -1601,15 +1580,13 @@ func TestFetcher_UnknownSizeRangeRemainsOpenEndedAfterPause(t *testing.T) {
|
||||
defer server.Close()
|
||||
|
||||
f := buildFetcher()
|
||||
f.Meta().Opts = &base.Options{
|
||||
Size: 3,
|
||||
if err := f.Resolve(&base.Request{URL: server.URL + "/pause-open-range.data"}, &base.Options{
|
||||
Path: t.TempDir(),
|
||||
Name: "pause-open-range.data",
|
||||
Extra: &http.OptsExtra{
|
||||
Connections: 1,
|
||||
},
|
||||
}
|
||||
if err := f.Resolve(&base.Request{URL: server.URL + "/pause-open-range.data"}); err != nil {
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer f.Close()
|
||||
@@ -1658,14 +1635,13 @@ func TestFetcher_UnknownSizeResolveReadErrorFailsTask(t *testing.T) {
|
||||
defer server.Close()
|
||||
|
||||
f := buildFetcher()
|
||||
f.Meta().Opts = &base.Options{
|
||||
if err := f.Resolve(&base.Request{URL: server.URL + "/broken-chunked.data"}, &base.Options{
|
||||
Path: t.TempDir(),
|
||||
Name: "broken-chunked.data",
|
||||
Extra: &http.OptsExtra{
|
||||
Connections: 1,
|
||||
},
|
||||
}
|
||||
if err := f.Resolve(&base.Request{URL: server.URL + "/broken-chunked.data"}); err != nil {
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer f.Close()
|
||||
@@ -1691,14 +1667,13 @@ func TestFetcher_UnknownSizeResolveCleanEOFUpdatesSize(t *testing.T) {
|
||||
defer server.Close()
|
||||
|
||||
f := buildFetcher()
|
||||
f.Meta().Opts = &base.Options{
|
||||
if err := f.Resolve(&base.Request{URL: server.URL + "/unknown-size.data"}, &base.Options{
|
||||
Path: t.TempDir(),
|
||||
Name: "unknown-size.data",
|
||||
Extra: &http.OptsExtra{
|
||||
Connections: 1,
|
||||
},
|
||||
}
|
||||
if err := f.Resolve(&base.Request{URL: server.URL + "/unknown-size.data"}); err != nil {
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer f.Close()
|
||||
@@ -2294,15 +2269,14 @@ func TestFetcher_DownloadOneTimeURL(t *testing.T) {
|
||||
defer listener.Close()
|
||||
|
||||
fetcher := buildFetcher()
|
||||
fetcher.Meta().Opts = &base.Options{
|
||||
err := fetcher.Resolve(&base.Request{
|
||||
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
|
||||
}, &base.Options{
|
||||
Name: test.DownloadName,
|
||||
Path: test.Dir,
|
||||
Extra: &http.OptsExtra{
|
||||
Connections: 4, // Try to use multiple connections, but only first should work
|
||||
},
|
||||
}
|
||||
err := fetcher.Resolve(&base.Request{
|
||||
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -2361,15 +2335,14 @@ func TestFetcher_SlowStartExpansion(t *testing.T) {
|
||||
Connections: tc.maxConns,
|
||||
})
|
||||
|
||||
fetcher.Meta().Opts = &base.Options{
|
||||
err := fetcher.Resolve(&base.Request{
|
||||
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
|
||||
}, &base.Options{
|
||||
Name: test.DownloadName,
|
||||
Path: test.Dir,
|
||||
Extra: &http.OptsExtra{
|
||||
Connections: tc.maxConns,
|
||||
},
|
||||
}
|
||||
err := fetcher.Resolve(&base.Request{
|
||||
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
|
||||
})
|
||||
if err != nil {
|
||||
cleanup()
|
||||
@@ -2425,15 +2398,14 @@ func TestFetcher_AsyncPrefetch(t *testing.T) {
|
||||
defer listener.Close()
|
||||
|
||||
fetcher := buildFetcher()
|
||||
fetcher.Meta().Opts = &base.Options{
|
||||
err := fetcher.Resolve(&base.Request{
|
||||
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
|
||||
}, &base.Options{
|
||||
Name: test.DownloadName,
|
||||
Path: test.Dir,
|
||||
Extra: &http.OptsExtra{
|
||||
Connections: 4,
|
||||
},
|
||||
}
|
||||
err := fetcher.Resolve(&base.Request{
|
||||
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -2497,15 +2469,14 @@ func TestFetcher_AsyncPrefetch(t *testing.T) {
|
||||
defer listener.Close()
|
||||
|
||||
fetcher := buildFetcher()
|
||||
fetcher.Meta().Opts = &base.Options{
|
||||
err := fetcher.Resolve(&base.Request{
|
||||
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
|
||||
}, &base.Options{
|
||||
Name: test.DownloadName,
|
||||
Path: test.Dir,
|
||||
Extra: &http.OptsExtra{
|
||||
Connections: 4,
|
||||
},
|
||||
}
|
||||
err := fetcher.Resolve(&base.Request{
|
||||
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -2580,15 +2551,14 @@ func TestFetcher_DownloadExpiringRedirectURL(t *testing.T) {
|
||||
defer listener.Close()
|
||||
|
||||
fetcher := buildFetcher()
|
||||
fetcher.Meta().Opts = &base.Options{
|
||||
err := fetcher.Resolve(&base.Request{
|
||||
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
|
||||
}, &base.Options{
|
||||
Name: test.DownloadName,
|
||||
Path: test.Dir,
|
||||
Extra: &http.OptsExtra{
|
||||
Connections: 4, // Use multiple connections to trigger redirect expiration
|
||||
},
|
||||
}
|
||||
err := fetcher.Resolve(&base.Request{
|
||||
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -2622,15 +2592,14 @@ func TestFetcher_DownloadExpiringRedirectURL(t *testing.T) {
|
||||
defer listener.Close()
|
||||
|
||||
fetcher := buildFetcher()
|
||||
fetcher.Meta().Opts = &base.Options{
|
||||
err := fetcher.Resolve(&base.Request{
|
||||
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
|
||||
}, &base.Options{
|
||||
Name: test.DownloadName,
|
||||
Path: test.Dir,
|
||||
Extra: &http.OptsExtra{
|
||||
Connections: 8,
|
||||
},
|
||||
}
|
||||
err := fetcher.Resolve(&base.Request{
|
||||
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -2670,15 +2639,14 @@ func TestFetcher_RetryAfterError(t *testing.T) {
|
||||
defer listener.Close()
|
||||
|
||||
fetcher := buildFetcher()
|
||||
fetcher.Meta().Opts = &base.Options{
|
||||
err := fetcher.Resolve(&base.Request{
|
||||
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
|
||||
}, &base.Options{
|
||||
Name: test.DownloadName,
|
||||
Path: test.Dir,
|
||||
Extra: &http.OptsExtra{
|
||||
Connections: 1, // Use single connection to simplify test
|
||||
},
|
||||
}
|
||||
err := fetcher.Resolve(&base.Request{
|
||||
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -3099,10 +3067,9 @@ func doDownloadReady(f fetcher.Fetcher, listener net.Listener, connections int,
|
||||
Path: test.Dir,
|
||||
Extra: extra,
|
||||
}
|
||||
f.Meta().Opts = opts
|
||||
err := f.Resolve(&base.Request{
|
||||
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
|
||||
})
|
||||
}, opts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -3151,8 +3118,7 @@ func downloadPost(listener net.Listener, connections int, t *testing.T) {
|
||||
Body: fmt.Sprintf(`{"name":"%s"}`, test.BuildName),
|
||||
},
|
||||
}
|
||||
f.Meta().Opts = opts
|
||||
err := f.Resolve(req)
|
||||
err := f.Resolve(req, opts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -3198,12 +3164,11 @@ func downloadContinue(listener net.Listener, connections int, t *testing.T) {
|
||||
|
||||
func downloadError(listener net.Listener, connections int, t *testing.T) {
|
||||
fetcher := buildFetcher()
|
||||
fetcher.Meta().Opts = &base.Options{
|
||||
Name: test.DownloadName,
|
||||
Path: test.Dir,
|
||||
}
|
||||
err := fetcher.Resolve(&base.Request{
|
||||
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
|
||||
}, &base.Options{
|
||||
Name: test.DownloadName,
|
||||
Path: test.Dir,
|
||||
})
|
||||
// With the new async design, Resolve may succeed (HTTP response received)
|
||||
// but errors occur during async download or Start/Wait
|
||||
@@ -3261,17 +3226,16 @@ func downloadResume(listener net.Listener, connections int, t *testing.T) {
|
||||
}
|
||||
|
||||
func downloadWithProxy(httpListener net.Listener, proxyListener net.Listener, t *testing.T) {
|
||||
// Setup must run before Resolve, as in the downloader, so the Resolve
|
||||
// request also goes through the proxy.
|
||||
f := buildFetcher()
|
||||
f.ctl.GetProxy = func(requestProxy *base.RequestProxy) func(*gohttp.Request) (*url.URL, error) {
|
||||
fetcher := downloadReady(httpListener, 4, t)
|
||||
ctl := controller.NewController()
|
||||
ctl.GetProxy = func(requestProxy *base.RequestProxy) func(*gohttp.Request) (*url.URL, error) {
|
||||
return (&base.DownloaderProxyConfig{
|
||||
Enable: true,
|
||||
Scheme: "socks5",
|
||||
Host: proxyListener.Addr().String(),
|
||||
}).ToHandler()
|
||||
}
|
||||
fetcher := doDownloadReady(f, httpListener, 4, t)
|
||||
fetcher.Setup(ctl)
|
||||
err := fetcher.Start()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -3328,8 +3292,7 @@ func TestFetcher_Patch_URLChange(t *testing.T) {
|
||||
}
|
||||
|
||||
// Step 1: Try to resolve with bad URL - should fail with error
|
||||
f.Meta().Opts = opts
|
||||
err := f.Resolve(&base.Request{URL: badURL})
|
||||
err := f.Resolve(&base.Request{URL: badURL}, opts)
|
||||
if err == nil {
|
||||
t.Fatal("Expected error for bad URL, got nil")
|
||||
}
|
||||
@@ -3339,8 +3302,7 @@ func TestFetcher_Patch_URLChange(t *testing.T) {
|
||||
f2 := buildFetcher()
|
||||
|
||||
// First resolve with good URL to create a valid fetcher state
|
||||
f2.Meta().Opts = opts
|
||||
err = f2.Resolve(&base.Request{URL: goodURL})
|
||||
err = f2.Resolve(&base.Request{URL: goodURL}, opts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -3392,14 +3354,13 @@ func TestFetcher_Patch_Labels(t *testing.T) {
|
||||
Extra: &http.OptsExtra{Connections: 1},
|
||||
}
|
||||
|
||||
f.Meta().Opts = opts
|
||||
err := f.Resolve(&base.Request{
|
||||
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
|
||||
Labels: map[string]string{
|
||||
"key1": "value1",
|
||||
"key3": "value3",
|
||||
},
|
||||
})
|
||||
}, opts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -3450,7 +3411,6 @@ func TestFetcher_Patch_Extra(t *testing.T) {
|
||||
}
|
||||
|
||||
// Resolve with initial Extra
|
||||
f.Meta().Opts = opts
|
||||
err := f.Resolve(&base.Request{
|
||||
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
|
||||
Extra: &http.ReqExtra{
|
||||
@@ -3461,7 +3421,7 @@ func TestFetcher_Patch_Extra(t *testing.T) {
|
||||
"X-Custom": "original",
|
||||
},
|
||||
},
|
||||
})
|
||||
}, opts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -3539,10 +3499,9 @@ func TestFetcher_Patch_NilData(t *testing.T) {
|
||||
Extra: &http.OptsExtra{Connections: 1},
|
||||
}
|
||||
|
||||
f.Meta().Opts = opts
|
||||
err := f.Resolve(&base.Request{
|
||||
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
|
||||
})
|
||||
}, opts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -3592,7 +3551,6 @@ func TestFetcher_Patch_CookieExpired(t *testing.T) {
|
||||
|
||||
// Step 1: Resolve with old_token - should succeed (first request accepts old_token)
|
||||
f := buildFetcher()
|
||||
f.Meta().Opts = opts
|
||||
err := f.Resolve(&base.Request{
|
||||
URL: downloadURL,
|
||||
Extra: &http.ReqExtra{
|
||||
@@ -3600,7 +3558,7 @@ func TestFetcher_Patch_CookieExpired(t *testing.T) {
|
||||
"Cookie": "session=old_token",
|
||||
},
|
||||
},
|
||||
})
|
||||
}, opts)
|
||||
if err != nil {
|
||||
t.Fatalf("Resolve should succeed with old_token: %v", err)
|
||||
}
|
||||
|
||||
@@ -1,52 +0,0 @@
|
||||
package http
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
gohttp "net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/GopeedLab/gopeed/pkg/base"
|
||||
"github.com/GopeedLab/gopeed/pkg/protocol/http"
|
||||
)
|
||||
|
||||
// Start can run after the Resolve prefetch wrote every byte but before it set
|
||||
// prefetchDone. The prefetched body must still be copied to the target.
|
||||
func TestFetcher_StartBeforePrefetchDoneCopiesFullBody(t *testing.T) {
|
||||
payload := []byte("prefetched before Start")
|
||||
server := httptest.NewServer(gohttp.HandlerFunc(func(w gohttp.ResponseWriter, r *gohttp.Request) {
|
||||
gohttp.ServeContent(w, r, "f.txt", time.Time{}, bytes.NewReader(payload))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
f := buildFetcher()
|
||||
f.Meta().Opts = &base.Options{Path: t.TempDir(), Name: "f.txt", Extra: &http.OptsExtra{Connections: 1}}
|
||||
if err := f.Resolve(&base.Request{URL: server.URL + "/f.txt"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for !f.prefetchDone.Load() {
|
||||
time.Sleep(time.Millisecond)
|
||||
}
|
||||
// Reopen the window between the last write and prefetchDone.
|
||||
f.prefetchDone.Store(false)
|
||||
go func() {
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
f.prefetchDone.Store(true)
|
||||
}()
|
||||
|
||||
if err := f.Start(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := f.Wait(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, err := os.ReadFile(f.meta.SingleFilepath())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !bytes.Equal(got, payload) {
|
||||
t.Fatalf("content = %q, want %q", got, payload)
|
||||
}
|
||||
}
|
||||
@@ -49,8 +49,7 @@ func TestResolveResponseSize(t *testing.T) {
|
||||
defer server.Close()
|
||||
f := buildFetcher()
|
||||
defer f.Close()
|
||||
f.Meta().Opts = &base.Options{Path: t.TempDir()}
|
||||
err := f.Resolve(&base.Request{URL: server.URL + "/file"})
|
||||
err := f.Resolve(&base.Request{URL: server.URL + "/file"}, &base.Options{Path: t.TempDir()})
|
||||
if tc.invalid {
|
||||
if !errors.Is(err, errInvalidRangeResponse) {
|
||||
t.Fatalf("Resolve error = %v, want invalid range", err)
|
||||
@@ -98,8 +97,7 @@ func TestDownloadPartialResolve(t *testing.T) {
|
||||
defer server.Close()
|
||||
f := buildFetcher()
|
||||
defer f.Close()
|
||||
f.Meta().Opts = &base.Options{Path: t.TempDir(), Name: "file", Extra: &fhttp.OptsExtra{Connections: connections}}
|
||||
if err := f.Resolve(&base.Request{URL: server.URL + "/file"}); err != nil {
|
||||
if err := f.Resolve(&base.Request{URL: server.URL + "/file"}, &base.Options{Path: t.TempDir(), Name: "file", Extra: &fhttp.OptsExtra{Connections: connections}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := f.Start(); err != nil {
|
||||
|
||||
@@ -1,245 +0,0 @@
|
||||
package http
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
|
||||
"github.com/GopeedLab/gopeed/pkg/base"
|
||||
)
|
||||
|
||||
func TestFetcher_OptionsSizeHintAndPatch(t *testing.T) {
|
||||
for _, known := range []bool{false, true} {
|
||||
for _, payload := range []string{"", "0123456789"} {
|
||||
t.Run(fmt.Sprintf("known=%v/bytes=%d", known, len(payload)), func(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
if known {
|
||||
w.Header().Set("Content-Length", fmt.Sprint(len(payload)))
|
||||
}
|
||||
w.(http.Flusher).Flush()
|
||||
fmt.Fprint(w, payload)
|
||||
}))
|
||||
defer server.Close()
|
||||
f := buildFetcher()
|
||||
defer f.Close()
|
||||
f.Meta().Opts = &base.Options{Size: 7, Path: t.TempDir(), Name: "output.bin"}
|
||||
if err := f.Resolve(&base.Request{URL: server.URL}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := int64(7)
|
||||
if known {
|
||||
want = int64(len(payload))
|
||||
}
|
||||
if f.meta.Res.Size != want || f.hasKnownSize() != known {
|
||||
t.Fatalf("resolved size=%d known=%v", f.meta.Res.Size, f.hasKnownSize())
|
||||
}
|
||||
if err := f.Patch(nil, &base.Options{Size: 9}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !known {
|
||||
want = 9
|
||||
}
|
||||
if f.meta.Res.Size != want {
|
||||
t.Fatal("hint update overrode a real size or failed to update an estimate")
|
||||
}
|
||||
if err := f.Start(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := f.Wait(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if f.meta.Res.Size != int64(len(payload)) || !f.hasKnownSize() {
|
||||
t.Fatalf("final size=%d known=%v", f.meta.Res.Size, f.hasKnownSize())
|
||||
}
|
||||
snapshot, err := new(FetcherManager).Store(f)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fd := snapshot.(*fetcherData)
|
||||
if fd.SizeKnown == nil || !*fd.SizeKnown {
|
||||
t.Fatal("snapshot lost confirmed size authority")
|
||||
}
|
||||
_, restore := new(FetcherManager).Restore()
|
||||
restored := restore(f.meta, fd).(*Fetcher)
|
||||
if !restored.hasKnownSize() || restored.knownSize() != int64(len(payload)) {
|
||||
t.Fatal("restored size lost authority, including known zero")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetcher_SizeHintCannotCompleteForbiddenUnknownRange(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Header.Get("Range") != "" {
|
||||
w.WriteHeader(http.StatusForbidden)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Accept-Ranges", "bytes")
|
||||
w.(http.Flusher).Flush()
|
||||
fmt.Fprint(w, "0123456789")
|
||||
w.(http.Flusher).Flush()
|
||||
<-r.Context().Done()
|
||||
}))
|
||||
defer server.Close()
|
||||
f := buildFetcher()
|
||||
defer f.Close()
|
||||
f.Meta().Opts = &base.Options{Size: 3, Path: t.TempDir(), Name: "output.bin"}
|
||||
if err := f.Resolve(&base.Request{URL: server.URL}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := f.Start(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := f.Wait(); err == nil {
|
||||
t.Fatal("a failed unknown-length Range request was completed from an estimate")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetcher_SizeHintPreservesHTTPIntegrity(t *testing.T) {
|
||||
for _, chunked := range []bool{false, true} {
|
||||
for _, broken := range []bool{false, true} {
|
||||
t.Run(fmt.Sprintf("chunked=%v/broken=%v", chunked, broken), func(t *testing.T) {
|
||||
payload := []byte("0123456789")
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
if chunked && broken {
|
||||
conn, rw, err := w.(http.Hijacker).Hijack()
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
return
|
||||
}
|
||||
defer conn.Close()
|
||||
fmt.Fprint(rw, "HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\nConnection: close\r\n\r\na\r\n0123456789\r\n")
|
||||
rw.Flush() // Omit the final chunk to simulate a connection interruption.
|
||||
return
|
||||
}
|
||||
if !chunked {
|
||||
length := len(payload)
|
||||
if broken {
|
||||
length++
|
||||
}
|
||||
w.Header().Set("Content-Length", fmt.Sprint(length))
|
||||
}
|
||||
w.(http.Flusher).Flush()
|
||||
w.Write(payload)
|
||||
}))
|
||||
defer server.Close()
|
||||
f := buildFetcher()
|
||||
defer f.Close()
|
||||
f.Meta().Opts = &base.Options{Size: 3, Path: t.TempDir(), Name: "output.bin"}
|
||||
if err := f.Resolve(&base.Request{URL: server.URL}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !chunked {
|
||||
length := int64(len(payload))
|
||||
if broken {
|
||||
length++
|
||||
}
|
||||
if f.meta.Res.Size != length || f.sequentialSizeUnknown {
|
||||
t.Fatal("HTTP length did not override size hint")
|
||||
}
|
||||
} else if f.meta.Res.Size != 3 || !f.sequentialSizeUnknown {
|
||||
t.Fatal("unknown HTTP length did not retain the display hint")
|
||||
}
|
||||
if err := f.Start(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
err := f.Wait()
|
||||
if broken {
|
||||
if err == nil {
|
||||
t.Fatal("truncated HTTP response succeeded after exceeding the size hint")
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if f.meta.Res.Size != int64(len(payload)) || f.sequentialSizeUnknown {
|
||||
t.Fatalf("completed size=%d unknown=%v", f.meta.Res.Size, f.sequentialSizeUnknown)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetcher_SizeHintSurvivesRestore(t *testing.T) {
|
||||
for _, partial := range []bool{false, true} {
|
||||
for _, fixedLength := range []bool{false, true} {
|
||||
t.Run(fmt.Sprintf("partial=%v/fixedLength=%v", partial, fixedLength), func(t *testing.T) {
|
||||
payload := []byte("0123456789")
|
||||
var requests atomic.Int64
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
request := requests.Add(1)
|
||||
if got := r.Header.Get("Range"); got != "" {
|
||||
t.Errorf("estimated sizes must not be used for resume ranges: %s", got)
|
||||
}
|
||||
if fixedLength && request > 1 {
|
||||
w.Header().Set("Content-Length", fmt.Sprint(len(payload)))
|
||||
}
|
||||
w.(http.Flusher).Flush()
|
||||
if fixedLength && request == 2 {
|
||||
w.Write(payload[:4]) // Retry must discard this interrupted attempt.
|
||||
return
|
||||
}
|
||||
w.Write(payload)
|
||||
}))
|
||||
defer server.Close()
|
||||
f := buildFetcher()
|
||||
opts := &base.Options{Size: 3, Path: t.TempDir(), Name: "output.bin"}
|
||||
f.Meta().Opts = opts
|
||||
if err := f.Resolve(&base.Request{URL: server.URL}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fm := new(FetcherManager)
|
||||
snapshot, err := fm.Store(f)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
data, err := json.Marshal(snapshot)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var saved fetcherData
|
||||
if err := json.Unmarshal(data, &saved); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !saved.SequentialSizeUnknown || saved.SizeKnown == nil || *saved.SizeKnown || *saved.ResourceSize != 3 {
|
||||
t.Fatal("snapshot lost the size hint or its advisory state")
|
||||
}
|
||||
f.Close()
|
||||
// An interrupted earlier attempt left a tail longer than the new output.
|
||||
if err := os.WriteFile(filepath.Join(opts.Path, opts.Name), bytes.Repeat([]byte("old"), 100), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if partial {
|
||||
chunk := newChunk(0, 0)
|
||||
chunk.Downloaded = 7 // Already exceeds the display estimate.
|
||||
saved.Connections = []*connection{{ID: 0, Role: rolePrimary, State: connNotStarted, Chunk: chunk, Downloaded: 7}}
|
||||
}
|
||||
_, restore := fm.Restore()
|
||||
restored := restore(f.meta, &saved).(*Fetcher)
|
||||
restored.Setup(f.ctl)
|
||||
defer restored.Close()
|
||||
if err := restored.Start(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := restored.Wait(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body, err := os.ReadFile(restored.meta.SingleFilepath())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !bytes.Equal(body, payload) || restored.meta.Res.Size != int64(len(payload)) || restored.Progress().TotalDownloaded() != int64(len(payload)) {
|
||||
t.Fatalf("restored stream retained an estimate or stale bytes: body=%q size=%d progress=%d", body, restored.meta.Res.Size, restored.Progress().TotalDownloaded())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,56 +0,0 @@
|
||||
package http
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
gohttp "net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/GopeedLab/gopeed/pkg/base"
|
||||
"github.com/GopeedLab/gopeed/pkg/protocol/http"
|
||||
)
|
||||
|
||||
// A stalled Resolve body must not keep the prefetch goroutine alive after
|
||||
// stopPrefetchAndCopyData returns, or it writes to a file we already removed.
|
||||
func TestFetcher_StopPrefetchUnblocksStalledRead(t *testing.T) {
|
||||
prefix := bytes.Repeat([]byte("p"), 64*1024)
|
||||
stall := make(chan struct{})
|
||||
defer close(stall)
|
||||
|
||||
server := httptest.NewServer(gohttp.HandlerFunc(func(w gohttp.ResponseWriter, r *gohttp.Request) {
|
||||
w.Header().Set(base.HttpHeaderAcceptRanges, base.HttpHeaderBytes)
|
||||
w.Header().Set(base.HttpHeaderContentLength, fmt.Sprintf("%d", len(prefix)*4))
|
||||
w.WriteHeader(gohttp.StatusOK)
|
||||
_, _ = w.Write(prefix)
|
||||
w.(gohttp.Flusher).Flush()
|
||||
select {
|
||||
case <-stall:
|
||||
case <-r.Context().Done():
|
||||
}
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
f := buildFetcher()
|
||||
f.Meta().Opts = &base.Options{
|
||||
Path: t.TempDir(),
|
||||
Name: "stalled.data",
|
||||
Extra: &http.OptsExtra{Connections: 1},
|
||||
}
|
||||
if err := f.Resolve(&base.Request{URL: server.URL + "/stalled.data"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for f.prefetchSize.Load() < int64(len(prefix)) {
|
||||
time.Sleep(time.Millisecond)
|
||||
}
|
||||
|
||||
start := time.Now()
|
||||
f.stopPrefetch()
|
||||
if elapsed := time.Since(start); elapsed > 2*time.Second {
|
||||
t.Fatalf("stopPrefetch took %v, want it to interrupt the stalled read", elapsed)
|
||||
}
|
||||
if !f.prefetchDone.Load() {
|
||||
t.Fatal("prefetch goroutine still running after stopPrefetch")
|
||||
}
|
||||
}
|
||||
@@ -6,56 +6,35 @@ import (
|
||||
"time"
|
||||
)
|
||||
|
||||
// TimeoutReader fails a Read that makes no progress within timeout.
|
||||
//
|
||||
// One timer per reader is armed when Read starts and stopped before it
|
||||
// returns, so time the caller spends between reads is not counted. If the
|
||||
// timer fires, it calls cancel, which should cancel the HTTP request the
|
||||
// reader belongs to so the transport unblocks the pending Read. Read still
|
||||
// waits for the underlying Read to return; that Read and every later one
|
||||
// return context.DeadlineExceeded. Errors from any other cancellation are
|
||||
// returned unchanged, and closing the response body is left to the caller.
|
||||
//
|
||||
// With a nil cancel the pending Read cannot be interrupted; it still reports
|
||||
// context.DeadlineExceeded, but only once it returns.
|
||||
//
|
||||
// TimeoutReader is not safe for concurrent use.
|
||||
type TimeoutReader struct {
|
||||
reader io.Reader
|
||||
timeout time.Duration
|
||||
cancel context.CancelFunc
|
||||
timer *time.Timer
|
||||
timedOut bool
|
||||
reader io.Reader
|
||||
timeout time.Duration
|
||||
}
|
||||
|
||||
func NewTimeoutReader(r io.Reader, timeout time.Duration, cancel context.CancelFunc) *TimeoutReader {
|
||||
if cancel == nil {
|
||||
cancel = func() {}
|
||||
}
|
||||
func NewTimeoutReader(r io.Reader, timeout time.Duration) *TimeoutReader {
|
||||
return &TimeoutReader{
|
||||
reader: r,
|
||||
timeout: timeout,
|
||||
cancel: cancel,
|
||||
}
|
||||
}
|
||||
|
||||
func (tr *TimeoutReader) Read(p []byte) (n int, err error) {
|
||||
if tr.timedOut {
|
||||
return 0, context.DeadlineExceeded
|
||||
}
|
||||
if tr.timer == nil {
|
||||
tr.timer = time.AfterFunc(tr.timeout, tr.cancel)
|
||||
} else {
|
||||
tr.timer.Reset(tr.timeout)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), tr.timeout)
|
||||
defer cancel()
|
||||
|
||||
n, err = tr.reader.Read(p)
|
||||
done := make(chan struct{})
|
||||
var readErr error
|
||||
var bytesRead int
|
||||
|
||||
if !tr.timer.Stop() {
|
||||
// The timer fired during this Read and cancelled the request, so err is
|
||||
// whatever the interrupted read produced. Report the timeout instead.
|
||||
tr.timedOut = true
|
||||
return n, context.DeadlineExceeded
|
||||
go func() {
|
||||
bytesRead, readErr = tr.reader.Read(p)
|
||||
close(done)
|
||||
}()
|
||||
|
||||
select {
|
||||
case <-done:
|
||||
return bytesRead, readErr
|
||||
case <-ctx.Done():
|
||||
return 0, ctx.Err()
|
||||
}
|
||||
return n, err
|
||||
}
|
||||
@@ -2,26 +2,17 @@ package http
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"errors"
|
||||
"io"
|
||||
gohttp "net/http"
|
||||
"net/http/httptest"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/GopeedLab/gopeed/internal/httpclient"
|
||||
)
|
||||
|
||||
func TestTimeoutReader_Read(t *testing.T) {
|
||||
data := []byte("Hello, World!")
|
||||
reader := bytes.NewReader(data)
|
||||
timeoutReader := NewTimeoutReader(reader, 1*time.Second, nil)
|
||||
timeoutReader := NewTimeoutReader(reader, 1*time.Second)
|
||||
|
||||
buf := make([]byte, len(data))
|
||||
n, err := timeoutReader.Read(buf)
|
||||
@@ -38,7 +29,7 @@ func TestTimeoutReader_Read(t *testing.T) {
|
||||
|
||||
func TestTimeoutReader_ReadTimeout(t *testing.T) {
|
||||
reader := &slowReader{delay: 2 * time.Second}
|
||||
timeoutReader := NewTimeoutReader(reader, 1*time.Second, nil)
|
||||
timeoutReader := NewTimeoutReader(reader, 1*time.Second)
|
||||
|
||||
buf := make([]byte, 8192)
|
||||
_, err := timeoutReader.Read(buf)
|
||||
@@ -58,340 +49,3 @@ func (sr *slowReader) Read(p []byte) (n int, err error) {
|
||||
time.Sleep(sr.delay)
|
||||
return 0, io.EOF
|
||||
}
|
||||
|
||||
// The timer must only cover time spent inside Read, not caller work between reads.
|
||||
func TestTimeoutReader_IdleBetweenReads(t *testing.T) {
|
||||
var cancelled atomic.Bool
|
||||
timeoutReader := NewTimeoutReader(bytes.NewReader([]byte("abcdef")), 50*time.Millisecond, func() {
|
||||
cancelled.Store(true)
|
||||
})
|
||||
|
||||
buf := make([]byte, 3)
|
||||
if _, err := timeoutReader.Read(buf); err != nil {
|
||||
t.Fatalf("first read: %v", err)
|
||||
}
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
if _, err := timeoutReader.Read(buf); err != nil {
|
||||
t.Fatalf("second read after idle: %v", err)
|
||||
}
|
||||
if cancelled.Load() {
|
||||
t.Fatal("request was cancelled while idle between reads")
|
||||
}
|
||||
}
|
||||
|
||||
// A timed-out Read must not return while the underlying Read can still write
|
||||
// into the caller's buffer, and must leave closing the reader to the caller.
|
||||
func TestTimeoutReader_TimedOutReadDoesNotOutliveCaller(t *testing.T) {
|
||||
reader := &lateWriteReader{cancelled: make(chan struct{})}
|
||||
timeoutReader := NewTimeoutReader(reader, 50*time.Millisecond, reader.cancel)
|
||||
|
||||
buf := make([]byte, 8)
|
||||
_, err := timeoutReader.Read(buf)
|
||||
if !errors.Is(err, context.DeadlineExceeded) {
|
||||
t.Fatalf("expected %v, got %v", context.DeadlineExceeded, err)
|
||||
}
|
||||
if !reader.returned.Load() {
|
||||
t.Fatal("Read returned before the underlying Read finished")
|
||||
}
|
||||
if _, err := timeoutReader.Read(buf); !errors.Is(err, context.DeadlineExceeded) {
|
||||
t.Fatalf("read after timeout: expected %v, got %v", context.DeadlineExceeded, err)
|
||||
}
|
||||
if reader.closed.Load() {
|
||||
t.Fatal("reader was closed by the timeout")
|
||||
}
|
||||
}
|
||||
|
||||
// lateWriteReader blocks until cancelled, then writes into p before returning.
|
||||
type lateWriteReader struct {
|
||||
cancelOnce sync.Once
|
||||
cancelled chan struct{}
|
||||
returned atomic.Bool
|
||||
closed atomic.Bool
|
||||
}
|
||||
|
||||
func (r *lateWriteReader) Read(p []byte) (int, error) {
|
||||
<-r.cancelled
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
p[0] = 'x'
|
||||
r.returned.Store(true)
|
||||
return 0, context.Canceled
|
||||
}
|
||||
|
||||
func (r *lateWriteReader) cancel() {
|
||||
r.cancelOnce.Do(func() { close(r.cancelled) })
|
||||
}
|
||||
|
||||
func (r *lateWriteReader) Close() error {
|
||||
r.closed.Store(true)
|
||||
return nil
|
||||
}
|
||||
|
||||
type timeoutReaderClientCase struct {
|
||||
name string
|
||||
http2 bool
|
||||
client func(t *testing.T, server *httptest.Server) *gohttp.Client
|
||||
}
|
||||
|
||||
func timeoutReaderClientCases() []timeoutReaderClientCase {
|
||||
newClient := func(mode httpclient.ImpersonationMode) func(t *testing.T, server *httptest.Server) *gohttp.Client {
|
||||
return func(t *testing.T, server *httptest.Server) *gohttp.Client {
|
||||
var tlsConfig *tls.Config
|
||||
if server.TLS != nil {
|
||||
roots := x509.NewCertPool()
|
||||
roots.AddCert(server.Certificate())
|
||||
tlsConfig = &tls.Config{RootCAs: roots}
|
||||
}
|
||||
client, err := httpclient.NewClient(httpclient.Options{
|
||||
Transport: httpclient.TransportOptions{
|
||||
TLSClientConfig: tlsConfig,
|
||||
},
|
||||
Impersonation: httpclient.ImpersonationOptions{
|
||||
Mode: mode,
|
||||
Browser: httpclient.BrowserChrome,
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return client
|
||||
}
|
||||
}
|
||||
var cases []timeoutReaderClientCase
|
||||
for _, mode := range []httpclient.ImpersonationMode{httpclient.ImpersonationNative, httpclient.ImpersonationFixed} {
|
||||
cases = append(cases,
|
||||
timeoutReaderClientCase{name: string(mode) + "/http1", client: newClient(mode)},
|
||||
timeoutReaderClientCase{name: string(mode) + "/http2", http2: true, client: newClient(mode)},
|
||||
)
|
||||
}
|
||||
return cases
|
||||
}
|
||||
|
||||
func startTimeoutReaderServer(t *testing.T, useHTTP2 bool, handler gohttp.HandlerFunc) *httptest.Server {
|
||||
server := httptest.NewUnstartedServer(handler)
|
||||
if useHTTP2 {
|
||||
server.EnableHTTP2 = true
|
||||
server.StartTLS()
|
||||
} else {
|
||||
server.Start()
|
||||
}
|
||||
t.Cleanup(server.Close)
|
||||
return server
|
||||
}
|
||||
|
||||
// readBodyWithTimeout sends the request under its own context derived from
|
||||
// ctx, as the fetcher does, and reads the body through a TimeoutReader.
|
||||
func readBodyWithTimeout(t *testing.T, ctx context.Context, client *gohttp.Client, url string, timeout time.Duration, wantProto int) ([]byte, error) {
|
||||
t.Helper()
|
||||
reqCtx, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
req, err := gohttp.NewRequestWithContext(reqCtx, gohttp.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.ProtoMajor != wantProto {
|
||||
t.Fatalf("expected HTTP/%d, got %s", wantProto, resp.Proto)
|
||||
}
|
||||
|
||||
type result struct {
|
||||
data []byte
|
||||
err error
|
||||
}
|
||||
done := make(chan result, 1)
|
||||
go func() {
|
||||
data, err := io.ReadAll(NewTimeoutReader(resp.Body, timeout, cancel))
|
||||
done <- result{data, err}
|
||||
}()
|
||||
select {
|
||||
case r := <-done:
|
||||
return r.data, r.err
|
||||
case <-time.After(timeout + 10*time.Second):
|
||||
t.Fatal("Read did not return after the read timeout")
|
||||
return nil, nil
|
||||
}
|
||||
}
|
||||
|
||||
func TestTimeoutReader_StalledHTTPBodyTimesOut(t *testing.T) {
|
||||
for _, tc := range timeoutReaderClientCases() {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
server := startTimeoutReaderServer(t, tc.http2, func(w gohttp.ResponseWriter, r *gohttp.Request) {
|
||||
w.Header().Set("Content-Length", "1048576")
|
||||
w.Write(bytes.Repeat([]byte("a"), 1024))
|
||||
w.(gohttp.Flusher).Flush()
|
||||
<-r.Context().Done()
|
||||
})
|
||||
|
||||
wantProto := 1
|
||||
if tc.http2 {
|
||||
wantProto = 2
|
||||
}
|
||||
const timeout = 200 * time.Millisecond
|
||||
start := time.Now()
|
||||
data, err := readBodyWithTimeout(t, context.Background(), tc.client(t, server), server.URL, timeout, wantProto)
|
||||
if !errors.Is(err, context.DeadlineExceeded) {
|
||||
t.Fatalf("expected %v, got %v", context.DeadlineExceeded, err)
|
||||
}
|
||||
if len(data) != 1024 {
|
||||
t.Fatalf("expected the 1024 bytes sent before the stall, got %d", len(data))
|
||||
}
|
||||
if elapsed := time.Since(start); elapsed < timeout {
|
||||
t.Fatalf("timed out after %v, before the %v read timeout", elapsed, timeout)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestTimeoutReader_SlowSteadyHTTPBodyDoesNotTimeOut(t *testing.T) {
|
||||
const (
|
||||
timeout = 300 * time.Millisecond
|
||||
interval = 30 * time.Millisecond
|
||||
chunks = 40 // 1.2s in total, four times the read timeout
|
||||
)
|
||||
for _, tc := range timeoutReaderClientCases() {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
server := startTimeoutReaderServer(t, tc.http2, func(w gohttp.ResponseWriter, r *gohttp.Request) {
|
||||
for i := 0; i < chunks; i++ {
|
||||
w.Write([]byte("0123456789"))
|
||||
w.(gohttp.Flusher).Flush()
|
||||
time.Sleep(interval)
|
||||
}
|
||||
})
|
||||
|
||||
wantProto := 1
|
||||
if tc.http2 {
|
||||
wantProto = 2
|
||||
}
|
||||
data, err := readBodyWithTimeout(t, context.Background(), tc.client(t, server), server.URL, timeout, wantProto)
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error, got %v", err)
|
||||
}
|
||||
if len(data) != chunks*10 {
|
||||
t.Fatalf("expected %d bytes, got %d", chunks*10, len(data))
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A gzip-encoded body that stalls must time out without racing the caller's
|
||||
// Body.Close against the decompressing reader.
|
||||
func TestTimeoutReader_StalledGzipHTTPBodyTimesOut(t *testing.T) {
|
||||
payload := bytes.Repeat([]byte("gopeed"), 1024)
|
||||
stages := []struct {
|
||||
name string
|
||||
write func(w io.Writer)
|
||||
want int
|
||||
}{
|
||||
{
|
||||
// The first bytes of a gzip header, so the client blocks inside
|
||||
// gzip.NewReader.
|
||||
name: "incomplete-header",
|
||||
write: func(w io.Writer) { w.Write([]byte{0x1f, 0x8b, 0x08, 0x00}) },
|
||||
want: 0,
|
||||
},
|
||||
{
|
||||
// A complete header and some compressed data, flushed without
|
||||
// finishing the gzip stream.
|
||||
name: "stalled-body",
|
||||
write: func(w io.Writer) {
|
||||
gz := gzip.NewWriter(w)
|
||||
gz.Write(payload)
|
||||
gz.Flush()
|
||||
},
|
||||
want: len(payload),
|
||||
},
|
||||
}
|
||||
for _, stage := range stages {
|
||||
for _, tc := range timeoutReaderClientCases() {
|
||||
t.Run(stage.name+"/"+tc.name, func(t *testing.T) {
|
||||
server := startTimeoutReaderServer(t, tc.http2, func(w gohttp.ResponseWriter, r *gohttp.Request) {
|
||||
w.Header().Set("Content-Encoding", "gzip")
|
||||
stage.write(w)
|
||||
w.(gohttp.Flusher).Flush()
|
||||
<-r.Context().Done()
|
||||
})
|
||||
|
||||
wantProto := 1
|
||||
if tc.http2 {
|
||||
wantProto = 2
|
||||
}
|
||||
data, err := readBodyWithTimeout(t, context.Background(), tc.client(t, server), server.URL, 200*time.Millisecond, wantProto)
|
||||
if !errors.Is(err, context.DeadlineExceeded) {
|
||||
t.Fatalf("expected %v, got %v", context.DeadlineExceeded, err)
|
||||
}
|
||||
if len(data) != stage.want {
|
||||
t.Fatalf("expected the %d bytes sent before the stall, got %d", stage.want, len(data))
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Cancelling the caller's context must not be reported as a read timeout.
|
||||
func TestTimeoutReader_CallerCancellationIsNotTimeout(t *testing.T) {
|
||||
for _, tc := range timeoutReaderClientCases() {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
server := startTimeoutReaderServer(t, tc.http2, func(w gohttp.ResponseWriter, r *gohttp.Request) {
|
||||
w.Header().Set("Content-Length", "1048576")
|
||||
w.Write(bytes.Repeat([]byte("a"), 1024))
|
||||
w.(gohttp.Flusher).Flush()
|
||||
<-r.Context().Done()
|
||||
})
|
||||
|
||||
wantProto := 1
|
||||
if tc.http2 {
|
||||
wantProto = 2
|
||||
}
|
||||
// Cancel once the client has read the first bytes, so the next
|
||||
// Read is the one blocked when the caller cancels.
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
client := tc.client(t, server)
|
||||
next := client.Transport
|
||||
client.Transport = roundTripFunc(func(r *gohttp.Request) (*gohttp.Response, error) {
|
||||
resp, err := next.RoundTrip(r)
|
||||
if err == nil {
|
||||
resp.Body = &cancelAfterFirstRead{ReadCloser: resp.Body, cancel: cancel}
|
||||
}
|
||||
return resp, err
|
||||
})
|
||||
|
||||
const timeout = 5 * time.Second
|
||||
start := time.Now()
|
||||
data, err := readBodyWithTimeout(t, ctx, client, server.URL, timeout, wantProto)
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("expected %v, got %v", context.Canceled, err)
|
||||
}
|
||||
if len(data) == 0 {
|
||||
t.Fatal("cancelled before any data was read")
|
||||
}
|
||||
if elapsed := time.Since(start); elapsed >= timeout {
|
||||
t.Fatalf("read returned after %v, not on cancellation", elapsed)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
type roundTripFunc func(*gohttp.Request) (*gohttp.Response, error)
|
||||
|
||||
func (f roundTripFunc) RoundTrip(r *gohttp.Request) (*gohttp.Response, error) {
|
||||
return f(r)
|
||||
}
|
||||
|
||||
// cancelAfterFirstRead calls cancel once a Read has returned data.
|
||||
type cancelAfterFirstRead struct {
|
||||
io.ReadCloser
|
||||
once sync.Once
|
||||
cancel context.CancelFunc
|
||||
}
|
||||
|
||||
func (b *cancelAfterFirstRead) Read(p []byte) (int, error) {
|
||||
n, err := b.ReadCloser.Read(p)
|
||||
if n > 0 {
|
||||
b.once.Do(b.cancel)
|
||||
}
|
||||
return n, err
|
||||
}
|
||||
+1
-6
@@ -2,7 +2,6 @@ package api
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/url"
|
||||
pathpkg "path"
|
||||
"strings"
|
||||
)
|
||||
@@ -76,11 +75,7 @@ func (r RouteSpec) Match(rawPath string) (map[string]string, bool) {
|
||||
params := make(map[string]string)
|
||||
for index, part := range r.parts {
|
||||
if part.param != "" {
|
||||
value, err := url.PathUnescape(segments[index])
|
||||
if err != nil {
|
||||
return nil, false
|
||||
}
|
||||
params[part.param] = value
|
||||
params[part.param] = segments[index]
|
||||
continue
|
||||
}
|
||||
if part.literal != segments[index] {
|
||||
|
||||
+4
-38
@@ -2,9 +2,7 @@ package api
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
@@ -64,8 +62,6 @@ func NewService(downloader *download.Downloader) (*Service, error) {
|
||||
newRoute(http.MethodGet, "/api/v1/tasks/{id}/status", service.getTaskStatus),
|
||||
newRoute(http.MethodGet, "/api/v1/tasks", service.getTasks),
|
||||
newRoute(http.MethodGet, "/api/v1/tasks/{id}/stats", service.getStats),
|
||||
newRoute(http.MethodPost, "/api/v1/protocols/{protocol}/jobs/{name}/trigger", service.triggerProtocolJob),
|
||||
newRoute(http.MethodPost, "/api/v1/mirrors/{provider}/candidates", service.mirrorCandidates),
|
||||
newRoute(http.MethodGet, "/api/v1/config", service.getConfig),
|
||||
newRoute(http.MethodPut, "/api/v1/config", service.putConfig),
|
||||
newRoute(http.MethodPost, "/api/v1/extensions", service.installExtension),
|
||||
@@ -143,7 +139,7 @@ func (s *Service) resolve(ctx *Context) *Response {
|
||||
if response := decodeRequest(ctx.Body, &request); response != nil {
|
||||
return response
|
||||
}
|
||||
result, err := s.Downloader.Resolve(request.Req)
|
||||
result, err := s.Downloader.Resolve(request.Req, request.Opts)
|
||||
if err != nil {
|
||||
return newErrorResponse(err.Error())
|
||||
}
|
||||
@@ -180,11 +176,11 @@ func (s *Service) createTaskBatch(ctx *Context) *Response {
|
||||
if len(request.Reqs) == 0 {
|
||||
return newErrorResponse("param invalid: reqs", restmodel.CodeInvalidParam)
|
||||
}
|
||||
result, err := s.Downloader.CreateDirectBatchWithResult(&request)
|
||||
taskIDs, err := s.Downloader.CreateDirectBatch(&request)
|
||||
if err != nil {
|
||||
return newErrorResponse(err.Error())
|
||||
}
|
||||
return newOKResponse(result)
|
||||
return newOKResponse(taskIDs)
|
||||
}
|
||||
|
||||
func (s *Service) patchTask(ctx *Context) *Response {
|
||||
@@ -192,7 +188,7 @@ func (s *Service) patchTask(ctx *Context) *Response {
|
||||
if taskID == "" {
|
||||
return invalidIDResponse()
|
||||
}
|
||||
var request restmodel.PatchTask
|
||||
var request restmodel.ResolveTask
|
||||
if response := decodeRequest(ctx.Body, &request); response != nil {
|
||||
return response
|
||||
}
|
||||
@@ -298,36 +294,6 @@ func (s *Service) getStats(ctx *Context) *Response {
|
||||
return newOKResponse(stats)
|
||||
}
|
||||
|
||||
func (s *Service) triggerProtocolJob(ctx *Context) *Response {
|
||||
if err := s.Downloader.TriggerProtocolJob(context.Background(), ctx.PathParam("protocol"), ctx.PathParam("name")); err != nil {
|
||||
status := http.StatusBadRequest
|
||||
if errors.Is(err, download.ErrUnSupportedProtocol) || errors.Is(err, download.ErrProtocolJobNotFound) {
|
||||
status = http.StatusNotFound
|
||||
}
|
||||
return newStatusResponse(status, restmodel.NewErrorResult(err.Error()))
|
||||
}
|
||||
return newNilResponse()
|
||||
}
|
||||
|
||||
func (s *Service) mirrorCandidates(ctx *Context) *Response {
|
||||
var request restmodel.MirrorCandidatesReq
|
||||
if response := decodeRequest(ctx.Body, &request); response != nil {
|
||||
return response
|
||||
}
|
||||
urls, err := s.Downloader.GetMirrorCandidates(ctx.PathParam("provider"), request.URL)
|
||||
if err != nil {
|
||||
status := http.StatusInternalServerError
|
||||
switch {
|
||||
case errors.Is(err, download.ErrMirrorProviderNotFound):
|
||||
status = http.StatusNotFound
|
||||
case errors.Is(err, download.ErrInvalidMirrorURL):
|
||||
status = http.StatusBadRequest
|
||||
}
|
||||
return newStatusResponse(status, restmodel.NewErrorResult(err.Error()))
|
||||
}
|
||||
return newOKResponse(&restmodel.MirrorCandidatesResp{URLs: urls})
|
||||
}
|
||||
|
||||
func (s *Service) getConfig(_ *Context) *Response {
|
||||
config, err := s.Downloader.GetConfig()
|
||||
if err != nil {
|
||||
|
||||
@@ -1,331 +0,0 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/GopeedLab/gopeed/internal/fetcher"
|
||||
httpProtocol "github.com/GopeedLab/gopeed/internal/protocol/http"
|
||||
"github.com/GopeedLab/gopeed/pkg/base"
|
||||
"github.com/GopeedLab/gopeed/pkg/download"
|
||||
restmodel "github.com/GopeedLab/gopeed/pkg/rest/model"
|
||||
)
|
||||
|
||||
func TestResolveThenCreateWithDownloadOptions(t *testing.T) {
|
||||
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.Header().Set("Content-Disposition", `attachment; filename="payload.txt"`)
|
||||
w.Header().Set("Content-Length", "4")
|
||||
_, _ = w.Write([]byte("test"))
|
||||
}))
|
||||
t.Cleanup(upstream.Close)
|
||||
d := download.NewDownloader(&download.DownloaderConfig{StorageDir: t.TempDir()})
|
||||
if err := d.Setup(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := d.Close(); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
})
|
||||
// Resolving must work even when the default download directory is unusable.
|
||||
blocked := filepath.Join(t.TempDir(), "not-a-directory")
|
||||
if err := os.WriteFile(blocked, nil, 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
current, err := d.GetConfig()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
config := *current
|
||||
config.DownloadDir = blocked
|
||||
config.MaxRunning = 0 // Inspect the created task before it starts downloading.
|
||||
if err := d.PutConfig(&config); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
service, err := NewService(d)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
resolved := serviceTestRequest[*download.ResolveResult](t, service, http.MethodPost, "/api/v1/resolve", &restmodel.ResolveTask{
|
||||
Req: &base.Request{URL: upstream.URL},
|
||||
})
|
||||
if resolved.ID == "" || resolved.Res.Size != 4 || resolved.Res.Files[0].Name != "payload.txt" {
|
||||
t.Fatalf("unexpected resolved resource: %+v", resolved)
|
||||
}
|
||||
dir := filepath.Join(t.TempDir(), "chosen-after-resolve")
|
||||
opts := &base.Options{Path: dir, Name: "renamed.txt", SelectFiles: []int{0}}
|
||||
id := serviceTestRequest[string](t, service, http.MethodPost, "/api/v1/tasks", &restmodel.CreateTask{Rid: resolved.ID, Opts: opts})
|
||||
task := serviceTestRequest[*download.Task](t, service, http.MethodGet, "/api/v1/tasks/"+id, nil)
|
||||
if task.Meta.Opts.Path != dir || task.Meta.Opts.Name != opts.Name || !reflect.DeepEqual(task.Meta.Opts.SelectFiles, opts.SelectFiles) {
|
||||
t.Fatalf("creation lost submitted download options: %+v", task.Meta.Opts)
|
||||
}
|
||||
}
|
||||
|
||||
func serviceTestRequest[T any](t *testing.T, service *Service, method, path string, body any) T {
|
||||
t.Helper()
|
||||
encoded, err := json.Marshal(body)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
status, response := service.JSON(method, path, nil, encoded, nil)
|
||||
var result restmodel.Result[T]
|
||||
if err := json.Unmarshal(response, &result); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if status != http.StatusOK || result.Code != restmodel.CodeOk {
|
||||
t.Fatalf("%s %s failed: status=%d response=%s", method, path, status, response)
|
||||
}
|
||||
return result.Data
|
||||
}
|
||||
|
||||
func TestFullConfigDispatch(t *testing.T) {
|
||||
d := download.NewDownloader(&download.DownloaderConfig{StorageDir: t.TempDir(), TempDir: t.TempDir(), FetchManagers: []fetcher.FetcherManager{&httpProtocol.FetcherManager{}}})
|
||||
if err := d.Setup(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer d.Close()
|
||||
service, err := NewService(d)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
response := service.Dispatch(http.MethodPut, "/api/v1/config", nil, []byte(`{"protocolConfig":{"http":{"connections":3,"useServerCtime":false}}}`), nil)
|
||||
if response.StatusCode != http.StatusOK {
|
||||
t.Fatalf("%+v", response)
|
||||
}
|
||||
status, body := service.JSON(http.MethodGet, "/api/v1/config", nil, nil, nil)
|
||||
var result struct {
|
||||
Data struct {
|
||||
ProtocolConfig map[string]struct {
|
||||
Connections int `json:"connections"`
|
||||
} `json:"protocolConfig"`
|
||||
} `json:"data"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &result); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if status != http.StatusOK || result.Data.ProtocolConfig["http"].Connections != 3 {
|
||||
t.Fatalf("%d %s", status, body)
|
||||
}
|
||||
if response := service.Dispatch(http.MethodPost, "/api/v1/protocols/http/jobs/missing/trigger", nil, nil, nil); response.StatusCode != http.StatusNotFound {
|
||||
t.Fatalf("%+v", response)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNativePathParametersAreDecodedOnce(t *testing.T) {
|
||||
route := newRoute(http.MethodGet, "/api/v1/tasks/{id}/status", func(c *Context) *Response { return newOKResponse(c.PathParam("id")) })
|
||||
for _, test := range []struct{ path, want string }{
|
||||
{"/api/v1/tasks/%E5%BC%80%E5%8F%91%E8%80%85%40name/status", "开发者@name"},
|
||||
{"/api/v1/tasks/literal%252Fname/status", "literal%2Fname"},
|
||||
} {
|
||||
response := (&Service{routes: []RouteSpec{route}}).Dispatch(http.MethodGet, test.path, nil, nil, nil)
|
||||
raw, _ := json.Marshal(response.Body)
|
||||
var body struct {
|
||||
Data string `json:"data"`
|
||||
}
|
||||
_ = json.Unmarshal(raw, &body)
|
||||
if body.Data != test.want {
|
||||
t.Fatalf("%s => %q", test.path, body.Data)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMirrorCandidates(t *testing.T) {
|
||||
d := download.NewDownloader(&download.DownloaderConfig{StorageDir: t.TempDir(), TempDir: t.TempDir(), FetchManagers: []fetcher.FetcherManager{&httpProtocol.FetcherManager{}}})
|
||||
if err := d.Setup(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer d.Close()
|
||||
cfg, _ := d.GetConfig()
|
||||
cfg.GithubMirror = &base.GithubMirrorConfig{Enabled: true, Mirrors: []base.GithubMirror{
|
||||
{Type: "jsdelivr", URL: "https://cdn.example/gh"},
|
||||
{Type: "ghProxy", URL: "https://proxy.example"},
|
||||
{Type: "ghProxy", URL: "https://proxy.example"},
|
||||
}}
|
||||
if err := d.PutConfig(cfg); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
service, err := NewService(d)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, official := range []string{
|
||||
"https://github.com/GopeedLab/gopeed/releases/download/v2.0.0/package.zip",
|
||||
"https://raw.githubusercontent.com/owner/repo/main/trackers.txt",
|
||||
} {
|
||||
request, _ := json.Marshal(map[string]string{"url": official})
|
||||
status, raw := service.JSON(http.MethodPost, "/api/v1/mirrors/github/candidates", nil, request, nil)
|
||||
var body struct {
|
||||
Data struct {
|
||||
URLs []string `json:"urls"`
|
||||
} `json:"data"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &body); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := 2
|
||||
if official == "https://raw.githubusercontent.com/owner/repo/main/trackers.txt" {
|
||||
want = 3
|
||||
}
|
||||
urls := body.Data.URLs
|
||||
if status != http.StatusOK || len(urls) != want || urls[0] != official || urls[len(urls)-1] != "https://proxy.example/"+official {
|
||||
t.Fatalf("%d %s", status, raw)
|
||||
}
|
||||
}
|
||||
for _, invalid := range []string{"https://api.github.com/repos/GopeedLab/gopeed/releases?per_page=10", "", "https://example.com/x", "http://github.com/x", "https://github.com.evil.test/x", "https://[email protected]/x", "https://github.com:443/x", "https://github.com:/x", "https://github.com/x#fragment", "https://github.com/x#", "/relative", "https://github.com%2fevil.test/x"} {
|
||||
request, _ := json.Marshal(map[string]string{"url": invalid})
|
||||
if response := service.Dispatch(http.MethodPost, "/api/v1/mirrors/github/candidates", nil, request, nil); response.StatusCode != http.StatusBadRequest {
|
||||
t.Fatalf("accepted %s: %+v", invalid, response)
|
||||
}
|
||||
}
|
||||
cfg.GithubMirror.Enabled = false
|
||||
if err := d.PutConfig(cfg); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
status, raw := service.JSON(http.MethodPost, "/api/v1/mirrors/github/candidates", nil, []byte(`{"url":"https://github.com/owner/repo"}`), nil)
|
||||
var body struct {
|
||||
Data struct {
|
||||
URLs []string `json:"urls"`
|
||||
} `json:"data"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &body); err != nil || status != http.StatusOK || len(body.Data.URLs) != 1 || body.Data.URLs[0] != "https://github.com/owner/repo" {
|
||||
t.Fatalf("%d %s", status, raw)
|
||||
}
|
||||
|
||||
for _, provider := range []string{"unknown", "GitHub"} {
|
||||
if response := service.Dispatch(http.MethodPost, "/api/v1/mirrors/"+provider+"/candidates", nil, []byte(`{"url":"https://github.com/owner/repo"}`), nil); response.StatusCode != http.StatusNotFound {
|
||||
t.Fatalf("unknown provider accepted: %+v", response)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type apiJobState struct {
|
||||
Runs int `json:"runs"`
|
||||
}
|
||||
type apiJobConfig struct {
|
||||
fetcher.ConfigState[apiJobState]
|
||||
Writable int `json:"writable"`
|
||||
}
|
||||
type apiStateJobManager struct{ httpProtocol.FetcherManager }
|
||||
|
||||
func (*apiStateJobManager) Name() string { return "state-api" }
|
||||
func (*apiStateJobManager) DefaultConfig() any { return &apiJobConfig{Writable: 7} }
|
||||
func (*apiStateJobManager) CronJobs(host fetcher.ProtocolHost) []fetcher.CronJob {
|
||||
return []fetcher.CronJob{{Name: "refresh", Every: time.Hour, Timeout: time.Second, Run: func(_ context.Context, source fetcher.TriggerSource) error {
|
||||
if source == fetcher.TriggerScheduled {
|
||||
return fetcher.ErrJobSkipped
|
||||
}
|
||||
return fetcher.UpdateState[apiJobConfig, apiJobState](host, func(cfg *apiJobConfig, state *apiJobState) error { cfg.Writable = 999; state.Runs++; return nil })
|
||||
}}}
|
||||
}
|
||||
|
||||
func TestFullConfigAndJobTriggerShareProtectedProtocolState(t *testing.T) {
|
||||
d := download.NewDownloader(&download.DownloaderConfig{StorageDir: t.TempDir(), TempDir: t.TempDir(), FetchManagers: []fetcher.FetcherManager{&apiStateJobManager{}}})
|
||||
if err := d.Setup(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer d.Close()
|
||||
service, err := NewService(d)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
put := []byte(`{"protocolConfig":{"state-api":{"writable":9,"state":"ignore forged state"}}}`)
|
||||
if response := service.Dispatch(http.MethodPut, "/api/v1/config", nil, put, nil); response.StatusCode != http.StatusOK {
|
||||
t.Fatalf("%+v", response)
|
||||
}
|
||||
if response := service.Dispatch(http.MethodPost, "/api/v1/protocols/state-api/jobs/refresh/trigger", nil, nil, nil); response.StatusCode != http.StatusOK {
|
||||
t.Fatalf("%+v", response)
|
||||
}
|
||||
status, raw := service.JSON(http.MethodGet, "/api/v1/config", nil, nil, nil)
|
||||
var response struct {
|
||||
Data struct {
|
||||
ProtocolConfig map[string]apiJobConfig `json:"protocolConfig"`
|
||||
} `json:"data"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &response); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cfg := response.Data.ProtocolConfig["state-api"]
|
||||
if status != http.StatusOK || cfg.Writable != 9 || cfg.State.Runs != 1 {
|
||||
t.Fatalf("%d %s", status, raw)
|
||||
}
|
||||
// Replaying an old full PUT must not undo the manual job's state change.
|
||||
if response := service.Dispatch(http.MethodPut, "/api/v1/config", nil, put, nil); response.StatusCode != http.StatusOK {
|
||||
t.Fatalf("%+v", response)
|
||||
}
|
||||
_, raw = service.JSON(http.MethodGet, "/api/v1/config", nil, nil, nil)
|
||||
if err := json.Unmarshal(raw, &response); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if response.Data.ProtocolConfig["state-api"].State.Runs != 1 {
|
||||
t.Fatalf("job state overwritten: %s", raw)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBatchResolvedAndDirectCreation(t *testing.T) {
|
||||
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.Header().Set("Content-Length", "4")
|
||||
_, _ = w.Write([]byte("test"))
|
||||
}))
|
||||
t.Cleanup(upstream.Close)
|
||||
d := download.NewDownloader(&download.DownloaderConfig{StorageDir: t.TempDir(), TempDir: t.TempDir(), FetchManagers: []fetcher.FetcherManager{&httpProtocol.FetcherManager{}}})
|
||||
if err := d.Setup(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { d.Close() })
|
||||
cfg, err := d.GetConfig()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cfg.MaxRunning = 0
|
||||
if err := d.PutConfig(cfg); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
service, err := NewService(d)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, grouped := range []bool{false, true} {
|
||||
resolved := serviceTestRequest[*download.ResolveResult](t, service, http.MethodPost, "/api/v1/resolve", &restmodel.ResolveTask{Req: &base.Request{URL: upstream.URL + "/resolved"}})
|
||||
result := serviceTestRequest[map[string]json.RawMessage](t, service, http.MethodPost, "/api/v1/tasks/batch", &base.CreateTaskBatch{
|
||||
Group: grouped, GroupName: "folder", Opts: &base.Options{Path: t.TempDir()},
|
||||
Reqs: []*base.CreateTaskBatchItem{
|
||||
{Rid: "invalid"},
|
||||
{Rid: resolved.ID, Req: &base.Request{URL: "unsupported://ignored"}, Opts: &base.Options{SelectFiles: []int{0}}},
|
||||
{Req: &base.Request{URL: upstream.URL + "/direct"}},
|
||||
},
|
||||
})
|
||||
var ids []string
|
||||
if err := json.Unmarshal(result["taskIds"], &ids); err != nil || len(ids) != 2 {
|
||||
t.Fatalf("unexpected task IDs: %+v, %v", result, err)
|
||||
}
|
||||
wantFields := 1
|
||||
if grouped {
|
||||
wantFields = 2
|
||||
}
|
||||
if len(result) != wantFields {
|
||||
t.Fatalf("unexpected response fields: %+v", result)
|
||||
}
|
||||
var parentID string
|
||||
if grouped {
|
||||
if err := json.Unmarshal(result["parentId"], &parentID); err != nil || parentID == "" {
|
||||
t.Fatalf("missing parent ID: %+v", result)
|
||||
}
|
||||
}
|
||||
task := serviceTestRequest[*download.Task](t, service, http.MethodGet, "/api/v1/tasks/"+ids[0], nil)
|
||||
if task.ParentID != parentID || !reflect.DeepEqual(task.Meta.Opts.SelectFiles, []int{0}) {
|
||||
t.Fatalf("resolved task lost membership or selection: %+v", task)
|
||||
}
|
||||
status, response := service.JSON(http.MethodPost, "/api/v1/tasks/batch", nil, []byte(`{"reqs":[{"rid":"missing"},{"req":{"url":"unsupported://bad"}}]}`), nil)
|
||||
var failed restmodel.Result[any]
|
||||
if err := json.Unmarshal(response, &failed); err != nil || status != http.StatusOK || failed.Code == restmodel.CodeOk {
|
||||
t.Fatalf("all-failed batch succeeded: %d %s", status, response)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,11 +0,0 @@
|
||||
package base
|
||||
|
||||
import "github.com/GopeedLab/gopeed/internal/githubmirror"
|
||||
|
||||
// Preserve downloader configuration type names and the existing JSON schema.
|
||||
type GithubMirror = githubmirror.Mirror
|
||||
type GithubMirrorConfig = githubmirror.Config
|
||||
|
||||
func DefaultGithubMirrorConfig() *GithubMirrorConfig {
|
||||
return githubmirror.DefaultConfig()
|
||||
}
|
||||
+2
-141
@@ -1,11 +1,9 @@
|
||||
package base
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
@@ -27,33 +25,6 @@ type Request struct {
|
||||
SkipVerifyCert bool `json:"skipVerifyCert"`
|
||||
}
|
||||
|
||||
// SetLabels replaces all labels on the request.
|
||||
//
|
||||
// Deprecated: In onResolve and onDone, assign request labels directly.
|
||||
// In onStart and onError, use task.meta.req.setLabels instead.
|
||||
func (r *Request) SetLabels(labels map[string]string) {
|
||||
r.Labels = labels
|
||||
}
|
||||
|
||||
// PutLabel sets a label on the request.
|
||||
//
|
||||
// Deprecated: In onResolve and onDone, assign request labels directly.
|
||||
// In onStart and onError, use task.meta.req.putLabel instead.
|
||||
func (r *Request) PutLabel(key, value string) {
|
||||
if r.Labels == nil {
|
||||
r.Labels = make(map[string]string)
|
||||
}
|
||||
r.Labels[key] = value
|
||||
}
|
||||
|
||||
// DelLabel deletes a label from the request.
|
||||
//
|
||||
// Deprecated: In onResolve and onDone, delete request labels directly.
|
||||
// In onStart and onError, use task.meta.req.delLabel instead.
|
||||
func (r *Request) DelLabel(key string) {
|
||||
delete(r.Labels, key)
|
||||
}
|
||||
|
||||
func (r *Request) Validate() error {
|
||||
if r.URL == "" {
|
||||
return fmt.Errorf("invalid request url")
|
||||
@@ -137,18 +108,12 @@ type FileInfo struct {
|
||||
|
||||
// Options for download
|
||||
type Options struct {
|
||||
// Size is an optional HTTP download size estimate. HTTP metadata and clean EOF take precedence.
|
||||
// Other protocols ignore it.
|
||||
Size int64 `json:"size,omitempty"`
|
||||
// Download file name
|
||||
Name string `json:"name"`
|
||||
// Download file path
|
||||
Path string `json:"path"`
|
||||
// Use this path as the default download directory after creation succeeds
|
||||
AsDefaultPath bool `json:"asDefaultPath,omitempty"`
|
||||
// SkipAutoCategorize keeps the base download directory of this task instead
|
||||
// of routing it to the category directory of its resolved file
|
||||
SkipAutoCategorize bool `json:"skipAutoCategorize,omitempty"`
|
||||
// Select file indexes to download
|
||||
SelectFiles []int `json:"selectFiles"`
|
||||
// Extra info for specific fetcher
|
||||
@@ -200,14 +165,11 @@ func ParseOptExtra[E any](opts *Options) error {
|
||||
}
|
||||
|
||||
type CreateTaskBatch struct {
|
||||
Reqs []*CreateTaskBatchItem `json:"reqs"`
|
||||
Opts *Options `json:"opts"`
|
||||
Group bool `json:"group"`
|
||||
GroupName string `json:"groupName"`
|
||||
Reqs []*CreateTaskBatchItem `json:"reqs"`
|
||||
Opts *Options `json:"opts"`
|
||||
}
|
||||
|
||||
type CreateTaskBatchItem struct {
|
||||
Rid string `json:"rid,omitempty"`
|
||||
Req *Request `json:"req"`
|
||||
Opts *Options `json:"opts"`
|
||||
}
|
||||
@@ -219,15 +181,12 @@ type DownloaderStoreConfig struct {
|
||||
DownloadDir string `json:"downloadDir"` // DownloadDir is the default directory to save the downloaded files
|
||||
MaxRunning int `json:"maxRunning"` // MaxRunning is the max running download count
|
||||
ProtocolConfig map[string]any `json:"protocolConfig"` // ProtocolConfig is special config for each protocol
|
||||
GithubMirror *GithubMirrorConfig `json:"githubMirror"`
|
||||
Extra map[string]any `json:"extra"`
|
||||
Proxy *DownloaderProxyConfig `json:"proxy"`
|
||||
Webhook *WebhookConfig `json:"webhook"` // Webhook is the webhook configuration
|
||||
Script *ScriptConfig `json:"script"` // Script is the script execution configuration
|
||||
AutoTorrent *AutoTorrentConfig `json:"autoTorrent"` // AutoTorrent is the auto torrent task creation configuration
|
||||
Archive *ArchiveConfig `json:"archive"` // Archive is the archive extraction configuration
|
||||
Categories []*DownloadCategory `json:"categories"` // Categories is the download directory category configuration
|
||||
AutoCategorize *bool `json:"autoCategorize,omitempty"` // AutoCategorize routes downloads to category directories by file extension; unset defaults to disabled
|
||||
API *APIServerConfig `json:"api"` // API is the optional REST server configuration
|
||||
AutoStartTasks bool `json:"autoStartTasks"` // AutoStartTasks continues all unfinished tasks when the backend starts
|
||||
AutoDeleteMissingFileTasks bool `json:"autoDeleteMissingFileTasks"` // AutoDeleteMissingFileTasks enables automatic deletion of tasks with missing files
|
||||
@@ -240,9 +199,6 @@ func (cfg *DownloaderStoreConfig) Init() *DownloaderStoreConfig {
|
||||
if cfg.ProtocolConfig == nil {
|
||||
cfg.ProtocolConfig = make(map[string]any)
|
||||
}
|
||||
if cfg.GithubMirror == nil {
|
||||
cfg.GithubMirror = DefaultGithubMirrorConfig()
|
||||
}
|
||||
if cfg.Proxy == nil {
|
||||
cfg.Proxy = &DownloaderProxyConfig{}
|
||||
}
|
||||
@@ -264,85 +220,9 @@ func (cfg *DownloaderStoreConfig) Init() *DownloaderStoreConfig {
|
||||
DeleteAfterExtract: false,
|
||||
}
|
||||
}
|
||||
// Seed the built-in categories for deployments that never run the flutter
|
||||
// startup flow, e.g. headless api servers.
|
||||
if len(cfg.Categories) == 0 && cfg.DownloadDir != "" {
|
||||
cfg.Categories = []*DownloadCategory{
|
||||
builtinSeedCategory("categoryMusic", "Music", cfg.DownloadDir),
|
||||
builtinSeedCategory("categoryVideo", "Video", cfg.DownloadDir),
|
||||
builtinSeedCategory("categoryDocument", "Document", cfg.DownloadDir),
|
||||
builtinSeedCategory("categoryProgram", "Program", cfg.DownloadDir),
|
||||
}
|
||||
}
|
||||
return cfg
|
||||
}
|
||||
|
||||
// builtinCategoryExtensions mirrors the flutter side
|
||||
// kDefaultCategoryExtensions, keep both maps in sync.
|
||||
var builtinCategoryExtensions = map[string][]string{
|
||||
"categoryProgram": {"exe", "msi", "msix", "apk", "dmg", "deb", "rpm", "pkg", "appimage"},
|
||||
"categoryVideo": {"mp4", "mkv", "avi", "mov", "wmv", "flv", "webm", "m4v", "mpg", "mpeg", "3gp"},
|
||||
"categoryMusic": {"mp3", "flac", "wav", "aac", "ogg", "m4a", "wma", "ape"},
|
||||
"categoryDocument": {"pdf", "doc", "docx", "xls", "xlsx", "ppt", "pptx", "txt", "md", "csv", "epub", "rtf"},
|
||||
}
|
||||
|
||||
func builtinSeedCategory(nameKey, name, downloadDir string) *DownloadCategory {
|
||||
return &DownloadCategory{
|
||||
Name: "",
|
||||
Path: filepath.Join(downloadDir, name),
|
||||
NameKey: nameKey,
|
||||
IsBuiltIn: true,
|
||||
Extensions: append([]string(nil), builtinCategoryExtensions[nameKey]...),
|
||||
}
|
||||
}
|
||||
|
||||
// MigrateLegacyExtraCategories moves the extra.downloadCategories list written
|
||||
// by v2.0.0-beta builds into the top level categories field and clears the
|
||||
// legacy key. The legacy list predates extension based routing, so built-in
|
||||
// entries regain their default extension lists from their name key, while user
|
||||
// created folders keep their empty list until edited in settings. It reports
|
||||
// whether the config changed so the caller can persist the cleared key.
|
||||
func (cfg *DownloaderStoreConfig) MigrateLegacyExtraCategories() bool {
|
||||
if len(cfg.Categories) > 0 || cfg.Extra == nil {
|
||||
return false
|
||||
}
|
||||
raw, ok := cfg.Extra["downloadCategories"]
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
data, err := json.Marshal(raw)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
var legacy []*DownloadCategory
|
||||
if err := json.Unmarshal(data, &legacy); err != nil {
|
||||
return false
|
||||
}
|
||||
categories := make([]*DownloadCategory, 0, len(legacy))
|
||||
for _, category := range legacy {
|
||||
if category == nil {
|
||||
continue
|
||||
}
|
||||
if extensions, exist := builtinCategoryExtensions[category.NameKey]; exist && len(category.Extensions) == 0 {
|
||||
category.Extensions = append([]string(nil), extensions...)
|
||||
}
|
||||
categories = append(categories, category)
|
||||
}
|
||||
if len(categories) == 0 {
|
||||
return false
|
||||
}
|
||||
cfg.Categories = categories
|
||||
delete(cfg.Extra, "downloadCategories")
|
||||
return true
|
||||
}
|
||||
|
||||
// AutoCategorizeEnabled reports whether downloads are routed to category
|
||||
// directories by file extension. The switch is optional: an unset value keeps
|
||||
// the feature off, so an older config that never saved the field stays off.
|
||||
func (cfg *DownloaderStoreConfig) AutoCategorizeEnabled() bool {
|
||||
return cfg.AutoCategorize != nil && *cfg.AutoCategorize
|
||||
}
|
||||
|
||||
func (cfg *DownloaderStoreConfig) Merge(beforeCfg *DownloaderStoreConfig) *DownloaderStoreConfig {
|
||||
if beforeCfg == nil {
|
||||
return cfg
|
||||
@@ -359,9 +239,6 @@ func (cfg *DownloaderStoreConfig) Merge(beforeCfg *DownloaderStoreConfig) *Downl
|
||||
if cfg.Extra == nil {
|
||||
cfg.Extra = beforeCfg.Extra
|
||||
}
|
||||
if cfg.GithubMirror == nil {
|
||||
cfg.GithubMirror = beforeCfg.GithubMirror
|
||||
}
|
||||
if cfg.Proxy == nil {
|
||||
cfg.Proxy = beforeCfg.Proxy
|
||||
}
|
||||
@@ -377,12 +254,6 @@ func (cfg *DownloaderStoreConfig) Merge(beforeCfg *DownloaderStoreConfig) *Downl
|
||||
if cfg.Archive == nil {
|
||||
cfg.Archive = beforeCfg.Archive
|
||||
}
|
||||
if cfg.Categories == nil {
|
||||
cfg.Categories = beforeCfg.Categories
|
||||
}
|
||||
if cfg.AutoCategorize == nil {
|
||||
cfg.AutoCategorize = beforeCfg.AutoCategorize
|
||||
}
|
||||
if cfg.API == nil {
|
||||
cfg.API = beforeCfg.API
|
||||
}
|
||||
@@ -431,16 +302,6 @@ type ArchiveConfig struct {
|
||||
DeleteAfterExtract bool `json:"deleteAfterExtract"` // DeleteAfterExtract deletes the archive after successful extraction
|
||||
}
|
||||
|
||||
// DownloadCategory is a download directory category that files can be routed to by extension
|
||||
type DownloadCategory struct {
|
||||
Name string `json:"name"` // Name is the category display name, empty for built-in categories that use NameKey
|
||||
Path string `json:"path"` // Path is the download directory of the category
|
||||
NameKey string `json:"nameKey,omitempty"` // NameKey is the i18n key of the built-in category name
|
||||
IsBuiltIn bool `json:"isBuiltIn,omitempty"` // IsBuiltIn marks the category as built-in
|
||||
IsDeleted bool `json:"isDeleted,omitempty"` // IsDeleted marks the built-in category as deleted
|
||||
Extensions []string `json:"extensions,omitempty"` // Extensions is the file extension list routed to the category, without leading dots
|
||||
}
|
||||
|
||||
type DownloaderProxyConfig struct {
|
||||
Enable bool `json:"enable"`
|
||||
// System is the flag that use system proxy
|
||||
|
||||
+1
-243
@@ -1,13 +1,8 @@
|
||||
package base
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/GopeedLab/gopeed/pkg/util"
|
||||
)
|
||||
|
||||
func TestDownloaderStoreConfig_Init(t *testing.T) {
|
||||
@@ -20,7 +15,6 @@ func TestDownloaderStoreConfig_Init(t *testing.T) {
|
||||
"Init",
|
||||
&DownloaderStoreConfig{},
|
||||
&DownloaderStoreConfig{
|
||||
GithubMirror: DefaultGithubMirrorConfig(),
|
||||
MaxRunning: 5,
|
||||
ProtocolConfig: map[string]any{},
|
||||
Proxy: &DownloaderProxyConfig{},
|
||||
@@ -42,7 +36,6 @@ func TestDownloaderStoreConfig_Init(t *testing.T) {
|
||||
MaxRunning: 10,
|
||||
},
|
||||
&DownloaderStoreConfig{
|
||||
GithubMirror: DefaultGithubMirrorConfig(),
|
||||
MaxRunning: 10,
|
||||
ProtocolConfig: map[string]any{},
|
||||
Proxy: &DownloaderProxyConfig{},
|
||||
@@ -66,8 +59,7 @@ func TestDownloaderStoreConfig_Init(t *testing.T) {
|
||||
},
|
||||
},
|
||||
&DownloaderStoreConfig{
|
||||
GithubMirror: DefaultGithubMirrorConfig(),
|
||||
MaxRunning: 5,
|
||||
MaxRunning: 5,
|
||||
ProtocolConfig: map[string]any{
|
||||
"key": "value",
|
||||
},
|
||||
@@ -92,7 +84,6 @@ func TestDownloaderStoreConfig_Init(t *testing.T) {
|
||||
},
|
||||
},
|
||||
&DownloaderStoreConfig{
|
||||
GithubMirror: DefaultGithubMirrorConfig(),
|
||||
MaxRunning: 5,
|
||||
ProtocolConfig: map[string]any{},
|
||||
Proxy: &DownloaderProxyConfig{
|
||||
@@ -119,7 +110,6 @@ func TestDownloaderStoreConfig_Init(t *testing.T) {
|
||||
},
|
||||
},
|
||||
&DownloaderStoreConfig{
|
||||
GithubMirror: DefaultGithubMirrorConfig(),
|
||||
MaxRunning: 5,
|
||||
ProtocolConfig: map[string]any{},
|
||||
Proxy: &DownloaderProxyConfig{},
|
||||
@@ -144,7 +134,6 @@ func TestDownloaderStoreConfig_Init(t *testing.T) {
|
||||
},
|
||||
},
|
||||
&DownloaderStoreConfig{
|
||||
GithubMirror: DefaultGithubMirrorConfig(),
|
||||
MaxRunning: 5,
|
||||
ProtocolConfig: map[string]any{},
|
||||
Proxy: &DownloaderProxyConfig{},
|
||||
@@ -160,20 +149,6 @@ func TestDownloaderStoreConfig_Init(t *testing.T) {
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "Init explicitly disabled mirrors with empty list",
|
||||
fields: &DownloaderStoreConfig{GithubMirror: &GithubMirrorConfig{Enabled: false, Mirrors: []GithubMirror{}}},
|
||||
want: &DownloaderStoreConfig{
|
||||
MaxRunning: 5,
|
||||
ProtocolConfig: map[string]any{},
|
||||
GithubMirror: &GithubMirrorConfig{Enabled: false, Mirrors: []GithubMirror{}},
|
||||
Proxy: &DownloaderProxyConfig{},
|
||||
Webhook: &WebhookConfig{},
|
||||
Script: &ScriptConfig{},
|
||||
AutoTorrent: &AutoTorrentConfig{},
|
||||
Archive: &ArchiveConfig{},
|
||||
},
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
@@ -182,7 +157,6 @@ func TestDownloaderStoreConfig_Init(t *testing.T) {
|
||||
DownloadDir: tt.fields.DownloadDir,
|
||||
MaxRunning: tt.fields.MaxRunning,
|
||||
ProtocolConfig: tt.fields.ProtocolConfig,
|
||||
GithubMirror: tt.fields.GithubMirror,
|
||||
Extra: tt.fields.Extra,
|
||||
Proxy: tt.fields.Proxy,
|
||||
Webhook: tt.fields.Webhook,
|
||||
@@ -197,202 +171,6 @@ func TestDownloaderStoreConfig_Init(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloaderStoreConfig_InitSeedsBuiltInCategories(t *testing.T) {
|
||||
t.Run("fresh config with a download dir gets the built-in categories", func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
cfg := &DownloaderStoreConfig{DownloadDir: dir}
|
||||
cfg.Init()
|
||||
|
||||
wantKeys := []string{"categoryMusic", "categoryVideo", "categoryDocument", "categoryProgram"}
|
||||
wantNames := []string{"Music", "Video", "Document", "Program"}
|
||||
if len(cfg.Categories) != len(wantKeys) {
|
||||
t.Fatalf("categories = %d, want %d", len(cfg.Categories), len(wantKeys))
|
||||
}
|
||||
for i, category := range cfg.Categories {
|
||||
if category.NameKey != wantKeys[i] {
|
||||
t.Errorf("category[%d].NameKey = %q, want %q", i, category.NameKey, wantKeys[i])
|
||||
}
|
||||
if !category.IsBuiltIn {
|
||||
t.Errorf("category[%d] %q is not built-in", i, category.NameKey)
|
||||
}
|
||||
if category.Path != filepath.Join(dir, wantNames[i]) {
|
||||
t.Errorf("category[%d].Path = %q, want %q", i, category.Path, filepath.Join(dir, wantNames[i]))
|
||||
}
|
||||
if len(category.Extensions) == 0 {
|
||||
t.Errorf("category[%d] %q has no default extensions", i, category.NameKey)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
// Init itself never reads the legacy key: Setup calls
|
||||
// MigrateLegacyExtraCategories first, and only a config that skipped the
|
||||
// migration ends up with the built-in seed.
|
||||
t.Run("legacy extra downloadCategories is not read by Init alone", func(t *testing.T) {
|
||||
cfg := &DownloaderStoreConfig{
|
||||
DownloadDir: t.TempDir(),
|
||||
Extra: map[string]any{
|
||||
"downloadCategories": []any{map[string]any{"name": "Music", "path": "/old/Music"}},
|
||||
},
|
||||
}
|
||||
cfg.Init()
|
||||
if len(cfg.Categories) != 4 {
|
||||
t.Fatalf("categories = %d, want the 4 built-ins seeded anyway", len(cfg.Categories))
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("existing categories and missing download dirs are untouched", func(t *testing.T) {
|
||||
existing := []*DownloadCategory{{Name: "Mine", Path: "/downloads/Mine"}}
|
||||
cfg := &DownloaderStoreConfig{DownloadDir: t.TempDir(), Categories: existing}
|
||||
cfg.Init()
|
||||
if len(cfg.Categories) != 1 || cfg.Categories[0].Name != "Mine" {
|
||||
t.Fatalf("categories = %v, want the existing one", cfg.Categories)
|
||||
}
|
||||
|
||||
empty := &DownloaderStoreConfig{}
|
||||
empty.Init()
|
||||
if len(empty.Categories) != 0 {
|
||||
t.Fatalf("categories = %d, want 0 without a download dir", len(empty.Categories))
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestDownloaderStoreConfig_AutoCategorizeEnabled(t *testing.T) {
|
||||
// A config written before the switch existed has no value, and stays off.
|
||||
if (&DownloaderStoreConfig{}).AutoCategorizeEnabled() {
|
||||
t.Fatal("unset auto categorize = enabled, want disabled by default")
|
||||
}
|
||||
if !(&DownloaderStoreConfig{AutoCategorize: util.BoolPtr(true)}).AutoCategorizeEnabled() {
|
||||
t.Fatal("explicit on = disabled, want the opt-in preserved")
|
||||
}
|
||||
if (&DownloaderStoreConfig{AutoCategorize: util.BoolPtr(false)}).AutoCategorizeEnabled() {
|
||||
t.Fatal("explicit off = enabled, want the opt-out preserved")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloaderStoreConfig_MigrateLegacyExtraCategories(t *testing.T) {
|
||||
t.Run("legacy built-ins move to the top level with their default extensions", func(t *testing.T) {
|
||||
cfg := &DownloaderStoreConfig{
|
||||
DownloadDir: "/downloads",
|
||||
Extra: map[string]any{
|
||||
"downloadCategories": []any{
|
||||
map[string]any{"name": "", "path": "/old/Music", "isBuiltIn": true, "nameKey": "categoryMusic"},
|
||||
map[string]any{"name": "Games", "path": "/old/Games"},
|
||||
},
|
||||
"themeMode": "dark",
|
||||
},
|
||||
}
|
||||
if !cfg.MigrateLegacyExtraCategories() {
|
||||
t.Fatal("migration reported no change")
|
||||
}
|
||||
if len(cfg.Categories) != 2 {
|
||||
t.Fatalf("categories = %d, want the 2 legacy entries", len(cfg.Categories))
|
||||
}
|
||||
builtIn := cfg.Categories[0]
|
||||
if builtIn.NameKey != "categoryMusic" || !builtIn.IsBuiltIn || builtIn.Path != "/old/Music" {
|
||||
t.Fatalf("built-in category = %+v, want the legacy entry", builtIn)
|
||||
}
|
||||
if len(builtIn.Extensions) == 0 {
|
||||
t.Fatal("migrated built-in must regain its default extensions")
|
||||
}
|
||||
custom := cfg.Categories[1]
|
||||
if custom.Name != "Games" || custom.Path != "/old/Games" || len(custom.Extensions) != 0 {
|
||||
t.Fatalf("custom category = %+v, want the legacy folder untouched", custom)
|
||||
}
|
||||
if _, ok := cfg.Extra["downloadCategories"]; ok {
|
||||
t.Fatal("legacy key must be cleared")
|
||||
}
|
||||
if cfg.Extra["themeMode"] != "dark" {
|
||||
t.Fatalf("unrelated extra entries must survive, got %v", cfg.Extra)
|
||||
}
|
||||
|
||||
// Setup runs Init after the migration, which must not seed over it.
|
||||
cfg.Init()
|
||||
if len(cfg.Categories) != 2 {
|
||||
t.Fatalf("categories after Init = %d, want the migrated list kept", len(cfg.Categories))
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("existing top level categories are left alone", func(t *testing.T) {
|
||||
cfg := &DownloaderStoreConfig{
|
||||
Extra: map[string]any{
|
||||
"downloadCategories": []any{map[string]any{"name": "Old", "path": "/old/Default"}},
|
||||
},
|
||||
Categories: []*DownloadCategory{{Name: "Mine", Path: "/downloads/Mine"}},
|
||||
}
|
||||
if cfg.MigrateLegacyExtraCategories() {
|
||||
t.Fatal("migration must not overwrite existing categories")
|
||||
}
|
||||
if len(cfg.Categories) != 1 || cfg.Categories[0].Name != "Mine" {
|
||||
t.Fatalf("categories = %v, want the existing one", cfg.Categories)
|
||||
}
|
||||
if _, ok := cfg.Extra["downloadCategories"]; !ok {
|
||||
t.Fatal("legacy key must stay for a manual move")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("missing, empty and malformed legacy lists are a no-op", func(t *testing.T) {
|
||||
if (&DownloaderStoreConfig{}).MigrateLegacyExtraCategories() {
|
||||
t.Fatal("empty config reported a change")
|
||||
}
|
||||
unrelated := &DownloaderStoreConfig{
|
||||
Extra: map[string]any{"themeMode": "dark"},
|
||||
}
|
||||
if unrelated.MigrateLegacyExtraCategories() {
|
||||
t.Fatal("config without the legacy key reported a change")
|
||||
}
|
||||
empty := &DownloaderStoreConfig{
|
||||
Extra: map[string]any{"downloadCategories": []any{}},
|
||||
}
|
||||
if empty.MigrateLegacyExtraCategories() {
|
||||
t.Fatal("empty legacy list reported a change")
|
||||
}
|
||||
malformed := &DownloaderStoreConfig{
|
||||
Extra: map[string]any{"downloadCategories": "not-a-list"},
|
||||
}
|
||||
if malformed.MigrateLegacyExtraCategories() {
|
||||
t.Fatal("malformed legacy list reported a change")
|
||||
}
|
||||
if _, ok := malformed.Extra["downloadCategories"]; !ok {
|
||||
t.Fatal("malformed legacy key must stay untouched")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("a nil entry in the legacy list is skipped", func(t *testing.T) {
|
||||
cfg := &DownloaderStoreConfig{
|
||||
DownloadDir: "/downloads",
|
||||
Extra: map[string]any{
|
||||
"downloadCategories": []any{
|
||||
nil,
|
||||
map[string]any{"name": "Games", "path": "/old/Games"},
|
||||
},
|
||||
},
|
||||
}
|
||||
if !cfg.MigrateLegacyExtraCategories() {
|
||||
t.Fatal("migration reported no change")
|
||||
}
|
||||
if len(cfg.Categories) != 1 || cfg.Categories[0].Name != "Games" {
|
||||
t.Fatalf("categories = %+v, want the single valid legacy entry", cfg.Categories)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestDownloaderStoreConfig_AutoCategorizeJSON(t *testing.T) {
|
||||
unset, err := json.Marshal(&DownloaderStoreConfig{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Contains(string(unset), `"autoCategorize"`) {
|
||||
t.Fatalf("unset autoCategorize must be omitted from the api json, got %s", unset)
|
||||
}
|
||||
off, err := json.Marshal(&DownloaderStoreConfig{AutoCategorize: util.BoolPtr(false)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(string(off), `"autoCategorize":false`) {
|
||||
t.Fatalf("explicit autoCategorize=false must be emitted, got %s", off)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloaderStoreConfig_Merge(t *testing.T) {
|
||||
type args struct {
|
||||
beforeCfg *DownloaderStoreConfig
|
||||
@@ -437,24 +215,6 @@ func TestDownloaderStoreConfig_Merge(t *testing.T) {
|
||||
DownloadDir: "after",
|
||||
},
|
||||
},
|
||||
{
|
||||
"Merge Categories and AutoCategorize Override",
|
||||
&DownloaderStoreConfig{},
|
||||
args{
|
||||
beforeCfg: &DownloaderStoreConfig{
|
||||
Categories: []*DownloadCategory{
|
||||
{Name: "Music", Path: "/downloads/Music"},
|
||||
},
|
||||
AutoCategorize: util.BoolPtr(true),
|
||||
},
|
||||
},
|
||||
&DownloaderStoreConfig{
|
||||
Categories: []*DownloadCategory{
|
||||
{Name: "Music", Path: "/downloads/Music"},
|
||||
},
|
||||
AutoCategorize: util.BoolPtr(true),
|
||||
},
|
||||
},
|
||||
{
|
||||
"Merge MaxRunning No Override",
|
||||
&DownloaderStoreConfig{
|
||||
@@ -668,8 +428,6 @@ func TestDownloaderStoreConfig_Merge(t *testing.T) {
|
||||
Webhook: tt.fields.Webhook,
|
||||
AutoTorrent: tt.fields.AutoTorrent,
|
||||
Archive: tt.fields.Archive,
|
||||
Categories: tt.fields.Categories,
|
||||
AutoCategorize: tt.fields.AutoCategorize,
|
||||
}
|
||||
if got := cfg.Merge(tt.args.beforeCfg); !reflect.DeepEqual(got, tt.want) {
|
||||
t.Errorf("Merge() = %v, want %v", got, tt.want)
|
||||
|
||||
@@ -1,245 +0,0 @@
|
||||
package download
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/GopeedLab/gopeed/internal/fetcher"
|
||||
btProtocol "github.com/GopeedLab/gopeed/internal/protocol/bt"
|
||||
httpProtocol "github.com/GopeedLab/gopeed/internal/protocol/http"
|
||||
"github.com/GopeedLab/gopeed/pkg/base"
|
||||
)
|
||||
|
||||
type testJobManager struct {
|
||||
httpProtocol.FetcherManager
|
||||
job fetcher.CronJob
|
||||
}
|
||||
|
||||
func (m *testJobManager) Name() string { return "test" }
|
||||
func (m *testJobManager) CronJobs(fetcher.ProtocolHost) []fetcher.CronJob {
|
||||
return []fetcher.CronJob{m.job}
|
||||
}
|
||||
func testBackendDownloader(t *testing.T, managers ...fetcher.FetcherManager) *Downloader {
|
||||
t.Helper()
|
||||
d := NewDownloader(&DownloaderConfig{StorageDir: t.TempDir(), TempDir: t.TempDir(), FetchManagers: managers})
|
||||
if err := d.Setup(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := d.Close(); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
})
|
||||
return d
|
||||
}
|
||||
func TestProtocolJobCoalescesAndCancelsOnClose(t *testing.T) {
|
||||
entered := make(chan struct{})
|
||||
var calls atomic.Int32
|
||||
manager := &testJobManager{job: fetcher.CronJob{Name: "refresh", Every: time.Hour, Timeout: time.Minute, Run: func(ctx context.Context, _ fetcher.TriggerSource) error {
|
||||
if calls.Add(1) == 1 {
|
||||
close(entered)
|
||||
}
|
||||
<-ctx.Done()
|
||||
return ctx.Err()
|
||||
}}}
|
||||
d := testBackendDownloader(t, manager)
|
||||
result := make(chan error, 2)
|
||||
go func() { result <- d.TriggerProtocolJob(context.Background(), "test", "refresh") }()
|
||||
<-entered
|
||||
waiterCtx, cancel := context.WithCancel(context.Background())
|
||||
// A canceled waiter joins the existing work without canceling it.
|
||||
cancel()
|
||||
if err := d.TriggerProtocolJob(waiterCtx, "test", "refresh"); !errors.Is(err, context.Canceled) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if calls.Load() != 1 {
|
||||
t.Fatalf("executed %d times", calls.Load())
|
||||
}
|
||||
if err := d.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := <-result; !errors.Is(err, context.Canceled) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := d.TriggerProtocolJob(context.Background(), "test", "missing"); !errors.Is(err, ErrProtocolJobNotFound) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
func TestProtocolJobPersistsSuccess(t *testing.T) {
|
||||
manager := &testJobManager{job: fetcher.CronJob{Name: "refresh", Every: time.Hour, Timeout: time.Minute, Run: func(context.Context, fetcher.TriggerSource) error { return nil }}}
|
||||
d := testBackendDownloader(t, manager)
|
||||
if err := d.TriggerProtocolJob(context.Background(), "test", "refresh"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var state protocolJobState
|
||||
exists, err := d.storage.Get(bucketProtocolJobs, "test/refresh", &state)
|
||||
if err != nil || !exists || state.LastSuccess.IsZero() {
|
||||
t.Fatalf("state %+v: %v", state, err)
|
||||
}
|
||||
}
|
||||
func TestRaceResourceValidatesResponsesAndCancelsLosers(t *testing.T) {
|
||||
loserCanceled := make(chan struct{})
|
||||
loserStarted := make(chan struct{})
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch {
|
||||
case r.Method == http.MethodConnect:
|
||||
http.Error(w, "official unavailable", http.StatusBadGateway)
|
||||
case strings.HasPrefix(r.URL.Path, "/bad/"):
|
||||
io.WriteString(w, "<html>proxy landing page</html>")
|
||||
case strings.HasPrefix(r.URL.Path, "/good/"):
|
||||
<-loserStarted
|
||||
io.WriteString(w, `{"ok":true}`)
|
||||
case strings.HasPrefix(r.URL.Path, "/slow/"):
|
||||
close(loserStarted)
|
||||
<-r.Context().Done()
|
||||
close(loserCanceled)
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
}))
|
||||
defer server.Close()
|
||||
d := testBackendDownloader(t, &httpProtocol.FetcherManager{})
|
||||
cfg, _ := d.GetConfig()
|
||||
cfg.Proxy = &base.DownloaderProxyConfig{Enable: true, Scheme: "http", Host: strings.TrimPrefix(server.URL, "http://")}
|
||||
cfg.GithubMirror = &base.GithubMirrorConfig{Enabled: true, Mirrors: []base.GithubMirror{
|
||||
{Type: "ghProxy", URL: server.URL + "/bad"},
|
||||
{Type: "ghProxy", URL: server.URL + "/slow"},
|
||||
{Type: "ghProxy", URL: server.URL + "/good"},
|
||||
}}
|
||||
if err := d.PutConfig(cfg); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body, err := d.fetchResource(context.Background(), "https://raw.githubusercontent.com/owner/repo/main/test.json", 1024, false, func(body []byte) error {
|
||||
if !json.Valid(body) {
|
||||
return errors.New("invalid JSON")
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil || string(body) != `{"ok":true}` {
|
||||
t.Fatalf("%s %v", body, err)
|
||||
}
|
||||
select {
|
||||
case <-loserCanceled:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("losing request was not canceled")
|
||||
}
|
||||
}
|
||||
|
||||
func TestProtocolConfigInitializesCurrentSchema(t *testing.T) {
|
||||
d := NewDownloader(&DownloaderConfig{StorageDir: t.TempDir(), TempDir: t.TempDir(), FetchManagers: []fetcher.FetcherManager{&httpProtocol.FetcherManager{}}})
|
||||
d.cfg.DownloaderStoreConfig = &base.DownloaderStoreConfig{
|
||||
GithubMirror: &base.GithubMirrorConfig{Enabled: false, Mirrors: []base.GithubMirror{}},
|
||||
ProtocolConfig: map[string]any{"http": map[string]any{"connections": 3}},
|
||||
}
|
||||
if err := d.storage.Setup([]string{bucketConfig}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer d.storage.Close()
|
||||
if err := d.initProtocolConfigs(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if d.cfg.GithubMirror.Enabled || len(d.cfg.GithubMirror.Mirrors) != 0 {
|
||||
t.Fatal("explicit disabled/empty mirrors lost")
|
||||
}
|
||||
value, err := d.readProtocolConfig("http")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
raw, _ := json.Marshal(value)
|
||||
var cfg struct {
|
||||
Connections int `json:"connections"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &cfg); err != nil || cfg.Connections != 3 {
|
||||
t.Fatalf("current protocol config: %s %v", raw, err)
|
||||
}
|
||||
if err := d.initProtocolConfigs(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProtocolConfigDoesNotImportOldExtraSettings(t *testing.T) {
|
||||
d := NewDownloader(&DownloaderConfig{StorageDir: t.TempDir(), TempDir: t.TempDir(), FetchManagers: []fetcher.FetcherManager{&btProtocol.FetcherManager{}}})
|
||||
d.cfg.DownloaderStoreConfig = &base.DownloaderStoreConfig{
|
||||
Extra: map[string]any{
|
||||
"githubMirror": "obsolete invalid value",
|
||||
"bt": map[string]any{"customTrackers": []string{"udp://old.example:80/announce"}, "autoUpdateTrackers": true},
|
||||
},
|
||||
ProtocolConfig: map[string]any{"bt": map[string]any{"autoUpdateTrackers": false, "listenPort": 6881}},
|
||||
}
|
||||
if err := d.storage.Setup([]string{bucketConfig}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer d.storage.Close()
|
||||
if err := d.initProtocolConfigs(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !d.cfg.GithubMirror.Enabled || len(d.cfg.GithubMirror.Mirrors) != 4 {
|
||||
t.Fatal("missing root mirror config must use new defaults")
|
||||
}
|
||||
value, err := d.readProtocolConfig("bt")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
raw, _ := json.Marshal(value)
|
||||
var cfg struct {
|
||||
ListenPort int `json:"listenPort"`
|
||||
AutoUpdateTrackers bool `json:"autoUpdateTrackers"`
|
||||
CustomTrackers []string `json:"customTrackers"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &cfg); err != nil || cfg.ListenPort != 6881 || cfg.AutoUpdateTrackers || len(cfg.CustomTrackers) != 0 {
|
||||
t.Fatalf("old extra settings affected protocol config: %s %v", raw, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProtocolJobsRunAutomaticallyAndRestoreSuccess(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
executed := make(chan fetcher.TriggerSource, 4)
|
||||
manager := func() *testJobManager {
|
||||
return &testJobManager{job: fetcher.CronJob{Name: "refresh", Every: time.Hour, Timeout: time.Second, Run: func(_ context.Context, source fetcher.TriggerSource) error { executed <- source; return nil }}}
|
||||
}
|
||||
makeDownloader := func() *Downloader {
|
||||
d := NewDownloader(&DownloaderConfig{Storage: NewBoltStorage(dir), StorageDir: dir, TempDir: t.TempDir(), FetchManagers: []fetcher.FetcherManager{manager()}})
|
||||
if err := d.Setup(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return d
|
||||
}
|
||||
d := makeDownloader()
|
||||
select {
|
||||
case source := <-executed:
|
||||
if source != fetcher.TriggerScheduled {
|
||||
t.Fatal(source)
|
||||
}
|
||||
case <-time.After(3 * time.Second):
|
||||
t.Fatal("overdue job was not run")
|
||||
}
|
||||
entry := d.protocolJobs.jobs["test/refresh"]
|
||||
entry.mu.Lock()
|
||||
run := entry.active
|
||||
entry.mu.Unlock()
|
||||
if run != nil {
|
||||
<-run.done
|
||||
}
|
||||
if err := d.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d = makeDownloader()
|
||||
defer d.Close()
|
||||
state := d.protocolJobs.jobs["test/refresh"].state
|
||||
if state.LastSuccess.IsZero() {
|
||||
t.Fatal("successful execution not restored")
|
||||
}
|
||||
select {
|
||||
case <-executed:
|
||||
t.Fatal("recently successful job ran again on restart")
|
||||
case <-time.After(30 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
@@ -1,193 +0,0 @@
|
||||
package download
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"path"
|
||||
"strings"
|
||||
"unicode"
|
||||
|
||||
"github.com/GopeedLab/gopeed/pkg/base"
|
||||
)
|
||||
|
||||
// categoryForResource returns the category that receives the resolved
|
||||
// resource, or nil when auto categorize does not apply. Only a single file
|
||||
// resource is routed: a folder or a multi file resource keeps its directory
|
||||
// layout, so its files are never matched one by one, not even when only a
|
||||
// single one of them is selected.
|
||||
func categoryForResource(cfg *base.DownloaderStoreConfig, res *base.Resource) *base.DownloadCategory {
|
||||
if !cfg.AutoCategorizeEnabled() || len(cfg.Categories) == 0 {
|
||||
return nil
|
||||
}
|
||||
if res == nil || res.Name != "" || len(res.Files) != 1 {
|
||||
return nil
|
||||
}
|
||||
return matchDownloadCategory(cfg, res.Files[0].Name)
|
||||
}
|
||||
|
||||
// decideDownloadDir returns the download directory of a task. The base
|
||||
// directory is the one the task asked for, or the configured download
|
||||
// directory when the task left it empty. A task that did not skip auto
|
||||
// categorize is routed to the category directory of its resolved resource
|
||||
// when one matches. Path placeholders are expanded and the white list check
|
||||
// runs on the result, so a category directory the white list rejects falls
|
||||
// back to the base directory of this task instead of failing it.
|
||||
func (d *Downloader) decideDownloadDir(opts *base.Options, res *base.Resource) (string, error) {
|
||||
return d.decideDownloadDirForTask(opts, res, nil)
|
||||
}
|
||||
|
||||
func (d *Downloader) decideDownloadDirForTask(opts *base.Options, res *base.Resource, group *TaskGroup) (string, error) {
|
||||
if opts == nil {
|
||||
opts = &base.Options{}
|
||||
}
|
||||
cfg, err := d.GetConfig()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
base := dartTrimSpace(opts.Path)
|
||||
if base == "" {
|
||||
base = cfg.DownloadDir
|
||||
}
|
||||
// Refuse to fall through to the process working directory when neither the
|
||||
// task nor the config names a directory.
|
||||
if base == "" {
|
||||
return "", errors.New("download directory is empty")
|
||||
}
|
||||
if group != nil && opts.SkipAutoCategorize {
|
||||
return d.initDownloadPathForTask(base, group)
|
||||
}
|
||||
target := base
|
||||
if !opts.SkipAutoCategorize {
|
||||
if category := categoryForResource(cfg, res); category != nil {
|
||||
target = dartTrimSpace(category.Path)
|
||||
}
|
||||
}
|
||||
dir, err := d.initDownloadPath(target)
|
||||
if err != nil && target != base {
|
||||
d.Logger.Warn().Err(err).Msgf("category directory %q rejected by white list, falling back to %q", target, base)
|
||||
dir, err = d.initDownloadPath(base)
|
||||
}
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return dir, nil
|
||||
}
|
||||
|
||||
// suggestedDownloadDir returns the category directory a task would be routed
|
||||
// to, or an empty string when auto categorize does not apply. It only reports
|
||||
// the decision of decideDownloadDir, it expands no directory and writes
|
||||
// nothing.
|
||||
func (d *Downloader) suggestedDownloadDir(res *base.Resource) string {
|
||||
cfg, err := d.GetConfig()
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
category := categoryForResource(cfg, res)
|
||||
if category == nil {
|
||||
return ""
|
||||
}
|
||||
dir, err := d.initDownloadPath(dartTrimSpace(category.Path))
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return dir
|
||||
}
|
||||
|
||||
// asciiLower lowercases the ascii letters of s and leaves the rest alone. The
|
||||
// ui lowercases the same way, because the unicode case tables of the two
|
||||
// languages disagree for some code points and would let the two sides pick a
|
||||
// different category for the same file.
|
||||
func asciiLower(s string) string {
|
||||
hasUpper := false
|
||||
for i := 0; i < len(s); i++ {
|
||||
if c := s[i]; c >= 'A' && c <= 'Z' {
|
||||
hasUpper = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !hasUpper {
|
||||
return s
|
||||
}
|
||||
b := []byte(s)
|
||||
for i, c := range b {
|
||||
if c >= 'A' && c <= 'Z' {
|
||||
b[i] = c + ('a' - 'A')
|
||||
}
|
||||
}
|
||||
return string(b)
|
||||
}
|
||||
|
||||
// matchDownloadCategory returns the first category whose extension list
|
||||
// contains the extension of the given file name, or nil when nothing matches.
|
||||
// An empty extension list never matches.
|
||||
func matchDownloadCategory(cfg *base.DownloaderStoreConfig, name string) *base.DownloadCategory {
|
||||
ext := fileExtension(name)
|
||||
if ext == "" {
|
||||
return nil
|
||||
}
|
||||
for _, category := range cfg.Categories {
|
||||
if category == nil || category.IsDeleted || dartTrimSpace(category.Path) == "" {
|
||||
continue
|
||||
}
|
||||
for _, e := range normalizeCategoryExtensions(category.Extensions) {
|
||||
if e == ext {
|
||||
return category
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// normalizeCategoryExtensions mirrors the dart side: entries are split on
|
||||
// commas and whitespace, trimmed, lowercased, stripped of leading dots and
|
||||
// deduplicated while keeping the input order, so a stored value like
|
||||
// "exe, msi" behaves the same in the backend as it does in the ui.
|
||||
func normalizeCategoryExtensions(raw []string) []string {
|
||||
var result []string
|
||||
seen := make(map[string]struct{}, len(raw))
|
||||
for _, item := range raw {
|
||||
for _, part := range strings.FieldsFunc(item, func(r rune) bool {
|
||||
return r == ',' || isECMAScriptSpace(r)
|
||||
}) {
|
||||
ext := asciiLower(strings.TrimLeft(dartTrimSpace(part), "."))
|
||||
if ext == "" {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[ext]; ok {
|
||||
continue
|
||||
}
|
||||
seen[ext] = struct{}{}
|
||||
result = append(result, ext)
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// fileExtension returns the lower case extension of a file name without the
|
||||
// leading dot, e.g. "setup.exe" -> "exe". Windows style separators are
|
||||
// treated like the ui does, so a dot inside a directory segment never counts
|
||||
// as an extension.
|
||||
func fileExtension(name string) string {
|
||||
name = strings.ReplaceAll(name, `\`, "/")
|
||||
return strings.TrimPrefix(asciiLower(path.Ext(name)), ".")
|
||||
}
|
||||
|
||||
// dartTrimSpace trims the whitespace the dart String.trim() removes, which is
|
||||
// the unicode white space set plus the byte order mark. The ui trims the
|
||||
// category paths and the extension lists with it, so the backend uses the same
|
||||
// set instead of the narrower unicode.IsSpace.
|
||||
func dartTrimSpace(s string) string {
|
||||
return strings.TrimFunc(s, func(r rune) bool {
|
||||
return unicode.IsSpace(r) || r == 0xFEFF
|
||||
})
|
||||
}
|
||||
|
||||
// isECMAScriptSpace reports whether r belongs to the whitespace class of the
|
||||
// ECMAScript regular expressions. The ui splits its extension lists with that
|
||||
// class, so the backend uses the same one instead of the wider unicode.IsSpace.
|
||||
func isECMAScriptSpace(r rune) bool {
|
||||
switch r {
|
||||
case '\t', '\n', '\v', '\f', '\r', ' ', 0x00A0, 0x1680, 0x2028, 0x2029, 0x202F, 0x205F, 0x3000, 0xFEFF:
|
||||
return true
|
||||
}
|
||||
return r >= 0x2000 && r <= 0x200A
|
||||
}
|
||||
@@ -1,566 +0,0 @@
|
||||
package download
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/GopeedLab/gopeed/pkg/base"
|
||||
"github.com/GopeedLab/gopeed/pkg/util"
|
||||
)
|
||||
|
||||
type failNextTaskPutStorage struct {
|
||||
Storage
|
||||
fail atomic.Bool
|
||||
failBucket string
|
||||
beforeTaskPut func(*Task)
|
||||
}
|
||||
|
||||
func (s *failNextTaskPutStorage) Put(bucket string, key string, value any) error {
|
||||
if bucket == bucketTask && s.beforeTaskPut != nil {
|
||||
s.beforeTaskPut(value.(*Task))
|
||||
}
|
||||
failBucket := s.failBucket
|
||||
if failBucket == "" {
|
||||
failBucket = bucketTask
|
||||
}
|
||||
if bucket == failBucket && (bucket != bucketTask || !value.(*Task).NeedsPreparation) && s.fail.CompareAndSwap(true, false) {
|
||||
return errors.New("task snapshot write failed")
|
||||
}
|
||||
return s.Storage.Put(bucket, key, value)
|
||||
}
|
||||
|
||||
func TestDownloader_PreparationSaveFailureCanRetry(t *testing.T) {
|
||||
for _, failBucket := range []string{bucketSave, bucketTask} {
|
||||
for _, restart := range []bool{false, true} {
|
||||
t.Run(fmt.Sprintf("%s/restart=%t", failBucket, restart), func(t *testing.T) {
|
||||
url := startCategoryFileServer(t, "setup.data", []byte("0123456789"))
|
||||
downloadDir := t.TempDir()
|
||||
storageDir := t.TempDir()
|
||||
storage := &failNextTaskPutStorage{Storage: NewBoltStorage(storageDir), failBucket: failBucket}
|
||||
downloader := NewDownloader(&DownloaderConfig{Storage: storage, StorageDir: storageDir})
|
||||
if err := downloader.Setup(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { downloader.Close() })
|
||||
categoryDir := putCategoryConfig(t, downloader, downloadDir, true, []string{"data"})
|
||||
if err := os.MkdirAll(categoryDir, 0755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(categoryDir, "setup.data"), []byte("existing"), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
failed := watchTaskError(t, downloader)
|
||||
storage.fail.Store(true)
|
||||
id, err := downloader.CreateDirect(&base.Request{URL: url}, &base.Options{Path: downloadDir})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waitTaskError(t, failed)
|
||||
task := downloader.GetTask(id)
|
||||
if !task.NeedsPreparation || task.Meta.Opts.Path != downloadDir || task.Meta.Opts.Name != "" {
|
||||
t.Fatal("failed preparation changed the original directory, name or flag")
|
||||
}
|
||||
if restart {
|
||||
if err := downloader.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
downloader = NewDownloader(&DownloaderConfig{Storage: NewBoltStorage(storageDir), StorageDir: storageDir})
|
||||
if err := downloader.Setup(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
done := watchTaskDone(t, downloader)
|
||||
if err := downloader.Continue(&TaskFilter{IDs: []string{id}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waitTaskDone(t, done)
|
||||
task = downloader.GetTask(id)
|
||||
if task.NeedsPreparation || task.Meta.Opts.Path != categoryDir || task.Meta.Opts.Name != "setup (1).data" {
|
||||
t.Fatalf("retry did not prepare the final directory and duplicate name: %+v", task.Meta.Opts)
|
||||
}
|
||||
data, err := os.ReadFile(filepath.Join(categoryDir, "setup (1).data"))
|
||||
if err != nil || string(data) != "0123456789" {
|
||||
t.Fatalf("downloaded content = %q, %v", data, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// watchTaskError subscribes to the error event, so a failing task can be
|
||||
// awaited instead of polled.
|
||||
func watchTaskError(t *testing.T, d *Downloader) <-chan struct{} {
|
||||
t.Helper()
|
||||
failed := make(chan struct{}, 1)
|
||||
d.Listener(func(event *Event) {
|
||||
if event.Key == EventKeyError {
|
||||
select {
|
||||
case failed <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
})
|
||||
return failed
|
||||
}
|
||||
|
||||
func waitTaskError(t *testing.T, failed <-chan struct{}) {
|
||||
t.Helper()
|
||||
select {
|
||||
case <-failed:
|
||||
case <-time.After(30 * time.Second):
|
||||
t.Fatal("task did not fail")
|
||||
}
|
||||
}
|
||||
|
||||
// waitTaskStatus waits until the task reaches one of the given statuses.
|
||||
func waitTaskStatus(t *testing.T, d *Downloader, task *Task, statuses ...base.Status) base.Status {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(30 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
status := d.taskStatus(task)
|
||||
for _, want := range statuses {
|
||||
if status == want {
|
||||
return status
|
||||
}
|
||||
}
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
}
|
||||
t.Fatalf("task status = %q, want one of %v", d.taskStatus(task), statuses)
|
||||
return ""
|
||||
}
|
||||
|
||||
// setMaxRunning limits the number of tasks that run at the same time.
|
||||
func setMaxRunning(t *testing.T, d *Downloader, max int) {
|
||||
t.Helper()
|
||||
cfg, err := d.GetConfig()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cfg.MaxRunning = max
|
||||
if err := d.PutConfig(cfg); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// A task that waits in the queue is prepared when it starts, the first
|
||||
// preparation must not be skipped by the wait status.
|
||||
func TestDownloader_QueuedTaskIsPreparedWhenItStarts(t *testing.T) {
|
||||
release := make(chan struct{})
|
||||
slow := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
<-release
|
||||
w.Header().Set("Content-Disposition", `attachment; filename="slow.bin"`)
|
||||
w.Write([]byte("slow"))
|
||||
}))
|
||||
defer slow.Close()
|
||||
fast := startCategoryFileServer(t, "setup.data", []byte("0123456789"))
|
||||
|
||||
downloadDir := t.TempDir()
|
||||
downloader := newSetupCategoryDownloader(t)
|
||||
categoryDir := putCategoryConfig(t, downloader, downloadDir, true, []string{"data"})
|
||||
setMaxRunning(t, downloader, 1)
|
||||
|
||||
// The blocker takes the only running slot.
|
||||
blockerID, err := downloader.CreateDirect(&base.Request{URL: slow.URL}, &base.Options{Path: downloadDir})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
blocker := downloader.GetTask(blockerID)
|
||||
if blocker == nil {
|
||||
t.Fatal("blocker task not found")
|
||||
}
|
||||
waitTaskStatus(t, downloader, blocker, base.DownloadStatusRunning)
|
||||
|
||||
queuedID, err := downloader.CreateDirect(&base.Request{URL: fast}, &base.Options{Path: downloadDir})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
queued := downloader.GetTask(queuedID)
|
||||
if queued == nil {
|
||||
t.Fatal("task not found")
|
||||
}
|
||||
if queued.Status != base.DownloadStatusWait {
|
||||
t.Fatalf("queued task status = %q, want wait", queued.Status)
|
||||
}
|
||||
|
||||
close(release)
|
||||
waitTaskStatus(t, downloader, queued, base.DownloadStatusDone, base.DownloadStatusError)
|
||||
|
||||
if queued.Meta.Opts.Path != categoryDir {
|
||||
t.Fatalf("queued task path = %q, want the category dir %q", queued.Meta.Opts.Path, categoryDir)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(categoryDir, "setup.data")); err != nil {
|
||||
t.Fatalf("categorized file missing: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// A task that failed before it was prepared is prepared when it continues.
|
||||
func TestDownloader_FailedTaskIsPreparedWhenItContinues(t *testing.T) {
|
||||
url := startCategoryFileServer(t, "setup.data", []byte("0123456789"))
|
||||
downloadDir := t.TempDir()
|
||||
allowedDir := t.TempDir()
|
||||
|
||||
downloader := newSetupCategoryDownloader(t)
|
||||
categoryDir := putCategoryConfig(t, downloader, downloadDir, true, []string{"data"})
|
||||
// The white list rejects the base directory, so the first start fails.
|
||||
downloader.cfg.WhiteDownloadDirs = []string{allowedDir}
|
||||
|
||||
failed := watchTaskError(t, downloader)
|
||||
taskID, err := downloader.CreateDirect(&base.Request{URL: url}, &base.Options{Path: downloadDir})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
task := downloader.GetTask(taskID)
|
||||
if task == nil {
|
||||
t.Fatal("task not found")
|
||||
}
|
||||
waitTaskError(t, failed)
|
||||
waitTaskStatus(t, downloader, task, base.DownloadStatusError)
|
||||
if !task.NeedsPreparation {
|
||||
t.Fatal("a failed preparation must keep the needs preparation flag")
|
||||
}
|
||||
|
||||
// The white list accepts the base directory now, the retry prepares the
|
||||
// task again.
|
||||
downloader.cfg.WhiteDownloadDirs = []string{allowedDir, downloadDir, categoryDir}
|
||||
done := watchTaskDone(t, downloader)
|
||||
if err := downloader.Continue(&TaskFilter{IDs: []string{taskID}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waitTaskDone(t, done)
|
||||
|
||||
if task.Meta.Opts.Path != categoryDir {
|
||||
t.Fatalf("retried task path = %q, want the category dir %q", task.Meta.Opts.Path, categoryDir)
|
||||
}
|
||||
if task.NeedsPreparation {
|
||||
t.Fatal("a prepared task must clear the needs preparation flag")
|
||||
}
|
||||
}
|
||||
|
||||
// A failed preparation save leaves the live task untouched, so the next
|
||||
// attempt starts from the original base directory and name.
|
||||
func TestDownloader_PreparationSaveFailureKeepsOriginalData(t *testing.T) {
|
||||
url := startCategoryFileServer(t, "setup.data", []byte("0123456789"))
|
||||
downloadDir := t.TempDir()
|
||||
storage := &failNextTaskPutStorage{Storage: NewMemStorage()}
|
||||
downloader := NewDownloader(&DownloaderConfig{Storage: storage})
|
||||
if err := downloader.Setup(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
downloader.Delete(nil, true)
|
||||
downloader.Clear()
|
||||
})
|
||||
categoryDir := putCategoryConfig(t, downloader, downloadDir, true, []string{"data"})
|
||||
setMaxRunning(t, downloader, 0)
|
||||
|
||||
result, err := downloader.Resolve(&base.Request{URL: url})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
taskID, err := downloader.CreateWithOptions(result.ID, &base.Options{
|
||||
Path: downloadDir,
|
||||
Name: "renamed?.data",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
task := downloader.GetTask(taskID)
|
||||
if task == nil {
|
||||
t.Fatal("task not found")
|
||||
}
|
||||
originalMeta := util.DeepClone(task.Meta)
|
||||
originalRequest := task.Meta.Req
|
||||
storage.beforeTaskPut = func(snapshot *Task) {
|
||||
if !task.NeedsPreparation || !reflect.DeepEqual(task.Meta, originalMeta) {
|
||||
t.Error("preparation was published before the snapshot was saved")
|
||||
}
|
||||
if snapshot.NeedsPreparation || snapshot.Meta.Opts.Path != categoryDir {
|
||||
t.Error("the snapshot must contain the prepared directory and cleared flag")
|
||||
}
|
||||
}
|
||||
|
||||
storage.fail.Store(true)
|
||||
if err := downloader.prepareTask(task); err == nil {
|
||||
t.Fatal("prepareTask() succeeded, want a storage error")
|
||||
}
|
||||
if !task.NeedsPreparation {
|
||||
t.Fatal("a failed save must keep the needs preparation flag")
|
||||
}
|
||||
if !reflect.DeepEqual(task.Meta, originalMeta) {
|
||||
t.Fatalf("task metadata changed after a failed save:\n got: %#v\nwant: %#v", task.Meta, originalMeta)
|
||||
}
|
||||
if !reflect.DeepEqual(task.fetcher.Meta(), originalMeta) {
|
||||
t.Fatalf("fetcher metadata changed after a failed save:\n got: %#v\nwant: %#v", task.fetcher.Meta(), originalMeta)
|
||||
}
|
||||
|
||||
if err := downloader.prepareTask(task); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if task.NeedsPreparation {
|
||||
t.Fatal("a successful retry must clear the needs preparation flag")
|
||||
}
|
||||
if task.Meta.Opts.Path != categoryDir {
|
||||
t.Fatalf("retried task path = %q, want %q", task.Meta.Opts.Path, categoryDir)
|
||||
}
|
||||
if task.Meta != task.fetcher.Meta() || task.Meta.Req != originalRequest {
|
||||
t.Fatal("preparation must preserve the shared metadata and resolved request")
|
||||
}
|
||||
storage.beforeTaskPut = nil
|
||||
task.Meta.Res.Size++
|
||||
var saved Task
|
||||
if _, err := storage.Get(bucketTask, taskID, &saved); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if saved.Meta.Res.Size != originalMeta.Res.Size {
|
||||
t.Fatal("live metadata changes must not alter the saved task snapshot")
|
||||
}
|
||||
}
|
||||
|
||||
// The white list is checked on every start, a task that was rejected once
|
||||
// keeps failing instead of writing into the rejected directory.
|
||||
func TestDownloader_WhiteListIsCheckedOnEveryStart(t *testing.T) {
|
||||
url := startCategoryFileServer(t, "setup.data", []byte("0123456789"))
|
||||
downloadDir := t.TempDir()
|
||||
allowedDir := t.TempDir()
|
||||
|
||||
downloader := newSetupCategoryDownloader(t)
|
||||
putCategoryConfig(t, downloader, downloadDir, true, []string{"data"})
|
||||
downloader.cfg.WhiteDownloadDirs = []string{allowedDir}
|
||||
|
||||
failed := watchTaskError(t, downloader)
|
||||
taskID, err := downloader.CreateDirect(&base.Request{URL: url}, &base.Options{Path: downloadDir})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
task := downloader.GetTask(taskID)
|
||||
if task == nil {
|
||||
t.Fatal("task not found")
|
||||
}
|
||||
waitTaskError(t, failed)
|
||||
waitTaskStatus(t, downloader, task, base.DownloadStatusError)
|
||||
|
||||
// The task is continued, the white list still rejects the directory.
|
||||
failed = watchTaskError(t, downloader)
|
||||
if err := downloader.Continue(&TaskFilter{IDs: []string{taskID}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waitTaskError(t, failed)
|
||||
waitTaskStatus(t, downloader, task, base.DownloadStatusError)
|
||||
|
||||
if _, err := os.Stat(filepath.Join(downloadDir, "setup.data")); err == nil {
|
||||
t.Fatalf("the file was written into the rejected dir %q", downloadDir)
|
||||
}
|
||||
}
|
||||
|
||||
// The white list is checked on every start, even for a task that was prepared
|
||||
// before the white list changed.
|
||||
func TestDownloader_WhiteListIsCheckedForAPreparedTask(t *testing.T) {
|
||||
// The first request is truncated, so the download fails after the task was
|
||||
// prepared and the retry can show that the directory is checked again.
|
||||
var requests int32
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Disposition", `attachment; filename="setup.data"`)
|
||||
w.Header().Set("Content-Length", "8")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
if atomic.AddInt32(&requests, 1) == 1 {
|
||||
w.Write([]byte("0123"))
|
||||
return
|
||||
}
|
||||
w.Write([]byte("01234567"))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
downloadDir := t.TempDir()
|
||||
allowedDir := t.TempDir()
|
||||
downloader := newSetupCategoryDownloader(t)
|
||||
categoryDir := putCategoryConfig(t, downloader, downloadDir, true, []string{"data"})
|
||||
downloader.cfg.WhiteDownloadDirs = []string{allowedDir, downloadDir, categoryDir}
|
||||
|
||||
failed := watchTaskError(t, downloader)
|
||||
taskID, err := downloader.CreateDirect(&base.Request{URL: server.URL}, &base.Options{Path: downloadDir})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
task := downloader.GetTask(taskID)
|
||||
if task == nil {
|
||||
t.Fatal("task not found")
|
||||
}
|
||||
waitTaskError(t, failed)
|
||||
waitTaskStatus(t, downloader, task, base.DownloadStatusError)
|
||||
if task.Meta.Opts.Path != categoryDir {
|
||||
t.Fatalf("task path = %q, want the category dir %q", task.Meta.Opts.Path, categoryDir)
|
||||
}
|
||||
|
||||
// The white list no longer accepts the prepared directory, the task must
|
||||
// not download into it.
|
||||
downloader.cfg.WhiteDownloadDirs = []string{allowedDir}
|
||||
requestsBeforeRetry := atomic.LoadInt32(&requests)
|
||||
failed = watchTaskError(t, downloader)
|
||||
if err := downloader.Continue(&TaskFilter{IDs: []string{taskID}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waitTaskError(t, failed)
|
||||
waitTaskStatus(t, downloader, task, base.DownloadStatusError)
|
||||
|
||||
if atomic.LoadInt32(&requests) != requestsBeforeRetry {
|
||||
t.Fatalf("the task downloaded into the rejected dir %q", categoryDir)
|
||||
}
|
||||
}
|
||||
|
||||
// A prepared task keeps its directory, a later start never classifies it
|
||||
// again.
|
||||
func TestDownloader_PreparedTaskKeepsItsDirectory(t *testing.T) {
|
||||
// The first request is truncated, so the download fails after the task was
|
||||
// prepared and the retry can show that the directory is kept.
|
||||
var requests int32
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Disposition", `attachment; filename="setup.data"`)
|
||||
w.Header().Set("Content-Length", "8")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
if atomic.AddInt32(&requests, 1) == 1 {
|
||||
w.Write([]byte("0123"))
|
||||
return
|
||||
}
|
||||
w.Write([]byte("01234567"))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
downloadDir := t.TempDir()
|
||||
downloader := newSetupCategoryDownloader(t)
|
||||
categoryDir := putCategoryConfig(t, downloader, downloadDir, true, []string{"data"})
|
||||
|
||||
failed := watchTaskError(t, downloader)
|
||||
taskID, err := downloader.CreateDirect(&base.Request{URL: server.URL}, &base.Options{Path: downloadDir})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
task := downloader.GetTask(taskID)
|
||||
if task == nil {
|
||||
t.Fatal("task not found")
|
||||
}
|
||||
waitTaskError(t, failed)
|
||||
waitTaskStatus(t, downloader, task, base.DownloadStatusError)
|
||||
if task.Meta.Opts.Path != categoryDir {
|
||||
t.Fatalf("task path = %q, want the category dir %q", task.Meta.Opts.Path, categoryDir)
|
||||
}
|
||||
if task.NeedsPreparation {
|
||||
t.Fatal("a prepared task must clear the needs preparation flag")
|
||||
}
|
||||
|
||||
// The category no longer matches the file, the prepared task keeps the
|
||||
// directory it was prepared with.
|
||||
putCategoryConfig(t, downloader, downloadDir, true, []string{"iso"})
|
||||
done := watchTaskDone(t, downloader)
|
||||
if err := downloader.Continue(&TaskFilter{IDs: []string{taskID}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waitTaskDone(t, done)
|
||||
|
||||
if task.Meta.Opts.Path != categoryDir {
|
||||
t.Fatalf("task path = %q, want the prepared dir %q", task.Meta.Opts.Path, categoryDir)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(categoryDir, "setup.data")); err != nil {
|
||||
t.Fatalf("the file was not written into the prepared dir: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// The global default download directory is only saved when the user named the
|
||||
// directory explicitly, an auto categorized task never becomes the default.
|
||||
func TestDownloader_DefaultPathNeedsTheSkipFlag(t *testing.T) {
|
||||
url := startCategoryFileServer(t, "setup.data", []byte("0123456789"))
|
||||
downloadDir := t.TempDir()
|
||||
chosenDir := t.TempDir()
|
||||
downloader := newSetupCategoryDownloader(t)
|
||||
putCategoryConfig(t, downloader, downloadDir, true, []string{"data"})
|
||||
done := watchTaskDone(t, downloader)
|
||||
|
||||
// The skip flag is missing, the task keeps the auto categorize decision
|
||||
// and the global default stays untouched.
|
||||
if _, err := downloader.CreateDirect(&base.Request{URL: url}, &base.Options{
|
||||
Path: chosenDir,
|
||||
AsDefaultPath: true,
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waitTaskDone(t, done)
|
||||
cfg, err := downloader.GetConfig()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if cfg.DownloadDir == chosenDir {
|
||||
t.Fatalf("default download dir = %q, want it unchanged", cfg.DownloadDir)
|
||||
}
|
||||
|
||||
// The user named the directory, so it becomes the global default.
|
||||
if _, err := downloader.CreateDirect(&base.Request{URL: url}, &base.Options{
|
||||
Path: chosenDir,
|
||||
AsDefaultPath: true,
|
||||
SkipAutoCategorize: true,
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waitTaskDone(t, done)
|
||||
cfg, err = downloader.GetConfig()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if cfg.DownloadDir != chosenDir {
|
||||
t.Fatalf("default download dir = %q, want %q", cfg.DownloadDir, chosenDir)
|
||||
}
|
||||
}
|
||||
|
||||
// Every task of a batch decides its own directory, the shared options of the
|
||||
// batch do not leak into the single tasks.
|
||||
func TestDownloader_BatchTasksDecideTheirOwnDirectory(t *testing.T) {
|
||||
url := startCategoryFileServer(t, "setup.data", []byte("0123456789"))
|
||||
downloadDir := t.TempDir()
|
||||
chosenDir := t.TempDir()
|
||||
downloader := newSetupCategoryDownloader(t)
|
||||
categoryDir := putCategoryConfig(t, downloader, downloadDir, true, []string{"data"})
|
||||
done := watchTaskDone(t, downloader)
|
||||
|
||||
taskIDs, err := downloader.CreateDirectBatch(&base.CreateTaskBatch{
|
||||
Reqs: []*base.CreateTaskBatchItem{
|
||||
{
|
||||
Req: &base.Request{URL: url},
|
||||
Opts: &base.Options{Path: chosenDir, SkipAutoCategorize: true},
|
||||
},
|
||||
{
|
||||
Req: &base.Request{URL: url},
|
||||
Opts: &base.Options{Path: downloadDir},
|
||||
},
|
||||
},
|
||||
Opts: &base.Options{Path: downloadDir},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(taskIDs) != 2 {
|
||||
t.Fatalf("task count = %d, want 2", len(taskIDs))
|
||||
}
|
||||
waitTaskDone(t, done)
|
||||
waitTaskDone(t, done)
|
||||
|
||||
skipped := downloader.GetTask(taskIDs[0])
|
||||
if skipped == nil {
|
||||
t.Fatal("task not found")
|
||||
}
|
||||
if skipped.Meta.Opts.Path != chosenDir {
|
||||
t.Fatalf("skipped task path = %q, want %q", skipped.Meta.Opts.Path, chosenDir)
|
||||
}
|
||||
categorized := downloader.GetTask(taskIDs[1])
|
||||
if categorized == nil {
|
||||
t.Fatal("task not found")
|
||||
}
|
||||
if categorized.Meta.Opts.Path != categoryDir {
|
||||
t.Fatalf("categorized task path = %q, want %q", categorized.Meta.Opts.Path, categoryDir)
|
||||
}
|
||||
}
|
||||
@@ -1,727 +0,0 @@
|
||||
package download
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/GopeedLab/gopeed/pkg/base"
|
||||
"github.com/GopeedLab/gopeed/pkg/util"
|
||||
)
|
||||
|
||||
func newTestCategoryConfig() *base.DownloaderStoreConfig {
|
||||
return &base.DownloaderStoreConfig{
|
||||
DownloadDir: "./downloads",
|
||||
AutoCategorize: util.BoolPtr(true),
|
||||
Categories: []*base.DownloadCategory{
|
||||
{Name: "Music", Path: filepath.Join("./downloads", "Music"), Extensions: []string{"mp3", "flac"}},
|
||||
{Name: "Video", Path: filepath.Join("./downloads", "Video"), Extensions: []string{"mp4", "mkv"}},
|
||||
{Name: "Program", Path: filepath.Join("./downloads", "Program"), Extensions: []string{"exe", "msi"}},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// singleFileResource is the resource shape of a one file task: no folder name
|
||||
// and a single file entry.
|
||||
func singleFileResource(name string) *base.Resource {
|
||||
return &base.Resource{Files: []*base.FileInfo{{Name: name, Size: 1}}}
|
||||
}
|
||||
|
||||
// folderResource is the resource shape of a multi file task: a folder name and
|
||||
// one entry per file, even when only one of them is selected later.
|
||||
func folderResource(name string, files ...string) *base.Resource {
|
||||
res := &base.Resource{Name: name}
|
||||
for _, file := range files {
|
||||
res.Files = append(res.Files, &base.FileInfo{Name: file, Size: 1})
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
||||
func newUnitCategoryDownloader(cfg *base.DownloaderStoreConfig) *Downloader {
|
||||
d := NewDownloader(nil)
|
||||
d.cfg.DownloaderStoreConfig = cfg
|
||||
return d
|
||||
}
|
||||
|
||||
func TestDecideDownloadDir(t *testing.T) {
|
||||
programDir := filepath.Join("./downloads", "Program")
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
cfg func(cfg *base.DownloaderStoreConfig)
|
||||
opts *base.Options
|
||||
res *base.Resource
|
||||
want string
|
||||
}{
|
||||
{
|
||||
name: "matching single file routes to the category",
|
||||
opts: &base.Options{},
|
||||
res: singleFileResource("setup.exe"),
|
||||
want: programDir,
|
||||
},
|
||||
{
|
||||
name: "an explicit base directory is routed too",
|
||||
opts: &base.Options{Path: "./elsewhere"},
|
||||
res: singleFileResource("setup.exe"),
|
||||
want: programDir,
|
||||
},
|
||||
{
|
||||
name: "skip flag keeps the configured download directory",
|
||||
opts: &base.Options{Path: "./downloads", SkipAutoCategorize: true},
|
||||
res: singleFileResource("setup.exe"),
|
||||
want: "./downloads",
|
||||
},
|
||||
{
|
||||
name: "skip flag keeps an explicit directory",
|
||||
opts: &base.Options{Path: "./elsewhere", SkipAutoCategorize: true},
|
||||
res: singleFileResource("setup.exe"),
|
||||
want: "./elsewhere",
|
||||
},
|
||||
{
|
||||
name: "switch off keeps the base directory",
|
||||
cfg: func(cfg *base.DownloaderStoreConfig) { cfg.AutoCategorize = util.BoolPtr(false) },
|
||||
opts: &base.Options{},
|
||||
res: singleFileResource("setup.exe"),
|
||||
want: "./downloads",
|
||||
},
|
||||
{
|
||||
name: "no categories keeps the base directory",
|
||||
cfg: func(cfg *base.DownloaderStoreConfig) { cfg.Categories = nil },
|
||||
opts: &base.Options{},
|
||||
res: singleFileResource("setup.exe"),
|
||||
want: "./downloads",
|
||||
},
|
||||
{
|
||||
name: "unknown extension keeps the base directory",
|
||||
opts: &base.Options{},
|
||||
res: singleFileResource("archive.zip"),
|
||||
want: "./downloads",
|
||||
},
|
||||
{
|
||||
name: "a name without an extension keeps the base directory",
|
||||
opts: &base.Options{},
|
||||
res: singleFileResource("README"),
|
||||
want: "./downloads",
|
||||
},
|
||||
{
|
||||
name: "a deleted category is skipped",
|
||||
cfg: func(cfg *base.DownloaderStoreConfig) {
|
||||
cfg.Categories = []*base.DownloadCategory{
|
||||
{Name: "Program", Path: programDir, Extensions: []string{"exe"}, IsDeleted: true},
|
||||
}
|
||||
},
|
||||
opts: &base.Options{},
|
||||
res: singleFileResource("setup.exe"),
|
||||
want: "./downloads",
|
||||
},
|
||||
{
|
||||
name: "a category without a path is skipped",
|
||||
cfg: func(cfg *base.DownloaderStoreConfig) {
|
||||
cfg.Categories = []*base.DownloadCategory{
|
||||
{Name: "Program", Path: " ", Extensions: []string{"exe"}},
|
||||
}
|
||||
},
|
||||
opts: &base.Options{},
|
||||
res: singleFileResource("setup.exe"),
|
||||
want: "./downloads",
|
||||
},
|
||||
{
|
||||
name: "a multi file resource keeps its layout",
|
||||
opts: &base.Options{},
|
||||
res: folderResource("bundle", "setup.exe", "readme.txt"),
|
||||
want: "./downloads",
|
||||
},
|
||||
{
|
||||
name: "a multi file resource with one match still keeps its layout",
|
||||
opts: &base.Options{SelectFiles: []int{0}},
|
||||
res: folderResource("bundle", "setup.exe", "readme.txt"),
|
||||
want: "./downloads",
|
||||
},
|
||||
{
|
||||
name: "a single file inside a folder resource is not routed",
|
||||
opts: &base.Options{},
|
||||
res: folderResource("bundle", "setup.exe"),
|
||||
want: "./downloads",
|
||||
},
|
||||
{
|
||||
name: "a missing resource keeps the base directory",
|
||||
opts: &base.Options{},
|
||||
res: nil,
|
||||
want: "./downloads",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
cfg := newTestCategoryConfig()
|
||||
if tt.cfg != nil {
|
||||
tt.cfg(cfg)
|
||||
}
|
||||
d := newUnitCategoryDownloader(cfg)
|
||||
got, err := d.decideDownloadDir(tt.opts, tt.res)
|
||||
if err != nil {
|
||||
t.Fatalf("decideDownloadDir() error = %v", err)
|
||||
}
|
||||
if got != tt.want {
|
||||
t.Fatalf("decideDownloadDir() = %q, want %q", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecideDownloadDirWhiteList(t *testing.T) {
|
||||
defaultDir := filepath.ToSlash(t.TempDir())
|
||||
categoryDir := defaultDir + "/Program"
|
||||
newConfig := func() *base.DownloaderStoreConfig {
|
||||
cfg := newTestCategoryConfig()
|
||||
cfg.DownloadDir = defaultDir
|
||||
cfg.Categories = []*base.DownloadCategory{
|
||||
{Name: "Program", Path: categoryDir, Extensions: []string{"exe"}},
|
||||
}
|
||||
return cfg
|
||||
}
|
||||
|
||||
t.Run("a rejected category directory falls back to the base directory", func(t *testing.T) {
|
||||
d := newUnitCategoryDownloader(newConfig())
|
||||
d.cfg.WhiteDownloadDirs = []string{defaultDir}
|
||||
|
||||
got, err := d.decideDownloadDir(&base.Options{Path: defaultDir}, singleFileResource("setup.exe"))
|
||||
if err != nil {
|
||||
t.Fatalf("expected white list fallback, got %v", err)
|
||||
}
|
||||
if got != defaultDir {
|
||||
t.Fatalf("path = %q, want the base directory %q", got, defaultDir)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("a rejected base directory is an error", func(t *testing.T) {
|
||||
d := newUnitCategoryDownloader(newConfig())
|
||||
d.cfg.WhiteDownloadDirs = []string{defaultDir}
|
||||
|
||||
_, err := d.decideDownloadDir(&base.Options{Path: defaultDir + "/Other", SkipAutoCategorize: true}, singleFileResource("setup.exe"))
|
||||
if err == nil {
|
||||
t.Fatal("expected a white list error for an explicit directory")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("equivalent category paths still fall back to the allowed spelling", func(t *testing.T) {
|
||||
cfg := newConfig()
|
||||
cfg.Categories[0].Path = defaultDir + "/."
|
||||
d := newUnitCategoryDownloader(cfg)
|
||||
d.cfg.WhiteDownloadDirs = []string{defaultDir}
|
||||
got, err := d.decideDownloadDir(&base.Options{Path: defaultDir}, singleFileResource("setup.exe"))
|
||||
if err != nil || got != defaultDir {
|
||||
t.Fatalf("decideDownloadDir() = %q, %v; want %q, nil", got, err, defaultDir)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("an empty base directory is an error", func(t *testing.T) {
|
||||
cfg := newConfig()
|
||||
cfg.DownloadDir = ""
|
||||
d := newUnitCategoryDownloader(cfg)
|
||||
|
||||
_, err := d.decideDownloadDir(&base.Options{SkipAutoCategorize: true}, singleFileResource("setup.exe"))
|
||||
if err == nil {
|
||||
t.Fatal("expected an error for an empty download directory")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestDecideGroupDownloadDirWhiteList(t *testing.T) {
|
||||
parent := t.TempDir()
|
||||
root := filepath.Join(parent, "folder")
|
||||
nested := filepath.Join(root, "nested")
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
path string
|
||||
group *TaskGroup
|
||||
allowed string
|
||||
wantErr bool
|
||||
}{
|
||||
{name: "group root", path: root, group: &TaskGroup{Path: root}, allowed: filepath.Join(parent, "*")},
|
||||
{name: "nested member", path: nested, group: &TaskGroup{Path: root}, allowed: filepath.Join(parent, "*")},
|
||||
{name: "ordinary nested task", path: nested, allowed: filepath.Join(parent, "*"), wantErr: true},
|
||||
{name: "member outside group", path: filepath.Join(parent, "other"), group: &TaskGroup{Path: root}, allowed: filepath.Join(parent, "*"), wantErr: true},
|
||||
{name: "group permission revoked", path: nested, group: &TaskGroup{Path: root}, allowed: nested, wantErr: true},
|
||||
{name: "literal group name", path: filepath.Join(parent, "%year%", "nested"), group: &TaskGroup{Path: filepath.Join(parent, "%year%")}, allowed: filepath.Join(parent, "*")},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
d := newUnitCategoryDownloader(newTestCategoryConfig())
|
||||
d.cfg.WhiteDownloadDirs = []string{test.allowed}
|
||||
got, err := d.decideDownloadDirForTask(&base.Options{Path: test.path, SkipAutoCategorize: true}, singleFileResource("file.txt"), test.group)
|
||||
if (err != nil) != test.wantErr {
|
||||
t.Fatalf("decideDownloadDirForTask() = %q, %v; want error = %v", got, err, test.wantErr)
|
||||
}
|
||||
if err == nil && got != test.path {
|
||||
t.Fatalf("path = %q, want %q", got, test.path)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSuggestedDownloadDir(t *testing.T) {
|
||||
programDir := filepath.Join("./downloads", "Program")
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
cfg func(cfg *base.DownloaderStoreConfig)
|
||||
res *base.Resource
|
||||
want string
|
||||
}{
|
||||
{name: "a matching single file is suggested", res: singleFileResource("setup.exe"), want: programDir},
|
||||
{
|
||||
name: "a padded category path matches the creation decision",
|
||||
cfg: func(cfg *base.DownloaderStoreConfig) { cfg.Categories[2].Path = " " + programDir + " " },
|
||||
res: singleFileResource("setup.exe"),
|
||||
want: programDir,
|
||||
},
|
||||
{
|
||||
name: "the switch off suggests nothing",
|
||||
cfg: func(cfg *base.DownloaderStoreConfig) { cfg.AutoCategorize = util.BoolPtr(false) },
|
||||
res: singleFileResource("setup.exe"),
|
||||
want: "",
|
||||
},
|
||||
{name: "a multi file resource suggests nothing", res: folderResource("bundle", "setup.exe"), want: ""},
|
||||
{name: "a folder resource suggests nothing", res: folderResource("bundle", "setup.exe", "a.txt"), want: ""},
|
||||
{name: "an unknown extension suggests nothing", res: singleFileResource("archive.zip"), want: ""},
|
||||
{name: "a missing resource suggests nothing", res: nil, want: ""},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
cfg := newTestCategoryConfig()
|
||||
if tt.cfg != nil {
|
||||
tt.cfg(cfg)
|
||||
}
|
||||
d := newUnitCategoryDownloader(cfg)
|
||||
if got := d.suggestedDownloadDir(tt.res); got != tt.want {
|
||||
t.Fatalf("suggestedDownloadDir() = %q, want %q", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
t.Run("a rejected category directory suggests nothing", func(t *testing.T) {
|
||||
dir := filepath.ToSlash(t.TempDir())
|
||||
cfg := newTestCategoryConfig()
|
||||
cfg.DownloadDir = dir
|
||||
cfg.Categories = []*base.DownloadCategory{
|
||||
{Name: "Program", Path: dir + "/Program", Extensions: []string{"exe"}},
|
||||
}
|
||||
d := newUnitCategoryDownloader(cfg)
|
||||
d.cfg.WhiteDownloadDirs = []string{dir}
|
||||
|
||||
if got := d.suggestedDownloadDir(singleFileResource("setup.exe")); got != "" {
|
||||
t.Fatalf("suggestedDownloadDir() = %q, want an empty suggestion", got)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestNormalizeCategoryExtensions(t *testing.T) {
|
||||
tests := []struct {
|
||||
raw []string
|
||||
want []string
|
||||
}{
|
||||
{[]string{"exe, msi"}, []string{"exe", "msi"}},
|
||||
{[]string{" mp3 ", "..exe", "MUSIC"}, []string{"mp3", "exe", "music"}},
|
||||
{[]string{"zip", "ZIP", "zip"}, []string{"zip"}},
|
||||
{[]string{" ", "..", ""}, nil},
|
||||
{nil, nil},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
got := normalizeCategoryExtensions(tt.raw)
|
||||
if len(got) != len(tt.want) {
|
||||
t.Fatalf("normalizeCategoryExtensions(%q) = %q, want %q", tt.raw, got, tt.want)
|
||||
}
|
||||
for i := range got {
|
||||
if got[i] != tt.want[i] {
|
||||
t.Fatalf("normalizeCategoryExtensions(%q) = %q, want %q", tt.raw, got, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// startCategoryFileServer serves a nameless url whose real file name only
|
||||
// shows up in the Content-Disposition header.
|
||||
func startCategoryFileServer(t *testing.T, fileName string, data []byte) string {
|
||||
t.Helper()
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/download", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Disposition", `attachment; filename="`+fileName+`"`)
|
||||
w.Header().Set("Content-Type", "application/octet-stream")
|
||||
http.ServeContent(w, r, fileName, time.Time{}, bytes.NewReader(data))
|
||||
})
|
||||
server := httptest.NewServer(mux)
|
||||
t.Cleanup(server.Close)
|
||||
return server.URL + "/download"
|
||||
}
|
||||
|
||||
func newSetupCategoryDownloader(t *testing.T) *Downloader {
|
||||
t.Helper()
|
||||
downloader := NewDownloader(nil)
|
||||
if err := downloader.Setup(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
downloader.Delete(nil, true)
|
||||
downloader.Clear()
|
||||
})
|
||||
return downloader
|
||||
}
|
||||
|
||||
func putCategoryConfig(t *testing.T, d *Downloader, downloadDir string, autoCategorize bool, extensions []string) string {
|
||||
t.Helper()
|
||||
categoryDir := filepath.Join(downloadDir, "Program")
|
||||
cfg, err := d.GetConfig()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cfg.DownloadDir = downloadDir
|
||||
cfg.AutoCategorize = util.BoolPtr(autoCategorize)
|
||||
cfg.Categories = []*base.DownloadCategory{
|
||||
{Name: "Program", Path: categoryDir, Extensions: extensions},
|
||||
}
|
||||
if err := d.PutConfig(cfg); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return categoryDir
|
||||
}
|
||||
|
||||
// watchTaskDone subscribes to the done event before the task is created, so a
|
||||
// fast task can not finish before the test starts listening.
|
||||
func watchTaskDone(t *testing.T, d *Downloader) <-chan struct{} {
|
||||
t.Helper()
|
||||
done := make(chan struct{}, 1)
|
||||
d.Listener(func(event *Event) {
|
||||
if event.Key == EventKeyDone {
|
||||
select {
|
||||
case done <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
})
|
||||
return done
|
||||
}
|
||||
|
||||
func waitTaskDone(t *testing.T, done <-chan struct{}) {
|
||||
t.Helper()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(30 * time.Second):
|
||||
t.Fatal("task did not finish")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloader_CreateDirectAutoCategorize(t *testing.T) {
|
||||
// The url carries no extension, the category is matched on the file name
|
||||
// of the resolved resource.
|
||||
url := startCategoryFileServer(t, "setup.data", []byte("0123456789"))
|
||||
downloadDir := t.TempDir()
|
||||
downloader := newSetupCategoryDownloader(t)
|
||||
categoryDir := putCategoryConfig(t, downloader, downloadDir, true, []string{"data"})
|
||||
done := watchTaskDone(t, downloader)
|
||||
|
||||
taskID, err := downloader.CreateDirect(&base.Request{URL: url}, &base.Options{Path: downloadDir})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
task := downloader.GetTask(taskID)
|
||||
if task == nil {
|
||||
t.Fatal("task not found")
|
||||
}
|
||||
waitTaskDone(t, done)
|
||||
if task.Meta.Opts.Path != categoryDir {
|
||||
t.Fatalf("task path = %q, want the category dir %q", task.Meta.Opts.Path, categoryDir)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(categoryDir, "setup.data")); err != nil {
|
||||
t.Fatalf("categorized file missing: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloader_CreateDirectDoesNotMatchTheRename(t *testing.T) {
|
||||
url := startCategoryFileServer(t, "setup.data", []byte("0123456789"))
|
||||
downloadDir := t.TempDir()
|
||||
downloader := newSetupCategoryDownloader(t)
|
||||
putCategoryConfig(t, downloader, downloadDir, true, []string{"exe"})
|
||||
done := watchTaskDone(t, downloader)
|
||||
|
||||
// opts.Name only renames the saved file, it never decides the category.
|
||||
taskID, err := downloader.CreateDirect(&base.Request{URL: url}, &base.Options{
|
||||
Path: downloadDir,
|
||||
Name: "renamed.exe",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
task := downloader.GetTask(taskID)
|
||||
if task == nil {
|
||||
t.Fatal("task not found")
|
||||
}
|
||||
waitTaskDone(t, done)
|
||||
if task.Meta.Opts.Path != downloadDir {
|
||||
t.Fatalf("task path = %q, want the base dir %q", task.Meta.Opts.Path, downloadDir)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(downloadDir, "renamed.exe")); err != nil {
|
||||
t.Fatalf("renamed file missing: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloader_CreateDirectSkipAutoCategorize(t *testing.T) {
|
||||
url := startCategoryFileServer(t, "setup.data", []byte("0123456789"))
|
||||
downloadDir := t.TempDir()
|
||||
chosenDir := t.TempDir()
|
||||
downloader := newSetupCategoryDownloader(t)
|
||||
putCategoryConfig(t, downloader, downloadDir, true, []string{"data"})
|
||||
done := watchTaskDone(t, downloader)
|
||||
|
||||
taskID, err := downloader.CreateDirect(&base.Request{URL: url}, &base.Options{
|
||||
Path: chosenDir,
|
||||
SkipAutoCategorize: true,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
task := downloader.GetTask(taskID)
|
||||
if task == nil {
|
||||
t.Fatal("task not found")
|
||||
}
|
||||
waitTaskDone(t, done)
|
||||
if task.Meta.Opts.Path != chosenDir {
|
||||
t.Fatalf("task path = %q, want the chosen dir %q", task.Meta.Opts.Path, chosenDir)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(chosenDir, "setup.data")); err != nil {
|
||||
t.Fatalf("file missing in the chosen dir: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloader_CreateWithOptionsAutoCategorize(t *testing.T) {
|
||||
url := startCategoryFileServer(t, "setup.data", []byte("0123456789"))
|
||||
downloadDir := t.TempDir()
|
||||
downloader := newSetupCategoryDownloader(t)
|
||||
categoryDir := putCategoryConfig(t, downloader, downloadDir, true, []string{"data"})
|
||||
done := watchTaskDone(t, downloader)
|
||||
|
||||
rr, err := downloader.Resolve(&base.Request{URL: url})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if rr.SuggestedDownloadDir != categoryDir {
|
||||
t.Fatalf("suggested dir = %q, want %q", rr.SuggestedDownloadDir, categoryDir)
|
||||
}
|
||||
taskID, err := downloader.CreateWithOptions(rr.ID, &base.Options{Path: downloadDir})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
task := downloader.GetTask(taskID)
|
||||
if task == nil {
|
||||
t.Fatal("task not found")
|
||||
}
|
||||
waitTaskDone(t, done)
|
||||
if task.Meta.Opts.Path != categoryDir {
|
||||
t.Fatalf("task path = %q, want the category dir %q", task.Meta.Opts.Path, categoryDir)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloader_CreateWithOptionsKeepsTheDefaultDirectory(t *testing.T) {
|
||||
url := startCategoryFileServer(t, "setup.data", []byte("0123456789"))
|
||||
downloadDir := t.TempDir()
|
||||
downloader := newSetupCategoryDownloader(t)
|
||||
putCategoryConfig(t, downloader, downloadDir, true, []string{"data"})
|
||||
done := watchTaskDone(t, downloader)
|
||||
|
||||
rr, err := downloader.Resolve(&base.Request{URL: url})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
taskID, err := downloader.CreateWithOptions(rr.ID, &base.Options{
|
||||
Path: downloadDir,
|
||||
AsDefaultPath: true,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waitTaskDone(t, done)
|
||||
|
||||
// The task lands in the category directory, but that must not become the
|
||||
// global default download directory.
|
||||
task := downloader.GetTask(taskID)
|
||||
if task == nil {
|
||||
t.Fatal("task not found")
|
||||
}
|
||||
if task.Meta.Opts.Path == downloadDir {
|
||||
t.Fatalf("task path = %q, want the category dir", task.Meta.Opts.Path)
|
||||
}
|
||||
cfg, err := downloader.GetConfig()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if cfg.DownloadDir != downloadDir {
|
||||
t.Fatalf("default download dir = %q, want the base dir %q", cfg.DownloadDir, downloadDir)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloader_ResolveSuggestsWithoutSideEffects(t *testing.T) {
|
||||
url := startCategoryFileServer(t, "setup.data", []byte("0123456789"))
|
||||
downloadDir := t.TempDir()
|
||||
downloader := newSetupCategoryDownloader(t)
|
||||
categoryDir := putCategoryConfig(t, downloader, downloadDir, true, []string{"data"})
|
||||
|
||||
rr, err := downloader.Resolve(&base.Request{URL: url})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if rr.SuggestedDownloadDir != categoryDir {
|
||||
t.Fatalf("suggested dir = %q, want %q", rr.SuggestedDownloadDir, categoryDir)
|
||||
}
|
||||
if _, err := os.Stat(categoryDir); !os.IsNotExist(err) {
|
||||
t.Fatalf("resolve created the suggested directory: %v", err)
|
||||
}
|
||||
cfg, err := downloader.GetConfig()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if cfg.DownloadDir != downloadDir {
|
||||
t.Fatalf("resolve changed the download dir to %q", cfg.DownloadDir)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloader_ResolveSuggestsNothingWhenTheSwitchIsOff(t *testing.T) {
|
||||
url := startCategoryFileServer(t, "setup.data", []byte("0123456789"))
|
||||
downloadDir := t.TempDir()
|
||||
downloader := newSetupCategoryDownloader(t)
|
||||
putCategoryConfig(t, downloader, downloadDir, false, []string{"data"})
|
||||
|
||||
rr, err := downloader.Resolve(&base.Request{URL: url})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if rr.SuggestedDownloadDir != "" {
|
||||
t.Fatalf("suggested dir = %q, want an empty suggestion", rr.SuggestedDownloadDir)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloader_SetupMigratesLegacyExtraCategories(t *testing.T) {
|
||||
storage := NewMemStorage()
|
||||
if err := storage.Setup([]string{bucketConfig}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
legacy := &base.DownloaderStoreConfig{
|
||||
DownloadDir: t.TempDir(),
|
||||
Extra: map[string]any{
|
||||
"downloadCategories": []any{
|
||||
map[string]any{"name": "", "path": filepath.Join(t.TempDir(), "Music"), "isBuiltIn": true, "nameKey": "categoryMusic"},
|
||||
},
|
||||
},
|
||||
}
|
||||
if err := storage.Put(bucketConfig, "config", legacy); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
downloader := NewDownloader(&DownloaderConfig{Storage: storage, StorageDir: t.TempDir()})
|
||||
if err := downloader.Setup(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() {
|
||||
downloader.Delete(nil, true)
|
||||
downloader.Clear()
|
||||
}()
|
||||
|
||||
cfg, err := downloader.GetConfig()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(cfg.Categories) != 1 || cfg.Categories[0].NameKey != "categoryMusic" || len(cfg.Categories[0].Extensions) == 0 {
|
||||
t.Fatalf("categories = %+v, want the migrated built-in with its default extensions", cfg.Categories)
|
||||
}
|
||||
if _, ok := cfg.Extra["downloadCategories"]; ok {
|
||||
t.Fatal("in memory config still carries the legacy key")
|
||||
}
|
||||
|
||||
// The cleared key has to be persisted, not only held in memory.
|
||||
var stored base.DownloaderStoreConfig
|
||||
exist, err := storage.Get(bucketConfig, "config", &stored)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !exist {
|
||||
t.Fatal("stored config missing")
|
||||
}
|
||||
if _, ok := stored.Extra["downloadCategories"]; ok {
|
||||
t.Fatal("stored config still carries the legacy key")
|
||||
}
|
||||
if len(stored.Categories) != 1 || stored.Categories[0].NameKey != "categoryMusic" {
|
||||
t.Fatalf("stored categories = %+v, want the migrated list", stored.Categories)
|
||||
}
|
||||
}
|
||||
|
||||
// readOnlyConfigStorage refuses every config write, so the setup can not
|
||||
// persist a migrated category list.
|
||||
type readOnlyConfigStorage struct {
|
||||
Storage
|
||||
}
|
||||
|
||||
func (s *readOnlyConfigStorage) Put(bucket string, key string, v any) error {
|
||||
if bucket == bucketConfig {
|
||||
return errors.New("config storage is read only")
|
||||
}
|
||||
return s.Storage.Put(bucket, key, v)
|
||||
}
|
||||
|
||||
func TestDownloader_SetupFailsWhenTheMigrationCanNotBePersisted(t *testing.T) {
|
||||
storage := NewMemStorage()
|
||||
if err := storage.Setup([]string{bucketConfig}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
legacy := &base.DownloaderStoreConfig{
|
||||
DownloadDir: t.TempDir(),
|
||||
Extra: map[string]any{
|
||||
"downloadCategories": []any{
|
||||
map[string]any{"name": "Music", "path": filepath.Join(t.TempDir(), "Music")},
|
||||
},
|
||||
},
|
||||
}
|
||||
if err := storage.Put(bucketConfig, "config", legacy); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// The migrated list can not be persisted, so the setup reports the write
|
||||
// error instead of continuing with a category list that is lost on restart.
|
||||
downloader := NewDownloader(&DownloaderConfig{Storage: &readOnlyConfigStorage{Storage: storage}, StorageDir: t.TempDir()})
|
||||
if err := downloader.Setup(); err == nil {
|
||||
t.Fatal("setup reported no error, want the failed config write")
|
||||
}
|
||||
}
|
||||
|
||||
// categoryMatchingCases is the shared extension rule table of the backend and
|
||||
// the ui. The ui runs the same file, see
|
||||
// ui/flutter/test/api/downloader_config_test.dart, so a rule that only changes
|
||||
// on one side breaks the other side's test.
|
||||
type categoryMatchingCases struct {
|
||||
NormalizeExtensions []struct {
|
||||
Input []string `json:"input"`
|
||||
Expected []string `json:"expected"`
|
||||
} `json:"normalizeExtensions"`
|
||||
}
|
||||
|
||||
func TestSharedCategoryMatchingRules(t *testing.T) {
|
||||
data, err := os.ReadFile(filepath.Join("testdata", "category_matching_cases.json"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var cases categoryMatchingCases
|
||||
if err := json.Unmarshal(data, &cases); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, c := range cases.NormalizeExtensions {
|
||||
got := normalizeCategoryExtensions(c.Input)
|
||||
if !slices.Equal(got, c.Expected) {
|
||||
t.Errorf("normalizeCategoryExtensions(%q) = %q, want %q", c.Input, got, c.Expected)
|
||||
}
|
||||
}
|
||||
}
|
||||
+188
-775
File diff suppressed because it is too large.
Load diff
+28
-1194
File diff suppressed because it is too large.
Load diff
@@ -30,21 +30,6 @@ support must remain disabled. Do not declare the sum of the input sizes as the
|
||||
output size. The downloader reads the merged output as one file; FFmpeg errors
|
||||
propagate through the stream instead of completing a partial download.
|
||||
|
||||
Extension resolve results can set `files[].size` to an estimated output size,
|
||||
for example the sum of the selected video and audio input lengths. Clients forward
|
||||
this value as the optional task creation `opts.size` for HTTP downloads only
|
||||
(including Blob URLs served over HTTP). Other protocols, including HLS, ignore
|
||||
this option. The estimate is persisted
|
||||
and displayed while queued and downloading. A valid HTTP total length (including
|
||||
zero) always overrides it; otherwise a clean EOF records the actual output length.
|
||||
Estimates never determine byte ranges, file allocation, integrity checks, or task
|
||||
completion. Size authority is persisted alongside HTTP connection state for
|
||||
pause/retry and process restart.
|
||||
|
||||
Blob options are unchanged: `createObjectURL`'s `size` is an exact source length,
|
||||
not an estimate. Leave it unset for generated merge output and pass any estimate
|
||||
through the extension resolve file instead.
|
||||
|
||||
## JS inputs, including SABR
|
||||
|
||||
Stream inputs **must** be created by an `inputs({ signal })` callback. It returns
|
||||
@@ -145,8 +130,7 @@ associates its input progress with that Blob. No public progress callback or
|
||||
extra `createObjectURL` argument is required. Return the merge stream itself;
|
||||
wrapping it in another stream loses this association. Downloaded bytes count
|
||||
unique input offsets; speed counts received input bytes, including re-fetches.
|
||||
The task stays downloading while queued or merging, displaying the optional
|
||||
`files[].size` estimate (or an unknown total when omitted);
|
||||
The task stays downloading with unknown total size while queued or merging;
|
||||
once input traffic stops the displayed speed settles to zero. Completion uses
|
||||
the actual merged output size. Generated output waits are governed by producer
|
||||
cancellation/errors rather than the HTTP body idle timeout; HTTP media inputs
|
||||
|
||||
+36
-84
@@ -1,19 +1,17 @@
|
||||
package download
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/url"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/GopeedLab/gopeed/internal/fetcher"
|
||||
"github.com/GopeedLab/gopeed/internal/githubmirror"
|
||||
"github.com/GopeedLab/gopeed/internal/logger"
|
||||
"github.com/GopeedLab/gopeed/pkg/base"
|
||||
"github.com/GopeedLab/gopeed/pkg/download/engine"
|
||||
@@ -198,85 +196,50 @@ func (d *Downloader) getExtension(identity string) *Extension {
|
||||
return nil
|
||||
}
|
||||
|
||||
// waitForExtensionGit waits for clones and their install/check handlers before
|
||||
// shutdown releases storage and temporary files. Close cancels backgroundCtx first.
|
||||
func (d *Downloader) waitForExtensionGit() {
|
||||
d.extensionGitLock.Lock()
|
||||
d.extensionGitLock.Unlock()
|
||||
d.extensionGitWG.Wait()
|
||||
}
|
||||
func (d *Downloader) fetchExtensionByGit(url string, handler func(tempExtPath string, devMode bool) (*Extension, error)) (*Extension, error) {
|
||||
if !strings.HasPrefix(url, "https://") && !strings.HasPrefix(url, "http://") {
|
||||
url = "https://" + url
|
||||
}
|
||||
|
||||
func (d *Downloader) fetchExtensionByGit(rawURL string, handler func(string, bool) (*Extension, error)) (*Extension, error) {
|
||||
// Pair admission with shutdown so no operation is added after its wait starts.
|
||||
d.extensionGitLock.Lock()
|
||||
if d.closed.Load() {
|
||||
d.extensionGitLock.Unlock()
|
||||
return nil, fmt.Errorf("downloader is closed")
|
||||
// resolve project path
|
||||
parentPath, projectPath := path.Split(url)
|
||||
// resolve project name and sub path
|
||||
pathArr := strings.SplitN(projectPath, "#", 2)
|
||||
projectPath = strings.TrimSuffix(pathArr[0], gitSuffix)
|
||||
subPath := ""
|
||||
if len(pathArr) > 1 {
|
||||
subPath = pathArr[1]
|
||||
}
|
||||
d.extensionGitWG.Add(1)
|
||||
d.extensionGitLock.Unlock()
|
||||
defer d.extensionGitWG.Done()
|
||||
ctx := d.backgroundCtx
|
||||
if !strings.HasPrefix(rawURL, "https://") && !strings.HasPrefix(rawURL, "http://") {
|
||||
rawURL = "https://" + rawURL
|
||||
}
|
||||
parsed, err := url.Parse(rawURL)
|
||||
if err != nil || parsed.Host == "" {
|
||||
return nil, fmt.Errorf("invalid extension repository URL")
|
||||
}
|
||||
subPath := filepath.FromSlash(parsed.Fragment)
|
||||
if filepath.IsAbs(subPath) || subPath == ".." || strings.HasPrefix(filepath.Clean(subPath), ".."+string(filepath.Separator)) {
|
||||
return nil, fmt.Errorf("invalid extension directory")
|
||||
}
|
||||
parsed.Fragment = ""
|
||||
parsed.Path = strings.TrimSuffix(strings.TrimRight(parsed.Path, "/"), gitSuffix) + gitSuffix
|
||||
officialURL := parsed.String()
|
||||
cfg, err := d.GetConfig()
|
||||
|
||||
// Use unique suffix to avoid concurrent conflicts when multiple git operations
|
||||
// target the same extension (e.g., install and update check happening simultaneously)
|
||||
tempExtDir, err := os.MkdirTemp(d.cfg.TempDir, "gopeed-extension-")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := d.tempFiles.Track(tempExtDir); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer d.tempFiles.Remove(tempExtDir)
|
||||
|
||||
proxyOptions := transport.ProxyOptions{}
|
||||
if proxyURL := cfg.Proxy.ToUrl(); proxyURL != nil {
|
||||
proxyOptions.URL = proxyURL.Scheme + "://" + proxyURL.Host
|
||||
if proxyURL.User != nil {
|
||||
proxyOptions.Username = proxyURL.User.Username()
|
||||
proxyOptions.Password, _ = proxyURL.User.Password()
|
||||
}
|
||||
proxyUrl := d.cfg.DownloaderStoreConfig.Proxy.ToUrl()
|
||||
if proxyUrl != nil {
|
||||
proxyOptions.URL = proxyUrl.Scheme + "://" + proxyUrl.Host
|
||||
proxyOptions.Username = proxyUrl.User.Username()
|
||||
proxyOptions.Password, _ = proxyUrl.User.Password()
|
||||
}
|
||||
tempDir, err := os.MkdirTemp(d.cfg.TempDir, "gopeed-extension-")
|
||||
if err != nil {
|
||||
// clone project to extension temp dir
|
||||
gitUrl := parentPath + projectPath + gitSuffix
|
||||
if _, err := git.PlainClone(tempExtDir, false, &git.CloneOptions{
|
||||
URL: gitUrl,
|
||||
Depth: 1,
|
||||
ProxyOptions: proxyOptions,
|
||||
}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err = d.tempFiles.Track(tempDir); err != nil {
|
||||
_ = os.RemoveAll(tempDir)
|
||||
return nil, err
|
||||
}
|
||||
defer d.tempFiles.Remove(tempDir)
|
||||
repoDir := filepath.Join(tempDir, "repo")
|
||||
repo, err := githubmirror.Fallback(ctx, cfg.GithubMirror, officialURL,
|
||||
githubmirror.Options[*git.Repository]{Kind: githubmirror.Repository},
|
||||
func(ctx context.Context, candidate string) (*git.Repository, error) {
|
||||
if err := os.RemoveAll(repoDir); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return git.PlainCloneContext(ctx, repoDir, false, &git.CloneOptions{URL: candidate, Depth: 1, ProxyOptions: proxyOptions})
|
||||
},
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Preserve the canonical source instead of persisting a selected mirror.
|
||||
repoConfig, err := repo.Config()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if remote := repoConfig.Remotes["origin"]; remote != nil {
|
||||
remote.URLs = []string{officialURL}
|
||||
}
|
||||
if err = repo.SetConfig(repoConfig); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return handler(filepath.Join(repoDir, subPath), false)
|
||||
|
||||
return handler(filepath.Join(tempExtDir, subPath), false)
|
||||
}
|
||||
|
||||
func (d *Downloader) parseExtensionByPath(path string) (*Extension, error) {
|
||||
@@ -790,10 +753,6 @@ type OnErrorExtensionTask struct {
|
||||
}
|
||||
|
||||
func cloneExtensionTask(task *Task) *Task {
|
||||
if task.statusLock != nil {
|
||||
task.statusLock.Lock()
|
||||
defer task.statusLock.Unlock()
|
||||
}
|
||||
newTask := task.clone()
|
||||
newTask.Meta.Req = task.Meta.Req
|
||||
return newTask
|
||||
@@ -825,13 +784,6 @@ func newOnDoneExtensionTask(task *Task) *Task {
|
||||
return cloneExtensionTask(task)
|
||||
}
|
||||
|
||||
// SetUrl replaces the task request URL.
|
||||
//
|
||||
// Deprecated: Use task.meta.req.setUrl instead.
|
||||
func (t *ExtensionTask) SetUrl(url string) {
|
||||
t.Meta.Req.SetUrl(url)
|
||||
}
|
||||
|
||||
func (t *OnErrorExtensionTask) Continue() error {
|
||||
return t.download.Continue(&TaskFilter{
|
||||
IDs: []string{t.ID},
|
||||
|
||||
@@ -1,89 +0,0 @@
|
||||
package download
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"github.com/GopeedLab/gopeed/internal/fetcher"
|
||||
"github.com/GopeedLab/gopeed/pkg/base"
|
||||
)
|
||||
|
||||
func TestExtensionDeprecatedRequestAPIs(t *testing.T) {
|
||||
for _, event := range []ActivationEvent{EventOnResolve, EventOnStart, EventOnError, EventOnDone} {
|
||||
for _, withLabels := range []bool{false, true} {
|
||||
t.Run(fmt.Sprintf("%s/labels=%v", event, withLabels), func(t *testing.T) {
|
||||
d := NewDownloader(&DownloaderConfig{Storage: NewMemStorage(), StorageDir: t.TempDir()})
|
||||
if err := d.Setup(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer d.Clear()
|
||||
|
||||
extDir := t.TempDir()
|
||||
manifest := fmt.Sprintf(`{"name":"legacy-request","title":"Legacy request APIs","version":"0.0.1","scripts":[{"event":%q,"match":{"urls":["*://example.com/*"]},"entry":"index.js"}]}`, event)
|
||||
request := "ctx.task.meta.req"
|
||||
urlMutation := ""
|
||||
if event == EventOnResolve {
|
||||
request = "ctx.req"
|
||||
} else if event == EventOnStart || event == EventOnError {
|
||||
urlMutation = `await ctx.task.setUrl('https://example.com/legacy');
|
||||
if (req.url !== 'https://example.com/legacy') throw new Error('legacy URL did not write through');
|
||||
await req.setUrl('https://example.com/new');
|
||||
if (req.url !== 'https://example.com/new') throw new Error('new URL did not write through');
|
||||
await ctx.task.setUrl('https://example.com/final');`
|
||||
}
|
||||
script := fmt.Sprintf(`gopeed.events.%s(async ctx => {
|
||||
const req = %s;
|
||||
await req.delLabel('missing');
|
||||
await req.putLabel('initial', 'yes');
|
||||
if (req.labels.initial !== 'yes') throw new Error('putLabel did not initialize labels');
|
||||
await req.setLabels({keep: 'yes', remove: 'yes'});
|
||||
if (req.labels.initial !== undefined || req.labels.old !== undefined) throw new Error('setLabels did not replace labels');
|
||||
await req.putLabel('added', 'yes');
|
||||
await req.delLabel('remove');
|
||||
if (req.labels.keep !== 'yes' || req.labels.added !== 'yes' || req.labels.remove !== undefined) throw new Error('label mutation failed');
|
||||
%s
|
||||
await req.putLabel('completed', 'yes');
|
||||
});`, event, request, urlMutation)
|
||||
for name, content := range map[string]string{"manifest.json": manifest, "index.js": script} {
|
||||
if err := os.WriteFile(filepath.Join(extDir, name), []byte(content), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if _, err := d.InstallExtensionByFolder(extDir, false); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
req := &base.Request{URL: "https://example.com/original"}
|
||||
if withLabels {
|
||||
req.Labels = map[string]string{"old": "yes"}
|
||||
}
|
||||
task := &Task{Protocol: "http", Meta: &fetcher.FetcherMeta{Req: req, Opts: &base.Options{}}}
|
||||
switch event {
|
||||
case EventOnResolve:
|
||||
if _, err := d.triggerOnResolve(req); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
case EventOnStart:
|
||||
d.triggerOnStart(task)
|
||||
case EventOnError:
|
||||
d.triggerOnError(task, errors.New("retry"))
|
||||
case EventOnDone:
|
||||
d.triggerOnDone(task)
|
||||
}
|
||||
if !reflect.DeepEqual(req.Labels, map[string]string{"keep": "yes", "added": "yes", "completed": "yes"}) {
|
||||
t.Fatalf("legacy handler did not update the original request: %#v", req.Labels)
|
||||
}
|
||||
expectedURL := "https://example.com/original"
|
||||
if urlMutation != "" {
|
||||
expectedURL = "https://example.com/final"
|
||||
}
|
||||
if req.URL != expectedURL {
|
||||
t.Fatalf("URL = %q, want %q", req.URL, expectedURL)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -152,7 +152,7 @@ func TestExtensionOnErrorRequestWritesToOriginalTask(t *testing.T) {
|
||||
request.setHeaders({Keep:'new'});
|
||||
request.putHeader('Authorization','Bearer new');
|
||||
request.setMethod('POST');
|
||||
if (typeof task.setUrl !== 'function' || typeof task.extra !== 'undefined') throw new Error('unexpected legacy task API');
|
||||
if (typeof task.setUrl !== 'undefined' || typeof task.extra !== 'undefined') throw new Error('legacy task API');
|
||||
request.setUrl('https://example.com/new');
|
||||
request.setLabels({keep:'yes',remove:'yes'});
|
||||
request.putLabel('new','value');
|
||||
|
||||
@@ -65,11 +65,11 @@ func TestExtensionExtraMutationHTTP(t *testing.T) {
|
||||
var id string
|
||||
var err error
|
||||
if resolved {
|
||||
rr, resolveErr := d.Resolve(req)
|
||||
rr, resolveErr := d.Resolve(req, opts)
|
||||
if resolveErr != nil {
|
||||
t.Fatal(resolveErr)
|
||||
}
|
||||
id, err = d.CreateWithOptions(rr.ID, opts)
|
||||
id, err = d.Create(rr.ID)
|
||||
} else {
|
||||
id, err = d.CreateDirect(req, opts)
|
||||
}
|
||||
|
||||
@@ -1,210 +0,0 @@
|
||||
package download
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/cgi"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
httpProtocol "github.com/GopeedLab/gopeed/internal/protocol/http"
|
||||
"github.com/GopeedLab/gopeed/pkg/base"
|
||||
)
|
||||
|
||||
// Exercise real Git clone/install/check/update without depending on GitHub or
|
||||
// public mirror availability. The proxy deliberately rejects the official URL.
|
||||
// github.com/gopeed-test/repo is a placeholder: the successful mirror serves
|
||||
// the temporary local repository through git http-backend instead.
|
||||
func newExtensionGitFixture(t *testing.T) (*Downloader, func(string), *atomic.Int32, *atomic.Int32) {
|
||||
t.Helper()
|
||||
git, err := exec.LookPath("git")
|
||||
if err != nil {
|
||||
t.Skip("Git executable required for HTTP Git fixture")
|
||||
}
|
||||
root := t.TempDir()
|
||||
bare := filepath.Join(root, "repo.git")
|
||||
work := t.TempDir()
|
||||
command := func(args ...string) {
|
||||
t.Helper()
|
||||
cmd := exec.Command(git, args...)
|
||||
if output, err := cmd.CombinedOutput(); err != nil {
|
||||
t.Fatalf("git %v: %v: %s", args, err, output)
|
||||
}
|
||||
}
|
||||
command("-c", "init.defaultBranch=main", "init", "--bare", bare)
|
||||
command("clone", bare, work)
|
||||
command("-C", work, "config", "user.name", "Gopeed Test")
|
||||
command("-C", work, "config", "user.email", "[email protected]")
|
||||
publish := func(version string) {
|
||||
t.Helper()
|
||||
directory := filepath.Join(work, "nested", "extension")
|
||||
if err := os.MkdirAll(directory, 0700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
manifest := map[string]any{"name": "mirror-test", "author": "test", "title": "Mirror test", "version": version, "repository": map[string]string{"url": "https://github.com/gopeed-test/repo", "directory": "nested/extension"}}
|
||||
body, _ := json.Marshal(manifest)
|
||||
if err := os.WriteFile(filepath.Join(directory, "manifest.json"), body, 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
command("-C", work, "add", ".")
|
||||
command("-C", work, "commit", "-m", version)
|
||||
command("-C", work, "push", "origin", "HEAD:main")
|
||||
}
|
||||
publish("1.0.0")
|
||||
var official, badMirror atomic.Int32
|
||||
backend := &cgi.Handler{Path: git, Args: []string{"http-backend"}, Env: []string{"GIT_PROJECT_ROOT=" + root, "GIT_HTTP_EXPORT_ALL=1"}}
|
||||
mirror := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
r.URL.Path = strings.TrimPrefix(r.URL.Path, "/https://github.com/gopeed-test")
|
||||
r.RequestURI = r.URL.RequestURI()
|
||||
backend.ServeHTTP(w, r)
|
||||
}))
|
||||
t.Cleanup(mirror.Close)
|
||||
failed := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { badMirror.Add(1); http.Error(w, "mirror offline", 502) }))
|
||||
t.Cleanup(failed.Close)
|
||||
direct := &http.Transport{}
|
||||
t.Cleanup(direct.CloseIdleConnections)
|
||||
proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method == http.MethodConnect || r.URL.Hostname() == "github.com" {
|
||||
official.Add(1)
|
||||
http.Error(w, "official unavailable", 502)
|
||||
return
|
||||
}
|
||||
request := r.Clone(r.Context())
|
||||
request.RequestURI = ""
|
||||
response, err := direct.RoundTrip(request)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), 502)
|
||||
return
|
||||
}
|
||||
defer response.Body.Close()
|
||||
for key, values := range response.Header {
|
||||
for _, value := range values {
|
||||
w.Header().Add(key, value)
|
||||
}
|
||||
}
|
||||
w.WriteHeader(response.StatusCode)
|
||||
_, _ = io.Copy(w, response.Body)
|
||||
}))
|
||||
t.Cleanup(proxy.Close)
|
||||
d := testBackendDownloader(t, &httpProtocol.FetcherManager{})
|
||||
config, _ := d.GetConfig()
|
||||
config.Proxy = &base.DownloaderProxyConfig{Enable: true, Scheme: "http", Host: strings.TrimPrefix(proxy.URL, "http://")}
|
||||
config.GithubMirror = &base.GithubMirrorConfig{Enabled: true, Mirrors: []base.GithubMirror{{Type: "ghProxy", URL: failed.URL}, {Type: "ghProxy", URL: mirror.URL}}}
|
||||
if err := d.PutConfig(config); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return d, publish, &official, &badMirror
|
||||
}
|
||||
func testInstallExtensionGitURL(t *testing.T, raw string) {
|
||||
d, _, official, badMirror := newExtensionGitFixture(t)
|
||||
extension, err := d.InstallExtensionByGit(raw)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if extension.Version != "1.0.0" || extension.Identity != "test@mirror-test" {
|
||||
t.Fatalf("%+v", extension)
|
||||
}
|
||||
if official.Load() == 0 || badMirror.Load() == 0 {
|
||||
t.Fatal("ordered fallback was not exercised")
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(d.ExtensionPath(extension), "manifest.json")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
func TestExtensionGitMirrorInstallAndUpdate(t *testing.T) {
|
||||
d, publish, official, badMirror := newExtensionGitFixture(t)
|
||||
extension, err := d.InstallExtensionByGit("https://github.com/gopeed-test/repo#nested/extension")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
publish("1.1.0")
|
||||
version, err := d.UpgradeCheckExtension(extension.Identity)
|
||||
if err != nil || version != "1.1.0" {
|
||||
t.Fatalf("%s %v", version, err)
|
||||
}
|
||||
if err = d.UpgradeExtension(extension.Identity); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
updated, err := d.GetExtension(extension.Identity)
|
||||
if err != nil || updated.Version != "1.1.0" {
|
||||
t.Fatalf("%+v %v", updated, err)
|
||||
}
|
||||
if updated.Repository.Url != "https://github.com/gopeed-test/repo" {
|
||||
t.Fatal("mirror URL replaced canonical repository")
|
||||
}
|
||||
if official.Load() != 3 || badMirror.Load() != 3 {
|
||||
t.Fatalf("attempts official=%d mirror=%d", official.Load(), badMirror.Load())
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtensionGitShutdownWaitsForHandler(t *testing.T) {
|
||||
d, _, _, _ := newExtensionGitFixture(t)
|
||||
started := make(chan string, 1)
|
||||
release := make(chan struct{})
|
||||
t.Cleanup(func() {
|
||||
select {
|
||||
case <-release:
|
||||
default:
|
||||
close(release)
|
||||
}
|
||||
})
|
||||
finished := make(chan error, 1)
|
||||
go func() {
|
||||
_, err := d.fetchExtensionByGit("https://github.com/gopeed-test/repo#nested/extension", func(path string, _ bool) (*Extension, error) {
|
||||
started <- path
|
||||
<-release
|
||||
return d.parseExtensionByPath(path)
|
||||
})
|
||||
finished <- err
|
||||
}()
|
||||
var path string
|
||||
select {
|
||||
case path = <-started:
|
||||
case err := <-finished:
|
||||
t.Fatalf("Git operation finished before handler: %v", err)
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("Git handler did not start")
|
||||
}
|
||||
closed := make(chan error, 1)
|
||||
go func() { closed <- d.Close() }()
|
||||
select {
|
||||
case <-d.backgroundCtx.Done():
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("shutdown did not cancel Git context")
|
||||
}
|
||||
select {
|
||||
case err := <-closed:
|
||||
t.Fatalf("shutdown returned before handler finished: %v", err)
|
||||
case <-time.After(100 * time.Millisecond):
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(path, "manifest.json")); err != nil {
|
||||
t.Fatalf("shutdown removed files still used by handler: %v", err)
|
||||
}
|
||||
if _, err := d.InstallExtensionByGit("https://github.com/gopeed-test/repo"); err == nil || !strings.Contains(err.Error(), "downloader is closed") {
|
||||
t.Fatalf("expected new Git operations to be rejected during shutdown: %v", err)
|
||||
}
|
||||
close(release)
|
||||
select {
|
||||
case err := <-finished:
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("Git handler did not finish")
|
||||
}
|
||||
select {
|
||||
case err := <-closed:
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("shutdown did not finish after Git handler")
|
||||
}
|
||||
}
|
||||
@@ -1,62 +0,0 @@
|
||||
package download
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
gitclient "github.com/go-git/go-git/v5/plumbing/transport/client"
|
||||
githttp "github.com/go-git/go-git/v5/plumbing/transport/http"
|
||||
)
|
||||
|
||||
func init() {
|
||||
// go-git exposes a process-wide protocol registry. Register once during
|
||||
// initialization, rather than changing it during concurrent clones. Proxy
|
||||
// settings still come from each clone's ProxyOptions.
|
||||
transport := githttp.NewClient(newExtensionGitHTTPClient(10*time.Second, 60*time.Second))
|
||||
gitclient.InstallProtocol("http", transport)
|
||||
gitclient.InstallProtocol("https", transport)
|
||||
}
|
||||
|
||||
func newExtensionGitHTTPClient(connectTimeout, idleTimeout time.Duration) *http.Client {
|
||||
dialer := &net.Dialer{Timeout: connectTimeout, KeepAlive: 30 * time.Second}
|
||||
// A socket read deadline must apply to one transfer, rather than a shared
|
||||
// HTTP/2 connection carrying traffic from other requests.
|
||||
protocols := new(http.Protocols)
|
||||
protocols.SetHTTP1(true)
|
||||
return &http.Client{Transport: &http.Transport{
|
||||
Proxy: http.ProxyFromEnvironment,
|
||||
DialContext: func(ctx context.Context, network, address string) (net.Conn, error) {
|
||||
conn, err := dialer.DialContext(ctx, network, address)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &gitIdleConn{Conn: conn, timeout: idleTimeout}, nil
|
||||
},
|
||||
TLSHandshakeTimeout: connectTimeout,
|
||||
IdleConnTimeout: idleTimeout,
|
||||
Protocols: protocols,
|
||||
}}
|
||||
}
|
||||
|
||||
type gitIdleConn struct {
|
||||
net.Conn
|
||||
timeout time.Duration
|
||||
}
|
||||
|
||||
func (c *gitIdleConn) Read(p []byte) (int, error) {
|
||||
if err := c.Conn.SetReadDeadline(time.Now().Add(c.timeout)); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return c.Conn.Read(p)
|
||||
}
|
||||
|
||||
// Refresh an already-pending read when a pooled HTTP/1 connection starts the
|
||||
// next request; time spent idle in the pool must not consume its read allowance.
|
||||
func (c *gitIdleConn) Write(p []byte) (int, error) {
|
||||
if err := c.Conn.SetReadDeadline(time.Now().Add(c.timeout)); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return c.Conn.Write(p)
|
||||
}
|
||||
@@ -1,189 +0,0 @@
|
||||
package download
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/http/httptrace"
|
||||
"net/url"
|
||||
"reflect"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/GopeedLab/gopeed/internal/githubmirror"
|
||||
)
|
||||
|
||||
func TestExtensionGitHTTPTransportAllowsSlowStreaming(t *testing.T) {
|
||||
const chunks = 24
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
for range chunks {
|
||||
if _, err := io.WriteString(w, "x"); err != nil {
|
||||
return
|
||||
}
|
||||
w.(http.Flusher).Flush()
|
||||
select {
|
||||
case <-time.After(50 * time.Millisecond):
|
||||
case <-r.Context().Done():
|
||||
return
|
||||
}
|
||||
}
|
||||
}))
|
||||
defer server.Close()
|
||||
client := newExtensionGitHTTPClient(time.Second, time.Second)
|
||||
defer client.CloseIdleConnections()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
request, _ := http.NewRequestWithContext(ctx, http.MethodGet, server.URL, nil)
|
||||
response, err := client.Do(request)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer response.Body.Close()
|
||||
// A regularly streaming response lasts longer than the idle allowance.
|
||||
body, err := io.ReadAll(response.Body)
|
||||
if err != nil || len(body) != chunks {
|
||||
t.Fatalf("received %d bytes, error=%v", len(body), err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtensionGitHTTPTransportTimesOutStalledReads(t *testing.T) {
|
||||
for _, bodyStarted := range []bool{false, true} {
|
||||
name := "headers"
|
||||
if bodyStarted {
|
||||
name = "body"
|
||||
}
|
||||
t.Run(name, func(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if bodyStarted {
|
||||
_, _ = io.WriteString(w, "partial")
|
||||
w.(http.Flusher).Flush()
|
||||
}
|
||||
<-r.Context().Done()
|
||||
}))
|
||||
defer server.Close()
|
||||
client := newExtensionGitHTTPClient(time.Second, 100*time.Millisecond)
|
||||
defer client.CloseIdleConnections()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
request, _ := http.NewRequestWithContext(ctx, http.MethodGet, server.URL, nil)
|
||||
response, err := client.Do(request)
|
||||
if err == nil {
|
||||
defer response.Body.Close()
|
||||
_, err = io.ReadAll(response.Body)
|
||||
}
|
||||
var timeout net.Error
|
||||
if !errors.As(err, &timeout) || !timeout.Timeout() || ctx.Err() != nil {
|
||||
t.Fatalf("expected idle timeout before parent deadline: error=%v context=%v", err, ctx.Err())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtensionGitIdleTimeoutFallsBackThroughProxy(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Hostname() == "github.com" {
|
||||
_, _ = io.WriteString(w, "partial")
|
||||
w.(http.Flusher).Flush()
|
||||
<-r.Context().Done()
|
||||
return
|
||||
}
|
||||
_, _ = io.WriteString(w, "complete")
|
||||
}))
|
||||
defer server.Close()
|
||||
client := newExtensionGitHTTPClient(time.Second, 100*time.Millisecond)
|
||||
defer client.CloseIdleConnections()
|
||||
proxy, _ := url.Parse(server.URL)
|
||||
client.Transport.(*http.Transport).Proxy = http.ProxyURL(proxy)
|
||||
config := &githubmirror.Config{Enabled: true, Mirrors: []githubmirror.Mirror{{Type: "ghProxy", URL: server.URL}}}
|
||||
source := "http://github.com/test/repo.git"
|
||||
var tried []string
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
result, err := githubmirror.Fallback(ctx, config, source, githubmirror.Options[string]{Kind: githubmirror.Repository}, func(ctx context.Context, candidate string) (string, error) {
|
||||
tried = append(tried, candidate)
|
||||
request, _ := http.NewRequestWithContext(ctx, http.MethodGet, candidate, nil)
|
||||
response, err := client.Do(request)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer response.Body.Close()
|
||||
body, err := io.ReadAll(response.Body)
|
||||
return string(body), err
|
||||
})
|
||||
if err != nil || result != "complete" || !reflect.DeepEqual(tried, []string{source, server.URL + "/" + source}) {
|
||||
t.Fatalf("result=%q tried=%v error=%v", result, tried, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtensionGitHTTPTransportCancelsStalledBody(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
w.(http.Flusher).Flush()
|
||||
<-r.Context().Done()
|
||||
}))
|
||||
defer server.Close()
|
||||
client := newExtensionGitHTTPClient(time.Second, time.Minute)
|
||||
defer client.CloseIdleConnections()
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
request, _ := http.NewRequestWithContext(ctx, http.MethodGet, server.URL, nil)
|
||||
response, err := client.Do(request)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer response.Body.Close()
|
||||
cancel()
|
||||
done := make(chan error, 1)
|
||||
go func() { _, err := io.ReadAll(response.Body); done <- err }()
|
||||
select {
|
||||
case err := <-done:
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("error=%v", err)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("parent cancellation did not interrupt the body")
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtensionGitHTTPTransportRefreshesReusedConnection(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path == "/slow" {
|
||||
select {
|
||||
case <-time.After(500 * time.Millisecond):
|
||||
case <-r.Context().Done():
|
||||
return
|
||||
}
|
||||
}
|
||||
_, _ = io.WriteString(w, "ok")
|
||||
}))
|
||||
defer server.Close()
|
||||
client := newExtensionGitHTTPClient(time.Second, time.Second)
|
||||
defer client.CloseIdleConnections()
|
||||
response, err := client.Get(server.URL)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err = io.Copy(io.Discard, response.Body)
|
||||
_ = response.Body.Close()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
time.Sleep(700 * time.Millisecond)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
reused := false
|
||||
ctx = httptrace.WithClientTrace(ctx, &httptrace.ClientTrace{GotConn: func(info httptrace.GotConnInfo) { reused = info.Reused }})
|
||||
request, _ := http.NewRequestWithContext(ctx, http.MethodGet, server.URL+"/slow", nil)
|
||||
response, err = client.Do(request)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer response.Body.Close()
|
||||
body, err := io.ReadAll(response.Body)
|
||||
if err != nil || string(body) != "ok" || !reused {
|
||||
t.Fatalf("reused=%v body=%q error=%v", reused, body, err)
|
||||
}
|
||||
}
|
||||
@@ -1,56 +0,0 @@
|
||||
package download
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/GopeedLab/gopeed/pkg/base"
|
||||
)
|
||||
|
||||
func TestDownloader_Extension_MagnetMatch(t *testing.T) {
|
||||
const magnetURL = "magnet:?xt=urn:btih:0123456789abcdef0123456789abcdef01234567"
|
||||
for _, pattern := range []string{"magnet:*", "magnet:?*"} {
|
||||
t.Run(pattern, func(t *testing.T) {
|
||||
d := NewDownloader(&DownloaderConfig{
|
||||
Storage: NewMemStorage(),
|
||||
StorageDir: t.TempDir(),
|
||||
TempDir: t.TempDir(),
|
||||
})
|
||||
if err := d.Setup(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer d.Clear()
|
||||
|
||||
extDir := t.TempDir()
|
||||
manifest := fmt.Sprintf(`{"name":"magnet-match","author":"test","title":"Magnet match","version":"1.0.0","scripts":[{"event":"onResolve","match":{"urls":[%q]},"entry":"index.js"}]}`, pattern)
|
||||
script := `gopeed.events.onResolve(ctx => {
|
||||
ctx.res = {files: [{name: "resolved.txt", size: 1, req: {url: "https://example.com/resolved.txt"}}]};
|
||||
});`
|
||||
for name, content := range map[string]string{"manifest.json": manifest, "index.js": script} {
|
||||
if err := os.WriteFile(filepath.Join(extDir, name), []byte(content), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if _, err := d.InstallExtensionByFolder(extDir, true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
for _, req := range []*base.Request{
|
||||
{URL: magnetURL},
|
||||
{URL: "https://example.com/rewritten", RawURL: magnetURL},
|
||||
} {
|
||||
// Exercise extension activation without falling back to a live
|
||||
// BitTorrent metadata lookup when the pattern fails to match.
|
||||
res, err := d.triggerOnResolve(req)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if res == nil || len(res.Files) != 1 || res.Files[0].Name != "resolved.txt" {
|
||||
t.Fatalf("magnet extension did not resolve request %+v: %+v", req, res)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,159 +0,0 @@
|
||||
package download
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/GopeedLab/gopeed/pkg/base"
|
||||
)
|
||||
|
||||
func TestExtensionResolveSizeThroughTaskCreation(t *testing.T) {
|
||||
for _, mode := range []string{"http", "blob"} {
|
||||
for _, known := range []bool{false, true} {
|
||||
t.Run(fmt.Sprintf("%s/known=%v", mode, known), func(t *testing.T) {
|
||||
payload := []byte("0123456789")
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
body := payload
|
||||
if r.URL.Path == "/empty" {
|
||||
body = nil
|
||||
}
|
||||
if known {
|
||||
w.Header().Set("Content-Length", fmt.Sprint(len(body)))
|
||||
}
|
||||
w.(http.Flusher).Flush()
|
||||
w.Write(body)
|
||||
}))
|
||||
defer server.Close()
|
||||
d := NewDownloader(&DownloaderConfig{Storage: NewMemStorage(), StorageDir: t.TempDir(), TempDir: t.TempDir()})
|
||||
if err := d.Setup(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer d.Clear()
|
||||
d.cfg.MaxRunning = 0 // Inspect persisted queued tasks before HTTP resolution.
|
||||
extDir := t.TempDir()
|
||||
manifest := `{"name":"size-hint","author":"test","title":"Size hint","version":"1.0.0","scripts":[{"event":"onResolve","match":{"urls":["https://example.com/size-hint"]},"entry":"index.js"}]}`
|
||||
script := fmt.Sprintf(`gopeed.events.onResolve(async ctx=>{
|
||||
const mode=%q, origin=%q;
|
||||
const file=async(name,size,empty)=>({name,size,req:{url:mode==='http'?origin+(empty?'/empty':'/data'):
|
||||
await gopeed.runtime.blob.createObjectURL(()=>new ReadableStream({start(c){if(!empty)c.enqueue(new Uint8Array([48,49,50,51,52,53,54,55,56,57]));c.close();}}))}});
|
||||
ctx.res={files:[await file('under.bin',3,false),await file('over.bin',1048576,false),await file('empty.bin',99,true)]};
|
||||
});`, mode, server.URL)
|
||||
for name, contents := range map[string]string{"manifest.json": manifest, "index.js": script} {
|
||||
if err := os.WriteFile(filepath.Join(extDir, name), []byte(contents), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if _, err := d.InstallExtensionByFolder(extDir, true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rr, err := d.Resolve(&base.Request{URL: "https://example.com/size-hint"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
output := t.TempDir()
|
||||
batch := &base.CreateTaskBatch{}
|
||||
for _, file := range rr.Res.Files {
|
||||
batch.Reqs = append(batch.Reqs, &base.CreateTaskBatchItem{Req: file.Req, Opts: &base.Options{Size: file.Size, Name: file.Name, Path: output}})
|
||||
}
|
||||
wire, err := json.Marshal(batch)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var submitted base.CreateTaskBatch
|
||||
if err := json.Unmarshal(wire, &submitted); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
batchResult, err := d.CreateDirectBatch(&submitted)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ids := batchResult
|
||||
for i, id := range ids {
|
||||
task := d.GetTask(id)
|
||||
if task.Meta.Res != nil {
|
||||
t.Fatal("a size estimate must not mark the task resolved")
|
||||
}
|
||||
status, err := d.RuntimeStatus(id)
|
||||
if err != nil || status.Total != rr.Res.Files[i].Size {
|
||||
t.Fatalf("queued size lost: status=%+v err=%v", status, err)
|
||||
}
|
||||
var saved Task
|
||||
if _, err := d.storage.Get(bucketTask, id, &saved); err != nil || saved.Meta.Opts.Size != status.Total {
|
||||
t.Fatal("queued estimate was not persisted", err)
|
||||
}
|
||||
if err := d.doStart(task); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waitForTaskTerminal(t, d, id, 5*time.Second)
|
||||
body, err := os.ReadFile(filepath.Join(output, rr.Res.Files[i].Name))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := payload
|
||||
if i == 2 {
|
||||
want = nil
|
||||
}
|
||||
if !bytes.Equal(body, want) {
|
||||
t.Fatalf("estimate changed output: %q", body)
|
||||
}
|
||||
status, err = d.RuntimeStatus(id)
|
||||
if err != nil || status.Total != int64(len(want)) || status.Downloaded != int64(len(want)) || task.Meta.Res.Files[0].Size != int64(len(want)) {
|
||||
t.Fatalf("final size not updated: %+v err=%v", status, err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolvedHTTPSizeEstimateAtCreation(t *testing.T) {
|
||||
for _, known := range []bool{false, true} {
|
||||
t.Run(fmt.Sprintf("known=%v", known), func(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
if known {
|
||||
w.Header().Set("Content-Length", "10")
|
||||
}
|
||||
w.(http.Flusher).Flush()
|
||||
w.Write([]byte("0123456789"))
|
||||
}))
|
||||
defer server.Close()
|
||||
d := NewDownloader(&DownloaderConfig{StorageDir: t.TempDir(), TempDir: t.TempDir()})
|
||||
if err := d.Setup(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer d.Clear()
|
||||
d.cfg.MaxRunning = 0
|
||||
rr, err := d.Resolve(&base.Request{URL: server.URL})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
id, err := d.CreateWithOptions(rr.ID, &base.Options{Size: 3, Path: t.TempDir(), Name: "output.bin"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := int64(3)
|
||||
if known {
|
||||
want = 10
|
||||
}
|
||||
status, err := d.RuntimeStatus(id)
|
||||
if err != nil || status.Total != want || d.GetTask(id).Meta.Res.Files[0].Size != want {
|
||||
t.Fatalf("resolve-before-create size=%+v want=%d err=%v", status, want, err)
|
||||
}
|
||||
if err := d.doStart(d.GetTask(id)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waitForTaskTerminal(t, d, id, 5*time.Second)
|
||||
task := d.GetTask(id)
|
||||
if task.Status != base.DownloadStatusDone || task.Meta.Res.Size != 10 {
|
||||
t.Fatalf("estimate changed completion: status=%s size=%d", task.Status, task.Meta.Res.Size)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,212 +0,0 @@
|
||||
package download
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/GopeedLab/gopeed/pkg/base"
|
||||
)
|
||||
|
||||
func TestExtensionFileSizeIsAdvisory(t *testing.T) {
|
||||
video, err := os.ReadFile("engine/testdata/ffmpeg/video.mp4")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
audio, err := os.ReadFile("engine/testdata/ffmpeg/audio.mp4")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, mode := range []string{"plain", "merge", "wrapped-merge"} {
|
||||
var expected []byte
|
||||
for _, size := range []int64{0, 3, 1 << 20} {
|
||||
t.Run(fmt.Sprintf("%s/size=%d", mode, size), func(t *testing.T) {
|
||||
d := NewDownloader(&DownloaderConfig{Storage: NewMemStorage(), StorageDir: t.TempDir(), TempDir: t.TempDir(), RefreshInterval: 50})
|
||||
if err := d.Setup(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer d.Clear()
|
||||
e, _ := d.newExtensionEngine()
|
||||
defer e.Close()
|
||||
e.Runtime.Set("videoBytes", e.Runtime.NewArrayBuffer(video))
|
||||
e.Runtime.Set("audioBytes", e.Runtime.NewArrayBuffer(audio))
|
||||
e.Runtime.Set("mode", mode)
|
||||
result, err := e.RunString(`__gopeed_blob_create_object_url(()=>{
|
||||
const source=b=>new ReadableStream({start(c){c.enqueue(new Uint8Array(b));c.close();}});
|
||||
if (mode==='plain') return source([0,1,2,3,4,5,6,7,8,9]);
|
||||
const output=__gopeed_ffmpeg.merge({inputs:()=>({video:source(videoBytes),audio:source(audioBytes)})});
|
||||
if (mode==='merge') return output;
|
||||
const reader=output.getReader();
|
||||
return new ReadableStream({async pull(c){const {done,value}=await reader.read();if(done)c.close();else c.enqueue(value);},cancel(r){return reader.cancel(r);}});
|
||||
},{contentType:'video/mp4',range:false})`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
output := t.TempDir()
|
||||
id, err := d.CreateDirect(&base.Request{URL: fmt.Sprint(result)}, &base.Options{Size: size, Path: output, Name: "output.bin"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waitForTaskTerminal(t, d, id, 30*time.Second)
|
||||
body, err := os.ReadFile(filepath.Join(output, "output.bin"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if size == 0 {
|
||||
expected = body
|
||||
if mode == "plain" && !bytes.Equal(body, []byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}) {
|
||||
t.Fatalf("unexpected stream output: %v", body)
|
||||
}
|
||||
if mode != "plain" && !bytes.Contains(body, []byte("moof")) {
|
||||
t.Fatal("invalid merged output")
|
||||
}
|
||||
} else if !bytes.Equal(body, expected) {
|
||||
t.Fatalf("size hint changed output: got %d bytes, want %d", len(body), len(expected))
|
||||
}
|
||||
task := d.GetTask(id)
|
||||
task.lock.Lock()
|
||||
defer task.lock.Unlock()
|
||||
if task.Meta.Res.Size != int64(len(body)) || task.Meta.Res.Files[0].Size != int64(len(body)) {
|
||||
t.Fatalf("final metadata must use actual size: %#v", task.Meta.Res)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtensionFileSizeYieldsToBlobContentLength(t *testing.T) {
|
||||
for _, materialized := range []bool{false, true} {
|
||||
for _, size := range []int64{3, 1 << 20} {
|
||||
t.Run(fmt.Sprintf("materialized=%v/estimate=%d", materialized, size), func(t *testing.T) {
|
||||
d := NewDownloader(&DownloaderConfig{Storage: NewMemStorage(), StorageDir: t.TempDir(), TempDir: t.TempDir()})
|
||||
if err := d.Setup(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer d.Clear()
|
||||
e, _ := d.newExtensionEngine()
|
||||
defer e.Close()
|
||||
e.Runtime.Set("materialized", materialized)
|
||||
result, err := e.RunString(`(()=>{
|
||||
const payload=new Uint8Array([0,1,2,3,4,5,6,7,8,9]);
|
||||
const source=materialized?new Blob([payload]):()=>new ReadableStream({start(c){c.enqueue(payload);c.close();}});
|
||||
return __gopeed_blob_create_object_url(source,{size:10});
|
||||
})()`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
output := t.TempDir()
|
||||
id, err := d.CreateDirect(&base.Request{URL: fmt.Sprint(result)}, &base.Options{Size: size, Path: output, Name: "output.bin"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waitForTaskTerminal(t, d, id, 5*time.Second)
|
||||
body, err := os.ReadFile(filepath.Join(output, "output.bin"))
|
||||
if err != nil || !bytes.Equal(body, []byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}) {
|
||||
t.Fatalf("unexpected body=%v err=%v", body, err)
|
||||
}
|
||||
task := d.GetTask(id)
|
||||
if task.Meta.Res.Size != 10 || task.Meta.Res.Files[0].Size != 10 {
|
||||
t.Fatalf("Blob Content-Length did not override estimate: %+v", task.Meta.Res)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtensionFileSizeHintWaitsForEOF(t *testing.T) {
|
||||
for _, fail := range []bool{false, true} {
|
||||
for _, size := range []int64{3, 1 << 20} {
|
||||
t.Run(fmt.Sprintf("fail=%v/size=%d", fail, size), func(t *testing.T) {
|
||||
d := NewDownloader(&DownloaderConfig{Storage: NewMemStorage(), StorageDir: t.TempDir(), TempDir: t.TempDir(), RefreshInterval: 50})
|
||||
if err := d.Setup(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer d.Clear()
|
||||
e, _ := d.newExtensionEngine()
|
||||
defer e.Close()
|
||||
e.Runtime.Set("fail", fail)
|
||||
result, err := e.RunString(`__gopeed_blob_create_object_url(()=>new ReadableStream({
|
||||
start(c){c.enqueue(new Uint8Array([0,1,2,3,4,5,6,7,8,9]));globalThis.finish=()=>{if(fail)c.error(new Error('upstream failed'));else c.close();};}
|
||||
}),{})`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
output := t.TempDir()
|
||||
id, err := d.CreateDirect(&base.Request{URL: fmt.Sprint(result)}, &base.Options{Size: size, Path: output, Name: "output.bin"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waitForTaskDownloadedBetween(t, d, id, 10, 10, 5*time.Second)
|
||||
task := d.GetTask(id)
|
||||
task.lock.Lock()
|
||||
if task.Meta.Res.Size != size || d.taskStatus(task) != base.DownloadStatusRunning {
|
||||
t.Errorf("hint must remain visible until EOF: size=%d status=%s", task.Meta.Res.Size, d.taskStatus(task))
|
||||
}
|
||||
task.lock.Unlock()
|
||||
stat, err := os.Stat(filepath.Join(output, "output.bin"))
|
||||
if err != nil || stat.Size() != 10 {
|
||||
t.Fatalf("hint must not preallocate or truncate output: stat=%v err=%v", stat, err)
|
||||
}
|
||||
if _, err := e.RunString(`finish()`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if fail {
|
||||
waitForTaskStatus(t, d, id, base.DownloadStatusError, 5*time.Second)
|
||||
} else {
|
||||
waitForTaskTerminal(t, d, id, 5*time.Second)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtensionFileSizeHintPauseContinue(t *testing.T) {
|
||||
for _, size := range []int64{3, 1 << 20} {
|
||||
t.Run(fmt.Sprint(size), func(t *testing.T) {
|
||||
d := NewDownloader(&DownloaderConfig{Storage: NewMemStorage(), StorageDir: t.TempDir(), TempDir: t.TempDir(), RefreshInterval: 50})
|
||||
if err := d.Setup(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer d.Clear()
|
||||
e, _ := d.newExtensionEngine()
|
||||
defer e.Close()
|
||||
result, err := e.RunString(`(()=>{
|
||||
let opens=0;
|
||||
return __gopeed_blob_create_object_url(()=>new ReadableStream({start(c){
|
||||
if (++opens===1) c.enqueue(new Uint8Array([0,1,2,3,4,5,6,7,8,9]));
|
||||
else {c.enqueue(new Uint8Array([5,6]));c.close();}
|
||||
}}),{});
|
||||
})()`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
output := t.TempDir()
|
||||
id, err := d.CreateDirect(&base.Request{URL: fmt.Sprint(result)}, &base.Options{Size: size, Path: output, Name: "output.bin"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waitForTaskDownloadedBetween(t, d, id, 10, 10, 5*time.Second)
|
||||
filter := &TaskFilter{IDs: []string{id}}
|
||||
if err := d.Pause(filter); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := d.Continue(filter); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waitForTaskTerminal(t, d, id, 5*time.Second)
|
||||
body, err := os.ReadFile(filepath.Join(output, "output.bin"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
task := d.GetTask(id)
|
||||
task.lock.Lock()
|
||||
defer task.lock.Unlock()
|
||||
if !bytes.Equal(body, []byte{5, 6}) || task.Meta.Res.Size != 2 {
|
||||
t.Fatalf("resumed stream kept an estimate or stale bytes: body=%v size=%d", body, task.Meta.Res.Size)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -32,7 +32,7 @@ func TestDownloader_InstallExtensionByFolder(t *testing.T) {
|
||||
URL: "https://github.com/test",
|
||||
Labels: map[string]string{"original": "true"},
|
||||
}
|
||||
rr, err := downloader.Resolve(req)
|
||||
rr, err := downloader.Resolve(req, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -58,7 +58,7 @@ func TestDownloader_InstallExtensionByFolderDevMode(t *testing.T) {
|
||||
}
|
||||
rr, err := downloader.Resolve(&base.Request{
|
||||
URL: "https://github.com/test",
|
||||
})
|
||||
}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -76,7 +76,7 @@ func TestDownloader_Extension_BlobBlob(t *testing.T) {
|
||||
|
||||
rr, err := downloader.Resolve(&base.Request{
|
||||
URL: "https://example.com/blob",
|
||||
})
|
||||
}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -130,7 +130,7 @@ func TestDownloader_Extension_BlobFunctionOpener(t *testing.T) {
|
||||
|
||||
rr, err := downloader.Resolve(&base.Request{
|
||||
URL: "https://example.com/opener",
|
||||
})
|
||||
}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -166,7 +166,7 @@ func TestDownloader_Extension_BlobFunctionOpenerUnknownSize(t *testing.T) {
|
||||
|
||||
rr, err := downloader.Resolve(&base.Request{
|
||||
URL: "https://example.com/opener-unknown",
|
||||
})
|
||||
}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -222,7 +222,7 @@ func TestDownloader_Extension_BlobSourceSizePropagatesToCreatedTask(t *testing.T
|
||||
|
||||
rr, err := downloader.Resolve(&base.Request{
|
||||
URL: "https://example.com/opener",
|
||||
})
|
||||
}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -303,7 +303,7 @@ func TestDownloader_Extension_BlobFetchDrainedStreamReportsEarlyProgress(t *test
|
||||
|
||||
rr, err := downloader.Resolve(&base.Request{
|
||||
URL: fmt.Sprintf("%s/single-fetch-stream?size=%d", server.URL, len(payload)),
|
||||
})
|
||||
}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -419,7 +419,7 @@ func TestDownloader_Extension_BlobRangeFetchDrainedStreamReportsEarlyProgress(t
|
||||
|
||||
rr, err := downloader.Resolve(&base.Request{
|
||||
URL: fmt.Sprintf("%s/range-fetch-stream?size=%d", server.URL, len(payload)),
|
||||
})
|
||||
}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -494,7 +494,7 @@ func TestDownloader_Extension_BlobMultiplexFetchStreamsReportEarlyProgress(t *te
|
||||
|
||||
rr, err := downloader.Resolve(&base.Request{
|
||||
URL: server.URL + "/multiplex-fetch-stream",
|
||||
})
|
||||
}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -550,7 +550,7 @@ func TestDownloader_Extension_BlobFunctionOpenerRangeResume(t *testing.T) {
|
||||
|
||||
rr, err := downloader.Resolve(&base.Request{
|
||||
URL: "https://example.com/opener-range",
|
||||
})
|
||||
}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -683,7 +683,7 @@ func TestDownloader_Extension_BlobResumeAfterRestartViaOnError(t *testing.T) {
|
||||
|
||||
rr, err := downloader.Resolve(&base.Request{
|
||||
URL: server.URL + "/restart-range",
|
||||
})
|
||||
}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -855,7 +855,7 @@ func TestDownloader_Extension_BlobPauseContinueKeepsSource(t *testing.T) {
|
||||
|
||||
rr, err := downloader.Resolve(&base.Request{
|
||||
URL: tc.url,
|
||||
})
|
||||
}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -932,7 +932,7 @@ func TestDownloader_Extension_BlobUnknownSizePauseContinueKeepsSource(t *testing
|
||||
|
||||
rr, err := downloader.Resolve(&base.Request{
|
||||
URL: "https://example.com/opener-unknown",
|
||||
})
|
||||
}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -1009,7 +1009,7 @@ func TestDownloader_Extension_BlobRecoverOnError(t *testing.T) {
|
||||
path := path
|
||||
t.Run(path, func(t *testing.T) {
|
||||
rawURL := "https://example.com/" + path
|
||||
rr, err := downloader.Resolve(&base.Request{URL: rawURL})
|
||||
rr, err := downloader.Resolve(&base.Request{URL: rawURL}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -1070,15 +1070,54 @@ func TestDownloader_Extension_BlobRecoverOnError(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestDownloader_InstallExtensionByGit(t *testing.T) {
|
||||
testInstallExtensionGitURL(t, "https://github.com/gopeed-test/repo#nested/extension")
|
||||
setupDownloader(func(downloader *Downloader) {
|
||||
if _, err := downloader.InstallExtensionByGit("https://github.com/GopeedLab/gopeed-extension-samples#github-release-sample"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rr, err := downloader.Resolve(&base.Request{
|
||||
URL: "https://github.com/GopeedLab/gopeed/releases",
|
||||
}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(rr.Res.Files) == 1 {
|
||||
t.Fatal("resolve error")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestDownloader_InstallExtensionByGitSimple(t *testing.T) {
|
||||
testInstallExtensionGitURL(t, "github.com/gopeed-test/repo#nested/extension")
|
||||
setupDownloader(func(downloader *Downloader) {
|
||||
if _, err := downloader.InstallExtensionByGit("github.com/GopeedLab/gopeed-extension-samples#github-release-sample"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rr, err := downloader.Resolve(&base.Request{
|
||||
URL: "https://github.com/GopeedLab/gopeed/releases",
|
||||
}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(rr.Res.Files) == 1 {
|
||||
t.Fatal("resolve error")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestDownloader_InstallExtensionByGitFull(t *testing.T) {
|
||||
testInstallExtensionGitURL(t, "https://github.com/gopeed-test/repo.git#nested/extension")
|
||||
setupDownloader(func(downloader *Downloader) {
|
||||
if _, err := downloader.InstallExtensionByGit("https://github.com/GopeedLab/gopeed-extension-samples.git#github-release-sample"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rr, err := downloader.Resolve(&base.Request{
|
||||
URL: "https://github.com/GopeedLab/gopeed/releases",
|
||||
}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(rr.Res.Files) == 1 {
|
||||
t.Fatal("resolve error")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestDownloader_UpgradeExtension(t *testing.T) {
|
||||
@@ -1141,7 +1180,7 @@ func TestDownloader_UpgradeExtension(t *testing.T) {
|
||||
|
||||
rr, err := downloader.Resolve(&base.Request{
|
||||
URL: "https://test.com",
|
||||
})
|
||||
}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -1316,7 +1355,7 @@ func TestDownloader_Extension_Errors(t *testing.T) {
|
||||
}
|
||||
rr, err := downloader.Resolve(&base.Request{
|
||||
URL: "https://github.com/test",
|
||||
})
|
||||
}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -1331,7 +1370,7 @@ func TestDownloader_Extension_Errors(t *testing.T) {
|
||||
}
|
||||
rr, err := downloader.Resolve(&base.Request{
|
||||
URL: "https://github.com/test",
|
||||
})
|
||||
}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -1346,7 +1385,7 @@ func TestDownloader_Extension_Errors(t *testing.T) {
|
||||
}
|
||||
_, err := downloader.Resolve(&base.Request{
|
||||
URL: "https://github.com/test",
|
||||
})
|
||||
}, nil)
|
||||
if err == nil {
|
||||
t.Fatalf("except error, but got nil")
|
||||
}
|
||||
@@ -1368,7 +1407,7 @@ func TestDownloader_Extension_Settings(t *testing.T) {
|
||||
}
|
||||
rr, err := downloader.Resolve(&base.Request{
|
||||
URL: "https://github.com/test",
|
||||
})
|
||||
}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -1389,7 +1428,7 @@ func TestDownloader_Extension_Settings(t *testing.T) {
|
||||
})
|
||||
rr, err := downloader.Resolve(&base.Request{
|
||||
URL: "https://github.com/test",
|
||||
})
|
||||
}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -1406,7 +1445,7 @@ func TestDownloader_ExtensionStorage(t *testing.T) {
|
||||
}
|
||||
rr, err := downloader.Resolve(&base.Request{
|
||||
URL: "https://github.com/test",
|
||||
})
|
||||
}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -2044,13 +2083,8 @@ func waitForTaskStatus(t *testing.T, downloader *Downloader, id string, status b
|
||||
deadline := time.Now().Add(timeout)
|
||||
for time.Now().Before(deadline) {
|
||||
task := downloader.GetTask(id)
|
||||
if task != nil && downloader.taskStatus(task) == status {
|
||||
if status != base.DownloadStatusDone {
|
||||
return
|
||||
}
|
||||
if _, watching := downloader.watchedTasks.Load(id); !watching {
|
||||
return
|
||||
}
|
||||
if task != nil && task.Status == status {
|
||||
return
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
@@ -2058,6 +2092,9 @@ func waitForTaskStatus(t *testing.T, downloader *Downloader, id string, status b
|
||||
if task == nil {
|
||||
t.Fatalf("timeout waiting for task %s status %s: task not found", id, status)
|
||||
}
|
||||
snapshot, err := downloader.RuntimeStatus(id)
|
||||
t.Fatalf("timeout waiting for task %s status %s: got %+v, error=%v", id, status, snapshot, err)
|
||||
total := int64(0)
|
||||
if task.Meta != nil && task.Meta.Res != nil {
|
||||
total = task.Meta.Res.Size
|
||||
}
|
||||
t.Fatalf("timeout waiting for task %s status %s: got %s downloaded=%d total=%d", id, status, task.Status, task.Progress.Downloaded, total)
|
||||
}
|
||||
@@ -24,7 +24,7 @@ func TestExtensionUnnamedResource(t *testing.T) {
|
||||
if _, err = d.InstallExtensionByFolder(dir, true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
res, err := d.Resolve(&base.Request{URL: "https://example.com/unnamed"})
|
||||
res, err := d.Resolve(&base.Request{URL: "https://example.com/unnamed"}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -1,57 +0,0 @@
|
||||
package download
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/GopeedLab/gopeed/internal/fetcher"
|
||||
"github.com/GopeedLab/gopeed/pkg/base"
|
||||
"github.com/GopeedLab/gopeed/pkg/protocol/ftp"
|
||||
"github.com/GopeedLab/gopeed/pkg/protocol/http"
|
||||
)
|
||||
|
||||
// TestTorrentNameIsFile checks the default manager order: an FTP URL whose
|
||||
// name looks like a torrent or a playlist is still an FTP download.
|
||||
func TestTorrentNameIsFile(t *testing.T) {
|
||||
d := NewDownloader(nil)
|
||||
for u, want := range map[string]string{
|
||||
"ftp://h/x.torrent": "ftp",
|
||||
"ftp://h/v.m3u8": "ftp",
|
||||
"ftps://h/x.torrent": "ftp",
|
||||
"ftpes://h/a.bin": "ftp",
|
||||
"https://h/v.m3u8": "hls",
|
||||
"https://h/x.torrent": "http",
|
||||
"magnet:?xt=urn:btih:": "bt",
|
||||
} {
|
||||
fm, err := d.parseFm(u)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", u, err)
|
||||
}
|
||||
if fm.Name() != want {
|
||||
t.Fatalf("%s goes to %q, want %q", u, fm.Name(), want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPostDownloadExtraAcceptsFTP(t *testing.T) {
|
||||
yes := true
|
||||
file := &base.Resource{Files: []*base.FileInfo{{Name: "a.zip"}}}
|
||||
folder := &base.Resource{Name: "dir", Files: []*base.FileInfo{{Name: "a.zip"}}}
|
||||
|
||||
httpExtra := &http.OptsExtra{AutoExtract: &yes}
|
||||
if got := postDownloadExtra(&fetcher.FetcherMeta{Res: file, Opts: &base.Options{Extra: httpExtra}}); got != httpExtra {
|
||||
t.Fatalf("HTTP options must pass through unchanged, got %+v", got)
|
||||
}
|
||||
|
||||
ftpExtra := &ftp.OptsExtra{Connections: 4, AutoTorrent: &yes, DeleteTorrentAfterDownload: &yes, AutoExtract: &yes, ArchivePassword: "pw", DeleteAfterExtract: true}
|
||||
got := postDownloadExtra(&fetcher.FetcherMeta{Res: file, Opts: &base.Options{Extra: ftpExtra}})
|
||||
if got == nil || got.AutoTorrent != &yes || got.DeleteTorrentAfterDownload != &yes || got.AutoExtract != &yes ||
|
||||
got.ArchivePassword != "pw" || !got.DeleteAfterExtract {
|
||||
t.Fatalf("FTP options were not carried over: %+v", got)
|
||||
}
|
||||
if got := postDownloadExtra(&fetcher.FetcherMeta{Res: folder, Opts: &base.Options{Extra: ftpExtra}}); got != nil {
|
||||
t.Fatalf("an FTP folder has no single file to extract, got %+v", got)
|
||||
}
|
||||
if got := postDownloadExtra(&fetcher.FetcherMeta{Res: file, Opts: &base.Options{}}); got != nil {
|
||||
t.Fatalf("no options, got %+v", got)
|
||||
}
|
||||
}
|
||||
@@ -1,151 +0,0 @@
|
||||
package download
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/GopeedLab/gopeed/internal/githubmirror"
|
||||
"github.com/GopeedLab/gopeed/internal/httpclient"
|
||||
)
|
||||
|
||||
func (d *Downloader) resourceClient(publicOnly bool) (*http.Client, error) {
|
||||
cfg, err := d.GetConfig()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
options := httpclient.Options{
|
||||
Client: httpclient.ClientOptions{Timeout: 30 * time.Second},
|
||||
Transport: httpclient.TransportOptions{Proxy: cfg.Proxy.ToHandler(), TLSHandshakeTimeout: 10 * time.Second},
|
||||
}
|
||||
if publicOnly {
|
||||
// A configured upstream proxy is trusted, but target URLs are still checked.
|
||||
options.Client.CheckRedirect = func(req *http.Request, via []*http.Request) error {
|
||||
if len(via) >= 5 {
|
||||
return errors.New("too many redirects")
|
||||
}
|
||||
return validatePublicURL(req.Context(), req.URL.String())
|
||||
}
|
||||
if cfg.Proxy == nil || !cfg.Proxy.Enable {
|
||||
options.Transport.DialContext = func(ctx context.Context, network, address string) (net.Conn, error) {
|
||||
host, port, err := net.SplitHostPort(address)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ips, err := publicIPs(ctx, host)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var last error
|
||||
for _, ip := range ips {
|
||||
conn, err := (&net.Dialer{Timeout: 10 * time.Second}).DialContext(ctx, network, net.JoinHostPort(ip.String(), port))
|
||||
if err == nil {
|
||||
return conn, nil
|
||||
}
|
||||
last = err
|
||||
}
|
||||
return nil, last
|
||||
}
|
||||
}
|
||||
}
|
||||
return httpclient.NewClient(options)
|
||||
}
|
||||
func publicIPs(ctx context.Context, host string) ([]net.IPAddr, error) {
|
||||
ips, err := net.DefaultResolver.LookupIPAddr(ctx, host)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(ips) == 0 {
|
||||
return nil, errors.New("target has no addresses")
|
||||
}
|
||||
for _, ip := range ips {
|
||||
if !ip.IP.IsGlobalUnicast() || ip.IP.IsPrivate() || ip.IP.IsLoopback() || ip.IP.IsLinkLocalUnicast() || ip.IP.IsUnspecified() {
|
||||
return nil, errors.New("private or local resource target is not allowed")
|
||||
}
|
||||
}
|
||||
return ips, nil
|
||||
}
|
||||
func validatePublicURL(ctx context.Context, raw string) error {
|
||||
u, err := url.Parse(raw)
|
||||
if err != nil || u == nil || (u.Scheme != "https" && u.Scheme != "http") || u.Host == "" || u.User != nil {
|
||||
return errors.New("invalid resource URL")
|
||||
}
|
||||
_, err = publicIPs(ctx, u.Hostname())
|
||||
return err
|
||||
}
|
||||
func (d *Downloader) fetchPublicResource(ctx context.Context, raw string, limit int64, validate func([]byte) error) ([]byte, error) {
|
||||
return d.fetchResource(ctx, raw, limit, true, validate)
|
||||
}
|
||||
|
||||
// fetchResource races complete, validated responses. A successful HEAD alone
|
||||
// cannot win, and failed candidates never discard a usable official response.
|
||||
func (d *Downloader) fetchResource(ctx context.Context, raw string, limit int64, publicOnly bool, validate func([]byte) error) ([]byte, error) {
|
||||
cfg, err := d.GetConfig()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
client, err := d.resourceClient(publicOnly)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer client.CloseIdleConnections()
|
||||
return githubmirror.Race(ctx, cfg.GithubMirror, raw,
|
||||
githubmirror.Options[[]byte]{Kind: githubmirror.Resource, Timeout: 30 * time.Second},
|
||||
func(ctx context.Context, candidate string) ([]byte, error) {
|
||||
if publicOnly {
|
||||
if err := validatePublicURL(ctx, candidate); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
request, err := http.NewRequestWithContext(ctx, http.MethodGet, candidate, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
response, err := client.Do(request)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer response.Body.Close()
|
||||
if response.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("resource returned HTTP %d", response.StatusCode)
|
||||
}
|
||||
body, err := io.ReadAll(io.LimitReader(response.Body, limit+1))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if int64(len(body)) > limit {
|
||||
return nil, errors.New("resource exceeds size limit")
|
||||
}
|
||||
if len(body) == 0 {
|
||||
return nil, errors.New("empty resource")
|
||||
}
|
||||
if validate != nil {
|
||||
if err := validate(body); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return body, nil
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
// githubResourceCandidates accepts official GitHub HTTPS resource URLs only.
|
||||
func githubResourceCandidates(config *githubmirror.Config, raw string) ([]string, error) {
|
||||
u, err := url.Parse(raw)
|
||||
if err != nil || u == nil || u.Scheme != "https" || u.User != nil || u.Port() != "" || u.Fragment != "" || strings.Contains(raw, "#") {
|
||||
return nil, ErrInvalidMirrorURL
|
||||
}
|
||||
u.Host = strings.ToLower(u.Host)
|
||||
switch u.Host {
|
||||
case "github.com", "raw.githubusercontent.com":
|
||||
default:
|
||||
return nil, ErrInvalidMirrorURL
|
||||
}
|
||||
return githubmirror.Candidates(config, u.String(), githubmirror.Resource), nil
|
||||
}
|
||||
@@ -40,7 +40,7 @@ func TestDownloader_HlsLiveSite(t *testing.T) {
|
||||
if referer != "" {
|
||||
req.Extra = map[string]any{"header": map[string]string{"Referer": referer}}
|
||||
}
|
||||
rr, err := downloader.Resolve(req)
|
||||
rr, err := downloader.Resolve(req, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("resolve: %v", err)
|
||||
}
|
||||
|
||||
@@ -89,7 +89,7 @@ func TestDownloader_HlsProtocolEndToEnd(t *testing.T) {
|
||||
|
||||
req := hlsResolveReferer()
|
||||
req.URL = server.URL + "/master.m3u8"
|
||||
rr, err := downloader.Resolve(req)
|
||||
rr, err := downloader.Resolve(req, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -1,35 +0,0 @@
|
||||
package download
|
||||
|
||||
import (
|
||||
"errors"
|
||||
|
||||
"github.com/GopeedLab/gopeed/pkg/base"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrMirrorProviderNotFound = errors.New("mirror provider not found")
|
||||
ErrInvalidMirrorURL = errors.New("invalid mirror resource URL")
|
||||
)
|
||||
|
||||
// Each provider owns URL validation and candidate generation using its own
|
||||
// configuration. Providers only generate URLs; they must not fetch resources.
|
||||
type mirrorCandidateProvider func(config *base.DownloaderStoreConfig, raw string) ([]string, error)
|
||||
|
||||
// Add providers here without changing the public API or its client transport.
|
||||
var mirrorCandidateProviders = map[string]mirrorCandidateProvider{
|
||||
"github": func(config *base.DownloaderStoreConfig, raw string) ([]string, error) {
|
||||
return githubResourceCandidates(config.GithubMirror, raw)
|
||||
},
|
||||
}
|
||||
|
||||
func (d *Downloader) GetMirrorCandidates(provider, raw string) ([]string, error) {
|
||||
candidates, ok := mirrorCandidateProviders[provider]
|
||||
if !ok {
|
||||
return nil, ErrMirrorProviderNotFound
|
||||
}
|
||||
config, err := d.GetConfig()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return candidates(config, raw)
|
||||
}
|
||||
+9
-47
@@ -10,7 +10,6 @@ import (
|
||||
"github.com/GopeedLab/gopeed/internal/production"
|
||||
"github.com/GopeedLab/gopeed/internal/protocol/bt"
|
||||
"github.com/GopeedLab/gopeed/internal/protocol/ed2k"
|
||||
"github.com/GopeedLab/gopeed/internal/protocol/ftp"
|
||||
"github.com/GopeedLab/gopeed/internal/protocol/hls"
|
||||
"github.com/GopeedLab/gopeed/internal/protocol/http"
|
||||
"github.com/GopeedLab/gopeed/pkg/base"
|
||||
@@ -22,55 +21,24 @@ import (
|
||||
type ResolveResult struct {
|
||||
ID string `json:"id"`
|
||||
Res *base.Resource `json:"res"`
|
||||
// SuggestedDownloadDir is the category directory the created task would be
|
||||
// routed to. It is only set when auto categorize is enabled, the resource is
|
||||
// a single file, a category matches and its directory passes the white list
|
||||
// check. It is a hint for the ui, the create call decides the final path.
|
||||
SuggestedDownloadDir string `json:"suggestedDownloadDir,omitempty"`
|
||||
}
|
||||
|
||||
// TaskGroup stores the directory shared by independently scheduled tasks.
|
||||
type TaskGroup struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Path string `json:"path"`
|
||||
CreatedAt time.Time `json:"createdAt"`
|
||||
}
|
||||
|
||||
// CreateTaskBatchResult is shared by ordinary and grouped batches.
|
||||
type CreateTaskBatchResult struct {
|
||||
TaskIDs []string `json:"taskIds"`
|
||||
ParentID string `json:"parentId,omitempty"`
|
||||
}
|
||||
|
||||
// Task represents a download unit.
|
||||
// Options flow path:
|
||||
// REST Request (model.CreateTask/ResolveTask) -> pkg/download Downloader -> Task.Meta.Opts (base.Options) -> Fetcher.Resolve/Start (e.g. internal/protocol/http/fetcher.go)
|
||||
type Task struct {
|
||||
ID string `json:"id"`
|
||||
Protocol string `json:"protocol"`
|
||||
Meta *fetcher.FetcherMeta `json:"meta"`
|
||||
// ParentID identifies the independently stored group.
|
||||
ParentID string `json:"parentId,omitempty"`
|
||||
Status base.Status `json:"status"`
|
||||
Uploading bool `json:"uploading"`
|
||||
Progress *Progress `json:"progress"`
|
||||
CreatedAt time.Time `json:"createdAt"`
|
||||
UpdatedAt time.Time `json:"updatedAt"`
|
||||
ID string `json:"id"`
|
||||
Protocol string `json:"protocol"`
|
||||
Meta *fetcher.FetcherMeta `json:"meta"`
|
||||
Status base.Status `json:"status"`
|
||||
Uploading bool `json:"uploading"`
|
||||
Progress *Progress `json:"progress"`
|
||||
CreatedAt time.Time `json:"createdAt"`
|
||||
UpdatedAt time.Time `json:"updatedAt"`
|
||||
// DoneAt is the time when the task completed; nil if it has never
|
||||
// finished. A done task never leaves the done state, so the value is
|
||||
// written once and kept.
|
||||
DoneAt *time.Time `json:"doneAt"`
|
||||
// NeedsPreparation marks a task that has not run its first preparation
|
||||
// yet. The preparation decides the download directory of the task and
|
||||
// resolves the duplicate name in it. It runs once, after the resource is
|
||||
// resolved, and the flag is cleared afterwards, so a queued, paused or
|
||||
// retried task is still prepared while a prepared task is never
|
||||
// classified or renamed again. A task stored before this flag existed
|
||||
// does not have it and keeps its directory and name.
|
||||
NeedsPreparation bool `json:"needsPreparation,omitempty"`
|
||||
|
||||
group *TaskGroup
|
||||
|
||||
producer production.Source
|
||||
producerReceived int64
|
||||
@@ -162,12 +130,10 @@ func (t *Task) MarshalJSON() ([]byte, error) {
|
||||
type rawTaskType Task
|
||||
jsonTask := struct {
|
||||
*rawTaskType
|
||||
Name string `json:"name"`
|
||||
Group *TaskGroup `json:"group,omitempty"`
|
||||
Name string `json:"name"`
|
||||
}{
|
||||
(*rawTaskType)(t),
|
||||
t.Name(),
|
||||
t.group,
|
||||
}
|
||||
return json.Marshal(jsonTask)
|
||||
}
|
||||
@@ -250,10 +216,6 @@ func (cfg *DownloaderConfig) Init() *DownloaderConfig {
|
||||
}
|
||||
if len(cfg.FetchManagers) == 0 {
|
||||
cfg.FetchManagers = []fetcher.FetcherManager{
|
||||
// FTP matches by URL scheme only. It comes before the managers that
|
||||
// match by file extension, so ftp://host/x.torrent or x.m3u8 is
|
||||
// downloaded as a file over FTP.
|
||||
new(ftp.FetcherManager),
|
||||
new(hls.FetcherManager),
|
||||
new(http.FetcherManager),
|
||||
new(bt.FetcherManager),
|
||||
|
||||
@@ -1,172 +0,0 @@
|
||||
package download
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"strings"
|
||||
|
||||
"github.com/GopeedLab/gopeed/internal/fetcher"
|
||||
"github.com/GopeedLab/gopeed/pkg/util"
|
||||
)
|
||||
|
||||
func (d *Downloader) protocolManager(name string) (fetcher.FetcherManager, error) {
|
||||
for _, fm := range d.cfg.FetchManagers {
|
||||
if fm.Name() == name {
|
||||
return fm, nil
|
||||
}
|
||||
}
|
||||
return nil, fmt.Errorf("%w: %s", ErrUnSupportedProtocol, name)
|
||||
}
|
||||
func jsonObject(v any) (map[string]json.RawMessage, error) {
|
||||
raw, err := json.Marshal(v)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var object map[string]json.RawMessage
|
||||
if err = json.Unmarshal(raw, &object); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if object == nil {
|
||||
object = map[string]json.RawMessage{}
|
||||
}
|
||||
return object, nil
|
||||
}
|
||||
func normalizeProtocol(fm fetcher.FetcherManager, v any) (any, error) {
|
||||
raw, err := json.Marshal(v)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
target := fm.DefaultConfig()
|
||||
if target == nil {
|
||||
return nil, errors.New("protocol does not expose configuration")
|
||||
}
|
||||
// Managers may return either typed pointers or value/map defaults.
|
||||
valueDefault := reflect.TypeOf(target).Kind() != reflect.Pointer
|
||||
if valueDefault {
|
||||
pointer := reflect.New(reflect.TypeOf(target))
|
||||
defaults, err := json.Marshal(target)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err = json.Unmarshal(defaults, pointer.Interface()); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
target = pointer.Interface()
|
||||
}
|
||||
if err = json.Unmarshal(raw, target); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if normalizer, ok := fm.(fetcher.ConfigNormalizer); ok {
|
||||
raw, err = json.Marshal(target)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return normalizer.NormalizeConfig(raw)
|
||||
}
|
||||
if valueDefault {
|
||||
return reflect.ValueOf(target).Elem().Interface(), nil
|
||||
}
|
||||
return target, nil
|
||||
}
|
||||
|
||||
// preserveProtocolState ignores all client-supplied state, including null or
|
||||
// malformed state values, before decoding/validating the writable config.
|
||||
func preserveProtocolState(next, previous any) (any, error) {
|
||||
object, err := jsonObject(next)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
before, err := jsonObject(previous)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Match encoding/json field folding as well as the canonical spelling.
|
||||
for key := range object {
|
||||
if strings.EqualFold(key, "state") {
|
||||
delete(object, key)
|
||||
}
|
||||
}
|
||||
if state, ok := before["state"]; ok {
|
||||
object["state"] = state
|
||||
}
|
||||
return object, nil
|
||||
}
|
||||
func (d *Downloader) readProtocolConfig(name string) (any, error) {
|
||||
if _, err := d.protocolManager(name); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
d.configLock.RLock()
|
||||
defer d.configLock.RUnlock()
|
||||
value := d.cfg.ProtocolConfig[name]
|
||||
return *util.DeepClone(&value), nil
|
||||
}
|
||||
|
||||
// updateProtocolState is the only internal write path for backend-owned state.
|
||||
// It never publishes callback edits outside the reserved state section.
|
||||
func (d *Downloader) updateProtocolState(name string, change func(config, state json.RawMessage) (any, error)) error {
|
||||
fm, err := d.protocolManager(name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
d.configLock.Lock()
|
||||
defer d.configLock.Unlock()
|
||||
if d.closed.Load() {
|
||||
return errors.New("downloader is closed")
|
||||
}
|
||||
next := util.DeepClone(d.cfg.DownloaderStoreConfig)
|
||||
object, err := jsonObject(next.ProtocolConfig[name])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
state, ok := object["state"]
|
||||
if !ok {
|
||||
return fmt.Errorf("protocol %s does not expose persistent state", name)
|
||||
}
|
||||
raw, err := json.Marshal(object)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
value, err := change(raw, state)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// Keep the transaction's original config isolated from callback mutations.
|
||||
object["state"], err = json.Marshal(value)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
normalized, err := normalizeProtocol(fm, object)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
next.ProtocolConfig[name] = normalized
|
||||
if err = d.storage.Put(bucketConfig, "config", next); err != nil {
|
||||
return err
|
||||
}
|
||||
d.cfg.DownloaderStoreConfig.ProtocolConfig = next.ProtocolConfig
|
||||
return nil
|
||||
}
|
||||
|
||||
// initProtocolConfigs loads the current schema and initializes missing defaults.
|
||||
// This release intentionally does not import settings from older config layouts.
|
||||
func (d *Downloader) initProtocolConfigs() error {
|
||||
cfg := d.cfg.DownloaderStoreConfig
|
||||
cfg.Init()
|
||||
if err := cfg.GithubMirror.Validate(); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, fm := range d.cfg.FetchManagers {
|
||||
value, ok := cfg.ProtocolConfig[fm.Name()]
|
||||
if !ok {
|
||||
value = map[string]any{}
|
||||
}
|
||||
normalized, err := normalizeProtocol(fm, value)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s config: %w", fm.Name(), err)
|
||||
}
|
||||
cfg.ProtocolConfig[fm.Name()] = normalized
|
||||
}
|
||||
return d.storage.Put(bucketConfig, "config", cfg)
|
||||
}
|
||||
@@ -1,250 +0,0 @@
|
||||
package download
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/GopeedLab/gopeed/internal/fetcher"
|
||||
"github.com/GopeedLab/gopeed/pkg/util"
|
||||
)
|
||||
|
||||
var ErrProtocolJobNotFound = errors.New("protocol job not found")
|
||||
var jobNamePattern = regexp.MustCompile(`^[a-z][a-z0-9-]*$`)
|
||||
|
||||
type protocolJobState struct {
|
||||
LastSuccess time.Time `json:"lastSuccess"`
|
||||
Failures int `json:"failures"`
|
||||
}
|
||||
type protocolJobRun struct {
|
||||
done chan struct{}
|
||||
err error
|
||||
source fetcher.TriggerSource
|
||||
}
|
||||
type registeredJob struct {
|
||||
protocol string
|
||||
spec fetcher.CronJob
|
||||
mu sync.Mutex
|
||||
state protocolJobState
|
||||
active *protocolJobRun
|
||||
next time.Time
|
||||
wake chan struct{}
|
||||
force atomic.Bool
|
||||
}
|
||||
type protocolScheduler struct {
|
||||
d *Downloader
|
||||
jobs map[string]*registeredJob
|
||||
wg sync.WaitGroup
|
||||
// gate prevents adding work while shutdown waits for it.
|
||||
gate sync.Mutex
|
||||
stopped bool
|
||||
}
|
||||
|
||||
func (d *Downloader) setupProtocolJobs() error {
|
||||
scheduler := &protocolScheduler{d: d, jobs: map[string]*registeredJob{}}
|
||||
for _, fm := range d.cfg.FetchManagers {
|
||||
provider, ok := fm.(fetcher.CronProvider)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
name := fm.Name()
|
||||
host := fetcher.ProtocolHost{
|
||||
Read: func(dst any) error {
|
||||
value, err := d.readProtocolConfig(name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return util.MapToStruct(value, dst)
|
||||
},
|
||||
UpdateState: func(change func(config, state json.RawMessage) (any, error)) error {
|
||||
return d.updateProtocolState(name, change)
|
||||
},
|
||||
Fetch: func(ctx context.Context, raw string, limit int64, validate func([]byte) error) ([]byte, error) {
|
||||
return d.fetchPublicResource(ctx, raw, limit, validate)
|
||||
},
|
||||
}
|
||||
for _, spec := range provider.CronJobs(host) {
|
||||
if !jobNamePattern.MatchString(spec.Name) || spec.Every <= 0 || spec.Timeout <= 0 || spec.Run == nil {
|
||||
return fmt.Errorf("invalid protocol job: %s/%s", name, spec.Name)
|
||||
}
|
||||
key := name + "/" + spec.Name
|
||||
if _, ok := scheduler.jobs[key]; ok {
|
||||
return fmt.Errorf("duplicate protocol job: %s", key)
|
||||
}
|
||||
entry := ®isteredJob{protocol: name, spec: spec, wake: make(chan struct{}, 1)}
|
||||
if _, err := d.storage.Get(bucketProtocolJobs, key, &entry.state); err != nil {
|
||||
return err
|
||||
}
|
||||
scheduler.jobs[key] = entry
|
||||
}
|
||||
}
|
||||
d.protocolJobs = scheduler
|
||||
return nil
|
||||
}
|
||||
func (s *protocolScheduler) start() {
|
||||
for key, entry := range s.jobs {
|
||||
s.wg.Add(1)
|
||||
go s.loop(key, entry)
|
||||
}
|
||||
}
|
||||
func (s *protocolScheduler) signal(entry *registeredJob) {
|
||||
select {
|
||||
case entry.wake <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
func (s *protocolScheduler) changed(protocol string) {
|
||||
for _, entry := range s.jobs {
|
||||
if protocol == "" || entry.protocol == protocol {
|
||||
entry.force.Store(true)
|
||||
s.signal(entry)
|
||||
}
|
||||
}
|
||||
}
|
||||
func (s *protocolScheduler) loop(key string, entry *registeredJob) {
|
||||
defer s.wg.Done()
|
||||
entry.mu.Lock()
|
||||
if entry.next.IsZero() {
|
||||
entry.next = entry.state.LastSuccess.Add(entry.spec.Every)
|
||||
if entry.next.Before(time.Now()) {
|
||||
entry.next = time.Now().Add(time.Second)
|
||||
}
|
||||
}
|
||||
next := entry.next
|
||||
entry.mu.Unlock()
|
||||
timer := time.NewTimer(time.Until(next))
|
||||
defer timer.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-s.d.backgroundCtx.Done():
|
||||
return
|
||||
case <-entry.wake:
|
||||
entry.mu.Lock()
|
||||
next = entry.next
|
||||
entry.mu.Unlock()
|
||||
if entry.force.Swap(false) {
|
||||
next = time.Now()
|
||||
}
|
||||
case <-timer.C:
|
||||
_ = s.trigger(s.d.backgroundCtx, key, fetcher.TriggerScheduled)
|
||||
entry.mu.Lock()
|
||||
next = entry.next
|
||||
entry.mu.Unlock()
|
||||
}
|
||||
|
||||
if !timer.Stop() {
|
||||
select {
|
||||
case <-timer.C:
|
||||
default:
|
||||
}
|
||||
}
|
||||
timer.Reset(max(time.Until(next), time.Millisecond))
|
||||
}
|
||||
}
|
||||
func (s *protocolScheduler) trigger(ctx context.Context, key string, source fetcher.TriggerSource) error {
|
||||
entry, ok := s.jobs[key]
|
||||
if !ok {
|
||||
return fmt.Errorf("%w: %s", ErrProtocolJobNotFound, key)
|
||||
}
|
||||
for {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
s.gate.Lock()
|
||||
if s.stopped || s.d.closed.Load() {
|
||||
s.gate.Unlock()
|
||||
return errors.New("downloader is closed")
|
||||
}
|
||||
entry.mu.Lock()
|
||||
run := entry.active
|
||||
if run == nil {
|
||||
run = &protocolJobRun{done: make(chan struct{}), source: source}
|
||||
entry.active = run
|
||||
s.wg.Add(1)
|
||||
go s.execute(key, entry, run, source)
|
||||
}
|
||||
entry.mu.Unlock()
|
||||
s.gate.Unlock()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-run.done:
|
||||
// A manual request must still run when the scheduled attempt it
|
||||
// joined was skipped (for example, automatic refresh is disabled).
|
||||
if source == fetcher.TriggerManual && run.source == fetcher.TriggerScheduled && errors.Is(run.err, fetcher.ErrJobSkipped) {
|
||||
continue
|
||||
}
|
||||
return run.err
|
||||
}
|
||||
}
|
||||
}
|
||||
func (s *protocolScheduler) execute(key string, entry *registeredJob, run *protocolJobRun, source fetcher.TriggerSource) {
|
||||
defer s.wg.Done()
|
||||
ctx, cancel := context.WithTimeout(s.d.backgroundCtx, entry.spec.Timeout)
|
||||
defer cancel()
|
||||
err := func() (err error) {
|
||||
defer func() {
|
||||
if v := recover(); v != nil {
|
||||
err = fmt.Errorf("protocol job panic: %v", v)
|
||||
}
|
||||
}()
|
||||
return entry.spec.Run(ctx, source)
|
||||
}()
|
||||
if err == nil {
|
||||
err = ctx.Err()
|
||||
}
|
||||
entry.mu.Lock()
|
||||
switch {
|
||||
case err == nil:
|
||||
entry.state.LastSuccess = time.Now().UTC()
|
||||
entry.state.Failures = 0
|
||||
case errors.Is(err, fetcher.ErrJobSkipped), errors.Is(err, fetcher.ErrConfigChanged), errors.Is(err, context.Canceled):
|
||||
default:
|
||||
entry.state.Failures++
|
||||
}
|
||||
if storeErr := s.d.storage.Put(bucketProtocolJobs, key, entry.state); storeErr != nil {
|
||||
err = errors.Join(err, storeErr)
|
||||
}
|
||||
switch {
|
||||
case err == nil:
|
||||
entry.next = entry.state.LastSuccess.Add(entry.spec.Every)
|
||||
case errors.Is(err, fetcher.ErrJobSkipped):
|
||||
entry.next = time.Now().Add(entry.spec.Every)
|
||||
case errors.Is(err, fetcher.ErrConfigChanged):
|
||||
entry.next = time.Now().Add(time.Second)
|
||||
default:
|
||||
retry := time.Minute * time.Duration(1<<min(max(entry.state.Failures-1, 0), 6))
|
||||
entry.next = time.Now().Add(retry)
|
||||
}
|
||||
run.err = err
|
||||
entry.active = nil
|
||||
close(run.done)
|
||||
entry.mu.Unlock()
|
||||
if err != nil && !errors.Is(err, fetcher.ErrJobSkipped) && !errors.Is(err, context.Canceled) {
|
||||
s.d.Logger.Warn().Err(err).Str("job", key).Str("source", string(source)).Msg("protocol job failed")
|
||||
}
|
||||
if source == fetcher.TriggerManual {
|
||||
s.signal(entry)
|
||||
}
|
||||
}
|
||||
func (s *protocolScheduler) close() {
|
||||
s.gate.Lock()
|
||||
s.stopped = true
|
||||
s.gate.Unlock()
|
||||
s.wg.Wait()
|
||||
}
|
||||
func (d *Downloader) TriggerProtocolJob(ctx context.Context, protocol, name string) error {
|
||||
if d.protocolJobs == nil {
|
||||
return ErrProtocolJobNotFound
|
||||
}
|
||||
err := d.protocolJobs.trigger(ctx, protocol+"/"+name, fetcher.TriggerManual)
|
||||
if errors.Is(err, fetcher.ErrJobSkipped) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
@@ -1,89 +0,0 @@
|
||||
package download
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/GopeedLab/gopeed/internal/fetcher"
|
||||
)
|
||||
|
||||
type jobWaitContext struct {
|
||||
context.Context
|
||||
waiting chan struct{}
|
||||
once sync.Once
|
||||
}
|
||||
|
||||
func (c *jobWaitContext) Done() <-chan struct{} {
|
||||
c.once.Do(func() { close(c.waiting) })
|
||||
return c.Context.Done()
|
||||
}
|
||||
|
||||
func TestManualProtocolJobRunsAfterJoinedScheduledSkip(t *testing.T) {
|
||||
entered := make(chan struct{})
|
||||
release := make(chan struct{})
|
||||
var manualCalls atomic.Int32
|
||||
manager := &testJobManager{job: fetcher.CronJob{Name: "refresh", Every: time.Hour, Timeout: time.Second,
|
||||
Run: func(ctx context.Context, source fetcher.TriggerSource) error {
|
||||
if source == fetcher.TriggerManual {
|
||||
manualCalls.Add(1)
|
||||
return nil
|
||||
}
|
||||
close(entered)
|
||||
select {
|
||||
case <-release:
|
||||
return fetcher.ErrJobSkipped
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
},
|
||||
}}
|
||||
d := testBackendDownloader(t, manager)
|
||||
scheduled := make(chan error, 1)
|
||||
go func() {
|
||||
scheduled <- d.protocolJobs.trigger(context.Background(), "test/refresh", fetcher.TriggerScheduled)
|
||||
}()
|
||||
<-entered
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
|
||||
defer cancel()
|
||||
waiter := &jobWaitContext{Context: ctx, waiting: make(chan struct{})}
|
||||
manual := make(chan error, 1)
|
||||
go func() { manual <- d.TriggerProtocolJob(waiter, "test", "refresh") }()
|
||||
<-waiter.waiting
|
||||
close(release)
|
||||
if err := <-scheduled; !errors.Is(err, fetcher.ErrJobSkipped) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := <-manual; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if manualCalls.Load() != 1 {
|
||||
t.Fatal("manual refresh was swallowed by skipped automatic work")
|
||||
}
|
||||
}
|
||||
|
||||
func TestManualProtocolJobFailureKeepsScheduledBackoff(t *testing.T) {
|
||||
var calls atomic.Int32
|
||||
failure := errors.New("offline")
|
||||
manager := &testJobManager{job: fetcher.CronJob{Name: "refresh", Every: time.Hour, Timeout: time.Second,
|
||||
Run: func(context.Context, fetcher.TriggerSource) error { calls.Add(1); return failure },
|
||||
}}
|
||||
d := testBackendDownloader(t, manager)
|
||||
if err := d.TriggerProtocolJob(context.Background(), "test", "refresh"); !errors.Is(err, failure) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
entry := d.protocolJobs.jobs["test/refresh"]
|
||||
entry.mu.Lock()
|
||||
next := entry.next
|
||||
entry.mu.Unlock()
|
||||
if time.Until(next) < 59*time.Second {
|
||||
t.Fatal("manual failure did not schedule retry backoff")
|
||||
}
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
if calls.Load() != 1 {
|
||||
t.Fatalf("manual failure immediately retried %d times", calls.Load())
|
||||
}
|
||||
}
|
||||
@@ -1,340 +0,0 @@
|
||||
package download
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"reflect"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/GopeedLab/gopeed/internal/fetcher"
|
||||
btProtocol "github.com/GopeedLab/gopeed/internal/protocol/bt"
|
||||
httpProtocol "github.com/GopeedLab/gopeed/internal/protocol/http"
|
||||
"github.com/GopeedLab/gopeed/pkg/base"
|
||||
)
|
||||
|
||||
// State tests initialize config without starting network clients or cron loops.
|
||||
func protocolConfigDownloader(t *testing.T, storage Storage, managers ...fetcher.FetcherManager) *Downloader {
|
||||
t.Helper()
|
||||
if storage == nil {
|
||||
storage = NewMemStorage()
|
||||
}
|
||||
d := NewDownloader(&DownloaderConfig{Storage: storage, StorageDir: t.TempDir(), FetchManagers: managers})
|
||||
d.cfg.DownloaderStoreConfig = &base.DownloaderStoreConfig{}
|
||||
if err := storage.Setup([]string{bucketConfig}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { d.cancelBackground(); _ = storage.Close() })
|
||||
if err := d.initProtocolConfigs(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return d
|
||||
}
|
||||
func protocolObject(t *testing.T, d *Downloader, name string) map[string]any {
|
||||
t.Helper()
|
||||
cfg, err := d.GetConfig()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
raw, err := json.Marshal(cfg.ProtocolConfig[name])
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var result map[string]any
|
||||
if err := json.Unmarshal(raw, &result); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return result
|
||||
}
|
||||
func configureBT(t *testing.T, d *Downloader, urls ...string) {
|
||||
t.Helper()
|
||||
cfg, _ := d.GetConfig()
|
||||
cfg.ProtocolConfig["bt"] = map[string]any{"autoUpdateTrackers": false, "trackerSubscribeUrls": urls, "customTrackers": []string{"udp://custom.example:80/announce"}}
|
||||
if err := d.PutConfig(cfg); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
func seedBTState(t *testing.T, d *Downloader) {
|
||||
t.Helper()
|
||||
if err := d.updateProtocolState("bt", func(_, _ json.RawMessage) (any, error) {
|
||||
return map[string]any{
|
||||
"trackerSources": map[string][]string{"https://example.com/list": {"udp://cached.example:80/announce"}},
|
||||
"trackerSubscriptionStatuses": map[string]any{"https://example.com/list": map[string]any{"success": true, "trackerCount": 1, "lastFetchTime": time.Date(2026, 10, 19, 1, 2, 3, 0, time.UTC)}},
|
||||
"lastTrackerUpdateTime": time.Date(2026, 10, 19, 1, 2, 3, 0, time.UTC),
|
||||
}, nil
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFullConfigValidatesProtocolsAndPreservesExplicitValues(t *testing.T) {
|
||||
d := protocolConfigDownloader(t, nil, &httpProtocol.FetcherManager{}, &btProtocol.FetcherManager{})
|
||||
configureBT(t, d)
|
||||
cfg, _ := d.GetConfig()
|
||||
cfg.ProtocolConfig["http"] = map[string]any{"connections": 0, "useServerCtime": false}
|
||||
if err := d.PutConfig(cfg); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := protocolObject(t, d, "http")
|
||||
if got["connections"] != float64(0) || got["useServerCtime"] != false {
|
||||
t.Fatal(got)
|
||||
}
|
||||
for _, invalid := range []any{map[string]any{"connections": "bad"}, map[string]any{"useServerCtime": "bad"}} {
|
||||
cfg, _ := d.GetConfig()
|
||||
cfg.ProtocolConfig["http"] = invalid
|
||||
if err := d.PutConfig(cfg); err == nil {
|
||||
t.Fatalf("invalid protocol config accepted: %v", invalid)
|
||||
}
|
||||
}
|
||||
cfg, _ = d.GetConfig()
|
||||
cfg.DownloadDir = "must-not-publish"
|
||||
cfg.ProtocolConfig["bt"] = map[string]any{"listenPort": -1}
|
||||
if err := d.PutConfig(cfg); err == nil {
|
||||
t.Fatal("invalid BT config accepted")
|
||||
}
|
||||
saved, _ := d.GetConfig()
|
||||
if saved.DownloadDir == "must-not-publish" {
|
||||
t.Fatal("invalid full PUT partially published")
|
||||
}
|
||||
cfg, _ = d.GetConfig()
|
||||
cfg.ProtocolConfig["unknown"] = map[string]any{"state": map[string]any{"injected": true}}
|
||||
if err := d.PutConfig(cfg); err == nil {
|
||||
t.Fatal("unregistered protocol accepted")
|
||||
}
|
||||
// Returned configs are detached snapshots.
|
||||
snapshot, _ := d.GetConfig()
|
||||
snapshot.ProtocolConfig["http"] = map[string]any{"connections": 999}
|
||||
if !reflect.DeepEqual(protocolObject(t, d, "http"), got) {
|
||||
t.Fatal("snapshot changed live config")
|
||||
}
|
||||
}
|
||||
|
||||
func TestProtocolConfigIgnoresUnknownFields(t *testing.T) {
|
||||
for _, mode := range []string{"stored", "put"} {
|
||||
t.Run(mode, func(t *testing.T) {
|
||||
d := protocolConfigDownloader(t, nil, &btProtocol.FetcherManager{})
|
||||
cfg, _ := d.GetConfig()
|
||||
cfg.ProtocolConfig["bt"] = map[string]any{
|
||||
"listenPort": 6881,
|
||||
"autoUpdateTrackers": false,
|
||||
"trackers": []string{"udp://obsolete.example:80/announce"},
|
||||
"state": map[string]any{"obsolete": true},
|
||||
}
|
||||
if mode == "stored" {
|
||||
if err := d.storage.Put(bucketConfig, "config", cfg); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var restored base.DownloaderStoreConfig
|
||||
if exists, err := d.storage.Get(bucketConfig, "config", &restored); err != nil || !exists {
|
||||
t.Fatalf("load stored config: %v %v", exists, err)
|
||||
}
|
||||
d.cfg.DownloaderStoreConfig = &restored
|
||||
if err := d.initProtocolConfigs(); err != nil {
|
||||
t.Fatalf("load config containing removed fields: %v", err)
|
||||
}
|
||||
} else if err := d.PutConfig(cfg); err != nil {
|
||||
t.Fatalf("save config containing unknown fields: %v", err)
|
||||
}
|
||||
actual := protocolObject(t, d, "bt")
|
||||
if actual["listenPort"] != float64(6881) || actual["autoUpdateTrackers"] != false {
|
||||
t.Fatalf("known fields changed: %v", actual)
|
||||
}
|
||||
if _, exists := actual["trackers"]; exists {
|
||||
t.Fatal("removed field retained")
|
||||
}
|
||||
if len(actual["customTrackers"].([]any)) != 0 {
|
||||
t.Fatal("removed trackers migrated into custom trackers")
|
||||
}
|
||||
if _, exists := actual["state"].(map[string]any)["obsolete"]; exists {
|
||||
t.Fatal("unknown state field retained")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestBTStateSurvivesOmittedNullForgedAndStaleFullConfig(t *testing.T) {
|
||||
for _, mode := range []string{"omitted", "null", "forged", "wrong-type", "stale", "omitted-protocol", "omitted-timestamp", "case-variant", "unicode-variant"} {
|
||||
t.Run(mode, func(t *testing.T) {
|
||||
d := protocolConfigDownloader(t, nil, &btProtocol.FetcherManager{})
|
||||
configureBT(t, d, "https://example.com/list")
|
||||
stale, _ := d.GetConfig()
|
||||
seedBTState(t, d)
|
||||
expected := protocolObject(t, d, "bt")["state"]
|
||||
cfg, _ := d.GetConfig()
|
||||
bt := protocolObject(t, d, "bt")
|
||||
switch mode {
|
||||
case "omitted":
|
||||
delete(bt, "state")
|
||||
case "null":
|
||||
bt["state"] = nil
|
||||
case "forged":
|
||||
bt["state"] = map[string]any{"lastTrackerUpdateTime": "2000-01-01T00:00:00Z", "trackerSources": map[string]any{}, "unknown": true}
|
||||
case "wrong-type":
|
||||
bt["state"] = "ignore this before decoding"
|
||||
case "stale":
|
||||
cfg = stale
|
||||
bt = map[string]any{"autoUpdateTrackers": false, "trackerSubscribeUrls": []string{"https://example.com/list"}, "customTrackers": []string{"udp://custom.example:80/announce"}, "state": map[string]any{}}
|
||||
case "omitted-protocol":
|
||||
delete(cfg.ProtocolConfig, "bt")
|
||||
case "case-variant":
|
||||
bt["STATE"] = "ignore this alias"
|
||||
case "unicode-variant":
|
||||
bt["ſtate"] = "ignore Unicode case folding"
|
||||
case "omitted-timestamp":
|
||||
bt["state"] = map[string]any{"trackerSources": map[string]any{}}
|
||||
}
|
||||
if mode != "omitted-protocol" {
|
||||
bt["listenPort"] = 6881
|
||||
cfg.ProtocolConfig["bt"] = bt
|
||||
}
|
||||
if err := d.PutConfig(cfg); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
actual := protocolObject(t, d, "bt")
|
||||
if !reflect.DeepEqual(actual["state"], expected) {
|
||||
t.Fatalf("backend state replaced: %v", actual)
|
||||
}
|
||||
if _, exists := actual["trackers"]; exists {
|
||||
t.Fatal("derived tracker list persisted")
|
||||
}
|
||||
if mode != "omitted-protocol" && actual["listenPort"] != float64(6881) {
|
||||
t.Fatal("writable config did not change")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestBTConfigRemovalPrunesOnlyRemovedState(t *testing.T) {
|
||||
d := protocolConfigDownloader(t, nil, &btProtocol.FetcherManager{})
|
||||
configureBT(t, d, "https://example.com/list")
|
||||
seedBTState(t, d)
|
||||
configureBT(t, d, "https://new.example/list")
|
||||
state := protocolObject(t, d, "bt")["state"].(map[string]any)
|
||||
if len(state["trackerSources"].(map[string]any)) != 0 || len(state["trackerSubscriptionStatuses"].(map[string]any)) != 0 {
|
||||
t.Fatal("removed subscription retained state")
|
||||
}
|
||||
}
|
||||
|
||||
type counterState struct {
|
||||
Count int `json:"count"`
|
||||
Other int `json:"other"`
|
||||
}
|
||||
type counterConfig struct {
|
||||
fetcher.ConfigState[counterState]
|
||||
Writable int `json:"writable"`
|
||||
}
|
||||
type counterManager struct{ httpProtocol.FetcherManager }
|
||||
|
||||
func (*counterManager) Name() string { return "counter" }
|
||||
func (*counterManager) DefaultConfig() any { return &counterConfig{Writable: 7} }
|
||||
func counterHost(d *Downloader) fetcher.ProtocolHost {
|
||||
return fetcher.ProtocolHost{UpdateState: func(change func(config, state json.RawMessage) (any, error)) error {
|
||||
return d.updateProtocolState("counter", change)
|
||||
}}
|
||||
}
|
||||
|
||||
func TestProtocolUpdateStateCommitsOnlyStateAndUsesLatestConfig(t *testing.T) {
|
||||
d := protocolConfigDownloader(t, nil, &counterManager{})
|
||||
host := counterHost(d)
|
||||
var wg sync.WaitGroup
|
||||
for i := 0; i < 20; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
if err := fetcher.UpdateState[counterConfig, counterState](host, func(cfg *counterConfig, state *counterState) error { cfg.Writable = 999; state.Count++; return nil }); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
actual := protocolObject(t, d, "counter")
|
||||
if actual["writable"] != float64(7) || actual["state"].(map[string]any)["count"] != float64(20) {
|
||||
t.Fatal(actual)
|
||||
}
|
||||
// Rejecting a callback must not publish its partial mutation.
|
||||
err := fetcher.UpdateState[counterConfig, counterState](host, func(_ *counterConfig, state *counterState) error { state.Count = 1000; return errors.New("reject") })
|
||||
if err == nil || !reflect.DeepEqual(protocolObject(t, d, "counter"), actual) {
|
||||
t.Fatal("failed callback published state")
|
||||
}
|
||||
if err := d.updateProtocolState("counter", func(_, _ json.RawMessage) (any, error) { return map[string]any{"count": "bad"}, nil }); err == nil {
|
||||
t.Fatal("invalid state accepted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestProtocolStateAndFullPutDoNotLoseConcurrentUpdates(t *testing.T) {
|
||||
d := protocolConfigDownloader(t, nil, &counterManager{})
|
||||
snapshot, _ := d.GetConfig()
|
||||
snapshot.ProtocolConfig["counter"] = map[string]any{"writable": 21}
|
||||
var wg sync.WaitGroup
|
||||
for i := 0; i < 20; i++ {
|
||||
wg.Add(2)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
if err := d.PutConfig(snapshot); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}()
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
if err := fetcher.UpdateState[counterConfig, counterState](counterHost(d), func(cfg *counterConfig, state *counterState) error { state.Count++; return nil }); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
actual := protocolObject(t, d, "counter")
|
||||
if actual["writable"] != float64(21) || actual["state"].(map[string]any)["count"] != float64(20) {
|
||||
t.Fatal(actual)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBTSubscriptionStatusesSurviveStorageRestart(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
storage := NewBoltStorage(dir)
|
||||
d := protocolConfigDownloader(t, storage, &btProtocol.FetcherManager{})
|
||||
configureBT(t, d, "https://example.com/list")
|
||||
seedBTState(t, d)
|
||||
expected := protocolObject(t, d, "bt")["state"]
|
||||
if err := storage.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
reopened := NewBoltStorage(dir)
|
||||
if err := reopened.Setup([]string{bucketConfig}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer reopened.Close()
|
||||
var persisted base.DownloaderStoreConfig
|
||||
if exists, err := reopened.Get(bucketConfig, "config", &persisted); err != nil || !exists {
|
||||
t.Fatalf("reload persisted config: %v %v", exists, err)
|
||||
}
|
||||
raw, _ := json.Marshal(persisted.ProtocolConfig["bt"])
|
||||
var cfg map[string]any
|
||||
if err := json.Unmarshal(raw, &cfg); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(cfg["state"], expected) {
|
||||
t.Fatalf("state lost after reopening storage: %+v", cfg)
|
||||
}
|
||||
}
|
||||
|
||||
type rejectedConfigStorage struct{ Storage }
|
||||
|
||||
func (*rejectedConfigStorage) Put(string, string, any) error {
|
||||
return errors.New("storage unavailable")
|
||||
}
|
||||
|
||||
func TestProtocolStateAndPutStorageFailureDoNotPublish(t *testing.T) {
|
||||
d := protocolConfigDownloader(t, nil, &counterManager{})
|
||||
before := protocolObject(t, d, "counter")
|
||||
d.storage = &rejectedConfigStorage{Storage: d.storage}
|
||||
err := fetcher.UpdateState[counterConfig, counterState](counterHost(d), func(_ *counterConfig, state *counterState) error { state.Count = 99; return nil })
|
||||
if err == nil || !reflect.DeepEqual(protocolObject(t, d, "counter"), before) {
|
||||
t.Fatal("failed state persistence published a change")
|
||||
}
|
||||
cfg, _ := d.GetConfig()
|
||||
cfg.ProtocolConfig["counter"] = map[string]any{"writable": 99}
|
||||
if err := d.PutConfig(cfg); err == nil || !reflect.DeepEqual(protocolObject(t, d, "counter"), before) {
|
||||
t.Fatal("failed config persistence published a change")
|
||||
}
|
||||
}
|
||||
@@ -27,10 +27,6 @@ type ScriptData struct {
|
||||
// ScriptPayload contains the task data
|
||||
type ScriptPayload struct {
|
||||
Task *Task
|
||||
// TaskName is computed before Task is cloned. The clone has no fetcher
|
||||
// manager, so for an unresolved task without Options.Name its Name()
|
||||
// would return "unknown".
|
||||
TaskName string
|
||||
}
|
||||
|
||||
// getScriptPaths extracts script paths from config
|
||||
@@ -107,7 +103,7 @@ func (d *Downloader) executeScriptAtPath(scriptPath string, data *ScriptData) er
|
||||
cmd.Env = append(os.Environ(),
|
||||
fmt.Sprintf("GOPEED_EVENT=%s", data.Event),
|
||||
fmt.Sprintf("GOPEED_TASK_ID=%s", data.Payload.Task.ID),
|
||||
fmt.Sprintf("GOPEED_TASK_NAME=%s", data.Payload.TaskName),
|
||||
fmt.Sprintf("GOPEED_TASK_NAME=%s", data.Payload.Task.Name()),
|
||||
fmt.Sprintf("GOPEED_TASK_STATUS=%s", data.Payload.Task.Status),
|
||||
)
|
||||
|
||||
@@ -141,8 +137,7 @@ func (d *Downloader) triggerScripts(event ScriptEvent, task *Task, err error) {
|
||||
Event: event,
|
||||
Time: time.Now().UnixMilli(),
|
||||
Payload: &ScriptPayload{
|
||||
Task: task.clone(),
|
||||
TaskName: task.Name(),
|
||||
Task: task.clone(),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@@ -3,8 +3,6 @@ package download
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -128,46 +126,6 @@ func TestScript_ExecuteScriptAtPath_NonExistentFile(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
func TestScript_TaskNameOfUnresolvedTask(t *testing.T) {
|
||||
scriptPath := getTestScriptPath(t, envDumpScriptName())
|
||||
ensureScriptExecutable(t, scriptPath)
|
||||
outputFile := filepath.Join(t.TempDir(), "env_output.txt")
|
||||
|
||||
setupScriptTest(t, func(downloader *Downloader) {
|
||||
cfg, _ := downloader.GetConfig()
|
||||
cfg.Script = &base.ScriptConfig{
|
||||
Enable: true,
|
||||
Paths: []string{scriptPath},
|
||||
}
|
||||
downloader.PutConfig(cfg)
|
||||
|
||||
// Without Options.Name and a resource, the name comes from the fetcher
|
||||
// manager, which the cloned task in the script data does not have.
|
||||
url := "http://127.0.0.1/error.bin"
|
||||
fm, err := downloader.parseFm(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
task := NewTask()
|
||||
task.Meta = &fetcher.FetcherMeta{
|
||||
Req: &base.Request{URL: url},
|
||||
Opts: &base.Options{Path: t.TempDir()},
|
||||
}
|
||||
task.fetcherManager = fm
|
||||
if name := task.Name(); name != "error.bin" {
|
||||
t.Fatalf("Expected task name error.bin, got %q", name)
|
||||
}
|
||||
|
||||
t.Setenv("GOPEED_TEST_OUTPUT_FILE", outputFile)
|
||||
downloader.triggerScripts(ScriptEventDownloadDone, task, nil)
|
||||
|
||||
output := waitForFileContains(t, outputFile, "GOPEED_TASK_PATH=", 3*time.Second)
|
||||
if !strings.Contains(output, "GOPEED_TASK_NAME=error.bin") {
|
||||
t.Errorf("Expected GOPEED_TASK_NAME=error.bin in output, got: %s", output)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func createDownloadDoneTask(t *testing.T, downloadDir, fileName string) (*Task, string) {
|
||||
t.Helper()
|
||||
content := []byte("downloaded file")
|
||||
@@ -213,31 +171,6 @@ func waitForFile(t *testing.T, path string, timeout time.Duration) {
|
||||
t.Fatalf("Timeout waiting for file: %s", path)
|
||||
}
|
||||
|
||||
// waitForFileContains waits until the file contains want and returns its content.
|
||||
// Test scripts write the file line by line, so waiting for the file to exist is not enough.
|
||||
func waitForFileContains(t *testing.T, path, want string, timeout time.Duration) string {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(timeout)
|
||||
for time.Now().Before(deadline) {
|
||||
if content, err := os.ReadFile(path); err == nil && strings.Contains(string(content), want) {
|
||||
return string(content)
|
||||
}
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
}
|
||||
content, _ := os.ReadFile(path)
|
||||
t.Fatalf("Timeout waiting for %q in file %s, got: %q", want, path, content)
|
||||
return ""
|
||||
}
|
||||
|
||||
// envDumpScriptName returns the test script that writes the GOPEED_* variables
|
||||
// into the file named by GOPEED_TEST_OUTPUT_FILE.
|
||||
func envDumpScriptName() string {
|
||||
if runtime.GOOS == "windows" {
|
||||
return "env_dump.bat"
|
||||
}
|
||||
return "env_dump.sh"
|
||||
}
|
||||
|
||||
func getTestScriptPath(t *testing.T, name string) string {
|
||||
t.Helper()
|
||||
path := filepath.Join("testdata", "scripts", name)
|
||||
@@ -258,16 +191,13 @@ func ensureScriptExecutable(t *testing.T, scriptPath string) {
|
||||
}
|
||||
|
||||
func setupScriptTest(t *testing.T, fn func(downloader *Downloader)) {
|
||||
t.Helper()
|
||||
// Each test owns its runtime; a closed Downloader cannot be reused.
|
||||
d := NewDownloader(&DownloaderConfig{StorageDir: t.TempDir(), TempDir: t.TempDir()})
|
||||
if err := d.Setup(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := d.Close(); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
})
|
||||
fn(d)
|
||||
defaultDownloader.Setup()
|
||||
defaultDownloader.cfg.StorageDir = ".test_storage"
|
||||
defaultDownloader.cfg.DownloadDir = ".test_download"
|
||||
defer func() {
|
||||
defaultDownloader.Clear()
|
||||
os.RemoveAll(defaultDownloader.cfg.StorageDir)
|
||||
os.RemoveAll(defaultDownloader.cfg.DownloadDir)
|
||||
}()
|
||||
fn(defaultDownloader)
|
||||
}
|
||||
Loaded 100 of 284 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user