mirror of
https://github.com/GopeedLab/gopeed.git
synced 2026-11-10 02:20:28 +00:00
Compare commits
18
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
abefe428ca | ||
|
|
9e9bcd4b01 | ||
|
|
2f378f7238 | ||
|
|
5bd57608d2 | ||
|
|
ebfd113360 | ||
|
|
d69874e13e | ||
|
|
b0d45226d7 | ||
|
|
cd19772099 | ||
|
|
1e36599d52 | ||
|
|
fc0a1e0ef9 | ||
|
|
8719166dbf | ||
|
|
c0b70d319d | ||
|
|
e0951de270 | ||
|
|
be0fb05edd | ||
|
|
91edba9af6 | ||
|
|
7d423e2182 | ||
|
|
903c5c46c4 | ||
|
|
b8ff5ff97f |
No files matched your search
@@ -84,17 +84,6 @@ jobs:
|
||||
- uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: ${{ env.GO_VERSION }}
|
||||
- name: Diagnose Windows compiler selection
|
||||
if: matrix.arch == 'amd64' && github.event.inputs.test == 'true'
|
||||
run: |
|
||||
$ccName = go env CC
|
||||
Get-Command -Name $ccName -CommandType Application | Format-Table Source
|
||||
$cc = (Get-Command -Name $ccName -CommandType Application -ErrorAction Stop | Select-Object -First 1).Source
|
||||
Write-Host "Selected cgo compiler: $cc"
|
||||
& $cc --version
|
||||
foreach ($dll in @("libstdc++-6.dll", "libgcc_s_seh-1.dll", "libwinpthread-1.dll")) {
|
||||
& $cc "-print-file-name=$dll"
|
||||
}
|
||||
- uses: subosito/flutter-action@v2
|
||||
with:
|
||||
channel: ${{ matrix.flutter_channel || 'stable' }}
|
||||
@@ -212,15 +201,7 @@ jobs:
|
||||
iscc.exe setup.iss
|
||||
mv "Output\gopeed.exe" "Output\Gopeed-$TAG-windows-${{ matrix.arch }}.exe"
|
||||
Compress-Archive -Path "Output\Gopeed-$TAG-windows-${{ matrix.arch }}.exe" -DestinationPath "Output\Gopeed-$TAG-windows-${{ matrix.arch }}.zip"
|
||||
- name: Upload Windows test packages
|
||||
if: github.event.inputs.test == 'true'
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: windows-${{ matrix.arch }}-packages
|
||||
path: ui/flutter/build/windows/Output/*
|
||||
if-no-files-found: error
|
||||
- name: Upload
|
||||
if: github.event.inputs.test != 'true'
|
||||
uses: shogo82148/[email protected]
|
||||
with:
|
||||
upload_url: ${{ needs.get-release.outputs.upload_url }}
|
||||
@@ -817,5 +798,5 @@ jobs:
|
||||
linux/arm64
|
||||
linux/arm/v7
|
||||
tags: |
|
||||
liwei2633/gopeed:latest
|
||||
${{ !contains(env.VERSION, 'beta') && 'liwei2633/gopeed:latest' || '' }}
|
||||
liwei2633/gopeed:v${{ env.VERSION }}
|
||||
@@ -86,6 +86,32 @@ jobs:
|
||||
flutter pub get
|
||||
flutter gen-l10n
|
||||
flutter test test/api/downloader_config_test.dart
|
||||
test-flutter-task-context-menu:
|
||||
name: Test task context menus and text inputs
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 20
|
||||
env:
|
||||
CHROME_EXECUTABLE: /usr/bin/google-chrome
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- uses: subosito/flutter-action@v2
|
||||
with:
|
||||
flutter-version: ${{ env.FLUTTER_VERSION }}
|
||||
- name: Prepare Flutter tests
|
||||
working-directory: ui/flutter
|
||||
run: |
|
||||
flutter pub get
|
||||
flutter gen-l10n
|
||||
- name: Test task menu actions and native input policy
|
||||
working-directory: ui/flutter
|
||||
run: |
|
||||
flutter test --no-pub test/features/tasks/presentation/widgets/task_context_menu_test.dart test/shared/widgets/app_text_field_test.dart
|
||||
flutter test --no-pub test/widget_test.dart --plain-name 'task context menu exposes'
|
||||
- name: Test browser menu cancellation and input policy
|
||||
working-directory: ui/flutter
|
||||
run: |
|
||||
"$CHROME_EXECUTABLE" --version
|
||||
flutter test --no-pub --platform chrome test/features/tasks/presentation/widgets/task_context_menu_test.dart test/shared/widgets/app_text_field_test.dart
|
||||
test-webview-integration:
|
||||
name: Test webview integration (${{ matrix.name }})
|
||||
strategy:
|
||||
|
||||
@@ -20,7 +20,7 @@ Beyond core download management, Gopeed offers browser integration, JavaScript e
|
||||
|
||||
Visit ✈ [Official Website](https://gopeed.com)
|
||||
|
||||

|
||||

|
||||
|
||||
## ✨ Features
|
||||
|
||||
|
||||
+1
-1
@@ -20,7 +20,7 @@ Gopeed(**Go Speed** の略)は、Go と Flutter で開発された高速で
|
||||
|
||||
✈ [公式サイト](https://gopeed.com)
|
||||
|
||||

|
||||

|
||||
|
||||
## ✨ 主な機能
|
||||
|
||||
|
||||
+1
-1
@@ -20,7 +20,7 @@ Ngoài việc quản lý tải xuống hằng ngày, Gopeed còn hỗ trợ tíc
|
||||
|
||||
Truy cập ✈ [Trang web chính thức](https://gopeed.com)
|
||||
|
||||

|
||||

|
||||
|
||||
## ✨ Tính năng chính
|
||||
|
||||
|
||||
+1
-1
@@ -20,7 +20,7 @@ Gopeed(全称 **Go Speed**,也叫“够快下载器”)是一款使用 Go
|
||||
|
||||
访问 ✈ [官方网站](https://gopeed.com)
|
||||
|
||||

|
||||

|
||||
|
||||
## ✨ 主要特性
|
||||
|
||||
|
||||
+1
-1
@@ -20,7 +20,7 @@ Gopeed(**Go Speed** 的縮寫)是一款以 Go 與 Flutter 開發的高速、
|
||||
|
||||
瀏覽 ✈ [官方網站](https://gopeed.com)
|
||||
|
||||

|
||||

|
||||
|
||||
## ✨ 主要功能
|
||||
|
||||
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 74 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 77 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 1.3 MiB |
Binary file not shown.
|
After Width: | Height: | Size: 792 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 1.3 MiB |
Binary file not shown.
|
After Width: | Height: | Size: 787 KiB |
@@ -0,0 +1,52 @@
|
||||
# Batch task creation
|
||||
|
||||
`POST /api/v1/tasks/batch` accepts ordinary and grouped batches:
|
||||
|
||||
```json
|
||||
{
|
||||
"group": true,
|
||||
"groupName": "album",
|
||||
"opts": {"path": "/downloads"},
|
||||
"reqs": [
|
||||
{"rid": "resolved-resource-id", "opts": {"selectFiles": [1]}},
|
||||
{"req": {"url": "https://example.com/cover.jpg"}}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
Each member accepts `rid` or `req`. A nonempty `rid` takes priority and creates a
|
||||
task from the cached resolve result. A member's `opts`, when present, replaces
|
||||
batch `opts` as a whole; omitted fields do not inherit individual batch values.
|
||||
Members without `opts` use batch `opts`. BT file selections use the indexes from
|
||||
the resolve result. Requests without `rid` continue to work.
|
||||
|
||||
For groups, batch `opts.path` selects the parent directory. Members inheriting
|
||||
batch options, or with an empty path, use the reserved group directory. Explicit
|
||||
relative member paths are resolved under that directory; do not prefix them with
|
||||
the batch parent path. For example, parent `downloads` and member path `nested`
|
||||
place the member in `downloads/<group name>/nested`. Absolute paths under the
|
||||
parent directory preserve their relative structure within the group. In ordinary
|
||||
batches, an empty member path uses the configured default download directory.
|
||||
|
||||
All members are attempted. If at least one task is persisted, the response is
|
||||
successful and contains the created task IDs. A task that fails to start after
|
||||
being persisted is included; use the existing task controls to retry it.
|
||||
Successful tasks are kept when other members fail. If every member fails, the
|
||||
API returns an error and removes any newly reserved empty group and directory.
|
||||
|
||||
Both ordinary and grouped batches now return an object in `data`:
|
||||
|
||||
```json
|
||||
{"code": 0, "msg": "", "data": {"taskIds": ["task-1", "task-2"], "parentId": "group-id"}}
|
||||
```
|
||||
|
||||
Ordinary batches omit `parentId`. **This replaces the former array response for
|
||||
ordinary batches.** The outer `code`/`msg`/`data` envelope is unchanged. Clients
|
||||
must read `data.taskIds`; there are no per-member results. A successful response
|
||||
completes the batch, including partial success, so clients must not automatically
|
||||
resubmit missing members.
|
||||
|
||||
With `group: true`, the server generates the group ID and reserves its directory.
|
||||
Creation does not accept a client group ID or append to an existing group. Group
|
||||
metadata remains available through the existing task list and detail responses;
|
||||
`parentId` identifies membership for group operations.
|
||||
+7
-1
@@ -1,6 +1,9 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"github.com/GopeedLab/gopeed/cmd"
|
||||
"github.com/GopeedLab/gopeed/pkg/rest/model"
|
||||
)
|
||||
@@ -13,5 +16,8 @@ func main() {
|
||||
Storage: model.StorageBolt,
|
||||
WebEnable: true,
|
||||
}
|
||||
cmd.Start(cfg)
|
||||
if err := cmd.Start(cfg); err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
+7
-5
@@ -16,15 +16,16 @@ import (
|
||||
//go:embed banner.txt
|
||||
var banner string
|
||||
|
||||
func Start(cfg *model.StartConfig) {
|
||||
fmt.Println(banner)
|
||||
func Start(cfg *model.StartConfig) error {
|
||||
srv, listener, err := rest.BuildServer(cfg)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
return err
|
||||
}
|
||||
fmt.Println(banner)
|
||||
downloadCfg, err := rest.Downloader.GetConfig()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
listener.Close()
|
||||
return err
|
||||
}
|
||||
if rest.Downloader.IsFirstLoad() {
|
||||
// Set default download config
|
||||
@@ -62,8 +63,9 @@ func Start(cfg *model.StartConfig) {
|
||||
|
||||
fmt.Printf("Server start success on http://%s\n", listener.Addr().String())
|
||||
if err := srv.Serve(listener); err != nil && err != http.ErrServerClosed {
|
||||
panic(err)
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func watchExit() {
|
||||
|
||||
@@ -0,0 +1,35 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/GopeedLab/gopeed/pkg/rest/model"
|
||||
)
|
||||
|
||||
func TestStartInvalidWebAPITokenReturnsError(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
webAuth *model.WebAuth
|
||||
}{
|
||||
{name: "no credentials"},
|
||||
{name: "missing username", webAuth: &model.WebAuth{Password: "secret"}},
|
||||
{name: "missing password", webAuth: &model.WebAuth{Username: "gopeed"}},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
err := Start(&model.StartConfig{
|
||||
WebEnable: true,
|
||||
ApiToken: "test-token",
|
||||
WebAuth: tt.webAuth,
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("Start() error = nil, want web authentication validation error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "web authentication username and password are required") {
|
||||
t.Fatalf("Start() error = %q, want missing web authentication credentials", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
+2
-2
@@ -52,8 +52,8 @@ func loadCliArgs() *args {
|
||||
flag.StringVar(cfg.Username, "username", "gopeed", "Web Authentication Username")
|
||||
cfg.Password = flag.String("p", "", "Web Authentication Password, if no password is set, web authentication will not be enabled")
|
||||
flag.StringVar(cfg.Password, "password", "", "Web Authentication Password, if no password is set, web authentication will not be enabled")
|
||||
cfg.ApiToken = flag.String("T", "", "API token, it must be configured when using HTTP API in the case of enabling web authentication")
|
||||
flag.StringVar(cfg.ApiToken, "api-token", "", "API token, it must be configured when using HTTP API in the case of enabling web authentication")
|
||||
cfg.ApiToken = flag.String("T", "", "API token (requires web authentication with username and password)")
|
||||
flag.StringVar(cfg.ApiToken, "api-token", "", "API token (requires web authentication with username and password)")
|
||||
flag.BoolVar(&cfg.MCPEnable, "mcp-enable", false, "Enable MCP endpoint")
|
||||
cfg.TempDir = flag.String("temp-dir", "", "Temporary directory (defaults to the OS temporary directory)")
|
||||
cfg.StorageDir = flag.String("d", "", "Storage directory")
|
||||
|
||||
+4
-1
@@ -62,7 +62,10 @@ func main() {
|
||||
WebFS: sub,
|
||||
WebAuth: webAuth,
|
||||
}
|
||||
cmd.Start(cfg)
|
||||
if err := cmd.Start(cfg); err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func isNotBlank(str *string) bool {
|
||||
|
||||
@@ -10,11 +10,13 @@ require (
|
||||
github.com/bodgit/sevenzip v1.6.1
|
||||
github.com/dop251/goja v0.0.0-20260106131823-651366fbe6e3
|
||||
github.com/dop251/goja_nodejs v0.0.0-20240728170619-29b559befffc
|
||||
github.com/fclairamb/ftpserverlib v0.31.0
|
||||
github.com/go-git/go-git/v5 v5.8.1
|
||||
github.com/google/jsonschema-go v0.4.3
|
||||
github.com/gorilla/handlers v1.5.1
|
||||
github.com/gorilla/mux v1.8.0
|
||||
github.com/imroc/req/v3 v3.52.2
|
||||
github.com/jlaffaye/ftp v0.2.4
|
||||
github.com/matoous/go-nanoid/v2 v2.0.0
|
||||
github.com/mattn/go-ieproxy v0.0.12
|
||||
github.com/mholt/archives v0.1.5
|
||||
@@ -23,6 +25,7 @@ require (
|
||||
github.com/pkg/errors v0.9.1
|
||||
github.com/rs/dnscache v0.0.0-20230804202142-fc85eb664529
|
||||
github.com/rs/zerolog v1.31.0
|
||||
github.com/spf13/afero v1.15.0
|
||||
github.com/tetratelabs/wazero v1.12.0
|
||||
github.com/xiaoqidun/setft v0.0.0-20220310121541-be86327699ad
|
||||
go.etcd.io/bbolt v1.4.3
|
||||
@@ -59,7 +62,6 @@ require (
|
||||
github.com/segmentio/asm v1.1.3 // indirect
|
||||
github.com/segmentio/encoding v0.5.4 // indirect
|
||||
github.com/sorairolake/lzip-go v0.3.8 // indirect
|
||||
github.com/spf13/afero v1.15.0 // indirect
|
||||
github.com/ulikunitz/xz v0.5.15 // indirect
|
||||
github.com/yosida95/uritemplate/v3 v3.0.2 // indirect
|
||||
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
|
||||
|
||||
@@ -205,6 +205,8 @@ github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc
|
||||
github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ=
|
||||
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
|
||||
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
|
||||
github.com/fclairamb/ftpserverlib v0.31.0 h1:ws4fLJvuU/mgg1uIgRjCJfc6CrFSgIBAOxVsnawXegE=
|
||||
github.com/fclairamb/ftpserverlib v0.31.0/go.mod h1:C5hUT0sd9P5XsHMxsmpcrT6wdZexu3qtUu1z0Ih6aG0=
|
||||
github.com/felixge/fgprof v0.9.5 h1:8+vR6yu2vvSKn08urWyEuxx75NWPEvybbkBirEpsbVY=
|
||||
github.com/felixge/fgprof v0.9.5/go.mod h1:yKl+ERSa++RYOs32d8K6WEXCB4uXdLls4ZaZPpayhMM=
|
||||
github.com/felixge/httpsnoop v1.0.1 h1:lvB5Jl89CsZtGIWuTcDM1E/vkVs49/Ml7JJe07l8SPQ=
|
||||
@@ -347,6 +349,8 @@ github.com/imroc/req/v3 v3.52.2 h1:xJocr1aIv0a2K9knfBQ4JnZHk+kWTITdjf0mgDg229I=
|
||||
github.com/imroc/req/v3 v3.52.2/go.mod h1:dBGsDloOSZJcFs6PnTjZXYBJK70OXbZpizHBLNqcH2k=
|
||||
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A=
|
||||
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo=
|
||||
github.com/jlaffaye/ftp v0.2.4 h1:JqI85DdkfZj8ntaHk8W9U2SC3jNfiPUU70+wtIWmlfE=
|
||||
github.com/jlaffaye/ftp v0.2.4/go.mod h1:Y1ZnkzxownGIuX7xQ1mQzzkZ21+DbjVIyeKL/V+IIz4=
|
||||
github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
|
||||
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
|
||||
github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
|
||||
@@ -527,6 +531,8 @@ github.com/rs/zerolog v1.31.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWR
|
||||
github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk=
|
||||
github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 h1:GHRpF1pTW19a8tTFrMLUcfWwyC0pnifVo2ClaLq+hP8=
|
||||
github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46/go.mod h1:uAQ5PCi+MFsC7HjREoAz1BU+Mq60+05gifQSsHSDG/8=
|
||||
github.com/secsy/goftp v0.0.0-20200609142545-aa2de14babf4 h1:PT+ElG/UUFMfqy5HrxJxNzj3QBOf7dZwupeVC+mG1Lo=
|
||||
github.com/secsy/goftp v0.0.0-20200609142545-aa2de14babf4/go.mod h1:MnkX001NG75g3p8bhFycnyIjeQoOjGL6CEIsdE/nKSY=
|
||||
github.com/segmentio/asm v1.1.3 h1:WM03sfUOENvvKexOLp+pCqgb/WDjsi7EK8gIsICtzhc=
|
||||
github.com/segmentio/asm v1.1.3/go.mod h1:Ld3L4ZXGNcSLRg4JBsZ3//1+f/TjYl0Mzen/DQy1EJg=
|
||||
github.com/segmentio/encoding v0.5.4 h1:OW1VRern8Nw6ITAtwSZ7Idrl3MXCFwXHPgqESYfvNt0=
|
||||
@@ -566,6 +572,8 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO
|
||||
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
|
||||
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
|
||||
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
||||
github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE=
|
||||
github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg=
|
||||
github.com/tetratelabs/wazero v1.12.0 h1:DuWcpNu/FzgEXgGBDp8J1Spc+CWOvvtvVyjKlaZopYU=
|
||||
github.com/tetratelabs/wazero v1.12.0/go.mod h1:LvKtzl2RqO4gyF27BiXU+nKAjcV8f38U+kP/q2vgxh0=
|
||||
github.com/tidwall/btree v1.8.1 h1:27ehoXvm5AG/g+1VxLS1SD3vRhp/H7LuEfwNvddEdmA=
|
||||
@@ -612,6 +620,8 @@ go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
|
||||
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
|
||||
go.uber.org/mock v0.5.1 h1:ASgazW/qBmR+A32MYFDB6E2POoTgOwT509VP0CT/fjs=
|
||||
go.uber.org/mock v0.5.1/go.mod h1:ge71pBPLYDk7QIi1LupWxdAykm7KIEFchiOqd6z7qMM=
|
||||
go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw=
|
||||
go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg=
|
||||
go4.org v0.0.0-20230225012048-214862532bf5 h1:nifaUDeh+rPaBCMPMQHZmvJf+QdpLFnuQPwx+LxVmtc=
|
||||
go4.org v0.0.0-20230225012048-214862532bf5/go.mod h1:F57wTi5Lrj6WLyswp5EYV1ncrEbFGHD4hhz6S1ZYeaU=
|
||||
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
|
||||
@@ -3,6 +3,7 @@ package controller
|
||||
import (
|
||||
"github.com/GopeedLab/gopeed/internal/tempfiles"
|
||||
"github.com/GopeedLab/gopeed/pkg/base"
|
||||
"github.com/rs/zerolog"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
@@ -15,6 +16,9 @@ type Controller struct {
|
||||
TempDir string
|
||||
GetConfig func(v any)
|
||||
GetProxy func(requestProxy *base.RequestProxy) func(*http.Request) (*url.URL, error)
|
||||
// Logger is the downloader's core logger. It may be nil, in which case
|
||||
// fetchers log nothing.
|
||||
Logger *zerolog.Logger
|
||||
FileController
|
||||
//ContextDialer() (proxy.Dialer, error)
|
||||
}
|
||||
|
||||
@@ -67,16 +67,11 @@ func (f *Fetcher) Setup(ctl *controller.Controller) {
|
||||
return
|
||||
}
|
||||
|
||||
// initClient must be called with lock held.
|
||||
func (f *Fetcher) initClient() (err error) {
|
||||
lock.Lock()
|
||||
defer lock.Unlock()
|
||||
|
||||
if client != nil {
|
||||
return
|
||||
}
|
||||
if closeCtx == nil {
|
||||
closeCtx, closeFunc = context.WithCancel(context.Background())
|
||||
}
|
||||
|
||||
cfg = torrent.NewDefaultClientConfig()
|
||||
cfg.Seed = true
|
||||
@@ -94,12 +89,23 @@ func (f *Fetcher) initClient() (err error) {
|
||||
}
|
||||
|
||||
closeCtx, closeFunc = context.WithCancel(context.Background())
|
||||
go func() {
|
||||
dnsResolver.Run(closeCtx)
|
||||
}()
|
||||
// closeClient may reset closeCtx before the goroutine runs, so pass it as an argument.
|
||||
go dnsResolver.Run(closeCtx)
|
||||
return
|
||||
}
|
||||
|
||||
// addTorrentSpec holds lock from client initialization until the torrent is added,
|
||||
// so closeClient cannot close or reset the client in between.
|
||||
func (f *Fetcher) addTorrentSpec(spec *torrent.TorrentSpec) (*torrent.Torrent, bool, error) {
|
||||
lock.Lock()
|
||||
defer lock.Unlock()
|
||||
|
||||
if err := f.initClient(); err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
return client.AddTorrentSpec(spec)
|
||||
}
|
||||
|
||||
func (f *Fetcher) Resolve(req *base.Request) error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
@@ -748,9 +754,6 @@ func (f *Fetcher) addTorrent(req *base.Request) (err error) {
|
||||
if err = base.ParseReqExtra[bt.ReqExtra](req); err != nil {
|
||||
return
|
||||
}
|
||||
if err = f.initClient(); err != nil {
|
||||
return
|
||||
}
|
||||
schema := util.ParseSchema(req.URL)
|
||||
privateTorrent := false
|
||||
var spec *torrent.TorrentSpec
|
||||
@@ -829,7 +832,7 @@ func (f *Fetcher) addTorrent(req *base.Request) (err error) {
|
||||
s := &deferredFileStorage{}
|
||||
spec.Storage = s
|
||||
var added bool
|
||||
f.torrent, added, err = client.AddTorrentSpec(spec)
|
||||
f.torrent, added, err = f.addTorrentSpec(spec)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/GopeedLab/gopeed/internal/controller"
|
||||
@@ -71,6 +72,82 @@ func TestFetcher_ResolveWithProxy(t *testing.T) {
|
||||
}))
|
||||
}
|
||||
|
||||
func TestFetcher_CloseAfterInit(t *testing.T) {
|
||||
if err := closeClient(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for i := 0; i < 100; i++ {
|
||||
f := buildFetcher(t).(*Fetcher)
|
||||
f.meta.Req = &base.Request{}
|
||||
lock.Lock()
|
||||
err := f.initClient()
|
||||
lock.Unlock()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := closeClient(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetcher_CloseWithoutClient(t *testing.T) {
|
||||
if err := closeClient(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
t.Run("restored task", func(t *testing.T) {
|
||||
_, build := new(FetcherManager).Restore()
|
||||
f := build(&fetcher.FetcherMeta{
|
||||
Req: &base.Request{URL: "./testdata/test.torrent"},
|
||||
Opts: &base.Options{},
|
||||
}, &fetcherData{})
|
||||
f.Setup(controller.NewController())
|
||||
if err := f.Close(); err != nil {
|
||||
t.Errorf("Close() got = %v, want nil", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("failed resolve", func(t *testing.T) {
|
||||
f := buildFetcher(t)
|
||||
if err := f.Resolve(&base.Request{URL: "./testdata/not-exist.torrent"}); err == nil {
|
||||
t.Fatal("Resolve() got = nil, want error")
|
||||
}
|
||||
if err := f.Close(); err != nil {
|
||||
t.Errorf("Close() got = %v, want nil", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestFetcher_ResolveWhileClientCloses(t *testing.T) {
|
||||
if err := closeClient(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { closeClient() })
|
||||
|
||||
var wg sync.WaitGroup
|
||||
for _, u := range []string{"./testdata/test.torrent", "./testdata/ubuntu-22.04-live-server-amd64.iso.torrent"} {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for i := 0; i < 10; i++ {
|
||||
f := buildFetcher(t)
|
||||
// The other goroutine may close the client after the torrent was added.
|
||||
if err := f.Resolve(&base.Request{URL: u}); err != nil && err.Error() != "torrent closed during resolution" {
|
||||
t.Errorf("Resolve(%s) got = %v", u, err)
|
||||
return
|
||||
}
|
||||
f.Close()
|
||||
if err := closeClient(); err != nil {
|
||||
t.Errorf("closeClient() got = %v", err)
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
func doResolve(t *testing.T, fetcher fetcher.Fetcher) {
|
||||
t.Run("Resolve Single File", func(t *testing.T) {
|
||||
err := fetcher.Resolve(&base.Request{
|
||||
|
||||
@@ -0,0 +1,404 @@
|
||||
package ftp
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/textproto"
|
||||
"net/url"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/jlaffaye/ftp"
|
||||
"golang.org/x/net/proxy"
|
||||
|
||||
pftp "github.com/GopeedLab/gopeed/pkg/protocol/ftp"
|
||||
)
|
||||
|
||||
const (
|
||||
connectTimeout = 15 * time.Second
|
||||
readTimeout = 15 * time.Second
|
||||
quitTimeout = 2 * time.Second
|
||||
)
|
||||
|
||||
// target is a parsed FTP URL. It holds the password, so it must never be
|
||||
// formatted into an error or a log line.
|
||||
type target struct {
|
||||
host string // host name or IP, the TLS server name
|
||||
addr string // host:port
|
||||
user string
|
||||
pass string
|
||||
path string // path relative to the login directory, "" for the directory itself
|
||||
tlsMode string
|
||||
}
|
||||
|
||||
// parseTarget reads an ftp, ftps or ftpes URL. The path follows RFC 1738: it
|
||||
// is relative to the login directory, and an encoded leading slash
|
||||
// (ftp://host/%2Fpub) makes it absolute.
|
||||
func parseTarget(rawURL, optTLS string) (*target, error) {
|
||||
u, err := url.Parse(rawURL)
|
||||
if err != nil {
|
||||
// url.Error repeats the whole URL, password included.
|
||||
return nil, errors.New("ftp: invalid URL")
|
||||
}
|
||||
t := &target{}
|
||||
switch strings.ToLower(u.Scheme) {
|
||||
case "ftps":
|
||||
t.tlsMode = pftp.TLSImplicit
|
||||
case "ftpes":
|
||||
t.tlsMode = pftp.TLSExplicit
|
||||
case "ftp":
|
||||
switch strings.ToLower(optTLS) {
|
||||
case "", pftp.TLSNone:
|
||||
t.tlsMode = pftp.TLSNone
|
||||
case pftp.TLSExplicit:
|
||||
t.tlsMode = pftp.TLSExplicit
|
||||
case pftp.TLSImplicit:
|
||||
t.tlsMode = pftp.TLSImplicit
|
||||
default:
|
||||
return nil, fmt.Errorf("ftp: unknown tls mode %q", optTLS)
|
||||
}
|
||||
default:
|
||||
return nil, fmt.Errorf("ftp: unsupported scheme %q", u.Scheme)
|
||||
}
|
||||
t.host = u.Hostname()
|
||||
if t.host == "" {
|
||||
return nil, errors.New("ftp: URL has no host")
|
||||
}
|
||||
port := u.Port()
|
||||
if port == "" {
|
||||
port = "21"
|
||||
if t.tlsMode == pftp.TLSImplicit {
|
||||
port = "990"
|
||||
}
|
||||
}
|
||||
t.addr = net.JoinHostPort(t.host, port)
|
||||
if u.User != nil {
|
||||
// Username and Password are already percent-decoded.
|
||||
t.user = u.User.Username()
|
||||
t.pass, _ = u.User.Password()
|
||||
}
|
||||
if t.user == "" {
|
||||
t.user, t.pass = "anonymous", "anonymous@"
|
||||
}
|
||||
t.path = strings.TrimPrefix(u.Path, "/")
|
||||
return t, nil
|
||||
}
|
||||
|
||||
type dialContextFunc func(ctx context.Context, network, addr string) (net.Conn, error)
|
||||
|
||||
// session is one logged-in control connection. It owns the network
|
||||
// connections it opens, so abort can unblock any read or write in progress.
|
||||
type session struct {
|
||||
*ftp.ServerConn
|
||||
|
||||
ctrl net.Conn // control connection as dialed, under any TLS layer
|
||||
dataHost string // where data connections go
|
||||
|
||||
mu sync.Mutex
|
||||
data net.Conn // current data connection as dialed
|
||||
closed bool
|
||||
stop func() bool
|
||||
}
|
||||
|
||||
// deadline bounds the next control-connection exchange. The control
|
||||
// connection is idle during a transfer, so every exchange sets a new one.
|
||||
func (s *session) deadline() {
|
||||
_ = s.ctrl.SetDeadline(time.Now().Add(readTimeout))
|
||||
}
|
||||
|
||||
// clearDeadline lets the control connection stay idle through a transfer.
|
||||
func (s *session) clearDeadline() {
|
||||
_ = s.ctrl.SetDeadline(time.Time{})
|
||||
}
|
||||
|
||||
// abort closes every connection at once. Pending calls fail immediately.
|
||||
func (s *session) abort() {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.closed = true
|
||||
_ = s.ctrl.Close()
|
||||
if s.data != nil {
|
||||
_ = s.data.Close()
|
||||
}
|
||||
}
|
||||
|
||||
// quit logs out and waits, briefly, for the server to hang up. Servers that
|
||||
// cap logins count a session until they have seen it end, so the next login
|
||||
// would otherwise be refused.
|
||||
func (s *session) quit() {
|
||||
if s.stop != nil {
|
||||
s.stop()
|
||||
}
|
||||
_ = s.ctrl.SetDeadline(time.Now().Add(quitTimeout))
|
||||
if s.ServerConn != nil {
|
||||
_ = s.ServerConn.Quit()
|
||||
}
|
||||
s.abort()
|
||||
}
|
||||
|
||||
func (s *session) setData(c net.Conn) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.closed {
|
||||
_ = c.Close()
|
||||
return net.ErrClosed
|
||||
}
|
||||
s.data = c
|
||||
return nil
|
||||
}
|
||||
|
||||
// ctrlConn is the control connection handed to the ftp library.
|
||||
type ctrlConn struct {
|
||||
net.Conn
|
||||
}
|
||||
|
||||
// RemoteAddr must be a *net.TCPAddr: the library asserts it. A connection
|
||||
// through a proxy may report something else, and its address is not used
|
||||
// for data connections anyway (see dialData).
|
||||
func (c ctrlConn) RemoteAddr() net.Addr {
|
||||
if a, ok := c.Conn.RemoteAddr().(*net.TCPAddr); ok {
|
||||
return a
|
||||
}
|
||||
return &net.TCPAddr{}
|
||||
}
|
||||
|
||||
// Close is called by the library's Quit right after it sends QUIT. Reading
|
||||
// until the server hangs up (bounded by the deadline quit set) tells us the
|
||||
// server has ended the session.
|
||||
func (c ctrlConn) Close() error {
|
||||
_, _ = io.Copy(io.Discard, c.Conn)
|
||||
return c.Conn.Close()
|
||||
}
|
||||
|
||||
// dial opens a control connection, upgrades it to TLS when asked, and logs in.
|
||||
//
|
||||
// The library is given a dial function for every connection. That is how
|
||||
// data connections go through the same proxy as the control connection, but
|
||||
// it also means the library does no TLS on them: with a dial function it
|
||||
// neither wraps an implicit-TLS control connection nor any data connection.
|
||||
// We do both here. Explicit TLS (AUTH TLS) is still done by the library, and
|
||||
// so are PBSZ and PROT P after login. For the same reason the library's
|
||||
// DialWithTimeout and DialWithContext would have no effect: connectTimeout
|
||||
// and ctx are applied to our own dials instead.
|
||||
func (f *Fetcher) dial(ctx context.Context, t *target) (*session, error) {
|
||||
netDial, proxied, err := f.netDialer(t)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var tlsConf *tls.Config
|
||||
if t.tlsMode != pftp.TLSNone {
|
||||
tlsConf = f.tlsConfig(t.host)
|
||||
}
|
||||
|
||||
dialCtx, cancel := context.WithTimeout(ctx, connectTimeout)
|
||||
defer cancel()
|
||||
raw, err := netDial(dialCtx, "tcp", t.addr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ctrl := raw
|
||||
if t.tlsMode == pftp.TLSImplicit {
|
||||
tc := tls.Client(raw, tlsConf)
|
||||
if err := tc.HandshakeContext(dialCtx); err != nil {
|
||||
_ = raw.Close()
|
||||
return nil, err
|
||||
}
|
||||
ctrl = tc
|
||||
}
|
||||
|
||||
s := &session{ctrl: raw, dataHost: t.host}
|
||||
if a, ok := raw.RemoteAddr().(*net.TCPAddr); ok && !proxied {
|
||||
// Same IP as the control connection, as the library itself does,
|
||||
// so a name with several addresses cannot send us elsewhere.
|
||||
s.dataHost = a.IP.String()
|
||||
}
|
||||
s.stop = context.AfterFunc(ctx, s.abort)
|
||||
s.deadline()
|
||||
|
||||
handedOut := false
|
||||
opts := []ftp.DialOption{
|
||||
ftp.DialWithShutTimeout(readTimeout),
|
||||
ftp.DialWithDialFunc(func(network, addr string) (net.Conn, error) {
|
||||
if !handedOut {
|
||||
// Dial asks for the control connection first, once.
|
||||
handedOut = true
|
||||
return ctrlConn{ctrl}, nil
|
||||
}
|
||||
return s.dialData(ctx, netDial, tlsConf, addr)
|
||||
}),
|
||||
}
|
||||
switch t.tlsMode {
|
||||
case pftp.TLSImplicit:
|
||||
opts = append(opts, ftp.DialWithTLS(tlsConf))
|
||||
case pftp.TLSExplicit:
|
||||
opts = append(opts, ftp.DialWithExplicitTLS(tlsConf))
|
||||
}
|
||||
c, err := ftp.Dial(t.addr, opts...)
|
||||
if err != nil {
|
||||
s.stop()
|
||||
s.abort()
|
||||
return nil, err
|
||||
}
|
||||
s.ServerConn = c
|
||||
s.deadline()
|
||||
if err := c.Login(t.user, t.pass); err != nil {
|
||||
s.stop()
|
||||
s.abort()
|
||||
return nil, err
|
||||
}
|
||||
return s, nil
|
||||
}
|
||||
|
||||
// dialData opens a passive data connection. The library takes the host from
|
||||
// the control connection's peer, which through a proxy is the proxy, so only
|
||||
// the port is used here.
|
||||
func (s *session) dialData(ctx context.Context, netDial dialContextFunc, tlsConf *tls.Config, addr string) (net.Conn, error) {
|
||||
_, port, err := net.SplitHostPort(addr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
dialCtx, cancel := context.WithTimeout(ctx, connectTimeout)
|
||||
defer cancel()
|
||||
raw, err := netDial(dialCtx, "tcp", net.JoinHostPort(s.dataHost, port))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := s.setData(raw); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if tlsConf != nil {
|
||||
// The handshake runs on the first read, as in the library, because
|
||||
// some servers only start it once the transfer command is answered.
|
||||
return tls.Client(raw, tlsConf), nil
|
||||
}
|
||||
return raw, nil
|
||||
}
|
||||
|
||||
// tlsConfig returns the TLS settings for one control connection and its data
|
||||
// connections. Certificates are verified unless the request sets
|
||||
// skipVerifyCert, as for HTTP.
|
||||
//
|
||||
// Each call gets its own one-entry session cache, so a data connection
|
||||
// resumes the TLS session of its own control connection. vsftpd
|
||||
// (require_ssl_reuse) only checks that some session was resumed, but ProFTPD's
|
||||
// mod_tls also checks that it is the control connection's; a cache shared by
|
||||
// several workers would hand a data connection another worker's session.
|
||||
// A cache in the manager's base config is kept as given.
|
||||
func (f *Fetcher) tlsConfig(host string) *tls.Config {
|
||||
cfg := &tls.Config{}
|
||||
if f.manager != nil && f.manager.baseTLS != nil {
|
||||
cfg = f.manager.baseTLS.Clone()
|
||||
}
|
||||
cfg.ServerName = host
|
||||
if f.meta.Req != nil && f.meta.Req.SkipVerifyCert {
|
||||
cfg.InsecureSkipVerify = true
|
||||
}
|
||||
if cfg.ClientSessionCache == nil {
|
||||
cfg.ClientSessionCache = tls.NewLRUClientSessionCache(1)
|
||||
}
|
||||
return cfg
|
||||
}
|
||||
|
||||
// netDialer picks how to reach the server, from the same proxy settings the
|
||||
// HTTP protocol reads (the task's proxy, otherwise the global one).
|
||||
func (f *Fetcher) netDialer(t *target) (dialContextFunc, bool, error) {
|
||||
direct := (&net.Dialer{Timeout: connectTimeout}).DialContext
|
||||
pu, err := f.proxyURL(t)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
if pu == nil {
|
||||
return direct, false, nil
|
||||
}
|
||||
switch strings.ToLower(pu.Scheme) {
|
||||
case "socks5", "socks5h":
|
||||
d, err := proxy.FromURL(pu, &net.Dialer{Timeout: connectTimeout})
|
||||
if err != nil {
|
||||
return nil, false, fmt.Errorf("ftp: socks5 proxy: %w", err)
|
||||
}
|
||||
if cd, ok := d.(proxy.ContextDialer); ok {
|
||||
return cd.DialContext, true, nil
|
||||
}
|
||||
return func(_ context.Context, network, addr string) (net.Conn, error) {
|
||||
return d.Dial(network, addr)
|
||||
}, true, nil
|
||||
default:
|
||||
// An HTTP proxy is not used. Every passive data connection goes to
|
||||
// a port the server picks, and HTTP proxies usually allow CONNECT to
|
||||
// port 443 only, so the transfer would fail after the login worked.
|
||||
f.warnOnce.Do(func() {
|
||||
if l := f.logger(); l != nil {
|
||||
l.Warn().Msgf("ftp: the %s proxy at %s cannot carry FTP data connections (HTTP proxy), connecting directly; use a SOCKS5 proxy for FTP", strings.ToLower(pu.Scheme), pu.Host)
|
||||
}
|
||||
})
|
||||
return direct, false, nil
|
||||
}
|
||||
}
|
||||
|
||||
func (f *Fetcher) proxyURL(t *target) (*url.URL, error) {
|
||||
if f.ctl == nil || f.ctl.GetProxy == nil {
|
||||
return nil, nil
|
||||
}
|
||||
handler := f.ctl.GetProxy(f.meta.Req.Proxy)
|
||||
if handler == nil {
|
||||
return nil, nil
|
||||
}
|
||||
// The proxy chooser reads only the URL; the credentials stay out.
|
||||
req := &http.Request{URL: &url.URL{Scheme: "ftp", Host: t.addr, Path: "/" + t.path}, Header: http.Header{}}
|
||||
pu, err := handler(req)
|
||||
if err != nil {
|
||||
return nil, errors.New("ftp: cannot read the proxy settings")
|
||||
}
|
||||
return pu, nil
|
||||
}
|
||||
|
||||
// isLoginRefusal tells a server that is full apart from bad credentials.
|
||||
// 421 always means "not now". 530 means a refused login; it is only a
|
||||
// capacity limit when these credentials have worked before. The library
|
||||
// drops the code when USER itself is refused, so a "too many" message counts
|
||||
// as well.
|
||||
func isLoginRefusal(err error, credentialsWorked bool) bool {
|
||||
var te *textproto.Error
|
||||
if errors.As(err, &te) {
|
||||
switch te.Code {
|
||||
case ftp.StatusNotAvailable:
|
||||
return true
|
||||
case ftp.StatusNotLoggedIn:
|
||||
return credentialsWorked || strings.Contains(strings.ToLower(te.Msg), "too many")
|
||||
}
|
||||
return false
|
||||
}
|
||||
return strings.Contains(strings.ToLower(err.Error()), "too many")
|
||||
}
|
||||
|
||||
// isPermanent reports errors a retry cannot fix: a 5xx reply and a
|
||||
// certificate that does not verify.
|
||||
func isPermanent(err error) bool {
|
||||
var te *textproto.Error
|
||||
if errors.As(err, &te) {
|
||||
return te.Code >= 500
|
||||
}
|
||||
var verifyErr *tls.CertificateVerificationError
|
||||
if errors.As(err, &verifyErr) {
|
||||
return true
|
||||
}
|
||||
var recordErr tls.RecordHeaderError
|
||||
if errors.As(err, &recordErr) {
|
||||
return true
|
||||
}
|
||||
return errors.Is(err, errSizeChanged)
|
||||
}
|
||||
|
||||
func replyCode(err error) int {
|
||||
var te *textproto.Error
|
||||
if errors.As(err, &te) {
|
||||
return te.Code
|
||||
}
|
||||
return 0
|
||||
}
|
||||
@@ -0,0 +1,146 @@
|
||||
package ftp_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/rs/zerolog"
|
||||
|
||||
"github.com/GopeedLab/gopeed/internal/logger"
|
||||
iftp "github.com/GopeedLab/gopeed/internal/protocol/ftp"
|
||||
"github.com/GopeedLab/gopeed/pkg/base"
|
||||
"github.com/GopeedLab/gopeed/pkg/download"
|
||||
pftp "github.com/GopeedLab/gopeed/pkg/protocol/ftp"
|
||||
)
|
||||
|
||||
// syncBuffer is a log sink that the downloader's goroutines can share.
|
||||
type syncBuffer struct {
|
||||
mu sync.Mutex
|
||||
buf bytes.Buffer
|
||||
}
|
||||
|
||||
func (b *syncBuffer) Write(p []byte) (int, error) {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
return b.buf.Write(p)
|
||||
}
|
||||
|
||||
func (b *syncBuffer) String() string {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
return b.buf.String()
|
||||
}
|
||||
|
||||
// taskEvents records the first done or error event of every task, so the
|
||||
// test never reads a task's status while the downloader writes it.
|
||||
type taskEvents struct {
|
||||
mu sync.Mutex
|
||||
final map[string]download.EventKey
|
||||
}
|
||||
|
||||
func (e *taskEvents) listen(ev *download.Event) {
|
||||
if ev.Key != download.EventKeyDone && ev.Key != download.EventKeyError {
|
||||
return
|
||||
}
|
||||
e.mu.Lock()
|
||||
defer e.mu.Unlock()
|
||||
if _, seen := e.final[ev.Task.ID]; !seen {
|
||||
e.final[ev.Task.ID] = ev.Key
|
||||
}
|
||||
}
|
||||
|
||||
func (e *taskEvents) wait(t *testing.T, id string) download.EventKey {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(60 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
e.mu.Lock()
|
||||
key, ok := e.final[id]
|
||||
e.mu.Unlock()
|
||||
if ok {
|
||||
return key
|
||||
}
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
}
|
||||
t.Fatalf("task %s did not finish", id)
|
||||
return ""
|
||||
}
|
||||
|
||||
// TestPasswordNeverLogged runs FTP tasks through the downloader with its core
|
||||
// logger captured: a success, a missing file and a wrong password, all behind
|
||||
// an HTTP proxy that FTP must skip with a warning.
|
||||
func TestPasswordNeverLogged(t *testing.T) {
|
||||
srv := iftp.StartFTPServer(t, iftp.ServerOptions{User: "u", Pass: "p@ss"})
|
||||
want := srv.WriteFile(t, "secret.bin", iftp.RandomBytes(300000, 21))
|
||||
|
||||
d := download.NewDownloader(&download.DownloaderConfig{Storage: download.NewMemStorage(), StorageDir: t.TempDir()})
|
||||
logs := &syncBuffer{}
|
||||
d.Logger = &logger.Logger{Logger: zerolog.New(logs)}
|
||||
events := &taskEvents{final: map[string]download.EventKey{}}
|
||||
d.Listener(events.listen)
|
||||
if err := d.Setup(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Close, not Clear: Clear resets the extension list without a lock while
|
||||
// the last task's done hooks may still read it.
|
||||
defer d.Close()
|
||||
cfg, err := d.GetConfig()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cfg.Proxy = &base.DownloaderProxyConfig{Enable: true, Scheme: "http", Host: "127.0.0.1:1", Usr: "proxyuser", Pwd: "proxysecret"}
|
||||
if err := d.PutConfig(cfg); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
dir := t.TempDir()
|
||||
create := func(rawURL string) string {
|
||||
id, err := d.CreateDirect(&base.Request{URL: rawURL}, &base.Options{Path: dir, Extra: &pftp.OptsExtra{Connections: 2}})
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "p@ss") || strings.Contains(err.Error(), "p%40ss") {
|
||||
t.Fatalf("the password leaked into an error: %v", err)
|
||||
}
|
||||
t.Fatal(err)
|
||||
}
|
||||
return id
|
||||
}
|
||||
root := "ftp://u:p%40ss@" + srv.Addr + "/"
|
||||
|
||||
okID := create(root + "secret.bin")
|
||||
if key := events.wait(t, okID); key != download.EventKeyDone {
|
||||
t.Fatalf("download ended with %q, logs:\n%s", key, logs.String())
|
||||
}
|
||||
ok := d.GetTask(okID)
|
||||
if ok.Protocol != "ftp" {
|
||||
t.Fatalf("protocol %q, want ftp", ok.Protocol)
|
||||
}
|
||||
if got := iftp.FileSHA256(t, ok.Meta.SingleFilepath()); got != want {
|
||||
t.Fatalf("sha256 %s, want %s", got, want)
|
||||
}
|
||||
missingID := create(root + "missing.bin")
|
||||
if key := events.wait(t, missingID); key != download.EventKeyError {
|
||||
t.Fatalf("a missing file must fail, got %q", key)
|
||||
}
|
||||
wrongID := create("ftp://u:wr0ng%40pw@" + srv.Addr + "/secret.bin")
|
||||
if key := events.wait(t, wrongID); key != download.EventKeyError {
|
||||
t.Fatalf("a wrong password must fail, got %q", key)
|
||||
}
|
||||
missing, wrong := d.GetTask(missingID), d.GetTask(wrongID)
|
||||
for _, task := range []*download.Task{ok, missing, wrong} {
|
||||
if name := task.Name(); strings.Contains(name, "p@ss") || strings.Contains(name, "p%40ss") || strings.Contains(name, "wr0ng") {
|
||||
t.Fatalf("task name %q shows the password", name)
|
||||
}
|
||||
}
|
||||
|
||||
out := logs.String()
|
||||
if !strings.Contains(out, "task id") || !strings.Contains(out, "HTTP proxy") {
|
||||
t.Fatalf("the captured log misses the expected lines, so it proves nothing:\n%s", out)
|
||||
}
|
||||
for _, secret := range []string{"p@ss", "p%40ss", "wr0ng", "proxysecret"} {
|
||||
if strings.Contains(out, secret) {
|
||||
t.Fatalf("%q appears in the log:\n%s", secret, out)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,810 @@
|
||||
// Package ftp downloads files and directory trees over FTP, FTPS (implicit
|
||||
// TLS) and FTPES (explicit TLS, AUTH TLS).
|
||||
//
|
||||
// A file is split into segments of at least 1 MiB. Each segment is fetched on
|
||||
// its own login with REST and RETR, and its data connection is closed once
|
||||
// the segment's last byte has arrived. A server that refuses a login lowers
|
||||
// the task's connection ceiling to the logins that worked, and a server that
|
||||
// refuses REST gets one sequential stream per file.
|
||||
package ftp
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/jlaffaye/ftp"
|
||||
"github.com/rs/zerolog"
|
||||
|
||||
"github.com/GopeedLab/gopeed/internal/controller"
|
||||
"github.com/GopeedLab/gopeed/internal/fetcher"
|
||||
"github.com/GopeedLab/gopeed/pkg/base"
|
||||
pftp "github.com/GopeedLab/gopeed/pkg/protocol/ftp"
|
||||
)
|
||||
|
||||
const (
|
||||
// Many servers cap logins per user, so FTP starts lower than HTTP.
|
||||
defaultConnections = 4
|
||||
maxConnections = 256
|
||||
minSegmentSize = 1 << 20
|
||||
maxRetries = 5
|
||||
copyBufferSize = 256 << 10
|
||||
)
|
||||
|
||||
// retryDelay is the first wait before a retry. Later retries wait longer.
|
||||
var retryDelay = 500 * time.Millisecond
|
||||
|
||||
var (
|
||||
errSizeChanged = errors.New("ftp: the remote file changed size since the task was resolved")
|
||||
errShortTransfer = errors.New("ftp: the transfer ended before the segment was complete")
|
||||
errRestUnsupported = errors.New("ftp: the server does not support REST")
|
||||
errDiskWrite = errors.New("ftp: cannot write the downloaded data")
|
||||
)
|
||||
|
||||
type config struct {
|
||||
Connections int `json:"connections"`
|
||||
}
|
||||
|
||||
// segment is a byte range of one file. End is inclusive, so an empty file
|
||||
// has End == Begin-1 and nothing to fetch.
|
||||
type segment struct {
|
||||
File int `json:",omitempty"` // index in Res.Files
|
||||
Begin int64
|
||||
End int64
|
||||
Downloaded int64
|
||||
}
|
||||
|
||||
func (s *segment) remain() int64 {
|
||||
return s.End - s.Begin + 1 - s.Downloaded
|
||||
}
|
||||
|
||||
// lane is one worker that logged in, shown as one connection in the stats.
|
||||
type lane struct {
|
||||
downloaded int64
|
||||
retryTimes int
|
||||
failed bool
|
||||
completed bool
|
||||
}
|
||||
|
||||
// loginError marks a failure to connect or log in, as opposed to a failure
|
||||
// during a transfer.
|
||||
type loginError struct{ err error }
|
||||
|
||||
func (e *loginError) Error() string { return e.err.Error() }
|
||||
func (e *loginError) Unwrap() error { return e.err }
|
||||
|
||||
type Fetcher struct {
|
||||
manager *FetcherManager
|
||||
ctl *controller.Controller
|
||||
config *config
|
||||
meta *fetcher.FetcherMeta
|
||||
doneCh chan error
|
||||
|
||||
warnOnce sync.Once
|
||||
// loginOK is set once these credentials have logged in, after which a
|
||||
// 530 on another login means the server is full.
|
||||
loginOK atomic.Bool
|
||||
|
||||
// mu guards the download state below.
|
||||
mu sync.Mutex
|
||||
segments []*segment
|
||||
maxConns int
|
||||
noRest bool
|
||||
lanes []*lane
|
||||
done bool
|
||||
|
||||
// runMu serializes Start and Pause.
|
||||
runMu sync.Mutex
|
||||
cancel context.CancelFunc
|
||||
runDone chan struct{}
|
||||
}
|
||||
|
||||
func (f *Fetcher) Setup(ctl *controller.Controller) {
|
||||
f.ctl = ctl
|
||||
f.doneCh = make(chan error, 1)
|
||||
if f.meta == nil {
|
||||
f.meta = &fetcher.FetcherMeta{}
|
||||
}
|
||||
if ctl != nil && ctl.GetConfig != nil {
|
||||
ctl.GetConfig(&f.config)
|
||||
}
|
||||
if f.config == nil {
|
||||
f.config = &config{Connections: defaultConnections}
|
||||
}
|
||||
}
|
||||
|
||||
func (f *Fetcher) logger() *zerolog.Logger {
|
||||
if f.ctl == nil {
|
||||
return nil
|
||||
}
|
||||
return f.ctl.Logger
|
||||
}
|
||||
|
||||
func (f *Fetcher) warn(format string, args ...any) {
|
||||
if l := f.logger(); l != nil {
|
||||
l.Warn().Msgf(format, args...)
|
||||
}
|
||||
}
|
||||
|
||||
func (f *Fetcher) initOptions() error {
|
||||
if f.meta.Opts == nil {
|
||||
f.meta.Opts = &base.Options{}
|
||||
}
|
||||
opts := f.meta.Opts
|
||||
if err := base.ParseOptExtra[pftp.OptsExtra](opts); err != nil {
|
||||
return err
|
||||
}
|
||||
if opts.Extra == nil {
|
||||
opts.Extra = &pftp.OptsExtra{}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *Fetcher) extra() *pftp.OptsExtra {
|
||||
if f.meta.Opts != nil {
|
||||
if e, ok := f.meta.Opts.Extra.(*pftp.OptsExtra); ok && e != nil {
|
||||
return e
|
||||
}
|
||||
}
|
||||
return &pftp.OptsExtra{}
|
||||
}
|
||||
|
||||
// tlsMode is the request's "tls" field, empty when the request has none.
|
||||
func (f *Fetcher) tlsMode() string {
|
||||
if f.meta.Req != nil {
|
||||
if e, ok := f.meta.Req.Extra.(*pftp.ReqExtra); ok && e != nil {
|
||||
return e.TLS
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// connections is the number of parallel logins to use. The caller holds mu.
|
||||
func (f *Fetcher) connections() int {
|
||||
if f.noRest {
|
||||
return 1
|
||||
}
|
||||
n := f.extra().Connections
|
||||
if n <= 0 && f.config != nil {
|
||||
n = f.config.Connections
|
||||
}
|
||||
if n <= 0 {
|
||||
n = defaultConnections
|
||||
}
|
||||
n = min(n, maxConnections)
|
||||
if f.maxConns > 0 {
|
||||
n = min(n, f.maxConns)
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
func (f *Fetcher) Resolve(req *base.Request) error {
|
||||
if err := base.ParseReqExtra[pftp.ReqExtra](req); err != nil {
|
||||
return err
|
||||
}
|
||||
f.meta.Req = req
|
||||
t, err := parseTarget(req.URL, f.tlsMode())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
s, err := f.dial(context.Background(), t)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// quit, not abort: a server that caps logins must see this session end
|
||||
// before the download's own logins arrive.
|
||||
defer s.quit()
|
||||
f.loginOK.Store(true)
|
||||
res, err := f.resolveTarget(s, t)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
f.meta.Res = res
|
||||
f.mu.Lock()
|
||||
f.segments = nil
|
||||
f.done = false
|
||||
f.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
// selected returns the indexes of the files to download, in SelectFiles
|
||||
// order. The caller holds mu or owns the fetcher.
|
||||
func (f *Fetcher) selected() []int {
|
||||
if f.meta == nil || f.meta.Res == nil {
|
||||
return nil
|
||||
}
|
||||
if f.meta.Opts != nil && len(f.meta.Opts.SelectFiles) > 0 {
|
||||
return f.meta.Opts.SelectFiles
|
||||
}
|
||||
all := make([]int, len(f.meta.Res.Files))
|
||||
for i := range all {
|
||||
all[i] = i
|
||||
}
|
||||
return all
|
||||
}
|
||||
|
||||
// localPath is where a file lands: under the task path, with the file's
|
||||
// directories below the URL kept.
|
||||
func (f *Fetcher) localPath(index int) string {
|
||||
if f.meta.Res.Name == "" {
|
||||
return f.meta.SingleFilepath()
|
||||
}
|
||||
file := f.meta.Res.Files[index]
|
||||
return path.Join(f.meta.FolderPath(), file.Path, file.Name)
|
||||
}
|
||||
|
||||
// remotePath is the server path of a file. A single file uses the URL's own
|
||||
// path, which a local rename never touches.
|
||||
func (f *Fetcher) remotePath(t *target, index int) string {
|
||||
if f.meta.Res.Name == "" {
|
||||
return t.path
|
||||
}
|
||||
file := f.meta.Res.Files[index]
|
||||
return path.Join(t.path, file.Path, file.Name)
|
||||
}
|
||||
|
||||
// plan splits every selected file into segments of at least minSegmentSize,
|
||||
// one per connection at most. The caller holds mu.
|
||||
func (f *Fetcher) plan() {
|
||||
conns := f.connections()
|
||||
f.segments = nil
|
||||
for _, index := range f.selected() {
|
||||
size := f.meta.Res.Files[index].Size
|
||||
n := int64(1)
|
||||
if !f.noRest {
|
||||
n = max(1, min(int64(conns), size/minSegmentSize))
|
||||
}
|
||||
for i := int64(0); i < n; i++ {
|
||||
f.segments = append(f.segments, &segment{
|
||||
File: index,
|
||||
Begin: i * size / n,
|
||||
End: (i+1)*size/n - 1,
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// planSequential gives every unfinished file one segment from byte 0,
|
||||
// because without REST a transfer cannot start in the middle. The caller
|
||||
// holds mu.
|
||||
func (f *Fetcher) planSequential() {
|
||||
complete := map[int]bool{}
|
||||
for _, s := range f.segments {
|
||||
if _, seen := complete[s.File]; !seen {
|
||||
complete[s.File] = true
|
||||
}
|
||||
if s.remain() > 0 {
|
||||
complete[s.File] = false
|
||||
}
|
||||
}
|
||||
var out []*segment
|
||||
for _, index := range f.selected() {
|
||||
size := f.meta.Res.Files[index].Size
|
||||
if complete[index] {
|
||||
out = append(out, &segment{File: index, Begin: 0, End: size - 1, Downloaded: size})
|
||||
continue
|
||||
}
|
||||
out = append(out, &segment{File: index, Begin: 0, End: size - 1})
|
||||
}
|
||||
f.segments = out
|
||||
// Bytes counted by the old segments are fetched again.
|
||||
f.lanes = nil
|
||||
}
|
||||
|
||||
// openFiles opens every selected file for writing. A fresh plan creates and
|
||||
// sizes them; a resumed one keeps their content, and a file that has gone
|
||||
// missing starts over.
|
||||
func (f *Fetcher) openFiles(fresh bool) (map[int]*os.File, error) {
|
||||
files := map[int]*os.File{}
|
||||
closeAll := func() {
|
||||
for _, file := range files {
|
||||
_ = file.Close()
|
||||
}
|
||||
}
|
||||
for _, index := range f.selected() {
|
||||
name := f.localPath(index)
|
||||
var (
|
||||
file *os.File
|
||||
err error
|
||||
)
|
||||
_, statErr := os.Stat(name)
|
||||
switch {
|
||||
case fresh || os.IsNotExist(statErr):
|
||||
file, err = f.ctl.Touch(name, f.meta.Res.Files[index].Size)
|
||||
if err == nil && !fresh {
|
||||
f.mu.Lock()
|
||||
for _, s := range f.segments {
|
||||
if s.File == index {
|
||||
s.Downloaded = 0
|
||||
}
|
||||
}
|
||||
f.mu.Unlock()
|
||||
}
|
||||
case statErr != nil:
|
||||
err = statErr
|
||||
default:
|
||||
file, err = os.OpenFile(name, os.O_RDWR, 0)
|
||||
}
|
||||
if err != nil {
|
||||
closeAll()
|
||||
return nil, err
|
||||
}
|
||||
files[index] = file
|
||||
}
|
||||
return files, nil
|
||||
}
|
||||
|
||||
func (f *Fetcher) Start() error {
|
||||
f.runMu.Lock()
|
||||
defer f.runMu.Unlock()
|
||||
if f.runDone != nil {
|
||||
select {
|
||||
case <-f.runDone:
|
||||
default:
|
||||
return nil // already running
|
||||
}
|
||||
}
|
||||
if err := f.initOptions(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := base.ParseReqExtra[pftp.ReqExtra](f.meta.Req); err != nil {
|
||||
return err
|
||||
}
|
||||
t, err := parseTarget(f.meta.Req.URL, f.tlsMode())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// A paused or failed run may have left a result nobody read.
|
||||
select {
|
||||
case <-f.doneCh:
|
||||
default:
|
||||
}
|
||||
|
||||
f.mu.Lock()
|
||||
if f.done {
|
||||
f.mu.Unlock()
|
||||
f.doneCh <- nil
|
||||
return nil
|
||||
}
|
||||
fresh := len(f.segments) == 0
|
||||
if fresh {
|
||||
f.plan()
|
||||
} else if f.noRest {
|
||||
f.planSequential()
|
||||
}
|
||||
f.mu.Unlock()
|
||||
|
||||
files, err := f.openFiles(fresh)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
f.cancel = cancel
|
||||
f.runDone = make(chan struct{})
|
||||
go f.run(ctx, t, files, f.runDone)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *Fetcher) run(ctx context.Context, t *target, files map[int]*os.File, done chan struct{}) {
|
||||
defer close(done)
|
||||
err := f.download(ctx, t, files)
|
||||
for _, file := range files {
|
||||
if cerr := file.Close(); cerr != nil && err == nil {
|
||||
err = cerr
|
||||
}
|
||||
}
|
||||
f.mu.Lock()
|
||||
if err == nil {
|
||||
f.done = true
|
||||
for _, l := range f.lanes {
|
||||
l.completed = true
|
||||
}
|
||||
}
|
||||
f.mu.Unlock()
|
||||
if err != nil && ctx.Err() != nil {
|
||||
return // paused
|
||||
}
|
||||
select {
|
||||
case f.doneCh <- err:
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
func (f *Fetcher) download(ctx context.Context, t *target, files map[int]*os.File) error {
|
||||
for {
|
||||
err := f.runWorkers(ctx, t, files)
|
||||
if !errors.Is(err, errRestUnsupported) {
|
||||
return err
|
||||
}
|
||||
f.mu.Lock()
|
||||
f.noRest = true
|
||||
f.planSequential()
|
||||
f.mu.Unlock()
|
||||
f.warn("ftp: the server refused REST, so each file is downloaded in one stream")
|
||||
}
|
||||
}
|
||||
|
||||
// pool hands out the segments of one run to its workers. Every decision to
|
||||
// leave the run is taken under mu together with the queue it depends on, so
|
||||
// a segment put back can never be left without a worker.
|
||||
type pool struct {
|
||||
mu sync.Mutex
|
||||
queue []*segment
|
||||
running int // workers still in the run
|
||||
}
|
||||
|
||||
// take returns the next segment. A worker that finds the queue empty leaves
|
||||
// the run.
|
||||
func (p *pool) take() *segment {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
if len(p.queue) == 0 {
|
||||
p.running--
|
||||
return nil
|
||||
}
|
||||
s := p.queue[0]
|
||||
p.queue = p.queue[1:]
|
||||
return s
|
||||
}
|
||||
|
||||
func (p *pool) putBack(s *segment) {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
p.queue = append([]*segment{s}, p.queue...)
|
||||
}
|
||||
|
||||
// refused puts back the segment of a worker whose login the server turned
|
||||
// away. The worker leaves when another worker is still in the run to take
|
||||
// the segment, and left is the number of workers that remain; the last
|
||||
// worker stays and retries.
|
||||
func (p *pool) refused(s *segment) (leave bool, left int) {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
p.queue = append([]*segment{s}, p.queue...)
|
||||
if p.running > 1 {
|
||||
p.running--
|
||||
return true, p.running
|
||||
}
|
||||
return false, p.running
|
||||
}
|
||||
|
||||
// leave takes a worker out of the run when it stops for any other reason.
|
||||
func (p *pool) leave() {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
p.running--
|
||||
}
|
||||
|
||||
func (f *Fetcher) runWorkers(ctx context.Context, t *target, files map[int]*os.File) error {
|
||||
runCtx, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
|
||||
f.mu.Lock()
|
||||
p := &pool{}
|
||||
for _, s := range f.segments {
|
||||
if s.remain() > 0 {
|
||||
p.queue = append(p.queue, s)
|
||||
}
|
||||
}
|
||||
workers := min(f.connections(), len(p.queue))
|
||||
f.mu.Unlock()
|
||||
if workers == 0 {
|
||||
return nil
|
||||
}
|
||||
p.running = workers
|
||||
|
||||
var (
|
||||
wg sync.WaitGroup
|
||||
errMu sync.Mutex
|
||||
firstErr error
|
||||
)
|
||||
for id := 0; id < workers; id++ {
|
||||
wg.Add(1)
|
||||
go func(id int) {
|
||||
defer wg.Done()
|
||||
if err := f.worker(runCtx, t, files, p, id); err != nil {
|
||||
errMu.Lock()
|
||||
if firstErr == nil {
|
||||
firstErr = err
|
||||
}
|
||||
errMu.Unlock()
|
||||
cancel()
|
||||
}
|
||||
}(id)
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if firstErr != nil {
|
||||
return firstErr
|
||||
}
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
for _, s := range f.segments {
|
||||
if s.remain() > 0 {
|
||||
return errors.New("ftp: the download stopped with data missing")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// worker fetches segments from the pool until it is empty. A worker whose
|
||||
// login the server refuses (421, or 530 once these credentials have worked)
|
||||
// hands its segment back and leaves while other workers remain, lowering the
|
||||
// task's ceiling to the workers left. This holds whether or not the worker
|
||||
// had logged in before: a slot can also go to another client mid-download.
|
||||
func (f *Fetcher) worker(ctx context.Context, t *target, files map[int]*os.File, p *pool, id int) error {
|
||||
var ln *lane
|
||||
f.mu.Lock()
|
||||
if id < len(f.lanes) {
|
||||
ln = f.lanes[id]
|
||||
}
|
||||
f.mu.Unlock()
|
||||
onLogin := func() {
|
||||
f.mu.Lock()
|
||||
if ln == nil {
|
||||
ln = &lane{}
|
||||
f.lanes = append(f.lanes, ln)
|
||||
}
|
||||
f.mu.Unlock()
|
||||
}
|
||||
|
||||
failures := 0
|
||||
pendingRetries := 0
|
||||
for {
|
||||
seg := p.take()
|
||||
if seg == nil {
|
||||
return nil
|
||||
}
|
||||
err := f.fetchSegment(ctx, t, files, seg, &ln, onLogin)
|
||||
if err == nil {
|
||||
failures = 0
|
||||
continue
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
p.putBack(seg)
|
||||
p.leave()
|
||||
return nil
|
||||
}
|
||||
|
||||
var le *loginError
|
||||
if errors.As(err, &le) && isLoginRefusal(le.err, f.loginOK.Load()) {
|
||||
if leave, left := p.refused(seg); leave {
|
||||
f.lowerCeiling(left)
|
||||
return nil
|
||||
}
|
||||
// The last worker: wait for a slot.
|
||||
} else {
|
||||
p.putBack(seg)
|
||||
if errors.Is(err, errRestUnsupported) {
|
||||
p.leave()
|
||||
return err
|
||||
}
|
||||
if isPermanent(err) || errors.Is(err, errDiskWrite) {
|
||||
f.markFailed(ln)
|
||||
p.leave()
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
failures++
|
||||
f.mu.Lock()
|
||||
if ln != nil {
|
||||
ln.retryTimes += pendingRetries + 1
|
||||
pendingRetries = 0
|
||||
} else {
|
||||
pendingRetries++
|
||||
}
|
||||
f.mu.Unlock()
|
||||
if failures > maxRetries {
|
||||
f.markFailed(ln)
|
||||
p.leave()
|
||||
return fmt.Errorf("ftp: giving up after %d retries: %w", maxRetries, err)
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
p.leave()
|
||||
return nil
|
||||
case <-time.After(retryDelay * time.Duration(failures)):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (f *Fetcher) lowerCeiling(n int) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if f.maxConns == 0 || n < f.maxConns {
|
||||
f.maxConns = n
|
||||
}
|
||||
}
|
||||
|
||||
func (f *Fetcher) markFailed(ln *lane) {
|
||||
if ln == nil {
|
||||
return
|
||||
}
|
||||
f.mu.Lock()
|
||||
ln.failed = true
|
||||
f.mu.Unlock()
|
||||
}
|
||||
|
||||
// isRestRefused tells a REST the server does not implement. RETR itself
|
||||
// fails with 550 and the like, never with these codes.
|
||||
func isRestRefused(err error) bool {
|
||||
switch replyCode(err) {
|
||||
case ftp.StatusBadCommand, ftp.StatusBadArguments, ftp.StatusNotImplemented, ftp.StatusNotImplementedParameter:
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// fetchSegment logs in, retrieves the segment from its first missing byte
|
||||
// and writes it in place. It stops reading at the segment's end and closes
|
||||
// the data connection; the server then reports the transfer aborted, which
|
||||
// is expected.
|
||||
func (f *Fetcher) fetchSegment(ctx context.Context, t *target, files map[int]*os.File, seg *segment, ln **lane, onLogin func()) error {
|
||||
s, err := f.dial(ctx, t)
|
||||
if err != nil {
|
||||
return &loginError{err}
|
||||
}
|
||||
defer s.quit()
|
||||
f.loginOK.Store(true)
|
||||
onLogin()
|
||||
|
||||
remote := f.remotePath(t, seg.File)
|
||||
f.mu.Lock()
|
||||
noRest := f.noRest
|
||||
wantSize := f.meta.Res.Files[seg.File].Size
|
||||
if noRest {
|
||||
seg.Downloaded = 0
|
||||
}
|
||||
pos := seg.Begin + seg.Downloaded
|
||||
end := seg.End
|
||||
f.mu.Unlock()
|
||||
|
||||
// A file that changed on the server would mix old and new bytes.
|
||||
s.deadline()
|
||||
if size, err := s.FileSize(remote); err == nil && size != wantSize {
|
||||
return errSizeChanged
|
||||
} else if err != nil && replyCode(err) == 0 {
|
||||
return err
|
||||
}
|
||||
|
||||
s.deadline()
|
||||
var resp *ftp.Response
|
||||
if noRest {
|
||||
resp, err = s.Retr(remote)
|
||||
} else {
|
||||
resp, err = s.RetrFrom(remote, uint64(pos))
|
||||
}
|
||||
if err != nil {
|
||||
if !noRest && pos > 0 && isRestRefused(err) {
|
||||
return errRestUnsupported
|
||||
}
|
||||
return err
|
||||
}
|
||||
defer resp.Close()
|
||||
s.clearDeadline()
|
||||
|
||||
file := files[seg.File]
|
||||
buf := make([]byte, copyBufferSize)
|
||||
for pos <= end {
|
||||
n := int(min(int64(len(buf)), end-pos+1))
|
||||
_ = resp.SetDeadline(time.Now().Add(readTimeout))
|
||||
nr, rerr := resp.Read(buf[:n])
|
||||
if nr > 0 {
|
||||
if _, werr := file.WriteAt(buf[:nr], pos); werr != nil {
|
||||
return fmt.Errorf("%w: %w", errDiskWrite, werr)
|
||||
}
|
||||
pos += int64(nr)
|
||||
f.mu.Lock()
|
||||
seg.Downloaded += int64(nr)
|
||||
if *ln != nil {
|
||||
(*ln).downloaded += int64(nr)
|
||||
}
|
||||
f.mu.Unlock()
|
||||
}
|
||||
if rerr != nil {
|
||||
if pos > end {
|
||||
break
|
||||
}
|
||||
if errors.Is(rerr, io.EOF) {
|
||||
return errShortTransfer
|
||||
}
|
||||
return rerr
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *Fetcher) Patch(req *base.Request, opts *base.Options) error {
|
||||
if req == nil {
|
||||
return nil
|
||||
}
|
||||
if req.URL != "" {
|
||||
f.meta.Req.URL = req.URL
|
||||
}
|
||||
if req.Labels != nil {
|
||||
if f.meta.Req.Labels == nil {
|
||||
f.meta.Req.Labels = make(map[string]string)
|
||||
}
|
||||
for k, v := range req.Labels {
|
||||
f.meta.Req.Labels[k] = v
|
||||
}
|
||||
}
|
||||
if req.Proxy != nil {
|
||||
f.meta.Req.Proxy = req.Proxy
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *Fetcher) Pause() error {
|
||||
f.runMu.Lock()
|
||||
defer f.runMu.Unlock()
|
||||
if f.cancel != nil {
|
||||
f.cancel()
|
||||
}
|
||||
if f.runDone != nil {
|
||||
<-f.runDone
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *Fetcher) Close() error {
|
||||
return f.Pause()
|
||||
}
|
||||
|
||||
func (f *Fetcher) Meta() *fetcher.FetcherMeta {
|
||||
return f.meta
|
||||
}
|
||||
|
||||
// Stats reports one connection per worker that logged in, in the HTTP
|
||||
// protocol's shape.
|
||||
func (f *Fetcher) Stats() *fetcher.Stats {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
var total int64
|
||||
if n := int64(len(f.lanes)); n > 0 && f.meta != nil && f.meta.Res != nil && f.meta.Res.Size > 0 {
|
||||
total = (f.meta.Res.Size + n - 1) / n
|
||||
}
|
||||
conns := make([]*pftp.StatsConnection, 0, len(f.lanes))
|
||||
for _, l := range f.lanes {
|
||||
conns = append(conns, &pftp.StatsConnection{
|
||||
Downloaded: l.downloaded,
|
||||
Total: total,
|
||||
Completed: l.completed || (f.done && !l.failed),
|
||||
Failed: l.failed,
|
||||
RetryTimes: l.retryTimes,
|
||||
})
|
||||
}
|
||||
return &fetcher.Stats{Snapshot: &pftp.Stats{Connections: conns}}
|
||||
}
|
||||
|
||||
// Progress returns the bytes downloaded of each selected file, in
|
||||
// SelectFiles order.
|
||||
func (f *Fetcher) Progress() fetcher.Progress {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
perFile := map[int]int64{}
|
||||
for _, s := range f.segments {
|
||||
perFile[s.File] += s.Downloaded
|
||||
}
|
||||
sel := f.selected()
|
||||
p := make(fetcher.Progress, len(sel))
|
||||
for i, index := range sel {
|
||||
p[i] = perFile[index]
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
func (f *Fetcher) Wait() error {
|
||||
return <-f.doneCh
|
||||
}
|
||||
@@ -0,0 +1,109 @@
|
||||
package ftp
|
||||
|
||||
import (
|
||||
"crypto/tls"
|
||||
"net/url"
|
||||
"path"
|
||||
"strings"
|
||||
|
||||
"github.com/GopeedLab/gopeed/internal/fetcher"
|
||||
"github.com/GopeedLab/gopeed/pkg/base"
|
||||
pftp "github.com/GopeedLab/gopeed/pkg/protocol/ftp"
|
||||
)
|
||||
|
||||
// fetcherData is what a task saves to resume later. Segments has the same
|
||||
// Begin, End and Downloaded fields every version writes; the other fields are
|
||||
// optional, so a save without them still restores.
|
||||
type fetcherData struct {
|
||||
Segments []*segment
|
||||
// MaxConns is the login ceiling the server enforced, 0 when none was hit.
|
||||
MaxConns int `json:",omitempty"`
|
||||
// NoRest records that the server refused REST, so the task downloads each
|
||||
// file in one sequential stream.
|
||||
NoRest bool `json:",omitempty"`
|
||||
}
|
||||
|
||||
// FetcherManager handles ftp://, ftps:// (implicit TLS) and ftpes:// (explicit
|
||||
// TLS) URLs.
|
||||
type FetcherManager struct {
|
||||
// baseTLS is the base for every TLS connection. Nil means the system
|
||||
// roots; the package's tests set it to trust their own certificate. The
|
||||
// task's skipVerifyCert still turns verification off.
|
||||
baseTLS *tls.Config
|
||||
}
|
||||
|
||||
func (fm *FetcherManager) Name() string {
|
||||
return "ftp"
|
||||
}
|
||||
|
||||
func (fm *FetcherManager) Filters() []*fetcher.SchemeFilter {
|
||||
return []*fetcher.SchemeFilter{
|
||||
{Type: fetcher.FilterTypeUrl, Pattern: "FTP"},
|
||||
{Type: fetcher.FilterTypeUrl, Pattern: "FTPS"},
|
||||
{Type: fetcher.FilterTypeUrl, Pattern: "FTPES"},
|
||||
}
|
||||
}
|
||||
|
||||
func (fm *FetcherManager) Build() fetcher.Fetcher {
|
||||
return &Fetcher{manager: fm}
|
||||
}
|
||||
|
||||
// ParseName names a task before it is resolved. It never includes the
|
||||
// userinfo of the URL.
|
||||
func (fm *FetcherManager) ParseName(u string) string {
|
||||
parsed, err := url.Parse(u)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
name := path.Base(strings.TrimSuffix(parsed.Path, "/"))
|
||||
if name == "" || name == "/" || name == "." {
|
||||
name = parsed.Hostname()
|
||||
}
|
||||
return name
|
||||
}
|
||||
|
||||
func (fm *FetcherManager) AutoRename() bool {
|
||||
return true
|
||||
}
|
||||
|
||||
func (fm *FetcherManager) DefaultConfig() any {
|
||||
return &config{Connections: defaultConnections}
|
||||
}
|
||||
|
||||
func (fm *FetcherManager) Store(f fetcher.Fetcher) (any, error) {
|
||||
ff := f.(*Fetcher)
|
||||
ff.mu.Lock()
|
||||
defer ff.mu.Unlock()
|
||||
segments := make([]*segment, len(ff.segments))
|
||||
for i, s := range ff.segments {
|
||||
c := *s
|
||||
segments[i] = &c
|
||||
}
|
||||
return &fetcherData{
|
||||
Segments: segments,
|
||||
MaxConns: ff.maxConns,
|
||||
NoRest: ff.noRest,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (fm *FetcherManager) Restore() (v any, f func(meta *fetcher.FetcherMeta, v any) fetcher.Fetcher) {
|
||||
return &fetcherData{}, func(meta *fetcher.FetcherMeta, v any) fetcher.Fetcher {
|
||||
fd := v.(*fetcherData)
|
||||
ff := fm.Build().(*Fetcher)
|
||||
ff.meta = meta
|
||||
if meta != nil && meta.Req != nil {
|
||||
_ = base.ParseReqExtra[pftp.ReqExtra](meta.Req)
|
||||
}
|
||||
if meta != nil && meta.Opts != nil {
|
||||
_ = base.ParseOptExtra[pftp.OptsExtra](meta.Opts)
|
||||
}
|
||||
ff.segments = fd.Segments
|
||||
ff.maxConns = fd.MaxConns
|
||||
ff.noRest = fd.NoRest
|
||||
return ff
|
||||
}
|
||||
}
|
||||
|
||||
func (fm *FetcherManager) Close() error {
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,818 @@
|
||||
package ftp
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/armon/go-socks5"
|
||||
"github.com/rs/zerolog"
|
||||
|
||||
"github.com/GopeedLab/gopeed/internal/controller"
|
||||
"github.com/GopeedLab/gopeed/internal/fetcher"
|
||||
"github.com/GopeedLab/gopeed/pkg/base"
|
||||
pftp "github.com/GopeedLab/gopeed/pkg/protocol/ftp"
|
||||
)
|
||||
|
||||
const mib = 1 << 20
|
||||
|
||||
// testManager trusts the test server's certificate, the way an embedding
|
||||
// application could add private roots.
|
||||
func testManager(srv *FTPServer) *FetcherManager {
|
||||
fm := &FetcherManager{}
|
||||
if srv != nil {
|
||||
fm.baseTLS = &tls.Config{RootCAs: srv.CA}
|
||||
}
|
||||
return fm
|
||||
}
|
||||
|
||||
func testController(fm *FetcherManager) *controller.Controller {
|
||||
ctl := controller.NewController()
|
||||
ctl.GetConfig = func(v any) {
|
||||
data, _ := json.Marshal(fm.DefaultConfig())
|
||||
_ = json.Unmarshal(data, v)
|
||||
}
|
||||
return ctl
|
||||
}
|
||||
|
||||
func buildFetcher(fm *FetcherManager) *Fetcher {
|
||||
f := fm.Build().(*Fetcher)
|
||||
f.Setup(testController(fm))
|
||||
return f
|
||||
}
|
||||
|
||||
func ftpURL(srv *FTPServer, scheme, userinfo, p string) string {
|
||||
if userinfo != "" {
|
||||
userinfo += "@"
|
||||
}
|
||||
return scheme + "://" + userinfo + srv.Addr + "/" + strings.TrimPrefix(p, "/")
|
||||
}
|
||||
|
||||
// resolveOpts resolves req and then gives the fetcher its options, in the
|
||||
// order the downloader uses.
|
||||
func resolveOpts(f *Fetcher, req *base.Request, opts *base.Options) error {
|
||||
if err := f.Resolve(req); err != nil {
|
||||
return err
|
||||
}
|
||||
f.Meta().Opts = opts
|
||||
return nil
|
||||
}
|
||||
|
||||
func resolveTask(t *testing.T, f *Fetcher, rawURL string, extra *pftp.OptsExtra) {
|
||||
t.Helper()
|
||||
resolveRequest(t, f, &base.Request{URL: rawURL}, extra)
|
||||
}
|
||||
|
||||
func resolveRequest(t *testing.T, f *Fetcher, req *base.Request, extra *pftp.OptsExtra) {
|
||||
t.Helper()
|
||||
opts := &base.Options{Path: t.TempDir()}
|
||||
if extra != nil {
|
||||
opts.Extra = extra
|
||||
}
|
||||
if err := resolveOpts(f, req, opts); err != nil {
|
||||
t.Fatalf("resolve %s: %v", req.URL, err)
|
||||
}
|
||||
}
|
||||
|
||||
func waitFetcher(t *testing.T, f fetcher.Fetcher) error {
|
||||
t.Helper()
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- f.Wait() }()
|
||||
select {
|
||||
case err := <-done:
|
||||
return err
|
||||
case <-time.After(60 * time.Second):
|
||||
t.Fatal("download did not finish in 60s")
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func startAndWait(t *testing.T, f *Fetcher) {
|
||||
t.Helper()
|
||||
if err := f.Start(); err != nil {
|
||||
t.Fatalf("start: %v", err)
|
||||
}
|
||||
if err := waitFetcher(t, f); err != nil {
|
||||
t.Fatalf("download: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func checkFile(t *testing.T, name, want string) {
|
||||
t.Helper()
|
||||
if got := FileSHA256(t, name); got != want {
|
||||
t.Fatalf("%s: sha256 %s, want %s", name, got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func ftpStats(t *testing.T, f *Fetcher) *pftp.Stats {
|
||||
t.Helper()
|
||||
stats, ok := f.Stats().Snapshot.(*pftp.Stats)
|
||||
if !ok {
|
||||
t.Fatalf("stats snapshot is %T, want *ftp.Stats", f.Stats().Snapshot)
|
||||
}
|
||||
return stats
|
||||
}
|
||||
|
||||
func storeData(t *testing.T, fm *FetcherManager, f *Fetcher) []byte {
|
||||
t.Helper()
|
||||
data, err := fm.Store(f)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
blob, err := json.Marshal(data)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return blob
|
||||
}
|
||||
|
||||
// restoreFetcher rebuilds a fetcher the way the downloader does after a
|
||||
// restart: the meta and the save blob both come back from JSON.
|
||||
func restoreFetcher(t *testing.T, fm *FetcherManager, meta *fetcher.FetcherMeta, blob []byte) *Fetcher {
|
||||
t.Helper()
|
||||
metaJSON, err := json.Marshal(meta)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var restoredMeta fetcher.FetcherMeta
|
||||
if err := json.Unmarshal(metaJSON, &restoredMeta); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
v, build := fm.Restore()
|
||||
if err := json.Unmarshal(blob, v); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f := build(&restoredMeta, v).(*Fetcher)
|
||||
f.Setup(testController(fm))
|
||||
return f
|
||||
}
|
||||
|
||||
func TestResolveFile(t *testing.T) {
|
||||
for _, noMLST := range []bool{false, true} {
|
||||
t.Run(fmt.Sprintf("noMLST=%v", noMLST), func(t *testing.T) {
|
||||
srv := StartFTPServer(t, ServerOptions{NoMLST: noMLST})
|
||||
data := RandomBytes(100*1024, 1)
|
||||
srv.WriteFile(t, "pub/file name.bin", data)
|
||||
|
||||
f := buildFetcher(testManager(srv))
|
||||
resolveTask(t, f, ftpURL(srv, "ftp", "", "pub/file%20name.bin"), nil)
|
||||
res := f.Meta().Res
|
||||
if res.Name != "" {
|
||||
t.Fatalf("a file must not be a folder resource, got name %q", res.Name)
|
||||
}
|
||||
if len(res.Files) != 1 {
|
||||
t.Fatalf("got %d files, want 1", len(res.Files))
|
||||
}
|
||||
file := res.Files[0]
|
||||
if file.Name != "file name.bin" || file.Path != "" || file.Size != int64(len(data)) {
|
||||
t.Fatalf("got file %+v, want name %q, empty path, size %d", file, "file name.bin", len(data))
|
||||
}
|
||||
if res.Size != int64(len(data)) {
|
||||
t.Fatalf("resource size %d, want %d", res.Size, len(data))
|
||||
}
|
||||
if file.Ctime == nil || time.Since(*file.Ctime) > time.Hour {
|
||||
t.Fatalf("modification time %v is missing or wrong", file.Ctime)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func writeTree(t *testing.T, srv *FTPServer) map[string]string {
|
||||
t.Helper()
|
||||
return map[string]string{
|
||||
"a.txt": srv.WriteFile(t, "pub/tree/a.txt", []byte("hello")),
|
||||
"sub/b.bin": srv.WriteFile(t, "pub/tree/sub/b.bin", RandomBytes(3*mib+17, 2)),
|
||||
"sub/deeper/c.dat": srv.WriteFile(t, "pub/tree/sub/deeper/c.dat", RandomBytes(2000, 3)),
|
||||
"sub/empty.txt": srv.WriteFile(t, "pub/tree/sub/empty.txt", nil),
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveDir(t *testing.T) {
|
||||
for _, noMLST := range []bool{false, true} {
|
||||
for _, p := range []string{"pub/tree", "pub/tree/"} {
|
||||
t.Run(fmt.Sprintf("noMLST=%v/%s", noMLST, p), func(t *testing.T) {
|
||||
srv := StartFTPServer(t, ServerOptions{NoMLST: noMLST})
|
||||
writeTree(t, srv)
|
||||
if err := os.MkdirAll(filepath.Join(srv.Root, "pub/tree/void"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
f := buildFetcher(testManager(srv))
|
||||
resolveTask(t, f, ftpURL(srv, "ftp", "", p), nil)
|
||||
res := f.Meta().Res
|
||||
if res.Name != "tree" {
|
||||
t.Fatalf("folder name %q, want tree", res.Name)
|
||||
}
|
||||
var got []string
|
||||
for _, file := range res.Files {
|
||||
got = append(got, fmt.Sprintf("%s|%s|%d", file.Path, file.Name, file.Size))
|
||||
}
|
||||
want := []string{
|
||||
"|a.txt|5",
|
||||
"sub|b.bin|3145745",
|
||||
"sub|empty.txt|0",
|
||||
"sub/deeper|c.dat|2000",
|
||||
}
|
||||
if !slices.Equal(got, want) {
|
||||
t.Fatalf("files\n got %q\nwant %q", got, want)
|
||||
}
|
||||
if res.Size != 5+3145745+2000 {
|
||||
t.Fatalf("resource size %d", res.Size)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveDirLimits(t *testing.T) {
|
||||
srv := StartFTPServer(t, ServerOptions{})
|
||||
srv.WriteFile(t, "deep/1/2/3/4/5/6/7/8/f.txt", []byte("ok"))
|
||||
srv.WriteFile(t, "deep9/1/2/3/4/5/6/7/8/9/f.txt", []byte("too deep"))
|
||||
for i := 0; i < 4; i++ {
|
||||
srv.WriteFile(t, fmt.Sprintf("many/f%d", i), []byte("x"))
|
||||
}
|
||||
|
||||
f := buildFetcher(testManager(srv))
|
||||
resolveTask(t, f, ftpURL(srv, "ftp", "", "deep"), nil)
|
||||
if got := f.Meta().Res.Files[0].Path; got != "1/2/3/4/5/6/7/8" {
|
||||
t.Fatalf("depth 8 path %q", got)
|
||||
}
|
||||
|
||||
f = buildFetcher(testManager(srv))
|
||||
err := resolveOpts(f, &base.Request{URL: ftpURL(srv, "ftp", "", "deep9")}, &base.Options{Path: t.TempDir()})
|
||||
if err == nil || !strings.Contains(err.Error(), "8 levels") {
|
||||
t.Fatalf("a tree deeper than 8 levels must fail and name the limit, got %v", err)
|
||||
}
|
||||
|
||||
saved := maxWalkFiles
|
||||
maxWalkFiles = 3
|
||||
defer func() { maxWalkFiles = saved }()
|
||||
f = buildFetcher(testManager(srv))
|
||||
err = resolveOpts(f, &base.Request{URL: ftpURL(srv, "ftp", "", "many")}, &base.Options{Path: t.TempDir()})
|
||||
if err == nil || !strings.Contains(err.Error(), "3 files") {
|
||||
t.Fatalf("a tree over the file cap must fail and name the limit, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloadPlain(t *testing.T) {
|
||||
srv := StartFTPServer(t, ServerOptions{})
|
||||
data := RandomBytes(3*mib+12345, 4)
|
||||
want := srv.WriteFile(t, "pub/plain.bin", data)
|
||||
|
||||
f := buildFetcher(testManager(srv))
|
||||
resolveTask(t, f, ftpURL(srv, "ftp", "", "pub/plain.bin"), &pftp.OptsExtra{Connections: 4})
|
||||
startAndWait(t, f)
|
||||
checkFile(t, f.Meta().SingleFilepath(), want)
|
||||
if got := f.Progress().TotalDownloaded(); got != int64(len(data)) {
|
||||
t.Fatalf("progress %d, want %d", got, len(data))
|
||||
}
|
||||
stats := ftpStats(t, f)
|
||||
if len(stats.Connections) == 0 || len(stats.Connections) > 4 {
|
||||
t.Fatalf("got %d connections in stats", len(stats.Connections))
|
||||
}
|
||||
var sum int64
|
||||
for _, c := range stats.Connections {
|
||||
sum += c.Downloaded
|
||||
if !c.Completed || c.Failed {
|
||||
t.Fatalf("connection stats %+v after success", c)
|
||||
}
|
||||
}
|
||||
if sum != int64(len(data)) {
|
||||
t.Fatalf("connections downloaded %d bytes, want %d", sum, len(data))
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloadDir(t *testing.T) {
|
||||
srv := StartFTPServer(t, ServerOptions{})
|
||||
want := writeTree(t, srv)
|
||||
|
||||
f := buildFetcher(testManager(srv))
|
||||
resolveTask(t, f, ftpURL(srv, "ftp", "", "pub/tree"), &pftp.OptsExtra{Connections: 2})
|
||||
startAndWait(t, f)
|
||||
root := f.Meta().FolderPath()
|
||||
if root != filepath.Join(f.Meta().Opts.Path, "tree") {
|
||||
t.Fatalf("folder path %q", root)
|
||||
}
|
||||
for rel, sum := range want {
|
||||
checkFile(t, filepath.Join(root, filepath.FromSlash(rel)), sum)
|
||||
}
|
||||
}
|
||||
|
||||
func testDownloadTLS(t *testing.T, mode string, schemes map[string]string) {
|
||||
srv := StartFTPServer(t, ServerOptions{TLS: mode})
|
||||
data := RandomBytes(2*mib+999, 5)
|
||||
want := srv.WriteFile(t, "secure.bin", data)
|
||||
for scheme, reqTLS := range schemes {
|
||||
t.Run(scheme+"+"+reqTLS, func(t *testing.T) {
|
||||
f := buildFetcher(testManager(srv))
|
||||
req := &base.Request{URL: ftpURL(srv, scheme, "", "secure.bin"), Extra: &pftp.ReqExtra{TLS: reqTLS}}
|
||||
resolveRequest(t, f, req, &pftp.OptsExtra{Connections: 2})
|
||||
startAndWait(t, f)
|
||||
checkFile(t, f.Meta().SingleFilepath(), want)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloadExplicitTLS(t *testing.T) {
|
||||
testDownloadTLS(t, pftp.TLSExplicit, map[string]string{
|
||||
"ftpes": "",
|
||||
"ftp": pftp.TLSExplicit,
|
||||
})
|
||||
}
|
||||
|
||||
func TestDownloadImplicitTLS(t *testing.T) {
|
||||
testDownloadTLS(t, pftp.TLSImplicit, map[string]string{
|
||||
"ftps": "",
|
||||
"ftp": pftp.TLSImplicit,
|
||||
})
|
||||
}
|
||||
|
||||
// The TLS mode of a plain ftp:// link lives on the request, so a task that is
|
||||
// saved and restored from JSON still logs in over TLS.
|
||||
func TestRestoredRequestKeepsTLSMode(t *testing.T) {
|
||||
srv := StartFTPServer(t, ServerOptions{TLS: pftp.TLSExplicit})
|
||||
want := srv.WriteFile(t, "keep.bin", RandomBytes(2*mib+5, 12))
|
||||
fm := testManager(srv)
|
||||
f := buildFetcher(fm)
|
||||
req := &base.Request{URL: ftpURL(srv, "ftp", "", "keep.bin"), Extra: &pftp.ReqExtra{TLS: pftp.TLSExplicit}}
|
||||
resolveRequest(t, f, req, &pftp.OptsExtra{Connections: 2})
|
||||
|
||||
restored := restoreFetcher(t, fm, f.Meta(), storeData(t, fm, f))
|
||||
if _, ok := restored.Meta().Req.Extra.(*pftp.ReqExtra); !ok {
|
||||
t.Fatalf("restored request extra is %T, want *ftp.ReqExtra", restored.Meta().Req.Extra)
|
||||
}
|
||||
// Explicit mode refuses plain logins, so the download only finishes over TLS.
|
||||
startAndWait(t, restored)
|
||||
checkFile(t, restored.Meta().SingleFilepath(), want)
|
||||
}
|
||||
|
||||
func TestBadCertRejectedByDefault(t *testing.T) {
|
||||
for mode, scheme := range map[string]string{pftp.TLSExplicit: "ftpes", pftp.TLSImplicit: "ftps"} {
|
||||
t.Run(mode, func(t *testing.T) {
|
||||
srv := StartFTPServer(t, ServerOptions{TLS: mode})
|
||||
want := srv.WriteFile(t, "f.bin", RandomBytes(4096, 6))
|
||||
rawURL := ftpURL(srv, scheme, "", "f.bin")
|
||||
|
||||
// No injected roots: the self-signed certificate is unknown.
|
||||
f := buildFetcher(testManager(nil))
|
||||
err := resolveOpts(f, &base.Request{URL: rawURL}, &base.Options{Path: t.TempDir()})
|
||||
var verifyErr *tls.CertificateVerificationError
|
||||
if !errors.As(err, &verifyErr) {
|
||||
t.Fatalf("want a certificate verification error, got %v", err)
|
||||
}
|
||||
|
||||
// The request's existing skipVerifyCert switch accepts it.
|
||||
f = buildFetcher(testManager(nil))
|
||||
if err := resolveOpts(f, &base.Request{URL: rawURL, SkipVerifyCert: true}, &base.Options{Path: t.TempDir()}); err != nil {
|
||||
t.Fatalf("skipVerifyCert resolve: %v", err)
|
||||
}
|
||||
startAndWait(t, f)
|
||||
checkFile(t, f.Meta().SingleFilepath(), want)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSegmentsUseRest(t *testing.T) {
|
||||
srv := StartFTPServer(t, ServerOptions{})
|
||||
srv.ReadDelay.Store(int64(time.Millisecond))
|
||||
size := 8 * mib
|
||||
want := srv.WriteFile(t, "big.bin", RandomBytes(size, 7))
|
||||
|
||||
f := buildFetcher(testManager(srv))
|
||||
resolveTask(t, f, ftpURL(srv, "ftp", "", "big.bin"), &pftp.OptsExtra{Connections: 4})
|
||||
startAndWait(t, f)
|
||||
checkFile(t, f.Meta().SingleFilepath(), want)
|
||||
|
||||
rests := srv.RestOffsets()
|
||||
for _, off := range []int64{2 * mib, 4 * mib, 6 * mib} {
|
||||
if !slices.Contains(rests, off) {
|
||||
t.Fatalf("REST offsets %v miss the segment start %d", rests, off)
|
||||
}
|
||||
}
|
||||
for _, off := range rests {
|
||||
if off <= 0 || off >= int64(size) {
|
||||
t.Fatalf("REST offset %d out of range", off)
|
||||
}
|
||||
}
|
||||
if got := len(ftpStats(t, f).Connections); got != 4 {
|
||||
t.Fatalf("stats report %d connections, want 4", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoginCapFallsBack(t *testing.T) {
|
||||
srv := StartFTPServer(t, ServerOptions{MaxLogins: 1})
|
||||
want := srv.WriteFile(t, "capped.bin", RandomBytes(4*mib+5, 8))
|
||||
|
||||
fm := testManager(srv)
|
||||
f := buildFetcher(fm)
|
||||
resolveTask(t, f, ftpURL(srv, "ftp", "", "capped.bin"), &pftp.OptsExtra{Connections: 4})
|
||||
startAndWait(t, f)
|
||||
checkFile(t, f.Meta().SingleFilepath(), want)
|
||||
|
||||
if srv.Refused() == 0 {
|
||||
t.Fatal("the server never refused a login, so the cap was not exercised")
|
||||
}
|
||||
if got := len(ftpStats(t, f).Connections); got != 1 {
|
||||
t.Fatalf("stats report %d connections, want 1", got)
|
||||
}
|
||||
var saved fetcherData
|
||||
if err := json.Unmarshal(storeData(t, fm, f), &saved); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if saved.MaxConns != 1 {
|
||||
t.Fatalf("saved login ceiling %d, want 1", saved.MaxConns)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNoRestSequential(t *testing.T) {
|
||||
srv := StartFTPServer(t, ServerOptions{NoRest: true})
|
||||
srv.ReadDelay.Store(int64(time.Millisecond))
|
||||
want := srv.WriteFile(t, "norest.bin", RandomBytes(4*mib+3, 9))
|
||||
|
||||
fm := testManager(srv)
|
||||
f := buildFetcher(fm)
|
||||
resolveTask(t, f, ftpURL(srv, "ftp", "", "norest.bin"), &pftp.OptsExtra{Connections: 4})
|
||||
startAndWait(t, f)
|
||||
checkFile(t, f.Meta().SingleFilepath(), want)
|
||||
|
||||
if rests := srv.RestOffsets(); len(rests) != 0 {
|
||||
t.Fatalf("the server cannot have seen a REST, got %v", rests)
|
||||
}
|
||||
var saved fetcherData
|
||||
if err := json.Unmarshal(storeData(t, fm, f), &saved); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !saved.NoRest {
|
||||
t.Fatal("the missing REST support was not recorded")
|
||||
}
|
||||
}
|
||||
|
||||
// TestResumeAfterKill stops the server's transfers at a fixed byte count,
|
||||
// waits until the client has written exactly those bytes, then pauses, saves,
|
||||
// restores and finishes. No timing decides where the pause lands.
|
||||
func TestResumeAfterKill(t *testing.T) {
|
||||
srv := StartFTPServer(t, ServerOptions{})
|
||||
size := int64(4*mib + 777)
|
||||
want := srv.WriteFile(t, "resume.bin", RandomBytes(int(size), 10))
|
||||
stallAt := size / 2
|
||||
srv.StallAfter(stallAt)
|
||||
|
||||
fm := testManager(srv)
|
||||
f := buildFetcher(fm)
|
||||
resolveTask(t, f, ftpURL(srv, "ftp", "", "resume.bin"), &pftp.OptsExtra{Connections: 2})
|
||||
if err := f.Start(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
select {
|
||||
case <-srv.Stalled():
|
||||
case <-time.After(30 * time.Second):
|
||||
t.Fatal("the server never reached the stall point")
|
||||
}
|
||||
deadline := time.Now().Add(30 * time.Second)
|
||||
for f.Progress().TotalDownloaded() != stallAt {
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("progress %d, want the %d bytes the server sent", f.Progress().TotalDownloaded(), stallAt)
|
||||
}
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
}
|
||||
if err := f.Pause(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
srv.Release()
|
||||
blob := storeData(t, fm, f)
|
||||
|
||||
var saved fetcherData
|
||||
if err := json.Unmarshal(blob, &saved); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var savedTotal int64
|
||||
resumeAt := map[int64]bool{}
|
||||
for _, seg := range saved.Segments {
|
||||
savedTotal += seg.Downloaded
|
||||
if seg.Begin+seg.Downloaded <= seg.End {
|
||||
resumeAt[seg.Begin+seg.Downloaded] = true
|
||||
}
|
||||
}
|
||||
if savedTotal != stallAt {
|
||||
t.Fatalf("saved progress %d, want %d", savedTotal, stallAt)
|
||||
}
|
||||
|
||||
before := len(srv.RetrOffsets())
|
||||
restored := restoreFetcher(t, fm, f.Meta(), blob)
|
||||
if got := restored.Progress().TotalDownloaded(); got != savedTotal {
|
||||
t.Fatalf("restored progress %d, want %d", got, savedTotal)
|
||||
}
|
||||
startAndWait(t, restored)
|
||||
checkFile(t, restored.Meta().SingleFilepath(), want)
|
||||
|
||||
resumed := srv.RetrOffsets()[before:]
|
||||
if len(resumed) == 0 {
|
||||
t.Fatal("no transfer after resume")
|
||||
}
|
||||
for _, off := range resumed {
|
||||
if !resumeAt[off] {
|
||||
t.Fatalf("resumed RETR at %d, want one of the saved offsets %v", off, resumeAt)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestLateRefusalAfterQueueDrained: worker B holds the second file's segment
|
||||
// while the server takes its time to refuse B's login. Worker A finishes the
|
||||
// first file meanwhile and finds the queue empty. B must not leave too, or
|
||||
// nobody fetches the second file.
|
||||
func TestLateRefusalAfterQueueDrained(t *testing.T) {
|
||||
srv := StartFTPServer(t, ServerOptions{MaxLogins: 1, RefuseDelay: 500 * time.Millisecond})
|
||||
want := map[string]string{
|
||||
"a.bin": srv.WriteFile(t, "dir/a.bin", RandomBytes(1000, 16)),
|
||||
"b.bin": srv.WriteFile(t, "dir/b.bin", RandomBytes(1000, 17)),
|
||||
}
|
||||
|
||||
f := buildFetcher(testManager(srv))
|
||||
resolveTask(t, f, ftpURL(srv, "ftp", "", "dir"), &pftp.OptsExtra{Connections: 2})
|
||||
startAndWait(t, f)
|
||||
for name, sum := range want {
|
||||
checkFile(t, filepath.Join(f.Meta().FolderPath(), name), sum)
|
||||
}
|
||||
if srv.Refused() == 0 {
|
||||
t.Fatal("the server never refused a login, so the late refusal was not exercised")
|
||||
}
|
||||
}
|
||||
|
||||
// TestLoginCapLoweredMidway: both workers have logged in when the server's
|
||||
// cap drops to one, as when another client takes a slot. The refused worker
|
||||
// leaves, the ceiling drops to one, and the task finishes.
|
||||
func TestLoginCapLoweredMidway(t *testing.T) {
|
||||
srv := StartFTPServer(t, ServerOptions{})
|
||||
srv.ReadDelay.Store(int64(time.Millisecond))
|
||||
want := map[string]string{}
|
||||
for i := 0; i < 8; i++ {
|
||||
name := fmt.Sprintf("f%d.bin", i)
|
||||
want[name] = srv.WriteFile(t, "dir/"+name, RandomBytes(512<<10, byte(30+i)))
|
||||
}
|
||||
|
||||
fm := testManager(srv)
|
||||
f := buildFetcher(fm)
|
||||
resolveTask(t, f, ftpURL(srv, "ftp", "", "dir"), &pftp.OptsExtra{Connections: 2})
|
||||
if err := f.Start(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
deadline := time.Now().Add(30 * time.Second)
|
||||
for len(ftpStats(t, f).Connections) < 2 || f.Progress().TotalDownloaded() < 512<<10 {
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatal("the two workers never got going")
|
||||
}
|
||||
time.Sleep(2 * time.Millisecond)
|
||||
}
|
||||
srv.SetMaxLogins(1)
|
||||
if err := waitFetcher(t, f); err != nil {
|
||||
t.Fatalf("download: %v", err)
|
||||
}
|
||||
for name, sum := range want {
|
||||
checkFile(t, filepath.Join(f.Meta().FolderPath(), name), sum)
|
||||
}
|
||||
if srv.Refused() == 0 {
|
||||
t.Fatal("the server never refused a login, so the lowered cap was not exercised")
|
||||
}
|
||||
for _, c := range ftpStats(t, f).Connections {
|
||||
if c.Failed {
|
||||
t.Fatalf("a refused worker failed: %+v", c)
|
||||
}
|
||||
}
|
||||
var saved fetcherData
|
||||
if err := json.Unmarshal(storeData(t, fm, f), &saved); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if saved.MaxConns != 1 {
|
||||
t.Fatalf("saved login ceiling %d, want 1", saved.MaxConns)
|
||||
}
|
||||
}
|
||||
|
||||
// TestDataConnectionsResumeOwnSession runs four FTPS workers against a server
|
||||
// that, like ProFTPD, accepts a data connection only when it resumes the TLS
|
||||
// session of its own control connection.
|
||||
func TestDataConnectionsResumeOwnSession(t *testing.T) {
|
||||
srv := StartFTPServer(t, ServerOptions{TLS: pftp.TLSImplicit, RequireSessionReuse: true})
|
||||
srv.ReadDelay.Store(int64(200 * time.Microsecond))
|
||||
want := srv.WriteFile(t, "reuse.bin", RandomBytes(8*mib, 18))
|
||||
|
||||
f := buildFetcher(testManager(srv))
|
||||
resolveTask(t, f, ftpURL(srv, "ftps", "", "reuse.bin"), &pftp.OptsExtra{Connections: 4})
|
||||
startAndWait(t, f)
|
||||
checkFile(t, f.Meta().SingleFilepath(), want)
|
||||
ok, bad := srv.SessionReuse()
|
||||
if ok < 4 || bad != 0 {
|
||||
t.Fatalf("data connections: %d resumed their own session, %d did not", ok, bad)
|
||||
}
|
||||
for _, c := range ftpStats(t, f).Connections {
|
||||
if c.RetryTimes != 0 {
|
||||
t.Fatalf("a worker needed retries: %+v", c)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRetryAfterDroppedConnection(t *testing.T) {
|
||||
srv := StartFTPServer(t, ServerOptions{})
|
||||
want := srv.WriteFile(t, "drop.bin", RandomBytes(3*mib, 11))
|
||||
srv.DropAfter.Store(mib + 100)
|
||||
|
||||
f := buildFetcher(testManager(srv))
|
||||
resolveTask(t, f, ftpURL(srv, "ftp", "", "drop.bin"), &pftp.OptsExtra{Connections: 1})
|
||||
startAndWait(t, f)
|
||||
checkFile(t, f.Meta().SingleFilepath(), want)
|
||||
|
||||
offsets := srv.RetrOffsets()
|
||||
if len(offsets) != 2 || offsets[0] != 0 || offsets[1] <= 0 || offsets[1] > mib+100 {
|
||||
t.Fatalf("RETR offsets %v, want a restart at the dropped offset", offsets)
|
||||
}
|
||||
if got := ftpStats(t, f).Connections[0].RetryTimes; got != 1 {
|
||||
t.Fatalf("retry count %d, want 1", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPercentEncodedPassword(t *testing.T) {
|
||||
srv := StartFTPServer(t, ServerOptions{User: "u", Pass: "p@ss"})
|
||||
want := srv.WriteFile(t, "secret.bin", RandomBytes(5000, 12))
|
||||
|
||||
f := buildFetcher(testManager(srv))
|
||||
resolveTask(t, f, ftpURL(srv, "ftp", "u:p%40ss", "secret.bin"), nil)
|
||||
startAndWait(t, f)
|
||||
checkFile(t, f.Meta().SingleFilepath(), want)
|
||||
|
||||
f = buildFetcher(testManager(srv))
|
||||
err := resolveOpts(f, &base.Request{URL: ftpURL(srv, "ftp", "u:wrong", "secret.bin")}, &base.Options{Path: t.TempDir()})
|
||||
if err == nil || !strings.Contains(err.Error(), "530") {
|
||||
t.Fatalf("a wrong password must fail with 530, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOldSaveFormatResumes(t *testing.T) {
|
||||
srv := StartFTPServer(t, ServerOptions{})
|
||||
data := RandomBytes(2*mib+1, 13)
|
||||
want := srv.WriteFile(t, "old.bin", data)
|
||||
|
||||
fm := testManager(srv)
|
||||
f := buildFetcher(fm)
|
||||
resolveTask(t, f, ftpURL(srv, "ftp", "", "old.bin"), &pftp.OptsExtra{Connections: 1})
|
||||
|
||||
// A save blob that has only the segment fields every version writes.
|
||||
const done = 700000
|
||||
blob := []byte(fmt.Sprintf(`{"Segments":[{"Begin":0,"End":%d,"Downloaded":%d}]}`, len(data)-1, done))
|
||||
name := f.Meta().SingleFilepath()
|
||||
if err := os.WriteFile(name, append(data[:done:done], make([]byte, len(data)-done)...), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
restored := restoreFetcher(t, fm, f.Meta(), blob)
|
||||
if got := restored.Progress().TotalDownloaded(); got != done {
|
||||
t.Fatalf("restored progress %d, want %d", got, done)
|
||||
}
|
||||
startAndWait(t, restored)
|
||||
checkFile(t, name, want)
|
||||
if got := srv.RetrOffsets(); !slices.Equal(got, []int64{done}) {
|
||||
t.Fatalf("RETR offsets %v, want [%d]", got, done)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloadThroughSOCKS5(t *testing.T) {
|
||||
srv := StartFTPServer(t, ServerOptions{TLS: pftp.TLSExplicit})
|
||||
want := srv.WriteFile(t, "proxied.bin", RandomBytes(2*mib+1, 14))
|
||||
|
||||
var dials atomic.Int32
|
||||
proxySrv, err := socks5.New(&socks5.Config{
|
||||
Dial: func(ctx context.Context, network, addr string) (net.Conn, error) {
|
||||
dials.Add(1)
|
||||
var d net.Dialer
|
||||
return d.DialContext(ctx, network, addr)
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ln, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer ln.Close()
|
||||
go proxySrv.Serve(ln)
|
||||
|
||||
f := buildFetcher(testManager(srv))
|
||||
req := &base.Request{
|
||||
URL: ftpURL(srv, "ftpes", "", "proxied.bin"),
|
||||
Proxy: &base.RequestProxy{
|
||||
Mode: base.RequestProxyModeCustom,
|
||||
Scheme: "socks5",
|
||||
Host: ln.Addr().String(),
|
||||
},
|
||||
}
|
||||
if err := resolveOpts(f, req, &base.Options{Path: t.TempDir(), Extra: &pftp.OptsExtra{Connections: 2}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
startAndWait(t, f)
|
||||
checkFile(t, f.Meta().SingleFilepath(), want)
|
||||
// Resolve's control and listing, then per worker one control and one
|
||||
// data connection at least: every one of them went through the proxy.
|
||||
if got := dials.Load(); got < 4 {
|
||||
t.Fatalf("the proxy saw %d connections, want every control and data connection", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHTTPProxyFallsBackToDirect(t *testing.T) {
|
||||
srv := StartFTPServer(t, ServerOptions{})
|
||||
want := srv.WriteFile(t, "direct.bin", RandomBytes(100000, 15))
|
||||
|
||||
var logs bytes.Buffer
|
||||
logger := zerolog.New(&logs)
|
||||
fm := testManager(srv)
|
||||
f := fm.Build().(*Fetcher)
|
||||
ctl := testController(fm)
|
||||
ctl.Logger = &logger
|
||||
f.Setup(ctl)
|
||||
req := &base.Request{
|
||||
URL: ftpURL(srv, "ftp", "", "direct.bin"),
|
||||
Proxy: &base.RequestProxy{
|
||||
Mode: base.RequestProxyModeCustom,
|
||||
Scheme: "http",
|
||||
Host: "127.0.0.1:1",
|
||||
Usr: "proxyuser",
|
||||
Pwd: "proxysecret",
|
||||
},
|
||||
}
|
||||
if err := resolveOpts(f, req, &base.Options{Path: t.TempDir(), Extra: &pftp.OptsExtra{Connections: 2}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
startAndWait(t, f)
|
||||
checkFile(t, f.Meta().SingleFilepath(), want)
|
||||
|
||||
out := logs.String()
|
||||
if n := strings.Count(out, "HTTP proxy"); n != 1 {
|
||||
t.Fatalf("want exactly one HTTP proxy warning, got %d in %q", n, out)
|
||||
}
|
||||
if strings.Contains(out, "proxysecret") {
|
||||
t.Fatalf("the proxy password leaked into the log: %q", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTorrentNameIsFile(t *testing.T) {
|
||||
fm := testManager(nil)
|
||||
for _, u := range []string{"ftp://h/x.torrent", "ftps://h/x.torrent", "ftpes://h/x.torrent", "FTP://h/v.m3u8"} {
|
||||
matched := false
|
||||
for _, filter := range fm.Filters() {
|
||||
if filter.Match(u) {
|
||||
matched = true
|
||||
}
|
||||
}
|
||||
if !matched {
|
||||
t.Fatalf("%s is not matched by the FTP filters", u)
|
||||
}
|
||||
}
|
||||
if fm.Name() != "ftp" {
|
||||
t.Fatalf("manager name %q", fm.Name())
|
||||
}
|
||||
|
||||
srv := StartFTPServer(t, ServerOptions{})
|
||||
want := srv.WriteFile(t, "x.torrent", []byte("d8:announce0:e"))
|
||||
f := buildFetcher(testManager(srv))
|
||||
resolveTask(t, f, ftpURL(srv, "ftp", "", "x.torrent"), nil)
|
||||
res := f.Meta().Res
|
||||
if res.Name != "" || len(res.Files) != 1 || res.Files[0].Name != "x.torrent" {
|
||||
t.Fatalf("x.torrent must resolve to one FTP file, got name %q and %d files", res.Name, len(res.Files))
|
||||
}
|
||||
startAndWait(t, f)
|
||||
checkFile(t, f.Meta().SingleFilepath(), want)
|
||||
}
|
||||
|
||||
func TestParseNameHidesCredentials(t *testing.T) {
|
||||
fm := testManager(nil)
|
||||
for u, want := range map[string]string{
|
||||
"ftp://u:p%[email protected]/pub/a.iso": "a.iso",
|
||||
"ftp://u:p%[email protected]/": "example.com",
|
||||
"ftps://u:p%[email protected]": "example.com",
|
||||
"ftp://u:p%[email protected]/pub/dir/": "dir",
|
||||
} {
|
||||
if got := fm.ParseName(u); got != want {
|
||||
t.Fatalf("ParseName(%q) = %q, want %q", u, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestUnsafeEntryNames(t *testing.T) {
|
||||
for name, ok := range map[string]bool{
|
||||
"a.txt": true, "..a": true, "a b": true,
|
||||
"": false, ".": false, "..": false, "a/b": false, `a\b`: false, "../x": false, "x\x00y": false,
|
||||
} {
|
||||
if got := safeEntryName(name); got != ok {
|
||||
t.Fatalf("safeEntryName(%q) = %v, want %v", name, got, ok)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,587 @@
|
||||
package ftp
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"crypto/ecdsa"
|
||||
"crypto/elliptic"
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"crypto/x509/pkix"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"io"
|
||||
"math/big"
|
||||
"net"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
ftpserver "github.com/fclairamb/ftpserverlib"
|
||||
"github.com/spf13/afero"
|
||||
|
||||
pftp "github.com/GopeedLab/gopeed/pkg/protocol/ftp"
|
||||
)
|
||||
|
||||
// ServerOptions are the knobs of the in-process FTP test server.
|
||||
type ServerOptions struct {
|
||||
// TLS is pftp.TLSNone (or empty), pftp.TLSExplicit or pftp.TLSImplicit.
|
||||
// Explicit mode refuses plain logins, so a passing test proves TLS was used.
|
||||
TLS string
|
||||
// MaxLogins caps concurrent control connections. Beyond it the server
|
||||
// greets with "421 Too many connections" and hangs up. Zero means no cap.
|
||||
MaxLogins int
|
||||
// NoRest answers "502" to REST. It filters the plain control stream, so it
|
||||
// works with TLSNone and TLSImplicit only.
|
||||
NoRest bool
|
||||
// NoMLST disables MLST and MLSD, so clients fall back to SIZE and LIST.
|
||||
NoMLST bool
|
||||
// User and Pass are the only accepted credentials. An empty User accepts
|
||||
// any login, anonymous included.
|
||||
User string
|
||||
Pass string
|
||||
// RefuseDelay holds every 421 refusal back this long, like a server
|
||||
// that answers slowly when it is full.
|
||||
RefuseDelay time.Duration
|
||||
// RequireSessionReuse makes every data connection resume the TLS
|
||||
// session of the control connection that opened it, as ProFTPD's
|
||||
// mod_tls does by default. A data connection that does not is refused.
|
||||
// It needs TLSImplicit, where this test's wrapper sees the plain
|
||||
// control stream and so learns which passive port belongs to whom.
|
||||
RequireSessionReuse bool
|
||||
}
|
||||
|
||||
// FTPServer is an ftpserverlib server over a temp directory.
|
||||
type FTPServer struct {
|
||||
Addr string // host:port
|
||||
Root string // directory served as "/"
|
||||
// CA trusts the server's self-signed certificate.
|
||||
CA *x509.CertPool
|
||||
|
||||
opts ServerOptions
|
||||
tlsConf *tls.Config
|
||||
server *ftpserver.FtpServer
|
||||
|
||||
// ReadDelay is slept before every read of a transferred file, in
|
||||
// nanoseconds, to slow downloads down. Tests may change it at any time.
|
||||
ReadDelay atomic.Int64
|
||||
|
||||
// DropAfter, when positive, makes the next transfer fail once after it
|
||||
// has sent that many bytes, as if the data connection were killed.
|
||||
DropAfter atomic.Int64
|
||||
|
||||
mu sync.Mutex
|
||||
active int
|
||||
retrs []int64 // offset of every RETR, 0 when no REST came first
|
||||
refused int
|
||||
logins int
|
||||
|
||||
// Passive port to the control connection that announced it, and the
|
||||
// outcome of every data connection's session check.
|
||||
pasvOwner map[int]string
|
||||
reuseOK int
|
||||
reuseBad int
|
||||
|
||||
// Stall state, see StallAfter. stallMu is held across file reads.
|
||||
stallMu sync.Mutex
|
||||
stallAt int64
|
||||
sent int64
|
||||
stallCh chan struct{}
|
||||
stalled chan struct{}
|
||||
stalledOnce *sync.Once
|
||||
}
|
||||
|
||||
// StartFTPServer starts a server and stops it when the test ends.
|
||||
func StartFTPServer(t *testing.T, opts ServerOptions) *FTPServer {
|
||||
t.Helper()
|
||||
s := &FTPServer{Root: t.TempDir(), opts: opts}
|
||||
s.tlsConf, s.CA = selfSignedTLS(t)
|
||||
|
||||
base, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var ln net.Listener = base
|
||||
if opts.RequireSessionReuse {
|
||||
if opts.TLS != pftp.TLSImplicit {
|
||||
t.Fatal("RequireSessionReuse needs TLSImplicit")
|
||||
}
|
||||
s.requireSessionReuse(t, base.Addr().(*net.TCPAddr).Port)
|
||||
}
|
||||
if opts.TLS == pftp.TLSImplicit {
|
||||
ln = tls.NewListener(ln, s.tlsConf)
|
||||
}
|
||||
ln = &limitListener{Listener: ln, srv: s}
|
||||
|
||||
s.Addr = base.Addr().String()
|
||||
s.server = ftpserver.NewFtpServer(&mainDriver{srv: s, listener: ln})
|
||||
if err := s.server.Listen(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
go s.server.Serve()
|
||||
t.Cleanup(func() { _ = s.server.Stop() })
|
||||
return s
|
||||
}
|
||||
|
||||
// WriteFile creates a file under the served root and returns its SHA-256.
|
||||
func (s *FTPServer) WriteFile(t *testing.T, rel string, data []byte) string {
|
||||
t.Helper()
|
||||
name := filepath.Join(s.Root, filepath.FromSlash(rel))
|
||||
if err := os.MkdirAll(filepath.Dir(name), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(name, data, 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return SHA256Hex(data)
|
||||
}
|
||||
|
||||
// RetrOffsets returns the start offset of every RETR the server received.
|
||||
func (s *FTPServer) RetrOffsets() []int64 {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return append([]int64(nil), s.retrs...)
|
||||
}
|
||||
|
||||
// RestOffsets returns the non-zero RETR offsets, which only a REST sets.
|
||||
func (s *FTPServer) RestOffsets() []int64 {
|
||||
var out []int64
|
||||
for _, off := range s.RetrOffsets() {
|
||||
if off != 0 {
|
||||
out = append(out, off)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Refused returns how many connections were turned away with 421.
|
||||
func (s *FTPServer) Refused() int {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return s.refused
|
||||
}
|
||||
|
||||
// Logins returns the number of successful logins.
|
||||
func (s *FTPServer) Logins() int {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return s.logins
|
||||
}
|
||||
|
||||
// SetMaxLogins changes the login cap while the server runs, as when other
|
||||
// clients take slots.
|
||||
func (s *FTPServer) SetMaxLogins(n int) {
|
||||
s.mu.Lock()
|
||||
s.opts.MaxLogins = n
|
||||
s.mu.Unlock()
|
||||
}
|
||||
|
||||
// SessionReuse returns how many data connections resumed their own control
|
||||
// connection's TLS session, and how many did not.
|
||||
func (s *FTPServer) SessionReuse() (ok, bad int) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return s.reuseOK, s.reuseBad
|
||||
}
|
||||
|
||||
// StallAfter makes every transfer stop sending once the server has sent n
|
||||
// bytes in total, until Release. Stalled is closed when that happens.
|
||||
func (s *FTPServer) StallAfter(n int64) {
|
||||
s.stallMu.Lock()
|
||||
defer s.stallMu.Unlock()
|
||||
s.stallAt, s.sent = n, 0
|
||||
s.stallCh = make(chan struct{})
|
||||
s.stalled = make(chan struct{})
|
||||
s.stalledOnce = &sync.Once{}
|
||||
}
|
||||
|
||||
// Stalled is closed once a transfer has stopped at the StallAfter limit.
|
||||
func (s *FTPServer) Stalled() <-chan struct{} {
|
||||
s.stallMu.Lock()
|
||||
defer s.stallMu.Unlock()
|
||||
return s.stalled
|
||||
}
|
||||
|
||||
// Sent returns the bytes sent since StallAfter.
|
||||
func (s *FTPServer) Sent() int64 {
|
||||
s.stallMu.Lock()
|
||||
defer s.stallMu.Unlock()
|
||||
return s.sent
|
||||
}
|
||||
|
||||
// Release ends the stall. The stalled transfers fail.
|
||||
func (s *FTPServer) Release() {
|
||||
s.stallMu.Lock()
|
||||
defer s.stallMu.Unlock()
|
||||
if s.stallAt > 0 {
|
||||
s.stallAt = 0
|
||||
close(s.stallCh)
|
||||
}
|
||||
}
|
||||
|
||||
var errStalled = errors.New("test server released a stalled transfer")
|
||||
|
||||
// stall limits a read to the StallAfter budget, and blocks once the budget
|
||||
// is spent.
|
||||
func (s *FTPServer) stall(read func([]byte) (int, error), p []byte) (int, error) {
|
||||
s.stallMu.Lock()
|
||||
if s.stallAt <= 0 {
|
||||
s.stallMu.Unlock()
|
||||
return read(p)
|
||||
}
|
||||
allowed := s.stallAt - s.sent
|
||||
if allowed <= 0 {
|
||||
ch := s.stallCh
|
||||
s.stalledOnce.Do(func() { close(s.stalled) })
|
||||
s.stallMu.Unlock()
|
||||
<-ch
|
||||
return 0, errStalled
|
||||
}
|
||||
if int64(len(p)) > allowed {
|
||||
p = p[:allowed]
|
||||
}
|
||||
n, err := read(p)
|
||||
s.sent += int64(n)
|
||||
s.stallMu.Unlock()
|
||||
return n, err
|
||||
}
|
||||
|
||||
var pasvReply = regexp.MustCompile(`^(?:229 [^(]*\(\|\|\|(\d+)\|\)|227 [^(]*\(\d+,\d+,\d+,\d+,(\d+),(\d+)\))`)
|
||||
|
||||
// notePassive records the passive port a control connection announced.
|
||||
func (s *FTPServer) notePassive(owner string, reply []byte) {
|
||||
m := pasvReply.FindSubmatch(reply)
|
||||
if m == nil {
|
||||
return
|
||||
}
|
||||
var port int
|
||||
if len(m[1]) > 0 {
|
||||
port, _ = strconv.Atoi(string(m[1]))
|
||||
} else {
|
||||
hi, _ := strconv.Atoi(string(m[2]))
|
||||
lo, _ := strconv.Atoi(string(m[3]))
|
||||
port = hi<<8 | lo
|
||||
}
|
||||
s.mu.Lock()
|
||||
s.pasvOwner[port] = owner
|
||||
s.mu.Unlock()
|
||||
}
|
||||
|
||||
// requireSessionReuse stamps each control connection's session tickets with
|
||||
// its address, and checks that each data connection resumed a ticket of the
|
||||
// control connection that announced its passive port.
|
||||
func (s *FTPServer) requireSessionReuse(t *testing.T, controlPort int) {
|
||||
base := s.tlsConf
|
||||
var key [32]byte
|
||||
if _, err := rand.Read(key[:]); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
base.SetSessionTicketKeys([][32]byte{key})
|
||||
s.pasvOwner = map[int]string{}
|
||||
stamp := func(owner string) func(tls.ConnectionState, *tls.SessionState) ([]byte, error) {
|
||||
return func(cs tls.ConnectionState, ss *tls.SessionState) ([]byte, error) {
|
||||
ss.Extra = [][]byte{[]byte(owner)}
|
||||
return base.EncryptTicket(cs, ss)
|
||||
}
|
||||
}
|
||||
base.GetConfigForClient = func(hello *tls.ClientHelloInfo) (*tls.Config, error) {
|
||||
cfg := base.Clone()
|
||||
cfg.GetConfigForClient = nil
|
||||
local, _ := hello.Conn.LocalAddr().(*net.TCPAddr)
|
||||
if local == nil || local.Port == controlPort {
|
||||
cfg.WrapSession = stamp(hello.Conn.RemoteAddr().String())
|
||||
return cfg, nil
|
||||
}
|
||||
s.mu.Lock()
|
||||
owner := s.pasvOwner[local.Port]
|
||||
s.mu.Unlock()
|
||||
var resumedFrom string
|
||||
cfg.UnwrapSession = func(identity []byte, cs tls.ConnectionState) (*tls.SessionState, error) {
|
||||
ss, err := base.DecryptTicket(identity, cs)
|
||||
if err != nil || ss == nil {
|
||||
return nil, nil
|
||||
}
|
||||
if len(ss.Extra) > 0 {
|
||||
resumedFrom = string(ss.Extra[0])
|
||||
}
|
||||
return ss, nil
|
||||
}
|
||||
cfg.WrapSession = stamp(owner)
|
||||
cfg.VerifyConnection = func(cs tls.ConnectionState) error {
|
||||
ok := owner != "" && cs.DidResume && resumedFrom == owner
|
||||
s.mu.Lock()
|
||||
if ok {
|
||||
s.reuseOK++
|
||||
} else {
|
||||
s.reuseBad++
|
||||
}
|
||||
s.mu.Unlock()
|
||||
if !ok {
|
||||
return errors.New("data connection did not resume its control connection's TLS session")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
return cfg, nil
|
||||
}
|
||||
}
|
||||
|
||||
func (s *FTPServer) acquire() bool {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.opts.MaxLogins > 0 && s.active >= s.opts.MaxLogins {
|
||||
s.refused++
|
||||
return false
|
||||
}
|
||||
s.active++
|
||||
return true
|
||||
}
|
||||
|
||||
func (s *FTPServer) release() {
|
||||
s.mu.Lock()
|
||||
s.active--
|
||||
s.mu.Unlock()
|
||||
}
|
||||
|
||||
// limitListener counts control connections and refuses those beyond
|
||||
// MaxLogins with a 421 greeting, like vsftpd's max_clients and max_per_ip.
|
||||
type limitListener struct {
|
||||
net.Listener
|
||||
srv *FTPServer
|
||||
}
|
||||
|
||||
func (l *limitListener) Accept() (net.Conn, error) {
|
||||
for {
|
||||
c, err := l.Listener.Accept()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !l.srv.acquire() {
|
||||
go func() {
|
||||
time.Sleep(l.srv.opts.RefuseDelay)
|
||||
_ = c.SetDeadline(time.Now().Add(5 * time.Second))
|
||||
_, _ = io.WriteString(c, "421 Too many connections\r\n")
|
||||
_ = c.Close()
|
||||
}()
|
||||
continue
|
||||
}
|
||||
cc := &countedConn{Conn: c, release: l.srv.release}
|
||||
if l.srv.opts.RequireSessionReuse {
|
||||
owner := c.RemoteAddr().String()
|
||||
cc.sniff = func(b []byte) { l.srv.notePassive(owner, b) }
|
||||
}
|
||||
var conn net.Conn = cc
|
||||
if l.srv.opts.NoRest {
|
||||
conn = &noRestConn{countedConn: conn.(*countedConn), in: bufio.NewReader(c)}
|
||||
}
|
||||
return conn, nil
|
||||
}
|
||||
}
|
||||
|
||||
type countedConn struct {
|
||||
net.Conn
|
||||
once sync.Once
|
||||
release func()
|
||||
sniff func([]byte) // sees every reply the server writes
|
||||
}
|
||||
|
||||
func (c *countedConn) Write(b []byte) (int, error) {
|
||||
if c.sniff != nil {
|
||||
c.sniff(b)
|
||||
}
|
||||
return c.Conn.Write(b)
|
||||
}
|
||||
|
||||
func (c *countedConn) Close() error {
|
||||
c.once.Do(c.release)
|
||||
return c.Conn.Close()
|
||||
}
|
||||
|
||||
// noRestConn answers REST itself with 502 and hides it from the server.
|
||||
type noRestConn struct {
|
||||
*countedConn
|
||||
in *bufio.Reader
|
||||
pending []byte
|
||||
}
|
||||
|
||||
func (c *noRestConn) Read(p []byte) (int, error) {
|
||||
for len(c.pending) == 0 {
|
||||
line, err := c.in.ReadBytes('\n')
|
||||
if len(line) > 0 {
|
||||
if strings.HasPrefix(strings.ToUpper(string(line)), "REST ") {
|
||||
if _, werr := io.WriteString(c.Conn, "502 REST not implemented\r\n"); werr != nil {
|
||||
return 0, werr
|
||||
}
|
||||
} else {
|
||||
c.pending = line
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
if len(c.pending) > 0 {
|
||||
break
|
||||
}
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
n := copy(p, c.pending)
|
||||
c.pending = c.pending[n:]
|
||||
return n, nil
|
||||
}
|
||||
|
||||
type mainDriver struct {
|
||||
srv *FTPServer
|
||||
listener net.Listener
|
||||
}
|
||||
|
||||
func (d *mainDriver) GetSettings() (*ftpserver.Settings, error) {
|
||||
settings := &ftpserver.Settings{
|
||||
Listener: d.listener,
|
||||
DisableMLSD: d.srv.opts.NoMLST,
|
||||
DisableMLST: d.srv.opts.NoMLST,
|
||||
}
|
||||
switch d.srv.opts.TLS {
|
||||
case pftp.TLSExplicit:
|
||||
settings.TLSRequired = ftpserver.MandatoryEncryption
|
||||
case pftp.TLSImplicit:
|
||||
settings.TLSRequired = ftpserver.ImplicitEncryption
|
||||
}
|
||||
return settings, nil
|
||||
}
|
||||
|
||||
func (d *mainDriver) ClientConnected(ftpserver.ClientContext) (string, error) {
|
||||
return "gopeed test server", nil
|
||||
}
|
||||
|
||||
func (d *mainDriver) ClientDisconnected(ftpserver.ClientContext) {}
|
||||
|
||||
func (d *mainDriver) AuthUser(_ ftpserver.ClientContext, user, pass string) (ftpserver.ClientDriver, error) {
|
||||
if d.srv.opts.User != "" && (user != d.srv.opts.User || pass != d.srv.opts.Pass) {
|
||||
return nil, errors.New("bad credentials")
|
||||
}
|
||||
d.srv.mu.Lock()
|
||||
d.srv.logins++
|
||||
d.srv.mu.Unlock()
|
||||
return &clientDriver{Fs: afero.NewBasePathFs(afero.NewOsFs(), d.srv.Root), srv: d.srv}, nil
|
||||
}
|
||||
|
||||
func (d *mainDriver) GetTLSConfig() (*tls.Config, error) {
|
||||
return d.srv.tlsConf, nil
|
||||
}
|
||||
|
||||
type clientDriver struct {
|
||||
afero.Fs
|
||||
srv *FTPServer
|
||||
}
|
||||
|
||||
// GetHandle sees the REST offset of every transfer. The server seeks the
|
||||
// returned file to that offset itself.
|
||||
func (d *clientDriver) GetHandle(name string, flags int, offset int64) (ftpserver.FileTransfer, error) {
|
||||
f, err := d.Fs.OpenFile(name, flags, 0o644)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if flags == os.O_RDONLY {
|
||||
d.srv.mu.Lock()
|
||||
d.srv.retrs = append(d.srv.retrs, offset)
|
||||
d.srv.mu.Unlock()
|
||||
}
|
||||
return &slowFile{File: f, srv: d.srv, dropAfter: d.srv.DropAfter.Swap(0)}, nil
|
||||
}
|
||||
|
||||
var errDropped = errors.New("test server dropped the transfer")
|
||||
|
||||
type slowFile struct {
|
||||
afero.File
|
||||
srv *FTPServer
|
||||
sent int64
|
||||
dropAfter int64
|
||||
}
|
||||
|
||||
func (f *slowFile) Read(p []byte) (int, error) {
|
||||
if delay := time.Duration(f.srv.ReadDelay.Load()); delay > 0 {
|
||||
time.Sleep(delay)
|
||||
if len(p) > 16<<10 {
|
||||
p = p[:16<<10]
|
||||
}
|
||||
}
|
||||
if f.dropAfter > 0 {
|
||||
if f.sent >= f.dropAfter {
|
||||
return 0, errDropped
|
||||
}
|
||||
if rest := f.dropAfter - f.sent; int64(len(p)) > rest {
|
||||
p = p[:rest]
|
||||
}
|
||||
}
|
||||
n, err := f.srv.stall(f.File.Read, p)
|
||||
f.sent += int64(n)
|
||||
return n, err
|
||||
}
|
||||
|
||||
// selfSignedTLS returns a server config with a fresh certificate for
|
||||
// 127.0.0.1 and localhost, and a pool that trusts it.
|
||||
func selfSignedTLS(t *testing.T) (*tls.Config, *x509.CertPool) {
|
||||
t.Helper()
|
||||
key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
tmpl := &x509.Certificate{
|
||||
SerialNumber: big.NewInt(time.Now().UnixNano()),
|
||||
Subject: pkix.Name{CommonName: "gopeed ftp test"},
|
||||
NotBefore: time.Now().Add(-time.Hour),
|
||||
NotAfter: time.Now().Add(time.Hour),
|
||||
IPAddresses: []net.IP{net.ParseIP("127.0.0.1")},
|
||||
DNSNames: []string{"localhost"},
|
||||
KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign,
|
||||
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
|
||||
IsCA: true,
|
||||
BasicConstraintsValid: true,
|
||||
}
|
||||
der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cert, err := x509.ParseCertificate(der)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pool := x509.NewCertPool()
|
||||
pool.AddCert(cert)
|
||||
return &tls.Config{
|
||||
Certificates: []tls.Certificate{{Certificate: [][]byte{der}, PrivateKey: key, Leaf: cert}},
|
||||
MinVersion: tls.VersionTLS12,
|
||||
}, pool
|
||||
}
|
||||
|
||||
func SHA256Hex(data []byte) string {
|
||||
sum := sha256.Sum256(data)
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
func FileSHA256(t *testing.T, name string) string {
|
||||
t.Helper()
|
||||
data, err := os.ReadFile(name)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return SHA256Hex(data)
|
||||
}
|
||||
|
||||
// RandomBytes returns n reproducible pseudo-random bytes.
|
||||
func RandomBytes(n int, seed byte) []byte {
|
||||
out := make([]byte, n)
|
||||
x := uint32(seed) + 1
|
||||
for i := range out {
|
||||
x ^= x << 13
|
||||
x ^= x >> 17
|
||||
x ^= x << 5
|
||||
out[i] = byte(x)
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,187 @@
|
||||
package ftp
|
||||
|
||||
import (
|
||||
"cmp"
|
||||
"fmt"
|
||||
"path"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/jlaffaye/ftp"
|
||||
|
||||
"github.com/GopeedLab/gopeed/pkg/base"
|
||||
)
|
||||
|
||||
// Limits of a directory download. A tree past either one is refused with an
|
||||
// error that names the limit; it is never cut short.
|
||||
const maxWalkDepth = 8
|
||||
|
||||
// maxWalkFiles is a variable so tests can lower it.
|
||||
var maxWalkFiles = 10000
|
||||
|
||||
// resolveTarget describes the URL's path: one file, or a directory walked
|
||||
// into a multi-file resource whose paths are relative to it.
|
||||
func (f *Fetcher) resolveTarget(s *session, t *target) (*base.Resource, error) {
|
||||
isDir, size, mtime, err := f.stat(s, t.path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !isDir {
|
||||
return &base.Resource{
|
||||
Size: size,
|
||||
Range: true,
|
||||
Files: []*base.FileInfo{{
|
||||
Name: path.Base(t.path),
|
||||
Size: size,
|
||||
Ctime: mtime,
|
||||
}},
|
||||
}, nil
|
||||
}
|
||||
|
||||
files, err := f.walk(s, t.path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
name := path.Base(strings.TrimSuffix(t.path, "/"))
|
||||
if name == "" || name == "." || name == "/" {
|
||||
name = t.host
|
||||
}
|
||||
res := &base.Resource{Name: name, Range: true, Files: files}
|
||||
res.CalcSize(nil)
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// stat tells a file from a directory. MLST answers both at once. Without
|
||||
// it, SIZE answers for a file, and a directory is one we can change into.
|
||||
func (f *Fetcher) stat(s *session, p string) (isDir bool, size int64, mtime *time.Time, err error) {
|
||||
if p == "" || strings.HasSuffix(p, "/") {
|
||||
return true, 0, nil, nil
|
||||
}
|
||||
s.deadline()
|
||||
if entry, mlstErr := s.GetEntry(p); mlstErr == nil {
|
||||
switch entry.Type {
|
||||
case ftp.EntryTypeFolder:
|
||||
return true, 0, nil, nil
|
||||
case ftp.EntryTypeFile:
|
||||
if !entry.Time.IsZero() {
|
||||
t := entry.Time
|
||||
mtime = &t
|
||||
}
|
||||
return false, int64(entry.Size), mtime, nil
|
||||
}
|
||||
// A link: SIZE below follows it on the server.
|
||||
}
|
||||
// MLST is missing or refused the path: the older commands decide. A
|
||||
// broken connection fails them too, with its own error.
|
||||
|
||||
s.deadline()
|
||||
size, sizeErr := s.FileSize(p)
|
||||
if sizeErr == nil {
|
||||
if s.IsGetTimeSupported() {
|
||||
s.deadline()
|
||||
if t, err := s.GetTime(p); err == nil {
|
||||
mtime = &t
|
||||
}
|
||||
}
|
||||
return false, size, mtime, nil
|
||||
}
|
||||
if replyCode(sizeErr) == 0 {
|
||||
return false, 0, nil, sizeErr
|
||||
}
|
||||
|
||||
// SIZE failed: a directory, or a missing path.
|
||||
s.deadline()
|
||||
cwd, err := s.CurrentDir()
|
||||
if err != nil {
|
||||
return false, 0, nil, err
|
||||
}
|
||||
s.deadline()
|
||||
if err := s.ChangeDir(p); err != nil {
|
||||
return false, 0, nil, sizeErr
|
||||
}
|
||||
s.deadline()
|
||||
if err := s.ChangeDir(cwd); err != nil {
|
||||
return false, 0, nil, err
|
||||
}
|
||||
return true, 0, nil, nil
|
||||
}
|
||||
|
||||
// safeEntryName reports whether a name from a listing can become a local
|
||||
// path element. A hostile server could otherwise list "../x" and write
|
||||
// outside the task directory.
|
||||
func safeEntryName(name string) bool {
|
||||
if name == "" || name == "." || name == ".." {
|
||||
return false
|
||||
}
|
||||
return !strings.ContainsAny(name, "/\\\x00")
|
||||
}
|
||||
|
||||
// walk lists a directory tree, at most maxWalkDepth levels below root and
|
||||
// maxWalkFiles files. Files come back sorted by path, then name.
|
||||
func (f *Fetcher) walk(s *session, root string) ([]*base.FileInfo, error) {
|
||||
type dir struct {
|
||||
remote string // path on the server
|
||||
rel string // path below root, "" for root itself
|
||||
depth int
|
||||
}
|
||||
var files []*base.FileInfo
|
||||
stack := []dir{{remote: strings.TrimSuffix(root, "/"), rel: "", depth: 0}}
|
||||
for len(stack) > 0 {
|
||||
d := stack[len(stack)-1]
|
||||
stack = stack[:len(stack)-1]
|
||||
|
||||
s.deadline()
|
||||
entries, err := s.List(d.remote)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, e := range entries {
|
||||
if e.Name == "." || e.Name == ".." {
|
||||
continue
|
||||
}
|
||||
if !safeEntryName(e.Name) {
|
||||
f.warn("ftp: skipping a listed name that is not a safe file name: %q", e.Name)
|
||||
continue
|
||||
}
|
||||
remote := path.Join(d.remote, e.Name)
|
||||
rel := path.Join(d.rel, e.Name)
|
||||
if d.remote == "" {
|
||||
remote = e.Name
|
||||
}
|
||||
entryType, size := e.Type, int64(e.Size)
|
||||
if entryType == ftp.EntryTypeLink {
|
||||
// A link to a file has a size; a link to a directory is
|
||||
// not followed, so a loop cannot make the walk endless.
|
||||
s.deadline()
|
||||
linkSize, err := s.FileSize(remote)
|
||||
if err != nil {
|
||||
f.warn("ftp: skipping link %q: %v", rel, err)
|
||||
continue
|
||||
}
|
||||
entryType, size = ftp.EntryTypeFile, linkSize
|
||||
}
|
||||
switch entryType {
|
||||
case ftp.EntryTypeFolder:
|
||||
if d.depth+1 > maxWalkDepth {
|
||||
return nil, fmt.Errorf("ftp: the folder is deeper than %d levels, the limit for a folder download", maxWalkDepth)
|
||||
}
|
||||
stack = append(stack, dir{remote: remote, rel: rel, depth: d.depth + 1})
|
||||
case ftp.EntryTypeFile:
|
||||
if len(files) >= maxWalkFiles {
|
||||
return nil, fmt.Errorf("ftp: the folder has more than %d files, the limit for a folder download", maxWalkFiles)
|
||||
}
|
||||
fi := &base.FileInfo{Name: e.Name, Path: d.rel, Size: size}
|
||||
if !e.Time.IsZero() {
|
||||
t := e.Time
|
||||
fi.Ctime = &t
|
||||
}
|
||||
files = append(files, fi)
|
||||
}
|
||||
}
|
||||
}
|
||||
slices.SortFunc(files, func(a, b *base.FileInfo) int {
|
||||
return cmp.Or(strings.Compare(a.Path, b.Path), strings.Compare(a.Name, b.Name))
|
||||
})
|
||||
return files, nil
|
||||
}
|
||||
@@ -303,8 +303,9 @@ type Fetcher struct {
|
||||
resolveDataPos atomic.Int64 // How many bytes downloaded during resolve
|
||||
|
||||
// Resolve response - kept open for one-time URLs
|
||||
resolveResp *http.Response
|
||||
resolveRespLock sync.Mutex
|
||||
resolveResp *http.Response
|
||||
resolveRespCancel context.CancelFunc // Cancels the Resolve request
|
||||
resolveRespLock sync.Mutex
|
||||
|
||||
// Async prefetch during resolve phase
|
||||
prefetchFile *os.File // Temporary file for prefetch data
|
||||
@@ -425,8 +426,11 @@ func (f *Fetcher) Resolve(req *base.Request) error {
|
||||
|
||||
// Build HTTP request WITHOUT Range header (normal request)
|
||||
// This allows the response to be reused for downloading (important for one-time URLs)
|
||||
httpReq, err := f.buildRequest(context.TODO(), req)
|
||||
// The body outlives Resolve, so the read timeout gets its own cancel func.
|
||||
reqCtx, reqCancel := context.WithCancel(context.Background())
|
||||
httpReq, err := f.buildRequest(reqCtx, req)
|
||||
if err != nil {
|
||||
reqCancel()
|
||||
f.setState(stateError)
|
||||
return err
|
||||
}
|
||||
@@ -438,6 +442,7 @@ func (f *Fetcher) Resolve(req *base.Request) error {
|
||||
connStartTime := time.Now()
|
||||
resp, err := client.Do(httpReq)
|
||||
if err != nil {
|
||||
reqCancel()
|
||||
f.setState(stateError)
|
||||
return err
|
||||
}
|
||||
@@ -452,6 +457,7 @@ func (f *Fetcher) Resolve(req *base.Request) error {
|
||||
|
||||
if resp.StatusCode != base.HttpCodeOK && resp.StatusCode != base.HttpCodePartialContent {
|
||||
resp.Body.Close()
|
||||
reqCancel()
|
||||
f.setState(stateError)
|
||||
return NewRequestError(resp.StatusCode)
|
||||
}
|
||||
@@ -469,6 +475,7 @@ func (f *Fetcher) Resolve(req *base.Request) error {
|
||||
start, _, total, err := parseRangeResponse(resp)
|
||||
if err != nil {
|
||||
resp.Body.Close()
|
||||
reqCancel()
|
||||
f.setState(stateError)
|
||||
return err
|
||||
}
|
||||
@@ -536,8 +543,10 @@ func (f *Fetcher) Resolve(req *base.Request) error {
|
||||
f.resolveRespLock.Lock()
|
||||
if reuseResolveBody {
|
||||
f.resolveResp = resp
|
||||
f.resolveRespCancel = reqCancel
|
||||
} else {
|
||||
resp.Body.Close()
|
||||
reqCancel()
|
||||
f.prefetchDone.Store(true)
|
||||
}
|
||||
f.resolveRespLock.Unlock()
|
||||
@@ -574,7 +583,7 @@ func (f *Fetcher) asyncPrefetch() {
|
||||
|
||||
// Get the resolve response
|
||||
f.resolveRespLock.Lock()
|
||||
resp := f.resolveResp
|
||||
resp, cancel := f.resolveResp, f.resolveRespCancel
|
||||
f.resolveRespLock.Unlock()
|
||||
|
||||
if resp == nil {
|
||||
@@ -585,10 +594,7 @@ func (f *Fetcher) asyncPrefetch() {
|
||||
// Close the response even if temporary-file setup fails or shutdown
|
||||
// rejects a late prefetch. Otherwise the server can remain blocked writing.
|
||||
f.resolveRespLock.Lock()
|
||||
if f.resolveResp != nil {
|
||||
f.resolveResp.Body.Close()
|
||||
f.resolveResp = nil
|
||||
}
|
||||
f.closeResolveRespLocked()
|
||||
f.resolveRespLock.Unlock()
|
||||
}()
|
||||
|
||||
@@ -618,7 +624,7 @@ func (f *Fetcher) asyncPrefetch() {
|
||||
f.prefetchFilePath = tmpFile.Name()
|
||||
|
||||
buf := make([]byte, 32*1024) // 32KB buffer
|
||||
reader := f.responseReader(resp)
|
||||
reader := f.responseReader(resp, cancel)
|
||||
|
||||
for {
|
||||
select {
|
||||
@@ -881,10 +887,7 @@ func (f *Fetcher) doStart() error {
|
||||
if !f.resolveFallback.Load() {
|
||||
// Also close resolve response if still open.
|
||||
f.resolveRespLock.Lock()
|
||||
if f.resolveResp != nil {
|
||||
f.resolveResp.Body.Close()
|
||||
f.resolveResp = nil
|
||||
}
|
||||
f.closeResolveRespLocked()
|
||||
f.resolveRespLock.Unlock()
|
||||
}
|
||||
}
|
||||
@@ -1337,7 +1340,11 @@ func (f *Fetcher) downloadChunkOnce(conn *connection, client *http.Client, buf [
|
||||
}
|
||||
f.connMu.Unlock()
|
||||
|
||||
httpReq, err := f.buildRequest(conn.ctx, f.meta.Req)
|
||||
// The read timeout cancels this request through its own context.
|
||||
reqCtx, reqCancel := context.WithCancel(conn.ctx)
|
||||
defer reqCancel()
|
||||
|
||||
httpReq, err := f.buildRequest(reqCtx, f.meta.Req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -1365,7 +1372,7 @@ func (f *Fetcher) downloadChunkOnce(conn *connection, client *http.Client, buf [
|
||||
// Check if this might be a redirect URL expiration error
|
||||
// If so, try falling back to the original URL
|
||||
if f.hasRedirectURL() && isRedirectExpiredError(originalErr) {
|
||||
fallbackResp, fallbackErr := f.tryFallbackToOriginalURL(conn.ctx, client, rangeStart, rangeEnd, rangeRequested, ifRange)
|
||||
fallbackResp, fallbackErr := f.tryFallbackToOriginalURL(reqCtx, client, rangeStart, rangeEnd, rangeRequested, ifRange)
|
||||
if fallbackErr == nil && fallbackResp != nil {
|
||||
// Fallback succeeded, use this response instead
|
||||
resp = fallbackResp
|
||||
@@ -1486,7 +1493,7 @@ func (f *Fetcher) downloadChunkOnce(conn *connection, client *http.Client, buf [
|
||||
f.slowStart.onConnectSuccess()
|
||||
}
|
||||
|
||||
reader := f.responseReader(resp)
|
||||
reader := f.responseReader(resp, reqCancel)
|
||||
var responseBytesRead int64
|
||||
for {
|
||||
if conn.ctx.Err() != nil {
|
||||
@@ -1840,8 +1847,8 @@ func (f *Fetcher) runConnectionWithResolveResp(conn *connection) {
|
||||
|
||||
// Get the resolve response
|
||||
f.resolveRespLock.Lock()
|
||||
resp := f.resolveResp
|
||||
f.resolveResp = nil // Take ownership
|
||||
resp, cancel := f.resolveResp, f.resolveRespCancel
|
||||
f.resolveResp, f.resolveRespCancel = nil, nil // Take ownership
|
||||
f.resolveRespLock.Unlock()
|
||||
|
||||
if resp == nil {
|
||||
@@ -1850,6 +1857,9 @@ func (f *Fetcher) runConnectionWithResolveResp(conn *connection) {
|
||||
return
|
||||
}
|
||||
|
||||
if cancel != nil {
|
||||
defer cancel()
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
f.connMu.Lock()
|
||||
@@ -1866,7 +1876,7 @@ func (f *Fetcher) runConnectionWithResolveResp(conn *connection) {
|
||||
}
|
||||
|
||||
// Download data from resolve response
|
||||
reader := f.responseReader(resp)
|
||||
reader := f.responseReader(resp, cancel)
|
||||
for {
|
||||
if conn.ctx.Err() != nil {
|
||||
return
|
||||
@@ -2481,10 +2491,7 @@ func (f *Fetcher) Pause() error {
|
||||
|
||||
// Clean up resolve response if still held
|
||||
f.resolveRespLock.Lock()
|
||||
if f.resolveResp != nil {
|
||||
f.resolveResp.Body.Close()
|
||||
f.resolveResp = nil
|
||||
}
|
||||
f.closeResolveRespLocked()
|
||||
f.resolveRespLock.Unlock()
|
||||
|
||||
f.fileMu.Lock()
|
||||
@@ -2602,9 +2609,25 @@ func (f *Fetcher) Wait() error {
|
||||
|
||||
// Generated output can pause while its producer waits for processing capacity.
|
||||
// Its producer owns upstream deadlines; cancellation still closes the response.
|
||||
func (f *Fetcher) responseReader(resp *http.Response) io.Reader {
|
||||
// Otherwise a stalled read cancels the request via cancel, which must belong
|
||||
// to the request that produced resp.
|
||||
func (f *Fetcher) responseReader(resp *http.Response, cancel context.CancelFunc) io.Reader {
|
||||
if f.ctl != nil && f.ctl.ManagedProduction != nil && resp.Request != nil && f.ctl.ManagedProduction(resp.Request.URL.String()) {
|
||||
return resp.Body
|
||||
}
|
||||
return NewTimeoutReader(resp.Body, readTimeout)
|
||||
return NewTimeoutReader(resp.Body, readTimeout, cancel)
|
||||
}
|
||||
|
||||
// closeResolveRespLocked closes the kept Resolve response, if any, and
|
||||
// releases its request context. The caller must hold resolveRespLock.
|
||||
func (f *Fetcher) closeResolveRespLocked() {
|
||||
if f.resolveResp == nil {
|
||||
return
|
||||
}
|
||||
f.resolveResp.Body.Close()
|
||||
f.resolveResp = nil
|
||||
if f.resolveRespCancel != nil {
|
||||
f.resolveRespCancel()
|
||||
f.resolveRespCancel = nil
|
||||
}
|
||||
}
|
||||
@@ -6,35 +6,56 @@ import (
|
||||
"time"
|
||||
)
|
||||
|
||||
// TimeoutReader fails a Read that makes no progress within timeout.
|
||||
//
|
||||
// One timer per reader is armed when Read starts and stopped before it
|
||||
// returns, so time the caller spends between reads is not counted. If the
|
||||
// timer fires, it calls cancel, which should cancel the HTTP request the
|
||||
// reader belongs to so the transport unblocks the pending Read. Read still
|
||||
// waits for the underlying Read to return; that Read and every later one
|
||||
// return context.DeadlineExceeded. Errors from any other cancellation are
|
||||
// returned unchanged, and closing the response body is left to the caller.
|
||||
//
|
||||
// With a nil cancel the pending Read cannot be interrupted; it still reports
|
||||
// context.DeadlineExceeded, but only once it returns.
|
||||
//
|
||||
// TimeoutReader is not safe for concurrent use.
|
||||
type TimeoutReader struct {
|
||||
reader io.Reader
|
||||
timeout time.Duration
|
||||
reader io.Reader
|
||||
timeout time.Duration
|
||||
cancel context.CancelFunc
|
||||
timer *time.Timer
|
||||
timedOut bool
|
||||
}
|
||||
|
||||
func NewTimeoutReader(r io.Reader, timeout time.Duration) *TimeoutReader {
|
||||
func NewTimeoutReader(r io.Reader, timeout time.Duration, cancel context.CancelFunc) *TimeoutReader {
|
||||
if cancel == nil {
|
||||
cancel = func() {}
|
||||
}
|
||||
return &TimeoutReader{
|
||||
reader: r,
|
||||
timeout: timeout,
|
||||
cancel: cancel,
|
||||
}
|
||||
}
|
||||
|
||||
func (tr *TimeoutReader) Read(p []byte) (n int, err error) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), tr.timeout)
|
||||
defer cancel()
|
||||
|
||||
done := make(chan struct{})
|
||||
var readErr error
|
||||
var bytesRead int
|
||||
|
||||
go func() {
|
||||
bytesRead, readErr = tr.reader.Read(p)
|
||||
close(done)
|
||||
}()
|
||||
|
||||
select {
|
||||
case <-done:
|
||||
return bytesRead, readErr
|
||||
case <-ctx.Done():
|
||||
return 0, ctx.Err()
|
||||
if tr.timedOut {
|
||||
return 0, context.DeadlineExceeded
|
||||
}
|
||||
if tr.timer == nil {
|
||||
tr.timer = time.AfterFunc(tr.timeout, tr.cancel)
|
||||
} else {
|
||||
tr.timer.Reset(tr.timeout)
|
||||
}
|
||||
|
||||
n, err = tr.reader.Read(p)
|
||||
|
||||
if !tr.timer.Stop() {
|
||||
// The timer fired during this Read and cancelled the request, so err is
|
||||
// whatever the interrupted read produced. Report the timeout instead.
|
||||
tr.timedOut = true
|
||||
return n, context.DeadlineExceeded
|
||||
}
|
||||
return n, err
|
||||
}
|
||||
@@ -2,17 +2,26 @@ package http
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"errors"
|
||||
"io"
|
||||
gohttp "net/http"
|
||||
"net/http/httptest"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/GopeedLab/gopeed/internal/httpclient"
|
||||
)
|
||||
|
||||
func TestTimeoutReader_Read(t *testing.T) {
|
||||
data := []byte("Hello, World!")
|
||||
reader := bytes.NewReader(data)
|
||||
timeoutReader := NewTimeoutReader(reader, 1*time.Second)
|
||||
timeoutReader := NewTimeoutReader(reader, 1*time.Second, nil)
|
||||
|
||||
buf := make([]byte, len(data))
|
||||
n, err := timeoutReader.Read(buf)
|
||||
@@ -29,7 +38,7 @@ func TestTimeoutReader_Read(t *testing.T) {
|
||||
|
||||
func TestTimeoutReader_ReadTimeout(t *testing.T) {
|
||||
reader := &slowReader{delay: 2 * time.Second}
|
||||
timeoutReader := NewTimeoutReader(reader, 1*time.Second)
|
||||
timeoutReader := NewTimeoutReader(reader, 1*time.Second, nil)
|
||||
|
||||
buf := make([]byte, 8192)
|
||||
_, err := timeoutReader.Read(buf)
|
||||
@@ -49,3 +58,340 @@ func (sr *slowReader) Read(p []byte) (n int, err error) {
|
||||
time.Sleep(sr.delay)
|
||||
return 0, io.EOF
|
||||
}
|
||||
|
||||
// The timer must only cover time spent inside Read, not caller work between reads.
|
||||
func TestTimeoutReader_IdleBetweenReads(t *testing.T) {
|
||||
var cancelled atomic.Bool
|
||||
timeoutReader := NewTimeoutReader(bytes.NewReader([]byte("abcdef")), 50*time.Millisecond, func() {
|
||||
cancelled.Store(true)
|
||||
})
|
||||
|
||||
buf := make([]byte, 3)
|
||||
if _, err := timeoutReader.Read(buf); err != nil {
|
||||
t.Fatalf("first read: %v", err)
|
||||
}
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
if _, err := timeoutReader.Read(buf); err != nil {
|
||||
t.Fatalf("second read after idle: %v", err)
|
||||
}
|
||||
if cancelled.Load() {
|
||||
t.Fatal("request was cancelled while idle between reads")
|
||||
}
|
||||
}
|
||||
|
||||
// A timed-out Read must not return while the underlying Read can still write
|
||||
// into the caller's buffer, and must leave closing the reader to the caller.
|
||||
func TestTimeoutReader_TimedOutReadDoesNotOutliveCaller(t *testing.T) {
|
||||
reader := &lateWriteReader{cancelled: make(chan struct{})}
|
||||
timeoutReader := NewTimeoutReader(reader, 50*time.Millisecond, reader.cancel)
|
||||
|
||||
buf := make([]byte, 8)
|
||||
_, err := timeoutReader.Read(buf)
|
||||
if !errors.Is(err, context.DeadlineExceeded) {
|
||||
t.Fatalf("expected %v, got %v", context.DeadlineExceeded, err)
|
||||
}
|
||||
if !reader.returned.Load() {
|
||||
t.Fatal("Read returned before the underlying Read finished")
|
||||
}
|
||||
if _, err := timeoutReader.Read(buf); !errors.Is(err, context.DeadlineExceeded) {
|
||||
t.Fatalf("read after timeout: expected %v, got %v", context.DeadlineExceeded, err)
|
||||
}
|
||||
if reader.closed.Load() {
|
||||
t.Fatal("reader was closed by the timeout")
|
||||
}
|
||||
}
|
||||
|
||||
// lateWriteReader blocks until cancelled, then writes into p before returning.
|
||||
type lateWriteReader struct {
|
||||
cancelOnce sync.Once
|
||||
cancelled chan struct{}
|
||||
returned atomic.Bool
|
||||
closed atomic.Bool
|
||||
}
|
||||
|
||||
func (r *lateWriteReader) Read(p []byte) (int, error) {
|
||||
<-r.cancelled
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
p[0] = 'x'
|
||||
r.returned.Store(true)
|
||||
return 0, context.Canceled
|
||||
}
|
||||
|
||||
func (r *lateWriteReader) cancel() {
|
||||
r.cancelOnce.Do(func() { close(r.cancelled) })
|
||||
}
|
||||
|
||||
func (r *lateWriteReader) Close() error {
|
||||
r.closed.Store(true)
|
||||
return nil
|
||||
}
|
||||
|
||||
type timeoutReaderClientCase struct {
|
||||
name string
|
||||
http2 bool
|
||||
client func(t *testing.T, server *httptest.Server) *gohttp.Client
|
||||
}
|
||||
|
||||
func timeoutReaderClientCases() []timeoutReaderClientCase {
|
||||
newClient := func(mode httpclient.ImpersonationMode) func(t *testing.T, server *httptest.Server) *gohttp.Client {
|
||||
return func(t *testing.T, server *httptest.Server) *gohttp.Client {
|
||||
var tlsConfig *tls.Config
|
||||
if server.TLS != nil {
|
||||
roots := x509.NewCertPool()
|
||||
roots.AddCert(server.Certificate())
|
||||
tlsConfig = &tls.Config{RootCAs: roots}
|
||||
}
|
||||
client, err := httpclient.NewClient(httpclient.Options{
|
||||
Transport: httpclient.TransportOptions{
|
||||
TLSClientConfig: tlsConfig,
|
||||
},
|
||||
Impersonation: httpclient.ImpersonationOptions{
|
||||
Mode: mode,
|
||||
Browser: httpclient.BrowserChrome,
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return client
|
||||
}
|
||||
}
|
||||
var cases []timeoutReaderClientCase
|
||||
for _, mode := range []httpclient.ImpersonationMode{httpclient.ImpersonationNative, httpclient.ImpersonationFixed} {
|
||||
cases = append(cases,
|
||||
timeoutReaderClientCase{name: string(mode) + "/http1", client: newClient(mode)},
|
||||
timeoutReaderClientCase{name: string(mode) + "/http2", http2: true, client: newClient(mode)},
|
||||
)
|
||||
}
|
||||
return cases
|
||||
}
|
||||
|
||||
func startTimeoutReaderServer(t *testing.T, useHTTP2 bool, handler gohttp.HandlerFunc) *httptest.Server {
|
||||
server := httptest.NewUnstartedServer(handler)
|
||||
if useHTTP2 {
|
||||
server.EnableHTTP2 = true
|
||||
server.StartTLS()
|
||||
} else {
|
||||
server.Start()
|
||||
}
|
||||
t.Cleanup(server.Close)
|
||||
return server
|
||||
}
|
||||
|
||||
// readBodyWithTimeout sends the request under its own context derived from
|
||||
// ctx, as the fetcher does, and reads the body through a TimeoutReader.
|
||||
func readBodyWithTimeout(t *testing.T, ctx context.Context, client *gohttp.Client, url string, timeout time.Duration, wantProto int) ([]byte, error) {
|
||||
t.Helper()
|
||||
reqCtx, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
req, err := gohttp.NewRequestWithContext(reqCtx, gohttp.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.ProtoMajor != wantProto {
|
||||
t.Fatalf("expected HTTP/%d, got %s", wantProto, resp.Proto)
|
||||
}
|
||||
|
||||
type result struct {
|
||||
data []byte
|
||||
err error
|
||||
}
|
||||
done := make(chan result, 1)
|
||||
go func() {
|
||||
data, err := io.ReadAll(NewTimeoutReader(resp.Body, timeout, cancel))
|
||||
done <- result{data, err}
|
||||
}()
|
||||
select {
|
||||
case r := <-done:
|
||||
return r.data, r.err
|
||||
case <-time.After(timeout + 10*time.Second):
|
||||
t.Fatal("Read did not return after the read timeout")
|
||||
return nil, nil
|
||||
}
|
||||
}
|
||||
|
||||
func TestTimeoutReader_StalledHTTPBodyTimesOut(t *testing.T) {
|
||||
for _, tc := range timeoutReaderClientCases() {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
server := startTimeoutReaderServer(t, tc.http2, func(w gohttp.ResponseWriter, r *gohttp.Request) {
|
||||
w.Header().Set("Content-Length", "1048576")
|
||||
w.Write(bytes.Repeat([]byte("a"), 1024))
|
||||
w.(gohttp.Flusher).Flush()
|
||||
<-r.Context().Done()
|
||||
})
|
||||
|
||||
wantProto := 1
|
||||
if tc.http2 {
|
||||
wantProto = 2
|
||||
}
|
||||
const timeout = 200 * time.Millisecond
|
||||
start := time.Now()
|
||||
data, err := readBodyWithTimeout(t, context.Background(), tc.client(t, server), server.URL, timeout, wantProto)
|
||||
if !errors.Is(err, context.DeadlineExceeded) {
|
||||
t.Fatalf("expected %v, got %v", context.DeadlineExceeded, err)
|
||||
}
|
||||
if len(data) != 1024 {
|
||||
t.Fatalf("expected the 1024 bytes sent before the stall, got %d", len(data))
|
||||
}
|
||||
if elapsed := time.Since(start); elapsed < timeout {
|
||||
t.Fatalf("timed out after %v, before the %v read timeout", elapsed, timeout)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestTimeoutReader_SlowSteadyHTTPBodyDoesNotTimeOut(t *testing.T) {
|
||||
const (
|
||||
timeout = 300 * time.Millisecond
|
||||
interval = 30 * time.Millisecond
|
||||
chunks = 40 // 1.2s in total, four times the read timeout
|
||||
)
|
||||
for _, tc := range timeoutReaderClientCases() {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
server := startTimeoutReaderServer(t, tc.http2, func(w gohttp.ResponseWriter, r *gohttp.Request) {
|
||||
for i := 0; i < chunks; i++ {
|
||||
w.Write([]byte("0123456789"))
|
||||
w.(gohttp.Flusher).Flush()
|
||||
time.Sleep(interval)
|
||||
}
|
||||
})
|
||||
|
||||
wantProto := 1
|
||||
if tc.http2 {
|
||||
wantProto = 2
|
||||
}
|
||||
data, err := readBodyWithTimeout(t, context.Background(), tc.client(t, server), server.URL, timeout, wantProto)
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error, got %v", err)
|
||||
}
|
||||
if len(data) != chunks*10 {
|
||||
t.Fatalf("expected %d bytes, got %d", chunks*10, len(data))
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A gzip-encoded body that stalls must time out without racing the caller's
|
||||
// Body.Close against the decompressing reader.
|
||||
func TestTimeoutReader_StalledGzipHTTPBodyTimesOut(t *testing.T) {
|
||||
payload := bytes.Repeat([]byte("gopeed"), 1024)
|
||||
stages := []struct {
|
||||
name string
|
||||
write func(w io.Writer)
|
||||
want int
|
||||
}{
|
||||
{
|
||||
// The first bytes of a gzip header, so the client blocks inside
|
||||
// gzip.NewReader.
|
||||
name: "incomplete-header",
|
||||
write: func(w io.Writer) { w.Write([]byte{0x1f, 0x8b, 0x08, 0x00}) },
|
||||
want: 0,
|
||||
},
|
||||
{
|
||||
// A complete header and some compressed data, flushed without
|
||||
// finishing the gzip stream.
|
||||
name: "stalled-body",
|
||||
write: func(w io.Writer) {
|
||||
gz := gzip.NewWriter(w)
|
||||
gz.Write(payload)
|
||||
gz.Flush()
|
||||
},
|
||||
want: len(payload),
|
||||
},
|
||||
}
|
||||
for _, stage := range stages {
|
||||
for _, tc := range timeoutReaderClientCases() {
|
||||
t.Run(stage.name+"/"+tc.name, func(t *testing.T) {
|
||||
server := startTimeoutReaderServer(t, tc.http2, func(w gohttp.ResponseWriter, r *gohttp.Request) {
|
||||
w.Header().Set("Content-Encoding", "gzip")
|
||||
stage.write(w)
|
||||
w.(gohttp.Flusher).Flush()
|
||||
<-r.Context().Done()
|
||||
})
|
||||
|
||||
wantProto := 1
|
||||
if tc.http2 {
|
||||
wantProto = 2
|
||||
}
|
||||
data, err := readBodyWithTimeout(t, context.Background(), tc.client(t, server), server.URL, 200*time.Millisecond, wantProto)
|
||||
if !errors.Is(err, context.DeadlineExceeded) {
|
||||
t.Fatalf("expected %v, got %v", context.DeadlineExceeded, err)
|
||||
}
|
||||
if len(data) != stage.want {
|
||||
t.Fatalf("expected the %d bytes sent before the stall, got %d", stage.want, len(data))
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Cancelling the caller's context must not be reported as a read timeout.
|
||||
func TestTimeoutReader_CallerCancellationIsNotTimeout(t *testing.T) {
|
||||
for _, tc := range timeoutReaderClientCases() {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
server := startTimeoutReaderServer(t, tc.http2, func(w gohttp.ResponseWriter, r *gohttp.Request) {
|
||||
w.Header().Set("Content-Length", "1048576")
|
||||
w.Write(bytes.Repeat([]byte("a"), 1024))
|
||||
w.(gohttp.Flusher).Flush()
|
||||
<-r.Context().Done()
|
||||
})
|
||||
|
||||
wantProto := 1
|
||||
if tc.http2 {
|
||||
wantProto = 2
|
||||
}
|
||||
// Cancel once the client has read the first bytes, so the next
|
||||
// Read is the one blocked when the caller cancels.
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
client := tc.client(t, server)
|
||||
next := client.Transport
|
||||
client.Transport = roundTripFunc(func(r *gohttp.Request) (*gohttp.Response, error) {
|
||||
resp, err := next.RoundTrip(r)
|
||||
if err == nil {
|
||||
resp.Body = &cancelAfterFirstRead{ReadCloser: resp.Body, cancel: cancel}
|
||||
}
|
||||
return resp, err
|
||||
})
|
||||
|
||||
const timeout = 5 * time.Second
|
||||
start := time.Now()
|
||||
data, err := readBodyWithTimeout(t, ctx, client, server.URL, timeout, wantProto)
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("expected %v, got %v", context.Canceled, err)
|
||||
}
|
||||
if len(data) == 0 {
|
||||
t.Fatal("cancelled before any data was read")
|
||||
}
|
||||
if elapsed := time.Since(start); elapsed >= timeout {
|
||||
t.Fatalf("read returned after %v, not on cancellation", elapsed)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
type roundTripFunc func(*gohttp.Request) (*gohttp.Response, error)
|
||||
|
||||
func (f roundTripFunc) RoundTrip(r *gohttp.Request) (*gohttp.Response, error) {
|
||||
return f(r)
|
||||
}
|
||||
|
||||
// cancelAfterFirstRead calls cancel once a Read has returned data.
|
||||
type cancelAfterFirstRead struct {
|
||||
io.ReadCloser
|
||||
once sync.Once
|
||||
cancel context.CancelFunc
|
||||
}
|
||||
|
||||
func (b *cancelAfterFirstRead) Read(p []byte) (int, error) {
|
||||
n, err := b.ReadCloser.Read(p)
|
||||
if n > 0 {
|
||||
b.once.Do(b.cancel)
|
||||
}
|
||||
return n, err
|
||||
}
|
||||
+2
-2
@@ -180,11 +180,11 @@ func (s *Service) createTaskBatch(ctx *Context) *Response {
|
||||
if len(request.Reqs) == 0 {
|
||||
return newErrorResponse("param invalid: reqs", restmodel.CodeInvalidParam)
|
||||
}
|
||||
taskIDs, err := s.Downloader.CreateDirectBatch(&request)
|
||||
result, err := s.Downloader.CreateDirectBatchWithResult(&request)
|
||||
if err != nil {
|
||||
return newErrorResponse(err.Error())
|
||||
}
|
||||
return newOKResponse(taskIDs)
|
||||
return newOKResponse(result)
|
||||
}
|
||||
|
||||
func (s *Service) patchTask(ctx *Context) *Response {
|
||||
|
||||
@@ -267,3 +267,65 @@ func TestFullConfigAndJobTriggerShareProtectedProtocolState(t *testing.T) {
|
||||
t.Fatalf("job state overwritten: %s", raw)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBatchResolvedAndDirectCreation(t *testing.T) {
|
||||
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.Header().Set("Content-Length", "4")
|
||||
_, _ = w.Write([]byte("test"))
|
||||
}))
|
||||
t.Cleanup(upstream.Close)
|
||||
d := download.NewDownloader(&download.DownloaderConfig{StorageDir: t.TempDir(), TempDir: t.TempDir(), FetchManagers: []fetcher.FetcherManager{&httpProtocol.FetcherManager{}}})
|
||||
if err := d.Setup(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { d.Close() })
|
||||
cfg, err := d.GetConfig()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cfg.MaxRunning = 0
|
||||
if err := d.PutConfig(cfg); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
service, err := NewService(d)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, grouped := range []bool{false, true} {
|
||||
resolved := serviceTestRequest[*download.ResolveResult](t, service, http.MethodPost, "/api/v1/resolve", &restmodel.ResolveTask{Req: &base.Request{URL: upstream.URL + "/resolved"}})
|
||||
result := serviceTestRequest[map[string]json.RawMessage](t, service, http.MethodPost, "/api/v1/tasks/batch", &base.CreateTaskBatch{
|
||||
Group: grouped, GroupName: "folder", Opts: &base.Options{Path: t.TempDir()},
|
||||
Reqs: []*base.CreateTaskBatchItem{
|
||||
{Rid: "invalid"},
|
||||
{Rid: resolved.ID, Req: &base.Request{URL: "unsupported://ignored"}, Opts: &base.Options{SelectFiles: []int{0}}},
|
||||
{Req: &base.Request{URL: upstream.URL + "/direct"}},
|
||||
},
|
||||
})
|
||||
var ids []string
|
||||
if err := json.Unmarshal(result["taskIds"], &ids); err != nil || len(ids) != 2 {
|
||||
t.Fatalf("unexpected task IDs: %+v, %v", result, err)
|
||||
}
|
||||
wantFields := 1
|
||||
if grouped {
|
||||
wantFields = 2
|
||||
}
|
||||
if len(result) != wantFields {
|
||||
t.Fatalf("unexpected response fields: %+v", result)
|
||||
}
|
||||
var parentID string
|
||||
if grouped {
|
||||
if err := json.Unmarshal(result["parentId"], &parentID); err != nil || parentID == "" {
|
||||
t.Fatalf("missing parent ID: %+v", result)
|
||||
}
|
||||
}
|
||||
task := serviceTestRequest[*download.Task](t, service, http.MethodGet, "/api/v1/tasks/"+ids[0], nil)
|
||||
if task.ParentID != parentID || !reflect.DeepEqual(task.Meta.Opts.SelectFiles, []int{0}) {
|
||||
t.Fatalf("resolved task lost membership or selection: %+v", task)
|
||||
}
|
||||
status, response := service.JSON(http.MethodPost, "/api/v1/tasks/batch", nil, []byte(`{"reqs":[{"rid":"missing"},{"req":{"url":"unsupported://bad"}}]}`), nil)
|
||||
var failed restmodel.Result[any]
|
||||
if err := json.Unmarshal(response, &failed); err != nil || status != http.StatusOK || failed.Code == restmodel.CodeOk {
|
||||
t.Fatalf("all-failed batch succeeded: %d %s", status, response)
|
||||
}
|
||||
}
|
||||
}
|
||||
+32
-2
@@ -27,6 +27,33 @@ type Request struct {
|
||||
SkipVerifyCert bool `json:"skipVerifyCert"`
|
||||
}
|
||||
|
||||
// SetLabels replaces all labels on the request.
|
||||
//
|
||||
// Deprecated: In onResolve and onDone, assign request labels directly.
|
||||
// In onStart and onError, use task.meta.req.setLabels instead.
|
||||
func (r *Request) SetLabels(labels map[string]string) {
|
||||
r.Labels = labels
|
||||
}
|
||||
|
||||
// PutLabel sets a label on the request.
|
||||
//
|
||||
// Deprecated: In onResolve and onDone, assign request labels directly.
|
||||
// In onStart and onError, use task.meta.req.putLabel instead.
|
||||
func (r *Request) PutLabel(key, value string) {
|
||||
if r.Labels == nil {
|
||||
r.Labels = make(map[string]string)
|
||||
}
|
||||
r.Labels[key] = value
|
||||
}
|
||||
|
||||
// DelLabel deletes a label from the request.
|
||||
//
|
||||
// Deprecated: In onResolve and onDone, delete request labels directly.
|
||||
// In onStart and onError, use task.meta.req.delLabel instead.
|
||||
func (r *Request) DelLabel(key string) {
|
||||
delete(r.Labels, key)
|
||||
}
|
||||
|
||||
func (r *Request) Validate() error {
|
||||
if r.URL == "" {
|
||||
return fmt.Errorf("invalid request url")
|
||||
@@ -173,11 +200,14 @@ func ParseOptExtra[E any](opts *Options) error {
|
||||
}
|
||||
|
||||
type CreateTaskBatch struct {
|
||||
Reqs []*CreateTaskBatchItem `json:"reqs"`
|
||||
Opts *Options `json:"opts"`
|
||||
Reqs []*CreateTaskBatchItem `json:"reqs"`
|
||||
Opts *Options `json:"opts"`
|
||||
Group bool `json:"group"`
|
||||
GroupName string `json:"groupName"`
|
||||
}
|
||||
|
||||
type CreateTaskBatchItem struct {
|
||||
Rid string `json:"rid,omitempty"`
|
||||
Req *Request `json:"req"`
|
||||
Opts *Options `json:"opts"`
|
||||
}
|
||||
|
||||
@@ -32,6 +32,10 @@ func categoryForResource(cfg *base.DownloaderStoreConfig, res *base.Resource) *b
|
||||
// runs on the result, so a category directory the white list rejects falls
|
||||
// back to the base directory of this task instead of failing it.
|
||||
func (d *Downloader) decideDownloadDir(opts *base.Options, res *base.Resource) (string, error) {
|
||||
return d.decideDownloadDirForTask(opts, res, nil)
|
||||
}
|
||||
|
||||
func (d *Downloader) decideDownloadDirForTask(opts *base.Options, res *base.Resource, group *TaskGroup) (string, error) {
|
||||
if opts == nil {
|
||||
opts = &base.Options{}
|
||||
}
|
||||
@@ -48,6 +52,9 @@ func (d *Downloader) decideDownloadDir(opts *base.Options, res *base.Resource) (
|
||||
if base == "" {
|
||||
return "", errors.New("download directory is empty")
|
||||
}
|
||||
if group != nil && opts.SkipAutoCategorize {
|
||||
return d.initDownloadPathForTask(base, group)
|
||||
}
|
||||
target := base
|
||||
if !opts.SkipAutoCategorize {
|
||||
if category := categoryForResource(cfg, res); category != nil {
|
||||
|
||||
@@ -234,6 +234,38 @@ func TestDecideDownloadDirWhiteList(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
func TestDecideGroupDownloadDirWhiteList(t *testing.T) {
|
||||
parent := t.TempDir()
|
||||
root := filepath.Join(parent, "folder")
|
||||
nested := filepath.Join(root, "nested")
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
path string
|
||||
group *TaskGroup
|
||||
allowed string
|
||||
wantErr bool
|
||||
}{
|
||||
{name: "group root", path: root, group: &TaskGroup{Path: root}, allowed: filepath.Join(parent, "*")},
|
||||
{name: "nested member", path: nested, group: &TaskGroup{Path: root}, allowed: filepath.Join(parent, "*")},
|
||||
{name: "ordinary nested task", path: nested, allowed: filepath.Join(parent, "*"), wantErr: true},
|
||||
{name: "member outside group", path: filepath.Join(parent, "other"), group: &TaskGroup{Path: root}, allowed: filepath.Join(parent, "*"), wantErr: true},
|
||||
{name: "group permission revoked", path: nested, group: &TaskGroup{Path: root}, allowed: nested, wantErr: true},
|
||||
{name: "literal group name", path: filepath.Join(parent, "%year%", "nested"), group: &TaskGroup{Path: filepath.Join(parent, "%year%")}, allowed: filepath.Join(parent, "*")},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
d := newUnitCategoryDownloader(newTestCategoryConfig())
|
||||
d.cfg.WhiteDownloadDirs = []string{test.allowed}
|
||||
got, err := d.decideDownloadDirForTask(&base.Options{Path: test.path, SkipAutoCategorize: true}, singleFileResource("file.txt"), test.group)
|
||||
if (err != nil) != test.wantErr {
|
||||
t.Fatalf("decideDownloadDirForTask() = %q, %v; want error = %v", got, err, test.wantErr)
|
||||
}
|
||||
if err == nil && got != test.path {
|
||||
t.Fatalf("path = %q, want %q", got, test.path)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSuggestedDownloadDir(t *testing.T) {
|
||||
programDir := filepath.Join("./downloads", "Program")
|
||||
|
||||
|
||||
+522
-123
@@ -25,6 +25,7 @@ import (
|
||||
"github.com/GopeedLab/gopeed/internal/tempfiles"
|
||||
webviewproxy "github.com/GopeedLab/gopeed/internal/webview/proxy"
|
||||
"github.com/GopeedLab/gopeed/pkg/base"
|
||||
"github.com/GopeedLab/gopeed/pkg/protocol/ftp"
|
||||
"github.com/GopeedLab/gopeed/pkg/protocol/http"
|
||||
"github.com/GopeedLab/gopeed/pkg/util"
|
||||
gonanoid "github.com/matoous/go-nanoid/v2"
|
||||
@@ -36,7 +37,8 @@ const (
|
||||
uploadCheckpointInterval = 30 * time.Second
|
||||
|
||||
// task info bucket
|
||||
bucketTask = "task"
|
||||
bucketTask = "task"
|
||||
bucketTaskGroup = "task_group"
|
||||
// task download data bucket
|
||||
bucketSave = "save"
|
||||
// task statistics snapshot bucket
|
||||
@@ -112,13 +114,16 @@ type Downloader struct {
|
||||
Logger *logger.Logger
|
||||
ExtensionLogger *logger.Logger
|
||||
|
||||
cfg *DownloaderConfig
|
||||
fetcherCache map[string]fetcher.Fetcher
|
||||
storage Storage
|
||||
tasks []*Task
|
||||
waitTasks []*Task
|
||||
watchedTasks sync.Map
|
||||
listener Listener
|
||||
cfg *DownloaderConfig
|
||||
fetcherCache map[string]fetcher.Fetcher
|
||||
storage Storage
|
||||
tasks []*Task
|
||||
waitTasks []*Task
|
||||
groups map[string]*TaskGroup
|
||||
creatingGroups map[string]struct{}
|
||||
deletingTasks map[string]*Task
|
||||
watchedTasks sync.Map
|
||||
listener Listener
|
||||
|
||||
lock *sync.Mutex
|
||||
configLock *sync.RWMutex
|
||||
@@ -149,6 +154,9 @@ func NewDownloader(cfg *DownloaderConfig) *Downloader {
|
||||
cfg: cfg,
|
||||
fetcherCache: make(map[string]fetcher.Fetcher),
|
||||
waitTasks: make([]*Task, 0),
|
||||
groups: make(map[string]*TaskGroup),
|
||||
creatingGroups: make(map[string]struct{}),
|
||||
deletingTasks: make(map[string]*Task),
|
||||
storage: cfg.Storage,
|
||||
|
||||
lock: &sync.Mutex{},
|
||||
@@ -175,7 +183,7 @@ func (d *Downloader) Setup() error {
|
||||
d.blob = internalblob.NewRegistry("")
|
||||
|
||||
// setup storage
|
||||
if err := d.storage.Setup([]string{bucketTask, bucketSave, bucketTaskStats, bucketProtocolState, bucketProtocolJobs, bucketConfig, bucketExtension, bucketExtensionStorage}); err != nil {
|
||||
if err := d.storage.Setup([]string{bucketTask, bucketTaskGroup, bucketSave, bucketTaskStats, bucketProtocolState, bucketProtocolJobs, bucketConfig, bucketExtension, bucketExtensionStorage}); err != nil {
|
||||
return err
|
||||
}
|
||||
// load config from storage
|
||||
@@ -219,6 +227,15 @@ func (d *Downloader) Setup() error {
|
||||
return err
|
||||
}
|
||||
|
||||
// Group metadata is independent of task/fetcher recovery records.
|
||||
var groups []*TaskGroup
|
||||
if err := d.storage.List(bucketTaskGroup, &groups); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, group := range groups {
|
||||
d.groups[group.ID] = group
|
||||
}
|
||||
|
||||
// load tasks from storage
|
||||
var tasks []*Task
|
||||
if err = d.storage.List(bucketTask, &tasks); err != nil {
|
||||
@@ -234,6 +251,7 @@ func (d *Downloader) Setup() error {
|
||||
tasks = append(tasks[:i], tasks[i+1:]...)
|
||||
continue
|
||||
}
|
||||
task.group = d.groups[task.ParentID]
|
||||
d.assignFetcherManager(task)
|
||||
initTask(task)
|
||||
if task.Status != base.DownloadStatusDone && task.Status != base.DownloadStatusError {
|
||||
@@ -242,6 +260,11 @@ func (d *Downloader) Setup() error {
|
||||
}
|
||||
}
|
||||
d.tasks = tasks
|
||||
groupIDs := make([]string, 0, len(d.groups))
|
||||
for id := range d.groups {
|
||||
groupIDs = append(groupIDs, id)
|
||||
}
|
||||
d.cleanupEmptyGroups(groupIDs)
|
||||
// sort by create time
|
||||
sort.Slice(d.tasks, func(i, j int) bool {
|
||||
return d.tasks[i].CreatedAt.Before(d.tasks[j].CreatedAt)
|
||||
@@ -459,6 +482,9 @@ func (d *Downloader) setupFetcher(fm fetcher.FetcherManager, fetcher fetcher.Fet
|
||||
ctl.GetConfig = func(v any) {
|
||||
d.getProtocolConfig(fm.Name(), v)
|
||||
}
|
||||
if d.Logger != nil {
|
||||
ctl.Logger = &d.Logger.Logger
|
||||
}
|
||||
// Get proxy config, task request proxy config has higher priority, then use global proxy config
|
||||
ctl.GetProxy = func(requestProxy *base.RequestProxy) func(*gohttp.Request) (*url.URL, error) {
|
||||
// Registry-backed blob URLs are loopback capability URLs and must never be
|
||||
@@ -603,14 +629,14 @@ func (d *Downloader) CreateDirect(req *base.Request, opts *base.Options) (taskId
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
taskId, err = d.createDirect(req, opts)
|
||||
taskId, err = d.createDirect(req, opts, "")
|
||||
if err == nil {
|
||||
d.persistDefaultPathBestEffort(defaultPath)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (d *Downloader) createDirect(req *base.Request, opts *base.Options) (taskId string, err error) {
|
||||
func (d *Downloader) createDirect(req *base.Request, opts *base.Options, groupID string) (taskId string, err error) {
|
||||
ensureRequestRawURL(req)
|
||||
var fetcher fetcher.Fetcher
|
||||
fetcher, err = d.buildFetcher(req.URL)
|
||||
@@ -622,31 +648,203 @@ func (d *Downloader) createDirect(req *base.Request, opts *base.Options) (taskId
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
return d.doCreate(fetcher, initOpt)
|
||||
return d.doCreate(fetcher, initOpt, groupID)
|
||||
}
|
||||
|
||||
func (d *Downloader) CreateDirectBatch(req *base.CreateTaskBatch) (taskId []string, err error) {
|
||||
// CreateDirectBatch retains the original Go API for callers that only need
|
||||
// the task IDs. APIs that expose group metadata use CreateDirectBatchWithResult.
|
||||
func (d *Downloader) CreateDirectBatch(req *base.CreateTaskBatch) ([]string, error) {
|
||||
result, err := d.CreateDirectBatchWithResult(req)
|
||||
if err != nil || result == nil {
|
||||
return nil, err
|
||||
}
|
||||
return result.TaskIDs, nil
|
||||
}
|
||||
|
||||
func (d *Downloader) CreateDirectBatchWithResult(req *base.CreateTaskBatch) (*CreateTaskBatchResult, error) {
|
||||
if req == nil || len(req.Reqs) == 0 {
|
||||
return nil, errors.New("batch requires at least one task")
|
||||
}
|
||||
defaultPath, err := d.prepareDefaultPath(req.Opts)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
taskIds := make([]string, 0)
|
||||
for _, ir := range req.Reqs {
|
||||
opts := ir.Opts
|
||||
if opts == nil {
|
||||
opts = req.Opts
|
||||
result := &CreateTaskBatchResult{TaskIDs: make([]string, 0)}
|
||||
var group *TaskGroup
|
||||
if req.Group {
|
||||
parentPath := ""
|
||||
if req.Opts != nil {
|
||||
parentPath = req.Opts.Path
|
||||
}
|
||||
if opts != nil {
|
||||
opts = opts.Clone()
|
||||
}
|
||||
taskId, err := d.createDirect(ir.Req, opts)
|
||||
group, err = d.createTaskGroup(req.GroupName, parentPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
taskIds = append(taskIds, taskId)
|
||||
result.ParentID = group.ID
|
||||
defer func() {
|
||||
d.lock.Lock()
|
||||
delete(d.creatingGroups, group.ID)
|
||||
d.lock.Unlock()
|
||||
d.cleanupEmptyGroups([]string{group.ID})
|
||||
}()
|
||||
}
|
||||
var failures []error
|
||||
for _, item := range req.Reqs {
|
||||
var taskID string
|
||||
var opts *base.Options
|
||||
if item != nil {
|
||||
opts = item.Opts
|
||||
}
|
||||
if opts == nil {
|
||||
opts = req.Opts
|
||||
}
|
||||
if opts == nil {
|
||||
opts = &base.Options{}
|
||||
} else {
|
||||
opts = opts.Clone()
|
||||
}
|
||||
var itemErr error
|
||||
groupID := ""
|
||||
if group != nil {
|
||||
groupID = group.ID
|
||||
parentPath := filepath.Dir(group.Path)
|
||||
relative := "."
|
||||
if item != nil && item.Opts != nil && opts.Path != "" {
|
||||
requestedPath := util.ReplacePathPlaceholders(opts.Path)
|
||||
if !filepath.IsAbs(requestedPath) {
|
||||
requestedPath = filepath.Join(parentPath, requestedPath)
|
||||
}
|
||||
requested, pathErr := filepath.Abs(requestedPath)
|
||||
if pathErr != nil {
|
||||
itemErr = pathErr
|
||||
} else {
|
||||
relative, itemErr = filepath.Rel(parentPath, requested)
|
||||
if itemErr == nil && !filepath.IsLocal(relative) {
|
||||
itemErr = errors.New("task path is outside the group directory")
|
||||
}
|
||||
}
|
||||
}
|
||||
opts.Path = filepath.Join(group.Path, relative)
|
||||
opts.SkipAutoCategorize = true
|
||||
}
|
||||
if itemErr == nil {
|
||||
switch {
|
||||
case item == nil:
|
||||
itemErr = errors.New("task request or rid is required")
|
||||
case item.Rid != "":
|
||||
taskID, itemErr = d.createResolved(item.Rid, opts, groupID)
|
||||
case item.Req != nil:
|
||||
taskID, itemErr = d.createDirect(item.Req, opts, groupID)
|
||||
default:
|
||||
itemErr = errors.New("task request or rid is required")
|
||||
}
|
||||
}
|
||||
if taskID != "" {
|
||||
result.TaskIDs = append(result.TaskIDs, taskID)
|
||||
} else if itemErr != nil {
|
||||
failures = append(failures, itemErr)
|
||||
}
|
||||
}
|
||||
if len(result.TaskIDs) == 0 {
|
||||
return nil, fmt.Errorf("all batch tasks failed: %w", errors.Join(failures...))
|
||||
}
|
||||
d.persistDefaultPathBestEffort(defaultPath)
|
||||
return taskIds, nil
|
||||
return result, nil
|
||||
}
|
||||
|
||||
const maxTaskGroupDirSuffix = 1000
|
||||
|
||||
func (d *Downloader) createTaskGroup(name, parentPath string) (*TaskGroup, error) {
|
||||
parentOpts, err := d.initOptions(&base.Options{Path: parentPath}, "")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
parentPath = util.ReplacePathPlaceholders(strings.TrimSpace(parentOpts.Path))
|
||||
if parentPath == "" {
|
||||
return nil, errors.New("download directory is empty")
|
||||
}
|
||||
parentPath, err = filepath.Abs(parentPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
name = strings.TrimSpace(name)
|
||||
if strings.ContainsAny(name, "/\\\x00") || filepath.IsAbs(name) {
|
||||
return nil, errors.New("invalid task group name")
|
||||
}
|
||||
if strings.ContainsAny(name, `:*?"<>|`) || strings.HasSuffix(name, ".") {
|
||||
return nil, errors.New("invalid task group name")
|
||||
}
|
||||
name = util.SafeFilename(name)
|
||||
if name == "" || name == "." || name == ".." {
|
||||
return nil, errors.New("invalid task group name")
|
||||
}
|
||||
d.lock.Lock()
|
||||
defer d.lock.Unlock()
|
||||
requested := filepath.Join(parentPath, name)
|
||||
var dir string
|
||||
for suffix := 0; suffix <= maxTaskGroupDirSuffix; suffix++ {
|
||||
candidate := requested
|
||||
if suffix > 0 {
|
||||
candidate = fmt.Sprintf("%s (%d)", requested, suffix)
|
||||
}
|
||||
if !d.isDownloadPathAllowed(candidate) {
|
||||
return nil, errors.New("task group directory is outside the white list")
|
||||
}
|
||||
reserved := false
|
||||
for _, existing := range d.groups {
|
||||
if existing.Path == candidate {
|
||||
reserved = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if reserved {
|
||||
continue
|
||||
}
|
||||
if err := os.MkdirAll(parentPath, 0755); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := os.Mkdir(candidate, 0755); err != nil {
|
||||
if errors.Is(err, os.ErrExist) {
|
||||
continue
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
dir = candidate
|
||||
break
|
||||
}
|
||||
if dir == "" {
|
||||
return nil, errors.New("too many duplicate task group directories")
|
||||
}
|
||||
group := &TaskGroup{ID: NewTask().ID, Name: filepath.Base(dir), Path: dir, CreatedAt: time.Now()}
|
||||
if err := d.storage.Put(bucketTaskGroup, group.ID, group); err != nil {
|
||||
_ = os.Remove(dir)
|
||||
return nil, err
|
||||
}
|
||||
d.groups[group.ID] = group
|
||||
d.creatingGroups[group.ID] = struct{}{}
|
||||
return group, nil
|
||||
}
|
||||
|
||||
// Group IDs expand through parentId, even when metadata is unavailable.
|
||||
// The caller holds d.lock.
|
||||
func (d *Downloader) expandGroupTaskIDsLocked(ids []string) []string {
|
||||
selected := make(map[string]bool, len(ids))
|
||||
for _, id := range ids {
|
||||
selected[id] = true
|
||||
}
|
||||
expanded := make([]string, 0, len(ids))
|
||||
for _, task := range d.tasks {
|
||||
if selected[task.ID] || (task.ParentID != "" && selected[task.ParentID]) {
|
||||
expanded = append(expanded, task.ID)
|
||||
}
|
||||
}
|
||||
return expanded
|
||||
}
|
||||
|
||||
func (d *Downloader) expandGroupTaskIDs(ids []string) []string {
|
||||
d.lock.Lock()
|
||||
defer d.lock.Unlock()
|
||||
return d.expandGroupTaskIDsLocked(ids)
|
||||
}
|
||||
|
||||
func (d *Downloader) Create(rrId string) (taskId string, err error) {
|
||||
@@ -655,22 +853,38 @@ func (d *Downloader) Create(rrId string) (taskId string, err error) {
|
||||
|
||||
// CreateWithOptions creates a resolved task using the submitted options, or the
|
||||
// default download options when opts is nil.
|
||||
func (d *Downloader) CreateWithOptions(rrId string, opts *base.Options) (taskId string, err error) {
|
||||
d.fetcherMapLock.RLock()
|
||||
fetcher, ok := d.fetcherCache[rrId]
|
||||
d.fetcherMapLock.RUnlock()
|
||||
if !ok {
|
||||
return "", errors.New("invalid resource id")
|
||||
}
|
||||
if opts == nil {
|
||||
opts = &base.Options{}
|
||||
}
|
||||
// The options of the caller decide the global default download directory,
|
||||
// the task options are prepared later, when the task starts.
|
||||
func (d *Downloader) CreateWithOptions(rrId string, opts *base.Options) (string, error) {
|
||||
defaultPath, err := d.prepareDefaultPath(opts)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
taskID, err := d.createResolved(rrId, opts, "")
|
||||
if taskID != "" {
|
||||
d.persistDefaultPathBestEffort(defaultPath)
|
||||
}
|
||||
return taskID, err
|
||||
}
|
||||
|
||||
func (d *Downloader) createResolved(rrId string, opts *base.Options, groupID string) (taskId string, err error) {
|
||||
d.fetcherMapLock.Lock()
|
||||
fetcher, ok := d.fetcherCache[rrId]
|
||||
delete(d.fetcherCache, rrId)
|
||||
d.fetcherMapLock.Unlock()
|
||||
if !ok {
|
||||
return "", errors.New("invalid resource id")
|
||||
}
|
||||
// Only one creator may use a resolved fetcher. Failed creation makes it
|
||||
// available again; a persisted task consumes it even if starting fails.
|
||||
defer func() {
|
||||
if taskId == "" {
|
||||
d.fetcherMapLock.Lock()
|
||||
d.fetcherCache[rrId] = fetcher
|
||||
d.fetcherMapLock.Unlock()
|
||||
}
|
||||
}()
|
||||
if opts == nil {
|
||||
opts = &base.Options{}
|
||||
}
|
||||
taskOpts, err := d.initOptions(opts.Clone(), fetcher.Meta().Req.URL)
|
||||
if err != nil {
|
||||
return "", err
|
||||
@@ -694,15 +908,7 @@ func (d *Downloader) CreateWithOptions(rrId string, opts *base.Options) (taskId
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
defer func() {
|
||||
d.fetcherMapLock.Lock()
|
||||
delete(d.fetcherCache, rrId)
|
||||
d.fetcherMapLock.Unlock()
|
||||
}()
|
||||
taskId, err = d.doCreate(fetcher, nil)
|
||||
if err == nil {
|
||||
d.persistDefaultPathBestEffort(defaultPath)
|
||||
}
|
||||
taskId, err = d.doCreate(fetcher, nil, groupID)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -758,11 +964,30 @@ func (d *Downloader) Pause(filter *TaskFilter) (err error) {
|
||||
return d.pauseAll()
|
||||
}
|
||||
|
||||
if len(filter.IDs) > 0 {
|
||||
filter.IDs = d.expandGroupTaskIDs(filter.IDs)
|
||||
if len(filter.IDs) == 0 {
|
||||
return ErrTaskNotFound
|
||||
}
|
||||
}
|
||||
filter.NotStatuses = []base.Status{base.DownloadStatusPause, base.DownloadStatusError, base.DownloadStatusDone}
|
||||
pauseTasks := d.GetTasksByFilter(filter)
|
||||
if len(pauseTasks) == 0 {
|
||||
return ErrTaskNotFound
|
||||
}
|
||||
d.lock.Lock()
|
||||
pausedIDs := make(map[string]struct{}, len(pauseTasks))
|
||||
for _, task := range pauseTasks {
|
||||
pausedIDs[task.ID] = struct{}{}
|
||||
}
|
||||
queued := d.waitTasks[:0]
|
||||
for _, task := range d.waitTasks {
|
||||
if _, paused := pausedIDs[task.ID]; !paused {
|
||||
queued = append(queued, task)
|
||||
}
|
||||
}
|
||||
d.waitTasks = queued
|
||||
d.lock.Unlock()
|
||||
|
||||
for _, task := range pauseTasks {
|
||||
if err = d.doPause(task); err != nil {
|
||||
@@ -798,6 +1023,12 @@ func (d *Downloader) Continue(filter *TaskFilter) (err error) {
|
||||
return d.continueAll()
|
||||
}
|
||||
|
||||
if len(filter.IDs) > 0 {
|
||||
filter.IDs = d.expandGroupTaskIDs(filter.IDs)
|
||||
if len(filter.IDs) == 0 {
|
||||
return ErrTaskNotFound
|
||||
}
|
||||
}
|
||||
filter.NotStatuses = []base.Status{base.DownloadStatusRunning, base.DownloadStatusDone}
|
||||
continueTasks := d.GetTasksByFilter(filter)
|
||||
if len(continueTasks) == 0 {
|
||||
@@ -897,6 +1128,12 @@ func (d *Downloader) ContinueBatch(filter *TaskFilter) (err error) {
|
||||
return d.continueAll()
|
||||
}
|
||||
|
||||
if len(filter.IDs) > 0 {
|
||||
filter.IDs = d.expandGroupTaskIDs(filter.IDs)
|
||||
if len(filter.IDs) == 0 {
|
||||
return ErrTaskNotFound
|
||||
}
|
||||
}
|
||||
continueTasks := d.GetTasksByFilter(filter)
|
||||
for _, task := range continueTasks {
|
||||
if err = d.doStart(task); err != nil {
|
||||
@@ -916,72 +1153,175 @@ func (d *Downloader) ContinueOnStartup() error {
|
||||
return d.Continue(nil)
|
||||
}
|
||||
|
||||
func (d *Downloader) Delete(filter *TaskFilter, force bool) (err error) {
|
||||
func (d *Downloader) Delete(filter *TaskFilter, force bool) error {
|
||||
d.lock.Lock()
|
||||
groups := make(map[string]*TaskGroup)
|
||||
if filter == nil || filter.IsEmpty() {
|
||||
return d.deleteAll(force)
|
||||
}
|
||||
|
||||
deleteTasks := d.GetTasksByFilter(filter)
|
||||
if len(deleteTasks) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
deleteIds := make([]string, 0)
|
||||
deleteTasksPtr := make([]*Task, 0)
|
||||
for _, task := range deleteTasks {
|
||||
deleteIds = append(deleteIds, task.ID)
|
||||
deleteTasksPtr = append(deleteTasksPtr, task)
|
||||
}
|
||||
func() {
|
||||
d.lock.Lock()
|
||||
defer d.lock.Unlock()
|
||||
|
||||
for _, id := range deleteIds {
|
||||
for i, t := range d.tasks {
|
||||
if t.ID == id {
|
||||
d.tasks = append(d.tasks[:i], d.tasks[i+1:]...)
|
||||
break
|
||||
}
|
||||
}
|
||||
for i, t := range d.waitTasks {
|
||||
if t.ID == id {
|
||||
d.waitTasks = append(d.waitTasks[:i], d.waitTasks[i+1:]...)
|
||||
break
|
||||
}
|
||||
for id, group := range d.groups {
|
||||
groups[id] = group
|
||||
}
|
||||
} else if len(filter.IDs) > 0 {
|
||||
copyFilter := *filter
|
||||
copyFilter.IDs = d.expandGroupTaskIDsLocked(filter.IDs)
|
||||
for _, id := range filter.IDs {
|
||||
if group := d.groups[id]; group != nil {
|
||||
groups[id] = group
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
for _, task := range deleteTasksPtr {
|
||||
err = d.doDelete(task, force)
|
||||
if err != nil {
|
||||
return
|
||||
if len(copyFilter.IDs) == 0 {
|
||||
if len(groups) == 0 {
|
||||
d.lock.Unlock()
|
||||
return nil
|
||||
}
|
||||
// An empty ID filter means all tasks; retain a nonmatching sentinel.
|
||||
copyFilter.IDs = []string{""}
|
||||
}
|
||||
filter = ©Filter
|
||||
}
|
||||
for id := range groups {
|
||||
if _, creating := d.creatingGroups[id]; creating {
|
||||
d.lock.Unlock()
|
||||
return errors.New("task group is being created")
|
||||
}
|
||||
}
|
||||
|
||||
tasks := append([]*Task(nil), d.getTasksByFilter(filter)...)
|
||||
deleted := make(map[string]bool, len(tasks))
|
||||
groupIDs := make([]string, 0, len(groups))
|
||||
for id := range groups {
|
||||
groupIDs = append(groupIDs, id)
|
||||
}
|
||||
for _, task := range tasks {
|
||||
d.deletingTasks[task.ID] = task
|
||||
deleted[task.ID] = true
|
||||
if task.ParentID != "" {
|
||||
groupIDs = append(groupIDs, task.ParentID)
|
||||
}
|
||||
}
|
||||
kept := d.tasks[:0]
|
||||
for _, task := range d.tasks {
|
||||
if !deleted[task.ID] {
|
||||
kept = append(kept, task)
|
||||
}
|
||||
}
|
||||
d.tasks = kept
|
||||
wasQueued := make(map[string]bool)
|
||||
queued := d.waitTasks[:0]
|
||||
for _, task := range d.waitTasks {
|
||||
if deleted[task.ID] {
|
||||
wasQueued[task.ID] = true
|
||||
}
|
||||
if !deleted[task.ID] {
|
||||
queued = append(queued, task)
|
||||
}
|
||||
}
|
||||
d.waitTasks = queued
|
||||
d.lock.Unlock()
|
||||
var firstErr error
|
||||
for _, task := range tasks {
|
||||
if err := d.doDelete(task, force); err != nil {
|
||||
// Deletion can fail after removing bucketTask. Keep recovery durable.
|
||||
if saveErr := d.storage.Put(bucketTask, task.ID, task.clone()); saveErr != nil {
|
||||
d.Logger.Error().Err(saveErr).Msg("restore failed task deletion failed")
|
||||
if firstErr == nil {
|
||||
firstErr = saveErr
|
||||
}
|
||||
}
|
||||
d.lock.Lock()
|
||||
d.tasks = append(d.tasks, task)
|
||||
if wasQueued[task.ID] {
|
||||
d.waitTasks = append(d.waitTasks, task)
|
||||
}
|
||||
d.lock.Unlock()
|
||||
if firstErr == nil {
|
||||
firstErr = err
|
||||
}
|
||||
}
|
||||
}
|
||||
d.lock.Lock()
|
||||
for _, task := range tasks {
|
||||
delete(d.deletingTasks, task.ID)
|
||||
}
|
||||
d.lock.Unlock()
|
||||
if !force {
|
||||
groups = nil
|
||||
}
|
||||
if err := d.cleanupTaskGroups(groupIDs, groups); err != nil && firstErr == nil {
|
||||
firstErr = err
|
||||
}
|
||||
d.notifyRunning()
|
||||
return
|
||||
return firstErr
|
||||
}
|
||||
|
||||
func (d *Downloader) deleteAll(force bool) (err error) {
|
||||
var deleteTasksTemp []*Task
|
||||
func() {
|
||||
d.lock.Lock()
|
||||
defer d.lock.Unlock()
|
||||
// Active batches keep their metadata until the final creation attempt finishes.
|
||||
func (d *Downloader) cleanupEmptyGroups(groupIDs []string) {
|
||||
if err := d.cleanupTaskGroups(groupIDs, nil); err != nil {
|
||||
d.Logger.Error().Err(err).Msg("cleanup empty task group failed")
|
||||
}
|
||||
}
|
||||
|
||||
for _, task := range d.tasks {
|
||||
deleteTasksTemp = append(deleteTasksTemp, task)
|
||||
func (d *Downloader) cleanupTaskGroups(groupIDs []string, forceGroups map[string]*TaskGroup) error {
|
||||
d.lock.Lock()
|
||||
defer d.lock.Unlock()
|
||||
candidates := make(map[string]bool, len(groupIDs))
|
||||
for _, id := range groupIDs {
|
||||
candidates[id] = true
|
||||
}
|
||||
for _, task := range d.tasks {
|
||||
delete(candidates, task.ParentID)
|
||||
}
|
||||
for _, task := range d.deletingTasks {
|
||||
delete(candidates, task.ParentID)
|
||||
}
|
||||
var firstErr error
|
||||
for id := range candidates {
|
||||
group := d.groups[id]
|
||||
if group == nil {
|
||||
continue
|
||||
}
|
||||
d.tasks = make([]*Task, 0)
|
||||
d.waitTasks = make([]*Task, 0)
|
||||
}()
|
||||
if _, creating := d.creatingGroups[id]; creating {
|
||||
continue
|
||||
}
|
||||
// Reserve the path until its final filesystem operation finishes.
|
||||
if forceGroups[id] != nil {
|
||||
if err := os.RemoveAll(group.Path); err != nil {
|
||||
if firstErr == nil {
|
||||
firstErr = err
|
||||
}
|
||||
continue
|
||||
}
|
||||
} else {
|
||||
removeEmptyGroupDirectories(group.Path)
|
||||
}
|
||||
if err := d.storage.Delete(bucketTaskGroup, id); err != nil {
|
||||
if firstErr == nil {
|
||||
firstErr = err
|
||||
}
|
||||
continue
|
||||
}
|
||||
delete(d.groups, id)
|
||||
}
|
||||
return firstErr
|
||||
}
|
||||
|
||||
for _, task := range deleteTasksTemp {
|
||||
if err = d.doDelete(task, force); err != nil {
|
||||
return
|
||||
// Remove empty directory trees without touching retained files or symlinks.
|
||||
func removeEmptyGroupDirectories(root string) {
|
||||
if root == "" {
|
||||
return
|
||||
}
|
||||
info, err := os.Lstat(root)
|
||||
if err != nil || !info.IsDir() {
|
||||
return
|
||||
}
|
||||
entries, err := os.ReadDir(root)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
for _, entry := range entries {
|
||||
if entry.IsDir() {
|
||||
removeEmptyGroupDirectories(filepath.Join(root, entry.Name()))
|
||||
}
|
||||
}
|
||||
return
|
||||
_ = os.Remove(root)
|
||||
}
|
||||
|
||||
func (d *Downloader) Stats(id string) (sr any, err error) {
|
||||
@@ -1330,6 +1670,11 @@ func (d *Downloader) GetTasksByFilter(filter *TaskFilter) []*Task {
|
||||
d.lock.Lock()
|
||||
defer d.lock.Unlock()
|
||||
|
||||
return d.getTasksByFilter(filter)
|
||||
}
|
||||
|
||||
// The caller holds d.lock.
|
||||
func (d *Downloader) getTasksByFilter(filter *TaskFilter) []*Task {
|
||||
if filter == nil || filter.IsEmpty() {
|
||||
return d.tasks
|
||||
}
|
||||
@@ -1594,7 +1939,7 @@ func (d *Downloader) watch(task *Task) {
|
||||
d.triggerWebhooks(WebhookEventDownloadDone, task, nil)
|
||||
d.triggerScripts(ScriptEventDownloadDone, task, nil)
|
||||
|
||||
if e, ok := task.Meta.Opts.Extra.(*http.OptsExtra); ok {
|
||||
if e := postDownloadExtra(task.Meta); e != nil {
|
||||
downloadFilePath := task.Meta.SingleFilepath()
|
||||
|
||||
cfg, _ := d.GetConfig()
|
||||
@@ -1654,6 +1999,33 @@ func (d *Downloader) watch(task *Task) {
|
||||
}
|
||||
}
|
||||
|
||||
// postDownloadExtra returns the auto-torrent and auto-extract options of a
|
||||
// finished single-file task, in the HTTP options shape the extraction code
|
||||
// takes. FTP options carry the same fields. An FTP folder has no single file
|
||||
// to act on, so it gets nil, as does any other protocol.
|
||||
func postDownloadExtra(meta *fetcher.FetcherMeta) *http.OptsExtra {
|
||||
if meta == nil || meta.Opts == nil {
|
||||
return nil
|
||||
}
|
||||
switch e := meta.Opts.Extra.(type) {
|
||||
case *http.OptsExtra:
|
||||
return e
|
||||
case *ftp.OptsExtra:
|
||||
if meta.Res == nil || meta.Res.Name != "" || len(meta.Res.Files) != 1 {
|
||||
return nil
|
||||
}
|
||||
return &http.OptsExtra{
|
||||
Connections: e.Connections,
|
||||
AutoTorrent: e.AutoTorrent,
|
||||
DeleteTorrentAfterDownload: e.DeleteTorrentAfterDownload,
|
||||
AutoExtract: e.AutoExtract,
|
||||
ArchivePassword: e.ArchivePassword,
|
||||
DeleteAfterExtract: e.DeleteAfterExtract,
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *Downloader) doOnError(task *Task, err error) {
|
||||
d.handleOnError(task, err, false)
|
||||
}
|
||||
@@ -1788,7 +2160,7 @@ func (d *Downloader) restoreFetcher(task *Task) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *Downloader) doCreate(f fetcher.Fetcher, opts *base.Options) (taskId string, err error) {
|
||||
func (d *Downloader) doCreate(f fetcher.Fetcher, opts *base.Options, groupID string) (taskId string, err error) {
|
||||
if f.Meta().Opts == nil {
|
||||
f.Meta().Opts = opts
|
||||
}
|
||||
@@ -1799,6 +2171,7 @@ func (d *Downloader) doCreate(f fetcher.Fetcher, opts *base.Options) (taskId str
|
||||
return
|
||||
}
|
||||
task := NewTask()
|
||||
task.ParentID = groupID
|
||||
task.fetcherManager = fm
|
||||
task.fetcher = f
|
||||
task.Protocol = fm.Name()
|
||||
@@ -1817,15 +2190,21 @@ func (d *Downloader) doCreate(f fetcher.Fetcher, opts *base.Options) (taskId str
|
||||
d.releaseBlobTask(task)
|
||||
}
|
||||
}()
|
||||
if err = d.storage.Put(bucketTask, task.ID, task.clone()); err != nil {
|
||||
return
|
||||
}
|
||||
taskId = task.ID
|
||||
|
||||
func() {
|
||||
d.lock.Lock()
|
||||
defer d.lock.Unlock()
|
||||
|
||||
if groupID != "" {
|
||||
task.group = d.groups[groupID]
|
||||
if task.group == nil {
|
||||
err = ErrTaskNotFound
|
||||
return
|
||||
}
|
||||
}
|
||||
if err = d.storage.Put(bucketTask, task.ID, task.clone()); err != nil {
|
||||
return
|
||||
}
|
||||
taskId = task.ID
|
||||
d.tasks = append(d.tasks, task)
|
||||
|
||||
remainRunningCount := d.remainRunningCount()
|
||||
@@ -1882,23 +2261,42 @@ func (d *Downloader) initOptions(opts *base.Options, requestURL string) (*base.O
|
||||
func (d *Downloader) initDownloadPath(path string) (string, error) {
|
||||
// Replace placeholders in download path (e.g., %year%, %month%, %day%, %date%)
|
||||
path = util.ReplacePathPlaceholders(path)
|
||||
|
||||
// if enable white download directory, check if the download directory is in the white list
|
||||
if len(d.cfg.WhiteDownloadDirs) > 0 {
|
||||
inWhiteList := false
|
||||
for _, dir := range d.cfg.WhiteDownloadDirs {
|
||||
if match, err := filepath.Match(dir, path); match && err == nil {
|
||||
inWhiteList = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !inWhiteList {
|
||||
return "", errors.New("download directory is not in white list")
|
||||
}
|
||||
if !d.isDownloadPathAllowed(path) {
|
||||
return "", errors.New("download directory is not in white list")
|
||||
}
|
||||
return path, nil
|
||||
}
|
||||
|
||||
func (d *Downloader) initDownloadPathForTask(path string, group *TaskGroup) (string, error) {
|
||||
if group == nil {
|
||||
return d.initDownloadPath(path)
|
||||
}
|
||||
// Group paths are finalized at creation; validate the reserved root and
|
||||
// containment without expanding placeholders again.
|
||||
relative, err := filepath.Rel(filepath.Clean(group.Path), filepath.Clean(path))
|
||||
if err != nil || !filepath.IsLocal(relative) {
|
||||
return "", errors.New("task path is outside the group directory")
|
||||
}
|
||||
if !d.isDownloadPathAllowed(group.Path) {
|
||||
return "", errors.New("task group directory is outside the white list")
|
||||
}
|
||||
return path, nil
|
||||
}
|
||||
|
||||
// isDownloadPathAllowed reports whether the white list accepts this exact
|
||||
// path. With no white list configured every path is allowed.
|
||||
func (d *Downloader) isDownloadPathAllowed(path string) bool {
|
||||
if len(d.cfg.WhiteDownloadDirs) == 0 {
|
||||
return true
|
||||
}
|
||||
for _, dir := range d.cfg.WhiteDownloadDirs {
|
||||
if match, err := filepath.Match(dir, path); match && err == nil {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (d *Downloader) statusMut(task *Task, fn func() (bool, error)) (bool, error) {
|
||||
task.statusLock.Lock()
|
||||
defer task.statusLock.Unlock()
|
||||
@@ -1919,7 +2317,7 @@ func (d *Downloader) prepareTask(task *Task) error {
|
||||
preparedOpts := *task.Meta.Opts
|
||||
preparedMeta.Opts = &preparedOpts
|
||||
preparedMeta.Res = util.DeepClone(task.Meta.Res)
|
||||
dir, err := d.decideDownloadDir(preparedMeta.Opts, preparedMeta.Res)
|
||||
dir, err := d.decideDownloadDirForTask(preparedMeta.Opts, preparedMeta.Res, task.group)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -2043,19 +2441,20 @@ func (d *Downloader) doStart(task *Task) (err error) {
|
||||
if err := d.prepareTask(task); err != nil {
|
||||
return err
|
||||
}
|
||||
} else if _, err := d.initDownloadPath(task.Meta.Opts.Path); err != nil {
|
||||
} else if _, err := d.initDownloadPathForTask(task.Meta.Opts.Path, task.group); err != nil {
|
||||
// A prepared task keeps its directory, but the directory still has
|
||||
// to pass the white list on every start.
|
||||
return err
|
||||
}
|
||||
|
||||
task.statusLock.Lock()
|
||||
task.Progress.Speed = 0
|
||||
task.statusLock.Unlock()
|
||||
if !d.taskIsRunningGeneration(task, generation) {
|
||||
if task.Status != base.DownloadStatusRunning || task.runGeneration != generation {
|
||||
task.statusLock.Unlock()
|
||||
return nil
|
||||
}
|
||||
task.Progress.Speed = 0
|
||||
task.timer.Start()
|
||||
task.statusLock.Unlock()
|
||||
if err := task.fetcher.Start(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
+1105
-2
File diff suppressed because it is too large.
Load diff
@@ -825,6 +825,13 @@ func newOnDoneExtensionTask(task *Task) *Task {
|
||||
return cloneExtensionTask(task)
|
||||
}
|
||||
|
||||
// SetUrl replaces the task request URL.
|
||||
//
|
||||
// Deprecated: Use task.meta.req.setUrl instead.
|
||||
func (t *ExtensionTask) SetUrl(url string) {
|
||||
t.Meta.Req.SetUrl(url)
|
||||
}
|
||||
|
||||
func (t *OnErrorExtensionTask) Continue() error {
|
||||
return t.download.Continue(&TaskFilter{
|
||||
IDs: []string{t.ID},
|
||||
|
||||
@@ -0,0 +1,89 @@
|
||||
package download
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"github.com/GopeedLab/gopeed/internal/fetcher"
|
||||
"github.com/GopeedLab/gopeed/pkg/base"
|
||||
)
|
||||
|
||||
func TestExtensionDeprecatedRequestAPIs(t *testing.T) {
|
||||
for _, event := range []ActivationEvent{EventOnResolve, EventOnStart, EventOnError, EventOnDone} {
|
||||
for _, withLabels := range []bool{false, true} {
|
||||
t.Run(fmt.Sprintf("%s/labels=%v", event, withLabels), func(t *testing.T) {
|
||||
d := NewDownloader(&DownloaderConfig{Storage: NewMemStorage(), StorageDir: t.TempDir()})
|
||||
if err := d.Setup(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer d.Clear()
|
||||
|
||||
extDir := t.TempDir()
|
||||
manifest := fmt.Sprintf(`{"name":"legacy-request","title":"Legacy request APIs","version":"0.0.1","scripts":[{"event":%q,"match":{"urls":["*://example.com/*"]},"entry":"index.js"}]}`, event)
|
||||
request := "ctx.task.meta.req"
|
||||
urlMutation := ""
|
||||
if event == EventOnResolve {
|
||||
request = "ctx.req"
|
||||
} else if event == EventOnStart || event == EventOnError {
|
||||
urlMutation = `await ctx.task.setUrl('https://example.com/legacy');
|
||||
if (req.url !== 'https://example.com/legacy') throw new Error('legacy URL did not write through');
|
||||
await req.setUrl('https://example.com/new');
|
||||
if (req.url !== 'https://example.com/new') throw new Error('new URL did not write through');
|
||||
await ctx.task.setUrl('https://example.com/final');`
|
||||
}
|
||||
script := fmt.Sprintf(`gopeed.events.%s(async ctx => {
|
||||
const req = %s;
|
||||
await req.delLabel('missing');
|
||||
await req.putLabel('initial', 'yes');
|
||||
if (req.labels.initial !== 'yes') throw new Error('putLabel did not initialize labels');
|
||||
await req.setLabels({keep: 'yes', remove: 'yes'});
|
||||
if (req.labels.initial !== undefined || req.labels.old !== undefined) throw new Error('setLabels did not replace labels');
|
||||
await req.putLabel('added', 'yes');
|
||||
await req.delLabel('remove');
|
||||
if (req.labels.keep !== 'yes' || req.labels.added !== 'yes' || req.labels.remove !== undefined) throw new Error('label mutation failed');
|
||||
%s
|
||||
await req.putLabel('completed', 'yes');
|
||||
});`, event, request, urlMutation)
|
||||
for name, content := range map[string]string{"manifest.json": manifest, "index.js": script} {
|
||||
if err := os.WriteFile(filepath.Join(extDir, name), []byte(content), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if _, err := d.InstallExtensionByFolder(extDir, false); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
req := &base.Request{URL: "https://example.com/original"}
|
||||
if withLabels {
|
||||
req.Labels = map[string]string{"old": "yes"}
|
||||
}
|
||||
task := &Task{Protocol: "http", Meta: &fetcher.FetcherMeta{Req: req, Opts: &base.Options{}}}
|
||||
switch event {
|
||||
case EventOnResolve:
|
||||
if _, err := d.triggerOnResolve(req); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
case EventOnStart:
|
||||
d.triggerOnStart(task)
|
||||
case EventOnError:
|
||||
d.triggerOnError(task, errors.New("retry"))
|
||||
case EventOnDone:
|
||||
d.triggerOnDone(task)
|
||||
}
|
||||
if !reflect.DeepEqual(req.Labels, map[string]string{"keep": "yes", "added": "yes", "completed": "yes"}) {
|
||||
t.Fatalf("legacy handler did not update the original request: %#v", req.Labels)
|
||||
}
|
||||
expectedURL := "https://example.com/original"
|
||||
if urlMutation != "" {
|
||||
expectedURL = "https://example.com/final"
|
||||
}
|
||||
if req.URL != expectedURL {
|
||||
t.Fatalf("URL = %q, want %q", req.URL, expectedURL)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -152,7 +152,7 @@ func TestExtensionOnErrorRequestWritesToOriginalTask(t *testing.T) {
|
||||
request.setHeaders({Keep:'new'});
|
||||
request.putHeader('Authorization','Bearer new');
|
||||
request.setMethod('POST');
|
||||
if (typeof task.setUrl !== 'undefined' || typeof task.extra !== 'undefined') throw new Error('legacy task API');
|
||||
if (typeof task.setUrl !== 'function' || typeof task.extra !== 'undefined') throw new Error('unexpected legacy task API');
|
||||
request.setUrl('https://example.com/new');
|
||||
request.setLabels({keep:'yes',remove:'yes'});
|
||||
request.putLabel('new','value');
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
package download
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/GopeedLab/gopeed/pkg/base"
|
||||
)
|
||||
|
||||
func TestDownloader_Extension_MagnetMatch(t *testing.T) {
|
||||
const magnetURL = "magnet:?xt=urn:btih:0123456789abcdef0123456789abcdef01234567"
|
||||
for _, pattern := range []string{"magnet:*", "magnet:?*"} {
|
||||
t.Run(pattern, func(t *testing.T) {
|
||||
d := NewDownloader(&DownloaderConfig{
|
||||
Storage: NewMemStorage(),
|
||||
StorageDir: t.TempDir(),
|
||||
TempDir: t.TempDir(),
|
||||
})
|
||||
if err := d.Setup(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer d.Clear()
|
||||
|
||||
extDir := t.TempDir()
|
||||
manifest := fmt.Sprintf(`{"name":"magnet-match","author":"test","title":"Magnet match","version":"1.0.0","scripts":[{"event":"onResolve","match":{"urls":[%q]},"entry":"index.js"}]}`, pattern)
|
||||
script := `gopeed.events.onResolve(ctx => {
|
||||
ctx.res = {files: [{name: "resolved.txt", size: 1, req: {url: "https://example.com/resolved.txt"}}]};
|
||||
});`
|
||||
for name, content := range map[string]string{"manifest.json": manifest, "index.js": script} {
|
||||
if err := os.WriteFile(filepath.Join(extDir, name), []byte(content), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if _, err := d.InstallExtensionByFolder(extDir, true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
for _, req := range []*base.Request{
|
||||
{URL: magnetURL},
|
||||
{URL: "https://example.com/rewritten", RawURL: magnetURL},
|
||||
} {
|
||||
// Exercise extension activation without falling back to a live
|
||||
// BitTorrent metadata lookup when the pattern fails to match.
|
||||
res, err := d.triggerOnResolve(req)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if res == nil || len(res.Files) != 1 || res.Files[0].Name != "resolved.txt" {
|
||||
t.Fatalf("magnet extension did not resolve request %+v: %+v", req, res)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -70,10 +70,11 @@ func TestExtensionResolveSizeThroughTaskCreation(t *testing.T) {
|
||||
if err := json.Unmarshal(wire, &submitted); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ids, err := d.CreateDirectBatch(&submitted)
|
||||
batchResult, err := d.CreateDirectBatch(&submitted)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ids := batchResult
|
||||
for i, id := range ids {
|
||||
task := d.GetTask(id)
|
||||
if task.Meta.Res != nil {
|
||||
|
||||
@@ -2044,8 +2044,13 @@ func waitForTaskStatus(t *testing.T, downloader *Downloader, id string, status b
|
||||
deadline := time.Now().Add(timeout)
|
||||
for time.Now().Before(deadline) {
|
||||
task := downloader.GetTask(id)
|
||||
if task != nil && task.Status == status {
|
||||
return
|
||||
if task != nil && downloader.taskStatus(task) == status {
|
||||
if status != base.DownloadStatusDone {
|
||||
return
|
||||
}
|
||||
if _, watching := downloader.watchedTasks.Load(id); !watching {
|
||||
return
|
||||
}
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
@@ -2053,9 +2058,6 @@ func waitForTaskStatus(t *testing.T, downloader *Downloader, id string, status b
|
||||
if task == nil {
|
||||
t.Fatalf("timeout waiting for task %s status %s: task not found", id, status)
|
||||
}
|
||||
total := int64(0)
|
||||
if task.Meta != nil && task.Meta.Res != nil {
|
||||
total = task.Meta.Res.Size
|
||||
}
|
||||
t.Fatalf("timeout waiting for task %s status %s: got %s downloaded=%d total=%d", id, status, task.Status, task.Progress.Downloaded, total)
|
||||
snapshot, err := downloader.RuntimeStatus(id)
|
||||
t.Fatalf("timeout waiting for task %s status %s: got %+v, error=%v", id, status, snapshot, err)
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
package download
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/GopeedLab/gopeed/internal/fetcher"
|
||||
"github.com/GopeedLab/gopeed/pkg/base"
|
||||
"github.com/GopeedLab/gopeed/pkg/protocol/ftp"
|
||||
"github.com/GopeedLab/gopeed/pkg/protocol/http"
|
||||
)
|
||||
|
||||
// TestTorrentNameIsFile checks the default manager order: an FTP URL whose
|
||||
// name looks like a torrent or a playlist is still an FTP download.
|
||||
func TestTorrentNameIsFile(t *testing.T) {
|
||||
d := NewDownloader(nil)
|
||||
for u, want := range map[string]string{
|
||||
"ftp://h/x.torrent": "ftp",
|
||||
"ftp://h/v.m3u8": "ftp",
|
||||
"ftps://h/x.torrent": "ftp",
|
||||
"ftpes://h/a.bin": "ftp",
|
||||
"https://h/v.m3u8": "hls",
|
||||
"https://h/x.torrent": "http",
|
||||
"magnet:?xt=urn:btih:": "bt",
|
||||
} {
|
||||
fm, err := d.parseFm(u)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", u, err)
|
||||
}
|
||||
if fm.Name() != want {
|
||||
t.Fatalf("%s goes to %q, want %q", u, fm.Name(), want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPostDownloadExtraAcceptsFTP(t *testing.T) {
|
||||
yes := true
|
||||
file := &base.Resource{Files: []*base.FileInfo{{Name: "a.zip"}}}
|
||||
folder := &base.Resource{Name: "dir", Files: []*base.FileInfo{{Name: "a.zip"}}}
|
||||
|
||||
httpExtra := &http.OptsExtra{AutoExtract: &yes}
|
||||
if got := postDownloadExtra(&fetcher.FetcherMeta{Res: file, Opts: &base.Options{Extra: httpExtra}}); got != httpExtra {
|
||||
t.Fatalf("HTTP options must pass through unchanged, got %+v", got)
|
||||
}
|
||||
|
||||
ftpExtra := &ftp.OptsExtra{Connections: 4, AutoTorrent: &yes, DeleteTorrentAfterDownload: &yes, AutoExtract: &yes, ArchivePassword: "pw", DeleteAfterExtract: true}
|
||||
got := postDownloadExtra(&fetcher.FetcherMeta{Res: file, Opts: &base.Options{Extra: ftpExtra}})
|
||||
if got == nil || got.AutoTorrent != &yes || got.DeleteTorrentAfterDownload != &yes || got.AutoExtract != &yes ||
|
||||
got.ArchivePassword != "pw" || !got.DeleteAfterExtract {
|
||||
t.Fatalf("FTP options were not carried over: %+v", got)
|
||||
}
|
||||
if got := postDownloadExtra(&fetcher.FetcherMeta{Res: folder, Opts: &base.Options{Extra: ftpExtra}}); got != nil {
|
||||
t.Fatalf("an FTP folder has no single file to extract, got %+v", got)
|
||||
}
|
||||
if got := postDownloadExtra(&fetcher.FetcherMeta{Res: file, Opts: &base.Options{}}); got != nil {
|
||||
t.Fatalf("no options, got %+v", got)
|
||||
}
|
||||
}
|
||||
+34
-9
@@ -10,6 +10,7 @@ import (
|
||||
"github.com/GopeedLab/gopeed/internal/production"
|
||||
"github.com/GopeedLab/gopeed/internal/protocol/bt"
|
||||
"github.com/GopeedLab/gopeed/internal/protocol/ed2k"
|
||||
"github.com/GopeedLab/gopeed/internal/protocol/ftp"
|
||||
"github.com/GopeedLab/gopeed/internal/protocol/hls"
|
||||
"github.com/GopeedLab/gopeed/internal/protocol/http"
|
||||
"github.com/GopeedLab/gopeed/pkg/base"
|
||||
@@ -28,18 +29,34 @@ type ResolveResult struct {
|
||||
SuggestedDownloadDir string `json:"suggestedDownloadDir,omitempty"`
|
||||
}
|
||||
|
||||
// TaskGroup stores the directory shared by independently scheduled tasks.
|
||||
type TaskGroup struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Path string `json:"path"`
|
||||
CreatedAt time.Time `json:"createdAt"`
|
||||
}
|
||||
|
||||
// CreateTaskBatchResult is shared by ordinary and grouped batches.
|
||||
type CreateTaskBatchResult struct {
|
||||
TaskIDs []string `json:"taskIds"`
|
||||
ParentID string `json:"parentId,omitempty"`
|
||||
}
|
||||
|
||||
// Task represents a download unit.
|
||||
// Options flow path:
|
||||
// REST Request (model.CreateTask/ResolveTask) -> pkg/download Downloader -> Task.Meta.Opts (base.Options) -> Fetcher.Resolve/Start (e.g. internal/protocol/http/fetcher.go)
|
||||
type Task struct {
|
||||
ID string `json:"id"`
|
||||
Protocol string `json:"protocol"`
|
||||
Meta *fetcher.FetcherMeta `json:"meta"`
|
||||
Status base.Status `json:"status"`
|
||||
Uploading bool `json:"uploading"`
|
||||
Progress *Progress `json:"progress"`
|
||||
CreatedAt time.Time `json:"createdAt"`
|
||||
UpdatedAt time.Time `json:"updatedAt"`
|
||||
ID string `json:"id"`
|
||||
Protocol string `json:"protocol"`
|
||||
Meta *fetcher.FetcherMeta `json:"meta"`
|
||||
// ParentID identifies the independently stored group.
|
||||
ParentID string `json:"parentId,omitempty"`
|
||||
Status base.Status `json:"status"`
|
||||
Uploading bool `json:"uploading"`
|
||||
Progress *Progress `json:"progress"`
|
||||
CreatedAt time.Time `json:"createdAt"`
|
||||
UpdatedAt time.Time `json:"updatedAt"`
|
||||
// DoneAt is the time when the task completed; nil if it has never
|
||||
// finished. A done task never leaves the done state, so the value is
|
||||
// written once and kept.
|
||||
@@ -53,6 +70,8 @@ type Task struct {
|
||||
// does not have it and keeps its directory and name.
|
||||
NeedsPreparation bool `json:"needsPreparation,omitempty"`
|
||||
|
||||
group *TaskGroup
|
||||
|
||||
producer production.Source
|
||||
producerReceived int64
|
||||
|
||||
@@ -143,10 +162,12 @@ func (t *Task) MarshalJSON() ([]byte, error) {
|
||||
type rawTaskType Task
|
||||
jsonTask := struct {
|
||||
*rawTaskType
|
||||
Name string `json:"name"`
|
||||
Name string `json:"name"`
|
||||
Group *TaskGroup `json:"group,omitempty"`
|
||||
}{
|
||||
(*rawTaskType)(t),
|
||||
t.Name(),
|
||||
t.group,
|
||||
}
|
||||
return json.Marshal(jsonTask)
|
||||
}
|
||||
@@ -229,6 +250,10 @@ func (cfg *DownloaderConfig) Init() *DownloaderConfig {
|
||||
}
|
||||
if len(cfg.FetchManagers) == 0 {
|
||||
cfg.FetchManagers = []fetcher.FetcherManager{
|
||||
// FTP matches by URL scheme only. It comes before the managers that
|
||||
// match by file extension, so ftp://host/x.torrent or x.m3u8 is
|
||||
// downloaded as a file over FTP.
|
||||
new(ftp.FetcherManager),
|
||||
new(hls.FetcherManager),
|
||||
new(http.FetcherManager),
|
||||
new(bt.FetcherManager),
|
||||
|
||||
@@ -27,6 +27,10 @@ type ScriptData struct {
|
||||
// ScriptPayload contains the task data
|
||||
type ScriptPayload struct {
|
||||
Task *Task
|
||||
// TaskName is computed before Task is cloned. The clone has no fetcher
|
||||
// manager, so for an unresolved task without Options.Name its Name()
|
||||
// would return "unknown".
|
||||
TaskName string
|
||||
}
|
||||
|
||||
// getScriptPaths extracts script paths from config
|
||||
@@ -103,7 +107,7 @@ func (d *Downloader) executeScriptAtPath(scriptPath string, data *ScriptData) er
|
||||
cmd.Env = append(os.Environ(),
|
||||
fmt.Sprintf("GOPEED_EVENT=%s", data.Event),
|
||||
fmt.Sprintf("GOPEED_TASK_ID=%s", data.Payload.Task.ID),
|
||||
fmt.Sprintf("GOPEED_TASK_NAME=%s", data.Payload.Task.Name()),
|
||||
fmt.Sprintf("GOPEED_TASK_NAME=%s", data.Payload.TaskName),
|
||||
fmt.Sprintf("GOPEED_TASK_STATUS=%s", data.Payload.Task.Status),
|
||||
)
|
||||
|
||||
@@ -137,7 +141,8 @@ func (d *Downloader) triggerScripts(event ScriptEvent, task *Task, err error) {
|
||||
Event: event,
|
||||
Time: time.Now().UnixMilli(),
|
||||
Payload: &ScriptPayload{
|
||||
Task: task.clone(),
|
||||
Task: task.clone(),
|
||||
TaskName: task.Name(),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@@ -3,6 +3,8 @@ package download
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -126,6 +128,46 @@ func TestScript_ExecuteScriptAtPath_NonExistentFile(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
func TestScript_TaskNameOfUnresolvedTask(t *testing.T) {
|
||||
scriptPath := getTestScriptPath(t, envDumpScriptName())
|
||||
ensureScriptExecutable(t, scriptPath)
|
||||
outputFile := filepath.Join(t.TempDir(), "env_output.txt")
|
||||
|
||||
setupScriptTest(t, func(downloader *Downloader) {
|
||||
cfg, _ := downloader.GetConfig()
|
||||
cfg.Script = &base.ScriptConfig{
|
||||
Enable: true,
|
||||
Paths: []string{scriptPath},
|
||||
}
|
||||
downloader.PutConfig(cfg)
|
||||
|
||||
// Without Options.Name and a resource, the name comes from the fetcher
|
||||
// manager, which the cloned task in the script data does not have.
|
||||
url := "http://127.0.0.1/error.bin"
|
||||
fm, err := downloader.parseFm(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
task := NewTask()
|
||||
task.Meta = &fetcher.FetcherMeta{
|
||||
Req: &base.Request{URL: url},
|
||||
Opts: &base.Options{Path: t.TempDir()},
|
||||
}
|
||||
task.fetcherManager = fm
|
||||
if name := task.Name(); name != "error.bin" {
|
||||
t.Fatalf("Expected task name error.bin, got %q", name)
|
||||
}
|
||||
|
||||
t.Setenv("GOPEED_TEST_OUTPUT_FILE", outputFile)
|
||||
downloader.triggerScripts(ScriptEventDownloadDone, task, nil)
|
||||
|
||||
output := waitForFileContains(t, outputFile, "GOPEED_TASK_PATH=", 3*time.Second)
|
||||
if !strings.Contains(output, "GOPEED_TASK_NAME=error.bin") {
|
||||
t.Errorf("Expected GOPEED_TASK_NAME=error.bin in output, got: %s", output)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func createDownloadDoneTask(t *testing.T, downloadDir, fileName string) (*Task, string) {
|
||||
t.Helper()
|
||||
content := []byte("downloaded file")
|
||||
@@ -171,6 +213,31 @@ func waitForFile(t *testing.T, path string, timeout time.Duration) {
|
||||
t.Fatalf("Timeout waiting for file: %s", path)
|
||||
}
|
||||
|
||||
// waitForFileContains waits until the file contains want and returns its content.
|
||||
// Test scripts write the file line by line, so waiting for the file to exist is not enough.
|
||||
func waitForFileContains(t *testing.T, path, want string, timeout time.Duration) string {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(timeout)
|
||||
for time.Now().Before(deadline) {
|
||||
if content, err := os.ReadFile(path); err == nil && strings.Contains(string(content), want) {
|
||||
return string(content)
|
||||
}
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
}
|
||||
content, _ := os.ReadFile(path)
|
||||
t.Fatalf("Timeout waiting for %q in file %s, got: %q", want, path, content)
|
||||
return ""
|
||||
}
|
||||
|
||||
// envDumpScriptName returns the test script that writes the GOPEED_* variables
|
||||
// into the file named by GOPEED_TEST_OUTPUT_FILE.
|
||||
func envDumpScriptName() string {
|
||||
if runtime.GOOS == "windows" {
|
||||
return "env_dump.bat"
|
||||
}
|
||||
return "env_dump.sh"
|
||||
}
|
||||
|
||||
func getTestScriptPath(t *testing.T, name string) string {
|
||||
t.Helper()
|
||||
path := filepath.Join("testdata", "scripts", name)
|
||||
|
||||
@@ -91,13 +91,8 @@ func TestScript_EnvironmentVariables(t *testing.T) {
|
||||
t.Setenv("GOPEED_TEST_OUTPUT_FILE", outputFile)
|
||||
downloader.triggerScripts(ScriptEventDownloadDone, task, nil)
|
||||
|
||||
waitForFile(t, outputFile, 3*time.Second)
|
||||
content, err := os.ReadFile(outputFile)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to read output file: %v", err)
|
||||
}
|
||||
|
||||
output := string(content)
|
||||
// GOPEED_TASK_PATH is the last line the script writes.
|
||||
output := waitForFileContains(t, outputFile, "GOPEED_TASK_PATH=", 3*time.Second)
|
||||
if !strings.Contains(output, "GOPEED_EVENT=DOWNLOAD_DONE") {
|
||||
t.Errorf("Expected GOPEED_EVENT in output, got: %s", output)
|
||||
}
|
||||
|
||||
@@ -87,13 +87,8 @@ func TestScript_EnvironmentVariables(t *testing.T) {
|
||||
t.Setenv("GOPEED_TEST_OUTPUT_FILE", outputFile)
|
||||
downloader.triggerScripts(ScriptEventDownloadDone, task, nil)
|
||||
|
||||
waitForFile(t, outputFile, 5*time.Second)
|
||||
content, err := os.ReadFile(outputFile)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to read output file: %v", err)
|
||||
}
|
||||
|
||||
output := string(content)
|
||||
// GOPEED_TASK_PATH is the last line the script writes.
|
||||
output := waitForFileContains(t, outputFile, "GOPEED_TASK_PATH=", 5*time.Second)
|
||||
if !strings.Contains(output, "GOPEED_EVENT=DOWNLOAD_DONE") {
|
||||
t.Errorf("Expected GOPEED_EVENT in output, got: %s", output)
|
||||
}
|
||||
|
||||
+1
-1
@@ -1,5 +1,5 @@
|
||||
gopeed.events.onResolve(async function (ctx) {
|
||||
for (const method of ['setUrl', 'setLabels', 'putLabel', 'delLabel', 'setMethod', 'putHeader', 'setTrackers']) {
|
||||
for (const method of ['setUrl', 'setMethod', 'putHeader', 'setTrackers']) {
|
||||
if (typeof ctx.req[method] !== 'undefined') throw new Error(`onResolve exposes ${method}`);
|
||||
}
|
||||
ctx.req.labels = { replaced: "true", removed: "true" };
|
||||
|
||||
@@ -0,0 +1,148 @@
|
||||
package mcpserver
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"github.com/GopeedLab/gopeed/internal/fetcher"
|
||||
"github.com/GopeedLab/gopeed/pkg/base"
|
||||
"github.com/GopeedLab/gopeed/pkg/download"
|
||||
"github.com/GopeedLab/gopeed/pkg/protocol/bt"
|
||||
"github.com/GopeedLab/gopeed/pkg/protocol/ed2k"
|
||||
phttp "github.com/GopeedLab/gopeed/pkg/protocol/http"
|
||||
"github.com/google/jsonschema-go/jsonschema"
|
||||
)
|
||||
|
||||
func TestToolSchemaPropertiesAreObjects(t *testing.T) {
|
||||
downloader := download.NewDownloader(&download.DownloaderConfig{Storage: download.NewMemStorage()})
|
||||
req := httptest.NewRequest(http.MethodPost, "http://localhost/mcp", bytes.NewBufferString(
|
||||
`{"jsonrpc":"2.0","id":1,"method":"tools/list","params":{}}`,
|
||||
))
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("Accept", "application/json, text/event-stream")
|
||||
recorder := httptest.NewRecorder()
|
||||
NewHandler(downloader).ServeHTTP(recorder, req)
|
||||
if recorder.Code != http.StatusOK {
|
||||
t.Fatalf("tools/list status = %d: %s", recorder.Code, recorder.Body.String())
|
||||
}
|
||||
var response struct {
|
||||
Result struct {
|
||||
Tools []struct {
|
||||
Name string `json:"name"`
|
||||
InputSchema map[string]any `json:"inputSchema"`
|
||||
OutputSchema map[string]any `json:"outputSchema"`
|
||||
} `json:"tools"`
|
||||
} `json:"result"`
|
||||
}
|
||||
if err := json.Unmarshal(recorder.Body.Bytes(), &response); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(response.Result.Tools) == 0 {
|
||||
t.Fatal("tools/list returned no tools")
|
||||
}
|
||||
for _, tool := range response.Result.Tools {
|
||||
t.Run(tool.Name, func(t *testing.T) {
|
||||
checkSchemaProperties(t, tool.InputSchema, "inputSchema")
|
||||
checkSchemaProperties(t, tool.OutputSchema, "outputSchema")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Check the serialized definitions, including nested file requests and union
|
||||
// branches. Boolean additionalProperties remains valid; boolean entries inside
|
||||
// properties are what clients with object-only schema parsers cannot consume.
|
||||
func checkSchemaProperties(t *testing.T, value any, path string) {
|
||||
t.Helper()
|
||||
switch value := value.(type) {
|
||||
case map[string]any:
|
||||
for key, child := range value {
|
||||
if key == "properties" {
|
||||
properties, ok := child.(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("%s.properties = %#v, want object", path, child)
|
||||
}
|
||||
for name, property := range properties {
|
||||
if _, ok := property.(map[string]any); !ok {
|
||||
t.Errorf("%s.properties.%s = %#v, want schema object", path, name, property)
|
||||
}
|
||||
}
|
||||
}
|
||||
checkSchemaProperties(t, child, path+"."+key)
|
||||
}
|
||||
case []any:
|
||||
for i, child := range value {
|
||||
checkSchemaProperties(t, child, fmt.Sprintf("%s[%d]", path, i))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestToolOutputSchemasPreserveDynamicValues(t *testing.T) {
|
||||
statsSchema := toolOutputSchema[getTaskStatsOutput]()
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
stats any
|
||||
}{
|
||||
{"unavailable", nil},
|
||||
{"typed nil", (*fetcher.Stats)(nil)},
|
||||
{"http", &fetcher.Stats{Snapshot: &phttp.Stats{Connections: []*phttp.StatsConnection{{Downloaded: 123}}}}},
|
||||
{"bittorrent", &fetcher.Stats{Snapshot: &bt.StatsSnapshot{}, Runtime: &bt.StatsRuntime{}}},
|
||||
{"ed2k", &fetcher.Stats{Snapshot: &ed2k.StatsSnapshot{}, Runtime: &ed2k.StatsRuntime{}}},
|
||||
{"restored snapshot", &fetcher.Stats{Snapshot: json.RawMessage(`{"segmentsTotal":3,"segmentsDone":2,"segmentsFailed":0}`)}},
|
||||
{"empty object", map[string]any{}},
|
||||
{"nested JSON", map[string]any{"values": []any{nil, true, "text", 1.5, map[string]any{"key": "value"}}}},
|
||||
{"array", []any{1, "two", false}},
|
||||
{"string", "protocol value"},
|
||||
{"number", 1.5},
|
||||
{"boolean", true},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
validateToolOutput(t, statsSchema, &getTaskStatsOutput{Stats: test.stats})
|
||||
})
|
||||
}
|
||||
|
||||
// Output extras can be empty, null, or extension-specific. In particular,
|
||||
// they need not satisfy the required settings in the download input schema.
|
||||
for _, extra := range []any{nil, map[string]any{}, &phttp.ReqExtra{}, &bt.ReqExtra{},
|
||||
map[string]any{"custom": []any{true, nil, 42}}, "custom value"} {
|
||||
req := &base.Request{URL: "https://example.com/file", Extra: extra}
|
||||
res := &base.Resource{Files: []*base.FileInfo{{Name: "file", Req: req}}}
|
||||
validateToolOutput(t, toolOutputSchema[resolveTaskOutput](), &resolveTaskOutput{Res: res})
|
||||
validateToolOutput(t, toolOutputSchema[getTaskOutput](), &getTaskOutput{Task: taskDetails{
|
||||
Req: req, Res: res, Opts: &base.Options{Extra: extra},
|
||||
}})
|
||||
}
|
||||
|
||||
// Supplying an explicit schema must retain validation of the output envelope.
|
||||
resolved, err := statsSchema.Resolve(nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, invalid := range []map[string]any{{}, {"stats": nil, "unexpected": true}} {
|
||||
if err := resolved.Validate(invalid); err == nil {
|
||||
t.Errorf("accepted invalid output envelope: %#v", invalid)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func validateToolOutput(t *testing.T, schema *jsonschema.Schema, output any) {
|
||||
t.Helper()
|
||||
data, err := json.Marshal(output)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var value any
|
||||
if err := json.Unmarshal(data, &value); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
resolved, err := schema.Resolve(nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := resolved.Validate(value); err != nil {
|
||||
t.Fatalf("output %s does not match schema: %v", data, err)
|
||||
}
|
||||
}
|
||||
+47
-13
@@ -190,6 +190,37 @@ func downloadInputSchema[T any]() *jsonschema.Schema {
|
||||
return schema
|
||||
}
|
||||
|
||||
// toolOutputSchema preserves arbitrary JSON values in protocol statistics and
|
||||
// task metadata. Unlike the boolean schema inferred for any, these object-form
|
||||
// schemas can also be parsed by clients that do not support boolean schemas in
|
||||
// properties. Output metadata may come from REST, extensions, or persisted tasks,
|
||||
// so it must not inherit the narrower download input restrictions.
|
||||
func toolOutputSchema[T any]() *jsonschema.Schema {
|
||||
schema, err := jsonschema.For[T](&jsonschema.ForOptions{
|
||||
TypeSchemas: map[reflect.Type]*jsonschema.Schema{
|
||||
// Request.Labels is serialized as null when the map is nil.
|
||||
reflect.TypeFor[map[string]string](): {
|
||||
Types: []string{"object", "null"},
|
||||
AdditionalProperties: &jsonschema.Schema{Type: "string"},
|
||||
},
|
||||
reflect.TypeFor[any](): {
|
||||
AnyOf: []*jsonschema.Schema{
|
||||
{Type: "object"},
|
||||
{Type: "array"},
|
||||
{Type: "string"},
|
||||
{Type: "number"},
|
||||
{Type: "boolean"},
|
||||
{Type: "null"},
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
panic(fmt.Sprintf("mcpserver: build tool output schema: %v", err))
|
||||
}
|
||||
return schema
|
||||
}
|
||||
|
||||
func (o *downloadOptions) baseOptions() *base.Options {
|
||||
if o == nil {
|
||||
return nil
|
||||
@@ -319,11 +350,12 @@ func registerTools(server *mcp.Server, downloader *download.Downloader) {
|
||||
destructive := true
|
||||
|
||||
mcp.AddTool(server, &mcp.Tool{
|
||||
Name: "resolve_task",
|
||||
Title: "Resolve task",
|
||||
Description: "Resolve an HTTP, HTTPS, BitTorrent, magnet, or ED2K download request and return its resource metadata and files. Use this before create_task when the user wants to inspect the resource or select files.",
|
||||
InputSchema: downloadInputSchema[resolveTaskInput](),
|
||||
Annotations: &mcp.ToolAnnotations{ReadOnlyHint: true, OpenWorldHint: &openWorld},
|
||||
Name: "resolve_task",
|
||||
Title: "Resolve task",
|
||||
Description: "Resolve an HTTP, HTTPS, BitTorrent, magnet, or ED2K download request and return its resource metadata and files. Use this before create_task when the user wants to inspect the resource or select files.",
|
||||
InputSchema: downloadInputSchema[resolveTaskInput](),
|
||||
OutputSchema: toolOutputSchema[resolveTaskOutput](),
|
||||
Annotations: &mcp.ToolAnnotations{ReadOnlyHint: true, OpenWorldHint: &openWorld},
|
||||
}, func(_ context.Context, _ *mcp.CallToolRequest, input *resolveTaskInput) (*mcp.CallToolResult, *resolveTaskOutput, error) {
|
||||
if input.Req == nil {
|
||||
return nil, nil, fmt.Errorf("req is required")
|
||||
@@ -403,10 +435,11 @@ func registerTools(server *mcp.Server, downloader *download.Downloader) {
|
||||
})
|
||||
|
||||
mcp.AddTool(server, &mcp.Tool{
|
||||
Name: "get_task",
|
||||
Title: "Get task",
|
||||
Description: "Get one Gopeed task including its request, resource, options, and current progress.",
|
||||
Annotations: &mcp.ToolAnnotations{ReadOnlyHint: true, OpenWorldHint: &closedWorld},
|
||||
Name: "get_task",
|
||||
Title: "Get task",
|
||||
Description: "Get one Gopeed task including its request, resource, options, and current progress.",
|
||||
OutputSchema: toolOutputSchema[getTaskOutput](),
|
||||
Annotations: &mcp.ToolAnnotations{ReadOnlyHint: true, OpenWorldHint: &closedWorld},
|
||||
}, func(_ context.Context, _ *mcp.CallToolRequest, input *taskIDInput) (*mcp.CallToolResult, *getTaskOutput, error) {
|
||||
task, err := requireTask(downloader, input.ID)
|
||||
if err != nil {
|
||||
@@ -442,10 +475,11 @@ func registerTools(server *mcp.Server, downloader *download.Downloader) {
|
||||
})
|
||||
|
||||
mcp.AddTool(server, &mcp.Tool{
|
||||
Name: "get_task_stats",
|
||||
Title: "Get task statistics",
|
||||
Description: "Get protocol-specific statistics for one Gopeed task, such as HTTP connections or BitTorrent peers and seeding data.",
|
||||
Annotations: &mcp.ToolAnnotations{ReadOnlyHint: true, OpenWorldHint: &closedWorld},
|
||||
Name: "get_task_stats",
|
||||
Title: "Get task statistics",
|
||||
Description: "Get protocol-specific statistics for one Gopeed task, such as HTTP connections or BitTorrent peers and seeding data.",
|
||||
OutputSchema: toolOutputSchema[getTaskStatsOutput](),
|
||||
Annotations: &mcp.ToolAnnotations{ReadOnlyHint: true, OpenWorldHint: &closedWorld},
|
||||
}, func(_ context.Context, _ *mcp.CallToolRequest, input *taskIDInput) (*mcp.CallToolResult, *getTaskStatsOutput, error) {
|
||||
if err := validateTaskID(input.ID); err != nil {
|
||||
return nil, nil, err
|
||||
|
||||
@@ -192,6 +192,36 @@ func TestStreamableHTTPTools(t *testing.T) {
|
||||
if tasks, ok := structured["tasks"].([]any); !ok || len(tasks) != 0 {
|
||||
t.Fatalf("tasks = %#v, want empty array", structured["tasks"])
|
||||
}
|
||||
|
||||
result, err = session.CallTool(context.Background(), &mcp.CallToolParams{
|
||||
Name: "create_task",
|
||||
Arguments: map[string]any{
|
||||
"req": map[string]any{
|
||||
"url": downloadServer.URL,
|
||||
"extra": map[string]any{"header": map[string]any{"X-Gopeed-Test": "mcp-object"}},
|
||||
},
|
||||
"opts": map[string]any{"path": t.TempDir()},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if result.IsError {
|
||||
t.Fatalf("create_task returned tool error: %+v", result.Content)
|
||||
}
|
||||
created := result.StructuredContent.(map[string]any)
|
||||
for _, name := range []string{"get_task", "get_task_stats"} {
|
||||
result, err := session.CallTool(context.Background(), &mcp.CallToolParams{
|
||||
Name: name,
|
||||
Arguments: map[string]any{"id": created["id"]},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", name, err)
|
||||
}
|
||||
if result.IsError {
|
||||
t.Fatalf("%s returned tool error: %+v", name, result.Content)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func nestedSchemaProperty(t *testing.T, schema map[string]any, path ...string) map[string]any {
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
package ftp
|
||||
|
||||
// TLS modes for the "tls" field of an ftp:// request. ftps:// URLs always use
|
||||
// implicit TLS and ftpes:// URLs always use explicit TLS.
|
||||
const (
|
||||
TLSNone = "none"
|
||||
TLSExplicit = "explicit"
|
||||
TLSImplicit = "implicit"
|
||||
)
|
||||
|
||||
// ReqExtra is the extra of an FTP request. The TLS mode is part of the request,
|
||||
// because resolving the link already needs a connection.
|
||||
type ReqExtra struct {
|
||||
// TLS upgrades a plain ftp:// URL: "none" (default), "explicit" (AUTH TLS)
|
||||
// or "implicit". It is ignored for ftps:// and ftpes:// URLs.
|
||||
TLS string `json:"tls"`
|
||||
}
|
||||
|
||||
type OptsExtra struct {
|
||||
// Connections is the maximum number of parallel logins. Servers often cap
|
||||
// logins per user, so the fetcher lowers it when the server refuses one.
|
||||
Connections int `json:"connections"`
|
||||
// AutoTorrent when task download complete, and it is a .torrent file, it will be auto create a new task for the torrent file
|
||||
// nil means use global config, true/false means explicit setting
|
||||
AutoTorrent *bool `json:"autoTorrent"`
|
||||
// DeleteTorrentAfterDownload when true, deletes the .torrent file after creating BT task
|
||||
// nil means use global config, true/false means explicit setting
|
||||
DeleteTorrentAfterDownload *bool `json:"deleteTorrentAfterDownload"`
|
||||
// AutoExtract when task download complete, and it is an archive file, it will be auto extracted
|
||||
// nil means use global config, true/false means explicit setting
|
||||
AutoExtract *bool `json:"autoExtract"`
|
||||
// ArchivePassword is the password for extracting password-protected archives
|
||||
ArchivePassword string `json:"archivePassword"`
|
||||
// DeleteAfterExtract when true, deletes the archive file after successful extraction
|
||||
DeleteAfterExtract bool `json:"deleteAfterExtract"`
|
||||
}
|
||||
|
||||
// Stats for download. It has the same shape as the HTTP protocol's stats, so
|
||||
// clients can show FTP connections the way they show HTTP connections.
|
||||
type Stats struct {
|
||||
Connections []*StatsConnection `json:"connections"`
|
||||
}
|
||||
|
||||
// StatsConnection describes one login that took part in the download.
|
||||
type StatsConnection struct {
|
||||
Downloaded int64 `json:"downloaded"`
|
||||
// Total is the resource size divided by the number of connections,
|
||||
// rounded up to a whole byte, as in the HTTP protocol's stats. It is zero
|
||||
// when the resource size is unknown.
|
||||
Total int64 `json:"total"`
|
||||
Completed bool `json:"completed"`
|
||||
Failed bool `json:"failed"`
|
||||
RetryTimes int `json:"retryTimes"`
|
||||
}
|
||||
+1
-1
@@ -63,7 +63,7 @@ func CreateTaskBatch(w http.ResponseWriter, r *http.Request) {
|
||||
WriteJson(w, model.NewErrorResult("param invalid: reqs", model.CodeInvalidParam))
|
||||
return
|
||||
}
|
||||
taskIds, err := Downloader.CreateDirectBatch(&req)
|
||||
taskIds, err := Downloader.CreateDirectBatchWithResult(&req)
|
||||
if err != nil {
|
||||
WriteJson(w, model.NewErrorResult(err.Error()))
|
||||
return
|
||||
|
||||
+38
-2
@@ -197,9 +197,10 @@ func TestCreateDirectTaskBatch(t *testing.T) {
|
||||
Req: createReq.Req,
|
||||
})
|
||||
}
|
||||
taskIds := httpRequestCheckOk[[]string](http.MethodPost, "/api/v1/tasks/batch", &base.CreateTaskBatch{
|
||||
result := httpRequestCheckOk[download.CreateTaskBatchResult](http.MethodPost, "/api/v1/tasks/batch", &base.CreateTaskBatch{
|
||||
Reqs: reqs,
|
||||
})
|
||||
taskIds := result.TaskIDs
|
||||
if len(taskIds) != len(reqs) {
|
||||
t.Errorf("CreateDirectTaskBatch() got = %v, want %v", len(taskIds), len(reqs))
|
||||
}
|
||||
@@ -220,12 +221,13 @@ func TestCreateDirectTaskBatchWithOpt(t *testing.T) {
|
||||
}
|
||||
reqs = append(reqs, item)
|
||||
}
|
||||
taskIds := httpRequestCheckOk[[]string](http.MethodPost, "/api/v1/tasks/batch", &base.CreateTaskBatch{
|
||||
result := httpRequestCheckOk[download.CreateTaskBatchResult](http.MethodPost, "/api/v1/tasks/batch", &base.CreateTaskBatch{
|
||||
Reqs: reqs,
|
||||
Opts: &base.Options{
|
||||
Name: "default_opt.data",
|
||||
},
|
||||
})
|
||||
taskIds := result.TaskIDs
|
||||
if len(taskIds) != len(reqs) {
|
||||
t.Errorf("CreateDirectTaskBatch() got = %v, want %v", len(taskIds), len(reqs))
|
||||
}
|
||||
@@ -245,6 +247,40 @@ func TestCreateDirectTaskBatchWithOpt(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
func TestCreateDirectTaskBatchWithGroup(t *testing.T) {
|
||||
doTest(func() {
|
||||
cfg, err := Downloader.GetConfig()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cfg.MaxRunning = 0
|
||||
if err := Downloader.PutConfig(cfg); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
parent := t.TempDir()
|
||||
result := httpRequestCheckOk[download.CreateTaskBatchResult](http.MethodPost, "/api/v1/tasks/batch", &base.CreateTaskBatch{
|
||||
Reqs: []*base.CreateTaskBatchItem{{Req: createReq.Req}, {Req: createReq.Req}},
|
||||
Opts: &base.Options{Path: parent}, Group: true, GroupName: "folder",
|
||||
})
|
||||
if result.ParentID == "" || len(result.TaskIDs) != 2 {
|
||||
t.Fatalf("unexpected group result: %+v", result)
|
||||
}
|
||||
all := httpRequestCheckOk[[]map[string]any](http.MethodGet, "/api/v1/tasks", nil)
|
||||
if len(all) != 2 {
|
||||
t.Fatalf("expected real tasks only, got %d", len(all))
|
||||
}
|
||||
for _, task := range all {
|
||||
if task["parentId"] != result.ParentID || task["type"] != nil {
|
||||
t.Fatalf("invalid member: %+v", task)
|
||||
}
|
||||
metadata, ok := task["group"].(map[string]any)
|
||||
if !ok || metadata["path"] != filepath.Join(parent, "folder") || metadata["name"] != "folder" {
|
||||
t.Fatalf("missing group metadata: %+v", task)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestPauseAndContinueTask(t *testing.T) {
|
||||
doTest(func() {
|
||||
var wg sync.WaitGroup
|
||||
|
||||
+15
-2
@@ -6,14 +6,27 @@ import (
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Match url with pattern by chrome extension match pattern style
|
||||
// Match url with pattern by chrome extension match pattern style.
|
||||
// Magnet URI patterns such as magnet:?* are also supported, with * as a wildcard.
|
||||
// https://developer.chrome.com/docs/extensions/mv3/match_patterns/
|
||||
func Match(pattern string, u string) bool {
|
||||
scheme, host, path := parsePattern(pattern)
|
||||
url, err := url.Parse(u)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
// Magnet URIs have no authority or path; match the raw scheme-specific
|
||||
// part instead, preserving query punctuation and percent-encoding.
|
||||
if strings.HasPrefix(pattern, "magnet:") && !strings.HasPrefix(pattern, "magnet://") {
|
||||
if url.Scheme != "magnet" {
|
||||
return false
|
||||
}
|
||||
_, target, _ := strings.Cut(u, ":")
|
||||
expression := regexp.QuoteMeta(strings.TrimPrefix(pattern, "magnet:"))
|
||||
expression = strings.ReplaceAll(expression, `\*`, ".*")
|
||||
matched, _ := regexp.MatchString("^"+expression+"$", target)
|
||||
return matched
|
||||
}
|
||||
scheme, host, path := parsePattern(pattern)
|
||||
if scheme != "*" && scheme != url.Scheme {
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -34,3 +34,35 @@ func TestMatch(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestMatchMagnet(t *testing.T) {
|
||||
const magnetURL = "magnet:?xt=urn:btih:0123456789abcdef0123456789abcdef01234567&dn=example.zip"
|
||||
tests := []struct {
|
||||
name string
|
||||
pattern string
|
||||
url string
|
||||
want bool
|
||||
}{
|
||||
{"scheme wildcard", "magnet:*", magnetURL, true},
|
||||
{"query wildcard", "magnet:?*", magnetURL, true},
|
||||
{"info hash wildcard", "magnet:?xt=urn:btih:*", magnetURL, true},
|
||||
{"exact URI", magnetURL, magnetURL, true},
|
||||
{"encoded name", "magnet:?*&dn=example%20file.zip", "magnet:?xt=urn:btih:abc&dn=example%20file.zip", true},
|
||||
{"literal query punctuation", "magnet:?*&dn=a+b[1].zip", "magnet:?xt=urn:btih:abc&dn=a+b[1].zip", true},
|
||||
{"literal question mark", "magnet:?*", "magnet:xt=urn:btih:abc", false},
|
||||
{"literal dot", "magnet:?*&dn=example.zip", "magnet:?xt=urn:btih:abc&dn=exampleXzip", false},
|
||||
{"different hash type", "magnet:?xt=urn:btmh:*", magnetURL, false},
|
||||
{"different hash", "magnet:?xt=urn:btih:other*", magnetURL, false},
|
||||
{"different scheme", "magnet:*", "https://example.org/file.zip", false},
|
||||
{"invalid URI", "magnet:*", "magnet:\x00", false},
|
||||
{"existing catch-all", "*://*/*", magnetURL, true},
|
||||
{"existing magnet pattern", "magnet://*", magnetURL, true},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if got := Match(tt.pattern, tt.url); got != tt.want {
|
||||
t.Errorf("Match(%q, %q) = %v, want %v", tt.pattern, tt.url, got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -94,6 +94,8 @@ State broadcasting must observe the owning Riverpod state centrally. Do not broa
|
||||
- Storage RPC methods expose business capabilities such as create-history operations, not raw `box.get` or `box.put` access.
|
||||
- Prefer batch mutations across the window boundary. For example, save all parsed URLs in one `saveCreateHistory(List<String>)` call.
|
||||
- New child-window storage requirements must be added to `StorageMethods` and bound in the main registry.
|
||||
- Config reads, full config saves, and create-history mutations share one main-window queue. Each history mutation reads, changes, and writes the config within that queue. History reads wait for earlier saves.
|
||||
- `StorageMethods` owns `extra.createHistory`; full config saves preserve its latest value inside the queue so stale settings or child-window snapshots cannot overwrite history.
|
||||
|
||||
## 7. Gopeed Backend Migration
|
||||
|
||||
@@ -128,3 +130,42 @@ Child-window code must not:
|
||||
- Parse RPC JSON directly inside a page or widget.
|
||||
|
||||
These constraints keep all backend and persistence ownership in the main window and allow multiple child windows to operate concurrently without database locks or duplicated Gopeed runtimes.
|
||||
|
||||
## 10. Batch Task Creation
|
||||
|
||||
`GopeedMethods.createTaskBatch` carries `CreateTaskBatch` through the existing
|
||||
codec registry. Each member accepts either `rid` from resolve or `req`, plus
|
||||
creation `opts`. A nonempty `rid` takes priority. BT selections retain their
|
||||
original file indexes in `opts.selectFiles`.
|
||||
|
||||
Both ordinary and grouped creation return `CreateTaskBatchResult`, an object
|
||||
with `taskIds` and an optional `parentId`. A successful response completes the
|
||||
batch, including partial success. The preview tracks completed batches so a
|
||||
failed batch can be retried without repeating successful batches. Creation does
|
||||
not accept a group ID to append members to an existing group.
|
||||
|
||||
## 11. Opening Child Surfaces
|
||||
|
||||
Use `AppRouter.openWindow(AppWindowPayload)` for child-window destinations.
|
||||
`AppRouter.openCreateTask(createTask: ...)` is the create-task convenience API.
|
||||
All entry points, including deep links, browser capture and the tray, use this
|
||||
API rather than selecting a presentation themselves:
|
||||
|
||||
- Desktop launches an independent native child window through
|
||||
`AppWindowLauncher.open(payload)` and the existing capability transport.
|
||||
- Web pushes an `AppWindowPage` on the root Navigator. This routed shadcn modal
|
||||
keeps the current section, filters and scroll state mounted underneath it.
|
||||
- Native mobile retains the full-page flow, including later share requests.
|
||||
- If native child-window creation fails, the main window is brought forward
|
||||
and the same in-app modal is used as a fallback.
|
||||
|
||||
`AppWindowContent` owns the destination paths and content builders shared with
|
||||
`ChildWindowApp`. For another child surface, add its payload type, register its
|
||||
path and content, and use `AppWindowPage` for its in-app route. The modal's
|
||||
preferred size can be supplied per destination. Keep backend and storage access
|
||||
on the existing capability API in either presentation.
|
||||
|
||||
Create-task content reports cancellation or completion through `onClose` when
|
||||
hosted in-app. Cancellation pops only the child surface. Completion selects the
|
||||
downloading list and returns home. The route payload uses a serializable string
|
||||
so browser history can restore a modal and its initial request.
|
||||
@@ -85,10 +85,10 @@ Future<String> createTask(CreateTask createTask) {
|
||||
return _parse<String>(() => _request('api/v1/tasks', method: 'POST', data: createTask), (data) => data as String);
|
||||
}
|
||||
|
||||
Future<List<String>> createTaskBatch(CreateTaskBatch createTaskBatch) {
|
||||
return _parse<List<String>>(
|
||||
Future<CreateTaskBatchResult> createTaskBatch(CreateTaskBatch createTaskBatch) {
|
||||
return _parse<CreateTaskBatchResult>(
|
||||
() => _request('api/v1/tasks/batch', method: 'POST', data: createTaskBatch),
|
||||
(data) => (data as List<dynamic>).map((e) => e as String).toList(),
|
||||
(json) => CreateTaskBatchResult.fromJson(json as Map<String, dynamic>),
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -10,7 +10,10 @@ class CreateTaskBatch {
|
||||
List<CreateTaskBatchItem>? reqs;
|
||||
Options? opts;
|
||||
|
||||
CreateTaskBatch({this.reqs, this.opts});
|
||||
bool group;
|
||||
String groupName;
|
||||
|
||||
CreateTaskBatch({this.reqs, this.opts, this.group = false, this.groupName = ''});
|
||||
|
||||
factory CreateTaskBatch.fromJson(Map<String, dynamic> json) => _$CreateTaskBatchFromJson(json);
|
||||
|
||||
@@ -19,11 +22,23 @@ class CreateTaskBatch {
|
||||
|
||||
@JsonSerializable()
|
||||
class CreateTaskBatchItem {
|
||||
String? rid;
|
||||
Request? req;
|
||||
Options? opts;
|
||||
|
||||
CreateTaskBatchItem({this.req, this.opts});
|
||||
CreateTaskBatchItem({this.rid, this.req, this.opts});
|
||||
|
||||
factory CreateTaskBatchItem.fromJson(Map<String, dynamic> json) => _$CreateTaskBatchItemFromJson(json);
|
||||
Map<String, dynamic> toJson() => _$CreateTaskBatchItemToJson(this);
|
||||
}
|
||||
|
||||
@JsonSerializable()
|
||||
class CreateTaskBatchResult {
|
||||
final List<String> taskIds;
|
||||
final String? parentId;
|
||||
|
||||
CreateTaskBatchResult({required this.taskIds, this.parentId});
|
||||
|
||||
factory CreateTaskBatchResult.fromJson(Map<String, dynamic> json) => _$CreateTaskBatchResultFromJson(json);
|
||||
Map<String, dynamic> toJson() => _$CreateTaskBatchResultToJson(this);
|
||||
}
|
||||
@@ -6,22 +6,55 @@ part of 'create_task_batch.dart';
|
||||
// JsonSerializableGenerator
|
||||
// **************************************************************************
|
||||
|
||||
CreateTaskBatch _$CreateTaskBatchFromJson(Map<String, dynamic> json) => CreateTaskBatch(
|
||||
reqs: (json['reqs'] as List<dynamic>?)?.map((e) => CreateTaskBatchItem.fromJson(e as Map<String, dynamic>)).toList(),
|
||||
opts: json['opts'] == null ? null : Options.fromJson(json['opts'] as Map<String, dynamic>),
|
||||
);
|
||||
CreateTaskBatch _$CreateTaskBatchFromJson(Map<String, dynamic> json) =>
|
||||
CreateTaskBatch(
|
||||
reqs: (json['reqs'] as List<dynamic>?)
|
||||
?.map((e) => CreateTaskBatchItem.fromJson(e as Map<String, dynamic>))
|
||||
.toList(),
|
||||
opts: json['opts'] == null
|
||||
? null
|
||||
: Options.fromJson(json['opts'] as Map<String, dynamic>),
|
||||
group: json['group'] as bool? ?? false,
|
||||
groupName: json['groupName'] as String? ?? '',
|
||||
);
|
||||
|
||||
Map<String, dynamic> _$CreateTaskBatchToJson(CreateTaskBatch instance) => <String, dynamic>{
|
||||
'reqs': ?instance.reqs?.map((e) => e.toJson()).toList(),
|
||||
'opts': ?instance.opts?.toJson(),
|
||||
};
|
||||
Map<String, dynamic> _$CreateTaskBatchToJson(CreateTaskBatch instance) =>
|
||||
<String, dynamic>{
|
||||
'reqs': ?instance.reqs?.map((e) => e.toJson()).toList(),
|
||||
'opts': ?instance.opts?.toJson(),
|
||||
'group': instance.group,
|
||||
'groupName': instance.groupName,
|
||||
};
|
||||
|
||||
CreateTaskBatchItem _$CreateTaskBatchItemFromJson(Map<String, dynamic> json) => CreateTaskBatchItem(
|
||||
req: json['req'] == null ? null : Request.fromJson(json['req'] as Map<String, dynamic>),
|
||||
opts: json['opts'] == null ? null : Options.fromJson(json['opts'] as Map<String, dynamic>),
|
||||
);
|
||||
CreateTaskBatchItem _$CreateTaskBatchItemFromJson(Map<String, dynamic> json) =>
|
||||
CreateTaskBatchItem(
|
||||
rid: json['rid'] as String?,
|
||||
req: json['req'] == null
|
||||
? null
|
||||
: Request.fromJson(json['req'] as Map<String, dynamic>),
|
||||
opts: json['opts'] == null
|
||||
? null
|
||||
: Options.fromJson(json['opts'] as Map<String, dynamic>),
|
||||
);
|
||||
|
||||
Map<String, dynamic> _$CreateTaskBatchItemToJson(CreateTaskBatchItem instance) => <String, dynamic>{
|
||||
Map<String, dynamic> _$CreateTaskBatchItemToJson(
|
||||
CreateTaskBatchItem instance,
|
||||
) => <String, dynamic>{
|
||||
'rid': ?instance.rid,
|
||||
'req': ?instance.req,
|
||||
'opts': ?instance.opts,
|
||||
};
|
||||
|
||||
CreateTaskBatchResult _$CreateTaskBatchResultFromJson(
|
||||
Map<String, dynamic> json,
|
||||
) => CreateTaskBatchResult(
|
||||
taskIds: (json['taskIds'] as List<dynamic>).map((e) => e as String).toList(),
|
||||
parentId: json['parentId'] as String?,
|
||||
);
|
||||
|
||||
Map<String, dynamic> _$CreateTaskBatchResultToJson(
|
||||
CreateTaskBatchResult instance,
|
||||
) => <String, dynamic>{
|
||||
'taskIds': instance.taskIds,
|
||||
'parentId': ?instance.parentId,
|
||||
};
|
||||
@@ -64,6 +64,10 @@ class ProtocolConfig {
|
||||
Ed2kConfig ed2k = Ed2kConfig();
|
||||
HlsConfig hls = HlsConfig();
|
||||
|
||||
/// A server without FTP support sends no `ftp` key; the defaults keep the rest of the config readable.
|
||||
@JsonKey(fromJson: _ftpConfigFromJson)
|
||||
FtpConfig ftp = FtpConfig();
|
||||
|
||||
ProtocolConfig();
|
||||
|
||||
factory ProtocolConfig.fromJson(Map<String, dynamic>? json) =>
|
||||
@@ -91,6 +95,20 @@ class HlsConfig {
|
||||
Map<String, dynamic> toJson() => _$HlsConfigToJson(this);
|
||||
}
|
||||
|
||||
FtpConfig _ftpConfigFromJson(Map<String, dynamic>? json) => json == null ? FtpConfig() : FtpConfig.fromJson(json);
|
||||
|
||||
/// Global FTP defaults. It mirrors the `ftp` protocol config of the server.
|
||||
@JsonSerializable()
|
||||
class FtpConfig {
|
||||
int connections;
|
||||
|
||||
FtpConfig({this.connections = 0});
|
||||
|
||||
factory FtpConfig.fromJson(Map<String, dynamic> json) => _$FtpConfigFromJson(json);
|
||||
|
||||
Map<String, dynamic> toJson() => _$FtpConfigToJson(this);
|
||||
}
|
||||
|
||||
@JsonSerializable()
|
||||
class HttpConfig {
|
||||
String userAgent;
|
||||
@@ -196,13 +214,15 @@ class ExtraConfig {
|
||||
List<String> createHistory;
|
||||
bool runAsMenubarApp;
|
||||
bool analyticsEnabled;
|
||||
String analyticsClientId;
|
||||
|
||||
/// Persisted UUID fallback; IDs returned by platform APIs are not stored here.
|
||||
String deviceId;
|
||||
|
||||
ExtraConfig({
|
||||
this.themeMode = '',
|
||||
this.themeColor = 'green',
|
||||
this.locale = '',
|
||||
this.lastDeleteTaskKeep = false,
|
||||
this.lastDeleteTaskKeep = true,
|
||||
this.defaultDirectDownload = false,
|
||||
this.defaultBtClient = true,
|
||||
this.notifyWhenNewVersion = true,
|
||||
@@ -214,7 +234,7 @@ class ExtraConfig {
|
||||
this.createHistory = const [],
|
||||
this.runAsMenubarApp = false,
|
||||
this.analyticsEnabled = true,
|
||||
this.analyticsClientId = '',
|
||||
this.deviceId = '',
|
||||
}) : windowState = windowState ?? WindowStateConfig();
|
||||
|
||||
factory ExtraConfig.fromJson(Map<String, dynamic>? json) =>
|
||||
|
||||
@@ -67,13 +67,15 @@ ProtocolConfig _$ProtocolConfigFromJson(Map<String, dynamic> json) => ProtocolCo
|
||||
..http = HttpConfig.fromJson(json['http'] as Map<String, dynamic>)
|
||||
..bt = BtConfig.fromJson(json['bt'] as Map<String, dynamic>)
|
||||
..ed2k = Ed2kConfig.fromJson(json['ed2k'] as Map<String, dynamic>)
|
||||
..hls = HlsConfig.fromJson(json['hls'] as Map<String, dynamic>);
|
||||
..hls = HlsConfig.fromJson(json['hls'] as Map<String, dynamic>)
|
||||
..ftp = _ftpConfigFromJson(json['ftp'] as Map<String, dynamic>?);
|
||||
|
||||
Map<String, dynamic> _$ProtocolConfigToJson(ProtocolConfig instance) => <String, dynamic>{
|
||||
'http': instance.http.toJson(),
|
||||
'bt': instance.bt.toJson(),
|
||||
'ed2k': instance.ed2k.toJson(),
|
||||
'hls': instance.hls.toJson(),
|
||||
'ftp': instance.ftp.toJson(),
|
||||
};
|
||||
|
||||
HlsConfig _$HlsConfigFromJson(Map<String, dynamic> json) => HlsConfig(
|
||||
@@ -90,6 +92,11 @@ Map<String, dynamic> _$HlsConfigToJson(HlsConfig instance) => <String, dynamic>{
|
||||
'prefetchContentLength': instance.prefetchContentLength,
|
||||
};
|
||||
|
||||
FtpConfig _$FtpConfigFromJson(Map<String, dynamic> json) =>
|
||||
FtpConfig(connections: (json['connections'] as num?)?.toInt() ?? 0);
|
||||
|
||||
Map<String, dynamic> _$FtpConfigToJson(FtpConfig instance) => <String, dynamic>{'connections': instance.connections};
|
||||
|
||||
HttpConfig _$HttpConfigFromJson(Map<String, dynamic> json) => HttpConfig(
|
||||
userAgent: json['userAgent'] as String? ?? '',
|
||||
connections: (json['connections'] as num?)?.toInt() ?? 0,
|
||||
@@ -179,7 +186,7 @@ ExtraConfig _$ExtraConfigFromJson(Map<String, dynamic> json) => ExtraConfig(
|
||||
themeMode: json['themeMode'] as String? ?? '',
|
||||
themeColor: json['themeColor'] as String? ?? 'green',
|
||||
locale: json['locale'] as String? ?? '',
|
||||
lastDeleteTaskKeep: json['lastDeleteTaskKeep'] as bool? ?? false,
|
||||
lastDeleteTaskKeep: json['lastDeleteTaskKeep'] as bool? ?? true,
|
||||
defaultDirectDownload: json['defaultDirectDownload'] as bool? ?? false,
|
||||
defaultBtClient: json['defaultBtClient'] as bool? ?? true,
|
||||
notifyWhenNewVersion: json['notifyWhenNewVersion'] as bool? ?? true,
|
||||
@@ -193,7 +200,7 @@ ExtraConfig _$ExtraConfigFromJson(Map<String, dynamic> json) => ExtraConfig(
|
||||
createHistory: (json['createHistory'] as List<dynamic>?)?.map((e) => e as String).toList() ?? const [],
|
||||
runAsMenubarApp: json['runAsMenubarApp'] as bool? ?? false,
|
||||
analyticsEnabled: json['analyticsEnabled'] as bool? ?? true,
|
||||
analyticsClientId: json['analyticsClientId'] as String? ?? '',
|
||||
deviceId: json['deviceId'] as String? ?? '',
|
||||
);
|
||||
|
||||
Map<String, dynamic> _$ExtraConfigToJson(ExtraConfig instance) => <String, dynamic>{
|
||||
@@ -212,7 +219,7 @@ Map<String, dynamic> _$ExtraConfigToJson(ExtraConfig instance) => <String, dynam
|
||||
'createHistory': instance.createHistory,
|
||||
'runAsMenubarApp': instance.runAsMenubarApp,
|
||||
'analyticsEnabled': instance.analyticsEnabled,
|
||||
'analyticsClientId': instance.analyticsClientId,
|
||||
'deviceId': instance.deviceId,
|
||||
};
|
||||
|
||||
WindowStateConfig _$WindowStateConfigFromJson(Map<String, dynamic> json) => WindowStateConfig(
|
||||
|
||||
@@ -67,3 +67,28 @@ class OptsExtraHttp {
|
||||
|
||||
Map<String, dynamic> toJson() => _$OptsExtraHttpToJson(this);
|
||||
}
|
||||
|
||||
/// Options for an FTP task. It mirrors `pkg/protocol/ftp.OptsExtra`. The TLS
|
||||
/// mode of an `ftp://` link is on the request, see `ReqExtraFtp`.
|
||||
@JsonSerializable()
|
||||
class OptsExtraFtp {
|
||||
int connections;
|
||||
bool? autoTorrent;
|
||||
bool? deleteTorrentAfterDownload;
|
||||
bool? autoExtract;
|
||||
String archivePassword;
|
||||
bool deleteAfterExtract;
|
||||
|
||||
OptsExtraFtp({
|
||||
this.connections = 0,
|
||||
this.autoTorrent,
|
||||
this.deleteTorrentAfterDownload,
|
||||
this.autoExtract,
|
||||
this.archivePassword = '',
|
||||
this.deleteAfterExtract = false,
|
||||
});
|
||||
|
||||
factory OptsExtraFtp.fromJson(Map<String, dynamic> json) => _$OptsExtraFtpFromJson(json);
|
||||
|
||||
Map<String, dynamic> toJson() => _$OptsExtraFtpToJson(this);
|
||||
}
|
||||
@@ -65,3 +65,21 @@ Map<String, dynamic> _$OptsExtraHttpToJson(OptsExtraHttp instance) =>
|
||||
'deleteAfterExtract': instance.deleteAfterExtract,
|
||||
'checksum': ?instance.checksum?.toJson(),
|
||||
};
|
||||
|
||||
OptsExtraFtp _$OptsExtraFtpFromJson(Map<String, dynamic> json) => OptsExtraFtp(
|
||||
connections: (json['connections'] as num?)?.toInt() ?? 0,
|
||||
autoTorrent: json['autoTorrent'] as bool?,
|
||||
deleteTorrentAfterDownload: json['deleteTorrentAfterDownload'] as bool?,
|
||||
autoExtract: json['autoExtract'] as bool?,
|
||||
archivePassword: json['archivePassword'] as String? ?? '',
|
||||
deleteAfterExtract: json['deleteAfterExtract'] as bool? ?? false,
|
||||
);
|
||||
|
||||
Map<String, dynamic> _$OptsExtraFtpToJson(OptsExtraFtp instance) => <String, dynamic>{
|
||||
'connections': instance.connections,
|
||||
'autoTorrent': ?instance.autoTorrent,
|
||||
'deleteTorrentAfterDownload': ?instance.deleteTorrentAfterDownload,
|
||||
'autoExtract': ?instance.autoExtract,
|
||||
'archivePassword': instance.archivePassword,
|
||||
'deleteAfterExtract': instance.deleteAfterExtract,
|
||||
};
|
||||
@@ -42,6 +42,20 @@ class ReqExtraBt {
|
||||
Map<String, dynamic> toJson() => _$ReqExtraBtToJson(this);
|
||||
}
|
||||
|
||||
/// The request extra of an FTP task. It mirrors `pkg/protocol/ftp.ReqExtra`;
|
||||
/// [tls] only applies to `ftp://` URLs, because `ftps://` always uses
|
||||
/// implicit TLS and `ftpes://` always uses explicit TLS.
|
||||
@JsonSerializable()
|
||||
class ReqExtraFtp {
|
||||
String tls;
|
||||
|
||||
ReqExtraFtp({this.tls = 'none'});
|
||||
|
||||
factory ReqExtraFtp.fromJson(Map<String, dynamic> json) => _$ReqExtraFtpFromJson(json);
|
||||
|
||||
Map<String, dynamic> toJson() => _$ReqExtraFtpToJson(this);
|
||||
}
|
||||
|
||||
enum RequestProxyMode { follow, none, custom }
|
||||
|
||||
@JsonSerializable()
|
||||
|
||||
@@ -41,6 +41,10 @@ ReqExtraBt _$ReqExtraBtFromJson(Map<String, dynamic> json) =>
|
||||
|
||||
Map<String, dynamic> _$ReqExtraBtToJson(ReqExtraBt instance) => <String, dynamic>{'trackers': instance.trackers};
|
||||
|
||||
ReqExtraFtp _$ReqExtraFtpFromJson(Map<String, dynamic> json) => ReqExtraFtp(tls: json['tls'] as String? ?? 'none');
|
||||
|
||||
Map<String, dynamic> _$ReqExtraFtpToJson(ReqExtraFtp instance) => <String, dynamic>{'tls': instance.tls};
|
||||
|
||||
RequestProxy _$RequestProxyFromJson(Map<String, dynamic> json) => RequestProxy(
|
||||
mode: $enumDecodeNullable(_$RequestProxyModeEnumMap, json['mode']) ?? RequestProxyMode.follow,
|
||||
scheme: json['scheme'] as String? ?? 'http',
|
||||
|
||||
@@ -6,7 +6,7 @@ part 'task.g.dart';
|
||||
|
||||
enum Status { ready, running, pause, wait, error, done }
|
||||
|
||||
enum Protocol { http, bt, ed2k, hls }
|
||||
enum Protocol { http, bt, ed2k, hls, ftp }
|
||||
|
||||
// ExtractStatus enum matching Go backend
|
||||
enum ExtractStatus {
|
||||
@@ -28,7 +28,10 @@ enum ExtractStatus {
|
||||
class Task {
|
||||
String id;
|
||||
String name;
|
||||
@JsonKey(unknownEnumValue: JsonKey.nullForUndefinedEnumValue)
|
||||
Protocol? protocol;
|
||||
TaskGroup? group;
|
||||
String parentId;
|
||||
Meta meta;
|
||||
Status status;
|
||||
bool uploading;
|
||||
@@ -47,6 +50,8 @@ class Task {
|
||||
required this.createdAt,
|
||||
required this.updatedAt,
|
||||
this.doneAt,
|
||||
this.group,
|
||||
this.parentId = '',
|
||||
});
|
||||
|
||||
factory Task.fromJson(Map<String, dynamic> json) => _$TaskFromJson(json);
|
||||
@@ -54,6 +59,18 @@ class Task {
|
||||
Map<String, dynamic> toJson() => _$TaskToJson(this);
|
||||
}
|
||||
|
||||
@JsonSerializable()
|
||||
class TaskGroup {
|
||||
final String id;
|
||||
final String name;
|
||||
final String path;
|
||||
final DateTime createdAt;
|
||||
|
||||
TaskGroup({required this.id, required this.name, required this.path, required this.createdAt});
|
||||
factory TaskGroup.fromJson(Map<String, dynamic> json) => _$TaskGroupFromJson(json);
|
||||
Map<String, dynamic> toJson() => _$TaskGroupToJson(this);
|
||||
}
|
||||
|
||||
@JsonSerializable()
|
||||
class Progress {
|
||||
int used;
|
||||
|
||||
@@ -16,12 +16,16 @@ Task _$TaskFromJson(Map<String, dynamic> json) => Task(
|
||||
createdAt: DateTime.parse(json['createdAt'] as String),
|
||||
updatedAt: DateTime.parse(json['updatedAt'] as String),
|
||||
doneAt: json['doneAt'] == null ? null : DateTime.parse(json['doneAt'] as String),
|
||||
)..protocol = $enumDecodeNullable(_$ProtocolEnumMap, json['protocol']);
|
||||
group: json['group'] == null ? null : TaskGroup.fromJson(json['group'] as Map<String, dynamic>),
|
||||
parentId: json['parentId'] as String? ?? '',
|
||||
)..protocol = $enumDecodeNullable(_$ProtocolEnumMap, json['protocol'], unknownValue: JsonKey.nullForUndefinedEnumValue);
|
||||
|
||||
Map<String, dynamic> _$TaskToJson(Task instance) => <String, dynamic>{
|
||||
'id': instance.id,
|
||||
'name': instance.name,
|
||||
'protocol': ?_$ProtocolEnumMap[instance.protocol],
|
||||
'group': ?instance.group?.toJson(),
|
||||
'parentId': instance.parentId,
|
||||
'meta': instance.meta.toJson(),
|
||||
'status': _$StatusEnumMap[instance.status]!,
|
||||
'uploading': instance.uploading,
|
||||
@@ -40,7 +44,27 @@ const _$StatusEnumMap = {
|
||||
Status.done: 'done',
|
||||
};
|
||||
|
||||
const _$ProtocolEnumMap = {Protocol.http: 'http', Protocol.bt: 'bt', Protocol.ed2k: 'ed2k', Protocol.hls: 'hls'};
|
||||
const _$ProtocolEnumMap = {
|
||||
Protocol.http: 'http',
|
||||
Protocol.bt: 'bt',
|
||||
Protocol.ed2k: 'ed2k',
|
||||
Protocol.hls: 'hls',
|
||||
Protocol.ftp: 'ftp',
|
||||
};
|
||||
|
||||
TaskGroup _$TaskGroupFromJson(Map<String, dynamic> json) => TaskGroup(
|
||||
id: json['id'] as String,
|
||||
name: json['name'] as String,
|
||||
path: json['path'] as String,
|
||||
createdAt: DateTime.parse(json['createdAt'] as String),
|
||||
);
|
||||
|
||||
Map<String, dynamic> _$TaskGroupToJson(TaskGroup instance) => <String, dynamic>{
|
||||
'id': instance.id,
|
||||
'name': instance.name,
|
||||
'path': instance.path,
|
||||
'createdAt': instance.createdAt.toIso8601String(),
|
||||
};
|
||||
|
||||
Progress _$ProgressFromJson(Map<String, dynamic> json) => Progress(
|
||||
used: (json['used'] as num).toInt(),
|
||||
|
||||
@@ -52,7 +52,8 @@ sealed class TaskStats {
|
||||
final runtime = _map(json['runtime']);
|
||||
if (snapshot.isEmpty && runtime.isEmpty) return null;
|
||||
return switch (protocol) {
|
||||
Protocol.http => HttpTaskStats.fromJson(snapshot),
|
||||
// FTP stats have the same shape as HTTP stats: one entry per connection.
|
||||
Protocol.http || Protocol.ftp => HttpTaskStats.fromJson(snapshot),
|
||||
Protocol.bt => BtTaskStats.fromJson(snapshot: snapshot, runtime: runtime),
|
||||
Protocol.ed2k => Ed2kTaskStats.fromJson(snapshot: snapshot, runtime: runtime),
|
||||
Protocol.hls || null => null,
|
||||
|
||||
@@ -13,9 +13,7 @@ import '../../api/model/install_extension.dart';
|
||||
import '../../api/model/request.dart';
|
||||
import '../../core/entry/app_startup_options.dart';
|
||||
import '../../core/utils/content_uri_resolver.dart';
|
||||
import '../../core/window/app_window_launcher.dart';
|
||||
import '../../features/extensions/application/pending_extension_install.dart';
|
||||
import '../../features/tasks/application/pending_create_task.dart';
|
||||
import '../../util/util.dart';
|
||||
import '../router/app_router.dart';
|
||||
import 'app_runtime_controller.dart';
|
||||
@@ -118,18 +116,7 @@ class AppDeepLinkController extends AsyncNotifier<AppDeepLinkState> {
|
||||
await _go('/');
|
||||
}
|
||||
|
||||
Future<void> _openCreate(CreateTask? createTask) async {
|
||||
if (Util.isDesktop()) {
|
||||
final opened = await AppWindowLauncher.openCreateTaskWindow(createTask: createTask);
|
||||
if (!opened) {
|
||||
ref.read(pendingCreateTaskProvider.notifier).set(createTask);
|
||||
await _go('/create');
|
||||
}
|
||||
return;
|
||||
}
|
||||
ref.read(pendingCreateTaskProvider.notifier).set(createTask);
|
||||
await _go('/create');
|
||||
}
|
||||
Future<void> _openCreate(CreateTask? createTask) => AppRouter.openCreateTask(createTask: createTask);
|
||||
|
||||
Future<void> _go(String route) async {
|
||||
if (Util.isDesktop()) {
|
||||
|
||||
@@ -14,9 +14,7 @@ import '../../api/api.dart' as api;
|
||||
import '../../api/model/downloader_config.dart';
|
||||
import '../../core/libgopeed_boot.dart';
|
||||
import '../../core/capabilities/app_capabilities.dart';
|
||||
import '../../core/window/app_window_launcher.dart';
|
||||
import '../../core/window/window_capability_transport.dart';
|
||||
import '../../features/tasks/application/pending_create_task.dart';
|
||||
import '../../features/tasks/application/pending_update_task.dart';
|
||||
import '../../l10n/l10n.dart';
|
||||
import '../../util/package_info.dart';
|
||||
@@ -211,11 +209,7 @@ class AppPlatformController extends AsyncNotifier<AppPlatformState> with WindowL
|
||||
await ref.read(gopeedServiceProvider).createTask(createTask);
|
||||
return;
|
||||
}
|
||||
final opened = await AppWindowLauncher.openCreateTaskWindow(createTask: createTask);
|
||||
if (!opened) {
|
||||
ref.read(pendingCreateTaskProvider.notifier).set(createTask);
|
||||
await _go('/create');
|
||||
}
|
||||
await AppRouter.openCreateTask(createTask: createTask);
|
||||
},
|
||||
onForward: ({required path, required method, data, query}) {
|
||||
return api.forward(path, method: method, data: data, queryParameters: query);
|
||||
@@ -348,10 +342,7 @@ class AppPlatformController extends AsyncNotifier<AppPlatformState> with WindowL
|
||||
|
||||
Future<void> _openCreateTask() async {
|
||||
await _showMainWindow();
|
||||
final opened = await AppWindowLauncher.openCreateTaskWindow();
|
||||
if (!opened) {
|
||||
_go('/create');
|
||||
}
|
||||
await AppRouter.openCreateTask();
|
||||
}
|
||||
|
||||
Future<void> _go(String route) async {
|
||||
|
||||
@@ -255,9 +255,9 @@ class AppRuntimeController extends AsyncNotifier<AppRuntimeState> {
|
||||
Future<void> _initAnalytics(DownloaderConfig config) async {
|
||||
if (!AnalyticsConfig.isConfigured || !config.extra.analyticsEnabled) return;
|
||||
try {
|
||||
await Analytics.instance.init(fallbackClientId: config.extra.analyticsClientId);
|
||||
if (config.extra.analyticsClientId.isEmpty && Analytics.instance.clientId.isNotEmpty) {
|
||||
config.extra.analyticsClientId = Analytics.instance.clientId;
|
||||
final generatedClientId = await Analytics.instance.init(fallbackClientId: config.extra.deviceId);
|
||||
if (generatedClientId != null) {
|
||||
config.extra.deviceId = generatedClientId;
|
||||
await ref.read(gopeedServiceProvider).putConfig(config);
|
||||
}
|
||||
unawaited(Analytics.instance.logAppOpen());
|
||||
|
||||
@@ -1,13 +1,12 @@
|
||||
import 'package:flutter/widgets.dart';
|
||||
import 'package:shadcn_flutter/shadcn_flutter.dart' as shad;
|
||||
|
||||
import '../api/model/create_task.dart';
|
||||
import '../app/application/app_appearance_controller.dart';
|
||||
import '../core/window/app_window_bootstrap.dart';
|
||||
import '../core/window/app_window_frame.dart';
|
||||
import '../core/window/app_window_payload.dart';
|
||||
import '../core/window/window_capability_transport.dart';
|
||||
import '../features/tasks/presentation/pages/create_task_window_page.dart';
|
||||
import 'router/app_window_content.dart';
|
||||
import '../l10n/l10n.dart';
|
||||
import '../shared/theme/app_component_themes.dart';
|
||||
import '../shared/theme/app_theme.dart';
|
||||
@@ -82,10 +81,7 @@ class _ChildWindowAppState extends State<ChildWindowApp> with WidgetsBindingObse
|
||||
|
||||
Widget _buildPage(BuildContext context) {
|
||||
return switch (widget.payload.type) {
|
||||
AppWindowType.createTask => CreateTaskWindowPage(
|
||||
windowController: widget.session.controller,
|
||||
initialTask: widget.payload.createTask == null ? null : CreateTask.fromJson(widget.payload.createTask!),
|
||||
),
|
||||
AppWindowType.createTask => AppWindowContent.build(widget.payload, windowController: widget.session.controller),
|
||||
_ => shad.Scaffold(child: Center(child: Text(context.l10n.unsupportedWindowPayload))),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import 'dart:async';
|
||||
|
||||
import 'package:flutter/foundation.dart';
|
||||
import 'package:flutter/widgets.dart';
|
||||
import 'package:go_router/go_router.dart';
|
||||
@@ -11,7 +13,13 @@ import '../../features/auth/application/web_auth_controller.dart';
|
||||
import '../../features/auth/presentation/pages/login_page.dart';
|
||||
import '../../features/home/presentation/pages/home_page.dart';
|
||||
import '../../features/settings/presentation/pages/settings_page.dart';
|
||||
import '../../features/tasks/presentation/pages/create_task_window_page.dart';
|
||||
import '../../api/model/create_task.dart';
|
||||
import '../../core/window/app_window_chrome.dart';
|
||||
import '../../core/window/app_window_launcher.dart';
|
||||
import '../../core/window/app_window_payload.dart';
|
||||
import '../../features/tasks/application/pending_create_task.dart';
|
||||
import 'app_window_content.dart';
|
||||
import 'app_window_page.dart';
|
||||
import '../../features/tasks/domain/task_record.dart';
|
||||
import '../../features/tasks/presentation/pages/task_details_page.dart';
|
||||
import '../../features/tasks/presentation/pages/task_files_page.dart';
|
||||
@@ -22,6 +30,37 @@ import 'runtime_page.dart';
|
||||
class AppRouter {
|
||||
static final GlobalKey<NavigatorState> rootNavigatorKey = GlobalKey<NavigatorState>();
|
||||
|
||||
static bool get _isMobile =>
|
||||
!kIsWeb && (defaultTargetPlatform == TargetPlatform.android || defaultTargetPlatform == TargetPlatform.iOS);
|
||||
|
||||
static Future<void> openCreateTask({CreateTask? createTask}) =>
|
||||
openWindow(AppWindowPayload.createTask(createTask: createTask?.toJson()));
|
||||
|
||||
/// Opens a desktop child window, a Web modal, or a native mobile page.
|
||||
/// Callers never need to choose a presentation or implement a fallback.
|
||||
static Future<void> openWindow(AppWindowPayload payload) async {
|
||||
final location = AppWindowContent.location(payload.type);
|
||||
if (AppWindowChrome.isDesktopWindow && await AppWindowLauncher.open(payload)) return;
|
||||
final context = rootNavigatorKey.currentContext;
|
||||
if (context == null || !context.mounted) return;
|
||||
final router = GoRouter.of(context);
|
||||
if (router.state.uri.path == location) {
|
||||
// Native share intents may arrive while the creation page is already open.
|
||||
if (payload.type == AppWindowType.createTask && payload.createTask != null) {
|
||||
ProviderScope.containerOf(
|
||||
context,
|
||||
listen: false,
|
||||
).read(pendingCreateTaskProvider.notifier).set(CreateTask.fromJson(payload.createTask!));
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (_isMobile) {
|
||||
router.go(location, extra: payload.toRaw());
|
||||
} else {
|
||||
unawaited(router.push<void>(location, extra: payload.toRaw()));
|
||||
}
|
||||
}
|
||||
|
||||
static void showDownloadingTasks() {
|
||||
final context = rootNavigatorKey.currentContext;
|
||||
if (context == null || !context.mounted) return;
|
||||
@@ -31,8 +70,7 @@ class AppRouter {
|
||||
|
||||
static GoRouter build(WebAuthController webAuthController) {
|
||||
// Keep desktop and web sections as peers; native mobile retains home below them.
|
||||
final isMobile =
|
||||
!kIsWeb && (defaultTargetPlatform == TargetPlatform.android || defaultTargetPlatform == TargetPlatform.iOS);
|
||||
final isMobile = _isMobile;
|
||||
final sectionRoutes = [
|
||||
GoRoute(
|
||||
path: isMobile ? 'extensions' : '/extensions',
|
||||
@@ -83,7 +121,33 @@ class AppRouter {
|
||||
path: '/',
|
||||
builder: _runtimePage((context, state) => const HomePage()),
|
||||
routes: [
|
||||
GoRoute(path: 'create', builder: _runtimePage((context, state) => const CreateTaskWindowPage())),
|
||||
GoRoute(
|
||||
path: 'create',
|
||||
parentNavigatorKey: rootNavigatorKey,
|
||||
pageBuilder: (context, state) {
|
||||
final payload = state.extra is String
|
||||
? AppWindowPayload.fromRaw(state.extra! as String)
|
||||
: AppWindowPayload.createTask();
|
||||
final child = RuntimePage(
|
||||
builder: (context) => AppWindowContent.build(
|
||||
payload,
|
||||
modal: !isMobile,
|
||||
onClose: (created) {
|
||||
if (created) {
|
||||
showDownloadingTasks();
|
||||
} else if (context.canPop()) {
|
||||
context.pop();
|
||||
} else {
|
||||
context.go('/');
|
||||
}
|
||||
},
|
||||
),
|
||||
);
|
||||
return isMobile
|
||||
? NoTransitionPage<void>(key: state.pageKey, child: child)
|
||||
: AppWindowPage<void>(key: state.pageKey, child: child);
|
||||
},
|
||||
),
|
||||
GoRoute(
|
||||
path: 'tasks/:id',
|
||||
builder: _runtimePage(
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
import 'package:desktop_multi_window/desktop_multi_window.dart';
|
||||
import 'package:flutter/widgets.dart';
|
||||
|
||||
import '../../api/model/create_task.dart';
|
||||
import '../../core/window/app_window_payload.dart';
|
||||
import '../../features/tasks/presentation/pages/create_task_window_page.dart';
|
||||
|
||||
/// Shared destination registry for desktop child engines and in-app windows.
|
||||
abstract final class AppWindowContent {
|
||||
static String location(AppWindowType type) => switch (type) {
|
||||
AppWindowType.createTask => '/create',
|
||||
AppWindowType.main => throw ArgumentError.value(type, 'type', 'Expected a child window'),
|
||||
};
|
||||
|
||||
static Widget build(
|
||||
AppWindowPayload payload, {
|
||||
WindowController? windowController,
|
||||
ValueChanged<bool>? onClose,
|
||||
bool modal = false,
|
||||
}) => switch (payload.type) {
|
||||
AppWindowType.createTask => CreateTaskWindowPage(
|
||||
windowController: windowController,
|
||||
initialTask: payload.createTask == null ? null : CreateTask.fromJson(payload.createTask!),
|
||||
onClose: onClose,
|
||||
modal: modal,
|
||||
),
|
||||
AppWindowType.main => throw ArgumentError.value(payload.type, 'payload.type', 'Expected a child window'),
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
import 'package:shadcn_flutter/shadcn_flutter.dart';
|
||||
|
||||
import '../../shared/theme/app_design_tokens.dart';
|
||||
import '../../shared/theme/app_palette.dart';
|
||||
import '../../core/utils/breakpoints.dart';
|
||||
|
||||
/// A routed modal keeps its underlying page mounted and participates in Back.
|
||||
class AppWindowPage<T> extends Page<T> {
|
||||
const AppWindowPage({super.key, required this.child, this.preferredSize = AppDesignTokens.childWindowSize});
|
||||
|
||||
final Widget child;
|
||||
final Size preferredSize;
|
||||
|
||||
@override
|
||||
Route<T> createRoute(BuildContext context) => DialogRoute<T>(
|
||||
context: context,
|
||||
settings: this,
|
||||
barrierDismissible: false,
|
||||
barrierColor: Theme.of(context).colorScheme.background.withValues(alpha: 0.6),
|
||||
traversalEdgeBehavior: TraversalEdgeBehavior.closedLoop,
|
||||
alignment: Alignment.center,
|
||||
fullScreen: true,
|
||||
transitionBuilder: (context, animation, secondaryAnimation, child) =>
|
||||
FadeTransition(opacity: animation, child: child),
|
||||
builder: (context) => Padding(
|
||||
padding: EdgeInsets.only(bottom: MediaQuery.viewInsetsOf(context).bottom),
|
||||
child: LayoutBuilder(
|
||||
builder: (context, constraints) {
|
||||
final page = ModalRoute.settingsOf(context)! as AppWindowPage<T>;
|
||||
final palette = AppPalette.of(context);
|
||||
return Padding(
|
||||
padding: EdgeInsets.all(
|
||||
constraints.maxWidth < Breakpoints.mobile ? AppDesignTokens.space12 : AppDesignTokens.space32,
|
||||
),
|
||||
child: Center(
|
||||
child: ConstrainedBox(
|
||||
constraints: BoxConstraints(maxWidth: page.preferredSize.width, maxHeight: page.preferredSize.height),
|
||||
child: ModalContainer(
|
||||
key: const ValueKey('app-window-modal'),
|
||||
padding: EdgeInsets.zero,
|
||||
filled: true,
|
||||
fillColor: palette.sideBg,
|
||||
borderColor: palette.border,
|
||||
borderRadius: BorderRadius.circular(AppDesignTokens.windowRadius),
|
||||
clipBehavior: Clip.antiAlias,
|
||||
child: MediaQuery.removeViewInsets(
|
||||
context: context,
|
||||
removeBottom: true,
|
||||
child: MediaQuery.removePadding(
|
||||
context: context,
|
||||
removeTop: true,
|
||||
removeBottom: true,
|
||||
child: page.child,
|
||||
),
|
||||
),
|
||||
),
|
||||
),
|
||||
),
|
||||
);
|
||||
},
|
||||
),
|
||||
),
|
||||
);
|
||||
}
|
||||
@@ -3,7 +3,6 @@ import 'dart:async';
|
||||
import 'package:flutter/foundation.dart';
|
||||
import 'package:flutter/widgets.dart';
|
||||
import 'package:flutter_riverpod/flutter_riverpod.dart';
|
||||
import 'package:go_router/go_router.dart';
|
||||
|
||||
import '../../../api/model/create_task.dart';
|
||||
import '../../application/app_deep_link_controller.dart';
|
||||
@@ -11,11 +10,10 @@ import '../../application/app_notification_controller.dart';
|
||||
import '../../application/app_runtime_controller.dart';
|
||||
import '../../application/app_platform_controller.dart';
|
||||
import '../../../features/settings/presentation/widgets/app_update_dialog.dart';
|
||||
import '../../../features/tasks/application/pending_create_task.dart';
|
||||
import '../../../features/tasks/application/pending_update_task.dart';
|
||||
import '../../../features/tasks/application/tasks_controller.dart';
|
||||
import '../../../features/tasks/presentation/widgets/pending_update_dialog.dart';
|
||||
import '../../../core/window/app_window_launcher.dart';
|
||||
import '../app_router.dart';
|
||||
import '../../../shared/widgets/app_toast.dart';
|
||||
|
||||
class MainShell extends ConsumerStatefulWidget {
|
||||
@@ -124,11 +122,5 @@ class _MainShellState extends ConsumerState<MainShell> {
|
||||
}
|
||||
}
|
||||
|
||||
Future<void> _openCapturedCreateTask(CreateTask createTask) async {
|
||||
final opened = await AppWindowLauncher.openCreateTaskWindow(createTask: createTask);
|
||||
if (!opened && mounted) {
|
||||
ref.read(pendingCreateTaskProvider.notifier).set(createTask);
|
||||
context.go('/create');
|
||||
}
|
||||
}
|
||||
Future<void> _openCapturedCreateTask(CreateTask createTask) => AppRouter.openCreateTask(createTask: createTask);
|
||||
}
|
||||
@@ -34,8 +34,9 @@ class AppCapabilities {
|
||||
class LocalAppCapabilities {
|
||||
LocalAppCapabilities._() : codecs = createAppCapabilityCodecs() {
|
||||
registry = CapabilityRegistry(codecs);
|
||||
_bindGopeed(registry);
|
||||
_bindStorage(registry);
|
||||
final config = _ConfigStore();
|
||||
_bindGopeed(registry, config);
|
||||
_bindStorage(registry, config);
|
||||
registry.bind(NavigationMethods.showDownloadingTasks, (_) {
|
||||
AppRouter.showDownloadingTasks();
|
||||
return const RpcUnit();
|
||||
@@ -62,6 +63,7 @@ RpcCodecRegistry createAppCapabilityCodecs() {
|
||||
..register<ResolveResult>((json) => ResolveResult.fromJson(json! as Map<String, dynamic>))
|
||||
..register<CreateTask>((json) => CreateTask.fromJson(json! as Map<String, dynamic>))
|
||||
..register<CreateTaskBatch>((json) => CreateTaskBatch.fromJson(json! as Map<String, dynamic>))
|
||||
..register<CreateTaskBatchResult>((json) => CreateTaskBatchResult.fromJson(json! as Map<String, dynamic>))
|
||||
..register<DownloaderConfig>((json) => DownloaderConfig.fromJson(json! as Map<String, dynamic>))
|
||||
..register<TaskRuntimeStatus>((json) => TaskRuntimeStatus.fromJson(json! as Map<String, dynamic>))
|
||||
..register<List<Task>>(
|
||||
@@ -80,7 +82,7 @@ RpcCodecRegistry createAppCapabilityCodecs() {
|
||||
return codecs;
|
||||
}
|
||||
|
||||
void _bindGopeed(CapabilityRegistry registry) {
|
||||
void _bindGopeed(CapabilityRegistry registry, _ConfigStore configStore) {
|
||||
registry
|
||||
..bind(GopeedMethods.triggerProtocolJob, (p) async {
|
||||
await api.triggerProtocolJob(p['protocol'] as String, p['name'] as String);
|
||||
@@ -127,9 +129,9 @@ void _bindGopeed(CapabilityRegistry registry) {
|
||||
await api.deleteTasks(_optionalStringList(params['ids']), params['force'] as bool? ?? false);
|
||||
return const RpcUnit();
|
||||
})
|
||||
..bind(GopeedMethods.getConfig, (_) => api.getConfig())
|
||||
..bind(GopeedMethods.getConfig, (_) => configStore.get())
|
||||
..bind(GopeedMethods.putConfig, (config) async {
|
||||
await api.putConfig(config);
|
||||
await configStore.put(config);
|
||||
return const RpcUnit();
|
||||
})
|
||||
..bind(GopeedMethods.installExtension, api.installExtension)
|
||||
@@ -163,35 +165,59 @@ void _bindGopeed(CapabilityRegistry registry) {
|
||||
});
|
||||
}
|
||||
|
||||
void _bindStorage(CapabilityRegistry registry) {
|
||||
// Shared by local and child-window handlers. History owns its config field;
|
||||
// full config snapshots must preserve the latest history inside the same queue.
|
||||
class _ConfigStore {
|
||||
Future<void> _tail = Future<void>.value();
|
||||
|
||||
Future<T> _run<T>(Future<T> Function() operation) {
|
||||
final result = _tail.then((_) => operation());
|
||||
_tail = result.then<void>((_) {}, onError: (Object _, StackTrace _) {});
|
||||
return result;
|
||||
}
|
||||
|
||||
Future<DownloaderConfig> get() => _run(api.getConfig);
|
||||
|
||||
Future<void> put(DownloaderConfig config) {
|
||||
final snapshot = DownloaderConfig.fromJson(config.toJson());
|
||||
return _run(() async {
|
||||
final latest = await api.getConfig();
|
||||
snapshot.extra.createHistory = latest.extra.createHistory;
|
||||
await api.putConfig(snapshot);
|
||||
});
|
||||
}
|
||||
|
||||
Future<void> updateHistory(void Function(List<String>) mutation) => _run(() async {
|
||||
final config = await api.getConfig();
|
||||
final history = List<String>.of(config.extra.createHistory);
|
||||
mutation(history);
|
||||
config.extra.createHistory = history;
|
||||
await api.putConfig(config);
|
||||
});
|
||||
}
|
||||
|
||||
void _bindStorage(CapabilityRegistry registry, _ConfigStore configStore) {
|
||||
registry
|
||||
..bind(StorageMethods.getCreateHistory, (_) async {
|
||||
final config = await api.getConfig();
|
||||
final config = await configStore.get();
|
||||
return config.extra.createHistory;
|
||||
})
|
||||
..bind(StorageMethods.saveCreateHistory, (urls) async {
|
||||
final config = await api.getConfig();
|
||||
final history = List<String>.of(config.extra.createHistory);
|
||||
for (final url in urls) {
|
||||
history.remove(url);
|
||||
history.insert(0, url);
|
||||
if (history.length > 64) history.removeLast();
|
||||
}
|
||||
config.extra.createHistory = history;
|
||||
await api.putConfig(config);
|
||||
await configStore.updateHistory((history) {
|
||||
for (final url in urls) {
|
||||
history.remove(url);
|
||||
history.insert(0, url);
|
||||
if (history.length > 64) history.removeLast();
|
||||
}
|
||||
});
|
||||
return const RpcUnit();
|
||||
})
|
||||
..bind(StorageMethods.removeCreateHistory, (url) async {
|
||||
final config = await api.getConfig();
|
||||
final history = List<String>.of(config.extra.createHistory)..remove(url);
|
||||
config.extra.createHistory = history;
|
||||
await api.putConfig(config);
|
||||
await configStore.updateHistory((history) => history.remove(url));
|
||||
return const RpcUnit();
|
||||
})
|
||||
..bind(StorageMethods.clearCreateHistory, (_) async {
|
||||
final config = await api.getConfig();
|
||||
config.extra.createHistory = const [];
|
||||
await api.putConfig(config);
|
||||
await configStore.updateHistory((history) => history.clear());
|
||||
return const RpcUnit();
|
||||
});
|
||||
}
|
||||
|
||||
@@ -17,7 +17,7 @@ abstract final class GopeedMethods {
|
||||
static const getMirrorCandidates = RpcMethod<Map<String, dynamic>, List<String>>('gopeed.mirrors.candidates');
|
||||
static const resolve = RpcMethod<ResolveTask, ResolveResult>('gopeed.resolve');
|
||||
static const createTask = RpcMethod<CreateTask, String>('gopeed.task.create');
|
||||
static const createTaskBatch = RpcMethod<CreateTaskBatch, List<String>>('gopeed.task.createBatch');
|
||||
static const createTaskBatch = RpcMethod<CreateTaskBatch, CreateTaskBatchResult>('gopeed.task.createBatch');
|
||||
static const patchTask = RpcMethod<Map<String, dynamic>, RpcUnit>('gopeed.task.patch');
|
||||
static const getTasks = RpcMethod<List<String>, List<Task>>('gopeed.task.list');
|
||||
static const getTaskStatus = RpcMethod<String, TaskRuntimeStatus>('gopeed.task.status');
|
||||
@@ -53,7 +53,7 @@ class GopeedService {
|
||||
|
||||
Future<String> createTask(CreateTask request) => _invoker.invoke(GopeedMethods.createTask, request);
|
||||
|
||||
Future<List<String>> createTaskBatch(CreateTaskBatch request) =>
|
||||
Future<CreateTaskBatchResult> createTaskBatch(CreateTaskBatch request) =>
|
||||
_invoker.invoke(GopeedMethods.createTaskBatch, request);
|
||||
|
||||
Future<void> patchTask(String id, PatchTask request) async {
|
||||
|
||||
@@ -4,6 +4,7 @@ import 'package:flutter/widgets.dart';
|
||||
import 'package:window_manager/window_manager.dart';
|
||||
|
||||
import '../../l10n/l10n.dart';
|
||||
import '../../shared/theme/app_design_tokens.dart';
|
||||
import 'app_window_chrome.dart';
|
||||
import 'app_window_payload.dart';
|
||||
|
||||
@@ -46,7 +47,7 @@ class AppWindowBootstrap {
|
||||
|
||||
final options = switch (type) {
|
||||
AppWindowType.createTask => WindowOptions(
|
||||
size: Size(820, 620),
|
||||
size: AppDesignTokens.childWindowSize,
|
||||
center: true,
|
||||
skipTaskbar: false,
|
||||
backgroundColor: Colors.transparent,
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
import 'package:flutter/foundation.dart' show TargetPlatform, defaultTargetPlatform;
|
||||
import 'package:flutter/material.dart' show Colors;
|
||||
import 'package:flutter/widgets.dart';
|
||||
import 'package:window_manager/window_manager.dart';
|
||||
|
||||
import '../../shared/theme/app_palette.dart';
|
||||
import '../../shared/widgets/window/desktop_window_header.dart';
|
||||
import 'app_window_chrome.dart';
|
||||
|
||||
@@ -16,6 +18,15 @@ class AppWindowFrame extends StatelessWidget {
|
||||
Widget build(BuildContext context) {
|
||||
Widget framedChild = child;
|
||||
|
||||
// System insets apply to mobile devices at every responsive breakpoint.
|
||||
// SafeArea also consumes the padding so page-level SafeAreas do not add it twice.
|
||||
if (defaultTargetPlatform == TargetPlatform.android || defaultTargetPlatform == TargetPlatform.iOS) {
|
||||
framedChild = ColoredBox(
|
||||
color: AppPalette.of(context).bg,
|
||||
child: SafeArea(child: framedChild),
|
||||
);
|
||||
}
|
||||
|
||||
if (AppWindowChrome.usesCustomChrome) {
|
||||
framedChild = Stack(
|
||||
children: [
|
||||
|
||||
@@ -2,22 +2,24 @@ import 'dart:io';
|
||||
|
||||
import 'package:desktop_multi_window/desktop_multi_window.dart';
|
||||
import 'package:flutter/foundation.dart';
|
||||
import 'package:window_manager/window_manager.dart';
|
||||
|
||||
import '../../api/model/create_task.dart';
|
||||
import '../../util/log_util.dart';
|
||||
import '../capabilities/app_capabilities.dart';
|
||||
import 'app_window_payload.dart';
|
||||
import 'window_capability_transport.dart';
|
||||
|
||||
class AppWindowLauncher {
|
||||
static Future<bool> openCreateTaskWindow({CreateTask? createTask}) async {
|
||||
static Future<bool> open(AppWindowPayload payload) async {
|
||||
if (payload.type == AppWindowType.main) {
|
||||
throw ArgumentError.value(payload.type, 'payload.type', 'Expected a child window');
|
||||
}
|
||||
if (kIsWeb || !(Platform.isWindows || Platform.isMacOS || Platform.isLinux)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
try {
|
||||
await AppWindowCapabilityHost.instance.start(LocalAppCapabilities.instance.registry);
|
||||
final payload = AppWindowPayload.createTask(createTask: createTask?.toJson());
|
||||
// Every request deliberately creates a new child window. Create-task
|
||||
// windows are independent and are never reused.
|
||||
await WindowController.create(WindowConfiguration(arguments: payload.toRaw(), hiddenAtLaunch: true));
|
||||
@@ -27,10 +29,16 @@ class AppWindowLauncher {
|
||||
return true;
|
||||
} catch (error, stackTrace) {
|
||||
try {
|
||||
logger.w('create task window failed', error, stackTrace);
|
||||
logger.w('child window failed', error, stackTrace);
|
||||
} catch (_) {
|
||||
debugPrint('create task window failed: $error');
|
||||
debugPrint('child window failed: $error');
|
||||
}
|
||||
// A tray/deep-link request can arrive while the main window is hidden.
|
||||
// Bring the in-app fallback into view if the child engine cannot start.
|
||||
try {
|
||||
await windowManager.show();
|
||||
await windowManager.focus();
|
||||
} catch (_) {}
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -52,6 +52,7 @@ class AppWindowCapabilityHost {
|
||||
CapabilityRegistry? _registry;
|
||||
AppWindowAppearance _appearance = const AppWindowAppearance.defaults();
|
||||
final Map<String, WindowController> _subscribers = {};
|
||||
StreamSubscription<void>? _windowsChanged;
|
||||
|
||||
bool get started => _registry != null;
|
||||
|
||||
@@ -59,14 +60,33 @@ class AppWindowCapabilityHost {
|
||||
if (_registry != null) return;
|
||||
_registry = registry;
|
||||
await _hostChannel.setMethodCallHandler(_handleCall);
|
||||
_windowsChanged = onWindowsChanged.listen((_) => unawaited(_removeClosedSubscribers()));
|
||||
}
|
||||
|
||||
Future<void> stop() async {
|
||||
_registry = null;
|
||||
await _windowsChanged?.cancel();
|
||||
_windowsChanged = null;
|
||||
_subscribers.clear();
|
||||
await _hostChannel.setMethodCallHandler(null);
|
||||
}
|
||||
|
||||
Future<void> _removeClosedSubscribers() async {
|
||||
// Engine shutdown does not run Widget.dispose, so a closed child cannot
|
||||
// always send window.unsubscribe itself.
|
||||
final registry = _registry;
|
||||
final candidates = _subscribers.keys.toSet();
|
||||
if (registry == null || candidates.isEmpty) return;
|
||||
try {
|
||||
final windows = await WindowController.getAll();
|
||||
if (!identical(registry, _registry)) return;
|
||||
final openIds = windows.map((window) => window.windowId).toSet();
|
||||
_subscribers.removeWhere((id, _) => candidates.contains(id) && !openIds.contains(id));
|
||||
} catch (_) {
|
||||
// The native window registry can already be gone during app shutdown.
|
||||
}
|
||||
}
|
||||
|
||||
void updateAppearance(AppWindowAppearance appearance) {
|
||||
if (_appearance == appearance) return;
|
||||
_appearance = appearance;
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import 'dart:async';
|
||||
|
||||
import 'package:flutter/foundation.dart' show TargetPlatform, defaultTargetPlatform;
|
||||
import 'package:flutter/material.dart' show Colors, Icons;
|
||||
import 'package:flutter/widgets.dart';
|
||||
import 'package:flutter_riverpod/flutter_riverpod.dart';
|
||||
@@ -10,7 +11,7 @@ import '../../../../app/application/app_runtime_controller.dart';
|
||||
import '../../../../core/capabilities/app_capabilities.dart';
|
||||
import '../../../../core/utils/breakpoints.dart';
|
||||
import '../../../../core/utils/file_explorer.dart';
|
||||
import '../../../../core/window/app_window_launcher.dart';
|
||||
import '../../../../app/router/app_router.dart';
|
||||
import '../../../../core/window/app_window_chrome.dart';
|
||||
import '../../../../shared/theme/app_design_tokens.dart';
|
||||
import '../../../../shared/theme/app_palette.dart';
|
||||
@@ -48,10 +49,10 @@ class _BatchActionAvailability {
|
||||
_BatchActionAvailability _batchActionAvailability(Iterable<TaskRecord> tasks, TaskBatchSelectionController selection) {
|
||||
var canPause = false;
|
||||
var canResume = false;
|
||||
for (final task in tasks) {
|
||||
for (final task in tasks.expand((task) => [task, ...task.members])) {
|
||||
if (!selection.contains(task.id)) continue;
|
||||
canPause = canPause || task.status == TaskStatus.downloading;
|
||||
canResume = canResume || task.status == TaskStatus.paused || task.status == TaskStatus.failed;
|
||||
canPause = canPause || task.canPause;
|
||||
canResume = canResume || task.canResume;
|
||||
if (canPause && canResume) break;
|
||||
}
|
||||
return _BatchActionAvailability(canPause: canPause, canResume: canResume);
|
||||
@@ -69,6 +70,7 @@ class _HomePageState extends ConsumerState<HomePage> {
|
||||
_TaskFilter _activeFilter = _TaskFilter.downloading;
|
||||
TaskRecord? _selectedTask;
|
||||
bool _batchMode = false;
|
||||
final Set<String> _expandedGroupIds = {};
|
||||
final TaskBatchSelectionController _batchSelection = TaskBatchSelectionController();
|
||||
|
||||
@override
|
||||
@@ -96,11 +98,16 @@ class _HomePageState extends ConsumerState<HomePage> {
|
||||
final tasksState = tasksAsync.value;
|
||||
final tasks = tasksState?.tasks ?? const <TaskRecord>[];
|
||||
final filteredTasks = _filteredTasks(tasks);
|
||||
final visibleTasks = _visibleTasks(filteredTasks);
|
||||
final memberIds = {
|
||||
for (final task in tasks)
|
||||
for (final member in task.members) member.id,
|
||||
};
|
||||
final emptyStateMessage = _searchController.text.trim().isEmpty
|
||||
? context.l10n.emptyTaskList
|
||||
: context.l10n.noMatchingTasks;
|
||||
final selectedTask = _latestSelectedTask(tasks);
|
||||
final visibleTaskIds = filteredTasks.map((task) => task.id).toList(growable: false);
|
||||
final selectedTask = _latestSelectedTask();
|
||||
final visibleTaskIds = visibleTasks.map((task) => task.id).toList(growable: false);
|
||||
final pendingUpdateTask = ref.watch(pendingUpdateTaskProvider);
|
||||
final navigationItems = _taskNavigationItems(tasks);
|
||||
final desktopBody = shad.Scaffold(
|
||||
@@ -158,7 +165,7 @@ class _HomePageState extends ConsumerState<HomePage> {
|
||||
listenable: _batchSelection,
|
||||
builder: (context, _) => _TaskBatchRow(
|
||||
selectedCount: _batchSelection.count,
|
||||
totalCount: filteredTasks.length,
|
||||
totalCount: visibleTasks.length,
|
||||
allSelected: _batchSelection.areAllSelected(visibleTaskIds),
|
||||
someSelected:
|
||||
_batchSelection.isNotEmpty &&
|
||||
@@ -179,16 +186,19 @@ class _HomePageState extends ConsumerState<HomePage> {
|
||||
)
|
||||
: ListView.separated(
|
||||
padding: const EdgeInsets.only(bottom: 16),
|
||||
itemCount: filteredTasks.length,
|
||||
itemCount: visibleTasks.length,
|
||||
separatorBuilder: (_, _) => const SizedBox(height: 8),
|
||||
itemBuilder: (context, index) {
|
||||
final task = filteredTasks[index];
|
||||
final task = visibleTasks[index];
|
||||
return TaskBatchSelectionBuilder(
|
||||
key: ValueKey(task.id),
|
||||
controller: _batchSelection,
|
||||
taskId: task.id,
|
||||
visibleTaskIds: visibleTaskIds,
|
||||
builder: (context, selectedInBatch, allSelected) => TaskCard(
|
||||
task: task,
|
||||
nested: memberIds.contains(task.id),
|
||||
expanded: _expandedGroupIds.contains(task.id),
|
||||
selected: _selectedTask?.id == task.id,
|
||||
batchMode: _batchMode,
|
||||
selectedInBatch: selectedInBatch,
|
||||
@@ -242,6 +252,8 @@ class _HomePageState extends ConsumerState<HomePage> {
|
||||
final mobileBody = tasksAsync.when(
|
||||
loading: () => _MobileTasksView(
|
||||
tasks: const [],
|
||||
memberIds: const {},
|
||||
expandedGroupIds: _expandedGroupIds,
|
||||
batchMode: _batchMode,
|
||||
batchSelection: _batchSelection,
|
||||
loading: true,
|
||||
@@ -262,7 +274,9 @@ class _HomePageState extends ConsumerState<HomePage> {
|
||||
),
|
||||
error: (error, _) => ColoredBox(color: palette.bg, child: const TaskEmptyState()),
|
||||
data: (_) => _MobileTasksView(
|
||||
tasks: filteredTasks,
|
||||
tasks: visibleTasks,
|
||||
memberIds: memberIds,
|
||||
expandedGroupIds: _expandedGroupIds,
|
||||
emptyStateMessage: emptyStateMessage,
|
||||
onCreateTask: _searchController.text.trim().isEmpty ? _handleAddTask : null,
|
||||
batchMode: _batchMode,
|
||||
@@ -331,20 +345,20 @@ class _HomePageState extends ConsumerState<HomePage> {
|
||||
_TaskFilter.completed => task.status == TaskStatus.completed,
|
||||
_TaskFilter.failed => task.status == TaskStatus.failed,
|
||||
};
|
||||
final passesSearch =
|
||||
query.isEmpty || task.name.toLowerCase().contains(query) || task.url.toLowerCase().contains(query);
|
||||
final passesSearch = task.matchesSearch(query);
|
||||
return passesFilter && passesSearch;
|
||||
})
|
||||
.toList(growable: false);
|
||||
}
|
||||
|
||||
TaskRecord? _latestSelectedTask(List<TaskRecord> tasks) {
|
||||
List<TaskRecord> _visibleTasks(List<TaskRecord> tasks) => [
|
||||
for (final task in tasks) ...[task, if (_expandedGroupIds.contains(task.id)) ...task.members],
|
||||
];
|
||||
|
||||
TaskRecord? _latestSelectedTask() {
|
||||
final selectedTask = _selectedTask;
|
||||
if (selectedTask == null) return null;
|
||||
for (final task in tasks) {
|
||||
if (task.id == selectedTask.id) return task;
|
||||
}
|
||||
return selectedTask;
|
||||
return ref.read(tasksControllerProvider).value?.findTask(selectedTask.id);
|
||||
}
|
||||
|
||||
List<ResponsiveNavigationItem<_TaskFilter>> _taskNavigationItems(List<TaskRecord> tasks) {
|
||||
@@ -376,7 +390,13 @@ class _HomePageState extends ConsumerState<HomePage> {
|
||||
if (_batchMode) {
|
||||
_toggleBatchTask(task.id);
|
||||
} else {
|
||||
_showTaskDetails(task);
|
||||
if (task.isGroup) {
|
||||
setState(() {
|
||||
if (!_expandedGroupIds.remove(task.id)) _expandedGroupIds.add(task.id);
|
||||
});
|
||||
} else {
|
||||
_showTaskDetails(task);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -406,8 +426,8 @@ class _HomePageState extends ConsumerState<HomePage> {
|
||||
void _toggleBatchTask(String taskId) => _batchSelection.toggle(taskId);
|
||||
|
||||
void _toggleSelectAll() {
|
||||
final visibleIds = _filteredTasks(
|
||||
ref.read(tasksControllerProvider).value?.tasks ?? const [],
|
||||
final visibleIds = _visibleTasks(
|
||||
_filteredTasks(ref.read(tasksControllerProvider).value?.tasks ?? const []),
|
||||
).map((task) => task.id).toList(growable: false);
|
||||
if (_batchSelection.areAllSelected(visibleIds)) {
|
||||
_batchSelection.clear();
|
||||
@@ -430,22 +450,20 @@ class _HomePageState extends ConsumerState<HomePage> {
|
||||
|
||||
void _pauseSelectedTasks() {
|
||||
if (_batchMode && _batchSelection.isEmpty) return;
|
||||
final ids = _batchMode ? _selectedTaskIdsFor((task) => task.status == TaskStatus.downloading) : null;
|
||||
final ids = _batchMode ? _selectedTaskIdsFor((task) => task.canPause) : null;
|
||||
if (_batchMode && ids!.isEmpty) return;
|
||||
unawaited(_runTaskAction(() => ref.read(tasksControllerProvider.notifier).pauseAll(ids)));
|
||||
}
|
||||
|
||||
void _resumeSelectedTasks() {
|
||||
if (_batchMode && _batchSelection.isEmpty) return;
|
||||
final ids = _batchMode
|
||||
? _selectedTaskIdsFor((task) => task.status == TaskStatus.paused || task.status == TaskStatus.failed)
|
||||
: null;
|
||||
final ids = _batchMode ? _selectedTaskIdsFor((task) => task.canResume) : null;
|
||||
if (_batchMode && ids!.isEmpty) return;
|
||||
unawaited(_runTaskAction(() => ref.read(tasksControllerProvider.notifier).resumeAll(ids)));
|
||||
}
|
||||
|
||||
List<String> _selectedTaskIdsFor(bool Function(TaskRecord task) test) {
|
||||
final tasks = ref.read(tasksControllerProvider).value?.tasks ?? const <TaskRecord>[];
|
||||
final tasks = ref.read(tasksControllerProvider).value?.allTasks ?? const <TaskRecord>[];
|
||||
return tasks.where((task) => _batchSelection.contains(task.id) && test(task)).map((task) => task.id).toList();
|
||||
}
|
||||
|
||||
@@ -479,8 +497,8 @@ class _HomePageState extends ConsumerState<HomePage> {
|
||||
}
|
||||
|
||||
List<String> _contextTaskIds(TaskRecord task) {
|
||||
final visibleIds = _filteredTasks(
|
||||
ref.read(tasksControllerProvider).value?.tasks ?? const [],
|
||||
final visibleIds = _visibleTasks(
|
||||
_filteredTasks(ref.read(tasksControllerProvider).value?.tasks ?? const []),
|
||||
).map((item) => item.id).toSet();
|
||||
return <String>{..._batchSelection.where(visibleIds.contains), task.id}.toList(growable: false);
|
||||
}
|
||||
@@ -502,10 +520,28 @@ class _HomePageState extends ConsumerState<HomePage> {
|
||||
if (ids.isEmpty) return;
|
||||
final runtime = ref.read(appRuntimeControllerProvider).value;
|
||||
final config = runtime?.downloaderConfig;
|
||||
final selected =
|
||||
ref.read(tasksControllerProvider).value?.allTasks.where((task) => ids.contains(task.id)).toList() ??
|
||||
<TaskRecord>[];
|
||||
final groupedMemberIds = {
|
||||
for (final task in selected)
|
||||
for (final member in task.members) member.id,
|
||||
};
|
||||
ids = ids.where((id) => !groupedMemberIds.contains(id)).toList();
|
||||
selected.removeWhere((task) => groupedMemberIds.contains(task.id));
|
||||
final hasGroups = selected.any((task) => task.isGroup);
|
||||
final keepFiles = await showTaskDeleteDialog(
|
||||
context,
|
||||
taskCount: ids.length,
|
||||
keepFiles: config?.extra.lastDeleteTaskKeep ?? false,
|
||||
memberCount: !hasGroups
|
||||
? null
|
||||
: selected.fold<int>(0, (total, task) => total + (task.isGroup ? task.members.length : 1)),
|
||||
// A task always downloads at least one file; records whose resource
|
||||
// has no file list (plain single-file downloads) still count as one.
|
||||
fileCount: !hasGroups
|
||||
? null
|
||||
: selected.fold<int>(0, (total, task) => total + (task.files.isEmpty ? 1 : task.files.length)),
|
||||
keepFiles: config?.extra.lastDeleteTaskKeep ?? true,
|
||||
);
|
||||
if (keepFiles == null || !mounted) return;
|
||||
|
||||
@@ -518,10 +554,11 @@ class _HomePageState extends ConsumerState<HomePage> {
|
||||
await ref.read(appRuntimeControllerProvider.notifier).reloadConfig();
|
||||
});
|
||||
if (!mounted) return;
|
||||
_batchSelection.removeAll(ids);
|
||||
_batchSelection.removeAll({...ids, ...groupedMemberIds});
|
||||
setState(() {
|
||||
_batchMode = false;
|
||||
if (_selectedTask != null && ids.contains(_selectedTask!.id)) {
|
||||
_expandedGroupIds.removeAll(ids);
|
||||
if (_selectedTask != null && (ids.contains(_selectedTask!.id) || groupedMemberIds.contains(_selectedTask!.id))) {
|
||||
_selectedTask = null;
|
||||
}
|
||||
});
|
||||
@@ -543,19 +580,7 @@ class _HomePageState extends ConsumerState<HomePage> {
|
||||
}
|
||||
}
|
||||
|
||||
void _handleAddTask() {
|
||||
if (!Util.isDesktop()) {
|
||||
context.go('/create');
|
||||
return;
|
||||
}
|
||||
unawaited(
|
||||
AppWindowLauncher.openCreateTaskWindow().then((opened) {
|
||||
if (!opened && mounted) {
|
||||
context.go('/create');
|
||||
}
|
||||
}),
|
||||
);
|
||||
}
|
||||
void _handleAddTask() => unawaited(AppRouter.openCreateTask());
|
||||
|
||||
Future<void> _runTaskAction(Future<void> Function() action) async {
|
||||
try {
|
||||
@@ -569,14 +594,8 @@ class _HomePageState extends ConsumerState<HomePage> {
|
||||
Future<void> _updateTaskUrl(String id, String url, {Map<String, String> headers = const {}}) async {
|
||||
try {
|
||||
await ref.read(tasksControllerProvider.notifier).updateUrl(id, url, headers: headers);
|
||||
final updatedTasks = ref.read(tasksControllerProvider).value?.tasks ?? const <TaskRecord>[];
|
||||
for (final task in updatedTasks) {
|
||||
if (task.id == id) {
|
||||
if (mounted) {
|
||||
setState(() => _selectedTask = task);
|
||||
}
|
||||
break;
|
||||
}
|
||||
if (mounted) {
|
||||
setState(() => _selectedTask = ref.read(tasksControllerProvider).value?.findTask(id));
|
||||
}
|
||||
} catch (error) {
|
||||
if (!mounted) rethrow;
|
||||
@@ -586,6 +605,16 @@ class _HomePageState extends ConsumerState<HomePage> {
|
||||
}
|
||||
|
||||
Future<void> _openTask(TaskRecord task) async {
|
||||
final nativeDirectory =
|
||||
!Util.isWeb() &&
|
||||
switch (defaultTargetPlatform) {
|
||||
TargetPlatform.windows || TargetPlatform.macOS || TargetPlatform.linux => true,
|
||||
_ => false,
|
||||
};
|
||||
if (task.isGroup && !nativeDirectory) {
|
||||
await browseTaskFiles(context, task, mobile: MediaQuery.sizeOf(context).width < Breakpoints.mobile);
|
||||
return;
|
||||
}
|
||||
final opened = task.isFolder
|
||||
? await FileExplorer.reveal(task.storagePath)
|
||||
: await FileExplorer.open(task.storagePath);
|
||||
@@ -674,6 +703,8 @@ class _TaskBatchRow extends StatelessWidget {
|
||||
class _MobileTasksView extends StatelessWidget {
|
||||
const _MobileTasksView({
|
||||
required this.tasks,
|
||||
required this.memberIds,
|
||||
required this.expandedGroupIds,
|
||||
required this.batchMode,
|
||||
required this.batchSelection,
|
||||
required this.onToggleSelectAll,
|
||||
@@ -695,6 +726,8 @@ class _MobileTasksView extends StatelessWidget {
|
||||
});
|
||||
|
||||
final List<TaskRecord> tasks;
|
||||
final Set<String> memberIds;
|
||||
final Set<String> expandedGroupIds;
|
||||
final bool batchMode;
|
||||
final TaskBatchSelectionController batchSelection;
|
||||
final VoidCallback onToggleSelectAll;
|
||||
@@ -760,11 +793,14 @@ class _MobileTasksView extends StatelessWidget {
|
||||
itemBuilder: (context, index) {
|
||||
final task = tasks[index];
|
||||
return TaskBatchSelectionBuilder(
|
||||
key: ValueKey(task.id),
|
||||
controller: batchSelection,
|
||||
taskId: task.id,
|
||||
visibleTaskIds: visibleTaskIds,
|
||||
builder: (context, selectedInBatch, allSelected) => TaskCard(
|
||||
task: task,
|
||||
nested: memberIds.contains(task.id),
|
||||
expanded: expandedGroupIds.contains(task.id),
|
||||
selected: false,
|
||||
batchMode: batchMode,
|
||||
selectedInBatch: selectedInBatch,
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
import 'package:flutter/material.dart' show Icons;
|
||||
import 'package:flutter/widgets.dart';
|
||||
import 'package:shadcn_flutter/shadcn_flutter.dart';
|
||||
import 'package:go_router/go_router.dart';
|
||||
|
||||
import '../../../../core/window/app_window_chrome.dart';
|
||||
@@ -28,7 +27,21 @@ class PrimaryRail extends StatelessWidget {
|
||||
children: [
|
||||
SizedBox(
|
||||
height: AppDesignTokens.contentHeaderHeight,
|
||||
child: const Center(child: GopeedAppMark(key: ValueKey('primary-rail-app-mark'))),
|
||||
child: Center(
|
||||
child: AppTooltip(
|
||||
message: context.l10n.task,
|
||||
child: Semantics(
|
||||
label: context.l10n.task,
|
||||
button: true,
|
||||
child: GhostButton(
|
||||
key: const ValueKey('primary-rail-home-button'),
|
||||
density: ButtonDensity.icon,
|
||||
onPressed: () => context.go('/'),
|
||||
child: const ExcludeSemantics(child: GopeedAppMark(key: ValueKey('primary-rail-app-mark'))),
|
||||
),
|
||||
),
|
||||
),
|
||||
),
|
||||
),
|
||||
const SizedBox(height: AppDesignTokens.space8),
|
||||
_RailItem(
|
||||
|
||||
@@ -56,7 +56,7 @@ class SettingsController extends AsyncNotifier<SettingsState> {
|
||||
..bookmarks = latest.extra.bookmarks
|
||||
..createHistory = latest.extra.createHistory
|
||||
..runAsMenubarApp = latest.extra.runAsMenubarApp
|
||||
..analyticsClientId = latest.extra.analyticsClientId;
|
||||
..deviceId = latest.extra.deviceId;
|
||||
// The backend preserves protocol-owned results and validates the whole PUT.
|
||||
await service.putConfig(config);
|
||||
final saved = await service.getConfig();
|
||||
|
||||
@@ -67,6 +67,7 @@ class _SettingsPageState extends ConsumerState<SettingsPage> {
|
||||
final _maxRunningController = TextEditingController();
|
||||
final _httpUserAgentController = TextEditingController();
|
||||
final _httpConnectionsController = TextEditingController();
|
||||
final _ftpConnectionsController = TextEditingController();
|
||||
final _hlsConnectionsController = TextEditingController();
|
||||
final _hlsMaxRetriesController = TextEditingController();
|
||||
final _btListenPortController = TextEditingController();
|
||||
@@ -101,6 +102,7 @@ class _SettingsPageState extends ConsumerState<SettingsPage> {
|
||||
_maxRunningController,
|
||||
_httpUserAgentController,
|
||||
_httpConnectionsController,
|
||||
_ftpConnectionsController,
|
||||
_hlsConnectionsController,
|
||||
_hlsMaxRetriesController,
|
||||
_btListenPortController,
|
||||
@@ -583,6 +585,23 @@ class _SettingsPageState extends ConsumerState<SettingsPage> {
|
||||
],
|
||||
),
|
||||
),
|
||||
_SettingsBlock(
|
||||
key: const ValueKey('settings-ftp-block'),
|
||||
title: 'FTP',
|
||||
child: _SettingsGroup(
|
||||
children: [
|
||||
SettingsItem(
|
||||
title: context.l10n.connections,
|
||||
child: _NumberSettingControl(
|
||||
fieldKey: const ValueKey('ftp-connections-input'),
|
||||
controller: _ftpConnectionsController,
|
||||
min: 1,
|
||||
max: _ftpMaxConnections,
|
||||
),
|
||||
),
|
||||
],
|
||||
),
|
||||
),
|
||||
_SettingsBlock(
|
||||
title: 'BitTorrent',
|
||||
child: _SettingsGroup(
|
||||
@@ -1010,6 +1029,7 @@ class _SettingsPageState extends ConsumerState<SettingsPage> {
|
||||
_maxRunningController.text = _config!.maxRunning.clamp(1, 256).toString();
|
||||
_httpUserAgentController.text = _config!.protocolConfig.http.userAgent;
|
||||
_httpConnectionsController.text = _config!.protocolConfig.http.connections.clamp(1, 256).toString();
|
||||
_ftpConnectionsController.text = _config!.protocolConfig.ftp.connections.clamp(1, _ftpMaxConnections).toString();
|
||||
_hlsConnectionsController.text = _config!.protocolConfig.hls.segmentConnections.clamp(1, 256).toString();
|
||||
_hlsMaxRetriesController.text = _config!.protocolConfig.hls.maxRetries.clamp(0, 30).toString();
|
||||
_btListenPortController.text = _config!.protocolConfig.bt.listenPort.clamp(0, 65535).toString();
|
||||
@@ -1113,6 +1133,12 @@ class _SettingsPageState extends ConsumerState<SettingsPage> {
|
||||
min: 1,
|
||||
max: 256,
|
||||
);
|
||||
config.protocolConfig.ftp.connections = _boundedInt(
|
||||
_ftpConnectionsController,
|
||||
fallback: config.protocolConfig.ftp.connections,
|
||||
min: 1,
|
||||
max: _ftpMaxConnections,
|
||||
);
|
||||
config.protocolConfig.hls.segmentConnections = _boundedInt(
|
||||
_hlsConnectionsController,
|
||||
fallback: config.protocolConfig.hls.segmentConnections,
|
||||
@@ -2245,3 +2271,7 @@ class _ErrorState extends StatelessWidget {
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// The highest number of parallel FTP logins offered, the same limit as on the create task page. FTP servers
|
||||
/// usually allow only a few logins per user.
|
||||
const _ftpMaxConnections = 32;
|
||||
@@ -0,0 +1,166 @@
|
||||
import 'package:path/path.dart' as path;
|
||||
|
||||
import '../../../api/model/create_task_batch.dart';
|
||||
import '../../../api/model/options.dart';
|
||||
import '../../../api/model/request.dart';
|
||||
import '../../../api/model/resource.dart';
|
||||
import '../../../api/model/resolve_result.dart';
|
||||
import '../../../api/model/resolve_task.dart';
|
||||
import '../../../core/capabilities/gopeed_capability.dart';
|
||||
|
||||
/// IDs identify sources within one preview and are never sent as group IDs.
|
||||
class PendingDownload {
|
||||
PendingDownload({required this.id, required this.request, required this.options});
|
||||
|
||||
final String id;
|
||||
final Request request;
|
||||
final Options options;
|
||||
ResolveResult? result;
|
||||
String? resolveError;
|
||||
String? createError;
|
||||
final Set<int> selected = {};
|
||||
bool created = false;
|
||||
bool resolving = false;
|
||||
|
||||
bool get loading => result == null && resolveError == null;
|
||||
bool get isGroup => result != null && result!.id.isEmpty && result!.res.name.trim().isNotEmpty && files.length > 1;
|
||||
bool get isFtp => RegExp(r'^(ftps?|ftpes):', caseSensitive: false).hasMatch(request.url.trim());
|
||||
bool get isBt {
|
||||
if (result?.id.isEmpty == true) return false;
|
||||
final url = request.url.trim().toLowerCase();
|
||||
if (url.startsWith('http:') || url.startsWith('https:') || isFtp) return false;
|
||||
return url.startsWith('magnet:') ||
|
||||
url.startsWith('data:application/x-bittorrent;base64,') ||
|
||||
url.split(RegExp(r'[?#]')).first.endsWith('.torrent');
|
||||
}
|
||||
|
||||
String get name {
|
||||
if (!isBt &&
|
||||
(!isFtp || result?.res.name.trim().isEmpty == true) &&
|
||||
result?.id.isNotEmpty == true &&
|
||||
files.length == 1 &&
|
||||
files.single.name.isNotEmpty) {
|
||||
return files.single.name;
|
||||
}
|
||||
if (result?.res.name.trim().isNotEmpty == true) return result!.res.name;
|
||||
return request.url;
|
||||
}
|
||||
|
||||
List<FileInfo> get files => result?.res.files ?? const [];
|
||||
bool get hasSelection => !created && result != null && selected.isNotEmpty;
|
||||
|
||||
Future<void> resolve(GopeedService service) async {
|
||||
if (result != null || resolving) return;
|
||||
resolving = true;
|
||||
resolveError = null;
|
||||
try {
|
||||
result = await service.resolve(ResolveTask(req: request));
|
||||
selected.addAll(List.generate(files.isEmpty ? 1 : files.length, (index) => index));
|
||||
} catch (error) {
|
||||
resolveError = error.toString();
|
||||
} finally {
|
||||
resolving = false;
|
||||
}
|
||||
}
|
||||
|
||||
List<CreateTaskBatchItem> creationItems() {
|
||||
final indexes = selected.toList()..sort();
|
||||
if (result!.id.isNotEmpty || files.isEmpty) {
|
||||
final opts = Options.fromJson(options.toJson());
|
||||
if (files.isNotEmpty) opts.selectFiles = indexes;
|
||||
if (indexes.length > 1) _omitChecksum(opts);
|
||||
return [
|
||||
CreateTaskBatchItem(
|
||||
rid: result!.id.isEmpty ? null : result!.id,
|
||||
req: result!.id.isEmpty ? request : null,
|
||||
opts: opts,
|
||||
),
|
||||
];
|
||||
}
|
||||
final group = isGroup;
|
||||
final resourceName = result!.res.name;
|
||||
final parentSegments = [
|
||||
if (!group && options.path.isNotEmpty) options.path,
|
||||
if (!group && resourceName.trim().isNotEmpty) resourceName,
|
||||
];
|
||||
final skipAutoCategorize = options.skipAutoCategorize || resourceName.isNotEmpty || files.length > 1;
|
||||
final httpUrl = RegExp(r'^https?:', caseSensitive: false);
|
||||
return indexes.map((index) {
|
||||
final file = files[index];
|
||||
final member = Options.fromJson(options.toJson());
|
||||
member.name = files.length == 1 && options.name.isNotEmpty ? options.name : file.name;
|
||||
final segments = [...parentSegments, if (file.path.isNotEmpty) file.path];
|
||||
member.path = segments.isEmpty ? '' : path.joinAll(segments);
|
||||
// The selected parent directory is persisted separately from members.
|
||||
member.asDefaultPath = false;
|
||||
member.skipAutoCategorize = skipAutoCategorize;
|
||||
final req = file.req ?? request;
|
||||
member.size = httpUrl.hasMatch(req.url) ? file.size : 0;
|
||||
if (RegExp(r'^(ftps?|ftpes):', caseSensitive: false).hasMatch(req.url)) {
|
||||
final extra = OptsExtraFtp.fromJson(Map<String, dynamic>.from(member.extra as Map? ?? const {}));
|
||||
extra.connections = extra.connections.clamp(0, 32);
|
||||
member.extra = extra.toJson();
|
||||
}
|
||||
if (indexes.length > 1) _omitChecksum(member);
|
||||
return CreateTaskBatchItem(req: req, opts: member);
|
||||
}).toList();
|
||||
}
|
||||
|
||||
void _omitChecksum(Options opts) {
|
||||
if (opts.extra is Map) opts.extra = Map<String, dynamic>.from(opts.extra! as Map)..remove('checksum');
|
||||
}
|
||||
}
|
||||
|
||||
/// A successful response completes the entire batch, including partial success.
|
||||
class DownloadSubmission {
|
||||
DownloadSubmission({required this.sources, required this.request});
|
||||
|
||||
final List<PendingDownload> sources;
|
||||
final CreateTaskBatch request;
|
||||
|
||||
Future<void> submit(GopeedService service) async {
|
||||
for (final source in sources) {
|
||||
source.createError = null;
|
||||
}
|
||||
try {
|
||||
await service.createTaskBatch(request);
|
||||
for (final source in sources) {
|
||||
source.created = true;
|
||||
}
|
||||
} catch (error) {
|
||||
for (final source in sources) {
|
||||
source.createError = error.toString();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
List<DownloadSubmission> planDownloads(List<PendingDownload> sources) {
|
||||
final groups = <DownloadSubmission>[];
|
||||
final independent = <PendingDownload>[];
|
||||
for (final source in sources.where((source) => source.hasSelection)) {
|
||||
if (source.isGroup) {
|
||||
groups.add(
|
||||
DownloadSubmission(
|
||||
sources: [source],
|
||||
request: CreateTaskBatch(
|
||||
group: true,
|
||||
groupName: source.options.name.isNotEmpty ? source.options.name : source.result!.res.name.trim(),
|
||||
opts: source.options,
|
||||
reqs: source.creationItems(),
|
||||
),
|
||||
),
|
||||
);
|
||||
} else {
|
||||
independent.add(source);
|
||||
}
|
||||
}
|
||||
return [
|
||||
...groups,
|
||||
if (independent.isNotEmpty)
|
||||
DownloadSubmission(
|
||||
sources: independent,
|
||||
request: CreateTaskBatch(reqs: [for (final source in independent) ...source.creationItems()]),
|
||||
),
|
||||
];
|
||||
}
|
||||
@@ -23,6 +23,10 @@ class TasksState {
|
||||
final int uploadSpeedBytesPerSecond;
|
||||
final Object? lastError;
|
||||
|
||||
Iterable<TaskRecord> get allTasks => tasks.expand((task) => [task, ...task.members]);
|
||||
|
||||
TaskRecord? findTask(String id) => allTasks.where((task) => task.id == id).firstOrNull;
|
||||
|
||||
TasksState copyWith({
|
||||
List<TaskRecord>? tasks,
|
||||
int? downloadSpeedBytesPerSecond,
|
||||
@@ -122,13 +126,14 @@ class TasksController extends AsyncNotifier<TasksState> {
|
||||
Future<TasksState> _fetch() async {
|
||||
final tasks = await ref.read(gopeedServiceProvider).getTasks(api_model.Status.values);
|
||||
final transferSpeeds = aggregateTaskTransferSpeeds(tasks);
|
||||
tasks.sort((a, b) {
|
||||
if (a.status == api_model.Status.running && b.status != api_model.Status.running) return -1;
|
||||
if (a.status != api_model.Status.running && b.status == api_model.Status.running) return 1;
|
||||
return b.updatedAt.compareTo(a.updatedAt);
|
||||
});
|
||||
return TasksState(
|
||||
tasks: tasks.map(TaskRecord.fromApi).toList(growable: false),
|
||||
tasks: taskRecordsFromApi(tasks)
|
||||
..sort((a, b) {
|
||||
final aRunning = a.status == TaskStatus.downloading && !a.waiting;
|
||||
final bRunning = b.status == TaskStatus.downloading && !b.waiting;
|
||||
if (aRunning != bRunning) return aRunning ? -1 : 1;
|
||||
return (b.updatedAt ?? b.createdAt!).compareTo(a.updatedAt ?? a.createdAt!);
|
||||
}),
|
||||
downloadSpeedBytesPerSecond: transferSpeeds.downloadBytesPerSecond,
|
||||
uploadSpeedBytesPerSecond: transferSpeeds.uploadBytesPerSecond,
|
||||
);
|
||||
|
||||
@@ -42,6 +42,10 @@ class TaskFileNode {
|
||||
required this.sizeBytes,
|
||||
this.downloadedBytes,
|
||||
this.resourceIndex,
|
||||
this.status,
|
||||
this.taskId,
|
||||
this.accessPath,
|
||||
this.localPath,
|
||||
});
|
||||
|
||||
final int? resourceIndex;
|
||||
@@ -49,6 +53,10 @@ class TaskFileNode {
|
||||
final String name;
|
||||
final int sizeBytes;
|
||||
final int? downloadedBytes;
|
||||
final TaskStatus? status;
|
||||
final String? taskId;
|
||||
final String? accessPath;
|
||||
final String? localPath;
|
||||
|
||||
double? get progress {
|
||||
final downloaded = downloadedBytes;
|
||||
@@ -68,6 +76,8 @@ class TaskRecord {
|
||||
required this.files,
|
||||
required this.uploading,
|
||||
this.isFolder = false,
|
||||
this.isGroup = false,
|
||||
this.members = const [],
|
||||
this.protocol,
|
||||
this.requestHeaders = const {},
|
||||
this.total,
|
||||
@@ -84,6 +94,8 @@ class TaskRecord {
|
||||
this.totalBytes,
|
||||
this.speedBytes,
|
||||
this.uploadedBytes,
|
||||
this.uploadSpeedBytes,
|
||||
this.updatedAt,
|
||||
this.remainingSeconds,
|
||||
this.waiting = false,
|
||||
}) : assert(!uploading || uploadSpeed != null);
|
||||
@@ -108,16 +120,31 @@ class TaskRecord {
|
||||
final int? totalBytes;
|
||||
final int? speedBytes;
|
||||
final int? uploadedBytes;
|
||||
final int? uploadSpeedBytes;
|
||||
final DateTime? updatedAt;
|
||||
final int? remainingSeconds;
|
||||
final bool waiting;
|
||||
final List<TaskFileNode> files;
|
||||
final bool uploading;
|
||||
final bool isFolder;
|
||||
final bool isGroup;
|
||||
final List<TaskRecord> members;
|
||||
final api_task.Protocol? protocol;
|
||||
final Map<String, String> requestHeaders;
|
||||
|
||||
bool get canUpdateUrl =>
|
||||
protocol == api_task.Protocol.http && (status == TaskStatus.paused || status == TaskStatus.failed);
|
||||
!isGroup && protocol == api_task.Protocol.http && (status == TaskStatus.paused || status == TaskStatus.failed);
|
||||
|
||||
bool get canPause => isGroup ? members.any((task) => task.canPause) : status == TaskStatus.downloading;
|
||||
bool get canResume =>
|
||||
isGroup ? members.any((task) => task.canResume) : status == TaskStatus.paused || status == TaskStatus.failed;
|
||||
|
||||
bool matchesSearch(String query) =>
|
||||
query.isEmpty ||
|
||||
name.toLowerCase().contains(query) ||
|
||||
url.toLowerCase().contains(query) ||
|
||||
files.any((file) => '${file.path}/${file.name}'.toLowerCase().contains(query)) ||
|
||||
members.any((member) => member.matchesSearch(query));
|
||||
|
||||
bool get isIndeterminate => status == TaskStatus.downloading && !waiting && total == null;
|
||||
|
||||
@@ -147,6 +174,7 @@ class TaskRecord {
|
||||
remainingSeconds: _remainingSeconds(task, totalBytes, downloadedBytes),
|
||||
downloadDuration: task.progress.used > 0 ? Duration(microseconds: (task.progress.used + 999) ~/ 1000) : null,
|
||||
createdAt: task.createdAt,
|
||||
updatedAt: task.updatedAt,
|
||||
doneAt: task.doneAt,
|
||||
waiting: task.status == api_task.Status.wait,
|
||||
progress: progress,
|
||||
@@ -156,6 +184,7 @@ class TaskRecord {
|
||||
totalBytes: totalBytes > 0 ? totalBytes : null,
|
||||
speedBytes: task.progress.speed,
|
||||
uploadedBytes: task.progress.uploaded,
|
||||
uploadSpeedBytes: task.progress.uploadSpeed,
|
||||
files: _fileNodes(task.meta.res?.files, task.meta.opts.selectFiles),
|
||||
uploading: task.uploading,
|
||||
isFolder: task.meta.res?.name.isNotEmpty ?? false,
|
||||
@@ -165,6 +194,186 @@ class TaskRecord {
|
||||
}
|
||||
}
|
||||
|
||||
List<TaskRecord> taskRecordsFromApi(List<api_task.Task> tasks) {
|
||||
final members = <String, List<TaskRecord>>{};
|
||||
final metadata = <String, api_task.TaskGroup>{};
|
||||
final records = <String, TaskRecord>{};
|
||||
for (final task in tasks) {
|
||||
final record = TaskRecord.fromApi(task);
|
||||
records[task.id] = record;
|
||||
if (task.parentId.isNotEmpty) {
|
||||
members.putIfAbsent(task.parentId, () => []).add(record);
|
||||
if (task.group != null) metadata[task.parentId] = task.group!;
|
||||
}
|
||||
}
|
||||
final emitted = <String>{};
|
||||
return [
|
||||
for (final task in tasks)
|
||||
if (task.parentId.isEmpty || !metadata.containsKey(task.parentId))
|
||||
records[task.id]!
|
||||
else if (emitted.add(task.parentId))
|
||||
_groupRecord(metadata[task.parentId]!, members[task.parentId]!),
|
||||
];
|
||||
}
|
||||
|
||||
TaskRecord _groupRecord(api_task.TaskGroup group, List<TaskRecord> members) {
|
||||
final running = members.any((task) => task.status == TaskStatus.downloading && !task.waiting);
|
||||
final failed = members.any((task) => task.status == TaskStatus.failed);
|
||||
final paused = members.isEmpty || members.any((task) => task.status == TaskStatus.paused);
|
||||
final completed = members.isNotEmpty && members.every((task) => task.status == TaskStatus.completed);
|
||||
final status = running
|
||||
? TaskStatus.downloading
|
||||
: failed
|
||||
? TaskStatus.failed
|
||||
: paused
|
||||
? TaskStatus.paused
|
||||
: completed
|
||||
? TaskStatus.completed
|
||||
: TaskStatus.downloading;
|
||||
final downloaded = members.fold(0, (total, task) => total + (task.downloadedBytes ?? 0));
|
||||
final total = members.every((task) => task.totalBytes != null || task.status == TaskStatus.completed)
|
||||
? members.fold(0, (total, task) => total + (task.totalBytes ?? 0))
|
||||
: null;
|
||||
final speed = members.fold(
|
||||
0,
|
||||
(total, task) => total + (task.status == TaskStatus.downloading && !task.waiting ? task.speedBytes ?? 0 : 0),
|
||||
);
|
||||
// The backend freezes a finished single task's speed at bytes over active
|
||||
// time (markTaskDone); a finished group reports the same ratio across its
|
||||
// members instead of collapsing to "0 B/s" when the last member completes.
|
||||
final reportedSpeed = speed > 0 ? speed : (completed ? _averageSpeed(members) ?? 0 : 0);
|
||||
final uploadSpeed = members.fold(0, (total, task) => total + (task.uploading ? task.uploadSpeedBytes ?? 0 : 0));
|
||||
final uploading = members.any((task) => task.uploading);
|
||||
final uploaded = members.fold(0, (total, task) => total + (task.uploadedBytes ?? 0));
|
||||
final pathContext = path.Context(
|
||||
style: group.path.contains('\\') || members.any((task) => task.storagePath.contains('\\'))
|
||||
? path.Style.windows
|
||||
: path.Style.posix,
|
||||
);
|
||||
final root = group.path;
|
||||
final files = <TaskFileNode>[];
|
||||
for (final member in members) {
|
||||
final sourceFiles = member.isFolder
|
||||
? member.files
|
||||
: [TaskFileNode(path: '', name: member.name, sizeBytes: member.totalBytes ?? 0)];
|
||||
// Folder members do not carry per-file progress from the backend; spread
|
||||
// the member's overall ratio over its files so the tree keeps moving
|
||||
// instead of showing flat zeros.
|
||||
final folderTotal = member.isFolder ? member.files.fold<int>(0, (total, file) => total + file.sizeBytes) : 0;
|
||||
final folderRatio = folderTotal > 0 && member.downloadedBytes != null
|
||||
? (member.downloadedBytes! / folderTotal).clamp(0.0, 1.0)
|
||||
: null;
|
||||
for (final file in sourceFiles) {
|
||||
final fullPath = member.isFolder
|
||||
? pathContext.join(member.storagePath, file.path, file.name)
|
||||
: member.storagePath;
|
||||
final relative = pathContext.relative(fullPath, from: root).replaceAll('\\', '/');
|
||||
final String nodePath;
|
||||
final String nodeName;
|
||||
if (pathContext.isAbsolute(relative) || relative == '..' || relative.startsWith('../')) {
|
||||
// Keep unexpected stored paths within the displayed tree.
|
||||
if (member.isFolder) {
|
||||
nodePath = path.posix.join(pathContext.basename(member.storagePath), file.path.replaceAll('\\', '/'));
|
||||
nodeName = file.name;
|
||||
} else {
|
||||
nodePath = '';
|
||||
nodeName = member.name;
|
||||
}
|
||||
} else {
|
||||
nodePath = path.posix.dirname(relative) == '.' ? '' : path.posix.dirname(relative);
|
||||
nodeName = path.posix.basename(relative);
|
||||
}
|
||||
files.add(
|
||||
TaskFileNode(
|
||||
resourceIndex: files.length,
|
||||
path: nodePath,
|
||||
name: nodeName,
|
||||
sizeBytes: file.sizeBytes,
|
||||
downloadedBytes: member.isFolder
|
||||
? folderRatio != null && file.sizeBytes > 0
|
||||
? (file.sizeBytes * folderRatio).round()
|
||||
: null
|
||||
: member.downloadedBytes,
|
||||
status: member.status,
|
||||
taskId: member.id,
|
||||
localPath: fullPath,
|
||||
accessPath: member.isFolder
|
||||
? path.posix.join(file.path.replaceAll('\\', '/'), file.name)
|
||||
: pathContext.relative(fullPath, from: pathContext.dirname(member.storagePath)).replaceAll('\\', '/'),
|
||||
),
|
||||
);
|
||||
}
|
||||
}
|
||||
var updatedAt = group.createdAt;
|
||||
DateTime? doneAt;
|
||||
for (final member in members) {
|
||||
final update = member.updatedAt;
|
||||
if (update != null && update.isAfter(updatedAt)) updatedAt = update;
|
||||
if (completed) {
|
||||
final done = member.doneAt;
|
||||
if (done != null && (doneAt == null || done.isAfter(doneAt))) doneAt = done;
|
||||
}
|
||||
}
|
||||
// The folder card answers "how long did this download take end to end", so
|
||||
// a completed group reports its wall-clock span; single tasks keep their
|
||||
// active-time semantics.
|
||||
final createdAt = group.createdAt;
|
||||
final downloadDuration = doneAt != null && doneAt.isAfter(createdAt) ? doneAt.difference(createdAt) : null;
|
||||
return TaskRecord(
|
||||
id: group.id,
|
||||
name: group.name,
|
||||
status: status,
|
||||
downloaded: _formatBytes(downloaded),
|
||||
total: total == null ? null : _formatBytes(total),
|
||||
downloadedBytes: downloaded,
|
||||
totalBytes: total,
|
||||
speed: TransferRateFormatter.format(reportedSpeed).text,
|
||||
speedBytes: reportedSpeed,
|
||||
uploadSpeed: uploading ? TransferRateFormatter.format(uploadSpeed).text : null,
|
||||
uploadedBytes: uploaded,
|
||||
uploadSpeedBytes: uploadSpeed,
|
||||
uploading: uploading,
|
||||
progress: completed
|
||||
? 1
|
||||
: total != null && total > 0
|
||||
? (downloaded / total).clamp(0.0, 1.0)
|
||||
: null,
|
||||
remainingSeconds: total != null && speed > 0 && total > downloaded ? ((total - downloaded) / speed).ceil() : null,
|
||||
waiting: status == TaskStatus.downloading && !running,
|
||||
downloadDuration: downloadDuration,
|
||||
url: '',
|
||||
storagePath: root,
|
||||
files: files,
|
||||
isFolder: true,
|
||||
isGroup: true,
|
||||
members: members,
|
||||
createdAt: group.createdAt,
|
||||
updatedAt: updatedAt,
|
||||
doneAt: doneAt,
|
||||
);
|
||||
}
|
||||
|
||||
// The combined average rate of a set of members: downloaded bytes over active
|
||||
// download time, the same ratio the backend freezes into a finished single
|
||||
// task's progress. Members without a measurable duration are skipped because
|
||||
// they would inflate the rate.
|
||||
int? _averageSpeed(List<TaskRecord> members) {
|
||||
var downloaded = 0;
|
||||
var microseconds = 0;
|
||||
for (final member in members) {
|
||||
final duration = member.downloadDuration;
|
||||
if (duration == null || duration.inMicroseconds <= 0) {
|
||||
continue;
|
||||
}
|
||||
downloaded += member.downloadedBytes ?? 0;
|
||||
microseconds += duration.inMicroseconds;
|
||||
}
|
||||
if (microseconds <= 0) {
|
||||
return null;
|
||||
}
|
||||
return (downloaded * Duration.microsecondsPerSecond / microseconds).round();
|
||||
}
|
||||
|
||||
Map<String, String> _requestHeaders(Object? extra) {
|
||||
if (extra is! Map) return const {};
|
||||
final header = extra['header'];
|
||||
@@ -304,7 +513,7 @@ TaskAssetType _assetType(String name, {required bool isFolder, required api_task
|
||||
return switch (protocol) {
|
||||
api_task.Protocol.bt => TaskAssetType.torrent,
|
||||
api_task.Protocol.ed2k => TaskAssetType.ed2k,
|
||||
api_task.Protocol.hls || api_task.Protocol.http || null => TaskAssetType.file,
|
||||
api_task.Protocol.hls || api_task.Protocol.http || api_task.Protocol.ftp || null => TaskAssetType.file,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -19,8 +19,6 @@ import '../../../../api/model/create_task_batch.dart';
|
||||
import '../../../../api/model/downloader_config.dart';
|
||||
import '../../../../api/model/options.dart' as api_options;
|
||||
import '../../../../api/model/request.dart';
|
||||
import '../../../../api/model/resolve_result.dart';
|
||||
import '../../../../api/model/resolve_task.dart';
|
||||
import '../../../../core/capabilities/app_capabilities.dart';
|
||||
import '../../../../core/utils/breakpoints.dart';
|
||||
import '../../../../core/utils/text_wrap.dart';
|
||||
@@ -39,16 +37,19 @@ import '../../../../shared/widgets/app_toast.dart';
|
||||
import '../../../../l10n/l10n.dart';
|
||||
import '../../../../util/util.dart';
|
||||
import '../../application/pending_create_task.dart';
|
||||
import '../../application/pending_download.dart';
|
||||
import '../../application/task_list_navigation.dart';
|
||||
import '../widgets/history_search_field.dart';
|
||||
import '../widgets/create_task_action_buttons.dart';
|
||||
import '../widgets/resolve_file_tree.dart';
|
||||
import '../widgets/resolve_download_dialog.dart';
|
||||
|
||||
class CreateTaskWindowPage extends ConsumerStatefulWidget {
|
||||
const CreateTaskWindowPage({super.key, this.windowController, this.initialTask});
|
||||
const CreateTaskWindowPage({super.key, this.windowController, this.initialTask, this.onClose, this.modal = false});
|
||||
|
||||
final multi_window.WindowController? windowController;
|
||||
final CreateTask? initialTask;
|
||||
final ValueChanged<bool>? onClose;
|
||||
final bool modal;
|
||||
|
||||
@override
|
||||
ConsumerState<CreateTaskWindowPage> createState() => _CreateTaskWindowPageState();
|
||||
@@ -82,6 +83,9 @@ class _CreateTaskWindowPageState extends ConsumerState<CreateTaskWindowPage> {
|
||||
bool? _deleteTorrentAfterDownload;
|
||||
bool? _autoExtract;
|
||||
bool _deleteAfterExtract = false;
|
||||
String _ftpTls = _ftpTlsNone;
|
||||
bool _allLinksFtp = false;
|
||||
bool _hasPlainFtpLink = false;
|
||||
List<DownloadCategory> _downloadCategories = const [];
|
||||
|
||||
bool _showAdvanced = false;
|
||||
@@ -89,6 +93,8 @@ class _CreateTaskWindowPageState extends ConsumerState<CreateTaskWindowPage> {
|
||||
bool _directDownload = false;
|
||||
int _protocolTab = 0;
|
||||
bool _creating = false;
|
||||
final Map<String, int> _completedSubmissionCounts = {};
|
||||
int _createSession = 0;
|
||||
bool _draggingTorrent = false;
|
||||
bool _asDefaultPath = false;
|
||||
String _configuredDownloadDirectory = '';
|
||||
@@ -129,7 +135,7 @@ class _CreateTaskWindowPageState extends ConsumerState<CreateTaskWindowPage> {
|
||||
if (widget.initialTask == null) {
|
||||
ref.read(pendingCreateTaskProvider.notifier).clear();
|
||||
}
|
||||
if (widget.windowController == null && widget.initialTask == null) {
|
||||
if (widget.windowController == null) {
|
||||
// Navigating to /create again reuses this page, so consume subsequent
|
||||
// browser/share requests as well as the initial pending task.
|
||||
ref.listenManual(pendingCreateTaskProvider, (_, next) {
|
||||
@@ -220,12 +226,33 @@ class _CreateTaskWindowPageState extends ConsumerState<CreateTaskWindowPage> {
|
||||
}
|
||||
}
|
||||
_recognizeMagnetUri(urlText.trim());
|
||||
_refreshFtpLinks();
|
||||
}
|
||||
|
||||
/// Shows the FTP options (the 32-connection limit and the TLS select) only
|
||||
/// while the input contains FTP links.
|
||||
void _refreshFtpLinks() {
|
||||
final links = Util.textToLines(_urlController.text).map((line) => line.trim()).where((line) => line.isNotEmpty);
|
||||
final allFtp = links.isNotEmpty && links.every((link) => _parseProtocol(link) == _TaskProtocol.ftp);
|
||||
final hasPlainFtp = links.any(_isPlainFtpUrl);
|
||||
if (allFtp == _allLinksFtp && hasPlainFtp == _hasPlainFtpLink) return;
|
||||
if (!mounted) {
|
||||
_allLinksFtp = allFtp;
|
||||
_hasPlainFtpLink = hasPlainFtp;
|
||||
return;
|
||||
}
|
||||
setState(() {
|
||||
_allLinksFtp = allFtp;
|
||||
_hasPlainFtpLink = hasPlainFtp;
|
||||
});
|
||||
}
|
||||
|
||||
void _applyInitialTask(CreateTask? task) {
|
||||
if (task == null || !mounted) return;
|
||||
final req = task.req;
|
||||
final opts = task.opts ?? api_options.Options();
|
||||
_createSession++;
|
||||
_completedSubmissionCounts.clear();
|
||||
setState(() {
|
||||
_initialSize = 0;
|
||||
_checksumAlgorithm = 'md5';
|
||||
@@ -237,6 +264,7 @@ class _CreateTaskWindowPageState extends ConsumerState<CreateTaskWindowPage> {
|
||||
_httpBody = '';
|
||||
_replaceHttpHeaders(const {'User-Agent': '', 'Cookie': '', 'Referer': ''});
|
||||
_trackersController.clear();
|
||||
_ftpTls = _ftpTlsNone;
|
||||
_protocolTab = 0;
|
||||
_urlController.text = req!.url;
|
||||
_initialRawUrl = req.rawUrl;
|
||||
@@ -261,6 +289,14 @@ class _CreateTaskWindowPageState extends ConsumerState<CreateTaskWindowPage> {
|
||||
_trackersController.text = extra.trackers.join('\n');
|
||||
}
|
||||
break;
|
||||
case _TaskProtocol.ftp:
|
||||
if (req.extra is Map) {
|
||||
final tls = ReqExtraFtp.fromJson(Map<String, dynamic>.from(req.extra! as Map)).tls;
|
||||
if (_ftpTlsModes.contains(tls)) {
|
||||
_ftpTls = tls;
|
||||
}
|
||||
}
|
||||
break;
|
||||
case _TaskProtocol.ed2k:
|
||||
case null:
|
||||
break;
|
||||
@@ -337,7 +373,9 @@ class _CreateTaskWindowPageState extends ConsumerState<CreateTaskWindowPage> {
|
||||
child: Padding(
|
||||
key: const ValueKey('create-task-safe-content'),
|
||||
padding: EdgeInsets.only(
|
||||
top: AppWindowChrome.reservesHeaderInset
|
||||
top: widget.modal
|
||||
? 0
|
||||
: AppWindowChrome.reservesHeaderInset
|
||||
? AppDesignTokens.windowHeaderHeight
|
||||
: MediaQuery.paddingOf(context).top,
|
||||
),
|
||||
@@ -437,10 +475,22 @@ class _CreateTaskWindowPageState extends ConsumerState<CreateTaskWindowPage> {
|
||||
fieldKey: const ValueKey('create-task-connections-input'),
|
||||
controller: _connectionsController,
|
||||
min: 1,
|
||||
max: 256,
|
||||
max: _allLinksFtp ? _ftpMaxConnections : 256,
|
||||
hintText: context.l10n.enterCount,
|
||||
),
|
||||
),
|
||||
if (_hasPlainFtpLink) ...[
|
||||
const SizedBox(height: 16),
|
||||
AppFormRow(
|
||||
direction: stacked ? Axis.vertical : Axis.horizontal,
|
||||
label: context.l10n.ftpTls,
|
||||
child: _FtpTlsSelect(
|
||||
key: const ValueKey('create-task-ftp-tls'),
|
||||
value: _ftpTls,
|
||||
onChanged: (value) => setState(() => _ftpTls = value),
|
||||
),
|
||||
),
|
||||
],
|
||||
const SizedBox(height: 16),
|
||||
AppFormRow(
|
||||
direction: stacked ? Axis.vertical : Axis.horizontal,
|
||||
@@ -729,7 +779,8 @@ class _CreateTaskWindowPageState extends ConsumerState<CreateTaskWindowPage> {
|
||||
),
|
||||
),
|
||||
],
|
||||
if (_inputUrls().length <= 1) ...[
|
||||
// Checksum verification is HTTP only.
|
||||
if (_inputUrls().length <= 1 && !_allLinksFtp) ...[
|
||||
const SizedBox(height: 14),
|
||||
AppFormPair(
|
||||
direction: stacked ? Axis.vertical : Axis.horizontal,
|
||||
@@ -820,6 +871,10 @@ class _CreateTaskWindowPageState extends ConsumerState<CreateTaskWindowPage> {
|
||||
}
|
||||
|
||||
Future<void> _closeWindow() async {
|
||||
if (widget.onClose != null) {
|
||||
widget.onClose!(false);
|
||||
return;
|
||||
}
|
||||
if (widget.windowController == null && mounted) {
|
||||
context.go('/');
|
||||
return;
|
||||
@@ -827,7 +882,24 @@ class _CreateTaskWindowPageState extends ConsumerState<CreateTaskWindowPage> {
|
||||
await windowManager.close();
|
||||
}
|
||||
|
||||
Future<void> _finishCreation() async {
|
||||
Future<void> _finishCreation(int session) async {
|
||||
if (!mounted || session != _createSession) return;
|
||||
if (_asDefaultPath && _directoryManuallySet && _directoryController.text.trim().isNotEmpty) {
|
||||
try {
|
||||
final service = ref.read(gopeedServiceProvider);
|
||||
final config = await service.getConfig();
|
||||
if (!mounted || session != _createSession) return;
|
||||
config.downloadDir = _directoryController.text.trim();
|
||||
await service.putConfig(config);
|
||||
} catch (error) {
|
||||
if (mounted && session == _createSession) _showToast(error.toString());
|
||||
}
|
||||
}
|
||||
if (!mounted || session != _createSession) return;
|
||||
if (widget.onClose != null) {
|
||||
widget.onClose!(true);
|
||||
return;
|
||||
}
|
||||
if (widget.windowController != null) {
|
||||
// Navigation is best-effort after a successful submission. A navigation
|
||||
// failure must not leave a completed form available for resubmission.
|
||||
@@ -839,6 +911,7 @@ class _CreateTaskWindowPageState extends ConsumerState<CreateTaskWindowPage> {
|
||||
} else {
|
||||
ref.read(taskListNavigationProvider.notifier).showDownloading();
|
||||
}
|
||||
if (!mounted || session != _createSession) return;
|
||||
await _closeWindow();
|
||||
}
|
||||
|
||||
@@ -945,50 +1018,74 @@ class _CreateTaskWindowPageState extends ConsumerState<CreateTaskWindowPage> {
|
||||
_showToast(context.l10n.downloadLinkValid);
|
||||
return;
|
||||
}
|
||||
if (!_validateAdvancedOptions()) {
|
||||
final occurrences = <String, int>{};
|
||||
final pendingUrls = <String>[];
|
||||
for (final url in urls) {
|
||||
final occurrence = (occurrences[url] ?? 0) + 1;
|
||||
occurrences[url] = occurrence;
|
||||
if (occurrence > (_completedSubmissionCounts[url] ?? 0)) {
|
||||
pendingUrls.add(url);
|
||||
}
|
||||
}
|
||||
if (pendingUrls.isNotEmpty && !_validateAdvancedOptions()) {
|
||||
return;
|
||||
}
|
||||
final session = _createSession;
|
||||
|
||||
setState(() {
|
||||
_creating = true;
|
||||
_draggingTorrent = false;
|
||||
});
|
||||
try {
|
||||
await _saveCreateHistory(urls);
|
||||
final isDirect = _directDownload || urls.length > 1;
|
||||
if (isDirect) {
|
||||
await Future.wait(
|
||||
urls.map((url) {
|
||||
return ref
|
||||
.read(gopeedServiceProvider)
|
||||
.createTask(
|
||||
CreateTask(
|
||||
req: _buildRequest(url, protocolUrl: _protocolUrlFor(url)),
|
||||
opts: _buildOptions(name: urls.length > 1 ? '' : _renameController.text.trim()),
|
||||
),
|
||||
);
|
||||
}),
|
||||
unawaited(
|
||||
_saveCreateHistory(urls).catchError((Object error) {
|
||||
if (mounted && session == _createSession) _showToast(error.toString());
|
||||
}),
|
||||
);
|
||||
if (pendingUrls.isEmpty) {
|
||||
await _finishCreation(session);
|
||||
return;
|
||||
}
|
||||
final sources = [
|
||||
for (final entry in pendingUrls.indexed)
|
||||
PendingDownload(
|
||||
id: 'source-${entry.$1}',
|
||||
request: _buildRequest(entry.$2, protocolUrl: _protocolUrlFor(entry.$2)),
|
||||
options: _buildOptions(
|
||||
url: _protocolUrlFor(entry.$2),
|
||||
name: urls.length == 1 ? _renameController.text.trim() : '',
|
||||
omitChecksum: urls.length > 1,
|
||||
),
|
||||
),
|
||||
];
|
||||
final service = ref.read(gopeedServiceProvider);
|
||||
if (_directDownload) {
|
||||
await service.createTaskBatch(
|
||||
CreateTaskBatch(
|
||||
reqs: [for (final source in sources) CreateTaskBatchItem(req: source.request, opts: source.options)],
|
||||
),
|
||||
);
|
||||
await _finishCreation();
|
||||
await _finishCreation(session);
|
||||
return;
|
||||
}
|
||||
|
||||
final request = _buildRequest(urls.first, protocolUrl: _protocolUrlFor(urls.first));
|
||||
final result = await ref.read(gopeedServiceProvider).resolve(ResolveTask(req: request));
|
||||
if (!mounted) return;
|
||||
final created = await _showResolveDialog(request, result);
|
||||
if (!created) {
|
||||
if (mounted) {
|
||||
setState(() => _creating = false);
|
||||
}
|
||||
return;
|
||||
}
|
||||
await _finishCreation();
|
||||
final created = await showResolveDownloadsDialog(
|
||||
context,
|
||||
sources: sources,
|
||||
service: service,
|
||||
onBatchCreated: (sources) {
|
||||
if (!mounted || session != _createSession) return;
|
||||
for (final source in sources) {
|
||||
_completedSubmissionCounts.update(source.request.url, (count) => count + 1, ifAbsent: () => 1);
|
||||
}
|
||||
},
|
||||
);
|
||||
if (!mounted || session != _createSession || !created) return;
|
||||
await _finishCreation(session);
|
||||
} catch (error) {
|
||||
if (mounted) {
|
||||
_showToast(error.toString());
|
||||
setState(() => _creating = false);
|
||||
}
|
||||
if (mounted && session == _createSession) _showToast(error.toString());
|
||||
} finally {
|
||||
if (mounted) setState(() => _creating = false);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1024,6 +1121,12 @@ class _CreateTaskWindowPageState extends ConsumerState<CreateTaskWindowPage> {
|
||||
extra = ReqExtraBt(trackers: trackers).toJson();
|
||||
}
|
||||
break;
|
||||
case _TaskProtocol.ftp:
|
||||
// ftps:// and ftpes:// fix the TLS mode; only a plain ftp:// link carries one.
|
||||
if (_isPlainFtpUrl(protocolUrl ?? url) && _ftpTls != _ftpTlsNone) {
|
||||
extra = ReqExtraFtp(tls: _ftpTls).toJson();
|
||||
}
|
||||
break;
|
||||
case _TaskProtocol.ed2k:
|
||||
case null:
|
||||
break;
|
||||
@@ -1039,7 +1142,7 @@ class _CreateTaskWindowPageState extends ConsumerState<CreateTaskWindowPage> {
|
||||
}
|
||||
|
||||
bool _validateAdvancedOptions() {
|
||||
if (_protocolTab == 0 && _inputUrls().length <= 1) {
|
||||
if (_protocolTab == 0 && _inputUrls().length <= 1 && !_allLinksFtp) {
|
||||
final checksumExpected = _checksumHashController.text.trim();
|
||||
if (checksumExpected.isNotEmpty) {
|
||||
final expectedLen = switch (_checksumAlgorithm.toLowerCase()) {
|
||||
@@ -1099,6 +1202,10 @@ class _CreateTaskWindowPageState extends ConsumerState<CreateTaskWindowPage> {
|
||||
if (uppercaseUrl.startsWith('HTTP:') || uppercaseUrl.startsWith('HTTPS:')) {
|
||||
return _TaskProtocol.http;
|
||||
}
|
||||
// Checked before BitTorrent, so ftp://host/file.torrent downloads the file over FTP.
|
||||
if (uppercaseUrl.startsWith('FTP:') || uppercaseUrl.startsWith('FTPS:') || uppercaseUrl.startsWith('FTPES:')) {
|
||||
return _TaskProtocol.ftp;
|
||||
}
|
||||
if (uppercaseUrl.startsWith('MAGNET:') || uppercaseUrl.endsWith('.TORRENT')) {
|
||||
return _TaskProtocol.bt;
|
||||
}
|
||||
@@ -1108,6 +1215,9 @@ class _CreateTaskWindowPageState extends ConsumerState<CreateTaskWindowPage> {
|
||||
return null;
|
||||
}
|
||||
|
||||
/// Whether [url] is a plain `ftp://` link, the only FTP scheme whose TLS mode is chosen by the user.
|
||||
bool _isPlainFtpUrl(String url) => url.trim().toUpperCase().startsWith('FTP:');
|
||||
|
||||
String _normalizeTaskUrl(String url) {
|
||||
if (_isBtHash(url)) {
|
||||
return 'magnet:?xt=urn:btih:$url';
|
||||
@@ -1130,7 +1240,10 @@ class _CreateTaskWindowPageState extends ConsumerState<CreateTaskWindowPage> {
|
||||
return text.length == 40 && RegExp(r'^[0-9a-fA-F]+$').hasMatch(text);
|
||||
}
|
||||
|
||||
/// Builds the task options for [url]. FTP links get the FTP options, with at
|
||||
/// most [_ftpMaxConnections] connections; the checksum is HTTP only.
|
||||
api_options.Options _buildOptions({
|
||||
required String url,
|
||||
String? name,
|
||||
List<int> selectFiles = const [],
|
||||
bool omitChecksum = false,
|
||||
@@ -1138,18 +1251,22 @@ class _CreateTaskWindowPageState extends ConsumerState<CreateTaskWindowPage> {
|
||||
}) {
|
||||
final urls = _inputUrls();
|
||||
final connections = int.tryParse(_connectionsController.text.trim()) ?? 0;
|
||||
final checksumExpected = _checksumHashController.text.trim();
|
||||
final checksum = (!omitChecksum && _inputUrls().length <= 1 && checksumExpected.isNotEmpty)
|
||||
? api_options.ChecksumOption(algorithm: _checksumAlgorithm, expected: checksumExpected)
|
||||
: null;
|
||||
return api_options.Options(
|
||||
size: urls.length == 1 && _parseProtocol(_protocolUrlFor(urls.first)) == _TaskProtocol.http ? _initialSize : 0,
|
||||
name: name ?? _renameController.text.trim(),
|
||||
path: _directoryController.text.trim(),
|
||||
asDefaultPath: _asDefaultPath,
|
||||
skipAutoCategorize: skipAutoCategorize ?? _directoryManuallySet,
|
||||
selectFiles: selectFiles,
|
||||
extra: api_options.OptsExtraHttp(
|
||||
final Map<String, dynamic> extra;
|
||||
if (_parseProtocol(url) == _TaskProtocol.ftp) {
|
||||
extra = api_options.OptsExtraFtp(
|
||||
connections: connections.clamp(0, _ftpMaxConnections),
|
||||
autoTorrent: _autoTorrent,
|
||||
deleteTorrentAfterDownload: _deleteTorrentAfterDownload,
|
||||
autoExtract: _autoExtract,
|
||||
archivePassword: _archivePasswordController.text,
|
||||
deleteAfterExtract: _deleteAfterExtract,
|
||||
).toJson();
|
||||
} else {
|
||||
final checksumExpected = _checksumHashController.text.trim();
|
||||
final checksum = (!omitChecksum && urls.length <= 1 && checksumExpected.isNotEmpty)
|
||||
? api_options.ChecksumOption(algorithm: _checksumAlgorithm, expected: checksumExpected)
|
||||
: null;
|
||||
extra = api_options.OptsExtraHttp(
|
||||
connections: connections,
|
||||
autoTorrent: _autoTorrent,
|
||||
deleteTorrentAfterDownload: _deleteTorrentAfterDownload,
|
||||
@@ -1157,7 +1274,16 @@ class _CreateTaskWindowPageState extends ConsumerState<CreateTaskWindowPage> {
|
||||
archivePassword: _archivePasswordController.text,
|
||||
deleteAfterExtract: _deleteAfterExtract,
|
||||
checksum: checksum,
|
||||
).toJson(),
|
||||
).toJson();
|
||||
}
|
||||
return api_options.Options(
|
||||
size: urls.length == 1 && _parseProtocol(_protocolUrlFor(urls.first)) == _TaskProtocol.http ? _initialSize : 0,
|
||||
name: name ?? _renameController.text.trim(),
|
||||
path: _directoryController.text.trim(),
|
||||
asDefaultPath: _asDefaultPath,
|
||||
skipAutoCategorize: skipAutoCategorize ?? _directoryManuallySet,
|
||||
selectFiles: selectFiles,
|
||||
extra: extra,
|
||||
);
|
||||
}
|
||||
|
||||
@@ -1222,183 +1348,6 @@ class _CreateTaskWindowPageState extends ConsumerState<CreateTaskWindowPage> {
|
||||
_applyCategoryHighlight();
|
||||
}
|
||||
|
||||
Future<void> _submitResolved(Request request, ResolveResult result, List<int> selectedIndexes) async {
|
||||
final selected = (selectedIndexes.isEmpty ? result.res.files.asMap().keys.toList() : selectedIndexes.toList())
|
||||
..sort();
|
||||
final isMultiFile = selected.length > 1;
|
||||
if (result.id.isNotEmpty) {
|
||||
await ref
|
||||
.read(gopeedServiceProvider)
|
||||
.createTask(
|
||||
CreateTask(
|
||||
rid: result.id,
|
||||
opts: _buildOptions(selectFiles: selected, omitChecksum: isMultiFile),
|
||||
),
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
if (result.res.files.isNotEmpty) {
|
||||
// The extension returned a file list, so the resource keeps a directory
|
||||
// structure. The split tasks must not be classified into a category, the
|
||||
// structure of the resolved resource is kept. The decision is based on
|
||||
// the resolved resource, not on the number of selected files.
|
||||
final keepStructure = _directoryManuallySet || result.res.name.isNotEmpty || result.res.files.length > 1;
|
||||
final reqs = selected.map((index) {
|
||||
final file = result.res.files[index];
|
||||
final fileRequest = file.req ?? request;
|
||||
return CreateTaskBatchItem(
|
||||
req: fileRequest,
|
||||
opts: api_options.Options(
|
||||
size: _parseProtocol(fileRequest.url) == _TaskProtocol.http ? file.size : 0,
|
||||
name: file.name,
|
||||
path: path.join(_directoryController.text.trim(), result.res.name, file.path),
|
||||
skipAutoCategorize: keepStructure,
|
||||
extra: _buildOptions(omitChecksum: isMultiFile).extra,
|
||||
),
|
||||
);
|
||||
}).toList();
|
||||
await ref
|
||||
.read(gopeedServiceProvider)
|
||||
.createTaskBatch(
|
||||
CreateTaskBatch(
|
||||
reqs: reqs,
|
||||
opts: _buildOptions(omitChecksum: isMultiFile, skipAutoCategorize: keepStructure),
|
||||
),
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
await ref.read(gopeedServiceProvider).createTask(CreateTask(req: request, opts: _buildOptions()));
|
||||
}
|
||||
|
||||
Future<bool> _showResolveDialog(Request request, ResolveResult result) async {
|
||||
final initialSelection = List<int>.generate(result.res.files.length, (index) => index);
|
||||
var selectedIndexes = List<int>.of(initialSelection);
|
||||
var submitting = false;
|
||||
String? errorText;
|
||||
// The directory the task will be saved to. A directory the user named
|
||||
// wins over the category suggestion of the resolve response, the backend
|
||||
// keeps it and does not classify the task.
|
||||
final effectiveDownloadDir = _directoryManuallySet
|
||||
? _directoryController.text.trim()
|
||||
: (result.suggestedDownloadDir.isNotEmpty ? result.suggestedDownloadDir : _directoryController.text.trim());
|
||||
|
||||
final dialog = const DialogOverlayHandler().show<bool>(
|
||||
context: context,
|
||||
alignment: Alignment.center,
|
||||
barrierDismissable: !submitting,
|
||||
builder: (dialogContext) {
|
||||
final palette = AppPalette.of(dialogContext);
|
||||
final screenSize = MediaQuery.sizeOf(dialogContext);
|
||||
final dialogWidth = screenSize.width < 760 ? screenSize.width - 32 : 720.0;
|
||||
final treeHeight = screenSize.height < 700 ? screenSize.height * 0.55 : 440.0;
|
||||
|
||||
return StatefulBuilder(
|
||||
builder: (dialogContext, setDialogState) {
|
||||
Future<void> submit() async {
|
||||
if (result.res.files.isNotEmpty && selectedIndexes.isEmpty) {
|
||||
setDialogState(() => errorText = dialogContext.l10n.selectAtLeastOneFile);
|
||||
return;
|
||||
}
|
||||
setDialogState(() {
|
||||
submitting = true;
|
||||
errorText = null;
|
||||
});
|
||||
try {
|
||||
await _submitResolved(request, result, selectedIndexes);
|
||||
if (dialogContext.mounted) {
|
||||
await closeOverlay(dialogContext, true);
|
||||
}
|
||||
} catch (error) {
|
||||
if (!dialogContext.mounted) return;
|
||||
setDialogState(() {
|
||||
submitting = false;
|
||||
errorText = error.toString();
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return AlertDialog(
|
||||
padding: const EdgeInsets.all(18),
|
||||
title: Text(
|
||||
result.res.name.trim().isEmpty ? dialogContext.l10n.resolvedFiles : result.res.name,
|
||||
maxLines: 1,
|
||||
overflow: TextOverflow.ellipsis,
|
||||
),
|
||||
content: SizedBox(
|
||||
width: dialogWidth,
|
||||
child: Column(
|
||||
mainAxisSize: MainAxisSize.min,
|
||||
crossAxisAlignment: CrossAxisAlignment.start,
|
||||
children: [
|
||||
Text(
|
||||
result.res.files.isEmpty
|
||||
? dialogContext.l10n.resourceHasNoFiles
|
||||
: dialogContext.l10n.resolvedFileCount(result.res.files.length),
|
||||
style: TextStyle(color: palette.textSecondary, fontSize: 12),
|
||||
),
|
||||
if (effectiveDownloadDir.isNotEmpty) ...[
|
||||
const SizedBox(height: 4),
|
||||
Text(
|
||||
dialogContext.l10n.resolvedCategoryDirectory(effectiveDownloadDir),
|
||||
key: const ValueKey('resolve-category-directory'),
|
||||
style: TextStyle(color: palette.textSecondary, fontSize: 12),
|
||||
maxLines: 1,
|
||||
overflow: TextOverflow.ellipsis,
|
||||
),
|
||||
],
|
||||
const SizedBox(height: 12),
|
||||
SizedBox(
|
||||
height: treeHeight.clamp(260.0, 460.0),
|
||||
child: result.res.files.isEmpty
|
||||
? Container(
|
||||
alignment: Alignment.center,
|
||||
decoration: BoxDecoration(
|
||||
color: palette.cardBg,
|
||||
border: Border.all(color: palette.border),
|
||||
borderRadius: BorderRadius.circular(AppDesignTokens.controlRadius),
|
||||
),
|
||||
child: Text(
|
||||
dialogContext.l10n.noFiles,
|
||||
style: TextStyle(color: palette.textMuted, fontSize: 13),
|
||||
),
|
||||
)
|
||||
: ResolveFileTree(
|
||||
files: result.res.files,
|
||||
initialSelection: initialSelection,
|
||||
onSelectionChanged: (values) => selectedIndexes = values,
|
||||
),
|
||||
),
|
||||
if (errorText != null) ...[
|
||||
const SizedBox(height: 10),
|
||||
Text(errorText!, style: TextStyle(color: palette.error, fontSize: 12)),
|
||||
],
|
||||
],
|
||||
),
|
||||
),
|
||||
actions: [
|
||||
flutter.Flexible(
|
||||
child: CreateTaskActionButtons(
|
||||
submitting: submitting,
|
||||
onCancel: () => closeOverlay(dialogContext, false),
|
||||
onSubmit: submit,
|
||||
cancelLabel: dialogContext.l10n.cancel,
|
||||
submitLabel: dialogContext.l10n.createAction,
|
||||
cancelButtonKey: const ValueKey('resolve-cancel-button'),
|
||||
submitButtonKey: const ValueKey('resolve-create-button'),
|
||||
),
|
||||
),
|
||||
],
|
||||
);
|
||||
},
|
||||
);
|
||||
},
|
||||
);
|
||||
final created = await dialog.future;
|
||||
return created == true;
|
||||
}
|
||||
|
||||
Future<void> _showCreateHistory() async {
|
||||
List<String> histories;
|
||||
try {
|
||||
@@ -1595,7 +1544,48 @@ class _CreateTaskWindowPageState extends ConsumerState<CreateTaskWindowPage> {
|
||||
}
|
||||
}
|
||||
|
||||
enum _TaskProtocol { http, bt, ed2k }
|
||||
enum _TaskProtocol { http, bt, ed2k, ftp }
|
||||
|
||||
/// Chooses how a plain `ftp://` link is encrypted. The values are the
|
||||
/// `tls` option of an FTP task.
|
||||
class _FtpTlsSelect extends StatelessWidget {
|
||||
const _FtpTlsSelect({super.key, required this.value, required this.onChanged});
|
||||
|
||||
final String value;
|
||||
final ValueChanged<String> onChanged;
|
||||
|
||||
@override
|
||||
Widget build(BuildContext context) {
|
||||
final labels = {
|
||||
_ftpTlsNone: context.l10n.ftpTlsNone,
|
||||
_ftpTlsExplicit: context.l10n.ftpTlsExplicit,
|
||||
_ftpTlsImplicit: context.l10n.ftpTlsImplicit,
|
||||
};
|
||||
return LayoutBuilder(
|
||||
builder: (context, constraints) => Select<String>(
|
||||
value: value,
|
||||
constraints: BoxConstraints.tightFor(width: constraints.maxWidth),
|
||||
itemBuilder: (context, selected) =>
|
||||
Text(labels[selected] ?? selected, maxLines: 1, overflow: TextOverflow.ellipsis),
|
||||
popup: (context) => SelectPopup<String>(
|
||||
items: SelectItemList(
|
||||
children: [
|
||||
for (final entry in labels.entries)
|
||||
SelectItemButton<String>(
|
||||
key: ValueKey('create-task-ftp-tls-${entry.key}'),
|
||||
value: entry.key,
|
||||
child: Text(entry.value),
|
||||
),
|
||||
],
|
||||
),
|
||||
),
|
||||
onChanged: (selected) {
|
||||
if (selected != null) onChanged(selected);
|
||||
},
|
||||
),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
class _DirectDownloadToggle extends StatelessWidget {
|
||||
const _DirectDownloadToggle({required this.value, required this.onChanged});
|
||||
@@ -2007,3 +1997,11 @@ const _advancedScrollDuration = Duration(milliseconds: 260);
|
||||
const _advancedScrollExtentRetryDelay = Duration(milliseconds: 40);
|
||||
const _advancedScrollExtentChecks = 4;
|
||||
const _advancedScrollDistance = 112.0;
|
||||
|
||||
/// The highest number of parallel logins offered for FTP links. FTP servers
|
||||
/// usually allow only a few logins per user.
|
||||
const _ftpMaxConnections = 32;
|
||||
const _ftpTlsNone = 'none';
|
||||
const _ftpTlsExplicit = 'explicit';
|
||||
const _ftpTlsImplicit = 'implicit';
|
||||
const _ftpTlsModes = {_ftpTlsNone, _ftpTlsExplicit, _ftpTlsImplicit};
|
||||
@@ -21,13 +21,8 @@ class TaskDetailsPage extends ConsumerWidget {
|
||||
@override
|
||||
Widget build(BuildContext context, WidgetRef ref) {
|
||||
final tasksAsync = ref.watch(tasksControllerProvider);
|
||||
TaskRecord? task = initialTask;
|
||||
for (final item in tasksAsync.value?.tasks ?? const <TaskRecord>[]) {
|
||||
if (item.id == taskId) {
|
||||
task = item;
|
||||
break;
|
||||
}
|
||||
}
|
||||
final state = tasksAsync.value;
|
||||
final task = state == null ? initialTask : state.findTask(taskId);
|
||||
final palette = AppPalette.of(context);
|
||||
|
||||
return AppDetailPage(
|
||||
@@ -43,12 +38,12 @@ class TaskDetailsPage extends ConsumerWidget {
|
||||
key: ValueKey(task.id),
|
||||
task: task,
|
||||
mobile: true,
|
||||
onOpenStorage: () => FileExplorer.reveal(task!.storagePath),
|
||||
onOpenStorage: () => FileExplorer.reveal(task.storagePath),
|
||||
onUpdateUrl: (url) async {
|
||||
try {
|
||||
await ref
|
||||
.read(tasksControllerProvider.notifier)
|
||||
.updateUrl(task!.id, url, headers: task.requestHeaders);
|
||||
.updateUrl(task.id, url, headers: task.requestHeaders);
|
||||
} catch (error) {
|
||||
if (context.mounted) {
|
||||
showAppToast(context, error.toString(), type: AppToastType.error);
|
||||
|
||||
@@ -20,13 +20,8 @@ class TaskFilesPage extends ConsumerWidget {
|
||||
@override
|
||||
Widget build(BuildContext context, WidgetRef ref) {
|
||||
final tasksAsync = ref.watch(tasksControllerProvider);
|
||||
TaskRecord? task = initialTask;
|
||||
for (final item in tasksAsync.value?.tasks ?? const <TaskRecord>[]) {
|
||||
if (item.id == taskId) {
|
||||
task = item;
|
||||
break;
|
||||
}
|
||||
}
|
||||
final state = tasksAsync.value;
|
||||
final task = state == null ? initialTask : state.findTask(taskId);
|
||||
final palette = AppPalette.of(context);
|
||||
|
||||
return shad.Scaffold(
|
||||
|
||||
@@ -18,7 +18,7 @@ class CreateTaskActionButtons extends StatelessWidget {
|
||||
|
||||
final bool submitting;
|
||||
final VoidCallback onCancel;
|
||||
final VoidCallback onSubmit;
|
||||
final VoidCallback? onSubmit;
|
||||
final String cancelLabel;
|
||||
final String submitLabel;
|
||||
final Key? cancelButtonKey;
|
||||
|
||||
Loaded 100 of 175 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user