mirror of
https://github.com/GopeedLab/gopeed.git
synced 2026-11-14 20:33:41 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Levi
|
44e2272ca4 |
No files matched your search
@@ -13,7 +13,7 @@ on:
|
||||
default: "false"
|
||||
|
||||
env:
|
||||
GO_VERSION: "1.26.11"
|
||||
GO_VERSION: "1.25.4"
|
||||
FLUTTER_VERSION: "3.41.6"
|
||||
GA4_MEASUREMENT_ID: ${{ secrets.GA4_MEASUREMENT_ID }}
|
||||
GA4_API_SECRET: ${{ secrets.GA4_API_SECRET }}
|
||||
@@ -798,5 +798,5 @@ jobs:
|
||||
linux/arm64
|
||||
linux/arm/v7
|
||||
tags: |
|
||||
${{ !contains(env.VERSION, 'beta') && 'liwei2633/gopeed:latest' || '' }}
|
||||
liwei2633/gopeed:latest
|
||||
liwei2633/gopeed:v${{ env.VERSION }}
|
||||
@@ -17,7 +17,7 @@ on:
|
||||
- "Dockerfile"
|
||||
|
||||
env:
|
||||
GO_VERSION: "1.26.11"
|
||||
GO_VERSION: "1.25.4"
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
|
||||
@@ -28,7 +28,7 @@ on:
|
||||
workflow_dispatch:
|
||||
|
||||
env:
|
||||
GO_VERSION: "1.26.11"
|
||||
GO_VERSION: "1.25.4"
|
||||
FLUTTER_VERSION: "3.41.6"
|
||||
|
||||
jobs:
|
||||
@@ -86,32 +86,6 @@ jobs:
|
||||
flutter pub get
|
||||
flutter gen-l10n
|
||||
flutter test test/api/downloader_config_test.dart
|
||||
test-flutter-task-context-menu:
|
||||
name: Test task context menus and text inputs
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 20
|
||||
env:
|
||||
CHROME_EXECUTABLE: /usr/bin/google-chrome
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- uses: subosito/flutter-action@v2
|
||||
with:
|
||||
flutter-version: ${{ env.FLUTTER_VERSION }}
|
||||
- name: Prepare Flutter tests
|
||||
working-directory: ui/flutter
|
||||
run: |
|
||||
flutter pub get
|
||||
flutter gen-l10n
|
||||
- name: Test task menu actions and native input policy
|
||||
working-directory: ui/flutter
|
||||
run: |
|
||||
flutter test --no-pub test/features/tasks/presentation/widgets/task_context_menu_test.dart test/shared/widgets/app_text_field_test.dart
|
||||
flutter test --no-pub test/widget_test.dart --plain-name 'task context menu exposes'
|
||||
- name: Test browser menu cancellation and input policy
|
||||
working-directory: ui/flutter
|
||||
run: |
|
||||
"$CHROME_EXECUTABLE" --version
|
||||
flutter test --no-pub --platform chrome test/features/tasks/presentation/widgets/task_context_menu_test.dart test/shared/widgets/app_text_field_test.dart
|
||||
test-webview-integration:
|
||||
name: Test webview integration (${{ matrix.name }})
|
||||
strategy:
|
||||
@@ -134,9 +108,6 @@ jobs:
|
||||
if: runner.os == 'macOS'
|
||||
with:
|
||||
flutter-version: ${{ env.FLUTTER_VERSION }}
|
||||
- name: Test Windows FFmpeg memory and stream lifecycle
|
||||
if: runner.os == 'Windows'
|
||||
run: go test -v -count=1 -timeout 180s ./pkg/download/engine/ffmpeg ./pkg/download/engine/inject/stream ./pkg/download/engine
|
||||
- name: Install Linux WebView dependencies
|
||||
if: runner.os == 'Linux'
|
||||
run: |
|
||||
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
FROM golang:1.26.11-alpine3.23 AS go
|
||||
FROM golang:1.25.4-alpine3.22 AS go
|
||||
WORKDIR /app
|
||||
COPY ./go.mod ./go.sum ./
|
||||
RUN go mod download
|
||||
|
||||
@@ -20,7 +20,7 @@ Beyond core download management, Gopeed offers browser integration, JavaScript e
|
||||
|
||||
Visit ✈ [Official Website](https://gopeed.com)
|
||||
|
||||
|
||||
|
||||
|
||||
## ✨ Features
|
||||
|
||||
|
||||
+1
-1
@@ -20,7 +20,7 @@ Gopeed（**Go Speed** の略）は、Go と Flutter で開発された高速で
|
||||
|
||||
✈ [公式サイト](https://gopeed.com)
|
||||
|
||||
|
||||
|
||||
|
||||
## ✨ 主な機能
|
||||
|
||||
|
||||
+1
-1
@@ -20,7 +20,7 @@ Ngoài việc quản lý tải xuống hằng ngày, Gopeed còn hỗ trợ tíc
|
||||
|
||||
Truy cập ✈ [Trang web chính thức](https://gopeed.com)
|
||||
|
||||
|
||||
|
||||
|
||||
## ✨ Tính năng chính
|
||||
|
||||
|
||||
+1
-1
@@ -20,7 +20,7 @@ Gopeed（全称 **Go Speed**，也叫“够快下载器”）是一款使用 Go
|
||||
|
||||
访问 ✈ [官方网站](https://gopeed.com)
|
||||
|
||||
|
||||
|
||||
|
||||
## ✨ 主要特性
|
||||
|
||||
|
||||
+1
-1
@@ -20,7 +20,7 @@ Gopeed（**Go Speed** 的縮寫）是一款以 Go 與 Flutter 開發的高速、
|
||||
|
||||
瀏覽 ✈ [官方網站](https://gopeed.com)
|
||||
|
||||
|
||||
|
||||
|
||||
## ✨ 主要功能
|
||||
|
||||
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 74 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 77 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 1.3 MiB |
Binary file not shown.
|
Before Width: | Height: | Size: 792 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 1.3 MiB |
Binary file not shown.
|
Before Width: | Height: | Size: 787 KiB |
@@ -1,52 +0,0 @@
|
||||
# Batch task creation
|
||||
|
||||
`POST /api/v1/tasks/batch` accepts ordinary and grouped batches:
|
||||
|
||||
```json
|
||||
{
|
||||
"group": true,
|
||||
"groupName": "album",
|
||||
"opts": {"path": "/downloads"},
|
||||
"reqs": [
|
||||
{"rid": "resolved-resource-id", "opts": {"selectFiles": [1]}},
|
||||
{"req": {"url": "https://example.com/cover.jpg"}}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
Each member accepts `rid` or `req`. A nonempty `rid` takes priority and creates a
|
||||
task from the cached resolve result. A member's `opts`, when present, replaces
|
||||
batch `opts` as a whole; omitted fields do not inherit individual batch values.
|
||||
Members without `opts` use batch `opts`. BT file selections use the indexes from
|
||||
the resolve result. Requests without `rid` continue to work.
|
||||
|
||||
For groups, batch `opts.path` selects the parent directory. Members inheriting
|
||||
batch options, or with an empty path, use the reserved group directory. Explicit
|
||||
relative member paths are resolved under that directory; do not prefix them with
|
||||
the batch parent path. For example, parent `downloads` and member path `nested`
|
||||
place the member in `downloads/<group name>/nested`. Absolute paths under the
|
||||
parent directory preserve their relative structure within the group. In ordinary
|
||||
batches, an empty member path uses the configured default download directory.
|
||||
|
||||
All members are attempted. If at least one task is persisted, the response is
|
||||
successful and contains the created task IDs. A task that fails to start after
|
||||
being persisted is included; use the existing task controls to retry it.
|
||||
Successful tasks are kept when other members fail. If every member fails, the
|
||||
API returns an error and removes any newly reserved empty group and directory.
|
||||
|
||||
Both ordinary and grouped batches now return an object in `data`:
|
||||
|
||||
```json
|
||||
{"code": 0, "msg": "", "data": {"taskIds": ["task-1", "task-2"], "parentId": "group-id"}}
|
||||
```
|
||||
|
||||
Ordinary batches omit `parentId`. **This replaces the former array response for
|
||||
ordinary batches.** The outer `code`/`msg`/`data` envelope is unchanged. Clients
|
||||
must read `data.taskIds`; there are no per-member results. A successful response
|
||||
completes the batch, including partial success, so clients must not automatically
|
||||
resubmit missing members.
|
||||
|
||||
With `group: true`, the server generates the group ID and reserves its directory.
|
||||
Creation does not accept a client group ID or append to an existing group. Group
|
||||
metadata remains available through the existing task list and detail responses;
|
||||
`parentId` identifies membership for group operations.
|
||||
+1
-7
@@ -1,9 +1,6 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"github.com/GopeedLab/gopeed/cmd"
|
||||
"github.com/GopeedLab/gopeed/pkg/rest/model"
|
||||
)
|
||||
@@ -16,8 +13,5 @@ func main() {
|
||||
Storage: model.StorageBolt,
|
||||
WebEnable: true,
|
||||
}
|
||||
if err := cmd.Start(cfg); err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
cmd.Start(cfg)
|
||||
}
|
||||
+5
-7
@@ -16,16 +16,15 @@ import (
|
||||
//go:embed banner.txt
|
||||
var banner string
|
||||
|
||||
func Start(cfg *model.StartConfig) error {
|
||||
func Start(cfg *model.StartConfig) {
|
||||
fmt.Println(banner)
|
||||
srv, listener, err := rest.BuildServer(cfg)
|
||||
if err != nil {
|
||||
return err
|
||||
panic(err)
|
||||
}
|
||||
fmt.Println(banner)
|
||||
downloadCfg, err := rest.Downloader.GetConfig()
|
||||
if err != nil {
|
||||
listener.Close()
|
||||
return err
|
||||
panic(err)
|
||||
}
|
||||
if rest.Downloader.IsFirstLoad() {
|
||||
// Set default download config
|
||||
@@ -63,9 +62,8 @@ func Start(cfg *model.StartConfig) error {
|
||||
|
||||
fmt.Printf("Server start success on http://%s\n", listener.Addr().String())
|
||||
if err := srv.Serve(listener); err != nil && err != http.ErrServerClosed {
|
||||
return err
|
||||
panic(err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func watchExit() {
|
||||
|
||||
@@ -1,35 +0,0 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/GopeedLab/gopeed/pkg/rest/model"
|
||||
)
|
||||
|
||||
func TestStartInvalidWebAPITokenReturnsError(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
webAuth *model.WebAuth
|
||||
}{
|
||||
{name: "no credentials"},
|
||||
{name: "missing username", webAuth: &model.WebAuth{Password: "secret"}},
|
||||
{name: "missing password", webAuth: &model.WebAuth{Username: "gopeed"}},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
err := Start(&model.StartConfig{
|
||||
WebEnable: true,
|
||||
ApiToken: "test-token",
|
||||
WebAuth: tt.webAuth,
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("Start() error = nil, want web authentication validation error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "web authentication username and password are required") {
|
||||
t.Fatalf("Start() error = %q, want missing web authentication credentials", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
+2
-2
@@ -52,8 +52,8 @@ func loadCliArgs() *args {
|
||||
flag.StringVar(cfg.Username, "username", "gopeed", "Web Authentication Username")
|
||||
cfg.Password = flag.String("p", "", "Web Authentication Password, if no password is set, web authentication will not be enabled")
|
||||
flag.StringVar(cfg.Password, "password", "", "Web Authentication Password, if no password is set, web authentication will not be enabled")
|
||||
cfg.ApiToken = flag.String("T", "", "API token (requires web authentication with username and password)")
|
||||
flag.StringVar(cfg.ApiToken, "api-token", "", "API token (requires web authentication with username and password)")
|
||||
cfg.ApiToken = flag.String("T", "", "API token, it must be configured when using HTTP API in the case of enabling web authentication")
|
||||
flag.StringVar(cfg.ApiToken, "api-token", "", "API token, it must be configured when using HTTP API in the case of enabling web authentication")
|
||||
flag.BoolVar(&cfg.MCPEnable, "mcp-enable", false, "Enable MCP endpoint")
|
||||
cfg.TempDir = flag.String("temp-dir", "", "Temporary directory (defaults to the OS temporary directory)")
|
||||
cfg.StorageDir = flag.String("d", "", "Storage directory")
|
||||
|
||||
+1
-4
@@ -62,10 +62,7 @@ func main() {
|
||||
WebFS: sub,
|
||||
WebAuth: webAuth,
|
||||
}
|
||||
if err := cmd.Start(cfg); err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
cmd.Start(cfg)
|
||||
}
|
||||
|
||||
func isNotBlank(str *string) bool {
|
||||
|
||||
@@ -1,23 +1,20 @@
|
||||
module github.com/GopeedLab/gopeed
|
||||
|
||||
go 1.26.0
|
||||
go 1.25.4
|
||||
|
||||
require (
|
||||
codeberg.org/gruf/go-ffmpreg v0.6.23-0.20260827095416-df1836cb5be0
|
||||
github.com/GopeedLab/webview_go v0.0.0-20260918010912-0d4c27bb7914
|
||||
github.com/RoaringBitmap/roaring/v2 v2.16.0
|
||||
github.com/anacrolix/torrent v1.62.0
|
||||
github.com/anacrolix/torrent v1.60.1-0.20251217073903-486bcbe758e0
|
||||
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5
|
||||
github.com/bodgit/sevenzip v1.6.1
|
||||
github.com/dop251/goja v0.0.0-20260106131823-651366fbe6e3
|
||||
github.com/dop251/goja_nodejs v0.0.0-20240728170619-29b559befffc
|
||||
github.com/fclairamb/ftpserverlib v0.31.0
|
||||
github.com/go-git/go-git/v5 v5.19.2
|
||||
github.com/go-git/go-git/v5 v5.8.1
|
||||
github.com/google/jsonschema-go v0.4.3
|
||||
github.com/gorilla/handlers v1.5.1
|
||||
github.com/gorilla/mux v1.8.0
|
||||
github.com/imroc/req/v3 v3.58.0
|
||||
github.com/jlaffaye/ftp v0.2.4
|
||||
github.com/imroc/req/v3 v3.52.2
|
||||
github.com/matoous/go-nanoid/v2 v2.0.0
|
||||
github.com/mattn/go-ieproxy v0.0.12
|
||||
github.com/mholt/archives v0.1.5
|
||||
@@ -26,65 +23,66 @@ require (
|
||||
github.com/pkg/errors v0.9.1
|
||||
github.com/rs/dnscache v0.0.0-20230804202142-fc85eb664529
|
||||
github.com/rs/zerolog v1.31.0
|
||||
github.com/spf13/afero v1.15.0
|
||||
github.com/tetratelabs/wazero v1.12.0
|
||||
github.com/xiaoqidun/setft v0.0.0-20220310121541-be86327699ad
|
||||
go.etcd.io/bbolt v1.4.3
|
||||
golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/cyphar/filepath-securejoin v0.6.1 // indirect
|
||||
github.com/google/go-querystring v1.1.0 // indirect
|
||||
github.com/onsi/gomega v1.36.3 // indirect
|
||||
golang.org/x/exp v0.0.0-20251219203646-944ab1f22d93
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/STARRY-S/zip v0.2.3 // indirect
|
||||
github.com/anacrolix/btree v0.1.1 // indirect
|
||||
github.com/anacrolix/go-utp v0.0.0-20260908033909-9f1664acf866 // indirect
|
||||
github.com/anacrolix/missinggo/v2 v2.11.0 // indirect
|
||||
github.com/anacrolix/missinggo/v2 v2.10.0 // indirect
|
||||
github.com/andybalholm/brotli v1.2.0 // indirect
|
||||
github.com/bodgit/plumbing v1.3.0 // indirect
|
||||
github.com/bodgit/windows v1.0.1 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.3.0 // indirect
|
||||
github.com/coder/websocket v1.8.15 // indirect
|
||||
github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707 // indirect
|
||||
github.com/felixge/fgprof v0.9.5 // indirect
|
||||
github.com/go-task/slim-sprig/v3 v3.0.0 // indirect
|
||||
github.com/google/go-cmp v0.7.0 // indirect
|
||||
github.com/hashicorp/errwrap v1.1.0 // indirect
|
||||
github.com/hashicorp/go-multierror v1.1.1 // indirect
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
|
||||
github.com/icholy/digest v1.1.0 // indirect
|
||||
github.com/klauspost/compress v1.18.2 // indirect
|
||||
github.com/klauspost/compress v1.18.0 // indirect
|
||||
github.com/klauspost/pgzip v1.2.6 // indirect
|
||||
github.com/mikelolasagasti/xz v1.0.1 // indirect
|
||||
github.com/minio/minlz v1.0.1 // indirect
|
||||
github.com/nwaples/rardecode/v2 v2.2.0 // indirect
|
||||
github.com/onsi/ginkgo/v2 v2.23.4 // indirect
|
||||
github.com/pierrec/lz4/v4 v4.1.22 // indirect
|
||||
github.com/pion/transport/v4 v4.0.2 // indirect
|
||||
github.com/quic-go/qpack v0.6.0 // indirect
|
||||
github.com/quic-go/quic-go v0.59.0 // indirect
|
||||
github.com/refraction-networking/utls v1.8.2 // indirect
|
||||
github.com/pion/transport/v4 v4.0.1 // indirect
|
||||
github.com/quic-go/qpack v0.5.1 // indirect
|
||||
github.com/quic-go/quic-go v0.51.0 // indirect
|
||||
github.com/refraction-networking/utls v1.6.7 // indirect
|
||||
github.com/segmentio/asm v1.1.3 // indirect
|
||||
github.com/segmentio/encoding v0.5.4 // indirect
|
||||
github.com/sorairolake/lzip-go v0.3.8 // indirect
|
||||
github.com/spf13/afero v1.15.0 // indirect
|
||||
github.com/ulikunitz/xz v0.5.15 // indirect
|
||||
github.com/yosida95/uritemplate/v3 v3.0.2 // indirect
|
||||
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
|
||||
go.uber.org/automaxprocs v1.6.0 // indirect
|
||||
go.uber.org/mock v0.5.1 // indirect
|
||||
go4.org v0.0.0-20230225012048-214862532bf5 // indirect
|
||||
golang.org/x/mod v0.33.0
|
||||
golang.org/x/oauth2 v0.35.0 // indirect
|
||||
golang.org/x/sync v0.23.0 // indirect
|
||||
golang.org/x/sync v0.20.0 // indirect
|
||||
)
|
||||
|
||||
require (
|
||||
dario.cat/mergo v1.0.0 // indirect
|
||||
github.com/Microsoft/go-winio v0.6.2
|
||||
github.com/ProtonMail/go-crypto v1.1.6 // indirect
|
||||
github.com/Microsoft/go-winio v0.6.1
|
||||
github.com/ProtonMail/go-crypto v0.0.0-20230717121422-5aa5874ade95 // indirect
|
||||
github.com/RoaringBitmap/roaring v1.9.4
|
||||
github.com/acomagu/bufpipe v1.0.4 // indirect
|
||||
github.com/alecthomas/atomic v0.1.0-alpha2 // indirect
|
||||
github.com/anacrolix/chansync v0.8.0 // indirect
|
||||
github.com/anacrolix/dht/v2 v2.23.1-0.20260525063928-ec3a9bd99456 // indirect
|
||||
github.com/anacrolix/chansync v0.7.0 // indirect
|
||||
github.com/anacrolix/dht/v2 v2.23.0 // indirect
|
||||
github.com/anacrolix/envpprof v1.5.0 // indirect
|
||||
github.com/anacrolix/generics v0.2.0
|
||||
github.com/anacrolix/go-libutp v1.5.2-0.20260908013213-836dd42cdde6 // indirect
|
||||
github.com/anacrolix/generics v0.2.0 // indirect
|
||||
github.com/anacrolix/go-libutp v1.3.2 // indirect
|
||||
github.com/anacrolix/log v0.17.1-0.20251118025802-918f1157b7bb // indirect
|
||||
github.com/anacrolix/missinggo v1.3.0 // indirect
|
||||
github.com/anacrolix/missinggo/perf v1.0.0 // indirect
|
||||
@@ -93,12 +91,13 @@ require (
|
||||
github.com/anacrolix/stm v0.5.0 // indirect
|
||||
github.com/anacrolix/sync v0.6.0 // indirect
|
||||
github.com/anacrolix/upnp v0.1.4 // indirect
|
||||
github.com/anacrolix/utp v0.2.0 // indirect
|
||||
github.com/bahlo/generic-list-go v0.2.0 // indirect
|
||||
github.com/benbjohnson/immutable v0.4.3 // indirect
|
||||
github.com/bits-and-blooms/bitset v1.24.4 // indirect
|
||||
github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8 // indirect
|
||||
github.com/cespare/xxhash v1.1.0 // indirect
|
||||
github.com/cloudflare/circl v1.6.3 // indirect
|
||||
github.com/cloudflare/circl v1.6.1 // indirect
|
||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||
github.com/dlclark/regexp2 v1.11.5 // indirect
|
||||
github.com/dustin/go-humanize v1.0.1 // indirect
|
||||
@@ -106,16 +105,17 @@ require (
|
||||
github.com/emirpasic/gods v1.18.1 // indirect
|
||||
github.com/felixge/httpsnoop v1.0.1 // indirect
|
||||
github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
|
||||
github.com/go-git/go-billy/v5 v5.9.2 // indirect
|
||||
github.com/go-git/go-billy/v5 v5.4.1 // indirect
|
||||
github.com/go-llsqlite/adapter v0.2.0 // indirect
|
||||
github.com/go-llsqlite/crawshaw v0.6.0 // indirect
|
||||
github.com/go-logr/logr v1.4.4 // indirect
|
||||
github.com/go-logr/logr v1.4.3 // indirect
|
||||
github.com/go-logr/stdr v1.2.2 // indirect
|
||||
github.com/go-sourcemap/sourcemap v2.1.4+incompatible // indirect
|
||||
github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect
|
||||
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
|
||||
github.com/google/btree v1.1.3 // indirect
|
||||
github.com/google/pprof v0.0.0-20260115054156-294ebfa9ad83 // indirect
|
||||
github.com/google/uuid v1.6.0 // indirect
|
||||
github.com/gorilla/websocket v1.5.3 // indirect
|
||||
github.com/huandu/xstrings v1.5.0 // indirect
|
||||
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
|
||||
github.com/kevinburke/ssh_config v1.2.0 // indirect
|
||||
@@ -129,7 +129,7 @@ require (
|
||||
github.com/multiformats/go-varint v0.1.0 // indirect
|
||||
github.com/ncruces/go-strftime v1.0.0 // indirect
|
||||
github.com/pion/datachannel v1.6.0 // indirect
|
||||
github.com/pion/dtls/v3 v3.1.4 // indirect
|
||||
github.com/pion/dtls/v3 v3.0.10 // indirect
|
||||
github.com/pion/ice/v4 v4.2.0 // indirect
|
||||
github.com/pion/interceptor v0.1.42 // indirect
|
||||
github.com/pion/logging v0.2.4 // indirect
|
||||
@@ -140,27 +140,28 @@ require (
|
||||
github.com/pion/sctp v1.9.1 // indirect
|
||||
github.com/pion/sdp/v3 v3.0.17 // indirect
|
||||
github.com/pion/srtp/v3 v3.0.10 // indirect
|
||||
github.com/pion/stun/v3 v3.1.5 // indirect
|
||||
github.com/pion/stun/v3 v3.1.1 // indirect
|
||||
github.com/pion/turn/v4 v4.1.4 // indirect
|
||||
github.com/pion/webrtc/v4 v4.2.2-0.20260109001657-a5962f314db7 // indirect
|
||||
github.com/pjbgf/sha1cd v0.6.0 // indirect
|
||||
github.com/pjbgf/sha1cd v0.3.0 // indirect
|
||||
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c
|
||||
github.com/protolambda/ctxlock v0.1.0 // indirect
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
|
||||
github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect
|
||||
github.com/skeema/knownhosts v1.3.1 // indirect
|
||||
github.com/sergi/go-diff v1.1.0 // indirect
|
||||
github.com/skeema/knownhosts v1.2.0 // indirect
|
||||
github.com/spaolacci/murmur3 v1.1.0 // indirect
|
||||
github.com/tidwall/btree v1.8.1 // indirect
|
||||
github.com/wlynxg/anet v0.0.5 // indirect
|
||||
github.com/xanzy/ssh-agent v0.3.3 // indirect
|
||||
go.opentelemetry.io/otel v1.45.0 // indirect
|
||||
go.opentelemetry.io/otel/metric v1.45.0 // indirect
|
||||
go.opentelemetry.io/otel/trace v1.45.0 // indirect
|
||||
golang.org/x/crypto v0.57.0 // indirect
|
||||
golang.org/x/net v0.60.0
|
||||
golang.org/x/sys v0.48.0 // indirect
|
||||
golang.org/x/text v0.42.0
|
||||
golang.org/x/time v0.15.0
|
||||
go.opentelemetry.io/otel v1.39.0 // indirect
|
||||
go.opentelemetry.io/otel/metric v1.39.0 // indirect
|
||||
go.opentelemetry.io/otel/trace v1.39.0 // indirect
|
||||
golang.org/x/crypto v0.48.0 // indirect
|
||||
golang.org/x/net v0.50.0
|
||||
golang.org/x/sys v0.44.0 // indirect
|
||||
golang.org/x/text v0.34.0
|
||||
golang.org/x/time v0.15.0 // indirect
|
||||
golang.org/x/tools v0.42.0 // indirect
|
||||
gopkg.in/warnings.v0 v0.1.2 // indirect
|
||||
lukechampine.com/blake3 v1.4.1 // indirect
|
||||
modernc.org/libc v1.67.4 // indirect
|
||||
|
||||
@@ -3,7 +3,6 @@ package controller
|
||||
import (
|
||||
"github.com/GopeedLab/gopeed/internal/tempfiles"
|
||||
"github.com/GopeedLab/gopeed/pkg/base"
|
||||
"github.com/rs/zerolog"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
@@ -16,9 +15,6 @@ type Controller struct {
|
||||
TempDir string
|
||||
GetConfig func(v any)
|
||||
GetProxy func(requestProxy *base.RequestProxy) func(*http.Request) (*url.URL, error)
|
||||
// Logger is the downloader's core logger. It may be nil, in which case
|
||||
// fetchers log nothing.
|
||||
Logger *zerolog.Logger
|
||||
FileController
|
||||
//ContextDialer() (proxy.Dialer, error)
|
||||
}
|
||||
|
||||
@@ -46,10 +46,6 @@ type Uploader interface {
|
||||
WaitUpload() error
|
||||
}
|
||||
|
||||
type DownloadCounter interface {
|
||||
ReceivedBytes() int64
|
||||
}
|
||||
|
||||
// FetcherMeta defines the meta information of a fetcher.
|
||||
type FetcherMeta struct {
|
||||
Req *base.Request `json:"req"`
|
||||
|
||||
@@ -290,82 +290,6 @@ func TestBTDownloadSelectedFiles(t *testing.T) {
|
||||
if f.Meta().Res.Size != 48*1024 {
|
||||
t.Fatal("selected resource size is incorrect")
|
||||
}
|
||||
if downloaded := f.Progress().TotalDownloaded(); downloaded != 48*1024 || f.ReceivedBytes() <= downloaded {
|
||||
t.Fatalf("selected progress = %d, received traffic = %d; want 48 KiB of progress and traffic across the shared piece", downloaded, f.ReceivedBytes())
|
||||
}
|
||||
}
|
||||
|
||||
func TestBTProgressOnlyCountsVerifiedFileBytes(t *testing.T) {
|
||||
const fileSize = 56 * 1024
|
||||
fixture := makeBTFixtureWithPieceSize(t, map[string][]byte{
|
||||
"a.bin": btPayload(fileSize, "verified-a"),
|
||||
"b.bin": btPayload(fileSize, "verified-b"),
|
||||
}, 16*1024, false, true, 64*1024)
|
||||
f := btDownloadFetcher(t, nil)
|
||||
btResolve(t, f, fixture.Magnet)
|
||||
dir := t.TempDir()
|
||||
f.Meta().Opts = &base.Options{Path: dir}
|
||||
btStart(t, f)
|
||||
deadline := time.Now().Add(8 * time.Second)
|
||||
for f.torrent.Files()[0].BytesCompleted() == 0 && time.Now().Before(deadline) {
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
btPause(t, f)
|
||||
if f.torrent.Files()[0].BytesCompleted() == 0 || f.torrent.Piece(0).State().Complete {
|
||||
t.Fatal("expected some received bytes in a piece that is still incomplete")
|
||||
}
|
||||
if f.ReceivedBytes() == 0 {
|
||||
t.Fatal("received traffic must be visible before a piece is verified")
|
||||
}
|
||||
if progress := f.Progress(); len(progress) != 2 || progress[0] != 0 {
|
||||
t.Errorf("progress before the first file's piece is verified = %v, want zero for the first file", progress)
|
||||
}
|
||||
fixture.Limiter.SetLimit(rate.Inf)
|
||||
btStart(t, f)
|
||||
btAwaitDone(t, f)
|
||||
fixture.AssertFiles(t, dir)
|
||||
progress := f.Progress()
|
||||
if len(progress) != 2 || progress[0] != fileSize || progress[1] != fileSize {
|
||||
t.Fatalf("verified progress at completion = %v, want %d for each file", progress, fileSize)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBTUploadedBytesSamplesLiveTraffic(t *testing.T) {
|
||||
payload := btPayload(48*1024, "upload-counter")
|
||||
fixture := newBTFixture(t, map[string][]byte{"": payload}, 0)
|
||||
const saved = 1024
|
||||
f := &Fetcher{
|
||||
torrent: fixture.Torrent,
|
||||
data: &fetcherData{SeedBytes: saved},
|
||||
seedBaseBytes: saved,
|
||||
}
|
||||
f.torrentReady.Store(true)
|
||||
f.torrentUpload.Store(true)
|
||||
|
||||
cfg := torrent.TestingConfig(t)
|
||||
cfg.DataDir = t.TempDir()
|
||||
cfg.DisableTrackers = true
|
||||
cfg.DisableUTP = true
|
||||
cfg.DisableIPv6 = true
|
||||
peer, err := torrent.NewClient(cfg)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { peer.Close() })
|
||||
download, _, err := peer.AddTorrentSpec(torrent.TorrentSpecFromMetaInfo(fixture.MetaInfo))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
download.AddPeers([]torrent.PeerInfo{{Addr: fixture.Seeder.ListenAddrs()[0]}})
|
||||
download.DownloadAll()
|
||||
select {
|
||||
case <-download.Complete().On():
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatal("loopback peer did not complete")
|
||||
}
|
||||
if got, want := f.UploadedBytes(), int64(saved+len(payload)); got != want {
|
||||
t.Fatalf("uploaded bytes = %d, want %d including the saved bytes", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBTPauseAndResume(t *testing.T) {
|
||||
|
||||
@@ -10,7 +10,6 @@ import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
@@ -21,7 +20,7 @@ import (
|
||||
"github.com/GopeedLab/gopeed/pkg/base"
|
||||
"github.com/GopeedLab/gopeed/pkg/protocol/bt"
|
||||
"github.com/GopeedLab/gopeed/pkg/util"
|
||||
"github.com/RoaringBitmap/roaring/v2"
|
||||
"github.com/RoaringBitmap/roaring"
|
||||
"github.com/anacrolix/torrent"
|
||||
"github.com/anacrolix/torrent/metainfo"
|
||||
"github.com/anacrolix/torrent/storage"
|
||||
@@ -43,10 +42,9 @@ type Fetcher struct {
|
||||
ctl *controller.Controller
|
||||
config *config
|
||||
|
||||
torrent *torrent.Torrent
|
||||
meta *fetcher.FetcherMeta
|
||||
data *fetcherData
|
||||
seedBaseBytes int64
|
||||
torrent *torrent.Torrent
|
||||
meta *fetcher.FetcherMeta
|
||||
data *fetcherData
|
||||
|
||||
torrentReady atomic.Bool
|
||||
torrentUpload atomic.Bool
|
||||
@@ -69,11 +67,16 @@ func (f *Fetcher) Setup(ctl *controller.Controller) {
|
||||
return
|
||||
}
|
||||
|
||||
// initClient must be called with lock held.
|
||||
func (f *Fetcher) initClient() (err error) {
|
||||
lock.Lock()
|
||||
defer lock.Unlock()
|
||||
|
||||
if client != nil {
|
||||
return
|
||||
}
|
||||
if closeCtx == nil {
|
||||
closeCtx, closeFunc = context.WithCancel(context.Background())
|
||||
}
|
||||
|
||||
cfg = torrent.NewDefaultClientConfig()
|
||||
cfg.Seed = true
|
||||
@@ -91,23 +94,12 @@ func (f *Fetcher) initClient() (err error) {
|
||||
}
|
||||
|
||||
closeCtx, closeFunc = context.WithCancel(context.Background())
|
||||
// closeClient may reset closeCtx before the goroutine runs, so pass it as an argument.
|
||||
go dnsResolver.Run(closeCtx)
|
||||
go func() {
|
||||
dnsResolver.Run(closeCtx)
|
||||
}()
|
||||
return
|
||||
}
|
||||
|
||||
// addTorrentSpec holds lock from client initialization until the torrent is added,
|
||||
// so closeClient cannot close or reset the client in between.
|
||||
func (f *Fetcher) addTorrentSpec(spec *torrent.TorrentSpec) (*torrent.Torrent, bool, error) {
|
||||
lock.Lock()
|
||||
defer lock.Unlock()
|
||||
|
||||
if err := f.initClient(); err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
return client.AddTorrentSpec(spec)
|
||||
}
|
||||
|
||||
func (f *Fetcher) Resolve(req *base.Request) error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
@@ -465,46 +457,14 @@ func (f *Fetcher) Progress() fetcher.Progress {
|
||||
if !f.torrentReady.Load() {
|
||||
return append(fetcher.Progress(nil), f.data.Progress...)
|
||||
}
|
||||
pieceSize := f.torrent.Info().PieceLength
|
||||
var offset int64
|
||||
var verified []verifiedByteRange
|
||||
for _, run := range f.torrent.PieceStateRuns() {
|
||||
end := offset + int64(run.Length)*pieceSize
|
||||
if pieceVerified(run.PieceState) {
|
||||
verified = append(verified, verifiedByteRange{start: offset, end: end})
|
||||
}
|
||||
offset = end
|
||||
}
|
||||
files := f.torrent.Files()
|
||||
for i := range f.data.Progress {
|
||||
selectIndex := f.meta.Opts.SelectFiles[i]
|
||||
file := files[selectIndex]
|
||||
f.data.Progress[i] = verifiedFileBytes(verified, file.Offset(), file.Length())
|
||||
file := f.torrent.Files()[selectIndex]
|
||||
f.data.Progress[i] = file.BytesCompleted()
|
||||
}
|
||||
return append(fetcher.Progress(nil), f.data.Progress...)
|
||||
}
|
||||
|
||||
type verifiedByteRange struct {
|
||||
start, end int64
|
||||
}
|
||||
|
||||
func verifiedFileBytes(verified []verifiedByteRange, offset, length int64) int64 {
|
||||
first := sort.Search(len(verified), func(i int) bool { return verified[i].end > offset })
|
||||
fileEnd := offset + length
|
||||
var completed int64
|
||||
for _, run := range verified[first:] {
|
||||
if run.start >= fileEnd {
|
||||
break
|
||||
}
|
||||
completed += min(fileEnd, run.end) - max(offset, run.start)
|
||||
}
|
||||
return completed
|
||||
}
|
||||
|
||||
func pieceVerified(state torrent.PieceState) bool {
|
||||
return state.Ok && state.Complete && !state.Hashing && !state.QueuedForHash
|
||||
}
|
||||
|
||||
func (f *Fetcher) Wait() error {
|
||||
f.mu.Lock()
|
||||
var closed <-chan struct{}
|
||||
@@ -621,7 +581,7 @@ func (f *Fetcher) isDoneLocked() bool {
|
||||
file := f.torrent.Files()[index]
|
||||
for i := file.BeginPieceIndex(); i < file.EndPieceIndex(); i++ {
|
||||
state := f.torrent.Piece(i).State()
|
||||
if !pieceVerified(state) {
|
||||
if !state.Ok || !state.Complete || state.Hashing || state.QueuedForHash {
|
||||
return false
|
||||
}
|
||||
}
|
||||
@@ -719,13 +679,11 @@ func (f *Fetcher) Upload() (err error) {
|
||||
}
|
||||
|
||||
func (f *Fetcher) doUpload(fromUpload bool) {
|
||||
f.mu.Lock()
|
||||
if !f.torrentUpload.CompareAndSwap(false, true) {
|
||||
f.mu.Unlock()
|
||||
return
|
||||
}
|
||||
f.seedBaseBytes = f.data.SeedBytes
|
||||
initialTime := f.data.SeedTime
|
||||
f.mu.Lock()
|
||||
initialBytes, initialTime := f.data.SeedBytes, f.data.SeedTime
|
||||
torrentClosed := f.torrent.Closed()
|
||||
f.mu.Unlock()
|
||||
var doneTime int64
|
||||
@@ -743,7 +701,8 @@ func (f *Fetcher) doUpload(fromUpload bool) {
|
||||
f.mu.Unlock()
|
||||
continue
|
||||
}
|
||||
f.refreshSeedBytesLocked()
|
||||
stats := f.torrentStats()
|
||||
f.data.SeedBytes = initialBytes + stats.BytesWrittenData.Int64()
|
||||
if !fromUpload && !f.isDoneLocked() {
|
||||
f.mu.Unlock()
|
||||
continue
|
||||
@@ -776,27 +735,9 @@ func (f *Fetcher) torrentStats() torrent.TorrentStats {
|
||||
func (f *Fetcher) UploadedBytes() int64 {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if f.torrentReady.Load() && f.torrentUpload.Load() {
|
||||
f.refreshSeedBytesLocked()
|
||||
}
|
||||
return f.data.SeedBytes
|
||||
}
|
||||
|
||||
func (f *Fetcher) refreshSeedBytesLocked() {
|
||||
stats := f.torrentStats()
|
||||
f.data.SeedBytes = max(f.data.SeedBytes, f.seedBaseBytes+stats.BytesWrittenData.Int64())
|
||||
}
|
||||
|
||||
func (f *Fetcher) ReceivedBytes() int64 {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if !f.torrentReady.Load() {
|
||||
return 0
|
||||
}
|
||||
stats := f.torrentStats()
|
||||
return stats.BytesReadUsefulData.Int64()
|
||||
}
|
||||
|
||||
func (f *Fetcher) WaitUpload() (err error) {
|
||||
<-f.uploadDoneCh
|
||||
return nil
|
||||
@@ -807,6 +748,9 @@ func (f *Fetcher) addTorrent(req *base.Request) (err error) {
|
||||
if err = base.ParseReqExtra[bt.ReqExtra](req); err != nil {
|
||||
return
|
||||
}
|
||||
if err = f.initClient(); err != nil {
|
||||
return
|
||||
}
|
||||
schema := util.ParseSchema(req.URL)
|
||||
privateTorrent := false
|
||||
var spec *torrent.TorrentSpec
|
||||
@@ -885,7 +829,7 @@ func (f *Fetcher) addTorrent(req *base.Request) (err error) {
|
||||
s := &deferredFileStorage{}
|
||||
spec.Storage = s
|
||||
var added bool
|
||||
f.torrent, added, err = f.addTorrentSpec(spec)
|
||||
f.torrent, added, err = client.AddTorrentSpec(spec)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -8,7 +8,6 @@ import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/GopeedLab/gopeed/internal/controller"
|
||||
@@ -72,82 +71,6 @@ func TestFetcher_ResolveWithProxy(t *testing.T) {
|
||||
}))
|
||||
}
|
||||
|
||||
func TestFetcher_CloseAfterInit(t *testing.T) {
|
||||
if err := closeClient(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for i := 0; i < 100; i++ {
|
||||
f := buildFetcher(t).(*Fetcher)
|
||||
f.meta.Req = &base.Request{}
|
||||
lock.Lock()
|
||||
err := f.initClient()
|
||||
lock.Unlock()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := closeClient(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetcher_CloseWithoutClient(t *testing.T) {
|
||||
if err := closeClient(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
t.Run("restored task", func(t *testing.T) {
|
||||
_, build := new(FetcherManager).Restore()
|
||||
f := build(&fetcher.FetcherMeta{
|
||||
Req: &base.Request{URL: "./testdata/test.torrent"},
|
||||
Opts: &base.Options{},
|
||||
}, &fetcherData{})
|
||||
f.Setup(controller.NewController())
|
||||
if err := f.Close(); err != nil {
|
||||
t.Errorf("Close() got = %v, want nil", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("failed resolve", func(t *testing.T) {
|
||||
f := buildFetcher(t)
|
||||
if err := f.Resolve(&base.Request{URL: "./testdata/not-exist.torrent"}); err == nil {
|
||||
t.Fatal("Resolve() got = nil, want error")
|
||||
}
|
||||
if err := f.Close(); err != nil {
|
||||
t.Errorf("Close() got = %v, want nil", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestFetcher_ResolveWhileClientCloses(t *testing.T) {
|
||||
if err := closeClient(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { closeClient() })
|
||||
|
||||
var wg sync.WaitGroup
|
||||
for _, u := range []string{"./testdata/test.torrent", "./testdata/ubuntu-22.04-live-server-amd64.iso.torrent"} {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for i := 0; i < 10; i++ {
|
||||
f := buildFetcher(t)
|
||||
// The other goroutine may close the client after the torrent was added.
|
||||
if err := f.Resolve(&base.Request{URL: u}); err != nil && err.Error() != "torrent closed during resolution" {
|
||||
t.Errorf("Resolve(%s) got = %v", u, err)
|
||||
return
|
||||
}
|
||||
f.Close()
|
||||
if err := closeClient(); err != nil {
|
||||
t.Errorf("closeClient() got = %v", err)
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
func doResolve(t *testing.T, fetcher fetcher.Fetcher) {
|
||||
t.Run("Resolve Single File", func(t *testing.T) {
|
||||
err := fetcher.Resolve(&base.Request{
|
||||
|
||||
@@ -46,11 +46,6 @@ func newBTHybridFixture(t testing.TB, payload []byte) *btFixture {
|
||||
}
|
||||
|
||||
func makeBTFixture(t testing.TB, files map[string][]byte, bytesPerSecond int, v2, private bool) *btFixture {
|
||||
t.Helper()
|
||||
return makeBTFixtureWithPieceSize(t, files, bytesPerSecond, v2, private, 16*1024)
|
||||
}
|
||||
|
||||
func makeBTFixtureWithPieceSize(t testing.TB, files map[string][]byte, bytesPerSecond int, v2, private bool, pieceSize int64) *btFixture {
|
||||
t.Helper()
|
||||
source := t.TempDir()
|
||||
name := "payload.bin"
|
||||
@@ -69,7 +64,7 @@ func makeBTFixtureWithPieceSize(t testing.TB, files map[string][]byte, bytesPerS
|
||||
}
|
||||
expected[filepath.Join(name, filepath.FromSlash(path))] = append([]byte(nil), data...)
|
||||
}
|
||||
info := metainfo.Info{PieceLength: pieceSize}
|
||||
info := metainfo.Info{PieceLength: 16 * 1024}
|
||||
if err := info.BuildFromFilePath(root); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -6,7 +6,7 @@ import (
|
||||
"github.com/GopeedLab/gopeed/internal/fetcher"
|
||||
"github.com/GopeedLab/gopeed/pkg/base"
|
||||
protocolbt "github.com/GopeedLab/gopeed/pkg/protocol/bt"
|
||||
"github.com/RoaringBitmap/roaring/v2"
|
||||
"github.com/RoaringBitmap/roaring"
|
||||
"github.com/anacrolix/torrent"
|
||||
"github.com/anacrolix/torrent/storage"
|
||||
)
|
||||
|
||||
@@ -1,404 +0,0 @@
|
||||
package ftp
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/textproto"
|
||||
"net/url"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/jlaffaye/ftp"
|
||||
"golang.org/x/net/proxy"
|
||||
|
||||
pftp "github.com/GopeedLab/gopeed/pkg/protocol/ftp"
|
||||
)
|
||||
|
||||
const (
|
||||
connectTimeout = 15 * time.Second
|
||||
readTimeout = 15 * time.Second
|
||||
quitTimeout = 2 * time.Second
|
||||
)
|
||||
|
||||
// target is a parsed FTP URL. It holds the password, so it must never be
|
||||
// formatted into an error or a log line.
|
||||
type target struct {
|
||||
host string // host name or IP, the TLS server name
|
||||
addr string // host:port
|
||||
user string
|
||||
pass string
|
||||
path string // path relative to the login directory, "" for the directory itself
|
||||
tlsMode string
|
||||
}
|
||||
|
||||
// parseTarget reads an ftp, ftps or ftpes URL. The path follows RFC 1738: it
|
||||
// is relative to the login directory, and an encoded leading slash
|
||||
// (ftp://host/%2Fpub) makes it absolute.
|
||||
func parseTarget(rawURL, optTLS string) (*target, error) {
|
||||
u, err := url.Parse(rawURL)
|
||||
if err != nil {
|
||||
// url.Error repeats the whole URL, password included.
|
||||
return nil, errors.New("ftp: invalid URL")
|
||||
}
|
||||
t := &target{}
|
||||
switch strings.ToLower(u.Scheme) {
|
||||
case "ftps":
|
||||
t.tlsMode = pftp.TLSImplicit
|
||||
case "ftpes":
|
||||
t.tlsMode = pftp.TLSExplicit
|
||||
case "ftp":
|
||||
switch strings.ToLower(optTLS) {
|
||||
case "", pftp.TLSNone:
|
||||
t.tlsMode = pftp.TLSNone
|
||||
case pftp.TLSExplicit:
|
||||
t.tlsMode = pftp.TLSExplicit
|
||||
case pftp.TLSImplicit:
|
||||
t.tlsMode = pftp.TLSImplicit
|
||||
default:
|
||||
return nil, fmt.Errorf("ftp: unknown tls mode %q", optTLS)
|
||||
}
|
||||
default:
|
||||
return nil, fmt.Errorf("ftp: unsupported scheme %q", u.Scheme)
|
||||
}
|
||||
t.host = u.Hostname()
|
||||
if t.host == "" {
|
||||
return nil, errors.New("ftp: URL has no host")
|
||||
}
|
||||
port := u.Port()
|
||||
if port == "" {
|
||||
port = "21"
|
||||
if t.tlsMode == pftp.TLSImplicit {
|
||||
port = "990"
|
||||
}
|
||||
}
|
||||
t.addr = net.JoinHostPort(t.host, port)
|
||||
if u.User != nil {
|
||||
// Username and Password are already percent-decoded.
|
||||
t.user = u.User.Username()
|
||||
t.pass, _ = u.User.Password()
|
||||
}
|
||||
if t.user == "" {
|
||||
t.user, t.pass = "anonymous", "anonymous@"
|
||||
}
|
||||
t.path = strings.TrimPrefix(u.Path, "/")
|
||||
return t, nil
|
||||
}
|
||||
|
||||
type dialContextFunc func(ctx context.Context, network, addr string) (net.Conn, error)
|
||||
|
||||
// session is one logged-in control connection. It owns the network
|
||||
// connections it opens, so abort can unblock any read or write in progress.
|
||||
type session struct {
|
||||
*ftp.ServerConn
|
||||
|
||||
ctrl net.Conn // control connection as dialed, under any TLS layer
|
||||
dataHost string // where data connections go
|
||||
|
||||
mu sync.Mutex
|
||||
data net.Conn // current data connection as dialed
|
||||
closed bool
|
||||
stop func() bool
|
||||
}
|
||||
|
||||
// deadline bounds the next control-connection exchange. The control
|
||||
// connection is idle during a transfer, so every exchange sets a new one.
|
||||
func (s *session) deadline() {
|
||||
_ = s.ctrl.SetDeadline(time.Now().Add(readTimeout))
|
||||
}
|
||||
|
||||
// clearDeadline lets the control connection stay idle through a transfer.
|
||||
func (s *session) clearDeadline() {
|
||||
_ = s.ctrl.SetDeadline(time.Time{})
|
||||
}
|
||||
|
||||
// abort closes every connection at once. Pending calls fail immediately.
|
||||
func (s *session) abort() {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.closed = true
|
||||
_ = s.ctrl.Close()
|
||||
if s.data != nil {
|
||||
_ = s.data.Close()
|
||||
}
|
||||
}
|
||||
|
||||
// quit logs out and waits, briefly, for the server to hang up. Servers that
|
||||
// cap logins count a session until they have seen it end, so the next login
|
||||
// would otherwise be refused.
|
||||
func (s *session) quit() {
|
||||
if s.stop != nil {
|
||||
s.stop()
|
||||
}
|
||||
_ = s.ctrl.SetDeadline(time.Now().Add(quitTimeout))
|
||||
if s.ServerConn != nil {
|
||||
_ = s.ServerConn.Quit()
|
||||
}
|
||||
s.abort()
|
||||
}
|
||||
|
||||
func (s *session) setData(c net.Conn) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.closed {
|
||||
_ = c.Close()
|
||||
return net.ErrClosed
|
||||
}
|
||||
s.data = c
|
||||
return nil
|
||||
}
|
||||
|
||||
// ctrlConn is the control connection handed to the ftp library.
|
||||
type ctrlConn struct {
|
||||
net.Conn
|
||||
}
|
||||
|
||||
// RemoteAddr must be a *net.TCPAddr: the library asserts it. A connection
|
||||
// through a proxy may report something else, and its address is not used
|
||||
// for data connections anyway (see dialData).
|
||||
func (c ctrlConn) RemoteAddr() net.Addr {
|
||||
if a, ok := c.Conn.RemoteAddr().(*net.TCPAddr); ok {
|
||||
return a
|
||||
}
|
||||
return &net.TCPAddr{}
|
||||
}
|
||||
|
||||
// Close is called by the library's Quit right after it sends QUIT. Reading
|
||||
// until the server hangs up (bounded by the deadline quit set) tells us the
|
||||
// server has ended the session.
|
||||
func (c ctrlConn) Close() error {
|
||||
_, _ = io.Copy(io.Discard, c.Conn)
|
||||
return c.Conn.Close()
|
||||
}
|
||||
|
||||
// dial opens a control connection, upgrades it to TLS when asked, and logs in.
|
||||
//
|
||||
// The library is given a dial function for every connection. That is how
|
||||
// data connections go through the same proxy as the control connection, but
|
||||
// it also means the library does no TLS on them: with a dial function it
|
||||
// neither wraps an implicit-TLS control connection nor any data connection.
|
||||
// We do both here. Explicit TLS (AUTH TLS) is still done by the library, and
|
||||
// so are PBSZ and PROT P after login. For the same reason the library's
|
||||
// DialWithTimeout and DialWithContext would have no effect: connectTimeout
|
||||
// and ctx are applied to our own dials instead.
|
||||
func (f *Fetcher) dial(ctx context.Context, t *target) (*session, error) {
|
||||
netDial, proxied, err := f.netDialer(t)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var tlsConf *tls.Config
|
||||
if t.tlsMode != pftp.TLSNone {
|
||||
tlsConf = f.tlsConfig(t.host)
|
||||
}
|
||||
|
||||
dialCtx, cancel := context.WithTimeout(ctx, connectTimeout)
|
||||
defer cancel()
|
||||
raw, err := netDial(dialCtx, "tcp", t.addr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ctrl := raw
|
||||
if t.tlsMode == pftp.TLSImplicit {
|
||||
tc := tls.Client(raw, tlsConf)
|
||||
if err := tc.HandshakeContext(dialCtx); err != nil {
|
||||
_ = raw.Close()
|
||||
return nil, err
|
||||
}
|
||||
ctrl = tc
|
||||
}
|
||||
|
||||
s := &session{ctrl: raw, dataHost: t.host}
|
||||
if a, ok := raw.RemoteAddr().(*net.TCPAddr); ok && !proxied {
|
||||
// Same IP as the control connection, as the library itself does,
|
||||
// so a name with several addresses cannot send us elsewhere.
|
||||
s.dataHost = a.IP.String()
|
||||
}
|
||||
s.stop = context.AfterFunc(ctx, s.abort)
|
||||
s.deadline()
|
||||
|
||||
handedOut := false
|
||||
opts := []ftp.DialOption{
|
||||
ftp.DialWithShutTimeout(readTimeout),
|
||||
ftp.DialWithDialFunc(func(network, addr string) (net.Conn, error) {
|
||||
if !handedOut {
|
||||
// Dial asks for the control connection first, once.
|
||||
handedOut = true
|
||||
return ctrlConn{ctrl}, nil
|
||||
}
|
||||
return s.dialData(ctx, netDial, tlsConf, addr)
|
||||
}),
|
||||
}
|
||||
switch t.tlsMode {
|
||||
case pftp.TLSImplicit:
|
||||
opts = append(opts, ftp.DialWithTLS(tlsConf))
|
||||
case pftp.TLSExplicit:
|
||||
opts = append(opts, ftp.DialWithExplicitTLS(tlsConf))
|
||||
}
|
||||
c, err := ftp.Dial(t.addr, opts...)
|
||||
if err != nil {
|
||||
s.stop()
|
||||
s.abort()
|
||||
return nil, err
|
||||
}
|
||||
s.ServerConn = c
|
||||
s.deadline()
|
||||
if err := c.Login(t.user, t.pass); err != nil {
|
||||
s.stop()
|
||||
s.abort()
|
||||
return nil, err
|
||||
}
|
||||
return s, nil
|
||||
}
|
||||
|
||||
// dialData opens a passive data connection. The library takes the host from
|
||||
// the control connection's peer, which through a proxy is the proxy, so only
|
||||
// the port is used here.
|
||||
func (s *session) dialData(ctx context.Context, netDial dialContextFunc, tlsConf *tls.Config, addr string) (net.Conn, error) {
|
||||
_, port, err := net.SplitHostPort(addr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
dialCtx, cancel := context.WithTimeout(ctx, connectTimeout)
|
||||
defer cancel()
|
||||
raw, err := netDial(dialCtx, "tcp", net.JoinHostPort(s.dataHost, port))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := s.setData(raw); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if tlsConf != nil {
|
||||
// The handshake runs on the first read, as in the library, because
|
||||
// some servers only start it once the transfer command is answered.
|
||||
return tls.Client(raw, tlsConf), nil
|
||||
}
|
||||
return raw, nil
|
||||
}
|
||||
|
||||
// tlsConfig returns the TLS settings for one control connection and its data
|
||||
// connections. Certificates are verified unless the request sets
|
||||
// skipVerifyCert, as for HTTP.
|
||||
//
|
||||
// Each call gets its own one-entry session cache, so a data connection
|
||||
// resumes the TLS session of its own control connection. vsftpd
|
||||
// (require_ssl_reuse) only checks that some session was resumed, but ProFTPD's
|
||||
// mod_tls also checks that it is the control connection's; a cache shared by
|
||||
// several workers would hand a data connection another worker's session.
|
||||
// A cache in the manager's base config is kept as given.
|
||||
func (f *Fetcher) tlsConfig(host string) *tls.Config {
|
||||
cfg := &tls.Config{}
|
||||
if f.manager != nil && f.manager.baseTLS != nil {
|
||||
cfg = f.manager.baseTLS.Clone()
|
||||
}
|
||||
cfg.ServerName = host
|
||||
if f.meta.Req != nil && f.meta.Req.SkipVerifyCert {
|
||||
cfg.InsecureSkipVerify = true
|
||||
}
|
||||
if cfg.ClientSessionCache == nil {
|
||||
cfg.ClientSessionCache = tls.NewLRUClientSessionCache(1)
|
||||
}
|
||||
return cfg
|
||||
}
|
||||
|
||||
// netDialer picks how to reach the server, from the same proxy settings the
|
||||
// HTTP protocol reads (the task's proxy, otherwise the global one).
|
||||
func (f *Fetcher) netDialer(t *target) (dialContextFunc, bool, error) {
|
||||
direct := (&net.Dialer{Timeout: connectTimeout}).DialContext
|
||||
pu, err := f.proxyURL(t)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
if pu == nil {
|
||||
return direct, false, nil
|
||||
}
|
||||
switch strings.ToLower(pu.Scheme) {
|
||||
case "socks5", "socks5h":
|
||||
d, err := proxy.FromURL(pu, &net.Dialer{Timeout: connectTimeout})
|
||||
if err != nil {
|
||||
return nil, false, fmt.Errorf("ftp: socks5 proxy: %w", err)
|
||||
}
|
||||
if cd, ok := d.(proxy.ContextDialer); ok {
|
||||
return cd.DialContext, true, nil
|
||||
}
|
||||
return func(_ context.Context, network, addr string) (net.Conn, error) {
|
||||
return d.Dial(network, addr)
|
||||
}, true, nil
|
||||
default:
|
||||
// An HTTP proxy is not used. Every passive data connection goes to
|
||||
// a port the server picks, and HTTP proxies usually allow CONNECT to
|
||||
// port 443 only, so the transfer would fail after the login worked.
|
||||
f.warnOnce.Do(func() {
|
||||
if l := f.logger(); l != nil {
|
||||
l.Warn().Msgf("ftp: the %s proxy at %s cannot carry FTP data connections (HTTP proxy), connecting directly; use a SOCKS5 proxy for FTP", strings.ToLower(pu.Scheme), pu.Host)
|
||||
}
|
||||
})
|
||||
return direct, false, nil
|
||||
}
|
||||
}
|
||||
|
||||
func (f *Fetcher) proxyURL(t *target) (*url.URL, error) {
|
||||
if f.ctl == nil || f.ctl.GetProxy == nil {
|
||||
return nil, nil
|
||||
}
|
||||
handler := f.ctl.GetProxy(f.meta.Req.Proxy)
|
||||
if handler == nil {
|
||||
return nil, nil
|
||||
}
|
||||
// The proxy chooser reads only the URL; the credentials stay out.
|
||||
req := &http.Request{URL: &url.URL{Scheme: "ftp", Host: t.addr, Path: "/" + t.path}, Header: http.Header{}}
|
||||
pu, err := handler(req)
|
||||
if err != nil {
|
||||
return nil, errors.New("ftp: cannot read the proxy settings")
|
||||
}
|
||||
return pu, nil
|
||||
}
|
||||
|
||||
// isLoginRefusal tells a server that is full apart from bad credentials.
|
||||
// 421 always means "not now". 530 means a refused login; it is only a
|
||||
// capacity limit when these credentials have worked before. The library
|
||||
// drops the code when USER itself is refused, so a "too many" message counts
|
||||
// as well.
|
||||
func isLoginRefusal(err error, credentialsWorked bool) bool {
|
||||
var te *textproto.Error
|
||||
if errors.As(err, &te) {
|
||||
switch te.Code {
|
||||
case ftp.StatusNotAvailable:
|
||||
return true
|
||||
case ftp.StatusNotLoggedIn:
|
||||
return credentialsWorked || strings.Contains(strings.ToLower(te.Msg), "too many")
|
||||
}
|
||||
return false
|
||||
}
|
||||
return strings.Contains(strings.ToLower(err.Error()), "too many")
|
||||
}
|
||||
|
||||
// isPermanent reports errors a retry cannot fix: a 5xx reply and a
|
||||
// certificate that does not verify.
|
||||
func isPermanent(err error) bool {
|
||||
var te *textproto.Error
|
||||
if errors.As(err, &te) {
|
||||
return te.Code >= 500
|
||||
}
|
||||
var verifyErr *tls.CertificateVerificationError
|
||||
if errors.As(err, &verifyErr) {
|
||||
return true
|
||||
}
|
||||
var recordErr tls.RecordHeaderError
|
||||
if errors.As(err, &recordErr) {
|
||||
return true
|
||||
}
|
||||
return errors.Is(err, errSizeChanged)
|
||||
}
|
||||
|
||||
func replyCode(err error) int {
|
||||
var te *textproto.Error
|
||||
if errors.As(err, &te) {
|
||||
return te.Code
|
||||
}
|
||||
return 0
|
||||
}
|
||||
@@ -1,146 +0,0 @@
|
||||
package ftp_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/rs/zerolog"
|
||||
|
||||
"github.com/GopeedLab/gopeed/internal/logger"
|
||||
iftp "github.com/GopeedLab/gopeed/internal/protocol/ftp"
|
||||
"github.com/GopeedLab/gopeed/pkg/base"
|
||||
"github.com/GopeedLab/gopeed/pkg/download"
|
||||
pftp "github.com/GopeedLab/gopeed/pkg/protocol/ftp"
|
||||
)
|
||||
|
||||
// syncBuffer is a log sink that the downloader's goroutines can share.
|
||||
type syncBuffer struct {
|
||||
mu sync.Mutex
|
||||
buf bytes.Buffer
|
||||
}
|
||||
|
||||
func (b *syncBuffer) Write(p []byte) (int, error) {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
return b.buf.Write(p)
|
||||
}
|
||||
|
||||
func (b *syncBuffer) String() string {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
return b.buf.String()
|
||||
}
|
||||
|
||||
// taskEvents records the first done or error event of every task, so the
|
||||
// test never reads a task's status while the downloader writes it.
|
||||
type taskEvents struct {
|
||||
mu sync.Mutex
|
||||
final map[string]download.EventKey
|
||||
}
|
||||
|
||||
func (e *taskEvents) listen(ev *download.Event) {
|
||||
if ev.Key != download.EventKeyDone && ev.Key != download.EventKeyError {
|
||||
return
|
||||
}
|
||||
e.mu.Lock()
|
||||
defer e.mu.Unlock()
|
||||
if _, seen := e.final[ev.Task.ID]; !seen {
|
||||
e.final[ev.Task.ID] = ev.Key
|
||||
}
|
||||
}
|
||||
|
||||
func (e *taskEvents) wait(t *testing.T, id string) download.EventKey {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(60 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
e.mu.Lock()
|
||||
key, ok := e.final[id]
|
||||
e.mu.Unlock()
|
||||
if ok {
|
||||
return key
|
||||
}
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
}
|
||||
t.Fatalf("task %s did not finish", id)
|
||||
return ""
|
||||
}
|
||||
|
||||
// TestPasswordNeverLogged runs FTP tasks through the downloader with its core
|
||||
// logger captured: a success, a missing file and a wrong password, all behind
|
||||
// an HTTP proxy that FTP must skip with a warning.
|
||||
func TestPasswordNeverLogged(t *testing.T) {
|
||||
srv := iftp.StartFTPServer(t, iftp.ServerOptions{User: "u", Pass: "p@ss"})
|
||||
want := srv.WriteFile(t, "secret.bin", iftp.RandomBytes(300000, 21))
|
||||
|
||||
d := download.NewDownloader(&download.DownloaderConfig{Storage: download.NewMemStorage(), StorageDir: t.TempDir()})
|
||||
logs := &syncBuffer{}
|
||||
d.Logger = &logger.Logger{Logger: zerolog.New(logs)}
|
||||
events := &taskEvents{final: map[string]download.EventKey{}}
|
||||
d.Listener(events.listen)
|
||||
if err := d.Setup(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Close, not Clear: Clear resets the extension list without a lock while
|
||||
// the last task's done hooks may still read it.
|
||||
defer d.Close()
|
||||
cfg, err := d.GetConfig()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cfg.Proxy = &base.DownloaderProxyConfig{Enable: true, Scheme: "http", Host: "127.0.0.1:1", Usr: "proxyuser", Pwd: "proxysecret"}
|
||||
if err := d.PutConfig(cfg); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
dir := t.TempDir()
|
||||
create := func(rawURL string) string {
|
||||
id, err := d.CreateDirect(&base.Request{URL: rawURL}, &base.Options{Path: dir, Extra: &pftp.OptsExtra{Connections: 2}})
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "p@ss") || strings.Contains(err.Error(), "p%40ss") {
|
||||
t.Fatalf("the password leaked into an error: %v", err)
|
||||
}
|
||||
t.Fatal(err)
|
||||
}
|
||||
return id
|
||||
}
|
||||
root := "ftp://u:p%40ss@" + srv.Addr + "/"
|
||||
|
||||
okID := create(root + "secret.bin")
|
||||
if key := events.wait(t, okID); key != download.EventKeyDone {
|
||||
t.Fatalf("download ended with %q, logs:\n%s", key, logs.String())
|
||||
}
|
||||
ok := d.GetTask(okID)
|
||||
if ok.Protocol != "ftp" {
|
||||
t.Fatalf("protocol %q, want ftp", ok.Protocol)
|
||||
}
|
||||
if got := iftp.FileSHA256(t, ok.Meta.SingleFilepath()); got != want {
|
||||
t.Fatalf("sha256 %s, want %s", got, want)
|
||||
}
|
||||
missingID := create(root + "missing.bin")
|
||||
if key := events.wait(t, missingID); key != download.EventKeyError {
|
||||
t.Fatalf("a missing file must fail, got %q", key)
|
||||
}
|
||||
wrongID := create("ftp://u:wr0ng%40pw@" + srv.Addr + "/secret.bin")
|
||||
if key := events.wait(t, wrongID); key != download.EventKeyError {
|
||||
t.Fatalf("a wrong password must fail, got %q", key)
|
||||
}
|
||||
missing, wrong := d.GetTask(missingID), d.GetTask(wrongID)
|
||||
for _, task := range []*download.Task{ok, missing, wrong} {
|
||||
if name := task.Name(); strings.Contains(name, "p@ss") || strings.Contains(name, "p%40ss") || strings.Contains(name, "wr0ng") {
|
||||
t.Fatalf("task name %q shows the password", name)
|
||||
}
|
||||
}
|
||||
|
||||
out := logs.String()
|
||||
if !strings.Contains(out, "task id") || !strings.Contains(out, "HTTP proxy") {
|
||||
t.Fatalf("the captured log misses the expected lines, so it proves nothing:\n%s", out)
|
||||
}
|
||||
for _, secret := range []string{"p@ss", "p%40ss", "wr0ng", "proxysecret"} {
|
||||
if strings.Contains(out, secret) {
|
||||
t.Fatalf("%q appears in the log:\n%s", secret, out)
|
||||
}
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
@@ -1,109 +0,0 @@
|
||||
package ftp
|
||||
|
||||
import (
|
||||
"crypto/tls"
|
||||
"net/url"
|
||||
"path"
|
||||
"strings"
|
||||
|
||||
"github.com/GopeedLab/gopeed/internal/fetcher"
|
||||
"github.com/GopeedLab/gopeed/pkg/base"
|
||||
pftp "github.com/GopeedLab/gopeed/pkg/protocol/ftp"
|
||||
)
|
||||
|
||||
// fetcherData is what a task saves to resume later. Segments has the same
|
||||
// Begin, End and Downloaded fields every version writes; the other fields are
|
||||
// optional, so a save without them still restores.
|
||||
type fetcherData struct {
|
||||
Segments []*segment
|
||||
// MaxConns is the login ceiling the server enforced, 0 when none was hit.
|
||||
MaxConns int `json:",omitempty"`
|
||||
// NoRest records that the server refused REST, so the task downloads each
|
||||
// file in one sequential stream.
|
||||
NoRest bool `json:",omitempty"`
|
||||
}
|
||||
|
||||
// FetcherManager handles ftp://, ftps:// (implicit TLS) and ftpes:// (explicit
|
||||
// TLS) URLs.
|
||||
type FetcherManager struct {
|
||||
// baseTLS is the base for every TLS connection. Nil means the system
|
||||
// roots; the package's tests set it to trust their own certificate. The
|
||||
// task's skipVerifyCert still turns verification off.
|
||||
baseTLS *tls.Config
|
||||
}
|
||||
|
||||
func (fm *FetcherManager) Name() string {
|
||||
return "ftp"
|
||||
}
|
||||
|
||||
func (fm *FetcherManager) Filters() []*fetcher.SchemeFilter {
|
||||
return []*fetcher.SchemeFilter{
|
||||
{Type: fetcher.FilterTypeUrl, Pattern: "FTP"},
|
||||
{Type: fetcher.FilterTypeUrl, Pattern: "FTPS"},
|
||||
{Type: fetcher.FilterTypeUrl, Pattern: "FTPES"},
|
||||
}
|
||||
}
|
||||
|
||||
func (fm *FetcherManager) Build() fetcher.Fetcher {
|
||||
return &Fetcher{manager: fm}
|
||||
}
|
||||
|
||||
// ParseName names a task before it is resolved. It never includes the
|
||||
// userinfo of the URL.
|
||||
func (fm *FetcherManager) ParseName(u string) string {
|
||||
parsed, err := url.Parse(u)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
name := path.Base(strings.TrimSuffix(parsed.Path, "/"))
|
||||
if name == "" || name == "/" || name == "." {
|
||||
name = parsed.Hostname()
|
||||
}
|
||||
return name
|
||||
}
|
||||
|
||||
func (fm *FetcherManager) AutoRename() bool {
|
||||
return true
|
||||
}
|
||||
|
||||
func (fm *FetcherManager) DefaultConfig() any {
|
||||
return &config{Connections: defaultConnections}
|
||||
}
|
||||
|
||||
func (fm *FetcherManager) Store(f fetcher.Fetcher) (any, error) {
|
||||
ff := f.(*Fetcher)
|
||||
ff.mu.Lock()
|
||||
defer ff.mu.Unlock()
|
||||
segments := make([]*segment, len(ff.segments))
|
||||
for i, s := range ff.segments {
|
||||
c := *s
|
||||
segments[i] = &c
|
||||
}
|
||||
return &fetcherData{
|
||||
Segments: segments,
|
||||
MaxConns: ff.maxConns,
|
||||
NoRest: ff.noRest,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (fm *FetcherManager) Restore() (v any, f func(meta *fetcher.FetcherMeta, v any) fetcher.Fetcher) {
|
||||
return &fetcherData{}, func(meta *fetcher.FetcherMeta, v any) fetcher.Fetcher {
|
||||
fd := v.(*fetcherData)
|
||||
ff := fm.Build().(*Fetcher)
|
||||
ff.meta = meta
|
||||
if meta != nil && meta.Req != nil {
|
||||
_ = base.ParseReqExtra[pftp.ReqExtra](meta.Req)
|
||||
}
|
||||
if meta != nil && meta.Opts != nil {
|
||||
_ = base.ParseOptExtra[pftp.OptsExtra](meta.Opts)
|
||||
}
|
||||
ff.segments = fd.Segments
|
||||
ff.maxConns = fd.MaxConns
|
||||
ff.noRest = fd.NoRest
|
||||
return ff
|
||||
}
|
||||
}
|
||||
|
||||
func (fm *FetcherManager) Close() error {
|
||||
return nil
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
@@ -1,187 +0,0 @@
|
||||
package ftp
|
||||
|
||||
import (
|
||||
"cmp"
|
||||
"fmt"
|
||||
"path"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/jlaffaye/ftp"
|
||||
|
||||
"github.com/GopeedLab/gopeed/pkg/base"
|
||||
)
|
||||
|
||||
// Limits of a directory download. A tree past either one is refused with an
|
||||
// error that names the limit; it is never cut short.
|
||||
const maxWalkDepth = 8
|
||||
|
||||
// maxWalkFiles is a variable so tests can lower it.
|
||||
var maxWalkFiles = 10000
|
||||
|
||||
// resolveTarget describes the URL's path: one file, or a directory walked
|
||||
// into a multi-file resource whose paths are relative to it.
|
||||
func (f *Fetcher) resolveTarget(s *session, t *target) (*base.Resource, error) {
|
||||
isDir, size, mtime, err := f.stat(s, t.path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !isDir {
|
||||
return &base.Resource{
|
||||
Size: size,
|
||||
Range: true,
|
||||
Files: []*base.FileInfo{{
|
||||
Name: path.Base(t.path),
|
||||
Size: size,
|
||||
Ctime: mtime,
|
||||
}},
|
||||
}, nil
|
||||
}
|
||||
|
||||
files, err := f.walk(s, t.path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
name := path.Base(strings.TrimSuffix(t.path, "/"))
|
||||
if name == "" || name == "." || name == "/" {
|
||||
name = t.host
|
||||
}
|
||||
res := &base.Resource{Name: name, Range: true, Files: files}
|
||||
res.CalcSize(nil)
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// stat tells a file from a directory. MLST answers both at once. Without
|
||||
// it, SIZE answers for a file, and a directory is one we can change into.
|
||||
func (f *Fetcher) stat(s *session, p string) (isDir bool, size int64, mtime *time.Time, err error) {
|
||||
if p == "" || strings.HasSuffix(p, "/") {
|
||||
return true, 0, nil, nil
|
||||
}
|
||||
s.deadline()
|
||||
if entry, mlstErr := s.GetEntry(p); mlstErr == nil {
|
||||
switch entry.Type {
|
||||
case ftp.EntryTypeFolder:
|
||||
return true, 0, nil, nil
|
||||
case ftp.EntryTypeFile:
|
||||
if !entry.Time.IsZero() {
|
||||
t := entry.Time
|
||||
mtime = &t
|
||||
}
|
||||
return false, int64(entry.Size), mtime, nil
|
||||
}
|
||||
// A link: SIZE below follows it on the server.
|
||||
}
|
||||
// MLST is missing or refused the path: the older commands decide. A
|
||||
// broken connection fails them too, with its own error.
|
||||
|
||||
s.deadline()
|
||||
size, sizeErr := s.FileSize(p)
|
||||
if sizeErr == nil {
|
||||
if s.IsGetTimeSupported() {
|
||||
s.deadline()
|
||||
if t, err := s.GetTime(p); err == nil {
|
||||
mtime = &t
|
||||
}
|
||||
}
|
||||
return false, size, mtime, nil
|
||||
}
|
||||
if replyCode(sizeErr) == 0 {
|
||||
return false, 0, nil, sizeErr
|
||||
}
|
||||
|
||||
// SIZE failed: a directory, or a missing path.
|
||||
s.deadline()
|
||||
cwd, err := s.CurrentDir()
|
||||
if err != nil {
|
||||
return false, 0, nil, err
|
||||
}
|
||||
s.deadline()
|
||||
if err := s.ChangeDir(p); err != nil {
|
||||
return false, 0, nil, sizeErr
|
||||
}
|
||||
s.deadline()
|
||||
if err := s.ChangeDir(cwd); err != nil {
|
||||
return false, 0, nil, err
|
||||
}
|
||||
return true, 0, nil, nil
|
||||
}
|
||||
|
||||
// safeEntryName reports whether a name from a listing can become a local
|
||||
// path element. A hostile server could otherwise list "../x" and write
|
||||
// outside the task directory.
|
||||
func safeEntryName(name string) bool {
|
||||
if name == "" || name == "." || name == ".." {
|
||||
return false
|
||||
}
|
||||
return !strings.ContainsAny(name, "/\\\x00")
|
||||
}
|
||||
|
||||
// walk lists a directory tree, at most maxWalkDepth levels below root and
|
||||
// maxWalkFiles files. Files come back sorted by path, then name.
|
||||
func (f *Fetcher) walk(s *session, root string) ([]*base.FileInfo, error) {
|
||||
type dir struct {
|
||||
remote string // path on the server
|
||||
rel string // path below root, "" for root itself
|
||||
depth int
|
||||
}
|
||||
var files []*base.FileInfo
|
||||
stack := []dir{{remote: strings.TrimSuffix(root, "/"), rel: "", depth: 0}}
|
||||
for len(stack) > 0 {
|
||||
d := stack[len(stack)-1]
|
||||
stack = stack[:len(stack)-1]
|
||||
|
||||
s.deadline()
|
||||
entries, err := s.List(d.remote)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, e := range entries {
|
||||
if e.Name == "." || e.Name == ".." {
|
||||
continue
|
||||
}
|
||||
if !safeEntryName(e.Name) {
|
||||
f.warn("ftp: skipping a listed name that is not a safe file name: %q", e.Name)
|
||||
continue
|
||||
}
|
||||
remote := path.Join(d.remote, e.Name)
|
||||
rel := path.Join(d.rel, e.Name)
|
||||
if d.remote == "" {
|
||||
remote = e.Name
|
||||
}
|
||||
entryType, size := e.Type, int64(e.Size)
|
||||
if entryType == ftp.EntryTypeLink {
|
||||
// A link to a file has a size; a link to a directory is
|
||||
// not followed, so a loop cannot make the walk endless.
|
||||
s.deadline()
|
||||
linkSize, err := s.FileSize(remote)
|
||||
if err != nil {
|
||||
f.warn("ftp: skipping link %q: %v", rel, err)
|
||||
continue
|
||||
}
|
||||
entryType, size = ftp.EntryTypeFile, linkSize
|
||||
}
|
||||
switch entryType {
|
||||
case ftp.EntryTypeFolder:
|
||||
if d.depth+1 > maxWalkDepth {
|
||||
return nil, fmt.Errorf("ftp: the folder is deeper than %d levels, the limit for a folder download", maxWalkDepth)
|
||||
}
|
||||
stack = append(stack, dir{remote: remote, rel: rel, depth: d.depth + 1})
|
||||
case ftp.EntryTypeFile:
|
||||
if len(files) >= maxWalkFiles {
|
||||
return nil, fmt.Errorf("ftp: the folder has more than %d files, the limit for a folder download", maxWalkFiles)
|
||||
}
|
||||
fi := &base.FileInfo{Name: e.Name, Path: d.rel, Size: size}
|
||||
if !e.Time.IsZero() {
|
||||
t := e.Time
|
||||
fi.Ctime = &t
|
||||
}
|
||||
files = append(files, fi)
|
||||
}
|
||||
}
|
||||
}
|
||||
slices.SortFunc(files, func(a, b *base.FileInfo) int {
|
||||
return cmp.Or(strings.Compare(a.Path, b.Path), strings.Compare(a.Name, b.Name))
|
||||
})
|
||||
return files, nil
|
||||
}
|
||||
@@ -303,9 +303,8 @@ type Fetcher struct {
|
||||
resolveDataPos atomic.Int64 // How many bytes downloaded during resolve
|
||||
|
||||
// Resolve response - kept open for one-time URLs
|
||||
resolveResp *http.Response
|
||||
resolveRespCancel context.CancelFunc // Cancels the Resolve request
|
||||
resolveRespLock sync.Mutex
|
||||
resolveResp *http.Response
|
||||
resolveRespLock sync.Mutex
|
||||
|
||||
// Async prefetch during resolve phase
|
||||
prefetchFile *os.File // Temporary file for prefetch data
|
||||
@@ -314,7 +313,6 @@ type Fetcher struct {
|
||||
prefetchDone atomic.Bool // Prefetch completed or stopped
|
||||
prefetchErr error // Error during prefetch (if any)
|
||||
prefetchStopCh chan struct{} // Signal to stop prefetch
|
||||
prefetchExitCh chan struct{} // Closed when the prefetch goroutine exits
|
||||
resolveFallback atomic.Bool // Resolve prefetch is still available until the first Range request takes over
|
||||
|
||||
// Target file
|
||||
@@ -427,11 +425,8 @@ func (f *Fetcher) Resolve(req *base.Request) error {
|
||||
|
||||
// Build HTTP request WITHOUT Range header (normal request)
|
||||
// This allows the response to be reused for downloading (important for one-time URLs)
|
||||
// The body outlives Resolve, so the read timeout gets its own cancel func.
|
||||
reqCtx, reqCancel := context.WithCancel(context.Background())
|
||||
httpReq, err := f.buildRequest(reqCtx, req)
|
||||
httpReq, err := f.buildRequest(context.TODO(), req)
|
||||
if err != nil {
|
||||
reqCancel()
|
||||
f.setState(stateError)
|
||||
return err
|
||||
}
|
||||
@@ -443,7 +438,6 @@ func (f *Fetcher) Resolve(req *base.Request) error {
|
||||
connStartTime := time.Now()
|
||||
resp, err := client.Do(httpReq)
|
||||
if err != nil {
|
||||
reqCancel()
|
||||
f.setState(stateError)
|
||||
return err
|
||||
}
|
||||
@@ -458,7 +452,6 @@ func (f *Fetcher) Resolve(req *base.Request) error {
|
||||
|
||||
if resp.StatusCode != base.HttpCodeOK && resp.StatusCode != base.HttpCodePartialContent {
|
||||
resp.Body.Close()
|
||||
reqCancel()
|
||||
f.setState(stateError)
|
||||
return NewRequestError(resp.StatusCode)
|
||||
}
|
||||
@@ -476,7 +469,6 @@ func (f *Fetcher) Resolve(req *base.Request) error {
|
||||
start, _, total, err := parseRangeResponse(resp)
|
||||
if err != nil {
|
||||
resp.Body.Close()
|
||||
reqCancel()
|
||||
f.setState(stateError)
|
||||
return err
|
||||
}
|
||||
@@ -544,10 +536,8 @@ func (f *Fetcher) Resolve(req *base.Request) error {
|
||||
f.resolveRespLock.Lock()
|
||||
if reuseResolveBody {
|
||||
f.resolveResp = resp
|
||||
f.resolveRespCancel = reqCancel
|
||||
} else {
|
||||
resp.Body.Close()
|
||||
reqCancel()
|
||||
f.prefetchDone.Store(true)
|
||||
}
|
||||
f.resolveRespLock.Unlock()
|
||||
@@ -563,7 +553,6 @@ func (f *Fetcher) Resolve(req *base.Request) error {
|
||||
// For non-range resources, the response will be used directly in Start
|
||||
if res.Range && reuseResolveBody {
|
||||
f.prefetchStopCh = make(chan struct{})
|
||||
f.prefetchExitCh = make(chan struct{})
|
||||
f.resolveFallback.Store(true)
|
||||
go f.asyncPrefetch()
|
||||
}
|
||||
@@ -581,14 +570,11 @@ func (f *Fetcher) Resolve(req *base.Request) error {
|
||||
func (f *Fetcher) asyncPrefetch() {
|
||||
defer func() {
|
||||
f.prefetchDone.Store(true)
|
||||
if f.prefetchExitCh != nil {
|
||||
close(f.prefetchExitCh)
|
||||
}
|
||||
}()
|
||||
|
||||
// Get the resolve response
|
||||
f.resolveRespLock.Lock()
|
||||
resp, cancel := f.resolveResp, f.resolveRespCancel
|
||||
resp := f.resolveResp
|
||||
f.resolveRespLock.Unlock()
|
||||
|
||||
if resp == nil {
|
||||
@@ -599,7 +585,10 @@ func (f *Fetcher) asyncPrefetch() {
|
||||
// Close the response even if temporary-file setup fails or shutdown
|
||||
// rejects a late prefetch. Otherwise the server can remain blocked writing.
|
||||
f.resolveRespLock.Lock()
|
||||
f.closeResolveRespLocked()
|
||||
if f.resolveResp != nil {
|
||||
f.resolveResp.Body.Close()
|
||||
f.resolveResp = nil
|
||||
}
|
||||
f.resolveRespLock.Unlock()
|
||||
}()
|
||||
|
||||
@@ -629,7 +618,7 @@ func (f *Fetcher) asyncPrefetch() {
|
||||
f.prefetchFilePath = tmpFile.Name()
|
||||
|
||||
buf := make([]byte, 32*1024) // 32KB buffer
|
||||
reader := f.responseReader(resp, cancel)
|
||||
reader := f.responseReader(resp)
|
||||
|
||||
for {
|
||||
select {
|
||||
@@ -659,38 +648,23 @@ func (f *Fetcher) asyncPrefetch() {
|
||||
}
|
||||
}
|
||||
|
||||
// stopPrefetch signals the async prefetch to stop and waits until it exits.
|
||||
// Closing the resolve body unblocks a pending Read, so a stalled server cannot
|
||||
// keep the prefetch goroutine writing to the temporary file after we reuse it.
|
||||
func (f *Fetcher) stopPrefetch() {
|
||||
if f.prefetchStopCh == nil {
|
||||
return
|
||||
}
|
||||
|
||||
// Pause and the first connection's takeover can stop concurrently, so the
|
||||
// check-and-close must be under the lock to avoid closing the channel twice.
|
||||
f.resolveRespLock.Lock()
|
||||
select {
|
||||
case <-f.prefetchStopCh:
|
||||
// Already closed
|
||||
default:
|
||||
close(f.prefetchStopCh)
|
||||
}
|
||||
if f.resolveResp != nil {
|
||||
f.resolveResp.Body.Close()
|
||||
f.resolveResp = nil
|
||||
}
|
||||
f.resolveRespLock.Unlock()
|
||||
|
||||
if f.prefetchExitCh != nil {
|
||||
<-f.prefetchExitCh
|
||||
}
|
||||
}
|
||||
|
||||
// stopPrefetchAndGetData stops the async prefetch and returns prefetched bytes
|
||||
// It also copies prefetched data to the target file
|
||||
func (f *Fetcher) stopPrefetchAndCopyData() int64 {
|
||||
f.stopPrefetch()
|
||||
// Signal prefetch to stop (safely)
|
||||
if f.prefetchStopCh != nil {
|
||||
select {
|
||||
case <-f.prefetchStopCh:
|
||||
// Already closed
|
||||
default:
|
||||
close(f.prefetchStopCh)
|
||||
}
|
||||
}
|
||||
|
||||
// Wait for prefetch to finish (with timeout)
|
||||
for i := 0; f.prefetchStopCh != nil && i < 1000 && !f.prefetchDone.Load(); i++ {
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
|
||||
prefetched := f.prefetchSize.Load()
|
||||
if prefetched == 0 {
|
||||
@@ -892,16 +866,12 @@ func (f *Fetcher) doStart() error {
|
||||
// For non-range resources, the response will be used directly
|
||||
var prefetchedBytes int64
|
||||
if f.meta.Res.Range {
|
||||
// Read the size once and judge completeness by it alone: Resolve may have
|
||||
// written every byte before it sets prefetchDone, and a snapshot that
|
||||
// covers the whole file would otherwise finish without copying it.
|
||||
prefetched := f.prefetchSize.Load()
|
||||
resolveComplete := f.knownSize() > 0 && prefetched >= f.knownSize()
|
||||
resolveComplete := f.knownSize() > 0 && f.prefetchDone.Load() && f.prefetchErr == nil && f.prefetchSize.Load() >= f.knownSize()
|
||||
if f.resolveFallback.Load() && !resolveComplete {
|
||||
// Keep Resolve downloading until the first Range response has been
|
||||
// validated. The Range request starts at a stable snapshot; a valid
|
||||
// takeover discards any overlap downloaded by Resolve in the meantime.
|
||||
prefetchedBytes = prefetched
|
||||
prefetchedBytes = f.prefetchSize.Load()
|
||||
} else {
|
||||
// Resolve already completed, or no live Resolve fallback remains.
|
||||
prefetchedBytes = f.stopPrefetchAndCopyData()
|
||||
@@ -911,7 +881,10 @@ func (f *Fetcher) doStart() error {
|
||||
if !f.resolveFallback.Load() {
|
||||
// Also close resolve response if still open.
|
||||
f.resolveRespLock.Lock()
|
||||
f.closeResolveRespLocked()
|
||||
if f.resolveResp != nil {
|
||||
f.resolveResp.Body.Close()
|
||||
f.resolveResp = nil
|
||||
}
|
||||
f.resolveRespLock.Unlock()
|
||||
}
|
||||
}
|
||||
@@ -1364,11 +1337,7 @@ func (f *Fetcher) downloadChunkOnce(conn *connection, client *http.Client, buf [
|
||||
}
|
||||
f.connMu.Unlock()
|
||||
|
||||
// The read timeout cancels this request through its own context.
|
||||
reqCtx, reqCancel := context.WithCancel(conn.ctx)
|
||||
defer reqCancel()
|
||||
|
||||
httpReq, err := f.buildRequest(reqCtx, f.meta.Req)
|
||||
httpReq, err := f.buildRequest(conn.ctx, f.meta.Req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -1396,7 +1365,7 @@ func (f *Fetcher) downloadChunkOnce(conn *connection, client *http.Client, buf [
|
||||
// Check if this might be a redirect URL expiration error
|
||||
// If so, try falling back to the original URL
|
||||
if f.hasRedirectURL() && isRedirectExpiredError(originalErr) {
|
||||
fallbackResp, fallbackErr := f.tryFallbackToOriginalURL(reqCtx, client, rangeStart, rangeEnd, rangeRequested, ifRange)
|
||||
fallbackResp, fallbackErr := f.tryFallbackToOriginalURL(conn.ctx, client, rangeStart, rangeEnd, rangeRequested, ifRange)
|
||||
if fallbackErr == nil && fallbackResp != nil {
|
||||
// Fallback succeeded, use this response instead
|
||||
resp = fallbackResp
|
||||
@@ -1517,7 +1486,7 @@ func (f *Fetcher) downloadChunkOnce(conn *connection, client *http.Client, buf [
|
||||
f.slowStart.onConnectSuccess()
|
||||
}
|
||||
|
||||
reader := f.responseReader(resp, reqCancel)
|
||||
reader := f.responseReader(resp)
|
||||
var responseBytesRead int64
|
||||
for {
|
||||
if conn.ctx.Err() != nil {
|
||||
@@ -1651,10 +1620,11 @@ func (f *Fetcher) completeFromResolveFallback(conn *connection) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
select {
|
||||
case <-f.prefetchExitCh:
|
||||
case <-conn.ctx.Done():
|
||||
return false
|
||||
for !f.prefetchDone.Load() {
|
||||
if conn.ctx.Err() != nil {
|
||||
return false
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
knownSize := f.hasKnownSize()
|
||||
if f.prefetchErr != nil || (knownSize && f.prefetchSize.Load() != f.meta.Res.Size) {
|
||||
@@ -1870,8 +1840,8 @@ func (f *Fetcher) runConnectionWithResolveResp(conn *connection) {
|
||||
|
||||
// Get the resolve response
|
||||
f.resolveRespLock.Lock()
|
||||
resp, cancel := f.resolveResp, f.resolveRespCancel
|
||||
f.resolveResp, f.resolveRespCancel = nil, nil // Take ownership
|
||||
resp := f.resolveResp
|
||||
f.resolveResp = nil // Take ownership
|
||||
f.resolveRespLock.Unlock()
|
||||
|
||||
if resp == nil {
|
||||
@@ -1880,9 +1850,6 @@ func (f *Fetcher) runConnectionWithResolveResp(conn *connection) {
|
||||
return
|
||||
}
|
||||
|
||||
if cancel != nil {
|
||||
defer cancel()
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
f.connMu.Lock()
|
||||
@@ -1899,7 +1866,7 @@ func (f *Fetcher) runConnectionWithResolveResp(conn *connection) {
|
||||
}
|
||||
|
||||
// Download data from resolve response
|
||||
reader := f.responseReader(resp, cancel)
|
||||
reader := f.responseReader(resp)
|
||||
for {
|
||||
if conn.ctx.Err() != nil {
|
||||
return
|
||||
@@ -2461,7 +2428,14 @@ func (f *Fetcher) Pause() error {
|
||||
}
|
||||
|
||||
// Stop prefetch if running
|
||||
f.stopPrefetch()
|
||||
if f.prefetchStopCh != nil {
|
||||
select {
|
||||
case <-f.prefetchStopCh:
|
||||
// Already closed
|
||||
default:
|
||||
close(f.prefetchStopCh)
|
||||
}
|
||||
}
|
||||
|
||||
// Wait for downloadLoop to exit first (it will call wg.Wait internally)
|
||||
if f.downloadLoopDone != nil {
|
||||
@@ -2471,6 +2445,11 @@ func (f *Fetcher) Pause() error {
|
||||
// Wait for all connection goroutines to stop
|
||||
f.wg.Wait()
|
||||
|
||||
// Wait for prefetch to finish
|
||||
for f.prefetchStopCh != nil && !f.prefetchDone.Load() {
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
|
||||
// If Pause wins before the first Range response is validated, materialize
|
||||
// the Resolve prefix before discarding its temporary file. The first
|
||||
// connection was allocated after the earlier prefetch snapshot, so advance
|
||||
@@ -2502,7 +2481,10 @@ func (f *Fetcher) Pause() error {
|
||||
|
||||
// Clean up resolve response if still held
|
||||
f.resolveRespLock.Lock()
|
||||
f.closeResolveRespLocked()
|
||||
if f.resolveResp != nil {
|
||||
f.resolveResp.Body.Close()
|
||||
f.resolveResp = nil
|
||||
}
|
||||
f.resolveRespLock.Unlock()
|
||||
|
||||
f.fileMu.Lock()
|
||||
@@ -2620,25 +2602,9 @@ func (f *Fetcher) Wait() error {
|
||||
|
||||
// Generated output can pause while its producer waits for processing capacity.
|
||||
// Its producer owns upstream deadlines; cancellation still closes the response.
|
||||
// Otherwise a stalled read cancels the request via cancel, which must belong
|
||||
// to the request that produced resp.
|
||||
func (f *Fetcher) responseReader(resp *http.Response, cancel context.CancelFunc) io.Reader {
|
||||
func (f *Fetcher) responseReader(resp *http.Response) io.Reader {
|
||||
if f.ctl != nil && f.ctl.ManagedProduction != nil && resp.Request != nil && f.ctl.ManagedProduction(resp.Request.URL.String()) {
|
||||
return resp.Body
|
||||
}
|
||||
return NewTimeoutReader(resp.Body, readTimeout, cancel)
|
||||
}
|
||||
|
||||
// closeResolveRespLocked closes the kept Resolve response, if any, and
|
||||
// releases its request context. The caller must hold resolveRespLock.
|
||||
func (f *Fetcher) closeResolveRespLocked() {
|
||||
if f.resolveResp == nil {
|
||||
return
|
||||
}
|
||||
f.resolveResp.Body.Close()
|
||||
f.resolveResp = nil
|
||||
if f.resolveRespCancel != nil {
|
||||
f.resolveRespCancel()
|
||||
f.resolveRespCancel = nil
|
||||
}
|
||||
return NewTimeoutReader(resp.Body, readTimeout)
|
||||
}
|
||||
@@ -3261,17 +3261,16 @@ func downloadResume(listener net.Listener, connections int, t *testing.T) {
|
||||
}
|
||||
|
||||
func downloadWithProxy(httpListener net.Listener, proxyListener net.Listener, t *testing.T) {
|
||||
// Setup must run before Resolve, as in the downloader, so the Resolve
|
||||
// request also goes through the proxy.
|
||||
f := buildFetcher()
|
||||
f.ctl.GetProxy = func(requestProxy *base.RequestProxy) func(*gohttp.Request) (*url.URL, error) {
|
||||
fetcher := downloadReady(httpListener, 4, t)
|
||||
ctl := controller.NewController()
|
||||
ctl.GetProxy = func(requestProxy *base.RequestProxy) func(*gohttp.Request) (*url.URL, error) {
|
||||
return (&base.DownloaderProxyConfig{
|
||||
Enable: true,
|
||||
Scheme: "socks5",
|
||||
Host: proxyListener.Addr().String(),
|
||||
}).ToHandler()
|
||||
}
|
||||
fetcher := doDownloadReady(f, httpListener, 4, t)
|
||||
fetcher.Setup(ctl)
|
||||
err := fetcher.Start()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
|
||||
@@ -125,26 +125,13 @@ func TestUnescapeHTMLEntities(t *testing.T) {
|
||||
in: "normal.txt",
|
||||
want: "normal.txt",
|
||||
},
|
||||
{
|
||||
name: "escaped ampersand before entity name is decoded once",
|
||||
in: "a&lt;b.txt",
|
||||
want: "a<b.txt",
|
||||
},
|
||||
{
|
||||
name: "escaped ampersand before quote entity is decoded once",
|
||||
in: "a&quot;b.txt",
|
||||
want: "a"b.txt",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
// Run repeatedly so that an order-dependent implementation fails reliably.
|
||||
for i := 0; i < 200; i++ {
|
||||
got := unescapeHTMLEntities(tt.in)
|
||||
if got != tt.want {
|
||||
t.Fatalf("unescapeHTMLEntities() = %q, want %q (run %d)", got, tt.want, i)
|
||||
}
|
||||
got := unescapeHTMLEntities(tt.in)
|
||||
if got != tt.want {
|
||||
t.Errorf("unescapeHTMLEntities() = %q, want %q", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
@@ -408,21 +408,24 @@ func decodeFilenameParam(filename string) string {
|
||||
return decoded
|
||||
}
|
||||
|
||||
// htmlEntityReplacer replaces every entity in a single left-to-right pass, so an
|
||||
// escaped ampersand such as "&lt;" becomes "<" and is not decoded twice.
|
||||
var htmlEntityReplacer = strings.NewReplacer(
|
||||
"&", "&",
|
||||
"<", "<",
|
||||
">", ">",
|
||||
""", "\"",
|
||||
"'", "'",
|
||||
"'", "'",
|
||||
)
|
||||
|
||||
// unescapeHTMLEntities unescapes common HTML entities in filenames
|
||||
// This handles cases where servers HTML-encode special characters like & to &
|
||||
func unescapeHTMLEntities(s string) string {
|
||||
return htmlEntityReplacer.Replace(s)
|
||||
// Common HTML entities that might appear in filenames
|
||||
replacements := map[string]string{
|
||||
"&": "&",
|
||||
"<": "<",
|
||||
">": ">",
|
||||
""": "\"",
|
||||
"'": "'",
|
||||
"'": "'",
|
||||
}
|
||||
|
||||
result := s
|
||||
for entity, char := range replacements {
|
||||
result = strings.ReplaceAll(result, entity, char)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// tryDecodeGBK attempts to decode string as GBK encoding
|
||||
|
||||
@@ -1,52 +0,0 @@
|
||||
package http
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
gohttp "net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/GopeedLab/gopeed/pkg/base"
|
||||
"github.com/GopeedLab/gopeed/pkg/protocol/http"
|
||||
)
|
||||
|
||||
// Start can run after the Resolve prefetch wrote every byte but before it set
|
||||
// prefetchDone. The prefetched body must still be copied to the target.
|
||||
func TestFetcher_StartBeforePrefetchDoneCopiesFullBody(t *testing.T) {
|
||||
payload := []byte("prefetched before Start")
|
||||
server := httptest.NewServer(gohttp.HandlerFunc(func(w gohttp.ResponseWriter, r *gohttp.Request) {
|
||||
gohttp.ServeContent(w, r, "f.txt", time.Time{}, bytes.NewReader(payload))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
f := buildFetcher()
|
||||
f.Meta().Opts = &base.Options{Path: t.TempDir(), Name: "f.txt", Extra: &http.OptsExtra{Connections: 1}}
|
||||
if err := f.Resolve(&base.Request{URL: server.URL + "/f.txt"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for !f.prefetchDone.Load() {
|
||||
time.Sleep(time.Millisecond)
|
||||
}
|
||||
// Reopen the window between the last write and prefetchDone.
|
||||
f.prefetchDone.Store(false)
|
||||
go func() {
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
f.prefetchDone.Store(true)
|
||||
}()
|
||||
|
||||
if err := f.Start(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := f.Wait(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, err := os.ReadFile(f.meta.SingleFilepath())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !bytes.Equal(got, payload) {
|
||||
t.Fatalf("content = %q, want %q", got, payload)
|
||||
}
|
||||
}
|
||||
@@ -1,56 +0,0 @@
|
||||
package http
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
gohttp "net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/GopeedLab/gopeed/pkg/base"
|
||||
"github.com/GopeedLab/gopeed/pkg/protocol/http"
|
||||
)
|
||||
|
||||
// A stalled Resolve body must not keep the prefetch goroutine alive after
|
||||
// stopPrefetchAndCopyData returns, or it writes to a file we already removed.
|
||||
func TestFetcher_StopPrefetchUnblocksStalledRead(t *testing.T) {
|
||||
prefix := bytes.Repeat([]byte("p"), 64*1024)
|
||||
stall := make(chan struct{})
|
||||
defer close(stall)
|
||||
|
||||
server := httptest.NewServer(gohttp.HandlerFunc(func(w gohttp.ResponseWriter, r *gohttp.Request) {
|
||||
w.Header().Set(base.HttpHeaderAcceptRanges, base.HttpHeaderBytes)
|
||||
w.Header().Set(base.HttpHeaderContentLength, fmt.Sprintf("%d", len(prefix)*4))
|
||||
w.WriteHeader(gohttp.StatusOK)
|
||||
_, _ = w.Write(prefix)
|
||||
w.(gohttp.Flusher).Flush()
|
||||
select {
|
||||
case <-stall:
|
||||
case <-r.Context().Done():
|
||||
}
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
f := buildFetcher()
|
||||
f.Meta().Opts = &base.Options{
|
||||
Path: t.TempDir(),
|
||||
Name: "stalled.data",
|
||||
Extra: &http.OptsExtra{Connections: 1},
|
||||
}
|
||||
if err := f.Resolve(&base.Request{URL: server.URL + "/stalled.data"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for f.prefetchSize.Load() < int64(len(prefix)) {
|
||||
time.Sleep(time.Millisecond)
|
||||
}
|
||||
|
||||
start := time.Now()
|
||||
f.stopPrefetch()
|
||||
if elapsed := time.Since(start); elapsed > 2*time.Second {
|
||||
t.Fatalf("stopPrefetch took %v, want it to interrupt the stalled read", elapsed)
|
||||
}
|
||||
if !f.prefetchDone.Load() {
|
||||
t.Fatal("prefetch goroutine still running after stopPrefetch")
|
||||
}
|
||||
}
|
||||
@@ -6,56 +6,35 @@ import (
|
||||
"time"
|
||||
)
|
||||
|
||||
// TimeoutReader fails a Read that makes no progress within timeout.
|
||||
//
|
||||
// One timer per reader is armed when Read starts and stopped before it
|
||||
// returns, so time the caller spends between reads is not counted. If the
|
||||
// timer fires, it calls cancel, which should cancel the HTTP request the
|
||||
// reader belongs to so the transport unblocks the pending Read. Read still
|
||||
// waits for the underlying Read to return; that Read and every later one
|
||||
// return context.DeadlineExceeded. Errors from any other cancellation are
|
||||
// returned unchanged, and closing the response body is left to the caller.
|
||||
//
|
||||
// With a nil cancel the pending Read cannot be interrupted; it still reports
|
||||
// context.DeadlineExceeded, but only once it returns.
|
||||
//
|
||||
// TimeoutReader is not safe for concurrent use.
|
||||
type TimeoutReader struct {
|
||||
reader io.Reader
|
||||
timeout time.Duration
|
||||
cancel context.CancelFunc
|
||||
timer *time.Timer
|
||||
timedOut bool
|
||||
reader io.Reader
|
||||
timeout time.Duration
|
||||
}
|
||||
|
||||
func NewTimeoutReader(r io.Reader, timeout time.Duration, cancel context.CancelFunc) *TimeoutReader {
|
||||
if cancel == nil {
|
||||
cancel = func() {}
|
||||
}
|
||||
func NewTimeoutReader(r io.Reader, timeout time.Duration) *TimeoutReader {
|
||||
return &TimeoutReader{
|
||||
reader: r,
|
||||
timeout: timeout,
|
||||
cancel: cancel,
|
||||
}
|
||||
}
|
||||
|
||||
func (tr *TimeoutReader) Read(p []byte) (n int, err error) {
|
||||
if tr.timedOut {
|
||||
return 0, context.DeadlineExceeded
|
||||
}
|
||||
if tr.timer == nil {
|
||||
tr.timer = time.AfterFunc(tr.timeout, tr.cancel)
|
||||
} else {
|
||||
tr.timer.Reset(tr.timeout)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), tr.timeout)
|
||||
defer cancel()
|
||||
|
||||
n, err = tr.reader.Read(p)
|
||||
done := make(chan struct{})
|
||||
var readErr error
|
||||
var bytesRead int
|
||||
|
||||
if !tr.timer.Stop() {
|
||||
// The timer fired during this Read and cancelled the request, so err is
|
||||
// whatever the interrupted read produced. Report the timeout instead.
|
||||
tr.timedOut = true
|
||||
return n, context.DeadlineExceeded
|
||||
go func() {
|
||||
bytesRead, readErr = tr.reader.Read(p)
|
||||
close(done)
|
||||
}()
|
||||
|
||||
select {
|
||||
case <-done:
|
||||
return bytesRead, readErr
|
||||
case <-ctx.Done():
|
||||
return 0, ctx.Err()
|
||||
}
|
||||
return n, err
|
||||
}
|
||||
@@ -2,26 +2,17 @@ package http
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"errors"
|
||||
"io"
|
||||
gohttp "net/http"
|
||||
"net/http/httptest"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/GopeedLab/gopeed/internal/httpclient"
|
||||
)
|
||||
|
||||
func TestTimeoutReader_Read(t *testing.T) {
|
||||
data := []byte("Hello, World!")
|
||||
reader := bytes.NewReader(data)
|
||||
timeoutReader := NewTimeoutReader(reader, 1*time.Second, nil)
|
||||
timeoutReader := NewTimeoutReader(reader, 1*time.Second)
|
||||
|
||||
buf := make([]byte, len(data))
|
||||
n, err := timeoutReader.Read(buf)
|
||||
@@ -38,7 +29,7 @@ func TestTimeoutReader_Read(t *testing.T) {
|
||||
|
||||
func TestTimeoutReader_ReadTimeout(t *testing.T) {
|
||||
reader := &slowReader{delay: 2 * time.Second}
|
||||
timeoutReader := NewTimeoutReader(reader, 1*time.Second, nil)
|
||||
timeoutReader := NewTimeoutReader(reader, 1*time.Second)
|
||||
|
||||
buf := make([]byte, 8192)
|
||||
_, err := timeoutReader.Read(buf)
|
||||
@@ -58,340 +49,3 @@ func (sr *slowReader) Read(p []byte) (n int, err error) {
|
||||
time.Sleep(sr.delay)
|
||||
return 0, io.EOF
|
||||
}
|
||||
|
||||
// The timer must only cover time spent inside Read, not caller work between reads.
|
||||
func TestTimeoutReader_IdleBetweenReads(t *testing.T) {
|
||||
var cancelled atomic.Bool
|
||||
timeoutReader := NewTimeoutReader(bytes.NewReader([]byte("abcdef")), 50*time.Millisecond, func() {
|
||||
cancelled.Store(true)
|
||||
})
|
||||
|
||||
buf := make([]byte, 3)
|
||||
if _, err := timeoutReader.Read(buf); err != nil {
|
||||
t.Fatalf("first read: %v", err)
|
||||
}
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
if _, err := timeoutReader.Read(buf); err != nil {
|
||||
t.Fatalf("second read after idle: %v", err)
|
||||
}
|
||||
if cancelled.Load() {
|
||||
t.Fatal("request was cancelled while idle between reads")
|
||||
}
|
||||
}
|
||||
|
||||
// A timed-out Read must not return while the underlying Read can still write
|
||||
// into the caller's buffer, and must leave closing the reader to the caller.
|
||||
func TestTimeoutReader_TimedOutReadDoesNotOutliveCaller(t *testing.T) {
|
||||
reader := &lateWriteReader{cancelled: make(chan struct{})}
|
||||
timeoutReader := NewTimeoutReader(reader, 50*time.Millisecond, reader.cancel)
|
||||
|
||||
buf := make([]byte, 8)
|
||||
_, err := timeoutReader.Read(buf)
|
||||
if !errors.Is(err, context.DeadlineExceeded) {
|
||||
t.Fatalf("expected %v, got %v", context.DeadlineExceeded, err)
|
||||
}
|
||||
if !reader.returned.Load() {
|
||||
t.Fatal("Read returned before the underlying Read finished")
|
||||
}
|
||||
if _, err := timeoutReader.Read(buf); !errors.Is(err, context.DeadlineExceeded) {
|
||||
t.Fatalf("read after timeout: expected %v, got %v", context.DeadlineExceeded, err)
|
||||
}
|
||||
if reader.closed.Load() {
|
||||
t.Fatal("reader was closed by the timeout")
|
||||
}
|
||||
}
|
||||
|
||||
// lateWriteReader blocks until cancelled, then writes into p before returning.
|
||||
type lateWriteReader struct {
|
||||
cancelOnce sync.Once
|
||||
cancelled chan struct{}
|
||||
returned atomic.Bool
|
||||
closed atomic.Bool
|
||||
}
|
||||
|
||||
func (r *lateWriteReader) Read(p []byte) (int, error) {
|
||||
<-r.cancelled
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
p[0] = 'x'
|
||||
r.returned.Store(true)
|
||||
return 0, context.Canceled
|
||||
}
|
||||
|
||||
func (r *lateWriteReader) cancel() {
|
||||
r.cancelOnce.Do(func() { close(r.cancelled) })
|
||||
}
|
||||
|
||||
func (r *lateWriteReader) Close() error {
|
||||
r.closed.Store(true)
|
||||
return nil
|
||||
}
|
||||
|
||||
type timeoutReaderClientCase struct {
|
||||
name string
|
||||
http2 bool
|
||||
client func(t *testing.T, server *httptest.Server) *gohttp.Client
|
||||
}
|
||||
|
||||
func timeoutReaderClientCases() []timeoutReaderClientCase {
|
||||
newClient := func(mode httpclient.ImpersonationMode) func(t *testing.T, server *httptest.Server) *gohttp.Client {
|
||||
return func(t *testing.T, server *httptest.Server) *gohttp.Client {
|
||||
var tlsConfig *tls.Config
|
||||
if server.TLS != nil {
|
||||
roots := x509.NewCertPool()
|
||||
roots.AddCert(server.Certificate())
|
||||
tlsConfig = &tls.Config{RootCAs: roots}
|
||||
}
|
||||
client, err := httpclient.NewClient(httpclient.Options{
|
||||
Transport: httpclient.TransportOptions{
|
||||
TLSClientConfig: tlsConfig,
|
||||
},
|
||||
Impersonation: httpclient.ImpersonationOptions{
|
||||
Mode: mode,
|
||||
Browser: httpclient.BrowserChrome,
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return client
|
||||
}
|
||||
}
|
||||
var cases []timeoutReaderClientCase
|
||||
for _, mode := range []httpclient.ImpersonationMode{httpclient.ImpersonationNative, httpclient.ImpersonationFixed} {
|
||||
cases = append(cases,
|
||||
timeoutReaderClientCase{name: string(mode) + "/http1", client: newClient(mode)},
|
||||
timeoutReaderClientCase{name: string(mode) + "/http2", http2: true, client: newClient(mode)},
|
||||
)
|
||||
}
|
||||
return cases
|
||||
}
|
||||
|
||||
func startTimeoutReaderServer(t *testing.T, useHTTP2 bool, handler gohttp.HandlerFunc) *httptest.Server {
|
||||
server := httptest.NewUnstartedServer(handler)
|
||||
if useHTTP2 {
|
||||
server.EnableHTTP2 = true
|
||||
server.StartTLS()
|
||||
} else {
|
||||
server.Start()
|
||||
}
|
||||
t.Cleanup(server.Close)
|
||||
return server
|
||||
}
|
||||
|
||||
// readBodyWithTimeout sends the request under its own context derived from
|
||||
// ctx, as the fetcher does, and reads the body through a TimeoutReader.
|
||||
func readBodyWithTimeout(t *testing.T, ctx context.Context, client *gohttp.Client, url string, timeout time.Duration, wantProto int) ([]byte, error) {
|
||||
t.Helper()
|
||||
reqCtx, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
req, err := gohttp.NewRequestWithContext(reqCtx, gohttp.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.ProtoMajor != wantProto {
|
||||
t.Fatalf("expected HTTP/%d, got %s", wantProto, resp.Proto)
|
||||
}
|
||||
|
||||
type result struct {
|
||||
data []byte
|
||||
err error
|
||||
}
|
||||
done := make(chan result, 1)
|
||||
go func() {
|
||||
data, err := io.ReadAll(NewTimeoutReader(resp.Body, timeout, cancel))
|
||||
done <- result{data, err}
|
||||
}()
|
||||
select {
|
||||
case r := <-done:
|
||||
return r.data, r.err
|
||||
case <-time.After(timeout + 10*time.Second):
|
||||
t.Fatal("Read did not return after the read timeout")
|
||||
return nil, nil
|
||||
}
|
||||
}
|
||||
|
||||
func TestTimeoutReader_StalledHTTPBodyTimesOut(t *testing.T) {
|
||||
for _, tc := range timeoutReaderClientCases() {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
server := startTimeoutReaderServer(t, tc.http2, func(w gohttp.ResponseWriter, r *gohttp.Request) {
|
||||
w.Header().Set("Content-Length", "1048576")
|
||||
w.Write(bytes.Repeat([]byte("a"), 1024))
|
||||
w.(gohttp.Flusher).Flush()
|
||||
<-r.Context().Done()
|
||||
})
|
||||
|
||||
wantProto := 1
|
||||
if tc.http2 {
|
||||
wantProto = 2
|
||||
}
|
||||
const timeout = 200 * time.Millisecond
|
||||
start := time.Now()
|
||||
data, err := readBodyWithTimeout(t, context.Background(), tc.client(t, server), server.URL, timeout, wantProto)
|
||||
if !errors.Is(err, context.DeadlineExceeded) {
|
||||
t.Fatalf("expected %v, got %v", context.DeadlineExceeded, err)
|
||||
}
|
||||
if len(data) != 1024 {
|
||||
t.Fatalf("expected the 1024 bytes sent before the stall, got %d", len(data))
|
||||
}
|
||||
if elapsed := time.Since(start); elapsed < timeout {
|
||||
t.Fatalf("timed out after %v, before the %v read timeout", elapsed, timeout)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestTimeoutReader_SlowSteadyHTTPBodyDoesNotTimeOut(t *testing.T) {
|
||||
const (
|
||||
timeout = 300 * time.Millisecond
|
||||
interval = 30 * time.Millisecond
|
||||
chunks = 40 // 1.2s in total, four times the read timeout
|
||||
)
|
||||
for _, tc := range timeoutReaderClientCases() {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
server := startTimeoutReaderServer(t, tc.http2, func(w gohttp.ResponseWriter, r *gohttp.Request) {
|
||||
for i := 0; i < chunks; i++ {
|
||||
w.Write([]byte("0123456789"))
|
||||
w.(gohttp.Flusher).Flush()
|
||||
time.Sleep(interval)
|
||||
}
|
||||
})
|
||||
|
||||
wantProto := 1
|
||||
if tc.http2 {
|
||||
wantProto = 2
|
||||
}
|
||||
data, err := readBodyWithTimeout(t, context.Background(), tc.client(t, server), server.URL, timeout, wantProto)
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error, got %v", err)
|
||||
}
|
||||
if len(data) != chunks*10 {
|
||||
t.Fatalf("expected %d bytes, got %d", chunks*10, len(data))
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A gzip-encoded body that stalls must time out without racing the caller's
|
||||
// Body.Close against the decompressing reader.
|
||||
func TestTimeoutReader_StalledGzipHTTPBodyTimesOut(t *testing.T) {
|
||||
payload := bytes.Repeat([]byte("gopeed"), 1024)
|
||||
stages := []struct {
|
||||
name string
|
||||
write func(w io.Writer)
|
||||
want int
|
||||
}{
|
||||
{
|
||||
// The first bytes of a gzip header, so the client blocks inside
|
||||
// gzip.NewReader.
|
||||
name: "incomplete-header",
|
||||
write: func(w io.Writer) { w.Write([]byte{0x1f, 0x8b, 0x08, 0x00}) },
|
||||
want: 0,
|
||||
},
|
||||
{
|
||||
// A complete header and some compressed data, flushed without
|
||||
// finishing the gzip stream.
|
||||
name: "stalled-body",
|
||||
write: func(w io.Writer) {
|
||||
gz := gzip.NewWriter(w)
|
||||
gz.Write(payload)
|
||||
gz.Flush()
|
||||
},
|
||||
want: len(payload),
|
||||
},
|
||||
}
|
||||
for _, stage := range stages {
|
||||
for _, tc := range timeoutReaderClientCases() {
|
||||
t.Run(stage.name+"/"+tc.name, func(t *testing.T) {
|
||||
server := startTimeoutReaderServer(t, tc.http2, func(w gohttp.ResponseWriter, r *gohttp.Request) {
|
||||
w.Header().Set("Content-Encoding", "gzip")
|
||||
stage.write(w)
|
||||
w.(gohttp.Flusher).Flush()
|
||||
<-r.Context().Done()
|
||||
})
|
||||
|
||||
wantProto := 1
|
||||
if tc.http2 {
|
||||
wantProto = 2
|
||||
}
|
||||
data, err := readBodyWithTimeout(t, context.Background(), tc.client(t, server), server.URL, 200*time.Millisecond, wantProto)
|
||||
if !errors.Is(err, context.DeadlineExceeded) {
|
||||
t.Fatalf("expected %v, got %v", context.DeadlineExceeded, err)
|
||||
}
|
||||
if len(data) != stage.want {
|
||||
t.Fatalf("expected the %d bytes sent before the stall, got %d", stage.want, len(data))
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Cancelling the caller's context must not be reported as a read timeout.
|
||||
func TestTimeoutReader_CallerCancellationIsNotTimeout(t *testing.T) {
|
||||
for _, tc := range timeoutReaderClientCases() {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
server := startTimeoutReaderServer(t, tc.http2, func(w gohttp.ResponseWriter, r *gohttp.Request) {
|
||||
w.Header().Set("Content-Length", "1048576")
|
||||
w.Write(bytes.Repeat([]byte("a"), 1024))
|
||||
w.(gohttp.Flusher).Flush()
|
||||
<-r.Context().Done()
|
||||
})
|
||||
|
||||
wantProto := 1
|
||||
if tc.http2 {
|
||||
wantProto = 2
|
||||
}
|
||||
// Cancel once the client has read the first bytes, so the next
|
||||
// Read is the one blocked when the caller cancels.
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
client := tc.client(t, server)
|
||||
next := client.Transport
|
||||
client.Transport = roundTripFunc(func(r *gohttp.Request) (*gohttp.Response, error) {
|
||||
resp, err := next.RoundTrip(r)
|
||||
if err == nil {
|
||||
resp.Body = &cancelAfterFirstRead{ReadCloser: resp.Body, cancel: cancel}
|
||||
}
|
||||
return resp, err
|
||||
})
|
||||
|
||||
const timeout = 5 * time.Second
|
||||
start := time.Now()
|
||||
data, err := readBodyWithTimeout(t, ctx, client, server.URL, timeout, wantProto)
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("expected %v, got %v", context.Canceled, err)
|
||||
}
|
||||
if len(data) == 0 {
|
||||
t.Fatal("cancelled before any data was read")
|
||||
}
|
||||
if elapsed := time.Since(start); elapsed >= timeout {
|
||||
t.Fatalf("read returned after %v, not on cancellation", elapsed)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
type roundTripFunc func(*gohttp.Request) (*gohttp.Response, error)
|
||||
|
||||
func (f roundTripFunc) RoundTrip(r *gohttp.Request) (*gohttp.Response, error) {
|
||||
return f(r)
|
||||
}
|
||||
|
||||
// cancelAfterFirstRead calls cancel once a Read has returned data.
|
||||
type cancelAfterFirstRead struct {
|
||||
io.ReadCloser
|
||||
once sync.Once
|
||||
cancel context.CancelFunc
|
||||
}
|
||||
|
||||
func (b *cancelAfterFirstRead) Read(p []byte) (int, error) {
|
||||
n, err := b.ReadCloser.Read(p)
|
||||
if n > 0 {
|
||||
b.once.Do(b.cancel)
|
||||
}
|
||||
return n, err
|
||||
}
|
||||
@@ -25,6 +25,7 @@ const defaultAvailabilityTimeout = time.Minute
|
||||
type ContractOptions struct {
|
||||
AvailabilityTimeout time.Duration
|
||||
CookieDomainMode CookieDomainMode
|
||||
CookieTestURL string
|
||||
}
|
||||
|
||||
type CookieDomainMode int
|
||||
@@ -43,7 +44,7 @@ func RunProviderContract(t *testing.T, provider enginewebview.Provider, opts Con
|
||||
}
|
||||
waitForProviderAvailable(t, provider, timeout)
|
||||
|
||||
page, _, testURL := openTestPage(t, provider, enginewebview.OpenOptions{
|
||||
page, server, testURL := openTestPage(t, provider, enginewebview.OpenOptions{
|
||||
Headless: true,
|
||||
Title: "Gopeed WebView Integration",
|
||||
Width: 640,
|
||||
@@ -154,9 +155,11 @@ func RunProviderContract(t *testing.T, provider enginewebview.Provider, opts Con
|
||||
t.Fatal("expected missing selector wait to time out")
|
||||
}
|
||||
|
||||
// Keep cookie navigation on the same local fixture as the DOM assertions:
|
||||
// this contract must not depend on public DNS, TLS, or site availability.
|
||||
runCookieLifecycleAssertions(t, page, testURL, opts.CookieDomainMode)
|
||||
cookieTestURL := opts.CookieTestURL
|
||||
if cookieTestURL == "" {
|
||||
cookieTestURL = testURL
|
||||
}
|
||||
runCookieLifecycleAssertions(t, page, server, cookieTestURL, opts.CookieDomainMode)
|
||||
}
|
||||
|
||||
func waitForProviderAvailable(t *testing.T, provider enginewebview.Provider, timeout time.Duration) {
|
||||
@@ -267,7 +270,7 @@ func openTestPage(t *testing.T, provider enginewebview.Provider, opts enginewebv
|
||||
return page, server, server.URL
|
||||
}
|
||||
|
||||
func runCookieLifecycleAssertions(t *testing.T, page *enginewebview.PageHandle, testURL string, domainMode CookieDomainMode) {
|
||||
func runCookieLifecycleAssertions(t *testing.T, page *enginewebview.PageHandle, _ *httptest.Server, testURL string, domainMode CookieDomainMode) {
|
||||
t.Helper()
|
||||
|
||||
parsedURL, err := url.Parse(testURL)
|
||||
|
||||
@@ -26,6 +26,7 @@ func TestProviderContract(t *testing.T) {
|
||||
})
|
||||
integrationtest.RunProviderContract(t, provider, integrationtest.ContractOptions{
|
||||
CookieDomainMode: integrationtest.CookieDomainModeRequired,
|
||||
CookieTestURL: "https://example.com/",
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
+2
-2
@@ -180,11 +180,11 @@ func (s *Service) createTaskBatch(ctx *Context) *Response {
|
||||
if len(request.Reqs) == 0 {
|
||||
return newErrorResponse("param invalid: reqs", restmodel.CodeInvalidParam)
|
||||
}
|
||||
result, err := s.Downloader.CreateDirectBatchWithResult(&request)
|
||||
taskIDs, err := s.Downloader.CreateDirectBatch(&request)
|
||||
if err != nil {
|
||||
return newErrorResponse(err.Error())
|
||||
}
|
||||
return newOKResponse(result)
|
||||
return newOKResponse(taskIDs)
|
||||
}
|
||||
|
||||
func (s *Service) patchTask(ctx *Context) *Response {
|
||||
|
||||
@@ -267,65 +267,3 @@ func TestFullConfigAndJobTriggerShareProtectedProtocolState(t *testing.T) {
|
||||
t.Fatalf("job state overwritten: %s", raw)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBatchResolvedAndDirectCreation(t *testing.T) {
|
||||
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.Header().Set("Content-Length", "4")
|
||||
_, _ = w.Write([]byte("test"))
|
||||
}))
|
||||
t.Cleanup(upstream.Close)
|
||||
d := download.NewDownloader(&download.DownloaderConfig{StorageDir: t.TempDir(), TempDir: t.TempDir(), FetchManagers: []fetcher.FetcherManager{&httpProtocol.FetcherManager{}}})
|
||||
if err := d.Setup(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { d.Close() })
|
||||
cfg, err := d.GetConfig()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cfg.MaxRunning = 0
|
||||
if err := d.PutConfig(cfg); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
service, err := NewService(d)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, grouped := range []bool{false, true} {
|
||||
resolved := serviceTestRequest[*download.ResolveResult](t, service, http.MethodPost, "/api/v1/resolve", &restmodel.ResolveTask{Req: &base.Request{URL: upstream.URL + "/resolved"}})
|
||||
result := serviceTestRequest[map[string]json.RawMessage](t, service, http.MethodPost, "/api/v1/tasks/batch", &base.CreateTaskBatch{
|
||||
Group: grouped, GroupName: "folder", Opts: &base.Options{Path: t.TempDir()},
|
||||
Reqs: []*base.CreateTaskBatchItem{
|
||||
{Rid: "invalid"},
|
||||
{Rid: resolved.ID, Req: &base.Request{URL: "unsupported://ignored"}, Opts: &base.Options{SelectFiles: []int{0}}},
|
||||
{Req: &base.Request{URL: upstream.URL + "/direct"}},
|
||||
},
|
||||
})
|
||||
var ids []string
|
||||
if err := json.Unmarshal(result["taskIds"], &ids); err != nil || len(ids) != 2 {
|
||||
t.Fatalf("unexpected task IDs: %+v, %v", result, err)
|
||||
}
|
||||
wantFields := 1
|
||||
if grouped {
|
||||
wantFields = 2
|
||||
}
|
||||
if len(result) != wantFields {
|
||||
t.Fatalf("unexpected response fields: %+v", result)
|
||||
}
|
||||
var parentID string
|
||||
if grouped {
|
||||
if err := json.Unmarshal(result["parentId"], &parentID); err != nil || parentID == "" {
|
||||
t.Fatalf("missing parent ID: %+v", result)
|
||||
}
|
||||
}
|
||||
task := serviceTestRequest[*download.Task](t, service, http.MethodGet, "/api/v1/tasks/"+ids[0], nil)
|
||||
if task.ParentID != parentID || !reflect.DeepEqual(task.Meta.Opts.SelectFiles, []int{0}) {
|
||||
t.Fatalf("resolved task lost membership or selection: %+v", task)
|
||||
}
|
||||
status, response := service.JSON(http.MethodPost, "/api/v1/tasks/batch", nil, []byte(`{"reqs":[{"rid":"missing"},{"req":{"url":"unsupported://bad"}}]}`), nil)
|
||||
var failed restmodel.Result[any]
|
||||
if err := json.Unmarshal(response, &failed); err != nil || status != http.StatusOK || failed.Code == restmodel.CodeOk {
|
||||
t.Fatalf("all-failed batch succeeded: %d %s", status, response)
|
||||
}
|
||||
}
|
||||
}
|
||||
+2
-32
@@ -27,33 +27,6 @@ type Request struct {
|
||||
SkipVerifyCert bool `json:"skipVerifyCert"`
|
||||
}
|
||||
|
||||
// SetLabels replaces all labels on the request.
|
||||
//
|
||||
// Deprecated: In onResolve and onDone, assign request labels directly.
|
||||
// In onStart and onError, use task.meta.req.setLabels instead.
|
||||
func (r *Request) SetLabels(labels map[string]string) {
|
||||
r.Labels = labels
|
||||
}
|
||||
|
||||
// PutLabel sets a label on the request.
|
||||
//
|
||||
// Deprecated: In onResolve and onDone, assign request labels directly.
|
||||
// In onStart and onError, use task.meta.req.putLabel instead.
|
||||
func (r *Request) PutLabel(key, value string) {
|
||||
if r.Labels == nil {
|
||||
r.Labels = make(map[string]string)
|
||||
}
|
||||
r.Labels[key] = value
|
||||
}
|
||||
|
||||
// DelLabel deletes a label from the request.
|
||||
//
|
||||
// Deprecated: In onResolve and onDone, delete request labels directly.
|
||||
// In onStart and onError, use task.meta.req.delLabel instead.
|
||||
func (r *Request) DelLabel(key string) {
|
||||
delete(r.Labels, key)
|
||||
}
|
||||
|
||||
func (r *Request) Validate() error {
|
||||
if r.URL == "" {
|
||||
return fmt.Errorf("invalid request url")
|
||||
@@ -200,14 +173,11 @@ func ParseOptExtra[E any](opts *Options) error {
|
||||
}
|
||||
|
||||
type CreateTaskBatch struct {
|
||||
Reqs []*CreateTaskBatchItem `json:"reqs"`
|
||||
Opts *Options `json:"opts"`
|
||||
Group bool `json:"group"`
|
||||
GroupName string `json:"groupName"`
|
||||
Reqs []*CreateTaskBatchItem `json:"reqs"`
|
||||
Opts *Options `json:"opts"`
|
||||
}
|
||||
|
||||
type CreateTaskBatchItem struct {
|
||||
Rid string `json:"rid,omitempty"`
|
||||
Req *Request `json:"req"`
|
||||
Opts *Options `json:"opts"`
|
||||
}
|
||||
|
||||
Loaded 50 of 215 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user