Compare commits

...
Author SHA1 Message Date
copilot-swe-agent[bot] cfda439b82 Clarify nil request validation error 2026-07-27 03:35:39 +00:00
copilot-swe-agent[bot] c2ada7dbaa Prevent nil request panic during resolve 2026-07-27 03:34:11 +00:00
copilot-swe-agent[bot] 6f0cfdcd91 Initial plan 2026-07-27 03:26:44 +00:00
Levi fdfc6902a7 feat: enhance extension API (#1413) 2026-07-14 20:52:21 +08:00
Levi 9fb3758203 feat: replace gblob with blob http transport (#1407) 2026-07-14 20:05:33 +08:00
moduvoice e2ac8ce64b i18n: add Korean (ko_KR) translation (#1408) 2026-07-13 10:29:05 +08:00
Locon213 c01d11305c feat: add Android dialog mode for external downloads (#1362) 2026-05-07 16:57:50 +08:00
Levi def54f8998 feat: add WebView support for extension (#1359) 2026-04-24 22:16:33 +08:00
Levi a205c0b8d7 feat: add gblob scheme support for extension (#1358) 2026-04-19 19:15:48 +08:00
Copilot ae70dc240c fix: persist download directory changes (#1352) 2026-04-19 16:56:07 +08:00
ScratchBuild d7f6c3b206 feat(lang): Update Japanese translation (#1344) 2026-04-07 10:07:21 +08:00
Levi a83e38dac4 feat: update flutter to 3.24.6 (#1336) 2026-04-03 14:42:32 +08:00
SBS123 d46c06f08d feat(lang): Create ar language (#1332) 2026-03-30 00:05:14 +08:00
Levi a61cdbed67 ci: fix windows asset upload (#1326) 2026-03-28 21:29:25 +08:00
Jaime Muñoz Martín daa8c39dad feat(lang): update Catalan language strings (#1322) 2026-03-23 11:01:59 +08:00
Levi a5cd53f94c release: v1.9.3 2026-03-18 20:00:59 +08:00
Levi 847e88881d fix: non-range HTTP downloaded bytes exceeding total file size (#1317) 2026-03-18 18:53:49 +08:00
Wang YiJia 3d20e818e5 fix: add IsCreated flag to reliably track task creation state (#1314) 2026-03-18 18:35:39 +08:00
Levi 09f7c88ed3 feat: ed2k protocol support (#1316) 2026-03-17 19:42:16 +08:00
Copilot f7b26753c1 docs: migrate docs.gopeed.com links to gopeed.com/docs (#1311) 2026-03-12 22:44:17 +08:00
Levi 9704d4af27 fix: update RPM icon path (#1309) 2026-03-12 21:33:31 +08:00
Jaime Muñoz Martín 1907f404ac feat(lang): Catalan language added (#1303) 2026-03-11 10:26:55 +08:00
Tuna Dinçer 72c5ba7d20 feat(lang): update Turkish translation (#1304) 2026-03-10 13:38:02 +08:00
Levi 5b0254f7c0 release: v1.9.2 2026-03-08 19:52:32 +08:00
Levi d36b345f92 feat: extension store (#1301) 2026-03-06 21:53:19 +08:00
Locon213 bc5c818eae feat(desktop): implement native desktop notifications for task completion and errors (#1293) 2026-03-01 11:12:40 +08:00
Levi 98bd3305b7 feat: update Flutter version to 3.41.2 (#1290) 2026-02-26 20:42:34 +08:00
Tuna Dinçer 3f38848051 feat: Turkish localization improvements 2026-02-25 09:54:15 +08:00
Levi 82d7f8ec81 feat: update debug extension IDs retrieval to use environment (#1281) 2026-02-14 17:25:46 +08:00
Levi a6981d58e9 release: v1.9.1 2026-02-08 18:40:59 +08:00
Copilot 1b1631b41a feat: support post-task script hooks (#1265) 2026-02-08 18:40:06 +08:00
Levi 3eb6df55df feat: implement task patching functionality (#1267) 2026-02-06 23:42:57 +08:00
Levi 6de63c3f63 fix: low FPS on Windows (#1279) 2026-02-06 22:38:31 +08:00
Inori 969615f353 ci: docker image optimization (#1274) 2026-02-05 16:25:59 +08:00
Levi 5a88b98314 fix: add missing lib to Windows ARM64 bundle (#1273) 2026-02-04 18:41:32 +08:00
Levi 2674b1c39e ci: upgrade flutter version on windows arm (#1269) 2026-02-03 21:13:25 +08:00
Copilot 4c90b38247 fix: gbk filename encoding in ZIP extraction (#1260) 2026-01-29 22:37:12 +08:00
Levi 93088b22f1 fix: improve retry handling for expired HTTP redirect url (#1263) 2026-01-29 22:13:42 +08:00
Levi fe90c1d261 fix: deep link hidden handling on MacOS (#1262) 2026-01-28 18:57:09 +08:00
173 changed files with 22485 additions and 3305 deletions

No files matched your search

+1 -1
View File
@@ -12,4 +12,4 @@
#custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2']
# github: monkeyWie
ko_fi: gopeed
custom: https://docs.gopeed.com/donate.html
custom: https://gopeed.com/docs/donate
+22 -24
View File
@@ -13,8 +13,8 @@ on:
default: "false"
env:
GO_VERSION: "1.24"
FLUTTER_VERSION: "3.38.7"
GO_VERSION: "1.25"
FLUTTER_VERSION: "3.41.6"
GA4_MEASUREMENT_ID: ${{ secrets.GA4_MEASUREMENT_ID }}
GA4_API_SECRET: ${{ secrets.GA4_API_SECRET }}
@@ -44,15 +44,17 @@ jobs:
- arch: arm64
os: windows-11-arm
llvm_ver: "20251202" # Only ARM64
flutter_channel: "master"
flutter_version: "8817d45220d" # Only ARM64
flutter_channel: "main"
flutter_version: "7e1c8868"
- arch: amd64
os: windows-2022
flutter_channel: "stable"
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v3
- name: Enable long paths for flutter main branch checks
run: |
git config --global core.longpaths true
# Install LLVM environment only on ARM64
- name: Install llvm-mingw-ucrt-aarch64 (ARM64 only)
@@ -79,12 +81,12 @@ jobs:
"CLANGARM64_ROOT=$target" >> $env:GITHUB_ENV
$b >> $env:GITHUB_PATH
- uses: actions/setup-go@v4
- uses: actions/setup-go@v5
with:
go-version: ${{ env.GO_VERSION }}
- uses: subosito/flutter-action@v2
with:
channel: ${{ matrix.flutter_channel }}
channel: ${{ matrix.flutter_channel || 'stable' }}
flutter-version: ${{ matrix.flutter_version || env.FLUTTER_VERSION }}
- name: Build
env:
@@ -112,6 +114,7 @@ jobs:
$release = "build\windows\arm64\runner\Release\"
$mingw = "$env:CLANGARM64_ROOT\aarch64-w64-mingw32\bin"
cp $mingw\libc++.dll $release
cp $mingw\libunwind.dll $release
}
cp $mingw\libwinpthread-1.dll $release
cp $system\msvcp140.dll $release
@@ -184,7 +187,7 @@ jobs:
mv "Output\gopeed.exe" "Output\Gopeed-$TAG-windows-${{ matrix.arch }}.exe"
Compress-Archive -Path "Output\Gopeed-$TAG-windows-${{ matrix.arch }}.exe" -DestinationPath "Output\Gopeed-$TAG-windows-${{ matrix.arch }}.zip"
- name: Upload
uses: shogo82148/actions-upload-release-asset@v1
uses: shogo82148/actions-upload-release-asset@v1.9.2
with:
upload_url: ${{ needs.get-release.outputs.upload_url }}
asset_path: ui/flutter/build/windows/Output/*
@@ -195,7 +198,7 @@ jobs:
needs: [get-release]
steps:
- uses: actions/checkout@v3
- uses: actions/setup-go@v4
- uses: actions/setup-go@v5
with:
go-version: ${{ env.GO_VERSION }}
- name: Build
@@ -221,7 +224,7 @@ jobs:
needs: [get-release, build-macos-amd64-lib]
steps:
- uses: actions/checkout@v3
- uses: actions/setup-go@v4
- uses: actions/setup-go@v5
with:
go-version: ${{ env.GO_VERSION }}
- uses: actions/setup-node@v3
@@ -312,7 +315,7 @@ jobs:
needs: [get-release]
steps:
- uses: actions/checkout@v3
- uses: actions/setup-go@v4
- uses: actions/setup-go@v5
with:
go-version: ${{ env.GO_VERSION }}
- uses: subosito/flutter-action@v2
@@ -321,7 +324,7 @@ jobs:
channel: master
- run: |
sudo apt update -y
sudo apt install -y ninja-build libgtk-3-dev libayatana-appindicator3-1 libayatana-appindicator3-dev rpm patchelf libfuse2 locate imagemagick potrace libkeybinder-3.0-dev
sudo apt install -y ninja-build libgtk-3-dev libwebkit2gtk-4.1-dev libayatana-appindicator3-1 libayatana-appindicator3-dev rpm patchelf libfuse2 locate libkeybinder-3.0-dev
arch=$(uname -m)
wget -O appimagetool "https://github.com/AppImage/appimagetool/releases/download/continuous/appimagetool-${arch}.AppImage"
chmod +x appimagetool
@@ -336,18 +339,11 @@ jobs:
cd ui/flutter
ICON_PATH=$(find assets -name "logo.png" -o -name "icon.png" | head -n 1)
[ -z "$ICON_PATH" ] && ICON_PATH="assets/icon.png" && touch assets/icon.png
convert "$ICON_PATH" assets/icon.svg
ICON_PATH="assets/icon.svg"
RPM_ARCH=$(uname -m)
mkdir -p linux/packaging/rpm
cat << EOF > linux/packaging/rpm/make_config.yaml
display_name: Gopeed
icon: $ICON_PATH
icon: assets/icon/icon.svg
summary: High speed downloader.
description: A high speed downloader that supports all platforms.
group: Applications/Internet
@@ -415,7 +411,7 @@ jobs:
needs: [get-release]
steps:
- uses: actions/checkout@v3
- uses: actions/setup-go@v4
- uses: actions/setup-go@v5
with:
go-version: ${{ env.GO_VERSION }}
- name: Setup LXD
@@ -484,6 +480,7 @@ jobs:
- cmake
- curl
- libgtk-3-dev
- libwebkit2gtk-4.1-dev
- libkeybinder-3.0-dev
- ninja-build
- unzip
@@ -499,6 +496,7 @@ jobs:
source: .
plugin: nil
stage-packages:
- libwebkit2gtk-4.1-0
- libgtk-3-dev
- libkeybinder-3.0-dev
- libappindicator3-dev
@@ -549,7 +547,7 @@ jobs:
needs: [get-release]
steps:
- uses: actions/checkout@v3
- uses: actions/setup-go@v4
- uses: actions/setup-go@v5
with:
go-version: ${{ env.GO_VERSION }}
- uses: actions/setup-java@v3
@@ -591,7 +589,7 @@ jobs:
needs: [get-release]
steps:
- uses: actions/checkout@v3
- uses: actions/setup-go@v4
- uses: actions/setup-go@v5
with:
go-version: ${{ env.GO_VERSION }}
- uses: subosito/flutter-action@v2
@@ -624,7 +622,7 @@ jobs:
needs: [get-release]
steps:
- uses: actions/checkout@v3
- uses: actions/setup-go@v4
- uses: actions/setup-go@v5
with:
go-version: ${{ env.GO_VERSION }}
- uses: subosito/flutter-action@v2
+2 -2
View File
@@ -17,7 +17,7 @@ on:
- "Dockerfile"
env:
GO_VERSION: "1.24"
GO_VERSION: "1.25"
permissions:
contents: write
@@ -31,7 +31,7 @@ jobs:
steps:
- uses: actions/checkout@v3
- uses: actions/setup-go@v4
- uses: actions/setup-go@v5
with:
go-version: ${{ env.GO_VERSION }}
- uses: release-drafter/release-drafter@v5
+112 -21
View File
@@ -28,8 +28,8 @@ on:
workflow_dispatch:
env:
GO_VERSION: "1.24"
FLUTTER_VERSION: "3.38.7"
GO_VERSION: "1.25"
FLUTTER_VERSION: "3.41.6"
jobs:
# lint:
@@ -50,14 +50,18 @@ jobs:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/setup-go@v4
- uses: actions/setup-go@v5
with:
go-version: ${{ env.GO_VERSION }}
- name: Install webview dependencies
run: |
sudo apt-get update -y
sudo apt-get install -y libgtk-3-dev libwebkit2gtk-4.1-dev xvfb
- name: Run Unit tests.
run: |
# Get packages for testing and coverage (including cmd/web for flags_test.go)
PACKAGES=$(go list ./... | grep -v /bind/ | grep -v -E '/cmd/(?!web)')
go test -v -coverpkg=$(echo "$PACKAGES" | paste -sd "," -) -covermode=count -coverprofile=coverage.txt $PACKAGES
PACKAGES=$(go list ./... | awk '!/\/bind\// && (!/\/cmd\// || /\/cmd\/web$/)')
env CGO_ENABLED=1 xvfb-run -a go test -tags webview -v -coverpkg=$(echo "$PACKAGES" | paste -sd "," -) -covermode=count -coverprofile=coverage.txt $PACKAGES
# Filter out main.go from coverage report to avoid lowering coverage percentage
grep -v "main.go" coverage.txt > coverage_filtered.txt || true
mv coverage_filtered.txt coverage.txt
@@ -65,22 +69,109 @@ jobs:
with:
files: ./coverage.txt
token: ${{ secrets.CODECOV_UPLOAD_TOKEN }}
build-desktop:
name: Build desktop check
runs-on: ${{ matrix.os }}
test-webview-integration:
name: Test webview integration (${{ matrix.name }})
strategy:
fail-fast: false
matrix:
os: [windows-2022, macos-latest, ubuntu-22.04, ubuntu-22.04-arm]
needs: [test]
include:
- name: Windows
os: windows-latest
- name: macOS
os: macos-latest
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v3
- uses: actions/setup-go@v4
- uses: actions/setup-go@v5
with:
go-version: ${{ env.GO_VERSION }}
- uses: subosito/flutter-action@v2
if: runner.os == 'macOS'
with:
flutter-version: ${{ env.FLUTTER_VERSION }}
- name: Test webview
if: runner.os == 'Windows'
run: go test -tags webview -v -timeout 300s ./internal/webview/goprovider/
env:
CGO_ENABLED: 1
- name: Test webview
if: runner.os == 'macOS'
shell: bash
run: |
set -euo pipefail
SOCKET_PATH="$HOME/Library/Application Support/com.gopeed.gopeed/gopeed_webview.sock"
LOG_PATH="$RUNNER_TEMP/gopeed-webview-rpc.log"
rm -f "$SOCKET_PATH"
go build -tags nosqlite -ldflags="-w -s" -buildmode=c-shared -o ui/flutter/macos/Frameworks/libgopeed.dylib github.com/GopeedLab/gopeed/bind/desktop
cd ui/flutter
flutter pub get
flutter run -d macos >"$LOG_PATH" 2>&1 &
FLUTTER_PID=$!
cd ../..
cleanup() {
kill "$FLUTTER_PID" >/dev/null 2>&1 || true
}
trap cleanup EXIT
if ! go test -tags webview -v -timeout 600s ./internal/webview/rpcprovider/ -args -webview-rpc-network unix -webview-rpc-address "$SOCKET_PATH"; then
cat "$LOG_PATH"
exit 1
fi
build-desktop:
strategy:
matrix:
include:
- os: windows-2022
- os: windows-11-arm
llvm_ver: "20251202" # Only ARM64
flutter_channel: "main"
flutter_version: "7e1c8868"
- os: macos-latest
- os: ubuntu-22.04
- os: ubuntu-22.04-arm
flutter_channel: "main"
name: Build desktop check (${{ matrix.os }})
runs-on: ${{ matrix.os }}
needs: [ test ]
steps:
- uses: actions/checkout@v3
- name: Enable long paths for flutter main branch checks
run: |
git config --global core.longpaths true
- name: Install llvm-mingw-ucrt-aarch64 (ARM64 only)
if: matrix.os == 'windows-11-arm'
run: |
$ver = "${{ matrix.llvm_ver }}"
$url = "https://github.com/mstorsjo/llvm-mingw/releases/download/$ver/llvm-mingw-$ver-ucrt-aarch64.zip"
$zip = "$env:RUNNER_TEMP\\llvm.zip"
$extract = "$env:RUNNER_TEMP\\extract"
$target = "C:\\clangarm64"
curl -L $url -o $zip
rm -r -fo $extract,$target -ea Ignore
mkdir $extract | Out-Null
tar -xf $zip -C $extract
mv (Get-ChildItem $extract)[0].FullName $target
$b = "$target\\bin"
"CC=$b\\clang.exe" >> $env:GITHUB_ENV
"CXX=$b\\clang++.exe" >> $env:GITHUB_ENV
"CLANGARM64_BIN=$b" >> $env:GITHUB_ENV
"CGO_ENABLED=1" >> $env:GITHUB_ENV
"CLANGARM64_ROOT=$target" >> $env:GITHUB_ENV
$b >> $env:GITHUB_PATH
- uses: actions/setup-go@v5
with:
go-version: ${{ env.GO_VERSION }}
- uses: subosito/flutter-action@v2
with:
flutter-version: ${{ env.FLUTTER_VERSION }}
channel: master
channel: ${{ matrix.flutter_channel || 'stable' }}
flutter-version: ${{ matrix.flutter_version || env.FLUTTER_VERSION }}
- if: runner.os == 'Windows'
run: |
go build -tags nosqlite -ldflags="-w -s" -buildmode=c-shared -o ui/flutter/windows/libgopeed.dll github.com/GopeedLab/gopeed/bind/desktop
@@ -94,7 +185,7 @@ jobs:
- if: runner.os == 'Linux'
run: |
sudo apt-get update -y
sudo apt-get install -y ninja-build libgtk-3-dev libayatana-appindicator3-dev libkeybinder-3.0-dev
sudo apt-get install -y ninja-build libgtk-3-dev libwebkit2gtk-4.1-dev libayatana-appindicator3-dev libkeybinder-3.0-dev
go build -tags nosqlite -ldflags="-w -s" -buildmode=c-shared -o ui/flutter/linux/bundle/lib/libgopeed.so github.com/GopeedLab/gopeed/bind/desktop
cd ui/flutter
flutter build linux
@@ -103,11 +194,11 @@ jobs:
runs-on: ${{ matrix.os }}
strategy:
matrix:
os: [macos-14, ubuntu-latest]
needs: [test]
os: [ macos-14, ubuntu-latest ]
needs: [ test ]
steps:
- uses: actions/checkout@v3
- uses: actions/setup-go@v4
- uses: actions/setup-go@v5
with:
go-version: ${{ env.GO_VERSION }}
- uses: subosito/flutter-action@v2
@@ -136,10 +227,10 @@ jobs:
build-web:
name: Build web check
runs-on: ubuntu-latest
needs: [test]
needs: [ test ]
steps:
- uses: actions/checkout@v3
- uses: actions/setup-go@v4
- uses: actions/setup-go@v5
with:
go-version: ${{ env.GO_VERSION }}
- uses: subosito/flutter-action@v2
@@ -157,10 +248,10 @@ jobs:
build-docker:
name: Build docker check
runs-on: ubuntu-latest
needs: [test]
needs: [ test ]
steps:
- uses: actions/checkout@v3
- uses: actions/setup-go@v4
- uses: actions/setup-go@v5
with:
go-version: ${{ env.GO_VERSION }}
- uses: subosito/flutter-action@v2
+2 -2
View File
@@ -1,4 +1,4 @@
FROM golang:1.24.11 AS go
FROM golang:1.25.4-alpine3.22 AS go
WORKDIR /app
COPY ./go.mod ./go.sum ./
RUN go mod download
@@ -8,7 +8,7 @@ RUN CGO_ENABLED=0 go build -tags nosqlite,web \
-ldflags="-s -w -X github.com/GopeedLab/gopeed/pkg/base.Version=$VERSION -X github.com/GopeedLab/gopeed/pkg/base.InDocker=true" \
-o dist/gopeed github.com/GopeedLab/gopeed/cmd/web
FROM alpine:3.18
FROM alpine:3.23
LABEL maintainer="monkeyWie"
WORKDIR /app
COPY --from=go /app/dist/gopeed ./
+7 -7
View File
@@ -1,10 +1,10 @@
# [![](_docs/img/banner.png)](https://gopeed.com)
# [![](_docs/img/banner.svg)](https://gopeed.com)
[![Test Status](https://github.com/GopeedLab/gopeed/workflows/test/badge.svg)](https://github.com/GopeedLab/gopeed/actions?query=workflow%3Atest)
[![Codecov](https://codecov.io/gh/GopeedLab/gopeed/branch/main/graph/badge.svg)](https://codecov.io/gh/GopeedLab/gopeed)
[![Release](https://img.shields.io/github/release/GopeedLab/gopeed.svg)](https://github.com/GopeedLab/gopeed/releases)
[![Download](https://img.shields.io/github/downloads/GopeedLab/gopeed/total.svg)](https://github.com/GopeedLab/gopeed/releases)
[![Donate](https://img.shields.io/badge/%24-donate-ff69b4.svg)](https://docs.gopeed.com/donate.html)
[![Donate](https://img.shields.io/badge/%24-donate-ff69b4.svg)](https://gopeed.com/docs/donate)
[![WeChat](https://img.shields.io/badge/WeChat%20Official%20Account-07C160?logo=wechat&logoColor=white)](https://raw.githubusercontent.com/GopeedLab/gopeed/main/_docs/img/weixin.png)
[![Discord](https://img.shields.io/discord/1037992631881449472?label=Discord&logo=discord&style=social)](https://discord.gg/ZUJqJrwCGB)
@@ -16,9 +16,9 @@
## 🚀 Introduction
Gopeed (full name Go Speed), a high-speed downloader developed by `Golang` + `Flutter`, supports (HTTP, BitTorrent, Magnet) protocol, and supports all platforms. In addition to basic download functions, Gopeed is also a highly customizable downloader that supports implementing more features through integration with [APIs](https://docs.gopeed.com/dev-api.html) or installation and development of [extensions](https://docs.gopeed.com/dev-extension.html).
Gopeed (full name Go Speed), a high-speed downloader developed by `Golang` + `Flutter`, supports (HTTP, BitTorrent, Magnet, ED2K) protocol, and supports all platforms. In addition to basic download functions, Gopeed is also a highly customizable downloader that supports implementing more features through integration with [APIs](https://gopeed.com/docs/dev-api) or installation and development of [extensions](https://gopeed.com/docs/dev-extension).
Visit ✈ [Official Website](https://gopeed.com) | 📖 [Official Docs](https://docs.gopeed.com)
Visit ✈ [Official Website](https://gopeed.com) | 📖 [Official Docs](https://gopeed.com/docs)
## ⬇️ Download
@@ -165,7 +165,7 @@ Visit ✈ [Official Website](https://gopeed.com) | 📖 [Official Docs](https://
</tbody>
</table>
More about installation, please refer to [Installation](https://docs.gopeed.com/install.html)
More about installation, please refer to [Installation](https://gopeed.com/docs/install)
### 🛠️ Command tool
@@ -187,7 +187,7 @@ Follow our WeChat Official Account to get the latest updates and news.
## 💝 Donate
If you like this project, please consider [donating](https://docs.gopeed.com/donate.html) to support the development of this project, thank you!
If you like this project, please consider [donating](https://gopeed.com/docs/donate) to support the development of this project, thank you!
## 🖼️ Showcase
@@ -201,7 +201,7 @@ This project is divided into two parts, the front end uses `flutter`, the back e
### 🌍 Environment
1. Golang 1.24+
1. Golang 1.25+
2. Flutter 3.38+
### 📋 Clone
+7 -7
View File
@@ -1,10 +1,10 @@
# [![](_docs/img/banner.png)](https://gopeed.com)
# [![](_docs/img/banner.svg)](https://gopeed.com)
[![Test Status](https://github.com/GopeedLab/gopeed/workflows/test/badge.svg)](https://github.com/GopeedLab/gopeed/actions?query=workflow%3Atest)
[![Codecov](https://codecov.io/gh/GopeedLab/gopeed/branch/main/graph/badge.svg)](https://codecov.io/gh/GopeedLab/gopeed)
[![Release](https://img.shields.io/github/release/GopeedLab/gopeed.svg)](https://github.com/GopeedLab/gopeed/releases)
[![Download](https://img.shields.io/github/downloads/GopeedLab/gopeed/total.svg)](https://github.com/GopeedLab/gopeed/releases)
[![Donate](https://img.shields.io/badge/%24-donate-ff69b4.svg)](https://docs.gopeed.com/donate.html)
[![Donate](https://img.shields.io/badge/%24-donate-ff69b4.svg)](https://gopeed.com/docs/donate)
[![WeChat](https://img.shields.io/badge/WeChat%20Official%20Account-07C160?logo=wechat&logoColor=white)](https://raw.githubusercontent.com/GopeedLab/gopeed/main/_docs/img/weixin.png)
[![Discord](https://img.shields.io/discord/1037992631881449472?label=Discord&logo=discord&style=social)](https://discord.gg/ZUJqJrwCGB)
@@ -16,9 +16,9 @@
## 🚀 はじめに
Gopeed (正式名 Go Speed) は `Golang` + `Flutter` によって開発された高速ダウンローダーで、(HTTP、BitTorrent、Magnet) プロトコルをサポートし、すべてのプラットフォームをサポートします。基本的なダウンロード機能に加え、[APIs](https://docs.gopeed.com/dev-api.html)との連動や[拡張機能](https://docs.gopeed.com/dev-extension.html)のインストール・開発による追加機能にも対応した、カスタマイズ性の高いダウンローダーです。
Gopeed (正式名 Go Speed) は `Golang` + `Flutter` によって開発された高速ダウンローダーで、(HTTP、BitTorrent、Magnet、ED2K) プロトコルをサポートし、すべてのプラットフォームをサポートします。基本的なダウンロード機能に加え、[APIs](https://gopeed.com/docs/dev-api)との連動や[拡張機能](https://gopeed.com/docs/dev-extension)のインストール・開発による追加機能にも対応した、カスタマイズ性の高いダウンローダーです。
見て下さい ✈ [公式ウェブサイト](https://gopeed.com) | 📖 [開発ドキュメント](https://docs.gopeed.com)
見て下さい ✈ [公式ウェブサイト](https://gopeed.com) | 📖 [開発ドキュメント](https://gopeed.com/docs)
## ⬇️ インストール
@@ -156,7 +156,7 @@ Gopeed (正式名 Go Speed) は `Golang` + `Flutter` によって開発された
</tr>
</tbody>
</table>
インストールについての詳細は、[インストール](https://docs.gopeed.com/install.html)を参照してください。
インストールについての詳細は、[インストール](https://gopeed.com/docs/install)を参照してください。
### 🛠️ コマンドツール
@@ -168,7 +168,7 @@ Gopeed (正式名 Go Speed) は `Golang` + `Flutter` によって開発された
## 💝 寄付
もしこのプロジェクトがお気に召しましたら、このプロジェクトの発展を支援するために[寄付](https://docs.gopeed.com/donate.html)をご検討ください!
もしこのプロジェクトがお気に召しましたら、このプロジェクトの発展を支援するために[寄付](https://gopeed.com/docs/donate)をご検討ください!
## 🖼️ ショーケース
@@ -182,7 +182,7 @@ Gopeed (正式名 Go Speed) は `Golang` + `Flutter` によって開発された
### 🌍 環境
1. Go 言語 1.24+
1. Go 言語 1.25+
2. Flutter 3.38+
### 📋 クローン
+7 -7
View File
@@ -1,10 +1,10 @@
# [![](_docs/img/banner.png)](https://gopeed.com)
# [![](_docs/img/banner.svg)](https://gopeed.com)
[![Trạng thái kiểm tra](https://github.com/GopeedLab/gopeed/workflows/test/badge.svg)](https://github.com/GopeedLab/gopeed/actions?query=workflow%3Atest)
[![Codecov](https://codecov.io/gh/GopeedLab/gopeed/branch/main/graph/badge.svg)](https://codecov.io/gh/GopeedLab/gopeed)
[![Phiên bản](https://img.shields.io/github/release/GopeedLab/gopeed.svg)](https://github.com/GopeedLab/gopeed/releases)
[![Tải về](https://img.shields.io/github/downloads/GopeedLab/gopeed/total.svg)](https://github.com/GopeedLab/gopeed/releases)
[![Ủng hộ](https://img.shields.io/badge/%24-ủng%20hộ-ff69b4.svg)](https://docs.gopeed.com/donate.html)
[![Ủng hộ](https://img.shields.io/badge/%24-ủng%20hộ-ff69b4.svg)](https://gopeed.com/docs/donate)
[![WeChat](https://img.shields.io/badge/WeChat%20Official%20Account-07C160?logo=wechat&logoColor=white)](https://raw.githubusercontent.com/GopeedLab/gopeed/main/_docs/img/weixin.png)
[![Discord](https://img.shields.io/discord/1037992631881449472?label=Discord&logo=discord&style=social)](https://discord.gg/ZUJqJrwCGB)
@@ -16,9 +16,9 @@
## 🚀 Giới thiệu
Gopeed (tên đầy đủ Go Speed), một công cụ tải xuống tốc độ cao được phát triển bởi `Golang` + `Flutter`, hỗ trợ giao thức (HTTP, BitTorrent, Magnet) và hỗ trợ tất cả các nền tảng. Ngoài các chức năng tải xuống cơ bản, Gopeed còn là một công cụ tải xuống có thể tùy chỉnh cao cho phép triển khai thêm tính năng thông qua việc tích hợp với [APIs](https://docs.gopeed.com/dev-api.html) hoặc cài đặt và phát triển các [tiện ích mở rộng](https://docs.gopeed.com/dev-extension.html).
Gopeed (tên đầy đủ Go Speed), một công cụ tải xuống tốc độ cao được phát triển bởi `Golang` + `Flutter`, hỗ trợ giao thức (HTTP, BitTorrent, Magnet, ED2K) và hỗ trợ tất cả các nền tảng. Ngoài các chức năng tải xuống cơ bản, Gopeed còn là một công cụ tải xuống có thể tùy chỉnh cao cho phép triển khai thêm tính năng thông qua việc tích hợp với [APIs](https://gopeed.com/docs/dev-api) hoặc cài đặt và phát triển các [tiện ích mở rộng](https://gopeed.com/docs/dev-extension).
Truy cập ✈ [Trang web chính thức](https://gopeed.com) | 📖 [Tài liệu chính thức](https://docs.gopeed.com)
Truy cập ✈ [Trang web chính thức](https://gopeed.com) | 📖 [Tài liệu chính thức](https://gopeed.com/docs)
## ⬇️ Tải về
@@ -156,7 +156,7 @@ Truy cập ✈ [Trang web chính thức](https://gopeed.com) | 📖 [Tài liệu
</tr>
</tbody>
</table>
Thêm thông tin về cài đặt, vui lòng tham khảo [Cài đặt](https://docs.gopeed.com/install.html)
Thêm thông tin về cài đặt, vui lòng tham khảo [Cài đặt](https://gopeed.com/docs/install)
### 🛠️ Công cụ lệnh
@@ -174,7 +174,7 @@ Theo dõi tài khoản chính thức để nhận các cập nhật và tin tứ
## 💝 Quyên góp
Nếu bạn thích dự án này, xin vui lòng xem xét [quyên góp](https://docs.gopeed.com/donate.html) để hỗ trợ phát triển dự án này, cảm ơn bạn!
Nếu bạn thích dự án này, xin vui lòng xem xét [quyên góp](https://gopeed.com/docs/donate) để hỗ trợ phát triển dự án này, cảm ơn bạn!
## 🖼️ Trưng bày
@@ -188,7 +188,7 @@ Dự án này được chia thành hai phần, phần giao diện sử dụng `f
### 🌍 Environment
1. Golang 1.24+
1. Golang 1.25+
2. Flutter 3.38+
### 📋 Clone
+7 -7
View File
@@ -1,10 +1,10 @@
# [![](_docs/img/banner.png)](https://gopeed.com)
# [![](_docs/img/banner.svg)](https://gopeed.com)
[![Test Status](https://github.com/GopeedLab/gopeed/workflows/test/badge.svg)](https://github.com/GopeedLab/gopeed/actions?query=workflow%3Atest)
[![Codecov](https://codecov.io/gh/GopeedLab/gopeed/branch/main/graph/badge.svg)](https://codecov.io/gh/GopeedLab/gopeed)
[![Release](https://img.shields.io/github/release/GopeedLab/gopeed.svg)](https://github.com/GopeedLab/gopeed/releases)
[![Download](https://img.shields.io/github/downloads/GopeedLab/gopeed/total.svg)](https://github.com/GopeedLab/gopeed/releases)
[![Donate](https://img.shields.io/badge/%24-donate-ff69b4.svg)](https://docs.gopeed.com/zh/donate.html)
[![Donate](https://img.shields.io/badge/%24-donate-ff69b4.svg)](https://gopeed.com/docs/donate)
[![WeChat](https://img.shields.io/badge/%E5%BE%AE%E4%BF%A1%E5%85%AC%E4%BC%97%E5%8F%B7-07C160?logo=wechat&logoColor=white)](https://raw.githubusercontent.com/GopeedLab/gopeed/main/_docs/img/weixin.png)
[![Discord](https://img.shields.io/discord/1037992631881449472?label=Discord&logo=discord&style=social)](https://discord.gg/ZUJqJrwCGB)
@@ -16,9 +16,9 @@
## 🚀 介绍
Gopeed(全称 Go Speed),直译过来中文名叫做`够快下载器`(不是狗屁下载器!),是一款由`Golang`+`Flutter`开发的高速下载器,支持(HTTP、BitTorrent、Magnet)协议下载,并且支持全平台使用。除了基本的下载功能外,Gopeed 还是一款高度可定制化的下载器,支持通过对接[APIs](https://docs.gopeed.com/zh/dev-api.html)或者安装和开发[扩展](https://docs.gopeed.com/zh/dev-extension.html)来实现更多的功能。
Gopeed(全称 Go Speed),直译过来中文名叫做`够快下载器`(不是狗屁下载器!),是一款由`Golang`+`Flutter`开发的高速下载器,支持(HTTP、BitTorrent、Magnet、ED2K)协议下载,并且支持全平台使用。除了基本的下载功能外,Gopeed 还是一款高度可定制化的下载器,支持通过对接[APIs](https://gopeed.com/docs/dev-api)或者安装和开发[扩展](https://gopeed.com/docs/dev-extension)来实现更多的功能。
访问 ✈ [官方网站](https://gopeed.com/zh-CN) | 📖 [官方文档](https://docs.gopeed.com/zh/)
访问 ✈ [官方网站](https://gopeed.com/zh-CN) | 📖 [官方文档](https://gopeed.com/docs)
## ⬇️ 下载
@@ -156,7 +156,7 @@ Gopeed(全称 Go Speed),直译过来中文名叫做`够快下载器`(不
</tr>
</tbody>
</table>
更多关于安装的内容请参考[安装文档](https://docs.gopeed.com/zh/install.html)
更多关于安装的内容请参考[安装文档](https://gopeed.com/docs/install)
### 🛠️ 命令行工具
@@ -178,7 +178,7 @@ Gopeed 还提供了浏览器扩展用于接管浏览器下载,支持 Chrome、
## 💝 赞助
如果觉得项目对你有帮助,请考虑[赞助](https://docs.gopeed.com/zh/donate)以支持这个项目的发展,非常感谢!
如果觉得项目对你有帮助,请考虑[赞助](https://gopeed.com/docs/donate)以支持这个项目的发展,非常感谢!
## 🖼️ 界面展示
@@ -192,7 +192,7 @@ Gopeed 还提供了浏览器扩展用于接管浏览器下载,支持 Chrome、
### 🌍 环境要求
1. Golang 1.24+
1. Golang 1.25+
2. Flutter 3.38+
### 📋 克隆项目
+7 -7
View File
@@ -1,10 +1,10 @@
# [![](_docs/img/banner.png)](https://gopeed.com)
# [![](_docs/img/banner.svg)](https://gopeed.com)
[![Test Status](https://github.com/GopeedLab/gopeed/workflows/test/badge.svg)](https://github.com/GopeedLab/gopeed/actions?query=workflow%3Atest)
[![Codecov](https://codecov.io/gh/GopeedLab/gopeed/branch/main/graph/badge.svg)](https://codecov.io/gh/GopeedLab/gopeed)
[![Release](https://img.shields.io/github/release/GopeedLab/gopeed.svg)](https://github.com/GopeedLab/gopeed/releases)
[![Download](https://img.shields.io/github/downloads/GopeedLab/gopeed/total.svg)](https://github.com/GopeedLab/gopeed/releases)
[![Donate](https://img.shields.io/badge/%24-donate-ff69b4.svg)](https://docs.gopeed.com/donate.html)
[![Donate](https://img.shields.io/badge/%24-donate-ff69b4.svg)](https://gopeed.com/docs/donate)
[![WeChat](https://img.shields.io/badge/%E5%BE%AE%E4%BF%A1%E5%85%AC%E4%BC%97%E5%8F%B7-07C160?logo=wechat&logoColor=white)](https://raw.githubusercontent.com/GopeedLab/gopeed/main/_docs/img/weixin.png)
[![Discord](https://img.shields.io/discord/1037992631881449472?label=Discord&logo=discord&style=social)](https://discord.gg/ZUJqJrwCGB)
@@ -16,9 +16,9 @@
## 🚀 簡介
Gopeed(全稱 Go Speed),是一款使用`Golang`+`Flutter`編寫的高速下載軟體,支援(HTTP、BitTorrent、Magnet)協定,同時支援所有的平台。
Gopeed(全稱 Go Speed),是一款使用`Golang`+`Flutter`編寫的高速下載軟體,支援(HTTP、BitTorrent、Magnet、ED2K)協定,同時支援所有的平台。
前往 ✈ [主頁](https://gopeed.com/zh-CN) | 📖 [文檔](https://docs.gopeed.com/zh/)
前往 ✈ [主頁](https://gopeed.com/zh-CN) | 📖 [文檔](https://gopeed.com/docs)
## ⬇️ 下載
@@ -157,7 +157,7 @@ Gopeed(全稱 Go Speed),是一款使用`Golang`+`Flutter`編寫的高速
</tbody>
</table>
更多關於安裝的內容請參考[安裝文檔](https://docs.gopeed.com/zh/install.html)
更多關於安裝的內容請參考[安裝文檔](https://gopeed.com/docs/install)
### 🛠️ 使用 CLI 安裝
@@ -175,7 +175,7 @@ go install github.com/GopeedLab/gopeed/cmd/gopeed@latest
## 💝 贊助
如果你認為該項目對你有所幫助,請考慮[贊助](https://docs.gopeed.com/zh/donate)以支持該項目的持續發展,謝謝!
如果你認為該項目對你有所幫助,請考慮[贊助](https://gopeed.com/docs/donate)以支持該項目的持續發展,謝謝!
## 🖼️ 軟體介面
@@ -189,7 +189,7 @@ go install github.com/GopeedLab/gopeed/cmd/gopeed@latest
### 🌍 開發環境
1. Golang 1.24+
1. Golang 1.25+
2. Flutter 3.38+
### 📋 克隆項目
Binary file not shown.

Before

Width:  |  Height:  |  Size: 51 KiB

+54
View File
@@ -0,0 +1,54 @@
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 1200 400" width="1200" height="400">
<rect width="1200" height="400" fill="#040605"/>
<g fill="#5dc76c">
<rect x="-100" y="50" width="150" height="2" opacity="0.6">
<animate attributeName="x" from="-200" to="1300" dur="0.8s" repeatCount="indefinite"/>
</rect>
<rect x="-200" y="150" width="200" height="1" opacity="0.4">
<animate attributeName="x" from="-300" to="1300" dur="1.2s" repeatCount="indefinite"/>
</rect>
<rect x="-50" y="250" width="180" height="3" opacity="0.8">
<animate attributeName="x" from="-200" to="1300" dur="0.6s" repeatCount="indefinite"/>
</rect>
<rect x="-150" y="350" width="220" height="2" opacity="0.5">
<animate attributeName="x" from="-300" to="1300" dur="1s" repeatCount="indefinite"/>
</rect>
<rect x="-100" y="100" width="100" height="1" opacity="0.7">
<animate attributeName="x" from="-150" to="1300" dur="0.9s" repeatCount="indefinite"/>
</rect>
<rect x="-250" y="300" width="250" height="2" opacity="0.3">
<animate attributeName="x" from="-400" to="1300" dur="1.5s" repeatCount="indefinite"/>
</rect>
</g>
<defs>
<filter id="logoGlow" x="-50%" y="-50%" width="200%" height="200%">
<feGaussianBlur stdDeviation="12" result="blur" />
<feComposite in="SourceGraphic" in2="blur" operator="over" />
</filter>
</defs>
<g transform="translate(600, 160)">
<!-- Floating animation -->
<g>
<animateTransform attributeName="transform" type="translate" values="0,-8; 0,8; 0,-8" dur="3s" repeatCount="indefinite" />
<!-- Pulsing glow behind logo -->
<circle cx="0" cy="0" r="45" fill="#5dc76c" filter="url(#logoGlow)" opacity="0.3">
<animate attributeName="opacity" values="0.1; 0.6; 0.1" dur="2s" repeatCount="indefinite" />
<animate attributeName="r" values="40; 65; 40" dur="2s" repeatCount="indefinite" />
</circle>
<!-- Breathing scale animation -->
<g>
<animateTransform attributeName="transform" type="scale" values="0.95; 1.05; 0.95" dur="2s" repeatCount="indefinite" />
<g transform="scale(0.12) translate(-512, -512)">
<path style="opacity:0.998" fill="#5dc76c" d="M 951.5,763.5 C 940.739,755.276 929.405,747.776 917.5,741C 908.477,736.319 899.143,732.485 889.5,729.5C 885.844,625.779 840.844,545.613 754.5,489C 674.802,441.453 590.135,430.12 500.5,455C 419.755,480.605 359.588,530.771 320,605.5C 278.585,565.816 230.752,555.649 176.5,575C 131.948,594.935 106.448,629.102 100,677.5C 99.3333,685.833 99.3333,694.167 100,702.5C 102.081,714.824 105.081,726.824 109,738.5C 95.4882,745.492 82.9882,753.826 71.5,763.5C 37.7121,705.639 16.5454,643.306 8,576.5C -9.41698,416.759 35.9164,278.093 144,160.5C 244.661,58.2827 367.161,5.94939 511.5,3.5C 678.975,7.48829 814.142,74.8216 917,205.5C 1011.33,336.426 1040,480.759 1003,638.5C 991.564,682.671 974.397,724.337 951.5,763.5 Z"/>
<path style="opacity:0.999" fill="#fefefe" d="M 951.5,763.5 C 949.28,768.272 946.78,772.938 944,777.5C 938.305,786.913 932.138,795.913 925.5,804.5C 915.954,796.888 905.288,791.055 893.5,787C 878.282,782.958 863.115,778.791 848,774.5C 854.013,716.891 841.346,663.891 810,615.5C 768.341,553.228 711.174,512.395 638.5,493C 560.311,479.566 490.311,497.566 428.5,547C 411.963,560.536 397.13,575.703 384,592.5C 366.745,616.923 348.911,640.923 330.5,664.5C 310.135,651.302 290.469,637.135 271.5,622C 238.448,602.307 206.782,604.307 176.5,628C 159.375,642.752 148.875,661.252 145,683.5C 143.968,692.716 144.968,701.716 148,710.5C 154.334,725.339 160.167,740.339 165.5,755.5C 138.118,764.688 114.451,779.688 94.5,800.5C 85.653,788.968 77.9863,776.635 71.5,763.5C 82.9882,753.826 95.4882,745.492 109,738.5C 105.081,726.824 102.081,714.824 100,702.5C 99.3333,694.167 99.3333,685.833 100,677.5C 106.448,629.102 131.948,594.935 176.5,575C 230.752,555.649 278.585,565.816 320,605.5C 359.588,530.771 419.755,480.605 500.5,455C 590.135,430.12 674.802,441.453 754.5,489C 840.844,545.613 885.844,625.779 889.5,729.5C 899.143,732.485 908.477,736.319 917.5,741C 929.405,747.776 940.739,755.276 951.5,763.5 Z"/>
<path style="opacity:0.998" fill="#5dc76c" d="M 925.5,804.5 C 850.791,908.608 750.791,975.775 625.5,1006C 609.308,1009.7 592.975,1012.53 576.5,1014.5C 615.708,969.938 655.042,925.438 694.5,881C 664.168,880.5 633.835,880.333 603.5,880.5C 603.5,811.167 603.5,741.833 603.5,672.5C 542.167,672.5 480.833,672.5 419.5,672.5C 419.5,741.5 419.5,810.5 419.5,879.5C 388.493,879.167 357.493,879.5 326.5,880.5C 326.624,881.107 326.957,881.44 327.5,881.5C 366.805,925.808 406.138,970.141 445.5,1014.5C 318.63,996.317 212.796,939.651 128,844.5C 115.85,830.525 104.683,815.859 94.5,800.5C 114.451,779.688 138.118,764.688 165.5,755.5C 160.167,740.339 154.334,725.339 148,710.5C 144.968,701.716 143.968,692.716 145,683.5C 148.875,661.252 159.375,642.752 176.5,628C 206.782,604.307 238.448,602.307 271.5,622C 290.469,637.135 310.135,651.302 330.5,664.5C 348.911,640.923 366.745,616.923 384,592.5C 397.13,575.703 411.963,560.536 428.5,547C 490.311,497.566 560.311,479.566 638.5,493C 711.174,512.395 768.341,553.228 810,615.5C 841.346,663.891 854.013,716.891 848,774.5C 863.115,778.791 878.282,782.958 893.5,787C 905.288,791.055 915.954,796.888 925.5,804.5 Z"/>
<path style="opacity:0.999" fill="#fefefe" d="M 576.5,1014.5 C 532.836,1019.82 489.169,1019.82 445.5,1014.5C 406.138,970.141 366.805,925.808 327.5,881.5C 358.167,880.833 388.833,880.167 419.5,879.5C 419.5,810.5 419.5,741.5 419.5,672.5C 480.833,672.5 542.167,672.5 603.5,672.5C 603.5,741.833 603.5,811.167 603.5,880.5C 633.835,880.333 664.168,880.5 694.5,881C 655.042,925.438 615.708,969.938 576.5,1014.5 Z"/>
<path style="opacity:1" fill="#ade2b5" d="M 419.5,879.5 C 388.833,880.167 358.167,880.833 327.5,881.5C 326.957,881.44 326.624,881.107 326.5,880.5C 357.493,879.5 388.493,879.167 419.5,879.5 Z"/>
</g>
</g>
</g>
</g>
<text x="600" y="300" font-family="sans-serif" font-size="72" font-style="italic" font-weight="900" text-anchor="middle" fill="#fff" letter-spacing="8">GOPEED</text>
</svg>

After

Width:  |  Height:  |  Size: 6.0 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 309 KiB

After

Width:  |  Height:  |  Size: 1.2 MiB

+1
View File
@@ -16,6 +16,7 @@ func Start(cfg *C.char) (int, *C.char) {
return 0, C.CString(err.Error())
}
config.ProductionMode = true
applyWebViewProvider(&config)
realPort, err := rest.Start(&config)
if err != nil {
return 0, C.CString(err.Error())
+23
View File
@@ -0,0 +1,23 @@
//go:build darwin
package main
import (
"github.com/GopeedLab/gopeed/internal/webview/rpcprovider"
enginewebview "github.com/GopeedLab/gopeed/pkg/download/engine/webview"
"github.com/GopeedLab/gopeed/pkg/rest/model"
)
func applyWebViewProvider(config *model.StartConfig) {
if config == nil {
return
}
config.WebViewProvider = newDesktopWebViewProvider(config.WebViewRPCConfig)
}
func newDesktopWebViewProvider(rpcCfg *enginewebview.RPCConfig) enginewebview.Provider {
if rpcCfg != nil && rpcCfg.Enabled() {
return rpcprovider.New(*rpcCfg)
}
return enginewebview.NewUnavailableProvider()
}
+15
View File
@@ -0,0 +1,15 @@
//go:build !darwin
package main
import (
"github.com/GopeedLab/gopeed/internal/webview/goprovider"
"github.com/GopeedLab/gopeed/pkg/rest/model"
)
func applyWebViewProvider(config *model.StartConfig) {
if config == nil {
return
}
config.WebViewProvider = goprovider.New()
}
+9
View File
@@ -4,6 +4,7 @@ package libgopeed
import "C"
import (
"encoding/json"
"github.com/GopeedLab/gopeed/internal/webview/rpcprovider"
"github.com/GopeedLab/gopeed/pkg/rest"
"github.com/GopeedLab/gopeed/pkg/rest/model"
)
@@ -14,9 +15,17 @@ func Start(cfg string) (int, error) {
return 0, err
}
config.ProductionMode = true
applyWebViewProvider(&config)
return rest.Start(&config)
}
func Stop() {
rest.Stop()
}
func applyWebViewProvider(config *model.StartConfig) {
if config == nil || config.WebViewRPCConfig == nil || !config.WebViewRPCConfig.Enabled() {
return
}
config.WebViewProvider = rpcprovider.New(*config.WebViewRPCConfig)
}
+723
View File
@@ -0,0 +1,723 @@
package main
import (
"encoding/json"
"flag"
"fmt"
"os"
"path/filepath"
"time"
"github.com/GopeedLab/gopeed/internal/webview/goprovider"
"github.com/GopeedLab/gopeed/internal/webview/rpcprovider"
"github.com/GopeedLab/gopeed/pkg/base"
"github.com/GopeedLab/gopeed/pkg/download"
enginewebview "github.com/GopeedLab/gopeed/pkg/download/engine/webview"
)
type kvFlag map[string]string
type webViewConfig struct {
provider string
network string
address string
token string
}
func (l *kvFlag) String() string {
if l == nil {
return ""
}
data, _ := json.Marshal(map[string]string(*l))
return string(data)
}
func (l *kvFlag) Set(value string) error {
if *l == nil {
*l = make(map[string]string)
}
idx := -1
for i := 0; i < len(value); i++ {
if value[i] == '=' {
idx = i
break
}
}
if idx <= 0 {
return fmt.Errorf("invalid label %q, expected key=value", value)
}
(*l)[value[:idx]] = value[idx+1:]
return nil
}
func main() {
os.Exit(goprovider.RunMainThreadLoop(run))
}
func run() int {
if len(os.Args) < 2 {
usage()
return 2
}
switch os.Args[1] {
case "resolve":
if err := runResolve(os.Args[2:]); err != nil {
fmt.Fprintln(os.Stderr, err)
return 1
}
case "download":
if err := runDownload(os.Args[2:]); err != nil {
fmt.Fprintln(os.Stderr, err)
return 1
}
case "eval":
if err := runEval(os.Args[2:]); err != nil {
fmt.Fprintln(os.Stderr, err)
return 1
}
case "-h", "--help", "help":
usage()
default:
fmt.Fprintf(os.Stderr, "unknown subcommand %q\n", os.Args[1])
usage()
return 2
}
return 0
}
func usage() {
fmt.Fprintf(os.Stderr, `gopeed extmock CLI
Usage:
extmock resolve [flags] <extension-dir> <url>
extmock download [flags] <extension-dir> <url> <output-dir>
extmock eval [flags] [script-file]
Flags:
-label key=value Request label, repeatable
-setting key=value Extension setting override, repeatable
-webview-provider WebView provider: go|rpc
-webview-network RPC network when provider=rpc: unix|tcp
-webview-address RPC address when provider=rpc
-webview-token RPC bearer token when provider=rpc
-verb Print download progress while running
-pretty Pretty-print JSON output
Examples:
extmock resolve ./ https://www.google.com
extmock download ./ https://example.com/video /tmp/extmock-download
extmock resolve ./ https://example.com/video -label env=test -setting token=abc
extmock eval ./test.js
extmock eval -e "new Blob(['hello']).size"
`)
}
func runResolve(args []string) error {
fs := flag.NewFlagSet("resolve", flag.ContinueOnError)
fs.SetOutput(os.Stderr)
var labels kvFlag
var settings kvFlag
var pretty bool
cfg := webViewConfig{}
fs.Var(&labels, "label", "Request label in key=value form (repeatable)")
fs.Var(&settings, "setting", "Extension setting override in key=value form (repeatable)")
fs.StringVar(&cfg.provider, "webview-provider", "", "WebView provider: go|rpc")
fs.StringVar(&cfg.network, "webview-network", "", "WebView RPC network: unix|tcp")
fs.StringVar(&cfg.address, "webview-address", "", "WebView RPC address")
fs.StringVar(&cfg.token, "webview-token", "", "WebView RPC bearer token")
fs.BoolVar(&pretty, "pretty", false, "Pretty-print JSON output")
flagArgs, positionalArgs, err := splitResolveArgs(args)
if err != nil {
return err
}
if err := fs.Parse(flagArgs); err != nil {
return err
}
if fs.NArg() != 0 {
return fmt.Errorf("unexpected trailing args: %v", fs.Args())
}
if len(positionalArgs) != 2 {
return fmt.Errorf("resolve expects exactly 2 args: <extension-dir> <url>")
}
extDir, err := filepath.Abs(positionalArgs[0])
if err != nil {
return err
}
url := positionalArgs[1]
d, cleanup, err := newExtMockDownloader(cfg)
if err != nil {
return err
}
defer cleanup()
ext, err := d.InstallExtensionByFolder(extDir, true)
if err != nil {
return err
}
if len(settings) > 0 {
if err := d.UpdateExtensionSettings(ext.Identity, toAnyMap(settings)); err != nil {
return err
}
}
result, err := d.Resolve(&base.Request{
URL: url,
Labels: map[string]string(labels),
}, nil)
if err != nil {
return err
}
return printJSON(result, pretty)
}
type downloadResult struct {
Resolve *download.ResolveResult `json:"resolve"`
OutputDir string `json:"outputDir"`
Files []*downloadedFileResult `json:"files"`
}
type downloadedFileResult struct {
Index int `json:"index"`
Name string `json:"name"`
Path string `json:"path"`
TaskID string `json:"taskId"`
Status base.Status `json:"status"`
Size int64 `json:"size"`
Req *base.Request `json:"req,omitempty"`
}
func runDownload(args []string) error {
fs := flag.NewFlagSet("download", flag.ContinueOnError)
fs.SetOutput(os.Stderr)
var labels kvFlag
var settings kvFlag
var pretty bool
var verbose bool
cfg := webViewConfig{}
fs.Var(&labels, "label", "Request label in key=value form (repeatable)")
fs.Var(&settings, "setting", "Extension setting override in key=value form (repeatable)")
fs.StringVar(&cfg.provider, "webview-provider", "", "WebView provider: go|rpc")
fs.StringVar(&cfg.network, "webview-network", "", "WebView RPC network: unix|tcp")
fs.StringVar(&cfg.address, "webview-address", "", "WebView RPC address")
fs.StringVar(&cfg.token, "webview-token", "", "WebView RPC bearer token")
fs.BoolVar(&verbose, "verb", false, "Print download progress while running")
fs.BoolVar(&verbose, "verbose", false, "Print download progress while running")
fs.BoolVar(&pretty, "pretty", false, "Pretty-print JSON output")
flagArgs, positionalArgs, err := splitDownloadArgs(args)
if err != nil {
return err
}
if err := fs.Parse(flagArgs); err != nil {
return err
}
if fs.NArg() != 0 {
return fmt.Errorf("unexpected trailing args: %v", fs.Args())
}
if len(positionalArgs) != 3 {
return fmt.Errorf("download expects exactly 3 args: <extension-dir> <url> <output-dir>")
}
extDir, err := filepath.Abs(positionalArgs[0])
if err != nil {
return err
}
url := positionalArgs[1]
outputDir, err := filepath.Abs(positionalArgs[2])
if err != nil {
return err
}
d, cleanup, err := newExtMockDownloader(cfg)
if err != nil {
return err
}
defer cleanup()
ext, err := d.InstallExtensionByFolder(extDir, true)
if err != nil {
return err
}
if len(settings) > 0 {
if err := d.UpdateExtensionSettings(ext.Identity, toAnyMap(settings)); err != nil {
return err
}
}
result, err := d.Resolve(&base.Request{
URL: url,
Labels: map[string]string(labels),
}, nil)
if err != nil {
return err
}
if result == nil || result.Res == nil {
return fmt.Errorf("resolve returned empty resource")
}
if len(result.Res.Files) == 0 {
return fmt.Errorf("resolve returned no files")
}
resourceOutputDir := outputDir
if result.Res.Name != "" {
resourceOutputDir = filepath.Join(outputDir, result.Res.Name)
}
if err := os.MkdirAll(resourceOutputDir, 0o755); err != nil {
return err
}
waiter := newTaskTerminalWaiter(d)
summary := &downloadResult{
Resolve: result,
OutputDir: resourceOutputDir,
Files: make([]*downloadedFileResult, 0, len(result.Res.Files)),
}
for idx, file := range result.Res.Files {
if file == nil {
return fmt.Errorf("file %d is nil", idx)
}
if file.Req == nil {
return fmt.Errorf("file %d request is nil", idx)
}
targetDir := resourceOutputDir
if file.Path != "" {
targetDir = filepath.Join(resourceOutputDir, file.Path)
}
if err := os.MkdirAll(targetDir, 0o755); err != nil {
return err
}
taskID, err := d.CreateDirect(file.Req, &base.Options{
Path: targetDir,
Name: file.Name,
})
if err != nil {
return fmt.Errorf("create task for %q failed: %w", file.Name, err)
}
waiter.Register(taskID)
summary.Files = append(summary.Files, &downloadedFileResult{
Index: idx,
Name: file.Name,
Path: filepath.Join(targetDir, file.Name),
TaskID: taskID,
Req: file.Req,
})
}
var stopVerbose func()
if verbose {
stopVerbose = startVerboseTaskListener(d, summary.Files)
defer stopVerbose()
}
for _, file := range summary.Files {
task, err := waiter.Wait(file.TaskID, 5*time.Minute)
if err != nil {
return fmt.Errorf("download %q failed: %w", file.Name, err)
}
file.Status = task.Status
if info, statErr := os.Stat(file.Path); statErr == nil {
file.Size = info.Size()
}
}
return printJSON(summary, pretty)
}
func runEval(args []string) error {
fs := flag.NewFlagSet("eval", flag.ContinueOnError)
fs.SetOutput(os.Stderr)
var (
script string
pretty bool
)
cfg := webViewConfig{}
fs.StringVar(&script, "e", "", "Inline JavaScript to execute")
fs.StringVar(&script, "eval", "", "Inline JavaScript to execute")
fs.StringVar(&cfg.provider, "webview-provider", "", "WebView provider: go|rpc")
fs.StringVar(&cfg.network, "webview-network", "", "WebView RPC network: unix|tcp")
fs.StringVar(&cfg.address, "webview-address", "", "WebView RPC address")
fs.StringVar(&cfg.token, "webview-token", "", "WebView RPC bearer token")
fs.BoolVar(&pretty, "pretty", false, "Pretty-print JSON output")
flagArgs, positionalArgs, err := splitEvalArgs(args)
if err != nil {
return err
}
if err := fs.Parse(flagArgs); err != nil {
return err
}
if fs.NArg() != 0 {
return fmt.Errorf("unexpected trailing args: %v", fs.Args())
}
if script == "" && len(positionalArgs) == 0 {
return fmt.Errorf("eval expects inline code via -e/--eval or a script file path")
}
if script != "" && len(positionalArgs) > 0 {
return fmt.Errorf("eval accepts either inline code or a script file path, not both")
}
if len(positionalArgs) > 1 {
return fmt.Errorf("eval expects at most 1 script file path")
}
d, cleanup, err := newExtMockDownloader(cfg)
if err != nil {
return err
}
defer cleanup()
runtime, err := d.NewExtensionEngine(newExtMockExtension(), map[string]any{})
if err != nil {
return err
}
defer runtime.Close()
var value any
if script != "" {
value, err = runtime.Eval(script)
} else {
scriptPath, pathErr := filepath.Abs(positionalArgs[0])
if pathErr != nil {
return pathErr
}
value, err = runtime.EvalFile(scriptPath)
}
if err != nil {
return err
}
return printJSON(value, pretty)
}
func splitResolveArgs(args []string) ([]string, []string, error) {
var flagArgs []string
var positionalArgs []string
for i := 0; i < len(args); i++ {
arg := args[i]
switch {
case arg == "-pretty":
flagArgs = append(flagArgs, arg)
case arg == "-webview-provider", arg == "-webview-network", arg == "-webview-address", arg == "-webview-token":
if i+1 >= len(args) {
return nil, nil, fmt.Errorf("flag %q requires a value", arg)
}
flagArgs = append(flagArgs, arg, args[i+1])
i++
case arg == "-label":
if i+1 >= len(args) {
return nil, nil, fmt.Errorf("flag %q requires a value", arg)
}
flagArgs = append(flagArgs, arg, args[i+1])
i++
case arg == "-setting":
if i+1 >= len(args) {
return nil, nil, fmt.Errorf("flag %q requires a value", arg)
}
flagArgs = append(flagArgs, arg, args[i+1])
i++
case len(arg) > len("-label=") && arg[:len("-label=")] == "-label=":
flagArgs = append(flagArgs, arg)
case len(arg) > len("-setting=") && arg[:len("-setting=")] == "-setting=":
flagArgs = append(flagArgs, arg)
case hasFlagValue(arg, "-webview-provider"), hasFlagValue(arg, "-webview-network"),
hasFlagValue(arg, "-webview-address"), hasFlagValue(arg, "-webview-token"):
flagArgs = append(flagArgs, arg)
case len(arg) > 0 && arg[0] == '-':
return nil, nil, fmt.Errorf("unknown flag %q", arg)
default:
positionalArgs = append(positionalArgs, arg)
}
}
return flagArgs, positionalArgs, nil
}
func splitDownloadArgs(args []string) ([]string, []string, error) {
var flagArgs []string
var positionalArgs []string
for i := 0; i < len(args); i++ {
arg := args[i]
switch {
case arg == "-pretty", arg == "-verb", arg == "-verbose":
flagArgs = append(flagArgs, arg)
case arg == "-webview-provider", arg == "-webview-network", arg == "-webview-address", arg == "-webview-token":
if i+1 >= len(args) {
return nil, nil, fmt.Errorf("flag %q requires a value", arg)
}
flagArgs = append(flagArgs, arg, args[i+1])
i++
case arg == "-label":
if i+1 >= len(args) {
return nil, nil, fmt.Errorf("flag %q requires a value", arg)
}
flagArgs = append(flagArgs, arg, args[i+1])
i++
case arg == "-setting":
if i+1 >= len(args) {
return nil, nil, fmt.Errorf("flag %q requires a value", arg)
}
flagArgs = append(flagArgs, arg, args[i+1])
i++
case len(arg) > len("-label=") && arg[:len("-label=")] == "-label=":
flagArgs = append(flagArgs, arg)
case len(arg) > len("-setting=") && arg[:len("-setting=")] == "-setting=":
flagArgs = append(flagArgs, arg)
case hasFlagValue(arg, "-webview-provider"), hasFlagValue(arg, "-webview-network"),
hasFlagValue(arg, "-webview-address"), hasFlagValue(arg, "-webview-token"):
flagArgs = append(flagArgs, arg)
case len(arg) > 0 && arg[0] == '-':
return nil, nil, fmt.Errorf("unknown flag %q", arg)
default:
positionalArgs = append(positionalArgs, arg)
}
}
return flagArgs, positionalArgs, nil
}
func splitEvalArgs(args []string) ([]string, []string, error) {
var flagArgs []string
var positionalArgs []string
for i := 0; i < len(args); i++ {
arg := args[i]
switch {
case arg == "-pretty":
flagArgs = append(flagArgs, arg)
case arg == "-webview-provider", arg == "-webview-network", arg == "-webview-address", arg == "-webview-token":
if i+1 >= len(args) {
return nil, nil, fmt.Errorf("flag %q requires a value", arg)
}
flagArgs = append(flagArgs, arg, args[i+1])
i++
case arg == "-e", arg == "--eval":
if i+1 >= len(args) {
return nil, nil, fmt.Errorf("flag %q requires a value", arg)
}
flagArgs = append(flagArgs, arg, args[i+1])
i++
case hasFlagValue(arg, "-e"), hasFlagValue(arg, "--eval"),
hasFlagValue(arg, "-webview-provider"), hasFlagValue(arg, "-webview-network"),
hasFlagValue(arg, "-webview-address"), hasFlagValue(arg, "-webview-token"):
flagArgs = append(flagArgs, arg)
case len(arg) > 0 && arg[0] == '-':
return nil, nil, fmt.Errorf("unknown flag %q", arg)
default:
positionalArgs = append(positionalArgs, arg)
}
}
return flagArgs, positionalArgs, nil
}
func hasFlagValue(arg string, prefix string) bool {
return len(arg) > len(prefix)+1 && arg[:len(prefix)+1] == prefix+"="
}
func toAnyMap(values kvFlag) map[string]any {
result := make(map[string]any, len(values))
for key, value := range values {
result[key] = value
}
return result
}
func newExtMockDownloader(cfg webViewConfig) (*download.Downloader, func(), error) {
webViewProvider, err := newExtMockWebViewProvider(cfg)
if err != nil {
return nil, nil, err
}
d := download.NewDownloader(&download.DownloaderConfig{
Storage: download.NewMemStorage(),
WebViewProvider: webViewProvider,
})
if err := d.Setup(); err != nil {
return nil, nil, err
}
cleanup := func() {
_ = d.Clear()
}
return d, cleanup, nil
}
func newExtMockExtension() *download.Extension {
return &download.Extension{
Name: "extmock",
Author: "gopeed",
Title: "Gopeed ExtMock Runtime",
Version: "0.0.0",
DevMode: true,
}
}
func newExtMockWebViewProvider(cfg webViewConfig) (enginewebview.Provider, error) {
switch cfg.provider {
case "", "go":
return goprovider.New(), nil
case "rpc":
rpcCfg, err := resolveRPCConfig(cfg)
if err != nil {
return nil, err
}
return rpcprovider.New(rpcCfg), nil
default:
return nil, fmt.Errorf("unsupported webview provider %q", cfg.provider)
}
}
func resolveRPCConfig(cfg webViewConfig) (enginewebview.RPCConfig, error) {
network := cfg.network
address := cfg.address
if network == "" && address == "" {
defaultAddress, err := defaultWebViewRPCUnixSocket()
if err != nil {
return enginewebview.RPCConfig{}, err
}
network = "unix"
address = defaultAddress
}
if network == "" || address == "" {
return enginewebview.RPCConfig{}, fmt.Errorf("webview rpc requires both network and address")
}
return enginewebview.RPCConfig{
Network: network,
Address: address,
Token: cfg.token,
}, nil
}
func defaultWebViewRPCUnixSocket() (string, error) {
homeDir, err := os.UserHomeDir()
if err != nil {
return "", err
}
return filepath.Join(homeDir, "gopeed_webview.sock"), nil
}
type taskTerminalWaiter struct {
downloader *download.Downloader
}
func newTaskTerminalWaiter(d *download.Downloader) *taskTerminalWaiter {
return &taskTerminalWaiter{downloader: d}
}
func (w *taskTerminalWaiter) Register(id string) {
}
func (w *taskTerminalWaiter) Wait(id string, timeout time.Duration) (*download.Task, error) {
deadline := time.Now().Add(timeout)
for time.Now().Before(deadline) {
if task, ok := w.currentTerminalTask(id); ok {
return task, w.taskError(task)
}
time.Sleep(50 * time.Millisecond)
}
return nil, fmt.Errorf("timeout waiting for task %s", id)
}
func (w *taskTerminalWaiter) currentTerminalTask(id string) (*download.Task, bool) {
task := w.downloader.GetTask(id)
if task == nil {
return nil, false
}
if task.Status != base.DownloadStatusDone && task.Status != base.DownloadStatusError {
return nil, false
}
return task, true
}
func (w *taskTerminalWaiter) taskError(task *download.Task) error {
if task == nil {
return fmt.Errorf("task is nil")
}
if task.Status == base.DownloadStatusError {
return fmt.Errorf("task %s finished with status %s", task.ID, task.Status)
}
return nil
}
func startVerboseTaskListener(d *download.Downloader, files []*downloadedFileResult) func() {
tracked := make(map[string]string, len(files))
for _, file := range files {
tracked[file.TaskID] = file.Name
}
d.Listener(func(event *download.Event) {
if event == nil || event.Task == nil {
return
}
name, ok := tracked[event.Task.ID]
if !ok {
return
}
switch event.Key {
case download.EventKeyStart, download.EventKeyProgress, download.EventKeyDone, download.EventKeyError:
printVerboseTaskEvent(event, name)
}
})
return func() {}
}
func printVerboseTaskEvent(event *download.Event, name string) {
task := event.Task
ts := time.Now().Format("15:04:05.000")
if task == nil {
return
}
total := int64(0)
if task.Meta != nil && task.Meta.Res != nil {
total = task.Meta.Res.Size
}
downloaded := int64(0)
speed := int64(0)
if task.Progress != nil {
downloaded = task.Progress.Downloaded
speed = task.Progress.Speed
}
if event.Key == download.EventKeyError && event.Err != nil {
fmt.Fprintf(
os.Stderr,
"[%s] event=%s task=%s status=%s downloaded=%d total=%d speed=%d name=%q err=%v\n",
ts,
event.Key,
task.ID,
task.Status,
downloaded,
total,
speed,
name,
event.Err,
)
return
}
fmt.Fprintf(
os.Stderr,
"[%s] event=%s task=%s status=%s downloaded=%d total=%d speed=%d name=%q\n",
ts,
event.Key,
task.ID,
task.Status,
downloaded,
total,
speed,
name,
)
}
func printJSON(v any, pretty bool) error {
var (
data []byte
err error
)
if pretty {
data, err = json.MarshalIndent(v, "", " ")
} else {
data, err = json.Marshal(v)
}
if err != nil {
return err
}
fmt.Println(string(data))
return nil
}
+3 -3
View File
@@ -1,10 +1,9 @@
module github.com/GopeedLab/gopeed
go 1.24.9
toolchain go1.24.11
go 1.25.4
require (
github.com/GopeedLab/webview_go v0.0.0-20260423085439-7a2f88b6e9b5
github.com/anacrolix/torrent v1.60.1-0.20251217073903-486bcbe758e0
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5
github.com/bodgit/sevenzip v1.6.1
@@ -17,6 +16,7 @@ require (
github.com/matoous/go-nanoid/v2 v2.0.0
github.com/mattn/go-ieproxy v0.0.12
github.com/mholt/archives v0.1.5
github.com/monkeyWie/goed2k v0.0.0-20260317100435-7a7575cf2447
github.com/pkg/errors v0.9.1
github.com/rs/dnscache v0.0.0-20230804202142-fc85eb664529
github.com/rs/zerolog v1.31.0
+4 -8
View File
@@ -23,6 +23,8 @@ filippo.io/edwards25519 v1.0.0-rc.1 h1:m0VOOB23frXZvAOK44usCgLWvtsxIoMCTBGJZlpmG
filippo.io/edwards25519 v1.0.0-rc.1/go.mod h1:N1IkdkCkiLB6tki+MYJoSx2JTY9NUlxZE7eHn5EwJns=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
github.com/GopeedLab/webview_go v0.0.0-20260423085439-7a2f88b6e9b5 h1:86LVTaNDM/j/uTAD2HedkXNO9LIQlt642VAdbU/0D/M=
github.com/GopeedLab/webview_go v0.0.0-20260423085439-7a2f88b6e9b5/go.mod h1:FvV19lu9rQ9NcbczN5y8pbWecURRRbWa8tbfFARBDa0=
github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0rYXWg0=
github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ=
github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY=
@@ -164,13 +166,9 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dlclark/regexp2 v1.11.4 h1:rPYF9/LECdNymJufQKmri9gV604RvvABwgOA8un7yAo=
github.com/dlclark/regexp2 v1.11.4/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
github.com/dlclark/regexp2 v1.11.5 h1:Q/sSnsKerHeCkc/jSTNq1oCm7KiVgUMZRDUoRu0JQZQ=
github.com/dlclark/regexp2 v1.11.5/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
github.com/dop251/goja v0.0.0-20240919115326-6c7d1df7ff05 h1:oK4+QcKsczZjHYTHD0JAdkvq5w74JEkG95J0XNBx/BI=
github.com/dop251/goja v0.0.0-20240919115326-6c7d1df7ff05/go.mod h1:MxLav0peU43GgvwVgNbLAj1s/bSGboKkhuULvq/7hx4=
github.com/dop251/goja v0.0.0-20260106131823-651366fbe6e3 h1:bVp3yUzvSAJzu9GqID+Z96P+eu5TKnIMJSV4QaZMauM=
github.com/dop251/goja v0.0.0-20260106131823-651366fbe6e3/go.mod h1:MxLav0peU43GgvwVgNbLAj1s/bSGboKkhuULvq/7hx4=
github.com/dop251/goja_nodejs v0.0.0-20240728170619-29b559befffc h1:MKYt39yZJi0Z9xEeRmDX2L4ocE0ETKcHKw6MVL3R+co=
@@ -286,8 +284,6 @@ github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OI
github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
github.com/google/pprof v0.0.0-20240227163752-401108e1b7e7/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik=
github.com/google/pprof v0.0.0-20260106004452-d7df1bf2cac7 h1:kmPAX+IJBcUAFTddx2+xC0H7sk2U9ijIIxZLLrPLNng=
github.com/google/pprof v0.0.0-20260106004452-d7df1bf2cac7/go.mod h1:67FPmZWbr+KDT/VlpWtw6sO9XSjpJmLuHpoLmWiTGgY=
github.com/google/pprof v0.0.0-20260115054156-294ebfa9ad83 h1:z2ogiKUYzX5Is6zr/vP9vJGqPwcdqsWjOt+V8J7+bTc=
github.com/google/pprof v0.0.0-20260115054156-294ebfa9ad83/go.mod h1:MxpfABSjhmINe3F1It9d+8exIHFvUqtLIRCdOGNXqiI=
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
@@ -392,6 +388,8 @@ github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJ
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/monkeyWie/goed2k v0.0.0-20260317100435-7a7575cf2447 h1:88kRsNwkDKPA4NzRUWdOoIL7SyMNDwtmAxzShKBvKTI=
github.com/monkeyWie/goed2k v0.0.0-20260317100435-7a7575cf2447/go.mod h1:Ry2y1QlzerUgA1hVmExBdXXzE4Sjk1M7w0nSh6dhDOg=
github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o=
github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc=
github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg=
@@ -741,8 +739,6 @@ golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU=
golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY=
golang.org/x/text v0.33.0 h1:B3njUFyqtHDUI5jMn1YIr5B0IE2U0qck04r6d4KPAxE=
golang.org/x/text v0.33.0/go.mod h1:LuMebE6+rBincTi9+xWTY8TztLzKHc/9C1uBCG27+q8=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+677
View File
@@ -0,0 +1,677 @@
package blob
import (
"bytes"
"context"
"crypto/rand"
"encoding/hex"
"errors"
"fmt"
"io"
"net"
"net/http"
"net/url"
"path"
"strconv"
"strings"
"sync"
"time"
)
const urlPathPrefix = "/__blob/"
const rangeSourceFailureLimit = 2
// unclaimedSourceTTL bounds how long a session-backed source may keep its
// engine alive without ever being claimed by a download task.
var unclaimedSourceTTL = 10 * time.Minute
var (
ErrInvalidURL = errors.New("invalid blob url")
ErrInvalidOptions = errors.New("invalid blob options")
ErrSourceNotFound = errors.New("blob source not found")
ErrSourceRevoked = errors.New("blob source revoked")
ErrSourceClosed = errors.New("blob source closed")
ErrRangeNotAllowed = errors.New("blob range not allowed")
)
type SessionRef interface {
Retain()
Release()
}
type OpenRequest struct {
Offset int64
End int64
}
type OpenFunc func(ctx context.Context, req OpenRequest) (io.ReadCloser, error)
type CreateOptions struct {
ContentType string
Size int64
Range bool
Session SessionRef
}
type Metadata struct {
ContentType string
Size int64
Range bool
}
type Source struct {
ID string
URL string
ContentType string
mu sync.Mutex
size int64
rangeEnabled bool
revoked bool
taskRefs int
readErr error
rangeFailures int
session SessionRef
open OpenFunc
unclaimed *time.Timer
}
type Registry struct {
dir string
mu sync.RWMutex
sources map[string]*Source
serverMu sync.Mutex
listener net.Listener
server *http.Server
baseURL string
}
func NewRegistry(dir string) *Registry {
return &Registry{
dir: dir,
sources: make(map[string]*Source),
}
}
func (r *Registry) Dir() string {
if r == nil {
return ""
}
return r.dir
}
func (r *Registry) IsURL(raw string) bool {
if r == nil {
return false
}
src, err := r.get(raw)
return err == nil && src != nil
}
func (r *Registry) CreateBlob(data []byte, contentType string) (string, error) {
buf := append([]byte(nil), data...)
return r.CreateOpener(func(ctx context.Context, req OpenRequest) (io.ReadCloser, error) {
if req.Offset < 0 || req.Offset > int64(len(buf)) {
return nil, ErrSourceNotFound
}
end := int64(len(buf))
if req.End >= req.Offset && req.End+1 < end {
end = req.End + 1
}
return io.NopCloser(bytes.NewReader(buf[req.Offset:end])), nil
}, &CreateOptions{
ContentType: contentType,
Size: int64(len(buf)),
Range: len(buf) > 0,
})
}
func (r *Registry) CreateOpener(open OpenFunc, opts *CreateOptions) (string, error) {
if r == nil {
return "", ErrSourceNotFound
}
if open == nil {
return "", ErrInvalidOptions
}
if opts == nil {
opts = &CreateOptions{}
}
if opts.Size < 0 {
opts.Size = 0
}
if opts.Range && opts.Size <= 0 {
return "", fmt.Errorf("%w: range requires positive size", ErrInvalidOptions)
}
// Keep server generation selection and source insertion atomic with Close.
r.serverMu.Lock()
baseURL, err := r.ensureServerLocked()
if err != nil {
r.serverMu.Unlock()
return "", err
}
id, err := randomID(18)
if err != nil {
r.serverMu.Unlock()
return "", err
}
if opts.Session != nil {
opts.Session.Retain()
}
srcURL := fmt.Sprintf("%s%s", baseURL, id)
src := &Source{
ID: id,
URL: srcURL,
ContentType: opts.ContentType,
size: opts.Size,
rangeEnabled: opts.Range,
session: opts.Session,
open: open,
}
r.mu.Lock()
r.sources[id] = src
r.mu.Unlock()
r.serverMu.Unlock()
if opts.Session != nil && unclaimedSourceTTL > 0 {
src.mu.Lock()
if !src.revoked {
src.unclaimed = time.AfterFunc(unclaimedSourceTTL, func() {
r.expireUnclaimed(src)
})
}
src.mu.Unlock()
}
return src.URL, nil
}
func (r *Registry) Metadata(raw string) (Metadata, error) {
src, err := r.get(raw)
if err != nil {
return Metadata{}, err
}
src.mu.Lock()
defer src.mu.Unlock()
return Metadata{
ContentType: src.ContentType,
Size: src.size,
Range: src.rangeEnabled,
}, nil
}
// Acquire retains a source for a task. A source remains available until every
// task that acquired it calls Release, unless it is explicitly revoked.
func (r *Registry) Acquire(raw string) error {
src, err := r.get(raw)
if err != nil {
return err
}
return src.acquireTask()
}
// Release releases a task's reference to a source. Releasing the final task
// reference revokes the source. Sources that have never been acquired remain
// registered until Revoke or Registry.Close is called; session-backed sources
// are also bounded by the unclaimed-source TTL.
func (r *Registry) Release(raw string) error {
src, err := r.get(raw)
if err != nil {
return err
}
session, last, err := src.releaseTask()
if err != nil {
return err
}
if !last {
return nil
}
r.deleteSource(src)
if session != nil {
session.Release()
}
return nil
}
// SourceError returns the first error encountered while opening or reading a
// source. Invalid and missing sources have no source error. HTTP response write
// errors are deliberately not recorded here.
func (r *Registry) SourceError(raw string) error {
src, err := r.get(raw)
if err != nil {
return nil
}
return src.sourceError()
}
func (r *Registry) Revoke(raw string) error {
src, err := r.get(raw)
if err != nil {
return err
}
r.removeSource(src)
return nil
}
func (r *Registry) Close() error {
if r == nil {
return nil
}
r.serverMu.Lock()
server := r.server
listener := r.listener
r.server = nil
r.listener = nil
r.baseURL = ""
r.mu.Lock()
sources := make([]*Source, 0, len(r.sources))
for _, src := range r.sources {
sources = append(sources, src)
}
r.sources = make(map[string]*Source)
r.mu.Unlock()
r.serverMu.Unlock()
if server != nil {
_ = server.Close()
}
if listener != nil {
_ = listener.Close()
}
for _, src := range sources {
src.close()
}
return nil
}
// ensureServerLocked returns the current server URL or starts a new server.
// The caller must hold serverMu.
func (r *Registry) ensureServerLocked() (string, error) {
if r.baseURL != "" {
return r.baseURL, nil
}
listener, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
return "", err
}
r.listener = listener
r.baseURL = "http://" + listener.Addr().String() + urlPathPrefix
server := &http.Server{Handler: r}
r.server = server
go func() {
if err := server.Serve(listener); err != nil && !errors.Is(err, http.ErrServerClosed) {
_ = r.Close()
}
}()
return r.baseURL, nil
}
func (r *Registry) ServeHTTP(w http.ResponseWriter, req *http.Request) {
if req.Method != http.MethodGet {
w.Header().Set("Allow", "GET")
http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
return
}
id := parseRequest(req)
if id == "" {
http.NotFound(w, req)
return
}
src, err := r.getByID(id)
if err != nil {
http.NotFound(w, req)
return
}
meta, open, session := src.acquireOpen()
if open == nil {
http.NotFound(w, req)
return
}
if session != nil {
defer session.Release()
}
if src.sourceError() != nil {
http.Error(w, "blob source unavailable", http.StatusGone)
return
}
start, end, ranged, err := parseRange(req.Header.Get("Range"), meta.Size, meta.Range)
if err != nil {
http.Error(w, err.Error(), http.StatusRequestedRangeNotSatisfiable)
return
}
if !meta.Range {
start, end, ranged = 0, -1, false
}
reader, err := open(req.Context(), OpenRequest{
Offset: start,
End: end,
})
if err != nil {
if req.Context().Err() == nil || !errors.Is(err, req.Context().Err()) {
src.recordSourceError(err, meta.Range)
}
http.Error(w, "blob source unavailable", http.StatusGone)
return
}
if reader == nil {
err = fmt.Errorf("%w: opener returned a nil reader", ErrSourceClosed)
src.recordSourceError(err, meta.Range)
http.Error(w, "blob source unavailable", http.StatusGone)
return
}
defer reader.Close()
writeHeaders(w, meta, start, end, ranged)
if ranged {
w.WriteHeader(http.StatusPartialContent)
}
limit := int64(-1)
if end >= start {
limit = end - start + 1
} else if meta.Size > 0 {
limit = meta.Size
}
_, readErr, writeErr := copyWithFlush(w, reader, limit)
if readErr != nil {
if ctxErr := req.Context().Err(); ctxErr == nil || !errors.Is(readErr, ctxErr) {
src.recordSourceError(readErr, meta.Range)
}
} else if writeErr == nil && req.Context().Err() == nil {
src.recordSourceSuccess(meta.Range)
}
}
func (s *Source) acquireOpen() (Metadata, OpenFunc, SessionRef) {
s.mu.Lock()
defer s.mu.Unlock()
if s.revoked {
return Metadata{}, nil, nil
}
if s.session != nil {
// The source's own retain guarantees that the session cannot become
// closed while this additional active-reader retain is acquired.
s.session.Retain()
}
return Metadata{
ContentType: s.ContentType,
Size: s.size,
Range: s.rangeEnabled,
}, s.open, s.session
}
func (s *Source) acquireTask() error {
s.mu.Lock()
if s.revoked {
s.mu.Unlock()
return ErrSourceRevoked
}
s.taskRefs++
timer := s.unclaimed
s.unclaimed = nil
s.mu.Unlock()
if timer != nil {
timer.Stop()
}
return nil
}
func (s *Source) releaseTask() (session SessionRef, last bool, err error) {
s.mu.Lock()
defer s.mu.Unlock()
if s.revoked {
return nil, false, ErrSourceRevoked
}
if s.taskRefs == 0 {
return nil, false, ErrSourceClosed
}
s.taskRefs--
if s.taskRefs > 0 {
return nil, false, nil
}
s.revoked = true
s.open = nil
session = s.session
s.session = nil
return session, true, nil
}
func (s *Source) sourceError() error {
s.mu.Lock()
defer s.mu.Unlock()
return s.readErr
}
func (s *Source) recordSourceError(err error, ranged bool) {
if err == nil {
return
}
s.mu.Lock()
if s.readErr == nil && ranged {
s.rangeFailures++
if s.rangeFailures < rangeSourceFailureLimit {
s.mu.Unlock()
return
}
}
if s.readErr == nil {
s.readErr = err
}
s.mu.Unlock()
}
func (s *Source) recordSourceSuccess(ranged bool) {
if !ranged {
return
}
s.mu.Lock()
if s.readErr == nil {
s.rangeFailures = 0
}
s.mu.Unlock()
}
func (s *Source) close() {
s.mu.Lock()
alreadyRevoked := s.revoked
s.revoked = true
s.open = nil
session := s.session
s.session = nil
timer := s.unclaimed
s.unclaimed = nil
s.mu.Unlock()
if timer != nil {
timer.Stop()
}
if !alreadyRevoked && session != nil {
session.Release()
}
}
func (s *Source) expireUnclaimed() (SessionRef, bool) {
s.mu.Lock()
defer s.mu.Unlock()
s.unclaimed = nil
if s.revoked || s.taskRefs > 0 {
return nil, false
}
s.revoked = true
s.open = nil
session := s.session
s.session = nil
return session, true
}
func (r *Registry) expireUnclaimed(src *Source) {
session, expired := src.expireUnclaimed()
if !expired {
return
}
r.deleteSource(src)
if session != nil {
session.Release()
}
}
func (r *Registry) removeSource(src *Source) {
src.close()
r.deleteSource(src)
}
func (r *Registry) deleteSource(src *Source) {
r.mu.Lock()
if current := r.sources[src.ID]; current == src {
delete(r.sources, src.ID)
}
r.mu.Unlock()
}
func writeHeaders(w http.ResponseWriter, meta Metadata, start, end int64, ranged bool) {
if meta.ContentType != "" {
w.Header().Set("Content-Type", meta.ContentType)
}
if meta.Range {
w.Header().Set("Accept-Ranges", "bytes")
}
if meta.Size > 0 {
if ranged {
w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, meta.Size))
w.Header().Set("Content-Length", strconv.FormatInt(end-start+1, 10))
return
}
w.Header().Set("Content-Length", strconv.FormatInt(meta.Size, 10))
}
}
func copyWithFlush(w http.ResponseWriter, reader io.Reader, limit int64) (written int64, readErr error, writeErr error) {
if limit >= 0 {
reader = io.LimitReader(reader, limit)
}
buf := make([]byte, 32*1024)
flusher, canFlush := w.(http.Flusher)
for {
nr, er := reader.Read(buf)
if er != nil && !errors.Is(er, io.EOF) {
readErr = er
}
if nr > 0 {
nw, ew := w.Write(buf[:nr])
if nw > 0 {
written += int64(nw)
if canFlush {
flusher.Flush()
}
}
if ew != nil {
return written, readErr, ew
}
if nr != nw {
return written, readErr, io.ErrShortWrite
}
}
if er != nil {
if errors.Is(er, io.EOF) {
if limit >= 0 && written < limit {
return written, io.ErrUnexpectedEOF, nil
}
return written, nil, nil
}
return written, readErr, nil
}
}
}
func parseRange(header string, size int64, rangeEnabled bool) (start int64, end int64, ranged bool, err error) {
if header == "" || !rangeEnabled {
return 0, -1, false, nil
}
if size <= 0 {
return 0, 0, false, ErrRangeNotAllowed
}
if !strings.HasPrefix(header, "bytes=") {
return 0, 0, false, fmt.Errorf("unsupported range")
}
parts := strings.SplitN(strings.TrimPrefix(header, "bytes="), "-", 2)
if len(parts) != 2 || parts[0] == "" {
return 0, 0, false, fmt.Errorf("unsupported range")
}
start, err = strconv.ParseInt(parts[0], 10, 64)
if err != nil || start < 0 {
return 0, 0, false, fmt.Errorf("invalid range")
}
if start >= size {
return 0, 0, false, fmt.Errorf("range out of bounds")
}
end = size - 1
if parts[1] != "" {
end, err = strconv.ParseInt(parts[1], 10, 64)
if err != nil || end < start {
return 0, 0, false, fmt.Errorf("invalid range")
}
if end >= size {
end = size - 1
}
}
return start, end, true, nil
}
func (r *Registry) get(raw string) (*Source, error) {
id, ok := r.parseURL(raw)
if !ok {
return nil, ErrInvalidURL
}
src, err := r.getByID(id)
if err != nil {
return nil, err
}
src.mu.Lock()
revoked := src.revoked
src.mu.Unlock()
if revoked {
return nil, ErrSourceRevoked
}
return src, nil
}
func (r *Registry) getByID(id string) (*Source, error) {
r.mu.RLock()
defer r.mu.RUnlock()
src := r.sources[id]
if src == nil {
return nil, ErrSourceNotFound
}
return src, nil
}
func (r *Registry) parseURL(raw string) (id string, ok bool) {
u, err := url.Parse(raw)
if err != nil || u.Scheme != "http" || !strings.HasPrefix(u.Path, urlPathPrefix) {
return "", false
}
r.serverMu.Lock()
baseURL := r.baseURL
r.serverMu.Unlock()
if baseURL == "" || !strings.HasPrefix(raw, baseURL) {
return "", false
}
id = strings.TrimPrefix(u.Path, urlPathPrefix)
if id == "" || path.Base(id) != id {
return "", false
}
return id, true
}
func parseRequest(req *http.Request) string {
id := strings.TrimPrefix(req.URL.Path, urlPathPrefix)
if id == "" || id == req.URL.Path || path.Base(id) != id {
return ""
}
return id
}
func randomID(size int) (string, error) {
buf := make([]byte, size)
if _, err := rand.Read(buf); err != nil {
return "", err
}
return hex.EncodeToString(buf), nil
}
+898
View File
@@ -0,0 +1,898 @@
package blob
import (
"bytes"
"context"
"errors"
"io"
"net/http"
"strings"
"sync"
"testing"
"time"
)
type terminalErrorReader struct {
err error
}
func (r *terminalErrorReader) Read([]byte) (int, error) {
return 0, r.err
}
type contextErrorReader struct {
ctx context.Context
}
func (r *contextErrorReader) Read([]byte) (int, error) {
<-r.ctx.Done()
return 0, r.ctx.Err()
}
func (r *contextErrorReader) Close() error { return nil }
type failingResponseWriter struct {
header http.Header
err error
}
func (w *failingResponseWriter) Header() http.Header {
return w.header
}
func (w *failingResponseWriter) Write([]byte) (int, error) {
return 0, w.err
}
func (w *failingResponseWriter) WriteHeader(int) {}
type countingSession struct {
mu sync.Mutex
refs int
zero chan struct{}
zeroOnce sync.Once
}
func newCountingSession() *countingSession {
return &countingSession{zero: make(chan struct{})}
}
func (s *countingSession) Retain() {
s.mu.Lock()
s.refs++
s.mu.Unlock()
}
func (s *countingSession) Release() {
s.mu.Lock()
s.refs--
refs := s.refs
s.mu.Unlock()
if refs == 0 {
s.zeroOnce.Do(func() { close(s.zero) })
}
}
func (s *countingSession) RefCount() int {
s.mu.Lock()
defer s.mu.Unlock()
return s.refs
}
func setUnclaimedSourceTTLForTest(t *testing.T, ttl time.Duration) {
t.Helper()
previous := unclaimedSourceTTL
unclaimedSourceTTL = ttl
t.Cleanup(func() {
unclaimedSourceTTL = previous
})
}
func TestRegistryBlobHTTPGetAndRange(t *testing.T) {
registry := NewRegistry(t.TempDir())
defer registry.Close()
url, err := registry.CreateBlob([]byte("hello world"), "text/plain")
if err != nil {
t.Fatal(err)
}
if !registry.IsURL(url) {
t.Fatal("expected registry to recognize created url")
}
resp, err := http.Get(url)
if err != nil {
t.Fatal(err)
}
body, err := io.ReadAll(resp.Body)
_ = resp.Body.Close()
if err != nil {
t.Fatal(err)
}
if resp.StatusCode != http.StatusOK {
t.Fatalf("unexpected GET status: %d", resp.StatusCode)
}
if got := resp.Header.Get("Content-Length"); got != "11" {
t.Fatalf("unexpected content length: %q", got)
}
if got := resp.Header.Get("Accept-Ranges"); got != "bytes" {
t.Fatalf("unexpected accept ranges: %q", got)
}
if got := resp.Header.Get("Content-Type"); !strings.HasPrefix(got, "text/plain") {
t.Fatalf("unexpected content type: %q", got)
}
if string(body) != "hello world" {
t.Fatalf("unexpected GET body: %q", string(body))
}
req, err := http.NewRequest(http.MethodGet, url, nil)
if err != nil {
t.Fatal(err)
}
req.Header.Set("Range", "bytes=6-10")
resp, err = http.DefaultClient.Do(req)
if err != nil {
t.Fatal(err)
}
body, err = io.ReadAll(resp.Body)
_ = resp.Body.Close()
if err != nil {
t.Fatal(err)
}
if resp.StatusCode != http.StatusPartialContent {
t.Fatalf("unexpected range status: %d", resp.StatusCode)
}
if got := resp.Header.Get("Content-Range"); got != "bytes 6-10/11" {
t.Fatalf("unexpected content range: %q", got)
}
if string(body) != "world" {
t.Fatalf("unexpected range body: %q", string(body))
}
}
func TestRegistryEmptyBlob(t *testing.T) {
registry := NewRegistry(t.TempDir())
defer registry.Close()
url, err := registry.CreateBlob(nil, "application/octet-stream")
if err != nil {
t.Fatal(err)
}
meta, err := registry.Metadata(url)
if err != nil {
t.Fatal(err)
}
if meta.Size != 0 || meta.Range {
t.Fatalf("unexpected empty blob metadata: %#v", meta)
}
resp, err := http.Get(url)
if err != nil {
t.Fatal(err)
}
body, readErr := io.ReadAll(resp.Body)
_ = resp.Body.Close()
if readErr != nil {
t.Fatal(readErr)
}
if resp.StatusCode != http.StatusOK {
t.Fatalf("unexpected status: %d", resp.StatusCode)
}
if len(body) != 0 {
t.Fatalf("unexpected body: %q", body)
}
if got := resp.Header.Get("Content-Length"); got != "0" {
t.Fatalf("unexpected content length: %q", got)
}
}
func TestRegistryOpenerSizeWithoutRange(t *testing.T) {
registry := NewRegistry(t.TempDir())
defer registry.Close()
payload := []byte("hello opener")
url, err := registry.CreateOpener(func(ctx context.Context, req OpenRequest) (io.ReadCloser, error) {
if req.Offset != 0 || req.End != -1 {
t.Fatalf("unexpected non-range request: %#v", req)
}
return io.NopCloser(bytes.NewReader(payload)), nil
}, &CreateOptions{
ContentType: "text/plain",
Size: int64(len(payload)),
Range: false,
})
if err != nil {
t.Fatal(err)
}
req, err := http.NewRequest(http.MethodGet, url, nil)
if err != nil {
t.Fatal(err)
}
req.Header.Set("Range", "bytes=6-11")
resp, err := http.DefaultClient.Do(req)
if err != nil {
t.Fatal(err)
}
body, err := io.ReadAll(resp.Body)
_ = resp.Body.Close()
if err != nil {
t.Fatal(err)
}
if resp.StatusCode != http.StatusOK {
t.Fatalf("unexpected status: %d", resp.StatusCode)
}
if got := resp.Header.Get("Content-Length"); got != "12" {
t.Fatalf("unexpected content length: %q", got)
}
if got := resp.Header.Get("Accept-Ranges"); got != "" {
t.Fatalf("unexpected accept ranges: %q", got)
}
if string(body) != string(payload) {
t.Fatalf("unexpected body: %q", string(body))
}
}
func TestRegistryOpenerRangeConcurrentRequests(t *testing.T) {
registry := NewRegistry(t.TempDir())
defer registry.Close()
payload := []byte("abcdefghijklmnopqrstuvwxyz")
var mu sync.Mutex
var calls []OpenRequest
url, err := registry.CreateOpener(func(ctx context.Context, req OpenRequest) (io.ReadCloser, error) {
mu.Lock()
calls = append(calls, req)
mu.Unlock()
start := int(req.Offset)
end := len(payload)
if req.End >= 0 {
end = int(req.End) + 1
}
if start < 0 || start > end || end > len(payload) {
return nil, errors.New("bad test range")
}
return io.NopCloser(bytes.NewReader(payload[start:end])), nil
}, &CreateOptions{
Size: int64(len(payload)),
Range: true,
})
if err != nil {
t.Fatal(err)
}
ranges := []struct {
header string
want string
}{
{header: "bytes=0-4", want: "abcde"},
{header: "bytes=10-15", want: "klmnop"},
{header: "bytes=20-25", want: "uvwxyz"},
}
var wg sync.WaitGroup
for _, item := range ranges {
item := item
wg.Add(1)
go func() {
defer wg.Done()
req, err := http.NewRequest(http.MethodGet, url, nil)
if err != nil {
t.Error(err)
return
}
req.Header.Set("Range", item.header)
resp, err := http.DefaultClient.Do(req)
if err != nil {
t.Error(err)
return
}
body, err := io.ReadAll(resp.Body)
_ = resp.Body.Close()
if err != nil {
t.Error(err)
return
}
if resp.StatusCode != http.StatusPartialContent {
t.Errorf("unexpected status for %s: %d", item.header, resp.StatusCode)
}
if string(body) != item.want {
t.Errorf("unexpected body for %s: %q", item.header, string(body))
}
}()
}
wg.Wait()
mu.Lock()
defer mu.Unlock()
if len(calls) != len(ranges) {
t.Fatalf("unexpected opener call count: %d", len(calls))
}
seen := map[OpenRequest]bool{}
for _, call := range calls {
seen[call] = true
}
for _, want := range []OpenRequest{{Offset: 0, End: 4}, {Offset: 10, End: 15}, {Offset: 20, End: 25}} {
if !seen[want] {
t.Fatalf("missing opener request %#v, got %#v", want, calls)
}
}
}
func TestRegistryRejectsRangeWithoutSize(t *testing.T) {
registry := NewRegistry(t.TempDir())
defer registry.Close()
_, err := registry.CreateOpener(func(ctx context.Context, req OpenRequest) (io.ReadCloser, error) {
return io.NopCloser(strings.NewReader("")), nil
}, &CreateOptions{Range: true})
if !errors.Is(err, ErrInvalidOptions) {
t.Fatalf("expected invalid options error, got %v", err)
}
}
func TestRegistryTracksSourceErrors(t *testing.T) {
t.Run("reader error", func(t *testing.T) {
registry := NewRegistry(t.TempDir())
defer registry.Close()
boom := errors.New("reader failed")
url, err := registry.CreateOpener(func(context.Context, OpenRequest) (io.ReadCloser, error) {
return io.NopCloser(io.MultiReader(strings.NewReader("partial"), &terminalErrorReader{err: boom})), nil
}, &CreateOptions{})
if err != nil {
t.Fatal(err)
}
resp, err := http.Get(url)
if err != nil {
t.Fatal(err)
}
body, readErr := io.ReadAll(resp.Body)
_ = resp.Body.Close()
if readErr != nil {
t.Fatalf("the HTTP transport should end normally, got %v", readErr)
}
if string(body) != "partial" {
t.Fatalf("unexpected partial body: %q", body)
}
if got := registry.SourceError(url); !errors.Is(got, boom) {
t.Fatalf("expected source error %v, got %v", boom, got)
}
})
t.Run("short known-size reader", func(t *testing.T) {
registry := NewRegistry(t.TempDir())
defer registry.Close()
url, err := registry.CreateOpener(func(context.Context, OpenRequest) (io.ReadCloser, error) {
return io.NopCloser(strings.NewReader("short")), nil
}, &CreateOptions{Size: 10})
if err != nil {
t.Fatal(err)
}
resp, err := http.Get(url)
if err != nil {
t.Fatal(err)
}
_, _ = io.ReadAll(resp.Body)
_ = resp.Body.Close()
if got := registry.SourceError(url); !errors.Is(got, io.ErrUnexpectedEOF) {
t.Fatalf("expected unexpected EOF, got %v", got)
}
})
t.Run("range reader error becomes terminal", func(t *testing.T) {
registry := NewRegistry(t.TempDir())
defer registry.Close()
boom := errors.New("range reader failed")
url, err := registry.CreateOpener(func(context.Context, OpenRequest) (io.ReadCloser, error) {
return io.NopCloser(io.MultiReader(strings.NewReader("partial"), &terminalErrorReader{err: boom})), nil
}, &CreateOptions{Size: 10, Range: true})
if err != nil {
t.Fatal(err)
}
for attempt := 0; attempt < rangeSourceFailureLimit; attempt++ {
req, err := http.NewRequest(http.MethodGet, url, nil)
if err != nil {
t.Fatal(err)
}
req.Header.Set("Range", "bytes=0-9")
resp, err := http.DefaultClient.Do(req)
if err != nil {
t.Fatal(err)
}
_, _ = io.ReadAll(resp.Body)
_ = resp.Body.Close()
}
if got := registry.SourceError(url); !errors.Is(got, boom) {
t.Fatalf("expected range source error %v, got %v", boom, got)
}
resp, err := http.Get(url)
if err != nil {
t.Fatal(err)
}
_ = resp.Body.Close()
if resp.StatusCode != http.StatusGone {
t.Fatalf("expected terminal source status %d, got %d", http.StatusGone, resp.StatusCode)
}
})
t.Run("short range reader", func(t *testing.T) {
registry := NewRegistry(t.TempDir())
defer registry.Close()
url, err := registry.CreateOpener(func(context.Context, OpenRequest) (io.ReadCloser, error) {
return io.NopCloser(strings.NewReader("short")), nil
}, &CreateOptions{Size: 10, Range: true})
if err != nil {
t.Fatal(err)
}
for attempt := 0; attempt < rangeSourceFailureLimit; attempt++ {
req, err := http.NewRequest(http.MethodGet, url, nil)
if err != nil {
t.Fatal(err)
}
req.Header.Set("Range", "bytes=0-9")
resp, err := http.DefaultClient.Do(req)
if err != nil {
t.Fatal(err)
}
_, _ = io.ReadAll(resp.Body)
_ = resp.Body.Close()
}
if got := registry.SourceError(url); !errors.Is(got, io.ErrUnexpectedEOF) {
t.Fatalf("expected range unexpected EOF, got %v", got)
}
})
t.Run("successful range retry clears transient failure", func(t *testing.T) {
registry := NewRegistry(t.TempDir())
defer registry.Close()
attempt := 0
boom := errors.New("transient range failure")
url, err := registry.CreateOpener(func(context.Context, OpenRequest) (io.ReadCloser, error) {
attempt++
if attempt == 1 {
return io.NopCloser(&terminalErrorReader{err: boom}), nil
}
return io.NopCloser(strings.NewReader("0123456789")), nil
}, &CreateOptions{Size: 10, Range: true})
if err != nil {
t.Fatal(err)
}
for i := 0; i < 2; i++ {
req, err := http.NewRequest(http.MethodGet, url, nil)
if err != nil {
t.Fatal(err)
}
req.Header.Set("Range", "bytes=0-9")
resp, err := http.DefaultClient.Do(req)
if err != nil {
t.Fatal(err)
}
_, _ = io.ReadAll(resp.Body)
_ = resp.Body.Close()
}
if got := registry.SourceError(url); got != nil {
t.Fatalf("successful retry left transient source error: %v", got)
}
})
t.Run("request cancellation is not a source error", func(t *testing.T) {
registry := NewRegistry(t.TempDir())
defer registry.Close()
opened := make(chan struct{})
url, err := registry.CreateOpener(func(ctx context.Context, _ OpenRequest) (io.ReadCloser, error) {
close(opened)
return &contextErrorReader{ctx: ctx}, nil
}, &CreateOptions{})
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithCancel(context.Background())
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
t.Fatal(err)
}
done := make(chan struct{})
go func() {
resp, _ := http.DefaultClient.Do(req)
if resp != nil {
_ = resp.Body.Close()
}
close(done)
}()
<-opened
cancel()
select {
case <-done:
case <-time.After(2 * time.Second):
t.Fatal("cancelled request did not finish")
}
if got := registry.SourceError(url); got != nil {
t.Fatalf("request cancellation was recorded as source error: %v", got)
}
})
t.Run("nil reader", func(t *testing.T) {
registry := NewRegistry(t.TempDir())
defer registry.Close()
url, err := registry.CreateOpener(func(context.Context, OpenRequest) (io.ReadCloser, error) {
return nil, nil
}, &CreateOptions{})
if err != nil {
t.Fatal(err)
}
resp, err := http.Get(url)
if err != nil {
t.Fatal(err)
}
_ = resp.Body.Close()
if resp.StatusCode != http.StatusGone {
t.Fatalf("expected nil reader status %d, got %d", http.StatusGone, resp.StatusCode)
}
if got := registry.SourceError(url); !errors.Is(got, ErrSourceClosed) {
t.Fatalf("expected closed source error, got %v", got)
}
})
}
func TestRegistryDoesNotRecordResponseWriteError(t *testing.T) {
registry := NewRegistry(t.TempDir())
defer registry.Close()
url, err := registry.CreateOpener(func(context.Context, OpenRequest) (io.ReadCloser, error) {
return io.NopCloser(strings.NewReader("payload")), nil
}, &CreateOptions{Size: 7})
if err != nil {
t.Fatal(err)
}
req, err := http.NewRequest(http.MethodGet, url, nil)
if err != nil {
t.Fatal(err)
}
writerErr := errors.New("client connection closed")
registry.ServeHTTP(&failingResponseWriter{header: make(http.Header), err: writerErr}, req)
if got := registry.SourceError(url); got != nil {
t.Fatalf("response write error was recorded as a source error: %v", got)
}
}
func TestRegistryTaskReferencesShareSource(t *testing.T) {
registry := NewRegistry(t.TempDir())
defer registry.Close()
url, err := registry.CreateBlob([]byte("shared"), "text/plain")
if err != nil {
t.Fatal(err)
}
if err := registry.Acquire(url); err != nil {
t.Fatal(err)
}
if err := registry.Acquire(url); err != nil {
t.Fatal(err)
}
if err := registry.Release(url); err != nil {
t.Fatal(err)
}
if !registry.IsURL(url) {
t.Fatal("the first task release revoked a source still used by another task")
}
resp, err := http.Get(url)
if err != nil {
t.Fatal(err)
}
body, readErr := io.ReadAll(resp.Body)
_ = resp.Body.Close()
if readErr != nil || string(body) != "shared" {
t.Fatalf("unexpected shared response body %q, error %v", body, readErr)
}
if err := registry.Release(url); err != nil {
t.Fatal(err)
}
if registry.IsURL(url) {
t.Fatal("the final task release did not revoke the source")
}
}
func TestRegistryUnclaimedSessionSourceExpires(t *testing.T) {
setUnclaimedSourceTTLForTest(t, 25*time.Millisecond)
registry := NewRegistry(t.TempDir())
defer registry.Close()
session := newCountingSession()
url, err := registry.CreateOpener(func(context.Context, OpenRequest) (io.ReadCloser, error) {
return io.NopCloser(strings.NewReader("unused")), nil
}, &CreateOptions{Size: 6, Session: session})
if err != nil {
t.Fatal(err)
}
if got := session.RefCount(); got != 1 {
t.Fatalf("unexpected initial session ref count: %d", got)
}
select {
case <-session.zero:
case <-time.After(2 * time.Second):
t.Fatal("unclaimed source did not expire")
}
if registry.IsURL(url) {
t.Fatal("expired source is still registered")
}
if got := session.RefCount(); got != 0 {
t.Fatalf("expired source retained its session: %d", got)
}
}
func TestRegistryAcquireStopsUnclaimedSourceExpiration(t *testing.T) {
const ttl = 25 * time.Millisecond
setUnclaimedSourceTTLForTest(t, ttl)
registry := NewRegistry(t.TempDir())
defer registry.Close()
session := newCountingSession()
url, err := registry.CreateOpener(func(context.Context, OpenRequest) (io.ReadCloser, error) {
return io.NopCloser(strings.NewReader("claimed")), nil
}, &CreateOptions{Size: 7, Session: session})
if err != nil {
t.Fatal(err)
}
if err := registry.Acquire(url); err != nil {
t.Fatal(err)
}
select {
case <-session.zero:
t.Fatal("claimed source expired")
case <-time.After(4 * ttl):
}
if !registry.IsURL(url) {
t.Fatal("claimed source was removed")
}
if err := registry.Release(url); err != nil {
t.Fatal(err)
}
select {
case <-session.zero:
case <-time.After(2 * time.Second):
t.Fatal("final task release did not release the session")
}
}
func TestRegistryUnclaimedExpirationKeepsActiveReaderSession(t *testing.T) {
setUnclaimedSourceTTLForTest(t, 100*time.Millisecond)
registry := NewRegistry(t.TempDir())
defer registry.Close()
session := newCountingSession()
reader, writer := io.Pipe()
opened := make(chan struct{})
url, err := registry.CreateOpener(func(context.Context, OpenRequest) (io.ReadCloser, error) {
close(opened)
return reader, nil
}, &CreateOptions{Size: 7, Session: session})
if err != nil {
t.Fatal(err)
}
type result struct {
body []byte
err error
}
resultCh := make(chan result, 1)
go func() {
resp, err := http.Get(url)
if err != nil {
resultCh <- result{err: err}
return
}
body, err := io.ReadAll(resp.Body)
_ = resp.Body.Close()
resultCh <- result{body: body, err: err}
}()
select {
case <-opened:
case got := <-resultCh:
t.Fatalf("request ended before the opener ran: body %q, error %v", got.body, got.err)
case <-time.After(2 * time.Second):
t.Fatal("active request did not open")
}
deadline := time.NewTimer(2 * time.Second)
ticker := time.NewTicker(5 * time.Millisecond)
defer deadline.Stop()
defer ticker.Stop()
for registry.IsURL(url) {
select {
case <-ticker.C:
case <-deadline.C:
t.Fatal("unclaimed source did not expire while its reader was active")
}
}
if got := session.RefCount(); got != 1 {
t.Fatalf("expiration released the active-reader session ref: %d", got)
}
if _, err := writer.Write([]byte("payload")); err != nil {
t.Fatal(err)
}
_ = writer.Close()
select {
case got := <-resultCh:
if got.err != nil || string(got.body) != "payload" {
t.Fatalf("active response was interrupted: body %q, error %v", got.body, got.err)
}
case <-time.After(2 * time.Second):
t.Fatal("active response did not finish after source expiration")
}
select {
case <-session.zero:
if got := session.RefCount(); got != 0 {
t.Fatalf("unexpected final session ref count: %d", got)
}
case <-time.After(2 * time.Second):
t.Fatal("active-reader session ref was not released")
}
}
func TestRegistryCloseCancelsUnclaimedExpiration(t *testing.T) {
const ttl = 25 * time.Millisecond
setUnclaimedSourceTTLForTest(t, ttl)
registry := NewRegistry(t.TempDir())
session := newCountingSession()
_, err := registry.CreateOpener(func(context.Context, OpenRequest) (io.ReadCloser, error) {
return io.NopCloser(strings.NewReader("unused")), nil
}, &CreateOptions{Size: 6, Session: session})
if err != nil {
t.Fatal(err)
}
if err := registry.Close(); err != nil {
t.Fatal(err)
}
select {
case <-session.zero:
case <-time.After(2 * time.Second):
t.Fatal("registry close did not release the session")
}
<-time.After(4 * ttl)
if got := session.RefCount(); got != 0 {
t.Fatalf("expiration released the session again after close: %d", got)
}
}
func TestRegistryRevokeKeepsActiveReaderSession(t *testing.T) {
registry := NewRegistry(t.TempDir())
defer registry.Close()
session := newCountingSession()
reader, writer := io.Pipe()
opened := make(chan struct{})
url, err := registry.CreateOpener(func(context.Context, OpenRequest) (io.ReadCloser, error) {
close(opened)
return reader, nil
}, &CreateOptions{Size: 7, Session: session})
if err != nil {
t.Fatal(err)
}
type result struct {
body []byte
err error
}
resultCh := make(chan result, 1)
go func() {
resp, err := http.Get(url)
if err != nil {
resultCh <- result{err: err}
return
}
body, err := io.ReadAll(resp.Body)
_ = resp.Body.Close()
resultCh <- result{body: body, err: err}
}()
<-opened
if got := session.RefCount(); got != 2 {
t.Fatalf("expected source and active-reader session refs, got %d", got)
}
if err := registry.Revoke(url); err != nil {
t.Fatal(err)
}
if got := session.RefCount(); got != 1 {
t.Fatalf("revoke released the active-reader session ref: %d", got)
}
if _, err := writer.Write([]byte("payload")); err != nil {
t.Fatal(err)
}
_ = writer.Close()
select {
case got := <-resultCh:
if got.err != nil || string(got.body) != "payload" {
t.Fatalf("active response was interrupted: body %q, error %v", got.body, got.err)
}
case <-time.After(2 * time.Second):
t.Fatal("active response did not finish after revoke")
}
select {
case <-session.zero:
if got := session.RefCount(); got != 0 {
t.Fatalf("unexpected final session ref count: %d", got)
}
case <-time.After(2 * time.Second):
t.Fatal("active-reader session ref was not released")
}
}
func TestRegistryHTTPErrorStatuses(t *testing.T) {
registry := NewRegistry(t.TempDir())
defer registry.Close()
url, err := registry.CreateBlob([]byte("secret"), "text/plain")
if err != nil {
t.Fatal(err)
}
badURL := url + "-missing"
resp, err := http.Get(badURL)
if err != nil {
t.Fatal(err)
}
_ = resp.Body.Close()
if resp.StatusCode != http.StatusNotFound {
t.Fatalf("unexpected missing source status: %d", resp.StatusCode)
}
req, err := http.NewRequest(http.MethodHead, url, nil)
if err != nil {
t.Fatal(err)
}
resp, err = http.DefaultClient.Do(req)
if err != nil {
t.Fatal(err)
}
_ = resp.Body.Close()
if resp.StatusCode != http.StatusMethodNotAllowed {
t.Fatalf("unexpected method status: %d", resp.StatusCode)
}
req, err = http.NewRequest(http.MethodGet, url, nil)
if err != nil {
t.Fatal(err)
}
req.Header.Set("Range", "bytes=99-100")
resp, err = http.DefaultClient.Do(req)
if err != nil {
t.Fatal(err)
}
_ = resp.Body.Close()
if resp.StatusCode != http.StatusRequestedRangeNotSatisfiable {
t.Fatalf("unexpected invalid range status: %d", resp.StatusCode)
}
if err := registry.Revoke(url); err != nil {
t.Fatal(err)
}
resp, err = http.Get(url)
if err != nil {
t.Fatal(err)
}
_ = resp.Body.Close()
if resp.StatusCode != http.StatusNotFound {
t.Fatalf("unexpected revoked source status: %d", resp.StatusCode)
}
if registry.IsURL(url) {
t.Fatal("revoked source should not be usable")
}
}
+19
View File
@@ -14,6 +14,10 @@ type Fetcher interface {
Setup(ctl *controller.Controller)
Resolve(req *base.Request, opts *base.Options) error
Start() error
// Patch modifies task-specific data based on the protocol.
// For HTTP: can modify Request info (URL, headers, etc.)
// For BT: can modify SelectFiles (via opts.SelectFiles)
Patch(req *base.Request, opts *base.Options) error
Pause() error
Close() error
@@ -123,6 +127,21 @@ type FetcherManager interface {
Close() error
}
// StatefulFetcherManager is an optional extension for protocols that keep
// shared client state outside individual task fetchers.
type StatefulFetcherManager interface {
SetStateStore(store ProtocolStateStore)
}
// ProtocolStateStore persists shared protocol state for a fetcher manager.
// Downloader provides the concrete storage backend, while the protocol decides
// when state should be loaded or flushed.
type ProtocolStateStore interface {
Load(v any) (bool, error)
Save(v any) error
Delete() error
}
type DefaultFetcher struct {
Ctl *controller.Controller
Meta *FetcherMeta
+53
View File
@@ -232,6 +232,59 @@ func (f *Fetcher) isDone() bool {
return true
}
// Patch modifies the BT task settings.
// Invalid file indices are silently ignored.
func (f *Fetcher) Patch(req *base.Request, opts *base.Options) error {
if opts == nil {
return nil
}
if opts.SelectFiles != nil {
selectFiles := opts.SelectFiles
// Get file count from resource metadata
fileCount := 0
if f.meta.Res != nil {
fileCount = len(f.meta.Res.Files)
}
// Filter out invalid indices (silently ignore)
validSelectFiles := make([]int, 0, len(selectFiles))
for _, idx := range selectFiles {
if idx >= 0 && idx < fileCount {
validSelectFiles = append(validSelectFiles, idx)
}
}
if f.torrent != nil {
files := f.torrent.Files()
// Cancel all current file downloads first
f.torrent.CancelPieces(0, f.torrent.NumPieces())
// Apply new file selection
if len(validSelectFiles) == len(files) {
f.torrent.DownloadAll()
} else {
for _, selectIndex := range validSelectFiles {
file := files[selectIndex]
file.Download()
}
}
}
f.meta.Opts.SelectFiles = validSelectFiles
// Recalculate the resource size based on new selection
if f.meta.Res != nil {
f.meta.Res.CalcSize(validSelectFiles)
}
// Reset progress tracking for new file selection
f.data.Progress = make(fetcher.Progress, len(validSelectFiles))
}
return nil
}
func (f *Fetcher) updateRes() {
res := &base.Resource{
Range: true,
+127
View File
@@ -239,3 +239,130 @@ func buildConfigFetcher(proxyConfig *base.DownloaderProxyConfig) fetcher.Fetcher
fetcher.Setup(newController)
return fetcher
}
// TestFetcher_Patch tests the Patch functionality for BT fetcher.
// It tests modifying selected files after Resolve (without downloading).
func TestFetcher_Patch(t *testing.T) {
f := buildFetcher()
// Resolve a multi-file torrent
err := f.Resolve(&base.Request{
URL: "./testdata/test.torrent",
}, nil)
if err != nil {
t.Fatalf("Resolve failed: %v", err)
}
// Verify initial state: 3 files, all selected by default
meta := f.Meta()
if len(meta.Res.Files) != 3 {
t.Fatalf("Expected 3 files, got %d", len(meta.Res.Files))
}
if len(meta.Opts.SelectFiles) != 3 {
t.Fatalf("Expected 3 selected files, got %d", len(meta.Opts.SelectFiles))
}
// Total size of all files: 107484864
totalSize := int64(107484864)
if meta.Res.Size != totalSize {
t.Fatalf("Expected total size %d, got %d", totalSize, meta.Res.Size)
}
t.Run("Patch with valid indices", func(t *testing.T) {
// Select only file 0 and 2 (c.txt and a.txt)
err := f.Patch(nil, &base.Options{
SelectFiles: []int{0, 2},
})
if err != nil {
t.Fatalf("Patch failed: %v", err)
}
meta := f.Meta()
if !reflect.DeepEqual(meta.Opts.SelectFiles, []int{0, 2}) {
t.Errorf("Expected SelectFiles [0, 2], got %v", meta.Opts.SelectFiles)
}
// Size should be recalculated: c.txt (98501754) + a.txt (78114) = 98579868
expectedSize := int64(98501754 + 78114)
if meta.Res.Size != expectedSize {
t.Errorf("Expected size %d, got %d", expectedSize, meta.Res.Size)
}
})
t.Run("Patch with invalid indices are silently ignored", func(t *testing.T) {
// Mix of valid (0, 1) and invalid (-1, 5, 100) indices
err := f.Patch(nil, &base.Options{
SelectFiles: []int{-1, 0, 5, 1, 100},
})
if err != nil {
t.Fatalf("Patch should not return error for invalid indices: %v", err)
}
meta := f.Meta()
// Only valid indices 0 and 1 should remain
if !reflect.DeepEqual(meta.Opts.SelectFiles, []int{0, 1}) {
t.Errorf("Expected SelectFiles [0, 1], got %v", meta.Opts.SelectFiles)
}
// Size should be: c.txt (98501754) + b.txt (8904996) = 107406750
expectedSize := int64(98501754 + 8904996)
if meta.Res.Size != expectedSize {
t.Errorf("Expected size %d, got %d", expectedSize, meta.Res.Size)
}
})
t.Run("Patch with all invalid indices results in empty selection", func(t *testing.T) {
err := f.Patch(nil, &base.Options{
SelectFiles: []int{-5, 10, 999},
})
if err != nil {
t.Fatalf("Patch should not return error: %v", err)
}
meta := f.Meta()
if len(meta.Opts.SelectFiles) != 0 {
t.Errorf("Expected empty SelectFiles, got %v", meta.Opts.SelectFiles)
}
// Note: CalcSize with empty selectFiles calculates total size of all files
// This is by design - empty selection in CalcSize means "all files"
// But SelectFiles being empty means no files are selected for download
if meta.Res.Size != totalSize {
t.Errorf("Expected size %d (CalcSize with empty slice = all files), got %d", totalSize, meta.Res.Size)
}
})
t.Run("Patch with nil opts does nothing", func(t *testing.T) {
// First set a known state
f.Patch(nil, &base.Options{SelectFiles: []int{1}})
prevSelectFiles := f.Meta().Opts.SelectFiles
// Patch with nil opts
err := f.Patch(nil, nil)
if err != nil {
t.Fatalf("Patch with nil opts should not fail: %v", err)
}
// Should remain unchanged
if !reflect.DeepEqual(f.Meta().Opts.SelectFiles, prevSelectFiles) {
t.Errorf("SelectFiles should remain unchanged after nil opts Patch")
}
})
t.Run("Patch progress array is resized", func(t *testing.T) {
btFetcher := f.(*Fetcher)
// Initialize progress array
btFetcher.data.Progress = make(fetcher.Progress, 3)
err := f.Patch(nil, &base.Options{
SelectFiles: []int{0, 2},
})
if err != nil {
t.Fatalf("Patch failed: %v", err)
}
if len(btFetcher.data.Progress) != 2 {
t.Errorf("Expected Progress length 2, got %d", len(btFetcher.data.Progress))
}
})
}
+15
View File
@@ -0,0 +1,15 @@
package ed2k
const (
defaultServerList = "45.82.80.155:5687,176.123.5.89:4725,85.121.5.137:4232,176.123.2.239:4232,145.239.2.134:4661,91.208.162.87:4232,37.15.61.236:4232"
defaultServerMet = "ed2k://|serverlist|http://upd.emule-security.org/server.met|/"
defaultNodesDat = "https://upd.emule-security.org/nodes.dat"
)
type config struct {
ListenPort int `json:"listenPort"`
UDPPort int `json:"udpPort"`
ServerAddr string `json:"serverAddr"`
ServerMet string `json:"serverMet"`
NodesDat string `json:"nodesDat"`
}
+505
View File
@@ -0,0 +1,505 @@
package ed2k
import (
"context"
"errors"
"os"
"path/filepath"
"strings"
"sync"
"time"
"github.com/GopeedLab/gopeed/internal/controller"
"github.com/GopeedLab/gopeed/internal/fetcher"
"github.com/GopeedLab/gopeed/pkg/base"
ped2k "github.com/GopeedLab/gopeed/pkg/protocol/ed2k"
"github.com/monkeyWie/goed2k"
gprotocol "github.com/monkeyWie/goed2k/protocol"
)
type clientStateStore struct {
store fetcher.ProtocolStateStore
}
func (s *clientStateStore) Load() (*goed2k.ClientState, error) {
if s == nil || s.store == nil {
return nil, nil
}
var state goed2k.ClientState
exist, err := s.store.Load(&state)
if err != nil {
return nil, err
}
if !exist {
return nil, nil
}
return &state, nil
}
func (s *clientStateStore) Save(state *goed2k.ClientState) error {
if s == nil || s.store == nil {
return nil
}
if state == nil {
return s.store.Delete()
}
return s.store.Save(state)
}
type Fetcher struct {
ctl *controller.Controller
config *config
manager *FetcherManager
meta *fetcher.FetcherMeta
handle goed2k.TransferHandle
waitCtx context.Context
waitCancel context.CancelFunc
}
func (f *Fetcher) Setup(ctl *controller.Controller) {
f.ctl = ctl
if f.meta == nil {
f.meta = &fetcher.FetcherMeta{}
}
f.waitCtx, f.waitCancel = context.WithCancel(context.Background())
f.ctl.GetConfig(&f.config)
}
func (f *Fetcher) Resolve(req *base.Request, opts *base.Options) error {
link, err := parseLink(req.URL)
if err != nil {
return err
}
f.meta.Req = req
f.meta.Opts = opts
if f.meta.Opts == nil {
f.meta.Opts = &base.Options{}
}
f.meta.Res = buildResource(link)
return nil
}
func (f *Fetcher) Start() error {
link, err := parseLink(f.meta.Req.URL)
if err != nil {
return err
}
if f.meta.Res == nil {
f.meta.Res = buildResource(link)
}
client, err := f.getClient()
if err != nil {
return err
}
targetPath := f.meta.SingleFilepath()
if err := os.MkdirAll(filepath.Dir(targetPath), 0o755); err != nil {
return err
}
handle := client.FindTransfer(link.Hash)
if handle.IsValid() {
f.handle = handle
if handle.IsPaused() {
if err := client.ResumeTransfer(handle.GetHash()); err != nil {
return err
}
}
return nil
}
atp := goed2k.AddTransferParams{
Hash: link.Hash,
CreateTime: time.Now().UnixMilli(),
Size: link.NumberValue,
FilePath: targetPath,
}
handle, err = client.AddTransfer(atp)
if err != nil {
return err
}
f.handle = handle
if handle.IsValid() && handle.IsPaused() {
if err := client.ResumeTransfer(handle.GetHash()); err != nil {
return err
}
}
return nil
}
func (f *Fetcher) Patch(req *base.Request, opts *base.Options) error {
handle := f.currentHandle()
if opts != nil && (opts.Name != "" || opts.Path != "") && handle.IsValid() {
return errors.New("cannot change ed2k target path after transfer started")
}
if req != nil && req.URL != "" && handle.IsValid() {
return errors.New("cannot change ed2k link after transfer started")
}
if req != nil {
if req.URL != "" {
link, err := parseLink(req.URL)
if err != nil {
return err
}
f.meta.Req.URL = req.URL
f.meta.Res = buildResource(link)
}
if req.Labels != nil {
if f.meta.Req.Labels == nil {
f.meta.Req.Labels = make(map[string]string)
}
for k, v := range req.Labels {
f.meta.Req.Labels[k] = v
}
}
if req.Proxy != nil {
f.meta.Req.Proxy = req.Proxy
}
}
if opts != nil {
if opts.Name != "" {
f.meta.Opts.Name = opts.Name
}
if opts.Path != "" {
f.meta.Opts.Path = opts.Path
}
}
return nil
}
func (f *Fetcher) Pause() error {
handle := f.currentHandle()
if !handle.IsValid() {
return nil
}
client, err := f.getClient()
if err != nil {
return err
}
if err := client.PauseTransfer(handle.GetHash()); err != nil {
return err
}
f.handle = handle
return nil
}
func (f *Fetcher) Close() error {
if f.waitCancel != nil {
f.waitCancel()
}
handle := f.currentHandle()
if !handle.IsValid() {
return nil
}
client, err := f.getClient()
if err != nil {
return err
}
f.handle = handle
return client.RemoveTransfer(handle.GetHash(), false)
}
func (f *Fetcher) Meta() *fetcher.FetcherMeta {
return f.meta
}
func (f *Fetcher) Stats() any {
handle := f.currentHandle()
if !handle.IsValid() {
return &ped2k.Stats{}
}
status := handle.GetStatus()
return &ped2k.Stats{
State: string(status.State),
ActivePeers: handle.ActiveConnections(),
TotalPeers: status.NumPeers,
DownloadRate: status.DownloadRate,
Upload: status.Upload,
UploadRate: status.UploadRate,
TotalDone: status.TotalDone,
TotalReceived: status.TotalReceived,
TotalWanted: status.TotalWanted,
}
}
func (f *Fetcher) Progress() fetcher.Progress {
handle := f.currentHandle()
if !handle.IsValid() {
return fetcher.Progress{0}
}
return fetcher.Progress{handle.GetStatus().TotalReceived}
}
func (f *Fetcher) Wait() error {
client, err := f.getClient()
if err != nil {
return err
}
hash, err := f.hash()
if err != nil {
return err
}
handle := f.currentHandle()
if handle.IsValid() && handle.IsFinished() {
return nil
}
progressCh, cancel := client.SubscribeTransferProgress()
defer cancel()
for {
select {
case <-f.waitCtx.Done():
return nil
case event, ok := <-progressCh:
if !ok {
return nil
}
for _, transfer := range event.Transfers {
if transfer.Hash.Compare(hash) != 0 {
continue
}
// Removal can happen during task deletion or client shutdown, both of
// which should unblock Wait without treating it as a download failure.
if transfer.Removed || transfer.State == goed2k.Finished {
return nil
}
}
}
}
}
func (f *Fetcher) getClient() (*goed2k.Client, error) {
if f.manager == nil {
f.manager = &FetcherManager{}
}
return f.manager.initClient(f.config)
}
func (f *Fetcher) currentHandle() goed2k.TransferHandle {
if f.handle.IsValid() {
return f.handle
}
if f.manager == nil {
return f.handle
}
client := f.manager.currentClient()
if client == nil {
return f.handle
}
hash, err := f.hash()
if err != nil {
return f.handle
}
handle := client.FindTransfer(hash)
if handle.IsValid() {
f.handle = handle
}
return f.handle
}
func (f *Fetcher) hash() (gprotocol.Hash, error) {
if f.meta == nil || f.meta.Req == nil {
return gprotocol.Invalid, errors.New("ed2k link is empty")
}
link, err := parseLink(f.meta.Req.URL)
if err != nil {
return gprotocol.Invalid, err
}
return link.Hash, nil
}
type FetcherManager struct {
mu sync.Mutex
client *goed2k.Client
stateStore *clientStateStore
}
func (fm *FetcherManager) SetStateStore(store fetcher.ProtocolStateStore) {
fm.mu.Lock()
defer fm.mu.Unlock()
if fm.stateStore == nil {
fm.stateStore = &clientStateStore{}
}
fm.stateStore.store = store
if fm.client != nil {
fm.client.SetStateStore(fm.stateStore)
}
}
func (fm *FetcherManager) Name() string {
return "ed2k"
}
func (fm *FetcherManager) Filters() []*fetcher.SchemeFilter {
return []*fetcher.SchemeFilter{
{
Type: fetcher.FilterTypeUrl,
Pattern: "ED2K",
},
}
}
func (fm *FetcherManager) Build() fetcher.Fetcher {
return &Fetcher{manager: fm}
}
func (fm *FetcherManager) ParseName(u string) string {
link, err := parseLink(u)
if err != nil {
return ""
}
return link.StringValue
}
func (fm *FetcherManager) AutoRename() bool {
return true
}
func (fm *FetcherManager) DefaultConfig() any {
return &config{
ListenPort: 0,
UDPPort: 0,
ServerAddr: defaultServerList,
ServerMet: defaultServerMet,
NodesDat: defaultNodesDat,
}
}
func (fm *FetcherManager) Store(f fetcher.Fetcher) (any, error) {
return nil, nil
}
func (fm *FetcherManager) Restore() (v any, f func(meta *fetcher.FetcherMeta, v any) fetcher.Fetcher) {
return nil, func(meta *fetcher.FetcherMeta, v any) fetcher.Fetcher {
return &Fetcher{
manager: fm,
meta: meta,
}
}
}
func (fm *FetcherManager) Close() error {
fm.mu.Lock()
defer fm.mu.Unlock()
if fm.client != nil {
fm.client.Close()
fm.client = nil
}
return nil
}
func parseLink(raw string) (goed2k.EMuleLink, error) {
link, err := goed2k.ParseEMuleLink(raw)
if err != nil {
return goed2k.EMuleLink{}, err
}
if link.Type != goed2k.LinkFile {
return goed2k.EMuleLink{}, errors.New("unsupported ed2k link type")
}
return link, nil
}
func buildResource(link goed2k.EMuleLink) *base.Resource {
return &base.Resource{
Size: link.NumberValue,
Range: false,
Hash: link.Hash.String(),
Files: []*base.FileInfo{
{
Name: link.StringValue,
Size: link.NumberValue,
},
},
}
}
func splitCommaList(value string) []string {
if strings.TrimSpace(value) == "" {
return nil
}
parts := strings.Split(value, ",")
out := make([]string, 0, len(parts))
for _, part := range parts {
part = strings.TrimSpace(part)
if part != "" {
out = append(out, part)
}
}
return out
}
func (fm *FetcherManager) getStateStoreLocked() *clientStateStore {
if fm.stateStore == nil {
fm.stateStore = &clientStateStore{}
}
return fm.stateStore
}
func (fm *FetcherManager) currentClient() *goed2k.Client {
fm.mu.Lock()
defer fm.mu.Unlock()
return fm.client
}
func (fm *FetcherManager) initClient(cfg *config) (*goed2k.Client, error) {
fm.mu.Lock()
defer fm.mu.Unlock()
if fm.client != nil {
return fm.client, nil
}
if cfg == nil {
cfg = fm.DefaultConfig().(*config)
}
settings := goed2k.NewSettings()
settings.ListenPort = cfg.ListenPort
settings.UDPPort = cfg.UDPPort
settings.EnableDHT = true
settings.EnableUPnP = true
settings.ReconnectToServer = true
client := goed2k.NewClient(settings)
client.SetStateStore(fm.getStateStoreLocked())
if err := client.LoadState(""); err != nil {
return nil, err
}
if err := client.Start(); err != nil {
return nil, err
}
fm.client = client
// Bootstrap is best-effort: downloads can still proceed later even if
// server list or DHT initialization fails during startup.
bootstrapClient(client, cfg)
return fm.client, nil
}
func bootstrapClient(client *goed2k.Client, cfg *config) {
for _, serverAddr := range splitCommaList(cfg.ServerAddr) {
go func(serverAddr string) {
_ = client.Connect(serverAddr)
}(serverAddr)
}
for _, source := range splitCommaList(cfg.ServerMet) {
go func(source string) {
_ = client.ConnectServerMet(source)
}(source)
}
if cfg.NodesDat != "" {
go func() {
_ = client.LoadDHTNodesDat(cfg.NodesDat)
}()
}
}
+42
View File
@@ -0,0 +1,42 @@
package ed2k
import (
"testing"
"github.com/GopeedLab/gopeed/internal/controller"
"github.com/GopeedLab/gopeed/pkg/base"
)
const testLink = "ed2k://|file|cn_windows_10_multi-edition_vl_version_1709_updated_sept_2017_x64_dvd_100090774.iso|4630972416|8867C5E54405FF9452225B66EFEE690A|/"
func TestFetcher_Resolve(t *testing.T) {
f := (&FetcherManager{}).Build()
f.Setup(controller.NewController())
err := f.Resolve(&base.Request{URL: testLink}, &base.Options{Path: t.TempDir()})
if err != nil {
t.Fatalf("Resolve() error = %v", err)
}
meta := f.Meta()
if meta.Res == nil {
t.Fatal("Resolve() resource is nil")
}
if got, want := meta.Res.Hash, "8867C5E54405FF9452225B66EFEE690A"; got != want {
t.Fatalf("Resolve() hash = %s, want %s", got, want)
}
if got, want := meta.Res.Size, int64(4630972416); got != want {
t.Fatalf("Resolve() size = %d, want %d", got, want)
}
if got, want := meta.Res.Files[0].Name, "cn_windows_10_multi-edition_vl_version_1709_updated_sept_2017_x64_dvd_100090774.iso"; got != want {
t.Fatalf("Resolve() name = %q, want %q", got, want)
}
}
func TestFetcherManager_ParseName(t *testing.T) {
got := (&FetcherManager{}).ParseName(testLink)
want := "cn_windows_10_multi-edition_vl_version_1709_updated_sept_2017_x64_dvd_100090774.iso"
if got != want {
t.Fatalf("ParseName() = %q, want %q", got, want)
}
}
+140 -4
View File
@@ -588,6 +588,10 @@ func (f *Fetcher) Start() error {
// Already downloading, this is a resume from pause
return f.doStart()
case stateError:
// Retry after error: reset and restart
return f.doStart()
default:
return fmt.Errorf("cannot start in current state: %v", state)
}
@@ -598,10 +602,32 @@ func (f *Fetcher) doStart() error {
<-f.resolvedCh
state := f.getState()
if state == stateDone || state == stateError {
if state == stateDone {
return nil
}
// If retrying after error, reset connection states for retry
if state == stateError {
// Drain any pending error from doneCh before retry
select {
case <-f.doneCh:
default:
}
f.connMu.Lock()
for _, conn := range f.connections {
// Reset connections that can be retried
if !conn.Completed && conn.State != connCompleted {
f.resetConnectionForRestart(conn)
conn.State = connNotStarted
conn.failed = false
conn.retryTimes = 0
conn.lastErr = nil
}
}
f.connMu.Unlock()
}
// Open or create target file first (needed for prefetch copy)
name := f.meta.SingleFilepath()
var err error
@@ -948,6 +974,8 @@ func (f *Fetcher) runConnection(conn *connection) {
}
// downloadChunkOnce performs a single HTTP request for the current chunk without retrying.
// If the redirect URL fails with an expiration-related error (401, 403, 410),
// it will automatically retry with the original URL and update the redirect URL on success.
func (f *Fetcher) downloadChunkOnce(conn *connection, client *http.Client, buf []byte) error {
if conn.ctx.Err() != nil {
return conn.ctx.Err()
@@ -981,11 +1009,34 @@ func (f *Fetcher) downloadChunkOnce(conn *connection, client *http.Client, buf [
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode != base.HttpCodeOK && resp.StatusCode != base.HttpCodePartialContent {
return NewRequestError(resp.StatusCode)
resp.Body.Close()
originalErr := NewRequestError(resp.StatusCode)
// Check if this might be a redirect URL expiration error
// If so, try falling back to the original URL
if f.hasRedirectURL() && isRedirectExpiredError(originalErr) {
fallbackResp, fallbackErr := f.tryFallbackToOriginalURL(conn.ctx, client, rangeStart, rangeEnd)
if fallbackErr == nil && fallbackResp != nil {
// Fallback succeeded, use this response instead
resp = fallbackResp
// Update the redirect URL from the response
if resp.Request != nil && resp.Request.URL != nil {
f.updateRedirectURL(resp.Request.URL.String())
}
} else {
// Fallback also failed, return the original error
if fallbackResp != nil {
fallbackResp.Body.Close()
}
return originalErr
}
} else {
return originalErr
}
}
defer resp.Body.Close()
// Record successful connection time for adaptive timeout
f.updateMaxConnTime(time.Since(connStartTime))
@@ -1363,6 +1414,27 @@ func (f *Fetcher) helpOtherConnection(helper *connection) bool {
return true
}
func (f *Fetcher) resetConnectionForRestart(conn *connection) {
if f.meta.Res.Range {
return
}
// Without range support a new request always starts from byte 0,
// so pause/retry must restart instead of continuing from the old offset.
if conn.Chunk == nil {
conn.Chunk = newChunk(0, 0)
} else {
conn.Chunk.Begin = 0
conn.Chunk.End = 0
conn.Chunk.Downloaded = 0
}
conn.Downloaded = 0
conn.Completed = false
conn.speed = 0
conn.lastSpeedCheck = 0
conn.lastSpeedDownload = 0
}
func (f *Fetcher) resumeConnections() {
// Collect connections to resume while holding the lock
var toResume []*connection
@@ -1386,6 +1458,7 @@ func (f *Fetcher) resumeConnections() {
continue
}
}
f.resetConnectionForRestart(conn)
// Reset the connection state for resume
conn.ctx, conn.cancel = context.WithCancel(f.ctx)
conn.State = connNotStarted
@@ -1425,7 +1498,16 @@ func (f *Fetcher) onDownloadComplete() {
// Check if all chunks are complete (no remaining bytes)
allChunksComplete := true
for _, conn := range f.connections {
if conn.Chunk != nil && conn.Chunk.remain() > 0 && !conn.Completed && conn.State != connCompleted {
needsMoreData := false
if f.meta.Res.Range {
needsMoreData = conn.Chunk != nil && conn.Chunk.remain() > 0
} else if f.meta.Res.Size > 0 {
needsMoreData = conn.Downloaded < f.meta.Res.Size
} else {
needsMoreData = !conn.Completed && conn.State != connCompleted
}
if needsMoreData && !conn.Completed && conn.State != connCompleted {
// This connection has remaining work and isn't done
// Check if it failed with 403 (server limit) - these can be ignored if other connections completed the work
if conn.State == connFailed && conn.failed {
@@ -1523,6 +1605,60 @@ func (f *Fetcher) checkCompletion() bool {
return false
}
// Patch modifies the HTTP request information.
func (f *Fetcher) Patch(req *base.Request, opts *base.Options) error {
// Patch request info
if req != nil {
if req.URL != "" {
f.meta.Req.URL = req.URL
// Clear redirect URL when URL is changed, so new requests use the new URL
f.updateRedirectURL("")
}
if req.Extra != nil {
if err := base.ParseReqExtra[fhttp.ReqExtra](req); err != nil {
return err
}
patchExtra := req.Extra.(*fhttp.ReqExtra)
// Merge Extra fields instead of replacing entirely
if f.meta.Req.Extra == nil {
f.meta.Req.Extra = &fhttp.ReqExtra{}
}
existingExtra := f.meta.Req.Extra.(*fhttp.ReqExtra)
// Update Method only if non-empty
if patchExtra.Method != "" {
existingExtra.Method = patchExtra.Method
}
// Update Body only if non-empty
if patchExtra.Body != "" {
existingExtra.Body = patchExtra.Body
}
// Merge Headers: existing keys are overwritten, new keys are added
if patchExtra.Header != nil {
if existingExtra.Header == nil {
existingExtra.Header = make(map[string]string)
}
for k, v := range patchExtra.Header {
existingExtra.Header[k] = v
}
}
}
// Merge Labels: existing keys are overwritten, new keys are added
if req.Labels != nil {
if f.meta.Req.Labels == nil {
f.meta.Req.Labels = make(map[string]string)
}
for k, v := range req.Labels {
f.meta.Req.Labels[k] = v
}
}
if req.Proxy != nil {
f.meta.Req.Proxy = req.Proxy
}
}
return nil
}
func (f *Fetcher) Pause() error {
if f.cancel != nil {
f.cancel()
@@ -15,6 +15,7 @@ import (
type fetcherData struct {
Connections []*connection
RedirectURL string // Saved redirect URL for resume
}
// ============================================================================
@@ -71,8 +72,12 @@ func (fm *FetcherManager) DefaultConfig() any {
func (fm *FetcherManager) Store(f fetcher.Fetcher) (data any, err error) {
_f := f.(*Fetcher)
_f.redirectLock.Lock()
redirectURL := _f.redirectURL
_f.redirectLock.Unlock()
return &fetcherData{
Connections: _f.connections,
RedirectURL: redirectURL,
}, nil
}
@@ -87,6 +92,10 @@ func (fm *FetcherManager) Restore() (v any, f func(meta *fetcher.FetcherMeta, v
if len(fd.Connections) > 0 {
fetcher.connections = fd.Connections
}
// Restore redirect URL for resume
if fd.RedirectURL != "" {
fetcher.redirectURL = fd.RedirectURL
}
return fetcher
}
}
+558 -3
View File
@@ -299,6 +299,53 @@ func TestFetcher_DownloadContinue(t *testing.T) {
downloadContinue(listener, 16, t)
}
func TestFetcher_DownloadContinue_NoRangeRestart(t *testing.T) {
listener := test.StartTestNoRangeSlowServer(time.Millisecond)
defer listener.Close()
fetcher := downloadReady(listener, 4, t)
if err := fetcher.Start(); err != nil {
t.Fatal(err)
}
time.Sleep(20 * time.Millisecond)
stats := fetcher.Stats().(*http.Stats)
if len(stats.Connections) != 1 {
t.Fatalf("expected a single non-range connection, got %d", len(stats.Connections))
}
if stats.Connections[0].Downloaded <= 0 || stats.Connections[0].Downloaded >= test.BuildSize {
t.Fatalf("expected partial download before pause, got %d", stats.Connections[0].Downloaded)
}
if err := fetcher.Pause(); err != nil {
t.Fatal(err)
}
if err := fetcher.Start(); err != nil {
t.Fatal(err)
}
if err := fetcher.Wait(); err != nil {
t.Fatal(err)
}
finalStats := fetcher.Stats().(*http.Stats)
if len(finalStats.Connections) != 1 {
t.Fatalf("expected a single non-range connection after resume, got %d", len(finalStats.Connections))
}
if finalStats.Connections[0].Downloaded != test.BuildSize {
t.Fatalf("downloaded bytes should restart cleanly: got %d, want %d", finalStats.Connections[0].Downloaded, test.BuildSize)
}
if total := fetcher.Progress().TotalDownloaded(); total != test.BuildSize {
t.Fatalf("progress total = %d, want %d", total, test.BuildSize)
}
want := test.FileMd5(test.BuildFile)
got := test.FileMd5(test.DownloadFile)
if want != got {
t.Errorf("Download() got = %v, want %v", got, want)
}
}
func TestFetcher_DownloadChunked(t *testing.T) {
listener := test.StartTestCustomServer()
defer listener.Close()
@@ -329,7 +376,7 @@ func TestFetcher_DownloadError(t *testing.T) {
}
func TestFetcher_DownloadLimit(t *testing.T) {
listener := test.StartTestLimitServer(4, 0)
listener := test.StartTestConLimitServer(4)
defer listener.Close()
downloadNormal(listener, 1, t)
@@ -567,7 +614,7 @@ func TestFetcher_SlowStartExpansion(t *testing.T) {
os.Remove(test.DownloadFile)
// Use 100ns delay per byte for faster test (~10MB/s theoretical)
listener := test.StartTestSlowStartServer(100 * time.Nanosecond)
listener := test.StartTestLowSpeedServer(100 * time.Nanosecond)
// Ensure cleanup happens before next subtest
cleanup := func() {
@@ -711,7 +758,7 @@ func TestFetcher_AsyncPrefetch(t *testing.T) {
t.Run("PrefetchPartial", func(t *testing.T) {
// Use slow server with 100 nanosecond delay per byte
// This means ~10MB/s speed, so 100ms should download ~1MB
listener := test.StartTestSlowStartServer(100 * time.Nanosecond)
listener := test.StartTestLowSpeedServer(100 * time.Nanosecond)
defer listener.Close()
fetcher := buildFetcher()
@@ -778,6 +825,168 @@ func TestFetcher_AsyncPrefetch(t *testing.T) {
})
}
// TestFetcher_DownloadExpiringRedirectURL tests that the fetcher correctly handles
// expiring redirect URLs by falling back to the original URL and getting a new redirect.
func TestFetcher_DownloadExpiringRedirectURL(t *testing.T) {
// Test with redirect expiring after 2 requests
// This means:
// - Request 1: Resolve (original URL redirects to temp URL v1)
// - Request 2: First download request to temp URL v1 succeeds
// - Request 3: Second download request to temp URL v1 returns 403
// - Fetcher should then retry with original URL, get temp URL v2
// - Continue downloading from temp URL v2
t.Run("RedirectExpiresAfter2Requests", func(t *testing.T) {
os.Remove(test.DownloadFile)
// Create server with redirect expiring after 2 requests, with slow transfer to ensure
// multiple connection attempts are needed
listener := test.StartTestExpiringRedirectServer(2, 100*time.Nanosecond)
defer listener.Close()
fetcher := buildFetcher()
err := fetcher.Resolve(&base.Request{
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
}, &base.Options{
Name: test.DownloadName,
Path: test.Dir,
Extra: &http.OptsExtra{
Connections: 4, // Use multiple connections to trigger redirect expiration
},
})
if err != nil {
t.Fatal(err)
}
err = fetcher.Start()
if err != nil {
t.Fatal(err)
}
err = fetcher.Wait()
if err != nil {
t.Fatal(err)
}
// Verify file content is correct despite redirect expiration
want := test.FileMd5(test.BuildFile)
got := test.FileMd5(test.DownloadFile)
if want != got {
t.Errorf("Download() got = %v, want %v", got, want)
}
os.Remove(test.DownloadFile)
})
// Test with redirect expiring after 5 requests (more room for initial connections)
t.Run("RedirectExpiresAfter5Requests", func(t *testing.T) {
os.Remove(test.DownloadFile)
listener := test.StartTestExpiringRedirectServer(5, 100*time.Nanosecond)
defer listener.Close()
fetcher := buildFetcher()
err := fetcher.Resolve(&base.Request{
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
}, &base.Options{
Name: test.DownloadName,
Path: test.Dir,
Extra: &http.OptsExtra{
Connections: 8,
},
})
if err != nil {
t.Fatal(err)
}
err = fetcher.Start()
if err != nil {
t.Fatal(err)
}
err = fetcher.Wait()
if err != nil {
t.Fatal(err)
}
// Verify file content
want := test.FileMd5(test.BuildFile)
got := test.FileMd5(test.DownloadFile)
if want != got {
t.Errorf("Download() got = %v, want %v", got, want)
}
os.Remove(test.DownloadFile)
})
}
// TestFetcher_RetryAfterError tests that the fetcher can retry downloading
// after a previous download attempt failed by calling Start() again.
func TestFetcher_RetryAfterError(t *testing.T) {
os.Remove(test.DownloadFile)
// Server fails first 3 requests (after resolve), then recovers
// With 1 connection and 3 retries:
// - First Start(): requests 2, 3, 4 → all fail (3 retries exhausted) → returns error
// - Second Start(): request 5 → succeeds (server recovered after 3 failures)
listener := test.StartTestFailThenRecoverServer(3)
defer listener.Close()
fetcher := buildFetcher()
err := fetcher.Resolve(&base.Request{
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
}, &base.Options{
Name: test.DownloadName,
Path: test.Dir,
Extra: &http.OptsExtra{
Connections: 1, // Use single connection to simplify test
},
})
if err != nil {
t.Fatal(err)
}
// First download attempt - should fail because server returns 416 after resolve
err = fetcher.Start()
if err != nil {
t.Fatal(err)
}
err = fetcher.Wait()
// First attempt should fail with 416 error
if err == nil {
t.Fatal("Expected first download attempt to fail, but it succeeded")
}
t.Logf("First attempt failed as expected: %v", err)
// Check that fetcher is in error state
state := fetcher.getState()
if state != stateError {
t.Errorf("Expected fetcher to be in stateError, got %v", state)
}
// Verify that we can call Start() again after error
// This tests the stateError handling in Start()
err = fetcher.Start()
if err != nil {
t.Fatalf("Start() after error failed: %v", err)
}
// Wait for second attempt - should succeed now that server has recovered
err = fetcher.Wait()
if err != nil {
t.Fatalf("Retry failed: %v", err)
}
// Verify file content
want := test.FileMd5(test.BuildFile)
got := test.FileMd5(test.DownloadFile)
if want != got {
t.Errorf("Download() got = %v, want %v", got, want)
}
os.Remove(test.DownloadFile)
}
func TestFetcherManager_ParseName(t *testing.T) {
type args struct {
u string
@@ -1046,3 +1255,349 @@ func buildConfigFetcher(cfg config) fetcher.Fetcher {
fetcher.Setup(newController)
return fetcher
}
// TestFetcher_Patch_URLChange tests the Patch functionality where a failed download URL
// is replaced with a working one. This simulates:
// 1. Initial download attempt with a bad URL (returns 404)
// 2. Patching the task with a new working URL
// 3. Successful download after URL modification
func TestFetcher_Patch_URLChange(t *testing.T) {
listener := test.StartTestPatchURLServer()
defer listener.Close()
f := buildFetcher()
badURL := "http://" + listener.Addr().String() + "/bad-url"
goodURL := "http://" + listener.Addr().String() + "/good-url"
opts := &base.Options{
Name: test.DownloadName,
Path: test.Dir,
Extra: &http.OptsExtra{Connections: 1},
}
// Step 1: Try to resolve with bad URL - should fail with error
err := f.Resolve(&base.Request{URL: badURL}, opts)
if err == nil {
t.Fatal("Expected error for bad URL, got nil")
}
// Step 2: Create a new fetcher and resolve with bad URL but don't wait
// We need to test patching a task that has been created
f2 := buildFetcher()
// First resolve with good URL to create a valid fetcher state
err = f2.Resolve(&base.Request{URL: goodURL}, opts)
if err != nil {
t.Fatal(err)
}
// Verify initial URL
if f2.meta.Req.URL != goodURL {
t.Errorf("Initial URL = %v, want %v", f2.meta.Req.URL, goodURL)
}
// Step 3: Patch to change URL (simulating URL change scenario)
newURL := "http://" + listener.Addr().String() + "/good-url"
err = f2.Patch(&base.Request{URL: newURL}, nil)
if err != nil {
t.Fatal(err)
}
// Verify URL was patched
if f2.meta.Req.URL != newURL {
t.Errorf("Patched URL = %v, want %v", f2.meta.Req.URL, newURL)
}
// Step 4: Start download and verify success
err = f2.Start()
if err != nil {
t.Fatal(err)
}
err = f2.Wait()
if err != nil {
t.Fatal(err)
}
// Verify file was downloaded correctly
want := test.FileMd5(test.BuildFile)
got := test.FileMd5(test.DownloadFile)
if want != got {
t.Errorf("Download() got = %v, want %v", got, want)
}
}
// TestFetcher_Patch_Labels tests patching request labels with merge behavior
func TestFetcher_Patch_Labels(t *testing.T) {
listener := test.StartTestFileServer()
defer listener.Close()
f := buildFetcher()
opts := &base.Options{
Name: test.DownloadName,
Path: test.Dir,
Extra: &http.OptsExtra{Connections: 1},
}
err := f.Resolve(&base.Request{
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
Labels: map[string]string{
"key1": "value1",
"key3": "value3",
},
}, opts)
if err != nil {
t.Fatal(err)
}
// Verify initial labels
if f.meta.Req.Labels["key1"] != "value1" {
t.Errorf("Initial label key1 = %v, want value1", f.meta.Req.Labels["key1"])
}
if f.meta.Req.Labels["key3"] != "value3" {
t.Errorf("Initial label key3 = %v, want value3", f.meta.Req.Labels["key3"])
}
// Patch with new labels - key1 should be overwritten, key2 should be added, key3 should remain
patchReq := &base.Request{
Labels: map[string]string{
"key1": "modified",
"key2": "newValue",
},
}
err = f.Patch(patchReq, nil)
if err != nil {
t.Fatal(err)
}
// Verify labels were merged correctly
if f.meta.Req.Labels["key1"] != "modified" {
t.Errorf("Patched label key1 = %v, want modified", f.meta.Req.Labels["key1"])
}
if f.meta.Req.Labels["key2"] != "newValue" {
t.Errorf("Patched label key2 = %v, want newValue", f.meta.Req.Labels["key2"])
}
// key3 should remain unchanged
if f.meta.Req.Labels["key3"] != "value3" {
t.Errorf("Label key3 = %v, want value3 (should remain unchanged)", f.meta.Req.Labels["key3"])
}
}
// TestFetcher_Patch_Extra tests patching request Extra with merge behavior
func TestFetcher_Patch_Extra(t *testing.T) {
listener := test.StartTestFileServer()
defer listener.Close()
f := buildFetcher()
opts := &base.Options{
Name: test.DownloadName,
Path: test.Dir,
Extra: &http.OptsExtra{Connections: 1},
}
// Resolve with initial Extra
err := f.Resolve(&base.Request{
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
Extra: &http.ReqExtra{
Method: "GET",
Body: "initial body",
Header: map[string]string{
"Authorization": "Bearer token123",
"X-Custom": "original",
},
},
}, opts)
if err != nil {
t.Fatal(err)
}
// Verify initial Extra
initialExtra := f.meta.Req.Extra.(*http.ReqExtra)
if initialExtra.Method != "GET" {
t.Errorf("Initial Method = %v, want GET", initialExtra.Method)
}
if initialExtra.Body != "initial body" {
t.Errorf("Initial Body = %v, want 'initial body'", initialExtra.Body)
}
if initialExtra.Header["Authorization"] != "Bearer token123" {
t.Errorf("Initial Authorization header = %v, want 'Bearer token123'", initialExtra.Header["Authorization"])
}
if initialExtra.Header["X-Custom"] != "original" {
t.Errorf("Initial X-Custom header = %v, want 'original'", initialExtra.Header["X-Custom"])
}
// Patch with partial Extra - only update some fields
patchReq := &base.Request{
Extra: &http.ReqExtra{
Method: "POST", // Update method
// Body is empty, should NOT update
Header: map[string]string{
"X-Custom": "modified", // Overwrite existing
"X-New": "added", // Add new
// Authorization is not in patch, should remain
},
},
}
err = f.Patch(patchReq, nil)
if err != nil {
t.Fatal(err)
}
// Verify Extra was merged correctly
patchedExtra := f.meta.Req.Extra.(*http.ReqExtra)
// Method should be updated
if patchedExtra.Method != "POST" {
t.Errorf("Patched Method = %v, want POST", patchedExtra.Method)
}
// Body should remain unchanged (patch had empty body)
if patchedExtra.Body != "initial body" {
t.Errorf("Patched Body = %v, want 'initial body' (should remain unchanged)", patchedExtra.Body)
}
// Authorization header should remain unchanged
if patchedExtra.Header["Authorization"] != "Bearer token123" {
t.Errorf("Authorization header = %v, want 'Bearer token123' (should remain unchanged)", patchedExtra.Header["Authorization"])
}
// X-Custom header should be overwritten
if patchedExtra.Header["X-Custom"] != "modified" {
t.Errorf("X-Custom header = %v, want 'modified'", patchedExtra.Header["X-Custom"])
}
// X-New header should be added
if patchedExtra.Header["X-New"] != "added" {
t.Errorf("X-New header = %v, want 'added'", patchedExtra.Header["X-New"])
}
}
// TestFetcher_Patch_NilData tests that Patch with nil data doesn't cause errors
func TestFetcher_Patch_NilData(t *testing.T) {
listener := test.StartTestFileServer()
defer listener.Close()
f := buildFetcher()
opts := &base.Options{
Name: test.DownloadName,
Path: test.Dir,
Extra: &http.OptsExtra{Connections: 1},
}
err := f.Resolve(&base.Request{
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
}, opts)
if err != nil {
t.Fatal(err)
}
originalURL := f.meta.Req.URL
// Patch with nil data - should not cause error
err = f.Patch(nil, nil)
if err != nil {
t.Fatal(err)
}
// Verify URL unchanged
if f.meta.Req.URL != originalURL {
t.Errorf("URL changed after nil patch: got %v, want %v", f.meta.Req.URL, originalURL)
}
// Patch with empty request - should not cause error
err = f.Patch(&base.Request{}, nil)
if err != nil {
t.Fatal(err)
}
// Verify URL still unchanged
if f.meta.Req.URL != originalURL {
t.Errorf("URL changed after empty patch: got %v, want %v", f.meta.Req.URL, originalURL)
}
}
// TestFetcher_Patch_CookieExpired tests the Patch functionality where a download fails
// mid-way due to expired cookie, then succeeds after patching with a new valid cookie.
// This simulates:
// 1. Initial resolve with valid cookie succeeds
// 2. Download starts but fails because cookie expires mid-download (server returns 401)
// 3. User patches the task with a new valid cookie
// 4. Download resumes and completes successfully
func TestFetcher_Patch_CookieExpired(t *testing.T) {
listener := test.StartTestCookieExpiringServer()
defer listener.Close()
downloadURL := "http://" + listener.Addr().String() + "/" + test.BuildName
opts := &base.Options{
Name: test.DownloadName,
Path: test.Dir,
Extra: &http.OptsExtra{Connections: 1},
}
// Step 1: Resolve with old_token - should succeed (first request accepts old_token)
f := buildFetcher()
err := f.Resolve(&base.Request{
URL: downloadURL,
Extra: &http.ReqExtra{
Header: map[string]string{
"Cookie": "session=old_token",
},
},
}, opts)
if err != nil {
t.Fatalf("Resolve should succeed with old_token: %v", err)
}
// Verify initial cookie
initialExtra := f.meta.Req.Extra.(*http.ReqExtra)
if initialExtra.Header["Cookie"] != "session=old_token" {
t.Errorf("Initial Cookie = %v, want session=old_token", initialExtra.Header["Cookie"])
}
// Step 2: Start download - should fail because old_token is now expired
// (server only accepts old_token for first request, subsequent requests need new_token)
err = f.Start()
if err != nil {
t.Fatalf("Start failed: %v", err)
}
err = f.Wait()
// Download should fail with 401 error
if err == nil {
t.Fatal("Expected download to fail with expired cookie, but it succeeded")
}
t.Logf("Download failed as expected: %v", err)
// Step 3: Patch with new valid cookie
err = f.Patch(&base.Request{
Extra: &http.ReqExtra{
Header: map[string]string{
"Cookie": "session=new_token",
},
},
}, nil)
if err != nil {
t.Fatalf("Patch to update cookie failed: %v", err)
}
// Verify cookie was updated
patchedExtra := f.meta.Req.Extra.(*http.ReqExtra)
if patchedExtra.Header["Cookie"] != "session=new_token" {
t.Errorf("Cookie should be updated: got %v, want session=new_token", patchedExtra.Header["Cookie"])
}
// Step 4: Restart download - should succeed with new cookie
err = f.Start()
if err != nil {
t.Fatalf("Restart failed: %v", err)
}
err = f.Wait()
if err != nil {
t.Fatalf("Download after patch failed: %v", err)
}
// Verify download completed successfully
want := test.FileMd5(test.BuildFile)
got := test.FileMd5(test.DownloadFile)
if want != got {
t.Errorf("File MD5 mismatch: got %v, want %v", got, want)
}
}
+73 -5
View File
@@ -65,14 +65,23 @@ func extractRequestError(err error) *RequestError {
return nil
}
// ============================================================================
// HTTP Request/Client Building
// ============================================================================
// buildRequest creates an HTTP request using the redirect URL if available.
func (f *Fetcher) buildRequest(ctx context.Context, req *base.Request) (httpReq *http.Request, err error) {
return f.buildRequestWithURL(ctx, req, true)
}
// buildRequestWithOriginalURL creates an HTTP request using the original URL.
// This is used for retrying when the redirect URL has expired.
func (f *Fetcher) buildRequestWithOriginalURL(ctx context.Context, req *base.Request) (httpReq *http.Request, err error) {
return f.buildRequestWithURL(ctx, req, false)
}
// buildRequestWithURL creates an HTTP request.
// If useRedirect is true and a redirect URL exists, it will be used; otherwise the original URL is used.
func (f *Fetcher) buildRequestWithURL(ctx context.Context, req *base.Request, useRedirect bool) (httpReq *http.Request, err error) {
var reqUrl string
f.redirectLock.Lock()
if f.redirectURL != "" {
if useRedirect && f.redirectURL != "" {
reqUrl = f.redirectURL
} else {
reqUrl = req.URL
@@ -121,6 +130,65 @@ func (f *Fetcher) buildRequest(ctx context.Context, req *base.Request) (httpReq
return httpReq, nil
}
// updateRedirectURL updates the redirect URL from the response.
// This is called when a request using the original URL succeeds after the redirect URL expired.
func (f *Fetcher) updateRedirectURL(url string) {
f.redirectLock.Lock()
f.redirectURL = url
f.redirectLock.Unlock()
}
// hasRedirectURL checks if a redirect URL exists and is different from the original URL.
func (f *Fetcher) hasRedirectURL() bool {
f.redirectLock.Lock()
defer f.redirectLock.Unlock()
return f.redirectURL != "" && f.redirectURL != f.meta.Req.URL
}
// isRedirectExpiredError checks if the error indicates that the redirect URL may have expired.
// This includes 403 (Forbidden), 401 (Unauthorized), 410 (Gone), and network errors.
func isRedirectExpiredError(err error) bool {
if err == nil {
return false
}
// Check for specific HTTP error codes that might indicate URL expiration
if re := extractRequestError(err); re != nil {
switch re.Code {
case 401, 403, 404, 410:
return true
}
}
return false
}
// tryFallbackToOriginalURL attempts to make a request using the original URL
// when the redirect URL has expired. Returns the response if successful.
func (f *Fetcher) tryFallbackToOriginalURL(ctx context.Context, client *http.Client, rangeStart, rangeEnd int64) (*http.Response, error) {
httpReq, err := f.buildRequestWithOriginalURL(ctx, f.meta.Req)
if err != nil {
return nil, err
}
if f.meta.Res.Range && rangeEnd > 0 {
httpReq.Header.Set(base.HttpHeaderRange,
fmt.Sprintf(base.HttpHeaderRangeFormat, rangeStart, rangeEnd))
}
resp, err := client.Do(httpReq)
if err != nil {
return nil, err
}
if resp.StatusCode != base.HttpCodeOK && resp.StatusCode != base.HttpCodePartialContent {
resp.Body.Close()
return nil, NewRequestError(resp.StatusCode)
}
return resp, nil
}
// buildClient creates an HTTP client with the default connection timeout.
// Used for resolve phase where we don't have connection time data yet.
func (f *Fetcher) buildClient() *http.Client {
+244 -19
View File
@@ -335,9 +335,106 @@ func StartTestOneTimeServer() net.Listener {
})
}
// StartTestSlowStartServer creates a server with configurable delay per request
// This allows testing slow-start connection expansion to reach max connections
func StartTestSlowStartServer(delayPerByte time.Duration) net.Listener {
// StartTestNoRangeSlowServer creates a server that always returns the full file
// with Content-Length but does not support Range requests.
func StartTestNoRangeSlowServer(delayPerChunk time.Duration) net.Listener {
return startTestServer(func(sl *shutdownListener) http.Handler {
mux := http.NewServeMux()
mux.HandleFunc("/"+BuildName, func(writer http.ResponseWriter, request *http.Request) {
writer.Header().Set("Content-Length", fmt.Sprintf("%d", BuildSize))
file, err := os.Open(BuildFile)
if err != nil {
return
}
defer file.Close()
buf := make([]byte, 256*1024)
for !sl.isShutdown {
n, readErr := file.Read(buf)
if n > 0 {
if _, writeErr := writer.Write(buf[:n]); writeErr != nil {
return
}
if flusher, ok := writer.(http.Flusher); ok {
flusher.Flush()
}
if delayPerChunk > 0 {
time.Sleep(delayPerChunk)
}
}
if readErr != nil {
return
}
}
})
return mux
})
}
// StartTestExpiringRedirectServer creates a server that simulates expiring redirect URLs.
// The original URL redirects to a temporary URL that expires after a specified number of requests.
// When the temporary URL expires (returns 403), the client should retry with the original URL
// to get a new redirect URL.
// Parameters:
// - requestsBeforeExpire: number of requests the temporary URL accepts before expiring
// - delayPerByte: optional delay per byte for slow transfer (use 0 for no delay)
func StartTestExpiringRedirectServer(requestsBeforeExpire int32, delayPerByte time.Duration) net.Listener {
var redirectVersion atomic.Int32
var requestCount atomic.Int32
redirectVersion.Store(1)
return startTestServer(func(sl *shutdownListener) http.Handler {
return http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
path := request.URL.Path
// Original URL - redirects to current version of temporary URL
if path == "/"+BuildName {
// Reset request count when original URL is accessed
requestCount.Store(0)
// Redirect to versioned temporary URL
version := redirectVersion.Load()
redirectURL := fmt.Sprintf("/redirect-v%d/%s", version, BuildName)
http.Redirect(writer, request, redirectURL, http.StatusFound)
return
}
// Temporary URL handler - matches /redirect-v{N}/... pattern
if strings.HasPrefix(path, "/redirect-v") {
// Check if the redirect has expired
count := requestCount.Add(1)
if count > requestsBeforeExpire {
// Redirect expired - increment version for next redirect
redirectVersion.Add(1)
writer.WriteHeader(403)
writer.Write([]byte("Redirect URL expired"))
return
}
// Serve the file with range support
rangeFileHandle(
writer,
request,
nil,
func(file *os.File, n int64) {
if delayPerByte > 0 {
slowCopyNWithDelay(sl, writer, file, n, delayPerByte)
} else {
io.CopyN(writer, file, n)
}
},
)
return
}
// Not found
writer.WriteHeader(404)
})
})
}
// StartTestLowSpeedServer creates a server with configurable delay per request
func StartTestLowSpeedServer(delayPerByte time.Duration) net.Listener {
return startTestServer(func(sl *shutdownListener) http.Handler {
mux := http.NewServeMux()
mux.HandleFunc("/"+BuildName, func(writer http.ResponseWriter, request *http.Request) {
@@ -386,10 +483,9 @@ func slowCopyNWithDelay(sl *shutdownListener, dst io.Writer, src io.Reader, n in
}
}
// StartTestLimitServer connections limit server
func StartTestLimitServer(maxConnections int32, delay int64) net.Listener {
// StartTestConLimitServer connections limit server
func StartTestConLimitServer(maxConnections int32) net.Listener {
var connections atomic.Int32
var slowOnce atomic.Bool
return startTestServer(func(sl *shutdownListener) http.Handler {
mux := http.NewServeMux()
@@ -402,19 +498,11 @@ func StartTestLimitServer(maxConnections int32, delay int64) net.Listener {
writer.WriteHeader(403)
return
}
// First request intentionally delays the first write to trigger a single read timeout,
// subsequent requests respond at normal speed.
useInitialDelay := delay > 0 && !slowOnce.Swap(true)
rangeFileHandle(
writer,
request,
nil,
func(file *os.File, n int64) {
if useInitialDelay {
slowCopyAfterDelay(sl, writer, file, n, delay)
return
}
slowCopyN(sl, writer, file, n, 0)
},
)
@@ -496,6 +584,57 @@ func StartTestTemporary500Server(errorDuration time.Duration) net.Listener {
})
}
// StartTestFailThenRecoverServer creates a server that fails all connections initially,
// then recovers after a specified number of failed requests.
// This tests the retry functionality when calling Start() again after a download fails.
// Parameters:
// - failedRequestsBeforeRecover: number of requests that will fail before server recovers
//
// Note: Uses 416 (Range Not Satisfiable) instead of 500 because 5xx errors are exempt
// from failure counting and will retry indefinitely. 416 is counted as a failure.
func StartTestFailThenRecoverServer(failedRequestsBeforeRecover int32) net.Listener {
var requestCount atomic.Int32
return startTestServer(func(sl *shutdownListener) http.Handler {
mux := http.NewServeMux()
mux.HandleFunc("/"+BuildName, func(writer http.ResponseWriter, request *http.Request) {
reqNum := requestCount.Add(1)
// First request (Resolve) always succeeds
if reqNum == 1 {
rangeFileHandle(
writer,
request,
nil,
func(file *os.File, n int64) {
io.CopyN(writer, file, n)
},
)
return
}
// Subsequent requests fail until we've had enough failures
// Use 416 Range Not Satisfiable - this error is counted towards failure limit
if reqNum <= failedRequestsBeforeRecover+1 { // +1 because first request is resolve
writer.WriteHeader(416)
writer.Write([]byte("Range Not Satisfiable"))
return
}
// After enough failures, server recovers
rangeFileHandle(
writer,
request,
nil,
func(file *os.File, n int64) {
io.CopyN(writer, file, n)
},
)
})
return mux
})
}
// StartTestRangeBugServer simulate bug server:
// Don't follow Range request rules, always return more data than range, e.g. Range: bytes=0-100, return 150 bytes
func StartTestRangeBugServer() net.Listener {
@@ -562,10 +701,13 @@ func rangeFileHandle(writer http.ResponseWriter, request *http.Request, modifyEn
writer.WriteHeader(400)
return
}
end, err := strconv.ParseInt(s[1], 10, 64)
if err != nil {
writer.WriteHeader(400)
return
end := int64(BuildSize - 1)
if s[1] != "" {
end, err = strconv.ParseInt(s[1], 10, 64)
if err != nil {
writer.WriteHeader(400)
return
}
}
if start < 0 || end < 0 || start > end {
writer.WriteHeader(400)
@@ -731,7 +873,7 @@ func StartSocks5Server(usr, pwd string) net.Listener {
panic(err)
}
listener, err := net.Listen("tcp", "127.0.0.1:0") // 你可以根据需要更改监听地址
listener, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
panic(err)
}
@@ -739,6 +881,89 @@ func StartSocks5Server(usr, pwd string) net.Listener {
return listener
}
// StartTestPatchURLServer creates a server with two endpoints:
// - /bad-url: always returns 404 (simulates a broken download link)
// - /good-url: returns the file successfully (simulates a working download link)
// This is used to test the Patch functionality where a failed download URL can be
// replaced with a working one.
func StartTestPatchURLServer() net.Listener {
return startTestServer(func(sl *shutdownListener) http.Handler {
mux := http.NewServeMux()
// Bad URL - always fails with 404
mux.HandleFunc("/bad-url", func(writer http.ResponseWriter, request *http.Request) {
writer.WriteHeader(404)
writer.Write([]byte("Not Found"))
})
// Good URL - returns the file successfully with range support
mux.HandleFunc("/good-url", func(writer http.ResponseWriter, request *http.Request) {
rangeFileHandle(
writer,
request,
nil,
func(file *os.File, n int64) {
io.CopyN(writer, file, n)
},
)
})
return mux
})
}
// StartTestCookieExpiringServer creates a server that simulates cookie expiration during download.
// - First request (Resolve): accepts "session=old_token" and succeeds
// - Subsequent requests: "session=old_token" is expired (returns 401), only "session=new_token" works
// This is used to test the Patch functionality where a cookie expires mid-download,
// requiring the user to patch with a new cookie to resume.
func StartTestCookieExpiringServer() net.Listener {
var requestCount atomic.Int32
return startTestServer(func(sl *shutdownListener) http.Handler {
mux := http.NewServeMux()
mux.HandleFunc("/"+BuildName, func(writer http.ResponseWriter, request *http.Request) {
reqNum := requestCount.Add(1)
cookie := request.Header.Get("Cookie")
// First request (Resolve): accept old_token
if reqNum == 1 {
if !strings.Contains(cookie, "session=old_token") {
writer.WriteHeader(401)
writer.Write([]byte("Unauthorized: Invalid cookie"))
return
}
// Return file info for resolve (with range support)
rangeFileHandle(
writer,
request,
nil,
func(file *os.File, n int64) {
io.CopyN(writer, file, n)
},
)
return
}
// Subsequent requests: old_token is expired, only new_token works
if strings.Contains(cookie, "session=new_token") {
// New token is valid - return the file with range support
rangeFileHandle(
writer,
request,
nil,
func(file *os.File, n int64) {
io.CopyN(writer, file, n)
},
)
return
}
// Old token or invalid token - return 401
writer.WriteHeader(401)
writer.Write([]byte("Unauthorized: Cookie expired"))
})
return mux
})
}
func AssertResourceEqual(want, got *base.Resource) bool {
// Ignore ctime
if got != nil && len(got.Files) > 0 {
@@ -0,0 +1,36 @@
//go:build darwin
package goprovider
var mainThreadTasks chan func()
func RunMainThreadLoop(run func() int) int {
if mainThreadTasks != nil {
return run()
}
mainThreadTasks = make(chan func())
resultCh := make(chan int, 1)
go func() {
resultCh <- run()
}()
for {
select {
case task := <-mainThreadTasks:
task()
case code := <-resultCh:
mainThreadTasks = nil
return code
}
}
}
func postMainThreadTask(task func()) bool {
if mainThreadTasks == nil {
return false
}
mainThreadTasks <- task
return true
}
@@ -0,0 +1,11 @@
//go:build !darwin
package goprovider
func RunMainThreadLoop(run func() int) int {
return run()
}
func postMainThreadTask(func()) bool {
return false
}
+608
View File
@@ -0,0 +1,608 @@
//go:build cgo
package goprovider
import (
"encoding/json"
"errors"
"fmt"
"net/url"
"strings"
"sync"
"sync/atomic"
"time"
runtimepkg "runtime"
enginewebview "github.com/GopeedLab/gopeed/pkg/download/engine/webview"
webview "github.com/GopeedLab/webview_go"
)
type Provider struct{}
func New() enginewebview.Provider {
return &Provider{}
}
func (p *Provider) IsAvailable() bool {
return webview.IsAvailable()
}
func (p *Provider) Open(opts enginewebview.OpenOptions) (enginewebview.Page, error) {
pw := newPageWrapper(opts)
if err := pw.start(); err != nil {
return nil, err
}
return pw, nil
}
type pageWrapper struct {
opts enginewebview.OpenOptions
callbackName string
readyID string
view webview.WebView
mu sync.Mutex
pending map[string]chan evalResult
url string
closed bool
ready chan error
done chan struct{}
loads chan string
}
type evalMessage struct {
ID string `json:"id"`
Ready bool `json:"ready,omitempty"`
Load bool `json:"load,omitempty"`
URL string `json:"url,omitempty"`
OK bool `json:"ok,omitempty"`
Value any `json:"value,omitempty"`
Error string `json:"error,omitempty"`
}
type evalResult struct {
value any
err error
}
type nativeCookieJSON struct {
Name string `json:"name"`
Value string `json:"value"`
Domain string `json:"domain"`
Path string `json:"path"`
Expires string `json:"expires"`
Secure bool `json:"secure"`
HTTPOnly bool `json:"httpOnly"`
}
var providerSeq atomic.Uint64
func newPageWrapper(opts enginewebview.OpenOptions) *pageWrapper {
seq := providerSeq.Add(1)
return &pageWrapper{
opts: opts,
callbackName: fmt.Sprintf("__gopeedWebViewCallback_%d", seq),
readyID: fmt.Sprintf("__ready__%d", seq),
pending: make(map[string]chan evalResult),
ready: make(chan error, 1),
done: make(chan struct{}),
loads: make(chan string, 8),
}
}
func (p *pageWrapper) start() error {
startedAt := time.Now()
run := func() {
runtimepkg.LockOSThread()
defer runtimepkg.UnlockOSThread()
var w webview.WebView
if p.opts.Headless {
w = webview.NewHeadless(p.opts.Debug)
} else {
w = webview.New(p.opts.Debug)
}
p.view = w
w.SetTitle(firstNonEmpty(p.opts.Title, "Gopeed WebView"))
w.SetSize(defaultWindowDimension(p.opts.Width, 1280), defaultWindowDimension(p.opts.Height, 800), webview.HintNone)
if p.opts.UserAgent != "" {
type userAgentSetter interface {
SetUserAgent(userAgent string)
}
if setter, ok := w.(userAgentSetter); ok {
setter.SetUserAgent(p.opts.UserAgent)
}
}
w.Init(buildLoadNotifyScript(p.callbackName))
if err := w.Bind(p.callbackName, func(payload string) error {
return p.handleCallback(payload)
}); err != nil {
p.ready <- err
return
}
applyWindowOptions(w, p.opts)
w.SetHtml(buildBootstrapHTML(p.callbackName, p.readyID))
w.Run()
w.Destroy()
close(p.done)
}
if !postMainThreadTask(run) {
go run()
}
select {
case err := <-p.ready:
p.tracef("open %dms", time.Since(startedAt).Milliseconds())
return err
case <-time.After(10 * time.Second):
p.tracef("open timeout %dms", time.Since(startedAt).Milliseconds())
return fmt.Errorf("webview startup timeout")
}
}
func (p *pageWrapper) AddInitScript(script string) error {
return p.dispatch(func(w webview.WebView) error {
w.Init(script)
return nil
})
}
func (p *pageWrapper) Goto(url string, opts enginewebview.GotoOptions) error {
startedAt := time.Now()
p.drainLoads()
if err := p.dispatch(func(w webview.WebView) error {
p.mu.Lock()
p.url = url
p.mu.Unlock()
w.Navigate(url)
return nil
}); err != nil {
return err
}
timeout := 30 * time.Second
if opts.TimeoutMS > 0 {
timeout = time.Duration(opts.TimeoutMS) * time.Millisecond
}
waitUntil := opts.WaitUntil
if waitUntil == "" {
waitUntil = "load"
}
err := p.waitForNavigation(url, timeout, waitUntil)
if err != nil {
p.tracef("goto failed %dms url=%s err=%v", time.Since(startedAt).Milliseconds(), url, err)
return err
}
p.tracef("goto %dms url=%s", time.Since(startedAt).Milliseconds(), url)
return nil
}
func (p *pageWrapper) Execute(expression string, args ...any) (any, error) {
startedAt := time.Now()
if args == nil {
args = []any{}
}
argsJSON, err := json.Marshal(args)
if err != nil {
return nil, err
}
reqID := fmt.Sprintf("req-%d", providerSeq.Add(1))
ch := make(chan evalResult, 1)
p.mu.Lock()
if p.closed {
p.mu.Unlock()
return nil, fmt.Errorf("webview page is closed")
}
p.pending[reqID] = ch
p.mu.Unlock()
script := buildExecuteScript(p.callbackName, reqID, expression, string(argsJSON))
if err := p.dispatch(func(w webview.WebView) error {
w.Eval(script)
return nil
}); err != nil {
p.mu.Lock()
delete(p.pending, reqID)
p.mu.Unlock()
return nil, err
}
result := <-ch
if result.err != nil {
p.tracef("execute failed %dms err=%v", time.Since(startedAt).Milliseconds(), result.err)
return result.value, result.err
}
p.tracef("execute %dms", time.Since(startedAt).Milliseconds())
return result.value, nil
}
func (p *pageWrapper) GetCookies() ([]enginewebview.Cookie, error) {
cookies, err := p.nativeCookies()
if err != nil {
return nil, err
}
result := make([]enginewebview.Cookie, 0, len(cookies))
for _, cookie := range cookies {
item := enginewebview.Cookie{
Name: cookie.Name,
Value: cookie.Value,
Domain: cookie.Domain,
Path: cookie.Path,
Secure: cookie.Secure,
HTTPOnly: cookie.HTTPOnly,
}
if cookie.Expires != "" {
if parsed, err := time.Parse(time.RFC3339Nano, cookie.Expires); err == nil {
item.Expires = parsed
}
}
result = append(result, item)
}
return result, nil
}
func (p *pageWrapper) SetCookie(cookie enginewebview.Cookie) error {
type cookieSetter interface {
SetCookie(cookie webview.Cookie) error
}
return p.dispatch(func(w webview.WebView) error {
setter, ok := w.(cookieSetter)
if !ok {
return fmt.Errorf("native cookie access is unavailable")
}
return setter.SetCookie(webview.Cookie{
Name: cookie.Name,
Value: cookie.Value,
Domain: cookie.Domain,
Path: cookie.Path,
Expires: cookie.Expires,
Secure: cookie.Secure,
HTTPOnly: cookie.HTTPOnly,
})
})
}
func (p *pageWrapper) DeleteCookie(cookie enginewebview.Cookie) error {
type cookieDeleter interface {
DeleteCookie(name string, domain string, path string) error
}
return p.dispatch(func(w webview.WebView) error {
deleter, ok := w.(cookieDeleter)
if !ok {
return fmt.Errorf("native cookie access is unavailable")
}
return deleter.DeleteCookie(cookie.Name, cookie.Domain, cookie.Path)
})
}
func (p *pageWrapper) ClearCookies() error {
type cookieClearer interface {
ClearCookies() error
}
return p.dispatch(func(w webview.WebView) error {
clearer, ok := w.(cookieClearer)
if !ok {
return fmt.Errorf("native cookie access is unavailable")
}
return clearer.ClearCookies()
})
}
func (p *pageWrapper) Close() error {
p.mu.Lock()
if p.closed {
p.mu.Unlock()
return nil
}
p.closed = true
w := p.view
wait := p.done
p.mu.Unlock()
if w != nil {
// PostQuitMessage (used by the Windows WebView2 terminate_impl)
// only takes effect on the thread that owns the message loop.
// Dispatch the termination to the webview's UI thread so that
// the quit message lands on the correct queue.
terminateDone := make(chan struct{})
w.Dispatch(func() {
w.Terminate()
close(terminateDone)
})
<-terminateDone
}
<-wait
return nil
}
func (p *pageWrapper) tracef(format string, args ...any) {
if !strings.Contains(p.opts.Title, "youtube-sabr") {
return
}
fmt.Printf("goprovider "+format+"\n", args...)
}
func (p *pageWrapper) waitForNavigation(targetURL string, timeout time.Duration, waitUntil string) error {
deadline := time.Now().Add(timeout)
ticker := time.NewTicker(100 * time.Millisecond)
defer ticker.Stop()
for {
select {
case <-p.loads:
if waitUntil == "load" {
return nil
}
case <-ticker.C:
state, err := p.navigationState()
if err == nil && state.ready(targetURL, waitUntil) {
return nil
}
if time.Now().After(deadline) {
return fmt.Errorf("webview navigation timeout")
}
}
}
}
type navigationState struct {
URL string `json:"url"`
ReadyState string `json:"readyState"`
}
func (s navigationState) ready(targetURL string, waitUntil string) bool {
if s.URL == "" || s.URL == "about:blank" {
return false
}
switch waitUntil {
case "domcontentloaded":
if s.ReadyState == "loading" || s.ReadyState == "" {
return false
}
default:
if s.ReadyState != "complete" {
return false
}
}
if targetURL == "" {
return true
}
if s.URL == targetURL {
return true
}
target, err := parseURL(targetURL)
if err != nil {
return true
}
current, err := parseURL(s.URL)
if err != nil {
return true
}
if current.Scheme == target.Scheme && current.Host == target.Host && current.Path == target.Path {
return true
}
return false
}
func parseURL(raw string) (*url.URL, error) {
return url.Parse(raw)
}
func (p *pageWrapper) navigationState() (navigationState, error) {
value, err := p.Execute(`() => ({
url: String(location.href || ""),
readyState: document.readyState || "",
})`)
if err != nil {
return navigationState{}, err
}
stateMap, ok := value.(map[string]any)
if !ok {
return navigationState{}, fmt.Errorf("invalid navigation state")
}
state := navigationState{}
if urlValue, ok := stateMap["url"].(string); ok {
state.URL = urlValue
}
if readyValue, ok := stateMap["readyState"].(string); ok {
state.ReadyState = readyValue
}
return state, nil
}
func (p *pageWrapper) dispatch(fn func(w webview.WebView) error) error {
p.mu.Lock()
if p.closed {
p.mu.Unlock()
return fmt.Errorf("webview page is closed")
}
w := p.view
p.mu.Unlock()
if w == nil {
return fmt.Errorf("webview is not initialized")
}
done := make(chan error, 1)
w.Dispatch(func() {
done <- fn(w)
})
return <-done
}
func (p *pageWrapper) nativeCookies() ([]nativeCookieJSON, error) {
type cookieGetter interface {
GetCookies(url string) ([]webview.Cookie, error)
}
var cookies []webview.Cookie
err := p.dispatch(func(w webview.WebView) error {
getter, ok := w.(cookieGetter)
if !ok {
return fmt.Errorf("native cookie access is unavailable")
}
p.mu.Lock()
url := p.url
p.mu.Unlock()
if url == "" {
url = "about:blank"
}
var err error
cookies, err = getter.GetCookies(url)
return err
})
if err != nil {
return nil, err
}
result := make([]nativeCookieJSON, 0, len(cookies))
for _, cookie := range cookies {
item := nativeCookieJSON{
Name: cookie.Name,
Value: cookie.Value,
Domain: cookie.Domain,
Path: cookie.Path,
Secure: cookie.Secure,
HTTPOnly: cookie.HTTPOnly,
}
if !cookie.Expires.IsZero() {
item.Expires = cookie.Expires.UTC().Format(time.RFC3339Nano)
}
result = append(result, item)
}
return result, nil
}
func (p *pageWrapper) handleCallback(payload string) error {
var msg evalMessage
if err := json.Unmarshal([]byte(payload), &msg); err != nil {
return err
}
if msg.Ready && msg.ID == p.readyID {
select {
case p.ready <- nil:
default:
}
return nil
}
if msg.Load {
select {
case p.loads <- msg.URL:
default:
}
return nil
}
p.mu.Lock()
ch := p.pending[msg.ID]
delete(p.pending, msg.ID)
p.mu.Unlock()
if ch == nil {
return nil
}
if !msg.OK {
ch <- evalResult{err: errors.New(strings.TrimSpace(msg.Error))}
return nil
}
ch <- evalResult{value: msg.Value}
return nil
}
func (p *pageWrapper) drainLoads() {
for {
select {
case <-p.loads:
default:
return
}
}
}
func buildBootstrapHTML(callbackName string, readyID string) string {
return fmt.Sprintf(`<!doctype html>
<html>
<head><meta charset="utf-8"></head>
<body>
<script>
window.addEventListener("load", () => {
const callback = globalThis[%q];
if (typeof callback === "function") {
callback(JSON.stringify({ id: %q, ready: true }));
}
}, { once: true });
</script>
</body>
</html>`, callbackName, readyID)
}
func buildExecuteScript(callbackName string, reqID string, expression string, argsJSON string) string {
callbackJSON, _ := json.Marshal(callbackName)
reqJSON, _ := json.Marshal(reqID)
exprJSON, _ := json.Marshal(expression)
argsLiteral := argsJSON
if argsLiteral == "" || argsLiteral == "null" {
argsLiteral = "[]"
}
return fmt.Sprintf(`(() => {
const __cb = globalThis[%s];
const __id = %s;
const __expression = %s;
const __args = %s;
Promise.resolve()
.then(async () => {
const __target = (0, eval)(__expression);
const __value = await (typeof __target === "function" ? __target(...__args) : __target);
return { id: __id, ok: true, value: __value ?? null };
})
.catch((error) => ({
id: __id,
ok: false,
error: [
error && error.name ? error.name : "Error",
error && error.message ? error.message : String(error),
error && error.stack ? error.stack : "",
].filter(Boolean).join(": ").replace(": @", "\n@"),
}))
.then((message) => __cb(JSON.stringify(message)));
})();`, string(callbackJSON), string(reqJSON), string(exprJSON), argsLiteral)
}
func buildLoadNotifyScript(callbackName string) string {
callbackJSON, _ := json.Marshal(callbackName)
return fmt.Sprintf(`(() => {
window.addEventListener("load", () => {
const __cb = globalThis[%s];
if (typeof __cb !== "function") {
return;
}
__cb(JSON.stringify({
load: true,
url: String(location.href || ""),
}));
}, { once: true });
})();`, string(callbackJSON))
}
func defaultWindowDimension(value int, fallback int) int {
if value > 0 {
return value
}
return fallback
}
func firstNonEmpty(values ...string) string {
for _, value := range values {
if strings.TrimSpace(value) != "" {
return value
}
}
return ""
}
@@ -0,0 +1,15 @@
//go:build cgo && webview && !darwin
package goprovider
import (
"testing"
"github.com/GopeedLab/gopeed/internal/webview/integrationtest"
)
func TestProviderContract(t *testing.T) {
integrationtest.RunProviderContract(t, New(), integrationtest.ContractOptions{
CookieDomainMode: integrationtest.CookieDomainModeRequired,
})
}
@@ -0,0 +1,9 @@
//go:build !cgo
package goprovider
import enginewebview "github.com/GopeedLab/gopeed/pkg/download/engine/webview"
func New() enginewebview.Provider {
return enginewebview.NewUnavailableProvider()
}
@@ -0,0 +1,38 @@
//go:build cgo && darwin
package goprovider
/*
#cgo darwin CFLAGS: -x objective-c
#cgo darwin LDFLAGS: -framework Cocoa
#include <Cocoa/Cocoa.h>
static void gopeedApplyHeadlessWindowOptions(void *windowPtr, int headless) {
if (windowPtr == NULL) {
return;
}
NSWindow *window = (__bridge NSWindow *)windowPtr;
if (headless) {
[window orderOut:nil];
}
}
*/
import "C"
import (
enginewebview "github.com/GopeedLab/gopeed/pkg/download/engine/webview"
webview "github.com/GopeedLab/webview_go"
)
func applyWindowOptions(w webview.WebView, opts enginewebview.OpenOptions) {
C.gopeedApplyHeadlessWindowOptions(w.Window(), C.int(boolToInt(opts.Headless)))
}
func boolToInt(value bool) int {
if value {
return 1
}
return 0
}
@@ -0,0 +1,10 @@
//go:build cgo && !darwin
package goprovider
import (
enginewebview "github.com/GopeedLab/gopeed/pkg/download/engine/webview"
webview "github.com/GopeedLab/webview_go"
)
func applyWindowOptions(_ webview.WebView, _ enginewebview.OpenOptions) {}
@@ -0,0 +1,378 @@
package integrationtest
import (
"fmt"
"net/http"
"net/http/httptest"
"net/url"
"strings"
"testing"
"time"
enginewebview "github.com/GopeedLab/gopeed/pkg/download/engine/webview"
)
const TestUserAgent = "GopeedWebViewIntegration/1.0"
type ContractOptions struct {
AvailabilityTimeout time.Duration
CookieDomainMode CookieDomainMode
CookieTestURL string
}
type CookieDomainMode int
const (
CookieDomainModeRequired CookieDomainMode = iota
CookieDomainModeOmit
)
func RunProviderContract(t *testing.T, provider enginewebview.Provider, opts ContractOptions) {
t.Helper()
timeout := opts.AvailabilityTimeout
if timeout <= 0 {
timeout = 10 * time.Second
}
waitForProviderAvailable(t, provider, timeout)
page, server, testURL := openTestPage(t, provider, enginewebview.OpenOptions{
Headless: true,
Title: "Gopeed WebView Integration",
Width: 640,
Height: 480,
UserAgent: TestUserAgent,
})
if err := page.Goto(testURL, map[string]any{"timeoutMs": 10000}); err != nil {
t.Fatalf("goto failed: %v", err)
}
ua, err := page.Execute(`() => navigator.userAgent`)
if err != nil {
t.Fatalf("navigator.userAgent execute failed: %v", err)
}
if ua != TestUserAgent {
t.Fatalf("unexpected user agent: got %v want %q", ua, TestUserAgent)
}
if err := page.Focus("#name"); err != nil {
t.Fatalf("focus failed: %v", err)
}
activeElement, err := page.Execute(`() => document.activeElement && document.activeElement.id`)
if err != nil {
t.Fatalf("activeElement execute failed: %v", err)
}
if activeElement != "name" {
t.Fatalf("expected active element to be #name, got %v", activeElement)
}
if err := page.Type("#name", "Gopeed", map[string]any{"delay": 5}); err != nil {
t.Fatalf("type failed: %v", err)
}
typeState, err := page.Execute(`() => ({
value: document.querySelector('#name').value,
mirror: document.querySelector('#name-value').textContent,
inputCount: window.__inputEvents.length
})`)
if err != nil {
t.Fatalf("type state execute failed: %v", err)
}
stateMap, ok := typeState.(map[string]any)
if !ok {
t.Fatalf("unexpected type state payload: %T", typeState)
}
if stateMap["value"] != "Gopeed" || stateMap["mirror"] != "Gopeed" {
t.Fatalf("unexpected type state: %#v", stateMap)
}
if stateMap["inputCount"] != float64(len("Gopeed")) {
t.Fatalf("unexpected input event count: %#v", stateMap["inputCount"])
}
if err := page.Click("#clicker"); err != nil {
t.Fatalf("click failed: %v", err)
}
clickCount, err := page.Execute(`() => document.querySelector('#click-count').textContent`)
if err != nil {
t.Fatalf("click count execute failed: %v", err)
}
if clickCount != "1" {
t.Fatalf("expected click count to be 1, got %v", clickCount)
}
found, err := page.WaitForSelector("#delayed", map[string]any{
"timeoutMs": 5000,
"pollIntervalMs": 25,
"visible": true,
})
if err != nil {
t.Fatalf("waitForSelector failed: %v", err)
}
if !found {
t.Fatal("expected delayed selector to appear")
}
value, err := page.WaitForFunction(`() => window.__readyValue`, map[string]any{
"timeoutMs": 5000,
"pollIntervalMs": 25,
})
if err != nil {
t.Fatalf("waitForFunction failed: %v", err)
}
readyValue, ok := value.(map[string]any)
if !ok {
t.Fatalf("unexpected waitForFunction value: %T", value)
}
if readyValue["status"] != "ready" {
t.Fatalf("unexpected waitForFunction payload: %#v", readyValue)
}
found, err = page.WaitForSelector("#missing", map[string]any{
"timeoutMs": 200,
"pollIntervalMs": 25,
})
if err != nil {
t.Fatalf("waitForSelector timeout case failed: %v", err)
}
if found {
t.Fatal("expected missing selector wait to time out")
}
cookieTestURL := opts.CookieTestURL
if cookieTestURL == "" {
cookieTestURL = testURL
}
runCookieLifecycleAssertions(t, page, server, cookieTestURL, opts.CookieDomainMode)
}
func waitForProviderAvailable(t *testing.T, provider enginewebview.Provider, timeout time.Duration) {
t.Helper()
deadline := time.Now().Add(timeout)
for time.Now().Before(deadline) {
if provider != nil && provider.IsAvailable() {
return
}
time.Sleep(200 * time.Millisecond)
}
t.Fatalf("provider did not become available within %s", timeout)
}
func openTestPage(t *testing.T, provider enginewebview.Provider, opts enginewebview.OpenOptions) (*enginewebview.PageHandle, *httptest.Server, string) {
t.Helper()
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/html; charset=utf-8")
switch r.URL.Path {
case "/next":
fmt.Fprint(w, `<!DOCTYPE html>
<html>
<head><title>Next Page</title></head>
<body>
<h1 id="next-page">Navigation complete</h1>
</body>
</html>`)
default:
fmt.Fprint(w, `<!DOCTYPE html>
<html>
<head><title>Test Page</title></head>
<body>
<input id="name" type="text" />
<div id="name-value"></div>
<button id="clicker" type="button">Click me</button>
<div id="click-count">0</div>
<button id="nav-btn" type="button">Navigate</button>
<script>
window.__inputEvents = [];
window.__readyValue = null;
const nameInput = document.getElementById('name');
const nameValue = document.getElementById('name-value');
nameInput.addEventListener('input', () => {
nameValue.textContent = nameInput.value;
window.__inputEvents.push(nameInput.value);
});
document.getElementById('clicker').addEventListener('click', () => {
const counter = document.getElementById('click-count');
counter.textContent = String(Number(counter.textContent || '0') + 1);
});
document.getElementById('nav-btn').addEventListener('click', () => {
setTimeout(() => {
window.location.href = '/next';
}, 50);
});
setTimeout(() => {
const marker = document.createElement('div');
marker.id = 'delayed';
marker.textContent = 'Loaded later';
document.body.appendChild(marker);
}, 150);
setTimeout(() => {
window.__readyValue = { status: 'ready', source: 'timer' };
}, 200);
</script>
</body>
</html>`)
}
}))
t.Cleanup(server.Close)
runtime := enginewebview.NewRuntime(provider, true)
page, err := runtime.Open(map[string]any{
"headless": opts.Headless,
"debug": opts.Debug,
"title": opts.Title,
"width": opts.Width,
"height": opts.Height,
"userAgent": opts.UserAgent,
})
if err != nil {
t.Fatalf("failed to open webview: %v", err)
}
t.Cleanup(func() {
_ = runtime.Close()
})
return page, server, server.URL
}
func runCookieLifecycleAssertions(t *testing.T, page *enginewebview.PageHandle, _ *httptest.Server, testURL string, domainMode CookieDomainMode) {
t.Helper()
parsedURL, err := url.Parse(testURL)
if err != nil {
t.Fatalf("parse cookie test url failed: %v", err)
}
cookieDomain := parsedURL.Hostname()
if err := page.ClearCookies(); err != nil {
t.Fatalf("clearCookies before cookie assertions failed: %v", err)
}
visibleCookie := map[string]any{
"name": "visible",
"value": "1",
"path": "/",
}
secretCookie := map[string]any{
"name": "secret",
"value": "2",
"path": "/",
"httpOnly": true,
"expires": time.Now().Add(time.Hour).UTC().Format(time.RFC3339Nano),
}
deleteCookie := map[string]any{
"name": "visible",
"path": "/",
}
if domainMode == CookieDomainModeRequired {
visibleCookie["domain"] = cookieDomain
secretCookie["domain"] = cookieDomain
deleteCookie["domain"] = cookieDomain
}
if err := page.SetCookie(visibleCookie); err != nil {
t.Fatalf("set visible cookie failed: %v", err)
}
if err := page.SetCookie(secretCookie); err != nil {
t.Fatalf("set secret cookie failed: %v", err)
}
if err := page.Goto(testURL, map[string]any{"timeoutMs": 10000}); err != nil {
t.Fatalf("goto after setCookie failed: %v", err)
}
documentCookie, err := page.Execute(`() => document.cookie`)
if err != nil {
t.Fatalf("document.cookie execute failed: %v", err)
}
documentCookieText, ok := documentCookie.(string)
if !ok {
t.Fatalf("unexpected document.cookie payload: %T", documentCookie)
}
if !strings.Contains(documentCookieText, "visible=1") {
t.Fatalf("expected visible cookie in document.cookie, got %q", documentCookieText)
}
if strings.Contains(documentCookieText, "secret=2") {
t.Fatalf("expected httpOnly cookie to be hidden from document.cookie, got %q", documentCookieText)
}
cookies, err := page.GetCookies()
if err != nil {
t.Fatalf("getCookies failed: %v", err)
}
if len(cookies) < 2 {
t.Fatalf("expected at least 2 cookies, got %#v", cookies)
}
if !hasCookie(cookies, "visible", "1") {
t.Fatalf("expected visible cookie in native cookie store, got %#v", cookies)
}
if !hasCookie(cookies, "secret", "2") {
t.Fatalf("expected secret cookie in native cookie store, got %#v", cookies)
}
if err := page.DeleteCookie(deleteCookie); err != nil {
t.Fatalf("deleteCookie failed: %v", err)
}
if err := page.Goto(testURL, map[string]any{"timeoutMs": 10000}); err != nil {
t.Fatalf("goto after deleteCookie failed: %v", err)
}
documentCookie, err = page.Execute(`() => document.cookie`)
if err != nil {
t.Fatalf("document.cookie execute after delete failed: %v", err)
}
documentCookieText, ok = documentCookie.(string)
if !ok {
t.Fatalf("unexpected document.cookie payload after delete: %T", documentCookie)
}
if strings.Contains(documentCookieText, "visible=1") {
t.Fatalf("expected visible cookie to be deleted, got %q", documentCookieText)
}
if strings.Contains(documentCookieText, "secret=2") {
t.Fatalf("expected httpOnly cookie to remain hidden after delete, got %q", documentCookieText)
}
cookies, err = page.GetCookies()
if err != nil {
t.Fatalf("getCookies after delete failed: %v", err)
}
if hasCookie(cookies, "visible", "1") {
t.Fatalf("expected visible cookie to be removed from native store, got %#v", cookies)
}
if !hasCookie(cookies, "secret", "2") {
t.Fatalf("expected secret cookie to remain in native store after delete, got %#v", cookies)
}
if err := page.ClearCookies(); err != nil {
t.Fatalf("clearCookies failed: %v", err)
}
if err := page.Goto(testURL, map[string]any{"timeoutMs": 10000}); err != nil {
t.Fatalf("goto after clearCookies failed: %v", err)
}
documentCookie, err = page.Execute(`() => document.cookie`)
if err != nil {
t.Fatalf("document.cookie execute after clear failed: %v", err)
}
documentCookieText, ok = documentCookie.(string)
if !ok {
t.Fatalf("unexpected document.cookie payload after clear: %T", documentCookie)
}
if documentCookieText != "" {
t.Fatalf("expected document.cookie to be empty after clear, got %q", documentCookieText)
}
cookies, err = page.GetCookies()
if err != nil {
t.Fatalf("getCookies after clear failed: %v", err)
}
if len(cookies) != 0 {
t.Fatalf("expected cookie store to be empty after clear, got %#v", cookies)
}
}
func hasCookie(cookies []enginewebview.Cookie, name string, value string) bool {
for _, cookie := range cookies {
if cookie.Name == name && cookie.Value == value {
return true
}
}
return false
}
+488
View File
@@ -0,0 +1,488 @@
# rpcprovider
`rpcprovider` is the RPC-backed implementation of Gopeed's `webview.Provider` interface.
It is intended for environments where WebView capability is owned by another process, most notably mobile hosts. This package does not implement a WebView itself. Instead, it translates the `pkg/download/engine/webview` page API into single-request `POST + JSON` calls sent to a local host-side RPC service.
## When to use it
Use `rpcprovider` for entrypoints such as `bind/mobile`, where the Go process cannot directly own a native WebView.
Do not use it for desktop or `cmd/web` local WebView integration. Those entrypoints should use `internal/webview/goprovider` instead.
## Configuration
The provider is configured through `pkg/download/engine/webview.RPCConfig`:
```go
type RPCConfig struct {
Network string `json:"network"`
Address string `json:"address"`
Token string `json:"token,omitempty"`
}
```
Notes:
- `Network` currently supports `tcp` and `unix`.
- `Address` examples:
- `127.0.0.1:38765`
- `/path/to/webview.sock`
- `Token` is optional. When set, the client sends `Authorization: Bearer <token>`.
Example:
```go
provider := rpcprovider.New(webview.RPCConfig{
Network: "tcp",
Address: "127.0.0.1:38765",
Token: "secret-token",
})
```
## Transport
- Endpoint: `POST /webview`
- Content type: `application/json`
- One request, one response
- No `id` field yet
- The envelope is intentionally close to JSON-RPC so it can evolve later without renaming the methods or reshaping the payloads
Request example:
```json
{
"method": "page.execute",
"params": {
"pageId": "page-1",
"expression": "document.title",
"args": []
}
}
```
Successful response:
```json
{
"result": {
"title": "Example"
},
"error": null
}
```
Failure response:
```json
{
"result": null,
"error": {
"code": "PAGE_NOT_FOUND",
"message": "page not found"
}
}
```
## Supported methods
- `webview.isAvailable`
- `page.open`
- `page.addInitScript`
- `page.goto`
- `page.execute`
- `page.getCookies`
- `page.setCookie`
- `page.deleteCookie`
- `page.clearCookies`
- `page.close`
The canonical wire models for these methods are defined in:
- [pkg/download/engine/webview/rpc.go](../../../pkg/download/engine/webview/rpc.go)
- [pkg/download/engine/webview/runtime.go](../../../pkg/download/engine/webview/runtime.go)
The README is only a guide. The Go source above is the authoritative protocol reference.
## Method reference
Each method below uses the common envelope:
```json
{
"method": "<method-name>",
"params": { ... }
}
```
and returns:
```json
{
"result": ...,
"error": null
}
```
or:
```json
{
"result": null,
"error": {
"code": "<ERROR_CODE>",
"message": "<message>"
}
}
```
### `webview.isAvailable`
Request:
```json
{
"method": "webview.isAvailable",
"params": {}
}
```
Success:
```json
{
"result": {
"available": true
},
"error": null
}
```
### `page.open`
Request:
```json
{
"method": "page.open",
"params": {
"headless": true,
"debug": false,
"title": "Gopeed WebView",
"width": 1280,
"height": 720,
"userAgent": "Mozilla/5.0 ..."
}
}
```
Success:
```json
{
"result": {
"pageId": "page-1"
},
"error": null
}
```
### `page.addInitScript`
Request:
```json
{
"method": "page.addInitScript",
"params": {
"pageId": "page-1",
"script": "window.__READY__ = true;"
}
}
```
Success:
```json
{
"result": {},
"error": null
}
```
### `page.goto`
Request:
```json
{
"method": "page.goto",
"params": {
"pageId": "page-1",
"url": "https://example.com",
"timeoutMs": 15000,
"waitUntil": "domcontentloaded"
}
}
```
Success:
```json
{
"result": {},
"error": null
}
```
`waitUntil` currently supports `load` and `domcontentloaded`. The default is `load`.
### `page.execute`
Request:
```json
{
"method": "page.execute",
"params": {
"pageId": "page-1",
"expression": "document.title",
"args": []
}
}
```
Success:
```json
{
"result": "Example Domain",
"error": null
}
```
### `page.getCookies`
Request:
```json
{
"method": "page.getCookies",
"params": {
"pageId": "page-1"
}
}
```
Success:
```json
{
"result": [
{
"name": "session",
"value": "abc",
"domain": ".example.com",
"path": "/",
"expires": "2026-03-26T12:00:00Z",
"secure": true,
"httpOnly": true
}
],
"error": null
}
```
### `page.setCookie`
Request:
```json
{
"method": "page.setCookie",
"params": {
"pageId": "page-1",
"cookie": {
"name": "session",
"value": "abc",
"domain": ".example.com",
"path": "/",
"expires": "2026-03-26T12:00:00Z",
"secure": true,
"httpOnly": true
}
}
}
```
Success:
```json
{
"result": {},
"error": null
}
```
### `page.deleteCookie`
Request:
```json
{
"method": "page.deleteCookie",
"params": {
"pageId": "page-1",
"cookie": {
"name": "session",
"domain": ".example.com",
"path": "/"
}
}
}
```
Success:
```json
{
"result": {},
"error": null
}
```
### `page.clearCookies`
Request:
```json
{
"method": "page.clearCookies",
"params": {
"pageId": "page-1"
}
}
```
Success:
```json
{
"result": {},
"error": null
}
```
### `page.close`
Request:
```json
{
"method": "page.close",
"params": {
"pageId": "page-1"
}
}
```
Success:
```json
{
"result": {},
"error": null
}
```
## Method semantics
- `page.open`
- Creates a page session
- Returns `pageId`
- `page.addInitScript`
- Registers a page initialization script
- `page.goto`
- Goes to the target URL
- The server should return only after navigation completes or fails
- `page.execute`
- Executes an expression or serialized function body
- The returned value must be JSON-serializable
- `page.getCookies`
- Returns cookies from the underlying native WebView cookie store
- `page.setCookie`
- Inserts or updates a cookie
- `page.deleteCookie`
- Deletes a cookie using `name/domain/path`
- `page.clearCookies`
- Clears the current cookie store
- `page.close`
- Closes the page and releases related resources
## Cookie model
Cookies use `runtime/webview.Cookie`.
Example payload:
```json
{
"name": "session",
"value": "abc",
"domain": ".example.com",
"path": "/",
"expires": "2026-03-26T12:00:00Z",
"secure": true,
"httpOnly": true
}
```
Notes:
- `expires` uses RFC3339 / RFC3339Nano string form.
- If the host platform supports a full native cookie store, it should return fields such as `httpOnly` and `secure`.
- `setCookie -> goto` is a valid pattern for preloading authenticated session state.
## Error codes
The current error code set is defined in `pkg/download/engine/webview/rpc.go`:
- `INVALID_REQUEST`
- `UNKNOWN_METHOD`
- `UNAVAILABLE`
- `BROWSER_NOT_FOUND`
- `PAGE_NOT_FOUND`
- `NAVIGATION_FAILED`
- `EVALUATION_FAILED`
- `TIMEOUT`
- `INTERNAL_ERROR`
`BROWSER_NOT_FOUND` is a historical leftover from an earlier shape of the protocol. The current protocol no longer has a separate browser model, so this can be cleaned up later if the protocol is tightened further.
## Current protocol boundary
This protocol is intentionally limited:
- Single-direction request/response only
- Not REST-style resource routing
- No bidirectional events
- No streaming event subscriptions
- No host-initiated push channel
If the protocol needs to evolve toward full JSON-RPC later, the intended migration path is to keep the existing `method/params/result/error` structure and add:
- `id`
- notifications
- a bidirectional transport model
## Host implementation guidance
A host implementation should usually:
- Listen on a local `tcp` address or `unix socket`
- Expose `POST /webview`
- Validate the bearer token if configured
- Maintain `pageId -> native webview/session`
- Own page lifecycle, cleanup, and timeout handling
With that split, Gopeed stays a pure RPC client and does not need to know how the host's native WebView is implemented.
+195
View File
@@ -0,0 +1,195 @@
package rpcprovider
import (
"bytes"
"context"
"encoding/json"
"fmt"
"net"
"net/http"
"strings"
enginewebview "github.com/GopeedLab/gopeed/pkg/download/engine/webview"
)
type Provider struct {
client *Client
}
func New(cfg enginewebview.RPCConfig) enginewebview.Provider {
return &Provider{
client: NewClient(cfg),
}
}
func (p *Provider) IsAvailable() bool {
if p == nil || p.client == nil || !p.client.Enabled() {
return false
}
var result enginewebview.IsAvailableResult
if err := p.client.Call(enginewebview.MethodIsAvailable, enginewebview.IsAvailableParams{}, &result); err != nil {
return false
}
return result.Available
}
func (p *Provider) Open(opts enginewebview.OpenOptions) (enginewebview.Page, error) {
if p == nil || p.client == nil || !p.IsAvailable() {
return nil, enginewebview.ErrUnavailable
}
var result enginewebview.PageOpenResult
if err := p.client.Call(enginewebview.MethodPageOpen, enginewebview.NewPageOpenParams(opts), &result); err != nil {
return nil, err
}
return &page{
client: p.client,
id: result.PageID,
}, nil
}
type page struct {
client *Client
id string
}
func (p *page) AddInitScript(script string) error {
return p.client.Call(enginewebview.MethodPageAddInitScript, enginewebview.PageAddInitScriptParams{
PageID: p.id,
Script: script,
}, nil)
}
func (p *page) Goto(url string, opts enginewebview.GotoOptions) error {
return p.client.Call(enginewebview.MethodPageGoto, enginewebview.NewPageGotoParams(p.id, url, opts), nil)
}
func (p *page) Execute(expression string, args ...any) (any, error) {
var result any
if err := p.client.Call(enginewebview.MethodPageExecute, enginewebview.PageExecuteParams{
PageID: p.id,
Expression: expression,
Args: args,
}, &result); err != nil {
return nil, err
}
return result, nil
}
func (p *page) GetCookies() ([]enginewebview.Cookie, error) {
var result []enginewebview.Cookie
if err := p.client.Call(enginewebview.MethodPageGetCookies, enginewebview.PageGetCookiesParams{
PageID: p.id,
}, &result); err != nil {
return nil, err
}
if result == nil {
return []enginewebview.Cookie{}, nil
}
return result, nil
}
func (p *page) SetCookie(cookie enginewebview.Cookie) error {
return p.client.Call(enginewebview.MethodPageSetCookie, enginewebview.PageSetCookieParams{
PageID: p.id,
Cookie: cookie,
}, nil)
}
func (p *page) DeleteCookie(cookie enginewebview.Cookie) error {
return p.client.Call(enginewebview.MethodPageDeleteCookie, enginewebview.PageDeleteCookieParams{
PageID: p.id,
Cookie: cookie,
}, nil)
}
func (p *page) ClearCookies() error {
return p.client.Call(enginewebview.MethodPageClearCookies, enginewebview.PageClearCookiesParams{
PageID: p.id,
}, nil)
}
func (p *page) Close() error {
return p.client.Call(enginewebview.MethodPageClose, enginewebview.PageCloseParams{
PageID: p.id,
}, nil)
}
type Client struct {
endpoint string
token string
http *http.Client
}
func NewClient(cfg enginewebview.RPCConfig) *Client {
client := &Client{token: cfg.Token}
if !cfg.Enabled() {
return client
}
switch strings.ToLower(cfg.Network) {
case "unix":
client.endpoint = "http://unix" + enginewebview.RPCEndpointPath
client.http = &http.Client{
Transport: &http.Transport{
DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) {
var d net.Dialer
return d.DialContext(ctx, "unix", cfg.Address)
},
},
}
default:
client.endpoint = "http://" + cfg.Address + enginewebview.RPCEndpointPath
client.http = &http.Client{}
}
return client
}
func (c *Client) Enabled() bool {
return c != nil && c.endpoint != "" && c.http != nil
}
func (c *Client) Call(method string, params any, result any) error {
if !c.Enabled() {
return enginewebview.ErrUnavailable
}
payload, err := json.Marshal(enginewebview.RPCRequest{
Method: method,
Params: params,
})
if err != nil {
return err
}
req, err := http.NewRequest(http.MethodPost, c.endpoint, bytes.NewReader(payload))
if err != nil {
return err
}
req.Header.Set("Content-Type", "application/json")
if c.token != "" {
req.Header.Set("Authorization", "Bearer "+c.token)
}
resp, err := c.http.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("webview rpc status %d", resp.StatusCode)
}
var body enginewebview.RPCResponse
if err := json.NewDecoder(resp.Body).Decode(&body); err != nil {
return err
}
if body.Error != nil {
return body.Error
}
if result == nil {
return nil
}
if len(body.Result) == 0 || string(body.Result) == "null" {
return nil
}
return json.Unmarshal(body.Result, result)
}
@@ -0,0 +1,41 @@
//go:build webview && darwin
package rpcprovider
import (
"flag"
"os"
"path/filepath"
"testing"
"time"
"github.com/GopeedLab/gopeed/internal/webview/integrationtest"
enginewebview "github.com/GopeedLab/gopeed/pkg/download/engine/webview"
)
var (
webViewRPCNetwork = flag.String("webview-rpc-network", "unix", "WebView RPC network for integration tests")
webViewRPCAddress = flag.String("webview-rpc-address", defaultWebViewRPCAddress(), "WebView RPC address for integration tests")
webViewRPCToken = flag.String("webview-rpc-token", "", "WebView RPC token for integration tests")
)
func TestProviderContract(t *testing.T) {
provider := New(enginewebview.RPCConfig{
Network: *webViewRPCNetwork,
Address: *webViewRPCAddress,
Token: *webViewRPCToken,
})
integrationtest.RunProviderContract(t, provider, integrationtest.ContractOptions{
AvailabilityTimeout: 5 * time.Minute,
CookieDomainMode: integrationtest.CookieDomainModeRequired,
CookieTestURL: "https://example.com/",
})
}
func defaultWebViewRPCAddress() string {
homeDir, err := os.UserHomeDir()
if err != nil {
return ""
}
return filepath.Join(homeDir, "Library", "Application Support", "com.gopeed.gopeed", "gopeed_webview.sock")
}
@@ -0,0 +1,50 @@
package rpcprovider
import (
"encoding/json"
"net"
"net/http"
"os"
"path/filepath"
"testing"
enginewebview "github.com/GopeedLab/gopeed/pkg/download/engine/webview"
)
func TestProviderIsAvailableOverUnixSocket(t *testing.T) {
socketPath := filepath.Join("/tmp", "gopeed-rpcwebview-"+t.Name()+".sock")
listener, err := net.Listen("unix", socketPath)
if err != nil {
t.Fatal(err)
}
defer listener.Close()
defer os.Remove(socketPath)
server := &http.Server{
Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
var req enginewebview.RPCRequest
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
t.Fatal(err)
}
if req.Method != enginewebview.MethodIsAvailable {
t.Fatalf("unexpected unix method: %s", req.Method)
}
if err := json.NewEncoder(w).Encode(map[string]any{
"result": enginewebview.IsAvailableResult{Available: true},
"error": nil,
}); err != nil {
t.Fatal(err)
}
}),
}
defer server.Close()
go server.Serve(listener)
provider := New(enginewebview.RPCConfig{
Network: "unix",
Address: socketPath,
})
if !provider.IsAvailable() {
t.Fatal("expected unix provider to be available")
}
}
@@ -0,0 +1,186 @@
package rpcprovider
import (
"encoding/json"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
enginewebview "github.com/GopeedLab/gopeed/pkg/download/engine/webview"
)
func TestProviderLifecycleOverHTTP(t *testing.T) {
var calls []enginewebview.RPCRequest
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost {
t.Fatalf("unexpected method: %s", r.Method)
}
if r.URL.Path != enginewebview.RPCEndpointPath {
t.Fatalf("unexpected path: %s", r.URL.Path)
}
if got := r.Header.Get("Authorization"); got != "Bearer secret-token" {
t.Fatalf("unexpected auth header: %q", got)
}
var req enginewebview.RPCRequest
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
t.Fatal(err)
}
calls = append(calls, req)
var body any
switch req.Method {
case enginewebview.MethodIsAvailable:
body = map[string]any{
"result": enginewebview.IsAvailableResult{Available: true},
"error": nil,
}
case enginewebview.MethodPageOpen:
body = map[string]any{
"result": enginewebview.PageOpenResult{PageID: "page-open-1"},
"error": nil,
}
case enginewebview.MethodPageAddInitScript, enginewebview.MethodPageGoto:
body = map[string]any{
"result": enginewebview.EmptyResult{},
"error": nil,
}
case enginewebview.MethodPageExecute:
body = map[string]any{
"result": map[string]any{"title": "Example"},
"error": nil,
}
case enginewebview.MethodPageClose:
body = map[string]any{
"result": enginewebview.EmptyResult{},
"error": nil,
}
default:
t.Fatalf("unexpected method: %s", req.Method)
}
if err := json.NewEncoder(w).Encode(body); err != nil {
t.Fatal(err)
}
}))
defer server.Close()
cfg := enginewebview.RPCConfig{
Network: "tcp",
Address: server.Listener.Addr().String(),
Token: "secret-token",
}
provider := New(cfg)
if !provider.IsAvailable() {
t.Fatal("expected provider to be available")
}
opener, ok := provider.(enginewebview.Opener)
if !ok {
t.Fatal("expected provider to implement runtime webview opener")
}
opened, err := opener.Open(enginewebview.OpenOptions{
Headless: true,
Width: 1280,
Height: 720,
Title: "Gopeed",
UserAgent: "UA",
})
if err != nil {
t.Fatal(err)
}
if err := opened.AddInitScript("window.__TEST__ = true"); err != nil {
t.Fatal(err)
}
if err := opened.Goto("https://example.com", enginewebview.GotoOptions{
TimeoutMS: 3210,
WaitUntil: "domcontentloaded",
}); err != nil {
t.Fatal(err)
}
value, err := opened.Execute("document.title")
if err != nil {
t.Fatal(err)
}
got, ok := value.(map[string]any)
if !ok || got["title"] != "Example" {
t.Fatalf("unexpected evaluate result: %#v", value)
}
if err := opened.Close(); err != nil {
t.Fatal(err)
}
expectedMethods := []string{
enginewebview.MethodIsAvailable,
enginewebview.MethodIsAvailable,
enginewebview.MethodPageOpen,
enginewebview.MethodPageAddInitScript,
enginewebview.MethodPageGoto,
enginewebview.MethodPageExecute,
enginewebview.MethodPageClose,
}
if len(calls) != len(expectedMethods) {
t.Fatalf("unexpected call count: got %d want %d", len(calls), len(expectedMethods))
}
for i, method := range expectedMethods {
if calls[i].Method != method {
t.Fatalf("unexpected method at %d: got %s want %s", i, calls[i].Method, method)
}
}
gotoParamsJSON, err := json.Marshal(calls[4].Params)
if err != nil {
t.Fatal(err)
}
var gotoParams enginewebview.PageGotoParams
if err := json.Unmarshal(gotoParamsJSON, &gotoParams); err != nil {
t.Fatal(err)
}
if gotoParams.WaitUntil != "domcontentloaded" {
t.Fatalf("unexpected waitUntil: %q", gotoParams.WaitUntil)
}
}
func TestProviderLaunchUnavailableWithoutEndpoint(t *testing.T) {
provider := New(enginewebview.RPCConfig{})
if provider.IsAvailable() {
t.Fatal("expected empty provider to be unavailable")
}
opener, ok := provider.(enginewebview.Opener)
if !ok {
t.Fatal("expected provider to implement runtime webview opener")
}
if _, err := opener.Open(enginewebview.OpenOptions{}); err == nil || err.Error() != enginewebview.ErrUnavailable.Error() {
t.Fatalf("unexpected launch error: %v", err)
}
}
func TestPageSetCookieParamsOmitsZeroExpires(t *testing.T) {
params := enginewebview.PageSetCookieParams{
PageID: "page-1",
Cookie: enginewebview.Cookie{
Name: "visible",
Value: "1",
Domain: "example.com",
Path: "/",
},
}
payload, err := json.Marshal(params)
if err != nil {
t.Fatal(err)
}
if string(payload) == "" {
t.Fatal("expected non-empty json payload")
}
if got := string(payload); strings.Contains(got, `"expires"`) {
t.Fatalf("unexpected zero expires in payload: %s", got)
}
params.Cookie.Expires = time.Now().UTC().Truncate(time.Second)
payload, err = json.Marshal(params)
if err != nil {
t.Fatal(err)
}
if got := string(payload); !strings.Contains(got, `"expires"`) {
t.Fatalf("expected expires in payload: %s", got)
}
}
+36 -4
View File
@@ -14,8 +14,9 @@ import (
// Request download request
type Request struct {
URL string `json:"url"`
Extra any `json:"extra"`
RawURL string `json:"rawUrl"`
URL string `json:"url"`
Extra any `json:"extra"`
// Labels is used to mark the download task
Labels map[string]string `json:"labels"`
// Proxy is special proxy config for request
@@ -24,6 +25,24 @@ type Request struct {
SkipVerifyCert bool `json:"skipVerifyCert"`
}
// SetLabels replaces all labels on the request.
func (r *Request) SetLabels(labels map[string]string) {
r.Labels = labels
}
// PutLabel sets a label on the request.
func (r *Request) PutLabel(key, value string) {
if r.Labels == nil {
r.Labels = make(map[string]string)
}
r.Labels[key] = value
}
// DelLabel deletes a label from the request.
func (r *Request) DelLabel(key string) {
delete(r.Labels, key)
}
func (r *Request) Validate() error {
if r.URL == "" {
return fmt.Errorf("invalid request url")
@@ -78,7 +97,7 @@ func (r *Resource) Validate() error {
if r.Name == "" {
return fmt.Errorf("invalid resource name")
}
if r.Files == nil || len(r.Files) == 0 {
if len(r.Files) == 0 {
return fmt.Errorf("invalid resource files")
}
for _, file := range r.Files {
@@ -124,7 +143,7 @@ func (o *Options) InitSelectFiles(fileSize int) {
// if selectFiles is empty, select all files
if len(o.SelectFiles) == 0 {
o.SelectFiles = make([]int, fileSize)
for i := 0; i < fileSize; i++ {
for i := range fileSize {
o.SelectFiles[i] = i
}
}
@@ -184,6 +203,7 @@ type DownloaderStoreConfig struct {
Extra map[string]any `json:"extra"`
Proxy *DownloaderProxyConfig `json:"proxy"`
Webhook *WebhookConfig `json:"webhook"` // Webhook is the webhook configuration
Script *ScriptConfig `json:"script"` // Script is the script execution configuration
AutoTorrent *AutoTorrentConfig `json:"autoTorrent"` // AutoTorrent is the auto torrent task creation configuration
Archive *ArchiveConfig `json:"archive"` // Archive is the archive extraction configuration
AutoDeleteMissingFileTasks bool `json:"autoDeleteMissingFileTasks"` // AutoDeleteMissingFileTasks enables automatic deletion of tasks with missing files
@@ -202,6 +222,9 @@ func (cfg *DownloaderStoreConfig) Init() *DownloaderStoreConfig {
if cfg.Webhook == nil {
cfg.Webhook = &WebhookConfig{}
}
if cfg.Script == nil {
cfg.Script = &ScriptConfig{}
}
if cfg.AutoTorrent == nil {
cfg.AutoTorrent = &AutoTorrentConfig{
Enable: false,
@@ -239,6 +262,9 @@ func (cfg *DownloaderStoreConfig) Merge(beforeCfg *DownloaderStoreConfig) *Downl
if cfg.Webhook == nil {
cfg.Webhook = beforeCfg.Webhook
}
if cfg.Script == nil {
cfg.Script = beforeCfg.Script
}
if cfg.AutoTorrent == nil {
cfg.AutoTorrent = beforeCfg.AutoTorrent
}
@@ -254,6 +280,12 @@ type WebhookConfig struct {
URLs []string `json:"urls"` // URLs is the list of webhook URLs
}
// ScriptConfig is the script execution configuration
type ScriptConfig struct {
Enable bool `json:"enable"` // Enable is the flag to enable/disable script execution
Paths []string `json:"paths"` // Paths is the list of script paths to execute
}
// AutoTorrentConfig is the auto torrent task creation configuration
type AutoTorrentConfig struct {
Enable bool `json:"enable"` // Enable enables automatic BT task creation when downloading .torrent files
+7
View File
@@ -19,6 +19,7 @@ func TestDownloaderStoreConfig_Init(t *testing.T) {
ProtocolConfig: map[string]any{},
Proxy: &DownloaderProxyConfig{},
Webhook: &WebhookConfig{},
Script: &ScriptConfig{},
AutoTorrent: &AutoTorrentConfig{
Enable: false,
DeleteAfterDownload: false,
@@ -39,6 +40,7 @@ func TestDownloaderStoreConfig_Init(t *testing.T) {
ProtocolConfig: map[string]any{},
Proxy: &DownloaderProxyConfig{},
Webhook: &WebhookConfig{},
Script: &ScriptConfig{},
AutoTorrent: &AutoTorrentConfig{
Enable: false,
DeleteAfterDownload: false,
@@ -63,6 +65,7 @@ func TestDownloaderStoreConfig_Init(t *testing.T) {
},
Proxy: &DownloaderProxyConfig{},
Webhook: &WebhookConfig{},
Script: &ScriptConfig{},
AutoTorrent: &AutoTorrentConfig{
Enable: false,
DeleteAfterDownload: false,
@@ -87,6 +90,7 @@ func TestDownloaderStoreConfig_Init(t *testing.T) {
Enable: true,
},
Webhook: &WebhookConfig{},
Script: &ScriptConfig{},
AutoTorrent: &AutoTorrentConfig{
Enable: false,
DeleteAfterDownload: false,
@@ -110,6 +114,7 @@ func TestDownloaderStoreConfig_Init(t *testing.T) {
ProtocolConfig: map[string]any{},
Proxy: &DownloaderProxyConfig{},
Webhook: &WebhookConfig{},
Script: &ScriptConfig{},
AutoTorrent: &AutoTorrentConfig{
Enable: true,
DeleteAfterDownload: true,
@@ -133,6 +138,7 @@ func TestDownloaderStoreConfig_Init(t *testing.T) {
ProtocolConfig: map[string]any{},
Proxy: &DownloaderProxyConfig{},
Webhook: &WebhookConfig{},
Script: &ScriptConfig{},
AutoTorrent: &AutoTorrentConfig{
Enable: false,
DeleteAfterDownload: false,
@@ -154,6 +160,7 @@ func TestDownloaderStoreConfig_Init(t *testing.T) {
Extra: tt.fields.Extra,
Proxy: tt.fields.Proxy,
Webhook: tt.fields.Webhook,
Script: tt.fields.Script,
AutoTorrent: tt.fields.AutoTorrent,
Archive: tt.fields.Archive,
}
+567 -118
View File
@@ -15,6 +15,7 @@ import (
"sync/atomic"
"time"
internalblob "github.com/GopeedLab/gopeed/internal/blob"
"github.com/GopeedLab/gopeed/internal/controller"
"github.com/GopeedLab/gopeed/internal/fetcher"
"github.com/GopeedLab/gopeed/internal/logger"
@@ -31,6 +32,8 @@ const (
bucketTask = "task"
// task download data bucket
bucketSave = "save"
// protocol-level shared client state bucket
bucketProtocolState = "protocol_state"
// downloader config bucket
bucketConfig = "config"
// downloader extension bucket
@@ -109,6 +112,7 @@ type Downloader struct {
claimedExtractions sync.Map
extensions []*Extension
blob *internalblob.Registry
}
func NewDownloader(cfg *DownloaderConfig) *Downloader {
@@ -140,8 +144,10 @@ func NewDownloader(cfg *DownloaderConfig) *Downloader {
}
func (d *Downloader) Setup() error {
d.blob = internalblob.NewRegistry("")
// setup storage
if err := d.storage.Setup([]string{bucketTask, bucketSave, bucketConfig, bucketExtension, bucketExtensionStorage}); err != nil {
if err := d.storage.Setup([]string{bucketTask, bucketSave, bucketProtocolState, bucketConfig, bucketExtension, bucketExtensionStorage}); err != nil {
return err
}
// load config from storage
@@ -165,6 +171,12 @@ func (d *Downloader) Setup() error {
if _, ok := d.cfg.DownloaderStoreConfig.ProtocolConfig[protocol]; !ok {
d.cfg.DownloaderStoreConfig.ProtocolConfig[protocol] = fm.DefaultConfig()
}
if sfm, ok := fm.(fetcher.StatefulFetcherManager); ok {
sfm.SetStateStore(&protocolStateStore{
storage: d.storage,
protocol: protocol,
})
}
}
// load tasks from storage
@@ -230,13 +242,18 @@ func (d *Downloader) Setup() error {
if len(d.tasks) > 0 {
for _, task := range d.tasks {
func() {
task.statusLock.Lock()
defer task.statusLock.Unlock()
if task.Status != base.DownloadStatusRunning && !task.Uploading {
// Do not acquire d.lock (via GetTask) while holding
// statusLock; scheduling uses the opposite lock order.
if d.GetTask(task.ID) == nil {
return
}
// check if task is deleted
if d.GetTask(task.ID) == nil || task.fetcher == nil {
task.statusLock.Lock()
if task.Status != base.DownloadStatusRunning && !task.Uploading {
task.statusLock.Unlock()
return
}
if task.fetcher == nil {
task.statusLock.Unlock()
return
}
@@ -258,6 +275,8 @@ func (d *Downloader) Setup() error {
task.Progress.UploadSpeed = task.updateUploadSpeed(currentUploaded-task.Progress.Uploaded, tick)
task.Progress.Uploaded = currentUploaded
}
task.statusLock.Unlock()
// Listener callbacks may Pause/Continue and acquire statusLock.
d.emit(EventKeyProgress, task)
// store fetcher progress when download/upload data changed
@@ -337,6 +356,12 @@ func (d *Downloader) setupFetcher(fm fetcher.FetcherManager, fetcher fetcher.Fet
}
// Get proxy config, task request proxy config has higher priority, then use global proxy config
ctl.GetProxy = func(requestProxy *base.RequestProxy) func(*gohttp.Request) (*url.URL, error) {
// Registry-backed blob URLs are loopback capability URLs and must never be
// sent through a configured proxy. Only exact, currently registered URLs
// take this path; everything else keeps normal HTTP proxy behavior.
if d.blob != nil && fetcher != nil && fetcher.Meta() != nil && fetcher.Meta().Req != nil && d.blob.IsURL(fetcher.Meta().Req.URL) {
return nil
}
if requestProxy == nil {
return d.cfg.Proxy.ToHandler()
}
@@ -358,9 +383,16 @@ func (d *Downloader) saveTask(task *Task) error {
d.Logger.Error().Stack().Err(err).Msgf("serialize fetcher failed: %s", task.ID)
return err
}
if err := d.storage.Put(bucketSave, task.ID, data); err != nil {
d.Logger.Error().Stack().Err(err).Msgf("persist fetcher failed: %s", task.ID)
return err
if data != nil {
if err := d.storage.Put(bucketSave, task.ID, data); err != nil {
d.Logger.Error().Stack().Err(err).Msgf("persist fetcher failed: %s", task.ID)
return err
}
} else {
if err := d.storage.Delete(bucketSave, task.ID); err != nil {
d.Logger.Error().Stack().Err(err).Msgf("clear fetcher state failed: %s", task.ID)
return err
}
}
if err := d.storage.Put(bucketTask, task.ID, task); err != nil {
d.Logger.Error().Stack().Err(err).Msgf("persist task failed: %s", task.ID)
@@ -369,11 +401,39 @@ func (d *Downloader) saveTask(task *Task) error {
return nil
}
func ensureRequestRawURL(req *base.Request) {
if req == nil {
return
}
if req.RawURL == "" {
req.RawURL = req.URL
}
}
func ensureResourceRequestRawURLs(parentReq *base.Request, res *base.Resource) {
if res == nil {
return
}
for _, file := range res.Files {
if file == nil || file.Req == nil {
continue
}
ensureRequestRawURL(file.Req)
if parentReq != nil && parentReq.RawURL != "" && file.Req.RawURL == file.Req.URL {
file.Req.RawURL = parentReq.RawURL
}
}
}
func (d *Downloader) Resolve(req *base.Request, opts *base.Options) (rr *ResolveResult, err error) {
if req == nil {
return nil, errors.New("request cannot be nil")
}
rrId, err := gonanoid.New()
if err != nil {
return
}
ensureRequestRawURL(req)
res, err := d.triggerOnResolve(req)
if err != nil {
@@ -436,6 +496,7 @@ func (d *Downloader) remainRunningCount() int {
}
func (d *Downloader) CreateDirect(req *base.Request, opts *base.Options) (taskId string, err error) {
ensureRequestRawURL(req)
var fetcher fetcher.Fetcher
fetcher, err = d.buildFetcher(req.URL)
if err != nil {
@@ -480,6 +541,53 @@ func (d *Downloader) Create(rrId string) (taskId string, err error) {
return d.doCreate(fetcher, nil)
}
// Patch modifies task-specific data based on the protocol.
// For HTTP protocol, it can modify Request info.
// For BT protocol, it can modify SelectFiles.
func (d *Downloader) Patch(id string, req *base.Request, opts *base.Options) error {
task := d.GetTask(id)
if task == nil {
return ErrTaskNotFound
}
if err := func() error {
task.lock.Lock()
defer task.lock.Unlock()
// Restore fetcher if not loaded
if task.fetcher == nil {
err := func() error {
task.statusLock.Lock()
defer task.statusLock.Unlock()
return d.restoreFetcher(task)
}()
if err != nil {
return err
}
}
// Call the fetcher's Patch method
if err := task.fetcher.Patch(req, opts); err != nil {
return err
}
// Update task meta from fetcher and transfer any live Blob lease.
task.Meta = task.fetcher.Meta()
if err := d.syncBlobTaskLease(task); err != nil {
return err
}
return d.saveTask(task)
}(); err != nil {
return err
}
// Emit outside task.lock so listeners may safely call back into Patch/Pause.
d.emit(EventKeyProgress, task)
return nil
}
func (d *Downloader) Pause(filter *TaskFilter) (err error) {
if filter == nil || filter.IsEmpty() {
return d.pauseAll()
@@ -532,6 +640,7 @@ func (d *Downloader) Continue(filter *TaskFilter) (err error) {
}
realContinueTasks := make([]*Task, 0)
pausedTasks := make([]*Task, 0)
func() {
d.lock.Lock()
defer d.lock.Unlock()
@@ -544,12 +653,16 @@ func (d *Downloader) Continue(filter *TaskFilter) (err error) {
pausedCount := 0
for _, task := range d.tasks {
if task.Status == base.DownloadStatusRunning {
if err = d.doPause(task); err != nil {
var queued bool
queued, err = d.doPauseForScheduling(task)
if err != nil {
return
}
task.Status = base.DownloadStatusWait
d.waitTasks = append(d.waitTasks, task)
pausedCount++
if queued {
d.waitTasks = append(d.waitTasks, task)
pausedTasks = append(pausedTasks, task)
pausedCount++
}
}
if pausedCount == needPauseCount {
break
@@ -566,6 +679,14 @@ func (d *Downloader) Continue(filter *TaskFilter) (err error) {
}
}
}()
if err != nil {
return
}
// Scheduling pauses run while d.lock is held, but events must not: listeners
// are allowed to call back into Downloader methods that acquire d.lock.
for _, task := range pausedTasks {
d.emit(EventKeyPause, task)
}
for _, task := range realContinueTasks {
if err = d.doStart(task); err != nil {
@@ -709,6 +830,7 @@ func (d *Downloader) Stats(id string) (sr any, err error) {
}
func (d *Downloader) doDelete(task *Task, force bool) (err error) {
defer d.releaseBlobTask(task)
err = func() error {
if err := d.storage.Delete(bucketTask, task.ID); err != nil {
return err
@@ -753,6 +875,9 @@ func (d *Downloader) Close() error {
for _, fm := range d.cfg.FetchManagers {
closeArr = append(closeArr, fm.Close)
}
if d.blob != nil {
closeArr = append(closeArr, d.blob.Close)
}
closeArr = append(closeArr, d.storage.Close)
// Make sure all resources are released, if had error, return the last error
var lastErr error
@@ -779,6 +904,26 @@ func (d *Downloader) Clear() error {
return nil
}
type protocolStateStore struct {
storage Storage
protocol string
}
func (s *protocolStateStore) Load(v any) (bool, error) {
return s.storage.Get(bucketProtocolState, s.protocol, v)
}
func (s *protocolStateStore) Save(v any) error {
if v == nil {
return s.Delete()
}
return s.storage.Put(bucketProtocolState, s.protocol, v)
}
func (s *protocolStateStore) Delete() error {
return s.storage.Delete(bucketProtocolState, s.protocol)
}
func (d *Downloader) Listener(fn Listener) {
d.listener = fn
}
@@ -898,7 +1043,6 @@ func (d *Downloader) watch(task *Task) {
if _, loaded := d.watchedTasks.LoadOrStore(task.ID, true); loaded {
return
}
defer func() {
d.watchedTasks.Delete(task.ID)
}()
@@ -921,103 +1065,175 @@ func (d *Downloader) watch(task *Task) {
}
}
if task.Status == base.DownloadStatusDone {
if d.taskStatus(task) == base.DownloadStatusDone {
return
}
err := task.fetcher.Wait()
if err != nil {
d.doOnError(task, err)
return
}
// When delete a not resolved task, need check if the task resource is nil
if task.Meta.Res == nil || d.GetTask(task.ID) == nil {
return
}
task.Progress.Used = task.timer.Used()
if task.Meta.Res.Size == 0 {
task.Meta.Res.Size = task.fetcher.Progress().TotalDownloaded()
}
used := task.Progress.Used / int64(time.Second)
if used == 0 {
used = 1
}
totalSize := task.Meta.Res.Size
task.Progress.Speed = totalSize / used
task.Progress.Downloaded = totalSize
task.updateStatus(base.DownloadStatusDone)
d.storage.Put(bucketTask, task.ID, task.clone())
d.emit(EventKeyDone, task)
d.emit(EventKeyFinally, task, err)
d.notifyRunning()
d.triggerOnDone(task)
d.triggerWebhooks(WebhookEventDownloadDone, task, nil)
if e, ok := task.Meta.Opts.Extra.(*http.OptsExtra); ok {
downloadFilePath := task.Meta.SingleFilepath()
cfg, _ := d.GetConfig()
// Determine if auto-torrent is enabled (use global config if not explicitly set)
autoTorrentEnabled := false
if e.AutoTorrent != nil {
autoTorrentEnabled = *e.AutoTorrent
} else if cfg != nil && cfg.AutoTorrent != nil {
autoTorrentEnabled = cfg.AutoTorrent.Enable
for {
err := task.fetcher.Wait()
if err != nil {
status := d.taskStatus(task)
if status == base.DownloadStatusPause || status == base.DownloadStatusWait {
return
}
var sourceErr error
if d.blob != nil && task.Meta != nil && task.Meta.Req != nil {
sourceErr = d.blob.SourceError(task.Meta.Req.URL)
}
if sourceErr != nil {
d.doOnBlobSourceError(task, sourceErr)
} else {
d.doOnError(task, err)
}
if d.taskStatus(task) == base.DownloadStatusRunning {
continue
}
return
}
if autoTorrentEnabled && strings.HasSuffix(downloadFilePath, ".torrent") {
// Determine if should delete torrent file after creating BT task
shouldDelete := false
if e.DeleteTorrentAfterDownload != nil {
shouldDelete = *e.DeleteTorrentAfterDownload
// When delete a not resolved task, need check if the task resource is nil
if task.Meta.Res == nil || d.GetTask(task.ID) == nil {
return
}
if d.blob != nil && task.Meta.Req != nil {
if err := d.blob.SourceError(task.Meta.Req.URL); err != nil {
d.doOnBlobSourceError(task, err)
if d.taskStatus(task) == base.DownloadStatusRunning {
continue
}
return
}
}
task.Progress.Used = task.timer.Used()
if task.Meta.Res.Size == 0 {
task.Meta.Res.Size = task.fetcher.Progress().TotalDownloaded()
}
used := task.Progress.Used / int64(time.Second)
if used == 0 {
used = 1
}
totalSize := task.Meta.Res.Size
task.Progress.Speed = totalSize / used
task.Progress.Downloaded = totalSize
if !d.markTaskDone(task) {
return
}
d.storage.Put(bucketTask, task.ID, task.clone())
d.emit(EventKeyDone, task)
d.emit(EventKeyFinally, task, err)
d.notifyRunning()
d.releaseBlobTask(task)
d.triggerOnDone(task)
d.triggerWebhooks(WebhookEventDownloadDone, task, nil)
d.triggerScripts(ScriptEventDownloadDone, task, nil)
if e, ok := task.Meta.Opts.Extra.(*http.OptsExtra); ok {
downloadFilePath := task.Meta.SingleFilepath()
cfg, _ := d.GetConfig()
// Determine if auto-torrent is enabled (use global config if not explicitly set)
autoTorrentEnabled := false
if e.AutoTorrent != nil {
autoTorrentEnabled = *e.AutoTorrent
} else if cfg != nil && cfg.AutoTorrent != nil {
shouldDelete = cfg.AutoTorrent.DeleteAfterDownload
autoTorrentEnabled = cfg.AutoTorrent.Enable
}
go func() {
_, err2 := d.CreateDirect(
&base.Request{
URL: downloadFilePath,
},
&base.Options{
Path: task.Meta.Opts.Path,
SelectFiles: make([]int, 0),
})
if err2 != nil {
d.Logger.Error().Err(err2).Msgf("auto create torrent task failed, task id: %s", task.ID)
return
if autoTorrentEnabled && strings.HasSuffix(downloadFilePath, ".torrent") {
// Determine if should delete torrent file after creating BT task
shouldDelete := false
if e.DeleteTorrentAfterDownload != nil {
shouldDelete = *e.DeleteTorrentAfterDownload
} else if cfg != nil && cfg.AutoTorrent != nil {
shouldDelete = cfg.AutoTorrent.DeleteAfterDownload
}
if shouldDelete {
d.Delete(&TaskFilter{IDs: []string{task.ID}}, true)
}
}()
}
go func() {
_, err2 := d.CreateDirect(
&base.Request{
URL: downloadFilePath,
},
&base.Options{
Path: task.Meta.Opts.Path,
SelectFiles: make([]int, 0),
})
if err2 != nil {
d.Logger.Error().Err(err2).Msgf("auto create torrent task failed, task id: %s", task.ID)
return
}
// Determine if auto-extract is enabled (use global config if not explicitly set)
autoExtractEnabled := false
if e.AutoExtract != nil {
autoExtractEnabled = *e.AutoExtract
} else if cfg != nil && cfg.Archive != nil {
autoExtractEnabled = cfg.Archive.AutoExtract
}
if shouldDelete {
d.Delete(&TaskFilter{IDs: []string{task.ID}}, true)
}
}()
}
// Auto-extract archive files using the extraction queue
// This ensures only one extraction runs at a time to prevent resource exhaustion
if autoExtractEnabled && isArchiveFile(downloadFilePath) {
d.enqueueExtraction(task, downloadFilePath, e)
// Determine if auto-extract is enabled (use global config if not explicitly set)
autoExtractEnabled := false
if e.AutoExtract != nil {
autoExtractEnabled = *e.AutoExtract
} else if cfg != nil && cfg.Archive != nil {
autoExtractEnabled = cfg.Archive.AutoExtract
}
// Auto-extract archive files using the extraction queue
// This ensures only one extraction runs at a time to prevent resource exhaustion
if autoExtractEnabled && isArchiveFile(downloadFilePath) {
d.enqueueExtraction(task, downloadFilePath, e)
}
}
return
}
}
func (d *Downloader) doOnError(task *Task, err error) {
d.handleOnError(task, err, false)
}
func (d *Downloader) doOnBlobSourceError(task *Task, err error) {
d.handleOnError(task, err, true)
}
func (d *Downloader) handleOnError(task *Task, err error, resetFetcher bool) {
if task == nil || task.lock == nil {
return
}
task.lock.Lock()
if !d.markTaskError(task) {
task.lock.Unlock()
return
}
d.Logger.Warn().Err(err).Msgf("task download failed, task id: %s", task.ID)
task.updateStatus(base.DownloadStatusError)
oldURL := ""
if task.Meta != nil && task.Meta.Req != nil {
oldURL = task.Meta.Req.URL
}
d.triggerOnError(task, err)
if task.Status == base.DownloadStatusError {
newURL := ""
if task.Meta != nil && task.Meta.Req != nil {
newURL = task.Meta.Req.URL
}
if oldURL != "" && newURL != oldURL {
// Extensions are allowed to replace an expired URL from onError. Patch the
// existing fetcher through its public API so protocol-owned redirect state
// is reset without teaching the HTTP fetcher about blob URLs.
if task.fetcher != nil {
if patchErr := task.fetcher.Patch(&base.Request{URL: newURL}, nil); patchErr != nil {
d.Logger.Warn().Err(patchErr).Msgf("patch recovered task url failed, task id: %s", task.ID)
}
}
d.releaseBlobTaskURL(task, oldURL)
}
if resetFetcher && d.taskStatus(task) == base.DownloadStatusRunning {
if resetErr := d.resetTaskFetcher(task); resetErr != nil {
d.Logger.Warn().Err(resetErr).Msgf("reset recovered task fetcher failed, task id: %s", task.ID)
}
}
task.lock.Unlock()
if d.taskStatus(task) == base.DownloadStatusError {
d.releaseBlobTask(task)
d.emit(EventKeyError, task, err)
d.emit(EventKeyFinally, task, err)
d.notifyRunning()
@@ -1025,13 +1241,43 @@ func (d *Downloader) doOnError(task *Task, err error) {
}
}
func (d *Downloader) resetTaskFetcher(task *Task) error {
if task == nil || task.Meta == nil || task.Meta.Req == nil {
return errors.New("task request is unavailable")
}
fm, err := d.parseFm(task.Meta.Req.URL)
if err != nil {
return err
}
opts := task.Meta.Opts
f := fm.Build()
d.setupFetcher(fm, f)
// A fresh source must never inherit bytes or preallocation from the failed
// source. HTTP opens existing targets without truncating them, so remove the
// old single-file target before installing a fresh fetcher.
if task.Meta.Res != nil && task.Meta.Opts != nil && len(task.Meta.Res.Files) > 0 {
if err := util.SafeRemove(task.Meta.SingleFilepath()); err != nil {
return err
}
}
f.Meta().Req = task.Meta.Req
f.Meta().Opts = opts
task.fetcherManager = fm
task.fetcher = f
task.Meta = f.Meta()
if task.Progress != nil {
task.Progress.Downloaded = 0
task.Progress.Speed = 0
}
return d.storage.Delete(bucketSave, task.ID)
}
func (d *Downloader) restoreTask(task *Task) error {
if task.fetcher == nil {
if err := d.restoreFetcher(task); err != nil {
return err
}
}
go d.watch(task)
return nil
}
@@ -1064,12 +1310,12 @@ func (d *Downloader) doCreate(f fetcher.Fetcher, opts *base.Options) (taskId str
if f.Meta().Opts == nil {
f.Meta().Opts = opts
}
ensureRequestRawURL(f.Meta().Req)
fm, err := d.parseFm(f.Meta().Req.URL)
if err != nil {
return
}
task := NewTask()
task.fetcherManager = fm
task.fetcher = f
@@ -1078,6 +1324,14 @@ func (d *Downloader) doCreate(f fetcher.Fetcher, opts *base.Options) (taskId str
task.Progress = &Progress{}
_, task.Uploading = f.(fetcher.Uploader)
initTask(task)
if err = d.syncBlobTaskLease(task); err != nil {
return "", err
}
defer func() {
if err != nil {
d.releaseBlobTask(task)
}
}()
if err = d.storage.Put(bucketTask, task.ID, task.clone()); err != nil {
return
}
@@ -1099,7 +1353,6 @@ func (d *Downloader) doCreate(f fetcher.Fetcher, opts *base.Options) (taskId str
err = d.doStart(task)
}()
go d.watch(task)
return
}
@@ -1145,6 +1398,7 @@ func (d *Downloader) statusMut(task *Task, fn func() (bool, error)) (bool, error
func (d *Downloader) doStart(task *Task) (err error) {
var isCreate bool
var generation uint64
isReturn, err := d.statusMut(task, func() (isReturn bool, err error) {
if task.Status == base.DownloadStatusRunning || task.Status == base.DownloadStatusDone {
isReturn = true
@@ -1158,6 +1412,8 @@ func (d *Downloader) doStart(task *Task) (err error) {
}
isCreate = task.Status == base.DownloadStatusReady
task.updateStatus(base.DownloadStatusRunning)
task.runGeneration++
generation = task.runGeneration
return
})
@@ -1169,11 +1425,31 @@ func (d *Downloader) doStart(task *Task) (err error) {
return
}
started := false
handler := func() error {
task.lock.Lock()
defer task.lock.Unlock()
if !d.taskIsRunningGeneration(task, generation) {
return nil
}
if task.fetcher == nil {
if err := d.restoreFetcher(task); err != nil {
return err
}
}
d.triggerOnStart(task)
if !d.taskIsRunningGeneration(task, generation) {
return nil
}
if task.fetcher != nil {
task.fetcher.Meta().Req = task.Meta.Req
task.fetcher.Meta().Res = task.Meta.Res
task.fetcher.Meta().Opts = task.Meta.Opts
}
if err := d.syncBlobTaskLease(task); err != nil {
return err
}
if task.Meta.Res == nil {
err := task.fetcher.Resolve(task.Meta.Req, task.Meta.Opts)
if err != nil {
@@ -1211,19 +1487,31 @@ func (d *Downloader) doStart(task *Task) (err error) {
}
task.Progress.Speed = 0
if !d.taskIsRunningGeneration(task, generation) {
return nil
}
task.timer.Start()
if err := task.fetcher.Start(); err != nil {
return err
}
started = true
if err := d.saveTask(task); err != nil {
return err
}
d.emit(EventKeyStart, task)
return nil
}
go func() {
if err := handler(); err != nil {
d.doOnError(task, err)
if d.taskIsRunningGeneration(task, generation) {
d.doOnError(task, err)
}
return
}
if started {
if d.taskIsRunningGeneration(task, generation) {
d.emit(EventKeyStart, task)
}
go d.watch(task)
}
}()
@@ -1231,45 +1519,137 @@ func (d *Downloader) doStart(task *Task) (err error) {
}
func (d *Downloader) doPause(task *Task) (err error) {
isReturn, err := d.statusMut(task, func() (isReturn bool, err error) {
generation, isReturn, err := d.preparePause(task)
if err != nil || isReturn {
return err
}
handled, err := d.runPauseHandler(task, generation, true)
if err != nil {
d.Logger.Error().Stack().Err(err).Msgf("pause task handle failed, task id: %s", task.ID)
return err
}
if handled {
d.emit(EventKeyPause, task)
}
return nil
}
// doPauseForScheduling physically pauses a running task without emitting while
// the caller holds d.lock, then atomically moves it to the wait queue state.
func (d *Downloader) doPauseForScheduling(task *Task) (bool, error) {
generation, isReturn, err := d.preparePause(task)
if err != nil || isReturn {
return false, err
}
handled, err := d.runPauseHandler(task, generation, true)
if err != nil || !handled {
return false, err
}
task.statusLock.Lock()
queued := task.Status == base.DownloadStatusPause && task.runGeneration == generation
if queued {
task.Status = base.DownloadStatusWait
}
task.statusLock.Unlock()
return queued, nil
}
func (d *Downloader) preparePause(task *Task) (generation uint64, isReturn bool, err error) {
isReturn, err = d.statusMut(task, func() (isReturn bool, err error) {
if task.Status == base.DownloadStatusPause || task.Status == base.DownloadStatusDone {
isReturn = true
return
}
task.updateStatus(base.DownloadStatusPause)
task.runGeneration++
generation = task.runGeneration
task.timer.Pause()
return
})
if err != nil {
d.Logger.Error().Stack().Err(err).Msgf("pause task failed, task id: %s", task.ID)
return
}
if isReturn {
return
return generation, isReturn, err
}
func (d *Downloader) runPauseHandler(task *Task, generation uint64, pauseFetcher bool) (bool, error) {
task.lock.Lock()
defer task.lock.Unlock()
if !d.taskIsPausedGeneration(task, generation) {
return false, nil
}
handler := func() error {
task.lock.Lock()
defer task.lock.Unlock()
if task.fetcher != nil {
if err := task.fetcher.Pause(); err != nil {
return err
}
if pauseFetcher && task.fetcher != nil {
if err := task.fetcher.Pause(); err != nil {
return false, err
}
}
if task.fetcherManager != nil && task.fetcher != nil {
if err := d.saveTask(task); err != nil {
return false, err
}
} else {
if err := d.storage.Put(bucketTask, task.ID, task.clone()); err != nil {
return err
return false, err
}
d.emit(EventKeyPause, task)
return nil
}
go func() {
if err := handler(); err != nil {
d.Logger.Error().Stack().Err(err).Msgf("pause task handle failed, task id: %s", task.ID)
}
}()
return
return true, nil
}
func (d *Downloader) taskIsRunningGeneration(task *Task, generation uint64) bool {
if task == nil || task.statusLock == nil {
return false
}
task.statusLock.Lock()
defer task.statusLock.Unlock()
return task.Status == base.DownloadStatusRunning && task.runGeneration == generation
}
func (d *Downloader) taskIsPausedGeneration(task *Task, generation uint64) bool {
if task == nil || task.statusLock == nil {
return false
}
task.statusLock.Lock()
defer task.statusLock.Unlock()
return (task.Status == base.DownloadStatusPause || task.Status == base.DownloadStatusWait) && task.runGeneration == generation
}
func (d *Downloader) taskStatus(task *Task) base.Status {
if task == nil || task.statusLock == nil {
return ""
}
task.statusLock.Lock()
defer task.statusLock.Unlock()
return task.Status
}
func (d *Downloader) markTaskDone(task *Task) bool {
if task == nil || task.statusLock == nil {
return false
}
task.statusLock.Lock()
defer task.statusLock.Unlock()
if task.Status != base.DownloadStatusRunning {
return false
}
task.updateStatus(base.DownloadStatusDone)
task.runGeneration++
return true
}
func (d *Downloader) markTaskError(task *Task) bool {
if task == nil || task.statusLock == nil {
return false
}
task.statusLock.Lock()
defer task.statusLock.Unlock()
if task.Status != base.DownloadStatusRunning {
return false
}
task.updateStatus(base.DownloadStatusError)
task.runGeneration++
return true
}
// redirect stderr to log file, when panic happened log it
@@ -1686,10 +2066,79 @@ func initTask(task *Task) {
task.statusLock = &sync.Mutex{}
task.lock = &sync.Mutex{}
task.blobRefLock = &sync.Mutex{}
task.speedArr = make([]int64, 0)
task.uploadSpeedArr = make([]int64, 0)
}
func (d *Downloader) releaseBlobTask(task *Task) {
if d.blob == nil || task == nil {
return
}
if task.blobRefLock != nil {
task.blobRefLock.Lock()
raw := task.blobURL
task.blobURL = ""
task.blobRefLock.Unlock()
if raw != "" {
_ = d.blob.Release(raw)
}
}
}
func (d *Downloader) syncBlobTaskLease(task *Task) error {
if d.blob == nil || task == nil || task.Meta == nil || task.Meta.Req == nil {
return nil
}
if task.blobRefLock == nil {
task.blobRefLock = &sync.Mutex{}
}
task.blobRefLock.Lock()
defer task.blobRefLock.Unlock()
terminal := false
if task.statusLock != nil {
task.statusLock.Lock()
terminal = task.Status == base.DownloadStatusDone || task.Status == base.DownloadStatusError
task.statusLock.Unlock()
}
newURL := ""
if !terminal && d.blob.IsURL(task.Meta.Req.URL) {
newURL = task.Meta.Req.URL
}
if task.blobURL == newURL {
return nil
}
if newURL != "" {
if err := d.blob.Acquire(newURL); err != nil {
return err
}
}
oldURL := task.blobURL
task.blobURL = newURL
if oldURL != "" {
_ = d.blob.Release(oldURL)
}
return nil
}
func (d *Downloader) releaseBlobTaskURL(task *Task, raw string) {
if d.blob == nil || task == nil || raw == "" {
return
}
if task.blobRefLock == nil {
task.blobRefLock = &sync.Mutex{}
}
task.blobRefLock.Lock()
claimed := task.blobURL == raw
if claimed {
task.blobURL = ""
}
task.blobRefLock.Unlock()
if claimed {
_ = d.blob.Release(raw)
}
}
var defaultDownloader = NewDownloader(nil)
type boot struct {
File diff suppressed because it is too large. Load diff
+137 -89
View File
@@ -3,16 +3,19 @@ package engine
import (
_ "embed"
"errors"
"fmt"
"sync"
"github.com/GopeedLab/gopeed/pkg/base"
gojaerror "github.com/GopeedLab/gopeed/pkg/download/engine/inject/error"
"github.com/GopeedLab/gopeed/pkg/download/engine/inject/file"
"github.com/GopeedLab/gopeed/pkg/download/engine/inject/formdata"
"github.com/GopeedLab/gopeed/pkg/download/engine/inject/stream"
"github.com/GopeedLab/gopeed/pkg/download/engine/inject/vm"
"github.com/GopeedLab/gopeed/pkg/download/engine/inject/xhr"
"github.com/dop251/goja"
"github.com/dop251/goja_nodejs/eventloop"
gojaurl "github.com/dop251/goja_nodejs/url"
"time"
)
//go:embed polyfill/out/index.js
@@ -24,93 +27,118 @@ type Engine struct {
Runtime *goja.Runtime
}
type JSFunction func(goja.FunctionCall) goja.Value
// RunString executes the script and returns the go type value
// if script result is promise, it will be resolved
func (e *Engine) RunString(script string) (value any, err error) {
defer func() {
if r := recover(); r != nil {
err = r.(error)
}
}()
var result goja.Value
e.loop.Run(func(runtime *goja.Runtime) {
result, err = runtime.RunString(script)
if err == nil {
go e.await(result)
}
return e.runOnLoop(func(runtime *goja.Runtime) (goja.Value, error) {
return runtime.RunString(script)
})
if err != nil {
return
}
return resolveResult(result)
}
// CallFunction calls the function and returns the go type value
// if function result is promise, it will be resolved
func (e *Engine) CallFunction(fn goja.Callable, args ...any) (value any, err error) {
defer func() {
if r := recover(); r != nil {
err = r.(error)
func (e *Engine) CallFunction(fn any, args ...any) (value any, err error) {
return e.runOnLoop(func(runtime *goja.Runtime) (goja.Value, error) {
var jsArgs []goja.Value
for _, arg := range args {
jsArgs = append(jsArgs, runtime.ToValue(arg))
}
}()
var result goja.Value
e.loop.Run(func(runtime *goja.Runtime) {
if args == nil {
result, err = fn(nil)
} else {
var jsArgs []goja.Value
for _, arg := range args {
jsArgs = append(jsArgs, runtime.ToValue(arg))
switch f := fn.(type) {
case goja.Callable:
if args == nil {
return f(nil)
}
result, err = fn(nil, jsArgs...)
}
if err == nil {
go e.await(result)
return f(nil, jsArgs...)
case JSFunction:
return callExportedFunction(func(call goja.FunctionCall) goja.Value {
return f(call)
}, jsArgs...)
default:
return nil, fmt.Errorf("unsupported function type: %T", fn)
}
})
if err != nil {
return
}
return resolveResult(result)
}
// loop.Run will hang if the script result has a non-stop code, such as setInterval.
// This method will stop the event loop when the promise result is resolved.
func (e *Engine) await(value any) {
if value == nil {
return
func (e *Engine) runOnLoop(fn func(runtime *goja.Runtime) (goja.Value, error)) (any, error) {
type result struct {
value any
err error
}
if v, ok := value.(goja.Value); ok {
// if result is promise, wait for it to be resolved
if p, ok := v.Export().(*goja.Promise); ok {
if p.State() != goja.PromiseStatePending {
ch := make(chan result, 1)
ok := e.loop.RunOnLoop(func(runtime *goja.Runtime) {
var once sync.Once
sendResult := func(res result) {
once.Do(func() {
ch <- res
})
}
defer func() {
if r := recover(); r != nil {
switch v := r.(type) {
case error:
sendResult(result{err: v})
case goja.Value:
sendResult(result{err: exportJSError(v)})
default:
sendResult(result{err: fmt.Errorf("panic: %v", r)})
}
}
}()
value, err := fn(runtime)
if err != nil {
sendResult(result{err: err})
return
}
if p, ok := value.Export().(*goja.Promise); ok {
switch p.State() {
case goja.PromiseStateFulfilled:
sendResult(result{value: exportJSValue(p.Result())})
return
case goja.PromiseStateRejected:
sendResult(result{err: exportJSError(p.Result())})
return
}
// check promise state every 100 milliseconds, until it is resolved
for {
time.Sleep(time.Millisecond * 100)
if p.State() == goja.PromiseStatePending {
continue
}
break
promiseObj := value.ToObject(runtime)
thenVal := promiseObj.Get("then")
thenFn, ok := goja.AssertFunction(thenVal)
if !ok {
sendResult(result{err: errors.New("promise.then is not callable")})
return
}
// stop the event loop
e.loop.StopNoWait()
onFulfilled := runtime.ToValue(func(call goja.FunctionCall) goja.Value {
sendResult(result{value: exportJSValue(call.Argument(0))})
return goja.Undefined()
})
onRejected := runtime.ToValue(func(call goja.FunctionCall) goja.Value {
sendResult(result{err: exportJSError(call.Argument(0))})
return goja.Undefined()
})
if _, err := thenFn(promiseObj, onFulfilled, onRejected); err != nil {
sendResult(result{err: err})
}
return
}
sendResult(result{value: exportJSValue(value)})
})
if !ok {
return nil, errors.New("engine loop terminated")
}
res := <-ch
if res.err != nil {
return nil, res.err
}
return res.value, nil
}
func (e *Engine) Close() {
e.loop.StopNoWait()
e.loop.Terminate()
}
type Config struct {
ProxyConfig *base.DownloaderProxyConfig
ProxyConfig *base.DownloaderProxyConfig
StreamConfig *stream.Config
}
func NewEngine(cfg *Config) *Engine {
@@ -121,7 +149,10 @@ func NewEngine(cfg *Config) *Engine {
engine := &Engine{
loop: loop,
}
loop.Run(func(runtime *goja.Runtime) {
loop.Start()
done := make(chan struct{})
loop.RunOnLoop(func(runtime *goja.Runtime) {
defer close(done)
engine.Runtime = runtime
runtime.SetFieldNameMapper(goja.TagFieldNameMapper("json", true))
vm.Enable(runtime)
@@ -153,8 +184,12 @@ func NewEngine(cfg *Config) *Engine {
if _, err := runtime.RunString("global.location = new URL('http://localhost');"); err != nil {
return
}
if err := stream.Enable(runtime, loop, cfg.StreamConfig); err != nil {
return
}
return
})
<-done
return engine
}
@@ -163,32 +198,45 @@ func Run(script string) (value any, err error) {
return engine.RunString(script)
}
// if the value is Promise, it will be resolved and return the result.
func resolveResult(value goja.Value) (any, error) {
export := value.Export()
switch export.(type) {
case *goja.Promise:
p := export.(*goja.Promise)
switch p.State() {
case goja.PromiseStatePending:
return nil, nil
case goja.PromiseStateFulfilled:
return p.Result().Export(), nil
case goja.PromiseStateRejected:
if err, ok := p.Result().Export().(error); ok {
return nil, err
} else {
stack := p.Result().String()
result := p.Result()
if ro, ok := result.(*goja.Object); ok {
stackVal := ro.Get("stack")
if stackVal != nil && stackVal.String() != "" {
stack = stackVal.String()
}
}
return nil, errors.New(stack)
}
func exportJSValue(value goja.Value) any {
if value == nil {
return nil
}
return value.Export()
}
func exportJSError(value goja.Value) error {
if value == nil || goja.IsUndefined(value) || goja.IsNull(value) {
return errors.New("promise rejected")
}
if err, ok := value.Export().(error); ok {
return err
}
stack := value.String()
if ro, ok := value.(*goja.Object); ok {
stackVal := ro.Get("stack")
if stackVal != nil && stackVal.String() != "" {
stack = stackVal.String()
}
}
return export, nil
return errors.New(stack)
}
func callExportedFunction(fn func(goja.FunctionCall) goja.Value, args ...goja.Value) (ret goja.Value, err error) {
defer func() {
if r := recover(); r != nil {
switch v := r.(type) {
case error:
err = v
case goja.Value:
err = exportJSError(v)
default:
err = fmt.Errorf("panic: %v", r)
}
}
}()
return fn(goja.FunctionCall{
This: nil,
Arguments: args,
}), nil
}
+149 -1
View File
@@ -26,7 +26,11 @@ func TestPolyfill(t *testing.T) {
doTestPolyfill(t, "MessageError")
doTestPolyfill(t, "XMLHttpRequest")
doTestPolyfill(t, "Blob")
doTestPolyfill(t, "Event")
doTestPolyfill(t, "EventTarget")
doTestPolyfill(t, "FormData")
doTestPolyfill(t, "AbortController")
doTestPolyfill(t, "AbortSignal")
doTestPolyfill(t, "TextDecoder")
doTestPolyfill(t, "TextEncoder")
doTestPolyfill(t, "fetch")
@@ -36,13 +40,75 @@ func TestPolyfill(t *testing.T) {
func TestError(t *testing.T) {
engine := NewEngine(nil)
_, err := engine.RunString(`
throw new MessageError('test');
throw new MessageError('test');
`)
if me, ok := gojautil.AssertError[*gojaerror.MessageError](err); !ok {
t.Fatalf("expect MessageError, but got %v", me)
}
}
func TestCallFunction_DetachedAsyncWorkDoesNotBlockReturn(t *testing.T) {
engine := NewEngine(nil)
defer engine.Close()
if _, err := engine.RunString(`
globalThis.__done = false;
async function testDetached() {
(async () => {
await new Promise((resolve) => setTimeout(resolve, 500));
globalThis.__done = true;
})();
return "ok";
}
`); err != nil {
t.Fatal(err)
}
fn, ok := goja.AssertFunction(engine.Runtime.Get("testDetached"))
if !ok {
t.Fatal("testDetached is not callable")
}
startedAt := time.Now()
value, err := engine.CallFunction(fn)
if err != nil {
t.Fatal(err)
}
elapsed := time.Since(startedAt)
if elapsed > 200*time.Millisecond {
t.Fatalf("detached async work blocked return for %s", elapsed)
}
if value != "ok" {
t.Fatalf("unexpected return value: %#v", value)
}
}
func TestCallFunction_AwaitedPromiseBlocksUntilSettled(t *testing.T) {
engine := NewEngine(nil)
defer engine.Close()
if _, err := engine.RunString(`
async function testAwaited() {
await new Promise((resolve) => setTimeout(resolve, 120));
return "ok";
}
`); err != nil {
t.Fatal(err)
}
fn, ok := goja.AssertFunction(engine.Runtime.Get("testAwaited"))
if !ok {
t.Fatal("testAwaited is not callable")
}
startedAt := time.Now()
value, err := engine.CallFunction(fn)
if err != nil {
t.Fatal(err)
}
elapsed := time.Since(startedAt)
if elapsed < 100*time.Millisecond {
t.Fatalf("awaited promise returned too early: %s", elapsed)
}
if value != "ok" {
t.Fatalf("unexpected return value: %#v", value)
}
}
func TestFetch(t *testing.T) {
server := startServer()
defer server.Close()
@@ -64,6 +130,50 @@ async function testText(){
return await resp.text();
}
async function testResponseBodyGetReader() {
const resp = await fetch(host+'/text', {
method: 'POST',
body: 'stream-body'
});
if (!resp.body || typeof resp.body.getReader !== 'function') {
throw new Error('response.body.getReader is not available');
}
const reader = resp.body.getReader();
const decoder = new TextDecoder();
let result = '';
while (true) {
const { done, value } = await reader.read();
if (done) {
break;
}
result += decoder.decode(value, { stream: true });
}
result += decoder.decode();
return result;
}
async function testStreamingFetchResponseBody() {
const startedAt = Date.now();
const resp = await fetch(host+'/stream');
const resolvedAt = Date.now() - startedAt;
const reader = resp.body.getReader();
const decoder = new TextDecoder();
const first = await reader.read();
const firstAt = Date.now() - startedAt;
const second = await reader.read();
const secondAt = Date.now() - startedAt;
let result = '';
result += decoder.decode(first.value, { stream: true });
result += decoder.decode(second.value, { stream: true });
result += decoder.decode();
return {
resolvedAt,
firstAt,
secondAt,
text: result
};
}
async function testOctetStream(file){
const resp = await fetch(host+'/octetStream',{
method: 'POST',
@@ -251,6 +361,32 @@ async function testFingerprintSafari(){
t.Fatalf("testText failed, want %s, got %s", "test", result)
}
result, err = callTestFun(engine, "testResponseBodyGetReader")
if err != nil {
t.Fatal(err)
}
if result != "stream-body" {
t.Fatalf("testResponseBodyGetReader failed, want %s, got %s", "stream-body", result)
}
result, err = callTestFun(engine, "testStreamingFetchResponseBody")
if err != nil {
t.Fatal(err)
}
streaming := result.(map[string]any)
if streaming["text"] != "chunk-1chunk-2" {
t.Fatalf("testStreamingFetchResponseBody failed, want %s, got %v", "chunk-1chunk-2", streaming["text"])
}
if int(streaming["resolvedAt"].(int64)) >= 120 {
t.Fatalf("fetch resolved too late, got %dms", streaming["resolvedAt"].(int64))
}
if int(streaming["firstAt"].(int64)) >= 120 {
t.Fatalf("first chunk arrived too late, got %dms", streaming["firstAt"].(int64))
}
if int(streaming["secondAt"].(int64)) < 120 {
t.Fatalf("second chunk arrived too early, got %dms", streaming["secondAt"].(int64))
}
func() {
jsFile, _, md5 := buildFile(t, engine.Runtime)
result, err = callTestFun(engine, "testOctetStream", jsFile)
@@ -464,6 +600,18 @@ func startServer() net.Listener {
w.WriteHeader(http.StatusOK)
w.Write(buf)
})
mux.HandleFunc("/stream", func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
if flusher, ok := w.(http.Flusher); ok {
_, _ = w.Write([]byte("chunk-1"))
flusher.Flush()
time.Sleep(150 * time.Millisecond)
_, _ = w.Write([]byte("chunk-2"))
flusher.Flush()
return
}
_, _ = w.Write([]byte("chunk-1chunk-2"))
})
mux.HandleFunc("/octetStream", func(w http.ResponseWriter, r *http.Request) {
md5 := calcMd5(r.Body)
w.WriteHeader(http.StatusOK)
+990
View File
@@ -0,0 +1,990 @@
package stream
import (
"context"
_ "embed"
"errors"
"fmt"
"io"
"net"
"net/http"
"net/url"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/GopeedLab/gopeed/pkg/download/engine/inject/file"
"github.com/GopeedLab/gopeed/pkg/download/engine/inject/formdata"
"github.com/GopeedLab/gopeed/pkg/download/engine/inject/xhr"
"github.com/GopeedLab/gopeed/pkg/download/engine/util"
"github.com/dop251/goja"
"github.com/dop251/goja_nodejs/eventloop"
"github.com/imroc/req/v3"
)
//go:embed stream.js
var script string
type Config struct {
CreateObjectURL func(opts *ObjectURLOptions, open ObjectURLOpener) (string, error)
RevokeObjectURL func(url string) error
ProxyHandler func(r *http.Request) (*url.URL, error)
RegisterCleanup func(cleanup func())
}
type ObjectURLOptions struct {
ContentType string
Size int64
Range bool
}
type ObjectURLOpenRequest struct {
Offset int64
End int64
}
type ObjectURLOpener func(ctx context.Context, req ObjectURLOpenRequest) (io.ReadCloser, error)
func Enable(runtime *goja.Runtime, loop *eventloop.EventLoop, cfg *Config) error {
if cfg == nil {
cfg = &Config{}
}
blobPipes := newBlobPipeRegistry()
if err := runtime.Set("__gopeed_create_blob_object_url", func(call goja.FunctionCall) goja.Value {
if cfg.CreateObjectURL == nil {
panic(runtime.NewGoError(fmt.Errorf("blob object url handler not configured")))
}
openValue := call.Argument(0)
if _, ok := goja.AssertFunction(openValue); !ok {
panic(runtime.NewGoError(fmt.Errorf("blob opener must be callable")))
}
optsObj := call.Argument(1).ToObject(runtime)
size := int64(0)
contentType := ""
rangeEnabled := false
if optsObj != nil {
size = optsObj.Get("size").ToInteger()
contentType = optsObj.Get("contentType").String()
rangeEnabled = optsObj.Get("range").ToBoolean()
}
if size < 0 {
size = 0
}
opts := &ObjectURLOptions{
ContentType: contentType,
Size: size,
Range: rangeEnabled,
}
url, err := cfg.CreateObjectURL(opts, func(ctx context.Context, req ObjectURLOpenRequest) (io.ReadCloser, error) {
return openBlobObjectURLReader(ctx, loop, blobPipes, openValue, req)
})
if err != nil {
panic(runtime.NewGoError(err))
}
return runtime.ToValue(url)
}); err != nil {
return err
}
if err := runtime.Set("__gopeed_revoke_object_url", func(url string) {
if cfg.RevokeObjectURL == nil {
panic(runtime.NewGoError(fmt.Errorf("blob revoke handler not configured")))
}
if err := cfg.RevokeObjectURL(url); err != nil {
panic(runtime.NewGoError(err))
}
}); err != nil {
return err
}
fetchRegistry := newFetchRegistry()
if cfg.RegisterCleanup != nil {
cfg.RegisterCleanup(fetchRegistry.CloseAll)
}
if err := runtime.Set("__gopeed_fetch_open", func(call goja.FunctionCall) goja.Value {
reqMeta, err := exportFetchRequest(runtime, call.Argument(0))
if err != nil {
panic(runtime.NewGoError(err))
}
fingerprint := util.SafeGet[string](runtime, xhr.FingerprintMagicKey)
promise, resolve, reject := runtime.NewPromise()
go func() {
meta, err := fetchRegistry.Open(fingerprint, cfg.ProxyHandler, reqMeta)
ok := loop.RunOnLoop(func(runtime *goja.Runtime) {
if err != nil {
reject(runtime.NewGoError(err))
return
}
resolve(runtime.ToValue(meta))
})
if !ok && meta != nil {
fetchRegistry.Close(meta.ID)
}
}()
return runtime.ToValue(promise)
}); err != nil {
return err
}
if err := runtime.Set("__gopeed_fetch_read", func(call goja.FunctionCall) goja.Value {
id := call.Argument(0).String()
chunkSize := int(call.Argument(1).ToInteger())
promise, resolve, reject := runtime.NewPromise()
go func() {
chunk, done, err := fetchRegistry.Read(id, chunkSize)
ok := loop.RunOnLoop(func(runtime *goja.Runtime) {
if err != nil {
reject(runtime.NewGoError(err))
return
}
if done {
resolve(goja.Null())
return
}
resolve(runtime.ToValue(runtime.NewArrayBuffer(chunk)))
})
if !ok {
fetchRegistry.Close(id)
}
}()
return runtime.ToValue(promise)
}); err != nil {
return err
}
if err := runtime.Set("__gopeed_fetch_close", func(id string) {
fetchRegistry.Close(id)
}); err != nil {
return err
}
if err := runtime.Set("__gopeed_fetch_abort", func(id string, reason string) {
fetchRegistry.Abort(id, reason)
}); err != nil {
return err
}
if err := runtime.Set("__gopeed_blob_pipe_chunk", func(call goja.FunctionCall) goja.Value {
id := call.Argument(0).String()
ok := blobPipes.Push(id, exportBytes(call.Argument(1)))
return runtime.ToValue(ok)
}); err != nil {
return err
}
if err := runtime.Set("__gopeed_blob_pipe_close", func(id string) {
blobPipes.Close(id, nil)
}); err != nil {
return err
}
if err := runtime.Set("__gopeed_blob_pipe_error", func(id string, message string) {
blobPipes.Close(id, errors.New(message))
}); err != nil {
return err
}
_, err := runtime.RunString(script)
return err
}
func openBlobObjectURLReader(ctx context.Context, loop *eventloop.EventLoop, pipes *blobPipeRegistry, openValue goja.Value, req ObjectURLOpenRequest) (io.ReadCloser, error) {
if pipes == nil {
return nil, fmt.Errorf("blob pipe registry not configured")
}
pipe := newBlobPipe(ctx, loop, pipes)
pipes.Add(pipe)
_, err := runOnLoop(loop, func(runtime *goja.Runtime) (goja.Value, error) {
fnVal := runtime.Get("__gopeed_blob_pipe_source")
fn, ok := goja.AssertFunction(fnVal)
if !ok {
return nil, fmt.Errorf("blob pipe helper is not callable")
}
request := map[string]any{
"offset": req.Offset,
"end": req.End,
}
return fn(nil, openValue, runtime.ToValue(request), runtime.ToValue(pipe.id))
})
if err != nil {
pipe.Close()
return nil, err
}
return pipe, nil
}
type blobPipeRegistry struct {
mu sync.Mutex
pipes map[string]*blobPipe
next atomic.Uint64
}
func newBlobPipeRegistry() *blobPipeRegistry {
return &blobPipeRegistry{
pipes: map[string]*blobPipe{},
}
}
func (r *blobPipeRegistry) Add(pipe *blobPipe) {
r.mu.Lock()
r.pipes[pipe.id] = pipe
r.mu.Unlock()
}
func (r *blobPipeRegistry) Remove(id string) {
r.mu.Lock()
delete(r.pipes, id)
r.mu.Unlock()
}
func (r *blobPipeRegistry) Push(id string, chunk []byte) bool {
r.mu.Lock()
pipe := r.pipes[id]
r.mu.Unlock()
if pipe == nil {
return false
}
return pipe.Push(chunk)
}
func (r *blobPipeRegistry) Close(id string, err error) {
r.mu.Lock()
pipe := r.pipes[id]
r.mu.Unlock()
if pipe != nil {
pipe.CloseWithError(err)
}
}
func (r *blobPipeRegistry) NewID() string {
return fmt.Sprintf("pipe-%d", r.next.Add(1))
}
type blobPipeItem struct {
chunk []byte
err error
done bool
}
type blobPipe struct {
id string
loop *eventloop.EventLoop
registry *blobPipeRegistry
ctx context.Context
cancel context.CancelFunc
ch chan blobPipeItem
mu sync.Mutex
buf []byte
closed bool
once sync.Once
}
func newBlobPipe(ctx context.Context, loop *eventloop.EventLoop, registry *blobPipeRegistry) *blobPipe {
pipeCtx, cancel := context.WithCancel(ctx)
return &blobPipe{
id: registry.NewID(),
loop: loop,
registry: registry,
ctx: pipeCtx,
cancel: cancel,
ch: make(chan blobPipeItem, 8),
}
}
func (p *blobPipe) Push(chunk []byte) bool {
if len(chunk) == 0 {
return true
}
buf := append([]byte(nil), chunk...)
select {
case p.ch <- blobPipeItem{chunk: buf}:
return true
case <-p.ctx.Done():
return false
}
}
func (p *blobPipe) CloseWithError(err error) {
p.once.Do(func() {
p.registry.Remove(p.id)
select {
case p.ch <- blobPipeItem{done: true, err: err}:
case <-p.ctx.Done():
}
})
}
func (p *blobPipe) Read(dst []byte) (int, error) {
p.mu.Lock()
if len(p.buf) > 0 {
n := copy(dst, p.buf)
p.buf = p.buf[n:]
p.mu.Unlock()
return n, nil
}
p.mu.Unlock()
select {
case item := <-p.ch:
if item.done {
if item.err != nil {
return 0, item.err
}
return 0, io.EOF
}
n := copy(dst, item.chunk)
if n < len(item.chunk) {
p.mu.Lock()
p.buf = append(p.buf, item.chunk[n:]...)
p.mu.Unlock()
}
return n, nil
case <-p.ctx.Done():
return 0, p.ctx.Err()
}
}
func (p *blobPipe) Close() error {
p.mu.Lock()
if p.closed {
p.mu.Unlock()
return nil
}
p.closed = true
p.mu.Unlock()
p.registry.Remove(p.id)
p.cancel()
p.cancelSourceReader()
return nil
}
func (p *blobPipe) cancelSourceReader() {
if p.loop == nil {
return
}
// Invoke the JavaScript cancellation before returning from Close. The
// source session may be released immediately after the HTTP handler closes
// this reader, so merely queueing the callback could let the engine stop
// before reader.cancel() is called.
_, _ = runOnLoop(p.loop, func(runtime *goja.Runtime) (goja.Value, error) {
fnVal := runtime.Get("__gopeed_blob_cancel_pipe_source")
fn, ok := goja.AssertFunction(fnVal)
if !ok {
return goja.Undefined(), nil
}
_, err := fn(nil, runtime.ToValue(p.id), runtime.ToValue("blob request closed"))
return goja.Undefined(), err
})
}
func exportBytes(value goja.Value) []byte {
if value == nil || goja.IsNull(value) || goja.IsUndefined(value) {
return nil
}
if ab, ok := value.Export().(goja.ArrayBuffer); ok {
return ab.Bytes()
}
if b, ok := value.Export().([]byte); ok {
return b
}
return nil
}
type blobObjectURLReader struct {
ctx context.Context
loop *eventloop.EventLoop
id string
mu sync.Mutex
buf []byte
closed bool
}
func (r *blobObjectURLReader) Read(p []byte) (int, error) {
r.mu.Lock()
if len(r.buf) > 0 {
n := copy(p, r.buf)
r.buf = r.buf[n:]
r.mu.Unlock()
return n, nil
}
closed := r.closed
r.mu.Unlock()
if closed {
return 0, io.EOF
}
if err := r.ctx.Err(); err != nil {
return 0, err
}
chunkSize := len(p)
if chunkSize <= 0 {
chunkSize = 32 * 1024
}
value, err := runOnLoop(r.loop, func(runtime *goja.Runtime) (goja.Value, error) {
fnVal := runtime.Get("__gopeed_blob_read_source")
fn, ok := goja.AssertFunction(fnVal)
if !ok {
return nil, fmt.Errorf("blob read helper is not callable")
}
return fn(nil, runtime.ToValue(r.id), runtime.ToValue(chunkSize))
})
if err != nil {
return 0, err
}
if goja.IsNull(value) || goja.IsUndefined(value) {
return 0, io.EOF
}
var chunk []byte
if ab, ok := value.Export().(goja.ArrayBuffer); ok {
chunk = ab.Bytes()
} else if b, ok := value.Export().([]byte); ok {
chunk = b
} else {
return 0, fmt.Errorf("blob read helper returned %T", value.Export())
}
if len(chunk) == 0 {
return 0, nil
}
n := copy(p, chunk)
if n < len(chunk) {
r.mu.Lock()
r.buf = append(r.buf, chunk[n:]...)
r.mu.Unlock()
}
return n, nil
}
func (r *blobObjectURLReader) Close() error {
r.mu.Lock()
if r.closed {
r.mu.Unlock()
return nil
}
r.closed = true
r.mu.Unlock()
_, err := runOnLoop(r.loop, func(runtime *goja.Runtime) (goja.Value, error) {
fnVal := runtime.Get("__gopeed_blob_close_source")
fn, ok := goja.AssertFunction(fnVal)
if !ok {
return nil, nil
}
return fn(nil, runtime.ToValue(r.id))
})
return err
}
func runOnLoop(loop *eventloop.EventLoop, fn func(runtime *goja.Runtime) (goja.Value, error)) (goja.Value, error) {
type result struct {
value goja.Value
err error
}
ch := make(chan result, 1)
ok := loop.RunOnLoop(func(runtime *goja.Runtime) {
send := func(value goja.Value, err error) {
select {
case ch <- result{value: value, err: err}:
default:
}
}
defer func() {
if r := recover(); r != nil {
switch v := r.(type) {
case error:
send(nil, v)
case goja.Value:
send(nil, exportJSError(v))
default:
send(nil, fmt.Errorf("panic: %v", r))
}
}
}()
value, err := fn(runtime)
if err != nil {
send(nil, err)
return
}
if promise, ok := value.Export().(*goja.Promise); ok {
switch promise.State() {
case goja.PromiseStateFulfilled:
send(promise.Result(), nil)
return
case goja.PromiseStateRejected:
send(nil, exportJSError(promise.Result()))
return
default:
thenVal := value.ToObject(runtime).Get("then")
thenFn, ok := goja.AssertFunction(thenVal)
if !ok {
send(nil, errors.New("promise.then is not callable"))
return
}
onFulfilled := runtime.ToValue(func(call goja.FunctionCall) goja.Value {
send(call.Argument(0), nil)
return goja.Undefined()
})
onRejected := runtime.ToValue(func(call goja.FunctionCall) goja.Value {
send(nil, exportJSError(call.Argument(0)))
return goja.Undefined()
})
if _, err := thenFn(value, onFulfilled, onRejected); err != nil {
send(nil, err)
}
return
}
}
send(value, nil)
})
if !ok {
return nil, errors.New("engine loop terminated")
}
res := <-ch
return res.value, res.err
}
func exportJSError(value goja.Value) error {
if value == nil || goja.IsUndefined(value) || goja.IsNull(value) {
return errors.New("promise rejected")
}
if err, ok := value.Export().(error); ok {
return err
}
stack := value.String()
if ro, ok := value.(*goja.Object); ok {
stackVal := ro.Get("stack")
if stackVal != nil && stackVal.String() != "" {
stack = stackVal.String()
}
}
return errors.New(stack)
}
type fetchRegistry struct {
mu sync.Mutex
streams map[string]*fetchStream
ctx context.Context
cancel context.CancelFunc
closed bool
}
type fetchStream struct {
body io.ReadCloser
cancel context.CancelFunc
closeOnce sync.Once
ctx context.Context
ch chan fetchChunk
readMu sync.Mutex
pending []byte
}
type fetchChunk struct {
data []byte
err error
}
type fetchRequest struct {
URL string
Method string
Headers [][2]string
Body any
Redirect string
Credentials string
}
type fetchFormDataEntry struct {
name string
value any
}
type fetchFormDataSnapshot struct {
entries []fetchFormDataEntry
}
type fetchOpenMeta struct {
ID string `json:"id"`
Status int `json:"status"`
StatusText string `json:"statusText"`
URL string `json:"url"`
Headers [][2]string `json:"headers"`
}
func newFetchRegistry() *fetchRegistry {
ctx, cancel := context.WithCancel(context.Background())
return &fetchRegistry{
streams: make(map[string]*fetchStream),
ctx: ctx,
cancel: cancel,
}
}
func exportFetchRequest(runtime *goja.Runtime, value goja.Value) (*fetchRequest, error) {
obj := value.ToObject(runtime)
if obj == nil {
return nil, fmt.Errorf("invalid fetch request")
}
meta := &fetchRequest{
URL: obj.Get("url").String(),
Method: obj.Get("method").String(),
Redirect: obj.Get("redirect").String(),
Credentials: obj.Get("credentials").String(),
}
if meta.Method == "" {
meta.Method = http.MethodGet
}
if headersVal := obj.Get("headers"); headersVal != nil && !goja.IsUndefined(headersVal) && !goja.IsNull(headersVal) {
if exported, ok := headersVal.Export().([]any); ok {
for _, item := range exported {
pair, ok := item.([]any)
if !ok || len(pair) != 2 {
continue
}
meta.Headers = append(meta.Headers, [2]string{fmt.Sprint(pair[0]), fmt.Sprint(pair[1])})
}
}
}
bodyVal := obj.Get("body")
if bodyVal != nil && !goja.IsUndefined(bodyVal) && !goja.IsNull(bodyVal) {
body, err := snapshotFetchBody(bodyVal.Export())
if err != nil {
return nil, err
}
meta.Body = body
}
return meta, nil
}
// snapshotFetchBody runs on the JavaScript event loop before fetch work is
// handed to a background goroutine. Values exported by goja may still share
// mutable storage with JavaScript (notably ArrayBuffer and FormData), so the
// background request must only retain immutable Go-owned data.
func snapshotFetchBody(body any) (any, error) {
switch v := body.(type) {
case nil, string:
return v, nil
case []byte:
return append([]byte(nil), v...), nil
case goja.ArrayBuffer:
return append([]byte(nil), v.Bytes()...), nil
case *file.File:
return snapshotFetchFile(v), nil
case *formdata.FormData:
entries := v.Entries()
snapshot := &fetchFormDataSnapshot{
entries: make([]fetchFormDataEntry, 0, len(entries)),
}
for _, entry := range entries {
pair, ok := entry.([]any)
if !ok || len(pair) != 2 {
return nil, fmt.Errorf("invalid FormData entry %T", entry)
}
name, ok := pair[0].(string)
if !ok {
return nil, fmt.Errorf("invalid FormData field name %T", pair[0])
}
value := pair[1]
if formFile, ok := value.(*file.File); ok {
value = snapshotFetchFile(formFile)
}
snapshot.entries = append(snapshot.entries, fetchFormDataEntry{
name: name,
value: value,
})
}
return snapshot, nil
default:
if typed, ok := v.(interface{ Bytes() []byte }); ok {
return append([]byte(nil), typed.Bytes()...), nil
}
return v, nil
}
}
func snapshotFetchFile(src *file.File) *file.File {
if src == nil {
return nil
}
return &file.File{
Reader: src.Reader,
Closer: src.Closer,
Name: src.Name,
Size: src.Size,
}
}
func (r *fetchRegistry) Open(fingerprint string, proxyHandler func(r *http.Request) (*url.URL, error), reqMeta *fetchRequest) (*fetchOpenMeta, error) {
r.mu.Lock()
if r.closed {
r.mu.Unlock()
return nil, context.Canceled
}
registryCtx := r.ctx
r.mu.Unlock()
client := req.C()
if proxyHandler != nil {
client.SetProxy(proxyHandler)
}
setFetchFingerprint(client, fingerprint)
contentType, body, err := buildFetchBody(reqMeta.Body)
if err != nil {
return nil, err
}
ctx, cancel := context.WithCancel(registryCtx)
keepCancel := false
defer func() {
if !keepCancel {
cancel()
}
}()
reqBuilder := client.R()
reqBuilder.SetContext(ctx)
reqBuilder.DisableAutoReadResponse()
for _, header := range reqMeta.Headers {
reqBuilder.SetHeader(header[0], header[1])
}
if body != nil && reqMeta.Method != http.MethodGet && reqMeta.Method != http.MethodHead {
reqBuilder.SetBody(body)
if contentType != "" && !hasHeader(reqMeta.Headers, "Content-Type") {
reqBuilder.SetHeader("Content-Type", contentType)
}
}
client.SetRedirectPolicy(func(req *http.Request, via []*http.Request) error {
switch reqMeta.Redirect {
case "manual":
return http.ErrUseLastResponse
case "error":
return fmt.Errorf("redirect failed")
default:
if len(via) > 20 {
return fmt.Errorf("too many redirects")
}
return nil
}
})
resp, err := reqBuilder.Send(reqMeta.Method, reqMeta.URL)
if err != nil {
var ne net.Error
if errorsAsTimeout(err, &ne) {
return nil, fmt.Errorf("Network request timed out")
}
return nil, fmt.Errorf("Network request failed: %w", err)
}
id := fmt.Sprintf("%d", time.Now().UnixNano())
meta := &fetchOpenMeta{
ID: id,
Status: resp.StatusCode,
StatusText: resp.Status,
URL: reqMeta.URL,
}
if resp.Response != nil && resp.Response.Request != nil && resp.Response.Request.URL != nil {
responseURL := *resp.Response.Request.URL
responseURL.Fragment = ""
meta.URL = responseURL.String()
}
for key, values := range resp.Header {
meta.Headers = append(meta.Headers, [2]string{key, strings.Join(values, ", ")})
}
bodyCloser := io.NopCloser(strings.NewReader(""))
if resp.Response != nil && resp.Response.Body != nil {
bodyCloser = resp.Response.Body
}
stream := &fetchStream{
body: bodyCloser,
cancel: cancel,
ctx: ctx,
ch: make(chan fetchChunk, 8),
}
r.mu.Lock()
if r.closed {
r.mu.Unlock()
closeFetchStream(stream)
return nil, context.Canceled
}
r.streams[id] = stream
r.mu.Unlock()
keepCancel = true
go stream.readLoop()
return meta, nil
}
func (r *fetchRegistry) Read(id string, chunkSize int) ([]byte, bool, error) {
stream := r.get(id)
if stream == nil {
return nil, true, nil
}
stream.readMu.Lock()
defer stream.readMu.Unlock()
if len(stream.pending) > 0 {
n := len(stream.pending)
if chunkSize > 0 && n > chunkSize {
n = chunkSize
}
chunk := append([]byte(nil), stream.pending[:n]...)
stream.pending = stream.pending[n:]
return chunk, false, nil
}
select {
case item, ok := <-stream.ch:
if !ok {
r.Close(id)
return nil, true, nil
}
if item.err != nil {
r.Close(id)
return nil, false, item.err
}
if chunkSize > 0 && len(item.data) > chunkSize {
head := append([]byte(nil), item.data[:chunkSize]...)
stream.pending = append(stream.pending[:0], item.data[chunkSize:]...)
return head, false, nil
}
return item.data, false, nil
case <-stream.ctx.Done():
return nil, false, stream.ctx.Err()
}
}
func (s *fetchStream) readLoop() {
defer close(s.ch)
buf := make([]byte, 64*1024)
for {
n, err := s.body.Read(buf)
if n > 0 {
chunk := append([]byte(nil), buf[:n]...)
select {
case s.ch <- fetchChunk{data: chunk}:
case <-s.ctx.Done():
return
}
}
if err == io.EOF {
return
}
if err != nil {
select {
case s.ch <- fetchChunk{err: err}:
case <-s.ctx.Done():
}
return
}
}
}
func (r *fetchRegistry) Close(id string) {
r.mu.Lock()
stream := r.streams[id]
delete(r.streams, id)
r.mu.Unlock()
closeFetchStream(stream)
}
// CloseAll permanently closes the registry and cancels both active and
// in-flight fetches. It is safe to call more than once.
func (r *fetchRegistry) CloseAll() {
if r == nil {
return
}
r.mu.Lock()
if r.closed {
r.mu.Unlock()
return
}
r.closed = true
if r.cancel != nil {
r.cancel()
}
streams := r.streams
r.streams = make(map[string]*fetchStream)
r.mu.Unlock()
for _, stream := range streams {
closeFetchStream(stream)
}
}
func closeFetchStream(stream *fetchStream) {
if stream == nil {
return
}
stream.closeOnce.Do(func() {
if stream.cancel != nil {
stream.cancel()
}
_ = stream.body.Close()
})
}
func (r *fetchRegistry) Abort(id string, _ string) {
r.Close(id)
}
func (r *fetchRegistry) get(id string) *fetchStream {
r.mu.Lock()
defer r.mu.Unlock()
return r.streams[id]
}
func buildFetchBody(body any) (string, any, error) {
switch v := body.(type) {
case nil:
return "", nil, nil
case string:
return "text/plain;charset=UTF-8", v, nil
case []byte:
return "application/octet-stream", v, nil
case goja.ArrayBuffer:
return "application/octet-stream", v.Bytes(), nil
case *file.File:
return "application/octet-stream", v.Reader, nil
case *fetchFormDataSnapshot:
pr, pw := io.Pipe()
mw := xhr.NewMultipart(pw)
for _, entry := range v.entries {
switch vv := entry.value.(type) {
case string:
if err := mw.WriteField(entry.name, vv); err != nil {
_ = pw.CloseWithError(err)
return "", nil, err
}
case *file.File:
if err := mw.WriteFile(entry.name, vv); err != nil {
_ = pw.CloseWithError(err)
return "", nil, err
}
}
}
go func() {
if err := mw.Send(); err != nil {
_ = pw.CloseWithError(err)
return
}
if err := mw.Close(); err != nil {
_ = pw.CloseWithError(err)
return
}
_ = pw.Close()
}()
return mw.FormDataContentType(), pr, nil
default:
if typed, ok := v.(interface{ Bytes() []byte }); ok {
return "application/octet-stream", typed.Bytes(), nil
}
return "", fmt.Sprint(v), nil
}
}
func hasHeader(headers [][2]string, key string) bool {
for _, header := range headers {
if strings.EqualFold(header[0], key) {
return true
}
}
return false
}
func setFetchFingerprint(client *req.Client, fingerprint string) {
switch fingerprint {
case "chrome":
client.ImpersonateChrome()
case "firefox":
client.ImpersonateFirefox()
case "safari":
client.ImpersonateSafari()
}
}
func errorsAsTimeout(err error, target *net.Error) bool {
return errors.As(err, target) && (*target).Timeout()
}
@@ -0,0 +1,278 @@
package stream_test
import (
"context"
"errors"
"fmt"
"io"
"net/http"
"net/http/httptest"
"sync/atomic"
"testing"
"time"
"github.com/GopeedLab/gopeed/pkg/download/engine"
"github.com/GopeedLab/gopeed/pkg/download/engine/inject/stream"
)
type createdObjectURL struct {
opts *stream.ObjectURLOptions
open stream.ObjectURLOpener
}
func TestResponseBlobObjectURLReadsResponseStream(t *testing.T) {
const payload = "response blob payload"
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.Header().Set("Content-Type", "text/plain")
_, _ = io.WriteString(w, payload[:8])
w.(http.Flusher).Flush() // force an unknown-length/chunked response
_, _ = io.WriteString(w, payload[8:])
}))
defer server.Close()
created := make(chan createdObjectURL, 1)
engine, cleanup := newStreamTestEngine(t, created)
defer func() {
cleanup()
engine.Close()
}()
value, err := engine.RunString(fmt.Sprintf(`
(async () => {
const response = await fetch(%q);
const blob = await response.blob();
return __gopeed_blob_create_object_url(blob);
})()
`, server.URL))
if err != nil {
t.Fatal(err)
}
if value != "blob:test-1" {
t.Fatalf("unexpected object URL: %#v", value)
}
object := <-created
if object.opts.Size != int64(len(payload)) || object.opts.ContentType != "text/plain" {
t.Fatalf("unexpected object URL options: %#v", object.opts)
}
if !object.opts.Range {
t.Fatal("materialized response Blob should advertise range support")
}
for i := 0; i < 2; i++ {
reader, err := object.open(context.Background(), stream.ObjectURLOpenRequest{Offset: 0, End: -1})
if err != nil {
t.Fatal(err)
}
data, readErr := io.ReadAll(reader)
_ = reader.Close()
if readErr != nil {
t.Fatal(readErr)
}
if got := string(data); got != payload {
t.Fatalf("unexpected response Blob data on open %d: %q", i+1, got)
}
}
rangeReader, err := object.open(context.Background(), stream.ObjectURLOpenRequest{Offset: 9, End: 12})
if err != nil {
t.Fatal(err)
}
rangeData, readErr := io.ReadAll(rangeReader)
_ = rangeReader.Close()
if readErr != nil {
t.Fatal(readErr)
}
if got, want := string(rangeData), payload[9:13]; got != want {
t.Fatalf("unexpected materialized Blob range: got %q want %q", got, want)
}
}
func TestResponseBlobRejectsTruncatedResponse(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.Header().Set("Content-Length", "64")
_, _ = io.WriteString(w, "partial")
}))
defer server.Close()
created := make(chan createdObjectURL, 1)
engine, cleanup := newStreamTestEngine(t, created)
defer func() {
cleanup()
engine.Close()
}()
value, err := engine.RunString(fmt.Sprintf(`
(async () => {
const response = await fetch(%q);
try {
await response.blob();
return "resolved";
} catch (_) {
return "rejected";
}
})()
`, server.URL))
if err != nil {
t.Fatal(err)
}
if value != "rejected" {
t.Fatalf("truncated response unexpectedly materialized a Blob: %#v", value)
}
}
func TestEmptyBlobObjectURL(t *testing.T) {
created := make(chan createdObjectURL, 1)
engine, cleanup := newStreamTestEngine(t, created)
defer func() {
cleanup()
engine.Close()
}()
value, err := engine.RunString(`
__gopeed_blob_create_object_url(new Blob([], { type: "application/empty" }))
`)
if err != nil {
t.Fatal(err)
}
if value != "blob:test-1" {
t.Fatalf("unexpected object URL: %#v", value)
}
object := <-created
if object.opts.Size != 0 || object.opts.Range {
t.Fatalf("unexpected empty Blob options: %#v", object.opts)
}
reader, err := object.open(context.Background(), stream.ObjectURLOpenRequest{Offset: 0, End: -1})
if err != nil {
t.Fatal(err)
}
defer reader.Close()
data, err := io.ReadAll(reader)
if err != nil {
t.Fatal(err)
}
if len(data) != 0 {
t.Fatalf("expected empty Blob, got %q", data)
}
}
func TestBlobObjectURLReaderCloseCancelsPendingJSReader(t *testing.T) {
created := make(chan createdObjectURL, 1)
engine, cleanup := newStreamTestEngine(t, created)
defer func() {
cleanup()
engine.Close()
}()
value, err := engine.RunString(`
globalThis.__blobPipeCancelState = {
readCalls: 0,
cancelCalls: 0,
releaseCalls: 0,
cancelReason: "",
resolveRead: null,
};
const pendingSource = {
getReader() {
return {
read() {
__blobPipeCancelState.readCalls++;
return new Promise((resolve) => {
__blobPipeCancelState.resolveRead = resolve;
});
},
cancel(reason) {
__blobPipeCancelState.cancelCalls++;
__blobPipeCancelState.cancelReason = String(reason);
if (__blobPipeCancelState.resolveRead) {
__blobPipeCancelState.resolveRead({ done: true, value: undefined });
}
return Promise.resolve();
},
releaseLock() {
__blobPipeCancelState.releaseCalls++;
},
};
},
};
__gopeed_blob_create_object_url(async () => pendingSource);
`)
if err != nil {
t.Fatal(err)
}
if value != "blob:test-1" {
t.Fatalf("unexpected object URL: %#v", value)
}
object := <-created
reader, err := object.open(context.Background(), stream.ObjectURLOpenRequest{Offset: 0, End: -1})
if err != nil {
t.Fatal(err)
}
readDone := make(chan error, 1)
go func() {
_, readErr := reader.Read(make([]byte, 1))
readDone <- readErr
}()
waitForJSValue(t, engine, `String(__blobPipeCancelState.readCalls)`, "1")
if err := reader.Close(); err != nil {
t.Fatal(err)
}
if err := reader.Close(); err != nil {
t.Fatal(err)
}
select {
case readErr := <-readDone:
if !errors.Is(readErr, context.Canceled) {
t.Fatalf("unexpected pending read result: %v", readErr)
}
case <-time.After(2 * time.Second):
t.Fatal("pending Go read was not released by Close")
}
waitForJSValue(t, engine, `JSON.stringify({
cancelCalls: __blobPipeCancelState.cancelCalls,
releaseCalls: __blobPipeCancelState.releaseCalls,
cancelReason: __blobPipeCancelState.cancelReason,
})`, `{"cancelCalls":1,"releaseCalls":1,"cancelReason":"blob request closed"}`)
}
func waitForJSValue(t *testing.T, engine *engine.Engine, expression, want string) {
t.Helper()
deadline := time.Now().Add(2 * time.Second)
var got any
for time.Now().Before(deadline) {
var err error
got, err = engine.RunString(expression)
if err != nil {
t.Fatal(err)
}
if got == want {
return
}
time.Sleep(10 * time.Millisecond)
}
t.Fatalf("timed out waiting for JavaScript value %q, got %#v", want, got)
}
func newStreamTestEngine(t *testing.T, created chan<- createdObjectURL) (*engine.Engine, func()) {
t.Helper()
var next atomic.Int64
var cleanup func()
e := engine.NewEngine(&engine.Config{
StreamConfig: &stream.Config{
CreateObjectURL: func(opts *stream.ObjectURLOptions, open stream.ObjectURLOpener) (string, error) {
created <- createdObjectURL{opts: opts, open: open}
return fmt.Sprintf("blob:test-%d", next.Add(1)), nil
},
RevokeObjectURL: func(string) error { return nil },
RegisterCleanup: func(fn func()) {
cleanup = fn
},
},
})
if cleanup == nil {
t.Fatal("stream cleanup was not registered")
}
return e, cleanup
}
@@ -0,0 +1,191 @@
package stream
import (
"context"
"io"
"strings"
"sync/atomic"
"testing"
"github.com/GopeedLab/gopeed/pkg/download/engine/inject/file"
"github.com/GopeedLab/gopeed/pkg/download/engine/inject/formdata"
"github.com/dop251/goja"
)
func TestExportFetchRequestSnapshotsArrayBuffer(t *testing.T) {
runtime := goja.New()
buffer := runtime.NewArrayBuffer([]byte("before"))
request := runtime.NewObject()
setFetchRequestDefaults(t, request)
if err := request.Set("body", buffer); err != nil {
t.Fatal(err)
}
meta, err := exportFetchRequest(runtime, request)
if err != nil {
t.Fatal(err)
}
buffer.Bytes()[0] = 'X'
body, ok := meta.Body.([]byte)
if !ok {
t.Fatalf("expected byte snapshot, got %T", meta.Body)
}
if got := string(body); got != "before" {
t.Fatalf("ArrayBuffer snapshot changed after export: %q", got)
}
}
func TestExportFetchRequestSnapshotsFormData(t *testing.T) {
runtime := goja.New()
runtime.SetFieldNameMapper(goja.TagFieldNameMapper("json", true))
if err := formdata.Enable(runtime); err != nil {
t.Fatal(err)
}
value, err := runtime.RunString(`
globalThis.form = new FormData();
form.append("field", "before");
form;
`)
if err != nil {
t.Fatal(err)
}
request := runtime.NewObject()
setFetchRequestDefaults(t, request)
if err := request.Set("body", value); err != nil {
t.Fatal(err)
}
meta, err := exportFetchRequest(runtime, request)
if err != nil {
t.Fatal(err)
}
if _, err := runtime.RunString(`form.set("field", "after")`); err != nil {
t.Fatal(err)
}
body, ok := meta.Body.(*fetchFormDataSnapshot)
if !ok {
t.Fatalf("expected FormData snapshot, got %T", meta.Body)
}
if len(body.entries) != 1 {
t.Fatalf("expected one FormData entry, got %d", len(body.entries))
}
if got := body.entries[0]; got.name != "field" || got.value != "before" {
t.Fatalf("unexpected FormData snapshot: %#v", got)
}
}
func setFetchRequestDefaults(t *testing.T, request *goja.Object) {
t.Helper()
for key, value := range map[string]any{
"url": "https://example.test",
"method": "POST",
"redirect": "follow",
"credentials": "same-origin",
} {
if err := request.Set(key, value); err != nil {
t.Fatal(err)
}
}
}
func TestSnapshotFetchBodyCopiesFileMetadata(t *testing.T) {
source := &file.File{
Reader: strings.NewReader("payload"),
Name: "before.txt",
Size: 7,
}
value, err := snapshotFetchBody(source)
if err != nil {
t.Fatal(err)
}
snapshot, ok := value.(*file.File)
if !ok {
t.Fatalf("expected file snapshot, got %T", value)
}
source.Name = "after.txt"
source.Size = 99
if snapshot.Name != "before.txt" || snapshot.Size != 7 {
t.Fatalf("file metadata was not snapshotted: %#v", snapshot)
}
}
func TestFetchRegistryCloseAll(t *testing.T) {
registry := newFetchRegistry()
streamCtx, cancel := context.WithCancel(context.Background())
body := &trackingReadCloser{}
registry.streams["active"] = &fetchStream{
body: body,
cancel: cancel,
ctx: streamCtx,
ch: make(chan fetchChunk),
}
registry.CloseAll()
registry.CloseAll()
select {
case <-streamCtx.Done():
default:
t.Fatal("active fetch context was not canceled")
}
if got := body.closeCalls.Load(); got != 1 {
t.Fatalf("expected body to close once, got %d", got)
}
registry.mu.Lock()
closed := registry.closed
remaining := len(registry.streams)
registry.mu.Unlock()
if !closed || remaining != 0 {
t.Fatalf("registry not fully closed: closed=%v remaining=%d", closed, remaining)
}
if _, err := registry.Open("", nil, &fetchRequest{URL: "https://example.test"}); err == nil {
t.Fatal("expected a closed registry to reject new fetches")
}
}
func TestFetchRegistryReadPreservesChunkOrder(t *testing.T) {
registry := newFetchRegistry()
ctx, cancel := context.WithCancel(context.Background())
stream := &fetchStream{
body: io.NopCloser(strings.NewReader("")),
cancel: cancel,
ctx: ctx,
ch: make(chan fetchChunk, 2),
}
stream.ch <- fetchChunk{data: []byte("abc")}
stream.ch <- fetchChunk{data: []byte("def")}
registry.streams["ordered"] = stream
defer registry.CloseAll()
var got strings.Builder
for range 4 {
chunk, done, err := registry.Read("ordered", 2)
if err != nil {
t.Fatal(err)
}
if done {
t.Fatal("stream ended before buffered chunks were read")
}
got.Write(chunk)
}
if got.String() != "abcdef" {
t.Fatalf("fetch chunks were reordered: %q", got.String())
}
}
type trackingReadCloser struct {
closeCalls atomic.Int32
}
func (r *trackingReadCloser) Read([]byte) (int, error) {
return 0, io.EOF
}
func (r *trackingReadCloser) Close() error {
r.closeCalls.Add(1)
return nil
}
+964
View File
@@ -0,0 +1,964 @@
(function () {
const createBlobObjectURL = globalThis.__gopeed_create_blob_object_url;
const revokeObjectURL = globalThis.__gopeed_revoke_object_url;
const fetchOpen = globalThis.__gopeed_fetch_open;
const fetchRead = globalThis.__gopeed_fetch_read;
const fetchClose = globalThis.__gopeed_fetch_close;
const fetchAbort = globalThis.__gopeed_fetch_abort;
if (typeof globalThis.ReadableStream === "undefined") {
class ReadableStreamDefaultController {
constructor(stream) {
this._stream = stream;
}
enqueue(chunk) {
this._stream._enqueue(chunk);
}
close() {
this._stream._close();
}
error(err) {
this._stream._error(err);
}
}
class ReadableStreamDefaultReader {
constructor(stream) {
this._stream = stream;
}
read() {
return this._stream._read();
}
cancel(reason) {
if (!this._stream) {
return Promise.resolve();
}
return this._stream.cancel(reason);
}
releaseLock() {
if (this._stream) {
this._stream._reader = null;
this._stream.locked = false;
this._stream = null;
}
}
}
class ReadableStream {
constructor(source = {}) {
this._source = source;
this._queue = [];
this._waiters = [];
this._closed = false;
this._errored = null;
this._reader = null;
this._pulling = false;
this.locked = false;
this._controller = new ReadableStreamDefaultController(this);
if (typeof source.start === "function") {
source.start(this._controller);
}
}
_enqueue(chunk) {
if (this._closed || this._errored) {
return;
}
if (this._waiters.length > 0) {
const waiter = this._waiters.shift();
waiter.resolve({ done: false, value: chunk });
return;
}
this._queue.push(chunk);
}
_close() {
this._closed = true;
while (this._waiters.length > 0) {
this._waiters.shift().resolve({ done: true, value: undefined });
}
}
_error(err) {
this._errored = err || new Error("ReadableStream error");
while (this._waiters.length > 0) {
this._waiters.shift().reject(this._errored);
}
}
_read() {
if (this._errored) {
return Promise.reject(this._errored);
}
this._markBodyUsed();
if (this._queue.length > 0) {
return Promise.resolve({ done: false, value: this._queue.shift() });
}
if (this._closed) {
return Promise.resolve({ done: true, value: undefined });
}
this._maybePull();
if (this._queue.length > 0) {
return Promise.resolve({ done: false, value: this._queue.shift() });
}
if (this._closed) {
return Promise.resolve({ done: true, value: undefined });
}
return new Promise((resolve, reject) => {
this._waiters.push({ resolve, reject });
this._maybePull();
});
}
_maybePull() {
if (this._closed || this._errored || this._pulling) {
return;
}
if (this._queue.length > 0) {
return;
}
if (!this._source || typeof this._source.pull !== "function") {
return;
}
this._pulling = true;
Promise.resolve(this._source.pull(this._controller))
.catch((err) => {
this._error(err);
})
.finally(() => {
this._pulling = false;
if (!this._closed && !this._errored && this._queue.length === 0 && this._waiters.length > 0) {
this._maybePull();
}
});
}
_markBodyUsed() {
if (typeof this.__gopeedMarkBodyUsed === "function") {
this.__gopeedMarkBodyUsed();
}
}
getReader() {
if (this.locked) {
throw new TypeError("ReadableStream is locked");
}
this.locked = true;
this._reader = new ReadableStreamDefaultReader(this);
return this._reader;
}
cancel(reason) {
this._markBodyUsed();
if (this._source && typeof this._source.cancel === "function") {
return Promise.resolve(this._source.cancel(reason)).then(() => {
this._close();
});
}
this._close();
return Promise.resolve();
}
async pipeTo(dest) {
const reader = this.getReader();
const writer = dest.getWriter();
try {
while (true) {
const { done, value } = await reader.read();
if (done) {
await writer.close();
break;
}
await writer.write(value);
}
} catch (err) {
await writer.abort(err);
throw err;
} finally {
reader.releaseLock();
writer.releaseLock();
}
}
}
globalThis.ReadableStream = ReadableStream;
}
if (typeof globalThis.WritableStream === "undefined") {
class WritableStreamDefaultWriter {
constructor(stream) {
if (stream.locked) {
throw new TypeError("WritableStream is locked");
}
this._stream = stream;
this._stream.locked = true;
}
write(chunk) {
return this._stream._write(chunk);
}
close() {
return this._stream._close();
}
abort(reason) {
return this._stream._abort(reason);
}
releaseLock() {
if (this._stream) {
this._stream.locked = false;
this._stream = null;
}
}
}
class WritableStream {
constructor(sink = {}) {
this._sink = sink;
this._observers = [];
this._state = "writable";
this.locked = false;
}
_targets() {
return [this._sink, ...this._observers];
}
_write(chunk) {
if (this._state !== "writable") {
return Promise.reject(new TypeError("WritableStream is not writable"));
}
let chain = Promise.resolve();
for (const target of this._targets()) {
if (target && typeof target.write === "function") {
chain = chain.then(() => target.write(chunk));
}
}
return chain;
}
_close() {
if (this._state !== "writable") {
return Promise.reject(new TypeError("WritableStream is not writable"));
}
this._state = "closed";
let chain = Promise.resolve();
for (const target of this._targets()) {
if (target && typeof target.close === "function") {
chain = chain.then(() => target.close());
}
}
return chain;
}
_abort(reason) {
if (this._state === "errored") {
return Promise.resolve();
}
this._state = "errored";
let chain = Promise.resolve();
for (const target of this._targets()) {
if (target && typeof target.abort === "function") {
chain = chain.then(() => target.abort(reason));
}
}
return chain;
}
_addObserver(observer) {
this._observers.push(observer);
}
getWriter() {
return new WritableStreamDefaultWriter(this);
}
}
globalThis.WritableStream = WritableStream;
}
if (typeof globalThis.TransformStream === "undefined") {
class TransformStream {
constructor(transformer = {}) {
let readableController;
this.readable = new ReadableStream({
start(controller) {
readableController = controller;
}
});
this.writable = new WritableStream({
async write(chunk) {
if (typeof transformer.transform === "function") {
await transformer.transform(chunk, readableController);
return;
}
readableController.enqueue(chunk);
},
async close() {
if (typeof transformer.flush === "function") {
await transformer.flush(readableController);
}
readableController.close();
},
async abort(reason) {
if (typeof transformer.abort === "function") {
await transformer.abort(reason);
}
readableController.error(reason || new Error("TransformStream aborted"));
}
});
}
}
globalThis.TransformStream = TransformStream;
}
if (typeof globalThis.ByteLengthQueuingStrategy === "undefined") {
globalThis.ByteLengthQueuingStrategy = class ByteLengthQueuingStrategy {
constructor({ highWaterMark }) {
this.highWaterMark = highWaterMark;
}
size(chunk) {
if (typeof chunk === "string") {
return chunk.length;
}
if (chunk && typeof chunk.byteLength === "number") {
return chunk.byteLength;
}
return 1;
}
};
}
if (typeof globalThis.CountQueuingStrategy === "undefined") {
globalThis.CountQueuingStrategy = class CountQueuingStrategy {
constructor({ highWaterMark }) {
this.highWaterMark = highWaterMark;
}
size() {
return 1;
}
};
}
function toUint8Array(chunk) {
if (chunk == null) {
return new Uint8Array(0);
}
if (chunk instanceof Uint8Array) {
return chunk;
}
if (typeof chunk === "string") {
return new TextEncoder().encode(chunk);
}
if (chunk instanceof ArrayBuffer) {
return new Uint8Array(chunk);
}
if (typeof ArrayBuffer !== "undefined" && ArrayBuffer.isView && ArrayBuffer.isView(chunk)) {
return new Uint8Array(chunk.buffer, chunk.byteOffset, chunk.byteLength);
}
if (typeof Blob !== "undefined" && chunk instanceof Blob) {
if (chunk._buffer instanceof Uint8Array) {
return chunk._buffer;
}
}
return new Uint8Array(0);
}
function createBodyReadableStream(owner) {
return new ReadableStream({
start(controller) {
Promise.resolve().then(async () => {
owner.bodyUsed = true;
if (owner._noBody) {
controller.close();
return;
}
if (owner._bodyArrayBuffer) {
controller.enqueue(toUint8Array(owner._bodyArrayBuffer));
controller.close();
return;
}
if (owner._bodyBlob) {
const data = await owner._bodyBlob.arrayBuffer();
controller.enqueue(new Uint8Array(data));
controller.close();
return;
}
if (owner._bodyText != null) {
controller.enqueue(toUint8Array(owner._bodyText));
controller.close();
return;
}
if (owner._bodyInit != null) {
controller.enqueue(toUint8Array(owner._bodyInit));
}
controller.close();
}).catch((err) => {
controller.error(err);
});
}
});
}
async function readAllFromStream(stream, asText) {
if (!stream) {
return asText ? "" : new Uint8Array(0);
}
const reader = stream.getReader();
const chunks = [];
let total = 0;
try {
while (true) {
const { done, value } = await reader.read();
if (done) {
break;
}
const chunk = toUint8Array(value);
chunks.push(chunk);
total += chunk.byteLength;
}
} finally {
reader.releaseLock();
}
const merged = new Uint8Array(total);
let offset = 0;
for (const chunk of chunks) {
merged.set(chunk, offset);
offset += chunk.byteLength;
}
if (asText) {
return new TextDecoder().decode(merged);
}
return merged;
}
function attachResponseStreaming(response, stream) {
response.__gopeedBodyStream = stream;
response.__gopeedBodyConsumed = false;
const ensureUnused = () => {
if (response.__gopeedBodyConsumed) {
throw new TypeError("Already read");
}
};
const markBodyUsed = () => {
ensureUnused();
response.__gopeedBodyConsumed = true;
response.bodyUsed = true;
};
if (stream) {
stream.__gopeedMarkBodyUsed = () => {
if (!response.__gopeedBodyConsumed) {
response.__gopeedBodyConsumed = true;
response.bodyUsed = true;
}
};
}
response.text = async function () {
ensureUnused();
markBodyUsed();
return readAllFromStream(stream, true);
};
response.arrayBuffer = async function () {
ensureUnused();
markBodyUsed();
const bytes = await readAllFromStream(stream, false);
return bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength);
};
response.blob = async function () {
ensureUnused();
markBodyUsed();
const contentType = this.headers && this.headers.get ? (this.headers.get("content-type") || "") : "";
// Response.blob() is a materializing body consumer. Keeping the live
// response stream behind an empty one-shot Blob makes retry, range and a
// second object-URL reader silently produce an empty file.
const bytes = await readAllFromStream(stream, false);
return new Blob([bytes], { type: contentType });
};
response.json = async function () {
const text = await this.text();
return JSON.parse(text);
};
return response;
}
if (typeof globalThis.Response === "function") {
const responseProto = globalThis.Response.prototype;
const bodyDescriptor = Object.getOwnPropertyDescriptor(responseProto, "body");
if (!bodyDescriptor || typeof bodyDescriptor.get !== "function") {
Object.defineProperty(responseProto, "body", {
configurable: true,
enumerable: true,
get() {
if (this.__gopeedBodyStream) {
return this.__gopeedBodyStream;
}
if (!this.__gopeedBodyStream) {
this.__gopeedBodyStream = createBodyReadableStream(this);
}
return this.__gopeedBodyStream;
}
});
}
}
const blobReaders = new Map();
function getValueTypeName(value) {
if (value === null) {
return "null";
}
if (value === undefined) {
return "undefined";
}
if (value && value.constructor && typeof value.constructor.name === "string" && value.constructor.name) {
return value.constructor.name;
}
return typeof value;
}
function toReadableStreamReader(value, sourceLabel) {
if (value && typeof value.getReader === "function") {
return value.getReader();
}
throw new TypeError(sourceLabel + " must return a ReadableStream, got " + getValueTypeName(value));
}
async function openBlobReadable(blob, request) {
const { offset, end } = request;
const sliced = blob.slice(offset, end >= offset ? end + 1 : undefined);
if (sliced && typeof sliced.stream === "function") {
try {
const stream = sliced.stream();
if (stream && typeof stream.getReader === "function") {
return stream;
}
} catch (_) {
}
}
if (sliced && typeof sliced.arrayBuffer === "function") {
const buffer = await sliced.arrayBuffer();
return new ReadableStream({
start(controller) {
controller.enqueue(new Uint8Array(buffer));
controller.close();
},
});
}
throw new TypeError("Blob source cannot be converted to a ReadableStream");
}
function describeObjectURLValue(value) {
if (value instanceof Blob) {
return {
kind: "blob",
value,
};
}
if (typeof value === "function") {
return {
kind: "opener",
openReadable: value,
sourceLabel: "gopeed.runtime.blob.createObjectURL opener function",
};
}
return {
kind: "other",
value,
};
}
function releaseReader(reader, reason) {
if (!reader) {
return;
}
try {
if (reason !== undefined && typeof reader.cancel === "function") {
const result = reader.cancel(reason);
if (result && typeof result.catch === "function") {
result.catch(function () {});
}
}
} catch (_) {
}
if (typeof reader.releaseLock === "function") {
try {
reader.releaseLock();
} catch (_) {
}
}
}
function normalizeBlobObjectURLOptions(value, options, defaultRange) {
const normalized = {
contentType: "",
size: 0,
range: !!defaultRange,
};
if (options && typeof options === "object") {
if (typeof options.contentType === "string") {
normalized.contentType = options.contentType;
}
if (Number.isFinite(Number(options.size)) && Number(options.size) > 0) {
normalized.size = Number(options.size);
}
if (typeof options.range === "boolean") {
normalized.range = options.range;
}
}
if (!normalized.contentType && value && typeof value.type === "string") {
normalized.contentType = value.type;
}
if (!normalized.size && value) {
if (Number.isFinite(Number(value.size)) && Number(value.size) > 0) {
normalized.size = Number(value.size);
} else if (value._buffer && Number.isFinite(Number(value._buffer.byteLength))) {
normalized.size = Number(value._buffer.byteLength);
}
}
return normalized;
}
function validateBlobObjectURLOptions(options) {
if (options.range && !(Number.isFinite(Number(options.size)) && Number(options.size) > 0)) {
throw new TypeError("gopeed.runtime.blob.createObjectURL options.range requires a positive options.size");
}
}
function normalizeOpenRequest(request) {
const offset = Math.max(0, Number(request && request.offset) || 0);
const endValue = Number(request && request.end);
const end = Number.isFinite(endValue) ? endValue : -1;
return { offset, end };
}
function yieldBlobPipeTask() {
return new Promise((resolve) => {
if (typeof setTimeout === "function") {
setTimeout(resolve, 0);
} else {
Promise.resolve().then(resolve);
}
});
}
function createPendingBlobReader(openReadable, request) {
let activeReader;
let ready;
function ensureReader() {
if (ready) {
return ready;
}
ready = (async function () {
const source = await openReadable(normalizeOpenRequest(request));
activeReader = toReadableStreamReader(source, "gopeed.runtime.blob.createObjectURL opener function");
return activeReader;
})();
return ready;
}
return {
async read() {
const reader = await ensureReader();
return reader.read();
},
cancel(reason) {
return ensureReader().then(function (reader) {
return reader.cancel(reason);
}, function () {});
},
releaseLock() {
if (!ready) {
return;
}
return ready.then(function (reader) {
return reader.releaseLock();
}, function () {});
}
};
}
globalThis.__gopeed_blob_open_source = function (openReadable, request) {
if (typeof openReadable !== "function") {
throw new TypeError("blob opener must be callable");
}
const id = crypto.randomUUID ? crypto.randomUUID() : String(Date.now()) + "-" + String(Math.random());
blobReaders.set(id, createPendingBlobReader(openReadable, request));
return id;
};
globalThis.__gopeed_blob_read_source = async function (id, chunkSize) {
const reader = blobReaders.get(id);
if (!reader) {
return null;
}
const { done, value } = await reader.read();
if (done) {
blobReaders.delete(id);
releaseReader(reader);
return null;
}
const chunk = value instanceof Uint8Array ? value : new Uint8Array(value);
const size = Math.max(1, Number(chunkSize) || chunk.byteLength);
if (chunk.byteLength <= size) {
return chunk.buffer.slice(chunk.byteOffset, chunk.byteOffset + chunk.byteLength);
}
const head = chunk.slice(0, size);
let offset = size;
blobReaders.set(id, {
read() {
if (offset >= chunk.byteLength) {
return reader.read();
}
const next = chunk.slice(offset, Math.min(offset + size, chunk.byteLength));
offset += next.byteLength;
return Promise.resolve({ done: false, value: next });
},
cancel(reason) {
return reader.cancel(reason);
},
releaseLock() {
return reader.releaseLock();
}
});
return head.buffer.slice(head.byteOffset, head.byteOffset + head.byteLength);
};
globalThis.__gopeed_blob_close_source = function (id) {
const reader = blobReaders.get(id);
if (!reader) {
return;
}
blobReaders.delete(id);
releaseReader(reader, "blob request closed");
};
const blobPipeReaderStates = new Map();
function releaseBlobPipeReader(state) {
if (!state || state.released) {
return;
}
state.released = true;
if (state.reader && typeof state.reader.releaseLock === "function") {
try {
state.reader.releaseLock();
} catch (_) {
}
}
if (blobPipeReaderStates.get(state.pipeId) === state) {
blobPipeReaderStates.delete(state.pipeId);
}
}
function finishBlobPipeReader(state, cancel, reason) {
if (!state) {
return Promise.resolve();
}
if (cancel) {
state.cancelRequested = true;
if (reason !== undefined) {
state.cancelReason = reason;
}
}
if (state.cleanupPromise) {
return state.cleanupPromise;
}
if (!state.reader) {
// A cancellation may arrive while the asynchronous opener is pending.
// Keep the state until the reader exists so it can still be canceled.
if (!state.cancelRequested || state.openSettled) {
releaseBlobPipeReader(state);
}
return Promise.resolve();
}
state.cleanupPromise = (async function () {
try {
if (state.cancelRequested && typeof state.reader.cancel === "function") {
await state.reader.cancel(state.cancelReason);
}
} catch (_) {
} finally {
releaseBlobPipeReader(state);
}
})();
return state.cleanupPromise;
}
globalThis.__gopeed_blob_cancel_pipe_source = function (pipeId, reason) {
const state = blobPipeReaderStates.get(pipeId);
if (!state) {
return;
}
// finishBlobPipeReader invokes reader.cancel() synchronously up to its
// first await. Do not return the cleanup Promise: Go only needs to ensure
// cancellation has started before releasing the source session.
finishBlobPipeReader(state, true, reason).catch(function () {});
};
globalThis.__gopeed_blob_pipe_source = function (openReadable, request, pipeId) {
if (typeof openReadable !== "function") {
throw new TypeError("blob opener must be callable");
}
const state = {
pipeId,
reader: null,
cancelRequested: false,
cancelReason: "blob pipe closed",
cleanupPromise: null,
openSettled: false,
released: false,
};
blobPipeReaderStates.set(pipeId, state);
const startPipe = async function () {
try {
const source = await openReadable(normalizeOpenRequest(request));
state.openSettled = true;
state.reader = toReadableStreamReader(source, "gopeed.runtime.blob.createObjectURL opener function");
if (state.cancelRequested) {
await finishBlobPipeReader(state, true, state.cancelReason);
return;
}
while (true) {
const { done, value } = await state.reader.read();
if (done) {
globalThis.__gopeed_blob_pipe_close(pipeId);
return;
}
const chunk = value instanceof Uint8Array ? value : new Uint8Array(value);
const buffer = chunk.buffer.slice(chunk.byteOffset, chunk.byteOffset + chunk.byteLength);
if (!globalThis.__gopeed_blob_pipe_chunk(pipeId, buffer)) {
state.cancelRequested = true;
state.cancelReason = "blob pipe closed";
return;
}
await yieldBlobPipeTask();
}
} catch (error) {
if (!state.cancelRequested) {
globalThis.__gopeed_blob_pipe_error(pipeId, error && error.stack ? error.stack : String(error));
}
} finally {
state.openSettled = true;
await finishBlobPipeReader(state, state.cancelRequested, state.cancelReason);
}
};
if (typeof setTimeout === "function") {
setTimeout(startPipe, 0);
} else {
Promise.resolve().then(startPipe);
}
};
globalThis.__gopeed_blob_create_object_url = function (value, options) {
const described = describeObjectURLValue(value);
if (described.kind === "blob") {
const blob = described.value;
const normalized = normalizeBlobObjectURLOptions(blob, options, Number(blob.size) > 0);
validateBlobObjectURLOptions(normalized);
const opener = async (request) => openBlobReadable(blob, request);
return createBlobObjectURL(opener, normalized);
}
if (described.kind === "opener") {
const normalized = normalizeBlobObjectURLOptions(null, options, false);
validateBlobObjectURLOptions(normalized);
return createBlobObjectURL(described.openReadable, normalized);
}
throw new TypeError("Unsupported object type for gopeed.runtime.blob.createObjectURL: " + getValueTypeName(described.value) + ". Expected Blob or opener function");
};
globalThis.__gopeed_blob_revoke_object_url = function (url) {
if (typeof url === "string") {
revokeObjectURL(url);
}
};
const originalFetch = typeof globalThis.fetch === "function"
? globalThis.fetch.bind(globalThis)
: null;
if (typeof fetchOpen === "function") {
globalThis.fetch = async function (input, init) {
const request = new Request(input, init);
if (originalFetch && (request.method === "HEAD" || request.redirect === "manual" || request.redirect === "error")) {
return originalFetch(input, init);
}
let body = null;
if (request._bodyFormData) {
body = request._bodyFormData;
} else if (request._bodyArrayBuffer) {
body = request._bodyArrayBuffer;
} else if (request._bodyBlob) {
body = await request._bodyBlob.arrayBuffer();
} else if (request._bodyInit != null && typeof request._bodyInit === "object") {
body = request._bodyInit;
} else if (request._bodyText != null) {
body = request._bodyText;
} else if (request._bodyInit != null) {
body = request._bodyInit;
}
const headers = [];
request.headers.forEach((value, key) => {
headers.push([key, value]);
});
let meta;
try {
meta = await fetchOpen({
url: request.url,
method: request.method,
headers,
body,
redirect: request.redirect,
credentials: request.credentials
});
} catch (error) {
throw error instanceof Error ? error : new TypeError(String(error));
}
const stream = new ReadableStream({
async pull(controller) {
let chunk;
try {
chunk = await fetchRead(meta.id, 64 * 1024);
} catch (error) {
fetchClose(meta.id);
controller.error(error instanceof Error ? error : new TypeError(String(error)));
return;
}
if (chunk == null) {
fetchClose(meta.id);
controller.close();
return;
}
const bytes = chunk instanceof Uint8Array ? chunk : new Uint8Array(chunk);
if (bytes.byteLength === 0) {
fetchClose(meta.id);
controller.close();
return;
}
controller.enqueue(bytes);
},
cancel(reason) {
fetchAbort(meta.id, reason == null ? "" : String(reason));
}
});
const response = new Response(null, {
status: meta.status,
statusText: meta.statusText,
headers: meta.headers,
url: meta.url
});
return attachResponseStreaming(response, stream);
};
globalThis.fetch.__gopeedOriginalFetch = originalFetch;
}
})();
+8 -1
View File
@@ -1,6 +1,8 @@
package vm
import (
"fmt"
"github.com/dop251/goja"
"github.com/dop251/goja_nodejs/eventloop"
)
@@ -25,7 +27,12 @@ func (vm *Vm) Get(name string) (value any) {
func (vm *Vm) RunString(script string) (value any, err error) {
defer func() {
if r := recover(); r != nil {
err = r.(error)
switch v := r.(type) {
case error:
err = v
default:
err = fmt.Errorf("panic: %v", r)
}
}
}()
+1 -1
View File
@@ -1 +1 @@
(()=>{var e={725:function(e,t,r){var o,n,i;i="undefined"!=typeof self&&self||"undefined"!=typeof window&&window||void 0!==r.g&&r.g||this,o=function(e){"use strict";var t=i.BlobBuilder||i.WebKitBlobBuilder||i.MSBlobBuilder||i.MozBlobBuilder,r=i.URL||i.webkitURL||function(e,t){return(t=document.createElement("a")).href=e,t},o=i.Blob,n=r.createObjectURL,a=r.revokeObjectURL,s=i.Symbol&&i.Symbol.toStringTag,u=!1,f=!1,l=t&&t.prototype.append&&t.prototype.getBlob;try{u=2===new Blob(["ä"]).size,f=2===new Blob([new Uint8Array([1,2])]).size}catch(e){}function c(e){return e.map((function(e){if(e.buffer instanceof ArrayBuffer){var t=e.buffer;if(e.byteLength!==t.byteLength){var r=new Uint8Array(e.byteLength);r.set(new Uint8Array(t,e.byteOffset,e.byteLength)),t=r.buffer}return t}return e}))}function h(e,r){r=r||{};var o=new t;return c(e).forEach((function(e){o.append(e)})),r.type?o.getBlob(r.type):o.getBlob()}function d(e,t){return new o(c(e),t||{})}i.Blob&&(h.prototype=Blob.prototype,d.prototype=Blob.prototype);var p="function"==typeof TextEncoder?TextEncoder.prototype.encode.bind(new TextEncoder):function(e){for(var t=0,r=e.length,o=i.Uint8Array||Array,n=0,a=Math.max(32,r+(r>>1)+7),s=new o(a>>3<<3);t<r;){var u=e.charCodeAt(t++);if(u>=55296&&u<=56319){if(t<r){var f=e.charCodeAt(t);56320==(64512&f)&&(++t,u=((1023&u)<<10)+(1023&f)+65536)}if(u>=55296&&u<=56319)continue}if(n+4>s.length){a+=8,a=(a*=1+t/e.length*2)>>3<<3;var l=new Uint8Array(a);l.set(s),s=l}if(0!=(4294967168&u)){if(0==(4294965248&u))s[n++]=u>>6&31|192;else if(0==(4294901760&u))s[n++]=u>>12&15|224,s[n++]=u>>6&63|128;else{if(0!=(4292870144&u))continue;s[n++]=u>>18&7|240,s[n++]=u>>12&63|128,s[n++]=u>>6&63|128}s[n++]=63&u|128}else s[n++]=u}return s.slice(0,n)},y="function"==typeof TextDecoder?TextDecoder.prototype.decode.bind(new TextDecoder):function(e){for(var t=e.length,r=[],o=0;o<t;){var n,i,a,s,u=e[o],f=null,l=u>239?4:u>223?3:u>191?2:1;if(o+l<=t)switch(l){case 1:u<128&&(f=u);break;case 2:128==(192&(n=e[o+1]))&&(s=(31&u)<<6|63&n)>127&&(f=s);break;case 3:n=e[o+1],i=e[o+2],128==(192&n)&&128==(192&i)&&(s=(15&u)<<12|(63&n)<<6|63&i)>2047&&(s<55296||s>57343)&&(f=s);break;case 4:n=e[o+1],i=e[o+2],a=e[o+3],128==(192&n)&&128==(192&i)&&128==(192&a)&&(s=(15&u)<<18|(63&n)<<12|(63&i)<<6|63&a)>65535&&s<1114112&&(f=s)}null===f?(f=65533,l=1):f>65535&&(f-=65536,r.push(f>>>10&1023|55296),f=56320|1023&f),r.push(f),o+=l}for(var c=r.length,h="",d=0;d<c;)h+=String.fromCharCode.apply(String,r.slice(d,d+=4096));return h};function b(){var t=!!i.ActiveXObject||"-ms-scroll-limit"in document.documentElement.style&&"-ms-ime-align"in document.documentElement.style,r=i.XMLHttpRequest&&i.XMLHttpRequest.prototype.send;t&&r&&(XMLHttpRequest.prototype.send=function(e){e instanceof Blob?(this.setRequestHeader("Content-Type",e.type),r.call(this,e)):r.call(this,e)});try{new File([],""),e.File=i.File,e.FileReader=i.FileReader}catch(t){try{e.File=new Function('class File extends Blob {constructor(chunks, name, opts) {opts = opts || {};super(chunks, opts || {});this.name = name.replace(/\\//g, ":");this.lastModifiedDate = opts.lastModified ? new Date(opts.lastModified) : new Date();this.lastModified = +this.lastModifiedDate;}};return new File([], ""), File')()}catch(t){e.File=function(e,t,r){var o=new Blob(e,r),n=r&&void 0!==r.lastModified?new Date(r.lastModified):new Date;return o.name=t.replace(/\//g,":"),o.lastModifiedDate=n,o.lastModified=+n,o.toString=function(){return"[object File]"},s&&(o[s]="File"),o}}}}u?(b(),e.Blob=f?i.Blob:d):l?(b(),e.Blob=h):function(){function t(e){for(var t=new Array(e.byteLength),r=new Uint8Array(e),o=t.length;o--;)t[o]=r[o];return t}function o(e){for(var t="ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=",r=[],o=0;o<e.length;o+=3){var n=e[o],i=o+1<e.length,a=i?e[o+1]:0,s=o+2<e.length,u=s?e[o+2]:0,f=n>>2,l=(3&n)<<4|a>>4,c=(15&a)<<2|u>>6,h=63&u;s||(h=64,i||(c=64)),r.push(t[f],t[l],t[c],t[h])}return r.join("")}var s=Object.create||function(e){function t(){}return t.prototype=e,new t};function u(e){return Object.prototype.toString.call(e).slice(8,-1)}function f(e,t){return"object"==typeof e&&Object.prototype.isPrototypeOf.call(e.prototype,t)}var l=["Int8Array","Uint8Array","Uint8ClampedArray","Int16Array","Uint16Array","Int32Array","Uint32Array","Float32Array","Float64Array","ArrayBuffer"];function c(e){return t=l,r=u(e),-1!==t.indexOf(r)||f(i.ArrayBuffer,e);var t,r}function h(e,r){r=null==r?{}:r;for(var o=0,n=(e=e?e.slice():[]).length;o<n;o++){var a=e[o];a instanceof h?e[o]=a._buffer:"string"==typeof a?e[o]=p(a):"DataView"===u(s=a)||f(i.DataView,s)?e[o]=t(a.buffer):c(a)?e[o]=t(a):e[o]=p(String(a))}var s;this._buffer=i.Uint8Array?function(e){for(var t=0,r=e.length;r--;)t+=e[r].length;for(var o=new Uint8Array(t),n=0,i=0;i<e.length;i++){var a=e[i];o.set(a,n),n+=a.byteLength||a.length}return o}(e):[].concat.apply([],e),this.size=this._buffer.length,this.type=r.type||"",/[^\u0020-\u007E]/.test(this.type)?this.type="":this.type=this.type.toLowerCase()}funcLine truncated
(()=>{var e={847(e,t,r){var n,o;n=function(){"use strict";function e(e,t){(null==t||t>e.length)&&(t=e.length);for(var r=0,n=Array(t);r<t;r++)n[r]=e[r];return n}function t(e,t,r){return t=l(t),function(e,t){if(t&&("object"==typeof t||"function"==typeof t))return t;if(void 0!==t)throw new TypeError("Derived constructors may only return object or undefined");return function(e){if(void 0===e)throw new ReferenceError("this hasn't been initialised - super() hasn't been called");return e}(e)}(e,c()?Reflect.construct(t,r||[],l(e).constructor):t.apply(e,r))}function n(e,t){if(!(e instanceof t))throw new TypeError("Cannot call a class as a function")}function o(e,t){for(var r=0;r<t.length;r++){var n=t[r];n.enumerable=n.enumerable||!1,n.configurable=!0,"value"in n&&(n.writable=!0),Object.defineProperty(e,f(n.key),n)}}function i(e,t,r){return t&&o(e.prototype,t),r&&o(e,r),Object.defineProperty(e,"prototype",{writable:!1}),e}function a(t,r){var n="undefined"!=typeof Symbol&&t[Symbol.iterator]||t["@@iterator"];if(!n){if(Array.isArray(t)||(n=function(t,r){if(t){if("string"==typeof t)return e(t,r);var n={}.toString.call(t).slice(8,-1);return"Object"===n&&t.constructor&&(n=t.constructor.name),"Map"===n||"Set"===n?Array.from(t):"Arguments"===n||/^(?:Ui|I)nt(?:8|16|32)(?:Clamped)?Array$/.test(n)?e(t,r):void 0}}(t))||r&&t&&"number"==typeof t.length){n&&(t=n);var o=0,i=function(){};return{s:i,n:function(){return o>=t.length?{done:!0}:{done:!1,value:t[o++]}},e:function(e){throw e},f:i}}throw new TypeError("Invalid attempt to iterate non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method.")}var a,s=!0,l=!1;return{s:function(){n=n.call(t)},n:function(){var e=n.next();return s=e.done,e},e:function(e){l=!0,a=e},f:function(){try{s||null==n.return||n.return()}finally{if(l)throw a}}}}function s(){return s="undefined"!=typeof Reflect&&Reflect.get?Reflect.get.bind():function(e,t,r){var n=function(e,t){for(;!{}.hasOwnProperty.call(e,t)&&null!==(e=l(e)););return e}(e,t);if(n){var o=Object.getOwnPropertyDescriptor(n,t);return o.get?o.get.call(arguments.length<3?e:r):o.value}},s.apply(null,arguments)}function l(e){return l=Object.setPrototypeOf?Object.getPrototypeOf.bind():function(e){return e.__proto__||Object.getPrototypeOf(e)},l(e)}function c(){try{var e=!Boolean.prototype.valueOf.call(Reflect.construct(Boolean,[],function(){}))}catch(e){}return(c=function(){return!!e})()}function u(e,t){return u=Object.setPrototypeOf?Object.setPrototypeOf.bind():function(e,t){return e.__proto__=t,e},u(e,t)}function f(e){var t=function(e,t){if("object"!=typeof e||!e)return e;var r=e[Symbol.toPrimitive];if(void 0!==r){var n=r.call(e,t||"default");if("object"!=typeof n)return n;throw new TypeError("@@toPrimitive must return a primitive value.")}return("string"===t?String:Number)(e)}(e,"string");return"symbol"==typeof t?t:t+""}!function(e){e.AbortSignal,e.AbortController}("undefined"!=typeof self?self:r.g);var p=function(){return i(function e(){n(this,e),Object.defineProperty(this,"listeners",{value:{},writable:!0,configurable:!0})},[{key:"addEventListener",value:function(e,t,r){e in this.listeners||(this.listeners[e]=[]),this.listeners[e].push({callback:t,options:r})}},{key:"removeEventListener",value:function(e,t){if(e in this.listeners)for(var r=this.listeners[e],n=0,o=r.length;n<o;n++)if(r[n].callback===t)return void r.splice(n,1)}},{key:"dispatchEvent",value:function(e){var t=this;if(e.type in this.listeners){for(var r=this.listeners[e.type].slice(),n=function(){var n=r[o];try{n.callback.call(t,e)}catch(e){Promise.resolve().then(function(){throw e})}n.options&&n.options.once&&t.removeEventListener(e.type,n.callback)},o=0,i=r.length;o<i;o++)n();return!e.defaultPrevented}}}])}(),d=function(e){function r(){var e;return n(this,r),(e=t(this,r)).listeners||p.call(e),Object.defineProperty(e,"aborted",{value:!1,writable:!0,configurable:!0}),Object.defineProperty(e,"onabort",{value:null,writable:!0,configurable:!0}),Object.defineProperty(e,"reason",{value:void 0,writable:!0,configurable:!0}),e}return function(e,t){if("function"!=typeof t&&null!==t)throw new TypeError("Super expression must either be null or a function");e.prototype=Object.create(t&&t.prototype,{constructor:{value:e,writable:!0,configurable:!0}}),Object.defineProperty(e,"prototype",{writable:!1}),t&&u(e,t)}(r,e),i(r,[{key:"toString",value:function(){return"[object AbortSignal]"}},{key:"dispatchEvent",value:function(e){var t,n,o,i,a;"abort"===e.type&&(this.aborted=!0,"function"==typeof this.onabort&&this.onabort.call(this,e)),(t=r,n="dispatchEvent",o=this,a=s(l(1&(i=3)?t.prototype:t),n,o),2&i&&"function"==typeof a?function(e){return a.apply(o,e)}:a)([e])}},{key:"throwIfAborted",value:function(){var e=this.aborted,t=this.reason;if(e)throw void 0===t?"Aborted":t}}],[{key:"timeout",value:function(e){var t=new h;return setTimeout(function(){return t.abort(new DOMException("This signal is timeout in ".concat(e,"ms"),"TimeoutError"))},e),t.signal}},{keLine truncated
File diff suppressed because it is too large. Load diff
@@ -18,7 +18,9 @@
"webpack-cli": "^5.0.1"
},
"dependencies": {
"abortcontroller-polyfill": "^1.7.8",
"blob-polyfill": "^7.0.20220408",
"event-target-shim": "^6.0.2",
"fastestsmallesttextencoderdecoder": "^1.0.22",
"whatwg-fetch": "^3.6.20"
}
@@ -0,0 +1,2 @@
import "../event/index.js";
import "abortcontroller-polyfill/dist/abortcontroller-polyfill-only.js";
@@ -0,0 +1,9 @@
import { Event, EventTarget } from "event-target-shim";
if (typeof globalThis.Event !== "function") {
globalThis.Event = Event;
}
if (typeof globalThis.EventTarget !== "function") {
globalThis.EventTarget = EventTarget;
}
+3 -1
View File
@@ -1,5 +1,7 @@
import "./event/index.js"
import "./abort/index.js"
import "./blob/index.js"
import "./crypto/index.js"
// polyfill TextEncoder
import 'fastestsmallesttextencoderdecoder';
import "./fetch/index.js"
import "./fetch/index.js"
+20
View File
@@ -0,0 +1,20 @@
package webview
type Provider interface {
IsAvailable() bool
Open(opts OpenOptions) (Page, error)
}
type unavailableProvider struct{}
func NewUnavailableProvider() Provider {
return unavailableProvider{}
}
func (unavailableProvider) IsAvailable() bool {
return false
}
func (unavailableProvider) Open(OpenOptions) (Page, error) {
return nil, ErrUnavailable
}
+143
View File
@@ -0,0 +1,143 @@
package webview
import "encoding/json"
const RPCEndpointPath = "/webview"
const (
MethodIsAvailable = "webview.isAvailable"
MethodPageOpen = "page.open"
MethodPageAddInitScript = "page.addInitScript"
MethodPageGoto = "page.goto"
MethodPageExecute = "page.execute"
MethodPageGetCookies = "page.getCookies"
MethodPageSetCookie = "page.setCookie"
MethodPageDeleteCookie = "page.deleteCookie"
MethodPageClearCookies = "page.clearCookies"
MethodPageClose = "page.close"
)
const (
ErrorCodeInvalidRequest = "INVALID_REQUEST"
ErrorCodeUnknownMethod = "UNKNOWN_METHOD"
ErrorCodeUnavailable = "UNAVAILABLE"
ErrorCodeBrowserNotFound = "BROWSER_NOT_FOUND"
ErrorCodePageNotFound = "PAGE_NOT_FOUND"
ErrorCodeNavigationFailed = "NAVIGATION_FAILED"
ErrorCodeEvaluationFailed = "EVALUATION_FAILED"
ErrorCodeTimeout = "TIMEOUT"
ErrorCodeInternal = "INTERNAL_ERROR"
)
type RPCConfig struct {
Network string `json:"network"`
Address string `json:"address"`
Token string `json:"token,omitempty"`
}
func (c RPCConfig) Enabled() bool {
return c.Network != "" && c.Address != ""
}
type RPCRequest struct {
Method string `json:"method"`
Params any `json:"params"`
}
type RPCResponse struct {
Result json.RawMessage `json:"result"`
Error *RPCError `json:"error"`
}
type RPCError struct {
Code string `json:"code"`
Message string `json:"message"`
}
func (e *RPCError) Error() string {
if e == nil {
return ""
}
return e.Message
}
type IsAvailableParams struct{}
type IsAvailableResult struct {
Available bool `json:"available"`
}
type PageOpenParams struct {
Headless bool `json:"headless,omitempty"`
Debug bool `json:"debug,omitempty"`
Title string `json:"title,omitempty"`
Width int `json:"width,omitempty"`
Height int `json:"height,omitempty"`
UserAgent string `json:"userAgent,omitempty"`
}
func NewPageOpenParams(opts OpenOptions) PageOpenParams {
return PageOpenParams{
Headless: opts.Headless,
Debug: opts.Debug,
Title: opts.Title,
Width: opts.Width,
Height: opts.Height,
UserAgent: opts.UserAgent,
}
}
type PageOpenResult struct {
PageID string `json:"pageId"`
}
type PageAddInitScriptParams struct {
PageID string `json:"pageId"`
Script string `json:"script"`
}
type PageGotoParams struct {
PageID string `json:"pageId"`
URL string `json:"url"`
TimeoutMS int64 `json:"timeoutMs,omitempty"`
WaitUntil string `json:"waitUntil,omitempty"`
}
func NewPageGotoParams(pageID string, url string, opts GotoOptions) PageGotoParams {
return PageGotoParams{
PageID: pageID,
URL: url,
TimeoutMS: opts.TimeoutMS,
WaitUntil: opts.WaitUntil,
}
}
type PageExecuteParams struct {
PageID string `json:"pageId"`
Expression string `json:"expression"`
Args []any `json:"args,omitempty"`
}
type PageGetCookiesParams struct {
PageID string `json:"pageId"`
}
type PageSetCookieParams struct {
PageID string `json:"pageId"`
Cookie Cookie `json:"cookie"`
}
type PageDeleteCookieParams struct {
PageID string `json:"pageId"`
Cookie Cookie `json:"cookie"`
}
type PageClearCookiesParams struct {
PageID string `json:"pageId"`
}
type PageCloseParams struct {
PageID string `json:"pageId"`
}
type EmptyResult struct{}
+765
View File
@@ -0,0 +1,765 @@
package webview
import (
"errors"
"fmt"
"strconv"
"strings"
"sync"
"time"
"github.com/dop251/goja"
)
var ErrUnavailable = errors.New("gopeed runtime webview is unavailable")
type Opener interface {
Open(opts OpenOptions) (Page, error)
}
type Page interface {
AddInitScript(script string) error
Goto(url string, opts GotoOptions) error
Execute(expression string, args ...any) (any, error)
GetCookies() ([]Cookie, error)
SetCookie(cookie Cookie) error
DeleteCookie(cookie Cookie) error
ClearCookies() error
Close() error
}
type OpenOptions struct {
Headless bool
Debug bool
Title string
Width int
Height int
UserAgent string
}
type GotoOptions struct {
TimeoutMS int64
WaitUntil string
}
type ClickOptions struct {
DelayMS int64
}
type TypeOptions struct {
DelayMS int64
}
type WaitOptions struct {
TimeoutMS int64
PollIntervalMS int64
}
type WaitForSelectorOptions struct {
TimeoutMS int64
PollIntervalMS int64
Visible bool
Hidden bool
}
type Cookie struct {
Name string `json:"name"`
Value string `json:"value"`
Domain string `json:"domain,omitempty"`
Path string `json:"path,omitempty"`
Expires time.Time `json:"expires,omitempty,omitzero"`
Secure bool `json:"secure,omitempty"`
HTTPOnly bool `json:"httpOnly,omitempty"`
}
type Runtime struct {
opener Opener
available bool
mu sync.Mutex
pageSeq uint64
pages map[string]Page
}
func NewRuntime(opener Opener, available bool) *Runtime {
if opener == nil {
available = false
}
return &Runtime{
opener: opener,
available: available,
pages: make(map[string]Page),
}
}
func (r *Runtime) IsAvailable() bool {
return r != nil && r.available && r.opener != nil
}
func (r *Runtime) Open(opts ...map[string]any) (*PageHandle, error) {
if !r.IsAvailable() {
return nil, ErrUnavailable
}
page, err := r.opener.Open(parseOpenOptions(firstMap(opts)))
if err != nil {
return nil, err
}
r.mu.Lock()
defer r.mu.Unlock()
r.pageSeq++
id := fmt.Sprintf("page-%d", r.pageSeq)
r.pages[id] = page
return &PageHandle{runtime: r, id: id}, nil
}
func (r *Runtime) Close() error {
r.mu.Lock()
pageIDs := make([]string, 0, len(r.pages))
for id := range r.pages {
pageIDs = append(pageIDs, id)
}
r.mu.Unlock()
var errs []error
for _, id := range pageIDs {
if err := r.closePage(id); err != nil {
errs = append(errs, err)
}
}
return errors.Join(errs...)
}
type PageHandle struct {
runtime *Runtime
id string
}
func (p *PageHandle) AddInitScript(script string) error {
page, err := p.page()
if err != nil {
return err
}
return page.AddInitScript(script)
}
func (p *PageHandle) Goto(url string, opts ...map[string]any) error {
page, err := p.page()
if err != nil {
return err
}
gotoOpts, err := parseGotoOptions(firstMap(opts))
if err != nil {
return err
}
return page.Goto(url, gotoOpts)
}
func (p *PageHandle) Execute(scriptOrFn any, args ...any) (any, error) {
page, err := p.page()
if err != nil {
return nil, err
}
expression, err := normalizeExecutable(scriptOrFn)
if err != nil {
return nil, err
}
return page.Execute(expression, args...)
}
func (p *PageHandle) Focus(selector string) error {
_, err := p.Execute(`(selector) => {
const element = document.querySelector(selector);
if (!element) {
throw new Error("element not found: " + selector);
}
if (typeof element.focus !== "function") {
throw new Error("element is not focusable: " + selector);
}
element.focus();
return true;
}`, selector)
return err
}
func (p *PageHandle) Click(selector string, opts ...map[string]any) error {
clickOpts := parseClickOptions(firstMap(opts))
_, err := p.Execute(`(selector, delayMS) => {
const element = document.querySelector(selector);
if (!element) {
throw new Error("element not found: " + selector);
}
if (typeof element.scrollIntoView === "function") {
element.scrollIntoView({ block: "center", inline: "center" });
}
const performClick = () => {
if (typeof element.click === "function") {
element.click();
return;
}
element.dispatchEvent(new MouseEvent("click", {
bubbles: true,
cancelable: true,
view: window,
}));
};
if (delayMS > 0) {
return new Promise((resolve) => {
setTimeout(() => {
performClick();
resolve(true);
}, delayMS);
});
}
performClick();
return true;
}`, selector, clickOpts.DelayMS)
return err
}
func (p *PageHandle) Type(selector string, text string, opts ...map[string]any) error {
typeOpts := parseTypeOptions(firstMap(opts))
_, err := p.Execute(`(selector, text, delayMS) => {
const element = document.querySelector(selector);
if (!element) {
throw new Error("element not found: " + selector);
}
if (typeof element.focus === "function") {
element.focus();
}
const emitInput = () => {
element.dispatchEvent(new Event("input", { bubbles: true }));
};
const insertIntoInput = (chunk) => {
if (element.disabled) {
throw new Error("element is disabled: " + selector);
}
if (element.readOnly) {
throw new Error("element is readonly: " + selector);
}
const value = typeof element.value === "string" ? element.value : "";
const hasSelection = typeof element.selectionStart === "number" && typeof element.selectionEnd === "number";
const start = hasSelection ? element.selectionStart : value.length;
const end = hasSelection ? element.selectionEnd : value.length;
element.value = value.slice(0, start) + chunk + value.slice(end);
const caret = start + chunk.length;
if (typeof element.setSelectionRange === "function") {
element.setSelectionRange(caret, caret);
}
emitInput();
};
const insertIntoEditable = (chunk) => {
if (element.isContentEditable) {
element.textContent = (element.textContent || "") + chunk;
emitInput();
return;
}
insertIntoInput(chunk);
};
const insertChunk = (chunk) => {
const tagName = (element.tagName || "").toUpperCase();
if (tagName === "INPUT" || tagName === "TEXTAREA" || typeof element.value === "string") {
insertIntoInput(chunk);
return;
}
if (element.isContentEditable) {
insertIntoEditable(chunk);
return;
}
throw new Error("element is not typable: " + selector);
};
const chars = Array.from(String(text));
if (delayMS > 0) {
return new Promise((resolve) => {
let index = 0;
const step = () => {
if (index >= chars.length) {
resolve(true);
return;
}
insertChunk(chars[index]);
index += 1;
setTimeout(step, delayMS);
};
step();
});
}
for (const char of chars) {
insertChunk(char);
}
return true;
}`, selector, text, typeOpts.DelayMS)
return err
}
func (p *PageHandle) WaitForSelector(selector string, opts ...map[string]any) (bool, error) {
waitOpts := parseWaitForSelectorOptions(firstMap(opts))
_, matched, err := p.poll(WaitOptions{
TimeoutMS: waitOpts.TimeoutMS,
PollIntervalMS: waitOpts.PollIntervalMS,
}, func() (any, bool, error) {
value, err := p.Execute(`(selector, visible, hidden) => {
const element = document.querySelector(selector);
if (hidden) {
if (!element) return true;
const style = window.getComputedStyle(element);
if (!style) return !!element.hidden;
const rect = element.getBoundingClientRect();
return element.hidden || style.display === "none" || style.visibility === "hidden" || rect.width === 0 || rect.height === 0;
}
if (!element) return false;
if (!visible) return true;
const style = window.getComputedStyle(element);
if (!style) return true;
const rect = element.getBoundingClientRect();
return !element.hidden && style.display !== "none" && style.visibility !== "hidden" && rect.width > 0 && rect.height > 0;
}`, selector, waitOpts.Visible, waitOpts.Hidden)
if err != nil {
return nil, false, err
}
return true, truthy(value), nil
})
if err != nil {
return false, err
}
return matched, nil
}
func (p *PageHandle) WaitForFunction(scriptOrFn any, args ...any) (any, error) {
expression, err := normalizeExecutable(scriptOrFn)
if err != nil {
return nil, err
}
waitOpts := WaitOptions{}
callArgs := args
if len(args) > 0 {
if raw, ok := args[0].(map[string]any); ok {
waitOpts = parseWaitOptions(raw)
callArgs = args[1:]
}
}
value, matched, err := p.poll(waitOpts, func() (any, bool, error) {
result, err := p.Execute(`(expression, args) => {
const target = (0, eval)(expression);
return Promise.resolve(typeof target === "function" ? target(...args) : target).then((value) => ({
matched: !!value,
value: value ?? null,
}));
}`, expression, callArgs)
if err != nil {
return nil, false, err
}
payload, ok := result.(map[string]any)
if !ok {
return nil, false, fmt.Errorf("unexpected waitForFunction payload: %T", result)
}
return payload["value"], truthy(payload["matched"]), nil
})
if err != nil {
return nil, err
}
if !matched {
return nil, nil
}
return value, nil
}
func (p *PageHandle) GetCookies() ([]Cookie, error) {
page, err := p.page()
if err != nil {
return nil, err
}
return page.GetCookies()
}
func (p *PageHandle) SetCookie(raw map[string]any) error {
page, err := p.page()
if err != nil {
return err
}
return page.SetCookie(parseCookie(raw))
}
func (p *PageHandle) DeleteCookie(raw map[string]any) error {
page, err := p.page()
if err != nil {
return err
}
return page.DeleteCookie(parseCookie(raw))
}
func (p *PageHandle) ClearCookies() error {
page, err := p.page()
if err != nil {
return err
}
return page.ClearCookies()
}
func (p *PageHandle) URL() (string, error) {
value, err := p.Execute(`() => location.href`)
if err != nil {
return "", err
}
return parseString(value), nil
}
func (p *PageHandle) Content() (string, error) {
value, err := p.Execute(`() => document.documentElement ? document.documentElement.outerHTML : ""`)
if err != nil {
return "", err
}
return parseString(value), nil
}
func (p *PageHandle) Close() error {
if p == nil || p.runtime == nil {
return nil
}
return p.runtime.closePage(p.id)
}
func (p *PageHandle) page() (Page, error) {
if p == nil || p.runtime == nil {
return nil, fmt.Errorf("webview page handle is not initialized")
}
return p.runtime.getPage(p.id)
}
func (p *PageHandle) poll(opts WaitOptions, fn func() (any, bool, error)) (any, bool, error) {
timeout := time.Duration(defaultWaitTimeoutMS(opts.TimeoutMS)) * time.Millisecond
pollInterval := time.Duration(defaultPollIntervalMS(opts.PollIntervalMS)) * time.Millisecond
deadline := time.Now().Add(timeout)
for {
value, matched, err := fn()
if err != nil {
return nil, false, err
}
if matched {
return value, true, nil
}
if time.Now().After(deadline) {
return nil, false, nil
}
time.Sleep(pollInterval)
}
}
func (r *Runtime) getPage(id string) (Page, error) {
r.mu.Lock()
defer r.mu.Unlock()
page, ok := r.pages[id]
if !ok {
return nil, fmt.Errorf("page not found")
}
return page, nil
}
func (r *Runtime) closePage(id string) error {
r.mu.Lock()
page, ok := r.pages[id]
if !ok {
r.mu.Unlock()
return nil
}
delete(r.pages, id)
r.mu.Unlock()
return page.Close()
}
func parseOpenOptions(raw map[string]any) OpenOptions {
return OpenOptions{
Headless: parseBool(raw["headless"]),
Debug: parseBool(raw["debug"]),
Title: parseString(raw["title"]),
Width: parseInt(raw["width"]),
Height: parseInt(raw["height"]),
UserAgent: parseString(raw["userAgent"]),
}
}
func parseGotoOptions(raw map[string]any) (GotoOptions, error) {
waitUntil, err := normalizeWaitUntil(parseString(raw["waitUntil"]))
if err != nil {
return GotoOptions{}, err
}
return GotoOptions{
TimeoutMS: parseInt64(raw["timeoutMs"]),
WaitUntil: waitUntil,
}, nil
}
func normalizeWaitUntil(raw string) (string, error) {
if raw == "" {
return "load", nil
}
switch strings.ToLower(strings.TrimSpace(raw)) {
case "load":
return "load", nil
case "domcontentloaded":
return "domcontentloaded", nil
default:
return "", fmt.Errorf("invalid waitUntil: %s", raw)
}
}
func parseClickOptions(raw map[string]any) ClickOptions {
return ClickOptions{DelayMS: parseInt64(raw["delay"])}
}
func parseTypeOptions(raw map[string]any) TypeOptions {
return TypeOptions{DelayMS: parseInt64(raw["delay"])}
}
func parseWaitOptions(raw map[string]any) WaitOptions {
return WaitOptions{
TimeoutMS: parseInt64(raw["timeoutMs"]),
PollIntervalMS: parseInt64(raw["pollIntervalMs"]),
}
}
func parseWaitForSelectorOptions(raw map[string]any) WaitForSelectorOptions {
return WaitForSelectorOptions{
TimeoutMS: parseInt64(raw["timeoutMs"]),
PollIntervalMS: parseInt64(raw["pollIntervalMs"]),
Visible: parseBool(raw["visible"]),
Hidden: parseBool(raw["hidden"]),
}
}
func parseCookie(raw map[string]any) Cookie {
cookie := Cookie{
Name: parseString(raw["name"]),
Value: parseString(raw["value"]),
Domain: parseString(raw["domain"]),
Path: parseString(raw["path"]),
Secure: parseBool(raw["secure"]),
HTTPOnly: parseBool(raw["httpOnly"]),
}
if rawExpires, ok := raw["expires"]; ok {
cookie.Expires = parseTime(rawExpires)
}
return cookie
}
func defaultWaitTimeoutMS(value int64) int64 {
if value <= 0 {
return 10000
}
return value
}
func defaultPollIntervalMS(value int64) int64 {
if value <= 0 {
return 100
}
return value
}
func normalizeExecutable(scriptOrFn any) (string, error) {
switch value := scriptOrFn.(type) {
case string:
return value, nil
case *goja.Object:
return NormalizeExecutableValue(value)
default:
return "", fmt.Errorf("execute expects a string or function, got %T", scriptOrFn)
}
}
func NormalizeExecutableValue(value goja.Value) (string, error) {
if value == nil {
return "", fmt.Errorf("execute expects a string or function")
}
switch raw := value.Export().(type) {
case string:
return raw, nil
}
obj, ok := value.(*goja.Object)
if ok {
if _, ok := goja.AssertFunction(obj); ok {
source, err := functionSource(obj)
if err != nil {
return "", err
}
return normalizeFunctionSource(source), nil
}
}
source := strings.TrimSpace(value.String())
if looksLikeFunctionSource(source) {
return normalizeFunctionSource(source), nil
}
return "", fmt.Errorf("execute expects a string or function, got %T (%q)", value, source)
}
func normalizeFunctionSource(source string) string {
repaired := repairArrowObjectLiteralSource(source)
return "(" + repaired + ")"
}
func functionSource(fn *goja.Object) (string, error) {
toStringValue := fn.Get("toString")
toString, ok := goja.AssertFunction(toStringValue)
if !ok {
return "", fmt.Errorf("function.toString is not callable")
}
value, err := toString(fn)
if err != nil {
return "", err
}
return value.String(), nil
}
func repairArrowObjectLiteralSource(source string) string {
trimmed := strings.TrimSpace(source)
if !strings.Contains(trimmed, "=>") || !strings.Contains(trimmed, "({") {
return source
}
if !strings.HasSuffix(trimmed, "}") {
return source
}
if strings.Count(trimmed, "(")-strings.Count(trimmed, ")") != 1 {
return source
}
if strings.Count(trimmed, "{") != strings.Count(trimmed, "}") {
return source
}
// TODO: Remove this narrow repair once goja fixes toString() for concise arrow
// functions that directly return object literals, then upstream the fix as a PR.
return source + ")"
}
func looksLikeFunctionSource(source string) bool {
if source == "" || source == "[object Object]" {
return false
}
return strings.Contains(source, "=>") ||
strings.HasPrefix(source, "function") ||
strings.HasPrefix(source, "async function") ||
strings.HasPrefix(source, "async (") ||
strings.HasPrefix(source, "async(")
}
func truthy(value any) bool {
switch v := value.(type) {
case nil:
return false
case bool:
return v
case string:
return v != ""
case int:
return v != 0
case int8:
return v != 0
case int16:
return v != 0
case int32:
return v != 0
case int64:
return v != 0
case uint:
return v != 0
case uint8:
return v != 0
case uint16:
return v != 0
case uint32:
return v != 0
case uint64:
return v != 0
case float32:
return v != 0
case float64:
return v != 0
default:
return true
}
}
func firstMap(values []map[string]any) map[string]any {
if len(values) == 0 || values[0] == nil {
return map[string]any{}
}
return values[0]
}
func parseString(value any) string {
if value == nil {
return ""
}
return fmt.Sprint(value)
}
func parseBool(value any) bool {
switch v := value.(type) {
case bool:
return v
case string:
parsed, _ := strconv.ParseBool(v)
return parsed
default:
return false
}
}
func parseInt(value any) int {
return int(parseInt64(value))
}
func parseInt64(value any) int64 {
switch v := value.(type) {
case int:
return int64(v)
case int8:
return int64(v)
case int16:
return int64(v)
case int32:
return int64(v)
case int64:
return v
case uint:
return int64(v)
case uint8:
return int64(v)
case uint16:
return int64(v)
case uint32:
return int64(v)
case uint64:
return int64(v)
case float32:
return int64(v)
case float64:
return int64(v)
case string:
parsed, _ := strconv.ParseInt(v, 10, 64)
return parsed
default:
return 0
}
}
func parseTime(value any) time.Time {
switch v := value.(type) {
case time.Time:
return v
case int64:
return time.UnixMilli(v)
case int:
return time.UnixMilli(int64(v))
case float64:
return time.UnixMilli(int64(v))
case string:
if parsed, err := time.Parse(time.RFC3339Nano, v); err == nil {
return parsed
}
if ms, err := strconv.ParseInt(v, 10, 64); err == nil {
return time.UnixMilli(ms)
}
}
return time.Time{}
}
+338
View File
@@ -0,0 +1,338 @@
package webview
import (
"strings"
"testing"
"github.com/dop251/goja"
)
func TestRuntimeOpenAndExecuteFunctionSource(t *testing.T) {
opener := &fakeOpener{
page: &fakePage{executeValue: "ok"},
}
runtime := NewRuntime(opener, true)
page, err := runtime.Open(map[string]any{
"headless": true,
"debug": true,
"title": "Gopeed",
"width": 1280,
"height": 720,
"userAgent": "UA",
})
if err != nil {
t.Fatal(err)
}
vm := goja.New()
fnValue, err := vm.RunString(`(a, b) => a + b`)
if err != nil {
t.Fatal(err)
}
value, err := page.Execute(fnValue, int64(1), int64(2))
if err != nil {
t.Fatal(err)
}
if value != "ok" {
t.Fatalf("unexpected execute result: %#v", value)
}
if opener.opts.Title != "Gopeed" || opener.opts.UserAgent != "UA" {
t.Fatalf("unexpected open options: %+v", opener.opts)
}
if opener.page.lastExecuteSource != "((a, b) => a + b)" {
t.Fatalf("unexpected function source: %q", opener.page.lastExecuteSource)
}
if len(opener.page.lastExecuteArgs) != 2 || opener.page.lastExecuteArgs[0] != int64(1) || opener.page.lastExecuteArgs[1] != int64(2) {
t.Fatalf("unexpected execute args: %#v", opener.page.lastExecuteArgs)
}
}
func TestRuntimeExecuteRepairsObjectLiteralArrowFunction(t *testing.T) {
opener := &fakeOpener{
page: &fakePage{executeValue: map[string]any{"ok": true}},
}
runtime := NewRuntime(opener, true)
page, err := runtime.Open()
if err != nil {
t.Fatal(err)
}
vm := goja.New()
fnValue, err := vm.RunString(`() => ({ ok: true })`)
if err != nil {
t.Fatal(err)
}
if _, err := page.Execute(fnValue); err != nil {
t.Fatal(err)
}
if opener.page.lastExecuteSource != "(() => ({ ok: true }))" {
t.Fatalf("expected repaired function source, got %q", opener.page.lastExecuteSource)
}
}
func TestRuntimeExecuteRepairsMultilineObjectLiteralArrowFunction(t *testing.T) {
opener := &fakeOpener{
page: &fakePage{executeValue: map[string]any{"ok": true}},
}
runtime := NewRuntime(opener, true)
page, err := runtime.Open()
if err != nil {
t.Fatal(err)
}
vm := goja.New()
fnValue, err := vm.RunString(`() => ({
title: document.title || "",
url: String(location.href || ""),
userAgent: navigator.userAgent,
readyState: document.readyState,
})`)
if err != nil {
t.Fatal(err)
}
if _, err := page.Execute(fnValue); err != nil {
t.Fatal(err)
}
expected := `(() => ({
title: document.title || "",
url: String(location.href || ""),
userAgent: navigator.userAgent,
readyState: document.readyState,
}))`
if opener.page.lastExecuteSource != expected {
t.Fatalf("expected repaired multiline function source, got %q", opener.page.lastExecuteSource)
}
}
func TestRuntimeHelpers(t *testing.T) {
opener := &fakeOpener{
page: &fakePage{
executeQueue: []any{
false,
true,
map[string]any{"matched": false, "value": nil},
map[string]any{"matched": true, "value": "ready"},
"https://example.com",
"<html></html>",
},
cookies: []Cookie{
{Name: "a", Value: "1", Domain: "example.com", Path: "/"},
{Name: "b", Value: "two", Domain: "example.com", Path: "/"},
},
},
}
runtime := NewRuntime(opener, true)
page, err := runtime.Open()
if err != nil {
t.Fatal(err)
}
if err := page.AddInitScript("window.__TEST__ = true"); err != nil {
t.Fatal(err)
}
if opener.page.lastInitScript != "window.__TEST__ = true" {
t.Fatalf("unexpected init script: %q", opener.page.lastInitScript)
}
if err := page.Goto("https://example.com", map[string]any{"waitUntil": "domcontentloaded"}); err != nil {
t.Fatal(err)
}
if opener.page.lastGotoURL != "https://example.com" {
t.Fatalf("unexpected goto url: %q", opener.page.lastGotoURL)
}
if opener.page.lastGotoOpts.WaitUntil != "domcontentloaded" {
t.Fatalf("unexpected goto waitUntil: %q", opener.page.lastGotoOpts.WaitUntil)
}
found, err := page.WaitForSelector("#app", map[string]any{"timeoutMs": 50, "pollIntervalMs": 1})
if err != nil {
t.Fatal(err)
}
if !found {
t.Fatal("expected selector to be found")
}
value, err := page.WaitForFunction(`() => window.__READY__`, map[string]any{"timeoutMs": 50, "pollIntervalMs": 1})
if err != nil {
t.Fatal(err)
}
if value != "ready" {
t.Fatalf("unexpected waitForFunction value: %#v", value)
}
cookies, err := page.GetCookies()
if err != nil {
t.Fatal(err)
}
if len(cookies) != 2 || cookies[0].Name != "a" || cookies[0].Value != "1" || cookies[1].Name != "b" || cookies[1].Value != "two" {
t.Fatalf("unexpected cookies: %#v", cookies)
}
if err := page.SetCookie(map[string]any{
"name": "session",
"value": "abc",
"domain": "example.com",
"path": "/",
"secure": true,
"httpOnly": true,
}); err != nil {
t.Fatal(err)
}
if opener.page.lastSetCookie.Name != "session" || !opener.page.lastSetCookie.Secure || !opener.page.lastSetCookie.HTTPOnly {
t.Fatalf("unexpected set cookie: %#v", opener.page.lastSetCookie)
}
if err := page.DeleteCookie(map[string]any{
"name": "session",
"domain": "example.com",
"path": "/",
}); err != nil {
t.Fatal(err)
}
if opener.page.lastDeleteCookie.Name != "session" || opener.page.lastDeleteCookie.Domain != "example.com" {
t.Fatalf("unexpected delete cookie: %#v", opener.page.lastDeleteCookie)
}
if err := page.ClearCookies(); err != nil {
t.Fatal(err)
}
if !opener.page.cookiesCleared {
t.Fatal("expected cookies to be cleared")
}
url, err := page.URL()
if err != nil {
t.Fatal(err)
}
if url != "https://example.com" {
t.Fatalf("unexpected url: %q", url)
}
content, err := page.Content()
if err != nil {
t.Fatal(err)
}
if content != "<html></html>" {
t.Fatalf("unexpected content: %q", content)
}
if err := page.Close(); err != nil {
t.Fatal(err)
}
if !opener.page.closed {
t.Fatal("expected page to be closed")
}
}
func TestRuntimeInteractionHelpers(t *testing.T) {
opener := &fakeOpener{
page: &fakePage{executeValue: true},
}
runtime := NewRuntime(opener, true)
page, err := runtime.Open()
if err != nil {
t.Fatal(err)
}
if err := page.Focus("#search"); err != nil {
t.Fatal(err)
}
if len(opener.page.lastExecuteArgs) != 1 || opener.page.lastExecuteArgs[0] != "#search" {
t.Fatalf("unexpected focus args: %#v", opener.page.lastExecuteArgs)
}
if !strings.Contains(opener.page.lastExecuteSource, "element.focus()") {
t.Fatalf("unexpected focus source: %q", opener.page.lastExecuteSource)
}
if err := page.Click("#submit", map[string]any{"delay": 12}); err != nil {
t.Fatal(err)
}
if len(opener.page.lastExecuteArgs) != 2 || opener.page.lastExecuteArgs[0] != "#submit" || opener.page.lastExecuteArgs[1] != int64(12) {
t.Fatalf("unexpected click args: %#v", opener.page.lastExecuteArgs)
}
if !strings.Contains(opener.page.lastExecuteSource, "performClick") {
t.Fatalf("unexpected click source: %q", opener.page.lastExecuteSource)
}
if err := page.Type("#search", "Gopeed", map[string]any{"delay": 5}); err != nil {
t.Fatal(err)
}
if len(opener.page.lastExecuteArgs) != 3 || opener.page.lastExecuteArgs[0] != "#search" || opener.page.lastExecuteArgs[1] != "Gopeed" || opener.page.lastExecuteArgs[2] != int64(5) {
t.Fatalf("unexpected type args: %#v", opener.page.lastExecuteArgs)
}
if !strings.Contains(opener.page.lastExecuteSource, "element is not typable") {
t.Fatalf("unexpected type source: %q", opener.page.lastExecuteSource)
}
}
func TestRuntimeGotoRejectsInvalidWaitUntil(t *testing.T) {
opener := &fakeOpener{
page: &fakePage{},
}
runtime := NewRuntime(opener, true)
page, err := runtime.Open()
if err != nil {
t.Fatal(err)
}
err = page.Goto("https://example.com", map[string]any{"waitUntil": "networkidle"})
if err == nil || !strings.Contains(err.Error(), "invalid waitUntil") {
t.Fatalf("unexpected error: %v", err)
}
}
type fakeOpener struct {
opts OpenOptions
page *fakePage
}
func (o *fakeOpener) Open(opts OpenOptions) (Page, error) {
o.opts = opts
return o.page, nil
}
type fakePage struct {
lastInitScript string
lastGotoURL string
lastGotoOpts GotoOptions
lastExecuteSource string
lastExecuteArgs []any
executeValue any
executeQueue []any
cookies []Cookie
lastSetCookie Cookie
lastDeleteCookie Cookie
cookiesCleared bool
closed bool
}
func (p *fakePage) AddInitScript(script string) error {
p.lastInitScript = script
return nil
}
func (p *fakePage) Goto(url string, opts GotoOptions) error {
p.lastGotoURL = url
p.lastGotoOpts = opts
return nil
}
func (p *fakePage) Execute(expression string, args ...any) (any, error) {
p.lastExecuteSource = expression
p.lastExecuteArgs = args
if len(p.executeQueue) > 0 {
value := p.executeQueue[0]
p.executeQueue = p.executeQueue[1:]
return value, nil
}
return p.executeValue, nil
}
func (p *fakePage) GetCookies() ([]Cookie, error) {
return append([]Cookie(nil), p.cookies...), nil
}
func (p *fakePage) SetCookie(cookie Cookie) error {
p.lastSetCookie = cookie
return nil
}
func (p *fakePage) DeleteCookie(cookie Cookie) error {
p.lastDeleteCookie = cookie
return nil
}
func (p *fakePage) ClearCookies() error {
p.cookiesCleared = true
return nil
}
func (p *fakePage) Close() error {
p.closed = true
return nil
}
+69 -38
View File
@@ -16,6 +16,7 @@ import (
"github.com/GopeedLab/gopeed/pkg/download/engine"
gojaerror "github.com/GopeedLab/gopeed/pkg/download/engine/inject/error"
gojautil "github.com/GopeedLab/gopeed/pkg/download/engine/util"
enginewebview "github.com/GopeedLab/gopeed/pkg/download/engine/webview"
"github.com/GopeedLab/gopeed/pkg/util"
"github.com/dop251/goja"
"github.com/go-git/go-git/v5"
@@ -268,6 +269,7 @@ func (d *Downloader) triggerOnResolve(req *base.Request) (res *base.Resource, er
for _, file := range ctx.Res.Files {
file.Name = util.SafeFilename(file.Name)
}
ensureResourceRequestRawURLs(req, ctx.Res)
ctx.Res.CalcSize(nil)
}
res = ctx.Res
@@ -281,7 +283,7 @@ func (d *Downloader) triggerOnStart(task *Task) {
EventOnStart,
task.Meta.Req,
&OnStartContext{
Task: NewExtensionTask(d, task),
Task: newOnStartExtensionTask(task),
},
func(ext *Extension, gopeed *Instance, ctx *OnStartContext) {
// Validate request structure
@@ -293,7 +295,6 @@ func (d *Downloader) triggerOnStart(task *Task) {
}
},
)
return
}
func (d *Downloader) triggerOnError(task *Task, err error) {
@@ -301,7 +302,7 @@ func (d *Downloader) triggerOnError(task *Task, err error) {
EventOnError,
task.Meta.Req,
&OnErrorContext{
Task: NewExtensionTask(d, task),
Task: newOnErrorExtensionTask(d, task),
Error: err,
},
nil,
@@ -312,8 +313,8 @@ func (d *Downloader) triggerOnDone(task *Task) {
doTrigger(d,
EventOnDone,
task.Meta.Req,
&OnErrorContext{
Task: NewExtensionTask(d, task),
&OnDoneContext{
Task: newOnDoneExtensionTask(task),
},
nil,
)
@@ -361,11 +362,12 @@ func doTrigger[T any](d *Downloader, event ActivationEvent, req *base.Request, c
if req.Labels == nil {
req.Labels = make(map[string]string)
}
engine := engine.NewEngine(&engine.Config{
ProxyConfig: d.cfg.Proxy,
})
defer engine.Close()
err = engine.Runtime.Set("gopeed", gopeed)
engine, session := d.newExtensionEngine()
defer session.CloseIfIdle()
gopeed.Runtime = &InstanceRuntime{
WebView: d.newExtensionWebViewRuntime(session),
}
err = injectGopeed(engine.Runtime, gopeed)
if err != nil {
gopeed.Logger.logger.Error().Err(err).Msgf("[%s] engine inject failed", ext.buildIdentity())
return
@@ -546,13 +548,17 @@ func (s *Script) match(event ActivationEvent, req *base.Request) bool {
if s.Event != string(event) {
return false
}
if s.Match == nil || (len(s.Match.Urls) == 0 && len(s.Match.Labels) == 0) {
if req == nil || s.Match == nil || (len(s.Match.Urls) == 0 && len(s.Match.Labels) == 0) {
return false
}
// match url
targetURL := req.RawURL
if targetURL == "" {
targetURL = req.URL
}
for _, url := range s.Match.Urls {
if util.Match(url, req.URL) {
if util.Match(url, targetURL) {
return true
}
}
@@ -601,32 +607,37 @@ type Option struct {
// Instance inject to js context when extension script is activated
type Instance struct {
Events InstanceEvents `json:"events"`
Info *ExtensionInfo `json:"info"`
Logger *InstanceLogger `json:"logger"`
Settings map[string]any `json:"settings"`
Storage *ContextStorage `json:"storage"`
Events InstanceEvents `json:"events"`
Info *ExtensionInfo `json:"info"`
Logger *InstanceLogger `json:"logger"`
Settings map[string]any `json:"settings"`
Storage *ContextStorage `json:"storage"`
Runtime *InstanceRuntime `json:"runtime"`
}
type InstanceEvents map[ActivationEvent]goja.Callable
type InstanceRuntime struct {
WebView *enginewebview.Runtime `json:"webview"`
}
func (h InstanceEvents) register(name ActivationEvent, fn goja.Callable) {
type InstanceEvents map[ActivationEvent]engine.JSFunction
func (h InstanceEvents) register(name ActivationEvent, fn engine.JSFunction) {
h[name] = fn
}
func (h InstanceEvents) OnResolve(fn goja.Callable) {
func (h InstanceEvents) OnResolve(fn engine.JSFunction) {
h.register(EventOnResolve, fn)
}
func (h InstanceEvents) OnStart(fn goja.Callable) {
func (h InstanceEvents) OnStart(fn engine.JSFunction) {
h.register(EventOnStart, fn)
}
func (h InstanceEvents) OnError(fn goja.Callable) {
func (h InstanceEvents) OnError(fn engine.JSFunction) {
h.register(EventOnError, fn)
}
func (h InstanceEvents) OnDone(fn goja.Callable) {
func (h InstanceEvents) OnDone(fn engine.JSFunction) {
h.register(EventOnDone, fn)
}
@@ -698,8 +709,8 @@ type OnStartContext struct {
}
type OnErrorContext struct {
Task *ExtensionTask `json:"task"`
Error error `json:"error"`
Task *OnErrorExtensionTask `json:"task"`
Error error `json:"error"`
}
type OnDoneContext struct {
@@ -710,29 +721,49 @@ type OnDoneContext struct {
// Avoid extension scripts modifying task directly, use ExtensionTask to encapsulate task,
// only some fields can be modified, such as request info.
type ExtensionTask struct {
download *Downloader
*Task
}
func NewExtensionTask(download *Downloader, task *Task) *ExtensionTask {
// restricts extension scripts to only modify request info
// OnErrorExtensionTask adds error-recovery controls to ExtensionTask.
// Continue is intentionally only exposed to onError handlers.
type OnErrorExtensionTask struct {
download *Downloader
*ExtensionTask
}
func cloneExtensionTask(task *Task) *Task {
newTask := task.clone()
newTask.Meta.Req = task.Meta.Req
return &ExtensionTask{
download: download,
Task: newTask,
return newTask
}
func newExtensionTask(task *Task) *ExtensionTask {
return &ExtensionTask{Task: cloneExtensionTask(task)}
}
func newOnStartExtensionTask(task *Task) *ExtensionTask {
return newExtensionTask(task)
}
func newOnErrorExtensionTask(download *Downloader, task *Task) *OnErrorExtensionTask {
return &OnErrorExtensionTask{
download: download,
ExtensionTask: newExtensionTask(task),
}
}
func (t *ExtensionTask) Continue() error {
return t.download.Continue(&TaskFilter{
IDs: []string{t.ID},
})
func newOnDoneExtensionTask(task *Task) *Task {
return cloneExtensionTask(task)
}
func (t *ExtensionTask) Pause() error {
return t.download.Pause(&TaskFilter{
// SetUrl replaces the task request URL.
func (t *ExtensionTask) SetUrl(url string) {
t.Meta.Req.URL = url
}
func (t *OnErrorExtensionTask) Continue() error {
return t.download.Continue(&TaskFilter{
IDs: []string{t.ID},
})
}
+75
View File
@@ -0,0 +1,75 @@
package download
import (
"fmt"
"os"
)
type extensionEngineRunner interface {
Close()
RunString(script string) (any, error)
}
type ExtensionEngine struct {
engine extensionEngineRunner
session *engineSession
}
func (d *Downloader) NewExtensionEngine(ext *Extension, settings map[string]any) (*ExtensionEngine, error) {
if ext == nil {
return nil, fmt.Errorf("extension is nil")
}
engine, session := d.newExtensionEngine()
gopeed := &Instance{
Events: make(InstanceEvents),
Info: NewExtensionInfo(ext),
Logger: newInstanceLogger(ext, d.ExtensionLogger),
Settings: settings,
Storage: &ContextStorage{
storage: d.storage,
identity: ext.buildIdentity(),
},
Runtime: &InstanceRuntime{
WebView: d.newExtensionWebViewRuntime(session),
},
}
if err := injectGopeed(engine.Runtime, gopeed); err != nil {
session.CloseIfIdle()
return nil, err
}
return &ExtensionEngine{
engine: engine,
session: session,
}, nil
}
func (r *ExtensionEngine) Eval(script string) (any, error) {
if r == nil || r.engine == nil {
return nil, fmt.Errorf("extension engine not initialized")
}
return r.engine.RunString(script)
}
func (r *ExtensionEngine) EvalFile(path string) (any, error) {
if path == "" {
return nil, fmt.Errorf("script path is empty")
}
buf, err := os.ReadFile(path)
if err != nil {
return nil, err
}
return r.Eval(string(buf))
}
func (r *ExtensionEngine) Close() {
if r == nil {
return
}
if r.session != nil {
r.session.CloseIfIdle()
} else if r.engine != nil {
r.engine.Close()
}
r.engine = nil
r.session = nil
}
+146
View File
@@ -0,0 +1,146 @@
package download
import (
"context"
"io"
"sync"
internalblob "github.com/GopeedLab/gopeed/internal/blob"
"github.com/GopeedLab/gopeed/pkg/download/engine"
"github.com/GopeedLab/gopeed/pkg/download/engine/inject/stream"
enginewebview "github.com/GopeedLab/gopeed/pkg/download/engine/webview"
)
type engineSession struct {
engine *engine.Engine
mu sync.Mutex
refs int
closed bool
close []func()
}
func newEngineSession(e *engine.Engine) *engineSession {
return &engineSession{engine: e}
}
func (s *engineSession) SetEngine(e *engine.Engine) {
s.mu.Lock()
defer s.mu.Unlock()
s.engine = e
}
func (s *engineSession) Retain() {
s.mu.Lock()
defer s.mu.Unlock()
if s.closed {
return
}
s.refs++
}
func (s *engineSession) Release() {
s.mu.Lock()
if s.refs > 0 {
s.refs--
}
shouldClose := s.refs == 0 && !s.closed
if shouldClose {
s.closed = true
}
s.mu.Unlock()
if shouldClose {
s.runClosers()
if s.engine != nil {
go s.engine.Close()
}
}
}
func (s *engineSession) CloseIfIdle() {
s.mu.Lock()
shouldClose := s.refs == 0 && !s.closed
if shouldClose {
s.closed = true
}
s.mu.Unlock()
if shouldClose {
s.runClosers()
if s.engine != nil {
s.engine.Close()
}
}
}
func (s *engineSession) OnClose(fn func()) {
if fn == nil {
return
}
s.mu.Lock()
if s.closed {
s.mu.Unlock()
fn()
return
}
s.close = append(s.close, fn)
s.mu.Unlock()
}
func (s *engineSession) runClosers() {
s.mu.Lock()
closeFns := s.close
s.close = nil
s.mu.Unlock()
for _, fn := range closeFns {
fn()
}
}
func (d *Downloader) newExtensionEngine() (*engine.Engine, *engineSession) {
session := newEngineSession(nil)
engineCfg := &stream.Config{
CreateObjectURL: func(opts *stream.ObjectURLOptions, open stream.ObjectURLOpener) (string, error) {
createOpts := &internalblob.CreateOptions{
Session: session,
}
if opts != nil {
createOpts.ContentType = opts.ContentType
createOpts.Size = opts.Size
createOpts.Range = opts.Range
}
return d.blob.CreateOpener(func(ctx context.Context, req internalblob.OpenRequest) (io.ReadCloser, error) {
return open(ctx, stream.ObjectURLOpenRequest{
Offset: req.Offset,
End: req.End,
})
}, createOpts)
},
RevokeObjectURL: func(url string) error {
return d.blob.Revoke(url)
},
ProxyHandler: d.cfg.Proxy.ToHandler(),
RegisterCleanup: session.OnClose,
}
e := engine.NewEngine(&engine.Config{
ProxyConfig: d.cfg.Proxy,
StreamConfig: engineCfg,
})
session.SetEngine(e)
return e, session
}
func (d *Downloader) newExtensionWebViewRuntime(session *engineSession) *enginewebview.Runtime {
var (
opener enginewebview.Opener
available bool
)
if provider := d.cfg.WebViewProvider; provider != nil && provider.IsAvailable() {
opener = provider
available = true
}
runtime := enginewebview.NewRuntime(opener, available)
session.OnClose(func() {
_ = runtime.Close()
})
return runtime
}
+289
View File
@@ -0,0 +1,289 @@
package download
import (
"fmt"
"github.com/GopeedLab/gopeed/pkg/download/engine"
enginewebview "github.com/GopeedLab/gopeed/pkg/download/engine/webview"
"github.com/dop251/goja"
)
func injectGopeed(vm *goja.Runtime, gopeed *Instance) error {
gopeedObject := vm.NewObject()
if gopeed == nil {
return vm.Set("gopeed", gopeedObject)
}
if err := gopeedObject.Set("events", newJSEventsRuntime(vm, gopeed.Events)); err != nil {
return err
}
if err := gopeedObject.Set("info", gopeed.Info); err != nil {
return err
}
if err := gopeedObject.Set("logger", gopeed.Logger); err != nil {
return err
}
if err := gopeedObject.Set("settings", gopeed.Settings); err != nil {
return err
}
if err := gopeedObject.Set("storage", gopeed.Storage); err != nil {
return err
}
runtimeObject := vm.NewObject()
if gopeed.Runtime != nil {
if err := runtimeObject.Set("blob", newJSBlobRuntime(vm)); err != nil {
return err
}
if gopeed.Runtime.WebView != nil {
if err := runtimeObject.Set("webview", newJSWebViewRuntime(vm, gopeed.Runtime.WebView)); err != nil {
return err
}
}
}
if err := gopeedObject.Set("runtime", runtimeObject); err != nil {
return err
}
return vm.Set("gopeed", gopeedObject)
}
func newJSBlobRuntime(vm *goja.Runtime) *goja.Object {
obj := vm.NewObject()
_ = obj.Set("createObjectURL", func(call goja.FunctionCall) goja.Value {
fn, ok := goja.AssertFunction(vm.Get("__gopeed_blob_create_object_url"))
if !ok {
panic(vm.ToValue(fmt.Errorf("blob runtime is not available")))
}
value, err := fn(goja.Undefined(), call.Argument(0), call.Argument(1))
if err != nil {
panic(vm.ToValue(err))
}
return value
})
_ = obj.Set("revokeObjectURL", func(call goja.FunctionCall) goja.Value {
fn, ok := goja.AssertFunction(vm.Get("__gopeed_blob_revoke_object_url"))
if !ok {
panic(vm.ToValue(fmt.Errorf("blob runtime is not available")))
}
if _, err := fn(goja.Undefined(), call.Argument(0)); err != nil {
panic(vm.ToValue(err))
}
return goja.Undefined()
})
return obj
}
func newJSEventsRuntime(vm *goja.Runtime, events InstanceEvents) *goja.Object {
obj := vm.NewObject()
register := func(event ActivationEvent, call goja.FunctionCall) goja.Value {
if len(call.Arguments) == 0 {
panic(vm.ToValue(fmt.Errorf("missing handler")))
}
fnValue := call.Argument(0)
exported, ok := fnValue.Export().(func(goja.FunctionCall) goja.Value)
if !ok {
panic(vm.ToValue(fmt.Errorf("handler must be a function")))
}
events.register(event, engine.JSFunction(exported))
return goja.Undefined()
}
_ = obj.Set("onResolve", func(call goja.FunctionCall) goja.Value {
return register(EventOnResolve, call)
})
_ = obj.Set("onStart", func(call goja.FunctionCall) goja.Value {
return register(EventOnStart, call)
})
_ = obj.Set("onError", func(call goja.FunctionCall) goja.Value {
return register(EventOnError, call)
})
_ = obj.Set("onDone", func(call goja.FunctionCall) goja.Value {
return register(EventOnDone, call)
})
return obj
}
func newJSWebViewRuntime(vm *goja.Runtime, runtime *enginewebview.Runtime) *goja.Object {
obj := vm.NewObject()
_ = obj.Set("isAvailable", func(goja.FunctionCall) goja.Value {
return vm.ToValue(runtime.IsAvailable())
})
_ = obj.Set("open", func(call goja.FunctionCall) goja.Value {
page, err := runtime.Open(optionalMap(call.Argument(0)))
if err != nil {
panic(vm.ToValue(err))
}
return newJSWebViewPage(vm, page)
})
return obj
}
func newJSWebViewPage(vm *goja.Runtime, page *enginewebview.PageHandle) *goja.Object {
obj := vm.NewObject()
_ = obj.Set("addInitScript", func(call goja.FunctionCall) goja.Value {
script, err := requireStringArg(call, 0, "script")
if err != nil {
panic(vm.ToValue(err))
}
if err := page.AddInitScript(script); err != nil {
panic(vm.ToValue(err))
}
return goja.Undefined()
})
_ = obj.Set("goto", func(call goja.FunctionCall) goja.Value {
url, err := requireStringArg(call, 0, "url")
if err != nil {
panic(vm.ToValue(err))
}
if err := page.Goto(
url,
optionalMap(call.Argument(1)),
); err != nil {
panic(vm.ToValue(err))
}
return goja.Undefined()
})
_ = obj.Set("execute", func(call goja.FunctionCall) goja.Value {
expression, err := enginewebview.NormalizeExecutableValue(call.Argument(0))
if err != nil {
panic(vm.ToValue(err))
}
result, err := page.Execute(expression, exportArgs(call.Arguments[1:])...)
if err != nil {
panic(vm.ToValue(err))
}
return vm.ToValue(result)
})
_ = obj.Set("focus", func(call goja.FunctionCall) goja.Value {
selector, err := requireStringArg(call, 0, "selector")
if err != nil {
panic(vm.ToValue(err))
}
if err := page.Focus(selector); err != nil {
panic(vm.ToValue(err))
}
return goja.Undefined()
})
_ = obj.Set("click", func(call goja.FunctionCall) goja.Value {
selector, err := requireStringArg(call, 0, "selector")
if err != nil {
panic(vm.ToValue(err))
}
if err := page.Click(selector, optionalMap(call.Argument(1))); err != nil {
panic(vm.ToValue(err))
}
return goja.Undefined()
})
_ = obj.Set("type", func(call goja.FunctionCall) goja.Value {
selector, err := requireStringArg(call, 0, "selector")
if err != nil {
panic(vm.ToValue(err))
}
text, err := requireStringArg(call, 1, "text")
if err != nil {
panic(vm.ToValue(err))
}
if err := page.Type(selector, text, optionalMap(call.Argument(2))); err != nil {
panic(vm.ToValue(err))
}
return goja.Undefined()
})
_ = obj.Set("waitForSelector", func(call goja.FunctionCall) goja.Value {
selector, err := requireStringArg(call, 0, "selector")
if err != nil {
panic(vm.ToValue(err))
}
result, err := page.WaitForSelector(
selector,
optionalMap(call.Argument(1)),
)
if err != nil {
panic(vm.ToValue(err))
}
return vm.ToValue(result)
})
_ = obj.Set("waitForFunction", func(call goja.FunctionCall) goja.Value {
expression, err := enginewebview.NormalizeExecutableValue(call.Argument(0))
if err != nil {
panic(vm.ToValue(err))
}
args := exportArgs(call.Arguments[1:])
result, err := page.WaitForFunction(expression, args...)
if err != nil {
panic(vm.ToValue(err))
}
return vm.ToValue(result)
})
_ = obj.Set("getCookies", func(goja.FunctionCall) goja.Value {
result, err := page.GetCookies()
if err != nil {
panic(vm.ToValue(err))
}
return vm.ToValue(result)
})
_ = obj.Set("setCookie", func(call goja.FunctionCall) goja.Value {
if err := page.SetCookie(optionalMap(call.Argument(0))); err != nil {
panic(vm.ToValue(err))
}
return goja.Undefined()
})
_ = obj.Set("deleteCookie", func(call goja.FunctionCall) goja.Value {
if err := page.DeleteCookie(optionalMap(call.Argument(0))); err != nil {
panic(vm.ToValue(err))
}
return goja.Undefined()
})
_ = obj.Set("clearCookies", func(goja.FunctionCall) goja.Value {
if err := page.ClearCookies(); err != nil {
panic(vm.ToValue(err))
}
return goja.Undefined()
})
_ = obj.Set("url", func(goja.FunctionCall) goja.Value {
result, err := page.URL()
if err != nil {
panic(vm.ToValue(err))
}
return vm.ToValue(result)
})
_ = obj.Set("content", func(goja.FunctionCall) goja.Value {
result, err := page.Content()
if err != nil {
panic(vm.ToValue(err))
}
return vm.ToValue(result)
})
_ = obj.Set("close", func(goja.FunctionCall) goja.Value {
if err := page.Close(); err != nil {
panic(vm.ToValue(err))
}
return goja.Undefined()
})
return obj
}
func optionalMap(value goja.Value) map[string]any {
if value == nil || goja.IsUndefined(value) || goja.IsNull(value) {
return nil
}
if exported, ok := value.Export().(map[string]any); ok {
return exported
}
return nil
}
func exportArgs(values []goja.Value) []any {
if len(values) == 0 {
return nil
}
args := make([]any, 0, len(values))
for _, value := range values {
args = append(args, value.Export())
}
return args
}
func requireStringArg(call goja.FunctionCall, index int, name string) (string, error) {
value := call.Argument(index)
if value == nil || goja.IsUndefined(value) || goja.IsNull(value) {
return "", fmt.Errorf(`missing or invalid "%s"`, name)
}
return value.String(), nil
}
File diff suppressed because it is too large. Load diff
+19 -1
View File
@@ -12,6 +12,7 @@ import (
"sync/atomic"
"github.com/mholt/archives"
"golang.org/x/text/encoding/simplifiedchinese"
)
// supportedArchiveExtensions contains file extensions supported by mholt/archives library
@@ -70,6 +71,16 @@ type ArchivePartInfo struct {
// ExtractProgressCallback is called to report extraction progress
type ExtractProgressCallback func(extractedFiles int, totalFiles int, progress int)
// newZipFormat creates a Zip format with proper character encoding support.
// It uses GB18030 encoding to handle Chinese characters in filenames that may
// be encoded with legacy GBK/GB18030 instead of UTF-8.
func newZipFormat() archives.Zip {
return archives.Zip{
// GB18030 is a superset of GBK and handles Chinese characters correctly
TextEncoding: simplifiedchinese.GB18030,
}
}
// isArchiveFile checks if a file is a supported archive format
func isArchiveFile(filename string) bool {
lowerName := strings.ToLower(filename)
@@ -109,7 +120,8 @@ func openArchive(archivePath string, password string) (*archiveInfo, error) {
return nil, err
}
// Handle password-protected archives
// Configure format-specific settings
// Handle password-protected archives and character encoding
if password != "" {
if rar, ok := format.(archives.Rar); ok {
rar.Password = password
@@ -121,6 +133,12 @@ func openArchive(archivePath string, password string) (*archiveInfo, error) {
}
}
// For ZIP files, configure character encoding to handle non-UTF8 filenames
// This is essential for Chinese characters encoded in GBK/GB18030
if _, ok := format.(archives.Zip); ok {
format = newZipFormat()
}
return &archiveInfo{
file: file,
stat: stat,
+109
View File
@@ -10,6 +10,8 @@ import (
"path/filepath"
"strings"
"testing"
"golang.org/x/text/encoding/simplifiedchinese"
)
func TestIsArchiveFile(t *testing.T) {
@@ -243,6 +245,69 @@ func createTestZipWithMultipleFiles(path string, numFiles int) error {
return w.Close()
}
// createTestZipWithChineseFilenames creates a test ZIP file with Chinese filenames encoded in GBK
func createTestZipWithChineseFilenames(path string) error {
zipFile, err := os.Create(path)
if err != nil {
return err
}
defer zipFile.Close()
w := zip.NewWriter(zipFile)
// Encode Chinese filenames in GBK (as some legacy Windows applications do)
encoder := simplifiedchinese.GBK.NewEncoder()
// Add a file with Chinese filename
chineseFilename := "测试文件.txt"
gbkFilename, err := encoder.String(chineseFilename)
if err != nil {
return err
}
// Create a FileHeader and manually set the Name with GBK encoding
// We need to mark it as non-UTF8 by not setting the UTF-8 flag
header := &zip.FileHeader{
Name: gbkFilename,
Method: zip.Deflate,
}
// Clear the UTF-8 bit (bit 11) to indicate non-UTF8 encoding
header.Flags = 0
f, err := w.CreateHeader(header)
if err != nil {
return err
}
_, err = f.Write([]byte("这是测试内容"))
if err != nil {
return err
}
// Add a file in a subdirectory with Chinese name
chineseDirAndFile := "文件夹/中文内容.txt"
gbkDirAndFile, err := encoder.String(chineseDirAndFile)
if err != nil {
return err
}
header2 := &zip.FileHeader{
Name: gbkDirAndFile,
Method: zip.Deflate,
}
header2.Flags = 0
f2, err := w.CreateHeader(header2)
if err != nil {
return err
}
_, err = f2.Write([]byte("中文子文件内容"))
if err != nil {
return err
}
return w.Close()
}
func TestOpenArchive_NonExistentFile(t *testing.T) {
_, err := openArchive("/nonexistent/path/file.zip", "")
if err == nil {
@@ -2382,3 +2447,47 @@ func TestExtractZipMultiPart_Progress(t *testing.T) {
t.Error("Expected progress callbacks")
}
}
// Test extracting ZIP files with Chinese filenames encoded in GBK/GB18030
func TestExtractArchive_ChineseFilenames(t *testing.T) {
tempDir, err := os.MkdirTemp("", "extract_chinese_test")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tempDir)
// Create a test ZIP file with Chinese filenames encoded in GBK
zipPath := filepath.Join(tempDir, "chinese.zip")
destDir := filepath.Join(tempDir, "extracted")
if err := createTestZipWithChineseFilenames(zipPath); err != nil {
t.Fatal(err)
}
// Extract the archive
err = extractArchive(zipPath, destDir, "", nil)
if err != nil {
t.Fatalf("extractArchive failed: %v", err)
}
// Verify the extracted files with proper Chinese filenames
expectedFiles := []string{
filepath.Join(destDir, "测试文件.txt"),
filepath.Join(destDir, "文件夹", "中文内容.txt"),
}
for _, path := range expectedFiles {
if _, err := os.Stat(path); os.IsNotExist(err) {
t.Errorf("expected file %q not found after extraction", path)
}
}
// Verify content of the Chinese file
content, err := os.ReadFile(filepath.Join(destDir, "测试文件.txt"))
if err != nil {
t.Fatal(err)
}
if string(content) != "这是测试内容" {
t.Errorf("unexpected content: %q", string(content))
}
}
+1 -1
View File
@@ -34,7 +34,7 @@ func extractZipMultiPart(firstPartPath string, destDir string, password string,
// First pass: count files for progress
totalFiles := 0
zip := archives.Zip{}
zip := newZipFormat()
err = zip.Extract(context.Background(), io.NewSectionReader(multiReader, 0, totalSize), func(ctx context.Context, fileInfo archives.FileInfo) error {
if !fileInfo.IsDir() {
totalFiles++
+15
View File
@@ -8,8 +8,10 @@ import (
"github.com/GopeedLab/gopeed/internal/controller"
"github.com/GopeedLab/gopeed/internal/fetcher"
"github.com/GopeedLab/gopeed/internal/protocol/bt"
"github.com/GopeedLab/gopeed/internal/protocol/ed2k"
"github.com/GopeedLab/gopeed/internal/protocol/http"
"github.com/GopeedLab/gopeed/pkg/base"
enginewebview "github.com/GopeedLab/gopeed/pkg/download/engine/webview"
"github.com/GopeedLab/gopeed/pkg/util"
gonanoid "github.com/matoous/go-nanoid/v2"
)
@@ -34,6 +36,9 @@ type Task struct {
timer *util.Timer
statusLock *sync.Mutex
lock *sync.Mutex
blobRefLock *sync.Mutex
blobURL string
runGeneration uint64
speedArr []int64
uploadSpeedArr []int64
}
@@ -110,6 +115,14 @@ func (t *Task) updateUploadSpeed(downloaded int64, usedTime float64) int64 {
}
func calcSpeed(speedArr *[]int64, downloaded int64, usedTime float64) int64 {
if usedTime <= 0 {
return 0
}
if downloaded < 0 {
*speedArr = (*speedArr)[:0]
return 0
}
*speedArr = append(*speedArr, downloaded)
// Record last 5 seconds of download speed to calculate the average speed
if len(*speedArr) > int(5.0/usedTime) {
@@ -142,6 +155,7 @@ type DownloaderConfig struct {
Storage Storage
StorageDir string
WhiteDownloadDirs []string
WebViewProvider enginewebview.Provider
ProductionMode bool
@@ -156,6 +170,7 @@ func (cfg *DownloaderConfig) Init() *DownloaderConfig {
cfg.FetchManagers = []fetcher.FetcherManager{
new(http.FetcherManager),
new(bt.FetcherManager),
new(ed2k.FetcherManager),
}
}
if cfg.RefreshInterval == 0 {
+18
View File
@@ -0,0 +1,18 @@
package download
import "testing"
func TestCalcSpeedResetOnRollback(t *testing.T) {
speedArr := []int64{1024, 2048, 4096}
if got := calcSpeed(&speedArr, -512, 1); got != 0 {
t.Fatalf("calcSpeed() = %d, want 0 after rollback", got)
}
if len(speedArr) != 0 {
t.Fatalf("speed window len = %d, want 0 after rollback", len(speedArr))
}
if got := calcSpeed(&speedArr, 1024, 1); got != 1024 {
t.Fatalf("calcSpeed() = %d, want 1024 after reset", got)
}
}
+161
View File
@@ -0,0 +1,161 @@
package download
import (
"fmt"
"os"
"os/exec"
"path/filepath"
"runtime"
"time"
)
// ScriptEvent represents the type of script event
type ScriptEvent string
const (
ScriptEventDownloadDone ScriptEvent = "DOWNLOAD_DONE"
ScriptEventDownloadError ScriptEvent = "DOWNLOAD_ERROR"
)
// ScriptData is the internal data structure for passing script information
type ScriptData struct {
Event ScriptEvent
Time int64 // Unix timestamp in milliseconds
Payload *ScriptPayload
}
// ScriptPayload contains the task data
type ScriptPayload struct {
Task *Task
}
// getScriptPaths extracts script paths from config
func (d *Downloader) getScriptPaths() []string {
cfg := d.cfg.DownloaderStoreConfig
if cfg == nil {
return nil
}
// Check new script config
if cfg.Script != nil && cfg.Script.Enable && len(cfg.Script.Paths) > 0 {
paths := make([]string, 0, len(cfg.Script.Paths))
for _, path := range cfg.Script.Paths {
if path != "" {
paths = append(paths, path)
}
}
if len(paths) > 0 {
return paths
}
}
return nil
}
// executeScriptAtPath executes a single script with the given data
// Returns any error that occurred during execution
func (d *Downloader) executeScriptAtPath(scriptPath string, data *ScriptData) error {
if scriptPath == "" {
return fmt.Errorf("script path is empty")
}
// Check if script file exists
if _, err := os.Stat(scriptPath); os.IsNotExist(err) {
return fmt.Errorf("script file does not exist: %s", scriptPath)
}
// Determine the script interpreter based on file extension
var cmd *exec.Cmd
ext := filepath.Ext(scriptPath)
switch ext {
case ".sh", ".bash":
cmd = exec.Command("bash", scriptPath)
case ".py":
cmd = exec.Command("python3", scriptPath)
case ".js":
cmd = exec.Command("node", scriptPath)
case ".bat", ".cmd":
// Windows batch files
if runtime.GOOS == "windows" {
cmd = exec.Command("cmd", "/c", scriptPath)
} else {
// Batch files are Windows-specific
return fmt.Errorf("batch files (.bat/.cmd) are only supported on Windows")
}
case ".ps1":
// PowerShell scripts
if runtime.GOOS == "windows" {
cmd = exec.Command("powershell", "-ExecutionPolicy", "Bypass", "-File", scriptPath)
} else {
// Try pwsh (PowerShell Core) on non-Windows systems
cmd = exec.Command("pwsh", "-File", scriptPath)
}
case "":
// No extension, try to execute directly (assumes shebang or executable)
cmd = exec.Command(scriptPath)
default:
// Unknown extension, try to execute directly
cmd = exec.Command(scriptPath)
}
// Set environment variables with task information
cmd.Env = append(os.Environ(),
fmt.Sprintf("GOPEED_EVENT=%s", data.Event),
fmt.Sprintf("GOPEED_TASK_ID=%s", data.Payload.Task.ID),
fmt.Sprintf("GOPEED_TASK_NAME=%s", data.Payload.Task.Name()),
fmt.Sprintf("GOPEED_TASK_STATUS=%s", data.Payload.Task.Status),
)
// Add task path using the same logic as task deletion
if data.Payload.Task.Meta != nil && data.Payload.Task.Meta.Res != nil {
var taskPath string
if data.Payload.Task.Meta.Res.Name != "" {
// Multi-file task (folder)
taskPath = data.Payload.Task.Meta.FolderPath()
} else {
// Single file task
taskPath = data.Payload.Task.Meta.SingleFilepath()
}
cmd.Env = append(cmd.Env,
fmt.Sprintf("GOPEED_TASK_PATH=%s", taskPath),
)
}
// Start and wait for the command to complete (no timeout)
return cmd.Run()
}
// triggerScripts executes all configured scripts
func (d *Downloader) triggerScripts(event ScriptEvent, task *Task, err error) {
paths := d.getScriptPaths()
if len(paths) == 0 {
return
}
data := &ScriptData{
Event: event,
Time: time.Now().UnixMilli(),
Payload: &ScriptPayload{
Task: task.clone(),
},
}
go d.executeScripts(paths, data)
}
func (d *Downloader) executeScripts(paths []string, data *ScriptData) {
for _, path := range paths {
if path == "" {
continue
}
go func(scriptPath string) {
err := d.executeScriptAtPath(scriptPath, data)
if err != nil {
d.Logger.Warn().Err(err).Str("path", scriptPath).Msg("script: failed to execute")
return
}
d.Logger.Debug().Str("path", scriptPath).Msg("script: executed successfully")
}(path)
}
}
+203
View File
@@ -0,0 +1,203 @@
package download
import (
"os"
"path/filepath"
"testing"
"time"
"github.com/GopeedLab/gopeed/internal/fetcher"
"github.com/GopeedLab/gopeed/pkg/base"
)
func TestScript_NoScriptConfigured(t *testing.T) {
setupScriptTest(t, func(downloader *Downloader) {
// Create a mock task
task := NewTask()
task.Protocol = "http"
task.Meta = &mockFetcherMeta
// Trigger script (should not panic with no scripts configured)
downloader.triggerScripts(ScriptEventDownloadDone, task, nil)
})
}
func TestScript_GetScriptPaths_EmptyConfig(t *testing.T) {
setupScriptTest(t, func(downloader *Downloader) {
paths := downloader.getScriptPaths()
if paths != nil {
t.Errorf("Expected nil, got %v", paths)
}
})
}
func TestScript_GetScriptPaths_NoScriptConfig(t *testing.T) {
setupScriptTest(t, func(downloader *Downloader) {
cfg, _ := downloader.GetConfig()
cfg.Script = nil
downloader.PutConfig(cfg)
paths := downloader.getScriptPaths()
if paths != nil {
t.Errorf("Expected nil, got %v", paths)
}
})
}
func TestScript_GetScriptPaths_DisabledScript(t *testing.T) {
setupScriptTest(t, func(downloader *Downloader) {
cfg, _ := downloader.GetConfig()
cfg.Script = &base.ScriptConfig{
Enable: false,
Paths: []string{"/path/to/script.sh"},
}
downloader.PutConfig(cfg)
paths := downloader.getScriptPaths()
if paths != nil {
t.Errorf("Expected nil for disabled script, got %v", paths)
}
})
}
func TestScript_GetScriptPaths_EmptyPaths(t *testing.T) {
setupScriptTest(t, func(downloader *Downloader) {
cfg, _ := downloader.GetConfig()
cfg.Script = &base.ScriptConfig{
Enable: true,
Paths: []string{},
}
downloader.PutConfig(cfg)
paths := downloader.getScriptPaths()
if paths != nil {
t.Errorf("Expected nil for empty paths, got %v", paths)
}
})
}
func TestScript_GetScriptPaths_WithEmptyStrings(t *testing.T) {
setupScriptTest(t, func(downloader *Downloader) {
cfg, _ := downloader.GetConfig()
cfg.Script = &base.ScriptConfig{
Enable: true,
Paths: []string{"/path/to/script1.sh", "", "/path/to/script2.sh", ""},
}
downloader.PutConfig(cfg)
paths := downloader.getScriptPaths()
if len(paths) != 2 {
t.Errorf("Expected 2 valid paths (ignoring empty strings), got %d: %v", len(paths), paths)
}
if paths[0] != "/path/to/script1.sh" || paths[1] != "/path/to/script2.sh" {
t.Errorf("Paths don't match expected values: %v", paths)
}
})
}
func TestScript_ExecuteScriptAtPath_EmptyPath(t *testing.T) {
setupScriptTest(t, func(downloader *Downloader) {
data := &ScriptData{
Event: ScriptEventDownloadDone,
Time: time.Now().UnixMilli(),
}
err := downloader.executeScriptAtPath("", data)
if err == nil {
t.Error("Expected error for empty path")
}
if err.Error() != "script path is empty" {
t.Errorf("Expected 'script path is empty' error, got: %v", err)
}
})
}
func TestScript_ExecuteScriptAtPath_NonExistentFile(t *testing.T) {
setupScriptTest(t, func(downloader *Downloader) {
data := &ScriptData{
Event: ScriptEventDownloadDone,
Time: time.Now().UnixMilli(),
}
err := downloader.executeScriptAtPath("/non/existent/script.sh", data)
if err == nil {
t.Error("Expected error for non-existent script")
}
})
}
func createDownloadDoneTask(t *testing.T, downloadDir, fileName string) (*Task, string) {
t.Helper()
content := []byte("downloaded file")
task := NewTask()
task.Protocol = "http"
task.Status = base.DownloadStatusDone
task.Meta = &fetcher.FetcherMeta{
Req: &base.Request{
URL: "https://example.com/" + fileName,
},
Opts: &base.Options{
Name: fileName,
Path: filepath.ToSlash(downloadDir),
},
Res: &base.Resource{
Size: int64(len(content)),
Files: []*base.FileInfo{
{Name: fileName, Size: int64(len(content))},
},
},
}
filePath := task.Meta.SingleFilepath()
filePathOS := filepath.FromSlash(filePath)
if err := os.MkdirAll(filepath.Dir(filePathOS), 0755); err != nil {
t.Fatalf("Failed to create download dir: %v", err)
}
if err := os.WriteFile(filePathOS, content, 0644); err != nil {
t.Fatalf("Failed to create download file: %v", err)
}
return task, filePath
}
func waitForFile(t *testing.T, path string, timeout time.Duration) {
t.Helper()
deadline := time.Now().Add(timeout)
for time.Now().Before(deadline) {
if _, err := os.Stat(path); err == nil {
return
}
time.Sleep(50 * time.Millisecond)
}
t.Fatalf("Timeout waiting for file: %s", path)
}
func getTestScriptPath(t *testing.T, name string) string {
t.Helper()
path := filepath.Join("testdata", "scripts", name)
if _, err := os.Stat(path); err != nil {
t.Fatalf("Missing test script %s: %v", path, err)
}
return path
}
func ensureScriptExecutable(t *testing.T, scriptPath string) {
t.Helper()
if filepath.Ext(scriptPath) != ".sh" {
return
}
if err := os.Chmod(scriptPath, 0755); err != nil {
t.Fatalf("Failed to chmod script: %v", err)
}
}
func setupScriptTest(t *testing.T, fn func(downloader *Downloader)) {
defaultDownloader.Setup()
defaultDownloader.cfg.StorageDir = ".test_storage"
defaultDownloader.cfg.DownloadDir = ".test_download"
defer func() {
defaultDownloader.Clear()
os.RemoveAll(defaultDownloader.cfg.StorageDir)
os.RemoveAll(defaultDownloader.cfg.DownloadDir)
}()
fn(defaultDownloader)
}
+117
View File
@@ -0,0 +1,117 @@
//go:build !windows
// +build !windows
package download
import (
"os"
"path/filepath"
"strings"
"testing"
"time"
"github.com/GopeedLab/gopeed/pkg/base"
)
func TestScript_TriggerOnDone_MoveFile(t *testing.T) {
tmpDir := t.TempDir()
downloadDir := filepath.Join(tmpDir, "downloads")
destDir := filepath.Join(tmpDir, "moved")
task, taskPath := createDownloadDoneTask(t, downloadDir, "test.txt")
srcPath := filepath.FromSlash(taskPath)
destFile := filepath.Join(destDir, "test.txt")
scriptPath := getTestScriptPath(t, "move.sh")
ensureScriptExecutable(t, scriptPath)
setupScriptTest(t, func(downloader *Downloader) {
cfg, _ := downloader.GetConfig()
cfg.Script = &base.ScriptConfig{
Enable: true,
Paths: []string{scriptPath},
}
downloader.PutConfig(cfg)
t.Setenv("GOPEED_TEST_DEST_DIR", destDir)
downloader.triggerScripts(ScriptEventDownloadDone, task, nil)
waitForFile(t, destFile, 3*time.Second)
if _, err := os.Stat(srcPath); !os.IsNotExist(err) {
t.Errorf("Expected source file to be moved, but it still exists: %s", srcPath)
}
})
}
func TestScript_MultipleScripts(t *testing.T) {
tmpDir := t.TempDir()
outputFile1 := filepath.Join(tmpDir, "output1.txt")
outputFile2 := filepath.Join(tmpDir, "output2.txt")
scriptPath1 := getTestScriptPath(t, "write_output1.sh")
ensureScriptExecutable(t, scriptPath1)
scriptPath2 := getTestScriptPath(t, "write_output2.sh")
ensureScriptExecutable(t, scriptPath2)
setupScriptTest(t, func(downloader *Downloader) {
cfg, _ := downloader.GetConfig()
cfg.Script = &base.ScriptConfig{
Enable: true,
Paths: []string{scriptPath1, scriptPath2},
}
downloader.PutConfig(cfg)
downloadDir := filepath.Join(tmpDir, "downloads")
task, _ := createDownloadDoneTask(t, downloadDir, "multi.txt")
t.Setenv("GOPEED_TEST_OUTPUT_FILE_1", outputFile1)
t.Setenv("GOPEED_TEST_OUTPUT_FILE_2", outputFile2)
downloader.triggerScripts(ScriptEventDownloadDone, task, nil)
waitForFile(t, outputFile1, 3*time.Second)
waitForFile(t, outputFile2, 3*time.Second)
})
}
func TestScript_EnvironmentVariables(t *testing.T) {
tmpDir := t.TempDir()
scriptPath := getTestScriptPath(t, "env_dump.sh")
ensureScriptExecutable(t, scriptPath)
outputFile := filepath.Join(tmpDir, "env_output.txt")
setupScriptTest(t, func(downloader *Downloader) {
cfg, _ := downloader.GetConfig()
cfg.Script = &base.ScriptConfig{
Enable: true,
Paths: []string{scriptPath},
}
downloader.PutConfig(cfg)
downloadDir := filepath.Join(tmpDir, "downloads")
task, taskPath := createDownloadDoneTask(t, downloadDir, "env.txt")
t.Setenv("GOPEED_TEST_OUTPUT_FILE", outputFile)
downloader.triggerScripts(ScriptEventDownloadDone, task, nil)
waitForFile(t, outputFile, 3*time.Second)
content, err := os.ReadFile(outputFile)
if err != nil {
t.Fatalf("Failed to read output file: %v", err)
}
output := string(content)
if !strings.Contains(output, "GOPEED_EVENT=DOWNLOAD_DONE") {
t.Errorf("Expected GOPEED_EVENT in output, got: %s", output)
}
if !strings.Contains(output, "GOPEED_TASK_ID="+task.ID) {
t.Errorf("Expected GOPEED_TASK_ID in output, got: %s", output)
}
if !strings.Contains(output, "GOPEED_TASK_NAME="+task.Name()) {
t.Errorf("Expected GOPEED_TASK_NAME in output, got: %s", output)
}
if !strings.Contains(output, "GOPEED_TASK_STATUS="+string(task.Status)) {
t.Errorf("Expected GOPEED_TASK_STATUS in output, got: %s", output)
}
if !strings.Contains(output, "GOPEED_TASK_PATH="+taskPath) {
t.Errorf("Expected GOPEED_TASK_PATH in output, got: %s", output)
}
})
}
+113
View File
@@ -0,0 +1,113 @@
//go:build windows
// +build windows
package download
import (
"os"
"path/filepath"
"strings"
"testing"
"time"
"github.com/GopeedLab/gopeed/pkg/base"
)
func TestScript_TriggerOnDone_MoveFile(t *testing.T) {
tmpDir := t.TempDir()
downloadDir := filepath.Join(tmpDir, "downloads")
destDir := filepath.Join(tmpDir, "moved")
task, taskPath := createDownloadDoneTask(t, downloadDir, "test.txt")
srcPath := filepath.FromSlash(taskPath)
destFile := filepath.Join(destDir, "test.txt")
scriptPath := getTestScriptPath(t, "move.bat")
setupScriptTest(t, func(downloader *Downloader) {
cfg, _ := downloader.GetConfig()
cfg.Script = &base.ScriptConfig{
Enable: true,
Paths: []string{scriptPath},
}
downloader.PutConfig(cfg)
t.Setenv("GOPEED_TEST_DEST_DIR", destDir)
downloader.triggerScripts(ScriptEventDownloadDone, task, nil)
waitForFile(t, destFile, 5*time.Second)
if _, err := os.Stat(srcPath); !os.IsNotExist(err) {
t.Errorf("Expected source file to be moved, but it still exists: %s", srcPath)
}
})
}
func TestScript_MultipleScripts(t *testing.T) {
tmpDir := t.TempDir()
outputFile1 := filepath.Join(tmpDir, "output1.txt")
outputFile2 := filepath.Join(tmpDir, "output2.txt")
scriptPath1 := getTestScriptPath(t, "write_output1.bat")
scriptPath2 := getTestScriptPath(t, "write_output2.bat")
setupScriptTest(t, func(downloader *Downloader) {
cfg, _ := downloader.GetConfig()
cfg.Script = &base.ScriptConfig{
Enable: true,
Paths: []string{scriptPath1, scriptPath2},
}
downloader.PutConfig(cfg)
downloadDir := filepath.Join(tmpDir, "downloads")
task, _ := createDownloadDoneTask(t, downloadDir, "multi.txt")
t.Setenv("GOPEED_TEST_OUTPUT_FILE_1", outputFile1)
t.Setenv("GOPEED_TEST_OUTPUT_FILE_2", outputFile2)
downloader.triggerScripts(ScriptEventDownloadDone, task, nil)
waitForFile(t, outputFile1, 5*time.Second)
waitForFile(t, outputFile2, 5*time.Second)
})
}
func TestScript_EnvironmentVariables(t *testing.T) {
tmpDir := t.TempDir()
scriptPath := getTestScriptPath(t, "env_dump.bat")
outputFile := filepath.Join(tmpDir, "env_output.txt")
setupScriptTest(t, func(downloader *Downloader) {
cfg, _ := downloader.GetConfig()
cfg.Script = &base.ScriptConfig{
Enable: true,
Paths: []string{scriptPath},
}
downloader.PutConfig(cfg)
downloadDir := filepath.Join(tmpDir, "downloads")
task, taskPath := createDownloadDoneTask(t, downloadDir, "env.txt")
t.Setenv("GOPEED_TEST_OUTPUT_FILE", outputFile)
downloader.triggerScripts(ScriptEventDownloadDone, task, nil)
waitForFile(t, outputFile, 5*time.Second)
content, err := os.ReadFile(outputFile)
if err != nil {
t.Fatalf("Failed to read output file: %v", err)
}
output := string(content)
if !strings.Contains(output, "GOPEED_EVENT=DOWNLOAD_DONE") {
t.Errorf("Expected GOPEED_EVENT in output, got: %s", output)
}
if !strings.Contains(output, "GOPEED_TASK_ID="+task.ID) {
t.Errorf("Expected GOPEED_TASK_ID in output, got: %s", output)
}
if !strings.Contains(output, "GOPEED_TASK_NAME="+task.Name()) {
t.Errorf("Expected GOPEED_TASK_NAME in output, got: %s", output)
}
if !strings.Contains(output, "GOPEED_TASK_STATUS="+string(task.Status)) {
t.Errorf("Expected GOPEED_TASK_STATUS in output, got: %s", output)
}
if !strings.Contains(output, "GOPEED_TASK_PATH="+taskPath) {
t.Errorf("Expected GOPEED_TASK_PATH in output, got: %s", output)
}
})
}
+3
View File
@@ -1,4 +1,7 @@
gopeed.events.onResolve(async function (ctx) {
ctx.req.setLabels({ replaced: "true", removed: "true" });
ctx.req.putLabel("modified", "true");
ctx.req.delLabel("removed");
ctx.res = {
name: "test",
files: Array(2).fill(true).map((_, i) => ({
+111
View File
@@ -0,0 +1,111 @@
gopeed.events.onResolve(async function (ctx) {
const encoder = new TextEncoder();
if (ctx.req.url.endsWith("/blob")) {
const blob = new Blob(["hello world"], {type: "text/plain"});
const url = gopeed.runtime.blob.createObjectURL(blob);
ctx.res = {
name: "blob-blob",
files: [
{
name: "hello.txt",
size: 11,
req: {
url,
}
}
]
};
return;
}
if (ctx.req.url.endsWith("/opener-range")) {
const payload = encoder.encode("line 1\nline 2\nline 3\n");
const firstChunkLength = encoder.encode("line 1\n").byteLength;
let firstAttempt = true;
const open = async ({ offset = 0, end = -1 }) => new ReadableStream({
async start(controller) {
try {
if (offset === 0 && firstAttempt) {
firstAttempt = false;
controller.enqueue(payload.slice(0, firstChunkLength));
await new Promise((resolve) => setTimeout(resolve, 120));
controller.error(new Error("resume required"));
return;
}
controller.enqueue(payload.slice(offset, end >= 0 ? end + 1 : undefined));
controller.close();
} catch (err) {
controller.error(err);
}
}
});
const url = gopeed.runtime.blob.createObjectURL(open, { size: payload.byteLength, range: true });
ctx.res = {
name: "blob-opener-range",
range: true,
files: [
{
name: "opener-range.txt",
size: payload.byteLength,
req: {
url,
}
}
]
};
return;
}
if (ctx.req.url.endsWith("/opener-unknown")) {
const url = gopeed.runtime.blob.createObjectURL(async ({ offset = 0 }) => new ReadableStream({
async start(controller) {
if (offset > 0) {
controller.close();
return;
}
controller.enqueue(encoder.encode("line 1\n"));
await new Promise((resolve) => setTimeout(resolve, 1500));
controller.enqueue(encoder.encode("line 2\n"));
controller.close();
},
}));
ctx.res = {
name: "blob-opener-unknown",
files: [
{
name: "opener-unknown.txt",
req: {
url,
}
}
]
};
return;
}
const url = gopeed.runtime.blob.createObjectURL(async ({ offset = 0, end = -1 }) => new ReadableStream({
start(controller) {
const payload = encoder.encode("line 1\nline 2\n");
controller.enqueue(payload.slice(offset, end >= 0 ? end + 1 : undefined));
controller.close();
},
}), { size: 14 });
ctx.res = {
name: "blob-opener",
files: [
{
name: "opener.txt",
size: 14,
req: {
url,
}
}
]
};
});
+16
View File
@@ -0,0 +1,16 @@
{
"name": "blob",
"title": "gopeed extension blob test",
"version": "0.0.1",
"scripts": [
{
"event": "onResolve",
"match": {
"urls": [
"*://example.com/*"
]
},
"entry": "index.js"
}
]
}
@@ -0,0 +1,139 @@
const payloadSize = 262144;
const chunkSize = 4096;
const payloadText = 'x'.repeat(payloadSize);
function delayedPayloadStream(offset = 0, end = -1) {
const limit = end >= 0 ? Math.min(end + 1, payloadSize) : payloadSize;
let pos = offset;
return new ReadableStream({
async pull(controller) {
if (pos >= limit) {
controller.close();
return;
}
const size = Math.min(chunkSize, limit - pos);
const chunk = new Uint8Array(size);
chunk.fill(120);
pos += size;
controller.enqueue(chunk);
await new Promise((resolve) => setTimeout(resolve, 20));
},
});
}
function fastPayloadStream(offset = 0, end = -1) {
const limit = end >= 0 ? Math.min(end + 1, payloadSize) : payloadSize;
let pos = offset;
return new ReadableStream({
pull(controller) {
if (pos >= limit) {
controller.close();
return;
}
const size = Math.min(chunkSize, limit - pos);
const chunk = new Uint8Array(size);
chunk.fill(120);
pos += size;
controller.enqueue(chunk);
},
});
}
function createBlobSourceURL(delayed) {
const blob = new Blob([payloadText], { type: 'application/octet-stream' });
const originalSlice = blob.slice.bind(blob);
blob.slice = function (offset = 0, end) {
if (!delayed) {
return originalSlice(offset, end);
}
const sliceEnd = Number.isFinite(Number(end)) ? Number(end) - 1 : -1;
return {
stream() {
return delayedPayloadStream(Number(offset) || 0, sliceEnd);
},
};
};
return gopeed.runtime.blob.createObjectURL(blob);
}
function createOpenerSourceURL(delayed) {
const open = async () => delayed ? delayedPayloadStream(0, -1) : fastPayloadStream(0, -1);
return gopeed.runtime.blob.createObjectURL(open, {
size: payloadSize,
contentType: 'application/octet-stream',
});
}
function createRangeSourceURL(delayed) {
const open = async ({ offset = 0, end = -1 }) => delayed ? delayedPayloadStream(offset, end) : fastPayloadStream(offset, end);
return gopeed.runtime.blob.createObjectURL(open, {
size: payloadSize,
range: true,
contentType: 'application/octet-stream',
});
}
function modeFromURL(url) {
if (url.includes('/pause-rebuild-blob')) {
return 'blob';
}
if (url.includes('/pause-rebuild-opener')) {
return 'opener';
}
if (url.includes('/pause-rebuild-range')) {
return 'range';
}
return '';
}
function createPayloadURL(mode, delayed) {
switch (mode) {
case 'blob':
return createBlobSourceURL(delayed);
case 'opener':
return createOpenerSourceURL(delayed);
case 'range':
return createRangeSourceURL(delayed);
default:
throw new Error('unsupported pause rebuild mode: ' + mode);
}
}
gopeed.events.onResolve(async function (ctx) {
const mode = modeFromURL(ctx.req.url);
if (!mode) {
return;
}
ctx.res = {
name: 'blob-pause-rebuild-' + mode,
range: mode !== 'opener',
files: [
{
name: 'pause-rebuild-' + mode + '.bin',
size: payloadSize,
req: {
url: createPayloadURL(mode, true),
rawUrl: ctx.req.rawUrl || ctx.req.url,
labels: {
mode: 'pause-rebuild',
source: mode,
},
},
},
],
};
});
gopeed.events.onError(async function (ctx) {
const req = ctx.task?.meta?.req;
const source = req?.labels?.source || modeFromURL(req?.rawUrl || '');
if (!req || req.labels?.mode !== 'pause-rebuild' || req.labels?.rebuilt === 'true' || !source) {
return;
}
ctx.task.setUrl(createPayloadURL(source, false));
req.putLabel('started', 'true');
req.putLabel('rebuilt', 'true');
ctx.task.continue();
});
@@ -0,0 +1,28 @@
{
"name": "blob-pause-rebuild",
"title": "gopeed extension blob pause rebuild test",
"version": "0.0.1",
"scripts": [
{
"event": "onResolve",
"match": {
"urls": [
"*://example.com/pause-rebuild*"
]
},
"entry": "index.js"
},
{
"event": "onError",
"match": {
"labels": [
"mode"
],
"urls": [
"*://example.com/pause-rebuild*"
]
},
"entry": "index.js"
}
]
}
+46
View File
@@ -0,0 +1,46 @@
gopeed.events.onResolve(async function (ctx) {
const isRange = ctx.req.url.endsWith("/recover-range");
if (!isRange && !ctx.req.url.endsWith("/recover")) {
return;
}
const url = gopeed.runtime.blob.createObjectURL(async ({ offset = 0 }) => new ReadableStream({
async start(controller) {
if (offset > 0) {
controller.close();
return;
}
controller.enqueue(new TextEncoder().encode("stale-prefix-that-must-be-removed\n"));
await new Promise((resolve) => setTimeout(resolve, 50));
controller.error(new Error("expired"));
},
}), { size: 64, range: isRange });
ctx.res = {
name: "blob-recover",
files: [
{
name: "recover.txt",
size: 64,
req: {
url,
}
}
]
};
});
gopeed.events.onError(async function (ctx) {
const req = ctx.task.meta.req;
if (!req.rawUrl || (!req.rawUrl.endsWith("/recover") && !req.rawUrl.endsWith("/recover-range"))) {
return;
}
req.labels = req.labels || {};
if (req.labels.recovered === "true") {
return;
}
req.putLabel("recovered", "true");
ctx.task.setUrl(gopeed.runtime.blob.createObjectURL(new Blob(["ok\n"], { type: "text/plain" })));
ctx.task.continue();
});
@@ -0,0 +1,25 @@
{
"name": "blob-recover",
"title": "gopeed extension blob recover test",
"version": "0.0.1",
"scripts": [
{
"event": "onResolve",
"match": {
"urls": [
"*://example.com/*"
]
},
"entry": "index.js"
},
{
"event": "onError",
"match": {
"urls": [
"*://example.com/*"
]
},
"entry": "index.js"
}
]
}
+57
View File
@@ -0,0 +1,57 @@
function createRangeBlobUrl(target) {
return gopeed.runtime.blob.createObjectURL(async ({ offset = 0, end = -1 }) => {
const headers = {};
if (offset > 0 || end >= 0) {
headers.Range = `bytes=${offset}-${end >= 0 ? end : ''}`;
}
const response = await fetch(target, { headers });
if (!response.body) {
throw new Error('empty response body');
}
return response.body;
}, { size: 262144, range: true });
}
gopeed.events.onResolve(async function (ctx) {
if (!ctx.req.url.includes('/restart-range')) {
return;
}
ctx.res = {
name: 'blob-restart',
range: true,
files: [
{
name: 'restart.bin',
size: 262144,
req: {
url: createRangeBlobUrl(ctx.req.rawUrl || ctx.req.url),
rawUrl: ctx.req.rawUrl || ctx.req.url,
labels: {
mode: 'restart',
},
},
},
],
};
});
gopeed.events.onError(async function (ctx) {
const req = ctx.task?.meta?.req;
if (!req || !req.rawUrl || !req.url || !req.url.includes('/__blob/')) {
return;
}
if (req.labels.rebuilt === 'true') {
return;
}
try {
ctx.task.setUrl(createRangeBlobUrl(req.rawUrl));
req.putLabel('started', 'true');
req.putLabel('rebuilt', 'true');
ctx.task.continue();
} catch (error) {
req.putLabel('rebuildError', String(error));
throw error;
}
});
@@ -0,0 +1,28 @@
{
"name": "blob-restart",
"title": "gopeed extension blob restart resume test",
"version": "0.0.1",
"scripts": [
{
"event": "onResolve",
"match": {
"urls": [
"*://*/restart-range*"
]
},
"entry": "index.js"
},
{
"event": "onError",
"match": {
"labels": [
"mode"
],
"urls": [
"*://*/restart-range*"
]
},
"entry": "index.js"
}
]
}
@@ -0,0 +1,198 @@
const encoder = new TextEncoder();
const decoder = new TextDecoder();
function byteLength(value) {
return encoder.encode(value).byteLength;
}
function sourceURL(rawURL) {
const url = new URL(rawURL);
url.searchParams.set("source", "1");
return url.toString();
}
function createFetchDrainedStream(url) {
return new ReadableStream({
async start(controller) {
try {
const response = await fetch(url);
const reader = response.body.getReader();
while (true) {
const { done, value } = await reader.read();
if (done) {
controller.close();
return;
}
controller.enqueue(value);
}
} catch (error) {
controller.error(error);
}
},
});
}
function createFetchDrainedRangeStream(url, offset, end) {
return new ReadableStream({
async start(controller) {
try {
const headers = {};
if (offset > 0 || end >= 0) {
headers.Range = `bytes=${offset}-${end >= 0 ? end : ""}`;
}
const response = await fetch(url, { headers });
const reader = response.body.getReader();
while (true) {
const { done, value } = await reader.read();
if (done) {
controller.close();
return;
}
controller.enqueue(value);
}
} catch (error) {
controller.error(error);
}
},
});
}
function createMultiplexStreams(url) {
let videoController;
let audioController;
let started = false;
async function pump() {
if (started || !videoController || !audioController) {
return;
}
started = true;
try {
const response = await fetch(url);
const reader = response.body.getReader();
let pending = "";
while (true) {
const { done, value } = await reader.read();
if (done) {
break;
}
pending += decoder.decode(value, { stream: true });
let newlineIndex;
while ((newlineIndex = pending.indexOf("\n")) >= 0) {
const line = pending.slice(0, newlineIndex);
pending = pending.slice(newlineIndex + 1);
if (line.startsWith("v:")) {
videoController.enqueue(encoder.encode(line.slice(2)));
} else if (line.startsWith("a:")) {
audioController.enqueue(encoder.encode(line.slice(2)));
}
}
}
if (pending.startsWith("v:")) {
videoController.enqueue(encoder.encode(pending.slice(2)));
} else if (pending.startsWith("a:")) {
audioController.enqueue(encoder.encode(pending.slice(2)));
}
videoController.close();
audioController.close();
} catch (error) {
videoController.error(error);
audioController.error(error);
}
}
return {
video: new ReadableStream({
start(controller) {
videoController = controller;
pump();
},
}),
audio: new ReadableStream({
start(controller) {
audioController = controller;
pump();
},
}),
};
}
gopeed.events.onResolve(async function (ctx) {
if (ctx.req.url.includes("/single-fetch-stream")) {
const payloadSize = Number(new URL(ctx.req.url).searchParams.get("size")) || 0;
const url = gopeed.runtime.blob.createObjectURL(async () => createFetchDrainedStream(sourceURL(ctx.req.url)), {
size: payloadSize,
});
ctx.res = {
name: "single-fetch-stream",
files: [
{
name: "single.bin",
size: payloadSize,
req: { url },
},
],
};
return;
}
if (ctx.req.url.includes("/range-fetch-stream")) {
const payloadSize = Number(new URL(ctx.req.url).searchParams.get("size")) || 0;
const url = gopeed.runtime.blob.createObjectURL(
async ({ offset = 0, end = -1 }) => createFetchDrainedRangeStream(sourceURL(ctx.req.url), offset, end),
{
size: payloadSize,
range: true,
}
);
ctx.res = {
name: "range-fetch-stream",
range: true,
files: [
{
name: "range.bin",
size: payloadSize,
req: { url },
},
],
};
return;
}
if (ctx.req.url.includes("/multiplex-fetch-stream")) {
const videoChunks = ["video-0", "video-1", "video-2", "video-3"];
const audioChunks = ["audio-0", "audio-1", "audio-2", "audio-3"];
let streams;
const openStreams = () => {
if (!streams) {
streams = createMultiplexStreams(sourceURL(ctx.req.url));
}
return streams;
};
const videoUrl = gopeed.runtime.blob.createObjectURL(async () => openStreams().video, {
size: videoChunks.reduce((sum, chunk) => sum + byteLength(chunk), 0),
});
const audioUrl = gopeed.runtime.blob.createObjectURL(async () => openStreams().audio, {
size: audioChunks.reduce((sum, chunk) => sum + byteLength(chunk), 0),
});
ctx.res = {
name: "multiplex-fetch-stream",
files: [
{
name: "video.bin",
size: videoChunks.reduce((sum, chunk) => sum + byteLength(chunk), 0),
req: { url: videoUrl },
},
{
name: "audio.bin",
size: audioChunks.reduce((sum, chunk) => sum + byteLength(chunk), 0),
req: { url: audioUrl },
},
],
};
}
});
@@ -0,0 +1,18 @@
{
"name": "blob-stream-bridge",
"title": "gopeed extension blob stream bridge test",
"version": "0.0.1",
"scripts": [
{
"event": "onResolve",
"match": {
"urls": [
"*://*/single-fetch-stream*",
"*://*/range-fetch-stream*",
"*://*/multiplex-fetch-stream*"
]
},
"entry": "index.js"
}
]
}
+1 -2
View File
@@ -1,7 +1,6 @@
gopeed.events.onError(async function (ctx) {
gopeed.logger.info("url", ctx.task.meta.req.url);
gopeed.logger.info("error", ctx.error);
ctx.task.meta.req.url = "https://github.com";
ctx.task.setUrl("https://github.com");
ctx.task.continue();
});
+2 -3
View File
@@ -1,6 +1,5 @@
gopeed.events.onStart(async function (ctx) {
gopeed.logger.info("url", ctx.task.meta.req.url);
ctx.task.meta.req.url = "https://github.com";
ctx.task.meta.req.labels['modified'] = 'true';
ctx.task.setUrl("https://github.com");
ctx.task.meta.req.putLabel('modified', 'true');
});
+9
View File
@@ -0,0 +1,9 @@
@echo off
setlocal
if "%GOPEED_TEST_OUTPUT_FILE%"=="" exit /b 2
echo GOPEED_EVENT=%GOPEED_EVENT% > "%GOPEED_TEST_OUTPUT_FILE%"
echo GOPEED_TASK_ID=%GOPEED_TASK_ID% >> "%GOPEED_TEST_OUTPUT_FILE%"
echo GOPEED_TASK_NAME=%GOPEED_TASK_NAME% >> "%GOPEED_TEST_OUTPUT_FILE%"
echo GOPEED_TASK_STATUS=%GOPEED_TASK_STATUS% >> "%GOPEED_TEST_OUTPUT_FILE%"
echo GOPEED_TASK_PATH=%GOPEED_TASK_PATH% >> "%GOPEED_TEST_OUTPUT_FILE%"
+13
View File
@@ -0,0 +1,13 @@
#!/bin/bash
set -e
if [ -z "$GOPEED_TEST_OUTPUT_FILE" ]; then
echo "GOPEED_TEST_OUTPUT_FILE is empty" >&2
exit 2
fi
echo "GOPEED_EVENT=$GOPEED_EVENT" > "$GOPEED_TEST_OUTPUT_FILE"
echo "GOPEED_TASK_ID=$GOPEED_TASK_ID" >> "$GOPEED_TEST_OUTPUT_FILE"
echo "GOPEED_TASK_NAME=$GOPEED_TASK_NAME" >> "$GOPEED_TEST_OUTPUT_FILE"
echo "GOPEED_TASK_STATUS=$GOPEED_TASK_STATUS" >> "$GOPEED_TEST_OUTPUT_FILE"
echo "GOPEED_TASK_PATH=$GOPEED_TASK_PATH" >> "$GOPEED_TEST_OUTPUT_FILE"
+8
View File
@@ -0,0 +1,8 @@
@echo off
setlocal
if "%GOPEED_TEST_DEST_DIR%"=="" exit /b 2
set "SRC=%GOPEED_TASK_PATH:/=\%"
set "DEST=%GOPEED_TEST_DEST_DIR%"
if not exist "%DEST%" mkdir "%DEST%"
move /Y "%SRC%" "%DEST%\" >nul
+10
View File
@@ -0,0 +1,10 @@
#!/bin/bash
set -e
if [ -z "$GOPEED_TEST_DEST_DIR" ]; then
echo "GOPEED_TEST_DEST_DIR is empty" >&2
exit 2
fi
mkdir -p "$GOPEED_TEST_DEST_DIR"
mv "$GOPEED_TASK_PATH" "$GOPEED_TEST_DEST_DIR/"
+5
View File
@@ -0,0 +1,5 @@
@echo off
setlocal
if "%GOPEED_TEST_OUTPUT_FILE_1%"=="" exit /b 2
echo Script 1 > "%GOPEED_TEST_OUTPUT_FILE_1%"
+9
View File
@@ -0,0 +1,9 @@
#!/bin/bash
set -e
if [ -z "$GOPEED_TEST_OUTPUT_FILE_1" ]; then
echo "GOPEED_TEST_OUTPUT_FILE_1 is empty" >&2
exit 2
fi
echo "Script 1" > "$GOPEED_TEST_OUTPUT_FILE_1"
Loaded 100 of 173 files, more files were not shown because too many files have changed in this diff. Show more