Compare commits

...
113 changed files with 15982 additions and 2642 deletions

No files matched your search

+14 -12
View File
@@ -13,8 +13,8 @@ on:
default: "false"
env:
GO_VERSION: "1.24"
FLUTTER_VERSION: "3.41.2"
GO_VERSION: "1.25"
FLUTTER_VERSION: "3.41.6"
GA4_MEASUREMENT_ID: ${{ secrets.GA4_MEASUREMENT_ID }}
GA4_API_SECRET: ${{ secrets.GA4_API_SECRET }}
@@ -81,7 +81,7 @@ jobs:
"CLANGARM64_ROOT=$target" >> $env:GITHUB_ENV
$b >> $env:GITHUB_PATH
- uses: actions/setup-go@v4
- uses: actions/setup-go@v5
with:
go-version: ${{ env.GO_VERSION }}
- uses: subosito/flutter-action@v2
@@ -187,7 +187,7 @@ jobs:
mv "Output\gopeed.exe" "Output\Gopeed-$TAG-windows-${{ matrix.arch }}.exe"
Compress-Archive -Path "Output\Gopeed-$TAG-windows-${{ matrix.arch }}.exe" -DestinationPath "Output\Gopeed-$TAG-windows-${{ matrix.arch }}.zip"
- name: Upload
uses: shogo82148/actions-upload-release-asset@v1
uses: shogo82148/actions-upload-release-asset@v1.9.2
with:
upload_url: ${{ needs.get-release.outputs.upload_url }}
asset_path: ui/flutter/build/windows/Output/*
@@ -198,7 +198,7 @@ jobs:
needs: [get-release]
steps:
- uses: actions/checkout@v3
- uses: actions/setup-go@v4
- uses: actions/setup-go@v5
with:
go-version: ${{ env.GO_VERSION }}
- name: Build
@@ -224,7 +224,7 @@ jobs:
needs: [get-release, build-macos-amd64-lib]
steps:
- uses: actions/checkout@v3
- uses: actions/setup-go@v4
- uses: actions/setup-go@v5
with:
go-version: ${{ env.GO_VERSION }}
- uses: actions/setup-node@v3
@@ -315,7 +315,7 @@ jobs:
needs: [get-release]
steps:
- uses: actions/checkout@v3
- uses: actions/setup-go@v4
- uses: actions/setup-go@v5
with:
go-version: ${{ env.GO_VERSION }}
- uses: subosito/flutter-action@v2
@@ -324,7 +324,7 @@ jobs:
channel: master
- run: |
sudo apt update -y
sudo apt install -y ninja-build libgtk-3-dev libayatana-appindicator3-1 libayatana-appindicator3-dev rpm patchelf libfuse2 locate libkeybinder-3.0-dev
sudo apt install -y ninja-build libgtk-3-dev libwebkit2gtk-4.1-dev libayatana-appindicator3-1 libayatana-appindicator3-dev rpm patchelf libfuse2 locate libkeybinder-3.0-dev
arch=$(uname -m)
wget -O appimagetool "https://github.com/AppImage/appimagetool/releases/download/continuous/appimagetool-${arch}.AppImage"
chmod +x appimagetool
@@ -411,7 +411,7 @@ jobs:
needs: [get-release]
steps:
- uses: actions/checkout@v3
- uses: actions/setup-go@v4
- uses: actions/setup-go@v5
with:
go-version: ${{ env.GO_VERSION }}
- name: Setup LXD
@@ -480,6 +480,7 @@ jobs:
- cmake
- curl
- libgtk-3-dev
- libwebkit2gtk-4.1-dev
- libkeybinder-3.0-dev
- ninja-build
- unzip
@@ -495,6 +496,7 @@ jobs:
source: .
plugin: nil
stage-packages:
- libwebkit2gtk-4.1-0
- libgtk-3-dev
- libkeybinder-3.0-dev
- libappindicator3-dev
@@ -545,7 +547,7 @@ jobs:
needs: [get-release]
steps:
- uses: actions/checkout@v3
- uses: actions/setup-go@v4
- uses: actions/setup-go@v5
with:
go-version: ${{ env.GO_VERSION }}
- uses: actions/setup-java@v3
@@ -587,7 +589,7 @@ jobs:
needs: [get-release]
steps:
- uses: actions/checkout@v3
- uses: actions/setup-go@v4
- uses: actions/setup-go@v5
with:
go-version: ${{ env.GO_VERSION }}
- uses: subosito/flutter-action@v2
@@ -620,7 +622,7 @@ jobs:
needs: [get-release]
steps:
- uses: actions/checkout@v3
- uses: actions/setup-go@v4
- uses: actions/setup-go@v5
with:
go-version: ${{ env.GO_VERSION }}
- uses: subosito/flutter-action@v2
+2 -2
View File
@@ -17,7 +17,7 @@ on:
- "Dockerfile"
env:
GO_VERSION: "1.24"
GO_VERSION: "1.25"
permissions:
contents: write
@@ -31,7 +31,7 @@ jobs:
steps:
- uses: actions/checkout@v3
- uses: actions/setup-go@v4
- uses: actions/setup-go@v5
with:
go-version: ${{ env.GO_VERSION }}
- uses: release-drafter/release-drafter@v5
+71 -15
View File
@@ -28,8 +28,8 @@ on:
workflow_dispatch:
env:
GO_VERSION: "1.24"
FLUTTER_VERSION: "3.41.2"
GO_VERSION: "1.25"
FLUTTER_VERSION: "3.41.6"
jobs:
# lint:
@@ -50,14 +50,18 @@ jobs:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/setup-go@v4
- uses: actions/setup-go@v5
with:
go-version: ${{ env.GO_VERSION }}
- name: Install webview dependencies
run: |
sudo apt-get update -y
sudo apt-get install -y libgtk-3-dev libwebkit2gtk-4.1-dev xvfb
- name: Run Unit tests.
run: |
# Get packages for testing and coverage (including cmd/web for flags_test.go)
PACKAGES=$(go list ./... | grep -v /bind/ | grep -v -E '/cmd/(?!web)')
go test -v -coverpkg=$(echo "$PACKAGES" | paste -sd "," -) -covermode=count -coverprofile=coverage.txt $PACKAGES
PACKAGES=$(go list ./... | awk '!/\/bind\// && (!/\/cmd\// || /\/cmd\/web$/)')
env CGO_ENABLED=1 xvfb-run -a go test -tags webview -v -coverpkg=$(echo "$PACKAGES" | paste -sd "," -) -covermode=count -coverprofile=coverage.txt $PACKAGES
# Filter out main.go from coverage report to avoid lowering coverage percentage
grep -v "main.go" coverage.txt > coverage_filtered.txt || true
mv coverage_filtered.txt coverage.txt
@@ -65,6 +69,58 @@ jobs:
with:
files: ./coverage.txt
token: ${{ secrets.CODECOV_UPLOAD_TOKEN }}
test-webview-integration:
name: Test webview integration (${{ matrix.name }})
strategy:
fail-fast: false
matrix:
include:
- name: Windows
os: windows-latest
- name: macOS
os: macos-latest
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v3
- uses: actions/setup-go@v5
with:
go-version: ${{ env.GO_VERSION }}
- uses: subosito/flutter-action@v2
if: runner.os == 'macOS'
with:
flutter-version: ${{ env.FLUTTER_VERSION }}
- name: Test webview
if: runner.os == 'Windows'
run: go test -tags webview -v -timeout 300s ./internal/webview/goprovider/
env:
CGO_ENABLED: 1
- name: Test webview
if: runner.os == 'macOS'
shell: bash
run: |
set -euo pipefail
SOCKET_PATH="$HOME/Library/Application Support/com.gopeed.gopeed/gopeed_webview.sock"
LOG_PATH="$RUNNER_TEMP/gopeed-webview-rpc.log"
rm -f "$SOCKET_PATH"
go build -tags nosqlite -ldflags="-w -s" -buildmode=c-shared -o ui/flutter/macos/Frameworks/libgopeed.dylib github.com/GopeedLab/gopeed/bind/desktop
cd ui/flutter
flutter pub get
flutter run -d macos >"$LOG_PATH" 2>&1 &
FLUTTER_PID=$!
cd ../..
cleanup() {
kill "$FLUTTER_PID" >/dev/null 2>&1 || true
}
trap cleanup EXIT
if ! go test -tags webview -v -timeout 600s ./internal/webview/rpcprovider/ -args -webview-rpc-network unix -webview-rpc-address "$SOCKET_PATH"; then
cat "$LOG_PATH"
exit 1
fi
build-desktop:
strategy:
matrix:
@@ -80,7 +136,7 @@ jobs:
flutter_channel: "main"
name: Build desktop check (${{ matrix.os }})
runs-on: ${{ matrix.os }}
needs: [test]
needs: [ test ]
steps:
- uses: actions/checkout@v3
- name: Enable long paths for flutter main branch checks
@@ -109,7 +165,7 @@ jobs:
"CGO_ENABLED=1" >> $env:GITHUB_ENV
"CLANGARM64_ROOT=$target" >> $env:GITHUB_ENV
$b >> $env:GITHUB_PATH
- uses: actions/setup-go@v4
- uses: actions/setup-go@v5
with:
go-version: ${{ env.GO_VERSION }}
- uses: subosito/flutter-action@v2
@@ -129,7 +185,7 @@ jobs:
- if: runner.os == 'Linux'
run: |
sudo apt-get update -y
sudo apt-get install -y ninja-build libgtk-3-dev libayatana-appindicator3-dev libkeybinder-3.0-dev
sudo apt-get install -y ninja-build libgtk-3-dev libwebkit2gtk-4.1-dev libayatana-appindicator3-dev libkeybinder-3.0-dev
go build -tags nosqlite -ldflags="-w -s" -buildmode=c-shared -o ui/flutter/linux/bundle/lib/libgopeed.so github.com/GopeedLab/gopeed/bind/desktop
cd ui/flutter
flutter build linux
@@ -138,11 +194,11 @@ jobs:
runs-on: ${{ matrix.os }}
strategy:
matrix:
os: [macos-14, ubuntu-latest]
needs: [test]
os: [ macos-14, ubuntu-latest ]
needs: [ test ]
steps:
- uses: actions/checkout@v3
- uses: actions/setup-go@v4
- uses: actions/setup-go@v5
with:
go-version: ${{ env.GO_VERSION }}
- uses: subosito/flutter-action@v2
@@ -171,10 +227,10 @@ jobs:
build-web:
name: Build web check
runs-on: ubuntu-latest
needs: [test]
needs: [ test ]
steps:
- uses: actions/checkout@v3
- uses: actions/setup-go@v4
- uses: actions/setup-go@v5
with:
go-version: ${{ env.GO_VERSION }}
- uses: subosito/flutter-action@v2
@@ -192,10 +248,10 @@ jobs:
build-docker:
name: Build docker check
runs-on: ubuntu-latest
needs: [test]
needs: [ test ]
steps:
- uses: actions/checkout@v3
- uses: actions/setup-go@v4
- uses: actions/setup-go@v5
with:
go-version: ${{ env.GO_VERSION }}
- uses: subosito/flutter-action@v2
+1 -1
View File
@@ -1,4 +1,4 @@
FROM golang:1.24.11-alpine3.23 AS go
FROM golang:1.25.4-alpine3.22 AS go
WORKDIR /app
COPY ./go.mod ./go.sum ./
RUN go mod download
+1 -1
View File
@@ -201,7 +201,7 @@ This project is divided into two parts, the front end uses `flutter`, the back e
### 🌍 Environment
1. Golang 1.24+
1. Golang 1.25+
2. Flutter 3.38+
### 📋 Clone
+1 -1
View File
@@ -182,7 +182,7 @@ Gopeed (正式名 Go Speed) は `Golang` + `Flutter` によって開発された
### 🌍 環境
1. Go 言語 1.24+
1. Go 言語 1.25+
2. Flutter 3.38+
### 📋 クローン
+1 -1
View File
@@ -188,7 +188,7 @@ Dự án này được chia thành hai phần, phần giao diện sử dụng `f
### 🌍 Environment
1. Golang 1.24+
1. Golang 1.25+
2. Flutter 3.38+
### 📋 Clone
+1 -1
View File
@@ -192,7 +192,7 @@ Gopeed 还提供了浏览器扩展用于接管浏览器下载,支持 Chrome、
### 🌍 环境要求
1. Golang 1.24+
1. Golang 1.25+
2. Flutter 3.38+
### 📋 克隆项目
+1 -1
View File
@@ -189,7 +189,7 @@ go install github.com/GopeedLab/gopeed/cmd/gopeed@latest
### 🌍 開發環境
1. Golang 1.24+
1. Golang 1.25+
2. Flutter 3.38+
### 📋 克隆項目
+1
View File
@@ -16,6 +16,7 @@ func Start(cfg *C.char) (int, *C.char) {
return 0, C.CString(err.Error())
}
config.ProductionMode = true
applyWebViewProvider(&config)
realPort, err := rest.Start(&config)
if err != nil {
return 0, C.CString(err.Error())
+23
View File
@@ -0,0 +1,23 @@
//go:build darwin
package main
import (
"github.com/GopeedLab/gopeed/internal/webview/rpcprovider"
enginewebview "github.com/GopeedLab/gopeed/pkg/download/engine/webview"
"github.com/GopeedLab/gopeed/pkg/rest/model"
)
func applyWebViewProvider(config *model.StartConfig) {
if config == nil {
return
}
config.WebViewProvider = newDesktopWebViewProvider(config.WebViewRPCConfig)
}
func newDesktopWebViewProvider(rpcCfg *enginewebview.RPCConfig) enginewebview.Provider {
if rpcCfg != nil && rpcCfg.Enabled() {
return rpcprovider.New(*rpcCfg)
}
return enginewebview.NewUnavailableProvider()
}
+15
View File
@@ -0,0 +1,15 @@
//go:build !darwin
package main
import (
"github.com/GopeedLab/gopeed/internal/webview/goprovider"
"github.com/GopeedLab/gopeed/pkg/rest/model"
)
func applyWebViewProvider(config *model.StartConfig) {
if config == nil {
return
}
config.WebViewProvider = goprovider.New()
}
+9
View File
@@ -4,6 +4,7 @@ package libgopeed
import "C"
import (
"encoding/json"
"github.com/GopeedLab/gopeed/internal/webview/rpcprovider"
"github.com/GopeedLab/gopeed/pkg/rest"
"github.com/GopeedLab/gopeed/pkg/rest/model"
)
@@ -14,9 +15,17 @@ func Start(cfg string) (int, error) {
return 0, err
}
config.ProductionMode = true
applyWebViewProvider(&config)
return rest.Start(&config)
}
func Stop() {
rest.Stop()
}
func applyWebViewProvider(config *model.StartConfig) {
if config == nil || config.WebViewRPCConfig == nil || !config.WebViewRPCConfig.Enabled() {
return
}
config.WebViewProvider = rpcprovider.New(*config.WebViewRPCConfig)
}
+723
View File
@@ -0,0 +1,723 @@
package main
import (
"encoding/json"
"flag"
"fmt"
"os"
"path/filepath"
"time"
"github.com/GopeedLab/gopeed/internal/webview/goprovider"
"github.com/GopeedLab/gopeed/internal/webview/rpcprovider"
"github.com/GopeedLab/gopeed/pkg/base"
"github.com/GopeedLab/gopeed/pkg/download"
enginewebview "github.com/GopeedLab/gopeed/pkg/download/engine/webview"
)
type kvFlag map[string]string
type webViewConfig struct {
provider string
network string
address string
token string
}
func (l *kvFlag) String() string {
if l == nil {
return ""
}
data, _ := json.Marshal(map[string]string(*l))
return string(data)
}
func (l *kvFlag) Set(value string) error {
if *l == nil {
*l = make(map[string]string)
}
idx := -1
for i := 0; i < len(value); i++ {
if value[i] == '=' {
idx = i
break
}
}
if idx <= 0 {
return fmt.Errorf("invalid label %q, expected key=value", value)
}
(*l)[value[:idx]] = value[idx+1:]
return nil
}
func main() {
os.Exit(goprovider.RunMainThreadLoop(run))
}
func run() int {
if len(os.Args) < 2 {
usage()
return 2
}
switch os.Args[1] {
case "resolve":
if err := runResolve(os.Args[2:]); err != nil {
fmt.Fprintln(os.Stderr, err)
return 1
}
case "download":
if err := runDownload(os.Args[2:]); err != nil {
fmt.Fprintln(os.Stderr, err)
return 1
}
case "eval":
if err := runEval(os.Args[2:]); err != nil {
fmt.Fprintln(os.Stderr, err)
return 1
}
case "-h", "--help", "help":
usage()
default:
fmt.Fprintf(os.Stderr, "unknown subcommand %q\n", os.Args[1])
usage()
return 2
}
return 0
}
func usage() {
fmt.Fprintf(os.Stderr, `gopeed extmock CLI
Usage:
extmock resolve [flags] <extension-dir> <url>
extmock download [flags] <extension-dir> <url> <output-dir>
extmock eval [flags] [script-file]
Flags:
-label key=value Request label, repeatable
-setting key=value Extension setting override, repeatable
-webview-provider WebView provider: go|rpc
-webview-network RPC network when provider=rpc: unix|tcp
-webview-address RPC address when provider=rpc
-webview-token RPC bearer token when provider=rpc
-verb Print download progress while running
-pretty Pretty-print JSON output
Examples:
extmock resolve ./ https://www.google.com
extmock download ./ https://example.com/video /tmp/extmock-download
extmock resolve ./ https://example.com/video -label env=test -setting token=abc
extmock eval ./test.js
extmock eval -e "new Blob(['hello']).size"
`)
}
func runResolve(args []string) error {
fs := flag.NewFlagSet("resolve", flag.ContinueOnError)
fs.SetOutput(os.Stderr)
var labels kvFlag
var settings kvFlag
var pretty bool
cfg := webViewConfig{}
fs.Var(&labels, "label", "Request label in key=value form (repeatable)")
fs.Var(&settings, "setting", "Extension setting override in key=value form (repeatable)")
fs.StringVar(&cfg.provider, "webview-provider", "", "WebView provider: go|rpc")
fs.StringVar(&cfg.network, "webview-network", "", "WebView RPC network: unix|tcp")
fs.StringVar(&cfg.address, "webview-address", "", "WebView RPC address")
fs.StringVar(&cfg.token, "webview-token", "", "WebView RPC bearer token")
fs.BoolVar(&pretty, "pretty", false, "Pretty-print JSON output")
flagArgs, positionalArgs, err := splitResolveArgs(args)
if err != nil {
return err
}
if err := fs.Parse(flagArgs); err != nil {
return err
}
if fs.NArg() != 0 {
return fmt.Errorf("unexpected trailing args: %v", fs.Args())
}
if len(positionalArgs) != 2 {
return fmt.Errorf("resolve expects exactly 2 args: <extension-dir> <url>")
}
extDir, err := filepath.Abs(positionalArgs[0])
if err != nil {
return err
}
url := positionalArgs[1]
d, cleanup, err := newExtMockDownloader(cfg)
if err != nil {
return err
}
defer cleanup()
ext, err := d.InstallExtensionByFolder(extDir, true)
if err != nil {
return err
}
if len(settings) > 0 {
if err := d.UpdateExtensionSettings(ext.Identity, toAnyMap(settings)); err != nil {
return err
}
}
result, err := d.Resolve(&base.Request{
URL: url,
Labels: map[string]string(labels),
}, nil)
if err != nil {
return err
}
return printJSON(result, pretty)
}
type downloadResult struct {
Resolve *download.ResolveResult `json:"resolve"`
OutputDir string `json:"outputDir"`
Files []*downloadedFileResult `json:"files"`
}
type downloadedFileResult struct {
Index int `json:"index"`
Name string `json:"name"`
Path string `json:"path"`
TaskID string `json:"taskId"`
Status base.Status `json:"status"`
Size int64 `json:"size"`
Req *base.Request `json:"req,omitempty"`
}
func runDownload(args []string) error {
fs := flag.NewFlagSet("download", flag.ContinueOnError)
fs.SetOutput(os.Stderr)
var labels kvFlag
var settings kvFlag
var pretty bool
var verbose bool
cfg := webViewConfig{}
fs.Var(&labels, "label", "Request label in key=value form (repeatable)")
fs.Var(&settings, "setting", "Extension setting override in key=value form (repeatable)")
fs.StringVar(&cfg.provider, "webview-provider", "", "WebView provider: go|rpc")
fs.StringVar(&cfg.network, "webview-network", "", "WebView RPC network: unix|tcp")
fs.StringVar(&cfg.address, "webview-address", "", "WebView RPC address")
fs.StringVar(&cfg.token, "webview-token", "", "WebView RPC bearer token")
fs.BoolVar(&verbose, "verb", false, "Print download progress while running")
fs.BoolVar(&verbose, "verbose", false, "Print download progress while running")
fs.BoolVar(&pretty, "pretty", false, "Pretty-print JSON output")
flagArgs, positionalArgs, err := splitDownloadArgs(args)
if err != nil {
return err
}
if err := fs.Parse(flagArgs); err != nil {
return err
}
if fs.NArg() != 0 {
return fmt.Errorf("unexpected trailing args: %v", fs.Args())
}
if len(positionalArgs) != 3 {
return fmt.Errorf("download expects exactly 3 args: <extension-dir> <url> <output-dir>")
}
extDir, err := filepath.Abs(positionalArgs[0])
if err != nil {
return err
}
url := positionalArgs[1]
outputDir, err := filepath.Abs(positionalArgs[2])
if err != nil {
return err
}
d, cleanup, err := newExtMockDownloader(cfg)
if err != nil {
return err
}
defer cleanup()
ext, err := d.InstallExtensionByFolder(extDir, true)
if err != nil {
return err
}
if len(settings) > 0 {
if err := d.UpdateExtensionSettings(ext.Identity, toAnyMap(settings)); err != nil {
return err
}
}
result, err := d.Resolve(&base.Request{
URL: url,
Labels: map[string]string(labels),
}, nil)
if err != nil {
return err
}
if result == nil || result.Res == nil {
return fmt.Errorf("resolve returned empty resource")
}
if len(result.Res.Files) == 0 {
return fmt.Errorf("resolve returned no files")
}
resourceOutputDir := outputDir
if result.Res.Name != "" {
resourceOutputDir = filepath.Join(outputDir, result.Res.Name)
}
if err := os.MkdirAll(resourceOutputDir, 0o755); err != nil {
return err
}
waiter := newTaskTerminalWaiter(d)
summary := &downloadResult{
Resolve: result,
OutputDir: resourceOutputDir,
Files: make([]*downloadedFileResult, 0, len(result.Res.Files)),
}
for idx, file := range result.Res.Files {
if file == nil {
return fmt.Errorf("file %d is nil", idx)
}
if file.Req == nil {
return fmt.Errorf("file %d request is nil", idx)
}
targetDir := resourceOutputDir
if file.Path != "" {
targetDir = filepath.Join(resourceOutputDir, file.Path)
}
if err := os.MkdirAll(targetDir, 0o755); err != nil {
return err
}
taskID, err := d.CreateDirect(file.Req, &base.Options{
Path: targetDir,
Name: file.Name,
})
if err != nil {
return fmt.Errorf("create task for %q failed: %w", file.Name, err)
}
waiter.Register(taskID)
summary.Files = append(summary.Files, &downloadedFileResult{
Index: idx,
Name: file.Name,
Path: filepath.Join(targetDir, file.Name),
TaskID: taskID,
Req: file.Req,
})
}
var stopVerbose func()
if verbose {
stopVerbose = startVerboseTaskListener(d, summary.Files)
defer stopVerbose()
}
for _, file := range summary.Files {
task, err := waiter.Wait(file.TaskID, 5*time.Minute)
if err != nil {
return fmt.Errorf("download %q failed: %w", file.Name, err)
}
file.Status = task.Status
if info, statErr := os.Stat(file.Path); statErr == nil {
file.Size = info.Size()
}
}
return printJSON(summary, pretty)
}
func runEval(args []string) error {
fs := flag.NewFlagSet("eval", flag.ContinueOnError)
fs.SetOutput(os.Stderr)
var (
script string
pretty bool
)
cfg := webViewConfig{}
fs.StringVar(&script, "e", "", "Inline JavaScript to execute")
fs.StringVar(&script, "eval", "", "Inline JavaScript to execute")
fs.StringVar(&cfg.provider, "webview-provider", "", "WebView provider: go|rpc")
fs.StringVar(&cfg.network, "webview-network", "", "WebView RPC network: unix|tcp")
fs.StringVar(&cfg.address, "webview-address", "", "WebView RPC address")
fs.StringVar(&cfg.token, "webview-token", "", "WebView RPC bearer token")
fs.BoolVar(&pretty, "pretty", false, "Pretty-print JSON output")
flagArgs, positionalArgs, err := splitEvalArgs(args)
if err != nil {
return err
}
if err := fs.Parse(flagArgs); err != nil {
return err
}
if fs.NArg() != 0 {
return fmt.Errorf("unexpected trailing args: %v", fs.Args())
}
if script == "" && len(positionalArgs) == 0 {
return fmt.Errorf("eval expects inline code via -e/--eval or a script file path")
}
if script != "" && len(positionalArgs) > 0 {
return fmt.Errorf("eval accepts either inline code or a script file path, not both")
}
if len(positionalArgs) > 1 {
return fmt.Errorf("eval expects at most 1 script file path")
}
d, cleanup, err := newExtMockDownloader(cfg)
if err != nil {
return err
}
defer cleanup()
runtime, err := d.NewExtensionEngine(newExtMockExtension(), map[string]any{})
if err != nil {
return err
}
defer runtime.Close()
var value any
if script != "" {
value, err = runtime.Eval(script)
} else {
scriptPath, pathErr := filepath.Abs(positionalArgs[0])
if pathErr != nil {
return pathErr
}
value, err = runtime.EvalFile(scriptPath)
}
if err != nil {
return err
}
return printJSON(value, pretty)
}
func splitResolveArgs(args []string) ([]string, []string, error) {
var flagArgs []string
var positionalArgs []string
for i := 0; i < len(args); i++ {
arg := args[i]
switch {
case arg == "-pretty":
flagArgs = append(flagArgs, arg)
case arg == "-webview-provider", arg == "-webview-network", arg == "-webview-address", arg == "-webview-token":
if i+1 >= len(args) {
return nil, nil, fmt.Errorf("flag %q requires a value", arg)
}
flagArgs = append(flagArgs, arg, args[i+1])
i++
case arg == "-label":
if i+1 >= len(args) {
return nil, nil, fmt.Errorf("flag %q requires a value", arg)
}
flagArgs = append(flagArgs, arg, args[i+1])
i++
case arg == "-setting":
if i+1 >= len(args) {
return nil, nil, fmt.Errorf("flag %q requires a value", arg)
}
flagArgs = append(flagArgs, arg, args[i+1])
i++
case len(arg) > len("-label=") && arg[:len("-label=")] == "-label=":
flagArgs = append(flagArgs, arg)
case len(arg) > len("-setting=") && arg[:len("-setting=")] == "-setting=":
flagArgs = append(flagArgs, arg)
case hasFlagValue(arg, "-webview-provider"), hasFlagValue(arg, "-webview-network"),
hasFlagValue(arg, "-webview-address"), hasFlagValue(arg, "-webview-token"):
flagArgs = append(flagArgs, arg)
case len(arg) > 0 && arg[0] == '-':
return nil, nil, fmt.Errorf("unknown flag %q", arg)
default:
positionalArgs = append(positionalArgs, arg)
}
}
return flagArgs, positionalArgs, nil
}
func splitDownloadArgs(args []string) ([]string, []string, error) {
var flagArgs []string
var positionalArgs []string
for i := 0; i < len(args); i++ {
arg := args[i]
switch {
case arg == "-pretty", arg == "-verb", arg == "-verbose":
flagArgs = append(flagArgs, arg)
case arg == "-webview-provider", arg == "-webview-network", arg == "-webview-address", arg == "-webview-token":
if i+1 >= len(args) {
return nil, nil, fmt.Errorf("flag %q requires a value", arg)
}
flagArgs = append(flagArgs, arg, args[i+1])
i++
case arg == "-label":
if i+1 >= len(args) {
return nil, nil, fmt.Errorf("flag %q requires a value", arg)
}
flagArgs = append(flagArgs, arg, args[i+1])
i++
case arg == "-setting":
if i+1 >= len(args) {
return nil, nil, fmt.Errorf("flag %q requires a value", arg)
}
flagArgs = append(flagArgs, arg, args[i+1])
i++
case len(arg) > len("-label=") && arg[:len("-label=")] == "-label=":
flagArgs = append(flagArgs, arg)
case len(arg) > len("-setting=") && arg[:len("-setting=")] == "-setting=":
flagArgs = append(flagArgs, arg)
case hasFlagValue(arg, "-webview-provider"), hasFlagValue(arg, "-webview-network"),
hasFlagValue(arg, "-webview-address"), hasFlagValue(arg, "-webview-token"):
flagArgs = append(flagArgs, arg)
case len(arg) > 0 && arg[0] == '-':
return nil, nil, fmt.Errorf("unknown flag %q", arg)
default:
positionalArgs = append(positionalArgs, arg)
}
}
return flagArgs, positionalArgs, nil
}
func splitEvalArgs(args []string) ([]string, []string, error) {
var flagArgs []string
var positionalArgs []string
for i := 0; i < len(args); i++ {
arg := args[i]
switch {
case arg == "-pretty":
flagArgs = append(flagArgs, arg)
case arg == "-webview-provider", arg == "-webview-network", arg == "-webview-address", arg == "-webview-token":
if i+1 >= len(args) {
return nil, nil, fmt.Errorf("flag %q requires a value", arg)
}
flagArgs = append(flagArgs, arg, args[i+1])
i++
case arg == "-e", arg == "--eval":
if i+1 >= len(args) {
return nil, nil, fmt.Errorf("flag %q requires a value", arg)
}
flagArgs = append(flagArgs, arg, args[i+1])
i++
case hasFlagValue(arg, "-e"), hasFlagValue(arg, "--eval"),
hasFlagValue(arg, "-webview-provider"), hasFlagValue(arg, "-webview-network"),
hasFlagValue(arg, "-webview-address"), hasFlagValue(arg, "-webview-token"):
flagArgs = append(flagArgs, arg)
case len(arg) > 0 && arg[0] == '-':
return nil, nil, fmt.Errorf("unknown flag %q", arg)
default:
positionalArgs = append(positionalArgs, arg)
}
}
return flagArgs, positionalArgs, nil
}
func hasFlagValue(arg string, prefix string) bool {
return len(arg) > len(prefix)+1 && arg[:len(prefix)+1] == prefix+"="
}
func toAnyMap(values kvFlag) map[string]any {
result := make(map[string]any, len(values))
for key, value := range values {
result[key] = value
}
return result
}
func newExtMockDownloader(cfg webViewConfig) (*download.Downloader, func(), error) {
webViewProvider, err := newExtMockWebViewProvider(cfg)
if err != nil {
return nil, nil, err
}
d := download.NewDownloader(&download.DownloaderConfig{
Storage: download.NewMemStorage(),
WebViewProvider: webViewProvider,
})
if err := d.Setup(); err != nil {
return nil, nil, err
}
cleanup := func() {
_ = d.Clear()
}
return d, cleanup, nil
}
func newExtMockExtension() *download.Extension {
return &download.Extension{
Name: "extmock",
Author: "gopeed",
Title: "Gopeed ExtMock Runtime",
Version: "0.0.0",
DevMode: true,
}
}
func newExtMockWebViewProvider(cfg webViewConfig) (enginewebview.Provider, error) {
switch cfg.provider {
case "", "go":
return goprovider.New(), nil
case "rpc":
rpcCfg, err := resolveRPCConfig(cfg)
if err != nil {
return nil, err
}
return rpcprovider.New(rpcCfg), nil
default:
return nil, fmt.Errorf("unsupported webview provider %q", cfg.provider)
}
}
func resolveRPCConfig(cfg webViewConfig) (enginewebview.RPCConfig, error) {
network := cfg.network
address := cfg.address
if network == "" && address == "" {
defaultAddress, err := defaultWebViewRPCUnixSocket()
if err != nil {
return enginewebview.RPCConfig{}, err
}
network = "unix"
address = defaultAddress
}
if network == "" || address == "" {
return enginewebview.RPCConfig{}, fmt.Errorf("webview rpc requires both network and address")
}
return enginewebview.RPCConfig{
Network: network,
Address: address,
Token: cfg.token,
}, nil
}
func defaultWebViewRPCUnixSocket() (string, error) {
homeDir, err := os.UserHomeDir()
if err != nil {
return "", err
}
return filepath.Join(homeDir, "gopeed_webview.sock"), nil
}
type taskTerminalWaiter struct {
downloader *download.Downloader
}
func newTaskTerminalWaiter(d *download.Downloader) *taskTerminalWaiter {
return &taskTerminalWaiter{downloader: d}
}
func (w *taskTerminalWaiter) Register(id string) {
}
func (w *taskTerminalWaiter) Wait(id string, timeout time.Duration) (*download.Task, error) {
deadline := time.Now().Add(timeout)
for time.Now().Before(deadline) {
if task, ok := w.currentTerminalTask(id); ok {
return task, w.taskError(task)
}
time.Sleep(50 * time.Millisecond)
}
return nil, fmt.Errorf("timeout waiting for task %s", id)
}
func (w *taskTerminalWaiter) currentTerminalTask(id string) (*download.Task, bool) {
task := w.downloader.GetTask(id)
if task == nil {
return nil, false
}
if task.Status != base.DownloadStatusDone && task.Status != base.DownloadStatusError {
return nil, false
}
return task, true
}
func (w *taskTerminalWaiter) taskError(task *download.Task) error {
if task == nil {
return fmt.Errorf("task is nil")
}
if task.Status == base.DownloadStatusError {
return fmt.Errorf("task %s finished with status %s", task.ID, task.Status)
}
return nil
}
func startVerboseTaskListener(d *download.Downloader, files []*downloadedFileResult) func() {
tracked := make(map[string]string, len(files))
for _, file := range files {
tracked[file.TaskID] = file.Name
}
d.Listener(func(event *download.Event) {
if event == nil || event.Task == nil {
return
}
name, ok := tracked[event.Task.ID]
if !ok {
return
}
switch event.Key {
case download.EventKeyStart, download.EventKeyProgress, download.EventKeyDone, download.EventKeyError:
printVerboseTaskEvent(event, name)
}
})
return func() {}
}
func printVerboseTaskEvent(event *download.Event, name string) {
task := event.Task
ts := time.Now().Format("15:04:05.000")
if task == nil {
return
}
total := int64(0)
if task.Meta != nil && task.Meta.Res != nil {
total = task.Meta.Res.Size
}
downloaded := int64(0)
speed := int64(0)
if task.Progress != nil {
downloaded = task.Progress.Downloaded
speed = task.Progress.Speed
}
if event.Key == download.EventKeyError && event.Err != nil {
fmt.Fprintf(
os.Stderr,
"[%s] event=%s task=%s status=%s downloaded=%d total=%d speed=%d name=%q err=%v\n",
ts,
event.Key,
task.ID,
task.Status,
downloaded,
total,
speed,
name,
event.Err,
)
return
}
fmt.Fprintf(
os.Stderr,
"[%s] event=%s task=%s status=%s downloaded=%d total=%d speed=%d name=%q\n",
ts,
event.Key,
task.ID,
task.Status,
downloaded,
total,
speed,
name,
)
}
func printJSON(v any, pretty bool) error {
var (
data []byte
err error
)
if pretty {
data, err = json.MarshalIndent(v, "", " ")
} else {
data, err = json.Marshal(v)
}
if err != nil {
return err
}
fmt.Println(string(data))
return nil
}
+3 -4
View File
@@ -1,10 +1,9 @@
module github.com/GopeedLab/gopeed
go 1.24.9
toolchain go1.24.11
go 1.25.4
require (
github.com/GopeedLab/webview_go v0.0.0-20260423085439-7a2f88b6e9b5
github.com/anacrolix/torrent v1.60.1-0.20251217073903-486bcbe758e0
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5
github.com/bodgit/sevenzip v1.6.1
@@ -17,6 +16,7 @@ require (
github.com/matoous/go-nanoid/v2 v2.0.0
github.com/mattn/go-ieproxy v0.0.12
github.com/mholt/archives v0.1.5
github.com/monkeyWie/goed2k v0.0.0-20260317100435-7a7575cf2447
github.com/pkg/errors v0.9.1
github.com/rs/dnscache v0.0.0-20230804202142-fc85eb664529
github.com/rs/zerolog v1.31.0
@@ -45,7 +45,6 @@ require (
github.com/klauspost/pgzip v1.2.6 // indirect
github.com/mikelolasagasti/xz v1.0.1 // indirect
github.com/minio/minlz v1.0.1 // indirect
github.com/monkeyWie/goed2k v0.0.0-20260317100435-7a7575cf2447 // indirect
github.com/nwaples/rardecode/v2 v2.2.0 // indirect
github.com/onsi/ginkgo/v2 v2.23.4 // indirect
github.com/pierrec/lz4/v4 v4.1.22 // indirect
+2 -2
View File
@@ -23,6 +23,8 @@ filippo.io/edwards25519 v1.0.0-rc.1 h1:m0VOOB23frXZvAOK44usCgLWvtsxIoMCTBGJZlpmG
filippo.io/edwards25519 v1.0.0-rc.1/go.mod h1:N1IkdkCkiLB6tki+MYJoSx2JTY9NUlxZE7eHn5EwJns=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
github.com/GopeedLab/webview_go v0.0.0-20260423085439-7a2f88b6e9b5 h1:86LVTaNDM/j/uTAD2HedkXNO9LIQlt642VAdbU/0D/M=
github.com/GopeedLab/webview_go v0.0.0-20260423085439-7a2f88b6e9b5/go.mod h1:FvV19lu9rQ9NcbczN5y8pbWecURRRbWa8tbfFARBDa0=
github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0rYXWg0=
github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ=
github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY=
@@ -386,8 +388,6 @@ github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJ
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/monkeyWie/goed2k v0.0.0-20260317094144-6e18d43056e5 h1:tEOucyKJzFsCz5Gr41jFHj8i2g1zemTyS4uyErJgFHc=
github.com/monkeyWie/goed2k v0.0.0-20260317094144-6e18d43056e5/go.mod h1:Ry2y1QlzerUgA1hVmExBdXXzE4Sjk1M7w0nSh6dhDOg=
github.com/monkeyWie/goed2k v0.0.0-20260317100435-7a7575cf2447 h1:88kRsNwkDKPA4NzRUWdOoIL7SyMNDwtmAxzShKBvKTI=
github.com/monkeyWie/goed2k v0.0.0-20260317100435-7a7575cf2447/go.mod h1:Ry2y1QlzerUgA1hVmExBdXXzE4Sjk1M7w0nSh6dhDOg=
github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o=
+677
View File
@@ -0,0 +1,677 @@
package blob
import (
"bytes"
"context"
"crypto/rand"
"encoding/hex"
"errors"
"fmt"
"io"
"net"
"net/http"
"net/url"
"path"
"strconv"
"strings"
"sync"
"time"
)
const urlPathPrefix = "/__blob/"
const rangeSourceFailureLimit = 2
// unclaimedSourceTTL bounds how long a session-backed source may keep its
// engine alive without ever being claimed by a download task.
var unclaimedSourceTTL = 10 * time.Minute
var (
ErrInvalidURL = errors.New("invalid blob url")
ErrInvalidOptions = errors.New("invalid blob options")
ErrSourceNotFound = errors.New("blob source not found")
ErrSourceRevoked = errors.New("blob source revoked")
ErrSourceClosed = errors.New("blob source closed")
ErrRangeNotAllowed = errors.New("blob range not allowed")
)
type SessionRef interface {
Retain()
Release()
}
type OpenRequest struct {
Offset int64
End int64
}
type OpenFunc func(ctx context.Context, req OpenRequest) (io.ReadCloser, error)
type CreateOptions struct {
ContentType string
Size int64
Range bool
Session SessionRef
}
type Metadata struct {
ContentType string
Size int64
Range bool
}
type Source struct {
ID string
URL string
ContentType string
mu sync.Mutex
size int64
rangeEnabled bool
revoked bool
taskRefs int
readErr error
rangeFailures int
session SessionRef
open OpenFunc
unclaimed *time.Timer
}
type Registry struct {
dir string
mu sync.RWMutex
sources map[string]*Source
serverMu sync.Mutex
listener net.Listener
server *http.Server
baseURL string
}
func NewRegistry(dir string) *Registry {
return &Registry{
dir: dir,
sources: make(map[string]*Source),
}
}
func (r *Registry) Dir() string {
if r == nil {
return ""
}
return r.dir
}
func (r *Registry) IsURL(raw string) bool {
if r == nil {
return false
}
src, err := r.get(raw)
return err == nil && src != nil
}
func (r *Registry) CreateBlob(data []byte, contentType string) (string, error) {
buf := append([]byte(nil), data...)
return r.CreateOpener(func(ctx context.Context, req OpenRequest) (io.ReadCloser, error) {
if req.Offset < 0 || req.Offset > int64(len(buf)) {
return nil, ErrSourceNotFound
}
end := int64(len(buf))
if req.End >= req.Offset && req.End+1 < end {
end = req.End + 1
}
return io.NopCloser(bytes.NewReader(buf[req.Offset:end])), nil
}, &CreateOptions{
ContentType: contentType,
Size: int64(len(buf)),
Range: len(buf) > 0,
})
}
func (r *Registry) CreateOpener(open OpenFunc, opts *CreateOptions) (string, error) {
if r == nil {
return "", ErrSourceNotFound
}
if open == nil {
return "", ErrInvalidOptions
}
if opts == nil {
opts = &CreateOptions{}
}
if opts.Size < 0 {
opts.Size = 0
}
if opts.Range && opts.Size <= 0 {
return "", fmt.Errorf("%w: range requires positive size", ErrInvalidOptions)
}
// Keep server generation selection and source insertion atomic with Close.
r.serverMu.Lock()
baseURL, err := r.ensureServerLocked()
if err != nil {
r.serverMu.Unlock()
return "", err
}
id, err := randomID(18)
if err != nil {
r.serverMu.Unlock()
return "", err
}
if opts.Session != nil {
opts.Session.Retain()
}
srcURL := fmt.Sprintf("%s%s", baseURL, id)
src := &Source{
ID: id,
URL: srcURL,
ContentType: opts.ContentType,
size: opts.Size,
rangeEnabled: opts.Range,
session: opts.Session,
open: open,
}
r.mu.Lock()
r.sources[id] = src
r.mu.Unlock()
r.serverMu.Unlock()
if opts.Session != nil && unclaimedSourceTTL > 0 {
src.mu.Lock()
if !src.revoked {
src.unclaimed = time.AfterFunc(unclaimedSourceTTL, func() {
r.expireUnclaimed(src)
})
}
src.mu.Unlock()
}
return src.URL, nil
}
func (r *Registry) Metadata(raw string) (Metadata, error) {
src, err := r.get(raw)
if err != nil {
return Metadata{}, err
}
src.mu.Lock()
defer src.mu.Unlock()
return Metadata{
ContentType: src.ContentType,
Size: src.size,
Range: src.rangeEnabled,
}, nil
}
// Acquire retains a source for a task. A source remains available until every
// task that acquired it calls Release, unless it is explicitly revoked.
func (r *Registry) Acquire(raw string) error {
src, err := r.get(raw)
if err != nil {
return err
}
return src.acquireTask()
}
// Release releases a task's reference to a source. Releasing the final task
// reference revokes the source. Sources that have never been acquired remain
// registered until Revoke or Registry.Close is called; session-backed sources
// are also bounded by the unclaimed-source TTL.
func (r *Registry) Release(raw string) error {
src, err := r.get(raw)
if err != nil {
return err
}
session, last, err := src.releaseTask()
if err != nil {
return err
}
if !last {
return nil
}
r.deleteSource(src)
if session != nil {
session.Release()
}
return nil
}
// SourceError returns the first error encountered while opening or reading a
// source. Invalid and missing sources have no source error. HTTP response write
// errors are deliberately not recorded here.
func (r *Registry) SourceError(raw string) error {
src, err := r.get(raw)
if err != nil {
return nil
}
return src.sourceError()
}
func (r *Registry) Revoke(raw string) error {
src, err := r.get(raw)
if err != nil {
return err
}
r.removeSource(src)
return nil
}
func (r *Registry) Close() error {
if r == nil {
return nil
}
r.serverMu.Lock()
server := r.server
listener := r.listener
r.server = nil
r.listener = nil
r.baseURL = ""
r.mu.Lock()
sources := make([]*Source, 0, len(r.sources))
for _, src := range r.sources {
sources = append(sources, src)
}
r.sources = make(map[string]*Source)
r.mu.Unlock()
r.serverMu.Unlock()
if server != nil {
_ = server.Close()
}
if listener != nil {
_ = listener.Close()
}
for _, src := range sources {
src.close()
}
return nil
}
// ensureServerLocked returns the current server URL or starts a new server.
// The caller must hold serverMu.
func (r *Registry) ensureServerLocked() (string, error) {
if r.baseURL != "" {
return r.baseURL, nil
}
listener, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
return "", err
}
r.listener = listener
r.baseURL = "http://" + listener.Addr().String() + urlPathPrefix
server := &http.Server{Handler: r}
r.server = server
go func() {
if err := server.Serve(listener); err != nil && !errors.Is(err, http.ErrServerClosed) {
_ = r.Close()
}
}()
return r.baseURL, nil
}
func (r *Registry) ServeHTTP(w http.ResponseWriter, req *http.Request) {
if req.Method != http.MethodGet {
w.Header().Set("Allow", "GET")
http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
return
}
id := parseRequest(req)
if id == "" {
http.NotFound(w, req)
return
}
src, err := r.getByID(id)
if err != nil {
http.NotFound(w, req)
return
}
meta, open, session := src.acquireOpen()
if open == nil {
http.NotFound(w, req)
return
}
if session != nil {
defer session.Release()
}
if src.sourceError() != nil {
http.Error(w, "blob source unavailable", http.StatusGone)
return
}
start, end, ranged, err := parseRange(req.Header.Get("Range"), meta.Size, meta.Range)
if err != nil {
http.Error(w, err.Error(), http.StatusRequestedRangeNotSatisfiable)
return
}
if !meta.Range {
start, end, ranged = 0, -1, false
}
reader, err := open(req.Context(), OpenRequest{
Offset: start,
End: end,
})
if err != nil {
if req.Context().Err() == nil || !errors.Is(err, req.Context().Err()) {
src.recordSourceError(err, meta.Range)
}
http.Error(w, "blob source unavailable", http.StatusGone)
return
}
if reader == nil {
err = fmt.Errorf("%w: opener returned a nil reader", ErrSourceClosed)
src.recordSourceError(err, meta.Range)
http.Error(w, "blob source unavailable", http.StatusGone)
return
}
defer reader.Close()
writeHeaders(w, meta, start, end, ranged)
if ranged {
w.WriteHeader(http.StatusPartialContent)
}
limit := int64(-1)
if end >= start {
limit = end - start + 1
} else if meta.Size > 0 {
limit = meta.Size
}
_, readErr, writeErr := copyWithFlush(w, reader, limit)
if readErr != nil {
if ctxErr := req.Context().Err(); ctxErr == nil || !errors.Is(readErr, ctxErr) {
src.recordSourceError(readErr, meta.Range)
}
} else if writeErr == nil && req.Context().Err() == nil {
src.recordSourceSuccess(meta.Range)
}
}
func (s *Source) acquireOpen() (Metadata, OpenFunc, SessionRef) {
s.mu.Lock()
defer s.mu.Unlock()
if s.revoked {
return Metadata{}, nil, nil
}
if s.session != nil {
// The source's own retain guarantees that the session cannot become
// closed while this additional active-reader retain is acquired.
s.session.Retain()
}
return Metadata{
ContentType: s.ContentType,
Size: s.size,
Range: s.rangeEnabled,
}, s.open, s.session
}
func (s *Source) acquireTask() error {
s.mu.Lock()
if s.revoked {
s.mu.Unlock()
return ErrSourceRevoked
}
s.taskRefs++
timer := s.unclaimed
s.unclaimed = nil
s.mu.Unlock()
if timer != nil {
timer.Stop()
}
return nil
}
func (s *Source) releaseTask() (session SessionRef, last bool, err error) {
s.mu.Lock()
defer s.mu.Unlock()
if s.revoked {
return nil, false, ErrSourceRevoked
}
if s.taskRefs == 0 {
return nil, false, ErrSourceClosed
}
s.taskRefs--
if s.taskRefs > 0 {
return nil, false, nil
}
s.revoked = true
s.open = nil
session = s.session
s.session = nil
return session, true, nil
}
func (s *Source) sourceError() error {
s.mu.Lock()
defer s.mu.Unlock()
return s.readErr
}
func (s *Source) recordSourceError(err error, ranged bool) {
if err == nil {
return
}
s.mu.Lock()
if s.readErr == nil && ranged {
s.rangeFailures++
if s.rangeFailures < rangeSourceFailureLimit {
s.mu.Unlock()
return
}
}
if s.readErr == nil {
s.readErr = err
}
s.mu.Unlock()
}
func (s *Source) recordSourceSuccess(ranged bool) {
if !ranged {
return
}
s.mu.Lock()
if s.readErr == nil {
s.rangeFailures = 0
}
s.mu.Unlock()
}
func (s *Source) close() {
s.mu.Lock()
alreadyRevoked := s.revoked
s.revoked = true
s.open = nil
session := s.session
s.session = nil
timer := s.unclaimed
s.unclaimed = nil
s.mu.Unlock()
if timer != nil {
timer.Stop()
}
if !alreadyRevoked && session != nil {
session.Release()
}
}
func (s *Source) expireUnclaimed() (SessionRef, bool) {
s.mu.Lock()
defer s.mu.Unlock()
s.unclaimed = nil
if s.revoked || s.taskRefs > 0 {
return nil, false
}
s.revoked = true
s.open = nil
session := s.session
s.session = nil
return session, true
}
func (r *Registry) expireUnclaimed(src *Source) {
session, expired := src.expireUnclaimed()
if !expired {
return
}
r.deleteSource(src)
if session != nil {
session.Release()
}
}
func (r *Registry) removeSource(src *Source) {
src.close()
r.deleteSource(src)
}
func (r *Registry) deleteSource(src *Source) {
r.mu.Lock()
if current := r.sources[src.ID]; current == src {
delete(r.sources, src.ID)
}
r.mu.Unlock()
}
func writeHeaders(w http.ResponseWriter, meta Metadata, start, end int64, ranged bool) {
if meta.ContentType != "" {
w.Header().Set("Content-Type", meta.ContentType)
}
if meta.Range {
w.Header().Set("Accept-Ranges", "bytes")
}
if meta.Size > 0 {
if ranged {
w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, meta.Size))
w.Header().Set("Content-Length", strconv.FormatInt(end-start+1, 10))
return
}
w.Header().Set("Content-Length", strconv.FormatInt(meta.Size, 10))
}
}
func copyWithFlush(w http.ResponseWriter, reader io.Reader, limit int64) (written int64, readErr error, writeErr error) {
if limit >= 0 {
reader = io.LimitReader(reader, limit)
}
buf := make([]byte, 32*1024)
flusher, canFlush := w.(http.Flusher)
for {
nr, er := reader.Read(buf)
if er != nil && !errors.Is(er, io.EOF) {
readErr = er
}
if nr > 0 {
nw, ew := w.Write(buf[:nr])
if nw > 0 {
written += int64(nw)
if canFlush {
flusher.Flush()
}
}
if ew != nil {
return written, readErr, ew
}
if nr != nw {
return written, readErr, io.ErrShortWrite
}
}
if er != nil {
if errors.Is(er, io.EOF) {
if limit >= 0 && written < limit {
return written, io.ErrUnexpectedEOF, nil
}
return written, nil, nil
}
return written, readErr, nil
}
}
}
func parseRange(header string, size int64, rangeEnabled bool) (start int64, end int64, ranged bool, err error) {
if header == "" || !rangeEnabled {
return 0, -1, false, nil
}
if size <= 0 {
return 0, 0, false, ErrRangeNotAllowed
}
if !strings.HasPrefix(header, "bytes=") {
return 0, 0, false, fmt.Errorf("unsupported range")
}
parts := strings.SplitN(strings.TrimPrefix(header, "bytes="), "-", 2)
if len(parts) != 2 || parts[0] == "" {
return 0, 0, false, fmt.Errorf("unsupported range")
}
start, err = strconv.ParseInt(parts[0], 10, 64)
if err != nil || start < 0 {
return 0, 0, false, fmt.Errorf("invalid range")
}
if start >= size {
return 0, 0, false, fmt.Errorf("range out of bounds")
}
end = size - 1
if parts[1] != "" {
end, err = strconv.ParseInt(parts[1], 10, 64)
if err != nil || end < start {
return 0, 0, false, fmt.Errorf("invalid range")
}
if end >= size {
end = size - 1
}
}
return start, end, true, nil
}
func (r *Registry) get(raw string) (*Source, error) {
id, ok := r.parseURL(raw)
if !ok {
return nil, ErrInvalidURL
}
src, err := r.getByID(id)
if err != nil {
return nil, err
}
src.mu.Lock()
revoked := src.revoked
src.mu.Unlock()
if revoked {
return nil, ErrSourceRevoked
}
return src, nil
}
func (r *Registry) getByID(id string) (*Source, error) {
r.mu.RLock()
defer r.mu.RUnlock()
src := r.sources[id]
if src == nil {
return nil, ErrSourceNotFound
}
return src, nil
}
func (r *Registry) parseURL(raw string) (id string, ok bool) {
u, err := url.Parse(raw)
if err != nil || u.Scheme != "http" || !strings.HasPrefix(u.Path, urlPathPrefix) {
return "", false
}
r.serverMu.Lock()
baseURL := r.baseURL
r.serverMu.Unlock()
if baseURL == "" || !strings.HasPrefix(raw, baseURL) {
return "", false
}
id = strings.TrimPrefix(u.Path, urlPathPrefix)
if id == "" || path.Base(id) != id {
return "", false
}
return id, true
}
func parseRequest(req *http.Request) string {
id := strings.TrimPrefix(req.URL.Path, urlPathPrefix)
if id == "" || id == req.URL.Path || path.Base(id) != id {
return ""
}
return id
}
func randomID(size int) (string, error) {
buf := make([]byte, size)
if _, err := rand.Read(buf); err != nil {
return "", err
}
return hex.EncodeToString(buf), nil
}
+898
View File
@@ -0,0 +1,898 @@
package blob
import (
"bytes"
"context"
"errors"
"io"
"net/http"
"strings"
"sync"
"testing"
"time"
)
type terminalErrorReader struct {
err error
}
func (r *terminalErrorReader) Read([]byte) (int, error) {
return 0, r.err
}
type contextErrorReader struct {
ctx context.Context
}
func (r *contextErrorReader) Read([]byte) (int, error) {
<-r.ctx.Done()
return 0, r.ctx.Err()
}
func (r *contextErrorReader) Close() error { return nil }
type failingResponseWriter struct {
header http.Header
err error
}
func (w *failingResponseWriter) Header() http.Header {
return w.header
}
func (w *failingResponseWriter) Write([]byte) (int, error) {
return 0, w.err
}
func (w *failingResponseWriter) WriteHeader(int) {}
type countingSession struct {
mu sync.Mutex
refs int
zero chan struct{}
zeroOnce sync.Once
}
func newCountingSession() *countingSession {
return &countingSession{zero: make(chan struct{})}
}
func (s *countingSession) Retain() {
s.mu.Lock()
s.refs++
s.mu.Unlock()
}
func (s *countingSession) Release() {
s.mu.Lock()
s.refs--
refs := s.refs
s.mu.Unlock()
if refs == 0 {
s.zeroOnce.Do(func() { close(s.zero) })
}
}
func (s *countingSession) RefCount() int {
s.mu.Lock()
defer s.mu.Unlock()
return s.refs
}
func setUnclaimedSourceTTLForTest(t *testing.T, ttl time.Duration) {
t.Helper()
previous := unclaimedSourceTTL
unclaimedSourceTTL = ttl
t.Cleanup(func() {
unclaimedSourceTTL = previous
})
}
func TestRegistryBlobHTTPGetAndRange(t *testing.T) {
registry := NewRegistry(t.TempDir())
defer registry.Close()
url, err := registry.CreateBlob([]byte("hello world"), "text/plain")
if err != nil {
t.Fatal(err)
}
if !registry.IsURL(url) {
t.Fatal("expected registry to recognize created url")
}
resp, err := http.Get(url)
if err != nil {
t.Fatal(err)
}
body, err := io.ReadAll(resp.Body)
_ = resp.Body.Close()
if err != nil {
t.Fatal(err)
}
if resp.StatusCode != http.StatusOK {
t.Fatalf("unexpected GET status: %d", resp.StatusCode)
}
if got := resp.Header.Get("Content-Length"); got != "11" {
t.Fatalf("unexpected content length: %q", got)
}
if got := resp.Header.Get("Accept-Ranges"); got != "bytes" {
t.Fatalf("unexpected accept ranges: %q", got)
}
if got := resp.Header.Get("Content-Type"); !strings.HasPrefix(got, "text/plain") {
t.Fatalf("unexpected content type: %q", got)
}
if string(body) != "hello world" {
t.Fatalf("unexpected GET body: %q", string(body))
}
req, err := http.NewRequest(http.MethodGet, url, nil)
if err != nil {
t.Fatal(err)
}
req.Header.Set("Range", "bytes=6-10")
resp, err = http.DefaultClient.Do(req)
if err != nil {
t.Fatal(err)
}
body, err = io.ReadAll(resp.Body)
_ = resp.Body.Close()
if err != nil {
t.Fatal(err)
}
if resp.StatusCode != http.StatusPartialContent {
t.Fatalf("unexpected range status: %d", resp.StatusCode)
}
if got := resp.Header.Get("Content-Range"); got != "bytes 6-10/11" {
t.Fatalf("unexpected content range: %q", got)
}
if string(body) != "world" {
t.Fatalf("unexpected range body: %q", string(body))
}
}
func TestRegistryEmptyBlob(t *testing.T) {
registry := NewRegistry(t.TempDir())
defer registry.Close()
url, err := registry.CreateBlob(nil, "application/octet-stream")
if err != nil {
t.Fatal(err)
}
meta, err := registry.Metadata(url)
if err != nil {
t.Fatal(err)
}
if meta.Size != 0 || meta.Range {
t.Fatalf("unexpected empty blob metadata: %#v", meta)
}
resp, err := http.Get(url)
if err != nil {
t.Fatal(err)
}
body, readErr := io.ReadAll(resp.Body)
_ = resp.Body.Close()
if readErr != nil {
t.Fatal(readErr)
}
if resp.StatusCode != http.StatusOK {
t.Fatalf("unexpected status: %d", resp.StatusCode)
}
if len(body) != 0 {
t.Fatalf("unexpected body: %q", body)
}
if got := resp.Header.Get("Content-Length"); got != "0" {
t.Fatalf("unexpected content length: %q", got)
}
}
func TestRegistryOpenerSizeWithoutRange(t *testing.T) {
registry := NewRegistry(t.TempDir())
defer registry.Close()
payload := []byte("hello opener")
url, err := registry.CreateOpener(func(ctx context.Context, req OpenRequest) (io.ReadCloser, error) {
if req.Offset != 0 || req.End != -1 {
t.Fatalf("unexpected non-range request: %#v", req)
}
return io.NopCloser(bytes.NewReader(payload)), nil
}, &CreateOptions{
ContentType: "text/plain",
Size: int64(len(payload)),
Range: false,
})
if err != nil {
t.Fatal(err)
}
req, err := http.NewRequest(http.MethodGet, url, nil)
if err != nil {
t.Fatal(err)
}
req.Header.Set("Range", "bytes=6-11")
resp, err := http.DefaultClient.Do(req)
if err != nil {
t.Fatal(err)
}
body, err := io.ReadAll(resp.Body)
_ = resp.Body.Close()
if err != nil {
t.Fatal(err)
}
if resp.StatusCode != http.StatusOK {
t.Fatalf("unexpected status: %d", resp.StatusCode)
}
if got := resp.Header.Get("Content-Length"); got != "12" {
t.Fatalf("unexpected content length: %q", got)
}
if got := resp.Header.Get("Accept-Ranges"); got != "" {
t.Fatalf("unexpected accept ranges: %q", got)
}
if string(body) != string(payload) {
t.Fatalf("unexpected body: %q", string(body))
}
}
func TestRegistryOpenerRangeConcurrentRequests(t *testing.T) {
registry := NewRegistry(t.TempDir())
defer registry.Close()
payload := []byte("abcdefghijklmnopqrstuvwxyz")
var mu sync.Mutex
var calls []OpenRequest
url, err := registry.CreateOpener(func(ctx context.Context, req OpenRequest) (io.ReadCloser, error) {
mu.Lock()
calls = append(calls, req)
mu.Unlock()
start := int(req.Offset)
end := len(payload)
if req.End >= 0 {
end = int(req.End) + 1
}
if start < 0 || start > end || end > len(payload) {
return nil, errors.New("bad test range")
}
return io.NopCloser(bytes.NewReader(payload[start:end])), nil
}, &CreateOptions{
Size: int64(len(payload)),
Range: true,
})
if err != nil {
t.Fatal(err)
}
ranges := []struct {
header string
want string
}{
{header: "bytes=0-4", want: "abcde"},
{header: "bytes=10-15", want: "klmnop"},
{header: "bytes=20-25", want: "uvwxyz"},
}
var wg sync.WaitGroup
for _, item := range ranges {
item := item
wg.Add(1)
go func() {
defer wg.Done()
req, err := http.NewRequest(http.MethodGet, url, nil)
if err != nil {
t.Error(err)
return
}
req.Header.Set("Range", item.header)
resp, err := http.DefaultClient.Do(req)
if err != nil {
t.Error(err)
return
}
body, err := io.ReadAll(resp.Body)
_ = resp.Body.Close()
if err != nil {
t.Error(err)
return
}
if resp.StatusCode != http.StatusPartialContent {
t.Errorf("unexpected status for %s: %d", item.header, resp.StatusCode)
}
if string(body) != item.want {
t.Errorf("unexpected body for %s: %q", item.header, string(body))
}
}()
}
wg.Wait()
mu.Lock()
defer mu.Unlock()
if len(calls) != len(ranges) {
t.Fatalf("unexpected opener call count: %d", len(calls))
}
seen := map[OpenRequest]bool{}
for _, call := range calls {
seen[call] = true
}
for _, want := range []OpenRequest{{Offset: 0, End: 4}, {Offset: 10, End: 15}, {Offset: 20, End: 25}} {
if !seen[want] {
t.Fatalf("missing opener request %#v, got %#v", want, calls)
}
}
}
func TestRegistryRejectsRangeWithoutSize(t *testing.T) {
registry := NewRegistry(t.TempDir())
defer registry.Close()
_, err := registry.CreateOpener(func(ctx context.Context, req OpenRequest) (io.ReadCloser, error) {
return io.NopCloser(strings.NewReader("")), nil
}, &CreateOptions{Range: true})
if !errors.Is(err, ErrInvalidOptions) {
t.Fatalf("expected invalid options error, got %v", err)
}
}
func TestRegistryTracksSourceErrors(t *testing.T) {
t.Run("reader error", func(t *testing.T) {
registry := NewRegistry(t.TempDir())
defer registry.Close()
boom := errors.New("reader failed")
url, err := registry.CreateOpener(func(context.Context, OpenRequest) (io.ReadCloser, error) {
return io.NopCloser(io.MultiReader(strings.NewReader("partial"), &terminalErrorReader{err: boom})), nil
}, &CreateOptions{})
if err != nil {
t.Fatal(err)
}
resp, err := http.Get(url)
if err != nil {
t.Fatal(err)
}
body, readErr := io.ReadAll(resp.Body)
_ = resp.Body.Close()
if readErr != nil {
t.Fatalf("the HTTP transport should end normally, got %v", readErr)
}
if string(body) != "partial" {
t.Fatalf("unexpected partial body: %q", body)
}
if got := registry.SourceError(url); !errors.Is(got, boom) {
t.Fatalf("expected source error %v, got %v", boom, got)
}
})
t.Run("short known-size reader", func(t *testing.T) {
registry := NewRegistry(t.TempDir())
defer registry.Close()
url, err := registry.CreateOpener(func(context.Context, OpenRequest) (io.ReadCloser, error) {
return io.NopCloser(strings.NewReader("short")), nil
}, &CreateOptions{Size: 10})
if err != nil {
t.Fatal(err)
}
resp, err := http.Get(url)
if err != nil {
t.Fatal(err)
}
_, _ = io.ReadAll(resp.Body)
_ = resp.Body.Close()
if got := registry.SourceError(url); !errors.Is(got, io.ErrUnexpectedEOF) {
t.Fatalf("expected unexpected EOF, got %v", got)
}
})
t.Run("range reader error becomes terminal", func(t *testing.T) {
registry := NewRegistry(t.TempDir())
defer registry.Close()
boom := errors.New("range reader failed")
url, err := registry.CreateOpener(func(context.Context, OpenRequest) (io.ReadCloser, error) {
return io.NopCloser(io.MultiReader(strings.NewReader("partial"), &terminalErrorReader{err: boom})), nil
}, &CreateOptions{Size: 10, Range: true})
if err != nil {
t.Fatal(err)
}
for attempt := 0; attempt < rangeSourceFailureLimit; attempt++ {
req, err := http.NewRequest(http.MethodGet, url, nil)
if err != nil {
t.Fatal(err)
}
req.Header.Set("Range", "bytes=0-9")
resp, err := http.DefaultClient.Do(req)
if err != nil {
t.Fatal(err)
}
_, _ = io.ReadAll(resp.Body)
_ = resp.Body.Close()
}
if got := registry.SourceError(url); !errors.Is(got, boom) {
t.Fatalf("expected range source error %v, got %v", boom, got)
}
resp, err := http.Get(url)
if err != nil {
t.Fatal(err)
}
_ = resp.Body.Close()
if resp.StatusCode != http.StatusGone {
t.Fatalf("expected terminal source status %d, got %d", http.StatusGone, resp.StatusCode)
}
})
t.Run("short range reader", func(t *testing.T) {
registry := NewRegistry(t.TempDir())
defer registry.Close()
url, err := registry.CreateOpener(func(context.Context, OpenRequest) (io.ReadCloser, error) {
return io.NopCloser(strings.NewReader("short")), nil
}, &CreateOptions{Size: 10, Range: true})
if err != nil {
t.Fatal(err)
}
for attempt := 0; attempt < rangeSourceFailureLimit; attempt++ {
req, err := http.NewRequest(http.MethodGet, url, nil)
if err != nil {
t.Fatal(err)
}
req.Header.Set("Range", "bytes=0-9")
resp, err := http.DefaultClient.Do(req)
if err != nil {
t.Fatal(err)
}
_, _ = io.ReadAll(resp.Body)
_ = resp.Body.Close()
}
if got := registry.SourceError(url); !errors.Is(got, io.ErrUnexpectedEOF) {
t.Fatalf("expected range unexpected EOF, got %v", got)
}
})
t.Run("successful range retry clears transient failure", func(t *testing.T) {
registry := NewRegistry(t.TempDir())
defer registry.Close()
attempt := 0
boom := errors.New("transient range failure")
url, err := registry.CreateOpener(func(context.Context, OpenRequest) (io.ReadCloser, error) {
attempt++
if attempt == 1 {
return io.NopCloser(&terminalErrorReader{err: boom}), nil
}
return io.NopCloser(strings.NewReader("0123456789")), nil
}, &CreateOptions{Size: 10, Range: true})
if err != nil {
t.Fatal(err)
}
for i := 0; i < 2; i++ {
req, err := http.NewRequest(http.MethodGet, url, nil)
if err != nil {
t.Fatal(err)
}
req.Header.Set("Range", "bytes=0-9")
resp, err := http.DefaultClient.Do(req)
if err != nil {
t.Fatal(err)
}
_, _ = io.ReadAll(resp.Body)
_ = resp.Body.Close()
}
if got := registry.SourceError(url); got != nil {
t.Fatalf("successful retry left transient source error: %v", got)
}
})
t.Run("request cancellation is not a source error", func(t *testing.T) {
registry := NewRegistry(t.TempDir())
defer registry.Close()
opened := make(chan struct{})
url, err := registry.CreateOpener(func(ctx context.Context, _ OpenRequest) (io.ReadCloser, error) {
close(opened)
return &contextErrorReader{ctx: ctx}, nil
}, &CreateOptions{})
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithCancel(context.Background())
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
t.Fatal(err)
}
done := make(chan struct{})
go func() {
resp, _ := http.DefaultClient.Do(req)
if resp != nil {
_ = resp.Body.Close()
}
close(done)
}()
<-opened
cancel()
select {
case <-done:
case <-time.After(2 * time.Second):
t.Fatal("cancelled request did not finish")
}
if got := registry.SourceError(url); got != nil {
t.Fatalf("request cancellation was recorded as source error: %v", got)
}
})
t.Run("nil reader", func(t *testing.T) {
registry := NewRegistry(t.TempDir())
defer registry.Close()
url, err := registry.CreateOpener(func(context.Context, OpenRequest) (io.ReadCloser, error) {
return nil, nil
}, &CreateOptions{})
if err != nil {
t.Fatal(err)
}
resp, err := http.Get(url)
if err != nil {
t.Fatal(err)
}
_ = resp.Body.Close()
if resp.StatusCode != http.StatusGone {
t.Fatalf("expected nil reader status %d, got %d", http.StatusGone, resp.StatusCode)
}
if got := registry.SourceError(url); !errors.Is(got, ErrSourceClosed) {
t.Fatalf("expected closed source error, got %v", got)
}
})
}
func TestRegistryDoesNotRecordResponseWriteError(t *testing.T) {
registry := NewRegistry(t.TempDir())
defer registry.Close()
url, err := registry.CreateOpener(func(context.Context, OpenRequest) (io.ReadCloser, error) {
return io.NopCloser(strings.NewReader("payload")), nil
}, &CreateOptions{Size: 7})
if err != nil {
t.Fatal(err)
}
req, err := http.NewRequest(http.MethodGet, url, nil)
if err != nil {
t.Fatal(err)
}
writerErr := errors.New("client connection closed")
registry.ServeHTTP(&failingResponseWriter{header: make(http.Header), err: writerErr}, req)
if got := registry.SourceError(url); got != nil {
t.Fatalf("response write error was recorded as a source error: %v", got)
}
}
func TestRegistryTaskReferencesShareSource(t *testing.T) {
registry := NewRegistry(t.TempDir())
defer registry.Close()
url, err := registry.CreateBlob([]byte("shared"), "text/plain")
if err != nil {
t.Fatal(err)
}
if err := registry.Acquire(url); err != nil {
t.Fatal(err)
}
if err := registry.Acquire(url); err != nil {
t.Fatal(err)
}
if err := registry.Release(url); err != nil {
t.Fatal(err)
}
if !registry.IsURL(url) {
t.Fatal("the first task release revoked a source still used by another task")
}
resp, err := http.Get(url)
if err != nil {
t.Fatal(err)
}
body, readErr := io.ReadAll(resp.Body)
_ = resp.Body.Close()
if readErr != nil || string(body) != "shared" {
t.Fatalf("unexpected shared response body %q, error %v", body, readErr)
}
if err := registry.Release(url); err != nil {
t.Fatal(err)
}
if registry.IsURL(url) {
t.Fatal("the final task release did not revoke the source")
}
}
func TestRegistryUnclaimedSessionSourceExpires(t *testing.T) {
setUnclaimedSourceTTLForTest(t, 25*time.Millisecond)
registry := NewRegistry(t.TempDir())
defer registry.Close()
session := newCountingSession()
url, err := registry.CreateOpener(func(context.Context, OpenRequest) (io.ReadCloser, error) {
return io.NopCloser(strings.NewReader("unused")), nil
}, &CreateOptions{Size: 6, Session: session})
if err != nil {
t.Fatal(err)
}
if got := session.RefCount(); got != 1 {
t.Fatalf("unexpected initial session ref count: %d", got)
}
select {
case <-session.zero:
case <-time.After(2 * time.Second):
t.Fatal("unclaimed source did not expire")
}
if registry.IsURL(url) {
t.Fatal("expired source is still registered")
}
if got := session.RefCount(); got != 0 {
t.Fatalf("expired source retained its session: %d", got)
}
}
func TestRegistryAcquireStopsUnclaimedSourceExpiration(t *testing.T) {
const ttl = 25 * time.Millisecond
setUnclaimedSourceTTLForTest(t, ttl)
registry := NewRegistry(t.TempDir())
defer registry.Close()
session := newCountingSession()
url, err := registry.CreateOpener(func(context.Context, OpenRequest) (io.ReadCloser, error) {
return io.NopCloser(strings.NewReader("claimed")), nil
}, &CreateOptions{Size: 7, Session: session})
if err != nil {
t.Fatal(err)
}
if err := registry.Acquire(url); err != nil {
t.Fatal(err)
}
select {
case <-session.zero:
t.Fatal("claimed source expired")
case <-time.After(4 * ttl):
}
if !registry.IsURL(url) {
t.Fatal("claimed source was removed")
}
if err := registry.Release(url); err != nil {
t.Fatal(err)
}
select {
case <-session.zero:
case <-time.After(2 * time.Second):
t.Fatal("final task release did not release the session")
}
}
func TestRegistryUnclaimedExpirationKeepsActiveReaderSession(t *testing.T) {
setUnclaimedSourceTTLForTest(t, 100*time.Millisecond)
registry := NewRegistry(t.TempDir())
defer registry.Close()
session := newCountingSession()
reader, writer := io.Pipe()
opened := make(chan struct{})
url, err := registry.CreateOpener(func(context.Context, OpenRequest) (io.ReadCloser, error) {
close(opened)
return reader, nil
}, &CreateOptions{Size: 7, Session: session})
if err != nil {
t.Fatal(err)
}
type result struct {
body []byte
err error
}
resultCh := make(chan result, 1)
go func() {
resp, err := http.Get(url)
if err != nil {
resultCh <- result{err: err}
return
}
body, err := io.ReadAll(resp.Body)
_ = resp.Body.Close()
resultCh <- result{body: body, err: err}
}()
select {
case <-opened:
case got := <-resultCh:
t.Fatalf("request ended before the opener ran: body %q, error %v", got.body, got.err)
case <-time.After(2 * time.Second):
t.Fatal("active request did not open")
}
deadline := time.NewTimer(2 * time.Second)
ticker := time.NewTicker(5 * time.Millisecond)
defer deadline.Stop()
defer ticker.Stop()
for registry.IsURL(url) {
select {
case <-ticker.C:
case <-deadline.C:
t.Fatal("unclaimed source did not expire while its reader was active")
}
}
if got := session.RefCount(); got != 1 {
t.Fatalf("expiration released the active-reader session ref: %d", got)
}
if _, err := writer.Write([]byte("payload")); err != nil {
t.Fatal(err)
}
_ = writer.Close()
select {
case got := <-resultCh:
if got.err != nil || string(got.body) != "payload" {
t.Fatalf("active response was interrupted: body %q, error %v", got.body, got.err)
}
case <-time.After(2 * time.Second):
t.Fatal("active response did not finish after source expiration")
}
select {
case <-session.zero:
if got := session.RefCount(); got != 0 {
t.Fatalf("unexpected final session ref count: %d", got)
}
case <-time.After(2 * time.Second):
t.Fatal("active-reader session ref was not released")
}
}
func TestRegistryCloseCancelsUnclaimedExpiration(t *testing.T) {
const ttl = 25 * time.Millisecond
setUnclaimedSourceTTLForTest(t, ttl)
registry := NewRegistry(t.TempDir())
session := newCountingSession()
_, err := registry.CreateOpener(func(context.Context, OpenRequest) (io.ReadCloser, error) {
return io.NopCloser(strings.NewReader("unused")), nil
}, &CreateOptions{Size: 6, Session: session})
if err != nil {
t.Fatal(err)
}
if err := registry.Close(); err != nil {
t.Fatal(err)
}
select {
case <-session.zero:
case <-time.After(2 * time.Second):
t.Fatal("registry close did not release the session")
}
<-time.After(4 * ttl)
if got := session.RefCount(); got != 0 {
t.Fatalf("expiration released the session again after close: %d", got)
}
}
func TestRegistryRevokeKeepsActiveReaderSession(t *testing.T) {
registry := NewRegistry(t.TempDir())
defer registry.Close()
session := newCountingSession()
reader, writer := io.Pipe()
opened := make(chan struct{})
url, err := registry.CreateOpener(func(context.Context, OpenRequest) (io.ReadCloser, error) {
close(opened)
return reader, nil
}, &CreateOptions{Size: 7, Session: session})
if err != nil {
t.Fatal(err)
}
type result struct {
body []byte
err error
}
resultCh := make(chan result, 1)
go func() {
resp, err := http.Get(url)
if err != nil {
resultCh <- result{err: err}
return
}
body, err := io.ReadAll(resp.Body)
_ = resp.Body.Close()
resultCh <- result{body: body, err: err}
}()
<-opened
if got := session.RefCount(); got != 2 {
t.Fatalf("expected source and active-reader session refs, got %d", got)
}
if err := registry.Revoke(url); err != nil {
t.Fatal(err)
}
if got := session.RefCount(); got != 1 {
t.Fatalf("revoke released the active-reader session ref: %d", got)
}
if _, err := writer.Write([]byte("payload")); err != nil {
t.Fatal(err)
}
_ = writer.Close()
select {
case got := <-resultCh:
if got.err != nil || string(got.body) != "payload" {
t.Fatalf("active response was interrupted: body %q, error %v", got.body, got.err)
}
case <-time.After(2 * time.Second):
t.Fatal("active response did not finish after revoke")
}
select {
case <-session.zero:
if got := session.RefCount(); got != 0 {
t.Fatalf("unexpected final session ref count: %d", got)
}
case <-time.After(2 * time.Second):
t.Fatal("active-reader session ref was not released")
}
}
func TestRegistryHTTPErrorStatuses(t *testing.T) {
registry := NewRegistry(t.TempDir())
defer registry.Close()
url, err := registry.CreateBlob([]byte("secret"), "text/plain")
if err != nil {
t.Fatal(err)
}
badURL := url + "-missing"
resp, err := http.Get(badURL)
if err != nil {
t.Fatal(err)
}
_ = resp.Body.Close()
if resp.StatusCode != http.StatusNotFound {
t.Fatalf("unexpected missing source status: %d", resp.StatusCode)
}
req, err := http.NewRequest(http.MethodHead, url, nil)
if err != nil {
t.Fatal(err)
}
resp, err = http.DefaultClient.Do(req)
if err != nil {
t.Fatal(err)
}
_ = resp.Body.Close()
if resp.StatusCode != http.StatusMethodNotAllowed {
t.Fatalf("unexpected method status: %d", resp.StatusCode)
}
req, err = http.NewRequest(http.MethodGet, url, nil)
if err != nil {
t.Fatal(err)
}
req.Header.Set("Range", "bytes=99-100")
resp, err = http.DefaultClient.Do(req)
if err != nil {
t.Fatal(err)
}
_ = resp.Body.Close()
if resp.StatusCode != http.StatusRequestedRangeNotSatisfiable {
t.Fatalf("unexpected invalid range status: %d", resp.StatusCode)
}
if err := registry.Revoke(url); err != nil {
t.Fatal(err)
}
resp, err = http.Get(url)
if err != nil {
t.Fatal(err)
}
_ = resp.Body.Close()
if resp.StatusCode != http.StatusNotFound {
t.Fatalf("unexpected revoked source status: %d", resp.StatusCode)
}
if registry.IsURL(url) {
t.Fatal("revoked source should not be usable")
}
}
+3 -3
View File
@@ -376,7 +376,7 @@ func TestFetcher_DownloadError(t *testing.T) {
}
func TestFetcher_DownloadLimit(t *testing.T) {
listener := test.StartTestLimitServer(4, 0)
listener := test.StartTestConLimitServer(4)
defer listener.Close()
downloadNormal(listener, 1, t)
@@ -614,7 +614,7 @@ func TestFetcher_SlowStartExpansion(t *testing.T) {
os.Remove(test.DownloadFile)
// Use 100ns delay per byte for faster test (~10MB/s theoretical)
listener := test.StartTestSlowStartServer(100 * time.Nanosecond)
listener := test.StartTestLowSpeedServer(100 * time.Nanosecond)
// Ensure cleanup happens before next subtest
cleanup := func() {
@@ -758,7 +758,7 @@ func TestFetcher_AsyncPrefetch(t *testing.T) {
t.Run("PrefetchPartial", func(t *testing.T) {
// Use slow server with 100 nanosecond delay per byte
// This means ~10MB/s speed, so 100ms should download ~1MB
listener := test.StartTestSlowStartServer(100 * time.Nanosecond)
listener := test.StartTestLowSpeedServer(100 * time.Nanosecond)
defer listener.Close()
fetcher := buildFetcher()
+12 -19
View File
@@ -433,9 +433,8 @@ func StartTestExpiringRedirectServer(requestsBeforeExpire int32, delayPerByte ti
})
}
// StartTestSlowStartServer creates a server with configurable delay per request
// This allows testing slow-start connection expansion to reach max connections
func StartTestSlowStartServer(delayPerByte time.Duration) net.Listener {
// StartTestLowSpeedServer creates a server with configurable delay per request
func StartTestLowSpeedServer(delayPerByte time.Duration) net.Listener {
return startTestServer(func(sl *shutdownListener) http.Handler {
mux := http.NewServeMux()
mux.HandleFunc("/"+BuildName, func(writer http.ResponseWriter, request *http.Request) {
@@ -484,10 +483,9 @@ func slowCopyNWithDelay(sl *shutdownListener, dst io.Writer, src io.Reader, n in
}
}
// StartTestLimitServer connections limit server
func StartTestLimitServer(maxConnections int32, delay int64) net.Listener {
// StartTestConLimitServer connections limit server
func StartTestConLimitServer(maxConnections int32) net.Listener {
var connections atomic.Int32
var slowOnce atomic.Bool
return startTestServer(func(sl *shutdownListener) http.Handler {
mux := http.NewServeMux()
@@ -500,19 +498,11 @@ func StartTestLimitServer(maxConnections int32, delay int64) net.Listener {
writer.WriteHeader(403)
return
}
// First request intentionally delays the first write to trigger a single read timeout,
// subsequent requests respond at normal speed.
useInitialDelay := delay > 0 && !slowOnce.Swap(true)
rangeFileHandle(
writer,
request,
nil,
func(file *os.File, n int64) {
if useInitialDelay {
slowCopyAfterDelay(sl, writer, file, n, delay)
return
}
slowCopyN(sl, writer, file, n, 0)
},
)
@@ -711,10 +701,13 @@ func rangeFileHandle(writer http.ResponseWriter, request *http.Request, modifyEn
writer.WriteHeader(400)
return
}
end, err := strconv.ParseInt(s[1], 10, 64)
if err != nil {
writer.WriteHeader(400)
return
end := int64(BuildSize - 1)
if s[1] != "" {
end, err = strconv.ParseInt(s[1], 10, 64)
if err != nil {
writer.WriteHeader(400)
return
}
}
if start < 0 || end < 0 || start > end {
writer.WriteHeader(400)
@@ -880,7 +873,7 @@ func StartSocks5Server(usr, pwd string) net.Listener {
panic(err)
}
listener, err := net.Listen("tcp", "127.0.0.1:0") // 你可以根据需要更改监听地址
listener, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
panic(err)
}
@@ -0,0 +1,36 @@
//go:build darwin
package goprovider
var mainThreadTasks chan func()
func RunMainThreadLoop(run func() int) int {
if mainThreadTasks != nil {
return run()
}
mainThreadTasks = make(chan func())
resultCh := make(chan int, 1)
go func() {
resultCh <- run()
}()
for {
select {
case task := <-mainThreadTasks:
task()
case code := <-resultCh:
mainThreadTasks = nil
return code
}
}
}
func postMainThreadTask(task func()) bool {
if mainThreadTasks == nil {
return false
}
mainThreadTasks <- task
return true
}
@@ -0,0 +1,11 @@
//go:build !darwin
package goprovider
func RunMainThreadLoop(run func() int) int {
return run()
}
func postMainThreadTask(func()) bool {
return false
}
+608
View File
@@ -0,0 +1,608 @@
//go:build cgo
package goprovider
import (
"encoding/json"
"errors"
"fmt"
"net/url"
"strings"
"sync"
"sync/atomic"
"time"
runtimepkg "runtime"
enginewebview "github.com/GopeedLab/gopeed/pkg/download/engine/webview"
webview "github.com/GopeedLab/webview_go"
)
type Provider struct{}
func New() enginewebview.Provider {
return &Provider{}
}
func (p *Provider) IsAvailable() bool {
return webview.IsAvailable()
}
func (p *Provider) Open(opts enginewebview.OpenOptions) (enginewebview.Page, error) {
pw := newPageWrapper(opts)
if err := pw.start(); err != nil {
return nil, err
}
return pw, nil
}
type pageWrapper struct {
opts enginewebview.OpenOptions
callbackName string
readyID string
view webview.WebView
mu sync.Mutex
pending map[string]chan evalResult
url string
closed bool
ready chan error
done chan struct{}
loads chan string
}
type evalMessage struct {
ID string `json:"id"`
Ready bool `json:"ready,omitempty"`
Load bool `json:"load,omitempty"`
URL string `json:"url,omitempty"`
OK bool `json:"ok,omitempty"`
Value any `json:"value,omitempty"`
Error string `json:"error,omitempty"`
}
type evalResult struct {
value any
err error
}
type nativeCookieJSON struct {
Name string `json:"name"`
Value string `json:"value"`
Domain string `json:"domain"`
Path string `json:"path"`
Expires string `json:"expires"`
Secure bool `json:"secure"`
HTTPOnly bool `json:"httpOnly"`
}
var providerSeq atomic.Uint64
func newPageWrapper(opts enginewebview.OpenOptions) *pageWrapper {
seq := providerSeq.Add(1)
return &pageWrapper{
opts: opts,
callbackName: fmt.Sprintf("__gopeedWebViewCallback_%d", seq),
readyID: fmt.Sprintf("__ready__%d", seq),
pending: make(map[string]chan evalResult),
ready: make(chan error, 1),
done: make(chan struct{}),
loads: make(chan string, 8),
}
}
func (p *pageWrapper) start() error {
startedAt := time.Now()
run := func() {
runtimepkg.LockOSThread()
defer runtimepkg.UnlockOSThread()
var w webview.WebView
if p.opts.Headless {
w = webview.NewHeadless(p.opts.Debug)
} else {
w = webview.New(p.opts.Debug)
}
p.view = w
w.SetTitle(firstNonEmpty(p.opts.Title, "Gopeed WebView"))
w.SetSize(defaultWindowDimension(p.opts.Width, 1280), defaultWindowDimension(p.opts.Height, 800), webview.HintNone)
if p.opts.UserAgent != "" {
type userAgentSetter interface {
SetUserAgent(userAgent string)
}
if setter, ok := w.(userAgentSetter); ok {
setter.SetUserAgent(p.opts.UserAgent)
}
}
w.Init(buildLoadNotifyScript(p.callbackName))
if err := w.Bind(p.callbackName, func(payload string) error {
return p.handleCallback(payload)
}); err != nil {
p.ready <- err
return
}
applyWindowOptions(w, p.opts)
w.SetHtml(buildBootstrapHTML(p.callbackName, p.readyID))
w.Run()
w.Destroy()
close(p.done)
}
if !postMainThreadTask(run) {
go run()
}
select {
case err := <-p.ready:
p.tracef("open %dms", time.Since(startedAt).Milliseconds())
return err
case <-time.After(10 * time.Second):
p.tracef("open timeout %dms", time.Since(startedAt).Milliseconds())
return fmt.Errorf("webview startup timeout")
}
}
func (p *pageWrapper) AddInitScript(script string) error {
return p.dispatch(func(w webview.WebView) error {
w.Init(script)
return nil
})
}
func (p *pageWrapper) Goto(url string, opts enginewebview.GotoOptions) error {
startedAt := time.Now()
p.drainLoads()
if err := p.dispatch(func(w webview.WebView) error {
p.mu.Lock()
p.url = url
p.mu.Unlock()
w.Navigate(url)
return nil
}); err != nil {
return err
}
timeout := 30 * time.Second
if opts.TimeoutMS > 0 {
timeout = time.Duration(opts.TimeoutMS) * time.Millisecond
}
waitUntil := opts.WaitUntil
if waitUntil == "" {
waitUntil = "load"
}
err := p.waitForNavigation(url, timeout, waitUntil)
if err != nil {
p.tracef("goto failed %dms url=%s err=%v", time.Since(startedAt).Milliseconds(), url, err)
return err
}
p.tracef("goto %dms url=%s", time.Since(startedAt).Milliseconds(), url)
return nil
}
func (p *pageWrapper) Execute(expression string, args ...any) (any, error) {
startedAt := time.Now()
if args == nil {
args = []any{}
}
argsJSON, err := json.Marshal(args)
if err != nil {
return nil, err
}
reqID := fmt.Sprintf("req-%d", providerSeq.Add(1))
ch := make(chan evalResult, 1)
p.mu.Lock()
if p.closed {
p.mu.Unlock()
return nil, fmt.Errorf("webview page is closed")
}
p.pending[reqID] = ch
p.mu.Unlock()
script := buildExecuteScript(p.callbackName, reqID, expression, string(argsJSON))
if err := p.dispatch(func(w webview.WebView) error {
w.Eval(script)
return nil
}); err != nil {
p.mu.Lock()
delete(p.pending, reqID)
p.mu.Unlock()
return nil, err
}
result := <-ch
if result.err != nil {
p.tracef("execute failed %dms err=%v", time.Since(startedAt).Milliseconds(), result.err)
return result.value, result.err
}
p.tracef("execute %dms", time.Since(startedAt).Milliseconds())
return result.value, nil
}
func (p *pageWrapper) GetCookies() ([]enginewebview.Cookie, error) {
cookies, err := p.nativeCookies()
if err != nil {
return nil, err
}
result := make([]enginewebview.Cookie, 0, len(cookies))
for _, cookie := range cookies {
item := enginewebview.Cookie{
Name: cookie.Name,
Value: cookie.Value,
Domain: cookie.Domain,
Path: cookie.Path,
Secure: cookie.Secure,
HTTPOnly: cookie.HTTPOnly,
}
if cookie.Expires != "" {
if parsed, err := time.Parse(time.RFC3339Nano, cookie.Expires); err == nil {
item.Expires = parsed
}
}
result = append(result, item)
}
return result, nil
}
func (p *pageWrapper) SetCookie(cookie enginewebview.Cookie) error {
type cookieSetter interface {
SetCookie(cookie webview.Cookie) error
}
return p.dispatch(func(w webview.WebView) error {
setter, ok := w.(cookieSetter)
if !ok {
return fmt.Errorf("native cookie access is unavailable")
}
return setter.SetCookie(webview.Cookie{
Name: cookie.Name,
Value: cookie.Value,
Domain: cookie.Domain,
Path: cookie.Path,
Expires: cookie.Expires,
Secure: cookie.Secure,
HTTPOnly: cookie.HTTPOnly,
})
})
}
func (p *pageWrapper) DeleteCookie(cookie enginewebview.Cookie) error {
type cookieDeleter interface {
DeleteCookie(name string, domain string, path string) error
}
return p.dispatch(func(w webview.WebView) error {
deleter, ok := w.(cookieDeleter)
if !ok {
return fmt.Errorf("native cookie access is unavailable")
}
return deleter.DeleteCookie(cookie.Name, cookie.Domain, cookie.Path)
})
}
func (p *pageWrapper) ClearCookies() error {
type cookieClearer interface {
ClearCookies() error
}
return p.dispatch(func(w webview.WebView) error {
clearer, ok := w.(cookieClearer)
if !ok {
return fmt.Errorf("native cookie access is unavailable")
}
return clearer.ClearCookies()
})
}
func (p *pageWrapper) Close() error {
p.mu.Lock()
if p.closed {
p.mu.Unlock()
return nil
}
p.closed = true
w := p.view
wait := p.done
p.mu.Unlock()
if w != nil {
// PostQuitMessage (used by the Windows WebView2 terminate_impl)
// only takes effect on the thread that owns the message loop.
// Dispatch the termination to the webview's UI thread so that
// the quit message lands on the correct queue.
terminateDone := make(chan struct{})
w.Dispatch(func() {
w.Terminate()
close(terminateDone)
})
<-terminateDone
}
<-wait
return nil
}
func (p *pageWrapper) tracef(format string, args ...any) {
if !strings.Contains(p.opts.Title, "youtube-sabr") {
return
}
fmt.Printf("goprovider "+format+"\n", args...)
}
func (p *pageWrapper) waitForNavigation(targetURL string, timeout time.Duration, waitUntil string) error {
deadline := time.Now().Add(timeout)
ticker := time.NewTicker(100 * time.Millisecond)
defer ticker.Stop()
for {
select {
case <-p.loads:
if waitUntil == "load" {
return nil
}
case <-ticker.C:
state, err := p.navigationState()
if err == nil && state.ready(targetURL, waitUntil) {
return nil
}
if time.Now().After(deadline) {
return fmt.Errorf("webview navigation timeout")
}
}
}
}
type navigationState struct {
URL string `json:"url"`
ReadyState string `json:"readyState"`
}
func (s navigationState) ready(targetURL string, waitUntil string) bool {
if s.URL == "" || s.URL == "about:blank" {
return false
}
switch waitUntil {
case "domcontentloaded":
if s.ReadyState == "loading" || s.ReadyState == "" {
return false
}
default:
if s.ReadyState != "complete" {
return false
}
}
if targetURL == "" {
return true
}
if s.URL == targetURL {
return true
}
target, err := parseURL(targetURL)
if err != nil {
return true
}
current, err := parseURL(s.URL)
if err != nil {
return true
}
if current.Scheme == target.Scheme && current.Host == target.Host && current.Path == target.Path {
return true
}
return false
}
func parseURL(raw string) (*url.URL, error) {
return url.Parse(raw)
}
func (p *pageWrapper) navigationState() (navigationState, error) {
value, err := p.Execute(`() => ({
url: String(location.href || ""),
readyState: document.readyState || "",
})`)
if err != nil {
return navigationState{}, err
}
stateMap, ok := value.(map[string]any)
if !ok {
return navigationState{}, fmt.Errorf("invalid navigation state")
}
state := navigationState{}
if urlValue, ok := stateMap["url"].(string); ok {
state.URL = urlValue
}
if readyValue, ok := stateMap["readyState"].(string); ok {
state.ReadyState = readyValue
}
return state, nil
}
func (p *pageWrapper) dispatch(fn func(w webview.WebView) error) error {
p.mu.Lock()
if p.closed {
p.mu.Unlock()
return fmt.Errorf("webview page is closed")
}
w := p.view
p.mu.Unlock()
if w == nil {
return fmt.Errorf("webview is not initialized")
}
done := make(chan error, 1)
w.Dispatch(func() {
done <- fn(w)
})
return <-done
}
func (p *pageWrapper) nativeCookies() ([]nativeCookieJSON, error) {
type cookieGetter interface {
GetCookies(url string) ([]webview.Cookie, error)
}
var cookies []webview.Cookie
err := p.dispatch(func(w webview.WebView) error {
getter, ok := w.(cookieGetter)
if !ok {
return fmt.Errorf("native cookie access is unavailable")
}
p.mu.Lock()
url := p.url
p.mu.Unlock()
if url == "" {
url = "about:blank"
}
var err error
cookies, err = getter.GetCookies(url)
return err
})
if err != nil {
return nil, err
}
result := make([]nativeCookieJSON, 0, len(cookies))
for _, cookie := range cookies {
item := nativeCookieJSON{
Name: cookie.Name,
Value: cookie.Value,
Domain: cookie.Domain,
Path: cookie.Path,
Secure: cookie.Secure,
HTTPOnly: cookie.HTTPOnly,
}
if !cookie.Expires.IsZero() {
item.Expires = cookie.Expires.UTC().Format(time.RFC3339Nano)
}
result = append(result, item)
}
return result, nil
}
func (p *pageWrapper) handleCallback(payload string) error {
var msg evalMessage
if err := json.Unmarshal([]byte(payload), &msg); err != nil {
return err
}
if msg.Ready && msg.ID == p.readyID {
select {
case p.ready <- nil:
default:
}
return nil
}
if msg.Load {
select {
case p.loads <- msg.URL:
default:
}
return nil
}
p.mu.Lock()
ch := p.pending[msg.ID]
delete(p.pending, msg.ID)
p.mu.Unlock()
if ch == nil {
return nil
}
if !msg.OK {
ch <- evalResult{err: errors.New(strings.TrimSpace(msg.Error))}
return nil
}
ch <- evalResult{value: msg.Value}
return nil
}
func (p *pageWrapper) drainLoads() {
for {
select {
case <-p.loads:
default:
return
}
}
}
func buildBootstrapHTML(callbackName string, readyID string) string {
return fmt.Sprintf(`<!doctype html>
<html>
<head><meta charset="utf-8"></head>
<body>
<script>
window.addEventListener("load", () => {
const callback = globalThis[%q];
if (typeof callback === "function") {
callback(JSON.stringify({ id: %q, ready: true }));
}
}, { once: true });
</script>
</body>
</html>`, callbackName, readyID)
}
func buildExecuteScript(callbackName string, reqID string, expression string, argsJSON string) string {
callbackJSON, _ := json.Marshal(callbackName)
reqJSON, _ := json.Marshal(reqID)
exprJSON, _ := json.Marshal(expression)
argsLiteral := argsJSON
if argsLiteral == "" || argsLiteral == "null" {
argsLiteral = "[]"
}
return fmt.Sprintf(`(() => {
const __cb = globalThis[%s];
const __id = %s;
const __expression = %s;
const __args = %s;
Promise.resolve()
.then(async () => {
const __target = (0, eval)(__expression);
const __value = await (typeof __target === "function" ? __target(...__args) : __target);
return { id: __id, ok: true, value: __value ?? null };
})
.catch((error) => ({
id: __id,
ok: false,
error: [
error && error.name ? error.name : "Error",
error && error.message ? error.message : String(error),
error && error.stack ? error.stack : "",
].filter(Boolean).join(": ").replace(": @", "\n@"),
}))
.then((message) => __cb(JSON.stringify(message)));
})();`, string(callbackJSON), string(reqJSON), string(exprJSON), argsLiteral)
}
func buildLoadNotifyScript(callbackName string) string {
callbackJSON, _ := json.Marshal(callbackName)
return fmt.Sprintf(`(() => {
window.addEventListener("load", () => {
const __cb = globalThis[%s];
if (typeof __cb !== "function") {
return;
}
__cb(JSON.stringify({
load: true,
url: String(location.href || ""),
}));
}, { once: true });
})();`, string(callbackJSON))
}
func defaultWindowDimension(value int, fallback int) int {
if value > 0 {
return value
}
return fallback
}
func firstNonEmpty(values ...string) string {
for _, value := range values {
if strings.TrimSpace(value) != "" {
return value
}
}
return ""
}
@@ -0,0 +1,15 @@
//go:build cgo && webview && !darwin
package goprovider
import (
"testing"
"github.com/GopeedLab/gopeed/internal/webview/integrationtest"
)
func TestProviderContract(t *testing.T) {
integrationtest.RunProviderContract(t, New(), integrationtest.ContractOptions{
CookieDomainMode: integrationtest.CookieDomainModeRequired,
})
}
@@ -0,0 +1,9 @@
//go:build !cgo
package goprovider
import enginewebview "github.com/GopeedLab/gopeed/pkg/download/engine/webview"
func New() enginewebview.Provider {
return enginewebview.NewUnavailableProvider()
}
@@ -0,0 +1,38 @@
//go:build cgo && darwin
package goprovider
/*
#cgo darwin CFLAGS: -x objective-c
#cgo darwin LDFLAGS: -framework Cocoa
#include <Cocoa/Cocoa.h>
static void gopeedApplyHeadlessWindowOptions(void *windowPtr, int headless) {
if (windowPtr == NULL) {
return;
}
NSWindow *window = (__bridge NSWindow *)windowPtr;
if (headless) {
[window orderOut:nil];
}
}
*/
import "C"
import (
enginewebview "github.com/GopeedLab/gopeed/pkg/download/engine/webview"
webview "github.com/GopeedLab/webview_go"
)
func applyWindowOptions(w webview.WebView, opts enginewebview.OpenOptions) {
C.gopeedApplyHeadlessWindowOptions(w.Window(), C.int(boolToInt(opts.Headless)))
}
func boolToInt(value bool) int {
if value {
return 1
}
return 0
}
@@ -0,0 +1,10 @@
//go:build cgo && !darwin
package goprovider
import (
enginewebview "github.com/GopeedLab/gopeed/pkg/download/engine/webview"
webview "github.com/GopeedLab/webview_go"
)
func applyWindowOptions(_ webview.WebView, _ enginewebview.OpenOptions) {}
@@ -0,0 +1,378 @@
package integrationtest
import (
"fmt"
"net/http"
"net/http/httptest"
"net/url"
"strings"
"testing"
"time"
enginewebview "github.com/GopeedLab/gopeed/pkg/download/engine/webview"
)
const TestUserAgent = "GopeedWebViewIntegration/1.0"
type ContractOptions struct {
AvailabilityTimeout time.Duration
CookieDomainMode CookieDomainMode
CookieTestURL string
}
type CookieDomainMode int
const (
CookieDomainModeRequired CookieDomainMode = iota
CookieDomainModeOmit
)
func RunProviderContract(t *testing.T, provider enginewebview.Provider, opts ContractOptions) {
t.Helper()
timeout := opts.AvailabilityTimeout
if timeout <= 0 {
timeout = 10 * time.Second
}
waitForProviderAvailable(t, provider, timeout)
page, server, testURL := openTestPage(t, provider, enginewebview.OpenOptions{
Headless: true,
Title: "Gopeed WebView Integration",
Width: 640,
Height: 480,
UserAgent: TestUserAgent,
})
if err := page.Goto(testURL, map[string]any{"timeoutMs": 10000}); err != nil {
t.Fatalf("goto failed: %v", err)
}
ua, err := page.Execute(`() => navigator.userAgent`)
if err != nil {
t.Fatalf("navigator.userAgent execute failed: %v", err)
}
if ua != TestUserAgent {
t.Fatalf("unexpected user agent: got %v want %q", ua, TestUserAgent)
}
if err := page.Focus("#name"); err != nil {
t.Fatalf("focus failed: %v", err)
}
activeElement, err := page.Execute(`() => document.activeElement && document.activeElement.id`)
if err != nil {
t.Fatalf("activeElement execute failed: %v", err)
}
if activeElement != "name" {
t.Fatalf("expected active element to be #name, got %v", activeElement)
}
if err := page.Type("#name", "Gopeed", map[string]any{"delay": 5}); err != nil {
t.Fatalf("type failed: %v", err)
}
typeState, err := page.Execute(`() => ({
value: document.querySelector('#name').value,
mirror: document.querySelector('#name-value').textContent,
inputCount: window.__inputEvents.length
})`)
if err != nil {
t.Fatalf("type state execute failed: %v", err)
}
stateMap, ok := typeState.(map[string]any)
if !ok {
t.Fatalf("unexpected type state payload: %T", typeState)
}
if stateMap["value"] != "Gopeed" || stateMap["mirror"] != "Gopeed" {
t.Fatalf("unexpected type state: %#v", stateMap)
}
if stateMap["inputCount"] != float64(len("Gopeed")) {
t.Fatalf("unexpected input event count: %#v", stateMap["inputCount"])
}
if err := page.Click("#clicker"); err != nil {
t.Fatalf("click failed: %v", err)
}
clickCount, err := page.Execute(`() => document.querySelector('#click-count').textContent`)
if err != nil {
t.Fatalf("click count execute failed: %v", err)
}
if clickCount != "1" {
t.Fatalf("expected click count to be 1, got %v", clickCount)
}
found, err := page.WaitForSelector("#delayed", map[string]any{
"timeoutMs": 5000,
"pollIntervalMs": 25,
"visible": true,
})
if err != nil {
t.Fatalf("waitForSelector failed: %v", err)
}
if !found {
t.Fatal("expected delayed selector to appear")
}
value, err := page.WaitForFunction(`() => window.__readyValue`, map[string]any{
"timeoutMs": 5000,
"pollIntervalMs": 25,
})
if err != nil {
t.Fatalf("waitForFunction failed: %v", err)
}
readyValue, ok := value.(map[string]any)
if !ok {
t.Fatalf("unexpected waitForFunction value: %T", value)
}
if readyValue["status"] != "ready" {
t.Fatalf("unexpected waitForFunction payload: %#v", readyValue)
}
found, err = page.WaitForSelector("#missing", map[string]any{
"timeoutMs": 200,
"pollIntervalMs": 25,
})
if err != nil {
t.Fatalf("waitForSelector timeout case failed: %v", err)
}
if found {
t.Fatal("expected missing selector wait to time out")
}
cookieTestURL := opts.CookieTestURL
if cookieTestURL == "" {
cookieTestURL = testURL
}
runCookieLifecycleAssertions(t, page, server, cookieTestURL, opts.CookieDomainMode)
}
func waitForProviderAvailable(t *testing.T, provider enginewebview.Provider, timeout time.Duration) {
t.Helper()
deadline := time.Now().Add(timeout)
for time.Now().Before(deadline) {
if provider != nil && provider.IsAvailable() {
return
}
time.Sleep(200 * time.Millisecond)
}
t.Fatalf("provider did not become available within %s", timeout)
}
func openTestPage(t *testing.T, provider enginewebview.Provider, opts enginewebview.OpenOptions) (*enginewebview.PageHandle, *httptest.Server, string) {
t.Helper()
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/html; charset=utf-8")
switch r.URL.Path {
case "/next":
fmt.Fprint(w, `<!DOCTYPE html>
<html>
<head><title>Next Page</title></head>
<body>
<h1 id="next-page">Navigation complete</h1>
</body>
</html>`)
default:
fmt.Fprint(w, `<!DOCTYPE html>
<html>
<head><title>Test Page</title></head>
<body>
<input id="name" type="text" />
<div id="name-value"></div>
<button id="clicker" type="button">Click me</button>
<div id="click-count">0</div>
<button id="nav-btn" type="button">Navigate</button>
<script>
window.__inputEvents = [];
window.__readyValue = null;
const nameInput = document.getElementById('name');
const nameValue = document.getElementById('name-value');
nameInput.addEventListener('input', () => {
nameValue.textContent = nameInput.value;
window.__inputEvents.push(nameInput.value);
});
document.getElementById('clicker').addEventListener('click', () => {
const counter = document.getElementById('click-count');
counter.textContent = String(Number(counter.textContent || '0') + 1);
});
document.getElementById('nav-btn').addEventListener('click', () => {
setTimeout(() => {
window.location.href = '/next';
}, 50);
});
setTimeout(() => {
const marker = document.createElement('div');
marker.id = 'delayed';
marker.textContent = 'Loaded later';
document.body.appendChild(marker);
}, 150);
setTimeout(() => {
window.__readyValue = { status: 'ready', source: 'timer' };
}, 200);
</script>
</body>
</html>`)
}
}))
t.Cleanup(server.Close)
runtime := enginewebview.NewRuntime(provider, true)
page, err := runtime.Open(map[string]any{
"headless": opts.Headless,
"debug": opts.Debug,
"title": opts.Title,
"width": opts.Width,
"height": opts.Height,
"userAgent": opts.UserAgent,
})
if err != nil {
t.Fatalf("failed to open webview: %v", err)
}
t.Cleanup(func() {
_ = runtime.Close()
})
return page, server, server.URL
}
func runCookieLifecycleAssertions(t *testing.T, page *enginewebview.PageHandle, _ *httptest.Server, testURL string, domainMode CookieDomainMode) {
t.Helper()
parsedURL, err := url.Parse(testURL)
if err != nil {
t.Fatalf("parse cookie test url failed: %v", err)
}
cookieDomain := parsedURL.Hostname()
if err := page.ClearCookies(); err != nil {
t.Fatalf("clearCookies before cookie assertions failed: %v", err)
}
visibleCookie := map[string]any{
"name": "visible",
"value": "1",
"path": "/",
}
secretCookie := map[string]any{
"name": "secret",
"value": "2",
"path": "/",
"httpOnly": true,
"expires": time.Now().Add(time.Hour).UTC().Format(time.RFC3339Nano),
}
deleteCookie := map[string]any{
"name": "visible",
"path": "/",
}
if domainMode == CookieDomainModeRequired {
visibleCookie["domain"] = cookieDomain
secretCookie["domain"] = cookieDomain
deleteCookie["domain"] = cookieDomain
}
if err := page.SetCookie(visibleCookie); err != nil {
t.Fatalf("set visible cookie failed: %v", err)
}
if err := page.SetCookie(secretCookie); err != nil {
t.Fatalf("set secret cookie failed: %v", err)
}
if err := page.Goto(testURL, map[string]any{"timeoutMs": 10000}); err != nil {
t.Fatalf("goto after setCookie failed: %v", err)
}
documentCookie, err := page.Execute(`() => document.cookie`)
if err != nil {
t.Fatalf("document.cookie execute failed: %v", err)
}
documentCookieText, ok := documentCookie.(string)
if !ok {
t.Fatalf("unexpected document.cookie payload: %T", documentCookie)
}
if !strings.Contains(documentCookieText, "visible=1") {
t.Fatalf("expected visible cookie in document.cookie, got %q", documentCookieText)
}
if strings.Contains(documentCookieText, "secret=2") {
t.Fatalf("expected httpOnly cookie to be hidden from document.cookie, got %q", documentCookieText)
}
cookies, err := page.GetCookies()
if err != nil {
t.Fatalf("getCookies failed: %v", err)
}
if len(cookies) < 2 {
t.Fatalf("expected at least 2 cookies, got %#v", cookies)
}
if !hasCookie(cookies, "visible", "1") {
t.Fatalf("expected visible cookie in native cookie store, got %#v", cookies)
}
if !hasCookie(cookies, "secret", "2") {
t.Fatalf("expected secret cookie in native cookie store, got %#v", cookies)
}
if err := page.DeleteCookie(deleteCookie); err != nil {
t.Fatalf("deleteCookie failed: %v", err)
}
if err := page.Goto(testURL, map[string]any{"timeoutMs": 10000}); err != nil {
t.Fatalf("goto after deleteCookie failed: %v", err)
}
documentCookie, err = page.Execute(`() => document.cookie`)
if err != nil {
t.Fatalf("document.cookie execute after delete failed: %v", err)
}
documentCookieText, ok = documentCookie.(string)
if !ok {
t.Fatalf("unexpected document.cookie payload after delete: %T", documentCookie)
}
if strings.Contains(documentCookieText, "visible=1") {
t.Fatalf("expected visible cookie to be deleted, got %q", documentCookieText)
}
if strings.Contains(documentCookieText, "secret=2") {
t.Fatalf("expected httpOnly cookie to remain hidden after delete, got %q", documentCookieText)
}
cookies, err = page.GetCookies()
if err != nil {
t.Fatalf("getCookies after delete failed: %v", err)
}
if hasCookie(cookies, "visible", "1") {
t.Fatalf("expected visible cookie to be removed from native store, got %#v", cookies)
}
if !hasCookie(cookies, "secret", "2") {
t.Fatalf("expected secret cookie to remain in native store after delete, got %#v", cookies)
}
if err := page.ClearCookies(); err != nil {
t.Fatalf("clearCookies failed: %v", err)
}
if err := page.Goto(testURL, map[string]any{"timeoutMs": 10000}); err != nil {
t.Fatalf("goto after clearCookies failed: %v", err)
}
documentCookie, err = page.Execute(`() => document.cookie`)
if err != nil {
t.Fatalf("document.cookie execute after clear failed: %v", err)
}
documentCookieText, ok = documentCookie.(string)
if !ok {
t.Fatalf("unexpected document.cookie payload after clear: %T", documentCookie)
}
if documentCookieText != "" {
t.Fatalf("expected document.cookie to be empty after clear, got %q", documentCookieText)
}
cookies, err = page.GetCookies()
if err != nil {
t.Fatalf("getCookies after clear failed: %v", err)
}
if len(cookies) != 0 {
t.Fatalf("expected cookie store to be empty after clear, got %#v", cookies)
}
}
func hasCookie(cookies []enginewebview.Cookie, name string, value string) bool {
for _, cookie := range cookies {
if cookie.Name == name && cookie.Value == value {
return true
}
}
return false
}
+488
View File
@@ -0,0 +1,488 @@
# rpcprovider
`rpcprovider` is the RPC-backed implementation of Gopeed's `webview.Provider` interface.
It is intended for environments where WebView capability is owned by another process, most notably mobile hosts. This package does not implement a WebView itself. Instead, it translates the `pkg/download/engine/webview` page API into single-request `POST + JSON` calls sent to a local host-side RPC service.
## When to use it
Use `rpcprovider` for entrypoints such as `bind/mobile`, where the Go process cannot directly own a native WebView.
Do not use it for desktop or `cmd/web` local WebView integration. Those entrypoints should use `internal/webview/goprovider` instead.
## Configuration
The provider is configured through `pkg/download/engine/webview.RPCConfig`:
```go
type RPCConfig struct {
Network string `json:"network"`
Address string `json:"address"`
Token string `json:"token,omitempty"`
}
```
Notes:
- `Network` currently supports `tcp` and `unix`.
- `Address` examples:
- `127.0.0.1:38765`
- `/path/to/webview.sock`
- `Token` is optional. When set, the client sends `Authorization: Bearer <token>`.
Example:
```go
provider := rpcprovider.New(webview.RPCConfig{
Network: "tcp",
Address: "127.0.0.1:38765",
Token: "secret-token",
})
```
## Transport
- Endpoint: `POST /webview`
- Content type: `application/json`
- One request, one response
- No `id` field yet
- The envelope is intentionally close to JSON-RPC so it can evolve later without renaming the methods or reshaping the payloads
Request example:
```json
{
"method": "page.execute",
"params": {
"pageId": "page-1",
"expression": "document.title",
"args": []
}
}
```
Successful response:
```json
{
"result": {
"title": "Example"
},
"error": null
}
```
Failure response:
```json
{
"result": null,
"error": {
"code": "PAGE_NOT_FOUND",
"message": "page not found"
}
}
```
## Supported methods
- `webview.isAvailable`
- `page.open`
- `page.addInitScript`
- `page.goto`
- `page.execute`
- `page.getCookies`
- `page.setCookie`
- `page.deleteCookie`
- `page.clearCookies`
- `page.close`
The canonical wire models for these methods are defined in:
- [pkg/download/engine/webview/rpc.go](../../../pkg/download/engine/webview/rpc.go)
- [pkg/download/engine/webview/runtime.go](../../../pkg/download/engine/webview/runtime.go)
The README is only a guide. The Go source above is the authoritative protocol reference.
## Method reference
Each method below uses the common envelope:
```json
{
"method": "<method-name>",
"params": { ... }
}
```
and returns:
```json
{
"result": ...,
"error": null
}
```
or:
```json
{
"result": null,
"error": {
"code": "<ERROR_CODE>",
"message": "<message>"
}
}
```
### `webview.isAvailable`
Request:
```json
{
"method": "webview.isAvailable",
"params": {}
}
```
Success:
```json
{
"result": {
"available": true
},
"error": null
}
```
### `page.open`
Request:
```json
{
"method": "page.open",
"params": {
"headless": true,
"debug": false,
"title": "Gopeed WebView",
"width": 1280,
"height": 720,
"userAgent": "Mozilla/5.0 ..."
}
}
```
Success:
```json
{
"result": {
"pageId": "page-1"
},
"error": null
}
```
### `page.addInitScript`
Request:
```json
{
"method": "page.addInitScript",
"params": {
"pageId": "page-1",
"script": "window.__READY__ = true;"
}
}
```
Success:
```json
{
"result": {},
"error": null
}
```
### `page.goto`
Request:
```json
{
"method": "page.goto",
"params": {
"pageId": "page-1",
"url": "https://example.com",
"timeoutMs": 15000,
"waitUntil": "domcontentloaded"
}
}
```
Success:
```json
{
"result": {},
"error": null
}
```
`waitUntil` currently supports `load` and `domcontentloaded`. The default is `load`.
### `page.execute`
Request:
```json
{
"method": "page.execute",
"params": {
"pageId": "page-1",
"expression": "document.title",
"args": []
}
}
```
Success:
```json
{
"result": "Example Domain",
"error": null
}
```
### `page.getCookies`
Request:
```json
{
"method": "page.getCookies",
"params": {
"pageId": "page-1"
}
}
```
Success:
```json
{
"result": [
{
"name": "session",
"value": "abc",
"domain": ".example.com",
"path": "/",
"expires": "2026-03-26T12:00:00Z",
"secure": true,
"httpOnly": true
}
],
"error": null
}
```
### `page.setCookie`
Request:
```json
{
"method": "page.setCookie",
"params": {
"pageId": "page-1",
"cookie": {
"name": "session",
"value": "abc",
"domain": ".example.com",
"path": "/",
"expires": "2026-03-26T12:00:00Z",
"secure": true,
"httpOnly": true
}
}
}
```
Success:
```json
{
"result": {},
"error": null
}
```
### `page.deleteCookie`
Request:
```json
{
"method": "page.deleteCookie",
"params": {
"pageId": "page-1",
"cookie": {
"name": "session",
"domain": ".example.com",
"path": "/"
}
}
}
```
Success:
```json
{
"result": {},
"error": null
}
```
### `page.clearCookies`
Request:
```json
{
"method": "page.clearCookies",
"params": {
"pageId": "page-1"
}
}
```
Success:
```json
{
"result": {},
"error": null
}
```
### `page.close`
Request:
```json
{
"method": "page.close",
"params": {
"pageId": "page-1"
}
}
```
Success:
```json
{
"result": {},
"error": null
}
```
## Method semantics
- `page.open`
- Creates a page session
- Returns `pageId`
- `page.addInitScript`
- Registers a page initialization script
- `page.goto`
- Goes to the target URL
- The server should return only after navigation completes or fails
- `page.execute`
- Executes an expression or serialized function body
- The returned value must be JSON-serializable
- `page.getCookies`
- Returns cookies from the underlying native WebView cookie store
- `page.setCookie`
- Inserts or updates a cookie
- `page.deleteCookie`
- Deletes a cookie using `name/domain/path`
- `page.clearCookies`
- Clears the current cookie store
- `page.close`
- Closes the page and releases related resources
## Cookie model
Cookies use `runtime/webview.Cookie`.
Example payload:
```json
{
"name": "session",
"value": "abc",
"domain": ".example.com",
"path": "/",
"expires": "2026-03-26T12:00:00Z",
"secure": true,
"httpOnly": true
}
```
Notes:
- `expires` uses RFC3339 / RFC3339Nano string form.
- If the host platform supports a full native cookie store, it should return fields such as `httpOnly` and `secure`.
- `setCookie -> goto` is a valid pattern for preloading authenticated session state.
## Error codes
The current error code set is defined in `pkg/download/engine/webview/rpc.go`:
- `INVALID_REQUEST`
- `UNKNOWN_METHOD`
- `UNAVAILABLE`
- `BROWSER_NOT_FOUND`
- `PAGE_NOT_FOUND`
- `NAVIGATION_FAILED`
- `EVALUATION_FAILED`
- `TIMEOUT`
- `INTERNAL_ERROR`
`BROWSER_NOT_FOUND` is a historical leftover from an earlier shape of the protocol. The current protocol no longer has a separate browser model, so this can be cleaned up later if the protocol is tightened further.
## Current protocol boundary
This protocol is intentionally limited:
- Single-direction request/response only
- Not REST-style resource routing
- No bidirectional events
- No streaming event subscriptions
- No host-initiated push channel
If the protocol needs to evolve toward full JSON-RPC later, the intended migration path is to keep the existing `method/params/result/error` structure and add:
- `id`
- notifications
- a bidirectional transport model
## Host implementation guidance
A host implementation should usually:
- Listen on a local `tcp` address or `unix socket`
- Expose `POST /webview`
- Validate the bearer token if configured
- Maintain `pageId -> native webview/session`
- Own page lifecycle, cleanup, and timeout handling
With that split, Gopeed stays a pure RPC client and does not need to know how the host's native WebView is implemented.
+195
View File
@@ -0,0 +1,195 @@
package rpcprovider
import (
"bytes"
"context"
"encoding/json"
"fmt"
"net"
"net/http"
"strings"
enginewebview "github.com/GopeedLab/gopeed/pkg/download/engine/webview"
)
type Provider struct {
client *Client
}
func New(cfg enginewebview.RPCConfig) enginewebview.Provider {
return &Provider{
client: NewClient(cfg),
}
}
func (p *Provider) IsAvailable() bool {
if p == nil || p.client == nil || !p.client.Enabled() {
return false
}
var result enginewebview.IsAvailableResult
if err := p.client.Call(enginewebview.MethodIsAvailable, enginewebview.IsAvailableParams{}, &result); err != nil {
return false
}
return result.Available
}
func (p *Provider) Open(opts enginewebview.OpenOptions) (enginewebview.Page, error) {
if p == nil || p.client == nil || !p.IsAvailable() {
return nil, enginewebview.ErrUnavailable
}
var result enginewebview.PageOpenResult
if err := p.client.Call(enginewebview.MethodPageOpen, enginewebview.NewPageOpenParams(opts), &result); err != nil {
return nil, err
}
return &page{
client: p.client,
id: result.PageID,
}, nil
}
type page struct {
client *Client
id string
}
func (p *page) AddInitScript(script string) error {
return p.client.Call(enginewebview.MethodPageAddInitScript, enginewebview.PageAddInitScriptParams{
PageID: p.id,
Script: script,
}, nil)
}
func (p *page) Goto(url string, opts enginewebview.GotoOptions) error {
return p.client.Call(enginewebview.MethodPageGoto, enginewebview.NewPageGotoParams(p.id, url, opts), nil)
}
func (p *page) Execute(expression string, args ...any) (any, error) {
var result any
if err := p.client.Call(enginewebview.MethodPageExecute, enginewebview.PageExecuteParams{
PageID: p.id,
Expression: expression,
Args: args,
}, &result); err != nil {
return nil, err
}
return result, nil
}
func (p *page) GetCookies() ([]enginewebview.Cookie, error) {
var result []enginewebview.Cookie
if err := p.client.Call(enginewebview.MethodPageGetCookies, enginewebview.PageGetCookiesParams{
PageID: p.id,
}, &result); err != nil {
return nil, err
}
if result == nil {
return []enginewebview.Cookie{}, nil
}
return result, nil
}
func (p *page) SetCookie(cookie enginewebview.Cookie) error {
return p.client.Call(enginewebview.MethodPageSetCookie, enginewebview.PageSetCookieParams{
PageID: p.id,
Cookie: cookie,
}, nil)
}
func (p *page) DeleteCookie(cookie enginewebview.Cookie) error {
return p.client.Call(enginewebview.MethodPageDeleteCookie, enginewebview.PageDeleteCookieParams{
PageID: p.id,
Cookie: cookie,
}, nil)
}
func (p *page) ClearCookies() error {
return p.client.Call(enginewebview.MethodPageClearCookies, enginewebview.PageClearCookiesParams{
PageID: p.id,
}, nil)
}
func (p *page) Close() error {
return p.client.Call(enginewebview.MethodPageClose, enginewebview.PageCloseParams{
PageID: p.id,
}, nil)
}
type Client struct {
endpoint string
token string
http *http.Client
}
func NewClient(cfg enginewebview.RPCConfig) *Client {
client := &Client{token: cfg.Token}
if !cfg.Enabled() {
return client
}
switch strings.ToLower(cfg.Network) {
case "unix":
client.endpoint = "http://unix" + enginewebview.RPCEndpointPath
client.http = &http.Client{
Transport: &http.Transport{
DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) {
var d net.Dialer
return d.DialContext(ctx, "unix", cfg.Address)
},
},
}
default:
client.endpoint = "http://" + cfg.Address + enginewebview.RPCEndpointPath
client.http = &http.Client{}
}
return client
}
func (c *Client) Enabled() bool {
return c != nil && c.endpoint != "" && c.http != nil
}
func (c *Client) Call(method string, params any, result any) error {
if !c.Enabled() {
return enginewebview.ErrUnavailable
}
payload, err := json.Marshal(enginewebview.RPCRequest{
Method: method,
Params: params,
})
if err != nil {
return err
}
req, err := http.NewRequest(http.MethodPost, c.endpoint, bytes.NewReader(payload))
if err != nil {
return err
}
req.Header.Set("Content-Type", "application/json")
if c.token != "" {
req.Header.Set("Authorization", "Bearer "+c.token)
}
resp, err := c.http.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("webview rpc status %d", resp.StatusCode)
}
var body enginewebview.RPCResponse
if err := json.NewDecoder(resp.Body).Decode(&body); err != nil {
return err
}
if body.Error != nil {
return body.Error
}
if result == nil {
return nil
}
if len(body.Result) == 0 || string(body.Result) == "null" {
return nil
}
return json.Unmarshal(body.Result, result)
}
@@ -0,0 +1,41 @@
//go:build webview && darwin
package rpcprovider
import (
"flag"
"os"
"path/filepath"
"testing"
"time"
"github.com/GopeedLab/gopeed/internal/webview/integrationtest"
enginewebview "github.com/GopeedLab/gopeed/pkg/download/engine/webview"
)
var (
webViewRPCNetwork = flag.String("webview-rpc-network", "unix", "WebView RPC network for integration tests")
webViewRPCAddress = flag.String("webview-rpc-address", defaultWebViewRPCAddress(), "WebView RPC address for integration tests")
webViewRPCToken = flag.String("webview-rpc-token", "", "WebView RPC token for integration tests")
)
func TestProviderContract(t *testing.T) {
provider := New(enginewebview.RPCConfig{
Network: *webViewRPCNetwork,
Address: *webViewRPCAddress,
Token: *webViewRPCToken,
})
integrationtest.RunProviderContract(t, provider, integrationtest.ContractOptions{
AvailabilityTimeout: 5 * time.Minute,
CookieDomainMode: integrationtest.CookieDomainModeRequired,
CookieTestURL: "https://example.com/",
})
}
func defaultWebViewRPCAddress() string {
homeDir, err := os.UserHomeDir()
if err != nil {
return ""
}
return filepath.Join(homeDir, "Library", "Application Support", "com.gopeed.gopeed", "gopeed_webview.sock")
}
@@ -0,0 +1,50 @@
package rpcprovider
import (
"encoding/json"
"net"
"net/http"
"os"
"path/filepath"
"testing"
enginewebview "github.com/GopeedLab/gopeed/pkg/download/engine/webview"
)
func TestProviderIsAvailableOverUnixSocket(t *testing.T) {
socketPath := filepath.Join("/tmp", "gopeed-rpcwebview-"+t.Name()+".sock")
listener, err := net.Listen("unix", socketPath)
if err != nil {
t.Fatal(err)
}
defer listener.Close()
defer os.Remove(socketPath)
server := &http.Server{
Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
var req enginewebview.RPCRequest
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
t.Fatal(err)
}
if req.Method != enginewebview.MethodIsAvailable {
t.Fatalf("unexpected unix method: %s", req.Method)
}
if err := json.NewEncoder(w).Encode(map[string]any{
"result": enginewebview.IsAvailableResult{Available: true},
"error": nil,
}); err != nil {
t.Fatal(err)
}
}),
}
defer server.Close()
go server.Serve(listener)
provider := New(enginewebview.RPCConfig{
Network: "unix",
Address: socketPath,
})
if !provider.IsAvailable() {
t.Fatal("expected unix provider to be available")
}
}
@@ -0,0 +1,186 @@
package rpcprovider
import (
"encoding/json"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
enginewebview "github.com/GopeedLab/gopeed/pkg/download/engine/webview"
)
func TestProviderLifecycleOverHTTP(t *testing.T) {
var calls []enginewebview.RPCRequest
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost {
t.Fatalf("unexpected method: %s", r.Method)
}
if r.URL.Path != enginewebview.RPCEndpointPath {
t.Fatalf("unexpected path: %s", r.URL.Path)
}
if got := r.Header.Get("Authorization"); got != "Bearer secret-token" {
t.Fatalf("unexpected auth header: %q", got)
}
var req enginewebview.RPCRequest
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
t.Fatal(err)
}
calls = append(calls, req)
var body any
switch req.Method {
case enginewebview.MethodIsAvailable:
body = map[string]any{
"result": enginewebview.IsAvailableResult{Available: true},
"error": nil,
}
case enginewebview.MethodPageOpen:
body = map[string]any{
"result": enginewebview.PageOpenResult{PageID: "page-open-1"},
"error": nil,
}
case enginewebview.MethodPageAddInitScript, enginewebview.MethodPageGoto:
body = map[string]any{
"result": enginewebview.EmptyResult{},
"error": nil,
}
case enginewebview.MethodPageExecute:
body = map[string]any{
"result": map[string]any{"title": "Example"},
"error": nil,
}
case enginewebview.MethodPageClose:
body = map[string]any{
"result": enginewebview.EmptyResult{},
"error": nil,
}
default:
t.Fatalf("unexpected method: %s", req.Method)
}
if err := json.NewEncoder(w).Encode(body); err != nil {
t.Fatal(err)
}
}))
defer server.Close()
cfg := enginewebview.RPCConfig{
Network: "tcp",
Address: server.Listener.Addr().String(),
Token: "secret-token",
}
provider := New(cfg)
if !provider.IsAvailable() {
t.Fatal("expected provider to be available")
}
opener, ok := provider.(enginewebview.Opener)
if !ok {
t.Fatal("expected provider to implement runtime webview opener")
}
opened, err := opener.Open(enginewebview.OpenOptions{
Headless: true,
Width: 1280,
Height: 720,
Title: "Gopeed",
UserAgent: "UA",
})
if err != nil {
t.Fatal(err)
}
if err := opened.AddInitScript("window.__TEST__ = true"); err != nil {
t.Fatal(err)
}
if err := opened.Goto("https://example.com", enginewebview.GotoOptions{
TimeoutMS: 3210,
WaitUntil: "domcontentloaded",
}); err != nil {
t.Fatal(err)
}
value, err := opened.Execute("document.title")
if err != nil {
t.Fatal(err)
}
got, ok := value.(map[string]any)
if !ok || got["title"] != "Example" {
t.Fatalf("unexpected evaluate result: %#v", value)
}
if err := opened.Close(); err != nil {
t.Fatal(err)
}
expectedMethods := []string{
enginewebview.MethodIsAvailable,
enginewebview.MethodIsAvailable,
enginewebview.MethodPageOpen,
enginewebview.MethodPageAddInitScript,
enginewebview.MethodPageGoto,
enginewebview.MethodPageExecute,
enginewebview.MethodPageClose,
}
if len(calls) != len(expectedMethods) {
t.Fatalf("unexpected call count: got %d want %d", len(calls), len(expectedMethods))
}
for i, method := range expectedMethods {
if calls[i].Method != method {
t.Fatalf("unexpected method at %d: got %s want %s", i, calls[i].Method, method)
}
}
gotoParamsJSON, err := json.Marshal(calls[4].Params)
if err != nil {
t.Fatal(err)
}
var gotoParams enginewebview.PageGotoParams
if err := json.Unmarshal(gotoParamsJSON, &gotoParams); err != nil {
t.Fatal(err)
}
if gotoParams.WaitUntil != "domcontentloaded" {
t.Fatalf("unexpected waitUntil: %q", gotoParams.WaitUntil)
}
}
func TestProviderLaunchUnavailableWithoutEndpoint(t *testing.T) {
provider := New(enginewebview.RPCConfig{})
if provider.IsAvailable() {
t.Fatal("expected empty provider to be unavailable")
}
opener, ok := provider.(enginewebview.Opener)
if !ok {
t.Fatal("expected provider to implement runtime webview opener")
}
if _, err := opener.Open(enginewebview.OpenOptions{}); err == nil || err.Error() != enginewebview.ErrUnavailable.Error() {
t.Fatalf("unexpected launch error: %v", err)
}
}
func TestPageSetCookieParamsOmitsZeroExpires(t *testing.T) {
params := enginewebview.PageSetCookieParams{
PageID: "page-1",
Cookie: enginewebview.Cookie{
Name: "visible",
Value: "1",
Domain: "example.com",
Path: "/",
},
}
payload, err := json.Marshal(params)
if err != nil {
t.Fatal(err)
}
if string(payload) == "" {
t.Fatal("expected non-empty json payload")
}
if got := string(payload); strings.Contains(got, `"expires"`) {
t.Fatalf("unexpected zero expires in payload: %s", got)
}
params.Cookie.Expires = time.Now().UTC().Truncate(time.Second)
payload, err = json.Marshal(params)
if err != nil {
t.Fatal(err)
}
if got := string(payload); !strings.Contains(got, `"expires"`) {
t.Fatalf("expected expires in payload: %s", got)
}
}
+21 -2
View File
@@ -14,8 +14,9 @@ import (
// Request download request
type Request struct {
URL string `json:"url"`
Extra any `json:"extra"`
RawURL string `json:"rawUrl"`
URL string `json:"url"`
Extra any `json:"extra"`
// Labels is used to mark the download task
Labels map[string]string `json:"labels"`
// Proxy is special proxy config for request
@@ -24,6 +25,24 @@ type Request struct {
SkipVerifyCert bool `json:"skipVerifyCert"`
}
// SetLabels replaces all labels on the request.
func (r *Request) SetLabels(labels map[string]string) {
r.Labels = labels
}
// PutLabel sets a label on the request.
func (r *Request) PutLabel(key, value string) {
if r.Labels == nil {
r.Labels = make(map[string]string)
}
r.Labels[key] = value
}
// DelLabel deletes a label from the request.
func (r *Request) DelLabel(key string) {
delete(r.Labels, key)
}
func (r *Request) Validate() error {
if r.URL == "" {
return fmt.Errorf("invalid request url")
+513 -146
View File
@@ -15,6 +15,7 @@ import (
"sync/atomic"
"time"
internalblob "github.com/GopeedLab/gopeed/internal/blob"
"github.com/GopeedLab/gopeed/internal/controller"
"github.com/GopeedLab/gopeed/internal/fetcher"
"github.com/GopeedLab/gopeed/internal/logger"
@@ -111,6 +112,7 @@ type Downloader struct {
claimedExtractions sync.Map
extensions []*Extension
blob *internalblob.Registry
}
func NewDownloader(cfg *DownloaderConfig) *Downloader {
@@ -142,6 +144,8 @@ func NewDownloader(cfg *DownloaderConfig) *Downloader {
}
func (d *Downloader) Setup() error {
d.blob = internalblob.NewRegistry("")
// setup storage
if err := d.storage.Setup([]string{bucketTask, bucketSave, bucketProtocolState, bucketConfig, bucketExtension, bucketExtensionStorage}); err != nil {
return err
@@ -238,13 +242,18 @@ func (d *Downloader) Setup() error {
if len(d.tasks) > 0 {
for _, task := range d.tasks {
func() {
task.statusLock.Lock()
defer task.statusLock.Unlock()
if task.Status != base.DownloadStatusRunning && !task.Uploading {
// Do not acquire d.lock (via GetTask) while holding
// statusLock; scheduling uses the opposite lock order.
if d.GetTask(task.ID) == nil {
return
}
// check if task is deleted
if d.GetTask(task.ID) == nil || task.fetcher == nil {
task.statusLock.Lock()
if task.Status != base.DownloadStatusRunning && !task.Uploading {
task.statusLock.Unlock()
return
}
if task.fetcher == nil {
task.statusLock.Unlock()
return
}
@@ -266,6 +275,8 @@ func (d *Downloader) Setup() error {
task.Progress.UploadSpeed = task.updateUploadSpeed(currentUploaded-task.Progress.Uploaded, tick)
task.Progress.Uploaded = currentUploaded
}
task.statusLock.Unlock()
// Listener callbacks may Pause/Continue and acquire statusLock.
d.emit(EventKeyProgress, task)
// store fetcher progress when download/upload data changed
@@ -345,6 +356,12 @@ func (d *Downloader) setupFetcher(fm fetcher.FetcherManager, fetcher fetcher.Fet
}
// Get proxy config, task request proxy config has higher priority, then use global proxy config
ctl.GetProxy = func(requestProxy *base.RequestProxy) func(*gohttp.Request) (*url.URL, error) {
// Registry-backed blob URLs are loopback capability URLs and must never be
// sent through a configured proxy. Only exact, currently registered URLs
// take this path; everything else keeps normal HTTP proxy behavior.
if d.blob != nil && fetcher != nil && fetcher.Meta() != nil && fetcher.Meta().Req != nil && d.blob.IsURL(fetcher.Meta().Req.URL) {
return nil
}
if requestProxy == nil {
return d.cfg.Proxy.ToHandler()
}
@@ -384,11 +401,39 @@ func (d *Downloader) saveTask(task *Task) error {
return nil
}
func ensureRequestRawURL(req *base.Request) {
if req == nil {
return
}
if req.RawURL == "" {
req.RawURL = req.URL
}
}
func ensureResourceRequestRawURLs(parentReq *base.Request, res *base.Resource) {
if res == nil {
return
}
for _, file := range res.Files {
if file == nil || file.Req == nil {
continue
}
ensureRequestRawURL(file.Req)
if parentReq != nil && parentReq.RawURL != "" && file.Req.RawURL == file.Req.URL {
file.Req.RawURL = parentReq.RawURL
}
}
}
func (d *Downloader) Resolve(req *base.Request, opts *base.Options) (rr *ResolveResult, err error) {
if req == nil {
return nil, errors.New("request cannot be nil")
}
rrId, err := gonanoid.New()
if err != nil {
return
}
ensureRequestRawURL(req)
res, err := d.triggerOnResolve(req)
if err != nil {
@@ -451,6 +496,7 @@ func (d *Downloader) remainRunningCount() int {
}
func (d *Downloader) CreateDirect(req *base.Request, opts *base.Options) (taskId string, err error) {
ensureRequestRawURL(req)
var fetcher fetcher.Fetcher
fetcher, err = d.buildFetcher(req.URL)
if err != nil {
@@ -503,34 +549,40 @@ func (d *Downloader) Patch(id string, req *base.Request, opts *base.Options) err
if task == nil {
return ErrTaskNotFound
}
if err := func() error {
task.lock.Lock()
defer task.lock.Unlock()
// Restore fetcher if not loaded
if task.fetcher == nil {
err := func() error {
task.statusLock.Lock()
defer task.statusLock.Unlock()
// Restore fetcher if not loaded
if task.fetcher == nil {
err := func() error {
task.statusLock.Lock()
defer task.statusLock.Unlock()
return d.restoreFetcher(task)
}()
if err != nil {
return d.restoreFetcher(task)
}()
if err != nil {
return err
}
}
// Call the fetcher's Patch method
if err := task.fetcher.Patch(req, opts); err != nil {
return err
}
}
// Call the fetcher's Patch method
if err := task.fetcher.Patch(req, opts); err != nil {
// Update task meta from fetcher and transfer any live Blob lease.
task.Meta = task.fetcher.Meta()
if err := d.syncBlobTaskLease(task); err != nil {
return err
}
return d.saveTask(task)
}(); err != nil {
return err
}
// Update task meta from fetcher
task.Meta = task.fetcher.Meta()
// Save task to storage
if err := d.saveTask(task); err != nil {
return err
}
// Emit progress event to notify listeners
// Emit outside task.lock so listeners may safely call back into Patch/Pause.
d.emit(EventKeyProgress, task)
return nil
@@ -588,6 +640,7 @@ func (d *Downloader) Continue(filter *TaskFilter) (err error) {
}
realContinueTasks := make([]*Task, 0)
pausedTasks := make([]*Task, 0)
func() {
d.lock.Lock()
defer d.lock.Unlock()
@@ -600,12 +653,16 @@ func (d *Downloader) Continue(filter *TaskFilter) (err error) {
pausedCount := 0
for _, task := range d.tasks {
if task.Status == base.DownloadStatusRunning {
if err = d.doPause(task); err != nil {
var queued bool
queued, err = d.doPauseForScheduling(task)
if err != nil {
return
}
task.Status = base.DownloadStatusWait
d.waitTasks = append(d.waitTasks, task)
pausedCount++
if queued {
d.waitTasks = append(d.waitTasks, task)
pausedTasks = append(pausedTasks, task)
pausedCount++
}
}
if pausedCount == needPauseCount {
break
@@ -622,6 +679,14 @@ func (d *Downloader) Continue(filter *TaskFilter) (err error) {
}
}
}()
if err != nil {
return
}
// Scheduling pauses run while d.lock is held, but events must not: listeners
// are allowed to call back into Downloader methods that acquire d.lock.
for _, task := range pausedTasks {
d.emit(EventKeyPause, task)
}
for _, task := range realContinueTasks {
if err = d.doStart(task); err != nil {
@@ -765,6 +830,7 @@ func (d *Downloader) Stats(id string) (sr any, err error) {
}
func (d *Downloader) doDelete(task *Task, force bool) (err error) {
defer d.releaseBlobTask(task)
err = func() error {
if err := d.storage.Delete(bucketTask, task.ID); err != nil {
return err
@@ -809,6 +875,9 @@ func (d *Downloader) Close() error {
for _, fm := range d.cfg.FetchManagers {
closeArr = append(closeArr, fm.Close)
}
if d.blob != nil {
closeArr = append(closeArr, d.blob.Close)
}
closeArr = append(closeArr, d.storage.Close)
// Make sure all resources are released, if had error, return the last error
var lastErr error
@@ -974,7 +1043,6 @@ func (d *Downloader) watch(task *Task) {
if _, loaded := d.watchedTasks.LoadOrStore(task.ID, true); loaded {
return
}
defer func() {
d.watchedTasks.Delete(task.ID)
}()
@@ -997,104 +1065,175 @@ func (d *Downloader) watch(task *Task) {
}
}
if task.Status == base.DownloadStatusDone {
if d.taskStatus(task) == base.DownloadStatusDone {
return
}
err := task.fetcher.Wait()
if err != nil {
d.doOnError(task, err)
return
}
// When delete a not resolved task, need check if the task resource is nil
if task.Meta.Res == nil || d.GetTask(task.ID) == nil {
return
}
task.Progress.Used = task.timer.Used()
if task.Meta.Res.Size == 0 {
task.Meta.Res.Size = task.fetcher.Progress().TotalDownloaded()
}
used := task.Progress.Used / int64(time.Second)
if used == 0 {
used = 1
}
totalSize := task.Meta.Res.Size
task.Progress.Speed = totalSize / used
task.Progress.Downloaded = totalSize
task.updateStatus(base.DownloadStatusDone)
d.storage.Put(bucketTask, task.ID, task.clone())
d.emit(EventKeyDone, task)
d.emit(EventKeyFinally, task, err)
d.notifyRunning()
d.triggerOnDone(task)
d.triggerWebhooks(WebhookEventDownloadDone, task, nil)
d.triggerScripts(ScriptEventDownloadDone, task, nil)
if e, ok := task.Meta.Opts.Extra.(*http.OptsExtra); ok {
downloadFilePath := task.Meta.SingleFilepath()
cfg, _ := d.GetConfig()
// Determine if auto-torrent is enabled (use global config if not explicitly set)
autoTorrentEnabled := false
if e.AutoTorrent != nil {
autoTorrentEnabled = *e.AutoTorrent
} else if cfg != nil && cfg.AutoTorrent != nil {
autoTorrentEnabled = cfg.AutoTorrent.Enable
for {
err := task.fetcher.Wait()
if err != nil {
status := d.taskStatus(task)
if status == base.DownloadStatusPause || status == base.DownloadStatusWait {
return
}
var sourceErr error
if d.blob != nil && task.Meta != nil && task.Meta.Req != nil {
sourceErr = d.blob.SourceError(task.Meta.Req.URL)
}
if sourceErr != nil {
d.doOnBlobSourceError(task, sourceErr)
} else {
d.doOnError(task, err)
}
if d.taskStatus(task) == base.DownloadStatusRunning {
continue
}
return
}
if autoTorrentEnabled && strings.HasSuffix(downloadFilePath, ".torrent") {
// Determine if should delete torrent file after creating BT task
shouldDelete := false
if e.DeleteTorrentAfterDownload != nil {
shouldDelete = *e.DeleteTorrentAfterDownload
// When delete a not resolved task, need check if the task resource is nil
if task.Meta.Res == nil || d.GetTask(task.ID) == nil {
return
}
if d.blob != nil && task.Meta.Req != nil {
if err := d.blob.SourceError(task.Meta.Req.URL); err != nil {
d.doOnBlobSourceError(task, err)
if d.taskStatus(task) == base.DownloadStatusRunning {
continue
}
return
}
}
task.Progress.Used = task.timer.Used()
if task.Meta.Res.Size == 0 {
task.Meta.Res.Size = task.fetcher.Progress().TotalDownloaded()
}
used := task.Progress.Used / int64(time.Second)
if used == 0 {
used = 1
}
totalSize := task.Meta.Res.Size
task.Progress.Speed = totalSize / used
task.Progress.Downloaded = totalSize
if !d.markTaskDone(task) {
return
}
d.storage.Put(bucketTask, task.ID, task.clone())
d.emit(EventKeyDone, task)
d.emit(EventKeyFinally, task, err)
d.notifyRunning()
d.releaseBlobTask(task)
d.triggerOnDone(task)
d.triggerWebhooks(WebhookEventDownloadDone, task, nil)
d.triggerScripts(ScriptEventDownloadDone, task, nil)
if e, ok := task.Meta.Opts.Extra.(*http.OptsExtra); ok {
downloadFilePath := task.Meta.SingleFilepath()
cfg, _ := d.GetConfig()
// Determine if auto-torrent is enabled (use global config if not explicitly set)
autoTorrentEnabled := false
if e.AutoTorrent != nil {
autoTorrentEnabled = *e.AutoTorrent
} else if cfg != nil && cfg.AutoTorrent != nil {
shouldDelete = cfg.AutoTorrent.DeleteAfterDownload
autoTorrentEnabled = cfg.AutoTorrent.Enable
}
go func() {
_, err2 := d.CreateDirect(
&base.Request{
URL: downloadFilePath,
},
&base.Options{
Path: task.Meta.Opts.Path,
SelectFiles: make([]int, 0),
})
if err2 != nil {
d.Logger.Error().Err(err2).Msgf("auto create torrent task failed, task id: %s", task.ID)
return
if autoTorrentEnabled && strings.HasSuffix(downloadFilePath, ".torrent") {
// Determine if should delete torrent file after creating BT task
shouldDelete := false
if e.DeleteTorrentAfterDownload != nil {
shouldDelete = *e.DeleteTorrentAfterDownload
} else if cfg != nil && cfg.AutoTorrent != nil {
shouldDelete = cfg.AutoTorrent.DeleteAfterDownload
}
if shouldDelete {
d.Delete(&TaskFilter{IDs: []string{task.ID}}, true)
}
}()
}
go func() {
_, err2 := d.CreateDirect(
&base.Request{
URL: downloadFilePath,
},
&base.Options{
Path: task.Meta.Opts.Path,
SelectFiles: make([]int, 0),
})
if err2 != nil {
d.Logger.Error().Err(err2).Msgf("auto create torrent task failed, task id: %s", task.ID)
return
}
// Determine if auto-extract is enabled (use global config if not explicitly set)
autoExtractEnabled := false
if e.AutoExtract != nil {
autoExtractEnabled = *e.AutoExtract
} else if cfg != nil && cfg.Archive != nil {
autoExtractEnabled = cfg.Archive.AutoExtract
}
if shouldDelete {
d.Delete(&TaskFilter{IDs: []string{task.ID}}, true)
}
}()
}
// Auto-extract archive files using the extraction queue
// This ensures only one extraction runs at a time to prevent resource exhaustion
if autoExtractEnabled && isArchiveFile(downloadFilePath) {
d.enqueueExtraction(task, downloadFilePath, e)
// Determine if auto-extract is enabled (use global config if not explicitly set)
autoExtractEnabled := false
if e.AutoExtract != nil {
autoExtractEnabled = *e.AutoExtract
} else if cfg != nil && cfg.Archive != nil {
autoExtractEnabled = cfg.Archive.AutoExtract
}
// Auto-extract archive files using the extraction queue
// This ensures only one extraction runs at a time to prevent resource exhaustion
if autoExtractEnabled && isArchiveFile(downloadFilePath) {
d.enqueueExtraction(task, downloadFilePath, e)
}
}
return
}
}
func (d *Downloader) doOnError(task *Task, err error) {
d.handleOnError(task, err, false)
}
func (d *Downloader) doOnBlobSourceError(task *Task, err error) {
d.handleOnError(task, err, true)
}
func (d *Downloader) handleOnError(task *Task, err error, resetFetcher bool) {
if task == nil || task.lock == nil {
return
}
task.lock.Lock()
if !d.markTaskError(task) {
task.lock.Unlock()
return
}
d.Logger.Warn().Err(err).Msgf("task download failed, task id: %s", task.ID)
task.updateStatus(base.DownloadStatusError)
oldURL := ""
if task.Meta != nil && task.Meta.Req != nil {
oldURL = task.Meta.Req.URL
}
d.triggerOnError(task, err)
if task.Status == base.DownloadStatusError {
newURL := ""
if task.Meta != nil && task.Meta.Req != nil {
newURL = task.Meta.Req.URL
}
if oldURL != "" && newURL != oldURL {
// Extensions are allowed to replace an expired URL from onError. Patch the
// existing fetcher through its public API so protocol-owned redirect state
// is reset without teaching the HTTP fetcher about blob URLs.
if task.fetcher != nil {
if patchErr := task.fetcher.Patch(&base.Request{URL: newURL}, nil); patchErr != nil {
d.Logger.Warn().Err(patchErr).Msgf("patch recovered task url failed, task id: %s", task.ID)
}
}
d.releaseBlobTaskURL(task, oldURL)
}
if resetFetcher && d.taskStatus(task) == base.DownloadStatusRunning {
if resetErr := d.resetTaskFetcher(task); resetErr != nil {
d.Logger.Warn().Err(resetErr).Msgf("reset recovered task fetcher failed, task id: %s", task.ID)
}
}
task.lock.Unlock()
if d.taskStatus(task) == base.DownloadStatusError {
d.releaseBlobTask(task)
d.emit(EventKeyError, task, err)
d.emit(EventKeyFinally, task, err)
d.notifyRunning()
@@ -1102,13 +1241,43 @@ func (d *Downloader) doOnError(task *Task, err error) {
}
}
func (d *Downloader) resetTaskFetcher(task *Task) error {
if task == nil || task.Meta == nil || task.Meta.Req == nil {
return errors.New("task request is unavailable")
}
fm, err := d.parseFm(task.Meta.Req.URL)
if err != nil {
return err
}
opts := task.Meta.Opts
f := fm.Build()
d.setupFetcher(fm, f)
// A fresh source must never inherit bytes or preallocation from the failed
// source. HTTP opens existing targets without truncating them, so remove the
// old single-file target before installing a fresh fetcher.
if task.Meta.Res != nil && task.Meta.Opts != nil && len(task.Meta.Res.Files) > 0 {
if err := util.SafeRemove(task.Meta.SingleFilepath()); err != nil {
return err
}
}
f.Meta().Req = task.Meta.Req
f.Meta().Opts = opts
task.fetcherManager = fm
task.fetcher = f
task.Meta = f.Meta()
if task.Progress != nil {
task.Progress.Downloaded = 0
task.Progress.Speed = 0
}
return d.storage.Delete(bucketSave, task.ID)
}
func (d *Downloader) restoreTask(task *Task) error {
if task.fetcher == nil {
if err := d.restoreFetcher(task); err != nil {
return err
}
}
go d.watch(task)
return nil
}
@@ -1141,12 +1310,12 @@ func (d *Downloader) doCreate(f fetcher.Fetcher, opts *base.Options) (taskId str
if f.Meta().Opts == nil {
f.Meta().Opts = opts
}
ensureRequestRawURL(f.Meta().Req)
fm, err := d.parseFm(f.Meta().Req.URL)
if err != nil {
return
}
task := NewTask()
task.fetcherManager = fm
task.fetcher = f
@@ -1155,6 +1324,14 @@ func (d *Downloader) doCreate(f fetcher.Fetcher, opts *base.Options) (taskId str
task.Progress = &Progress{}
_, task.Uploading = f.(fetcher.Uploader)
initTask(task)
if err = d.syncBlobTaskLease(task); err != nil {
return "", err
}
defer func() {
if err != nil {
d.releaseBlobTask(task)
}
}()
if err = d.storage.Put(bucketTask, task.ID, task.clone()); err != nil {
return
}
@@ -1176,7 +1353,6 @@ func (d *Downloader) doCreate(f fetcher.Fetcher, opts *base.Options) (taskId str
err = d.doStart(task)
}()
go d.watch(task)
return
}
@@ -1221,7 +1397,8 @@ func (d *Downloader) statusMut(task *Task, fn func() (bool, error)) (bool, error
}
func (d *Downloader) doStart(task *Task) (err error) {
var needCreate bool
var isCreate bool
var generation uint64
isReturn, err := d.statusMut(task, func() (isReturn bool, err error) {
if task.Status == base.DownloadStatusRunning || task.Status == base.DownloadStatusDone {
isReturn = true
@@ -1233,8 +1410,10 @@ func (d *Downloader) doStart(task *Task) (err error) {
d.Logger.Error().Stack().Err(err).Msgf("restore fetcher failed, task id: %s", task.ID)
return
}
needCreate = !task.IsCreated
isCreate = task.Status == base.DownloadStatusReady
task.updateStatus(base.DownloadStatusRunning)
task.runGeneration++
generation = task.runGeneration
return
})
@@ -1246,11 +1425,31 @@ func (d *Downloader) doStart(task *Task) (err error) {
return
}
started := false
handler := func() error {
task.lock.Lock()
defer task.lock.Unlock()
if !d.taskIsRunningGeneration(task, generation) {
return nil
}
if task.fetcher == nil {
if err := d.restoreFetcher(task); err != nil {
return err
}
}
d.triggerOnStart(task)
if !d.taskIsRunningGeneration(task, generation) {
return nil
}
if task.fetcher != nil {
task.fetcher.Meta().Req = task.Meta.Req
task.fetcher.Meta().Res = task.Meta.Res
task.fetcher.Meta().Opts = task.Meta.Opts
}
if err := d.syncBlobTaskLease(task); err != nil {
return err
}
if task.Meta.Res == nil {
err := task.fetcher.Resolve(task.Meta.Req, task.Meta.Opts)
if err != nil {
@@ -1259,7 +1458,7 @@ func (d *Downloader) doStart(task *Task) (err error) {
task.Meta.Res = task.fetcher.Meta().Res
}
if needCreate {
if isCreate {
if task.fetcherManager.AutoRename() {
d.checkDuplicateLock.Lock()
defer d.checkDuplicateLock.Unlock()
@@ -1283,23 +1482,36 @@ func (d *Downloader) doStart(task *Task) (err error) {
task.Meta.Opts.Name = newName
}
}
task.IsCreated = true
task.Meta.Res.CalcSize(task.Meta.Opts.SelectFiles)
}
task.Progress.Speed = 0
if !d.taskIsRunningGeneration(task, generation) {
return nil
}
task.timer.Start()
if err := task.fetcher.Start(); err != nil {
return err
}
started = true
if err := d.saveTask(task); err != nil {
return err
}
d.emit(EventKeyStart, task)
return nil
}
go func() {
if err := handler(); err != nil {
d.doOnError(task, err)
if d.taskIsRunningGeneration(task, generation) {
d.doOnError(task, err)
}
return
}
if started {
if d.taskIsRunningGeneration(task, generation) {
d.emit(EventKeyStart, task)
}
go d.watch(task)
}
}()
@@ -1307,51 +1519,137 @@ func (d *Downloader) doStart(task *Task) (err error) {
}
func (d *Downloader) doPause(task *Task) (err error) {
isReturn, err := d.statusMut(task, func() (isReturn bool, err error) {
generation, isReturn, err := d.preparePause(task)
if err != nil || isReturn {
return err
}
handled, err := d.runPauseHandler(task, generation, true)
if err != nil {
d.Logger.Error().Stack().Err(err).Msgf("pause task handle failed, task id: %s", task.ID)
return err
}
if handled {
d.emit(EventKeyPause, task)
}
return nil
}
// doPauseForScheduling physically pauses a running task without emitting while
// the caller holds d.lock, then atomically moves it to the wait queue state.
func (d *Downloader) doPauseForScheduling(task *Task) (bool, error) {
generation, isReturn, err := d.preparePause(task)
if err != nil || isReturn {
return false, err
}
handled, err := d.runPauseHandler(task, generation, true)
if err != nil || !handled {
return false, err
}
task.statusLock.Lock()
queued := task.Status == base.DownloadStatusPause && task.runGeneration == generation
if queued {
task.Status = base.DownloadStatusWait
}
task.statusLock.Unlock()
return queued, nil
}
func (d *Downloader) preparePause(task *Task) (generation uint64, isReturn bool, err error) {
isReturn, err = d.statusMut(task, func() (isReturn bool, err error) {
if task.Status == base.DownloadStatusPause || task.Status == base.DownloadStatusDone {
isReturn = true
return
}
task.updateStatus(base.DownloadStatusPause)
task.runGeneration++
generation = task.runGeneration
task.timer.Pause()
return
})
if err != nil {
d.Logger.Error().Stack().Err(err).Msgf("pause task failed, task id: %s", task.ID)
return
}
if isReturn {
return
return generation, isReturn, err
}
func (d *Downloader) runPauseHandler(task *Task, generation uint64, pauseFetcher bool) (bool, error) {
task.lock.Lock()
defer task.lock.Unlock()
if !d.taskIsPausedGeneration(task, generation) {
return false, nil
}
handler := func() error {
task.lock.Lock()
defer task.lock.Unlock()
if task.fetcher != nil {
if err := task.fetcher.Pause(); err != nil {
return err
}
if pauseFetcher && task.fetcher != nil {
if err := task.fetcher.Pause(); err != nil {
return false, err
}
if task.fetcherManager != nil && task.fetcher != nil {
if err := d.saveTask(task); err != nil {
return err
}
} else {
if err := d.storage.Put(bucketTask, task.ID, task.clone()); err != nil {
return err
}
}
d.emit(EventKeyPause, task)
return nil
}
go func() {
if err := handler(); err != nil {
d.Logger.Error().Stack().Err(err).Msgf("pause task handle failed, task id: %s", task.ID)
if task.fetcherManager != nil && task.fetcher != nil {
if err := d.saveTask(task); err != nil {
return false, err
}
}()
return
} else {
if err := d.storage.Put(bucketTask, task.ID, task.clone()); err != nil {
return false, err
}
}
return true, nil
}
func (d *Downloader) taskIsRunningGeneration(task *Task, generation uint64) bool {
if task == nil || task.statusLock == nil {
return false
}
task.statusLock.Lock()
defer task.statusLock.Unlock()
return task.Status == base.DownloadStatusRunning && task.runGeneration == generation
}
func (d *Downloader) taskIsPausedGeneration(task *Task, generation uint64) bool {
if task == nil || task.statusLock == nil {
return false
}
task.statusLock.Lock()
defer task.statusLock.Unlock()
return (task.Status == base.DownloadStatusPause || task.Status == base.DownloadStatusWait) && task.runGeneration == generation
}
func (d *Downloader) taskStatus(task *Task) base.Status {
if task == nil || task.statusLock == nil {
return ""
}
task.statusLock.Lock()
defer task.statusLock.Unlock()
return task.Status
}
func (d *Downloader) markTaskDone(task *Task) bool {
if task == nil || task.statusLock == nil {
return false
}
task.statusLock.Lock()
defer task.statusLock.Unlock()
if task.Status != base.DownloadStatusRunning {
return false
}
task.updateStatus(base.DownloadStatusDone)
task.runGeneration++
return true
}
func (d *Downloader) markTaskError(task *Task) bool {
if task == nil || task.statusLock == nil {
return false
}
task.statusLock.Lock()
defer task.statusLock.Unlock()
if task.Status != base.DownloadStatusRunning {
return false
}
task.updateStatus(base.DownloadStatusError)
task.runGeneration++
return true
}
// redirect stderr to log file, when panic happened log it
@@ -1768,10 +2066,79 @@ func initTask(task *Task) {
task.statusLock = &sync.Mutex{}
task.lock = &sync.Mutex{}
task.blobRefLock = &sync.Mutex{}
task.speedArr = make([]int64, 0)
task.uploadSpeedArr = make([]int64, 0)
}
func (d *Downloader) releaseBlobTask(task *Task) {
if d.blob == nil || task == nil {
return
}
if task.blobRefLock != nil {
task.blobRefLock.Lock()
raw := task.blobURL
task.blobURL = ""
task.blobRefLock.Unlock()
if raw != "" {
_ = d.blob.Release(raw)
}
}
}
func (d *Downloader) syncBlobTaskLease(task *Task) error {
if d.blob == nil || task == nil || task.Meta == nil || task.Meta.Req == nil {
return nil
}
if task.blobRefLock == nil {
task.blobRefLock = &sync.Mutex{}
}
task.blobRefLock.Lock()
defer task.blobRefLock.Unlock()
terminal := false
if task.statusLock != nil {
task.statusLock.Lock()
terminal = task.Status == base.DownloadStatusDone || task.Status == base.DownloadStatusError
task.statusLock.Unlock()
}
newURL := ""
if !terminal && d.blob.IsURL(task.Meta.Req.URL) {
newURL = task.Meta.Req.URL
}
if task.blobURL == newURL {
return nil
}
if newURL != "" {
if err := d.blob.Acquire(newURL); err != nil {
return err
}
}
oldURL := task.blobURL
task.blobURL = newURL
if oldURL != "" {
_ = d.blob.Release(oldURL)
}
return nil
}
func (d *Downloader) releaseBlobTaskURL(task *Task, raw string) {
if d.blob == nil || task == nil || raw == "" {
return
}
if task.blobRefLock == nil {
task.blobRefLock = &sync.Mutex{}
}
task.blobRefLock.Lock()
claimed := task.blobURL == raw
if claimed {
task.blobURL = ""
}
task.blobRefLock.Unlock()
if claimed {
_ = d.blob.Release(raw)
}
}
var defaultDownloader = NewDownloader(nil)
type boot struct {
+693 -39
View File
@@ -2,18 +2,26 @@ package download
import (
"archive/zip"
"bytes"
"context"
"errors"
"fmt"
"io"
"net"
gohttp "net/http"
"net/http/httptest"
"os"
"path/filepath"
"reflect"
"strconv"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
internalblob "github.com/GopeedLab/gopeed/internal/blob"
"github.com/GopeedLab/gopeed/internal/controller"
"github.com/GopeedLab/gopeed/internal/fetcher"
"github.com/GopeedLab/gopeed/internal/test"
"github.com/GopeedLab/gopeed/pkg/base"
@@ -21,12 +29,147 @@ import (
"github.com/GopeedLab/gopeed/pkg/util"
)
var testDownloadOpt = &base.Options{
Path: test.Dir,
Name: test.DownloadName,
Extra: http.OptsExtra{
Connections: 4,
},
type generationTestManager struct {
starts atomic.Int32
pauses atomic.Int32
holdOpen bool
resolveStarted chan struct{}
resolveRelease <-chan struct{}
resolveErr error
resolveOnce sync.Once
pauseStarted chan struct{}
pauseRelease <-chan struct{}
pauseOnce sync.Once
}
func TestExtensionTaskMethodsMatchEventCapabilities(t *testing.T) {
taskType := reflect.TypeOf((*Task)(nil))
if _, ok := taskType.MethodByName("Continue"); ok {
t.Fatal("regular task unexpectedly exposes Continue")
}
if _, ok := taskType.MethodByName("SetUrl"); ok {
t.Fatal("regular task unexpectedly exposes SetUrl")
}
extensionTaskType := reflect.TypeOf((*ExtensionTask)(nil))
if _, ok := extensionTaskType.MethodByName("SetUrl"); !ok {
t.Fatal("extension task does not expose SetUrl")
}
if _, ok := extensionTaskType.MethodByName("Continue"); ok {
t.Fatal("non-error extension task unexpectedly exposes Continue")
}
errorTaskType := reflect.TypeOf((*OnErrorExtensionTask)(nil))
if _, ok := errorTaskType.MethodByName("SetUrl"); !ok {
t.Fatal("onError task does not expose SetUrl")
}
if _, ok := errorTaskType.MethodByName("Continue"); !ok {
t.Fatal("onError task does not expose Continue")
}
startTaskField, _ := reflect.TypeOf(OnStartContext{}).FieldByName("Task")
if startTaskField.Type != extensionTaskType {
t.Fatalf("onStart does not inject the request mutation wrapper: %v", startTaskField.Type)
}
doneTaskField, _ := reflect.TypeOf(OnDoneContext{}).FieldByName("Task")
if doneTaskField.Type != taskType {
t.Fatalf("onDone unexpectedly injects a mutation wrapper: %v", doneTaskField.Type)
}
errorTaskField, _ := reflect.TypeOf(OnErrorContext{}).FieldByName("Task")
if errorTaskField.Type != errorTaskType {
t.Fatalf("onError does not inject the recovery wrapper: %v", errorTaskField.Type)
}
}
func (m *generationTestManager) Name() string { return "generation" }
func (m *generationTestManager) Filters() []*fetcher.SchemeFilter {
return []*fetcher.SchemeFilter{{Type: fetcher.FilterTypeUrl, Pattern: "generation"}}
}
func (m *generationTestManager) Build() fetcher.Fetcher {
return &generationTestFetcher{manager: m, meta: &fetcher.FetcherMeta{}}
}
func (m *generationTestManager) ParseName(string) string { return "generation.bin" }
func (m *generationTestManager) AutoRename() bool { return false }
func (m *generationTestManager) DefaultConfig() any { return map[string]any{} }
func (m *generationTestManager) Store(fetcher.Fetcher) (any, error) { return nil, nil }
func (m *generationTestManager) Restore() (any, func(*fetcher.FetcherMeta, any) fetcher.Fetcher) {
return nil, func(meta *fetcher.FetcherMeta, _ any) fetcher.Fetcher {
return &generationTestFetcher{manager: m, meta: meta}
}
}
func (m *generationTestManager) Close() error { return nil }
type generationTestFetcher struct {
manager *generationTestManager
meta *fetcher.FetcherMeta
done chan error
}
func (f *generationTestFetcher) Setup(*controller.Controller) {
if f.done == nil {
f.done = make(chan error, 2)
}
}
func (f *generationTestFetcher) Resolve(req *base.Request, opts *base.Options) error {
if f.manager.resolveStarted != nil {
f.manager.resolveOnce.Do(func() { close(f.manager.resolveStarted) })
}
if f.manager.resolveRelease != nil {
<-f.manager.resolveRelease
}
if f.manager.resolveErr != nil {
return f.manager.resolveErr
}
f.meta.Req = req
f.meta.Opts = opts
f.meta.Res = &base.Resource{Files: []*base.FileInfo{{Name: "generation.bin", Size: 1}}}
return nil
}
func (f *generationTestFetcher) Start() error {
f.manager.starts.Add(1)
if !f.manager.holdOpen {
f.done <- nil
}
return nil
}
func (f *generationTestFetcher) Patch(req *base.Request, opts *base.Options) error {
if req != nil && req.URL != "" {
f.meta.Req.URL = req.URL
}
if opts != nil {
f.meta.Opts = opts
}
return nil
}
func (f *generationTestFetcher) Pause() error {
f.manager.pauses.Add(1)
if f.manager.pauseStarted != nil {
f.manager.pauseOnce.Do(func() { close(f.manager.pauseStarted) })
}
if f.manager.pauseRelease != nil {
<-f.manager.pauseRelease
}
return nil
}
func (f *generationTestFetcher) Close() error { return nil }
func (f *generationTestFetcher) Stats() any { return nil }
func (f *generationTestFetcher) Meta() *fetcher.FetcherMeta { return f.meta }
func (f *generationTestFetcher) Progress() fetcher.Progress { return fetcher.Progress{1} }
func (f *generationTestFetcher) Wait() error { return <-f.done }
func newTestDownloadOpt(t *testing.T) *base.Options {
t.Helper()
return newTestDownloadOptAt(t.TempDir())
}
func newTestDownloadOptAt(path string) *base.Options {
return &base.Options{
Path: path,
Name: test.DownloadName,
Extra: http.OptsExtra{
Connections: 4,
},
}
}
func TestDownloader_Resolve(t *testing.T) {
@@ -61,6 +204,500 @@ func TestDownloader_Resolve(t *testing.T) {
}
}
func TestDownloader_BlobRegistryDoesNotUseStorageDirForSpooling(t *testing.T) {
storageDir := t.TempDir()
downloader := NewDownloader(&DownloaderConfig{
Storage: NewMemStorage(),
StorageDir: storageDir,
})
if err := downloader.Setup(); err != nil {
t.Fatal(err)
}
defer downloader.Clear()
if downloader.blob == nil {
t.Fatal("expected blob registry to be initialized")
}
if downloader.blob.Dir() != "" {
t.Fatalf("expected blob registry not to use a spool dir, got %s", downloader.blob.Dir())
}
if _, err := os.Stat(filepath.Join(storageDir, "blob")); !os.IsNotExist(err) {
t.Fatalf("expected blob spool dir not to be created, got err=%v", err)
}
}
func TestDownloader_BlobTaskKeepsConfiguredHTTPConnections(t *testing.T) {
downloader := NewDownloader(&DownloaderConfig{
Storage: NewMemStorage(),
StorageDir: t.TempDir(),
})
if err := downloader.Setup(); err != nil {
t.Fatal(err)
}
defer downloader.Clear()
url, err := downloader.blob.CreateBlob([]byte("hello"), "text/plain")
if err != nil {
t.Fatal(err)
}
id, err := downloader.CreateDirect(&base.Request{URL: url}, &base.Options{
Path: t.TempDir(),
Name: "blob-unmarked.txt",
Extra: &http.OptsExtra{Connections: 8},
})
if err != nil {
t.Fatal(err)
}
task := downloader.GetTask(id)
if task == nil {
t.Fatal("task not found")
}
if task.Protocol != "http" {
t.Fatalf("expected blob task protocol http, got %s", task.Protocol)
}
extra, ok := task.Meta.Opts.Extra.(*http.OptsExtra)
if !ok {
t.Fatalf("expected http extra, got %T", task.Meta.Opts.Extra)
}
if extra.Connections != 8 {
t.Fatalf("expected blob task connections to stay 8, got %d", extra.Connections)
}
waitForTaskStatus(t, downloader, id, base.DownloadStatusDone, 5*time.Second)
}
func TestDownloader_ExternalBlobLikePathUsesNormalHTTP(t *testing.T) {
payload := []byte("external blob-like path")
server := httptest.NewServer(gohttp.HandlerFunc(func(w gohttp.ResponseWriter, r *gohttp.Request) {
if r.URL.Path != "/__blob/file.bin" {
gohttp.NotFound(w, r)
return
}
w.Header().Set("Content-Length", strconv.Itoa(len(payload)))
_, _ = w.Write(payload)
}))
defer server.Close()
downloader := NewDownloader(nil)
if err := downloader.Setup(); err != nil {
t.Fatal(err)
}
defer downloader.Clear()
dir := t.TempDir()
id, err := downloader.CreateDirect(&base.Request{URL: server.URL + "/__blob/file.bin"}, &base.Options{
Path: dir,
Name: "external.bin",
})
if err != nil {
t.Fatal(err)
}
waitForTaskStatus(t, downloader, id, base.DownloadStatusDone, 5*time.Second)
got, err := os.ReadFile(filepath.Join(dir, "external.bin"))
if err != nil {
t.Fatal(err)
}
if string(got) != string(payload) {
t.Fatalf("unexpected external payload %q", string(got))
}
}
func TestDownloader_InternalBlobBypassesGlobalProxy(t *testing.T) {
downloader := NewDownloader(nil)
if err := downloader.Setup(); err != nil {
t.Fatal(err)
}
defer downloader.Clear()
downloader.cfg.DownloaderStoreConfig.Proxy = &base.DownloaderProxyConfig{
Enable: true,
Scheme: "http",
Host: "127.0.0.1:1",
}
payload := []byte("direct blob")
blobURL, err := downloader.blob.CreateBlob(payload, "application/octet-stream")
if err != nil {
t.Fatal(err)
}
dir := t.TempDir()
id, err := downloader.CreateDirect(&base.Request{URL: blobURL}, &base.Options{
Path: dir,
Name: "direct.bin",
})
if err != nil {
t.Fatal(err)
}
waitForTaskStatus(t, downloader, id, base.DownloadStatusDone, 5*time.Second)
got, err := os.ReadFile(filepath.Join(dir, "direct.bin"))
if err != nil {
t.Fatal(err)
}
if string(got) != string(payload) {
t.Fatalf("unexpected blob payload %q", string(got))
}
}
func TestDownloader_RegistryMissUsesOrdinaryHTTPError(t *testing.T) {
downloader := NewDownloader(nil)
if err := downloader.Setup(); err != nil {
t.Fatal(err)
}
defer downloader.Clear()
liveURL, err := downloader.blob.CreateBlob([]byte("unused"), "text/plain")
if err != nil {
t.Fatal(err)
}
missingURL := liveURL[:len(liveURL)-1] + "x"
if downloader.blob.IsURL(missingURL) {
t.Fatal("test URL unexpectedly matched the Registry")
}
errorCh := make(chan error, 1)
downloader.Listener(func(event *Event) {
if event.Key == EventKeyError {
select {
case errorCh <- event.Err:
default:
}
}
})
_, err = downloader.CreateDirect(&base.Request{URL: missingURL}, &base.Options{
Path: t.TempDir(),
Name: "missing.bin",
})
if err != nil {
t.Fatalf("Registry miss was rejected before ordinary HTTP handling: %v", err)
}
select {
case got := <-errorCh:
if errors.Is(got, internalblob.ErrSourceNotFound) {
t.Fatalf("Registry miss surfaced Blob error instead of HTTP failure: %v", got)
}
if got == nil || !strings.Contains(got.Error(), "404") {
t.Fatalf("expected ordinary HTTP 404 failure, got %v", got)
}
case <-time.After(10 * time.Second):
t.Fatal("Registry miss did not reach the ordinary HTTP error path")
}
}
func TestDownloader_TasksShareBlobLease(t *testing.T) {
downloader := NewDownloader(nil)
if err := downloader.Setup(); err != nil {
t.Fatal(err)
}
defer downloader.Clear()
downloader.cfg.MaxRunning = 1
payload := bytes.Repeat([]byte("shared-blob-"), 8192)
allowLaterOpens := make(chan struct{})
var opens atomic.Int32
blobURL, err := downloader.blob.CreateOpener(func(ctx context.Context, req internalblob.OpenRequest) (io.ReadCloser, error) {
if opens.Add(1) > 1 {
select {
case <-allowLaterOpens:
case <-ctx.Done():
return nil, ctx.Err()
}
}
start := req.Offset
end := int64(len(payload))
if req.End >= start && req.End+1 < end {
end = req.End + 1
}
return io.NopCloser(bytes.NewReader(payload[start:end])), nil
}, &internalblob.CreateOptions{Size: int64(len(payload))})
if err != nil {
t.Fatal(err)
}
dirA, dirB := t.TempDir(), t.TempDir()
idA, err := downloader.CreateDirect(&base.Request{URL: blobURL}, &base.Options{Path: dirA, Name: "a.bin"})
if err != nil {
t.Fatal(err)
}
idB, err := downloader.CreateDirect(&base.Request{URL: blobURL}, &base.Options{Path: dirB, Name: "b.bin"})
if err != nil {
t.Fatal(err)
}
waitForTaskStatus(t, downloader, idA, base.DownloadStatusDone, 5*time.Second)
if err := downloader.Delete(&TaskFilter{IDs: []string{idA}}, false); err != nil {
t.Fatal(err)
}
if !downloader.blob.IsURL(blobURL) {
t.Fatal("deleting the first task revoked a Blob still leased by the second task")
}
close(allowLaterOpens)
waitForTaskStatus(t, downloader, idB, base.DownloadStatusDone, 5*time.Second)
if downloader.blob.IsURL(blobURL) {
t.Fatal("final task completion did not release the shared Blob")
}
for _, file := range []string{filepath.Join(dirA, "a.bin"), filepath.Join(dirB, "b.bin")} {
got, err := os.ReadFile(file)
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(got, payload) {
t.Fatalf("unexpected shared Blob content in %s", file)
}
}
}
func TestDownloader_EmptyBlob(t *testing.T) {
downloader := NewDownloader(nil)
if err := downloader.Setup(); err != nil {
t.Fatal(err)
}
defer downloader.Clear()
blobURL, err := downloader.blob.CreateBlob(nil, "application/octet-stream")
if err != nil {
t.Fatal(err)
}
dir := t.TempDir()
id, err := downloader.CreateDirect(&base.Request{URL: blobURL}, &base.Options{Path: dir, Name: "empty.bin"})
if err != nil {
t.Fatal(err)
}
waitForTaskStatus(t, downloader, id, base.DownloadStatusDone, 5*time.Second)
info, err := os.Stat(filepath.Join(dir, "empty.bin"))
if err != nil {
t.Fatal(err)
}
if info.Size() != 0 {
t.Fatalf("unexpected empty Blob size: %d", info.Size())
}
}
func TestDownloader_PatchTransfersBlobLease(t *testing.T) {
downloader := NewDownloader(nil)
if err := downloader.Setup(); err != nil {
t.Fatal(err)
}
defer downloader.Clear()
// Keep the task waiting so Patch exercises only metadata and lease transfer.
downloader.cfg.MaxRunning = 0
urlA, err := downloader.blob.CreateBlob([]byte("a"), "text/plain")
if err != nil {
t.Fatal(err)
}
urlB, err := downloader.blob.CreateBlob([]byte("b"), "text/plain")
if err != nil {
t.Fatal(err)
}
id, err := downloader.CreateDirect(&base.Request{URL: urlA}, &base.Options{Path: t.TempDir(), Name: "patched.bin"})
if err != nil {
t.Fatal(err)
}
if err := downloader.Patch(id, &base.Request{URL: urlB}, nil); err != nil {
t.Fatal(err)
}
if downloader.blob.IsURL(urlA) {
t.Fatal("Patch retained the old Blob lease")
}
if !downloader.blob.IsURL(urlB) {
t.Fatal("Patch did not acquire the new Blob lease")
}
if err := downloader.Delete(&TaskFilter{IDs: []string{id}}, false); err != nil {
t.Fatal(err)
}
if downloader.blob.IsURL(urlB) {
t.Fatal("deleting patched task did not release the new Blob lease")
}
}
func TestDownloader_StaleStartGenerationCannotRestartTask(t *testing.T) {
manager := &generationTestManager{}
downloader := NewDownloader(&DownloaderConfig{FetchManagers: []fetcher.FetcherManager{manager}})
if err := downloader.Setup(); err != nil {
t.Fatal(err)
}
defer downloader.Clear()
downloader.cfg.MaxRunning = 0
id, err := downloader.CreateDirect(&base.Request{URL: "generation://test"}, &base.Options{
Path: t.TempDir(),
Name: "generation.bin",
})
if err != nil {
t.Fatal(err)
}
task := downloader.GetTask(id)
if task == nil || task.Status != base.DownloadStatusWait {
t.Fatalf("expected queued task, got %#v", task)
}
// Hold task.lock so the first Start, its Pause handler, and the second Start
// are all queued. Only the newest start generation may reach Fetcher.Start.
task.lock.Lock()
downloader.cfg.MaxRunning = 1
if err := downloader.Continue(&TaskFilter{IDs: []string{id}}); err != nil {
t.Fatal(err)
}
pauseDone := make(chan error, 1)
go func() {
pauseDone <- downloader.Pause(&TaskFilter{IDs: []string{id}})
}()
deadline := time.Now().Add(2 * time.Second)
for downloader.taskStatus(task) != base.DownloadStatusPause && time.Now().Before(deadline) {
time.Sleep(5 * time.Millisecond)
}
if downloader.taskStatus(task) != base.DownloadStatusPause {
t.Fatal("Pause did not update task status")
}
if err := downloader.Continue(&TaskFilter{IDs: []string{id}}); err != nil {
t.Fatal(err)
}
task.lock.Unlock()
select {
case err := <-pauseDone:
if err != nil {
t.Fatal(err)
}
case <-time.After(2 * time.Second):
t.Fatal("Pause did not finish")
}
waitForTaskStatus(t, downloader, id, base.DownloadStatusDone, 3*time.Second)
if got := manager.starts.Load(); got != 1 {
t.Fatalf("stale start generation reached Fetcher.Start: got %d starts, want 1", got)
}
}
func TestDownloader_PauseWinsAgainstStaleResolveError(t *testing.T) {
releaseResolve := make(chan struct{})
manager := &generationTestManager{
resolveStarted: make(chan struct{}),
resolveRelease: releaseResolve,
resolveErr: errors.New("stale resolve failed"),
}
downloader := NewDownloader(&DownloaderConfig{FetchManagers: []fetcher.FetcherManager{manager}})
if err := downloader.Setup(); err != nil {
t.Fatal(err)
}
defer downloader.Clear()
errorEvent := make(chan error, 1)
downloader.Listener(func(event *Event) {
if event.Key == EventKeyError {
errorEvent <- event.Err
}
})
id, err := downloader.CreateDirect(&base.Request{URL: "generation://stale-resolve"}, &base.Options{
Path: t.TempDir(),
Name: "stale.bin",
})
if err != nil {
t.Fatal(err)
}
select {
case <-manager.resolveStarted:
case <-time.After(2 * time.Second):
t.Fatal("Resolve did not start")
}
pauseDone := make(chan error, 1)
go func() {
pauseDone <- downloader.Pause(&TaskFilter{IDs: []string{id}})
}()
deadline := time.Now().Add(2 * time.Second)
for downloader.taskStatus(downloader.GetTask(id)) != base.DownloadStatusPause && time.Now().Before(deadline) {
time.Sleep(5 * time.Millisecond)
}
if got := downloader.taskStatus(downloader.GetTask(id)); got != base.DownloadStatusPause {
t.Fatalf("Pause did not win the status transition: %s", got)
}
close(releaseResolve)
select {
case err := <-pauseDone:
if err != nil {
t.Fatal(err)
}
case <-time.After(2 * time.Second):
t.Fatal("Pause did not finish after stale Resolve returned")
}
if got := downloader.taskStatus(downloader.GetTask(id)); got != base.DownloadStatusPause {
t.Fatalf("stale Resolve error overwrote Pause: %s", got)
}
select {
case err := <-errorEvent:
t.Fatalf("stale Resolve emitted onError after Pause: %v", err)
case <-time.After(100 * time.Millisecond):
}
}
func TestDownloader_SchedulingPauseCompletesBeforeReplacementStart(t *testing.T) {
releasePause := make(chan struct{})
manager := &generationTestManager{
holdOpen: true,
pauseStarted: make(chan struct{}),
pauseRelease: releasePause,
}
downloader := NewDownloader(&DownloaderConfig{FetchManagers: []fetcher.FetcherManager{manager}})
if err := downloader.Setup(); err != nil {
t.Fatal(err)
}
defer downloader.Clear()
downloader.cfg.MaxRunning = 1
_, err := downloader.CreateDirect(&base.Request{URL: "generation://a"}, &base.Options{
Path: t.TempDir(), Name: "a.bin",
})
if err != nil {
t.Fatal(err)
}
deadline := time.Now().Add(2 * time.Second)
for manager.starts.Load() != 1 && time.Now().Before(deadline) {
time.Sleep(5 * time.Millisecond)
}
if manager.starts.Load() != 1 {
t.Fatal("first task did not physically start")
}
idB, err := downloader.CreateDirect(&base.Request{URL: "generation://b"}, &base.Options{
Path: t.TempDir(), Name: "b.bin",
})
if err != nil {
t.Fatal(err)
}
if downloader.taskStatus(downloader.GetTask(idB)) != base.DownloadStatusWait {
t.Fatal("second task was not queued")
}
continueDone := make(chan error, 1)
go func() {
continueDone <- downloader.Continue(&TaskFilter{IDs: []string{idB}})
}()
select {
case <-manager.pauseStarted:
case <-time.After(2 * time.Second):
t.Fatal("running task was not physically paused")
}
if got := manager.starts.Load(); got != 1 {
t.Fatalf("replacement started before Pause completed: %d starts", got)
}
close(releasePause)
select {
case err := <-continueDone:
if err != nil {
t.Fatal(err)
}
case <-time.After(2 * time.Second):
t.Fatal("Continue did not return after Pause completed")
}
deadline = time.Now().Add(2 * time.Second)
for manager.starts.Load() != 2 && time.Now().Before(deadline) {
time.Sleep(5 * time.Millisecond)
}
if got := manager.starts.Load(); got != 2 {
t.Fatalf("replacement task did not start: %d starts", got)
}
if got := manager.pauses.Load(); got != 1 {
t.Fatalf("unexpected physical Pause count: %d", got)
}
}
func TestDownloader_Create(t *testing.T) {
listener := test.StartTestFileServer()
defer listener.Close()
@@ -73,7 +710,8 @@ func TestDownloader_Create(t *testing.T) {
req := &base.Request{
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
}
rr, err := downloader.Resolve(req, testDownloadOpt)
opts := newTestDownloadOpt(t)
rr, err := downloader.Resolve(req, opts)
if err != nil {
t.Fatal(err)
}
@@ -91,7 +729,7 @@ func TestDownloader_Create(t *testing.T) {
}
wg.Wait()
want := test.FileMd5(test.BuildFile)
got := test.FileMd5(test.DownloadFile)
got := test.FileMd5(filepath.Join(opts.Path, opts.Name))
if want != got {
t.Errorf("Downloader_Create() got = %v, want %v", got, want)
}
@@ -113,7 +751,7 @@ func TestDownloader_CreateNotInWhite(t *testing.T) {
}
// With new fetcher design, white list check happens during Resolve (not Create)
// because Resolve now requires Options which includes the download path
_, err := downloader.Resolve(req, testDownloadOpt)
_, err := downloader.Resolve(req, newTestDownloadOpt(t))
if err == nil {
t.Error("TestDownloader_CreateNotInWhite() expected error but got nil")
}
@@ -160,9 +798,10 @@ func TestDownloader_CreateDirectBatch(t *testing.T) {
}
})
opts := newTestDownloadOpt(t)
_, err := downloader.CreateDirectBatch(&base.CreateTaskBatch{
Reqs: reqs,
Opts: testDownloadOpt,
Opts: opts,
})
if err != nil {
t.Fatal(err)
@@ -188,7 +827,7 @@ func TestDownloader_CreateDirectBatch(t *testing.T) {
// Check that all task files exist
for name := range taskNames {
if _, err := os.Stat(test.Dir + "/" + name); os.IsNotExist(err) {
if _, err := os.Stat(filepath.Join(opts.Path, name)); os.IsNotExist(err) {
t.Errorf("CreateDirectBatch() file not exist: %v", name)
}
}
@@ -266,6 +905,9 @@ func TestDownloader_CreateWithProxy(t *testing.T) {
}
func doTestDownloaderCreateWithProxy(t *testing.T, auth bool, buildReqProxy func(reqProxy *base.RequestProxy) *base.RequestProxy, buildProxyConfig func(proxyCfg *base.DownloaderProxyConfig) *base.DownloaderProxyConfig, errHandler func(err error)) {
listener := test.StartTestFileServer()
defer listener.Close()
usr, pwd := "", ""
if auth {
usr, pwd = "admin", "123"
@@ -291,7 +933,7 @@ func doTestDownloaderCreateWithProxy(t *testing.T, auth bool, buildReqProxy func
downloader.cfg.DownloaderStoreConfig.Proxy = globalProxyCfg
req := &base.Request{
URL: test.ExternalDownloadUrl,
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
}
if buildReqProxy != nil {
req.Proxy = buildReqProxy(&base.RequestProxy{
@@ -310,13 +952,13 @@ func doTestDownloaderCreateWithProxy(t *testing.T, auth bool, buildReqProxy func
return
}
want := &base.Resource{
Size: test.ExternalDownloadSize,
Size: test.BuildSize,
Range: true,
Files: []*base.FileInfo{
{
Name: test.ExternalDownloadName,
Name: test.BuildName,
Path: "",
Size: test.ExternalDownloadSize,
Size: test.BuildSize,
},
},
}
@@ -338,6 +980,7 @@ func TestDownloader_CreateRename(t *testing.T) {
req := &base.Request{
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
}
downloadDir := t.TempDir()
var wg sync.WaitGroup
wg.Add(2)
downloader.Listener(func(event *Event) {
@@ -347,7 +990,7 @@ func TestDownloader_CreateRename(t *testing.T) {
})
for i := 0; i < 2; i++ {
_, err := downloader.CreateDirect(req, &base.Options{
Path: test.Dir,
Path: downloadDir,
Name: test.DownloadName,
Extra: http.OptsExtra{
Connections: 4,
@@ -360,11 +1003,11 @@ func TestDownloader_CreateRename(t *testing.T) {
wg.Wait()
want := test.FileMd5(test.BuildFile)
got := test.FileMd5(test.DownloadFile)
got := test.FileMd5(filepath.Join(downloadDir, test.DownloadName))
if want != got {
t.Errorf("Downloader_CreateRename() got = %v, want %v", got, want)
}
got = test.FileMd5(test.DownloadRenameFile)
got = test.FileMd5(filepath.Join(downloadDir, test.DownloadRename))
if want != got {
t.Errorf("Downloader_CreateRename() got = %v, want %v", got, want)
}
@@ -374,8 +1017,9 @@ func TestDownloader_StoreAndRestore(t *testing.T) {
listener := test.StartTestSlowFileServer(time.Millisecond * 2000)
defer listener.Close()
storageDir := t.TempDir()
downloader := NewDownloader(&DownloaderConfig{
Storage: NewBoltStorage("./"),
Storage: NewBoltStorage(storageDir),
})
if err := downloader.Setup(); err != nil {
t.Fatal(err)
@@ -384,24 +1028,37 @@ func TestDownloader_StoreAndRestore(t *testing.T) {
req := &base.Request{
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
}
rr, err := downloader.Resolve(req, testDownloadOpt)
opts := newTestDownloadOpt(t)
rr, err := downloader.Resolve(req, opts)
if err != nil {
t.Fatal(err)
}
pauseResult := make(chan error, 1)
var pauseOnce sync.Once
downloader.Listener(func(event *Event) {
if event.Key == EventKeyStart {
pauseOnce.Do(func() {
pauseResult <- downloader.Pause(&TaskFilter{IDs: []string{event.Task.ID}})
})
}
})
id, err := downloader.Create(rr.ID)
if err != nil {
t.Fatal(err)
}
time.Sleep(time.Millisecond * 1001)
err = downloader.Pause(&TaskFilter{IDs: []string{id}})
if err != nil {
t.Fatal(err)
select {
case err = <-pauseResult:
if err != nil {
t.Fatal(err)
}
case <-time.After(10 * time.Second):
t.Fatal("timed out waiting for task to pause")
}
downloader.Close()
downloader = NewDownloader(&DownloaderConfig{
Storage: NewBoltStorage("./"),
Storage: NewBoltStorage(storageDir),
})
if err := downloader.Setup(); err != nil {
t.Fatal(err)
@@ -424,7 +1081,7 @@ func TestDownloader_StoreAndRestore(t *testing.T) {
t.Fatal(err)
}
want := test.FileMd5(test.BuildFile)
got := test.FileMd5(test.DownloadFile)
got := test.FileMd5(filepath.Join(opts.Path, opts.Name))
if want != got {
t.Errorf("StoreAndResume() got = %v, want %v", got, want)
}
@@ -518,15 +1175,10 @@ func TestDownloader_GetTasksByFilter(t *testing.T) {
}
})
batchOpts := newTestDownloadOpt(t)
taskIds, err := downloader.CreateDirectBatch(&base.CreateTaskBatch{
Reqs: reqs,
Opts: &base.Options{
Path: test.Dir,
Name: test.DownloadName,
Extra: http.OptsExtra{
Connections: 4,
},
},
Opts: batchOpts,
})
if err != nil {
t.Fatal(err)
@@ -653,7 +1305,7 @@ func TestDownloader_Stats(t *testing.T) {
req := &base.Request{
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
}
rr, err := downloader.Resolve(req, testDownloadOpt)
rr, err := downloader.Resolve(req, newTestDownloadOpt(t))
if err != nil {
t.Fatal(err)
}
@@ -696,6 +1348,7 @@ func TestDownloader_Delete(t *testing.T) {
// Create multiple tasks
var wg sync.WaitGroup
taskCount := 3
downloadDir := t.TempDir()
wg.Add(taskCount)
downloader.Listener(func(event *Event) {
if event.Key == EventKeyDone {
@@ -708,7 +1361,7 @@ func TestDownloader_Delete(t *testing.T) {
req := &base.Request{
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
}
taskId, err := downloader.CreateDirect(req, testDownloadOpt)
taskId, err := downloader.CreateDirect(req, newTestDownloadOptAt(downloadDir))
if err != nil {
t.Fatal(err)
}
@@ -772,7 +1425,7 @@ func TestDownloader_PauseAndContinue(t *testing.T) {
req := &base.Request{
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
}
rr, err := downloader.Resolve(req, testDownloadOpt)
rr, err := downloader.Resolve(req, newTestDownloadOpt(t))
if err != nil {
t.Fatal(err)
}
@@ -843,7 +1496,7 @@ func TestDownloader_PauseAllAndContinueAll(t *testing.T) {
req := &base.Request{
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
}
rr, err := downloader.Resolve(req, testDownloadOpt)
rr, err := downloader.Resolve(req, newTestDownloadOpt(t))
if err != nil {
t.Fatal(err)
}
@@ -1503,6 +2156,7 @@ func TestDownloader_DeleteAll(t *testing.T) {
// Create multiple tasks
var wg sync.WaitGroup
taskCount := 3
downloadDir := t.TempDir()
wg.Add(taskCount)
downloader.Listener(func(event *Event) {
if event.Key == EventKeyDone {
@@ -1515,7 +2169,7 @@ func TestDownloader_DeleteAll(t *testing.T) {
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
}
_, err := downloader.CreateDirect(req, &base.Options{
Path: test.Dir,
Path: downloadDir,
Name: test.DownloadName,
Extra: http.OptsExtra{
Connections: 4,
@@ -2669,7 +3323,7 @@ func TestDownloader_PatchTask_HTTP(t *testing.T) {
},
}
opts := &base.Options{
Path: test.Dir,
Path: t.TempDir(),
Name: test.DownloadName,
}
+137 -89
View File
@@ -3,16 +3,19 @@ package engine
import (
_ "embed"
"errors"
"fmt"
"sync"
"github.com/GopeedLab/gopeed/pkg/base"
gojaerror "github.com/GopeedLab/gopeed/pkg/download/engine/inject/error"
"github.com/GopeedLab/gopeed/pkg/download/engine/inject/file"
"github.com/GopeedLab/gopeed/pkg/download/engine/inject/formdata"
"github.com/GopeedLab/gopeed/pkg/download/engine/inject/stream"
"github.com/GopeedLab/gopeed/pkg/download/engine/inject/vm"
"github.com/GopeedLab/gopeed/pkg/download/engine/inject/xhr"
"github.com/dop251/goja"
"github.com/dop251/goja_nodejs/eventloop"
gojaurl "github.com/dop251/goja_nodejs/url"
"time"
)
//go:embed polyfill/out/index.js
@@ -24,93 +27,118 @@ type Engine struct {
Runtime *goja.Runtime
}
type JSFunction func(goja.FunctionCall) goja.Value
// RunString executes the script and returns the go type value
// if script result is promise, it will be resolved
func (e *Engine) RunString(script string) (value any, err error) {
defer func() {
if r := recover(); r != nil {
err = r.(error)
}
}()
var result goja.Value
e.loop.Run(func(runtime *goja.Runtime) {
result, err = runtime.RunString(script)
if err == nil {
go e.await(result)
}
return e.runOnLoop(func(runtime *goja.Runtime) (goja.Value, error) {
return runtime.RunString(script)
})
if err != nil {
return
}
return resolveResult(result)
}
// CallFunction calls the function and returns the go type value
// if function result is promise, it will be resolved
func (e *Engine) CallFunction(fn goja.Callable, args ...any) (value any, err error) {
defer func() {
if r := recover(); r != nil {
err = r.(error)
func (e *Engine) CallFunction(fn any, args ...any) (value any, err error) {
return e.runOnLoop(func(runtime *goja.Runtime) (goja.Value, error) {
var jsArgs []goja.Value
for _, arg := range args {
jsArgs = append(jsArgs, runtime.ToValue(arg))
}
}()
var result goja.Value
e.loop.Run(func(runtime *goja.Runtime) {
if args == nil {
result, err = fn(nil)
} else {
var jsArgs []goja.Value
for _, arg := range args {
jsArgs = append(jsArgs, runtime.ToValue(arg))
switch f := fn.(type) {
case goja.Callable:
if args == nil {
return f(nil)
}
result, err = fn(nil, jsArgs...)
}
if err == nil {
go e.await(result)
return f(nil, jsArgs...)
case JSFunction:
return callExportedFunction(func(call goja.FunctionCall) goja.Value {
return f(call)
}, jsArgs...)
default:
return nil, fmt.Errorf("unsupported function type: %T", fn)
}
})
if err != nil {
return
}
return resolveResult(result)
}
// loop.Run will hang if the script result has a non-stop code, such as setInterval.
// This method will stop the event loop when the promise result is resolved.
func (e *Engine) await(value any) {
if value == nil {
return
func (e *Engine) runOnLoop(fn func(runtime *goja.Runtime) (goja.Value, error)) (any, error) {
type result struct {
value any
err error
}
if v, ok := value.(goja.Value); ok {
// if result is promise, wait for it to be resolved
if p, ok := v.Export().(*goja.Promise); ok {
if p.State() != goja.PromiseStatePending {
ch := make(chan result, 1)
ok := e.loop.RunOnLoop(func(runtime *goja.Runtime) {
var once sync.Once
sendResult := func(res result) {
once.Do(func() {
ch <- res
})
}
defer func() {
if r := recover(); r != nil {
switch v := r.(type) {
case error:
sendResult(result{err: v})
case goja.Value:
sendResult(result{err: exportJSError(v)})
default:
sendResult(result{err: fmt.Errorf("panic: %v", r)})
}
}
}()
value, err := fn(runtime)
if err != nil {
sendResult(result{err: err})
return
}
if p, ok := value.Export().(*goja.Promise); ok {
switch p.State() {
case goja.PromiseStateFulfilled:
sendResult(result{value: exportJSValue(p.Result())})
return
case goja.PromiseStateRejected:
sendResult(result{err: exportJSError(p.Result())})
return
}
// check promise state every 100 milliseconds, until it is resolved
for {
time.Sleep(time.Millisecond * 100)
if p.State() == goja.PromiseStatePending {
continue
}
break
promiseObj := value.ToObject(runtime)
thenVal := promiseObj.Get("then")
thenFn, ok := goja.AssertFunction(thenVal)
if !ok {
sendResult(result{err: errors.New("promise.then is not callable")})
return
}
// stop the event loop
e.loop.StopNoWait()
onFulfilled := runtime.ToValue(func(call goja.FunctionCall) goja.Value {
sendResult(result{value: exportJSValue(call.Argument(0))})
return goja.Undefined()
})
onRejected := runtime.ToValue(func(call goja.FunctionCall) goja.Value {
sendResult(result{err: exportJSError(call.Argument(0))})
return goja.Undefined()
})
if _, err := thenFn(promiseObj, onFulfilled, onRejected); err != nil {
sendResult(result{err: err})
}
return
}
sendResult(result{value: exportJSValue(value)})
})
if !ok {
return nil, errors.New("engine loop terminated")
}
res := <-ch
if res.err != nil {
return nil, res.err
}
return res.value, nil
}
func (e *Engine) Close() {
e.loop.StopNoWait()
e.loop.Terminate()
}
type Config struct {
ProxyConfig *base.DownloaderProxyConfig
ProxyConfig *base.DownloaderProxyConfig
StreamConfig *stream.Config
}
func NewEngine(cfg *Config) *Engine {
@@ -121,7 +149,10 @@ func NewEngine(cfg *Config) *Engine {
engine := &Engine{
loop: loop,
}
loop.Run(func(runtime *goja.Runtime) {
loop.Start()
done := make(chan struct{})
loop.RunOnLoop(func(runtime *goja.Runtime) {
defer close(done)
engine.Runtime = runtime
runtime.SetFieldNameMapper(goja.TagFieldNameMapper("json", true))
vm.Enable(runtime)
@@ -153,8 +184,12 @@ func NewEngine(cfg *Config) *Engine {
if _, err := runtime.RunString("global.location = new URL('http://localhost');"); err != nil {
return
}
if err := stream.Enable(runtime, loop, cfg.StreamConfig); err != nil {
return
}
return
})
<-done
return engine
}
@@ -163,32 +198,45 @@ func Run(script string) (value any, err error) {
return engine.RunString(script)
}
// if the value is Promise, it will be resolved and return the result.
func resolveResult(value goja.Value) (any, error) {
export := value.Export()
switch export.(type) {
case *goja.Promise:
p := export.(*goja.Promise)
switch p.State() {
case goja.PromiseStatePending:
return nil, nil
case goja.PromiseStateFulfilled:
return p.Result().Export(), nil
case goja.PromiseStateRejected:
if err, ok := p.Result().Export().(error); ok {
return nil, err
} else {
stack := p.Result().String()
result := p.Result()
if ro, ok := result.(*goja.Object); ok {
stackVal := ro.Get("stack")
if stackVal != nil && stackVal.String() != "" {
stack = stackVal.String()
}
}
return nil, errors.New(stack)
}
func exportJSValue(value goja.Value) any {
if value == nil {
return nil
}
return value.Export()
}
func exportJSError(value goja.Value) error {
if value == nil || goja.IsUndefined(value) || goja.IsNull(value) {
return errors.New("promise rejected")
}
if err, ok := value.Export().(error); ok {
return err
}
stack := value.String()
if ro, ok := value.(*goja.Object); ok {
stackVal := ro.Get("stack")
if stackVal != nil && stackVal.String() != "" {
stack = stackVal.String()
}
}
return export, nil
return errors.New(stack)
}
func callExportedFunction(fn func(goja.FunctionCall) goja.Value, args ...goja.Value) (ret goja.Value, err error) {
defer func() {
if r := recover(); r != nil {
switch v := r.(type) {
case error:
err = v
case goja.Value:
err = exportJSError(v)
default:
err = fmt.Errorf("panic: %v", r)
}
}
}()
return fn(goja.FunctionCall{
This: nil,
Arguments: args,
}), nil
}
+149 -1
View File
@@ -26,7 +26,11 @@ func TestPolyfill(t *testing.T) {
doTestPolyfill(t, "MessageError")
doTestPolyfill(t, "XMLHttpRequest")
doTestPolyfill(t, "Blob")
doTestPolyfill(t, "Event")
doTestPolyfill(t, "EventTarget")
doTestPolyfill(t, "FormData")
doTestPolyfill(t, "AbortController")
doTestPolyfill(t, "AbortSignal")
doTestPolyfill(t, "TextDecoder")
doTestPolyfill(t, "TextEncoder")
doTestPolyfill(t, "fetch")
@@ -36,13 +40,75 @@ func TestPolyfill(t *testing.T) {
func TestError(t *testing.T) {
engine := NewEngine(nil)
_, err := engine.RunString(`
throw new MessageError('test');
throw new MessageError('test');
`)
if me, ok := gojautil.AssertError[*gojaerror.MessageError](err); !ok {
t.Fatalf("expect MessageError, but got %v", me)
}
}
func TestCallFunction_DetachedAsyncWorkDoesNotBlockReturn(t *testing.T) {
engine := NewEngine(nil)
defer engine.Close()
if _, err := engine.RunString(`
globalThis.__done = false;
async function testDetached() {
(async () => {
await new Promise((resolve) => setTimeout(resolve, 500));
globalThis.__done = true;
})();
return "ok";
}
`); err != nil {
t.Fatal(err)
}
fn, ok := goja.AssertFunction(engine.Runtime.Get("testDetached"))
if !ok {
t.Fatal("testDetached is not callable")
}
startedAt := time.Now()
value, err := engine.CallFunction(fn)
if err != nil {
t.Fatal(err)
}
elapsed := time.Since(startedAt)
if elapsed > 200*time.Millisecond {
t.Fatalf("detached async work blocked return for %s", elapsed)
}
if value != "ok" {
t.Fatalf("unexpected return value: %#v", value)
}
}
func TestCallFunction_AwaitedPromiseBlocksUntilSettled(t *testing.T) {
engine := NewEngine(nil)
defer engine.Close()
if _, err := engine.RunString(`
async function testAwaited() {
await new Promise((resolve) => setTimeout(resolve, 120));
return "ok";
}
`); err != nil {
t.Fatal(err)
}
fn, ok := goja.AssertFunction(engine.Runtime.Get("testAwaited"))
if !ok {
t.Fatal("testAwaited is not callable")
}
startedAt := time.Now()
value, err := engine.CallFunction(fn)
if err != nil {
t.Fatal(err)
}
elapsed := time.Since(startedAt)
if elapsed < 100*time.Millisecond {
t.Fatalf("awaited promise returned too early: %s", elapsed)
}
if value != "ok" {
t.Fatalf("unexpected return value: %#v", value)
}
}
func TestFetch(t *testing.T) {
server := startServer()
defer server.Close()
@@ -64,6 +130,50 @@ async function testText(){
return await resp.text();
}
async function testResponseBodyGetReader() {
const resp = await fetch(host+'/text', {
method: 'POST',
body: 'stream-body'
});
if (!resp.body || typeof resp.body.getReader !== 'function') {
throw new Error('response.body.getReader is not available');
}
const reader = resp.body.getReader();
const decoder = new TextDecoder();
let result = '';
while (true) {
const { done, value } = await reader.read();
if (done) {
break;
}
result += decoder.decode(value, { stream: true });
}
result += decoder.decode();
return result;
}
async function testStreamingFetchResponseBody() {
const startedAt = Date.now();
const resp = await fetch(host+'/stream');
const resolvedAt = Date.now() - startedAt;
const reader = resp.body.getReader();
const decoder = new TextDecoder();
const first = await reader.read();
const firstAt = Date.now() - startedAt;
const second = await reader.read();
const secondAt = Date.now() - startedAt;
let result = '';
result += decoder.decode(first.value, { stream: true });
result += decoder.decode(second.value, { stream: true });
result += decoder.decode();
return {
resolvedAt,
firstAt,
secondAt,
text: result
};
}
async function testOctetStream(file){
const resp = await fetch(host+'/octetStream',{
method: 'POST',
@@ -251,6 +361,32 @@ async function testFingerprintSafari(){
t.Fatalf("testText failed, want %s, got %s", "test", result)
}
result, err = callTestFun(engine, "testResponseBodyGetReader")
if err != nil {
t.Fatal(err)
}
if result != "stream-body" {
t.Fatalf("testResponseBodyGetReader failed, want %s, got %s", "stream-body", result)
}
result, err = callTestFun(engine, "testStreamingFetchResponseBody")
if err != nil {
t.Fatal(err)
}
streaming := result.(map[string]any)
if streaming["text"] != "chunk-1chunk-2" {
t.Fatalf("testStreamingFetchResponseBody failed, want %s, got %v", "chunk-1chunk-2", streaming["text"])
}
if int(streaming["resolvedAt"].(int64)) >= 120 {
t.Fatalf("fetch resolved too late, got %dms", streaming["resolvedAt"].(int64))
}
if int(streaming["firstAt"].(int64)) >= 120 {
t.Fatalf("first chunk arrived too late, got %dms", streaming["firstAt"].(int64))
}
if int(streaming["secondAt"].(int64)) < 120 {
t.Fatalf("second chunk arrived too early, got %dms", streaming["secondAt"].(int64))
}
func() {
jsFile, _, md5 := buildFile(t, engine.Runtime)
result, err = callTestFun(engine, "testOctetStream", jsFile)
@@ -464,6 +600,18 @@ func startServer() net.Listener {
w.WriteHeader(http.StatusOK)
w.Write(buf)
})
mux.HandleFunc("/stream", func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
if flusher, ok := w.(http.Flusher); ok {
_, _ = w.Write([]byte("chunk-1"))
flusher.Flush()
time.Sleep(150 * time.Millisecond)
_, _ = w.Write([]byte("chunk-2"))
flusher.Flush()
return
}
_, _ = w.Write([]byte("chunk-1chunk-2"))
})
mux.HandleFunc("/octetStream", func(w http.ResponseWriter, r *http.Request) {
md5 := calcMd5(r.Body)
w.WriteHeader(http.StatusOK)
+990
View File
@@ -0,0 +1,990 @@
package stream
import (
"context"
_ "embed"
"errors"
"fmt"
"io"
"net"
"net/http"
"net/url"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/GopeedLab/gopeed/pkg/download/engine/inject/file"
"github.com/GopeedLab/gopeed/pkg/download/engine/inject/formdata"
"github.com/GopeedLab/gopeed/pkg/download/engine/inject/xhr"
"github.com/GopeedLab/gopeed/pkg/download/engine/util"
"github.com/dop251/goja"
"github.com/dop251/goja_nodejs/eventloop"
"github.com/imroc/req/v3"
)
//go:embed stream.js
var script string
type Config struct {
CreateObjectURL func(opts *ObjectURLOptions, open ObjectURLOpener) (string, error)
RevokeObjectURL func(url string) error
ProxyHandler func(r *http.Request) (*url.URL, error)
RegisterCleanup func(cleanup func())
}
type ObjectURLOptions struct {
ContentType string
Size int64
Range bool
}
type ObjectURLOpenRequest struct {
Offset int64
End int64
}
type ObjectURLOpener func(ctx context.Context, req ObjectURLOpenRequest) (io.ReadCloser, error)
func Enable(runtime *goja.Runtime, loop *eventloop.EventLoop, cfg *Config) error {
if cfg == nil {
cfg = &Config{}
}
blobPipes := newBlobPipeRegistry()
if err := runtime.Set("__gopeed_create_blob_object_url", func(call goja.FunctionCall) goja.Value {
if cfg.CreateObjectURL == nil {
panic(runtime.NewGoError(fmt.Errorf("blob object url handler not configured")))
}
openValue := call.Argument(0)
if _, ok := goja.AssertFunction(openValue); !ok {
panic(runtime.NewGoError(fmt.Errorf("blob opener must be callable")))
}
optsObj := call.Argument(1).ToObject(runtime)
size := int64(0)
contentType := ""
rangeEnabled := false
if optsObj != nil {
size = optsObj.Get("size").ToInteger()
contentType = optsObj.Get("contentType").String()
rangeEnabled = optsObj.Get("range").ToBoolean()
}
if size < 0 {
size = 0
}
opts := &ObjectURLOptions{
ContentType: contentType,
Size: size,
Range: rangeEnabled,
}
url, err := cfg.CreateObjectURL(opts, func(ctx context.Context, req ObjectURLOpenRequest) (io.ReadCloser, error) {
return openBlobObjectURLReader(ctx, loop, blobPipes, openValue, req)
})
if err != nil {
panic(runtime.NewGoError(err))
}
return runtime.ToValue(url)
}); err != nil {
return err
}
if err := runtime.Set("__gopeed_revoke_object_url", func(url string) {
if cfg.RevokeObjectURL == nil {
panic(runtime.NewGoError(fmt.Errorf("blob revoke handler not configured")))
}
if err := cfg.RevokeObjectURL(url); err != nil {
panic(runtime.NewGoError(err))
}
}); err != nil {
return err
}
fetchRegistry := newFetchRegistry()
if cfg.RegisterCleanup != nil {
cfg.RegisterCleanup(fetchRegistry.CloseAll)
}
if err := runtime.Set("__gopeed_fetch_open", func(call goja.FunctionCall) goja.Value {
reqMeta, err := exportFetchRequest(runtime, call.Argument(0))
if err != nil {
panic(runtime.NewGoError(err))
}
fingerprint := util.SafeGet[string](runtime, xhr.FingerprintMagicKey)
promise, resolve, reject := runtime.NewPromise()
go func() {
meta, err := fetchRegistry.Open(fingerprint, cfg.ProxyHandler, reqMeta)
ok := loop.RunOnLoop(func(runtime *goja.Runtime) {
if err != nil {
reject(runtime.NewGoError(err))
return
}
resolve(runtime.ToValue(meta))
})
if !ok && meta != nil {
fetchRegistry.Close(meta.ID)
}
}()
return runtime.ToValue(promise)
}); err != nil {
return err
}
if err := runtime.Set("__gopeed_fetch_read", func(call goja.FunctionCall) goja.Value {
id := call.Argument(0).String()
chunkSize := int(call.Argument(1).ToInteger())
promise, resolve, reject := runtime.NewPromise()
go func() {
chunk, done, err := fetchRegistry.Read(id, chunkSize)
ok := loop.RunOnLoop(func(runtime *goja.Runtime) {
if err != nil {
reject(runtime.NewGoError(err))
return
}
if done {
resolve(goja.Null())
return
}
resolve(runtime.ToValue(runtime.NewArrayBuffer(chunk)))
})
if !ok {
fetchRegistry.Close(id)
}
}()
return runtime.ToValue(promise)
}); err != nil {
return err
}
if err := runtime.Set("__gopeed_fetch_close", func(id string) {
fetchRegistry.Close(id)
}); err != nil {
return err
}
if err := runtime.Set("__gopeed_fetch_abort", func(id string, reason string) {
fetchRegistry.Abort(id, reason)
}); err != nil {
return err
}
if err := runtime.Set("__gopeed_blob_pipe_chunk", func(call goja.FunctionCall) goja.Value {
id := call.Argument(0).String()
ok := blobPipes.Push(id, exportBytes(call.Argument(1)))
return runtime.ToValue(ok)
}); err != nil {
return err
}
if err := runtime.Set("__gopeed_blob_pipe_close", func(id string) {
blobPipes.Close(id, nil)
}); err != nil {
return err
}
if err := runtime.Set("__gopeed_blob_pipe_error", func(id string, message string) {
blobPipes.Close(id, errors.New(message))
}); err != nil {
return err
}
_, err := runtime.RunString(script)
return err
}
func openBlobObjectURLReader(ctx context.Context, loop *eventloop.EventLoop, pipes *blobPipeRegistry, openValue goja.Value, req ObjectURLOpenRequest) (io.ReadCloser, error) {
if pipes == nil {
return nil, fmt.Errorf("blob pipe registry not configured")
}
pipe := newBlobPipe(ctx, loop, pipes)
pipes.Add(pipe)
_, err := runOnLoop(loop, func(runtime *goja.Runtime) (goja.Value, error) {
fnVal := runtime.Get("__gopeed_blob_pipe_source")
fn, ok := goja.AssertFunction(fnVal)
if !ok {
return nil, fmt.Errorf("blob pipe helper is not callable")
}
request := map[string]any{
"offset": req.Offset,
"end": req.End,
}
return fn(nil, openValue, runtime.ToValue(request), runtime.ToValue(pipe.id))
})
if err != nil {
pipe.Close()
return nil, err
}
return pipe, nil
}
type blobPipeRegistry struct {
mu sync.Mutex
pipes map[string]*blobPipe
next atomic.Uint64
}
func newBlobPipeRegistry() *blobPipeRegistry {
return &blobPipeRegistry{
pipes: map[string]*blobPipe{},
}
}
func (r *blobPipeRegistry) Add(pipe *blobPipe) {
r.mu.Lock()
r.pipes[pipe.id] = pipe
r.mu.Unlock()
}
func (r *blobPipeRegistry) Remove(id string) {
r.mu.Lock()
delete(r.pipes, id)
r.mu.Unlock()
}
func (r *blobPipeRegistry) Push(id string, chunk []byte) bool {
r.mu.Lock()
pipe := r.pipes[id]
r.mu.Unlock()
if pipe == nil {
return false
}
return pipe.Push(chunk)
}
func (r *blobPipeRegistry) Close(id string, err error) {
r.mu.Lock()
pipe := r.pipes[id]
r.mu.Unlock()
if pipe != nil {
pipe.CloseWithError(err)
}
}
func (r *blobPipeRegistry) NewID() string {
return fmt.Sprintf("pipe-%d", r.next.Add(1))
}
type blobPipeItem struct {
chunk []byte
err error
done bool
}
type blobPipe struct {
id string
loop *eventloop.EventLoop
registry *blobPipeRegistry
ctx context.Context
cancel context.CancelFunc
ch chan blobPipeItem
mu sync.Mutex
buf []byte
closed bool
once sync.Once
}
func newBlobPipe(ctx context.Context, loop *eventloop.EventLoop, registry *blobPipeRegistry) *blobPipe {
pipeCtx, cancel := context.WithCancel(ctx)
return &blobPipe{
id: registry.NewID(),
loop: loop,
registry: registry,
ctx: pipeCtx,
cancel: cancel,
ch: make(chan blobPipeItem, 8),
}
}
func (p *blobPipe) Push(chunk []byte) bool {
if len(chunk) == 0 {
return true
}
buf := append([]byte(nil), chunk...)
select {
case p.ch <- blobPipeItem{chunk: buf}:
return true
case <-p.ctx.Done():
return false
}
}
func (p *blobPipe) CloseWithError(err error) {
p.once.Do(func() {
p.registry.Remove(p.id)
select {
case p.ch <- blobPipeItem{done: true, err: err}:
case <-p.ctx.Done():
}
})
}
func (p *blobPipe) Read(dst []byte) (int, error) {
p.mu.Lock()
if len(p.buf) > 0 {
n := copy(dst, p.buf)
p.buf = p.buf[n:]
p.mu.Unlock()
return n, nil
}
p.mu.Unlock()
select {
case item := <-p.ch:
if item.done {
if item.err != nil {
return 0, item.err
}
return 0, io.EOF
}
n := copy(dst, item.chunk)
if n < len(item.chunk) {
p.mu.Lock()
p.buf = append(p.buf, item.chunk[n:]...)
p.mu.Unlock()
}
return n, nil
case <-p.ctx.Done():
return 0, p.ctx.Err()
}
}
func (p *blobPipe) Close() error {
p.mu.Lock()
if p.closed {
p.mu.Unlock()
return nil
}
p.closed = true
p.mu.Unlock()
p.registry.Remove(p.id)
p.cancel()
p.cancelSourceReader()
return nil
}
func (p *blobPipe) cancelSourceReader() {
if p.loop == nil {
return
}
// Invoke the JavaScript cancellation before returning from Close. The
// source session may be released immediately after the HTTP handler closes
// this reader, so merely queueing the callback could let the engine stop
// before reader.cancel() is called.
_, _ = runOnLoop(p.loop, func(runtime *goja.Runtime) (goja.Value, error) {
fnVal := runtime.Get("__gopeed_blob_cancel_pipe_source")
fn, ok := goja.AssertFunction(fnVal)
if !ok {
return goja.Undefined(), nil
}
_, err := fn(nil, runtime.ToValue(p.id), runtime.ToValue("blob request closed"))
return goja.Undefined(), err
})
}
func exportBytes(value goja.Value) []byte {
if value == nil || goja.IsNull(value) || goja.IsUndefined(value) {
return nil
}
if ab, ok := value.Export().(goja.ArrayBuffer); ok {
return ab.Bytes()
}
if b, ok := value.Export().([]byte); ok {
return b
}
return nil
}
type blobObjectURLReader struct {
ctx context.Context
loop *eventloop.EventLoop
id string
mu sync.Mutex
buf []byte
closed bool
}
func (r *blobObjectURLReader) Read(p []byte) (int, error) {
r.mu.Lock()
if len(r.buf) > 0 {
n := copy(p, r.buf)
r.buf = r.buf[n:]
r.mu.Unlock()
return n, nil
}
closed := r.closed
r.mu.Unlock()
if closed {
return 0, io.EOF
}
if err := r.ctx.Err(); err != nil {
return 0, err
}
chunkSize := len(p)
if chunkSize <= 0 {
chunkSize = 32 * 1024
}
value, err := runOnLoop(r.loop, func(runtime *goja.Runtime) (goja.Value, error) {
fnVal := runtime.Get("__gopeed_blob_read_source")
fn, ok := goja.AssertFunction(fnVal)
if !ok {
return nil, fmt.Errorf("blob read helper is not callable")
}
return fn(nil, runtime.ToValue(r.id), runtime.ToValue(chunkSize))
})
if err != nil {
return 0, err
}
if goja.IsNull(value) || goja.IsUndefined(value) {
return 0, io.EOF
}
var chunk []byte
if ab, ok := value.Export().(goja.ArrayBuffer); ok {
chunk = ab.Bytes()
} else if b, ok := value.Export().([]byte); ok {
chunk = b
} else {
return 0, fmt.Errorf("blob read helper returned %T", value.Export())
}
if len(chunk) == 0 {
return 0, nil
}
n := copy(p, chunk)
if n < len(chunk) {
r.mu.Lock()
r.buf = append(r.buf, chunk[n:]...)
r.mu.Unlock()
}
return n, nil
}
func (r *blobObjectURLReader) Close() error {
r.mu.Lock()
if r.closed {
r.mu.Unlock()
return nil
}
r.closed = true
r.mu.Unlock()
_, err := runOnLoop(r.loop, func(runtime *goja.Runtime) (goja.Value, error) {
fnVal := runtime.Get("__gopeed_blob_close_source")
fn, ok := goja.AssertFunction(fnVal)
if !ok {
return nil, nil
}
return fn(nil, runtime.ToValue(r.id))
})
return err
}
func runOnLoop(loop *eventloop.EventLoop, fn func(runtime *goja.Runtime) (goja.Value, error)) (goja.Value, error) {
type result struct {
value goja.Value
err error
}
ch := make(chan result, 1)
ok := loop.RunOnLoop(func(runtime *goja.Runtime) {
send := func(value goja.Value, err error) {
select {
case ch <- result{value: value, err: err}:
default:
}
}
defer func() {
if r := recover(); r != nil {
switch v := r.(type) {
case error:
send(nil, v)
case goja.Value:
send(nil, exportJSError(v))
default:
send(nil, fmt.Errorf("panic: %v", r))
}
}
}()
value, err := fn(runtime)
if err != nil {
send(nil, err)
return
}
if promise, ok := value.Export().(*goja.Promise); ok {
switch promise.State() {
case goja.PromiseStateFulfilled:
send(promise.Result(), nil)
return
case goja.PromiseStateRejected:
send(nil, exportJSError(promise.Result()))
return
default:
thenVal := value.ToObject(runtime).Get("then")
thenFn, ok := goja.AssertFunction(thenVal)
if !ok {
send(nil, errors.New("promise.then is not callable"))
return
}
onFulfilled := runtime.ToValue(func(call goja.FunctionCall) goja.Value {
send(call.Argument(0), nil)
return goja.Undefined()
})
onRejected := runtime.ToValue(func(call goja.FunctionCall) goja.Value {
send(nil, exportJSError(call.Argument(0)))
return goja.Undefined()
})
if _, err := thenFn(value, onFulfilled, onRejected); err != nil {
send(nil, err)
}
return
}
}
send(value, nil)
})
if !ok {
return nil, errors.New("engine loop terminated")
}
res := <-ch
return res.value, res.err
}
func exportJSError(value goja.Value) error {
if value == nil || goja.IsUndefined(value) || goja.IsNull(value) {
return errors.New("promise rejected")
}
if err, ok := value.Export().(error); ok {
return err
}
stack := value.String()
if ro, ok := value.(*goja.Object); ok {
stackVal := ro.Get("stack")
if stackVal != nil && stackVal.String() != "" {
stack = stackVal.String()
}
}
return errors.New(stack)
}
type fetchRegistry struct {
mu sync.Mutex
streams map[string]*fetchStream
ctx context.Context
cancel context.CancelFunc
closed bool
}
type fetchStream struct {
body io.ReadCloser
cancel context.CancelFunc
closeOnce sync.Once
ctx context.Context
ch chan fetchChunk
readMu sync.Mutex
pending []byte
}
type fetchChunk struct {
data []byte
err error
}
type fetchRequest struct {
URL string
Method string
Headers [][2]string
Body any
Redirect string
Credentials string
}
type fetchFormDataEntry struct {
name string
value any
}
type fetchFormDataSnapshot struct {
entries []fetchFormDataEntry
}
type fetchOpenMeta struct {
ID string `json:"id"`
Status int `json:"status"`
StatusText string `json:"statusText"`
URL string `json:"url"`
Headers [][2]string `json:"headers"`
}
func newFetchRegistry() *fetchRegistry {
ctx, cancel := context.WithCancel(context.Background())
return &fetchRegistry{
streams: make(map[string]*fetchStream),
ctx: ctx,
cancel: cancel,
}
}
func exportFetchRequest(runtime *goja.Runtime, value goja.Value) (*fetchRequest, error) {
obj := value.ToObject(runtime)
if obj == nil {
return nil, fmt.Errorf("invalid fetch request")
}
meta := &fetchRequest{
URL: obj.Get("url").String(),
Method: obj.Get("method").String(),
Redirect: obj.Get("redirect").String(),
Credentials: obj.Get("credentials").String(),
}
if meta.Method == "" {
meta.Method = http.MethodGet
}
if headersVal := obj.Get("headers"); headersVal != nil && !goja.IsUndefined(headersVal) && !goja.IsNull(headersVal) {
if exported, ok := headersVal.Export().([]any); ok {
for _, item := range exported {
pair, ok := item.([]any)
if !ok || len(pair) != 2 {
continue
}
meta.Headers = append(meta.Headers, [2]string{fmt.Sprint(pair[0]), fmt.Sprint(pair[1])})
}
}
}
bodyVal := obj.Get("body")
if bodyVal != nil && !goja.IsUndefined(bodyVal) && !goja.IsNull(bodyVal) {
body, err := snapshotFetchBody(bodyVal.Export())
if err != nil {
return nil, err
}
meta.Body = body
}
return meta, nil
}
// snapshotFetchBody runs on the JavaScript event loop before fetch work is
// handed to a background goroutine. Values exported by goja may still share
// mutable storage with JavaScript (notably ArrayBuffer and FormData), so the
// background request must only retain immutable Go-owned data.
func snapshotFetchBody(body any) (any, error) {
switch v := body.(type) {
case nil, string:
return v, nil
case []byte:
return append([]byte(nil), v...), nil
case goja.ArrayBuffer:
return append([]byte(nil), v.Bytes()...), nil
case *file.File:
return snapshotFetchFile(v), nil
case *formdata.FormData:
entries := v.Entries()
snapshot := &fetchFormDataSnapshot{
entries: make([]fetchFormDataEntry, 0, len(entries)),
}
for _, entry := range entries {
pair, ok := entry.([]any)
if !ok || len(pair) != 2 {
return nil, fmt.Errorf("invalid FormData entry %T", entry)
}
name, ok := pair[0].(string)
if !ok {
return nil, fmt.Errorf("invalid FormData field name %T", pair[0])
}
value := pair[1]
if formFile, ok := value.(*file.File); ok {
value = snapshotFetchFile(formFile)
}
snapshot.entries = append(snapshot.entries, fetchFormDataEntry{
name: name,
value: value,
})
}
return snapshot, nil
default:
if typed, ok := v.(interface{ Bytes() []byte }); ok {
return append([]byte(nil), typed.Bytes()...), nil
}
return v, nil
}
}
func snapshotFetchFile(src *file.File) *file.File {
if src == nil {
return nil
}
return &file.File{
Reader: src.Reader,
Closer: src.Closer,
Name: src.Name,
Size: src.Size,
}
}
func (r *fetchRegistry) Open(fingerprint string, proxyHandler func(r *http.Request) (*url.URL, error), reqMeta *fetchRequest) (*fetchOpenMeta, error) {
r.mu.Lock()
if r.closed {
r.mu.Unlock()
return nil, context.Canceled
}
registryCtx := r.ctx
r.mu.Unlock()
client := req.C()
if proxyHandler != nil {
client.SetProxy(proxyHandler)
}
setFetchFingerprint(client, fingerprint)
contentType, body, err := buildFetchBody(reqMeta.Body)
if err != nil {
return nil, err
}
ctx, cancel := context.WithCancel(registryCtx)
keepCancel := false
defer func() {
if !keepCancel {
cancel()
}
}()
reqBuilder := client.R()
reqBuilder.SetContext(ctx)
reqBuilder.DisableAutoReadResponse()
for _, header := range reqMeta.Headers {
reqBuilder.SetHeader(header[0], header[1])
}
if body != nil && reqMeta.Method != http.MethodGet && reqMeta.Method != http.MethodHead {
reqBuilder.SetBody(body)
if contentType != "" && !hasHeader(reqMeta.Headers, "Content-Type") {
reqBuilder.SetHeader("Content-Type", contentType)
}
}
client.SetRedirectPolicy(func(req *http.Request, via []*http.Request) error {
switch reqMeta.Redirect {
case "manual":
return http.ErrUseLastResponse
case "error":
return fmt.Errorf("redirect failed")
default:
if len(via) > 20 {
return fmt.Errorf("too many redirects")
}
return nil
}
})
resp, err := reqBuilder.Send(reqMeta.Method, reqMeta.URL)
if err != nil {
var ne net.Error
if errorsAsTimeout(err, &ne) {
return nil, fmt.Errorf("Network request timed out")
}
return nil, fmt.Errorf("Network request failed: %w", err)
}
id := fmt.Sprintf("%d", time.Now().UnixNano())
meta := &fetchOpenMeta{
ID: id,
Status: resp.StatusCode,
StatusText: resp.Status,
URL: reqMeta.URL,
}
if resp.Response != nil && resp.Response.Request != nil && resp.Response.Request.URL != nil {
responseURL := *resp.Response.Request.URL
responseURL.Fragment = ""
meta.URL = responseURL.String()
}
for key, values := range resp.Header {
meta.Headers = append(meta.Headers, [2]string{key, strings.Join(values, ", ")})
}
bodyCloser := io.NopCloser(strings.NewReader(""))
if resp.Response != nil && resp.Response.Body != nil {
bodyCloser = resp.Response.Body
}
stream := &fetchStream{
body: bodyCloser,
cancel: cancel,
ctx: ctx,
ch: make(chan fetchChunk, 8),
}
r.mu.Lock()
if r.closed {
r.mu.Unlock()
closeFetchStream(stream)
return nil, context.Canceled
}
r.streams[id] = stream
r.mu.Unlock()
keepCancel = true
go stream.readLoop()
return meta, nil
}
func (r *fetchRegistry) Read(id string, chunkSize int) ([]byte, bool, error) {
stream := r.get(id)
if stream == nil {
return nil, true, nil
}
stream.readMu.Lock()
defer stream.readMu.Unlock()
if len(stream.pending) > 0 {
n := len(stream.pending)
if chunkSize > 0 && n > chunkSize {
n = chunkSize
}
chunk := append([]byte(nil), stream.pending[:n]...)
stream.pending = stream.pending[n:]
return chunk, false, nil
}
select {
case item, ok := <-stream.ch:
if !ok {
r.Close(id)
return nil, true, nil
}
if item.err != nil {
r.Close(id)
return nil, false, item.err
}
if chunkSize > 0 && len(item.data) > chunkSize {
head := append([]byte(nil), item.data[:chunkSize]...)
stream.pending = append(stream.pending[:0], item.data[chunkSize:]...)
return head, false, nil
}
return item.data, false, nil
case <-stream.ctx.Done():
return nil, false, stream.ctx.Err()
}
}
func (s *fetchStream) readLoop() {
defer close(s.ch)
buf := make([]byte, 64*1024)
for {
n, err := s.body.Read(buf)
if n > 0 {
chunk := append([]byte(nil), buf[:n]...)
select {
case s.ch <- fetchChunk{data: chunk}:
case <-s.ctx.Done():
return
}
}
if err == io.EOF {
return
}
if err != nil {
select {
case s.ch <- fetchChunk{err: err}:
case <-s.ctx.Done():
}
return
}
}
}
func (r *fetchRegistry) Close(id string) {
r.mu.Lock()
stream := r.streams[id]
delete(r.streams, id)
r.mu.Unlock()
closeFetchStream(stream)
}
// CloseAll permanently closes the registry and cancels both active and
// in-flight fetches. It is safe to call more than once.
func (r *fetchRegistry) CloseAll() {
if r == nil {
return
}
r.mu.Lock()
if r.closed {
r.mu.Unlock()
return
}
r.closed = true
if r.cancel != nil {
r.cancel()
}
streams := r.streams
r.streams = make(map[string]*fetchStream)
r.mu.Unlock()
for _, stream := range streams {
closeFetchStream(stream)
}
}
func closeFetchStream(stream *fetchStream) {
if stream == nil {
return
}
stream.closeOnce.Do(func() {
if stream.cancel != nil {
stream.cancel()
}
_ = stream.body.Close()
})
}
func (r *fetchRegistry) Abort(id string, _ string) {
r.Close(id)
}
func (r *fetchRegistry) get(id string) *fetchStream {
r.mu.Lock()
defer r.mu.Unlock()
return r.streams[id]
}
func buildFetchBody(body any) (string, any, error) {
switch v := body.(type) {
case nil:
return "", nil, nil
case string:
return "text/plain;charset=UTF-8", v, nil
case []byte:
return "application/octet-stream", v, nil
case goja.ArrayBuffer:
return "application/octet-stream", v.Bytes(), nil
case *file.File:
return "application/octet-stream", v.Reader, nil
case *fetchFormDataSnapshot:
pr, pw := io.Pipe()
mw := xhr.NewMultipart(pw)
for _, entry := range v.entries {
switch vv := entry.value.(type) {
case string:
if err := mw.WriteField(entry.name, vv); err != nil {
_ = pw.CloseWithError(err)
return "", nil, err
}
case *file.File:
if err := mw.WriteFile(entry.name, vv); err != nil {
_ = pw.CloseWithError(err)
return "", nil, err
}
}
}
go func() {
if err := mw.Send(); err != nil {
_ = pw.CloseWithError(err)
return
}
if err := mw.Close(); err != nil {
_ = pw.CloseWithError(err)
return
}
_ = pw.Close()
}()
return mw.FormDataContentType(), pr, nil
default:
if typed, ok := v.(interface{ Bytes() []byte }); ok {
return "application/octet-stream", typed.Bytes(), nil
}
return "", fmt.Sprint(v), nil
}
}
func hasHeader(headers [][2]string, key string) bool {
for _, header := range headers {
if strings.EqualFold(header[0], key) {
return true
}
}
return false
}
func setFetchFingerprint(client *req.Client, fingerprint string) {
switch fingerprint {
case "chrome":
client.ImpersonateChrome()
case "firefox":
client.ImpersonateFirefox()
case "safari":
client.ImpersonateSafari()
}
}
func errorsAsTimeout(err error, target *net.Error) bool {
return errors.As(err, target) && (*target).Timeout()
}
@@ -0,0 +1,278 @@
package stream_test
import (
"context"
"errors"
"fmt"
"io"
"net/http"
"net/http/httptest"
"sync/atomic"
"testing"
"time"
"github.com/GopeedLab/gopeed/pkg/download/engine"
"github.com/GopeedLab/gopeed/pkg/download/engine/inject/stream"
)
type createdObjectURL struct {
opts *stream.ObjectURLOptions
open stream.ObjectURLOpener
}
func TestResponseBlobObjectURLReadsResponseStream(t *testing.T) {
const payload = "response blob payload"
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.Header().Set("Content-Type", "text/plain")
_, _ = io.WriteString(w, payload[:8])
w.(http.Flusher).Flush() // force an unknown-length/chunked response
_, _ = io.WriteString(w, payload[8:])
}))
defer server.Close()
created := make(chan createdObjectURL, 1)
engine, cleanup := newStreamTestEngine(t, created)
defer func() {
cleanup()
engine.Close()
}()
value, err := engine.RunString(fmt.Sprintf(`
(async () => {
const response = await fetch(%q);
const blob = await response.blob();
return __gopeed_blob_create_object_url(blob);
})()
`, server.URL))
if err != nil {
t.Fatal(err)
}
if value != "blob:test-1" {
t.Fatalf("unexpected object URL: %#v", value)
}
object := <-created
if object.opts.Size != int64(len(payload)) || object.opts.ContentType != "text/plain" {
t.Fatalf("unexpected object URL options: %#v", object.opts)
}
if !object.opts.Range {
t.Fatal("materialized response Blob should advertise range support")
}
for i := 0; i < 2; i++ {
reader, err := object.open(context.Background(), stream.ObjectURLOpenRequest{Offset: 0, End: -1})
if err != nil {
t.Fatal(err)
}
data, readErr := io.ReadAll(reader)
_ = reader.Close()
if readErr != nil {
t.Fatal(readErr)
}
if got := string(data); got != payload {
t.Fatalf("unexpected response Blob data on open %d: %q", i+1, got)
}
}
rangeReader, err := object.open(context.Background(), stream.ObjectURLOpenRequest{Offset: 9, End: 12})
if err != nil {
t.Fatal(err)
}
rangeData, readErr := io.ReadAll(rangeReader)
_ = rangeReader.Close()
if readErr != nil {
t.Fatal(readErr)
}
if got, want := string(rangeData), payload[9:13]; got != want {
t.Fatalf("unexpected materialized Blob range: got %q want %q", got, want)
}
}
func TestResponseBlobRejectsTruncatedResponse(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.Header().Set("Content-Length", "64")
_, _ = io.WriteString(w, "partial")
}))
defer server.Close()
created := make(chan createdObjectURL, 1)
engine, cleanup := newStreamTestEngine(t, created)
defer func() {
cleanup()
engine.Close()
}()
value, err := engine.RunString(fmt.Sprintf(`
(async () => {
const response = await fetch(%q);
try {
await response.blob();
return "resolved";
} catch (_) {
return "rejected";
}
})()
`, server.URL))
if err != nil {
t.Fatal(err)
}
if value != "rejected" {
t.Fatalf("truncated response unexpectedly materialized a Blob: %#v", value)
}
}
func TestEmptyBlobObjectURL(t *testing.T) {
created := make(chan createdObjectURL, 1)
engine, cleanup := newStreamTestEngine(t, created)
defer func() {
cleanup()
engine.Close()
}()
value, err := engine.RunString(`
__gopeed_blob_create_object_url(new Blob([], { type: "application/empty" }))
`)
if err != nil {
t.Fatal(err)
}
if value != "blob:test-1" {
t.Fatalf("unexpected object URL: %#v", value)
}
object := <-created
if object.opts.Size != 0 || object.opts.Range {
t.Fatalf("unexpected empty Blob options: %#v", object.opts)
}
reader, err := object.open(context.Background(), stream.ObjectURLOpenRequest{Offset: 0, End: -1})
if err != nil {
t.Fatal(err)
}
defer reader.Close()
data, err := io.ReadAll(reader)
if err != nil {
t.Fatal(err)
}
if len(data) != 0 {
t.Fatalf("expected empty Blob, got %q", data)
}
}
func TestBlobObjectURLReaderCloseCancelsPendingJSReader(t *testing.T) {
created := make(chan createdObjectURL, 1)
engine, cleanup := newStreamTestEngine(t, created)
defer func() {
cleanup()
engine.Close()
}()
value, err := engine.RunString(`
globalThis.__blobPipeCancelState = {
readCalls: 0,
cancelCalls: 0,
releaseCalls: 0,
cancelReason: "",
resolveRead: null,
};
const pendingSource = {
getReader() {
return {
read() {
__blobPipeCancelState.readCalls++;
return new Promise((resolve) => {
__blobPipeCancelState.resolveRead = resolve;
});
},
cancel(reason) {
__blobPipeCancelState.cancelCalls++;
__blobPipeCancelState.cancelReason = String(reason);
if (__blobPipeCancelState.resolveRead) {
__blobPipeCancelState.resolveRead({ done: true, value: undefined });
}
return Promise.resolve();
},
releaseLock() {
__blobPipeCancelState.releaseCalls++;
},
};
},
};
__gopeed_blob_create_object_url(async () => pendingSource);
`)
if err != nil {
t.Fatal(err)
}
if value != "blob:test-1" {
t.Fatalf("unexpected object URL: %#v", value)
}
object := <-created
reader, err := object.open(context.Background(), stream.ObjectURLOpenRequest{Offset: 0, End: -1})
if err != nil {
t.Fatal(err)
}
readDone := make(chan error, 1)
go func() {
_, readErr := reader.Read(make([]byte, 1))
readDone <- readErr
}()
waitForJSValue(t, engine, `String(__blobPipeCancelState.readCalls)`, "1")
if err := reader.Close(); err != nil {
t.Fatal(err)
}
if err := reader.Close(); err != nil {
t.Fatal(err)
}
select {
case readErr := <-readDone:
if !errors.Is(readErr, context.Canceled) {
t.Fatalf("unexpected pending read result: %v", readErr)
}
case <-time.After(2 * time.Second):
t.Fatal("pending Go read was not released by Close")
}
waitForJSValue(t, engine, `JSON.stringify({
cancelCalls: __blobPipeCancelState.cancelCalls,
releaseCalls: __blobPipeCancelState.releaseCalls,
cancelReason: __blobPipeCancelState.cancelReason,
})`, `{"cancelCalls":1,"releaseCalls":1,"cancelReason":"blob request closed"}`)
}
func waitForJSValue(t *testing.T, engine *engine.Engine, expression, want string) {
t.Helper()
deadline := time.Now().Add(2 * time.Second)
var got any
for time.Now().Before(deadline) {
var err error
got, err = engine.RunString(expression)
if err != nil {
t.Fatal(err)
}
if got == want {
return
}
time.Sleep(10 * time.Millisecond)
}
t.Fatalf("timed out waiting for JavaScript value %q, got %#v", want, got)
}
func newStreamTestEngine(t *testing.T, created chan<- createdObjectURL) (*engine.Engine, func()) {
t.Helper()
var next atomic.Int64
var cleanup func()
e := engine.NewEngine(&engine.Config{
StreamConfig: &stream.Config{
CreateObjectURL: func(opts *stream.ObjectURLOptions, open stream.ObjectURLOpener) (string, error) {
created <- createdObjectURL{opts: opts, open: open}
return fmt.Sprintf("blob:test-%d", next.Add(1)), nil
},
RevokeObjectURL: func(string) error { return nil },
RegisterCleanup: func(fn func()) {
cleanup = fn
},
},
})
if cleanup == nil {
t.Fatal("stream cleanup was not registered")
}
return e, cleanup
}
@@ -0,0 +1,191 @@
package stream
import (
"context"
"io"
"strings"
"sync/atomic"
"testing"
"github.com/GopeedLab/gopeed/pkg/download/engine/inject/file"
"github.com/GopeedLab/gopeed/pkg/download/engine/inject/formdata"
"github.com/dop251/goja"
)
func TestExportFetchRequestSnapshotsArrayBuffer(t *testing.T) {
runtime := goja.New()
buffer := runtime.NewArrayBuffer([]byte("before"))
request := runtime.NewObject()
setFetchRequestDefaults(t, request)
if err := request.Set("body", buffer); err != nil {
t.Fatal(err)
}
meta, err := exportFetchRequest(runtime, request)
if err != nil {
t.Fatal(err)
}
buffer.Bytes()[0] = 'X'
body, ok := meta.Body.([]byte)
if !ok {
t.Fatalf("expected byte snapshot, got %T", meta.Body)
}
if got := string(body); got != "before" {
t.Fatalf("ArrayBuffer snapshot changed after export: %q", got)
}
}
func TestExportFetchRequestSnapshotsFormData(t *testing.T) {
runtime := goja.New()
runtime.SetFieldNameMapper(goja.TagFieldNameMapper("json", true))
if err := formdata.Enable(runtime); err != nil {
t.Fatal(err)
}
value, err := runtime.RunString(`
globalThis.form = new FormData();
form.append("field", "before");
form;
`)
if err != nil {
t.Fatal(err)
}
request := runtime.NewObject()
setFetchRequestDefaults(t, request)
if err := request.Set("body", value); err != nil {
t.Fatal(err)
}
meta, err := exportFetchRequest(runtime, request)
if err != nil {
t.Fatal(err)
}
if _, err := runtime.RunString(`form.set("field", "after")`); err != nil {
t.Fatal(err)
}
body, ok := meta.Body.(*fetchFormDataSnapshot)
if !ok {
t.Fatalf("expected FormData snapshot, got %T", meta.Body)
}
if len(body.entries) != 1 {
t.Fatalf("expected one FormData entry, got %d", len(body.entries))
}
if got := body.entries[0]; got.name != "field" || got.value != "before" {
t.Fatalf("unexpected FormData snapshot: %#v", got)
}
}
func setFetchRequestDefaults(t *testing.T, request *goja.Object) {
t.Helper()
for key, value := range map[string]any{
"url": "https://example.test",
"method": "POST",
"redirect": "follow",
"credentials": "same-origin",
} {
if err := request.Set(key, value); err != nil {
t.Fatal(err)
}
}
}
func TestSnapshotFetchBodyCopiesFileMetadata(t *testing.T) {
source := &file.File{
Reader: strings.NewReader("payload"),
Name: "before.txt",
Size: 7,
}
value, err := snapshotFetchBody(source)
if err != nil {
t.Fatal(err)
}
snapshot, ok := value.(*file.File)
if !ok {
t.Fatalf("expected file snapshot, got %T", value)
}
source.Name = "after.txt"
source.Size = 99
if snapshot.Name != "before.txt" || snapshot.Size != 7 {
t.Fatalf("file metadata was not snapshotted: %#v", snapshot)
}
}
func TestFetchRegistryCloseAll(t *testing.T) {
registry := newFetchRegistry()
streamCtx, cancel := context.WithCancel(context.Background())
body := &trackingReadCloser{}
registry.streams["active"] = &fetchStream{
body: body,
cancel: cancel,
ctx: streamCtx,
ch: make(chan fetchChunk),
}
registry.CloseAll()
registry.CloseAll()
select {
case <-streamCtx.Done():
default:
t.Fatal("active fetch context was not canceled")
}
if got := body.closeCalls.Load(); got != 1 {
t.Fatalf("expected body to close once, got %d", got)
}
registry.mu.Lock()
closed := registry.closed
remaining := len(registry.streams)
registry.mu.Unlock()
if !closed || remaining != 0 {
t.Fatalf("registry not fully closed: closed=%v remaining=%d", closed, remaining)
}
if _, err := registry.Open("", nil, &fetchRequest{URL: "https://example.test"}); err == nil {
t.Fatal("expected a closed registry to reject new fetches")
}
}
func TestFetchRegistryReadPreservesChunkOrder(t *testing.T) {
registry := newFetchRegistry()
ctx, cancel := context.WithCancel(context.Background())
stream := &fetchStream{
body: io.NopCloser(strings.NewReader("")),
cancel: cancel,
ctx: ctx,
ch: make(chan fetchChunk, 2),
}
stream.ch <- fetchChunk{data: []byte("abc")}
stream.ch <- fetchChunk{data: []byte("def")}
registry.streams["ordered"] = stream
defer registry.CloseAll()
var got strings.Builder
for range 4 {
chunk, done, err := registry.Read("ordered", 2)
if err != nil {
t.Fatal(err)
}
if done {
t.Fatal("stream ended before buffered chunks were read")
}
got.Write(chunk)
}
if got.String() != "abcdef" {
t.Fatalf("fetch chunks were reordered: %q", got.String())
}
}
type trackingReadCloser struct {
closeCalls atomic.Int32
}
func (r *trackingReadCloser) Read([]byte) (int, error) {
return 0, io.EOF
}
func (r *trackingReadCloser) Close() error {
r.closeCalls.Add(1)
return nil
}
+964
View File
@@ -0,0 +1,964 @@
(function () {
const createBlobObjectURL = globalThis.__gopeed_create_blob_object_url;
const revokeObjectURL = globalThis.__gopeed_revoke_object_url;
const fetchOpen = globalThis.__gopeed_fetch_open;
const fetchRead = globalThis.__gopeed_fetch_read;
const fetchClose = globalThis.__gopeed_fetch_close;
const fetchAbort = globalThis.__gopeed_fetch_abort;
if (typeof globalThis.ReadableStream === "undefined") {
class ReadableStreamDefaultController {
constructor(stream) {
this._stream = stream;
}
enqueue(chunk) {
this._stream._enqueue(chunk);
}
close() {
this._stream._close();
}
error(err) {
this._stream._error(err);
}
}
class ReadableStreamDefaultReader {
constructor(stream) {
this._stream = stream;
}
read() {
return this._stream._read();
}
cancel(reason) {
if (!this._stream) {
return Promise.resolve();
}
return this._stream.cancel(reason);
}
releaseLock() {
if (this._stream) {
this._stream._reader = null;
this._stream.locked = false;
this._stream = null;
}
}
}
class ReadableStream {
constructor(source = {}) {
this._source = source;
this._queue = [];
this._waiters = [];
this._closed = false;
this._errored = null;
this._reader = null;
this._pulling = false;
this.locked = false;
this._controller = new ReadableStreamDefaultController(this);
if (typeof source.start === "function") {
source.start(this._controller);
}
}
_enqueue(chunk) {
if (this._closed || this._errored) {
return;
}
if (this._waiters.length > 0) {
const waiter = this._waiters.shift();
waiter.resolve({ done: false, value: chunk });
return;
}
this._queue.push(chunk);
}
_close() {
this._closed = true;
while (this._waiters.length > 0) {
this._waiters.shift().resolve({ done: true, value: undefined });
}
}
_error(err) {
this._errored = err || new Error("ReadableStream error");
while (this._waiters.length > 0) {
this._waiters.shift().reject(this._errored);
}
}
_read() {
if (this._errored) {
return Promise.reject(this._errored);
}
this._markBodyUsed();
if (this._queue.length > 0) {
return Promise.resolve({ done: false, value: this._queue.shift() });
}
if (this._closed) {
return Promise.resolve({ done: true, value: undefined });
}
this._maybePull();
if (this._queue.length > 0) {
return Promise.resolve({ done: false, value: this._queue.shift() });
}
if (this._closed) {
return Promise.resolve({ done: true, value: undefined });
}
return new Promise((resolve, reject) => {
this._waiters.push({ resolve, reject });
this._maybePull();
});
}
_maybePull() {
if (this._closed || this._errored || this._pulling) {
return;
}
if (this._queue.length > 0) {
return;
}
if (!this._source || typeof this._source.pull !== "function") {
return;
}
this._pulling = true;
Promise.resolve(this._source.pull(this._controller))
.catch((err) => {
this._error(err);
})
.finally(() => {
this._pulling = false;
if (!this._closed && !this._errored && this._queue.length === 0 && this._waiters.length > 0) {
this._maybePull();
}
});
}
_markBodyUsed() {
if (typeof this.__gopeedMarkBodyUsed === "function") {
this.__gopeedMarkBodyUsed();
}
}
getReader() {
if (this.locked) {
throw new TypeError("ReadableStream is locked");
}
this.locked = true;
this._reader = new ReadableStreamDefaultReader(this);
return this._reader;
}
cancel(reason) {
this._markBodyUsed();
if (this._source && typeof this._source.cancel === "function") {
return Promise.resolve(this._source.cancel(reason)).then(() => {
this._close();
});
}
this._close();
return Promise.resolve();
}
async pipeTo(dest) {
const reader = this.getReader();
const writer = dest.getWriter();
try {
while (true) {
const { done, value } = await reader.read();
if (done) {
await writer.close();
break;
}
await writer.write(value);
}
} catch (err) {
await writer.abort(err);
throw err;
} finally {
reader.releaseLock();
writer.releaseLock();
}
}
}
globalThis.ReadableStream = ReadableStream;
}
if (typeof globalThis.WritableStream === "undefined") {
class WritableStreamDefaultWriter {
constructor(stream) {
if (stream.locked) {
throw new TypeError("WritableStream is locked");
}
this._stream = stream;
this._stream.locked = true;
}
write(chunk) {
return this._stream._write(chunk);
}
close() {
return this._stream._close();
}
abort(reason) {
return this._stream._abort(reason);
}
releaseLock() {
if (this._stream) {
this._stream.locked = false;
this._stream = null;
}
}
}
class WritableStream {
constructor(sink = {}) {
this._sink = sink;
this._observers = [];
this._state = "writable";
this.locked = false;
}
_targets() {
return [this._sink, ...this._observers];
}
_write(chunk) {
if (this._state !== "writable") {
return Promise.reject(new TypeError("WritableStream is not writable"));
}
let chain = Promise.resolve();
for (const target of this._targets()) {
if (target && typeof target.write === "function") {
chain = chain.then(() => target.write(chunk));
}
}
return chain;
}
_close() {
if (this._state !== "writable") {
return Promise.reject(new TypeError("WritableStream is not writable"));
}
this._state = "closed";
let chain = Promise.resolve();
for (const target of this._targets()) {
if (target && typeof target.close === "function") {
chain = chain.then(() => target.close());
}
}
return chain;
}
_abort(reason) {
if (this._state === "errored") {
return Promise.resolve();
}
this._state = "errored";
let chain = Promise.resolve();
for (const target of this._targets()) {
if (target && typeof target.abort === "function") {
chain = chain.then(() => target.abort(reason));
}
}
return chain;
}
_addObserver(observer) {
this._observers.push(observer);
}
getWriter() {
return new WritableStreamDefaultWriter(this);
}
}
globalThis.WritableStream = WritableStream;
}
if (typeof globalThis.TransformStream === "undefined") {
class TransformStream {
constructor(transformer = {}) {
let readableController;
this.readable = new ReadableStream({
start(controller) {
readableController = controller;
}
});
this.writable = new WritableStream({
async write(chunk) {
if (typeof transformer.transform === "function") {
await transformer.transform(chunk, readableController);
return;
}
readableController.enqueue(chunk);
},
async close() {
if (typeof transformer.flush === "function") {
await transformer.flush(readableController);
}
readableController.close();
},
async abort(reason) {
if (typeof transformer.abort === "function") {
await transformer.abort(reason);
}
readableController.error(reason || new Error("TransformStream aborted"));
}
});
}
}
globalThis.TransformStream = TransformStream;
}
if (typeof globalThis.ByteLengthQueuingStrategy === "undefined") {
globalThis.ByteLengthQueuingStrategy = class ByteLengthQueuingStrategy {
constructor({ highWaterMark }) {
this.highWaterMark = highWaterMark;
}
size(chunk) {
if (typeof chunk === "string") {
return chunk.length;
}
if (chunk && typeof chunk.byteLength === "number") {
return chunk.byteLength;
}
return 1;
}
};
}
if (typeof globalThis.CountQueuingStrategy === "undefined") {
globalThis.CountQueuingStrategy = class CountQueuingStrategy {
constructor({ highWaterMark }) {
this.highWaterMark = highWaterMark;
}
size() {
return 1;
}
};
}
function toUint8Array(chunk) {
if (chunk == null) {
return new Uint8Array(0);
}
if (chunk instanceof Uint8Array) {
return chunk;
}
if (typeof chunk === "string") {
return new TextEncoder().encode(chunk);
}
if (chunk instanceof ArrayBuffer) {
return new Uint8Array(chunk);
}
if (typeof ArrayBuffer !== "undefined" && ArrayBuffer.isView && ArrayBuffer.isView(chunk)) {
return new Uint8Array(chunk.buffer, chunk.byteOffset, chunk.byteLength);
}
if (typeof Blob !== "undefined" && chunk instanceof Blob) {
if (chunk._buffer instanceof Uint8Array) {
return chunk._buffer;
}
}
return new Uint8Array(0);
}
function createBodyReadableStream(owner) {
return new ReadableStream({
start(controller) {
Promise.resolve().then(async () => {
owner.bodyUsed = true;
if (owner._noBody) {
controller.close();
return;
}
if (owner._bodyArrayBuffer) {
controller.enqueue(toUint8Array(owner._bodyArrayBuffer));
controller.close();
return;
}
if (owner._bodyBlob) {
const data = await owner._bodyBlob.arrayBuffer();
controller.enqueue(new Uint8Array(data));
controller.close();
return;
}
if (owner._bodyText != null) {
controller.enqueue(toUint8Array(owner._bodyText));
controller.close();
return;
}
if (owner._bodyInit != null) {
controller.enqueue(toUint8Array(owner._bodyInit));
}
controller.close();
}).catch((err) => {
controller.error(err);
});
}
});
}
async function readAllFromStream(stream, asText) {
if (!stream) {
return asText ? "" : new Uint8Array(0);
}
const reader = stream.getReader();
const chunks = [];
let total = 0;
try {
while (true) {
const { done, value } = await reader.read();
if (done) {
break;
}
const chunk = toUint8Array(value);
chunks.push(chunk);
total += chunk.byteLength;
}
} finally {
reader.releaseLock();
}
const merged = new Uint8Array(total);
let offset = 0;
for (const chunk of chunks) {
merged.set(chunk, offset);
offset += chunk.byteLength;
}
if (asText) {
return new TextDecoder().decode(merged);
}
return merged;
}
function attachResponseStreaming(response, stream) {
response.__gopeedBodyStream = stream;
response.__gopeedBodyConsumed = false;
const ensureUnused = () => {
if (response.__gopeedBodyConsumed) {
throw new TypeError("Already read");
}
};
const markBodyUsed = () => {
ensureUnused();
response.__gopeedBodyConsumed = true;
response.bodyUsed = true;
};
if (stream) {
stream.__gopeedMarkBodyUsed = () => {
if (!response.__gopeedBodyConsumed) {
response.__gopeedBodyConsumed = true;
response.bodyUsed = true;
}
};
}
response.text = async function () {
ensureUnused();
markBodyUsed();
return readAllFromStream(stream, true);
};
response.arrayBuffer = async function () {
ensureUnused();
markBodyUsed();
const bytes = await readAllFromStream(stream, false);
return bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength);
};
response.blob = async function () {
ensureUnused();
markBodyUsed();
const contentType = this.headers && this.headers.get ? (this.headers.get("content-type") || "") : "";
// Response.blob() is a materializing body consumer. Keeping the live
// response stream behind an empty one-shot Blob makes retry, range and a
// second object-URL reader silently produce an empty file.
const bytes = await readAllFromStream(stream, false);
return new Blob([bytes], { type: contentType });
};
response.json = async function () {
const text = await this.text();
return JSON.parse(text);
};
return response;
}
if (typeof globalThis.Response === "function") {
const responseProto = globalThis.Response.prototype;
const bodyDescriptor = Object.getOwnPropertyDescriptor(responseProto, "body");
if (!bodyDescriptor || typeof bodyDescriptor.get !== "function") {
Object.defineProperty(responseProto, "body", {
configurable: true,
enumerable: true,
get() {
if (this.__gopeedBodyStream) {
return this.__gopeedBodyStream;
}
if (!this.__gopeedBodyStream) {
this.__gopeedBodyStream = createBodyReadableStream(this);
}
return this.__gopeedBodyStream;
}
});
}
}
const blobReaders = new Map();
function getValueTypeName(value) {
if (value === null) {
return "null";
}
if (value === undefined) {
return "undefined";
}
if (value && value.constructor && typeof value.constructor.name === "string" && value.constructor.name) {
return value.constructor.name;
}
return typeof value;
}
function toReadableStreamReader(value, sourceLabel) {
if (value && typeof value.getReader === "function") {
return value.getReader();
}
throw new TypeError(sourceLabel + " must return a ReadableStream, got " + getValueTypeName(value));
}
async function openBlobReadable(blob, request) {
const { offset, end } = request;
const sliced = blob.slice(offset, end >= offset ? end + 1 : undefined);
if (sliced && typeof sliced.stream === "function") {
try {
const stream = sliced.stream();
if (stream && typeof stream.getReader === "function") {
return stream;
}
} catch (_) {
}
}
if (sliced && typeof sliced.arrayBuffer === "function") {
const buffer = await sliced.arrayBuffer();
return new ReadableStream({
start(controller) {
controller.enqueue(new Uint8Array(buffer));
controller.close();
},
});
}
throw new TypeError("Blob source cannot be converted to a ReadableStream");
}
function describeObjectURLValue(value) {
if (value instanceof Blob) {
return {
kind: "blob",
value,
};
}
if (typeof value === "function") {
return {
kind: "opener",
openReadable: value,
sourceLabel: "gopeed.runtime.blob.createObjectURL opener function",
};
}
return {
kind: "other",
value,
};
}
function releaseReader(reader, reason) {
if (!reader) {
return;
}
try {
if (reason !== undefined && typeof reader.cancel === "function") {
const result = reader.cancel(reason);
if (result && typeof result.catch === "function") {
result.catch(function () {});
}
}
} catch (_) {
}
if (typeof reader.releaseLock === "function") {
try {
reader.releaseLock();
} catch (_) {
}
}
}
function normalizeBlobObjectURLOptions(value, options, defaultRange) {
const normalized = {
contentType: "",
size: 0,
range: !!defaultRange,
};
if (options && typeof options === "object") {
if (typeof options.contentType === "string") {
normalized.contentType = options.contentType;
}
if (Number.isFinite(Number(options.size)) && Number(options.size) > 0) {
normalized.size = Number(options.size);
}
if (typeof options.range === "boolean") {
normalized.range = options.range;
}
}
if (!normalized.contentType && value && typeof value.type === "string") {
normalized.contentType = value.type;
}
if (!normalized.size && value) {
if (Number.isFinite(Number(value.size)) && Number(value.size) > 0) {
normalized.size = Number(value.size);
} else if (value._buffer && Number.isFinite(Number(value._buffer.byteLength))) {
normalized.size = Number(value._buffer.byteLength);
}
}
return normalized;
}
function validateBlobObjectURLOptions(options) {
if (options.range && !(Number.isFinite(Number(options.size)) && Number(options.size) > 0)) {
throw new TypeError("gopeed.runtime.blob.createObjectURL options.range requires a positive options.size");
}
}
function normalizeOpenRequest(request) {
const offset = Math.max(0, Number(request && request.offset) || 0);
const endValue = Number(request && request.end);
const end = Number.isFinite(endValue) ? endValue : -1;
return { offset, end };
}
function yieldBlobPipeTask() {
return new Promise((resolve) => {
if (typeof setTimeout === "function") {
setTimeout(resolve, 0);
} else {
Promise.resolve().then(resolve);
}
});
}
function createPendingBlobReader(openReadable, request) {
let activeReader;
let ready;
function ensureReader() {
if (ready) {
return ready;
}
ready = (async function () {
const source = await openReadable(normalizeOpenRequest(request));
activeReader = toReadableStreamReader(source, "gopeed.runtime.blob.createObjectURL opener function");
return activeReader;
})();
return ready;
}
return {
async read() {
const reader = await ensureReader();
return reader.read();
},
cancel(reason) {
return ensureReader().then(function (reader) {
return reader.cancel(reason);
}, function () {});
},
releaseLock() {
if (!ready) {
return;
}
return ready.then(function (reader) {
return reader.releaseLock();
}, function () {});
}
};
}
globalThis.__gopeed_blob_open_source = function (openReadable, request) {
if (typeof openReadable !== "function") {
throw new TypeError("blob opener must be callable");
}
const id = crypto.randomUUID ? crypto.randomUUID() : String(Date.now()) + "-" + String(Math.random());
blobReaders.set(id, createPendingBlobReader(openReadable, request));
return id;
};
globalThis.__gopeed_blob_read_source = async function (id, chunkSize) {
const reader = blobReaders.get(id);
if (!reader) {
return null;
}
const { done, value } = await reader.read();
if (done) {
blobReaders.delete(id);
releaseReader(reader);
return null;
}
const chunk = value instanceof Uint8Array ? value : new Uint8Array(value);
const size = Math.max(1, Number(chunkSize) || chunk.byteLength);
if (chunk.byteLength <= size) {
return chunk.buffer.slice(chunk.byteOffset, chunk.byteOffset + chunk.byteLength);
}
const head = chunk.slice(0, size);
let offset = size;
blobReaders.set(id, {
read() {
if (offset >= chunk.byteLength) {
return reader.read();
}
const next = chunk.slice(offset, Math.min(offset + size, chunk.byteLength));
offset += next.byteLength;
return Promise.resolve({ done: false, value: next });
},
cancel(reason) {
return reader.cancel(reason);
},
releaseLock() {
return reader.releaseLock();
}
});
return head.buffer.slice(head.byteOffset, head.byteOffset + head.byteLength);
};
globalThis.__gopeed_blob_close_source = function (id) {
const reader = blobReaders.get(id);
if (!reader) {
return;
}
blobReaders.delete(id);
releaseReader(reader, "blob request closed");
};
const blobPipeReaderStates = new Map();
function releaseBlobPipeReader(state) {
if (!state || state.released) {
return;
}
state.released = true;
if (state.reader && typeof state.reader.releaseLock === "function") {
try {
state.reader.releaseLock();
} catch (_) {
}
}
if (blobPipeReaderStates.get(state.pipeId) === state) {
blobPipeReaderStates.delete(state.pipeId);
}
}
function finishBlobPipeReader(state, cancel, reason) {
if (!state) {
return Promise.resolve();
}
if (cancel) {
state.cancelRequested = true;
if (reason !== undefined) {
state.cancelReason = reason;
}
}
if (state.cleanupPromise) {
return state.cleanupPromise;
}
if (!state.reader) {
// A cancellation may arrive while the asynchronous opener is pending.
// Keep the state until the reader exists so it can still be canceled.
if (!state.cancelRequested || state.openSettled) {
releaseBlobPipeReader(state);
}
return Promise.resolve();
}
state.cleanupPromise = (async function () {
try {
if (state.cancelRequested && typeof state.reader.cancel === "function") {
await state.reader.cancel(state.cancelReason);
}
} catch (_) {
} finally {
releaseBlobPipeReader(state);
}
})();
return state.cleanupPromise;
}
globalThis.__gopeed_blob_cancel_pipe_source = function (pipeId, reason) {
const state = blobPipeReaderStates.get(pipeId);
if (!state) {
return;
}
// finishBlobPipeReader invokes reader.cancel() synchronously up to its
// first await. Do not return the cleanup Promise: Go only needs to ensure
// cancellation has started before releasing the source session.
finishBlobPipeReader(state, true, reason).catch(function () {});
};
globalThis.__gopeed_blob_pipe_source = function (openReadable, request, pipeId) {
if (typeof openReadable !== "function") {
throw new TypeError("blob opener must be callable");
}
const state = {
pipeId,
reader: null,
cancelRequested: false,
cancelReason: "blob pipe closed",
cleanupPromise: null,
openSettled: false,
released: false,
};
blobPipeReaderStates.set(pipeId, state);
const startPipe = async function () {
try {
const source = await openReadable(normalizeOpenRequest(request));
state.openSettled = true;
state.reader = toReadableStreamReader(source, "gopeed.runtime.blob.createObjectURL opener function");
if (state.cancelRequested) {
await finishBlobPipeReader(state, true, state.cancelReason);
return;
}
while (true) {
const { done, value } = await state.reader.read();
if (done) {
globalThis.__gopeed_blob_pipe_close(pipeId);
return;
}
const chunk = value instanceof Uint8Array ? value : new Uint8Array(value);
const buffer = chunk.buffer.slice(chunk.byteOffset, chunk.byteOffset + chunk.byteLength);
if (!globalThis.__gopeed_blob_pipe_chunk(pipeId, buffer)) {
state.cancelRequested = true;
state.cancelReason = "blob pipe closed";
return;
}
await yieldBlobPipeTask();
}
} catch (error) {
if (!state.cancelRequested) {
globalThis.__gopeed_blob_pipe_error(pipeId, error && error.stack ? error.stack : String(error));
}
} finally {
state.openSettled = true;
await finishBlobPipeReader(state, state.cancelRequested, state.cancelReason);
}
};
if (typeof setTimeout === "function") {
setTimeout(startPipe, 0);
} else {
Promise.resolve().then(startPipe);
}
};
globalThis.__gopeed_blob_create_object_url = function (value, options) {
const described = describeObjectURLValue(value);
if (described.kind === "blob") {
const blob = described.value;
const normalized = normalizeBlobObjectURLOptions(blob, options, Number(blob.size) > 0);
validateBlobObjectURLOptions(normalized);
const opener = async (request) => openBlobReadable(blob, request);
return createBlobObjectURL(opener, normalized);
}
if (described.kind === "opener") {
const normalized = normalizeBlobObjectURLOptions(null, options, false);
validateBlobObjectURLOptions(normalized);
return createBlobObjectURL(described.openReadable, normalized);
}
throw new TypeError("Unsupported object type for gopeed.runtime.blob.createObjectURL: " + getValueTypeName(described.value) + ". Expected Blob or opener function");
};
globalThis.__gopeed_blob_revoke_object_url = function (url) {
if (typeof url === "string") {
revokeObjectURL(url);
}
};
const originalFetch = typeof globalThis.fetch === "function"
? globalThis.fetch.bind(globalThis)
: null;
if (typeof fetchOpen === "function") {
globalThis.fetch = async function (input, init) {
const request = new Request(input, init);
if (originalFetch && (request.method === "HEAD" || request.redirect === "manual" || request.redirect === "error")) {
return originalFetch(input, init);
}
let body = null;
if (request._bodyFormData) {
body = request._bodyFormData;
} else if (request._bodyArrayBuffer) {
body = request._bodyArrayBuffer;
} else if (request._bodyBlob) {
body = await request._bodyBlob.arrayBuffer();
} else if (request._bodyInit != null && typeof request._bodyInit === "object") {
body = request._bodyInit;
} else if (request._bodyText != null) {
body = request._bodyText;
} else if (request._bodyInit != null) {
body = request._bodyInit;
}
const headers = [];
request.headers.forEach((value, key) => {
headers.push([key, value]);
});
let meta;
try {
meta = await fetchOpen({
url: request.url,
method: request.method,
headers,
body,
redirect: request.redirect,
credentials: request.credentials
});
} catch (error) {
throw error instanceof Error ? error : new TypeError(String(error));
}
const stream = new ReadableStream({
async pull(controller) {
let chunk;
try {
chunk = await fetchRead(meta.id, 64 * 1024);
} catch (error) {
fetchClose(meta.id);
controller.error(error instanceof Error ? error : new TypeError(String(error)));
return;
}
if (chunk == null) {
fetchClose(meta.id);
controller.close();
return;
}
const bytes = chunk instanceof Uint8Array ? chunk : new Uint8Array(chunk);
if (bytes.byteLength === 0) {
fetchClose(meta.id);
controller.close();
return;
}
controller.enqueue(bytes);
},
cancel(reason) {
fetchAbort(meta.id, reason == null ? "" : String(reason));
}
});
const response = new Response(null, {
status: meta.status,
statusText: meta.statusText,
headers: meta.headers,
url: meta.url
});
return attachResponseStreaming(response, stream);
};
globalThis.fetch.__gopeedOriginalFetch = originalFetch;
}
})();
+8 -1
View File
@@ -1,6 +1,8 @@
package vm
import (
"fmt"
"github.com/dop251/goja"
"github.com/dop251/goja_nodejs/eventloop"
)
@@ -25,7 +27,12 @@ func (vm *Vm) Get(name string) (value any) {
func (vm *Vm) RunString(script string) (value any, err error) {
defer func() {
if r := recover(); r != nil {
err = r.(error)
switch v := r.(type) {
case error:
err = v
default:
err = fmt.Errorf("panic: %v", r)
}
}
}()
+1 -1
View File
@@ -1 +1 @@
(()=>{var e={725:function(e,t,r){var o,n,i;i="undefined"!=typeof self&&self||"undefined"!=typeof window&&window||void 0!==r.g&&r.g||this,o=function(e){"use strict";var t=i.BlobBuilder||i.WebKitBlobBuilder||i.MSBlobBuilder||i.MozBlobBuilder,r=i.URL||i.webkitURL||function(e,t){return(t=document.createElement("a")).href=e,t},o=i.Blob,n=r.createObjectURL,a=r.revokeObjectURL,s=i.Symbol&&i.Symbol.toStringTag,u=!1,f=!1,l=t&&t.prototype.append&&t.prototype.getBlob;try{u=2===new Blob(["ä"]).size,f=2===new Blob([new Uint8Array([1,2])]).size}catch(e){}function c(e){return e.map((function(e){if(e.buffer instanceof ArrayBuffer){var t=e.buffer;if(e.byteLength!==t.byteLength){var r=new Uint8Array(e.byteLength);r.set(new Uint8Array(t,e.byteOffset,e.byteLength)),t=r.buffer}return t}return e}))}function h(e,r){r=r||{};var o=new t;return c(e).forEach((function(e){o.append(e)})),r.type?o.getBlob(r.type):o.getBlob()}function d(e,t){return new o(c(e),t||{})}i.Blob&&(h.prototype=Blob.prototype,d.prototype=Blob.prototype);var p="function"==typeof TextEncoder?TextEncoder.prototype.encode.bind(new TextEncoder):function(e){for(var t=0,r=e.length,o=i.Uint8Array||Array,n=0,a=Math.max(32,r+(r>>1)+7),s=new o(a>>3<<3);t<r;){var u=e.charCodeAt(t++);if(u>=55296&&u<=56319){if(t<r){var f=e.charCodeAt(t);56320==(64512&f)&&(++t,u=((1023&u)<<10)+(1023&f)+65536)}if(u>=55296&&u<=56319)continue}if(n+4>s.length){a+=8,a=(a*=1+t/e.length*2)>>3<<3;var l=new Uint8Array(a);l.set(s),s=l}if(0!=(4294967168&u)){if(0==(4294965248&u))s[n++]=u>>6&31|192;else if(0==(4294901760&u))s[n++]=u>>12&15|224,s[n++]=u>>6&63|128;else{if(0!=(4292870144&u))continue;s[n++]=u>>18&7|240,s[n++]=u>>12&63|128,s[n++]=u>>6&63|128}s[n++]=63&u|128}else s[n++]=u}return s.slice(0,n)},y="function"==typeof TextDecoder?TextDecoder.prototype.decode.bind(new TextDecoder):function(e){for(var t=e.length,r=[],o=0;o<t;){var n,i,a,s,u=e[o],f=null,l=u>239?4:u>223?3:u>191?2:1;if(o+l<=t)switch(l){case 1:u<128&&(f=u);break;case 2:128==(192&(n=e[o+1]))&&(s=(31&u)<<6|63&n)>127&&(f=s);break;case 3:n=e[o+1],i=e[o+2],128==(192&n)&&128==(192&i)&&(s=(15&u)<<12|(63&n)<<6|63&i)>2047&&(s<55296||s>57343)&&(f=s);break;case 4:n=e[o+1],i=e[o+2],a=e[o+3],128==(192&n)&&128==(192&i)&&128==(192&a)&&(s=(15&u)<<18|(63&n)<<12|(63&i)<<6|63&a)>65535&&s<1114112&&(f=s)}null===f?(f=65533,l=1):f>65535&&(f-=65536,r.push(f>>>10&1023|55296),f=56320|1023&f),r.push(f),o+=l}for(var c=r.length,h="",d=0;d<c;)h+=String.fromCharCode.apply(String,r.slice(d,d+=4096));return h};function b(){var t=!!i.ActiveXObject||"-ms-scroll-limit"in document.documentElement.style&&"-ms-ime-align"in document.documentElement.style,r=i.XMLHttpRequest&&i.XMLHttpRequest.prototype.send;t&&r&&(XMLHttpRequest.prototype.send=function(e){e instanceof Blob?(this.setRequestHeader("Content-Type",e.type),r.call(this,e)):r.call(this,e)});try{new File([],""),e.File=i.File,e.FileReader=i.FileReader}catch(t){try{e.File=new Function('class File extends Blob {constructor(chunks, name, opts) {opts = opts || {};super(chunks, opts || {});this.name = name.replace(/\\//g, ":");this.lastModifiedDate = opts.lastModified ? new Date(opts.lastModified) : new Date();this.lastModified = +this.lastModifiedDate;}};return new File([], ""), File')()}catch(t){e.File=function(e,t,r){var o=new Blob(e,r),n=r&&void 0!==r.lastModified?new Date(r.lastModified):new Date;return o.name=t.replace(/\//g,":"),o.lastModifiedDate=n,o.lastModified=+n,o.toString=function(){return"[object File]"},s&&(o[s]="File"),o}}}}u?(b(),e.Blob=f?i.Blob:d):l?(b(),e.Blob=h):function(){function t(e){for(var t=new Array(e.byteLength),r=new Uint8Array(e),o=t.length;o--;)t[o]=r[o];return t}function o(e){for(var t="ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=",r=[],o=0;o<e.length;o+=3){var n=e[o],i=o+1<e.length,a=i?e[o+1]:0,s=o+2<e.length,u=s?e[o+2]:0,f=n>>2,l=(3&n)<<4|a>>4,c=(15&a)<<2|u>>6,h=63&u;s||(h=64,i||(c=64)),r.push(t[f],t[l],t[c],t[h])}return r.join("")}var s=Object.create||function(e){function t(){}return t.prototype=e,new t};function u(e){return Object.prototype.toString.call(e).slice(8,-1)}function f(e,t){return"object"==typeof e&&Object.prototype.isPrototypeOf.call(e.prototype,t)}var l=["Int8Array","Uint8Array","Uint8ClampedArray","Int16Array","Uint16Array","Int32Array","Uint32Array","Float32Array","Float64Array","ArrayBuffer"];function c(e){return t=l,r=u(e),-1!==t.indexOf(r)||f(i.ArrayBuffer,e);var t,r}function h(e,r){r=null==r?{}:r;for(var o=0,n=(e=e?e.slice():[]).length;o<n;o++){var a=e[o];a instanceof h?e[o]=a._buffer:"string"==typeof a?e[o]=p(a):"DataView"===u(s=a)||f(i.DataView,s)?e[o]=t(a.buffer):c(a)?e[o]=t(a):e[o]=p(String(a))}var s;this._buffer=i.Uint8Array?function(e){for(var t=0,r=e.length;r--;)t+=e[r].length;for(var o=new Uint8Array(t),n=0,i=0;i<e.length;i++){var a=e[i];o.set(a,n),n+=a.byteLength||a.length}return o}(e):[].concat.apply([],e),this.size=this._buffer.length,this.type=r.type||"",/[^\u0020-\u007E]/.test(this.type)?this.type="":this.type=this.type.toLowerCase()}funcLine truncated
(()=>{var e={847(e,t,r){var n,o;n=function(){"use strict";function e(e,t){(null==t||t>e.length)&&(t=e.length);for(var r=0,n=Array(t);r<t;r++)n[r]=e[r];return n}function t(e,t,r){return t=l(t),function(e,t){if(t&&("object"==typeof t||"function"==typeof t))return t;if(void 0!==t)throw new TypeError("Derived constructors may only return object or undefined");return function(e){if(void 0===e)throw new ReferenceError("this hasn't been initialised - super() hasn't been called");return e}(e)}(e,c()?Reflect.construct(t,r||[],l(e).constructor):t.apply(e,r))}function n(e,t){if(!(e instanceof t))throw new TypeError("Cannot call a class as a function")}function o(e,t){for(var r=0;r<t.length;r++){var n=t[r];n.enumerable=n.enumerable||!1,n.configurable=!0,"value"in n&&(n.writable=!0),Object.defineProperty(e,f(n.key),n)}}function i(e,t,r){return t&&o(e.prototype,t),r&&o(e,r),Object.defineProperty(e,"prototype",{writable:!1}),e}function a(t,r){var n="undefined"!=typeof Symbol&&t[Symbol.iterator]||t["@@iterator"];if(!n){if(Array.isArray(t)||(n=function(t,r){if(t){if("string"==typeof t)return e(t,r);var n={}.toString.call(t).slice(8,-1);return"Object"===n&&t.constructor&&(n=t.constructor.name),"Map"===n||"Set"===n?Array.from(t):"Arguments"===n||/^(?:Ui|I)nt(?:8|16|32)(?:Clamped)?Array$/.test(n)?e(t,r):void 0}}(t))||r&&t&&"number"==typeof t.length){n&&(t=n);var o=0,i=function(){};return{s:i,n:function(){return o>=t.length?{done:!0}:{done:!1,value:t[o++]}},e:function(e){throw e},f:i}}throw new TypeError("Invalid attempt to iterate non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method.")}var a,s=!0,l=!1;return{s:function(){n=n.call(t)},n:function(){var e=n.next();return s=e.done,e},e:function(e){l=!0,a=e},f:function(){try{s||null==n.return||n.return()}finally{if(l)throw a}}}}function s(){return s="undefined"!=typeof Reflect&&Reflect.get?Reflect.get.bind():function(e,t,r){var n=function(e,t){for(;!{}.hasOwnProperty.call(e,t)&&null!==(e=l(e)););return e}(e,t);if(n){var o=Object.getOwnPropertyDescriptor(n,t);return o.get?o.get.call(arguments.length<3?e:r):o.value}},s.apply(null,arguments)}function l(e){return l=Object.setPrototypeOf?Object.getPrototypeOf.bind():function(e){return e.__proto__||Object.getPrototypeOf(e)},l(e)}function c(){try{var e=!Boolean.prototype.valueOf.call(Reflect.construct(Boolean,[],function(){}))}catch(e){}return(c=function(){return!!e})()}function u(e,t){return u=Object.setPrototypeOf?Object.setPrototypeOf.bind():function(e,t){return e.__proto__=t,e},u(e,t)}function f(e){var t=function(e,t){if("object"!=typeof e||!e)return e;var r=e[Symbol.toPrimitive];if(void 0!==r){var n=r.call(e,t||"default");if("object"!=typeof n)return n;throw new TypeError("@@toPrimitive must return a primitive value.")}return("string"===t?String:Number)(e)}(e,"string");return"symbol"==typeof t?t:t+""}!function(e){e.AbortSignal,e.AbortController}("undefined"!=typeof self?self:r.g);var p=function(){return i(function e(){n(this,e),Object.defineProperty(this,"listeners",{value:{},writable:!0,configurable:!0})},[{key:"addEventListener",value:function(e,t,r){e in this.listeners||(this.listeners[e]=[]),this.listeners[e].push({callback:t,options:r})}},{key:"removeEventListener",value:function(e,t){if(e in this.listeners)for(var r=this.listeners[e],n=0,o=r.length;n<o;n++)if(r[n].callback===t)return void r.splice(n,1)}},{key:"dispatchEvent",value:function(e){var t=this;if(e.type in this.listeners){for(var r=this.listeners[e.type].slice(),n=function(){var n=r[o];try{n.callback.call(t,e)}catch(e){Promise.resolve().then(function(){throw e})}n.options&&n.options.once&&t.removeEventListener(e.type,n.callback)},o=0,i=r.length;o<i;o++)n();return!e.defaultPrevented}}}])}(),d=function(e){function r(){var e;return n(this,r),(e=t(this,r)).listeners||p.call(e),Object.defineProperty(e,"aborted",{value:!1,writable:!0,configurable:!0}),Object.defineProperty(e,"onabort",{value:null,writable:!0,configurable:!0}),Object.defineProperty(e,"reason",{value:void 0,writable:!0,configurable:!0}),e}return function(e,t){if("function"!=typeof t&&null!==t)throw new TypeError("Super expression must either be null or a function");e.prototype=Object.create(t&&t.prototype,{constructor:{value:e,writable:!0,configurable:!0}}),Object.defineProperty(e,"prototype",{writable:!1}),t&&u(e,t)}(r,e),i(r,[{key:"toString",value:function(){return"[object AbortSignal]"}},{key:"dispatchEvent",value:function(e){var t,n,o,i,a;"abort"===e.type&&(this.aborted=!0,"function"==typeof this.onabort&&this.onabort.call(this,e)),(t=r,n="dispatchEvent",o=this,a=s(l(1&(i=3)?t.prototype:t),n,o),2&i&&"function"==typeof a?function(e){return a.apply(o,e)}:a)([e])}},{key:"throwIfAborted",value:function(){var e=this.aborted,t=this.reason;if(e)throw void 0===t?"Aborted":t}}],[{key:"timeout",value:function(e){var t=new h;return setTimeout(function(){return t.abort(new DOMException("This signal is timeout in ".concat(e,"ms"),"TimeoutError"))},e),t.signal}},{keLine truncated
File diff suppressed because it is too large. Load diff
@@ -18,7 +18,9 @@
"webpack-cli": "^5.0.1"
},
"dependencies": {
"abortcontroller-polyfill": "^1.7.8",
"blob-polyfill": "^7.0.20220408",
"event-target-shim": "^6.0.2",
"fastestsmallesttextencoderdecoder": "^1.0.22",
"whatwg-fetch": "^3.6.20"
}
@@ -0,0 +1,2 @@
import "../event/index.js";
import "abortcontroller-polyfill/dist/abortcontroller-polyfill-only.js";
@@ -0,0 +1,9 @@
import { Event, EventTarget } from "event-target-shim";
if (typeof globalThis.Event !== "function") {
globalThis.Event = Event;
}
if (typeof globalThis.EventTarget !== "function") {
globalThis.EventTarget = EventTarget;
}
+3 -1
View File
@@ -1,5 +1,7 @@
import "./event/index.js"
import "./abort/index.js"
import "./blob/index.js"
import "./crypto/index.js"
// polyfill TextEncoder
import 'fastestsmallesttextencoderdecoder';
import "./fetch/index.js"
import "./fetch/index.js"
+20
View File
@@ -0,0 +1,20 @@
package webview
type Provider interface {
IsAvailable() bool
Open(opts OpenOptions) (Page, error)
}
type unavailableProvider struct{}
func NewUnavailableProvider() Provider {
return unavailableProvider{}
}
func (unavailableProvider) IsAvailable() bool {
return false
}
func (unavailableProvider) Open(OpenOptions) (Page, error) {
return nil, ErrUnavailable
}
+143
View File
@@ -0,0 +1,143 @@
package webview
import "encoding/json"
const RPCEndpointPath = "/webview"
const (
MethodIsAvailable = "webview.isAvailable"
MethodPageOpen = "page.open"
MethodPageAddInitScript = "page.addInitScript"
MethodPageGoto = "page.goto"
MethodPageExecute = "page.execute"
MethodPageGetCookies = "page.getCookies"
MethodPageSetCookie = "page.setCookie"
MethodPageDeleteCookie = "page.deleteCookie"
MethodPageClearCookies = "page.clearCookies"
MethodPageClose = "page.close"
)
const (
ErrorCodeInvalidRequest = "INVALID_REQUEST"
ErrorCodeUnknownMethod = "UNKNOWN_METHOD"
ErrorCodeUnavailable = "UNAVAILABLE"
ErrorCodeBrowserNotFound = "BROWSER_NOT_FOUND"
ErrorCodePageNotFound = "PAGE_NOT_FOUND"
ErrorCodeNavigationFailed = "NAVIGATION_FAILED"
ErrorCodeEvaluationFailed = "EVALUATION_FAILED"
ErrorCodeTimeout = "TIMEOUT"
ErrorCodeInternal = "INTERNAL_ERROR"
)
type RPCConfig struct {
Network string `json:"network"`
Address string `json:"address"`
Token string `json:"token,omitempty"`
}
func (c RPCConfig) Enabled() bool {
return c.Network != "" && c.Address != ""
}
type RPCRequest struct {
Method string `json:"method"`
Params any `json:"params"`
}
type RPCResponse struct {
Result json.RawMessage `json:"result"`
Error *RPCError `json:"error"`
}
type RPCError struct {
Code string `json:"code"`
Message string `json:"message"`
}
func (e *RPCError) Error() string {
if e == nil {
return ""
}
return e.Message
}
type IsAvailableParams struct{}
type IsAvailableResult struct {
Available bool `json:"available"`
}
type PageOpenParams struct {
Headless bool `json:"headless,omitempty"`
Debug bool `json:"debug,omitempty"`
Title string `json:"title,omitempty"`
Width int `json:"width,omitempty"`
Height int `json:"height,omitempty"`
UserAgent string `json:"userAgent,omitempty"`
}
func NewPageOpenParams(opts OpenOptions) PageOpenParams {
return PageOpenParams{
Headless: opts.Headless,
Debug: opts.Debug,
Title: opts.Title,
Width: opts.Width,
Height: opts.Height,
UserAgent: opts.UserAgent,
}
}
type PageOpenResult struct {
PageID string `json:"pageId"`
}
type PageAddInitScriptParams struct {
PageID string `json:"pageId"`
Script string `json:"script"`
}
type PageGotoParams struct {
PageID string `json:"pageId"`
URL string `json:"url"`
TimeoutMS int64 `json:"timeoutMs,omitempty"`
WaitUntil string `json:"waitUntil,omitempty"`
}
func NewPageGotoParams(pageID string, url string, opts GotoOptions) PageGotoParams {
return PageGotoParams{
PageID: pageID,
URL: url,
TimeoutMS: opts.TimeoutMS,
WaitUntil: opts.WaitUntil,
}
}
type PageExecuteParams struct {
PageID string `json:"pageId"`
Expression string `json:"expression"`
Args []any `json:"args,omitempty"`
}
type PageGetCookiesParams struct {
PageID string `json:"pageId"`
}
type PageSetCookieParams struct {
PageID string `json:"pageId"`
Cookie Cookie `json:"cookie"`
}
type PageDeleteCookieParams struct {
PageID string `json:"pageId"`
Cookie Cookie `json:"cookie"`
}
type PageClearCookiesParams struct {
PageID string `json:"pageId"`
}
type PageCloseParams struct {
PageID string `json:"pageId"`
}
type EmptyResult struct{}
+765
View File
@@ -0,0 +1,765 @@
package webview
import (
"errors"
"fmt"
"strconv"
"strings"
"sync"
"time"
"github.com/dop251/goja"
)
var ErrUnavailable = errors.New("gopeed runtime webview is unavailable")
type Opener interface {
Open(opts OpenOptions) (Page, error)
}
type Page interface {
AddInitScript(script string) error
Goto(url string, opts GotoOptions) error
Execute(expression string, args ...any) (any, error)
GetCookies() ([]Cookie, error)
SetCookie(cookie Cookie) error
DeleteCookie(cookie Cookie) error
ClearCookies() error
Close() error
}
type OpenOptions struct {
Headless bool
Debug bool
Title string
Width int
Height int
UserAgent string
}
type GotoOptions struct {
TimeoutMS int64
WaitUntil string
}
type ClickOptions struct {
DelayMS int64
}
type TypeOptions struct {
DelayMS int64
}
type WaitOptions struct {
TimeoutMS int64
PollIntervalMS int64
}
type WaitForSelectorOptions struct {
TimeoutMS int64
PollIntervalMS int64
Visible bool
Hidden bool
}
type Cookie struct {
Name string `json:"name"`
Value string `json:"value"`
Domain string `json:"domain,omitempty"`
Path string `json:"path,omitempty"`
Expires time.Time `json:"expires,omitempty,omitzero"`
Secure bool `json:"secure,omitempty"`
HTTPOnly bool `json:"httpOnly,omitempty"`
}
type Runtime struct {
opener Opener
available bool
mu sync.Mutex
pageSeq uint64
pages map[string]Page
}
func NewRuntime(opener Opener, available bool) *Runtime {
if opener == nil {
available = false
}
return &Runtime{
opener: opener,
available: available,
pages: make(map[string]Page),
}
}
func (r *Runtime) IsAvailable() bool {
return r != nil && r.available && r.opener != nil
}
func (r *Runtime) Open(opts ...map[string]any) (*PageHandle, error) {
if !r.IsAvailable() {
return nil, ErrUnavailable
}
page, err := r.opener.Open(parseOpenOptions(firstMap(opts)))
if err != nil {
return nil, err
}
r.mu.Lock()
defer r.mu.Unlock()
r.pageSeq++
id := fmt.Sprintf("page-%d", r.pageSeq)
r.pages[id] = page
return &PageHandle{runtime: r, id: id}, nil
}
func (r *Runtime) Close() error {
r.mu.Lock()
pageIDs := make([]string, 0, len(r.pages))
for id := range r.pages {
pageIDs = append(pageIDs, id)
}
r.mu.Unlock()
var errs []error
for _, id := range pageIDs {
if err := r.closePage(id); err != nil {
errs = append(errs, err)
}
}
return errors.Join(errs...)
}
type PageHandle struct {
runtime *Runtime
id string
}
func (p *PageHandle) AddInitScript(script string) error {
page, err := p.page()
if err != nil {
return err
}
return page.AddInitScript(script)
}
func (p *PageHandle) Goto(url string, opts ...map[string]any) error {
page, err := p.page()
if err != nil {
return err
}
gotoOpts, err := parseGotoOptions(firstMap(opts))
if err != nil {
return err
}
return page.Goto(url, gotoOpts)
}
func (p *PageHandle) Execute(scriptOrFn any, args ...any) (any, error) {
page, err := p.page()
if err != nil {
return nil, err
}
expression, err := normalizeExecutable(scriptOrFn)
if err != nil {
return nil, err
}
return page.Execute(expression, args...)
}
func (p *PageHandle) Focus(selector string) error {
_, err := p.Execute(`(selector) => {
const element = document.querySelector(selector);
if (!element) {
throw new Error("element not found: " + selector);
}
if (typeof element.focus !== "function") {
throw new Error("element is not focusable: " + selector);
}
element.focus();
return true;
}`, selector)
return err
}
func (p *PageHandle) Click(selector string, opts ...map[string]any) error {
clickOpts := parseClickOptions(firstMap(opts))
_, err := p.Execute(`(selector, delayMS) => {
const element = document.querySelector(selector);
if (!element) {
throw new Error("element not found: " + selector);
}
if (typeof element.scrollIntoView === "function") {
element.scrollIntoView({ block: "center", inline: "center" });
}
const performClick = () => {
if (typeof element.click === "function") {
element.click();
return;
}
element.dispatchEvent(new MouseEvent("click", {
bubbles: true,
cancelable: true,
view: window,
}));
};
if (delayMS > 0) {
return new Promise((resolve) => {
setTimeout(() => {
performClick();
resolve(true);
}, delayMS);
});
}
performClick();
return true;
}`, selector, clickOpts.DelayMS)
return err
}
func (p *PageHandle) Type(selector string, text string, opts ...map[string]any) error {
typeOpts := parseTypeOptions(firstMap(opts))
_, err := p.Execute(`(selector, text, delayMS) => {
const element = document.querySelector(selector);
if (!element) {
throw new Error("element not found: " + selector);
}
if (typeof element.focus === "function") {
element.focus();
}
const emitInput = () => {
element.dispatchEvent(new Event("input", { bubbles: true }));
};
const insertIntoInput = (chunk) => {
if (element.disabled) {
throw new Error("element is disabled: " + selector);
}
if (element.readOnly) {
throw new Error("element is readonly: " + selector);
}
const value = typeof element.value === "string" ? element.value : "";
const hasSelection = typeof element.selectionStart === "number" && typeof element.selectionEnd === "number";
const start = hasSelection ? element.selectionStart : value.length;
const end = hasSelection ? element.selectionEnd : value.length;
element.value = value.slice(0, start) + chunk + value.slice(end);
const caret = start + chunk.length;
if (typeof element.setSelectionRange === "function") {
element.setSelectionRange(caret, caret);
}
emitInput();
};
const insertIntoEditable = (chunk) => {
if (element.isContentEditable) {
element.textContent = (element.textContent || "") + chunk;
emitInput();
return;
}
insertIntoInput(chunk);
};
const insertChunk = (chunk) => {
const tagName = (element.tagName || "").toUpperCase();
if (tagName === "INPUT" || tagName === "TEXTAREA" || typeof element.value === "string") {
insertIntoInput(chunk);
return;
}
if (element.isContentEditable) {
insertIntoEditable(chunk);
return;
}
throw new Error("element is not typable: " + selector);
};
const chars = Array.from(String(text));
if (delayMS > 0) {
return new Promise((resolve) => {
let index = 0;
const step = () => {
if (index >= chars.length) {
resolve(true);
return;
}
insertChunk(chars[index]);
index += 1;
setTimeout(step, delayMS);
};
step();
});
}
for (const char of chars) {
insertChunk(char);
}
return true;
}`, selector, text, typeOpts.DelayMS)
return err
}
func (p *PageHandle) WaitForSelector(selector string, opts ...map[string]any) (bool, error) {
waitOpts := parseWaitForSelectorOptions(firstMap(opts))
_, matched, err := p.poll(WaitOptions{
TimeoutMS: waitOpts.TimeoutMS,
PollIntervalMS: waitOpts.PollIntervalMS,
}, func() (any, bool, error) {
value, err := p.Execute(`(selector, visible, hidden) => {
const element = document.querySelector(selector);
if (hidden) {
if (!element) return true;
const style = window.getComputedStyle(element);
if (!style) return !!element.hidden;
const rect = element.getBoundingClientRect();
return element.hidden || style.display === "none" || style.visibility === "hidden" || rect.width === 0 || rect.height === 0;
}
if (!element) return false;
if (!visible) return true;
const style = window.getComputedStyle(element);
if (!style) return true;
const rect = element.getBoundingClientRect();
return !element.hidden && style.display !== "none" && style.visibility !== "hidden" && rect.width > 0 && rect.height > 0;
}`, selector, waitOpts.Visible, waitOpts.Hidden)
if err != nil {
return nil, false, err
}
return true, truthy(value), nil
})
if err != nil {
return false, err
}
return matched, nil
}
func (p *PageHandle) WaitForFunction(scriptOrFn any, args ...any) (any, error) {
expression, err := normalizeExecutable(scriptOrFn)
if err != nil {
return nil, err
}
waitOpts := WaitOptions{}
callArgs := args
if len(args) > 0 {
if raw, ok := args[0].(map[string]any); ok {
waitOpts = parseWaitOptions(raw)
callArgs = args[1:]
}
}
value, matched, err := p.poll(waitOpts, func() (any, bool, error) {
result, err := p.Execute(`(expression, args) => {
const target = (0, eval)(expression);
return Promise.resolve(typeof target === "function" ? target(...args) : target).then((value) => ({
matched: !!value,
value: value ?? null,
}));
}`, expression, callArgs)
if err != nil {
return nil, false, err
}
payload, ok := result.(map[string]any)
if !ok {
return nil, false, fmt.Errorf("unexpected waitForFunction payload: %T", result)
}
return payload["value"], truthy(payload["matched"]), nil
})
if err != nil {
return nil, err
}
if !matched {
return nil, nil
}
return value, nil
}
func (p *PageHandle) GetCookies() ([]Cookie, error) {
page, err := p.page()
if err != nil {
return nil, err
}
return page.GetCookies()
}
func (p *PageHandle) SetCookie(raw map[string]any) error {
page, err := p.page()
if err != nil {
return err
}
return page.SetCookie(parseCookie(raw))
}
func (p *PageHandle) DeleteCookie(raw map[string]any) error {
page, err := p.page()
if err != nil {
return err
}
return page.DeleteCookie(parseCookie(raw))
}
func (p *PageHandle) ClearCookies() error {
page, err := p.page()
if err != nil {
return err
}
return page.ClearCookies()
}
func (p *PageHandle) URL() (string, error) {
value, err := p.Execute(`() => location.href`)
if err != nil {
return "", err
}
return parseString(value), nil
}
func (p *PageHandle) Content() (string, error) {
value, err := p.Execute(`() => document.documentElement ? document.documentElement.outerHTML : ""`)
if err != nil {
return "", err
}
return parseString(value), nil
}
func (p *PageHandle) Close() error {
if p == nil || p.runtime == nil {
return nil
}
return p.runtime.closePage(p.id)
}
func (p *PageHandle) page() (Page, error) {
if p == nil || p.runtime == nil {
return nil, fmt.Errorf("webview page handle is not initialized")
}
return p.runtime.getPage(p.id)
}
func (p *PageHandle) poll(opts WaitOptions, fn func() (any, bool, error)) (any, bool, error) {
timeout := time.Duration(defaultWaitTimeoutMS(opts.TimeoutMS)) * time.Millisecond
pollInterval := time.Duration(defaultPollIntervalMS(opts.PollIntervalMS)) * time.Millisecond
deadline := time.Now().Add(timeout)
for {
value, matched, err := fn()
if err != nil {
return nil, false, err
}
if matched {
return value, true, nil
}
if time.Now().After(deadline) {
return nil, false, nil
}
time.Sleep(pollInterval)
}
}
func (r *Runtime) getPage(id string) (Page, error) {
r.mu.Lock()
defer r.mu.Unlock()
page, ok := r.pages[id]
if !ok {
return nil, fmt.Errorf("page not found")
}
return page, nil
}
func (r *Runtime) closePage(id string) error {
r.mu.Lock()
page, ok := r.pages[id]
if !ok {
r.mu.Unlock()
return nil
}
delete(r.pages, id)
r.mu.Unlock()
return page.Close()
}
func parseOpenOptions(raw map[string]any) OpenOptions {
return OpenOptions{
Headless: parseBool(raw["headless"]),
Debug: parseBool(raw["debug"]),
Title: parseString(raw["title"]),
Width: parseInt(raw["width"]),
Height: parseInt(raw["height"]),
UserAgent: parseString(raw["userAgent"]),
}
}
func parseGotoOptions(raw map[string]any) (GotoOptions, error) {
waitUntil, err := normalizeWaitUntil(parseString(raw["waitUntil"]))
if err != nil {
return GotoOptions{}, err
}
return GotoOptions{
TimeoutMS: parseInt64(raw["timeoutMs"]),
WaitUntil: waitUntil,
}, nil
}
func normalizeWaitUntil(raw string) (string, error) {
if raw == "" {
return "load", nil
}
switch strings.ToLower(strings.TrimSpace(raw)) {
case "load":
return "load", nil
case "domcontentloaded":
return "domcontentloaded", nil
default:
return "", fmt.Errorf("invalid waitUntil: %s", raw)
}
}
func parseClickOptions(raw map[string]any) ClickOptions {
return ClickOptions{DelayMS: parseInt64(raw["delay"])}
}
func parseTypeOptions(raw map[string]any) TypeOptions {
return TypeOptions{DelayMS: parseInt64(raw["delay"])}
}
func parseWaitOptions(raw map[string]any) WaitOptions {
return WaitOptions{
TimeoutMS: parseInt64(raw["timeoutMs"]),
PollIntervalMS: parseInt64(raw["pollIntervalMs"]),
}
}
func parseWaitForSelectorOptions(raw map[string]any) WaitForSelectorOptions {
return WaitForSelectorOptions{
TimeoutMS: parseInt64(raw["timeoutMs"]),
PollIntervalMS: parseInt64(raw["pollIntervalMs"]),
Visible: parseBool(raw["visible"]),
Hidden: parseBool(raw["hidden"]),
}
}
func parseCookie(raw map[string]any) Cookie {
cookie := Cookie{
Name: parseString(raw["name"]),
Value: parseString(raw["value"]),
Domain: parseString(raw["domain"]),
Path: parseString(raw["path"]),
Secure: parseBool(raw["secure"]),
HTTPOnly: parseBool(raw["httpOnly"]),
}
if rawExpires, ok := raw["expires"]; ok {
cookie.Expires = parseTime(rawExpires)
}
return cookie
}
func defaultWaitTimeoutMS(value int64) int64 {
if value <= 0 {
return 10000
}
return value
}
func defaultPollIntervalMS(value int64) int64 {
if value <= 0 {
return 100
}
return value
}
func normalizeExecutable(scriptOrFn any) (string, error) {
switch value := scriptOrFn.(type) {
case string:
return value, nil
case *goja.Object:
return NormalizeExecutableValue(value)
default:
return "", fmt.Errorf("execute expects a string or function, got %T", scriptOrFn)
}
}
func NormalizeExecutableValue(value goja.Value) (string, error) {
if value == nil {
return "", fmt.Errorf("execute expects a string or function")
}
switch raw := value.Export().(type) {
case string:
return raw, nil
}
obj, ok := value.(*goja.Object)
if ok {
if _, ok := goja.AssertFunction(obj); ok {
source, err := functionSource(obj)
if err != nil {
return "", err
}
return normalizeFunctionSource(source), nil
}
}
source := strings.TrimSpace(value.String())
if looksLikeFunctionSource(source) {
return normalizeFunctionSource(source), nil
}
return "", fmt.Errorf("execute expects a string or function, got %T (%q)", value, source)
}
func normalizeFunctionSource(source string) string {
repaired := repairArrowObjectLiteralSource(source)
return "(" + repaired + ")"
}
func functionSource(fn *goja.Object) (string, error) {
toStringValue := fn.Get("toString")
toString, ok := goja.AssertFunction(toStringValue)
if !ok {
return "", fmt.Errorf("function.toString is not callable")
}
value, err := toString(fn)
if err != nil {
return "", err
}
return value.String(), nil
}
func repairArrowObjectLiteralSource(source string) string {
trimmed := strings.TrimSpace(source)
if !strings.Contains(trimmed, "=>") || !strings.Contains(trimmed, "({") {
return source
}
if !strings.HasSuffix(trimmed, "}") {
return source
}
if strings.Count(trimmed, "(")-strings.Count(trimmed, ")") != 1 {
return source
}
if strings.Count(trimmed, "{") != strings.Count(trimmed, "}") {
return source
}
// TODO: Remove this narrow repair once goja fixes toString() for concise arrow
// functions that directly return object literals, then upstream the fix as a PR.
return source + ")"
}
func looksLikeFunctionSource(source string) bool {
if source == "" || source == "[object Object]" {
return false
}
return strings.Contains(source, "=>") ||
strings.HasPrefix(source, "function") ||
strings.HasPrefix(source, "async function") ||
strings.HasPrefix(source, "async (") ||
strings.HasPrefix(source, "async(")
}
func truthy(value any) bool {
switch v := value.(type) {
case nil:
return false
case bool:
return v
case string:
return v != ""
case int:
return v != 0
case int8:
return v != 0
case int16:
return v != 0
case int32:
return v != 0
case int64:
return v != 0
case uint:
return v != 0
case uint8:
return v != 0
case uint16:
return v != 0
case uint32:
return v != 0
case uint64:
return v != 0
case float32:
return v != 0
case float64:
return v != 0
default:
return true
}
}
func firstMap(values []map[string]any) map[string]any {
if len(values) == 0 || values[0] == nil {
return map[string]any{}
}
return values[0]
}
func parseString(value any) string {
if value == nil {
return ""
}
return fmt.Sprint(value)
}
func parseBool(value any) bool {
switch v := value.(type) {
case bool:
return v
case string:
parsed, _ := strconv.ParseBool(v)
return parsed
default:
return false
}
}
func parseInt(value any) int {
return int(parseInt64(value))
}
func parseInt64(value any) int64 {
switch v := value.(type) {
case int:
return int64(v)
case int8:
return int64(v)
case int16:
return int64(v)
case int32:
return int64(v)
case int64:
return v
case uint:
return int64(v)
case uint8:
return int64(v)
case uint16:
return int64(v)
case uint32:
return int64(v)
case uint64:
return int64(v)
case float32:
return int64(v)
case float64:
return int64(v)
case string:
parsed, _ := strconv.ParseInt(v, 10, 64)
return parsed
default:
return 0
}
}
func parseTime(value any) time.Time {
switch v := value.(type) {
case time.Time:
return v
case int64:
return time.UnixMilli(v)
case int:
return time.UnixMilli(int64(v))
case float64:
return time.UnixMilli(int64(v))
case string:
if parsed, err := time.Parse(time.RFC3339Nano, v); err == nil {
return parsed
}
if ms, err := strconv.ParseInt(v, 10, 64); err == nil {
return time.UnixMilli(ms)
}
}
return time.Time{}
}
+338
View File
@@ -0,0 +1,338 @@
package webview
import (
"strings"
"testing"
"github.com/dop251/goja"
)
func TestRuntimeOpenAndExecuteFunctionSource(t *testing.T) {
opener := &fakeOpener{
page: &fakePage{executeValue: "ok"},
}
runtime := NewRuntime(opener, true)
page, err := runtime.Open(map[string]any{
"headless": true,
"debug": true,
"title": "Gopeed",
"width": 1280,
"height": 720,
"userAgent": "UA",
})
if err != nil {
t.Fatal(err)
}
vm := goja.New()
fnValue, err := vm.RunString(`(a, b) => a + b`)
if err != nil {
t.Fatal(err)
}
value, err := page.Execute(fnValue, int64(1), int64(2))
if err != nil {
t.Fatal(err)
}
if value != "ok" {
t.Fatalf("unexpected execute result: %#v", value)
}
if opener.opts.Title != "Gopeed" || opener.opts.UserAgent != "UA" {
t.Fatalf("unexpected open options: %+v", opener.opts)
}
if opener.page.lastExecuteSource != "((a, b) => a + b)" {
t.Fatalf("unexpected function source: %q", opener.page.lastExecuteSource)
}
if len(opener.page.lastExecuteArgs) != 2 || opener.page.lastExecuteArgs[0] != int64(1) || opener.page.lastExecuteArgs[1] != int64(2) {
t.Fatalf("unexpected execute args: %#v", opener.page.lastExecuteArgs)
}
}
func TestRuntimeExecuteRepairsObjectLiteralArrowFunction(t *testing.T) {
opener := &fakeOpener{
page: &fakePage{executeValue: map[string]any{"ok": true}},
}
runtime := NewRuntime(opener, true)
page, err := runtime.Open()
if err != nil {
t.Fatal(err)
}
vm := goja.New()
fnValue, err := vm.RunString(`() => ({ ok: true })`)
if err != nil {
t.Fatal(err)
}
if _, err := page.Execute(fnValue); err != nil {
t.Fatal(err)
}
if opener.page.lastExecuteSource != "(() => ({ ok: true }))" {
t.Fatalf("expected repaired function source, got %q", opener.page.lastExecuteSource)
}
}
func TestRuntimeExecuteRepairsMultilineObjectLiteralArrowFunction(t *testing.T) {
opener := &fakeOpener{
page: &fakePage{executeValue: map[string]any{"ok": true}},
}
runtime := NewRuntime(opener, true)
page, err := runtime.Open()
if err != nil {
t.Fatal(err)
}
vm := goja.New()
fnValue, err := vm.RunString(`() => ({
title: document.title || "",
url: String(location.href || ""),
userAgent: navigator.userAgent,
readyState: document.readyState,
})`)
if err != nil {
t.Fatal(err)
}
if _, err := page.Execute(fnValue); err != nil {
t.Fatal(err)
}
expected := `(() => ({
title: document.title || "",
url: String(location.href || ""),
userAgent: navigator.userAgent,
readyState: document.readyState,
}))`
if opener.page.lastExecuteSource != expected {
t.Fatalf("expected repaired multiline function source, got %q", opener.page.lastExecuteSource)
}
}
func TestRuntimeHelpers(t *testing.T) {
opener := &fakeOpener{
page: &fakePage{
executeQueue: []any{
false,
true,
map[string]any{"matched": false, "value": nil},
map[string]any{"matched": true, "value": "ready"},
"https://example.com",
"<html></html>",
},
cookies: []Cookie{
{Name: "a", Value: "1", Domain: "example.com", Path: "/"},
{Name: "b", Value: "two", Domain: "example.com", Path: "/"},
},
},
}
runtime := NewRuntime(opener, true)
page, err := runtime.Open()
if err != nil {
t.Fatal(err)
}
if err := page.AddInitScript("window.__TEST__ = true"); err != nil {
t.Fatal(err)
}
if opener.page.lastInitScript != "window.__TEST__ = true" {
t.Fatalf("unexpected init script: %q", opener.page.lastInitScript)
}
if err := page.Goto("https://example.com", map[string]any{"waitUntil": "domcontentloaded"}); err != nil {
t.Fatal(err)
}
if opener.page.lastGotoURL != "https://example.com" {
t.Fatalf("unexpected goto url: %q", opener.page.lastGotoURL)
}
if opener.page.lastGotoOpts.WaitUntil != "domcontentloaded" {
t.Fatalf("unexpected goto waitUntil: %q", opener.page.lastGotoOpts.WaitUntil)
}
found, err := page.WaitForSelector("#app", map[string]any{"timeoutMs": 50, "pollIntervalMs": 1})
if err != nil {
t.Fatal(err)
}
if !found {
t.Fatal("expected selector to be found")
}
value, err := page.WaitForFunction(`() => window.__READY__`, map[string]any{"timeoutMs": 50, "pollIntervalMs": 1})
if err != nil {
t.Fatal(err)
}
if value != "ready" {
t.Fatalf("unexpected waitForFunction value: %#v", value)
}
cookies, err := page.GetCookies()
if err != nil {
t.Fatal(err)
}
if len(cookies) != 2 || cookies[0].Name != "a" || cookies[0].Value != "1" || cookies[1].Name != "b" || cookies[1].Value != "two" {
t.Fatalf("unexpected cookies: %#v", cookies)
}
if err := page.SetCookie(map[string]any{
"name": "session",
"value": "abc",
"domain": "example.com",
"path": "/",
"secure": true,
"httpOnly": true,
}); err != nil {
t.Fatal(err)
}
if opener.page.lastSetCookie.Name != "session" || !opener.page.lastSetCookie.Secure || !opener.page.lastSetCookie.HTTPOnly {
t.Fatalf("unexpected set cookie: %#v", opener.page.lastSetCookie)
}
if err := page.DeleteCookie(map[string]any{
"name": "session",
"domain": "example.com",
"path": "/",
}); err != nil {
t.Fatal(err)
}
if opener.page.lastDeleteCookie.Name != "session" || opener.page.lastDeleteCookie.Domain != "example.com" {
t.Fatalf("unexpected delete cookie: %#v", opener.page.lastDeleteCookie)
}
if err := page.ClearCookies(); err != nil {
t.Fatal(err)
}
if !opener.page.cookiesCleared {
t.Fatal("expected cookies to be cleared")
}
url, err := page.URL()
if err != nil {
t.Fatal(err)
}
if url != "https://example.com" {
t.Fatalf("unexpected url: %q", url)
}
content, err := page.Content()
if err != nil {
t.Fatal(err)
}
if content != "<html></html>" {
t.Fatalf("unexpected content: %q", content)
}
if err := page.Close(); err != nil {
t.Fatal(err)
}
if !opener.page.closed {
t.Fatal("expected page to be closed")
}
}
func TestRuntimeInteractionHelpers(t *testing.T) {
opener := &fakeOpener{
page: &fakePage{executeValue: true},
}
runtime := NewRuntime(opener, true)
page, err := runtime.Open()
if err != nil {
t.Fatal(err)
}
if err := page.Focus("#search"); err != nil {
t.Fatal(err)
}
if len(opener.page.lastExecuteArgs) != 1 || opener.page.lastExecuteArgs[0] != "#search" {
t.Fatalf("unexpected focus args: %#v", opener.page.lastExecuteArgs)
}
if !strings.Contains(opener.page.lastExecuteSource, "element.focus()") {
t.Fatalf("unexpected focus source: %q", opener.page.lastExecuteSource)
}
if err := page.Click("#submit", map[string]any{"delay": 12}); err != nil {
t.Fatal(err)
}
if len(opener.page.lastExecuteArgs) != 2 || opener.page.lastExecuteArgs[0] != "#submit" || opener.page.lastExecuteArgs[1] != int64(12) {
t.Fatalf("unexpected click args: %#v", opener.page.lastExecuteArgs)
}
if !strings.Contains(opener.page.lastExecuteSource, "performClick") {
t.Fatalf("unexpected click source: %q", opener.page.lastExecuteSource)
}
if err := page.Type("#search", "Gopeed", map[string]any{"delay": 5}); err != nil {
t.Fatal(err)
}
if len(opener.page.lastExecuteArgs) != 3 || opener.page.lastExecuteArgs[0] != "#search" || opener.page.lastExecuteArgs[1] != "Gopeed" || opener.page.lastExecuteArgs[2] != int64(5) {
t.Fatalf("unexpected type args: %#v", opener.page.lastExecuteArgs)
}
if !strings.Contains(opener.page.lastExecuteSource, "element is not typable") {
t.Fatalf("unexpected type source: %q", opener.page.lastExecuteSource)
}
}
func TestRuntimeGotoRejectsInvalidWaitUntil(t *testing.T) {
opener := &fakeOpener{
page: &fakePage{},
}
runtime := NewRuntime(opener, true)
page, err := runtime.Open()
if err != nil {
t.Fatal(err)
}
err = page.Goto("https://example.com", map[string]any{"waitUntil": "networkidle"})
if err == nil || !strings.Contains(err.Error(), "invalid waitUntil") {
t.Fatalf("unexpected error: %v", err)
}
}
type fakeOpener struct {
opts OpenOptions
page *fakePage
}
func (o *fakeOpener) Open(opts OpenOptions) (Page, error) {
o.opts = opts
return o.page, nil
}
type fakePage struct {
lastInitScript string
lastGotoURL string
lastGotoOpts GotoOptions
lastExecuteSource string
lastExecuteArgs []any
executeValue any
executeQueue []any
cookies []Cookie
lastSetCookie Cookie
lastDeleteCookie Cookie
cookiesCleared bool
closed bool
}
func (p *fakePage) AddInitScript(script string) error {
p.lastInitScript = script
return nil
}
func (p *fakePage) Goto(url string, opts GotoOptions) error {
p.lastGotoURL = url
p.lastGotoOpts = opts
return nil
}
func (p *fakePage) Execute(expression string, args ...any) (any, error) {
p.lastExecuteSource = expression
p.lastExecuteArgs = args
if len(p.executeQueue) > 0 {
value := p.executeQueue[0]
p.executeQueue = p.executeQueue[1:]
return value, nil
}
return p.executeValue, nil
}
func (p *fakePage) GetCookies() ([]Cookie, error) {
return append([]Cookie(nil), p.cookies...), nil
}
func (p *fakePage) SetCookie(cookie Cookie) error {
p.lastSetCookie = cookie
return nil
}
func (p *fakePage) DeleteCookie(cookie Cookie) error {
p.lastDeleteCookie = cookie
return nil
}
func (p *fakePage) ClearCookies() error {
p.cookiesCleared = true
return nil
}
func (p *fakePage) Close() error {
p.closed = true
return nil
}
+69 -38
View File
@@ -16,6 +16,7 @@ import (
"github.com/GopeedLab/gopeed/pkg/download/engine"
gojaerror "github.com/GopeedLab/gopeed/pkg/download/engine/inject/error"
gojautil "github.com/GopeedLab/gopeed/pkg/download/engine/util"
enginewebview "github.com/GopeedLab/gopeed/pkg/download/engine/webview"
"github.com/GopeedLab/gopeed/pkg/util"
"github.com/dop251/goja"
"github.com/go-git/go-git/v5"
@@ -268,6 +269,7 @@ func (d *Downloader) triggerOnResolve(req *base.Request) (res *base.Resource, er
for _, file := range ctx.Res.Files {
file.Name = util.SafeFilename(file.Name)
}
ensureResourceRequestRawURLs(req, ctx.Res)
ctx.Res.CalcSize(nil)
}
res = ctx.Res
@@ -281,7 +283,7 @@ func (d *Downloader) triggerOnStart(task *Task) {
EventOnStart,
task.Meta.Req,
&OnStartContext{
Task: NewExtensionTask(d, task),
Task: newOnStartExtensionTask(task),
},
func(ext *Extension, gopeed *Instance, ctx *OnStartContext) {
// Validate request structure
@@ -293,7 +295,6 @@ func (d *Downloader) triggerOnStart(task *Task) {
}
},
)
return
}
func (d *Downloader) triggerOnError(task *Task, err error) {
@@ -301,7 +302,7 @@ func (d *Downloader) triggerOnError(task *Task, err error) {
EventOnError,
task.Meta.Req,
&OnErrorContext{
Task: NewExtensionTask(d, task),
Task: newOnErrorExtensionTask(d, task),
Error: err,
},
nil,
@@ -312,8 +313,8 @@ func (d *Downloader) triggerOnDone(task *Task) {
doTrigger(d,
EventOnDone,
task.Meta.Req,
&OnErrorContext{
Task: NewExtensionTask(d, task),
&OnDoneContext{
Task: newOnDoneExtensionTask(task),
},
nil,
)
@@ -361,11 +362,12 @@ func doTrigger[T any](d *Downloader, event ActivationEvent, req *base.Request, c
if req.Labels == nil {
req.Labels = make(map[string]string)
}
engine := engine.NewEngine(&engine.Config{
ProxyConfig: d.cfg.Proxy,
})
defer engine.Close()
err = engine.Runtime.Set("gopeed", gopeed)
engine, session := d.newExtensionEngine()
defer session.CloseIfIdle()
gopeed.Runtime = &InstanceRuntime{
WebView: d.newExtensionWebViewRuntime(session),
}
err = injectGopeed(engine.Runtime, gopeed)
if err != nil {
gopeed.Logger.logger.Error().Err(err).Msgf("[%s] engine inject failed", ext.buildIdentity())
return
@@ -546,13 +548,17 @@ func (s *Script) match(event ActivationEvent, req *base.Request) bool {
if s.Event != string(event) {
return false
}
if s.Match == nil || (len(s.Match.Urls) == 0 && len(s.Match.Labels) == 0) {
if req == nil || s.Match == nil || (len(s.Match.Urls) == 0 && len(s.Match.Labels) == 0) {
return false
}
// match url
targetURL := req.RawURL
if targetURL == "" {
targetURL = req.URL
}
for _, url := range s.Match.Urls {
if util.Match(url, req.URL) {
if util.Match(url, targetURL) {
return true
}
}
@@ -601,32 +607,37 @@ type Option struct {
// Instance inject to js context when extension script is activated
type Instance struct {
Events InstanceEvents `json:"events"`
Info *ExtensionInfo `json:"info"`
Logger *InstanceLogger `json:"logger"`
Settings map[string]any `json:"settings"`
Storage *ContextStorage `json:"storage"`
Events InstanceEvents `json:"events"`
Info *ExtensionInfo `json:"info"`
Logger *InstanceLogger `json:"logger"`
Settings map[string]any `json:"settings"`
Storage *ContextStorage `json:"storage"`
Runtime *InstanceRuntime `json:"runtime"`
}
type InstanceEvents map[ActivationEvent]goja.Callable
type InstanceRuntime struct {
WebView *enginewebview.Runtime `json:"webview"`
}
func (h InstanceEvents) register(name ActivationEvent, fn goja.Callable) {
type InstanceEvents map[ActivationEvent]engine.JSFunction
func (h InstanceEvents) register(name ActivationEvent, fn engine.JSFunction) {
h[name] = fn
}
func (h InstanceEvents) OnResolve(fn goja.Callable) {
func (h InstanceEvents) OnResolve(fn engine.JSFunction) {
h.register(EventOnResolve, fn)
}
func (h InstanceEvents) OnStart(fn goja.Callable) {
func (h InstanceEvents) OnStart(fn engine.JSFunction) {
h.register(EventOnStart, fn)
}
func (h InstanceEvents) OnError(fn goja.Callable) {
func (h InstanceEvents) OnError(fn engine.JSFunction) {
h.register(EventOnError, fn)
}
func (h InstanceEvents) OnDone(fn goja.Callable) {
func (h InstanceEvents) OnDone(fn engine.JSFunction) {
h.register(EventOnDone, fn)
}
@@ -698,8 +709,8 @@ type OnStartContext struct {
}
type OnErrorContext struct {
Task *ExtensionTask `json:"task"`
Error error `json:"error"`
Task *OnErrorExtensionTask `json:"task"`
Error error `json:"error"`
}
type OnDoneContext struct {
@@ -710,29 +721,49 @@ type OnDoneContext struct {
// Avoid extension scripts modifying task directly, use ExtensionTask to encapsulate task,
// only some fields can be modified, such as request info.
type ExtensionTask struct {
download *Downloader
*Task
}
func NewExtensionTask(download *Downloader, task *Task) *ExtensionTask {
// restricts extension scripts to only modify request info
// OnErrorExtensionTask adds error-recovery controls to ExtensionTask.
// Continue is intentionally only exposed to onError handlers.
type OnErrorExtensionTask struct {
download *Downloader
*ExtensionTask
}
func cloneExtensionTask(task *Task) *Task {
newTask := task.clone()
newTask.Meta.Req = task.Meta.Req
return &ExtensionTask{
download: download,
Task: newTask,
return newTask
}
func newExtensionTask(task *Task) *ExtensionTask {
return &ExtensionTask{Task: cloneExtensionTask(task)}
}
func newOnStartExtensionTask(task *Task) *ExtensionTask {
return newExtensionTask(task)
}
func newOnErrorExtensionTask(download *Downloader, task *Task) *OnErrorExtensionTask {
return &OnErrorExtensionTask{
download: download,
ExtensionTask: newExtensionTask(task),
}
}
func (t *ExtensionTask) Continue() error {
return t.download.Continue(&TaskFilter{
IDs: []string{t.ID},
})
func newOnDoneExtensionTask(task *Task) *Task {
return cloneExtensionTask(task)
}
func (t *ExtensionTask) Pause() error {
return t.download.Pause(&TaskFilter{
// SetUrl replaces the task request URL.
func (t *ExtensionTask) SetUrl(url string) {
t.Meta.Req.URL = url
}
func (t *OnErrorExtensionTask) Continue() error {
return t.download.Continue(&TaskFilter{
IDs: []string{t.ID},
})
}
+75
View File
@@ -0,0 +1,75 @@
package download
import (
"fmt"
"os"
)
type extensionEngineRunner interface {
Close()
RunString(script string) (any, error)
}
type ExtensionEngine struct {
engine extensionEngineRunner
session *engineSession
}
func (d *Downloader) NewExtensionEngine(ext *Extension, settings map[string]any) (*ExtensionEngine, error) {
if ext == nil {
return nil, fmt.Errorf("extension is nil")
}
engine, session := d.newExtensionEngine()
gopeed := &Instance{
Events: make(InstanceEvents),
Info: NewExtensionInfo(ext),
Logger: newInstanceLogger(ext, d.ExtensionLogger),
Settings: settings,
Storage: &ContextStorage{
storage: d.storage,
identity: ext.buildIdentity(),
},
Runtime: &InstanceRuntime{
WebView: d.newExtensionWebViewRuntime(session),
},
}
if err := injectGopeed(engine.Runtime, gopeed); err != nil {
session.CloseIfIdle()
return nil, err
}
return &ExtensionEngine{
engine: engine,
session: session,
}, nil
}
func (r *ExtensionEngine) Eval(script string) (any, error) {
if r == nil || r.engine == nil {
return nil, fmt.Errorf("extension engine not initialized")
}
return r.engine.RunString(script)
}
func (r *ExtensionEngine) EvalFile(path string) (any, error) {
if path == "" {
return nil, fmt.Errorf("script path is empty")
}
buf, err := os.ReadFile(path)
if err != nil {
return nil, err
}
return r.Eval(string(buf))
}
func (r *ExtensionEngine) Close() {
if r == nil {
return
}
if r.session != nil {
r.session.CloseIfIdle()
} else if r.engine != nil {
r.engine.Close()
}
r.engine = nil
r.session = nil
}
+146
View File
@@ -0,0 +1,146 @@
package download
import (
"context"
"io"
"sync"
internalblob "github.com/GopeedLab/gopeed/internal/blob"
"github.com/GopeedLab/gopeed/pkg/download/engine"
"github.com/GopeedLab/gopeed/pkg/download/engine/inject/stream"
enginewebview "github.com/GopeedLab/gopeed/pkg/download/engine/webview"
)
type engineSession struct {
engine *engine.Engine
mu sync.Mutex
refs int
closed bool
close []func()
}
func newEngineSession(e *engine.Engine) *engineSession {
return &engineSession{engine: e}
}
func (s *engineSession) SetEngine(e *engine.Engine) {
s.mu.Lock()
defer s.mu.Unlock()
s.engine = e
}
func (s *engineSession) Retain() {
s.mu.Lock()
defer s.mu.Unlock()
if s.closed {
return
}
s.refs++
}
func (s *engineSession) Release() {
s.mu.Lock()
if s.refs > 0 {
s.refs--
}
shouldClose := s.refs == 0 && !s.closed
if shouldClose {
s.closed = true
}
s.mu.Unlock()
if shouldClose {
s.runClosers()
if s.engine != nil {
go s.engine.Close()
}
}
}
func (s *engineSession) CloseIfIdle() {
s.mu.Lock()
shouldClose := s.refs == 0 && !s.closed
if shouldClose {
s.closed = true
}
s.mu.Unlock()
if shouldClose {
s.runClosers()
if s.engine != nil {
s.engine.Close()
}
}
}
func (s *engineSession) OnClose(fn func()) {
if fn == nil {
return
}
s.mu.Lock()
if s.closed {
s.mu.Unlock()
fn()
return
}
s.close = append(s.close, fn)
s.mu.Unlock()
}
func (s *engineSession) runClosers() {
s.mu.Lock()
closeFns := s.close
s.close = nil
s.mu.Unlock()
for _, fn := range closeFns {
fn()
}
}
func (d *Downloader) newExtensionEngine() (*engine.Engine, *engineSession) {
session := newEngineSession(nil)
engineCfg := &stream.Config{
CreateObjectURL: func(opts *stream.ObjectURLOptions, open stream.ObjectURLOpener) (string, error) {
createOpts := &internalblob.CreateOptions{
Session: session,
}
if opts != nil {
createOpts.ContentType = opts.ContentType
createOpts.Size = opts.Size
createOpts.Range = opts.Range
}
return d.blob.CreateOpener(func(ctx context.Context, req internalblob.OpenRequest) (io.ReadCloser, error) {
return open(ctx, stream.ObjectURLOpenRequest{
Offset: req.Offset,
End: req.End,
})
}, createOpts)
},
RevokeObjectURL: func(url string) error {
return d.blob.Revoke(url)
},
ProxyHandler: d.cfg.Proxy.ToHandler(),
RegisterCleanup: session.OnClose,
}
e := engine.NewEngine(&engine.Config{
ProxyConfig: d.cfg.Proxy,
StreamConfig: engineCfg,
})
session.SetEngine(e)
return e, session
}
func (d *Downloader) newExtensionWebViewRuntime(session *engineSession) *enginewebview.Runtime {
var (
opener enginewebview.Opener
available bool
)
if provider := d.cfg.WebViewProvider; provider != nil && provider.IsAvailable() {
opener = provider
available = true
}
runtime := enginewebview.NewRuntime(opener, available)
session.OnClose(func() {
_ = runtime.Close()
})
return runtime
}
+289
View File
@@ -0,0 +1,289 @@
package download
import (
"fmt"
"github.com/GopeedLab/gopeed/pkg/download/engine"
enginewebview "github.com/GopeedLab/gopeed/pkg/download/engine/webview"
"github.com/dop251/goja"
)
func injectGopeed(vm *goja.Runtime, gopeed *Instance) error {
gopeedObject := vm.NewObject()
if gopeed == nil {
return vm.Set("gopeed", gopeedObject)
}
if err := gopeedObject.Set("events", newJSEventsRuntime(vm, gopeed.Events)); err != nil {
return err
}
if err := gopeedObject.Set("info", gopeed.Info); err != nil {
return err
}
if err := gopeedObject.Set("logger", gopeed.Logger); err != nil {
return err
}
if err := gopeedObject.Set("settings", gopeed.Settings); err != nil {
return err
}
if err := gopeedObject.Set("storage", gopeed.Storage); err != nil {
return err
}
runtimeObject := vm.NewObject()
if gopeed.Runtime != nil {
if err := runtimeObject.Set("blob", newJSBlobRuntime(vm)); err != nil {
return err
}
if gopeed.Runtime.WebView != nil {
if err := runtimeObject.Set("webview", newJSWebViewRuntime(vm, gopeed.Runtime.WebView)); err != nil {
return err
}
}
}
if err := gopeedObject.Set("runtime", runtimeObject); err != nil {
return err
}
return vm.Set("gopeed", gopeedObject)
}
func newJSBlobRuntime(vm *goja.Runtime) *goja.Object {
obj := vm.NewObject()
_ = obj.Set("createObjectURL", func(call goja.FunctionCall) goja.Value {
fn, ok := goja.AssertFunction(vm.Get("__gopeed_blob_create_object_url"))
if !ok {
panic(vm.ToValue(fmt.Errorf("blob runtime is not available")))
}
value, err := fn(goja.Undefined(), call.Argument(0), call.Argument(1))
if err != nil {
panic(vm.ToValue(err))
}
return value
})
_ = obj.Set("revokeObjectURL", func(call goja.FunctionCall) goja.Value {
fn, ok := goja.AssertFunction(vm.Get("__gopeed_blob_revoke_object_url"))
if !ok {
panic(vm.ToValue(fmt.Errorf("blob runtime is not available")))
}
if _, err := fn(goja.Undefined(), call.Argument(0)); err != nil {
panic(vm.ToValue(err))
}
return goja.Undefined()
})
return obj
}
func newJSEventsRuntime(vm *goja.Runtime, events InstanceEvents) *goja.Object {
obj := vm.NewObject()
register := func(event ActivationEvent, call goja.FunctionCall) goja.Value {
if len(call.Arguments) == 0 {
panic(vm.ToValue(fmt.Errorf("missing handler")))
}
fnValue := call.Argument(0)
exported, ok := fnValue.Export().(func(goja.FunctionCall) goja.Value)
if !ok {
panic(vm.ToValue(fmt.Errorf("handler must be a function")))
}
events.register(event, engine.JSFunction(exported))
return goja.Undefined()
}
_ = obj.Set("onResolve", func(call goja.FunctionCall) goja.Value {
return register(EventOnResolve, call)
})
_ = obj.Set("onStart", func(call goja.FunctionCall) goja.Value {
return register(EventOnStart, call)
})
_ = obj.Set("onError", func(call goja.FunctionCall) goja.Value {
return register(EventOnError, call)
})
_ = obj.Set("onDone", func(call goja.FunctionCall) goja.Value {
return register(EventOnDone, call)
})
return obj
}
func newJSWebViewRuntime(vm *goja.Runtime, runtime *enginewebview.Runtime) *goja.Object {
obj := vm.NewObject()
_ = obj.Set("isAvailable", func(goja.FunctionCall) goja.Value {
return vm.ToValue(runtime.IsAvailable())
})
_ = obj.Set("open", func(call goja.FunctionCall) goja.Value {
page, err := runtime.Open(optionalMap(call.Argument(0)))
if err != nil {
panic(vm.ToValue(err))
}
return newJSWebViewPage(vm, page)
})
return obj
}
func newJSWebViewPage(vm *goja.Runtime, page *enginewebview.PageHandle) *goja.Object {
obj := vm.NewObject()
_ = obj.Set("addInitScript", func(call goja.FunctionCall) goja.Value {
script, err := requireStringArg(call, 0, "script")
if err != nil {
panic(vm.ToValue(err))
}
if err := page.AddInitScript(script); err != nil {
panic(vm.ToValue(err))
}
return goja.Undefined()
})
_ = obj.Set("goto", func(call goja.FunctionCall) goja.Value {
url, err := requireStringArg(call, 0, "url")
if err != nil {
panic(vm.ToValue(err))
}
if err := page.Goto(
url,
optionalMap(call.Argument(1)),
); err != nil {
panic(vm.ToValue(err))
}
return goja.Undefined()
})
_ = obj.Set("execute", func(call goja.FunctionCall) goja.Value {
expression, err := enginewebview.NormalizeExecutableValue(call.Argument(0))
if err != nil {
panic(vm.ToValue(err))
}
result, err := page.Execute(expression, exportArgs(call.Arguments[1:])...)
if err != nil {
panic(vm.ToValue(err))
}
return vm.ToValue(result)
})
_ = obj.Set("focus", func(call goja.FunctionCall) goja.Value {
selector, err := requireStringArg(call, 0, "selector")
if err != nil {
panic(vm.ToValue(err))
}
if err := page.Focus(selector); err != nil {
panic(vm.ToValue(err))
}
return goja.Undefined()
})
_ = obj.Set("click", func(call goja.FunctionCall) goja.Value {
selector, err := requireStringArg(call, 0, "selector")
if err != nil {
panic(vm.ToValue(err))
}
if err := page.Click(selector, optionalMap(call.Argument(1))); err != nil {
panic(vm.ToValue(err))
}
return goja.Undefined()
})
_ = obj.Set("type", func(call goja.FunctionCall) goja.Value {
selector, err := requireStringArg(call, 0, "selector")
if err != nil {
panic(vm.ToValue(err))
}
text, err := requireStringArg(call, 1, "text")
if err != nil {
panic(vm.ToValue(err))
}
if err := page.Type(selector, text, optionalMap(call.Argument(2))); err != nil {
panic(vm.ToValue(err))
}
return goja.Undefined()
})
_ = obj.Set("waitForSelector", func(call goja.FunctionCall) goja.Value {
selector, err := requireStringArg(call, 0, "selector")
if err != nil {
panic(vm.ToValue(err))
}
result, err := page.WaitForSelector(
selector,
optionalMap(call.Argument(1)),
)
if err != nil {
panic(vm.ToValue(err))
}
return vm.ToValue(result)
})
_ = obj.Set("waitForFunction", func(call goja.FunctionCall) goja.Value {
expression, err := enginewebview.NormalizeExecutableValue(call.Argument(0))
if err != nil {
panic(vm.ToValue(err))
}
args := exportArgs(call.Arguments[1:])
result, err := page.WaitForFunction(expression, args...)
if err != nil {
panic(vm.ToValue(err))
}
return vm.ToValue(result)
})
_ = obj.Set("getCookies", func(goja.FunctionCall) goja.Value {
result, err := page.GetCookies()
if err != nil {
panic(vm.ToValue(err))
}
return vm.ToValue(result)
})
_ = obj.Set("setCookie", func(call goja.FunctionCall) goja.Value {
if err := page.SetCookie(optionalMap(call.Argument(0))); err != nil {
panic(vm.ToValue(err))
}
return goja.Undefined()
})
_ = obj.Set("deleteCookie", func(call goja.FunctionCall) goja.Value {
if err := page.DeleteCookie(optionalMap(call.Argument(0))); err != nil {
panic(vm.ToValue(err))
}
return goja.Undefined()
})
_ = obj.Set("clearCookies", func(goja.FunctionCall) goja.Value {
if err := page.ClearCookies(); err != nil {
panic(vm.ToValue(err))
}
return goja.Undefined()
})
_ = obj.Set("url", func(goja.FunctionCall) goja.Value {
result, err := page.URL()
if err != nil {
panic(vm.ToValue(err))
}
return vm.ToValue(result)
})
_ = obj.Set("content", func(goja.FunctionCall) goja.Value {
result, err := page.Content()
if err != nil {
panic(vm.ToValue(err))
}
return vm.ToValue(result)
})
_ = obj.Set("close", func(goja.FunctionCall) goja.Value {
if err := page.Close(); err != nil {
panic(vm.ToValue(err))
}
return goja.Undefined()
})
return obj
}
func optionalMap(value goja.Value) map[string]any {
if value == nil || goja.IsUndefined(value) || goja.IsNull(value) {
return nil
}
if exported, ok := value.Export().(map[string]any); ok {
return exported
}
return nil
}
func exportArgs(values []goja.Value) []any {
if len(values) == 0 {
return nil
}
args := make([]any, 0, len(values))
for _, value := range values {
args = append(args, value.Export())
}
return args
}
func requireStringArg(call goja.FunctionCall, index int, name string) (string, error) {
value := call.Argument(index)
if value == nil || goja.IsUndefined(value) || goja.IsNull(value) {
return "", fmt.Errorf(`missing or invalid "%s"`, name)
}
return value.String(), nil
}
File diff suppressed because it is too large. Load diff
+5 -2
View File
@@ -11,6 +11,7 @@ import (
"github.com/GopeedLab/gopeed/internal/protocol/ed2k"
"github.com/GopeedLab/gopeed/internal/protocol/http"
"github.com/GopeedLab/gopeed/pkg/base"
enginewebview "github.com/GopeedLab/gopeed/pkg/download/engine/webview"
"github.com/GopeedLab/gopeed/pkg/util"
gonanoid "github.com/matoous/go-nanoid/v2"
)
@@ -27,7 +28,6 @@ type Task struct {
Status base.Status `json:"status"`
Uploading bool `json:"uploading"`
Progress *Progress `json:"progress"`
IsCreated bool `json:"isCreated"`
CreatedAt time.Time `json:"createdAt"`
UpdatedAt time.Time `json:"updatedAt"`
@@ -36,6 +36,9 @@ type Task struct {
timer *util.Timer
statusLock *sync.Mutex
lock *sync.Mutex
blobRefLock *sync.Mutex
blobURL string
runGeneration uint64
speedArr []int64
uploadSpeedArr []int64
}
@@ -50,7 +53,6 @@ func NewTask() *Task {
Status: base.DownloadStatusReady,
CreatedAt: time.Now(),
UpdatedAt: time.Now(),
IsCreated: false,
}
}
@@ -153,6 +155,7 @@ type DownloaderConfig struct {
Storage Storage
StorageDir string
WhiteDownloadDirs []string
WebViewProvider enginewebview.Provider
ProductionMode bool
+3
View File
@@ -1,4 +1,7 @@
gopeed.events.onResolve(async function (ctx) {
ctx.req.setLabels({ replaced: "true", removed: "true" });
ctx.req.putLabel("modified", "true");
ctx.req.delLabel("removed");
ctx.res = {
name: "test",
files: Array(2).fill(true).map((_, i) => ({
+111
View File
@@ -0,0 +1,111 @@
gopeed.events.onResolve(async function (ctx) {
const encoder = new TextEncoder();
if (ctx.req.url.endsWith("/blob")) {
const blob = new Blob(["hello world"], {type: "text/plain"});
const url = gopeed.runtime.blob.createObjectURL(blob);
ctx.res = {
name: "blob-blob",
files: [
{
name: "hello.txt",
size: 11,
req: {
url,
}
}
]
};
return;
}
if (ctx.req.url.endsWith("/opener-range")) {
const payload = encoder.encode("line 1\nline 2\nline 3\n");
const firstChunkLength = encoder.encode("line 1\n").byteLength;
let firstAttempt = true;
const open = async ({ offset = 0, end = -1 }) => new ReadableStream({
async start(controller) {
try {
if (offset === 0 && firstAttempt) {
firstAttempt = false;
controller.enqueue(payload.slice(0, firstChunkLength));
await new Promise((resolve) => setTimeout(resolve, 120));
controller.error(new Error("resume required"));
return;
}
controller.enqueue(payload.slice(offset, end >= 0 ? end + 1 : undefined));
controller.close();
} catch (err) {
controller.error(err);
}
}
});
const url = gopeed.runtime.blob.createObjectURL(open, { size: payload.byteLength, range: true });
ctx.res = {
name: "blob-opener-range",
range: true,
files: [
{
name: "opener-range.txt",
size: payload.byteLength,
req: {
url,
}
}
]
};
return;
}
if (ctx.req.url.endsWith("/opener-unknown")) {
const url = gopeed.runtime.blob.createObjectURL(async ({ offset = 0 }) => new ReadableStream({
async start(controller) {
if (offset > 0) {
controller.close();
return;
}
controller.enqueue(encoder.encode("line 1\n"));
await new Promise((resolve) => setTimeout(resolve, 1500));
controller.enqueue(encoder.encode("line 2\n"));
controller.close();
},
}));
ctx.res = {
name: "blob-opener-unknown",
files: [
{
name: "opener-unknown.txt",
req: {
url,
}
}
]
};
return;
}
const url = gopeed.runtime.blob.createObjectURL(async ({ offset = 0, end = -1 }) => new ReadableStream({
start(controller) {
const payload = encoder.encode("line 1\nline 2\n");
controller.enqueue(payload.slice(offset, end >= 0 ? end + 1 : undefined));
controller.close();
},
}), { size: 14 });
ctx.res = {
name: "blob-opener",
files: [
{
name: "opener.txt",
size: 14,
req: {
url,
}
}
]
};
});
+16
View File
@@ -0,0 +1,16 @@
{
"name": "blob",
"title": "gopeed extension blob test",
"version": "0.0.1",
"scripts": [
{
"event": "onResolve",
"match": {
"urls": [
"*://example.com/*"
]
},
"entry": "index.js"
}
]
}
@@ -0,0 +1,139 @@
const payloadSize = 262144;
const chunkSize = 4096;
const payloadText = 'x'.repeat(payloadSize);
function delayedPayloadStream(offset = 0, end = -1) {
const limit = end >= 0 ? Math.min(end + 1, payloadSize) : payloadSize;
let pos = offset;
return new ReadableStream({
async pull(controller) {
if (pos >= limit) {
controller.close();
return;
}
const size = Math.min(chunkSize, limit - pos);
const chunk = new Uint8Array(size);
chunk.fill(120);
pos += size;
controller.enqueue(chunk);
await new Promise((resolve) => setTimeout(resolve, 20));
},
});
}
function fastPayloadStream(offset = 0, end = -1) {
const limit = end >= 0 ? Math.min(end + 1, payloadSize) : payloadSize;
let pos = offset;
return new ReadableStream({
pull(controller) {
if (pos >= limit) {
controller.close();
return;
}
const size = Math.min(chunkSize, limit - pos);
const chunk = new Uint8Array(size);
chunk.fill(120);
pos += size;
controller.enqueue(chunk);
},
});
}
function createBlobSourceURL(delayed) {
const blob = new Blob([payloadText], { type: 'application/octet-stream' });
const originalSlice = blob.slice.bind(blob);
blob.slice = function (offset = 0, end) {
if (!delayed) {
return originalSlice(offset, end);
}
const sliceEnd = Number.isFinite(Number(end)) ? Number(end) - 1 : -1;
return {
stream() {
return delayedPayloadStream(Number(offset) || 0, sliceEnd);
},
};
};
return gopeed.runtime.blob.createObjectURL(blob);
}
function createOpenerSourceURL(delayed) {
const open = async () => delayed ? delayedPayloadStream(0, -1) : fastPayloadStream(0, -1);
return gopeed.runtime.blob.createObjectURL(open, {
size: payloadSize,
contentType: 'application/octet-stream',
});
}
function createRangeSourceURL(delayed) {
const open = async ({ offset = 0, end = -1 }) => delayed ? delayedPayloadStream(offset, end) : fastPayloadStream(offset, end);
return gopeed.runtime.blob.createObjectURL(open, {
size: payloadSize,
range: true,
contentType: 'application/octet-stream',
});
}
function modeFromURL(url) {
if (url.includes('/pause-rebuild-blob')) {
return 'blob';
}
if (url.includes('/pause-rebuild-opener')) {
return 'opener';
}
if (url.includes('/pause-rebuild-range')) {
return 'range';
}
return '';
}
function createPayloadURL(mode, delayed) {
switch (mode) {
case 'blob':
return createBlobSourceURL(delayed);
case 'opener':
return createOpenerSourceURL(delayed);
case 'range':
return createRangeSourceURL(delayed);
default:
throw new Error('unsupported pause rebuild mode: ' + mode);
}
}
gopeed.events.onResolve(async function (ctx) {
const mode = modeFromURL(ctx.req.url);
if (!mode) {
return;
}
ctx.res = {
name: 'blob-pause-rebuild-' + mode,
range: mode !== 'opener',
files: [
{
name: 'pause-rebuild-' + mode + '.bin',
size: payloadSize,
req: {
url: createPayloadURL(mode, true),
rawUrl: ctx.req.rawUrl || ctx.req.url,
labels: {
mode: 'pause-rebuild',
source: mode,
},
},
},
],
};
});
gopeed.events.onError(async function (ctx) {
const req = ctx.task?.meta?.req;
const source = req?.labels?.source || modeFromURL(req?.rawUrl || '');
if (!req || req.labels?.mode !== 'pause-rebuild' || req.labels?.rebuilt === 'true' || !source) {
return;
}
ctx.task.setUrl(createPayloadURL(source, false));
req.putLabel('started', 'true');
req.putLabel('rebuilt', 'true');
ctx.task.continue();
});
@@ -0,0 +1,28 @@
{
"name": "blob-pause-rebuild",
"title": "gopeed extension blob pause rebuild test",
"version": "0.0.1",
"scripts": [
{
"event": "onResolve",
"match": {
"urls": [
"*://example.com/pause-rebuild*"
]
},
"entry": "index.js"
},
{
"event": "onError",
"match": {
"labels": [
"mode"
],
"urls": [
"*://example.com/pause-rebuild*"
]
},
"entry": "index.js"
}
]
}
+46
View File
@@ -0,0 +1,46 @@
gopeed.events.onResolve(async function (ctx) {
const isRange = ctx.req.url.endsWith("/recover-range");
if (!isRange && !ctx.req.url.endsWith("/recover")) {
return;
}
const url = gopeed.runtime.blob.createObjectURL(async ({ offset = 0 }) => new ReadableStream({
async start(controller) {
if (offset > 0) {
controller.close();
return;
}
controller.enqueue(new TextEncoder().encode("stale-prefix-that-must-be-removed\n"));
await new Promise((resolve) => setTimeout(resolve, 50));
controller.error(new Error("expired"));
},
}), { size: 64, range: isRange });
ctx.res = {
name: "blob-recover",
files: [
{
name: "recover.txt",
size: 64,
req: {
url,
}
}
]
};
});
gopeed.events.onError(async function (ctx) {
const req = ctx.task.meta.req;
if (!req.rawUrl || (!req.rawUrl.endsWith("/recover") && !req.rawUrl.endsWith("/recover-range"))) {
return;
}
req.labels = req.labels || {};
if (req.labels.recovered === "true") {
return;
}
req.putLabel("recovered", "true");
ctx.task.setUrl(gopeed.runtime.blob.createObjectURL(new Blob(["ok\n"], { type: "text/plain" })));
ctx.task.continue();
});
@@ -0,0 +1,25 @@
{
"name": "blob-recover",
"title": "gopeed extension blob recover test",
"version": "0.0.1",
"scripts": [
{
"event": "onResolve",
"match": {
"urls": [
"*://example.com/*"
]
},
"entry": "index.js"
},
{
"event": "onError",
"match": {
"urls": [
"*://example.com/*"
]
},
"entry": "index.js"
}
]
}
+57
View File
@@ -0,0 +1,57 @@
function createRangeBlobUrl(target) {
return gopeed.runtime.blob.createObjectURL(async ({ offset = 0, end = -1 }) => {
const headers = {};
if (offset > 0 || end >= 0) {
headers.Range = `bytes=${offset}-${end >= 0 ? end : ''}`;
}
const response = await fetch(target, { headers });
if (!response.body) {
throw new Error('empty response body');
}
return response.body;
}, { size: 262144, range: true });
}
gopeed.events.onResolve(async function (ctx) {
if (!ctx.req.url.includes('/restart-range')) {
return;
}
ctx.res = {
name: 'blob-restart',
range: true,
files: [
{
name: 'restart.bin',
size: 262144,
req: {
url: createRangeBlobUrl(ctx.req.rawUrl || ctx.req.url),
rawUrl: ctx.req.rawUrl || ctx.req.url,
labels: {
mode: 'restart',
},
},
},
],
};
});
gopeed.events.onError(async function (ctx) {
const req = ctx.task?.meta?.req;
if (!req || !req.rawUrl || !req.url || !req.url.includes('/__blob/')) {
return;
}
if (req.labels.rebuilt === 'true') {
return;
}
try {
ctx.task.setUrl(createRangeBlobUrl(req.rawUrl));
req.putLabel('started', 'true');
req.putLabel('rebuilt', 'true');
ctx.task.continue();
} catch (error) {
req.putLabel('rebuildError', String(error));
throw error;
}
});
@@ -0,0 +1,28 @@
{
"name": "blob-restart",
"title": "gopeed extension blob restart resume test",
"version": "0.0.1",
"scripts": [
{
"event": "onResolve",
"match": {
"urls": [
"*://*/restart-range*"
]
},
"entry": "index.js"
},
{
"event": "onError",
"match": {
"labels": [
"mode"
],
"urls": [
"*://*/restart-range*"
]
},
"entry": "index.js"
}
]
}
@@ -0,0 +1,198 @@
const encoder = new TextEncoder();
const decoder = new TextDecoder();
function byteLength(value) {
return encoder.encode(value).byteLength;
}
function sourceURL(rawURL) {
const url = new URL(rawURL);
url.searchParams.set("source", "1");
return url.toString();
}
function createFetchDrainedStream(url) {
return new ReadableStream({
async start(controller) {
try {
const response = await fetch(url);
const reader = response.body.getReader();
while (true) {
const { done, value } = await reader.read();
if (done) {
controller.close();
return;
}
controller.enqueue(value);
}
} catch (error) {
controller.error(error);
}
},
});
}
function createFetchDrainedRangeStream(url, offset, end) {
return new ReadableStream({
async start(controller) {
try {
const headers = {};
if (offset > 0 || end >= 0) {
headers.Range = `bytes=${offset}-${end >= 0 ? end : ""}`;
}
const response = await fetch(url, { headers });
const reader = response.body.getReader();
while (true) {
const { done, value } = await reader.read();
if (done) {
controller.close();
return;
}
controller.enqueue(value);
}
} catch (error) {
controller.error(error);
}
},
});
}
function createMultiplexStreams(url) {
let videoController;
let audioController;
let started = false;
async function pump() {
if (started || !videoController || !audioController) {
return;
}
started = true;
try {
const response = await fetch(url);
const reader = response.body.getReader();
let pending = "";
while (true) {
const { done, value } = await reader.read();
if (done) {
break;
}
pending += decoder.decode(value, { stream: true });
let newlineIndex;
while ((newlineIndex = pending.indexOf("\n")) >= 0) {
const line = pending.slice(0, newlineIndex);
pending = pending.slice(newlineIndex + 1);
if (line.startsWith("v:")) {
videoController.enqueue(encoder.encode(line.slice(2)));
} else if (line.startsWith("a:")) {
audioController.enqueue(encoder.encode(line.slice(2)));
}
}
}
if (pending.startsWith("v:")) {
videoController.enqueue(encoder.encode(pending.slice(2)));
} else if (pending.startsWith("a:")) {
audioController.enqueue(encoder.encode(pending.slice(2)));
}
videoController.close();
audioController.close();
} catch (error) {
videoController.error(error);
audioController.error(error);
}
}
return {
video: new ReadableStream({
start(controller) {
videoController = controller;
pump();
},
}),
audio: new ReadableStream({
start(controller) {
audioController = controller;
pump();
},
}),
};
}
gopeed.events.onResolve(async function (ctx) {
if (ctx.req.url.includes("/single-fetch-stream")) {
const payloadSize = Number(new URL(ctx.req.url).searchParams.get("size")) || 0;
const url = gopeed.runtime.blob.createObjectURL(async () => createFetchDrainedStream(sourceURL(ctx.req.url)), {
size: payloadSize,
});
ctx.res = {
name: "single-fetch-stream",
files: [
{
name: "single.bin",
size: payloadSize,
req: { url },
},
],
};
return;
}
if (ctx.req.url.includes("/range-fetch-stream")) {
const payloadSize = Number(new URL(ctx.req.url).searchParams.get("size")) || 0;
const url = gopeed.runtime.blob.createObjectURL(
async ({ offset = 0, end = -1 }) => createFetchDrainedRangeStream(sourceURL(ctx.req.url), offset, end),
{
size: payloadSize,
range: true,
}
);
ctx.res = {
name: "range-fetch-stream",
range: true,
files: [
{
name: "range.bin",
size: payloadSize,
req: { url },
},
],
};
return;
}
if (ctx.req.url.includes("/multiplex-fetch-stream")) {
const videoChunks = ["video-0", "video-1", "video-2", "video-3"];
const audioChunks = ["audio-0", "audio-1", "audio-2", "audio-3"];
let streams;
const openStreams = () => {
if (!streams) {
streams = createMultiplexStreams(sourceURL(ctx.req.url));
}
return streams;
};
const videoUrl = gopeed.runtime.blob.createObjectURL(async () => openStreams().video, {
size: videoChunks.reduce((sum, chunk) => sum + byteLength(chunk), 0),
});
const audioUrl = gopeed.runtime.blob.createObjectURL(async () => openStreams().audio, {
size: audioChunks.reduce((sum, chunk) => sum + byteLength(chunk), 0),
});
ctx.res = {
name: "multiplex-fetch-stream",
files: [
{
name: "video.bin",
size: videoChunks.reduce((sum, chunk) => sum + byteLength(chunk), 0),
req: { url: videoUrl },
},
{
name: "audio.bin",
size: audioChunks.reduce((sum, chunk) => sum + byteLength(chunk), 0),
req: { url: audioUrl },
},
],
};
}
});
@@ -0,0 +1,18 @@
{
"name": "blob-stream-bridge",
"title": "gopeed extension blob stream bridge test",
"version": "0.0.1",
"scripts": [
{
"event": "onResolve",
"match": {
"urls": [
"*://*/single-fetch-stream*",
"*://*/range-fetch-stream*",
"*://*/multiplex-fetch-stream*"
]
},
"entry": "index.js"
}
]
}
+1 -2
View File
@@ -1,7 +1,6 @@
gopeed.events.onError(async function (ctx) {
gopeed.logger.info("url", ctx.task.meta.req.url);
gopeed.logger.info("error", ctx.error);
ctx.task.meta.req.url = "https://github.com";
ctx.task.setUrl("https://github.com");
ctx.task.continue();
});
+2 -3
View File
@@ -1,6 +1,5 @@
gopeed.events.onStart(async function (ctx) {
gopeed.logger.info("url", ctx.task.meta.req.url);
ctx.task.meta.req.url = "https://github.com";
ctx.task.meta.req.labels['modified'] = 'true';
ctx.task.setUrl("https://github.com");
ctx.task.meta.req.putLabel('modified', 'true');
});
Vendored Regular → Executable
View File
File mode changed.
Vendored Regular → Executable
View File
File mode changed.
Vendored Regular → Executable
View File
File mode changed.
Vendored Regular → Executable
View File
File mode changed.
+4 -1
View File
@@ -2,6 +2,7 @@ package model
import (
"github.com/GopeedLab/gopeed/pkg/base"
enginewebview "github.com/GopeedLab/gopeed/pkg/download/engine/webview"
"io/fs"
)
@@ -21,8 +22,10 @@ type StartConfig struct {
WhiteDownloadDirs []string `json:"whiteDownloadDirs"`
ApiToken string `json:"apiToken"`
DownloadConfig *base.DownloaderStoreConfig `json:"downloadConfig"`
WebViewRPCConfig *enginewebview.RPCConfig `json:"webViewRpcConfig,omitempty"`
ProductionMode bool
ProductionMode bool
WebViewProvider enginewebview.Provider `json:"-"`
WebEnable bool
WebFS fs.FS
+1
View File
@@ -89,6 +89,7 @@ func BuildServer(startCfg *model.StartConfig) (*http.Server, net.Listener, error
ProductionMode: startCfg.ProductionMode,
RefreshInterval: startCfg.RefreshInterval,
WhiteDownloadDirs: startCfg.WhiteDownloadDirs,
WebViewProvider: startCfg.WebViewProvider,
}
if startCfg.Storage == model.StorageBolt {
downloadCfg.Storage = download.NewBoltStorage(startCfg.StorageDir)
+120 -4
View File
@@ -20,6 +20,7 @@ import (
"github.com/GopeedLab/gopeed/internal/test"
"github.com/GopeedLab/gopeed/pkg/base"
"github.com/GopeedLab/gopeed/pkg/download"
enginewebview "github.com/GopeedLab/gopeed/pkg/download/engine/webview"
"github.com/GopeedLab/gopeed/pkg/rest/model"
)
@@ -66,6 +67,10 @@ var (
}
)
func currentTestDownloadFile() string {
return filepath.Join(createOpts.Path, createOpts.Name)
}
func TestInfo(t *testing.T) {
matchKeys := []string{"version", "runtime", "os", "arch", "inDocker"}
doTest(func() {
@@ -108,7 +113,7 @@ func TestCreateTask(t *testing.T) {
wg.Wait()
want := test.FileMd5(test.BuildFile)
got := test.FileMd5(test.DownloadFile)
got := test.FileMd5(currentTestDownloadFile())
if want != got {
t.Errorf("CreateTask() got = %v, want %v", got, want)
}
@@ -132,7 +137,7 @@ func TestCreateDirectTask(t *testing.T) {
wg.Wait()
want := test.FileMd5(test.BuildFile)
got := test.FileMd5(test.DownloadFile)
got := test.FileMd5(currentTestDownloadFile())
if want != got {
t.Errorf("CreateDirectTask() got = %v, want %v", got, want)
}
@@ -225,7 +230,7 @@ func TestPauseAndContinueTask(t *testing.T) {
wg.Wait()
want := test.FileMd5(test.BuildFile)
got := test.FileMd5(test.DownloadFile)
got := test.FileMd5(currentTestDownloadFile())
if !reflect.DeepEqual(got, want) {
t.Errorf("PauseAndContinueTask() got = %v, want %v", got, want)
}
@@ -288,6 +293,10 @@ func TestPatchTaskNotFound(t *testing.T) {
func TestPauseAllAndContinueALLTasks(t *testing.T) {
doTest(func() {
slowListener := test.StartTestLowSpeedServer(5 * time.Nanosecond)
defer slowListener.Close()
taskReq.URL = "http://" + slowListener.Addr().String() + "/" + test.BuildName
cfg, err := Downloader.GetConfig()
if err != nil {
t.Fatal(err)
@@ -339,7 +348,7 @@ func TestDeleteTaskForce(t *testing.T) {
httpRequestCheckOk[any](http.MethodDelete, "/api/v1/tasks/"+taskId+"?force=true", nil)
code, _ := httpRequest[*download.Task](http.MethodGet, "/api/v1/tasks/"+taskId, nil)
checkCode(code, model.CodeTaskNotFound)
if _, err := os.Stat(test.DownloadFile); !errors.Is(err, os.ErrNotExist) {
if _, err := os.Stat(currentTestDownloadFile()); !errors.Is(err, os.ErrNotExist) {
t.Errorf("DeleteTaskForce() got = %v, want %v", err, os.ErrNotExist)
}
})
@@ -875,16 +884,77 @@ func TestAuthorization(t *testing.T) {
}
}
func TestBuildServerPropagatesWebViewProvider(t *testing.T) {
listener, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
addr := listener.Addr().String()
listener.Close()
cfg := &model.StartConfig{
Network: "tcp",
Address: addr,
Storage: model.StorageMem,
WebViewProvider: fakeRestWebViewProvider{available: true},
}
server, serverListener, err := BuildServer(cfg)
if err != nil {
t.Fatal(err)
}
defer serverListener.Close()
defer server.Close()
defer func() {
if Downloader != nil {
Downloader.Clear()
Downloader = nil
}
}()
runtime, err := Downloader.NewExtensionEngine(&download.Extension{
Name: "test-runtime",
Author: "gopeed",
Title: "Gopeed Test Script Runtime",
Version: "0.0.0",
DevMode: true,
}, map[string]any{})
if err != nil {
t.Fatal(err)
}
defer runtime.Close()
value, err := runtime.Eval("gopeed.runtime.webview.isAvailable()")
if err != nil {
t.Fatal(err)
}
if value != true {
t.Fatalf("expected provider to reach downloader runtime, got %#v", value)
}
}
func doTest(handler func()) {
doTest0(nil, handler)
}
func doTest0(onStart func(cfg *model.StartConfig), handler func()) {
testFunc := func(storage model.Storage) {
downloadDir, err := os.MkdirTemp("", "gopeed-rest-test-download-")
if err != nil {
panic(err)
}
defer os.RemoveAll(downloadDir)
oldCreatePath := createOpts.Path
createOpts.Path = downloadDir
defer func() {
createOpts.Path = oldCreatePath
}()
var cfg = &model.StartConfig{}
cfg.Init()
cfg.Storage = storage
cfg.StorageDir = ".test_storage"
cfg.DownloadConfig = (&base.DownloaderStoreConfig{DownloadDir: downloadDir}).Init()
cfg.WebEnable = true
if onStart != nil {
onStart(cfg)
@@ -919,6 +989,52 @@ func doStart(cfg *model.StartConfig) net.Listener {
return test.StartTestFileServer()
}
type fakeRestWebViewProvider struct {
available bool
}
func (p fakeRestWebViewProvider) IsAvailable() bool {
return p.available
}
func (p fakeRestWebViewProvider) Open(enginewebview.OpenOptions) (enginewebview.Page, error) {
return fakeRestPage{}, nil
}
type fakeRestPage struct{}
func (fakeRestPage) AddInitScript(string) error {
return nil
}
func (fakeRestPage) Goto(string, enginewebview.GotoOptions) error {
return nil
}
func (fakeRestPage) Execute(string, ...any) (any, error) {
return nil, nil
}
func (fakeRestPage) GetCookies() ([]enginewebview.Cookie, error) {
return nil, nil
}
func (fakeRestPage) SetCookie(enginewebview.Cookie) error {
return nil
}
func (fakeRestPage) DeleteCookie(enginewebview.Cookie) error {
return nil
}
func (fakeRestPage) ClearCookies() error {
return nil
}
func (fakeRestPage) Close() error {
return nil
}
func doHttpRequest0(method string, path string, headers map[string]string, body any) (int, []byte) {
r1, _, r3 := doHttpRequest1(method, path, headers, body)
return r1, r3
+1 -1
View File
@@ -24,7 +24,7 @@ if (flutterVersionName == null) {
android {
namespace 'com.gopeed.gopeed'
compileSdk 35
compileSdk 36
ndkVersion flutter.ndkVersion
compileOptions {
@@ -27,6 +27,30 @@
android:theme="@style/LaunchTheme"
android:windowSoftInputMode="adjustResize">
<!-- Specifies an Android theme to apply to this Activity as soon as
the Android process has started. This theme is visible to the user
while the Flutter UI initializes. After that, this theme continues
to determine the Window background behind the Flutter UI. -->
<meta-data
android:name="io.flutter.embedding.android.NormalTheme"
android:resource="@style/NormalTheme" />
<intent-filter>
<action android:name="android.intent.action.MAIN" />
<category android:name="android.intent.category.LAUNCHER" />
</intent-filter>
</activity>
<activity
android:name=".CreateDialogActivity"
android:configChanges="orientation|keyboardHidden|keyboard|screenSize|smallestScreenSize|locale|layoutDirection|fontScale|screenLayout|density|uiMode"
android:excludeFromRecents="true"
android:exported="true"
android:finishOnTaskLaunch="true"
android:hardwareAccelerated="true"
android:launchMode="singleTask"
android:theme="@style/DialogLaunchTheme"
android:windowSoftInputMode="adjustResize">
<intent-filter>
<action android:name="android.intent.action.VIEW" />
<category android:name="android.intent.category.DEFAULT" />
@@ -83,18 +107,6 @@
<action android:name="android.intent.action.VIEW_DOWNLOADS" />
<category android:name="android.intent.category.DEFAULT" />
</intent-filter>
<!-- Specifies an Android theme to apply to this Activity as soon as
the Android process has started. This theme is visible to the user
while the Flutter UI initializes. After that, this theme continues
to determine the Window background behind the Flutter UI. -->
<meta-data
android:name="io.flutter.embedding.android.NormalTheme"
android:resource="@style/NormalTheme" />
<intent-filter>
<action android:name="android.intent.action.MAIN" />
<category android:name="android.intent.category.LAUNCHER" />
</intent-filter>
<intent-filter>
<action android:name="android.intent.action.SEND" />
<action android:name="android.intent.action.SEND_MULTIPLE" />
@@ -102,6 +114,10 @@
<category android:name="android.intent.category.DEFAULT" />
<data android:mimeType="*/*" />
</intent-filter>
<meta-data
android:name="io.flutter.embedding.android.NormalTheme"
android:resource="@style/DialogNormalTheme" />
</activity>
<!-- Add android:stopWithTask option only when necessary. -->
@@ -116,4 +132,4 @@
android:name="flutterEmbedding"
android:value="2" />
</application>
</manifest>
</manifest>
@@ -0,0 +1,31 @@
package com.gopeed.gopeed
import android.os.Bundle
import android.view.Gravity
import android.view.WindowManager
import io.flutter.embedding.android.FlutterActivityLaunchConfigs
class CreateDialogActivity : MainActivity() {
override fun isDialogMode(): Boolean = true
override fun getBackgroundMode(): FlutterActivityLaunchConfigs.BackgroundMode {
return FlutterActivityLaunchConfigs.BackgroundMode.transparent
}
override fun onCreate(savedInstanceState: Bundle?) {
super.onCreate(savedInstanceState)
val metrics = resources.displayMetrics
val width = (metrics.widthPixels * 0.92f).toInt()
val maxHeight = (metrics.heightPixels * 0.62f).toInt()
val preferredHeight = (420 * metrics.density).toInt()
window.setGravity(Gravity.CENTER)
window.setDimAmount(0.28f)
window.setLayout(width, minOf(maxHeight, preferredHeight))
val attrs = window.attributes
attrs.flags = attrs.flags or WindowManager.LayoutParams.FLAG_DIM_BEHIND
window.attributes = attrs
}
}
@@ -7,9 +7,11 @@ import io.flutter.embedding.engine.FlutterEngine
import io.flutter.plugin.common.MethodChannel
import io.flutter.plugin.common.StandardMethodCodec
class MainActivity : FlutterActivity() {
open class MainActivity : FlutterActivity() {
private val CHANNEL = "gopeed.com/libgopeed"
protected open fun isDialogMode(): Boolean = false
override fun configureFlutterEngine(@NonNull flutterEngine: FlutterEngine) {
super.configureFlutterEngine(flutterEngine)
val taskQueue =
@@ -34,6 +36,9 @@ class MainActivity : FlutterActivity() {
Libgopeed.stop()
result.success(null)
}
"isDialogMode" -> {
result.success(isDialogMode())
}
else -> {
result.notImplemented()
}
@@ -15,4 +15,18 @@
<style name="NormalTheme" parent="@android:style/Theme.Black.NoTitleBar">
<item name="android:windowBackground">?android:colorBackground</item>
</style>
<style name="DialogLaunchTheme" parent="@android:style/Theme.Translucent.NoTitleBar">
<item name="android:windowIsTranslucent">true</item>
<item name="android:windowIsFloating">true</item>
<item name="android:windowBackground">@android:color/transparent</item>
<item name="android:backgroundDimEnabled">true</item>
<item name="android:windowNoTitle">true</item>
</style>
<style name="DialogNormalTheme" parent="@android:style/Theme.Translucent.NoTitleBar">
<item name="android:windowIsTranslucent">true</item>
<item name="android:windowIsFloating">true</item>
<item name="android:windowBackground">@android:color/transparent</item>
<item name="android:backgroundDimEnabled">true</item>
<item name="android:windowNoTitle">true</item>
</style>
</resources>
@@ -15,4 +15,18 @@
<style name="NormalTheme" parent="@android:style/Theme.Light.NoTitleBar">
<item name="android:windowBackground">?android:colorBackground</item>
</style>
<style name="DialogLaunchTheme" parent="@android:style/Theme.Translucent.NoTitleBar">
<item name="android:windowIsTranslucent">true</item>
<item name="android:windowIsFloating">true</item>
<item name="android:windowBackground">@android:color/transparent</item>
<item name="android:backgroundDimEnabled">true</item>
<item name="android:windowNoTitle">true</item>
</style>
<style name="DialogNormalTheme" parent="@android:style/Theme.Translucent.NoTitleBar">
<item name="android:windowIsTranslucent">true</item>
<item name="android:windowIsFloating">true</item>
<item name="android:windowBackground">@android:color/transparent</item>
<item name="android:backgroundDimEnabled">true</item>
<item name="android:windowNoTitle">true</item>
</style>
</resources>
+1 -1
View File
@@ -46,7 +46,7 @@ class _Client {
dio.options.contentType = Headers.jsonContentType;
dio.options.sendTimeout = const Duration(seconds: 5);
dio.options.connectTimeout = const Duration(seconds: 5);
dio.options.receiveTimeout = const Duration(seconds: 60);
dio.options.receiveTimeout = const Duration(seconds: 120);
dio.interceptors.add(InterceptorsWrapper(
onRequest: (options, handler) {
if (apiToken.isNotEmpty) {
@@ -49,7 +49,7 @@ Map<String, dynamic> _$CreateTaskBatchItemToJson(CreateTaskBatchItem instance) {
}
}
writeNotNull('req', instance.req);
writeNotNull('opts', instance.opts);
writeNotNull('req', instance.req?.toJson());
writeNotNull('opts', instance.opts?.toJson());
return val;
}
+2
View File
@@ -4,6 +4,7 @@ part 'request.g.dart';
@JsonSerializable(explicitToJson: true)
class Request {
String? rawUrl;
String url;
Object? extra;
Map<String, String>? labels;
@@ -11,6 +12,7 @@ class Request {
bool skipVerifyCert;
Request({
this.rawUrl,
required this.url,
this.extra,
this.labels,
+2
View File
@@ -7,6 +7,7 @@ part of 'request.dart';
// **************************************************************************
Request _$RequestFromJson(Map<String, dynamic> json) => Request(
rawUrl: json['rawUrl'] as String?,
url: json['url'] as String,
extra: json['extra'],
labels: (json['labels'] as Map<String, dynamic>?)?.map(
@@ -20,6 +21,7 @@ Request _$RequestFromJson(Map<String, dynamic> json) => Request(
Map<String, dynamic> _$RequestToJson(Request instance) {
final val = <String, dynamic>{
'rawUrl': instance.rawUrl,
'url': instance.url,
};
@@ -4,8 +4,8 @@ import 'dart:io';
import 'dart:ui';
import 'package:app_links/app_links.dart';
import 'package:dio/dio.dart';
import 'package:flutter/material.dart';
import 'package:flutter/services.dart';
import 'package:flutter_foreground_task/flutter_foreground_task.dart';
import 'package:get/get.dart';
import 'package:launch_at_startup/launch_at_startup.dart';
@@ -17,13 +17,11 @@ import 'package:uri_to_file/uri_to_file.dart';
import 'package:url_launcher/url_launcher.dart';
import 'package:window_manager/window_manager.dart';
import '../../../../api/api.dart';
import '../../../../api/model/create_task.dart';
import '../../../../api/model/downloader_config.dart';
import '../../../../api/model/install_extension.dart';
import '../../../../api/model/request.dart';
import '../../../../api/model/result.dart';
import '../../../../core/common/start_config.dart';
import '../../../../core/libgopeed_boot.dart';
import '../../../../database/database.dart';
@@ -37,7 +35,8 @@ import '../../../../util/package_info.dart';
import '../../../../util/updater.dart';
import '../../../../util/util.dart';
import '../../../routes/app_pages.dart';
import '../../../rpc/rpc.dart';
import '../../../rpc/host_rpc_service.dart';
import '../../../rpc/webview_rpc_service.dart';
import '../../redirect/views/redirect_view.dart';
import '../../../services/notification_service.dart';
@@ -71,6 +70,7 @@ class PendingUpdateTask {
class AppController extends GetxController with WindowListener, TrayListener {
static StartConfig? _defaultStartConfig;
static const _nativeChannel = MethodChannel('gopeed.com/libgopeed');
/// Command line --hidden flag passed from main.dart
final bool hiddenFromArgs;
@@ -81,6 +81,7 @@ class AppController extends GetxController with WindowListener, TrayListener {
final startConfig = StartConfig().obs;
final runningPort = 0.obs;
final downloaderConfig = DownloaderConfig().obs;
var androidDialogMode = false;
/// The task that is pending URL update via listen mode.
/// Stored here in AppController to persist across page navigations.
@@ -98,7 +99,7 @@ class AppController extends GetxController with WindowListener, TrayListener {
_initWindows().onError((error, stackTrace) =>
logger.w("initWindows error", error, stackTrace));
if (Util.isDesktop()) {
Get.put(NotificationService());
}
@@ -126,7 +127,11 @@ class AppController extends GetxController with WindowListener, TrayListener {
void onClose() {
_linkSubscription?.cancel();
trayManager.removeListener(this);
LibgopeedBoot.instance.stop();
if (!androidDialogMode) {
HostRpcService.instance.stop();
WebViewRpcService.instance.stop();
LibgopeedBoot.instance.stop();
}
}
@override
@@ -182,6 +187,7 @@ class AppController extends GetxController with WindowListener, TrayListener {
// For web, just show window
return;
}
androidDialogMode = await _isAndroidDialogMode();
// Handle deep link
_appLinks = AppLinks();
@@ -241,6 +247,18 @@ class AppController extends GetxController with WindowListener, TrayListener {
}
}
Future<bool> _isAndroidDialogMode() async {
if (!Util.isAndroid()) {
return false;
}
try {
return await _nativeChannel.invokeMethod<bool>('isDialogMode') ?? false;
} catch (e) {
logger.w("get android dialog mode fail", e);
return false;
}
}
Future<void> _initWindows() async {
if (!Util.isDesktop()) {
return;
@@ -330,54 +348,33 @@ class AppController extends GetxController with WindowListener, TrayListener {
return;
}
try {
await startRpcServer({
"/create": (ctx) async {
final meta =
ctx.request.headers["X-Gopeed-Host-Meta"]?.firstOrNull ?? "{}";
final jsonMeta = jsonDecode(meta);
final silent = jsonMeta['silent'] as bool? ?? false;
final params = await ctx.readText();
final createTaskParams = CreateTask.fromJson(_decodeParams(params));
await HostRpcService.instance.start(
onCreate: (createTaskParams, silent) async {
if (!silent || pendingUpdateTask.value != null) {
await windowManager.show();
_handleToCreate0(createTaskParams);
} else {
try {
await createTask(createTaskParams);
} catch (e) {
logger.w(
"create task from extension fail", e, StackTrace.current);
}
return;
}
},
"/forward": (ctx) async {
try {
final body = await ctx.readJSON();
final method = (body['method'] as String?)?.toUpperCase() ?? 'GET';
final path = (body['path'] as String?) ?? "/";
final data = body['data'];
final query = body['query'] as Map<String, dynamic>?;
// Forward request to gopeed REST API
final response = await forward(
path,
method: method,
data: data,
queryParameters: query,
);
// Return raw response
await ctx.writeJSON(response.data);
await createTask(createTaskParams);
} catch (e) {
if (e is DioException && e.response != null) {
// Return API error response
await ctx.writeJSON(e.response!.data);
} else {
await ctx.writeJSON(Result(code: 1, msg: e.toString()).toJson());
}
logger.w("create task from extension fail", e, StackTrace.current);
}
},
});
onForward: ({
required String path,
required String method,
data,
Map<String, dynamic>? query,
}) {
return forward(
path,
method: method,
data: data,
queryParameters: query,
);
},
);
} catch (e) {
logger.w("start rpc server fail", e, StackTrace.current);
}
@@ -461,9 +458,7 @@ class AppController extends GetxController with WindowListener, TrayListener {
} else {
path = (await toFile(uri.toString())).path;
}
Get.rootDelegate.offAndToNamed(Routes.REDIRECT,
arguments: RedirectArgs(Routes.CREATE,
arguments: CreateTask(req: Request(url: path))));
_handleToCreate0(CreateTask(req: Request(url: path)));
}
String runningAddress() {
@@ -718,8 +713,10 @@ class AppController extends GetxController with WindowListener, TrayListener {
}
_handleToCreate0(CreateTask createTaskParams) {
final targetRoute =
androidDialogMode ? Routes.QUICK_CREATE : Routes.CREATE;
Get.rootDelegate.offAndToNamed(Routes.REDIRECT,
arguments: RedirectArgs(Routes.CREATE, arguments: createTaskParams));
arguments: RedirectArgs(targetRoute, arguments: createTaskParams));
}
_handleToExtension(String params) {
@@ -8,6 +8,8 @@ import '../../../../i18n/message.dart';
import '../../../../theme/theme.dart';
import '../../../../util/locale_manager.dart';
import '../../../../util/util.dart'; // Import the required packages
import '../../../rpc/webview_rpc_overlay.dart';
import '../../../rpc/webview_rpc_service.dart';
import '../../../routes/app_pages.dart';
import '../controllers/app_controller.dart';
@@ -46,9 +48,13 @@ class AppView extends GetView<AppController> {
}
// Fix for GetX Overlay issue with Flutter 3.38.1+
// Reference: https://github.com/jonataslaw/getx/issues/3425
return Overlay(
initialEntries: [OverlayEntry(builder: (_) => child!)],
);
final entries = <OverlayEntry>[
OverlayEntry(builder: (_) => child!),
];
if (WebViewRpcService.instance.supported) {
entries.add(OverlayEntry(builder: (_) => const WebViewRpcOverlay()));
}
return Overlay(initialEntries: entries);
},
),
);
@@ -0,0 +1,299 @@
import 'package:flutter/material.dart';
import 'package:flutter/services.dart';
import 'package:get/get.dart';
import 'package:path/path.dart' as path;
import '../../../../api/api.dart';
import '../../../../api/model/create_task.dart';
import '../../../../api/model/create_task_batch.dart';
import '../../../../api/model/options.dart';
import '../../../../api/model/resolve_result.dart';
import '../../../../api/model/resolve_task.dart';
import '../../../../util/message.dart';
import '../../../views/directory_selector.dart';
import '../../app/controllers/app_controller.dart';
class CreateDialogView extends StatefulWidget {
const CreateDialogView({Key? key}) : super(key: key);
@override
State<CreateDialogView> createState() => _CreateDialogViewState();
}
class _CreateDialogViewState extends State<CreateDialogView> {
final _pathController = TextEditingController();
final _nameController = TextEditingController();
CreateTask? _createTask;
ResolveResult? _resolveResult;
Object? _error;
bool _resolving = true;
bool _creating = false;
@override
void initState() {
super.initState();
final appController = Get.find<AppController>();
_pathController.text = renderPathPlaceholders(
appController.downloaderConfig.value.downloadDir,
);
WidgetsBinding.instance.addPostFrameCallback((_) => _loadArguments());
}
@override
void dispose() {
_pathController.dispose();
_nameController.dispose();
super.dispose();
}
Future<void> _loadArguments() async {
final args = Get.rootDelegate.arguments();
if (args is! CreateTask || args.req?.url.isNotEmpty != true) {
if (mounted) {
setState(() {
_resolving = false;
});
}
return;
}
_createTask = args;
if (args.opts?.path.isNotEmpty == true) {
_pathController.text = args.opts!.path;
}
if (args.opts?.name.isNotEmpty == true) {
_nameController.text = args.opts!.name;
} else {
_nameController.text = _fallbackName(args.req!.url);
}
await _resolve(args);
}
Future<void> _resolve(CreateTask createTask) async {
setState(() {
_resolving = true;
_error = null;
});
try {
final opt = Options(
name: createTask.opts?.name ?? '',
path: _pathController.text,
selectFiles: const [],
extra: _optsExtra(),
);
final result = await resolve(
ResolveTask(req: createTask.req!, opts: opt),
);
if (!mounted) return;
setState(() {
_resolveResult = result;
if (_nameController.text.trim().isEmpty ||
_nameController.text == _fallbackName(createTask.req!.url)) {
_nameController.text = _resolvedName(result, createTask.req!.url);
}
});
} catch (e) {
if (!mounted) return;
setState(() {
_error = e;
});
} finally {
if (mounted) {
setState(() {
_resolving = false;
});
}
}
}
Future<void> _confirm() async {
final createTaskParams = _createTask;
if (createTaskParams?.req == null) {
return;
}
setState(() {
_creating = true;
});
try {
final rr = _resolveResult;
final optExtra = _optsExtra();
if (rr == null) {
await createTask(
CreateTask(
req: createTaskParams!.req,
opts: Options(
name: _nameController.text.trim(),
path: _pathController.text,
selectFiles: const [],
extra: optExtra,
),
),
);
} else if (rr.id.isEmpty) {
final reqs = rr.res.files.map((file) {
return CreateTaskBatchItem(
req: file.req ?? createTaskParams!.req,
opts: Options(
name: file.name,
path: path.join(_pathController.text, rr.res.name, file.path),
selectFiles: const [],
extra: optExtra,
),
);
}).toList();
await createTaskBatch(CreateTaskBatch(reqs: reqs));
} else {
await createTask(
CreateTask(
rid: rr.id,
opts: Options(
name: _nameController.text.trim(),
path: _pathController.text,
selectFiles: rr.res.files.asMap().keys.toList(),
extra: optExtra,
),
),
);
}
await SystemNavigator.pop();
} catch (e) {
showErrorMessage(e);
if (mounted) {
setState(() {
_creating = false;
});
}
}
}
Object _optsExtra() {
final httpConfig =
Get.find<AppController>().downloaderConfig.value.protocolConfig.http;
return OptsExtraHttp(connections: httpConfig.connections);
}
String _resolvedName(ResolveResult result, String url) {
if (result.res.name.trim().isNotEmpty) {
return result.res.name;
}
if (result.res.files.length == 1) {
return result.res.files.first.name;
}
return _fallbackName(url);
}
String _fallbackName(String url) {
final uri = Uri.tryParse(url);
final uriName = uri == null ? '' : path.basename(uri.path);
if (uriName.isNotEmpty && uriName != '.') {
return uriName;
}
return url;
}
@override
Widget build(BuildContext context) {
final theme = Theme.of(context);
return Scaffold(
backgroundColor: Colors.transparent,
body: SafeArea(
child: Center(
child: Material(
color: theme.colorScheme.surface,
elevation: 16,
borderRadius: BorderRadius.circular(8),
clipBehavior: Clip.antiAlias,
child: ConstrainedBox(
constraints: const BoxConstraints(maxWidth: 420),
child: Padding(
padding: const EdgeInsets.fromLTRB(20, 16, 20, 16),
child: Column(
mainAxisSize: MainAxisSize.min,
crossAxisAlignment: CrossAxisAlignment.stretch,
children: [
Row(
children: [
Expanded(
child: Text(
'create'.tr,
style: theme.textTheme.titleMedium,
overflow: TextOverflow.ellipsis,
),
),
IconButton(
icon: const Icon(Icons.close),
onPressed: _creating ? null : SystemNavigator.pop,
tooltip: 'cancel'.tr,
),
],
),
const SizedBox(height: 8),
if (_resolving)
const LinearProgressIndicator(minHeight: 2)
else
const SizedBox(height: 2),
const SizedBox(height: 14),
TextField(
controller: _nameController,
enabled: !_creating,
decoration: InputDecoration(
labelText: 'rename'.tr,
prefixIcon: const Icon(Icons.insert_drive_file),
),
maxLines: 1,
),
const SizedBox(height: 12),
DirectorySelector(
controller: _pathController,
showAndoirdToggle: true,
allowEdit: true,
),
if (_error != null) ...[
const SizedBox(height: 12),
Text(
_error.toString(),
style: theme.textTheme.bodySmall?.copyWith(
color: theme.colorScheme.error,
),
maxLines: 3,
overflow: TextOverflow.ellipsis,
),
],
const SizedBox(height: 18),
Row(
mainAxisAlignment: MainAxisAlignment.end,
children: [
TextButton(
onPressed: _creating ? null : SystemNavigator.pop,
child: Text('cancel'.tr),
),
const SizedBox(width: 8),
ElevatedButton(
onPressed:
(_resolving || _creating) ? null : _confirm,
child: _creating
? const SizedBox.square(
dimension: 18,
child: CircularProgressIndicator(
strokeWidth: 2,
),
)
: Text('download'.tr),
),
],
),
],
),
),
),
),
),
),
);
}
}
@@ -17,6 +17,14 @@ import 'task_downloading_view.dart';
class TaskView extends GetView<TaskController> {
const TaskView({Key? key}) : super(key: key);
String? _displayTaskUrl(Task? task) {
final rawUrl = task?.meta.req.rawUrl;
if (rawUrl != null && rawUrl.isNotEmpty) {
return rawUrl;
}
return task?.meta.req.url;
}
@override
Widget build(BuildContext context) {
final selectTask = controller.selectTask;
@@ -85,9 +93,8 @@ class TaskView extends GetView<TaskController> {
subtitle: buildTooltipSubtitle(selectTask.value?.name)),
ListTile(
title: Text('taskUrl'.tr),
subtitle:
buildTooltipSubtitle(selectTask.value?.meta.req.url),
trailing: CopyButton(selectTask.value?.meta.req.url),
subtitle: buildTooltipSubtitle(_displayTaskUrl(selectTask.value)),
trailing: CopyButton(_displayTaskUrl(selectTask.value)),
),
ListTile(
title: Text('downloadPath'.tr),
+7
View File
@@ -1,6 +1,7 @@
import 'package:get/get.dart';
import '../modules/create/bindings/create_binding.dart';
import '../modules/create/views/create_dialog_view.dart';
import '../modules/create/views/create_view.dart';
import '../modules/extension/bindings/extension_binding.dart';
import '../modules/extension/views/extension_view.dart';
@@ -78,6 +79,12 @@ class AppPages {
page: () => CreateView(),
binding: CreateBinding(),
),
GetPage(
name: _Paths.QUICK_CREATE,
transition: Transition.noTransition,
page: () => const CreateDialogView(),
binding: CreateBinding(),
),
GetPage(
name: _Paths.REDIRECT,
page: () => const RedirectView(),
@@ -7,6 +7,7 @@ abstract class Routes {
static const ROOT = _Paths.ROOT;
static const HOME = _Paths.HOME;
static const CREATE = _Paths.CREATE;
static const QUICK_CREATE = _Paths.QUICK_CREATE;
static const LOGIN = _Paths.LOGIN;
static const TASK = _Paths.HOME + _Paths.TASK;
static const TASK_FILES = TASK + _Paths.TASK_FILES;
@@ -21,6 +22,7 @@ abstract class _Paths {
static const ROOT = '/';
static const HOME = '/home';
static const CREATE = '/create';
static const QUICK_CREATE = '/quick-create';
static const LOGIN = '/login';
static const TASK = '/task';
static const TASK_FILES = '/files';
@@ -0,0 +1,90 @@
import 'dart:convert';
import 'package:dio/dio.dart';
import '../../api/model/create_task.dart';
import '../../api/model/result.dart';
import 'server.dart';
typedef HostCreateHandler = Future<void> Function(
CreateTask createTask,
bool silent,
);
typedef HostForwardHandler = Future<dynamic> Function({
required String path,
required String method,
dynamic data,
Map<String, dynamic>? query,
});
class HostRpcService {
HostRpcService._();
static final HostRpcService instance = HostRpcService._();
RpcServerHandle? _server;
Future<void> start({
required HostCreateHandler onCreate,
required HostForwardHandler onForward,
}) async {
if (_server != null) {
return;
}
_server = await startRpcServer(
routes: {
'/create': (ctx) async {
final meta =
ctx.request.headers['X-Gopeed-Host-Meta']?.firstOrNull ?? '{}';
final jsonMeta = jsonDecode(meta);
final silent = jsonMeta['silent'] as bool? ?? false;
final params = await ctx.readText();
final createTask = CreateTask.fromJson(_decodeParams(params));
await onCreate(createTask, silent);
},
'/forward': (ctx) async {
try {
final body = await ctx.readJSON();
final method = (body['method'] as String?)?.toUpperCase() ?? 'GET';
final path = (body['path'] as String?) ?? '/';
final data = body['data'];
final query = body['query'] as Map<String, dynamic>?;
final response = await onForward(
path: path,
method: method,
data: data,
query: query,
);
if (response is Map<String, dynamic>) {
await ctx.writeJSON(response);
} else {
await ctx.writeJSON((response as dynamic).data);
}
} catch (e) {
if (e is DioException && e.response != null) {
await ctx.writeJSON(e.response!.data);
} else {
await ctx.writeJSON(Result(code: 1, msg: e.toString()).toJson());
}
}
},
},
);
}
Future<void> stop() async {
final server = _server;
_server = null;
if (server != null) {
await server.close();
}
}
Map<String, dynamic> _decodeParams(String params) {
final safeParams = params.replaceAll('"', '').replaceAll(' ', '+');
final paramsJson =
String.fromCharCodes(base64Decode(base64.normalize(safeParams)));
return jsonDecode(paramsJson);
}
}
+198
View File
@@ -0,0 +1,198 @@
import 'dart:convert';
import 'dart:io';
import 'package:dart_ipc/dart_ipc.dart';
import '../../util/util.dart';
class RpcContext {
final HttpRequest request;
final HttpResponse response;
String? _bodyCache;
RpcContext(this.request, this.response);
Future<Map<String, dynamic>> readJSON() async {
if (_bodyCache == null) {
final content = <int>[];
await for (final data in request) {
content.addAll(data);
}
_bodyCache = String.fromCharCodes(content);
}
if (_bodyCache!.isEmpty) {
return {};
}
return jsonDecode(_bodyCache!) as Map<String, dynamic>;
}
Future<String> readText() async {
if (_bodyCache == null) {
final content = <int>[];
await for (final data in request) {
content.addAll(data);
}
_bodyCache = String.fromCharCodes(content);
}
return _bodyCache ?? '';
}
Future<void> writeJSON(Map<String, dynamic> data) async {
response.headers.contentType = ContentType.json;
response.write(jsonEncode(data));
}
Future<void> writeError(String message, [int statusCode = 500]) async {
response.statusCode = statusCode;
await writeJSON({'error': message});
}
}
typedef RouteHandler = Future<void> Function(RpcContext ctx);
class RouteRegistry {
final Map<String, RouteHandler> _routes = {};
void register(String path, RouteHandler handler) {
_routes[path] = handler;
}
RouteHandler? getHandler(String path) {
return _routes[path];
}
}
class RpcBinding {
final String network;
final String address;
const RpcBinding({
required this.network,
required this.address,
});
}
class RpcServerHandle {
final RpcBinding binding;
final HttpServer server;
final Future<void> Function()? _cleanup;
RpcServerHandle({
required this.binding,
required this.server,
Future<void> Function()? cleanup,
}) : _cleanup = cleanup;
Future<void> close({bool force = true}) async {
await server.close(force: force);
if (_cleanup != null) {
await _cleanup!();
}
}
}
Future<RpcBinding> defaultHostRpcBinding() async {
if (Util.isWindows()) {
return const RpcBinding(
network: 'pipe',
address: r'\\.\pipe\gopeed_host',
);
}
return RpcBinding(
network: 'unix',
address: await Util.homePathJoin('gopeed_host.sock'),
);
}
Future<RpcBinding> defaultWebViewRpcBinding() async {
if (Util.supportUnixSocket()) {
return RpcBinding(
network: 'unix',
address: await Util.homePathJoin('gopeed_webview.sock'),
);
}
return const RpcBinding(
network: 'tcp',
address: '127.0.0.1:0',
);
}
Future<RpcServerHandle> startRpcServer({
RpcBinding? binding,
Map<String, RouteHandler>? routes,
}) async {
final rpcBinding = binding ?? await defaultHostRpcBinding();
Future<void> Function()? cleanup;
late final HttpServer httpServer;
switch (rpcBinding.network) {
case 'pipe':
final serverSocket = await bind(rpcBinding.address);
httpServer = HttpServer.listenOn(serverSocket);
break;
case 'unix':
final socketFile = File(rpcBinding.address);
if (await socketFile.exists()) {
try {
await socketFile.delete();
} catch (_) {}
}
final serverSocket = await bind(rpcBinding.address);
httpServer = HttpServer.listenOn(serverSocket);
cleanup = () async {
if (await socketFile.exists()) {
try {
await socketFile.delete();
} catch (_) {}
}
};
break;
case 'tcp':
final separator = rpcBinding.address.lastIndexOf(':');
if (separator <= 0 || separator == rpcBinding.address.length - 1) {
throw ArgumentError('invalid tcp rpc address: ${rpcBinding.address}');
}
final host = rpcBinding.address.substring(0, separator);
final port = int.parse(rpcBinding.address.substring(separator + 1));
httpServer = await HttpServer.bind(host, port);
break;
default:
throw ArgumentError('unsupported rpc network: ${rpcBinding.network}');
}
final registry = RouteRegistry();
routes?.forEach(registry.register);
httpServer.forEach((request) async {
final ctx = RpcContext(request, request.response);
try {
final handler = registry.getHandler(request.uri.path);
if (handler != null) {
await handler(ctx);
} else {
await ctx.writeError('Route not found: ${request.uri.path}', 404);
}
} catch (e) {
await ctx.writeError('Internal server error: $e', 500);
} finally {
await request.response.close();
}
});
var resolvedBinding = rpcBinding;
if (rpcBinding.network == 'tcp') {
resolvedBinding = RpcBinding(
network: rpcBinding.network,
address: '${httpServer.address.address}:${httpServer.port}',
);
}
return RpcServerHandle(
binding: resolvedBinding,
server: httpServer,
cleanup: cleanup,
);
}
@@ -0,0 +1,97 @@
import 'package:flutter/material.dart';
import 'webview_rpc_service.dart';
class WebViewRpcOverlay extends StatefulWidget {
const WebViewRpcOverlay({super.key});
@override
State<WebViewRpcOverlay> createState() => _WebViewRpcOverlayState();
}
class _WebViewRpcOverlayState extends State<WebViewRpcOverlay> {
@override
void initState() {
super.initState();
WidgetsBinding.instance.addPostFrameCallback((_) {
WebViewRpcService.instance.markOverlayReady();
});
}
@override
Widget build(BuildContext context) {
return ValueListenableBuilder<List<WebViewRpcPageSession>>(
valueListenable: WebViewRpcService.instance.pages,
builder: (context, pages, _) {
if (pages.isEmpty) {
return const SizedBox.shrink();
}
return IgnorePointer(
ignoring: false,
child: Stack(
children: pages
.map((page) => _WebViewRpcPageView(page: page))
.toList(growable: false),
),
);
},
);
}
}
class _WebViewRpcPageView extends StatefulWidget {
const _WebViewRpcPageView({required this.page});
final WebViewRpcPageSession page;
@override
State<_WebViewRpcPageView> createState() => _WebViewRpcPageViewState();
}
class _WebViewRpcPageViewState extends State<_WebViewRpcPageView> {
@override
Widget build(BuildContext context) {
final page = widget.page;
final content = page.buildWebView();
return Positioned.fill(
child: Material(
color: Colors.black54,
child: Center(
child: Container(
width: page.width > 0 ? page.width.toDouble() : 960,
height: page.height > 0 ? page.height.toDouble() : 720,
clipBehavior: Clip.antiAlias,
decoration: BoxDecoration(
color: Theme.of(context).colorScheme.surface,
borderRadius: BorderRadius.circular(12),
boxShadow: const [
BoxShadow(
blurRadius: 24,
color: Colors.black26,
),
],
),
child: Column(
children: [
Container(
height: 44,
padding: const EdgeInsets.symmetric(horizontal: 16),
alignment: Alignment.centerLeft,
color: Theme.of(context).colorScheme.surfaceContainerHighest,
child: Text(
page.title.isNotEmpty ? page.title : 'WebView',
maxLines: 1,
overflow: TextOverflow.ellipsis,
style: Theme.of(context).textTheme.titleSmall,
),
),
Expanded(child: content),
],
),
),
),
),
);
}
}
@@ -0,0 +1,776 @@
import 'dart:async';
import 'dart:collection';
import 'dart:convert';
import 'package:flutter/material.dart';
import 'package:flutter_inappwebview/flutter_inappwebview.dart';
import '../../core/common/start_config.dart';
import '../../util/log_util.dart';
import '../../util/util.dart';
import 'server.dart';
String buildWebViewExecuteScript({
required String channelName,
required String requestId,
required String expression,
required List<dynamic> args,
}) {
return '''
(() => {
var __bridge = window.flutter_inappwebview;
var __id = ${jsonEncode(requestId)};
var __expr = ${jsonEncode(expression)};
var __args = ${jsonEncode(args)};
var __serializeError = function(error) {
return [
error && error.name ? error.name : 'Error',
error && error.message ? error.message : String(error),
error && error.stack ? error.stack : ''
].filter(Boolean).join(': ').replace(': @', '\\n@');
};
var __post = function(message) {
if (__bridge && typeof __bridge.callHandler === 'function') {
__bridge.callHandler(${jsonEncode(channelName)}, JSON.stringify(message));
}
};
try {
Promise.resolve()
.then(function() {
var __target = (0, eval)(__expr);
if (typeof __target === 'function') {
return __target.apply(null, __args || []);
}
return __target;
})
.then(function(__value) {
__post({ id: __id, ok: true, value: __value == null ? null : __value });
})
.catch(function(error) {
__post({
id: __id,
ok: false,
error: __serializeError(error)
});
});
} catch (error) {
__post({
id: __id,
ok: false,
error: __serializeError(error)
});
}
})();
''';
}
class WebViewRpcService {
WebViewRpcService._();
static final WebViewRpcService instance = WebViewRpcService._();
final ValueNotifier<List<WebViewRpcPageSession>> pages =
ValueNotifier<List<WebViewRpcPageSession>>([]);
final Map<String, WebViewRpcPageSession> _pagesById = {};
final Completer<void> _overlayReady = Completer<void>();
final CookieManager _cookieManager = CookieManager.instance();
RpcServerHandle? _server;
int _pageSeq = 0;
bool get supported => Util.isMobile() || Util.isMacos();
void markOverlayReady() {
if (!_overlayReady.isCompleted) {
_overlayReady.complete();
}
}
Future<WebViewRpcConfig?> start() async {
if (!supported) {
return null;
}
if (_server != null) {
return _toConfig(_server!.binding);
}
final binding = await defaultWebViewRpcBinding();
_server = await startRpcServer(
binding: binding,
routes: {
'/webview': _handleWebViewRequest,
},
);
return _toConfig(_server!.binding);
}
Future<void> stop() async {
for (final page in _pagesById.values.toList()) {
await page.dispose();
}
_pagesById.clear();
pages.value = const [];
final server = _server;
_server = null;
if (server != null) {
await server.close();
}
}
Future<void> _handleWebViewRequest(RpcContext ctx) async {
await _overlayReady.future;
final body = await ctx.readJSON();
final method = body['method'] as String? ?? '';
final params = (body['params'] as Map?)?.cast<String, dynamic>() ??
const <String, dynamic>{};
try {
final result = await _dispatch(method, params);
await ctx.writeJSON({
'result': result,
'error': null,
});
} catch (e, stackTrace) {
logger.w('webview rpc request failed: $method', e, stackTrace);
await ctx.writeJSON({
'result': null,
'error': _toRpcError(method, e),
});
}
}
Future<dynamic> _dispatch(String method, Map<String, dynamic> params) async {
switch (method) {
case 'webview.isAvailable':
return {'available': true};
case 'page.open':
return _openPage(params);
case 'page.addInitScript':
return _page(params).addInitScript(_string(params, 'script'));
case 'page.goto':
await _page(params).goto(
_string(params, 'url'),
timeoutMs: _int(params, 'timeoutMs'),
waitUntil: _stringOrNull(params, 'waitUntil'),
);
return {};
case 'page.execute':
return _page(params).execute(
_string(params, 'expression'),
(_list(params, 'args') ?? const []),
);
case 'page.getCookies':
return _page(params).getCookies();
case 'page.setCookie':
await _page(params).setCookie(_cookie(params));
return {};
case 'page.deleteCookie':
await _page(params).deleteCookie(_cookie(params));
return {};
case 'page.clearCookies':
await _page(params).clearCookies();
return {};
case 'page.close':
await _closePage(_pageId(params));
return {};
default:
throw WebViewRpcException(
code: 'UNKNOWN_METHOD',
message: 'unknown method: $method',
);
}
}
Future<Map<String, dynamic>> _openPage(Map<String, dynamic> params) async {
final pageId = 'page-${++_pageSeq}';
final session = WebViewRpcPageSession(
pageId: pageId,
cookieManager: _cookieManager,
headless: params['headless'] as bool? ?? false,
debug: params['debug'] as bool? ?? false,
title: params['title'] as String? ?? '',
width: (params['width'] as num?)?.toInt() ?? 1280,
height: (params['height'] as num?)?.toInt() ?? 800,
userAgent: params['userAgent'] as String? ?? '',
);
_pagesById[pageId] = session;
_syncPages();
try {
await session.init();
} catch (e) {
_pagesById.remove(pageId);
_syncPages();
rethrow;
}
return {'pageId': pageId};
}
WebViewRpcPageSession _page(Map<String, dynamic> params) {
final page = _pagesById[_pageId(params)];
if (page == null) {
throw WebViewRpcException(
code: 'PAGE_NOT_FOUND',
message: 'page not found',
);
}
return page;
}
String _pageId(Map<String, dynamic> params) => _string(params, 'pageId');
String _string(Map<String, dynamic> params, String key) {
final value = params[key];
if (value is String && value.isNotEmpty) {
return value;
}
throw WebViewRpcException(
code: 'INVALID_REQUEST',
message: 'missing or invalid "$key"',
);
}
String? _stringOrNull(Map<String, dynamic> params, String key) {
final value = params[key];
if (value == null) {
return null;
}
if (value is String && value.isNotEmpty) {
return value;
}
throw WebViewRpcException(
code: 'INVALID_REQUEST',
message: 'invalid "$key"',
);
}
int? _int(Map<String, dynamic> params, String key) {
final value = params[key];
if (value == null) {
return null;
}
if (value is num) {
return value.toInt();
}
throw WebViewRpcException(
code: 'INVALID_REQUEST',
message: 'invalid "$key"',
);
}
List<dynamic>? _list(Map<String, dynamic> params, String key) {
final value = params[key];
if (value == null) {
return null;
}
if (value is List<dynamic>) {
return value;
}
throw WebViewRpcException(
code: 'INVALID_REQUEST',
message: 'invalid "$key"',
);
}
Map<String, dynamic> _cookie(Map<String, dynamic> params) {
final value = params['cookie'];
if (value is Map<String, dynamic>) {
return value;
}
if (value is Map) {
return value.cast<String, dynamic>();
}
throw WebViewRpcException(
code: 'INVALID_REQUEST',
message: 'missing or invalid "cookie"',
);
}
Future<void> _closePage(String pageId) async {
final page = _pagesById.remove(pageId);
if (page == null) {
throw WebViewRpcException(
code: 'PAGE_NOT_FOUND',
message: 'page not found',
);
}
_syncPages();
await page.dispose();
}
void _syncPages() {
pages.value = _pagesById.values
.where((page) => !page.headless)
.toList(growable: false);
}
WebViewRpcConfig _toConfig(RpcBinding binding) {
final config = WebViewRpcConfig()
..network = binding.network
..address = binding.address;
return config;
}
Map<String, dynamic> _toRpcError(String method, Object error) {
if (error is WebViewRpcException) {
return {
'code': error.code,
'message': error.message,
};
}
final code = switch (method) {
'page.goto' => 'NAVIGATION_FAILED',
'page.execute' => 'EVALUATION_FAILED',
_ => 'INTERNAL_ERROR',
};
return {
'code': code,
'message': error.toString(),
};
}
}
class WebViewRpcPageSession {
WebViewRpcPageSession({
required this.pageId,
required this.cookieManager,
required this.headless,
required this.debug,
required this.title,
required this.width,
required this.height,
required this.userAgent,
});
final String pageId;
final CookieManager cookieManager;
final bool headless;
final bool debug;
final String title;
final int width;
final int height;
final String userAgent;
late final String callbackChannelName;
final List<UserScript> _initScripts = [];
final Map<String, Completer<dynamic>> _pendingExecutions =
<String, Completer<dynamic>>{};
final Completer<void> _ready = Completer<void>();
HeadlessInAppWebView? _headlessWebView;
InAppWebViewController? _controller;
Completer<void>? _navigation;
String _currentUrl = '';
int _executeSeq = 0;
bool _disposed = false;
Future<void> init() async {
callbackChannelName =
'__gopeedWebViewCallback_${pageId.replaceAll('-', '_')}';
if (headless) {
_headlessWebView = HeadlessInAppWebView(
initialSize: Size(
width > 0 ? width.toDouble() : 1,
height > 0 ? height.toDouble() : 1,
),
initialSettings: _settings,
initialUserScripts: UnmodifiableListView(_initScripts),
onWebViewCreated: _attachController,
onLoadStart: _handleLoadStart,
onLoadStop: _handleLoadStop,
onReceivedError: _handleReceivedError,
onReceivedHttpError: _handleReceivedHttpError,
);
await _headlessWebView!.run();
}
await _ready.future;
}
Future<void> addInitScript(String script) async {
final userScript = UserScript(
source: script,
injectionTime: UserScriptInjectionTime.AT_DOCUMENT_START,
);
_initScripts.add(userScript);
final controller = await _controllerOrThrow();
await controller.addUserScript(userScript: userScript);
if (_currentUrl.isNotEmpty && !_disposed) {
try {
await controller.evaluateJavascript(source: script);
} catch (_) {
// Best-effort only for the current page. The user script persists
// for future navigations through the native user script registry.
}
}
}
Future<void> goto(String url, {int? timeoutMs, String? waitUntil}) async {
final controller = await _controllerOrThrow();
final waitStrategy = _normalizeWaitUntil(waitUntil);
final navigation = _ensureNavigationCompleter();
try {
await controller.loadUrl(urlRequest: URLRequest(url: WebUri(url)));
} catch (e) {
if (identical(_navigation, navigation)) {
_navigation = null;
}
throw WebViewRpcException(
code: 'NAVIGATION_FAILED',
message: e.toString(),
);
}
final future = switch (waitStrategy) {
'domcontentloaded' => Future.any<void>([
_waitForDomReady(),
navigation.future,
]),
_ => navigation.future,
};
if (timeoutMs == null || timeoutMs <= 0) {
await future;
return;
}
await future.timeout(
Duration(milliseconds: timeoutMs),
onTimeout: () {
_navigation = null;
throw WebViewRpcException(
code: 'TIMEOUT',
message: 'navigation timeout after ${timeoutMs}ms',
);
},
);
}
Future<dynamic> execute(String expression, List<dynamic> args) async {
final controller = await _controllerOrThrow();
final requestId = 'exec-$pageId-${++_executeSeq}';
final completer = Completer<dynamic>();
_pendingExecutions[requestId] = completer;
try {
await controller.evaluateJavascript(
source: buildWebViewExecuteScript(
channelName: callbackChannelName,
requestId: requestId,
expression: expression,
args: args,
),
);
} catch (e) {
_pendingExecutions.remove(requestId);
throw WebViewRpcException(
code: 'EVALUATION_FAILED',
message: e.toString(),
);
}
try {
final result = await completer.future.timeout(
const Duration(seconds: 30),
onTimeout: () {
_pendingExecutions.remove(requestId);
throw WebViewRpcException(
code: 'TIMEOUT',
message: 'javascript execution timeout',
);
},
);
return result;
} finally {
_pendingExecutions.remove(requestId);
}
}
String _normalizeWaitUntil(String? waitUntil) {
if (waitUntil == null || waitUntil.isEmpty) {
return 'load';
}
switch (waitUntil.toLowerCase()) {
case 'load':
return 'load';
case 'domcontentloaded':
return 'domcontentloaded';
default:
throw WebViewRpcException(
code: 'INVALID_REQUEST',
message: 'invalid "waitUntil"',
);
}
}
Future<void> _waitForDomReady() async {
final controller = await _controllerOrThrow();
while (true) {
final readyState = await controller.evaluateJavascript(
source: 'document.readyState',
);
final state = readyState?.toString() ?? '';
if (state == 'interactive' || state == 'complete') {
return;
}
await Future<void>.delayed(const Duration(milliseconds: 50));
}
}
Future<List<Map<String, dynamic>>> getCookies() async {
if (_currentUrl.isEmpty) {
return const [];
}
final cookies = await cookieManager.getCookies(url: WebUri(_currentUrl));
return cookies.map((cookie) {
return <String, dynamic>{
'name': cookie.name,
'value': cookie.value,
'domain': cookie.domain ?? '',
'path': cookie.path ?? '/',
'secure': cookie.isSecure,
'httpOnly': cookie.isHttpOnly,
};
}).toList(growable: false);
}
Future<void> setCookie(Map<String, dynamic> cookie) async {
final targetUrl = _cookieUrl(cookie);
await cookieManager.setCookie(
url: targetUrl,
name: cookie['name'] as String? ?? '',
value: cookie['value'] as String? ?? '',
domain: cookie['domain'] as String?,
path: cookie['path'] as String? ?? '/',
expiresDate: _cookieExpires(cookie),
isSecure: cookie['secure'] as bool? ?? false,
isHttpOnly: cookie['httpOnly'] as bool? ?? false,
);
}
Future<void> deleteCookie(Map<String, dynamic> cookie) async {
final name = cookie['name'] as String? ?? '';
final path = cookie['path'] as String? ?? '/';
final domain = cookie['domain'] as String? ?? '';
if (name.isEmpty) {
throw WebViewRpcException(
code: 'INVALID_REQUEST',
message: 'cookie.name is required',
);
}
await cookieManager.deleteCookie(
url: _cookieUrl(cookie),
name: name,
domain: domain.isNotEmpty ? domain : null,
path: path,
);
}
Future<void> clearCookies() async {
await cookieManager.deleteAllCookies();
}
Future<void> dispose() async {
_disposed = true;
if (!_ready.isCompleted) {
_ready.completeError(
WebViewRpcException(
code: 'PAGE_NOT_FOUND',
message: 'webview page closed',
),
);
}
if (_navigation != null) {
_completeNavigationError(
WebViewRpcException(
code: 'NAVIGATION_FAILED',
message: 'webview page closed',
),
);
}
for (final completer in _pendingExecutions.values) {
if (!completer.isCompleted) {
completer.completeError(
WebViewRpcException(
code: 'EVALUATION_FAILED',
message: 'webview page closed',
),
);
}
}
_pendingExecutions.clear();
await Future.sync(() => _headlessWebView?.dispose());
_headlessWebView = null;
_controller = null;
}
Completer<void> _ensureNavigationCompleter() {
return _navigation ??= Completer<void>();
}
Widget buildWebView() {
return InAppWebView(
key: ValueKey<String>('webview-$pageId'),
initialSettings: _settings,
initialUserScripts: UnmodifiableListView(_initScripts),
onWebViewCreated: _attachController,
onLoadStart: _handleLoadStart,
onLoadStop: _handleLoadStop,
onReceivedError: _handleReceivedError,
onReceivedHttpError: _handleReceivedHttpError,
);
}
InAppWebViewSettings get _settings => InAppWebViewSettings(
javaScriptEnabled: true,
transparentBackground: true,
isInspectable: debug,
userAgent: userAgent.isNotEmpty ? userAgent : null,
);
Future<InAppWebViewController> _controllerOrThrow() async {
await _ready.future;
final controller = _controller;
if (controller != null && !_disposed) {
return controller;
}
throw WebViewRpcException(
code: 'PAGE_NOT_FOUND',
message: 'page not found',
);
}
void _attachController(InAppWebViewController controller) {
_controller = controller;
controller.addJavaScriptHandler(
handlerName: callbackChannelName,
callback: (args) {
if (args.isNotEmpty) {
_handleExecuteMessage(args.first?.toString() ?? '');
}
return null;
},
);
if (!_ready.isCompleted) {
_ready.complete();
}
}
void _handleLoadStart(InAppWebViewController controller, WebUri? url) {
_currentUrl = url?.toString() ?? _currentUrl;
_ensureNavigationCompleter();
}
Future<void> _handleLoadStop(
InAppWebViewController controller,
WebUri? url,
) async {
_currentUrl = url?.toString() ?? _currentUrl;
_completeNavigation();
}
void _handleReceivedError(
InAppWebViewController controller,
WebResourceRequest request,
WebResourceError error,
) {
_completeNavigationError(
WebViewRpcException(
code: 'NAVIGATION_FAILED',
message: error.description,
),
);
}
void _handleReceivedHttpError(
InAppWebViewController controller,
WebResourceRequest request,
WebResourceResponse errorResponse,
) {
_completeNavigationError(
WebViewRpcException(
code: 'NAVIGATION_FAILED',
message:
'HTTP ${errorResponse.statusCode}: ${errorResponse.reasonPhrase ?? 'navigation failed'}',
),
);
}
void _completeNavigation() {
if (_navigation != null && !_navigation!.isCompleted) {
_navigation!.complete();
}
_navigation = null;
}
void _completeNavigationError(Object error) {
if (_navigation != null && !_navigation!.isCompleted) {
_navigation!.completeError(error);
}
_navigation = null;
}
WebUri _cookieUrl(Map<String, dynamic> cookie) {
final domain = (cookie['domain'] as String? ?? '').trim();
final path = cookie['path'] as String? ?? '/';
if (domain.isNotEmpty) {
final host = domain.startsWith('.') ? domain.substring(1) : domain;
final scheme = (cookie['secure'] as bool? ?? false) ? 'https' : 'http';
return WebUri('$scheme://$host$path');
}
if (_currentUrl.isNotEmpty) {
return WebUri(_currentUrl);
}
throw WebViewRpcException(
code: 'INVALID_REQUEST',
message: 'cookie target url is unavailable',
);
}
int? _cookieExpires(Map<String, dynamic> cookie) {
final raw = cookie['expires'];
if (raw is int) {
return raw;
}
if (raw is String && raw.isNotEmpty) {
return DateTime.tryParse(raw)?.millisecondsSinceEpoch;
}
return null;
}
void _handleExecuteMessage(String payload) {
if (payload.isEmpty) {
return;
}
final decoded = jsonDecode(payload);
if (decoded is! Map) {
return;
}
final message = decoded.cast<String, dynamic>();
final id = message['id']?.toString();
if (id == null || id.isEmpty) {
return;
}
final completer = _pendingExecutions[id];
if (completer == null || completer.isCompleted) {
return;
}
if (message['ok'] == true) {
completer.complete(message['value']);
return;
}
completer.completeError(
WebViewRpcException(
code: 'EVALUATION_FAILED',
message: message['error']?.toString() ?? 'javascript execution failed',
),
);
}
}
class WebViewRpcException implements Exception {
final String code;
final String message;
WebViewRpcException({
required this.code,
required this.message,
});
@override
String toString() => message;
}
@@ -119,7 +119,10 @@ class BuildTaskListView extends GetView {
}
Future<void> showUpdateUrlDialog(BuildContext context, Task task) async {
final urlController = TextEditingController(text: task.meta.req.url);
final displayUrl = (task.meta.req.rawUrl?.isNotEmpty ?? false)
? task.meta.req.rawUrl!
: task.meta.req.url;
final urlController = TextEditingController(text: displayUrl);
final headerControllers =
<MapEntry<TextEditingController, TextEditingController>>[];
Loaded 100 of 113 files, more files were not shown because too many files have changed in this diff. Show more