mirror of
https://github.com/GopeedLab/gopeed.git
synced 2026-11-10 10:30:28 +00:00
Compare commits
23
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
fe25ccab1d | ||
|
|
8301b1f9ce | ||
|
|
49aab4ea7c | ||
|
|
5d8267fa56 | ||
|
|
1ab452ced8 | ||
|
|
743f356567 | ||
|
|
b1dd46cc7c | ||
|
|
38750d8505 | ||
|
|
7c1447f659 | ||
|
|
21702d5d46 | ||
|
|
e99ef6c7f4 | ||
|
|
c667f097dc | ||
|
|
46c34f2e92 | ||
|
|
e47a20a1bd | ||
|
|
cee5f3336e | ||
|
|
a760290213 | ||
|
|
a29f4bb42a | ||
|
|
62e8ab3cef | ||
|
|
8fa9a1e902 | ||
|
|
46675f26af | ||
|
|
0d52b840a2 | ||
|
|
091e7b8936 | ||
|
|
ff8f61423e |
No files matched your search
@@ -0,0 +1,2 @@
|
||||
# Keep upstream plugin sources recognizable and collapsed in GitHub reviews.
|
||||
ui/flutter/third_party/** linguist-vendored whitespace=-blank-at-eol,-blank-at-eof
|
||||
@@ -77,6 +77,8 @@ jobs:
|
||||
include:
|
||||
- name: Windows
|
||||
os: windows-latest
|
||||
- name: Linux
|
||||
os: ubuntu-latest
|
||||
- name: macOS
|
||||
os: macos-latest
|
||||
runs-on: ${{ matrix.os }}
|
||||
@@ -89,9 +91,25 @@ jobs:
|
||||
if: runner.os == 'macOS'
|
||||
with:
|
||||
flutter-version: ${{ env.FLUTTER_VERSION }}
|
||||
- name: Install Linux WebView dependencies
|
||||
if: runner.os == 'Linux'
|
||||
run: |
|
||||
sudo apt-get update -y
|
||||
sudo apt-get install -y libgtk-3-dev libwebkit2gtk-4.1-dev xvfb
|
||||
- name: Test webview
|
||||
if: runner.os == 'Linux'
|
||||
# Match the full suite's coverage instrumentation and repeat on the same
|
||||
# GTK event loop to catch queued callbacks after engine destruction.
|
||||
run: |
|
||||
PACKAGES=$(go list ./... | awk '!/\/bind\// && (!/\/cmd\// || /\/cmd\/web$/)')
|
||||
go test -c -tags webview -coverpkg=$(echo "$PACKAGES" | paste -sd "," -) -covermode=count -o "$RUNNER_TEMP/goprovider.test" ./internal/webview/goprovider
|
||||
xvfb-run -a "$RUNNER_TEMP/goprovider.test" -test.v -test.count=5 -test.timeout=300s -test.coverprofile="$RUNNER_TEMP/webview-coverage.out"
|
||||
env:
|
||||
CGO_ENABLED: 1
|
||||
- name: Test webview
|
||||
if: runner.os == 'Windows'
|
||||
run: go test -tags webview -v -count=1 -timeout 300s ./internal/webview/goprovider/
|
||||
# Repeat close/reopen coverage to catch browser shutdown races.
|
||||
run: go test -tags webview -v -count=3 -timeout 300s ./internal/webview/goprovider/
|
||||
env:
|
||||
CGO_ENABLED: 1
|
||||
- name: Test webview
|
||||
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 391 KiB After Width: | Height: | Size: 1.3 MiB |
Binary file not shown.
|
Before Width: | Height: | Size: 357 KiB After Width: | Height: | Size: 1.3 MiB |
@@ -7,6 +7,7 @@ import (
|
||||
|
||||
nativebridge "github.com/GopeedLab/gopeed/bind/native"
|
||||
"github.com/GopeedLab/gopeed/internal/webview/rpcprovider"
|
||||
goapi "github.com/GopeedLab/gopeed/pkg/api"
|
||||
"github.com/GopeedLab/gopeed/pkg/rest"
|
||||
"github.com/GopeedLab/gopeed/pkg/rest/model"
|
||||
)
|
||||
@@ -90,3 +91,14 @@ func applyWebViewProvider(config *model.StartConfig) {
|
||||
}
|
||||
config.WebViewProvider = rpcprovider.New(*config.WebViewRPCConfig)
|
||||
}
|
||||
|
||||
func LiveActivityTaskEventMask() int64 {
|
||||
return int64(
|
||||
goapi.TaskEventDone |
|
||||
goapi.TaskEventError |
|
||||
goapi.TaskEventStart |
|
||||
goapi.TaskEventProgress |
|
||||
goapi.TaskEventPause |
|
||||
goapi.TaskEventDelete,
|
||||
)
|
||||
}
|
||||
@@ -37,15 +37,17 @@ func installByInstaller(killSignalChan chan<- any, packagePath, destDir string)
|
||||
}
|
||||
defer reader.Close()
|
||||
|
||||
// Find the installer file
|
||||
var installerPath string
|
||||
for _, file := range reader.File {
|
||||
cleanName := filepath.Clean(file.Name)
|
||||
if strings.HasPrefix(cleanName, "..") || filepath.IsAbs(cleanName) {
|
||||
continue
|
||||
}
|
||||
if file.FileInfo().IsDir() {
|
||||
continue
|
||||
}
|
||||
|
||||
// Extract file
|
||||
path := filepath.Join(tempDir, file.Name)
|
||||
path := filepath.Join(tempDir, cleanName)
|
||||
|
||||
if err := os.MkdirAll(filepath.Dir(path), os.ModePerm); err != nil {
|
||||
return err
|
||||
@@ -101,7 +103,11 @@ func installByPortable(killSignalChan chan<- any, packagePath, destDir string) e
|
||||
defer reader.Close()
|
||||
|
||||
for _, file := range reader.File {
|
||||
path := filepath.Join(destDir, file.Name)
|
||||
cleanName := filepath.Clean(file.Name)
|
||||
if strings.HasPrefix(cleanName, "..") || filepath.IsAbs(cleanName) {
|
||||
continue
|
||||
}
|
||||
path := filepath.Join(destDir, cleanName)
|
||||
|
||||
if file.FileInfo().IsDir() {
|
||||
os.MkdirAll(path, file.Mode())
|
||||
|
||||
@@ -13,6 +13,7 @@ import (
|
||||
)
|
||||
|
||||
type args struct {
|
||||
TempDir *string `json:"tempDir"`
|
||||
Address *string `json:"address"`
|
||||
Port *int `json:"port"`
|
||||
Username *string `json:"username"`
|
||||
@@ -54,6 +55,7 @@ func loadCliArgs() *args {
|
||||
cfg.ApiToken = flag.String("T", "", "API token, it must be configured when using HTTP API in the case of enabling web authentication")
|
||||
flag.StringVar(cfg.ApiToken, "api-token", "", "API token, it must be configured when using HTTP API in the case of enabling web authentication")
|
||||
flag.BoolVar(&cfg.MCPEnable, "mcp-enable", false, "Enable MCP endpoint")
|
||||
cfg.TempDir = flag.String("temp-dir", "", "Temporary directory (defaults to the OS temporary directory)")
|
||||
cfg.StorageDir = flag.String("d", "", "Storage directory")
|
||||
flag.StringVar(cfg.StorageDir, "storage-dir", "", "Storage directory")
|
||||
whiteDownloadDirs := flag.String("white-download-dirs", "", "White download directories, comma-separated")
|
||||
@@ -89,6 +91,8 @@ func overrideWithCliArgs(cfg *args, cliConfig *args) {
|
||||
cfg.ApiToken = cliConfig.ApiToken
|
||||
case "mcp-enable":
|
||||
cfg.MCPEnable = cliConfig.MCPEnable
|
||||
case "temp-dir":
|
||||
cfg.TempDir = cliConfig.TempDir
|
||||
case "d", "storage-dir":
|
||||
cfg.StorageDir = cliConfig.StorageDir
|
||||
case "white-download-dirs":
|
||||
@@ -101,6 +105,9 @@ func overrideWithCliArgs(cfg *args, cliConfig *args) {
|
||||
|
||||
// setDefaults sets default values for any unset configuration fields
|
||||
func setDefaults(cfg *args, cliConfig *args) {
|
||||
if cfg.TempDir == nil {
|
||||
cfg.TempDir = cliConfig.TempDir
|
||||
}
|
||||
if cfg.Address == nil {
|
||||
cfg.Address = cliConfig.Address
|
||||
}
|
||||
|
||||
@@ -43,11 +43,16 @@ func main() {
|
||||
storageDir = filepath.Join(filepath.Dir(exe), "storage")
|
||||
}
|
||||
|
||||
tempDir := ""
|
||||
if args.TempDir != nil {
|
||||
tempDir = *args.TempDir
|
||||
}
|
||||
cfg := &model.StartConfig{
|
||||
Network: "tcp",
|
||||
Address: fmt.Sprintf("%s:%d", *args.Address, *args.Port),
|
||||
Storage: model.StorageBolt,
|
||||
StorageDir: storageDir,
|
||||
TempDir: tempDir,
|
||||
WhiteDownloadDirs: args.WhiteDownloadDirs,
|
||||
ApiToken: *args.ApiToken,
|
||||
MCPEnable: args.MCPEnable,
|
||||
|
||||
@@ -3,13 +3,15 @@ module github.com/GopeedLab/gopeed
|
||||
go 1.25.4
|
||||
|
||||
require (
|
||||
github.com/GopeedLab/webview_go v0.0.0-20260423085439-7a2f88b6e9b5
|
||||
codeberg.org/gruf/go-ffmpreg v0.6.23-0.20260827095416-df1836cb5be0
|
||||
github.com/GopeedLab/webview_go v0.0.0-20260918010912-0d4c27bb7914
|
||||
github.com/anacrolix/torrent v1.60.1-0.20251217073903-486bcbe758e0
|
||||
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5
|
||||
github.com/bodgit/sevenzip v1.6.1
|
||||
github.com/dop251/goja v0.0.0-20260106131823-651366fbe6e3
|
||||
github.com/dop251/goja_nodejs v0.0.0-20240728170619-29b559befffc
|
||||
github.com/go-git/go-git/v5 v5.8.1
|
||||
github.com/google/jsonschema-go v0.4.3
|
||||
github.com/gorilla/handlers v1.5.1
|
||||
github.com/gorilla/mux v1.8.0
|
||||
github.com/imroc/req/v3 v3.52.2
|
||||
@@ -21,6 +23,7 @@ require (
|
||||
github.com/pkg/errors v0.9.1
|
||||
github.com/rs/dnscache v0.0.0-20230804202142-fc85eb664529
|
||||
github.com/rs/zerolog v1.31.0
|
||||
github.com/tetratelabs/wazero v1.12.0
|
||||
github.com/xiaoqidun/setft v0.0.0-20220310121541-be86327699ad
|
||||
go.etcd.io/bbolt v1.4.3
|
||||
golang.org/x/exp v0.0.0-20251219203646-944ab1f22d93
|
||||
@@ -38,7 +41,6 @@ require (
|
||||
github.com/felixge/fgprof v0.9.5 // indirect
|
||||
github.com/go-task/slim-sprig/v3 v3.0.0 // indirect
|
||||
github.com/google/go-cmp v0.7.0 // indirect
|
||||
github.com/google/jsonschema-go v0.4.3 // indirect
|
||||
github.com/hashicorp/errwrap v1.1.0 // indirect
|
||||
github.com/hashicorp/go-multierror v1.1.1 // indirect
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
|
||||
@@ -156,7 +158,7 @@ require (
|
||||
golang.org/x/crypto v0.48.0 // indirect
|
||||
golang.org/x/mod v0.33.0 // indirect
|
||||
golang.org/x/net v0.50.0
|
||||
golang.org/x/sys v0.41.0 // indirect
|
||||
golang.org/x/sys v0.44.0 // indirect
|
||||
golang.org/x/text v0.34.0
|
||||
golang.org/x/time v0.15.0 // indirect
|
||||
golang.org/x/tools v0.42.0 // indirect
|
||||
|
||||
@@ -14,6 +14,8 @@ cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2k
|
||||
cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw=
|
||||
cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
|
||||
cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=
|
||||
codeberg.org/gruf/go-ffmpreg v0.6.23-0.20260827095416-df1836cb5be0 h1:AqNFl60o4xhTTOxmzAvSf7pazXHKBGuV5ATw7sETIRA=
|
||||
codeberg.org/gruf/go-ffmpreg v0.6.23-0.20260827095416-df1836cb5be0/go.mod h1:XM6eW8CSgAuZRKRynm5nDiSNedm0rav30IychqygOHE=
|
||||
crawshaw.io/iox v0.0.0-20181124134642-c51c3df30797/go.mod h1:sXBiorCo8c46JlQV3oXPKINnZ8mcqnye1EkVkqsectk=
|
||||
crawshaw.io/sqlite v0.3.2/go.mod h1:igAO5JulrQ1DbdZdtVq48mnZUBAPOeFzer7VhDWNtW4=
|
||||
dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk=
|
||||
@@ -23,8 +25,20 @@ filippo.io/edwards25519 v1.0.0-rc.1 h1:m0VOOB23frXZvAOK44usCgLWvtsxIoMCTBGJZlpmG
|
||||
filippo.io/edwards25519 v1.0.0-rc.1/go.mod h1:N1IkdkCkiLB6tki+MYJoSx2JTY9NUlxZE7eHn5EwJns=
|
||||
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
|
||||
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
|
||||
github.com/GopeedLab/webview_go v0.0.0-20260423085439-7a2f88b6e9b5 h1:86LVTaNDM/j/uTAD2HedkXNO9LIQlt642VAdbU/0D/M=
|
||||
github.com/GopeedLab/webview_go v0.0.0-20260423085439-7a2f88b6e9b5/go.mod h1:FvV19lu9rQ9NcbczN5y8pbWecURRRbWa8tbfFARBDa0=
|
||||
github.com/GopeedLab/webview_go v0.0.0-20260915010614-2567e492369a h1:Vzz+8aMVX9ZPPpkxgxa7mCorrAgdXCZL5EunOpOBvao=
|
||||
github.com/GopeedLab/webview_go v0.0.0-20260915010614-2567e492369a/go.mod h1:FvV19lu9rQ9NcbczN5y8pbWecURRRbWa8tbfFARBDa0=
|
||||
github.com/GopeedLab/webview_go v0.0.0-20260917153919-dcf00737f32a h1:nz1bC6TGwQ1kv/OVLahNZ0EEtb7v5LHfceAkW4MvS6I=
|
||||
github.com/GopeedLab/webview_go v0.0.0-20260917153919-dcf00737f32a/go.mod h1:FvV19lu9rQ9NcbczN5y8pbWecURRRbWa8tbfFARBDa0=
|
||||
github.com/GopeedLab/webview_go v0.0.0-20260917155039-1ee7ffb4169a h1:yudcfXpipKLnMaaw1losn6YeOBjvj2rAFicfjqYJexY=
|
||||
github.com/GopeedLab/webview_go v0.0.0-20260917155039-1ee7ffb4169a/go.mod h1:FvV19lu9rQ9NcbczN5y8pbWecURRRbWa8tbfFARBDa0=
|
||||
github.com/GopeedLab/webview_go v0.0.0-20260917155218-bd16da0d6f53 h1:nMSHKhbrEaWPE/x3mYl/270GPjswbUmS+E7yYjly4xk=
|
||||
github.com/GopeedLab/webview_go v0.0.0-20260917155218-bd16da0d6f53/go.mod h1:oNO07SCX8xgoHLds2OBwML8TU4xils88vMQ4gadKUHQ=
|
||||
github.com/GopeedLab/webview_go v0.0.0-20260917161756-2a4f989dcb1d h1:4fjlhJpBWa6IcdKVPkEMMIDd6Xo9JM09dJHdKU+1V7k=
|
||||
github.com/GopeedLab/webview_go v0.0.0-20260917161756-2a4f989dcb1d/go.mod h1:oNO07SCX8xgoHLds2OBwML8TU4xils88vMQ4gadKUHQ=
|
||||
github.com/GopeedLab/webview_go v0.0.0-20260917163929-ca26c3e6bcfa h1:VkdBMcNHYTqCMwOOCimA7b3xZKV/qFIGrhh3+vm9n3s=
|
||||
github.com/GopeedLab/webview_go v0.0.0-20260917163929-ca26c3e6bcfa/go.mod h1:oNO07SCX8xgoHLds2OBwML8TU4xils88vMQ4gadKUHQ=
|
||||
github.com/GopeedLab/webview_go v0.0.0-20260918010912-0d4c27bb7914 h1:9VJQZIZ1R6T4m+4vlNQYNXSPNUv2gf2PiTfIXiQ3MTw=
|
||||
github.com/GopeedLab/webview_go v0.0.0-20260918010912-0d4c27bb7914/go.mod h1:oNO07SCX8xgoHLds2OBwML8TU4xils88vMQ4gadKUHQ=
|
||||
github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0rYXWg0=
|
||||
github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ=
|
||||
github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY=
|
||||
@@ -552,6 +566,8 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO
|
||||
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
|
||||
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
|
||||
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
||||
github.com/tetratelabs/wazero v1.12.0 h1:DuWcpNu/FzgEXgGBDp8J1Spc+CWOvvtvVyjKlaZopYU=
|
||||
github.com/tetratelabs/wazero v1.12.0/go.mod h1:LvKtzl2RqO4gyF27BiXU+nKAjcV8f38U+kP/q2vgxh0=
|
||||
github.com/tidwall/btree v1.8.1 h1:27ehoXvm5AG/g+1VxLS1SD3vRhp/H7LuEfwNvddEdmA=
|
||||
github.com/tidwall/btree v1.8.1/go.mod h1:jBbTdUWhSZClZWoDg54VnvV7/54modSOzDN7VXftj1A=
|
||||
github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE=
|
||||
@@ -732,6 +748,8 @@ golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k=
|
||||
golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
|
||||
golang.org/x/sys v0.44.0 h1:ildZl3J4uzeKP07r2F++Op7E9B29JRUy+a27EibtBTQ=
|
||||
golang.org/x/sys v0.44.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc=
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"github.com/GopeedLab/gopeed/internal/production"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
@@ -61,6 +62,7 @@ type Metadata struct {
|
||||
}
|
||||
|
||||
type Source struct {
|
||||
producer production.Reader
|
||||
ID string
|
||||
URL string
|
||||
ContentType string
|
||||
@@ -359,11 +361,35 @@ func (r *Registry) ServeHTTP(w http.ResponseWriter, req *http.Request) {
|
||||
return
|
||||
}
|
||||
defer reader.Close()
|
||||
if producer, ok := reader.(production.Reader); ok {
|
||||
src.mu.Lock()
|
||||
src.producer = producer
|
||||
src.mu.Unlock()
|
||||
if err := producer.WaitProduction(req.Context()); err != nil {
|
||||
return
|
||||
}
|
||||
if producer.Production() != nil {
|
||||
meta.Size = 0
|
||||
meta.Range = false
|
||||
start = 0
|
||||
end = -1
|
||||
ranged = false
|
||||
src.mu.Lock()
|
||||
src.size = 0
|
||||
src.rangeEnabled = false
|
||||
src.mu.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
writeHeaders(w, meta, start, end, ranged)
|
||||
if ranged {
|
||||
w.WriteHeader(http.StatusPartialContent)
|
||||
}
|
||||
if producer, ok := reader.(production.Reader); ok && producer.Production() != nil {
|
||||
if flusher, ok := w.(http.Flusher); ok {
|
||||
flusher.Flush()
|
||||
}
|
||||
}
|
||||
limit := int64(-1)
|
||||
if end >= start {
|
||||
limit = end - start + 1
|
||||
@@ -675,3 +701,21 @@ func randomID(size int) (string, error) {
|
||||
}
|
||||
return hex.EncodeToString(buf), nil
|
||||
}
|
||||
|
||||
// Production returns the active opener's input counter, if it generates output.
|
||||
func (r *Registry) Production(raw string) production.Source {
|
||||
if r == nil {
|
||||
return nil
|
||||
}
|
||||
src, err := r.get(raw)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
src.mu.Lock()
|
||||
reader := src.producer
|
||||
src.mu.Unlock()
|
||||
if reader == nil {
|
||||
return nil
|
||||
}
|
||||
return reader.Production()
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
package controller
|
||||
|
||||
import (
|
||||
"github.com/GopeedLab/gopeed/internal/tempfiles"
|
||||
"github.com/GopeedLab/gopeed/pkg/base"
|
||||
"net/http"
|
||||
"net/url"
|
||||
@@ -9,8 +10,11 @@ import (
|
||||
)
|
||||
|
||||
type Controller struct {
|
||||
GetConfig func(v any)
|
||||
GetProxy func(requestProxy *base.RequestProxy) func(*http.Request) (*url.URL, error)
|
||||
TempFiles *tempfiles.Scope
|
||||
ManagedProduction func(string) bool
|
||||
TempDir string
|
||||
GetConfig func(v any)
|
||||
GetProxy func(requestProxy *base.RequestProxy) func(*http.Request) (*url.URL, error)
|
||||
FileController
|
||||
//ContextDialer() (proxy.Dialer, error)
|
||||
}
|
||||
|
||||
@@ -106,7 +106,13 @@ func (s *SchemeFilter) Match(uri string) bool {
|
||||
case FilterTypeUrl:
|
||||
return strings.HasPrefix(uriUpper, patternUpper+":")
|
||||
case FilterTypeFile:
|
||||
return strings.HasSuffix(uriUpper, "."+patternUpper)
|
||||
// Strip query and fragment before matching so URLs like
|
||||
// https://example.com/video.m3u8?sign=abc still route by extension.
|
||||
clean := uri
|
||||
if idx := strings.IndexAny(clean, "?#"); idx >= 0 {
|
||||
clean = clean[:idx]
|
||||
}
|
||||
return strings.HasSuffix(strings.ToUpper(clean), "."+patternUpper)
|
||||
case FilterTypeBase64:
|
||||
return strings.HasPrefix(uriUpper, "DATA:"+patternUpper+";BASE64,")
|
||||
}
|
||||
|
||||
@@ -0,0 +1,35 @@
|
||||
// Package production associates generated resources with their input progress.
|
||||
package production
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// Source counts unique input bytes and total received bytes (including refetches).
|
||||
// Implementations must support concurrent snapshots.
|
||||
type Source interface {
|
||||
Progress() (downloaded, received int64)
|
||||
}
|
||||
|
||||
// Reader resolves its optional producer before response headers are sent.
|
||||
// A nil producer preserves ordinary Blob behavior.
|
||||
type Reader interface {
|
||||
Production() Source
|
||||
WaitProduction(context.Context) error
|
||||
}
|
||||
type Registry struct {
|
||||
mu sync.Mutex
|
||||
sources map[string]Source
|
||||
}
|
||||
|
||||
func (r *Registry) Set(id string, source Source) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
if r.sources == nil {
|
||||
r.sources = make(map[string]Source)
|
||||
}
|
||||
r.sources[id] = source
|
||||
}
|
||||
func (r *Registry) Get(id string) Source { r.mu.Lock(); defer r.mu.Unlock(); return r.sources[id] }
|
||||
func (r *Registry) Remove(id string) { r.mu.Lock(); defer r.mu.Unlock(); delete(r.sources, id) }
|
||||
@@ -399,7 +399,10 @@ func (f *Fetcher) Wait() (err error) {
|
||||
}
|
||||
}
|
||||
if !selected {
|
||||
util.SafeRemove(filepath.Join(f.meta.Opts.Path, f.meta.Res.Name, file.Path()))
|
||||
root, name := torrentFileLayout(f.torrent.Info(), file.FileInfo())
|
||||
if err := removeUnselectedFile(f.meta.Opts.Path, root, name); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return
|
||||
@@ -409,6 +412,62 @@ func (f *Fetcher) Wait() (err error) {
|
||||
}
|
||||
}
|
||||
|
||||
// Match the file storage layout, including its NoName convention for rootless
|
||||
// torrents. File.Path() always prepends BestName, even when storage omits it.
|
||||
func torrentFileLayout(info *metainfo.Info, file metainfo.FileInfo) (root, name string) {
|
||||
parts := file.BestPath()
|
||||
if info.BestName() != metainfo.NoName {
|
||||
if len(parts) > 0 {
|
||||
root = info.BestName()
|
||||
}
|
||||
parts = append([]string{info.BestName()}, parts...)
|
||||
}
|
||||
// A v2 file tree can itself have a named root without an extra name wrapper.
|
||||
if root == "" && info.HasV2() && len(info.FileTree.Dir) == 1 {
|
||||
for dir, tree := range info.FileTree.Dir {
|
||||
if tree.IsDir() {
|
||||
root = dir
|
||||
}
|
||||
}
|
||||
}
|
||||
return root, filepath.Join(parts...)
|
||||
}
|
||||
|
||||
// Preserve the actual torrent root, or the download directory for rootless
|
||||
// torrents. Only prune empty directories strictly below that boundary.
|
||||
func removeUnselectedFile(downloadDir, torrentRoot, torrentPath string) error {
|
||||
if !filepath.IsLocal(torrentPath) || (torrentRoot != "" && !filepath.IsLocal(torrentRoot)) {
|
||||
return fmt.Errorf("invalid torrent file path %q", torrentPath)
|
||||
}
|
||||
root := filepath.Join(downloadDir, torrentRoot)
|
||||
name := filepath.Join(downloadDir, torrentPath)
|
||||
rel, err := filepath.Rel(root, name)
|
||||
if err != nil || rel == "." || !filepath.IsLocal(rel) {
|
||||
return fmt.Errorf("torrent file %q is outside cleanup root %q", name, root)
|
||||
}
|
||||
for _, candidate := range []string{name, name + ".part"} {
|
||||
if err := util.SafeRemove(candidate); err != nil {
|
||||
return fmt.Errorf("remove unselected torrent file %q: %w", candidate, err)
|
||||
}
|
||||
}
|
||||
for dir := filepath.Dir(name); dir != root && dir != filepath.Dir(dir); dir = filepath.Dir(dir) {
|
||||
entries, err := os.ReadDir(dir)
|
||||
if os.IsNotExist(err) {
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("read torrent directory %q: %w", dir, err)
|
||||
}
|
||||
if len(entries) != 0 {
|
||||
break
|
||||
}
|
||||
if err := util.SafeRemove(dir); err != nil {
|
||||
return fmt.Errorf("remove empty torrent directory %q: %w", dir, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *Fetcher) isDone() bool {
|
||||
if f.meta.Opts == nil {
|
||||
return false
|
||||
@@ -638,6 +697,10 @@ func (f *Fetcher) addTorrent(req *base.Request, fromUpload bool) (err error) {
|
||||
}
|
||||
spec.Storage = storage.NewFileOpts(storage.NewFileClientOpts{
|
||||
ClientBaseDir: cfg.DataDir,
|
||||
FilePathMaker: func(opts storage.FilePathMakerOpts) string {
|
||||
_, name := torrentFileLayout(opts.Info, *opts.File)
|
||||
return name
|
||||
},
|
||||
TorrentDirMaker: func(baseDir string, info *metainfo.Info, infoHash metainfo.Hash) string {
|
||||
return f.meta.Opts.Path
|
||||
},
|
||||
|
||||
@@ -0,0 +1,302 @@
|
||||
package bt
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha1"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/GopeedLab/gopeed/internal/fetcher"
|
||||
"github.com/GopeedLab/gopeed/pkg/base"
|
||||
"github.com/anacrolix/torrent"
|
||||
"github.com/anacrolix/torrent/bencode"
|
||||
"github.com/anacrolix/torrent/metainfo"
|
||||
"github.com/anacrolix/torrent/storage"
|
||||
)
|
||||
|
||||
func TestStartUsesSelectionSubmittedAfterResolve(t *testing.T) {
|
||||
config := torrent.TestingConfig(t)
|
||||
config.DataDir = t.TempDir()
|
||||
config.DisableTCP = true
|
||||
config.DisableUTP = true
|
||||
c, err := torrent.NewClient(config)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer c.Close()
|
||||
tor, err := c.AddTorrentFromFile("testdata/test.torrent")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f := &Fetcher{torrent: tor, meta: &fetcher.FetcherMeta{Opts: &base.Options{}}, data: &fetcherData{}}
|
||||
f.torrentReady.Store(true)
|
||||
f.updateRes()
|
||||
// Resolve initializes all files; creation replaces the selection before Start.
|
||||
f.meta.Opts = &base.Options{SelectFiles: []int{2}}
|
||||
if err := f.Start(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for i, file := range tor.Files() {
|
||||
want := torrent.PiecePriorityNone
|
||||
if i == 2 {
|
||||
want = torrent.PiecePriorityNormal
|
||||
}
|
||||
if file.Priority() != want {
|
||||
t.Errorf("file %d priority = %v, want %v", i, file.Priority(), want)
|
||||
}
|
||||
}
|
||||
if len(f.data.Progress) != 1 {
|
||||
t.Fatalf("progress tracks %d files, want 1", len(f.data.Progress))
|
||||
}
|
||||
if err := f.Pause(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := f.Start(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if tor.Files()[0].Priority() != torrent.PiecePriorityNone {
|
||||
t.Fatal("resume selected an unselected file")
|
||||
}
|
||||
}
|
||||
|
||||
func TestWaitCleansUnselectedFilesFromSharedPiece(t *testing.T) {
|
||||
for _, mode := range []string{"partial", "part only", "both paths", "all", "already removed", "remove error"} {
|
||||
t.Run(mode, func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
keepPath := filepath.Join(dir, "bundle", "keep.txt")
|
||||
skipPath := filepath.Join(dir, "bundle", "nested", "skip.txt")
|
||||
if err := os.MkdirAll(filepath.Dir(skipPath), 0755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(keepPath, []byte("keep"), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(skipPath, []byte("skip"), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Both files occupy the same piece, so downloading keep.txt also writes skip.txt.
|
||||
hash := sha1.Sum([]byte("keepskip"))
|
||||
info := metainfo.Info{Name: "bundle", PieceLength: 16, Pieces: hash[:], Files: []metainfo.FileInfo{
|
||||
{Length: 4, Path: []string{"keep.txt"}},
|
||||
{Length: 4, Path: []string{"nested", "skip.txt"}},
|
||||
}}
|
||||
store := storage.NewFileOpts(storage.NewFileClientOpts{
|
||||
ClientBaseDir: dir,
|
||||
TorrentDirMaker: func(baseDir string, _ *metainfo.Info, _ metainfo.Hash) string { return baseDir },
|
||||
})
|
||||
defer store.Close()
|
||||
config := torrent.TestingConfig(t)
|
||||
config.DisableTCP = true
|
||||
config.DisableUTP = true
|
||||
c, err := torrent.NewClient(config)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer c.Close()
|
||||
spec, err := torrent.TorrentSpecFromMetaInfoErr(&metainfo.MetaInfo{InfoBytes: bencode.MustMarshal(&info)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
spec.Storage = store
|
||||
tor, _, err := c.AddTorrentSpec(spec)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
if err := tor.VerifyDataContext(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f := &Fetcher{torrent: tor, meta: &fetcher.FetcherMeta{Opts: &base.Options{Path: dir, SelectFiles: []int{0}}}, data: &fetcherData{}, torrentDropCtx: ctx}
|
||||
f.torrentReady.Store(true)
|
||||
f.updateRes()
|
||||
if mode == "all" {
|
||||
f.meta.Opts.SelectFiles = []int{0, 1}
|
||||
}
|
||||
if !f.isDone() {
|
||||
t.Fatal("selected file is not complete")
|
||||
}
|
||||
if mode == "part only" || mode == "both paths" || mode == "all" {
|
||||
if err := os.WriteFile(skipPath+".part", []byte("skip"), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if mode == "part only" || mode == "already removed" || mode == "remove error" {
|
||||
if err := os.Remove(skipPath); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if mode == "remove error" {
|
||||
if err := os.Mkdir(skipPath, 0755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(skipPath, "child"), []byte("preserve"), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
err = f.Wait()
|
||||
if ctx.Err() != nil {
|
||||
t.Fatal(ctx.Err())
|
||||
}
|
||||
if mode == "remove error" {
|
||||
if err == nil {
|
||||
t.Fatal("cleanup failure was silently ignored")
|
||||
}
|
||||
} else if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
data, err := os.ReadFile(keepPath)
|
||||
if err != nil || string(data) != "keep" {
|
||||
t.Fatalf("selected file changed: %q, %v", data, err)
|
||||
}
|
||||
if mode == "all" {
|
||||
if _, err := os.Stat(skipPath + ".part"); err != nil {
|
||||
t.Fatalf("selected part file was removed: %v", err)
|
||||
}
|
||||
data, err := os.ReadFile(skipPath)
|
||||
if err != nil || string(data) != "skip" {
|
||||
t.Fatalf("selected file changed: %q, %v", data, err)
|
||||
}
|
||||
} else if mode != "remove error" {
|
||||
for _, removed := range []string{skipPath, skipPath + ".part", filepath.Dir(skipPath)} {
|
||||
if _, err := os.Stat(removed); !os.IsNotExist(err) {
|
||||
t.Fatalf("unselected file or empty directory remains: %s: %v", removed, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemoveUnselectedFilePrunesOnlyEmptyParents(t *testing.T) {
|
||||
for _, preserveSibling := range []bool{false, true} {
|
||||
t.Run(fmt.Sprint(preserveSibling), func(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
relative := filepath.Join("bundle", "nested", "deep", "skip.txt")
|
||||
name := filepath.Join(root, relative)
|
||||
if err := os.MkdirAll(filepath.Dir(name), 0755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(name+".part", []byte("partial"), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sibling := filepath.Join(root, "bundle", "keep.txt")
|
||||
if preserveSibling {
|
||||
if err := os.WriteFile(sibling, []byte("keep"), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := removeUnselectedFile(root, "bundle", relative); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(root, "bundle", "nested")); !os.IsNotExist(err) {
|
||||
t.Fatalf("empty ancestors remain: %v", err)
|
||||
}
|
||||
if preserveSibling {
|
||||
data, err := os.ReadFile(sibling)
|
||||
if err != nil || string(data) != "keep" {
|
||||
t.Fatalf("sibling changed: %q, %v", data, err)
|
||||
}
|
||||
}
|
||||
if info, err := os.Stat(filepath.Join(root, "bundle")); err != nil || !info.IsDir() {
|
||||
t.Fatalf("torrent root directory was removed: %v", err)
|
||||
}
|
||||
if _, err := os.Stat(root); err != nil {
|
||||
t.Fatalf("download directory was removed: %v", err)
|
||||
}
|
||||
// A file directly inside the torrent root must not cause that root to be pruned.
|
||||
rootFile := filepath.Join("bundle", "root.txt")
|
||||
if err := os.WriteFile(filepath.Join(root, rootFile)+".part", []byte("partial"), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := removeUnselectedFile(root, "bundle", rootFile); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(root, "bundle")); err != nil {
|
||||
t.Fatalf("torrent root directory was removed after root-level cleanup: %v", err)
|
||||
}
|
||||
// Repeating cleanup after the file and its subdirectories are gone is harmless.
|
||||
if err := removeUnselectedFile(root, "bundle", relative); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCleanupMatchesNamedAndRootlessStorage(t *testing.T) {
|
||||
for _, version := range []int64{1, 2} {
|
||||
for _, layout := range []string{"rootless", "wrapped", "file-tree-root"} {
|
||||
if version == 1 && layout == "file-tree-root" {
|
||||
continue
|
||||
}
|
||||
t.Run(fmt.Sprintf("v%d/%s", version, layout), func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
info := metainfo.Info{Name: metainfo.NoName, PieceLength: 16384}
|
||||
root := dir
|
||||
if layout == "wrapped" {
|
||||
info.Name = "bundle"
|
||||
root = filepath.Join(dir, "bundle")
|
||||
}
|
||||
if version == 1 {
|
||||
info.Files = []metainfo.FileInfo{{Path: []string{"keep.txt"}}, {Path: []string{"nested", "deep", "skip.txt"}}}
|
||||
} else {
|
||||
info.MetaVersion = 2
|
||||
info.FileTree = metainfo.FileTree{Dir: map[string]metainfo.FileTree{
|
||||
"keep.txt": {},
|
||||
"nested": {Dir: map[string]metainfo.FileTree{"deep": {Dir: map[string]metainfo.FileTree{"skip.txt": {}}}}},
|
||||
}}
|
||||
}
|
||||
if layout == "file-tree-root" {
|
||||
info.FileTree = metainfo.FileTree{Dir: map[string]metainfo.FileTree{"tree-root": info.FileTree}}
|
||||
root = filepath.Join(dir, "tree-root")
|
||||
}
|
||||
// Keep the library's default FilePathMaker here to independently verify
|
||||
// that cleanup agrees with its real disk layout, including NoName.
|
||||
store := storage.NewFileOpts(storage.NewFileClientOpts{
|
||||
ClientBaseDir: dir,
|
||||
TorrentDirMaker: func(baseDir string, _ *metainfo.Info, _ metainfo.Hash) string { return baseDir },
|
||||
})
|
||||
defer store.Close()
|
||||
torrentStorage, err := store.OpenTorrent(context.Background(), &info, metainfo.Hash{1})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer torrentStorage.Close()
|
||||
skip := filepath.Join(root, "nested", "deep", "skip.txt")
|
||||
if _, err := os.Stat(skip); err != nil {
|
||||
t.Fatalf("unexpected storage layout: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(skip+".part", []byte("partial"), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
files := info.UpvertedFiles()
|
||||
boundary, name := torrentFileLayout(&info, files[1])
|
||||
if err := removeUnselectedFile(dir, boundary, name); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(root, "nested")); !os.IsNotExist(err) {
|
||||
t.Fatalf("empty subtree remains: %v", err)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(root, "keep.txt")); err != nil {
|
||||
t.Fatalf("selected file removed: %v", err)
|
||||
}
|
||||
// Also verify the boundary when the protected directory becomes empty.
|
||||
if err := os.Remove(filepath.Join(root, "keep.txt")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, file := range files {
|
||||
boundary, name := torrentFileLayout(&info, file)
|
||||
if err := removeUnselectedFile(dir, boundary, name); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if stat, err := os.Stat(root); err != nil || !stat.IsDir() {
|
||||
t.Fatalf("cleanup boundary was removed: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
package hls
|
||||
|
||||
const (
|
||||
defaultSegmentConnections = 8
|
||||
defaultMaxRetries = 5
|
||||
defaultTimeoutSeconds = 30
|
||||
|
||||
// maxPrefetchSizeSegments caps the HEAD probing done at resolve time.
|
||||
// Playlists larger than this skip size prefetch entirely.
|
||||
maxPrefetchSizeSegments = 500
|
||||
)
|
||||
|
||||
type config struct {
|
||||
SegmentConnections int `json:"segmentConnections"`
|
||||
MaxRetries int `json:"maxRetries"`
|
||||
TimeoutSeconds int `json:"timeoutSeconds"`
|
||||
PrefetchContentLength bool `json:"prefetchContentLength"`
|
||||
}
|
||||
|
||||
func (c *config) normalize() {
|
||||
if c.SegmentConnections <= 0 {
|
||||
c.SegmentConnections = defaultSegmentConnections
|
||||
}
|
||||
if c.MaxRetries < 0 {
|
||||
c.MaxRetries = defaultMaxRetries
|
||||
}
|
||||
if c.TimeoutSeconds <= 0 {
|
||||
c.TimeoutSeconds = defaultTimeoutSeconds
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
package hls
|
||||
|
||||
import (
|
||||
"crypto/aes"
|
||||
"crypto/cipher"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// decryptAES128 decrypts an AES-128-CBC encrypted segment and strips PKCS7
|
||||
// padding. data length must be a multiple of the AES block size.
|
||||
func decryptAES128(data, key, iv []byte) ([]byte, error) {
|
||||
if len(key) != 16 {
|
||||
return nil, fmt.Errorf("invalid AES-128 key size %d", len(key))
|
||||
}
|
||||
if len(iv) != 16 {
|
||||
return nil, fmt.Errorf("invalid AES-128 IV size %d", len(iv))
|
||||
}
|
||||
if len(data) == 0 || len(data)%aes.BlockSize != 0 {
|
||||
return nil, fmt.Errorf("invalid AES-128 data size %d", len(data))
|
||||
}
|
||||
block, err := aes.NewCipher(key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
plain := make([]byte, len(data))
|
||||
cipher.NewCBCDecrypter(block, iv).CryptBlocks(plain, data)
|
||||
return pkcs7Unpad(plain, aes.BlockSize)
|
||||
}
|
||||
|
||||
func pkcs7Unpad(data []byte, blockSize int) ([]byte, error) {
|
||||
if len(data) == 0 {
|
||||
return nil, fmt.Errorf("empty plaintext")
|
||||
}
|
||||
pad := int(data[len(data)-1])
|
||||
if pad == 0 || pad > blockSize || pad > len(data) {
|
||||
return nil, fmt.Errorf("invalid PKCS7 padding")
|
||||
}
|
||||
for _, b := range data[len(data)-pad:] {
|
||||
if int(b) != pad {
|
||||
return nil, fmt.Errorf("invalid PKCS7 padding")
|
||||
}
|
||||
}
|
||||
return data[:len(data)-pad], nil
|
||||
}
|
||||
|
||||
// sequenceIV derives the default per-segment IV mandated by RFC 8216: the
|
||||
// segment media sequence number as a 16-byte big-endian integer.
|
||||
func sequenceIV(sequence int64) []byte {
|
||||
iv := make([]byte, 16)
|
||||
iv[8] = byte(sequence >> 56)
|
||||
iv[9] = byte(sequence >> 48)
|
||||
iv[10] = byte(sequence >> 40)
|
||||
iv[11] = byte(sequence >> 32)
|
||||
iv[12] = byte(sequence >> 24)
|
||||
iv[13] = byte(sequence >> 16)
|
||||
iv[14] = byte(sequence >> 8)
|
||||
iv[15] = byte(sequence)
|
||||
return iv
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
package hls
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/aes"
|
||||
"crypto/cipher"
|
||||
"crypto/rand"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func pkcs7Pad(data []byte, blockSize int) []byte {
|
||||
pad := blockSize - len(data)%blockSize
|
||||
return append(data, bytes.Repeat([]byte{byte(pad)}, pad)...)
|
||||
}
|
||||
|
||||
func TestDecryptAES128Roundtrip(t *testing.T) {
|
||||
key := make([]byte, 16)
|
||||
iv := make([]byte, 16)
|
||||
if _, err := rand.Read(key); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := rand.Read(iv); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
plain := []byte("hello hls segment payload, padded to block size!")
|
||||
padded := pkcs7Pad(plain, aes.BlockSize)
|
||||
block, err := aes.NewCipher(key)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
encrypted := make([]byte, len(padded))
|
||||
cipher.NewCBCEncrypter(block, iv).CryptBlocks(encrypted, padded)
|
||||
|
||||
got, err := decryptAES128(encrypted, key, iv)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !bytes.Equal(got, plain) {
|
||||
t.Errorf("roundtrip mismatch: %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecryptAES128InvalidInput(t *testing.T) {
|
||||
if _, err := decryptAES128([]byte{1, 2, 3}, make([]byte, 16), make([]byte, 16)); err == nil {
|
||||
t.Error("non-block-aligned data should fail")
|
||||
}
|
||||
if _, err := decryptAES128(make([]byte, 16), make([]byte, 15), make([]byte, 16)); err == nil {
|
||||
t.Error("short key should fail")
|
||||
}
|
||||
if _, err := decryptAES128(make([]byte, 16), make([]byte, 16), make([]byte, 16)); err == nil {
|
||||
t.Error("all-zero block has invalid padding and should fail")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSequenceIV(t *testing.T) {
|
||||
iv := sequenceIV(1)
|
||||
if len(iv) != 16 || iv[15] != 1 {
|
||||
t.Errorf("sequence 1 IV wrong: %v", iv)
|
||||
}
|
||||
iv = sequenceIV(0x0102030405060708)
|
||||
for i, want := range []byte{1, 2, 3, 4, 5, 6, 7, 8} {
|
||||
if iv[8+i] != want {
|
||||
t.Errorf("sequence IV byte %d: want %d got %d", i, want, iv[8+i])
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
// Package hls implements native HTTP Live Streaming downloads for m3u8
|
||||
// playlists. A task produces exactly one merged, playable output file:
|
||||
// MPEG-2 TS segments are concatenated as-is, fMP4 streams are merged as
|
||||
// init section followed by the media fragments, and AES-128 encrypted
|
||||
// segments are decrypted before they are stored.
|
||||
//
|
||||
// Supported subset:
|
||||
// - VOD media playlists (EXT-X-ENDLIST present) and master playlists whose
|
||||
// selected variant carries its own audio
|
||||
// - AES-128 with the identity KEYFORMAT, explicit IV or media-sequence
|
||||
// derived IV, including key rotation mid-playlist
|
||||
// - EXT-X-BYTERANGE (including offset inheritance) and fMP4 init sections
|
||||
// via a single, constant EXT-X-MAP
|
||||
//
|
||||
// Everything the merge pipeline cannot reproduce faithfully is rejected with
|
||||
// an explicit error instead of silently producing a broken output file:
|
||||
// - live streams (no EXT-X-ENDLIST) and SAMPLE-AES / DRM encryption
|
||||
// - EXT-X-GAP (would leave a hole) and EXT-X-DISCONTINUITY (broken timeline)
|
||||
// - changing init sections (a repeated EXT-X-MAP is compared by all its
|
||||
// attributes, not just the URI) and encrypted EXT-X-MAP without an
|
||||
// explicit IV
|
||||
// - separate audio renditions referenced by the selected variant (the
|
||||
// merged output would have no audio) and non-identity KEYFORMATs
|
||||
package hls
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,100 @@
|
||||
package hls
|
||||
|
||||
import (
|
||||
"net/url"
|
||||
"path"
|
||||
"strings"
|
||||
|
||||
"github.com/GopeedLab/gopeed/internal/fetcher"
|
||||
)
|
||||
|
||||
type FetcherManager struct {
|
||||
}
|
||||
|
||||
func (fm *FetcherManager) Name() string {
|
||||
return "hls"
|
||||
}
|
||||
|
||||
// Filters registers the .m3u8 extension. The manager must be registered BEFORE
|
||||
// the http manager in the downloader config: protocol routing picks the first
|
||||
// manager whose filter matches, and the http filter claims every http(s) URL.
|
||||
func (fm *FetcherManager) Filters() []*fetcher.SchemeFilter {
|
||||
return []*fetcher.SchemeFilter{
|
||||
{
|
||||
Type: fetcher.FilterTypeFile,
|
||||
Pattern: "M3U8",
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (fm *FetcherManager) Build() fetcher.Fetcher {
|
||||
return &Fetcher{}
|
||||
}
|
||||
|
||||
func (fm *FetcherManager) ParseName(u string) string {
|
||||
parsed, err := url.Parse(u)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
name := path.Base(parsed.Path)
|
||||
name = strings.TrimSuffix(name, path.Ext(name))
|
||||
if name == "" || name == "/" || name == "." {
|
||||
name = parsed.Hostname()
|
||||
}
|
||||
return name
|
||||
}
|
||||
|
||||
func (fm *FetcherManager) AutoRename() bool {
|
||||
return true
|
||||
}
|
||||
|
||||
func (fm *FetcherManager) DefaultConfig() any {
|
||||
return &config{
|
||||
SegmentConnections: defaultSegmentConnections,
|
||||
MaxRetries: defaultMaxRetries,
|
||||
TimeoutSeconds: defaultTimeoutSeconds,
|
||||
PrefetchContentLength: true,
|
||||
}
|
||||
}
|
||||
|
||||
func (fm *FetcherManager) Store(f fetcher.Fetcher) (data any, err error) {
|
||||
_f := f.(*Fetcher)
|
||||
_f.mu.Lock()
|
||||
defer _f.mu.Unlock()
|
||||
if _f.state == nil {
|
||||
return nil, nil
|
||||
}
|
||||
// The engine's periodic checkpoint calls Store without holding the task
|
||||
// lock and serializes the returned value afterwards, while Resolve/Patch/
|
||||
// Start mutate the live state under the fetcher mutex: return a snapshot
|
||||
// copy taken under that mutex, mirroring the HTTP protocol's Store.
|
||||
return cloneState(_f.state), nil
|
||||
}
|
||||
|
||||
func cloneState(state *fetcherState) *fetcherState {
|
||||
cloned := *state
|
||||
cloned.Segments = make([]*Segment, len(state.Segments))
|
||||
for i, seg := range state.Segments {
|
||||
s := *seg
|
||||
cloned.Segments[i] = &s
|
||||
}
|
||||
return &cloned
|
||||
}
|
||||
|
||||
func (fm *FetcherManager) Restore() (v any, f func(meta *fetcher.FetcherMeta, v any) fetcher.Fetcher) {
|
||||
return &fetcherState{}, func(meta *fetcher.FetcherMeta, v any) fetcher.Fetcher {
|
||||
fb := &FetcherManager{}
|
||||
restored := fb.Build().(*Fetcher)
|
||||
restored.meta = meta
|
||||
if state, ok := v.(*fetcherState); ok && state.MediaURL != "" {
|
||||
restored.state = state
|
||||
}
|
||||
// Setup is invoked by the engine (restoreFetcher) after this builder
|
||||
// returns; ReqExtra parsing happens there as well.
|
||||
return restored
|
||||
}
|
||||
}
|
||||
|
||||
func (fm *FetcherManager) Close() error {
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,857 @@
|
||||
package hls
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/aes"
|
||||
"crypto/cipher"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/GopeedLab/gopeed/internal/controller"
|
||||
"github.com/GopeedLab/gopeed/internal/fetcher"
|
||||
"github.com/GopeedLab/gopeed/pkg/base"
|
||||
)
|
||||
|
||||
func pkcs7PadForTest(data []byte, blockSize int) []byte {
|
||||
pad := blockSize - len(data)%blockSize
|
||||
return append(data, bytes.Repeat([]byte{byte(pad)}, pad)...)
|
||||
}
|
||||
|
||||
func encryptSegment(plain, key, iv []byte) []byte {
|
||||
padded := pkcs7PadForTest(plain, aes.BlockSize)
|
||||
block, _ := aes.NewCipher(key)
|
||||
encrypted := make([]byte, len(padded))
|
||||
cipher.NewCBCEncrypter(block, iv).CryptBlocks(encrypted, padded)
|
||||
return encrypted
|
||||
}
|
||||
|
||||
// serveContentWithReferer is a helper handler that enforces an optional
|
||||
// Referer requirement and reports request counts.
|
||||
type cdnHandler struct {
|
||||
mux *http.ServeMux
|
||||
requireReferer string
|
||||
requests atomic.Int64
|
||||
}
|
||||
|
||||
func (h *cdnHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
h.requests.Add(1)
|
||||
if h.requireReferer != "" && r.Header.Get("Referer") != h.requireReferer {
|
||||
w.WriteHeader(http.StatusForbidden)
|
||||
return
|
||||
}
|
||||
h.mux.ServeHTTP(w, r)
|
||||
}
|
||||
|
||||
func newTestFetcher(t *testing.T, cfg config) *Fetcher {
|
||||
t.Helper()
|
||||
fm := &FetcherManager{}
|
||||
f := fm.Build().(*Fetcher)
|
||||
ctl := controller.NewController()
|
||||
ctl.GetConfig = func(v any) {
|
||||
if c, ok := v.(*config); ok {
|
||||
*c = cfg
|
||||
}
|
||||
}
|
||||
f.Setup(ctl)
|
||||
return f
|
||||
}
|
||||
|
||||
func testConfig() config {
|
||||
return config{
|
||||
SegmentConnections: 4,
|
||||
MaxRetries: 2,
|
||||
TimeoutSeconds: 5,
|
||||
PrefetchContentLength: true,
|
||||
}
|
||||
}
|
||||
|
||||
func mustResolve(t *testing.T, f *Fetcher, playlistURL, referer string) {
|
||||
t.Helper()
|
||||
req := &base.Request{URL: playlistURL}
|
||||
if referer != "" {
|
||||
req.Extra = &ReqExtra{Header: map[string]string{"Referer": referer}}
|
||||
}
|
||||
if err := f.Resolve(req, &base.Options{Path: t.TempDir()}); err != nil {
|
||||
t.Fatalf("resolve: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func mustRun(t *testing.T, f *Fetcher) {
|
||||
t.Helper()
|
||||
if err := f.Start(); err != nil {
|
||||
t.Fatalf("start: %v", err)
|
||||
}
|
||||
if err := f.Wait(); err != nil {
|
||||
t.Fatalf("wait: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func statsOf(t *testing.T, f *Fetcher) Stats {
|
||||
t.Helper()
|
||||
s, ok := f.Stats().Snapshot.(*Stats)
|
||||
if !ok {
|
||||
t.Fatalf("unexpected stats snapshot type %T", f.Stats().Snapshot)
|
||||
}
|
||||
return *s
|
||||
}
|
||||
|
||||
func readOutput(t *testing.T, f *Fetcher) []byte {
|
||||
t.Helper()
|
||||
data, err := os.ReadFile(f.meta.SingleFilepath())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return data
|
||||
}
|
||||
|
||||
func TestFetcherE2E_MasterTS(t *testing.T) {
|
||||
seg := func(i int) []byte {
|
||||
return []byte(fmt.Sprintf("SEGMENT-%02d-PAYLOAD", i))
|
||||
}
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/master.m3u8", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Write([]byte("#EXTM3U\n" +
|
||||
"#EXT-X-STREAM-INF:BANDWIDTH=800000\nv1/index.m3u8\n" +
|
||||
"#EXT-X-STREAM-INF:BANDWIDTH=2000000\nv2/index.m3u8\n"))
|
||||
})
|
||||
mux.HandleFunc("/v1/index.m3u8", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Write([]byte("#EXTM3U\n#EXTINF:4,\nlow-0.ts\n#EXT-X-ENDLIST\n"))
|
||||
})
|
||||
mux.HandleFunc("/v2/index.m3u8", func(w http.ResponseWriter, r *http.Request) {
|
||||
var b strings.Builder
|
||||
b.WriteString("#EXTM3U\n")
|
||||
for i := 0; i < 5; i++ {
|
||||
b.WriteString(fmt.Sprintf("#EXTINF:4,\nseg-%d.ts\n", i))
|
||||
}
|
||||
b.WriteString("#EXT-X-ENDLIST\n")
|
||||
w.Write([]byte(b.String()))
|
||||
})
|
||||
for i := 0; i < 5; i++ {
|
||||
content := seg(i)
|
||||
path := "/v2/seg-" + strconv.Itoa(i) + ".ts"
|
||||
mux.HandleFunc(path, func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Length", strconv.Itoa(len(content)))
|
||||
w.Write(content)
|
||||
})
|
||||
}
|
||||
server := httptest.NewServer(&cdnHandler{mux: mux})
|
||||
defer server.Close()
|
||||
|
||||
f := newTestFetcher(t, testConfig())
|
||||
mustResolve(t, f, server.URL+"/master.m3u8", "")
|
||||
// Master resolution picked the 2000000 variant.
|
||||
if f.state.MediaURL != server.URL+"/v2/index.m3u8" {
|
||||
t.Fatalf("wrong variant picked: %s", f.state.MediaURL)
|
||||
}
|
||||
// HEAD prefetch should have computed the exact total size.
|
||||
if f.meta.Res.Size != int64(5*len(seg(0))) {
|
||||
t.Fatalf("prefetched size wrong: %d", f.meta.Res.Size)
|
||||
}
|
||||
mustRun(t, f)
|
||||
|
||||
var want []byte
|
||||
for i := 0; i < 5; i++ {
|
||||
want = append(want, seg(i)...)
|
||||
}
|
||||
got := readOutput(t, f)
|
||||
if !bytes.Equal(got, want) {
|
||||
t.Errorf("merged output mismatch: %q", got)
|
||||
}
|
||||
// The staging folder must be cleaned up after a successful merge; only
|
||||
// the output file remains in the task directory.
|
||||
if entries, _ := os.ReadDir(filepath.Dir(f.meta.SingleFilepath())); len(entries) != 1 {
|
||||
for _, e := range entries {
|
||||
t.Logf("leftover entry: %s", e.Name())
|
||||
}
|
||||
t.Errorf("staging dir should be cleaned up after merge")
|
||||
}
|
||||
if f.meta.Res.Size != int64(len(want)) {
|
||||
t.Errorf("final size wrong: %d", f.meta.Res.Size)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetcherE2E_RefererEnforced(t *testing.T) {
|
||||
page := "https://player.example.com/watch/123"
|
||||
var playlist strings.Builder
|
||||
playlist.WriteString("#EXTM3U\n")
|
||||
segments := make(map[string][]byte)
|
||||
for i := 0; i < 3; i++ {
|
||||
name := fmt.Sprintf("seg-%d.ts", i)
|
||||
segments["/"+name] = []byte(fmt.Sprintf("DATA-%d", i))
|
||||
playlist.WriteString(fmt.Sprintf("#EXTINF:4,\n%s\n", name))
|
||||
}
|
||||
playlist.WriteString("#EXT-X-ENDLIST\n")
|
||||
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/video.m3u8", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Write([]byte(playlist.String()))
|
||||
})
|
||||
for path, content := range segments {
|
||||
c := content
|
||||
mux.HandleFunc(path, func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Write(c)
|
||||
})
|
||||
}
|
||||
server := httptest.NewServer(&cdnHandler{mux: mux, requireReferer: page})
|
||||
defer server.Close()
|
||||
|
||||
// Without Referer the resolve itself must fail (403 on the playlist).
|
||||
blocked := newTestFetcher(t, testConfig())
|
||||
req := &base.Request{URL: server.URL + "/video.m3u8"}
|
||||
if err := blocked.Resolve(req, &base.Options{Path: t.TempDir()}); err == nil {
|
||||
t.Fatal("resolve without Referer should fail")
|
||||
}
|
||||
|
||||
// With the Referer set through the task extra headers it must succeed.
|
||||
allowed := newTestFetcher(t, testConfig())
|
||||
allowedReq := &base.Request{
|
||||
URL: server.URL + "/video.m3u8",
|
||||
Extra: &ReqExtra{Header: map[string]string{"Referer": page}},
|
||||
}
|
||||
if err := allowed.Resolve(allowedReq, &base.Options{Path: t.TempDir()}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
mustRun(t, allowed)
|
||||
want := []byte("DATA-0DATA-1DATA-2")
|
||||
if !bytes.Equal(readOutput(t, allowed), want) {
|
||||
t.Error("output mismatch with Referer download")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetcherE2E_AES128KeyRotation(t *testing.T) {
|
||||
key1 := []byte("0123456789abcdef")
|
||||
key2 := []byte("fedcba9876543210")
|
||||
iv1 := []byte("aaaaaaaaaaaaaaaa")
|
||||
plain1 := []byte("ENCRYPTED-SEGMENT-ONE!!") // padded on encrypt
|
||||
plain2 := []byte("ENCRYPTED-SEGMENT-TWO!!")
|
||||
// seg-2 has no explicit IV: derived from media sequence 1.
|
||||
enc1 := encryptSegment(plain1, key1, iv1)
|
||||
enc2 := encryptSegment(plain2, key2, sequenceIV(1))
|
||||
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/video.m3u8", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Write([]byte("#EXTM3U\n" +
|
||||
"#EXT-X-MEDIA-SEQUENCE:0\n" +
|
||||
"#EXT-X-KEY:METHOD=AES-128,URI=\"key-1.bin\",IV=0x" + hexEncode(iv1) + "\n" +
|
||||
"#EXTINF:5,\nseg-1.ts\n" +
|
||||
"#EXT-X-KEY:METHOD=AES-128,URI=\"key-2.bin\"\n" +
|
||||
"#EXTINF:5,\nseg-2.ts\n" +
|
||||
"#EXT-X-ENDLIST\n"))
|
||||
})
|
||||
mux.HandleFunc("/key-1.bin", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Write(key1)
|
||||
})
|
||||
mux.HandleFunc("/key-2.bin", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Write(key2)
|
||||
})
|
||||
mux.HandleFunc("/seg-1.ts", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Write(enc1)
|
||||
})
|
||||
mux.HandleFunc("/seg-2.ts", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Write(enc2)
|
||||
})
|
||||
server := httptest.NewServer(mux)
|
||||
defer server.Close()
|
||||
|
||||
f := newTestFetcher(t, testConfig())
|
||||
mustResolve(t, f, server.URL+"/video.m3u8", "")
|
||||
// Ciphertext sizes overstate the final output after decryption, so an
|
||||
// encrypted plan must report an indeterminate size.
|
||||
if f.meta.Res.Size != 0 {
|
||||
t.Errorf("encrypted plan should report size 0, got %d", f.meta.Res.Size)
|
||||
}
|
||||
mustRun(t, f)
|
||||
|
||||
want := append(append([]byte{}, plain1...), plain2...)
|
||||
if !bytes.Equal(readOutput(t, f), want) {
|
||||
t.Errorf("decrypted output mismatch: %q", readOutput(t, f))
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetcherE2E_ByteRange(t *testing.T) {
|
||||
data := []byte("0123456789ABCDEFGHIJ") // 20 bytes
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/video.m3u8", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Write([]byte("#EXTM3U\n" +
|
||||
"#EXT-X-BYTERANGE:10@0\nblob.bin\n" +
|
||||
"#EXT-X-BYTERANGE:10\nblob.bin\n" +
|
||||
"#EXT-X-ENDLIST\n"))
|
||||
})
|
||||
mux.HandleFunc("/blob.bin", func(w http.ResponseWriter, r *http.Request) {
|
||||
rng := r.Header.Get("Range")
|
||||
if rng == "" {
|
||||
w.Write(data)
|
||||
return
|
||||
}
|
||||
// bytes=start-end
|
||||
var start, end int
|
||||
fmt.Sscanf(rng, "bytes=%d-%d", &start, &end)
|
||||
w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, len(data)))
|
||||
w.WriteHeader(http.StatusPartialContent)
|
||||
w.Write(data[start : end+1])
|
||||
})
|
||||
server := httptest.NewServer(mux)
|
||||
defer server.Close()
|
||||
|
||||
f := newTestFetcher(t, testConfig())
|
||||
mustResolve(t, f, server.URL+"/video.m3u8", "")
|
||||
if f.meta.Res.Size != 20 {
|
||||
t.Fatalf("byterange sizes should be known from the playlist, got %d", f.meta.Res.Size)
|
||||
}
|
||||
mustRun(t, f)
|
||||
if !bytes.Equal(readOutput(t, f), data) {
|
||||
t.Errorf("byterange merge mismatch: %q", readOutput(t, f))
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetcherE2E_PauseResume(t *testing.T) {
|
||||
const total = 8
|
||||
var served atomic.Int64
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/video.m3u8", func(w http.ResponseWriter, r *http.Request) {
|
||||
var b strings.Builder
|
||||
b.WriteString("#EXTM3U\n")
|
||||
for i := 0; i < total; i++ {
|
||||
b.WriteString(fmt.Sprintf("#EXTINF:4,\nseg-%d.ts\n", i))
|
||||
}
|
||||
b.WriteString("#EXT-X-ENDLIST\n")
|
||||
w.Write([]byte(b.String()))
|
||||
})
|
||||
segments := make(map[int][]byte)
|
||||
for i := 0; i < total; i++ {
|
||||
segments[i] = []byte(fmt.Sprintf("SLOW-SEGMENT-%02d", i))
|
||||
}
|
||||
for i := 0; i < total; i++ {
|
||||
idx := i
|
||||
mux.HandleFunc(fmt.Sprintf("/seg-%d.ts", i), func(w http.ResponseWriter, r *http.Request) {
|
||||
time.Sleep(150 * time.Millisecond)
|
||||
served.Add(1)
|
||||
w.Write(segments[idx])
|
||||
})
|
||||
}
|
||||
server := httptest.NewServer(mux)
|
||||
defer server.Close()
|
||||
|
||||
cfg := testConfig()
|
||||
cfg.SegmentConnections = 2
|
||||
cfg.PrefetchContentLength = false
|
||||
f := newTestFetcher(t, cfg)
|
||||
mustResolve(t, f, server.URL+"/video.m3u8", "")
|
||||
|
||||
if err := f.Start(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Let a couple of segments finish, then pause.
|
||||
time.Sleep(400 * time.Millisecond)
|
||||
if err := f.Pause(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
doneBefore := statsOf(t, f).SegmentsDone
|
||||
if doneBefore == 0 {
|
||||
t.Fatal("expected some segments to complete before pause")
|
||||
}
|
||||
if doneBefore == total {
|
||||
t.Fatal("all segments finished before pause, test is not exercising resume")
|
||||
}
|
||||
|
||||
// Resume must finish the rest and produce the full output.
|
||||
if err := f.Start(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := f.Wait(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var want []byte
|
||||
for i := 0; i < total; i++ {
|
||||
want = append(want, segments[i]...)
|
||||
}
|
||||
if !bytes.Equal(readOutput(t, f), want) {
|
||||
t.Errorf("resumed output mismatch: %q", readOutput(t, f))
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetcherE2E_StoreRestoreResume(t *testing.T) {
|
||||
const total = 6
|
||||
var served atomic.Int64
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/video.m3u8", func(w http.ResponseWriter, r *http.Request) {
|
||||
var b strings.Builder
|
||||
b.WriteString("#EXTM3U\n")
|
||||
for i := 0; i < total; i++ {
|
||||
b.WriteString(fmt.Sprintf("#EXTINF:4,\nseg-%d.ts\n", i))
|
||||
}
|
||||
b.WriteString("#EXT-X-ENDLIST\n")
|
||||
w.Write([]byte(b.String()))
|
||||
})
|
||||
segments := make(map[int][]byte)
|
||||
for i := 0; i < total; i++ {
|
||||
segments[i] = []byte(fmt.Sprintf("RESTORE-SEGMENT-%02d", i))
|
||||
}
|
||||
for i := 0; i < total; i++ {
|
||||
idx := i
|
||||
mux.HandleFunc(fmt.Sprintf("/seg-%d.ts", i), func(w http.ResponseWriter, r *http.Request) {
|
||||
time.Sleep(120 * time.Millisecond)
|
||||
served.Add(1)
|
||||
w.Write(segments[idx])
|
||||
})
|
||||
}
|
||||
server := httptest.NewServer(mux)
|
||||
defer server.Close()
|
||||
|
||||
cfg := testConfig()
|
||||
cfg.SegmentConnections = 2
|
||||
cfg.PrefetchContentLength = false
|
||||
|
||||
f := newTestFetcher(t, cfg)
|
||||
mustResolve(t, f, server.URL+"/video.m3u8", "")
|
||||
if err := f.Start(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
time.Sleep(350 * time.Millisecond)
|
||||
if err := f.Pause(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
doneBefore := statsOf(t, f).SegmentsDone
|
||||
if doneBefore == 0 {
|
||||
t.Fatal("expected some segments to finish before simulated restart")
|
||||
}
|
||||
|
||||
// Simulate an app restart: persist, then restore into a fresh fetcher.
|
||||
fm := &FetcherManager{}
|
||||
state, err := fm.Store(f)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if state == nil {
|
||||
t.Fatal("state should be persisted")
|
||||
}
|
||||
ctl := controller.NewController()
|
||||
ctl.GetConfig = func(v any) {
|
||||
if c, ok := v.(*config); ok {
|
||||
*c = cfg
|
||||
}
|
||||
}
|
||||
holder := fm.Build().(*Fetcher)
|
||||
holder.Setup(ctl)
|
||||
holder.meta = &fetcher.FetcherMeta{
|
||||
Req: f.meta.Req,
|
||||
Opts: f.meta.Opts,
|
||||
Res: f.meta.Res,
|
||||
}
|
||||
_, builder := fm.Restore()
|
||||
rf := builder(holder.meta, state).(*Fetcher)
|
||||
// The engine calls Setup on the builder result (restoreFetcher).
|
||||
rf.Setup(ctl)
|
||||
if rf.state == nil || len(rf.state.Segments) != total {
|
||||
t.Fatal("restored state missing segment plan")
|
||||
}
|
||||
|
||||
if err := rf.Start(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := rf.Wait(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if done := statsOf(t, rf).SegmentsDone; done != total {
|
||||
t.Errorf("want %d done, got %d", total, done)
|
||||
}
|
||||
var want []byte
|
||||
for i := 0; i < total; i++ {
|
||||
want = append(want, segments[i]...)
|
||||
}
|
||||
if !bytes.Equal(readOutput(t, rf), want) {
|
||||
t.Errorf("restored output mismatch: %q", readOutput(t, rf))
|
||||
}
|
||||
}
|
||||
|
||||
func hexEncode(b []byte) string {
|
||||
const digits = "0123456789abcdef"
|
||||
out := make([]byte, 0, len(b)*2)
|
||||
for _, v := range b {
|
||||
out = append(out, digits[v>>4], digits[v&0x0f])
|
||||
}
|
||||
return string(out)
|
||||
}
|
||||
|
||||
// restoreLikeEngine round-trips the persisted state and the task meta through
|
||||
// JSON the way the engine storage does, then builds a fresh fetcher. This is
|
||||
// the path that historically lost the request extra headers.
|
||||
func restoreLikeEngine(t *testing.T, f *Fetcher, cfg config) *Fetcher {
|
||||
t.Helper()
|
||||
fm := &FetcherManager{}
|
||||
stateAny, err := fm.Store(f)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rawState, err := json.Marshal(stateAny)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
storedState := &fetcherState{}
|
||||
if err := json.Unmarshal(rawState, storedState); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rawMeta, err := json.Marshal(f.meta)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
restoredMeta := &fetcher.FetcherMeta{}
|
||||
if err := json.Unmarshal(rawMeta, restoredMeta); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, ok := restoredMeta.Req.Extra.(*ReqExtra); ok {
|
||||
t.Fatal("storage round trip should yield a map-shaped extra, like the engine")
|
||||
}
|
||||
ctl := controller.NewController()
|
||||
ctl.GetConfig = func(v any) {
|
||||
if c, ok := v.(*config); ok {
|
||||
*c = cfg
|
||||
}
|
||||
}
|
||||
_, builder := fm.Restore()
|
||||
rf := builder(restoredMeta, storedState).(*Fetcher)
|
||||
// The engine calls Setup on the builder result (restoreFetcher).
|
||||
rf.Setup(ctl)
|
||||
return rf
|
||||
}
|
||||
|
||||
func TestFetcherE2E_StoreRestoreAuthenticated(t *testing.T) {
|
||||
const total = 4
|
||||
page := "https://player.example.com/watch/42"
|
||||
var served atomic.Int64
|
||||
mux := http.NewServeMux()
|
||||
var playlist strings.Builder
|
||||
playlist.WriteString("#EXTM3U\n")
|
||||
for i := 0; i < total; i++ {
|
||||
playlist.WriteString(fmt.Sprintf("#EXTINF:4,\nseg-%d.ts\n", i))
|
||||
}
|
||||
playlist.WriteString("#EXT-X-ENDLIST\n")
|
||||
mux.HandleFunc("/video.m3u8", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Write([]byte(playlist.String()))
|
||||
})
|
||||
segments := make(map[int][]byte)
|
||||
for i := 0; i < total; i++ {
|
||||
segments[i] = []byte(fmt.Sprintf("AUTH-SEGMENT-%02d", i))
|
||||
idx := i
|
||||
mux.HandleFunc(fmt.Sprintf("/seg-%d.ts", idx), func(w http.ResponseWriter, r *http.Request) {
|
||||
time.Sleep(80 * time.Millisecond)
|
||||
served.Add(1)
|
||||
w.Write(segments[idx])
|
||||
})
|
||||
}
|
||||
server := httptest.NewServer(&cdnHandler{mux: mux, requireReferer: page})
|
||||
defer server.Close()
|
||||
|
||||
cfg := testConfig()
|
||||
cfg.SegmentConnections = 2
|
||||
cfg.PrefetchContentLength = false
|
||||
|
||||
f := newTestFetcher(t, cfg)
|
||||
req := &base.Request{
|
||||
URL: server.URL + "/video.m3u8",
|
||||
Extra: &ReqExtra{Header: map[string]string{"Referer": page}},
|
||||
}
|
||||
if err := f.Resolve(req, &base.Options{Path: t.TempDir()}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := f.Start(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
time.Sleep(150 * time.Millisecond)
|
||||
if err := f.Pause(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
rf := restoreLikeEngine(t, f, cfg)
|
||||
if rf.extra == nil || rf.extra.Header["Referer"] != page {
|
||||
t.Fatalf("restored fetcher lost the request extra headers: %+v", rf.extra)
|
||||
}
|
||||
if err := rf.Start(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := rf.Wait(); err != nil {
|
||||
t.Fatalf("resumed download failed (request headers lost?): %v", err)
|
||||
}
|
||||
var want []byte
|
||||
for i := 0; i < total; i++ {
|
||||
want = append(want, segments[i]...)
|
||||
}
|
||||
if !bytes.Equal(readOutput(t, rf), want) {
|
||||
t.Errorf("restored output mismatch: %q", readOutput(t, rf))
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetcherE2E_PatchPartialUpdate(t *testing.T) {
|
||||
page := "https://player.example.com/watch/7"
|
||||
var requiredReferer atomic.Value // string
|
||||
requiredReferer.Store(page)
|
||||
var playlist strings.Builder
|
||||
playlist.WriteString("#EXTM3U\n")
|
||||
segments := make(map[string][]byte)
|
||||
for i := 0; i < 3; i++ {
|
||||
segments[fmt.Sprintf("/s%d.ts", i)] = []byte(fmt.Sprintf("PATCH-%d", i))
|
||||
playlist.WriteString(fmt.Sprintf("#EXTINF:4,\ns%d.ts\n", i))
|
||||
}
|
||||
playlist.WriteString("#EXT-X-ENDLIST\n")
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/video.m3u8", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Write([]byte(playlist.String()))
|
||||
})
|
||||
for path, content := range segments {
|
||||
c := content
|
||||
mux.HandleFunc(path, func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Write(c)
|
||||
})
|
||||
}
|
||||
// The Referer requirement can be flipped mid-test to prove that a patched
|
||||
// header is actually sent by the next run.
|
||||
dyn := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if want, _ := requiredReferer.Load().(string); want != "" && r.Header.Get("Referer") != want {
|
||||
w.WriteHeader(http.StatusForbidden)
|
||||
return
|
||||
}
|
||||
mux.ServeHTTP(w, r)
|
||||
})
|
||||
server := httptest.NewServer(dyn)
|
||||
defer server.Close()
|
||||
|
||||
f := newTestFetcher(t, testConfig())
|
||||
req := &base.Request{
|
||||
URL: server.URL + "/video.m3u8",
|
||||
Extra: &ReqExtra{Header: map[string]string{"Referer": page}},
|
||||
}
|
||||
opts := &base.Options{Path: t.TempDir()}
|
||||
if err := f.Resolve(req, opts); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// A nil/nil patch must be a no-op, not a panic.
|
||||
if err := f.Patch(nil, nil); err != nil {
|
||||
t.Fatalf("nil patch should be accepted: %v", err)
|
||||
}
|
||||
|
||||
// Header-only patch: the resolved plan and output options are retained
|
||||
// while the new header takes effect on the next run.
|
||||
newReferer := "https://player.example.com/watch/9"
|
||||
if err := f.Patch(&base.Request{
|
||||
URL: server.URL + "/video.m3u8",
|
||||
Extra: &ReqExtra{Header: map[string]string{"Referer": newReferer}},
|
||||
}, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if f.state == nil || f.meta.Res == nil {
|
||||
t.Fatal("header-only patch must keep the resolved plan")
|
||||
}
|
||||
if f.meta.Opts.Path != opts.Path {
|
||||
t.Fatalf("header-only patch must keep the output path: %s", f.meta.Opts.Path)
|
||||
}
|
||||
if f.extra.Header["Referer"] != newReferer {
|
||||
t.Fatalf("patched header not applied: %+v", f.extra.Header)
|
||||
}
|
||||
requiredReferer.Store(newReferer)
|
||||
if err := f.Start(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := f.Wait(); err != nil {
|
||||
t.Fatalf("run with patched header failed: %v", err)
|
||||
}
|
||||
if got := readOutput(t, f); string(got) != "PATCH-0PATCH-1PATCH-2" {
|
||||
t.Errorf("output mismatch after header patch: %q", got)
|
||||
}
|
||||
|
||||
// Options-only patch: renaming the task keeps the resolved plan.
|
||||
if err := f.Patch(nil, &base.Options{Name: "renamed.ts"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if f.state == nil {
|
||||
t.Fatal("options-only patch must keep the resolved plan")
|
||||
}
|
||||
if f.meta.Opts.Name != "renamed.ts" || f.meta.Opts.Path != opts.Path {
|
||||
t.Fatalf("options patch misapplied: %+v", f.meta.Opts)
|
||||
}
|
||||
|
||||
// URL-only patch: the plan is dropped for a re-resolve but the output
|
||||
// path and renamed file survive.
|
||||
if err := f.Patch(&base.Request{URL: server.URL + "/refreshed.m3u8"}, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if f.state != nil || f.meta.Res != nil {
|
||||
t.Fatal("URL patch must drop the resolved plan")
|
||||
}
|
||||
if f.meta.Opts.Path != opts.Path || f.meta.Opts.Name != "renamed.ts" {
|
||||
t.Fatal("URL patch must keep the output options")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetcherE2E_PatchWhileRunningRejected(t *testing.T) {
|
||||
mux := http.NewServeMux()
|
||||
var playlist strings.Builder
|
||||
playlist.WriteString("#EXTM3U\n")
|
||||
for i := 0; i < 6; i++ {
|
||||
playlist.WriteString(fmt.Sprintf("#EXTINF:4,\nseg-%d.ts\n", i))
|
||||
}
|
||||
playlist.WriteString("#EXT-X-ENDLIST\n")
|
||||
mux.HandleFunc("/video.m3u8", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Write([]byte(playlist.String()))
|
||||
})
|
||||
for i := 0; i < 6; i++ {
|
||||
mux.HandleFunc(fmt.Sprintf("/seg-%d.ts", i), func(w http.ResponseWriter, r *http.Request) {
|
||||
time.Sleep(150 * time.Millisecond)
|
||||
w.Write([]byte("X"))
|
||||
})
|
||||
}
|
||||
server := httptest.NewServer(mux)
|
||||
defer server.Close()
|
||||
|
||||
f := newTestFetcher(t, testConfig())
|
||||
if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: t.TempDir()}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := f.Start(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := f.Patch(&base.Request{URL: server.URL + "/other.m3u8"}, nil); err == nil {
|
||||
t.Fatal("patching a running task must be rejected")
|
||||
}
|
||||
if err := f.Pause(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// After the pause the patch goes through and only then drops the plan.
|
||||
if err := f.Patch(&base.Request{URL: server.URL + "/other.m3u8"}, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if f.state != nil {
|
||||
t.Fatal("URL patch after pause must drop the plan")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetcherE2E_RetryAfterFailure(t *testing.T) {
|
||||
const total = 4
|
||||
var healthy atomic.Bool
|
||||
mux := http.NewServeMux()
|
||||
var playlist strings.Builder
|
||||
playlist.WriteString("#EXTM3U\n")
|
||||
for i := 0; i < total; i++ {
|
||||
playlist.WriteString(fmt.Sprintf("#EXTINF:4,\nseg-%d.ts\n", i))
|
||||
}
|
||||
playlist.WriteString("#EXT-X-ENDLIST\n")
|
||||
mux.HandleFunc("/video.m3u8", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Write([]byte(playlist.String()))
|
||||
})
|
||||
segments := make(map[int][]byte)
|
||||
for i := 0; i < total; i++ {
|
||||
segments[i] = []byte(fmt.Sprintf("RETRY-SEGMENT-%02d", i))
|
||||
idx := i
|
||||
mux.HandleFunc(fmt.Sprintf("/seg-%d.ts", idx), func(w http.ResponseWriter, r *http.Request) {
|
||||
if idx == 1 && !healthy.Load() {
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
time.Sleep(60 * time.Millisecond)
|
||||
w.Write(segments[idx])
|
||||
})
|
||||
}
|
||||
server := httptest.NewServer(mux)
|
||||
defer server.Close()
|
||||
|
||||
cfg := testConfig()
|
||||
cfg.MaxRetries = 0 // fail the run on the first bad segment
|
||||
cfg.PrefetchContentLength = false
|
||||
f := newTestFetcher(t, cfg)
|
||||
if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: t.TempDir()}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := f.Start(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := f.Wait(); err == nil {
|
||||
t.Fatal("first run should fail while the segment endpoint is down")
|
||||
}
|
||||
if got := statsOf(t, f).SegmentsFailed; got == 0 {
|
||||
t.Fatal("failed segments should be reported in stats")
|
||||
}
|
||||
|
||||
// Server recovered: a fresh run must succeed and report zero failures.
|
||||
healthy.Store(true)
|
||||
if err := f.Start(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := f.Wait(); err != nil {
|
||||
t.Fatalf("retry run failed: %v", err)
|
||||
}
|
||||
stats := statsOf(t, f)
|
||||
if stats.SegmentsFailed != 0 {
|
||||
t.Errorf("failures must reset per run, got %d", stats.SegmentsFailed)
|
||||
}
|
||||
if stats.SegmentsDone != total {
|
||||
t.Errorf("want %d done, got %d", total, stats.SegmentsDone)
|
||||
}
|
||||
var want []byte
|
||||
for i := 0; i < total; i++ {
|
||||
want = append(want, segments[i]...)
|
||||
}
|
||||
if !bytes.Equal(readOutput(t, f), want) {
|
||||
t.Errorf("retry output mismatch: %q", readOutput(t, f))
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetcherE2E_StateMissingSelfHeal(t *testing.T) {
|
||||
const total = 3
|
||||
mux := http.NewServeMux()
|
||||
var playlist strings.Builder
|
||||
playlist.WriteString("#EXTM3U\n")
|
||||
for i := 0; i < total; i++ {
|
||||
playlist.WriteString(fmt.Sprintf("#EXTINF:4,\nseg-%d.ts\n", i))
|
||||
}
|
||||
playlist.WriteString("#EXT-X-ENDLIST\n")
|
||||
mux.HandleFunc("/video.m3u8", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Write([]byte(playlist.String()))
|
||||
})
|
||||
segments := make(map[int][]byte)
|
||||
for i := 0; i < total; i++ {
|
||||
segments[i] = []byte(fmt.Sprintf("SELFHEAL-%02d", i))
|
||||
idx := i
|
||||
mux.HandleFunc(fmt.Sprintf("/seg-%d.ts", idx), func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Write(segments[idx])
|
||||
})
|
||||
}
|
||||
server := httptest.NewServer(mux)
|
||||
defer server.Close()
|
||||
|
||||
f := newTestFetcher(t, testConfig())
|
||||
if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: t.TempDir()}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Simulate the crash window where the engine persisted the task with its
|
||||
// resource but the HLS plan was never checkpointed.
|
||||
f.state = nil
|
||||
outputName := f.meta.Res.Files[0].Name
|
||||
if err := f.Start(); err != nil {
|
||||
t.Fatalf("start should self-heal a missing plan: %v", err)
|
||||
}
|
||||
if err := f.Wait(); err != nil {
|
||||
t.Fatalf("self-healed run failed: %v", err)
|
||||
}
|
||||
if f.state == nil || len(f.state.Segments) != total {
|
||||
t.Fatal("self-heal should rebuild the segment plan")
|
||||
}
|
||||
if f.meta.Res.Files[0].Name != outputName {
|
||||
t.Fatalf("self-heal renamed the output: %s", f.meta.Res.Files[0].Name)
|
||||
}
|
||||
var want []byte
|
||||
for i := 0; i < total; i++ {
|
||||
want = append(want, segments[i]...)
|
||||
}
|
||||
if !bytes.Equal(readOutput(t, f), want) {
|
||||
t.Errorf("self-healed output mismatch: %q", readOutput(t, f))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,100 @@
|
||||
package hls
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/cookiejar"
|
||||
"time"
|
||||
|
||||
"github.com/GopeedLab/gopeed/internal/httpclient"
|
||||
"github.com/GopeedLab/gopeed/pkg/base"
|
||||
)
|
||||
|
||||
const defaultUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36"
|
||||
|
||||
const maxPlaylistSize = 10 << 20 // 10MB, playlists are small text files
|
||||
|
||||
func (f *Fetcher) timeout() time.Duration {
|
||||
return time.Duration(f.config.TimeoutSeconds) * time.Second
|
||||
}
|
||||
|
||||
// buildClient builds an HTTP client honoring the task proxy and TLS settings.
|
||||
// The client has no total timeout on purpose: large segments over slow links
|
||||
// must not be aborted; body stalls are handled by stallReader instead. It is
|
||||
// only called while the fetcher is quiescent (Resolve/Start), never from
|
||||
// inside running worker goroutines.
|
||||
func (f *Fetcher) buildClient() *http.Client {
|
||||
var proxy *base.RequestProxy
|
||||
skipVerify := false
|
||||
if req := f.meta.Req; req != nil {
|
||||
proxy = req.Proxy
|
||||
skipVerify = req.SkipVerifyCert
|
||||
}
|
||||
jar, _ := cookiejar.New(nil)
|
||||
client, err := httpclient.NewClient(httpclient.Options{
|
||||
Client: httpclient.ClientOptions{
|
||||
Jar: jar,
|
||||
},
|
||||
Transport: httpclient.TransportOptions{
|
||||
DialContext: (&net.Dialer{
|
||||
Timeout: f.timeout(),
|
||||
}).DialContext,
|
||||
Proxy: f.Ctl.GetProxy(proxy),
|
||||
TLSClientConfig: &tls.Config{
|
||||
InsecureSkipVerify: skipVerify,
|
||||
},
|
||||
TLSHandshakeTimeout: f.timeout(),
|
||||
},
|
||||
Impersonation: httpclient.ImpersonationOptions{
|
||||
Mode: httpclient.ImpersonationAuto,
|
||||
Session: f.impSession,
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
panic(fmt.Sprintf("build HTTP client: %v", err))
|
||||
}
|
||||
return client
|
||||
}
|
||||
|
||||
// buildRequest builds a request with task extra headers, the default user
|
||||
// agent and an optional byte range. extra may be nil.
|
||||
func buildRequest(ctx context.Context, extra *ReqExtra, method, u string, body io.Reader, byterange *ByteRange) (*http.Request, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, method, u, body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if extra != nil {
|
||||
for key, value := range extra.Header {
|
||||
req.Header.Set(key, value)
|
||||
}
|
||||
}
|
||||
if req.Header.Get("User-Agent") == "" {
|
||||
req.Header.Set("User-Agent", defaultUserAgent)
|
||||
}
|
||||
if byterange != nil {
|
||||
req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", byterange.Offset, byterange.Offset+byterange.Length-1))
|
||||
}
|
||||
return req, nil
|
||||
}
|
||||
|
||||
// stallReader aborts reads that see no data for the configured timeout.
|
||||
type stallReader struct {
|
||||
rc io.ReadCloser
|
||||
timeout time.Duration
|
||||
}
|
||||
|
||||
func (r *stallReader) Read(p []byte) (int, error) {
|
||||
timer := time.AfterFunc(r.timeout, func() {
|
||||
r.rc.Close()
|
||||
})
|
||||
defer timer.Stop()
|
||||
return r.rc.Read(p)
|
||||
}
|
||||
|
||||
func (r *stallReader) Close() error {
|
||||
return r.rc.Close()
|
||||
}
|
||||
@@ -0,0 +1,112 @@
|
||||
package hls
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"crypto/sha1"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"strconv"
|
||||
)
|
||||
|
||||
// journalFile is the persisted completed-segment ledger. It is bound to the
|
||||
// exact download plan: a journal whose fingerprint does not match the current
|
||||
// plan is discarded instead of being applied to the wrong segments.
|
||||
type journalFile struct {
|
||||
StagingID string `json:"stagingID"`
|
||||
PlanHash string `json:"planHash"`
|
||||
SegmentCount int `json:"segmentCount"`
|
||||
Sizes map[string]int64 `json:"sizes"`
|
||||
}
|
||||
|
||||
// randomStagingID returns a random identifier used to give every task its own
|
||||
// staging folder, stable across app restarts through the persisted state.
|
||||
func randomStagingID() string {
|
||||
buf := make([]byte, 6)
|
||||
if _, err := rand.Read(buf); err != nil {
|
||||
// crypto/rand failing is catastrophic anyway; fall back to address
|
||||
// entropy rather than crashing the fetch.
|
||||
return fmt.Sprintf("%x", sha1.Sum([]byte(fmt.Sprintf("%p", buf))))[:12]
|
||||
}
|
||||
return hex.EncodeToString(buf)
|
||||
}
|
||||
|
||||
// planFingerprint hashes everything that defines the download plan: the media
|
||||
// URL, the ordered segments with their sequence numbers, byte ranges, keys
|
||||
// and init markers. Two tasks resolve to different fingerprints even when
|
||||
// they point at the same playlist URL.
|
||||
func planFingerprint(state *fetcherState) string {
|
||||
h := sha1.New()
|
||||
io.WriteString(h, state.MediaURL+"\n")
|
||||
io.WriteString(h, strconv.Itoa(len(state.Segments))+"\n")
|
||||
for _, seg := range state.Segments {
|
||||
fmt.Fprintf(h, "%d|%s|", seg.Sequence, seg.URI)
|
||||
if seg.Byterange != nil {
|
||||
fmt.Fprintf(h, "%d@%d|", seg.Byterange.Offset, seg.Byterange.Length)
|
||||
}
|
||||
if seg.Key != nil {
|
||||
// The IV is part of the plan: same URI with a rotated IV would
|
||||
// silently decrypt to garbage if the journal were reused.
|
||||
fmt.Fprintf(h, "key=%s iv=%x|", seg.Key.URI, seg.Key.IV)
|
||||
}
|
||||
if seg.IsInit {
|
||||
io.WriteString(h, "init|")
|
||||
}
|
||||
io.WriteString(h, "\n")
|
||||
}
|
||||
return hex.EncodeToString(h.Sum(nil))[:16]
|
||||
}
|
||||
|
||||
// loadJournal reads the journal of a run, discarding anything that cannot be
|
||||
// proven to belong to the current plan: foreign or legacy journals, corrupted
|
||||
// files, and entries whose recorded size no longer matches the plan.
|
||||
func loadJournal(run *hlsRun) error {
|
||||
data, err := os.ReadFile(run.journalPath)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
var file journalFile
|
||||
if err := json.Unmarshal(data, &file); err != nil {
|
||||
// A broken or legacy journal must not kill a restartable download.
|
||||
return nil
|
||||
}
|
||||
if file.StagingID != run.stagingID || file.PlanHash != run.planHash || file.SegmentCount != len(run.segs) {
|
||||
return nil
|
||||
}
|
||||
for key, size := range file.Sizes {
|
||||
idx, err := strconv.ParseInt(key, 10, 64)
|
||||
if err != nil || idx < 0 || idx >= int64(len(run.segs)) || size <= 0 {
|
||||
continue
|
||||
}
|
||||
run.journal[idx] = size
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// saveJournal atomically rewrites the journal through a temp file + rename so
|
||||
// a crash never leaves a half-written ledger behind.
|
||||
func saveJournal(run *hlsRun) error {
|
||||
file := journalFile{
|
||||
StagingID: run.stagingID,
|
||||
PlanHash: run.planHash,
|
||||
SegmentCount: len(run.segs),
|
||||
Sizes: make(map[string]int64, len(run.journal)),
|
||||
}
|
||||
for idx, size := range run.journal {
|
||||
file.Sizes[strconv.FormatInt(idx, 10)] = size
|
||||
}
|
||||
data, err := json.Marshal(file)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tmp := run.journalPath + ".tmp"
|
||||
if err := os.WriteFile(tmp, data, 0666); err != nil {
|
||||
return err
|
||||
}
|
||||
return os.Rename(tmp, run.journalPath)
|
||||
}
|
||||
@@ -0,0 +1,297 @@
|
||||
package hls
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/GopeedLab/gopeed/pkg/base"
|
||||
)
|
||||
|
||||
// The validators below check real media structure, not just byte
|
||||
// concatenation: TS packets are verified on the 188 byte grid with
|
||||
// continuity-counter rules, PAT/PMT presence and PES start codes; fMP4
|
||||
// output is verified by walking the ISO-BMFF box hierarchy.
|
||||
|
||||
func be32(b []byte) int {
|
||||
return int(b[0])<<24 | int(b[1])<<16 | int(b[2])<<8 | int(b[3])
|
||||
}
|
||||
|
||||
// validateTS asserts structural validity of a merged MPEG-2 TS stream.
|
||||
func validateTS(t *testing.T, data []byte) {
|
||||
t.Helper()
|
||||
if len(data) == 0 || len(data)%188 != 0 {
|
||||
t.Fatalf("TS size %d is not a positive multiple of 188", len(data))
|
||||
}
|
||||
type pidState struct{ cc int }
|
||||
ccByPID := map[int]int{}
|
||||
patSeen, pmtSeen, pesSeen := false, false, false
|
||||
for off := 0; off < len(data); off += 188 {
|
||||
pkt := data[off : off+188]
|
||||
if pkt[0] != 0x47 {
|
||||
t.Fatalf("bad TS sync byte at offset %d", off)
|
||||
}
|
||||
pid := (int(pkt[1]&0x1F) << 8) | int(pkt[2])
|
||||
pusi := pkt[1]&0x40 != 0
|
||||
afc := (pkt[3] >> 4) & 0x3
|
||||
if afc == 0 {
|
||||
continue // reserved
|
||||
}
|
||||
idx := 4
|
||||
if afc&0x2 != 0 { // adaptation field present
|
||||
afLen := int(pkt[4])
|
||||
if afLen > 0 && pkt[5]&0x80 != 0 {
|
||||
// discontinuity_indicator: restart the CC expectation
|
||||
delete(ccByPID, pid)
|
||||
}
|
||||
idx += 1 + afLen
|
||||
if afc == 2 {
|
||||
continue // adaptation only, no payload: no CC rule
|
||||
}
|
||||
}
|
||||
if idx > 188 {
|
||||
t.Fatalf("adaptation field overruns packet at offset %d", off)
|
||||
}
|
||||
if pid == 0x1FFF {
|
||||
continue // null packet
|
||||
}
|
||||
payload := pkt[idx:]
|
||||
if pusi && len(payload) >= 3 && pid != 0 && payload[0] == 0 && payload[1] == 0 && payload[2] == 1 {
|
||||
pesSeen = true
|
||||
}
|
||||
if pid == 0 {
|
||||
if ptr := int(payload[0]); 1+ptr < len(payload) && payload[1+ptr] == 0x00 {
|
||||
patSeen = true // program_association_section table_id
|
||||
}
|
||||
} else if pusi && len(payload) >= 2 && payload[0] == 0 && payload[1] == 0x02 {
|
||||
pmtSeen = true // program_map_section table_id
|
||||
}
|
||||
cc := int(pkt[3] & 0x0F)
|
||||
prev, ok := ccByPID[pid]
|
||||
if !ok {
|
||||
ccByPID[pid] = cc
|
||||
continue
|
||||
}
|
||||
// CC increments by one per payload-bearing packet; a repeated value
|
||||
// models the single legal duplicate packet.
|
||||
if cc != (prev+1)%16 && cc != prev {
|
||||
t.Fatalf("continuity error on PID %d: %d after %d at offset %d", pid, cc, prev, off)
|
||||
}
|
||||
ccByPID[pid] = cc
|
||||
}
|
||||
if !patSeen {
|
||||
t.Error("no PAT found in merged TS")
|
||||
}
|
||||
if !pmtSeen {
|
||||
t.Error("no PMT found in merged TS")
|
||||
}
|
||||
if !pesSeen {
|
||||
t.Error("no PES start code found in merged TS")
|
||||
}
|
||||
}
|
||||
|
||||
// walkBoxes iterates the top-level ISO-BMFF boxes of data[off:end].
|
||||
func walkBoxes(t *testing.T, data []byte, off, end int) []string {
|
||||
t.Helper()
|
||||
var types []string
|
||||
for off+8 <= end {
|
||||
size := be32(data[off:])
|
||||
typ := string(data[off+4 : off+8])
|
||||
if size < 8 || off+size > end {
|
||||
t.Fatalf("invalid box %q at offset %d (size %d)", typ, off, size)
|
||||
}
|
||||
types = append(types, typ)
|
||||
off += size
|
||||
}
|
||||
return types
|
||||
}
|
||||
|
||||
// validateFMP4 asserts the merged fMP4 output is init section followed by
|
||||
// well-formed moof+mdat fragments.
|
||||
func validateFMP4(t *testing.T, merged []byte, wantFragments int) {
|
||||
t.Helper()
|
||||
types := walkBoxes(t, merged, 0, len(merged))
|
||||
hasFtyp, hasMoov := false, false
|
||||
fragments := 0
|
||||
expectFragment := false
|
||||
for i, typ := range types {
|
||||
switch typ {
|
||||
case "ftyp":
|
||||
if i != 0 {
|
||||
t.Errorf("ftyp not at stream start")
|
||||
}
|
||||
hasFtyp = true
|
||||
case "moov":
|
||||
if i != 1 {
|
||||
t.Errorf("moov must follow ftyp, found at index %d", i)
|
||||
}
|
||||
hasMoov = true
|
||||
case "styp":
|
||||
expectFragment = true
|
||||
case "moof":
|
||||
if !hasMoov {
|
||||
t.Errorf("fragment before moov")
|
||||
}
|
||||
expectFragment = true
|
||||
case "mdat":
|
||||
if !expectFragment {
|
||||
t.Errorf("mdat outside of a fragment at index %d", i)
|
||||
}
|
||||
expectFragment = false
|
||||
fragments++
|
||||
case "mfra":
|
||||
// optional trailer
|
||||
default:
|
||||
t.Errorf("unexpected top-level box %q", typ)
|
||||
}
|
||||
}
|
||||
if !hasFtyp || !hasMoov {
|
||||
t.Error("merged fMP4 missing ftyp/moov init section")
|
||||
}
|
||||
if fragments != wantFragments {
|
||||
t.Errorf("want %d fragments in merged fMP4, got %d", wantFragments, fragments)
|
||||
}
|
||||
}
|
||||
|
||||
func mediaFixture(t *testing.T, name string) []byte {
|
||||
t.Helper()
|
||||
data, err := os.ReadFile("testdata/media/" + name)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return data
|
||||
}
|
||||
|
||||
func serveFixtureServer(t *testing.T) *httptest.Server {
|
||||
t.Helper()
|
||||
mux := http.NewServeMux()
|
||||
fs := http.FileServer(http.Dir("testdata/media"))
|
||||
mux.Handle("/", fs)
|
||||
return httptest.NewServer(mux)
|
||||
}
|
||||
|
||||
func TestHLSRealMedia_TS(t *testing.T) {
|
||||
seg0 := mediaFixture(t, "sample-00.ts")
|
||||
seg1 := mediaFixture(t, "sample-01.ts")
|
||||
playlist := "#EXTM3U\n" +
|
||||
"#EXTINF:0.5,\nsample-00.ts\n" +
|
||||
"#EXTINF:0.5,\nsample-01.ts\n" +
|
||||
"#EXT-X-ENDLIST\n"
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/video.m3u8", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Write([]byte(playlist))
|
||||
})
|
||||
mux.Handle("/", http.FileServer(http.Dir("testdata/media")))
|
||||
server := httptest.NewServer(mux)
|
||||
defer server.Close()
|
||||
|
||||
f := newTestFetcher(t, testConfig())
|
||||
if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: t.TempDir()}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := f.Start(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := f.Wait(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := readOutput(t, f)
|
||||
if !bytes.Equal(got, append(append([]byte{}, seg0...), seg1...)) {
|
||||
t.Fatal("merged TS is not the exact concatenation of the segments")
|
||||
}
|
||||
validateTS(t, got)
|
||||
}
|
||||
|
||||
func TestHLSRealMedia_FMP4(t *testing.T) {
|
||||
init := mediaFixture(t, "fmp4-init.mp4")
|
||||
frag1 := mediaFixture(t, "fmp4-seg1.m4s")
|
||||
frag2 := mediaFixture(t, "fmp4-seg2.m4s")
|
||||
frag3 := mediaFixture(t, "fmp4-seg3.m4s")
|
||||
playlist := "#EXTM3U\n" +
|
||||
"#EXT-X-MAP:URI=\"fmp4-init.mp4\"\n" +
|
||||
"#EXTINF:0.5,\nfmp4-seg1.m4s\n" +
|
||||
"#EXTINF:0.5,\nfmp4-seg2.m4s\n" +
|
||||
"#EXTINF:0.5,\nfmp4-seg3.m4s\n" +
|
||||
"#EXT-X-ENDLIST\n"
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/video.m3u8", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Write([]byte(playlist))
|
||||
})
|
||||
mux.Handle("/", http.FileServer(http.Dir("testdata/media")))
|
||||
server := httptest.NewServer(mux)
|
||||
defer server.Close()
|
||||
|
||||
f := newTestFetcher(t, testConfig())
|
||||
if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: t.TempDir()}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := f.Start(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := f.Wait(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := readOutput(t, f)
|
||||
want := append(append(append(append([]byte{}, init...), frag1...), frag2...), frag3...)
|
||||
if !bytes.Equal(got, want) {
|
||||
t.Fatal("merged fMP4 is not the exact concatenation of init + fragments")
|
||||
}
|
||||
validateFMP4(t, got, 3)
|
||||
}
|
||||
|
||||
func TestHLSRealMedia_AES128(t *testing.T) {
|
||||
keys := [][]byte{[]byte("key-A-0123456789"), []byte("key-B-9876543210")}
|
||||
iv := []byte("iv-1337-01234567")
|
||||
plain := [][]byte{mediaFixture(t, "sample-00.ts"), mediaFixture(t, "sample-01.ts")}
|
||||
|
||||
playlist := "#EXTM3U\n" +
|
||||
"#EXT-X-MEDIA-SEQUENCE:0\n" +
|
||||
"#EXT-X-KEY:METHOD=AES-128,URI=\"key-0.bin\",IV=0x" + hexEncode(iv) + "\n" +
|
||||
"#EXTINF:0.5,\nenc-0.ts\n" +
|
||||
"#EXT-X-KEY:METHOD=AES-128,URI=\"key-1.bin\"\n" +
|
||||
"#EXTINF:0.5,\nenc-1.ts\n" +
|
||||
"#EXT-X-ENDLIST\n"
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/video.m3u8", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Write([]byte(playlist))
|
||||
})
|
||||
for i, key := range keys {
|
||||
k := key
|
||||
mux.HandleFunc(fmt.Sprintf("/key-%d.bin", i), func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Write(k)
|
||||
})
|
||||
// Segment 0 has an explicit IV, segment 1 derives it from its media
|
||||
// sequence number (1).
|
||||
enc := encryptSegment(plain[i], k, map[bool][]byte{true: iv, false: sequenceIV(int64(i))}[i == 0])
|
||||
mux.HandleFunc(fmt.Sprintf("/enc-%d.ts", i), func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Write(enc)
|
||||
})
|
||||
}
|
||||
server := httptest.NewServer(mux)
|
||||
defer server.Close()
|
||||
|
||||
f := newTestFetcher(t, testConfig())
|
||||
if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: t.TempDir()}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if f.meta.Res.Size != 0 {
|
||||
t.Errorf("encrypted plan must report indeterminate size, got %d", f.meta.Res.Size)
|
||||
}
|
||||
if err := f.Start(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := f.Wait(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := readOutput(t, f)
|
||||
if !bytes.Equal(got, append(append([]byte{}, plain[0]...), plain[1]...)) {
|
||||
t.Fatal("decrypted output mismatch")
|
||||
}
|
||||
validateTS(t, got)
|
||||
if p := f.Progress(); p.TotalDownloaded() != int64(len(got)) {
|
||||
t.Errorf("final progress %d != merged size %d", p.TotalDownloaded(), len(got))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,479 @@
|
||||
package hls
|
||||
|
||||
import (
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"path"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Variant is one rendition entry of a master playlist.
|
||||
type Variant struct {
|
||||
URI string
|
||||
Bandwidth int64
|
||||
AvgBandwidth int64
|
||||
Resolution string
|
||||
Codecs string
|
||||
Name string
|
||||
AudioGroup string // AUDIO attribute: group id of the audio renditions
|
||||
}
|
||||
|
||||
// MediaRendition is one EXT-X-MEDIA entry of a master playlist. Only the
|
||||
// fields needed to detect unsupported track layouts are kept.
|
||||
type MediaRendition struct {
|
||||
Type string
|
||||
GroupID string
|
||||
URI string
|
||||
}
|
||||
|
||||
// MasterPlaylist is a parsed master playlist with its rendition groups.
|
||||
type MasterPlaylist struct {
|
||||
Variants []*Variant
|
||||
Rendition map[string][]*MediaRendition // group id -> all EXT-X-MEDIA members
|
||||
}
|
||||
|
||||
// Key is the decryption key applied to the segments that follow its tag, until
|
||||
// the next EXT-X-KEY tag or the end of the playlist.
|
||||
type Key struct {
|
||||
Method string
|
||||
URI string // absolute URI
|
||||
IV []byte // explicit IV; nil means derive from the media sequence number
|
||||
}
|
||||
|
||||
// ByteRange is an EXT-X-BYTERANGE sub range of the segment resource.
|
||||
type ByteRange struct {
|
||||
Length int64
|
||||
Offset int64
|
||||
}
|
||||
|
||||
// Segment is one downloadable piece of a media playlist. The init section of
|
||||
// an fMP4 stream is represented as a segment with IsInit set.
|
||||
type Segment struct {
|
||||
URI string
|
||||
Sequence int64 // media sequence number, used to derive the default AES IV
|
||||
Byterange *ByteRange
|
||||
Key *Key // nil means unencrypted
|
||||
IsInit bool
|
||||
Gap bool
|
||||
Duration float64
|
||||
Size int64 // exact size in bytes when known (byterange / HEAD probe), 0 when unknown
|
||||
}
|
||||
|
||||
// Media is a parsed media playlist.
|
||||
type Media struct {
|
||||
Segments []*Segment
|
||||
Version int
|
||||
Live bool // no EXT-X-ENDLIST tag
|
||||
IsFMP4 bool // playlist declares an init section via EXT-X-MAP
|
||||
Discontinuity bool // playlist contains EXT-X-DISCONTINUITY
|
||||
}
|
||||
|
||||
const (
|
||||
keyMethodNone = "NONE"
|
||||
keyMethodAES128 = "AES-128"
|
||||
keyMethodSampleAES = "SAMPLE-AES"
|
||||
)
|
||||
|
||||
// IsMasterPlaylist reports whether the content looks like a master playlist.
|
||||
func IsMasterPlaylist(content string) bool {
|
||||
return strings.Contains(content, "#EXT-X-STREAM-INF")
|
||||
}
|
||||
|
||||
func validateM3U8Header(content string) error {
|
||||
for _, line := range strings.Split(content, "\n") {
|
||||
line = strings.TrimSpace(line)
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
if line != "#EXTM3U" {
|
||||
return errors.New("content is not a valid m3u8 playlist")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
return errors.New("empty m3u8 playlist")
|
||||
}
|
||||
|
||||
// ParseMaster parses a master playlist and returns all variants.
|
||||
func ParseMaster(content string, baseURL *url.URL) ([]*Variant, error) {
|
||||
master, err := ParseMasterPlaylist(content, baseURL)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return master.Variants, nil
|
||||
}
|
||||
|
||||
// ParseMasterPlaylist parses a master playlist including its EXT-X-MEDIA
|
||||
// rendition groups.
|
||||
func ParseMasterPlaylist(content string, baseURL *url.URL) (*MasterPlaylist, error) {
|
||||
if err := validateM3U8Header(content); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
master := &MasterPlaylist{Rendition: make(map[string][]*MediaRendition)}
|
||||
var pending *Variant
|
||||
for _, line := range strings.Split(content, "\n") {
|
||||
line = strings.TrimSpace(line)
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
if strings.HasPrefix(line, "#EXT-X-STREAM-INF:") {
|
||||
attrs := parseAttributes(line[len("#EXT-X-STREAM-INF:"):])
|
||||
pending = &Variant{
|
||||
Bandwidth: parseInt64(attrs["BANDWIDTH"]),
|
||||
AvgBandwidth: parseInt64(attrs["AVERAGE-BANDWIDTH"]),
|
||||
Resolution: attrs["RESOLUTION"],
|
||||
Codecs: unquote(attrs["CODECS"]),
|
||||
Name: unquote(attrs["NAME"]),
|
||||
AudioGroup: unquote(attrs["AUDIO"]),
|
||||
}
|
||||
continue
|
||||
}
|
||||
if strings.HasPrefix(line, "#EXT-X-MEDIA:") {
|
||||
attrs := parseAttributes(line[len("#EXT-X-MEDIA:"):])
|
||||
rendition := &MediaRendition{
|
||||
Type: strings.ToUpper(attrs["TYPE"]),
|
||||
GroupID: unquote(attrs["GROUP-ID"]),
|
||||
}
|
||||
// Only an explicit URI means a separate track; resolving an empty
|
||||
// reference would fabricate one out of the playlist URL.
|
||||
if uri := unquote(attrs["URI"]); uri != "" {
|
||||
rendition.URI = resolveURL(baseURL, uri)
|
||||
}
|
||||
if rendition.GroupID != "" {
|
||||
// Keep every member: a group is unsupported as a whole when
|
||||
// ANY of its members carries its own URI.
|
||||
master.Rendition[rendition.GroupID] = append(master.Rendition[rendition.GroupID], rendition)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if pending != nil {
|
||||
if line[0] != '#' {
|
||||
pending.URI = resolveURL(baseURL, line)
|
||||
master.Variants = append(master.Variants, pending)
|
||||
}
|
||||
pending = nil
|
||||
}
|
||||
}
|
||||
if len(master.Variants) == 0 {
|
||||
return nil, errors.New("master playlist has no variants")
|
||||
}
|
||||
return master, nil
|
||||
}
|
||||
|
||||
// checkAudioRendition rejects the selected variant when its AUDIO group
|
||||
// carries a separate audio track: downloading only the video variant would
|
||||
// silently produce a silent file. Groups the variant does not reference are
|
||||
// ignored (variants without an AUDIO attribute are assumed self-contained).
|
||||
func checkAudioRendition(master *MasterPlaylist, best *Variant) error {
|
||||
if best == nil || best.AudioGroup == "" {
|
||||
return nil
|
||||
}
|
||||
for _, rendition := range master.Rendition[best.AudioGroup] {
|
||||
if rendition == nil || rendition.Type != "AUDIO" || rendition.URI == "" {
|
||||
continue
|
||||
}
|
||||
return fmt.Errorf(
|
||||
"selected variant uses a separate audio rendition (group %q), which is not supported: the merged output would have no audio",
|
||||
best.AudioGroup)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// PickBestVariant selects the highest-bandwidth variant.
|
||||
func PickBestVariant(variants []*Variant) *Variant {
|
||||
var best *Variant
|
||||
for _, v := range variants {
|
||||
if best == nil || v.Bandwidth > best.Bandwidth {
|
||||
best = v
|
||||
}
|
||||
}
|
||||
return best
|
||||
}
|
||||
|
||||
// ParseMedia parses a media playlist into an ordered segment plan.
|
||||
func ParseMedia(content string, baseURL *url.URL) (*Media, error) {
|
||||
if err := validateM3U8Header(content); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
media := &Media{Live: true}
|
||||
var currentKey *Key
|
||||
var pendingByterange *ByteRange
|
||||
var pendingGap bool
|
||||
var pendingDuration float64
|
||||
var lastRangeEnd int64
|
||||
var sequence int64
|
||||
var mapIdentity string // normalized identity of the current EXT-X-MAP
|
||||
|
||||
for _, line := range strings.Split(content, "\n") {
|
||||
line = strings.TrimSpace(line)
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
if line[0] != '#' {
|
||||
// Every non-comment line in a media playlist is a segment URI.
|
||||
seg := &Segment{
|
||||
URI: resolveURL(baseURL, line),
|
||||
Sequence: sequence,
|
||||
Byterange: pendingByterange,
|
||||
Key: currentKey,
|
||||
Gap: pendingGap,
|
||||
Duration: pendingDuration,
|
||||
}
|
||||
if pendingByterange != nil {
|
||||
// Byterange segments have exact sizes straight from the playlist.
|
||||
seg.Size = pendingByterange.Length
|
||||
}
|
||||
sequence++
|
||||
pendingByterange = nil
|
||||
pendingGap = false
|
||||
pendingDuration = 0
|
||||
media.Segments = append(media.Segments, seg)
|
||||
continue
|
||||
}
|
||||
|
||||
switch {
|
||||
case strings.HasPrefix(line, "#EXTINF:"):
|
||||
value := line[len("#EXTINF:"):]
|
||||
if idx := strings.Index(value, ","); idx >= 0 {
|
||||
value = value[:idx]
|
||||
}
|
||||
pendingDuration = parseFloat(value)
|
||||
case strings.HasPrefix(line, "#EXT-X-KEY:"):
|
||||
key, err := parseKey(line[len("#EXT-X-KEY:"):], baseURL)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
currentKey = key
|
||||
case strings.HasPrefix(line, "#EXT-X-MAP:"):
|
||||
attrs := parseAttributes(line[len("#EXT-X-MAP:"):])
|
||||
uriAttr := unquote(attrs["URI"])
|
||||
if uriAttr == "" {
|
||||
return nil, errors.New("EXT-X-MAP without URI is not supported")
|
||||
}
|
||||
resolved := resolveURL(baseURL, uriAttr)
|
||||
// The init section identity covers every attribute, not just the
|
||||
// URI: a repeated tag with a different BYTERANGE addresses
|
||||
// different bytes and changes the init section. NUL keeps the
|
||||
// two parts unambiguously separated.
|
||||
identity := resolved + "\x00" + attrs["BYTERANGE"]
|
||||
if mapIdentity != "" {
|
||||
if mapIdentity != identity {
|
||||
return nil, errors.New("changing init sections are not supported")
|
||||
}
|
||||
media.IsFMP4 = true
|
||||
continue
|
||||
}
|
||||
mapIdentity = identity
|
||||
init := &Segment{
|
||||
URI: resolved,
|
||||
Sequence: sequence,
|
||||
IsInit: true,
|
||||
Key: currentKey,
|
||||
}
|
||||
if attrs["BYTERANGE"] != "" {
|
||||
init.Byterange = parseByteRange(attrs["BYTERANGE"], 0)
|
||||
if init.Byterange == nil {
|
||||
return nil, fmt.Errorf("invalid EXT-X-MAP BYTERANGE: %s", attrs["BYTERANGE"])
|
||||
}
|
||||
}
|
||||
if init.Key != nil && init.Key.IV == nil {
|
||||
// RFC 8216 only defines sequence-derived IVs for media
|
||||
// segments; guessing one for the init section produces
|
||||
// corrupt output, so require the playlist to be explicit.
|
||||
return nil, errors.New("encrypted EXT-X-MAP without an explicit IV is not supported")
|
||||
}
|
||||
// The init section is prepended to the segment plan, it takes
|
||||
// the media sequence number of the playlist for IV derivation.
|
||||
media.Segments = append(media.Segments, init)
|
||||
media.IsFMP4 = true
|
||||
case strings.HasPrefix(line, "#EXT-X-BYTERANGE:"):
|
||||
pendingByterange = parseByteRange(line[len("#EXT-X-BYTERANGE:"):], lastRangeEnd)
|
||||
if pendingByterange == nil {
|
||||
return nil, fmt.Errorf("invalid EXT-X-BYTERANGE: %s", line)
|
||||
}
|
||||
case line == "#EXT-X-GAP":
|
||||
pendingGap = true
|
||||
case line == "#EXT-X-DISCONTINUITY":
|
||||
media.Discontinuity = true
|
||||
case line == "#EXT-X-ENDLIST":
|
||||
media.Live = false
|
||||
case strings.HasPrefix(line, "#EXT-X-MEDIA-SEQUENCE:"):
|
||||
sequence = parseInt64(line[len("#EXT-X-MEDIA-SEQUENCE:"):])
|
||||
case strings.HasPrefix(line, "#EXT-X-VERSION:"):
|
||||
media.Version = int(parseInt64(line[len("#EXT-X-VERSION:"):]))
|
||||
}
|
||||
|
||||
if pendingByterange != nil {
|
||||
// Non-segment tags between BYTERANGE and its URI must not reset the
|
||||
// running offset, so it is tracked here after each parse.
|
||||
lastRangeEnd = pendingByterange.Offset + pendingByterange.Length
|
||||
}
|
||||
}
|
||||
|
||||
if len(media.Segments) == 0 {
|
||||
return nil, errors.New("media playlist has no segments")
|
||||
}
|
||||
return media, nil
|
||||
}
|
||||
|
||||
// validateSupported rejects playlist features the byte-concatenation merge
|
||||
// cannot reproduce faithfully. Skipping such features silently would report
|
||||
// success while producing a broken file.
|
||||
func validateSupported(media *Media) error {
|
||||
if media.Live {
|
||||
return errors.New("live streams are not supported yet")
|
||||
}
|
||||
if media.Discontinuity {
|
||||
return errors.New("EXT-X-DISCONTINUITY is not supported: the merged output would have a broken timeline")
|
||||
}
|
||||
for _, seg := range media.Segments {
|
||||
if seg.Gap {
|
||||
return errors.New("EXT-X-GAP is not supported: the missing segment would leave a hole in the merged output")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func parseKey(attrs string, baseURL *url.URL) (*Key, error) {
|
||||
parsed := parseAttributes(attrs)
|
||||
switch method := parsed["METHOD"]; method {
|
||||
case keyMethodNone:
|
||||
return nil, nil
|
||||
case keyMethodAES128:
|
||||
uri := unquote(parsed["URI"])
|
||||
if uri == "" {
|
||||
return nil, errors.New("AES-128 EXT-X-KEY without URI")
|
||||
}
|
||||
if format := unquote(parsed["KEYFORMAT"]); format != "" && format != "identity" {
|
||||
return nil, fmt.Errorf("unsupported EXT-X-KEY KEYFORMAT: %s", format)
|
||||
}
|
||||
key := &Key{Method: keyMethodAES128, URI: resolveURL(baseURL, uri)}
|
||||
if iv, ok := parsed["IV"]; ok {
|
||||
value := strings.TrimPrefix(strings.ToLower(iv), "0x")
|
||||
decoded, err := hex.DecodeString(value)
|
||||
if err != nil || len(decoded) != 16 {
|
||||
return nil, fmt.Errorf("invalid EXT-X-KEY IV: %s", iv)
|
||||
}
|
||||
key.IV = decoded
|
||||
}
|
||||
return key, nil
|
||||
case keyMethodSampleAES:
|
||||
return nil, errors.New("SAMPLE-AES encryption is not supported")
|
||||
default:
|
||||
return nil, fmt.Errorf("unsupported encryption method: %s", method)
|
||||
}
|
||||
}
|
||||
|
||||
func parseByteRange(value string, defaultOffset int64) *ByteRange {
|
||||
lengthPart := value
|
||||
offsetPart := ""
|
||||
if idx := strings.Index(value, "@"); idx >= 0 {
|
||||
lengthPart = value[:idx]
|
||||
offsetPart = value[idx+1:]
|
||||
}
|
||||
length := parseInt64(lengthPart)
|
||||
if length <= 0 {
|
||||
return nil
|
||||
}
|
||||
offset := defaultOffset
|
||||
if offsetPart != "" {
|
||||
offset = parseInt64(offsetPart)
|
||||
if offset < 0 {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return &ByteRange{Length: length, Offset: offset}
|
||||
}
|
||||
|
||||
// resolveURL resolves a playlist URI reference against the playlist URL
|
||||
// following RFC 3986.
|
||||
func resolveURL(baseURL *url.URL, ref string) string {
|
||||
refURL, err := url.Parse(ref)
|
||||
if err != nil {
|
||||
return ref
|
||||
}
|
||||
return baseURL.ResolveReference(refURL).String()
|
||||
}
|
||||
|
||||
// parseAttributes parses an HLS attribute list like FOO="bar",NUM=42.
|
||||
func parseAttributes(s string) map[string]string {
|
||||
attrs := make(map[string]string)
|
||||
var key, val strings.Builder
|
||||
inKey := true
|
||||
inQuote := false
|
||||
flush := func() {
|
||||
name := strings.ToUpper(strings.TrimSpace(key.String()))
|
||||
if name != "" {
|
||||
attrs[name] = strings.TrimSpace(val.String())
|
||||
}
|
||||
key.Reset()
|
||||
val.Reset()
|
||||
inKey = true
|
||||
}
|
||||
for i := 0; i < len(s); i++ {
|
||||
c := s[i]
|
||||
if inKey {
|
||||
if c == '=' {
|
||||
inKey = false
|
||||
} else {
|
||||
key.WriteByte(c)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if c == '"' {
|
||||
inQuote = !inQuote
|
||||
continue
|
||||
}
|
||||
if c == ',' && !inQuote {
|
||||
flush()
|
||||
continue
|
||||
}
|
||||
val.WriteByte(c)
|
||||
}
|
||||
flush()
|
||||
return attrs
|
||||
}
|
||||
|
||||
func unquote(s string) string {
|
||||
s = strings.TrimSpace(s)
|
||||
if len(s) >= 2 && strings.HasPrefix(s, "\"") && strings.HasSuffix(s, "\"") {
|
||||
return s[1 : len(s)-1]
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func parseInt64(s string) int64 {
|
||||
value, _ := strconv.ParseInt(strings.TrimSpace(s), 10, 64)
|
||||
return value
|
||||
}
|
||||
|
||||
func parseFloat(s string) float64 {
|
||||
value, _ := strconv.ParseFloat(strings.TrimSpace(s), 64)
|
||||
return value
|
||||
}
|
||||
|
||||
// OutputExtension returns the container file extension for the playlist.
|
||||
func (m *Media) OutputExtension() string {
|
||||
if m.IsFMP4 {
|
||||
return ".mp4"
|
||||
}
|
||||
return ".ts"
|
||||
}
|
||||
|
||||
// DeriveOutputName builds the merged output file name from a playlist URL.
|
||||
func DeriveOutputName(playlistURL string, media *Media) string {
|
||||
name := ""
|
||||
if u, err := url.Parse(playlistURL); err == nil {
|
||||
name = path.Base(u.Path)
|
||||
name = strings.TrimSuffix(name, path.Ext(name))
|
||||
}
|
||||
if name == "" || name == "/" || name == "." {
|
||||
if u, err := url.Parse(playlistURL); err == nil {
|
||||
name = u.Hostname()
|
||||
}
|
||||
}
|
||||
return name + media.OutputExtension()
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
package hls
|
||||
|
||||
import (
|
||||
_ "embed"
|
||||
"net/url"
|
||||
"testing"
|
||||
)
|
||||
|
||||
//go:embed testdata/olemovienews.m3u8
|
||||
var realWorldPlaylist string
|
||||
|
||||
// TestParseMediaRealWorld guards parsing against a real site playlist
|
||||
// (olemovienews.com VOD, 138 TS segments).
|
||||
func TestParseMediaRealWorld(t *testing.T) {
|
||||
base, err := url.Parse("https://europe.olemovienews.com/ts2/20230125/afAxfjit/mp4/afAxfjit.mp4/index-v1-a1.m3u8")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
media, err := ParseMedia(realWorldPlaylist, base)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if media.Live {
|
||||
t.Error("real VOD playlist should not be live")
|
||||
}
|
||||
if media.IsFMP4 {
|
||||
t.Error("playlist should be detected as TS")
|
||||
}
|
||||
if len(media.Segments) != 138 {
|
||||
t.Errorf("want 138 segments, got %d", len(media.Segments))
|
||||
}
|
||||
first := media.Segments[0]
|
||||
if first.URI != "https://europe.olemovienews.com/ts2/20230125/afAxfjit/mp4/afAxfjit.mp4/seg-1-v1-a1.ts" {
|
||||
t.Errorf("segment URI resolution wrong: %s", first.URI)
|
||||
}
|
||||
if media.Segments[0].Sequence != 1 {
|
||||
t.Errorf("first sequence = %d, want 1", media.Segments[0].Sequence)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,252 @@
|
||||
package hls
|
||||
|
||||
import (
|
||||
"net/url"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func mustURL(t *testing.T, raw string) *url.URL {
|
||||
t.Helper()
|
||||
u, err := url.Parse(raw)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return u
|
||||
}
|
||||
|
||||
func TestParseMediaTS(t *testing.T) {
|
||||
content := "#EXTM3U\n" +
|
||||
"#EXT-X-TARGETDURATION:20\n" +
|
||||
"#EXT-X-VERSION:3\n" +
|
||||
"#EXT-X-MEDIA-SEQUENCE:1\n" +
|
||||
"#EXTINF:20.000,\n" +
|
||||
"seg-1.ts\n" +
|
||||
"#EXTINF:20.000,\n" +
|
||||
"seg-2.ts\n" +
|
||||
"#EXT-X-ENDLIST\n"
|
||||
media, err := ParseMedia(content, mustURL(t, "https://cdn.example.com/vod/index.m3u8"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if media.Live {
|
||||
t.Error("VOD playlist should not be live")
|
||||
}
|
||||
if media.IsFMP4 {
|
||||
t.Error("TS playlist should not be fMP4")
|
||||
}
|
||||
if len(media.Segments) != 2 {
|
||||
t.Fatalf("want 2 segments, got %d", len(media.Segments))
|
||||
}
|
||||
want := "https://cdn.example.com/vod/seg-1.ts"
|
||||
if media.Segments[0].URI != want {
|
||||
t.Errorf("want uri %s, got %s", want, media.Segments[0].URI)
|
||||
}
|
||||
if media.Segments[0].Sequence != 1 {
|
||||
t.Errorf("want sequence 1, got %d", media.Segments[0].Sequence)
|
||||
}
|
||||
if media.Segments[0].Key != nil {
|
||||
t.Error("segment should be unencrypted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseMediaWithoutExtinf(t *testing.T) {
|
||||
// Some playlists list segment URIs without EXTINF lines.
|
||||
content := "#EXTM3U\nseg-1.ts\nseg-2.ts\n#EXT-X-ENDLIST\n"
|
||||
media, err := ParseMedia(content, mustURL(t, "https://cdn.example.com/a/index.m3u8"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(media.Segments) != 2 {
|
||||
t.Fatalf("want 2 segments, got %d", len(media.Segments))
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseMediaLive(t *testing.T) {
|
||||
content := "#EXTM3U\n#EXT-X-TARGETDURATION:6\n#EXTINF:6.0,\nseg-1.ts\n"
|
||||
media, err := ParseMedia(content, mustURL(t, "https://cdn.example.com/live/index.m3u8"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !media.Live {
|
||||
t.Error("playlist without ENDLIST should be live")
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseMediaFMP4(t *testing.T) {
|
||||
content := "#EXTM3U\n" +
|
||||
"#EXT-X-VERSION:6\n" +
|
||||
"#EXT-X-MAP:URI=\"init.mp4\"\n" +
|
||||
"#EXTINF:5.0,\n" +
|
||||
"seg-1.m4s\n" +
|
||||
"#EXT-X-ENDLIST\n"
|
||||
media, err := ParseMedia(content, mustURL(t, "https://cdn.example.com/vod/index.m3u8"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !media.IsFMP4 {
|
||||
t.Error("playlist with MAP should be fMP4")
|
||||
}
|
||||
if len(media.Segments) != 2 {
|
||||
t.Fatalf("want 2 segments (init + 1), got %d", len(media.Segments))
|
||||
}
|
||||
if !media.Segments[0].IsInit {
|
||||
t.Error("first segment should be the init section")
|
||||
}
|
||||
if media.Segments[0].URI != "https://cdn.example.com/vod/init.mp4" {
|
||||
t.Errorf("init uri wrong: %s", media.Segments[0].URI)
|
||||
}
|
||||
if media.OutputExtension() != ".mp4" {
|
||||
t.Errorf("want .mp4, got %s", media.OutputExtension())
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseMediaMultiMapError(t *testing.T) {
|
||||
content := "#EXTM3U\n" +
|
||||
"#EXT-X-MAP:URI=\"init1.mp4\"\n" +
|
||||
"#EXTINF:5.0,\nseg-1.m4s\n" +
|
||||
"#EXT-X-DISCONTINUITY\n" +
|
||||
"#EXT-X-MAP:URI=\"init2.mp4\"\n" +
|
||||
"#EXTINF:5.0,\nseg-2.m4s\n" +
|
||||
"#EXT-X-ENDLIST\n"
|
||||
if _, err := ParseMedia(content, mustURL(t, "https://cdn.example.com/vod/index.m3u8")); err == nil {
|
||||
t.Error("multi-period fMP4 should be rejected")
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseMediaKeyRotation(t *testing.T) {
|
||||
content := "#EXTM3U\n" +
|
||||
"#EXT-X-KEY:METHOD=AES-128,URI=\"key1.bin\",IV=0x00000000000000000000000000000001\n" +
|
||||
"#EXTINF:5.0,\nseg-1.ts\n" +
|
||||
"#EXT-X-KEY:METHOD=AES-128,URI=\"key2.bin\"\n" +
|
||||
"#EXTINF:5.0,\nseg-2.ts\n" +
|
||||
"#EXT-X-KEY:METHOD=NONE\n" +
|
||||
"#EXTINF:5.0,\nseg-3.ts\n" +
|
||||
"#EXT-X-ENDLIST\n"
|
||||
media, err := ParseMedia(content, mustURL(t, "https://cdn.example.com/vod/index.m3u8"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if media.Segments[0].Key == nil || media.Segments[0].Key.URI != "https://cdn.example.com/vod/key1.bin" {
|
||||
t.Fatalf("segment 1 key wrong: %+v", media.Segments[0].Key)
|
||||
}
|
||||
if len(media.Segments[0].Key.IV) != 16 || media.Segments[0].Key.IV[15] != 1 {
|
||||
t.Error("explicit IV not parsed")
|
||||
}
|
||||
if media.Segments[1].Key == nil || media.Segments[1].Key.URI != "https://cdn.example.com/vod/key2.bin" {
|
||||
t.Fatalf("segment 2 key wrong: %+v", media.Segments[1].Key)
|
||||
}
|
||||
if media.Segments[1].Key.IV != nil {
|
||||
t.Error("key without IV should leave IV nil (sequence-derived)")
|
||||
}
|
||||
if media.Segments[2].Key != nil {
|
||||
t.Error("segment 3 should be unencrypted after METHOD=NONE")
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseMediaSampleAESError(t *testing.T) {
|
||||
content := "#EXTM3U\n#EXT-X-KEY:METHOD=SAMPLE-AES,URI=\"skd://key\"\n#EXTINF:5.0,\nseg-1.ts\n"
|
||||
if _, err := ParseMedia(content, mustURL(t, "https://cdn.example.com/vod/index.m3u8")); err == nil {
|
||||
t.Error("SAMPLE-AES should be rejected")
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseMediaByterange(t *testing.T) {
|
||||
content := "#EXTM3U\n" +
|
||||
"#EXT-X-BYTERANGE:1000@0\nseg.ts\n" +
|
||||
"#EXT-X-BYTERANGE:2000\nseg.ts\n" +
|
||||
"#EXT-X-BYTERANGE:3000\nseg.ts\n" +
|
||||
"#EXT-X-ENDLIST\n"
|
||||
media, err := ParseMedia(content, mustURL(t, "https://cdn.example.com/vod/index.m3u8"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ranges := [][2]int64{{0, 1000}, {1000, 2000}, {3000, 3000}}
|
||||
for i, want := range ranges {
|
||||
br := media.Segments[i].Byterange
|
||||
if br == nil {
|
||||
t.Fatalf("segment %d byterange missing", i)
|
||||
}
|
||||
if br.Offset != want[0] || br.Length != want[1] {
|
||||
t.Errorf("segment %d: want offset %d length %d, got %d/%d", i, want[0], want[1], br.Offset, br.Length)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseMediaGap(t *testing.T) {
|
||||
content := "#EXTM3U\n#EXTINF:5.0,\nseg-1.ts\n#EXT-X-GAP\n#EXTINF:5.0,\nseg-2.ts\n#EXT-X-ENDLIST\n"
|
||||
media, err := ParseMedia(content, mustURL(t, "https://cdn.example.com/vod/index.m3u8"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if media.Segments[0].Gap || !media.Segments[1].Gap {
|
||||
t.Error("gap flags wrong")
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseMediaInvalidHeader(t *testing.T) {
|
||||
if _, err := ParseMedia("not a playlist", mustURL(t, "https://cdn.example.com/a.m3u8")); err == nil {
|
||||
t.Error("invalid content should be rejected")
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseMasterPickBest(t *testing.T) {
|
||||
content := "#EXTM3U\n" +
|
||||
"#EXT-X-MEDIA:TYPE=AUDIO,GROUP-ID=\"aud\",NAME=\"en\",DEFAULT=YES,URI=\"audio/index.m3u8\"\n" +
|
||||
"#EXT-X-STREAM-INF:BANDWIDTH=800000,RESOLUTION=640x360\n" +
|
||||
"360p/index.m3u8\n" +
|
||||
"#EXT-X-STREAM-INF:BANDWIDTH=2500000,RESOLUTION=1920x1080,CODECS=\"avc1.640028,mp4a.40.2\"\n" +
|
||||
"1080p/index.m3u8?sign=abc\n" +
|
||||
"#EXT-X-ENDLIST\n"
|
||||
variants, err := ParseMaster(content, mustURL(t, "https://cdn.example.com/vod/master.m3u8"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(variants) != 2 {
|
||||
t.Fatalf("want 2 variants, got %d", len(variants))
|
||||
}
|
||||
best := PickBestVariant(variants)
|
||||
if best.Bandwidth != 2500000 {
|
||||
t.Errorf("want highest bandwidth variant, got %d", best.Bandwidth)
|
||||
}
|
||||
if best.URI != "https://cdn.example.com/vod/1080p/index.m3u8?sign=abc" {
|
||||
t.Errorf("variant uri wrong: %s", best.URI)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveURLQueryNotInherited(t *testing.T) {
|
||||
base := mustURL(t, "https://cdn.example.com/vod/index.m3u8?sign=abc")
|
||||
got := resolveURL(base, "seg-1.ts")
|
||||
if got != "https://cdn.example.com/vod/seg-1.ts" {
|
||||
t.Errorf("relative resolution should follow RFC3986 (no query inheritance), got %s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeriveOutputName(t *testing.T) {
|
||||
media := &Media{}
|
||||
if got := DeriveOutputName("https://cdn.example.com/ts2/a/index-v1-a1.m3u8", media); got != "index-v1-a1.ts" {
|
||||
t.Errorf("want index-v1-a1.ts, got %s", got)
|
||||
}
|
||||
fmp4 := &Media{IsFMP4: true}
|
||||
if got := DeriveOutputName("https://cdn.example.com/ts2/a/index.m3u8", fmp4); got != "index.mp4" {
|
||||
t.Errorf("want index.mp4, got %s", got)
|
||||
}
|
||||
if got := DeriveOutputName("https://cdn.example.com/play?token=x", media); got != "play.ts" {
|
||||
t.Errorf("want play.ts, got %s", got)
|
||||
}
|
||||
if got := DeriveOutputName("https://cdn.example.com/", media); got != "cdn.example.com.ts" {
|
||||
t.Errorf("want host fallback, got %s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseAttributesQuotedComma(t *testing.T) {
|
||||
attrs := parseAttributes(`URI="a,b.ts",IV=0xABCDEF00000000000000000000000001,NAME="x"`)
|
||||
if attrs["URI"] != "a,b.ts" {
|
||||
t.Errorf("quoted comma value wrong: %q", attrs["URI"])
|
||||
}
|
||||
if attrs["IV"] != "0xABCDEF00000000000000000000000001" {
|
||||
t.Errorf("hex value wrong: %q", attrs["IV"])
|
||||
}
|
||||
if attrs["NAME"] != "x" {
|
||||
t.Errorf("quoted value wrong: %q", attrs["NAME"])
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,296 @@
|
||||
package hls
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/GopeedLab/gopeed/pkg/base"
|
||||
)
|
||||
|
||||
func TestFetcherE2E_SameURLTwoTasks(t *testing.T) {
|
||||
const total = 6
|
||||
mux := http.NewServeMux()
|
||||
var playlist strings.Builder
|
||||
playlist.WriteString("#EXTM3U\n")
|
||||
for i := 0; i < total; i++ {
|
||||
playlist.WriteString(fmt.Sprintf("#EXTINF:4,\nseg-%d.ts\n", i))
|
||||
}
|
||||
playlist.WriteString("#EXT-X-ENDLIST\n")
|
||||
mux.HandleFunc("/video.m3u8", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Write([]byte(playlist.String()))
|
||||
})
|
||||
segments := make(map[int][]byte)
|
||||
for i := 0; i < total; i++ {
|
||||
segments[i] = []byte(fmt.Sprintf("SHARED-SEGMENT-%02d", i))
|
||||
idx := i
|
||||
mux.HandleFunc(fmt.Sprintf("/seg-%d.ts", idx), func(w http.ResponseWriter, r *http.Request) {
|
||||
time.Sleep(120 * time.Millisecond)
|
||||
w.Write(segments[idx])
|
||||
})
|
||||
}
|
||||
server := httptest.NewServer(mux)
|
||||
defer server.Close()
|
||||
|
||||
cfg := testConfig()
|
||||
cfg.SegmentConnections = 2
|
||||
cfg.PrefetchContentLength = false
|
||||
dir := t.TempDir()
|
||||
|
||||
// Two tasks for the same playlist URL into the same directory, like a
|
||||
// user duplicating a task. The engine would rename the second output
|
||||
// (AutoRename); mirror that here.
|
||||
f1 := newTestFetcher(t, cfg)
|
||||
if err := f1.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: dir}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f2 := newTestFetcher(t, cfg)
|
||||
if err := f2.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: dir, Name: "video-2.ts"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if f1.state.StagingDir == "" || f1.state.StagingDir == f2.state.StagingDir {
|
||||
t.Fatalf("staging dirs must be distinct per task: %q vs %q", f1.state.StagingDir, f2.state.StagingDir)
|
||||
}
|
||||
|
||||
// Start task 1, let it stage some segments, then pause it.
|
||||
if err := f1.Start(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
time.Sleep(250 * time.Millisecond)
|
||||
if err := f1.Pause(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
staged1, err := os.ReadDir(f1.state.StagingDir)
|
||||
if err != nil || len(staged1) == 0 {
|
||||
t.Fatalf("task 1 should have staged segments, dir: %s", f1.state.StagingDir)
|
||||
}
|
||||
|
||||
// Task 2 must complete without touching task 1's staging data...
|
||||
if err := f2.Start(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := f2.Wait(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
stillThere, err := os.ReadDir(f1.state.StagingDir)
|
||||
if err != nil || len(stillThere) != len(staged1) {
|
||||
t.Fatalf("task 2 disturbed task 1 staging: had %d entries, now %d", len(staged1), len(stillThere))
|
||||
}
|
||||
|
||||
// ...and task 1 resumes to a complete, correct output of its own.
|
||||
if err := f1.Start(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := f1.Wait(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var want []byte
|
||||
for i := 0; i < total; i++ {
|
||||
want = append(want, segments[i]...)
|
||||
}
|
||||
if !bytes.Equal(readOutput(t, f1), want) || !bytes.Equal(readOutput(t, f2), want) {
|
||||
t.Error("same-URL task outputs mismatch")
|
||||
}
|
||||
entries, _ := os.ReadDir(dir)
|
||||
if len(entries) != 2 {
|
||||
names := make([]string, 0, len(entries))
|
||||
for _, e := range entries {
|
||||
names = append(names, e.Name())
|
||||
}
|
||||
t.Errorf("expected only the two outputs to remain, got %v", names)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetcherE2E_ForeignJournalDiscarded(t *testing.T) {
|
||||
const total = 3
|
||||
var served atomic.Int64
|
||||
mux := http.NewServeMux()
|
||||
var playlist strings.Builder
|
||||
playlist.WriteString("#EXTM3U\n")
|
||||
for i := 0; i < total; i++ {
|
||||
playlist.WriteString(fmt.Sprintf("#EXTINF:4,\nseg-%d.ts\n", i))
|
||||
}
|
||||
playlist.WriteString("#EXT-X-ENDLIST\n")
|
||||
mux.HandleFunc("/video.m3u8", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Write([]byte(playlist.String()))
|
||||
})
|
||||
for i := 0; i < total; i++ {
|
||||
mux.HandleFunc(fmt.Sprintf("/seg-%d.ts", i), func(w http.ResponseWriter, r *http.Request) {
|
||||
served.Add(1)
|
||||
w.Write([]byte("DATA"))
|
||||
})
|
||||
}
|
||||
server := httptest.NewServer(mux)
|
||||
defer server.Close()
|
||||
|
||||
cfg := testConfig()
|
||||
cfg.PrefetchContentLength = false
|
||||
f := newTestFetcher(t, cfg)
|
||||
if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: t.TempDir()}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Plant a journal that does not belong to this plan (wrong fingerprint).
|
||||
tempDir := f.resolveStagingDirLocked()
|
||||
if err := os.MkdirAll(tempDir, 0777); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
foreign := journalFile{
|
||||
StagingID: "some-other-task",
|
||||
PlanHash: "deadbeef",
|
||||
SegmentCount: total,
|
||||
Sizes: map[string]int64{"0": 4, "1": 4, "2": 4},
|
||||
}
|
||||
data, _ := json.Marshal(foreign)
|
||||
if err := os.WriteFile(filepath.Join(tempDir, "journal.json"), data, 0666); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if err := f.Start(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := f.Wait(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := served.Load(); got != total {
|
||||
t.Errorf("foreign journal must be discarded, want %d downloads, got %d", total, got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetcherE2E_MergeFailurePreservesStaging(t *testing.T) {
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/video.m3u8", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Write([]byte("#EXTM3U\n#EXTINF:4,\na.ts\n#EXTINF:4,\nb.ts\n#EXT-X-ENDLIST\n"))
|
||||
})
|
||||
mux.HandleFunc("/a.ts", func(w http.ResponseWriter, r *http.Request) { w.Write([]byte("AAAA")) })
|
||||
mux.HandleFunc("/b.ts", func(w http.ResponseWriter, r *http.Request) { w.Write([]byte("BBBB")) })
|
||||
server := httptest.NewServer(mux)
|
||||
defer server.Close()
|
||||
|
||||
f := newTestFetcher(t, testConfig())
|
||||
if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: t.TempDir()}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
run := &hlsRun{
|
||||
ctx: context.Background(),
|
||||
cancel: func() {},
|
||||
done: make(chan struct{}),
|
||||
segs: f.state.Segments,
|
||||
outputPath: f.meta.SingleFilepath(),
|
||||
stagingID: f.state.StagingID,
|
||||
planHash: planFingerprint(f.state),
|
||||
tempDir: f.resolveStagingDirLocked(),
|
||||
keyCache: make(map[string][]byte),
|
||||
journal: make(map[int64]int64),
|
||||
}
|
||||
run.journalPath = filepath.Join(run.tempDir, "journal.json")
|
||||
if err := os.MkdirAll(run.tempDir, 0777); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(segmentPath(run.tempDir, 0), []byte("AAAA"), 0666); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Segment 1 is missing on disk: the merge must fail without deleting the
|
||||
// staged data or leaving a partial output behind.
|
||||
if err := f.merge(run); err == nil {
|
||||
t.Fatal("merge with a missing segment should fail")
|
||||
}
|
||||
if _, err := os.Stat(segmentPath(run.tempDir, 0)); err != nil {
|
||||
t.Fatal("merge failure must preserve staged segments")
|
||||
}
|
||||
entries, _ := os.ReadDir(filepath.Dir(run.outputPath))
|
||||
for _, e := range entries {
|
||||
if strings.HasSuffix(e.Name(), ".part") {
|
||||
t.Errorf("failed merge left a partial output behind: %s", e.Name())
|
||||
}
|
||||
}
|
||||
|
||||
// Recover the segment: the retried merge succeeds.
|
||||
if err := os.WriteFile(segmentPath(run.tempDir, 1), []byte("BBBB"), 0666); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := f.merge(run); err != nil {
|
||||
t.Fatalf("retried merge failed: %v", err)
|
||||
}
|
||||
got, err := os.ReadFile(run.outputPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if string(got) != "AAAABBBB" {
|
||||
t.Errorf("retried merge output mismatch: %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetcherE2E_ReMergeOverExistingOutput(t *testing.T) {
|
||||
const total = 2
|
||||
mux := http.NewServeMux()
|
||||
var playlist strings.Builder
|
||||
playlist.WriteString("#EXTM3U\n")
|
||||
for i := 0; i < total; i++ {
|
||||
playlist.WriteString(fmt.Sprintf("#EXTINF:4,\nseg-%d.ts\n", i))
|
||||
}
|
||||
playlist.WriteString("#EXT-X-ENDLIST\n")
|
||||
mux.HandleFunc("/video.m3u8", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Write([]byte(playlist.String()))
|
||||
})
|
||||
for i := 0; i < total; i++ {
|
||||
mux.HandleFunc(fmt.Sprintf("/seg-%d.ts", i), func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Write([]byte("DATA"))
|
||||
})
|
||||
}
|
||||
server := httptest.NewServer(mux)
|
||||
defer server.Close()
|
||||
|
||||
f := newTestFetcher(t, testConfig())
|
||||
if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: t.TempDir()}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
tempDir := f.resolveStagingDirLocked()
|
||||
if err := os.MkdirAll(tempDir, 0777); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Simulate the crash window after a completed rename but before the task
|
||||
// was marked done: a stale output file, a complete journal and all
|
||||
// segments staged.
|
||||
stale := filepath.Join(filepath.Dir(f.meta.SingleFilepath()), f.state.OutputName)
|
||||
if err := os.WriteFile(stale, []byte("STALE-OUTPUT"), 0666); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
jr := &hlsRun{
|
||||
segs: f.state.Segments,
|
||||
stagingID: f.state.StagingID,
|
||||
planHash: planFingerprint(f.state),
|
||||
journalPath: filepath.Join(tempDir, "journal.json"),
|
||||
journal: map[int64]int64{0: 4, 1: 4},
|
||||
}
|
||||
if err := saveJournal(jr); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for i := 0; i < total; i++ {
|
||||
if err := os.WriteFile(segmentPath(tempDir, int64(i)), []byte("DATA"), 0666); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
if err := f.Start(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := f.Wait(); err != nil {
|
||||
t.Fatalf("re-merge over an existing output failed: %v", err)
|
||||
}
|
||||
got, err := os.ReadFile(stale)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if string(got) != "DATADATA" {
|
||||
t.Errorf("stale output was not replaced: %q", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,146 @@
|
||||
package hls
|
||||
|
||||
import (
|
||||
"net/url"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func mustParseURL(t *testing.T, raw string) *url.URL {
|
||||
t.Helper()
|
||||
u, err := url.Parse(raw)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return u
|
||||
}
|
||||
|
||||
// subsetCase is a media playlist plus the error substring the pipeline must
|
||||
// reject it with; wantOK playlists must parse AND validate cleanly.
|
||||
type subsetCase struct {
|
||||
name string
|
||||
playlist string
|
||||
wantErr string
|
||||
}
|
||||
|
||||
func TestSubsetRejections(t *testing.T) {
|
||||
cases := []subsetCase{
|
||||
{
|
||||
name: "gap",
|
||||
playlist: "#EXTM3U\n#EXTINF:4,\na.ts\n#EXT-X-GAP\n#EXTINF:4,\nb.ts\n#EXT-X-ENDLIST\n",
|
||||
wantErr: "EXT-X-GAP",
|
||||
},
|
||||
{
|
||||
name: "gap-before-first-segment",
|
||||
playlist: "#EXTM3U\n#EXT-X-GAP\n#EXTINF:4,\na.ts\n#EXT-X-ENDLIST\n",
|
||||
wantErr: "EXT-X-GAP",
|
||||
},
|
||||
{
|
||||
name: "discontinuity",
|
||||
playlist: "#EXTM3U\n#EXTINF:4,\na.ts\n#EXT-X-DISCONTINUITY\n#EXTINF:4,\nb.ts\n#EXT-X-ENDLIST\n",
|
||||
wantErr: "EXT-X-DISCONTINUITY",
|
||||
},
|
||||
{
|
||||
name: "live",
|
||||
playlist: "#EXTM3U\n#EXTINF:4,\na.ts\n",
|
||||
wantErr: "live",
|
||||
},
|
||||
{
|
||||
name: "keyformat",
|
||||
playlist: "#EXTM3U\n#EXT-X-KEY:METHOD=AES-128,URI=\"k\",KEYFORMAT=\"com.apple.streamingkeydelivery\"\n#EXTINF:4,\na.ts\n#EXT-X-ENDLIST\n",
|
||||
wantErr: "KEYFORMAT",
|
||||
},
|
||||
{
|
||||
name: "encrypted-map-without-iv",
|
||||
playlist: "#EXTM3U\n#EXT-X-KEY:METHOD=AES-128,URI=\"k\"\n#EXT-X-MAP:URI=\"init.mp4\"\n#EXTINF:4,\na.m4s\n#EXT-X-ENDLIST\n",
|
||||
wantErr: "explicit IV",
|
||||
},
|
||||
{
|
||||
name: "map-change-byterange",
|
||||
playlist: "#EXTM3U\n#EXT-X-MAP:URI=\"init.mp4\",BYTERANGE=\"100@0\"\n#EXTINF:4,\na.m4s\n#EXT-X-MAP:URI=\"init.mp4\",BYTERANGE=\"100@100\"\n#EXTINF:4,\nb.m4s\n#EXT-X-ENDLIST\n",
|
||||
wantErr: "changing init sections",
|
||||
},
|
||||
{
|
||||
name: "map-change-uri",
|
||||
playlist: "#EXTM3U\n#EXT-X-MAP:URI=\"init1.mp4\"\n#EXTINF:4,\na.m4s\n#EXT-X-MAP:URI=\"init2.mp4\"\n#EXTINF:4,\nb.m4s\n#EXT-X-ENDLIST\n",
|
||||
wantErr: "changing init sections",
|
||||
},
|
||||
}
|
||||
baseURL := mustParseURL(t, "https://cdn.example.com/vod/index.m3u8")
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
media, parseErr := ParseMedia(tc.playlist, baseURL)
|
||||
if parseErr != nil {
|
||||
if !strings.Contains(parseErr.Error(), tc.wantErr) {
|
||||
t.Fatalf("parse error %q does not mention %q", parseErr, tc.wantErr)
|
||||
}
|
||||
return
|
||||
}
|
||||
err := validateSupported(media)
|
||||
if err == nil {
|
||||
t.Fatalf("playlist should be rejected with %q", tc.wantErr)
|
||||
}
|
||||
if !strings.Contains(err.Error(), tc.wantErr) {
|
||||
t.Fatalf("error %q does not mention %q", err, tc.wantErr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSubsetRepeatedSameMapAllowed(t *testing.T) {
|
||||
playlist := "#EXTM3U\n" +
|
||||
"#EXT-X-MAP:URI=\"init.mp4\",BYTERANGE=\"100@0\"\n" +
|
||||
"#EXTINF:4,\na.m4s\n" +
|
||||
"#EXT-X-MAP:URI=\"init.mp4\",BYTERANGE=\"100@0\"\n" +
|
||||
"#EXTINF:4,\nb.m4s\n" +
|
||||
"#EXT-X-ENDLIST\n"
|
||||
media, err := ParseMedia(playlist, mustParseURL(t, "https://cdn.example.com/vod/index.m3u8"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := validateSupported(media); err != nil {
|
||||
t.Fatalf("identical repeated MAP must be allowed: %v", err)
|
||||
}
|
||||
if len(media.Segments) != 3 { // init + 2 segments, no duplicate init
|
||||
t.Fatalf("want 3 segments (init + 2), got %d", len(media.Segments))
|
||||
}
|
||||
}
|
||||
|
||||
func TestMasterAudioRenditionDetection(t *testing.T) {
|
||||
masterURL := mustParseURL(t, "https://cdn.example.com/master.m3u8")
|
||||
|
||||
separateAudio := "#EXTM3U\n" +
|
||||
"#EXT-X-MEDIA:TYPE=AUDIO,GROUP-ID=\"aud\",NAME=\"en\",DEFAULT=YES,URI=\"audio/index.m3u8\"\n" +
|
||||
"#EXT-X-STREAM-INF:BANDWIDTH=1000000,AUDIO=\"aud\"\nvideo/index.m3u8\n"
|
||||
master, err := ParseMasterPlaylist(separateAudio, masterURL)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := checkAudioRendition(master, PickBestVariant(master.Variants)); err == nil {
|
||||
t.Fatal("selected variant with a separate audio rendition must be rejected")
|
||||
}
|
||||
|
||||
// A group the variant does not reference must not cause a false positive.
|
||||
unrelatedGroup := "#EXTM3U\n" +
|
||||
"#EXT-X-MEDIA:TYPE=AUDIO,GROUP-ID=\"aud\",NAME=\"en\",DEFAULT=YES,URI=\"audio/index.m3u8\"\n" +
|
||||
"#EXT-X-STREAM-INF:BANDWIDTH=1000000\nvideo/index.m3u8\n"
|
||||
master, err = ParseMasterPlaylist(unrelatedGroup, masterURL)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := checkAudioRendition(master, PickBestVariant(master.Variants)); err != nil {
|
||||
t.Fatalf("unreferenced audio group must not be rejected: %v", err)
|
||||
}
|
||||
|
||||
// A rendition group without URIs (muxed tracks) is fine.
|
||||
groupWithoutURI := "#EXTM3U\n" +
|
||||
"#EXT-X-MEDIA:TYPE=AUDIO,GROUP-ID=\"aud\",NAME=\"en\",DEFAULT=YES\n" +
|
||||
"#EXT-X-STREAM-INF:BANDWIDTH=1000000,AUDIO=\"aud\"\nvideo/index.m3u8\n"
|
||||
master, err = ParseMasterPlaylist(groupWithoutURI, masterURL)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := checkAudioRendition(master, PickBestVariant(master.Variants)); err != nil {
|
||||
t.Fatalf("audio group without URI must not be rejected: %v", err)
|
||||
}
|
||||
}
|
||||
+10
@@ -0,0 +1,10 @@
|
||||
Media fixtures generated locally with FFmpeg 7.0.2 (testsrc + sine sources,
|
||||
no third-party content):
|
||||
|
||||
- sample-00.ts / sample-01.ts: one continuous MPEG-2 TS encode split at a
|
||||
packet boundary, so continuity counters and PCR are consistent across the
|
||||
segment join exactly like segments cut from a single real-time stream.
|
||||
- fmp4-init.mp4: ftyp+moov init section of a fragmented MP4.
|
||||
- fmp4-seg1.m4s .. fmp4-seg3.m4s: moof+mdat media fragments (styp omitted by
|
||||
the muxer, which is legal) split at moof boundaries from one fragmented MP4
|
||||
(-movflags +frag_keyframe+empty_moov+default_base_moof -frag_duration 500000).
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
+283
@@ -0,0 +1,283 @@
|
||||
#EXTM3U
|
||||
#EXT-X-TARGETDURATION:20
|
||||
#EXT-X-ALLOW-CACHE:YES
|
||||
#EXT-X-PLAYLIST-TYPE:VOD
|
||||
#EXT-X-VERSION:3
|
||||
#EXT-X-MEDIA-SEQUENCE:1
|
||||
#EXTINF:20.000,
|
||||
seg-1-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-2-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-3-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-4-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-5-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-6-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-7-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-8-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-9-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-10-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-11-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-12-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-13-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-14-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-15-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-16-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-17-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-18-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-19-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-20-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-21-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-22-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-23-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-24-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-25-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-26-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-27-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-28-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-29-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-30-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-31-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-32-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-33-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-34-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-35-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-36-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-37-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-38-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-39-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-40-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-41-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-42-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-43-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-44-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-45-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-46-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-47-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-48-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-49-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-50-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-51-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-52-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-53-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-54-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-55-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-56-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-57-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-58-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-59-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-60-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-61-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-62-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-63-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-64-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-65-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-66-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-67-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-68-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-69-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-70-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-71-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-72-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-73-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-74-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-75-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-76-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-77-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-78-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-79-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-80-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-81-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-82-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-83-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-84-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-85-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-86-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-87-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-88-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-89-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-90-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-91-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-92-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-93-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-94-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-95-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-96-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-97-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-98-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-99-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-100-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-101-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-102-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-103-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-104-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-105-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-106-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-107-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-108-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-109-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-110-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-111-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-112-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-113-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-114-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-115-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-116-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-117-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-118-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-119-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-120-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-121-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-122-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-123-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-124-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-125-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-126-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-127-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-128-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-129-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-130-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-131-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-132-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-133-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-134-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-135-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-136-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-137-v1-a1.ts
|
||||
#EXTINF:4.111,
|
||||
seg-138-v1-a1.ts
|
||||
#EXT-X-ENDLIST
|
||||
@@ -1,5 +1,8 @@
|
||||
package http
|
||||
|
||||
// DefaultUserAgent is shared by HTTP downloads and extension HTTP requests.
|
||||
const DefaultUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36"
|
||||
|
||||
type config struct {
|
||||
UserAgent string `json:"userAgent"`
|
||||
Connections int `json:"connections"`
|
||||
|
||||
@@ -150,7 +150,7 @@ type slowStartController struct {
|
||||
totalLaunched int
|
||||
batchPending int // Connections in current batch waiting for HTTP response
|
||||
batchReady int // Connections in current batch that succeeded
|
||||
nextBatchSize int // Next batch size: 1, 2, 4, 8...
|
||||
nextBatchSize int // Next batch size: 1, 2, 4, 16, 256... (capped)
|
||||
expansionCh chan struct{} // Signal to trigger next expansion
|
||||
paused bool // Pause expansion (e.g., on 429)
|
||||
}
|
||||
@@ -229,7 +229,15 @@ func (s *slowStartController) commitBatch(count int) {
|
||||
defer s.mu.Unlock()
|
||||
|
||||
s.totalLaunched += count
|
||||
s.nextBatchSize = s.nextBatchSize * 2 // Exponential growth: 1, 2, 4, 8...
|
||||
if s.nextBatchSize == 1 {
|
||||
// Bootstrap growth: squaring one would never increase the batch size.
|
||||
s.nextBatchSize = min(2, s.maxConnections)
|
||||
} else if s.nextBatchSize > s.maxConnections/s.nextBatchSize {
|
||||
// Cap before multiplying to avoid integer overflow.
|
||||
s.nextBatchSize = s.maxConnections
|
||||
} else {
|
||||
s.nextBatchSize *= s.nextBatchSize
|
||||
}
|
||||
s.batchPending = count
|
||||
s.batchReady = 0
|
||||
}
|
||||
@@ -363,20 +371,9 @@ func (f *Fetcher) Resolve(req *base.Request, opts *base.Options) error {
|
||||
f.meta.Opts = &base.Options{}
|
||||
}
|
||||
|
||||
// Parse options
|
||||
if err := base.ParseOptExtra[fhttp.OptsExtra](opts); err != nil {
|
||||
if err := f.initOptions(); err != nil {
|
||||
return err
|
||||
}
|
||||
if opts.Extra == nil {
|
||||
opts.Extra = &fhttp.OptsExtra{}
|
||||
}
|
||||
extra := opts.Extra.(*fhttp.OptsExtra)
|
||||
if extra.Connections <= 0 {
|
||||
extra.Connections = f.config.Connections
|
||||
if extra.Connections <= 0 {
|
||||
extra.Connections = 1
|
||||
}
|
||||
}
|
||||
|
||||
f.setState(stateResolving)
|
||||
|
||||
@@ -512,17 +509,9 @@ func (f *Fetcher) asyncPrefetch() {
|
||||
return
|
||||
}
|
||||
|
||||
// Create temporary file for prefetch data
|
||||
tmpFile, err := os.CreateTemp("", "gopeed-prefetch-*")
|
||||
if err != nil {
|
||||
f.prefetchErr = err
|
||||
return
|
||||
}
|
||||
f.prefetchFile = tmpFile
|
||||
f.prefetchFilePath = tmpFile.Name()
|
||||
|
||||
defer func() {
|
||||
// Close response body when prefetch stops
|
||||
// Close the response even if temporary-file setup fails or shutdown
|
||||
// rejects a late prefetch. Otherwise the server can remain blocked writing.
|
||||
f.resolveRespLock.Lock()
|
||||
if f.resolveResp != nil {
|
||||
f.resolveResp.Body.Close()
|
||||
@@ -531,8 +520,33 @@ func (f *Fetcher) asyncPrefetch() {
|
||||
f.resolveRespLock.Unlock()
|
||||
}()
|
||||
|
||||
// Create temporary file for prefetch data
|
||||
tempDir := ""
|
||||
if f.ctl != nil && f.ctl.TempDir != "" {
|
||||
tempDir = f.ctl.TempDir
|
||||
if err := os.MkdirAll(tempDir, 0700); err != nil {
|
||||
f.prefetchErr = err
|
||||
return
|
||||
}
|
||||
}
|
||||
tmpFile, err := os.CreateTemp(tempDir, "gopeed-prefetch-*")
|
||||
if err != nil {
|
||||
f.prefetchErr = err
|
||||
return
|
||||
}
|
||||
if f.ctl != nil {
|
||||
if err := f.ctl.TempFiles.Track(tmpFile.Name()); err != nil {
|
||||
tmpFile.Close()
|
||||
os.Remove(tmpFile.Name())
|
||||
f.prefetchErr = err
|
||||
return
|
||||
}
|
||||
}
|
||||
f.prefetchFile = tmpFile
|
||||
f.prefetchFilePath = tmpFile.Name()
|
||||
|
||||
buf := make([]byte, 32*1024) // 32KB buffer
|
||||
reader := NewTimeoutReader(resp.Body, readTimeout)
|
||||
reader := f.responseReader(resp)
|
||||
|
||||
for {
|
||||
select {
|
||||
@@ -617,7 +631,11 @@ func (f *Fetcher) cleanupPrefetchFile() {
|
||||
f.prefetchFile = nil
|
||||
}
|
||||
if f.prefetchFilePath != "" {
|
||||
os.Remove(f.prefetchFilePath)
|
||||
if f.ctl != nil {
|
||||
f.ctl.TempFiles.Remove(f.prefetchFilePath)
|
||||
} else {
|
||||
os.Remove(f.prefetchFilePath)
|
||||
}
|
||||
f.prefetchFilePath = ""
|
||||
}
|
||||
// The byte count is only reusable while the backing prefetch file exists.
|
||||
@@ -627,7 +645,31 @@ func (f *Fetcher) cleanupPrefetchFile() {
|
||||
f.resolveFallback.Store(false)
|
||||
}
|
||||
|
||||
// Options may be replaced between Resolve and Start when creating a resolved task.
|
||||
func (f *Fetcher) initOptions() error {
|
||||
opts := f.meta.Opts
|
||||
// Parse options
|
||||
if err := base.ParseOptExtra[fhttp.OptsExtra](opts); err != nil {
|
||||
return err
|
||||
}
|
||||
if opts.Extra == nil {
|
||||
opts.Extra = &fhttp.OptsExtra{}
|
||||
}
|
||||
extra := opts.Extra.(*fhttp.OptsExtra)
|
||||
if extra.Connections <= 0 {
|
||||
extra.Connections = f.config.Connections
|
||||
if extra.Connections <= 0 {
|
||||
extra.Connections = 1
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *Fetcher) Start() error {
|
||||
if err := f.initOptions(); err != nil {
|
||||
return err
|
||||
}
|
||||
state := f.getState()
|
||||
|
||||
switch state {
|
||||
@@ -1329,7 +1371,7 @@ func (f *Fetcher) downloadChunkOnce(conn *connection, client *http.Client, buf [
|
||||
f.slowStart.onConnectSuccess()
|
||||
}
|
||||
|
||||
reader := NewTimeoutReader(resp.Body, readTimeout)
|
||||
reader := f.responseReader(resp)
|
||||
var responseBytesRead int64
|
||||
for {
|
||||
if conn.ctx.Err() != nil {
|
||||
@@ -1711,7 +1753,7 @@ func (f *Fetcher) runConnectionWithResolveResp(conn *connection) {
|
||||
}
|
||||
|
||||
// Download data from resolve response
|
||||
reader := NewTimeoutReader(resp.Body, readTimeout)
|
||||
reader := f.responseReader(resp)
|
||||
for {
|
||||
if conn.ctx.Err() != nil {
|
||||
return
|
||||
@@ -1828,7 +1870,7 @@ func (f *Fetcher) runConnectionFallback(conn *connection) {
|
||||
f.slowStart.onConnectSuccess()
|
||||
}
|
||||
|
||||
reader := NewTimeoutReader(resp.Body, readTimeout)
|
||||
reader := f.responseReader(resp)
|
||||
for {
|
||||
if conn.ctx.Err() != nil {
|
||||
return conn.ctx.Err()
|
||||
@@ -2451,3 +2493,12 @@ func (f *Fetcher) Progress() fetcher.Progress {
|
||||
func (f *Fetcher) Wait() error {
|
||||
return <-f.doneCh
|
||||
}
|
||||
|
||||
// Generated output can pause while its producer waits for processing capacity.
|
||||
// Its producer owns upstream deadlines; cancellation still closes the response.
|
||||
func (f *Fetcher) responseReader(resp *http.Response) io.Reader {
|
||||
if f.ctl != nil && f.ctl.ManagedProduction != nil && resp.Request != nil && f.ctl.ManagedProduction(resp.Request.URL.String()) {
|
||||
return resp.Body
|
||||
}
|
||||
return NewTimeoutReader(resp.Body, readTimeout)
|
||||
}
|
||||
@@ -72,7 +72,7 @@ func (fm *FetcherManager) AutoRename() bool {
|
||||
|
||||
func (fm *FetcherManager) DefaultConfig() any {
|
||||
return &config{
|
||||
UserAgent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36",
|
||||
UserAgent: DefaultUserAgent,
|
||||
Connections: 16,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,74 @@
|
||||
package http
|
||||
|
||||
import (
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/GopeedLab/gopeed/internal/controller"
|
||||
"github.com/GopeedLab/gopeed/internal/tempfiles"
|
||||
)
|
||||
|
||||
type prefetchSetupBody struct {
|
||||
io.Reader
|
||||
closed bool
|
||||
}
|
||||
|
||||
func (b *prefetchSetupBody) Close() error {
|
||||
b.closed = true
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestPrefetchSetupFailureClosesResponse(t *testing.T) {
|
||||
for _, closedScope := range []bool{false, true} {
|
||||
name := "invalid_temp_directory"
|
||||
if closedScope {
|
||||
name = "downloader_already_closed"
|
||||
}
|
||||
t.Run(name, func(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
scope := &tempfiles.Scope{}
|
||||
dir := root
|
||||
if closedScope {
|
||||
if err := scope.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
} else {
|
||||
dir = filepath.Join(root, "file")
|
||||
if err := os.WriteFile(dir, nil, 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
body := &prefetchSetupBody{Reader: strings.NewReader("unused response")}
|
||||
f := &Fetcher{
|
||||
ctl: &controller.Controller{TempDir: dir, TempFiles: scope},
|
||||
resolveResp: &http.Response{Body: body},
|
||||
}
|
||||
f.asyncPrefetch()
|
||||
if f.prefetchErr == nil {
|
||||
t.Fatal("expected temporary-file initialization to fail")
|
||||
}
|
||||
if !body.closed {
|
||||
t.Error("prefetch setup failure left the response body open")
|
||||
}
|
||||
if f.resolveResp != nil {
|
||||
t.Error("prefetch retained its failed resolve response")
|
||||
}
|
||||
if !f.prefetchDone.Load() {
|
||||
t.Error("prefetch did not report completion")
|
||||
}
|
||||
if closedScope {
|
||||
entries, err := os.ReadDir(root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(entries) != 0 {
|
||||
t.Errorf("late prefetch left temporary files: %v", entries)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
// Package tempfiles tracks temporary paths owned by one downloader, without
|
||||
// requiring a private parent directory or scanning the shared system temp root.
|
||||
package tempfiles
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"io"
|
||||
"os"
|
||||
"sync"
|
||||
)
|
||||
|
||||
type Scope struct {
|
||||
mu sync.Mutex
|
||||
paths map[string]struct{}
|
||||
closed bool
|
||||
}
|
||||
|
||||
// Track adopts a newly created unique path. Late creations are removed when
|
||||
// shutdown has already begun. A nil scope leaves cleanup to the resource owner.
|
||||
func (s *Scope) Track(path string) error {
|
||||
if s == nil {
|
||||
return nil
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.closed {
|
||||
return errors.Join(io.ErrClosedPipe, os.RemoveAll(path))
|
||||
}
|
||||
if s.paths == nil {
|
||||
s.paths = make(map[string]struct{})
|
||||
}
|
||||
s.paths[path] = struct{}{}
|
||||
return nil
|
||||
}
|
||||
func (s *Scope) Remove(path string) error {
|
||||
if s == nil {
|
||||
return os.RemoveAll(path)
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if err := os.RemoveAll(path); err != nil {
|
||||
return err
|
||||
}
|
||||
delete(s.paths, path)
|
||||
return nil
|
||||
}
|
||||
func (s *Scope) Close() error {
|
||||
if s == nil {
|
||||
return nil
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.closed = true
|
||||
var result error
|
||||
for path := range s.paths {
|
||||
if err := os.RemoveAll(path); err != nil {
|
||||
result = errors.Join(result, err)
|
||||
} else {
|
||||
delete(s.paths, path)
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
package tempfiles
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"io"
|
||||
"os"
|
||||
"sync"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestSharedDirectoryOwnersAndLateCreation(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
a, b := &Scope{}, &Scope{}
|
||||
own := func(scope *Scope) string {
|
||||
t.Helper()
|
||||
dir, err := os.MkdirTemp(root, "gopeed-media-")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := scope.Track(dir); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return dir
|
||||
}
|
||||
first, second := own(a), own(b)
|
||||
if err := a.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := os.Stat(first); !os.IsNotExist(err) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := os.Stat(second); err != nil {
|
||||
t.Fatal("removed other owner's path", err)
|
||||
}
|
||||
late, err := os.MkdirTemp(root, "gopeed-media-")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := a.Track(late); !errors.Is(err, io.ErrClosedPipe) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := os.Stat(late); !os.IsNotExist(err) {
|
||||
t.Fatal("late creation leaked", err)
|
||||
}
|
||||
if err := b.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConcurrentCreationAndShutdown(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
scope := &Scope{}
|
||||
var wg sync.WaitGroup
|
||||
for i := 0; i < 20; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
dir, err := os.MkdirTemp(root, "gopeed-media-")
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
return
|
||||
}
|
||||
if err := scope.Track(dir); err != nil && !errors.Is(err, io.ErrClosedPipe) {
|
||||
t.Error(err)
|
||||
}
|
||||
}()
|
||||
}
|
||||
if err := scope.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
wg.Wait()
|
||||
entries, err := os.ReadDir(root)
|
||||
if err != nil || len(entries) != 0 {
|
||||
t.Fatal("shutdown leaked paths", entries, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,109 @@
|
||||
# Extension browser host configuration
|
||||
|
||||
The downloader wraps each provider with the extension's stable identity before
|
||||
constructing the runtime. `OpenOptions.ProfileID`, `DataPath`, and `ProxyURL` are
|
||||
host-only fields. `parseOpenOptions` never reads them from extension JavaScript.
|
||||
The extension API is unchanged. The Flutter host uses the fork's public
|
||||
`WebViewProfile`, `InAppWebViewSettings.profileId`, and profile-scoped
|
||||
`CookieManager` APIs; the library has no Gopeed-specific method-channel bridge.
|
||||
Only the Gopeed extension JavaScript boundary keeps these options host-only.
|
||||
|
||||
Profiles are persistent and shared across invocations/pages of the same extension.
|
||||
Different identities use different native stores. Go providers use a SHA-256-based
|
||||
path under `<StorageDir>/webview`; Apple RPC hosts use a stable UUID data store;
|
||||
Android uses a named WebView profile. Cookie operations select that same store,
|
||||
including `clearCookies`. Closing an execution closes its pages, not its profile.
|
||||
Uninstall closes the extension's pages, prevents its old runtimes from opening
|
||||
new pages, and removes its native profile before deleting the extension record.
|
||||
Cleanup failures abort uninstall and can be retried. Reinstallation starts with
|
||||
empty browser data; an ordinary extension update keeps the existing profile.
|
||||
Windows/Linux delete the profile directory; Apple removes the named WebKit store.
|
||||
Android cannot delete a profile already loaded by the current process: when
|
||||
DELETE_BROWSING_DATA is supported it clears all site data immediately, and records
|
||||
an empty-profile deletion in SharedPreferences for the next process start. A
|
||||
reinstallation may reuse that cleared profile and cancels the pending deletion.
|
||||
If the full-data clearing API is unavailable, uninstall fails with a restart
|
||||
instruction; the pending profile is deleted at startup before it can be loaded.
|
||||
No removal operation is exposed through extension JavaScript.
|
||||
|
||||
Existing shared browser data is not copied into every extension's new profile.
|
||||
|
||||
The downloader lazily owns one loopback forward proxy with HTTP and SOCKS5
|
||||
listeners. Windows/Linux/Android use HTTP; Apple uses SOCKS5 because its HTTP
|
||||
CONNECT configuration may bypass plain HTTP. The forwarder reads the current
|
||||
Gopeed proxy config for each new request/tunnel and supports direct, system,
|
||||
HTTP(S) upstream proxies with Basic authentication, and authenticated SOCKS5.
|
||||
Credentials remain in Go and are never sent over the WebView RPC.
|
||||
|
||||
Changes affect new requests/connections. Existing CONNECT/SOCKS tunnels keep their
|
||||
route until closed; opening another page in the same profile does not reset it.
|
||||
SOCKS carries host/port rather than a full URL: system proxy/PAC selection for
|
||||
Apple is host-based, with port 80 treated as HTTP and other ports as HTTPS.
|
||||
Path-dependent PAC rules cannot be reproduced from a SOCKS connection.
|
||||
|
||||
Apple requires macOS 14 / iOS 17. Android checks MULTI_PROFILE and PROXY_OVERRIDE.
|
||||
Unsupported hosts fail explicitly instead of silently sharing browser storage or
|
||||
ignoring the proxy.
|
||||
|
||||
## Validation
|
||||
|
||||
- `go test -race ./internal/webview/proxy ./pkg/download/engine/webview`
|
||||
- `go test ./pkg/download -run 'TestWebViewProfileBound|TestDownloader_ExtensionRuntimeWebView'`
|
||||
- Windows/Linux: `go test -tags webview -v ./internal/webview/goprovider` (Linux: run under `xvfb-run -a`).
|
||||
- macOS: start the normal Flutter app, then run `go test -tags webview -v ./internal/webview/rpcprovider` with its socket address. See `rpcprovider/README.md`.
|
||||
- In webview_go: `go test -tags webview_integration -v -count=1 .` (Linux: run under `xvfb-run -a`).
|
||||
|
||||
The shared provider contract covers HTTP/HTTPS proxy routing, concurrent pages,
|
||||
HttpOnly cookies, LocalStorage, IndexedDB, profile-scoped clearing, reopening,
|
||||
and removal without affecting another profile. HTTPS routing is checked with a
|
||||
rejecting upstream tunnel; no test-only certificate bypass is added to the app.
|
||||
Go proxy unit tests separately cover successful TLS forwarding and authentication.
|
||||
These cases run in the existing `test.yml` WebView jobs; existing mobile build jobs
|
||||
compile the Android/iOS bridge. No separate Flutter test application is required.
|
||||
|
||||
## Extension events and host environment
|
||||
|
||||
`gopeed.host.env.version` is the host's build version (`dev` for development
|
||||
builds). `gopeed.info.version` continues to identify the extension's own version.
|
||||
`gopeed.host.env.os` and `gopeed.host.env.arch` match the REST info endpoint's `os` and
|
||||
`arch`: the host's Go `runtime.GOOS` and `runtime.GOARCH` values (for example,
|
||||
`windows` / `amd64` or `darwin` / `arm64`).
|
||||
See [extension-api.d.ts](extension-api.d.ts) for the event payloads.
|
||||
|
||||
```js
|
||||
const page = await gopeed.runtime.webview.open();
|
||||
const unsubscribe = await page.on("url-changed", ({ url, sameDocument }) => {
|
||||
gopeed.logger.info(url, sameDocument);
|
||||
});
|
||||
await page.on("closed", ({ reason }) => gopeed.logger.info(reason));
|
||||
await page.addInitScript("globalThis.extensionReady = true");
|
||||
await page.goto("https://example.com");
|
||||
unsubscribe();
|
||||
await page.close();
|
||||
```
|
||||
|
||||
Registration resolves only after the listener is installed; it does not replay
|
||||
current state. `url-changed` reports actual address changes, including hash,
|
||||
pushState, replaceState and history navigation. `load` follows the main document's
|
||||
`window.load`, including reloads, and does not wait for business-level async work.
|
||||
Iframe loads and subresource failures do not become main-page notifications.
|
||||
`closed` distinguishes the user's close action from API closure and is terminal.
|
||||
Unsubscribe is idempotent. Notifications cannot block navigation; the engine does
|
||||
not await promises returned by handlers. Catch async errors inside the handler.
|
||||
|
||||
`addInitScript` applies to future document contexts only. Use `execute` to run
|
||||
code on the current page. Executable functions are serialized and cannot capture
|
||||
extension-side lexical variables; pass values explicitly as arguments.
|
||||
|
||||
The RPC host uses an NDJSON `page.events` response whose first record is
|
||||
`{"ready":true}`. Subsequent records are `{event,data}`. Closing a page sends the
|
||||
terminal record before ending the stream. The native backend uses document-start
|
||||
notifications for main-document URL/load events and native callbacks for
|
||||
navigation failures and user closure.
|
||||
|
||||
Additional validation:
|
||||
|
||||
- `go test -race ./pkg/download/engine/webview ./internal/webview/rpcprovider`
|
||||
- `go test ./pkg/download -run 'TestExtension(WebViewEvents|InfoHostVersion)'`
|
||||
- macOS native: `go test -tags webview_native ./internal/webview/goprovider -run TestProviderEvents`
|
||||
- Running Flutter host: `go test -tags webview ./internal/webview/rpcprovider -run TestProviderEvents`
|
||||
Vendored
+60
@@ -0,0 +1,60 @@
|
||||
/** Public extension API additions. info.version remains the extension version. */
|
||||
export interface ExtensionInfo {
|
||||
identity: string;
|
||||
name: string;
|
||||
author: string;
|
||||
title: string;
|
||||
version: string;
|
||||
}
|
||||
|
||||
/** Available as gopeed.host. */
|
||||
export interface Host {
|
||||
env: HostEnv;
|
||||
}
|
||||
|
||||
export interface HostEnv {
|
||||
/** Host build version; "dev" for development builds. */
|
||||
version: string;
|
||||
/** Host OS and architecture, matching the REST info endpoint (Go GOOS/GOARCH). */
|
||||
os: string;
|
||||
arch: string;
|
||||
}
|
||||
|
||||
/** Functions are serialized; they cannot capture extension-side variables. */
|
||||
export type WebviewExecutable<T = unknown> =
|
||||
| string
|
||||
| ((...args: unknown[]) => T | Promise<T>);
|
||||
|
||||
export type WebviewUnsubscribe = () => void;
|
||||
|
||||
/** Navigation and load notifications are for the main document only. */
|
||||
export interface WebviewEventMap {
|
||||
"url-changed": { url: string; sameDocument: boolean };
|
||||
/** window.load, including reloads, excluding hash/History API navigation. */
|
||||
"load": { url: string };
|
||||
"load-error": { url: string; message: string };
|
||||
"closed": { reason: "user" | "api" };
|
||||
}
|
||||
|
||||
export interface WebviewGotoOptions {
|
||||
timeoutMs?: number;
|
||||
waitUntil?: "load" | "domcontentloaded";
|
||||
}
|
||||
|
||||
/** These members extend the existing WebviewPage API. */
|
||||
export interface WebviewPage {
|
||||
/**
|
||||
* Resolves when listening is ready. Does not replay current state.
|
||||
* Register before goto() to observe navigation. Unsubscribe is idempotent.
|
||||
* Closed is delivered once, then listeners are released automatically.
|
||||
* Callbacks cannot cancel navigation; returned promises are not awaited.
|
||||
* Callers handle errors in asynchronous callbacks.
|
||||
*/
|
||||
on<K extends keyof WebviewEventMap>(
|
||||
event: K,
|
||||
handler: (data: WebviewEventMap[K]) => void,
|
||||
): Promise<WebviewUnsubscribe>;
|
||||
/** Runs at document creation on future navigations, never immediately. */
|
||||
addInitScript(script: string): Promise<void>;
|
||||
goto(url: string, opts?: WebviewGotoOptions): Promise<void>;
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
//go:build cgo && darwin && webview_native
|
||||
|
||||
package goprovider
|
||||
|
||||
import (
|
||||
"github.com/GopeedLab/gopeed/internal/webview/integrationtest"
|
||||
"os"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestMain(m *testing.M) { os.Exit(RunMainThreadLoop(m.Run)) }
|
||||
func TestProviderEvents(t *testing.T) { integrationtest.RunEventContract(t, New()) }
|
||||
@@ -0,0 +1,49 @@
|
||||
//go:build cgo
|
||||
|
||||
package goprovider
|
||||
|
||||
import (
|
||||
enginewebview "github.com/GopeedLab/gopeed/pkg/download/engine/webview"
|
||||
webview "github.com/GopeedLab/webview_go"
|
||||
"github.com/dop251/goja"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestInternalErrorDocumentDoesNotReportPageEvents(t *testing.T) {
|
||||
vm := goja.New()
|
||||
if _, err := vm.RunString(`
|
||||
var window = globalThis;
|
||||
window.top = window;
|
||||
var location = {protocol: "chrome-error:", href: "chrome-error://chromewebdata/"};
|
||||
var notifications = [];
|
||||
var listeners = [];
|
||||
function notify(payload) { notifications.push(payload); }
|
||||
window.addEventListener = (name) => listeners.push(name);
|
||||
`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := vm.RunString(buildLoadNotifyScript("notify")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, name := range []string{"notifications", "listeners"} {
|
||||
value, err := vm.RunString(name + ".length")
|
||||
if err != nil || value.ToInteger() != 0 {
|
||||
t.Fatalf("internal error page registered %s: %v (%v)", name, value, err)
|
||||
}
|
||||
}
|
||||
// The provider must wait for the native failure, preserving the failed URL.
|
||||
page := newPageWrapper(enginewebview.OpenOptions{})
|
||||
got := make(chan string, 1)
|
||||
_, err := page.On("load-error", func(e enginewebview.Event) { got <- e.Data["url"].(string) })
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
page.handleNativeEvent(webview.Event{Name: "load-error", URL: "https://example.test/failure", Message: "TLS handshake failed"})
|
||||
if err := page.waitForNavigation(time.Second, "load"); err == nil {
|
||||
t.Fatal("native failure completed navigation successfully")
|
||||
}
|
||||
if url := <-got; url != "https://example.test/failure" {
|
||||
t.Fatal(url)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
//go:build linux && cgo
|
||||
|
||||
package goprovider
|
||||
|
||||
/*
|
||||
#cgo pkg-config: gtk+-3.0
|
||||
#include <gtk/gtk.h>
|
||||
static void pumpGTK(void) {
|
||||
// Bound each batch so Go creation/removal requests cannot be starved.
|
||||
for (int i = 0; i < 64 && g_main_context_pending(NULL); ++i)
|
||||
g_main_context_iteration(NULL, FALSE);
|
||||
}
|
||||
*/
|
||||
import "C"
|
||||
|
||||
import (
|
||||
"runtime"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
var gtkThreadOnce sync.Once
|
||||
var gtkThreadTasks = make(chan func())
|
||||
|
||||
func RunMainThreadLoop(run func() int) int { return run() }
|
||||
|
||||
func postMainThreadTask(task func()) bool {
|
||||
gtkThreadOnce.Do(func() {
|
||||
go func() {
|
||||
runtime.LockOSThread()
|
||||
// WebKit must always use its original UI thread, including after
|
||||
// the last page closes. A page cannot stop this shared event loop.
|
||||
ticker := time.NewTicker(10 * time.Millisecond)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case task := <-gtkThreadTasks:
|
||||
task()
|
||||
case <-ticker.C:
|
||||
}
|
||||
C.pumpGTK()
|
||||
}
|
||||
}()
|
||||
})
|
||||
gtkThreadTasks <- task
|
||||
return true
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
//go:build !darwin
|
||||
//go:build !darwin && (!linux || !cgo)
|
||||
|
||||
package goprovider
|
||||
|
||||
|
||||
@@ -6,7 +6,6 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
@@ -37,7 +36,9 @@ func (p *Provider) Open(opts enginewebview.OpenOptions) (enginewebview.Page, err
|
||||
}
|
||||
|
||||
type pageWrapper struct {
|
||||
opts enginewebview.OpenOptions
|
||||
opts enginewebview.OpenOptions
|
||||
events enginewebview.EventHub
|
||||
navigationErrors chan error
|
||||
|
||||
callbackName string
|
||||
readyID string
|
||||
@@ -49,19 +50,23 @@ type pageWrapper struct {
|
||||
url string
|
||||
closed bool
|
||||
|
||||
ready chan error
|
||||
done chan struct{}
|
||||
loads chan string
|
||||
ready chan error
|
||||
done chan struct{}
|
||||
loads chan string
|
||||
domReady chan struct{}
|
||||
sameDocument chan struct{}
|
||||
}
|
||||
|
||||
type evalMessage struct {
|
||||
ID string `json:"id"`
|
||||
Ready bool `json:"ready,omitempty"`
|
||||
Load bool `json:"load,omitempty"`
|
||||
URL string `json:"url,omitempty"`
|
||||
OK bool `json:"ok,omitempty"`
|
||||
Value any `json:"value,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
Event string `json:"event,omitempty"`
|
||||
SameDocument bool `json:"sameDocument,omitempty"`
|
||||
ID string `json:"id"`
|
||||
Ready bool `json:"ready,omitempty"`
|
||||
Load bool `json:"load,omitempty"`
|
||||
URL string `json:"url,omitempty"`
|
||||
OK bool `json:"ok,omitempty"`
|
||||
Value any `json:"value,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
type evalResult struct {
|
||||
@@ -88,13 +93,16 @@ const startupTimeout = time.Minute
|
||||
func newPageWrapper(opts enginewebview.OpenOptions) *pageWrapper {
|
||||
seq := providerSeq.Add(1)
|
||||
return &pageWrapper{
|
||||
opts: opts,
|
||||
callbackName: fmt.Sprintf("__gopeedWebViewCallback_%d", seq),
|
||||
readyID: fmt.Sprintf("__ready__%d", seq),
|
||||
pending: make(map[string]chan evalResult),
|
||||
ready: make(chan error, 1),
|
||||
done: make(chan struct{}),
|
||||
loads: make(chan string, 8),
|
||||
opts: opts,
|
||||
navigationErrors: make(chan error, 8),
|
||||
callbackName: fmt.Sprintf("__gopeedWebViewCallback_%d", seq),
|
||||
readyID: fmt.Sprintf("__ready__%d", seq),
|
||||
pending: make(map[string]chan evalResult),
|
||||
ready: make(chan error, 1),
|
||||
done: make(chan struct{}),
|
||||
loads: make(chan string, 8),
|
||||
domReady: make(chan struct{}, 8),
|
||||
sameDocument: make(chan struct{}, 8),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -105,12 +113,21 @@ func (p *pageWrapper) start() error {
|
||||
defer runtimepkg.UnlockOSThread()
|
||||
|
||||
var w webview.WebView
|
||||
if p.opts.Headless {
|
||||
if p.opts.DataPath != "" {
|
||||
var err error
|
||||
w, err = webview.NewWithOptions(webview.Options{Debug: p.opts.Debug, Headless: p.opts.Headless, DataPath: p.opts.DataPath, ProxyURL: p.opts.ProxyURL})
|
||||
if err != nil {
|
||||
p.ready <- err
|
||||
close(p.done)
|
||||
return
|
||||
}
|
||||
} else if p.opts.Headless {
|
||||
w = webview.NewHeadless(p.opts.Debug)
|
||||
} else {
|
||||
w = webview.New(p.opts.Debug)
|
||||
}
|
||||
p.view = w
|
||||
w.SetEventHandler(p.handleNativeEvent)
|
||||
w.SetTitle(firstNonEmpty(p.opts.Title, "Gopeed WebView"))
|
||||
w.SetSize(defaultWindowDimension(p.opts.Width, 1280), defaultWindowDimension(p.opts.Height, 800), webview.HintNone)
|
||||
|
||||
@@ -122,18 +139,25 @@ func (p *pageWrapper) start() error {
|
||||
setter.SetUserAgent(p.opts.UserAgent)
|
||||
}
|
||||
}
|
||||
w.Init(buildLoadNotifyScript(p.callbackName))
|
||||
|
||||
if err := w.Bind(p.callbackName, func(payload string) error {
|
||||
return p.handleCallback(payload)
|
||||
}); err != nil {
|
||||
w.Destroy()
|
||||
p.ready <- err
|
||||
close(p.done)
|
||||
return
|
||||
}
|
||||
|
||||
w.Init(buildLoadNotifyScript(p.callbackName))
|
||||
applyWindowOptions(w, p.opts)
|
||||
w.SetHtml(buildBootstrapHTML(p.callbackName, p.readyID))
|
||||
if runtimepkg.GOOS == "linux" {
|
||||
// All GTK pages share the process-wide event loop below.
|
||||
return
|
||||
}
|
||||
w.Run()
|
||||
p.handleNativeEvent(webview.Event{Name: "closed"})
|
||||
w.Destroy()
|
||||
close(p.done)
|
||||
}
|
||||
@@ -162,10 +186,15 @@ func (p *pageWrapper) AddInitScript(script string) error {
|
||||
func (p *pageWrapper) Goto(url string, opts enginewebview.GotoOptions) error {
|
||||
startedAt := time.Now()
|
||||
p.drainLoads()
|
||||
for {
|
||||
select {
|
||||
case <-p.navigationErrors:
|
||||
continue
|
||||
default:
|
||||
}
|
||||
break
|
||||
}
|
||||
if err := p.dispatch(func(w webview.WebView) error {
|
||||
p.mu.Lock()
|
||||
p.url = url
|
||||
p.mu.Unlock()
|
||||
w.Navigate(url)
|
||||
return nil
|
||||
}); err != nil {
|
||||
@@ -179,7 +208,7 @@ func (p *pageWrapper) Goto(url string, opts enginewebview.GotoOptions) error {
|
||||
if waitUntil == "" {
|
||||
waitUntil = "load"
|
||||
}
|
||||
err := p.waitForNavigation(url, timeout, waitUntil)
|
||||
err := p.waitForNavigation(timeout, waitUntil)
|
||||
if err != nil {
|
||||
p.tracef("goto failed %dms url=%s err=%v", time.Since(startedAt).Milliseconds(), url, err)
|
||||
return err
|
||||
@@ -219,7 +248,12 @@ func (p *pageWrapper) Execute(expression string, args ...any) (any, error) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
result := <-ch
|
||||
var result evalResult
|
||||
select {
|
||||
case result = <-ch:
|
||||
case <-p.done:
|
||||
return nil, fmt.Errorf("webview page is closed")
|
||||
}
|
||||
if result.err != nil {
|
||||
p.tracef("execute failed %dms err=%v", time.Since(startedAt).Milliseconds(), result.err)
|
||||
return result.value, result.err
|
||||
@@ -307,6 +341,7 @@ func (p *pageWrapper) Close() error {
|
||||
return nil
|
||||
}
|
||||
p.closed = true
|
||||
p.failPendingLocked()
|
||||
w := p.view
|
||||
wait := p.done
|
||||
p.mu.Unlock()
|
||||
@@ -318,12 +353,18 @@ func (p *pageWrapper) Close() error {
|
||||
// the quit message lands on the correct queue.
|
||||
terminateDone := make(chan struct{})
|
||||
w.Dispatch(func() {
|
||||
w.Terminate()
|
||||
if runtimepkg.GOOS == "linux" {
|
||||
w.Destroy()
|
||||
close(p.done)
|
||||
} else {
|
||||
w.Terminate()
|
||||
}
|
||||
close(terminateDone)
|
||||
})
|
||||
<-terminateDone
|
||||
}
|
||||
<-wait
|
||||
p.events.Emit(enginewebview.Event{Name: "closed", Data: map[string]any{"reason": "api"}})
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -334,94 +375,29 @@ func (p *pageWrapper) tracef(format string, args ...any) {
|
||||
fmt.Printf("goprovider "+format+"\n", args...)
|
||||
}
|
||||
|
||||
func (p *pageWrapper) waitForNavigation(targetURL string, timeout time.Duration, waitUntil string) error {
|
||||
deadline := time.Now().Add(timeout)
|
||||
ticker := time.NewTicker(100 * time.Millisecond)
|
||||
defer ticker.Stop()
|
||||
|
||||
func (p *pageWrapper) waitForNavigation(timeout time.Duration, waitUntil string) error {
|
||||
timer := time.NewTimer(timeout)
|
||||
defer timer.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-p.done:
|
||||
return fmt.Errorf("webview page is closed")
|
||||
case err := <-p.navigationErrors:
|
||||
return err
|
||||
case <-timer.C:
|
||||
return fmt.Errorf("webview navigation timeout")
|
||||
case <-p.sameDocument:
|
||||
return nil
|
||||
case <-p.domReady:
|
||||
if waitUntil == "domcontentloaded" {
|
||||
return nil
|
||||
}
|
||||
case <-p.loads:
|
||||
if waitUntil == "load" {
|
||||
return nil
|
||||
}
|
||||
case <-ticker.C:
|
||||
state, err := p.navigationState()
|
||||
if err == nil && state.ready(targetURL, waitUntil) {
|
||||
return nil
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
return fmt.Errorf("webview navigation timeout")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type navigationState struct {
|
||||
URL string `json:"url"`
|
||||
ReadyState string `json:"readyState"`
|
||||
}
|
||||
|
||||
func (s navigationState) ready(targetURL string, waitUntil string) bool {
|
||||
if s.URL == "" || s.URL == "about:blank" {
|
||||
return false
|
||||
}
|
||||
switch waitUntil {
|
||||
case "domcontentloaded":
|
||||
if s.ReadyState == "loading" || s.ReadyState == "" {
|
||||
return false
|
||||
}
|
||||
default:
|
||||
if s.ReadyState != "complete" {
|
||||
return false
|
||||
}
|
||||
}
|
||||
if targetURL == "" {
|
||||
return true
|
||||
}
|
||||
if s.URL == targetURL {
|
||||
return true
|
||||
}
|
||||
target, err := parseURL(targetURL)
|
||||
if err != nil {
|
||||
return true
|
||||
}
|
||||
current, err := parseURL(s.URL)
|
||||
if err != nil {
|
||||
return true
|
||||
}
|
||||
if current.Scheme == target.Scheme && current.Host == target.Host && current.Path == target.Path {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func parseURL(raw string) (*url.URL, error) {
|
||||
return url.Parse(raw)
|
||||
}
|
||||
|
||||
func (p *pageWrapper) navigationState() (navigationState, error) {
|
||||
value, err := p.Execute(`() => ({
|
||||
url: String(location.href || ""),
|
||||
readyState: document.readyState || "",
|
||||
})`)
|
||||
if err != nil {
|
||||
return navigationState{}, err
|
||||
}
|
||||
stateMap, ok := value.(map[string]any)
|
||||
if !ok {
|
||||
return navigationState{}, fmt.Errorf("invalid navigation state")
|
||||
}
|
||||
state := navigationState{}
|
||||
if urlValue, ok := stateMap["url"].(string); ok {
|
||||
state.URL = urlValue
|
||||
}
|
||||
if readyValue, ok := stateMap["readyState"].(string); ok {
|
||||
state.ReadyState = readyValue
|
||||
}
|
||||
return state, nil
|
||||
}
|
||||
|
||||
func (p *pageWrapper) dispatch(fn func(w webview.WebView) error) error {
|
||||
p.mu.Lock()
|
||||
if p.closed {
|
||||
@@ -438,7 +414,12 @@ func (p *pageWrapper) dispatch(fn func(w webview.WebView) error) error {
|
||||
w.Dispatch(func() {
|
||||
done <- fn(w)
|
||||
})
|
||||
return <-done
|
||||
select {
|
||||
case err := <-done:
|
||||
return err
|
||||
case <-p.done:
|
||||
return fmt.Errorf("webview page is closed")
|
||||
}
|
||||
}
|
||||
|
||||
func (p *pageWrapper) nativeCookies() ([]nativeCookieJSON, error) {
|
||||
@@ -496,7 +477,31 @@ func (p *pageWrapper) handleCallback(payload string) error {
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if msg.Event == "domcontentloaded" {
|
||||
select {
|
||||
case p.domReady <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if msg.Event == "url-changed" {
|
||||
if msg.SameDocument {
|
||||
select {
|
||||
case p.sameDocument <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
p.mu.Lock()
|
||||
changed := p.url != msg.URL
|
||||
p.url = msg.URL
|
||||
p.mu.Unlock()
|
||||
if changed {
|
||||
p.events.Emit(enginewebview.Event{Name: "url-changed", Data: map[string]any{"url": msg.URL, "sameDocument": msg.SameDocument}})
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if msg.Load {
|
||||
p.events.Emit(enginewebview.Event{Name: "load", Data: map[string]any{"url": msg.URL}})
|
||||
select {
|
||||
case p.loads <- msg.URL:
|
||||
default:
|
||||
@@ -521,6 +526,22 @@ func (p *pageWrapper) handleCallback(payload string) error {
|
||||
}
|
||||
|
||||
func (p *pageWrapper) drainLoads() {
|
||||
for {
|
||||
select {
|
||||
case <-p.domReady:
|
||||
continue
|
||||
default:
|
||||
}
|
||||
break
|
||||
}
|
||||
for {
|
||||
select {
|
||||
case <-p.sameDocument:
|
||||
continue
|
||||
default:
|
||||
}
|
||||
break
|
||||
}
|
||||
for {
|
||||
select {
|
||||
case <-p.loads:
|
||||
@@ -582,19 +603,77 @@ func buildExecuteScript(callbackName string, reqID string, expression string, ar
|
||||
func buildLoadNotifyScript(callbackName string) string {
|
||||
callbackJSON, _ := json.Marshal(callbackName)
|
||||
return fmt.Sprintf(`(() => {
|
||||
window.addEventListener("load", () => {
|
||||
const __cb = globalThis[%s];
|
||||
if (typeof __cb !== "function") {
|
||||
return;
|
||||
}
|
||||
__cb(JSON.stringify({
|
||||
load: true,
|
||||
url: String(location.href || ""),
|
||||
}));
|
||||
}, { once: true });
|
||||
if (window !== window.top) return;
|
||||
// WebView2 runs init scripts in its internal error document too. Its load
|
||||
// must not complete goto() before the native NavigationCompleted failure.
|
||||
if (location.protocol === "chrome-error:") return;
|
||||
const send = (message) => {
|
||||
const cb = globalThis[%s];
|
||||
if (typeof cb === "function") cb(JSON.stringify(message));
|
||||
};
|
||||
let lastURL = String(location.href);
|
||||
const changed = () => {
|
||||
const url = String(location.href);
|
||||
if (url === lastURL) return;
|
||||
lastURL = url;
|
||||
send({event: "url-changed", url, sameDocument: true});
|
||||
};
|
||||
send({event: "url-changed", url: lastURL, sameDocument: false});
|
||||
for (const name of ["pushState", "replaceState"]) {
|
||||
const original = history[name];
|
||||
history[name] = function(...args) {
|
||||
const result = Reflect.apply(original, this, args);
|
||||
changed();
|
||||
return result;
|
||||
};
|
||||
}
|
||||
window.addEventListener("hashchange", changed);
|
||||
window.addEventListener("popstate", changed);
|
||||
window.addEventListener("pageshow", (event) => {
|
||||
if (event.persisted) send({event: "url-changed", url: String(location.href), sameDocument: false});
|
||||
else changed();
|
||||
});
|
||||
window.addEventListener("DOMContentLoaded", () => send({event: "domcontentloaded"}), {once:true});
|
||||
window.addEventListener("load", () => send({load: true, url: String(location.href)}), {once: true});
|
||||
})();`, string(callbackJSON))
|
||||
}
|
||||
|
||||
func (p *pageWrapper) On(name string, handler func(enginewebview.Event)) (func(), error) {
|
||||
return p.events.On(name, handler)
|
||||
}
|
||||
|
||||
func (p *pageWrapper) failPendingLocked() {
|
||||
for id, ch := range p.pending {
|
||||
ch <- evalResult{err: fmt.Errorf("webview page is closed")}
|
||||
delete(p.pending, id)
|
||||
}
|
||||
}
|
||||
|
||||
func (p *pageWrapper) handleNativeEvent(event webview.Event) {
|
||||
switch event.Name {
|
||||
case "load-error":
|
||||
p.events.Emit(enginewebview.Event{Name: "load-error", Data: map[string]any{"url": event.URL, "message": event.Message}})
|
||||
select {
|
||||
case p.navigationErrors <- errors.New(event.Message):
|
||||
default:
|
||||
}
|
||||
case "closed":
|
||||
p.mu.Lock()
|
||||
api := p.closed
|
||||
p.closed = true
|
||||
p.failPendingLocked()
|
||||
p.mu.Unlock()
|
||||
reason := "user"
|
||||
if api {
|
||||
reason = "api"
|
||||
}
|
||||
p.events.Emit(enginewebview.Event{Name: "closed", Data: map[string]any{"reason": reason}})
|
||||
if runtimepkg.GOOS == "linux" && !api {
|
||||
p.view.Dispatch(func() { p.view.Destroy(); close(p.done) })
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func defaultWindowDimension(value int, fallback int) int {
|
||||
if value > 0 {
|
||||
return value
|
||||
@@ -610,3 +689,12 @@ func firstNonEmpty(values ...string) string {
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (p *Provider) RemoveProfile(profileID, dataPath string) error {
|
||||
done := make(chan error, 1)
|
||||
run := func() { done <- webview.RemoveProfile(dataPath) }
|
||||
if !postMainThreadTask(run) {
|
||||
run()
|
||||
}
|
||||
return <-done
|
||||
}
|
||||
@@ -13,3 +13,7 @@ func TestProviderContract(t *testing.T) {
|
||||
CookieDomainMode: integrationtest.CookieDomainModeRequired,
|
||||
})
|
||||
}
|
||||
|
||||
func TestProviderProfiles(t *testing.T) { integrationtest.RunProfileContract(t, New()) }
|
||||
|
||||
func TestProviderEvents(t *testing.T) { integrationtest.RunEventContract(t, New()) }
|
||||
@@ -1,10 +1,14 @@
|
||||
package integrationtest
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"fmt"
|
||||
webviewproxy "github.com/GopeedLab/gopeed/internal/webview/proxy"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -104,8 +108,10 @@ func RunProviderContract(t *testing.T, provider enginewebview.Provider, opts Con
|
||||
t.Fatalf("expected click count to be 1, got %v", clickCount)
|
||||
}
|
||||
|
||||
// Hidden WebViews can throttle timers on shared CI runners. Keep the
|
||||
// positive asynchronous waits within the same budget as navigation.
|
||||
found, err := page.WaitForSelector("#delayed", map[string]any{
|
||||
"timeoutMs": 5000,
|
||||
"timeoutMs": 10000,
|
||||
"pollIntervalMs": 25,
|
||||
"visible": true,
|
||||
})
|
||||
@@ -113,11 +119,18 @@ func RunProviderContract(t *testing.T, provider enginewebview.Provider, opts Con
|
||||
t.Fatalf("waitForSelector failed: %v", err)
|
||||
}
|
||||
if !found {
|
||||
t.Fatal("expected delayed selector to appear")
|
||||
diagnostic, diagnosticErr := page.Execute(`() => {
|
||||
const element = document.querySelector('#delayed');
|
||||
const rect = element && element.getBoundingClientRect();
|
||||
return {url: location.href, readyState: document.readyState,
|
||||
visibility: document.visibilityState, exists: !!element,
|
||||
width: rect && rect.width, height: rect && rect.height};
|
||||
}`)
|
||||
t.Fatalf("expected delayed selector to appear: state=%v, diagnostic error=%v", diagnostic, diagnosticErr)
|
||||
}
|
||||
|
||||
value, err := page.WaitForFunction(`() => window.__readyValue`, map[string]any{
|
||||
"timeoutMs": 5000,
|
||||
"timeoutMs": 10000,
|
||||
"pollIntervalMs": 25,
|
||||
})
|
||||
if err != nil {
|
||||
@@ -219,7 +232,19 @@ func openTestPage(t *testing.T, provider enginewebview.Provider, opts enginewebv
|
||||
}))
|
||||
t.Cleanup(server.Close)
|
||||
|
||||
runtime := enginewebview.NewRuntime(provider, true)
|
||||
proxy, err := webviewproxy.New(nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { proxy.Close() })
|
||||
path := filepath.Join(t.TempDir(), "profile")
|
||||
sum := sha256.Sum256([]byte(path))
|
||||
profileID := fmt.Sprintf("%x-%x-%x-%x-%x", sum[:4], sum[4:6], sum[6:8], sum[8:10], sum[10:16])
|
||||
proxyURL := proxy.URL()
|
||||
if runtime.GOOS == "darwin" {
|
||||
proxyURL = proxy.SOCKSURL()
|
||||
}
|
||||
runtime := enginewebview.NewRuntime(&configuredOpener{provider: provider, profileID: profileID, dataPath: path, proxyURL: proxyURL}, true)
|
||||
page, err := runtime.Open(map[string]any{
|
||||
"headless": opts.Headless,
|
||||
"debug": opts.Debug,
|
||||
@@ -232,7 +257,14 @@ func openTestPage(t *testing.T, provider enginewebview.Provider, opts enginewebv
|
||||
t.Fatalf("failed to open webview: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
_ = runtime.Close()
|
||||
if err := runtime.Close(); err != nil {
|
||||
t.Errorf("close contract page: %v", err)
|
||||
}
|
||||
if remover, ok := provider.(enginewebview.ProfileRemover); ok {
|
||||
if err := remover.RemoveProfile(profileID, path); err != nil {
|
||||
t.Errorf("remove contract profile: %v", err)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
return page, server, server.URL
|
||||
@@ -380,3 +412,15 @@ func hasCookie(cookies []enginewebview.Cookie, name string, value string) bool {
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
type configuredOpener struct {
|
||||
provider enginewebview.Provider
|
||||
profileID, dataPath, proxyURL string
|
||||
}
|
||||
|
||||
func (o *configuredOpener) Open(opts enginewebview.OpenOptions) (enginewebview.Page, error) {
|
||||
opts.ProfileID = o.profileID
|
||||
opts.DataPath = o.dataPath
|
||||
opts.ProxyURL = o.proxyURL
|
||||
return o.provider.Open(opts)
|
||||
}
|
||||
@@ -0,0 +1,110 @@
|
||||
package integrationtest
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
wv "github.com/GopeedLab/gopeed/pkg/download/engine/webview"
|
||||
)
|
||||
|
||||
// RunEventContract exercises the same public semantics on native and RPC hosts.
|
||||
func RunEventContract(t *testing.T, provider wv.Provider) {
|
||||
t.Helper()
|
||||
page, _, url := openTestPage(t, provider, wv.OpenOptions{Headless: true, Title: "WebView event contract"})
|
||||
events := make(chan wv.Event, 100)
|
||||
for _, name := range []string{"url-changed", "load", "load-error", "closed"} {
|
||||
if _, err := page.On(name, func(e wv.Event) { events <- e }); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
next := func(name string) wv.Event {
|
||||
t.Helper()
|
||||
select {
|
||||
case event := <-events:
|
||||
if event.Name != name {
|
||||
t.Fatalf("got event %s (%v), want %s", event.Name, event.Data, name)
|
||||
}
|
||||
return event
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatalf("timed out waiting for %s", name)
|
||||
}
|
||||
return wv.Event{}
|
||||
}
|
||||
quiet := func() {
|
||||
t.Helper()
|
||||
select {
|
||||
case e := <-events:
|
||||
t.Fatalf("unexpected event: %+v", e)
|
||||
case <-time.After(150 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
quiet()
|
||||
if err := page.AddInitScript(`globalThis.__initBeforePage = document.readyState`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := page.Goto(url, map[string]any{"timeoutMs": 10000}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
e := next("url-changed")
|
||||
if strings.TrimRight(fmt.Sprint(e.Data["url"]), "/") != strings.TrimRight(url, "/") || e.Data["sameDocument"] != false {
|
||||
t.Fatal(e)
|
||||
}
|
||||
next("load")
|
||||
if result, err := page.Execute(`() => globalThis.__initBeforePage`); err != nil || result != "loading" {
|
||||
t.Fatalf("init script: %v %v", result, err)
|
||||
}
|
||||
if err := page.AddInitScript(`globalThis.__onlyNext = true`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if result, err := page.Execute(`() => typeof globalThis.__onlyNext`); err != nil || result != "undefined" {
|
||||
t.Fatalf("init ran in current page: %v %v", result, err)
|
||||
}
|
||||
for _, expression := range []string{
|
||||
`() => { history.pushState({}, '', '#push'); }`,
|
||||
`() => { history.replaceState({}, '', '#replace'); }`,
|
||||
`() => { location.hash = 'hash'; }`,
|
||||
} {
|
||||
if _, err := page.Execute(expression); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
e := next("url-changed")
|
||||
if e.Data["sameDocument"] != true {
|
||||
t.Fatal(e)
|
||||
}
|
||||
quiet()
|
||||
}
|
||||
if _, err := page.Execute(`() => { location.reload(); }`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
next("load") // Same address on refresh is not a URL change.
|
||||
if result, err := page.Execute(`() => globalThis.__onlyNext`); err != nil || result != true {
|
||||
t.Fatalf("next init: %v %v", result, err)
|
||||
}
|
||||
if _, err := page.Execute(`() => { const f=document.createElement('iframe'); f.src='/next'; document.body.append(f); }`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
quiet()
|
||||
// The fixture serves plain HTTP. Attempting TLS against it produces a
|
||||
// real navigation failure through both HTTP CONNECT and SOCKS proxies.
|
||||
// An unreachable HTTP origin would instead become a valid 502 document
|
||||
// when the host's HTTP proxy handles the connection error.
|
||||
failedURL := strings.Replace(url, "http://", "https://", 1) + "/failure"
|
||||
if err := page.Goto(failedURL, map[string]any{"timeoutMs": 3000}); err == nil {
|
||||
t.Fatal("failed navigation succeeded")
|
||||
}
|
||||
e = next("load-error")
|
||||
if e.Data["message"] == "" || !strings.Contains(fmt.Sprint(e.Data["url"]), "/failure") {
|
||||
t.Fatal(e)
|
||||
}
|
||||
quiet() // A handled failure must not load a browser-generated fallback page.
|
||||
if err := page.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
e = next("closed")
|
||||
if e.Data["reason"] != "api" {
|
||||
t.Fatal(e)
|
||||
}
|
||||
quiet()
|
||||
}
|
||||
@@ -0,0 +1,247 @@
|
||||
package integrationtest
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"runtime"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
webviewproxy "github.com/GopeedLab/gopeed/internal/webview/proxy"
|
||||
webview "github.com/GopeedLab/gopeed/pkg/download/engine/webview"
|
||||
)
|
||||
|
||||
// RunProfileContract exercises the same host API through both native and RPC
|
||||
// providers. It never bypasses the host bridge with direct Flutter plugin calls.
|
||||
func RunProfileContract(t *testing.T, provider webview.Provider) {
|
||||
t.Helper()
|
||||
waitForProviderAvailable(t, provider, defaultAvailabilityTimeout)
|
||||
remover, ok := provider.(webview.ProfileRemover)
|
||||
if !ok {
|
||||
t.Fatal("provider does not implement host profile removal")
|
||||
}
|
||||
proxyURL, secureHits := profileProxy(t)
|
||||
a := newContractProfile(t, provider, remover, proxyURL)
|
||||
b := newContractProfile(t, provider, remover, proxyURL)
|
||||
first := a.open(t)
|
||||
assertProfileValue(t, first, `async () => { localStorage.setItem('owner','a'); return 'a'; }`, "a")
|
||||
assertProfileValue(t, first, profileDatabaseScript(true), "saved")
|
||||
if err := first.SetCookie(webview.Cookie{Name: "session", Value: "a", Domain: "profiles.invalid", Path: "/", HTTPOnly: true, Expires: time.Now().Add(time.Hour)}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
second := b.open(t)
|
||||
assertProfileValue(t, second, `() => localStorage.getItem('owner')`, nil)
|
||||
assertProfileValue(t, second, profileDatabaseScript(false), nil)
|
||||
assertProfileCookies(t, second, "", "new profile B")
|
||||
assertProfileValue(t, second, `() => { localStorage.setItem('owner','b'); return 'b'; }`, "b")
|
||||
if err := second.SetCookie(webview.Cookie{Name: "session", Value: "b", Domain: "profiles.invalid", Path: "/", HTTPOnly: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
shared := a.open(t)
|
||||
assertProfileValue(t, shared, `() => localStorage.getItem('owner')`, "a")
|
||||
assertProfileValue(t, shared, profileDatabaseScript(false), "saved")
|
||||
assertProfileCookies(t, shared, "a", "shared profile A")
|
||||
storedCookies, err := shared.GetCookies()
|
||||
if err != nil || len(storedCookies) != 1 || storedCookies[0].Expires.Before(time.Now()) {
|
||||
t.Fatalf("persistent cookie lost its expiration: %#v, error %v", storedCookies, err)
|
||||
}
|
||||
assertProfileValue(t, shared, `() => document.cookie`, "") // HttpOnly stays hidden.
|
||||
if err := second.ClearCookies(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
assertProfileCookies(t, first, "a", "clearing B leaves A intact")
|
||||
if err := second.SetCookie(webview.Cookie{Name: "session", Value: "b", Domain: "profiles.invalid", Path: "/", HTTPOnly: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a.close(t)
|
||||
reopened := a.open(t)
|
||||
assertProfileValue(t, reopened, `() => localStorage.getItem('owner')`, "a")
|
||||
assertProfileValue(t, reopened, profileDatabaseScript(false), "saved")
|
||||
assertProfileCookies(t, reopened, "a", "reopened profile A")
|
||||
// Include a session cookie immediately before deletion to catch stale
|
||||
// in-process CookieStore caches when the same profile is recreated.
|
||||
if err := reopened.SetCookie(webview.Cookie{Name: "session", Value: "remove-me", Domain: "profiles.invalid", Path: "/", HTTPOnly: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a.close(t)
|
||||
a.remove(t)
|
||||
a.remove(t) // Retrying a successful uninstall is harmless.
|
||||
reinstalled := a.open(t)
|
||||
assertProfileValue(t, reinstalled, `() => localStorage.getItem('owner')`, nil)
|
||||
assertProfileValue(t, reinstalled, profileDatabaseScript(false), nil)
|
||||
assertProfileCookies(t, reinstalled, "", "reinstalled profile A")
|
||||
assertProfileValue(t, second, `() => localStorage.getItem('owner')`, "b")
|
||||
assertProfileCookies(t, second, "b", "B after removing A")
|
||||
// Refuse the HTTPS tunnel deliberately. Seeing it at our proxy proves HTTPS
|
||||
// routing without adding a certificate-validation bypass to the test host.
|
||||
// WebView2 may commit its built-in error page and report navigation as
|
||||
// complete. The invariant here is that the HTTPS tunnel reaches the proxy.
|
||||
_ = reinstalled.Goto("https://profiles.invalid/", webview.GotoOptions{TimeoutMS: 5000})
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
for secureHits.Load() == 0 && time.Now().Before(deadline) {
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
if secureHits.Load() == 0 {
|
||||
t.Fatal("HTTPS bypassed the host proxy")
|
||||
}
|
||||
}
|
||||
|
||||
type contractProfile struct {
|
||||
provider webview.Provider
|
||||
remover webview.ProfileRemover
|
||||
options webview.OpenOptions
|
||||
pages []webview.Page
|
||||
}
|
||||
|
||||
func newContractProfile(t *testing.T, provider webview.Provider, remover webview.ProfileRemover, proxyURL string) *contractProfile {
|
||||
t.Helper()
|
||||
path := filepath.Join(t.TempDir(), "profile")
|
||||
hash := sha256.Sum256([]byte(path))
|
||||
id := fmt.Sprintf("%x-%x-%x-%x-%x", hash[:4], hash[4:6], hash[6:8], hash[8:10], hash[10:16])
|
||||
p := &contractProfile{provider: provider, remover: remover, options: webview.OpenOptions{ProfileID: id, DataPath: path, ProxyURL: proxyURL, Headless: true, Title: "Gopeed profile contract"}}
|
||||
t.Cleanup(func() { p.close(t); p.remove(t) })
|
||||
return p
|
||||
}
|
||||
func (p *contractProfile) open(t *testing.T) webview.Page {
|
||||
t.Helper()
|
||||
page, err := p.provider.Open(p.options)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
p.pages = append(p.pages, page)
|
||||
if err := page.Goto("http://profiles.invalid/", webview.GotoOptions{TimeoutMS: 10000}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return page
|
||||
}
|
||||
func (p *contractProfile) close(t *testing.T) {
|
||||
t.Helper()
|
||||
for _, page := range p.pages {
|
||||
if err := page.Close(); err != nil {
|
||||
t.Errorf("close profile page: %v", err)
|
||||
}
|
||||
}
|
||||
p.pages = nil
|
||||
}
|
||||
func (p *contractProfile) remove(t *testing.T) {
|
||||
t.Helper()
|
||||
if err := p.remover.RemoveProfile(p.options.ProfileID, p.options.DataPath); err != nil {
|
||||
t.Errorf("remove profile: %v", err)
|
||||
}
|
||||
}
|
||||
func assertProfileValue(t *testing.T, page webview.Page, script string, want any) {
|
||||
t.Helper()
|
||||
got, err := page.Execute(script)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("script %s: got %#v, want %#v", script, got, want)
|
||||
}
|
||||
}
|
||||
func assertProfileCookies(t *testing.T, page webview.Page, want, stage string) {
|
||||
t.Helper()
|
||||
// Native cookie managers propagate updates across browser processes
|
||||
// asynchronously. Wait for the expected state rather than assuming the
|
||||
// first IPC read already includes the preceding write.
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
for {
|
||||
cookies, err := page.GetCookies()
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", stage, err)
|
||||
}
|
||||
matches := len(cookies) == 0 && want == "" || len(cookies) == 1 && want != "" && cookies[0].Name == "session" && cookies[0].Value == want && cookies[0].HTTPOnly
|
||||
if matches {
|
||||
return
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("%s: cookies %#v, want %q", stage, cookies, want)
|
||||
}
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
func profileDatabaseScript(write bool) string {
|
||||
operation := "store.get('owner')"
|
||||
if write {
|
||||
operation = "store.put('saved','owner')"
|
||||
}
|
||||
return fmt.Sprintf(`async () => await new Promise((resolve,reject)=>{
|
||||
const request=indexedDB.open('profile-contract',1);
|
||||
request.onupgradeneeded=()=>request.result.createObjectStore('state');
|
||||
request.onerror=()=>reject(String(request.error));
|
||||
request.onsuccess=()=>{
|
||||
const db=request.result, tx=db.transaction('state','readwrite'), store=tx.objectStore('state');
|
||||
const op=%s; let value;
|
||||
op.onsuccess=()=>value=%s;
|
||||
tx.oncomplete=()=>{db.close();resolve(value===undefined?null:value)};
|
||||
tx.onerror=()=>{db.close();reject(String(tx.error))};
|
||||
};
|
||||
})`, operation, map[bool]string{true: "'saved'", false: "op.result"}[write])
|
||||
}
|
||||
|
||||
func profileProxy(t *testing.T) (string, *atomic.Int32) {
|
||||
t.Helper()
|
||||
origin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "text/html")
|
||||
w.Header().Set("Cache-Control", "no-store")
|
||||
fmt.Fprint(w, "<!doctype html><title>Profile contract</title>")
|
||||
}))
|
||||
t.Cleanup(origin.Close)
|
||||
secureHits := &atomic.Int32{}
|
||||
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodConnect {
|
||||
origin.Config.Handler.ServeHTTP(w, r)
|
||||
return
|
||||
}
|
||||
if r.Host == "profiles.invalid:443" {
|
||||
secureHits.Add(1)
|
||||
http.Error(w, "HTTPS routing fixture rejects tunnels", http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
if r.Host != "profiles.invalid:80" {
|
||||
http.Error(w, "unexpected target", http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
remote, err := net.DialTimeout("tcp", origin.Listener.Addr().String(), 5*time.Second)
|
||||
if err != nil {
|
||||
http.Error(w, "origin unavailable", http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
defer remote.Close()
|
||||
client, buffer, err := w.(http.Hijacker).Hijack()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer client.Close()
|
||||
if _, err := buffer.WriteString("HTTP/1.1 200 Connection Established\r\n\r\n"); err != nil {
|
||||
return
|
||||
}
|
||||
if err := buffer.Flush(); err != nil {
|
||||
return
|
||||
}
|
||||
go func() { io.Copy(remote, buffer); remote.Close() }()
|
||||
io.Copy(client, remote)
|
||||
}))
|
||||
t.Cleanup(upstream.Close)
|
||||
upstreamURL, err := url.Parse(upstream.URL)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
proxy, err := webviewproxy.New(func(*http.Request) (*url.URL, error) { return upstreamURL, nil })
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { proxy.Close() })
|
||||
if runtime.GOOS == "darwin" {
|
||||
return proxy.SOCKSURL(), secureHits
|
||||
}
|
||||
return proxy.URL(), secureHits
|
||||
}
|
||||
@@ -0,0 +1,341 @@
|
||||
// Package proxy provides the host-owned WebView forward proxy. Browser engines
|
||||
// always use this endpoint; Gopeed decides the upstream route for each request.
|
||||
package proxy
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/armon/go-socks5"
|
||||
xproxy "golang.org/x/net/proxy"
|
||||
)
|
||||
|
||||
type Selector func(*http.Request) (*url.URL, error)
|
||||
|
||||
type Server struct {
|
||||
listener net.Listener
|
||||
socksListener net.Listener
|
||||
server *http.Server
|
||||
transport *http.Transport
|
||||
selectProxy Selector
|
||||
mu sync.Mutex
|
||||
connections map[io.Closer]struct{}
|
||||
closed bool
|
||||
}
|
||||
|
||||
func New(selectProxy Selector) (*Server, error) {
|
||||
if selectProxy == nil {
|
||||
selectProxy = func(*http.Request) (*url.URL, error) { return nil, nil }
|
||||
}
|
||||
listener, err := net.Listen("tcp4", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
s := &Server{listener: listener, selectProxy: selectProxy, connections: make(map[io.Closer]struct{})}
|
||||
s.transport = &http.Transport{Proxy: selectProxy, DisableKeepAlives: true, TLSHandshakeTimeout: 15 * time.Second, ResponseHeaderTimeout: 30 * time.Second}
|
||||
s.server = &http.Server{Handler: s, ReadHeaderTimeout: 15 * time.Second}
|
||||
s.socksListener, err = net.Listen("tcp4", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
listener.Close()
|
||||
return nil, err
|
||||
}
|
||||
socks, err := socks5.New(&socks5.Config{Logger: log.New(io.Discard, "", 0), Resolver: unresolvedDNS{}, Dial: func(ctx context.Context, network, address string) (net.Conn, error) {
|
||||
_, port, _ := net.SplitHostPort(address)
|
||||
scheme := "https"
|
||||
if port == "80" {
|
||||
scheme = "http"
|
||||
}
|
||||
request := &http.Request{URL: &url.URL{Scheme: scheme, Host: address}, Header: make(http.Header)}
|
||||
upstream, err := selectProxy(request)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
timed, cancel := context.WithTimeout(ctx, 30*time.Second)
|
||||
defer cancel()
|
||||
conn, err := dialTunnel(timed, address, upstream)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return s.track(conn)
|
||||
}})
|
||||
if err != nil {
|
||||
listener.Close()
|
||||
s.socksListener.Close()
|
||||
return nil, err
|
||||
}
|
||||
go s.server.Serve(listener)
|
||||
go socks.Serve(&trackedListener{Listener: s.socksListener, server: s})
|
||||
return s, nil
|
||||
}
|
||||
|
||||
func (s *Server) URL() string { return "http://" + s.listener.Addr().String() }
|
||||
|
||||
func (s *Server) SOCKSURL() string { return "socks5://" + s.socksListener.Addr().String() }
|
||||
|
||||
func (s *Server) Close() error {
|
||||
s.mu.Lock()
|
||||
s.closed = true
|
||||
connections := make([]io.Closer, 0, len(s.connections))
|
||||
for c := range s.connections {
|
||||
connections = append(connections, c)
|
||||
}
|
||||
s.mu.Unlock()
|
||||
for _, c := range connections {
|
||||
c.Close()
|
||||
}
|
||||
s.socksListener.Close()
|
||||
s.transport.CloseIdleConnections()
|
||||
return s.server.Close()
|
||||
}
|
||||
|
||||
func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method == http.MethodConnect {
|
||||
s.connect(w, r)
|
||||
return
|
||||
}
|
||||
if !r.URL.IsAbs() || (r.URL.Scheme != "http" && r.URL.Scheme != "https") {
|
||||
http.Error(w, "absolute HTTP URL required", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
out := r.Clone(r.Context())
|
||||
out.RequestURI = ""
|
||||
upgrade := out.Header.Get("Upgrade")
|
||||
stripHopHeaders(out.Header)
|
||||
if upgrade != "" {
|
||||
out.Header.Set("Connection", "Upgrade")
|
||||
out.Header.Set("Upgrade", upgrade)
|
||||
}
|
||||
resp, err := s.transport.RoundTrip(out)
|
||||
if err != nil {
|
||||
http.Error(w, "upstream request failed", http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode == http.StatusSwitchingProtocols {
|
||||
remote, ok := resp.Body.(io.ReadWriteCloser)
|
||||
if !ok {
|
||||
http.Error(w, "invalid upstream upgrade", 502)
|
||||
return
|
||||
}
|
||||
client, buffer, err := w.(http.Hijacker).Hijack()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer client.Close()
|
||||
resp.Body = nil
|
||||
if err := resp.Write(buffer); err != nil {
|
||||
return
|
||||
}
|
||||
if err := buffer.Flush(); err != nil {
|
||||
return
|
||||
}
|
||||
s.bridge(client, buffer, remote)
|
||||
return
|
||||
}
|
||||
stripHopHeaders(resp.Header)
|
||||
for k, v := range resp.Header {
|
||||
w.Header()[k] = v
|
||||
}
|
||||
w.WriteHeader(resp.StatusCode)
|
||||
io.Copy(w, resp.Body)
|
||||
}
|
||||
|
||||
func stripHopHeaders(h http.Header) {
|
||||
for _, key := range strings.Split(h.Get("Connection"), ",") {
|
||||
h.Del(strings.TrimSpace(key))
|
||||
}
|
||||
for _, key := range []string{"Connection", "Proxy-Connection", "Proxy-Authorization", "Proxy-Authenticate", "Keep-Alive", "TE", "Trailer", "Transfer-Encoding", "Upgrade"} {
|
||||
h.Del(key)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Server) connect(w http.ResponseWriter, r *http.Request) {
|
||||
if _, _, err := net.SplitHostPort(r.Host); err != nil {
|
||||
http.Error(w, "invalid tunnel destination", 400)
|
||||
return
|
||||
}
|
||||
target := r.Clone(r.Context())
|
||||
target.URL = &url.URL{Scheme: "https", Host: r.Host}
|
||||
upstream, err := s.selectProxy(target)
|
||||
if err != nil {
|
||||
http.Error(w, "proxy selection failed", 502)
|
||||
return
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(r.Context(), 30*time.Second)
|
||||
defer cancel()
|
||||
remote, err := dialTunnel(ctx, r.Host, upstream)
|
||||
if err != nil {
|
||||
http.Error(w, "upstream connection failed", 502)
|
||||
return
|
||||
}
|
||||
defer remote.Close()
|
||||
client, buffer, err := w.(http.Hijacker).Hijack()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer client.Close()
|
||||
if _, err := buffer.WriteString("HTTP/1.1 200 Connection Established\r\n\r\n"); err != nil {
|
||||
return
|
||||
}
|
||||
if err := buffer.Flush(); err != nil {
|
||||
return
|
||||
}
|
||||
s.bridge(client, buffer, remote)
|
||||
}
|
||||
|
||||
func (s *Server) bridge(client net.Conn, buffer *bufio.ReadWriter, remote io.ReadWriteCloser) {
|
||||
s.mu.Lock()
|
||||
if s.closed {
|
||||
s.mu.Unlock()
|
||||
return
|
||||
}
|
||||
s.connections[client] = struct{}{}
|
||||
s.connections[remote] = struct{}{}
|
||||
s.mu.Unlock()
|
||||
defer func() { s.mu.Lock(); delete(s.connections, client); delete(s.connections, remote); s.mu.Unlock() }()
|
||||
|
||||
done := make(chan struct{})
|
||||
go func() { io.Copy(remote, buffer); remote.Close(); close(done) }()
|
||||
io.Copy(client, remote)
|
||||
client.Close()
|
||||
<-done
|
||||
}
|
||||
|
||||
type bufferedConn struct {
|
||||
net.Conn
|
||||
reader *bufio.Reader
|
||||
}
|
||||
|
||||
func (c *bufferedConn) Read(p []byte) (int, error) { return c.reader.Read(p) }
|
||||
|
||||
func dialTunnel(ctx context.Context, target string, upstream *url.URL) (net.Conn, error) {
|
||||
dialer := &net.Dialer{}
|
||||
if upstream == nil {
|
||||
return dialer.DialContext(ctx, "tcp", target)
|
||||
}
|
||||
if upstream.Scheme == "socks5" || upstream.Scheme == "socks5h" {
|
||||
u := *upstream
|
||||
u.Scheme = "socks5"
|
||||
d, err := xproxy.FromURL(&u, dialer)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return d.(xproxy.ContextDialer).DialContext(ctx, "tcp", target)
|
||||
}
|
||||
if upstream.Scheme != "http" && upstream.Scheme != "https" {
|
||||
return nil, errors.New("unsupported proxy scheme")
|
||||
}
|
||||
host := upstream.Host
|
||||
if upstream.Port() == "" {
|
||||
port := "80"
|
||||
if upstream.Scheme == "https" {
|
||||
port = "443"
|
||||
}
|
||||
host = net.JoinHostPort(upstream.Hostname(), port)
|
||||
}
|
||||
conn, err := dialer.DialContext(ctx, "tcp", host)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ok := false
|
||||
defer func() {
|
||||
if !ok {
|
||||
conn.Close()
|
||||
}
|
||||
}()
|
||||
if deadline, exists := ctx.Deadline(); exists {
|
||||
conn.SetDeadline(deadline)
|
||||
}
|
||||
if upstream.Scheme == "https" {
|
||||
secure := tls.Client(conn, &tls.Config{ServerName: upstream.Hostname(), MinVersion: tls.VersionTLS12})
|
||||
if err := secure.HandshakeContext(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
conn = secure
|
||||
}
|
||||
request := &http.Request{Method: http.MethodConnect, URL: &url.URL{Opaque: target}, Host: target, Header: make(http.Header)}
|
||||
if upstream.User != nil {
|
||||
password, _ := upstream.User.Password()
|
||||
request.Header.Set("Proxy-Authorization", "Basic "+base64.StdEncoding.EncodeToString([]byte(upstream.User.Username()+":"+password)))
|
||||
}
|
||||
if err := request.Write(conn); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
reader := bufio.NewReader(conn)
|
||||
response, err := http.ReadResponse(reader, request)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if response.StatusCode != 200 {
|
||||
response.Body.Close()
|
||||
return nil, fmt.Errorf("upstream CONNECT status %d", response.StatusCode)
|
||||
}
|
||||
conn.SetDeadline(time.Time{})
|
||||
ok = true
|
||||
return &bufferedConn{Conn: conn, reader: reader}, nil
|
||||
}
|
||||
|
||||
// Leave hostname resolution to the selected upstream (SOCKS remote DNS / CONNECT).
|
||||
type unresolvedDNS struct{}
|
||||
|
||||
func (unresolvedDNS) Resolve(ctx context.Context, name string) (context.Context, net.IP, error) {
|
||||
return ctx, nil, nil
|
||||
}
|
||||
|
||||
type trackedListener struct {
|
||||
net.Listener
|
||||
server *Server
|
||||
}
|
||||
|
||||
func (l *trackedListener) Accept() (net.Conn, error) {
|
||||
c, err := l.Listener.Accept()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return l.server.track(c)
|
||||
}
|
||||
|
||||
type trackedConn struct {
|
||||
net.Conn
|
||||
server *Server
|
||||
}
|
||||
|
||||
func (c *trackedConn) Close() error {
|
||||
err := c.Conn.Close()
|
||||
c.server.mu.Lock()
|
||||
delete(c.server.connections, c)
|
||||
c.server.mu.Unlock()
|
||||
return err
|
||||
}
|
||||
func (s *Server) track(c net.Conn) (net.Conn, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.closed {
|
||||
c.Close()
|
||||
return nil, net.ErrClosed
|
||||
}
|
||||
tracked := &trackedConn{Conn: c, server: s}
|
||||
s.connections[tracked] = struct{}{}
|
||||
return tracked, nil
|
||||
}
|
||||
|
||||
func (c *trackedConn) CloseWrite() error { return closeWrite(c.Conn) }
|
||||
func (c *bufferedConn) CloseWrite() error { return closeWrite(c.Conn) }
|
||||
func closeWrite(c net.Conn) error {
|
||||
if writer, ok := c.(interface{ CloseWrite() error }); ok {
|
||||
return writer.CloseWrite()
|
||||
}
|
||||
return c.Close()
|
||||
}
|
||||
@@ -0,0 +1,315 @@
|
||||
package proxy
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"errors"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/armon/go-socks5"
|
||||
)
|
||||
|
||||
func TestRejectedProxyRequests(t *testing.T) {
|
||||
secret := "upstream-password"
|
||||
for _, tc := range []struct {
|
||||
name, method, target string
|
||||
status int
|
||||
}{
|
||||
{"relative URL", "GET", "/relative", 400},
|
||||
{"unsupported URL", "GET", "ftp://example.com/file", 400},
|
||||
{"invalid CONNECT", "CONNECT", "example.com", 400},
|
||||
{"HTTP selection failure", "GET", "http://example.com/", 502},
|
||||
{"CONNECT selection failure", "CONNECT", "example.com:443", 502},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
s := &Server{selectProxy: func(*http.Request) (*url.URL, error) { return nil, errors.New(secret) }}
|
||||
s.transport = &http.Transport{Proxy: s.selectProxy}
|
||||
defer s.transport.CloseIdleConnections()
|
||||
response := httptest.NewRecorder()
|
||||
s.ServeHTTP(response, httptest.NewRequest(tc.method, tc.target, nil))
|
||||
if response.Code != tc.status || strings.Contains(response.Body.String(), secret) {
|
||||
t.Fatalf("unexpected proxy rejection: %d %q", response.Code, response.Body.String())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestTunnelRejectsUnsafeUpstreams(t *testing.T) {
|
||||
rejected := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
http.Error(w, "authentication required", http.StatusProxyAuthRequired)
|
||||
}))
|
||||
defer rejected.Close()
|
||||
secure := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
t.Error("untrusted TLS proxy received a request")
|
||||
}))
|
||||
defer secure.Close()
|
||||
for _, upstream := range []string{"ftp://127.0.0.1", rejected.URL, secure.URL} {
|
||||
t.Run(upstream, func(t *testing.T) {
|
||||
selected, err := url.Parse(upstream)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
|
||||
defer cancel()
|
||||
conn, err := dialTunnel(ctx, "example.com:443", selected)
|
||||
if conn != nil {
|
||||
conn.Close()
|
||||
}
|
||||
if err == nil {
|
||||
t.Fatal("unsafe upstream unexpectedly accepted")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRoutesAndAuthentication(t *testing.T) {
|
||||
origin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Header.Get("Proxy-Authorization") != "" {
|
||||
t.Error("proxy credentials leaked to origin")
|
||||
}
|
||||
w.Write([]byte("origin"))
|
||||
}))
|
||||
defer origin.Close()
|
||||
secure := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Write([]byte("secure")) }))
|
||||
defer secure.Close()
|
||||
upstream, err := New(func(*http.Request) (*url.URL, error) { return nil, nil })
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer upstream.Close()
|
||||
var authenticated atomic.Int32
|
||||
authProxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Header.Get("Proxy-Authorization") != "Basic dXNlcjpwYXNz" {
|
||||
http.Error(w, "authentication required", 407)
|
||||
return
|
||||
}
|
||||
authenticated.Add(1)
|
||||
upstream.ServeHTTP(w, r)
|
||||
}))
|
||||
defer authProxy.Close()
|
||||
socks, err := socks5.New(&socks5.Config{Credentials: socks5.StaticCredentials{"user": "pass"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
listener, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer listener.Close()
|
||||
go socks.Serve(listener)
|
||||
for _, route := range []string{"", authProxy.URL, "socks5://" + listener.Addr().String()} {
|
||||
t.Run(route, func(t *testing.T) {
|
||||
var selected *url.URL
|
||||
if route != "" {
|
||||
selected, _ = url.Parse(route)
|
||||
selected.User = url.UserPassword("user", "pass")
|
||||
}
|
||||
local, err := New(func(r *http.Request) (*url.URL, error) { return selected, nil })
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer local.Close()
|
||||
proxyURL, _ := url.Parse(local.URL())
|
||||
transport := &http.Transport{Proxy: http.ProxyURL(proxyURL), TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}
|
||||
defer transport.CloseIdleConnections()
|
||||
client := &http.Client{Transport: transport, Timeout: 5 * time.Second}
|
||||
for _, test := range []struct{ url, want string }{{origin.URL, "origin"}, {secure.URL, "secure"}} {
|
||||
response, err := client.Get(test.url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body, _ := io.ReadAll(response.Body)
|
||||
response.Body.Close()
|
||||
if string(body) != test.want {
|
||||
t.Fatalf("got status %d, body %q", response.StatusCode, body)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
if authenticated.Load() != 2 {
|
||||
t.Fatalf("expected authenticated HTTP and CONNECT, got %d", authenticated.Load())
|
||||
}
|
||||
}
|
||||
|
||||
func TestRoutingChangesAndFailureDoesNotBypassProxy(t *testing.T) {
|
||||
var hits atomic.Int32
|
||||
origin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { hits.Add(1); w.Write([]byte("direct")) }))
|
||||
defer origin.Close()
|
||||
var useProxy atomic.Bool
|
||||
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Write([]byte("proxied")) }))
|
||||
upstreamURL, _ := url.Parse(upstream.URL)
|
||||
local, err := New(func(*http.Request) (*url.URL, error) {
|
||||
if useProxy.Load() {
|
||||
return upstreamURL, nil
|
||||
}
|
||||
return nil, nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer local.Close()
|
||||
endpoint, _ := url.Parse(local.URL())
|
||||
transport := &http.Transport{Proxy: http.ProxyURL(endpoint)}
|
||||
defer transport.CloseIdleConnections()
|
||||
client := &http.Client{Transport: transport, Timeout: 3 * time.Second}
|
||||
for _, want := range []string{"direct", "proxied", "upstream request failed\n"} {
|
||||
resp, err := client.Get(origin.URL)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
resp.Body.Close()
|
||||
if string(body) != want {
|
||||
t.Fatalf("got %q want %q", body, want)
|
||||
}
|
||||
if want == "direct" {
|
||||
useProxy.Store(true)
|
||||
} else if want == "proxied" {
|
||||
upstream.Close()
|
||||
}
|
||||
}
|
||||
if hits.Load() != 1 {
|
||||
t.Fatal("failed proxy must not fall back to direct")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCloseTerminatesTunnels(t *testing.T) {
|
||||
echo, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer echo.Close()
|
||||
go func() {
|
||||
c, e := echo.Accept()
|
||||
if e == nil {
|
||||
defer c.Close()
|
||||
io.Copy(c, c)
|
||||
}
|
||||
}()
|
||||
s, err := New(func(*http.Request) (*url.URL, error) { return nil, nil })
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
endpoint, _ := url.Parse(s.URL())
|
||||
c, err := dialTunnel(context.Background(), echo.Addr().String(), endpoint)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer c.Close()
|
||||
c.SetDeadline(time.Now().Add(3 * time.Second))
|
||||
c.Write([]byte("hello\n"))
|
||||
line, err := bufio.NewReader(c).ReadString('\n')
|
||||
if err != nil || line != "hello\n" {
|
||||
t.Fatalf("echo %q %v", line, err)
|
||||
}
|
||||
s.Close()
|
||||
if _, err := c.Read(make([]byte, 1)); err == nil {
|
||||
t.Fatal("tunnel stayed open")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSOCKSIngressPreservesRemoteDNS(t *testing.T) {
|
||||
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != "CONNECT" || r.Host != "unresolved.invalid:80" {
|
||||
t.Errorf("lost destination: %s %s", r.Method, r.Host)
|
||||
}
|
||||
c, b, err := w.(http.Hijacker).Hijack()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer c.Close()
|
||||
b.WriteString("HTTP/1.1 200 Connection Established\r\n\r\n")
|
||||
b.Flush()
|
||||
request, err := http.ReadRequest(b.Reader)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
return
|
||||
}
|
||||
request.Body.Close()
|
||||
b.WriteString("HTTP/1.1 200 OK\r\nContent-Length: 7\r\nConnection: close\r\n\r\nproxied")
|
||||
b.Flush()
|
||||
}))
|
||||
defer upstream.Close()
|
||||
target, _ := url.Parse(upstream.URL)
|
||||
local, err := New(func(r *http.Request) (*url.URL, error) {
|
||||
if r.URL.Host != "unresolved.invalid:80" {
|
||||
t.Errorf("selector lost hostname: %s", r.URL)
|
||||
}
|
||||
return target, nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer local.Close()
|
||||
endpoint, _ := url.Parse(local.SOCKSURL())
|
||||
transport := &http.Transport{Proxy: http.ProxyURL(endpoint)}
|
||||
defer transport.CloseIdleConnections()
|
||||
client := &http.Client{Transport: transport, Timeout: 5 * time.Second}
|
||||
response, err := client.Get("http://unresolved.invalid/")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer response.Body.Close()
|
||||
body, _ := io.ReadAll(response.Body)
|
||||
if string(body) != "proxied" {
|
||||
t.Fatalf("unexpected body %q", body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHTTPUpgradeIsForwarded(t *testing.T) {
|
||||
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Header.Get("Upgrade") != "fixture" {
|
||||
http.Error(w, "upgrade lost", 400)
|
||||
return
|
||||
}
|
||||
c, b, err := w.(http.Hijacker).Hijack()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer c.Close()
|
||||
b.WriteString("HTTP/1.1 101 Switching Protocols\r\nConnection: Upgrade\r\nUpgrade: fixture\r\n\r\n")
|
||||
b.Flush()
|
||||
line, err := b.ReadString('\n')
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
b.WriteString(line)
|
||||
b.Flush()
|
||||
}))
|
||||
defer upstream.Close()
|
||||
local, err := New(nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer local.Close()
|
||||
endpoint, _ := url.Parse(local.URL())
|
||||
c, err := net.Dial("tcp", endpoint.Host)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer c.Close()
|
||||
c.SetDeadline(time.Now().Add(3 * time.Second))
|
||||
io.WriteString(c, "GET "+upstream.URL+"/ HTTP/1.1\r\nHost: fixture\r\nConnection: Upgrade\r\nUpgrade: fixture\r\n\r\n")
|
||||
reader := bufio.NewReader(c)
|
||||
response, err := http.ReadResponse(reader, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if response.StatusCode != 101 {
|
||||
t.Fatalf("upgrade failed: %s", response.Status)
|
||||
}
|
||||
io.WriteString(c, "hello\n")
|
||||
line, err := reader.ReadString('\n')
|
||||
if err != nil || line != "hello\n" {
|
||||
t.Fatalf("upgraded echo %q %v", line, err)
|
||||
}
|
||||
}
|
||||
@@ -1,5 +1,11 @@
|
||||
# rpcprovider
|
||||
|
||||
The private Flutter native profile bridge is maintained in the
|
||||
[GopeedLab flutter_inappwebview fork](https://github.com/GopeedLab/flutter_inappwebview/tree/codex/gopeed-profiles).
|
||||
All three platform overrides pin one commit in `ui/flutter/pubspec.yaml`;
|
||||
see the fork’s [GOPEED.md](https://github.com/GopeedLab/flutter_inappwebview/blob/codex/gopeed-profiles/GOPEED.md)
|
||||
for patch scope and upstream synchronization.
|
||||
|
||||
`rpcprovider` is the RPC-backed implementation of Gopeed's `webview.Provider` interface.
|
||||
|
||||
It is intended for environments where WebView capability is owned by another process, most notably mobile hosts. This package does not implement a WebView itself. Instead, it translates the `pkg/download/engine/webview` page API into single-request `POST + JSON` calls sent to a local host-side RPC service.
|
||||
|
||||
@@ -0,0 +1,207 @@
|
||||
package rpcprovider
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
wv "github.com/GopeedLab/gopeed/pkg/download/engine/webview"
|
||||
)
|
||||
|
||||
func TestPageEventStream(t *testing.T) {
|
||||
events := make(chan wv.Event, 8)
|
||||
disconnected := make(chan struct{})
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
var request wv.RPCRequest
|
||||
if err := json.NewDecoder(r.Body).Decode(&request); err != nil {
|
||||
t.Error(err)
|
||||
return
|
||||
}
|
||||
if request.Method != "page.events" {
|
||||
t.Errorf("unexpected method %s", request.Method)
|
||||
return
|
||||
}
|
||||
if r.Header.Get("Authorization") != "Bearer test" {
|
||||
t.Error("missing authorization")
|
||||
}
|
||||
defer close(disconnected)
|
||||
enc := json.NewEncoder(w)
|
||||
_ = enc.Encode(map[string]any{"ready": true})
|
||||
w.(http.Flusher).Flush()
|
||||
for {
|
||||
select {
|
||||
case e := <-events:
|
||||
_ = enc.Encode(e)
|
||||
w.(http.Flusher).Flush()
|
||||
if e.Name == "closed" {
|
||||
return
|
||||
}
|
||||
case <-r.Context().Done():
|
||||
return
|
||||
}
|
||||
}
|
||||
}))
|
||||
defer server.Close()
|
||||
p := &page{client: NewClient(wv.RPCConfig{Network: "tcp", Address: strings.TrimPrefix(server.URL, "http://"), Token: "test"}), id: "page-1"}
|
||||
got := make(chan wv.Event, 8)
|
||||
off, err := p.On("load", func(e wv.Event) { got <- e })
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err = p.On("closed", func(e wv.Event) { got <- e })
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
events <- wv.Event{Name: "load", Data: map[string]any{"url": "https://example.test"}}
|
||||
select {
|
||||
case e := <-got:
|
||||
if e.Name != "load" {
|
||||
t.Fatal(e)
|
||||
}
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("no load event")
|
||||
}
|
||||
off()
|
||||
off()
|
||||
events <- wv.Event{Name: "load"}
|
||||
events <- wv.Event{Name: "closed", Data: map[string]any{"reason": "user"}}
|
||||
select {
|
||||
case e := <-got:
|
||||
if e.Name != "closed" || e.Data["reason"] != "user" {
|
||||
t.Fatal(e)
|
||||
}
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("no closed event")
|
||||
}
|
||||
select {
|
||||
case <-disconnected:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("stream retained after close")
|
||||
}
|
||||
if _, err = p.On("load", func(wv.Event) {}); err == nil {
|
||||
t.Fatal("registration after close")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPageEventStreamHandshakeFailureCanRetry(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
status int
|
||||
body string
|
||||
}{
|
||||
{"http_error", http.StatusUnauthorized, `{"ready":true}`},
|
||||
{"invalid_json", http.StatusOK, `not json`},
|
||||
{"not_ready", http.StatusOK, `{"ready":false}`},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
var requests atomic.Int32
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if requests.Add(1) == 1 {
|
||||
w.WriteHeader(tc.status)
|
||||
_, _ = io.WriteString(w, tc.body)
|
||||
return
|
||||
}
|
||||
_, _ = io.WriteString(w, "{\"ready\":true}\n{\"event\":\"load\",\"data\":{\"url\":\"https://example.test\"}}\n")
|
||||
w.(http.Flusher).Flush()
|
||||
<-r.Context().Done()
|
||||
}))
|
||||
defer server.Close()
|
||||
p := &page{client: NewClient(wv.RPCConfig{Network: "tcp", Address: strings.TrimPrefix(server.URL, "http://")}), id: "retry-page"}
|
||||
if off, err := p.On("load", func(wv.Event) { t.Error("failed registration retained its listener") }); err == nil || off != nil {
|
||||
t.Fatal("failed handshake accepted")
|
||||
}
|
||||
got := make(chan wv.Event, 1)
|
||||
off, err := p.On("load", func(e wv.Event) { got <- e })
|
||||
if err != nil {
|
||||
t.Fatalf("retry: %v", err)
|
||||
}
|
||||
defer func() {
|
||||
p.eventCancel()
|
||||
<-p.eventDone
|
||||
}()
|
||||
defer off()
|
||||
select {
|
||||
case e := <-got:
|
||||
if e.Data["url"] != "https://example.test" {
|
||||
t.Fatal(e)
|
||||
}
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("retry did not receive load notification")
|
||||
}
|
||||
if requests.Load() != 2 {
|
||||
t.Fatalf("unexpected request count: %d", requests.Load())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPageEventStreamDisconnectAndClose(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
var request wv.RPCRequest
|
||||
if err := json.NewDecoder(r.Body).Decode(&request); err != nil {
|
||||
t.Error(err)
|
||||
return
|
||||
}
|
||||
switch request.Method {
|
||||
case wv.MethodPageEvents:
|
||||
// End the stream without a terminal event, as if the host dropped it.
|
||||
_, _ = io.WriteString(w, "{\"ready\":true}\n")
|
||||
case wv.MethodPageClose:
|
||||
_ = json.NewEncoder(w).Encode(wv.RPCResponse{})
|
||||
default:
|
||||
t.Errorf("unexpected method: %s", request.Method)
|
||||
}
|
||||
}))
|
||||
defer server.Close()
|
||||
p := &page{client: NewClient(wv.RPCConfig{Network: "tcp", Address: strings.TrimPrefix(server.URL, "http://")}), id: "disconnected-page"}
|
||||
closed := make(chan wv.Event, 1)
|
||||
if _, err := p.On("closed", func(e wv.Event) { closed <- e }); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
select {
|
||||
case <-p.eventDone:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("disconnected stream did not stop")
|
||||
}
|
||||
if _, err := p.On("load", func(wv.Event) {}); err == nil || !strings.Contains(err.Error(), "disconnected") {
|
||||
t.Fatalf("registration hid stream failure: %v", err)
|
||||
}
|
||||
if err := p.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
select {
|
||||
case event := <-closed:
|
||||
if event.Data["reason"] != "api" {
|
||||
t.Fatal(event)
|
||||
}
|
||||
default:
|
||||
t.Fatal("API close did not release listeners after disconnect")
|
||||
}
|
||||
if err := p.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
select {
|
||||
case event := <-closed:
|
||||
t.Fatalf("duplicate close notification: %+v", event)
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
func TestPageEventStreamConnectionFailureCanRetry(t *testing.T) {
|
||||
server := httptest.NewServer(http.NotFoundHandler())
|
||||
server.Close()
|
||||
p := &page{client: NewClient(wv.RPCConfig{Network: "tcp", Address: strings.TrimPrefix(server.URL, "http://")}), id: "offline-page"}
|
||||
for i := 0; i < 2; i++ {
|
||||
if off, err := p.On("load", func(wv.Event) {}); err == nil || off != nil {
|
||||
t.Fatal("registration to unavailable host succeeded")
|
||||
}
|
||||
}
|
||||
if p.eventCancel != nil {
|
||||
t.Fatal("failed connection retained an active stream")
|
||||
}
|
||||
}
|
||||
@@ -4,10 +4,13 @@ import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
enginewebview "github.com/GopeedLab/gopeed/pkg/download/engine/webview"
|
||||
)
|
||||
@@ -48,8 +51,13 @@ func (p *Provider) Open(opts enginewebview.OpenOptions) (enginewebview.Page, err
|
||||
}
|
||||
|
||||
type page struct {
|
||||
client *Client
|
||||
id string
|
||||
eventMu sync.Mutex
|
||||
events enginewebview.EventHub
|
||||
eventCancel context.CancelFunc
|
||||
eventDone chan struct{}
|
||||
eventErr error
|
||||
client *Client
|
||||
id string
|
||||
}
|
||||
|
||||
func (p *page) AddInitScript(script string) error {
|
||||
@@ -109,9 +117,28 @@ func (p *page) ClearCookies() error {
|
||||
}
|
||||
|
||||
func (p *page) Close() error {
|
||||
return p.client.Call(enginewebview.MethodPageClose, enginewebview.PageCloseParams{
|
||||
PageID: p.id,
|
||||
}, nil)
|
||||
err := p.client.Call(enginewebview.MethodPageClose, enginewebview.PageCloseParams{PageID: p.id}, nil)
|
||||
var rpcErr *enginewebview.RPCError
|
||||
if errors.As(err, &rpcErr) && rpcErr.Code == enginewebview.ErrorCodePageNotFound {
|
||||
err = nil
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
p.eventMu.Lock()
|
||||
cancel, done := p.eventCancel, p.eventDone
|
||||
p.eventMu.Unlock()
|
||||
if cancel != nil {
|
||||
// Drain the terminal notification before cancelling the transport, so
|
||||
// already-sent navigation events keep their order before closed.
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(5 * time.Second):
|
||||
}
|
||||
cancel()
|
||||
}
|
||||
p.events.Emit(enginewebview.Event{Name: "closed", Data: map[string]any{"reason": "api"}})
|
||||
return nil
|
||||
}
|
||||
|
||||
type Client struct {
|
||||
@@ -193,3 +220,83 @@ func (c *Client) Call(method string, params any, result any) error {
|
||||
}
|
||||
return json.Unmarshal(body.Result, result)
|
||||
}
|
||||
|
||||
func (p *Provider) RemoveProfile(profileID, dataPath string) error {
|
||||
return p.client.Call(enginewebview.MethodProfileRemove, enginewebview.ProfileRemoveParams{ProfileID: profileID}, nil)
|
||||
}
|
||||
|
||||
// On starts one ordered notification stream per page. The ready record is sent
|
||||
// only after the host has installed its subscription.
|
||||
func (p *page) On(name string, handler func(enginewebview.Event)) (func(), error) {
|
||||
p.eventMu.Lock()
|
||||
defer p.eventMu.Unlock()
|
||||
if p.eventErr != nil {
|
||||
return nil, p.eventErr
|
||||
}
|
||||
unsubscribe, err := p.events.On(name, handler)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if p.eventCancel != nil {
|
||||
return unsubscribe, nil
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
payload, err := json.Marshal(enginewebview.RPCRequest{Method: enginewebview.MethodPageEvents, Params: enginewebview.PageCloseParams{PageID: p.id}})
|
||||
if err != nil {
|
||||
cancel()
|
||||
unsubscribe()
|
||||
return nil, err
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, p.client.endpoint, bytes.NewReader(payload))
|
||||
if err != nil {
|
||||
cancel()
|
||||
unsubscribe()
|
||||
return nil, err
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
if p.client.token != "" {
|
||||
req.Header.Set("Authorization", "Bearer "+p.client.token)
|
||||
}
|
||||
// A broken host must not leave registration pending indefinitely.
|
||||
timer := time.AfterFunc(30*time.Second, cancel)
|
||||
resp, err := p.client.http.Do(req)
|
||||
if err != nil {
|
||||
timer.Stop()
|
||||
cancel()
|
||||
unsubscribe()
|
||||
return nil, err
|
||||
}
|
||||
decoder := json.NewDecoder(resp.Body)
|
||||
var ready struct {
|
||||
Ready bool `json:"ready"`
|
||||
}
|
||||
err = decoder.Decode(&ready)
|
||||
timer.Stop()
|
||||
if err != nil || !ready.Ready || resp.StatusCode != http.StatusOK {
|
||||
resp.Body.Close()
|
||||
cancel()
|
||||
unsubscribe()
|
||||
return nil, fmt.Errorf("webview event stream did not become ready: %v", err)
|
||||
}
|
||||
p.eventCancel = cancel
|
||||
p.eventDone = make(chan struct{})
|
||||
go func() {
|
||||
defer resp.Body.Close()
|
||||
defer cancel()
|
||||
defer close(p.eventDone)
|
||||
for {
|
||||
var event enginewebview.Event
|
||||
if err := decoder.Decode(&event); err != nil {
|
||||
p.eventMu.Lock()
|
||||
p.eventErr = fmt.Errorf("webview event stream disconnected: %w", err)
|
||||
p.eventMu.Unlock()
|
||||
return
|
||||
}
|
||||
p.events.Emit(event)
|
||||
if event.Name == "closed" {
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
return unsubscribe, nil
|
||||
}
|
||||
@@ -37,3 +37,12 @@ func defaultWebViewRPCAddress() string {
|
||||
}
|
||||
return filepath.Join(homeDir, "Library", "Application Support", "com.gopeed.gopeed", "gopeed_webview.sock")
|
||||
}
|
||||
|
||||
func TestProviderProfiles(t *testing.T) {
|
||||
provider := New(enginewebview.RPCConfig{Network: *webViewRPCNetwork, Address: *webViewRPCAddress, Token: *webViewRPCToken})
|
||||
integrationtest.RunProfileContract(t, provider)
|
||||
}
|
||||
|
||||
func TestProviderEvents(t *testing.T) {
|
||||
integrationtest.RunEventContract(t, New(enginewebview.RPCConfig{Network: *webViewRPCNetwork, Address: *webViewRPCAddress, Token: *webViewRPCToken}))
|
||||
}
|
||||
@@ -184,3 +184,29 @@ func TestPageSetCookieParamsOmitsZeroExpires(t *testing.T) {
|
||||
t.Fatalf("expected expires in payload: %s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemoveProfileUsesPrivateHostRPC(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
var request struct {
|
||||
Method string `json:"method"`
|
||||
Params map[string]string `json:"params"`
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&request); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
if request.Method != enginewebview.MethodProfileRemove || request.Params["profileId"] != "extension-profile" || len(request.Params) != 1 {
|
||||
t.Errorf("unexpected profile removal request: %+v", request)
|
||||
}
|
||||
if r.Header.Get("Authorization") != "Bearer secret" {
|
||||
t.Error("missing host authentication")
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.Write([]byte(`{"error":{"code":"PROFILE_REMOVE_FAILED","message":"store busy"}}`))
|
||||
}))
|
||||
defer server.Close()
|
||||
provider := New(enginewebview.RPCConfig{Network: "tcp", Address: strings.TrimPrefix(server.URL, "http://"), Token: "secret"})
|
||||
err := provider.(enginewebview.ProfileRemover).RemoveProfile("extension-profile", "/host-only/path")
|
||||
if err == nil || err.Error() != "store busy" {
|
||||
t.Fatalf("cleanup failure was lost: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -10,8 +10,15 @@ import (
|
||||
type TaskEventMask uint64
|
||||
|
||||
const (
|
||||
// Keep these first two unchanged because Flutter already uses
|
||||
// bit 0 for done and bit 1 for error.
|
||||
TaskEventDone TaskEventMask = 1 << iota
|
||||
TaskEventError
|
||||
|
||||
TaskEventStart
|
||||
TaskEventProgress
|
||||
TaskEventPause
|
||||
TaskEventDelete
|
||||
)
|
||||
|
||||
type TaskEvent struct {
|
||||
@@ -42,11 +49,26 @@ func (s *Service) emitTaskEvent(event *download.Event) {
|
||||
}
|
||||
|
||||
var mask TaskEventMask
|
||||
|
||||
switch event.Key {
|
||||
case download.EventKeyDone:
|
||||
mask = TaskEventDone
|
||||
|
||||
case download.EventKeyError:
|
||||
mask = TaskEventError
|
||||
|
||||
case download.EventKeyStart:
|
||||
mask = TaskEventStart
|
||||
|
||||
case download.EventKeyProgress:
|
||||
mask = TaskEventProgress
|
||||
|
||||
case download.EventKeyPause:
|
||||
mask = TaskEventPause
|
||||
|
||||
case download.EventKeyDelete:
|
||||
mask = TaskEventDelete
|
||||
|
||||
default:
|
||||
return
|
||||
}
|
||||
|
||||
+1
-1
@@ -156,7 +156,7 @@ func (s *Service) createTask(ctx *Context) *Response {
|
||||
err error
|
||||
)
|
||||
if request.Rid != "" {
|
||||
taskID, err = s.Downloader.Create(request.Rid)
|
||||
taskID, err = s.Downloader.CreateWithOptions(request.Rid, request.Opts)
|
||||
} else if request.Req != nil {
|
||||
taskID, err = s.Downloader.CreateDirect(request.Req, request.Opts)
|
||||
} else {
|
||||
|
||||
@@ -94,9 +94,6 @@ type Resource struct {
|
||||
}
|
||||
|
||||
func (r *Resource) Validate() error {
|
||||
if r.Name == "" {
|
||||
return fmt.Errorf("invalid resource name")
|
||||
}
|
||||
if len(r.Files) == 0 {
|
||||
return fmt.Errorf("invalid resource files")
|
||||
}
|
||||
|
||||
+86
-15
@@ -21,6 +21,8 @@ import (
|
||||
"github.com/GopeedLab/gopeed/internal/controller"
|
||||
"github.com/GopeedLab/gopeed/internal/fetcher"
|
||||
"github.com/GopeedLab/gopeed/internal/logger"
|
||||
"github.com/GopeedLab/gopeed/internal/tempfiles"
|
||||
webviewproxy "github.com/GopeedLab/gopeed/internal/webview/proxy"
|
||||
"github.com/GopeedLab/gopeed/pkg/base"
|
||||
"github.com/GopeedLab/gopeed/pkg/protocol/http"
|
||||
"github.com/GopeedLab/gopeed/pkg/util"
|
||||
@@ -97,8 +99,13 @@ type Progress struct {
|
||||
}
|
||||
|
||||
type Downloader struct {
|
||||
Logger *logger.Logger
|
||||
ExtensionLogger *logger.Logger
|
||||
tempFiles *tempfiles.Scope
|
||||
webviewProfilesLock sync.Mutex
|
||||
webviewProfiles map[string]*extensionWebViewProfile
|
||||
webviewProxyLock sync.Mutex
|
||||
webviewProxy *webviewproxy.Server
|
||||
Logger *logger.Logger
|
||||
ExtensionLogger *logger.Logger
|
||||
|
||||
cfg *DownloaderConfig
|
||||
fetcherCache map[string]fetcher.Fetcher
|
||||
@@ -152,6 +159,9 @@ func NewDownloader(cfg *DownloaderConfig) *Downloader {
|
||||
}
|
||||
|
||||
func (d *Downloader) Setup() error {
|
||||
if err := d.setupTempDir(); err != nil {
|
||||
return err
|
||||
}
|
||||
d.blob = internalblob.NewRegistry("")
|
||||
|
||||
// setup storage
|
||||
@@ -263,24 +273,45 @@ func (d *Downloader) Setup() error {
|
||||
if d.GetTask(task.ID) == nil {
|
||||
return
|
||||
}
|
||||
// Startup can replace resource metadata while resolving a Blob.
|
||||
// Skip that short transition instead of reading half-initialized state.
|
||||
if !task.lock.TryLock() {
|
||||
return
|
||||
}
|
||||
task.statusLock.Lock()
|
||||
if task.Status != base.DownloadStatusRunning && !task.Uploading {
|
||||
task.statusLock.Unlock()
|
||||
task.lock.Unlock()
|
||||
return
|
||||
}
|
||||
if task.fetcher == nil {
|
||||
task.statusLock.Unlock()
|
||||
task.lock.Unlock()
|
||||
return
|
||||
}
|
||||
|
||||
current := task.fetcher.Progress().TotalDownloaded()
|
||||
delta := current - task.Progress.Downloaded
|
||||
if task.Meta != nil && task.Meta.Req != nil {
|
||||
if producer := d.blob.Production(task.Meta.Req.URL); producer != nil {
|
||||
downloaded, received := producer.Progress()
|
||||
if task.producer != producer {
|
||||
task.producer = producer
|
||||
task.producerReceived = 0
|
||||
task.speedArr = nil
|
||||
}
|
||||
current = downloaded
|
||||
delta = max(0, received-task.producerReceived)
|
||||
task.producerReceived = received
|
||||
}
|
||||
}
|
||||
tick := float64(d.cfg.RefreshInterval) / 1000
|
||||
downloadDataChanged := false
|
||||
uploading := task.Uploading
|
||||
if task.Status == base.DownloadStatusRunning {
|
||||
downloadDataChanged = current != task.Progress.Downloaded
|
||||
task.Progress.Used = task.timer.Used()
|
||||
task.Progress.Speed = task.updateSpeed(current-task.Progress.Downloaded, tick)
|
||||
task.Progress.Speed = task.updateSpeed(max(0, delta), tick)
|
||||
task.Progress.Downloaded = current
|
||||
}
|
||||
|
||||
@@ -299,6 +330,7 @@ func (d *Downloader) Setup() error {
|
||||
if err := d.captureTaskStats(task, true, false); err != nil {
|
||||
d.Logger.Warn().Err(err).Msgf("persist task stats failed, task id: %s", task.ID)
|
||||
}
|
||||
task.lock.Unlock()
|
||||
// Listener callbacks may Pause/Continue and acquire statusLock.
|
||||
d.emit(EventKeyProgress, task)
|
||||
|
||||
@@ -389,6 +421,9 @@ func (d *Downloader) parseFm(url string) (fetcher.FetcherManager, error) {
|
||||
|
||||
func (d *Downloader) setupFetcher(fm fetcher.FetcherManager, fetcher fetcher.Fetcher) {
|
||||
ctl := controller.NewController()
|
||||
ctl.TempDir = d.cfg.TempDir
|
||||
ctl.TempFiles = d.tempFiles
|
||||
ctl.ManagedProduction = func(raw string) bool { return d.blob.Production(raw) != nil }
|
||||
ctl.GetConfig = func(v any) {
|
||||
d.getProtocolConfig(fm.Name(), v)
|
||||
}
|
||||
@@ -582,12 +617,36 @@ func (d *Downloader) CreateDirectBatch(req *base.CreateTaskBatch) (taskId []stri
|
||||
}
|
||||
|
||||
func (d *Downloader) Create(rrId string) (taskId string, err error) {
|
||||
return d.CreateWithOptions(rrId, nil)
|
||||
}
|
||||
|
||||
// CreateWithOptions creates a resolved task using the submitted options, or the
|
||||
// resolve-time options when opts is nil.
|
||||
func (d *Downloader) CreateWithOptions(rrId string, opts *base.Options) (taskId string, err error) {
|
||||
d.fetcherMapLock.RLock()
|
||||
fetcher, ok := d.fetcherCache[rrId]
|
||||
d.fetcherMapLock.RUnlock()
|
||||
if !ok {
|
||||
return "", errors.New("invalid resource id")
|
||||
}
|
||||
if opts != nil {
|
||||
opts, err = d.initOptions(opts.Clone())
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if res := fetcher.Meta().Res; res != nil {
|
||||
for _, index := range opts.SelectFiles {
|
||||
if index < 0 || index >= len(res.Files) {
|
||||
return "", fmt.Errorf("invalid selected file index: %d", index)
|
||||
}
|
||||
}
|
||||
opts.InitSelectFiles(len(res.Files))
|
||||
}
|
||||
fetcher.Meta().Opts = opts
|
||||
}
|
||||
if res := fetcher.Meta().Res; res != nil {
|
||||
res.CalcSize(fetcher.Meta().Opts.SelectFiles)
|
||||
}
|
||||
defer func() {
|
||||
d.fetcherMapLock.Lock()
|
||||
delete(d.fetcherCache, rrId)
|
||||
@@ -1114,6 +1173,11 @@ func (d *Downloader) doDelete(task *Task, force bool) (err error) {
|
||||
|
||||
func (d *Downloader) Close() error {
|
||||
d.closed.Store(true)
|
||||
d.webviewProxyLock.Lock()
|
||||
if d.webviewProxy != nil {
|
||||
_ = d.webviewProxy.Close()
|
||||
}
|
||||
d.webviewProxyLock.Unlock()
|
||||
|
||||
closeArr := []func() error{
|
||||
d.pauseAll,
|
||||
@@ -1124,7 +1188,7 @@ func (d *Downloader) Close() error {
|
||||
if d.blob != nil {
|
||||
closeArr = append(closeArr, d.blob.Close)
|
||||
}
|
||||
closeArr = append(closeArr, d.storage.Close)
|
||||
closeArr = append(closeArr, d.storage.Close, d.tempFiles.Close)
|
||||
// Make sure all resources are released, if had error, return the last error
|
||||
var lastErr error
|
||||
for i, close := range closeArr {
|
||||
@@ -1399,17 +1463,6 @@ func (d *Downloader) watch(task *Task) {
|
||||
}
|
||||
}
|
||||
|
||||
task.Progress.Used = task.timer.Used()
|
||||
if task.Meta.Res.Size == 0 {
|
||||
task.Meta.Res.Size = task.fetcher.Progress().TotalDownloaded()
|
||||
}
|
||||
used := task.Progress.Used / int64(time.Second)
|
||||
if used == 0 {
|
||||
used = 1
|
||||
}
|
||||
totalSize := task.Meta.Res.Size
|
||||
task.Progress.Speed = totalSize / used
|
||||
task.Progress.Downloaded = totalSize
|
||||
if !d.markTaskDone(task) {
|
||||
return
|
||||
}
|
||||
@@ -1808,7 +1861,9 @@ func (d *Downloader) doStart(task *Task) (err error) {
|
||||
task.Meta.Res.CalcSize(task.Meta.Opts.SelectFiles)
|
||||
}
|
||||
|
||||
task.statusLock.Lock()
|
||||
task.Progress.Speed = 0
|
||||
task.statusLock.Unlock()
|
||||
if !d.taskIsRunningGeneration(task, generation) {
|
||||
return nil
|
||||
}
|
||||
@@ -1962,11 +2017,27 @@ func (d *Downloader) markTaskDone(task *Task) bool {
|
||||
if task == nil || task.statusLock == nil {
|
||||
return false
|
||||
}
|
||||
task.lock.Lock()
|
||||
defer task.lock.Unlock()
|
||||
task.statusLock.Lock()
|
||||
defer task.statusLock.Unlock()
|
||||
if task.Status != base.DownloadStatusRunning {
|
||||
return false
|
||||
}
|
||||
task.Progress.Used = task.timer.Used()
|
||||
if task.Meta.Res.Size == 0 {
|
||||
task.Meta.Res.Size = task.fetcher.Progress().TotalDownloaded()
|
||||
}
|
||||
used := task.Progress.Used / int64(time.Second)
|
||||
if used == 0 {
|
||||
used = 1
|
||||
}
|
||||
totalSize := task.Meta.Res.Size
|
||||
task.Progress.Speed = totalSize / used
|
||||
task.Progress.Downloaded = totalSize
|
||||
|
||||
task.producer = nil
|
||||
task.producerReceived = 0
|
||||
task.updateStatus(base.DownloadStatusDone)
|
||||
task.runGeneration++
|
||||
return true
|
||||
|
||||
@@ -6,9 +6,12 @@ import (
|
||||
"fmt"
|
||||
"sync"
|
||||
|
||||
"github.com/GopeedLab/gopeed/internal/production"
|
||||
"github.com/GopeedLab/gopeed/internal/tempfiles"
|
||||
"github.com/GopeedLab/gopeed/pkg/base"
|
||||
gojaerror "github.com/GopeedLab/gopeed/pkg/download/engine/inject/error"
|
||||
fetchapi "github.com/GopeedLab/gopeed/pkg/download/engine/inject/fetch"
|
||||
ffmpegapi "github.com/GopeedLab/gopeed/pkg/download/engine/inject/ffmpeg"
|
||||
"github.com/GopeedLab/gopeed/pkg/download/engine/inject/file"
|
||||
"github.com/GopeedLab/gopeed/pkg/download/engine/inject/stream"
|
||||
"github.com/GopeedLab/gopeed/pkg/download/engine/inject/vm"
|
||||
@@ -159,8 +162,12 @@ func (e *Engine) addCleanup(cleanup func()) {
|
||||
}
|
||||
|
||||
type Config struct {
|
||||
ProxyConfig *base.DownloaderProxyConfig
|
||||
StreamConfig *stream.Config
|
||||
TempFiles *tempfiles.Scope
|
||||
// HTTPUserAgent is a snapshot of the downloader HTTP default. Nil leaves standalone engines unchanged.
|
||||
TempDir string
|
||||
HTTPUserAgent *string
|
||||
ProxyConfig *base.DownloaderProxyConfig
|
||||
StreamConfig *stream.Config
|
||||
}
|
||||
|
||||
func NewEngine(cfg *Config) *Engine {
|
||||
@@ -200,12 +207,29 @@ func NewEngine(cfg *Config) *Engine {
|
||||
if _, err := runtime.RunString("global.location = new URL('http://localhost');"); err != nil {
|
||||
return
|
||||
}
|
||||
if err := stream.Enable(runtime, loop, cfg.StreamConfig); err != nil {
|
||||
producers := &production.Registry{}
|
||||
streamCfg := stream.Config{}
|
||||
if cfg.StreamConfig != nil {
|
||||
streamCfg = *cfg.StreamConfig
|
||||
}
|
||||
streamCfg.Producers = producers
|
||||
if err := stream.Enable(runtime, loop, &streamCfg); err != nil {
|
||||
return
|
||||
}
|
||||
if err := ffmpegapi.Enable(runtime, loop, &ffmpegapi.Config{
|
||||
Producers: producers,
|
||||
TempDir: cfg.TempDir,
|
||||
TempFiles: cfg.TempFiles,
|
||||
ProxyHandler: cfg.ProxyConfig.ToHandler(),
|
||||
DefaultUserAgent: cfg.HTTPUserAgent,
|
||||
RegisterCleanup: engine.addCleanup,
|
||||
}); err != nil {
|
||||
return
|
||||
}
|
||||
if err := fetchapi.Enable(runtime, loop, &fetchapi.Config{
|
||||
ProxyHandler: cfg.ProxyConfig.ToHandler(),
|
||||
RegisterCleanup: engine.addCleanup,
|
||||
ProxyHandler: cfg.ProxyConfig.ToHandler(),
|
||||
DefaultUserAgent: cfg.HTTPUserAgent,
|
||||
RegisterCleanup: engine.addCleanup,
|
||||
}); err != nil {
|
||||
return
|
||||
}
|
||||
@@ -265,3 +289,8 @@ func callExportedFunction(fn func(goja.FunctionCall) goja.Value, args ...goja.Va
|
||||
Arguments: args,
|
||||
}), nil
|
||||
}
|
||||
|
||||
// Post queues a notification without waiting for its callback or returned promise.
|
||||
func (e *Engine) Post(fn func(*goja.Runtime)) bool {
|
||||
return e.loop.RunOnLoop(fn)
|
||||
}
|
||||
@@ -0,0 +1,160 @@
|
||||
# Extension FFmpeg runtime
|
||||
|
||||
`gopeed.runtime.ffmpeg.merge(options)` returns a `ReadableStream<Uint8Array>`.
|
||||
It merges one video track and one audio track without re-encoding, using
|
||||
[go-ffmpreg](https://codeberg.org/gruf/go-ffmpreg)'s embedded FFmpeg 5.1.10 and
|
||||
wazero. No system FFmpeg installation is required. go-ffmpreg and its embedded
|
||||
GPL-enabled FFmpeg are distributed under GPLv3; see the pinned dependency's
|
||||
LICENSE, README and build scripts for its source and included components.
|
||||
|
||||
## HTTP inputs and one downloadable output
|
||||
|
||||
```js
|
||||
gopeed.events.onResolve(async (ctx) => {
|
||||
// Resolve these URLs and headers using the site's API in your extension.
|
||||
const video = { url: videoURL, headers: { Referer: pageURL } };
|
||||
const audio = { url: audioURL, headers: { Referer: pageURL } };
|
||||
const url = await gopeed.runtime.blob.createObjectURL(
|
||||
() => gopeed.runtime.ffmpeg.merge({ video, audio }),
|
||||
{ contentType: "video/mp4" }
|
||||
);
|
||||
ctx.res = {
|
||||
name: "video",
|
||||
files: [{ name: "video.mp4", req: { url } }]
|
||||
};
|
||||
});
|
||||
```
|
||||
|
||||
The opener creates a new merge on each open. Output size is unknown and range
|
||||
support must remain disabled. Do not declare the sum of the input sizes as the
|
||||
output size. The downloader reads the merged output as one file; FFmpeg errors
|
||||
propagate through the stream instead of completing a partial download.
|
||||
|
||||
## JS inputs, including SABR
|
||||
|
||||
Stream inputs **must** be created by an `inputs({ signal })` callback. It returns
|
||||
`{ video, audio }`, each an HTTP descriptor or `ReadableStream<Uint8Array>`,
|
||||
including mixed inputs. Do not start producers before this callback. SABR/UMP decoding belongs
|
||||
to the extension: pass the resulting media streams, with initialization data,
|
||||
not the SABR protocol bytes. Keep each selected track's codec/configuration
|
||||
stable for the duration of a merge.
|
||||
|
||||
```js
|
||||
const output = gopeed.runtime.ffmpeg.merge({
|
||||
inputs: async ({ signal }) => {
|
||||
const session = await prepareSession({ signal });
|
||||
const { videoStream, audioStream, abort } = await session.openStreams();
|
||||
if (signal.aborted) { abort(); throw new Error("Cancelled"); }
|
||||
signal.addEventListener("abort", abort, { once: true });
|
||||
return { video: videoStream, audio: audioStream };
|
||||
},
|
||||
signal: abortController.signal,
|
||||
args: ["-shortest", "-metadata", "title=Example"]
|
||||
});
|
||||
```
|
||||
|
||||
Creating the output does not start requests or acquire input readers until the
|
||||
output is pulled. Inputs then download to temporary disk chunks even while the
|
||||
job waits for a global FFmpeg execution slot. Only merging occupies a slot.
|
||||
The callback runs once per output; its signal aborts on cancellation, engine
|
||||
shutdown, failure, and completion. A callback must cooperate with cancellation;
|
||||
returned streams are cancelled even if the callback finishes after cancellation.
|
||||
A stream input is single-use. To retry a Blob-backed task,
|
||||
its opener must create a new SABR session and fresh input streams. Arbitrary
|
||||
output byte ranges / resumable merged output are not supported.
|
||||
|
||||
`gopeed.runtime.ffmpeg.supportsInputFactory === true` identifies the callback API.
|
||||
`gopeed.runtime.ffmpeg.supportsProducerProgress === true` identifies disk-backed
|
||||
inputs and automatic Blob production progress. Require the latter before using
|
||||
one shared SABR session for both tracks; older bounded buffers can deadlock.
|
||||
|
||||
## Options
|
||||
|
||||
- `inputs`: callback returning `{ video, audio }`, synchronously or asynchronously.
|
||||
Required for stream inputs. Cannot be combined with top-level `video`/`audio`.
|
||||
- `video`, `audio`: HTTP descriptors only when supplied at the top level; both
|
||||
are required unless `inputs` is supplied. HTTP descriptors have `url` and optional
|
||||
`headers` (standard `HeadersInit`). HTTP requests use Gopeed's configured
|
||||
proxy. Supply required cookies/authorization explicitly; browser or `fetch`
|
||||
cookie jars are not implicitly shared. When `User-Agent` is absent, requests
|
||||
use the downloader HTTP protocol's configured UA, captured when the extension
|
||||
engine is created, matching fetch and XHR. Explicit UA values (including an
|
||||
empty string) take precedence.
|
||||
- `format`: defaults to `"mp4"`. MP4 output always enables fragmentation
|
||||
(`frag_keyframe+empty_moov+default_base_moof`) for non-seekable output. The
|
||||
other accepted muxers are `"mkv"`/`"matroska"`, `"webm"`, and `"mpegts"`.
|
||||
Codec/container incompatibility is an error, not an automatic conversion.
|
||||
Choose the matching filename and Blob content type in the extension.
|
||||
- `args`: optional array of **output options**, not a shell command or a full
|
||||
FFmpeg invocation. Defaults select `0:v:0`, `1:a:0`, and `-c copy`. Additional
|
||||
`-map` options are additive. Allowed options are listed in `outputOptions` in
|
||||
`runtime.go`: metadata, dispositions, bitstream filters, duration/shortest,
|
||||
fragmentation, timestamp/interleaving options and `-strict`. Codec options
|
||||
must remain `copy`. Extra inputs, output paths, reports and arbitrary global
|
||||
options are rejected. Required MP4 fragmentation is enforced by the host.
|
||||
- `signal`: optional `AbortSignal`. Aborting or cancelling the output stops
|
||||
network requests and native execution, cancels input readers, and removes
|
||||
temporary buffers. Await/consume the output inside a live engine or retain
|
||||
it through a Blob URL; unowned work ends with the extension engine.
|
||||
|
||||
The backend defaults to strict input errors (`-xerror`). Native errors include
|
||||
the FFmpeg exit code and a bounded stderr tail. No host directories are exposed
|
||||
to the WASM module; it can access only its two input files and standard output.
|
||||
|
||||
## Input behavior and resource limits
|
||||
|
||||
HTTP inputs start with a 1 MiB Range GET, reusing the bytes for media reads.
|
||||
A validated 206 response with a known total size enables seeking; a 200
|
||||
response is consumed as-is, sequentially. Subsequent ranges verify positions,
|
||||
lengths, total size and available ETag/Last-Modified validators. Without a
|
||||
server validator, same-size remote changes cannot be reliably detected.
|
||||
Content encoding is forced to identity. Authentication/status failures and
|
||||
malformed range responses are errors. There is no full-file seek fallback.
|
||||
|
||||
HTTP and sequential JS inputs are drained into **8 MiB disk chunks** beneath
|
||||
`TempDir/gopeed-media-<random>`. Range downloads stream each response into as many chunks as
|
||||
needed, tracking valid byte intervals independently of sparse file length.
|
||||
Verified HTTP Range inputs can seek, skip missing blocks, and re-fetch deleted
|
||||
blocks. Sequential inputs cannot seek even though their bytes are on disk.
|
||||
Consumed chunks are deleted; cancellation, errors and completion remove the
|
||||
remaining input directories. Disk space is not capped; write failures stop the
|
||||
merge. Media size does not determine the size of in-memory input queues.
|
||||
|
||||
JS writes await disk writes, and Blob output writes await bounded downstream
|
||||
capacity. Extensions must still propagate backpressure through upstream queues
|
||||
and network reads. Both tracks can share a SABR response because disk spooling
|
||||
continues independently of which track FFmpeg currently reads.
|
||||
|
||||
When a Blob opener returns the merge stream directly, the runtime automatically
|
||||
associates its input progress with that Blob. No public progress callback or
|
||||
extra `createObjectURL` argument is required. Return the merge stream itself;
|
||||
wrapping it in another stream loses this association. Downloaded bytes count
|
||||
unique input offsets; speed counts received input bytes, including re-fetches.
|
||||
The task stays downloading with unknown total size while queued or merging;
|
||||
once input traffic stops the displayed speed settles to zero. Completion uses
|
||||
the actual merged output size. Generated output waits are governed by producer
|
||||
cancellation/errors rather than the HTTP body idle timeout; HTTP media inputs
|
||||
still enforce a 15-second network idle deadline.
|
||||
|
||||
`tempDir` defaults to Go's `os.TempDir()` when omitted. Flutter only supplies
|
||||
an explicit directory on Android/iOS, using `getTemporaryDirectory()` directly;
|
||||
no extra `gopeed` parent directory is added. Desktop/server callers can override
|
||||
it via startup JSON or `--temp-dir`.
|
||||
|
||||
Initialization never clears this directory. Media inputs, HTTP prefetch and
|
||||
extension installation staging create unique `gopeed-media-*`,
|
||||
`gopeed-prefetch-*`, and `gopeed-extension-*` paths directly beneath it. The
|
||||
downloader tracks only its own paths and removes remaining ones on normal
|
||||
shutdown, without scanning or deleting other files in a shared temporary
|
||||
directory. Resources still clean up on completion/cancellation, and consumed
|
||||
media chunks are deleted promptly. Forced process termination can leave files
|
||||
behind; the next startup does not delete them. Persistent HLS resume data and
|
||||
WebView profiles are unchanged.
|
||||
|
||||
At most two WASM merges execute concurrently per process, with a 512 MiB
|
||||
linear-memory limit per instance. Compilation is shared and lazy; the upstream
|
||||
embedded package still decompresses its WASM at process initialization. Native
|
||||
compilation memory, host buffers and JS allocations are additional to the
|
||||
linear-memory limit. Playback support depends on the selected codecs, not just
|
||||
the `.mp4` suffix. Platform execution and performance must be tested on target
|
||||
devices, especially mobile/32-bit devices.
|
||||
@@ -0,0 +1,248 @@
|
||||
package ffmpeg
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"sync"
|
||||
|
||||
"github.com/GopeedLab/gopeed/internal/tempfiles"
|
||||
)
|
||||
|
||||
// HTTPSpool downloads ahead independently of the FFmpeg semaphore. Range
|
||||
// sources can fetch a missing block on demand, including blocks already evicted.
|
||||
type HTTPSpool struct {
|
||||
demand chan int64
|
||||
demandDone chan struct{}
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
source Input
|
||||
disk *DiskInput
|
||||
network sync.Mutex
|
||||
readMu sync.Mutex
|
||||
pos int64
|
||||
done chan struct{}
|
||||
}
|
||||
|
||||
func OpenSpoolingHTTP(ctx context.Context, client *http.Client, source HTTPSource, root string, owners ...*tempfiles.Scope) (Input, error) {
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
in, err := OpenHTTP(ctx, client, source)
|
||||
if err != nil {
|
||||
cancel()
|
||||
return nil, err
|
||||
}
|
||||
disk, err := NewDiskInput(ctx, root, owners...)
|
||||
if err != nil {
|
||||
cancel()
|
||||
in.Close()
|
||||
return nil, err
|
||||
}
|
||||
if ranged := in.(*httpInput); ranged.ranged {
|
||||
if _, err = disk.spool.WriteAt(ranged.block, 0); err != nil {
|
||||
cancel()
|
||||
in.Close()
|
||||
disk.Close()
|
||||
return nil, err
|
||||
}
|
||||
ranged.block = nil
|
||||
}
|
||||
h := &HTTPSpool{ctx: ctx, cancel: cancel, source: in, disk: disk, done: make(chan struct{}), demand: make(chan int64, 1), demandDone: make(chan struct{})}
|
||||
go func() {
|
||||
defer close(h.demandDone)
|
||||
for {
|
||||
select {
|
||||
case offset := <-h.demand:
|
||||
// A prefetch may have filled this position while demand was queued.
|
||||
var probe [1]byte
|
||||
if n, err, _ := h.disk.spool.readAt(probe[:], offset); n > 0 || err != nil {
|
||||
continue
|
||||
}
|
||||
if err := h.fetch(offset / SpoolBlockSize * SpoolBlockSize); err != nil {
|
||||
h.disk.End(err)
|
||||
return
|
||||
}
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
go func() {
|
||||
defer close(h.done)
|
||||
var err error
|
||||
if !in.Seekable() {
|
||||
_, err = io.CopyBuffer(disk, in, make([]byte, 64*1024))
|
||||
} else {
|
||||
for offset := int64(0); offset < in.Size(); offset += 3 * SpoolBlockSize {
|
||||
if err = h.fetch(offset); err != nil {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
disk.End(err)
|
||||
}()
|
||||
return h, nil
|
||||
}
|
||||
func (h *HTTPSpool) fetch(offset int64) error {
|
||||
h.network.Lock()
|
||||
defer h.network.Unlock()
|
||||
if err := h.ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
source := h.source.(*httpInput)
|
||||
end := offset + min(3*SpoolBlockSize, h.source.Size()-offset)
|
||||
buffer := make([]byte, 64*1024)
|
||||
for offset < end {
|
||||
n, err, _ := h.disk.spool.readAt(buffer[:min(int64(len(buffer)), end-offset)], offset)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if n > 0 {
|
||||
offset += int64(n)
|
||||
continue
|
||||
}
|
||||
resp, err := source.request(offset, end-1)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = func() error {
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusPartialContent {
|
||||
return fmt.Errorf("ffmpeg: Range request returned %d; source changed or stopped supporting Range", resp.StatusCode)
|
||||
}
|
||||
start, last, size, err := contentRange(resp.Header.Get("Content-Range"))
|
||||
if err != nil || start != offset || last >= end || size != source.size {
|
||||
return errors.New("ffmpeg: inconsistent Content-Range")
|
||||
}
|
||||
if source.etag != "" && resp.Header.Get("ETag") != source.etag {
|
||||
return errors.New("ffmpeg: HTTP input ETag changed")
|
||||
}
|
||||
if source.etag == "" && source.modified != "" && resp.Header.Get("Last-Modified") != source.modified {
|
||||
return errors.New("ffmpeg: HTTP input Last-Modified changed")
|
||||
}
|
||||
if err := identityEncoding(resp); err != nil {
|
||||
return err
|
||||
}
|
||||
remaining := last - start + 1
|
||||
if resp.ContentLength >= 0 && resp.ContentLength != remaining {
|
||||
return errors.New("ffmpeg: Range response length mismatch")
|
||||
}
|
||||
for remaining > 0 {
|
||||
n, err := resp.Body.Read(buffer[:min(int64(len(buffer)), remaining)])
|
||||
if n > 0 {
|
||||
if _, e := h.disk.spool.WriteAt(buffer[:n], offset); e != nil {
|
||||
return e
|
||||
}
|
||||
offset += int64(n)
|
||||
remaining -= int64(n)
|
||||
}
|
||||
if err != nil {
|
||||
if err == io.EOF && remaining == 0 {
|
||||
return nil
|
||||
}
|
||||
if err == io.EOF {
|
||||
return io.ErrUnexpectedEOF
|
||||
}
|
||||
return err
|
||||
}
|
||||
if n == 0 {
|
||||
return io.ErrNoProgress
|
||||
}
|
||||
}
|
||||
n, err := resp.Body.Read(buffer[:1])
|
||||
if n != 0 || err != io.EOF {
|
||||
if err != nil && err != io.EOF {
|
||||
return err
|
||||
}
|
||||
return errors.New("ffmpeg: oversized Range response")
|
||||
}
|
||||
return nil
|
||||
}()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
func (h *HTTPSpool) Read(p []byte) (int, error) {
|
||||
if !h.Seekable() {
|
||||
return h.disk.Read(p)
|
||||
}
|
||||
h.readMu.Lock()
|
||||
defer h.readMu.Unlock()
|
||||
if len(p) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
if h.pos >= h.Size() {
|
||||
if err := h.disk.spool.discard((h.Size() - 1) / SpoolBlockSize); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return 0, io.EOF
|
||||
}
|
||||
for {
|
||||
if err := h.ctx.Err(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
n, err, wake := h.disk.spool.readAt(p, h.pos)
|
||||
if n > 0 {
|
||||
block := h.pos / SpoolBlockSize
|
||||
h.pos += int64(n)
|
||||
if h.pos/SpoolBlockSize > block {
|
||||
if e := h.disk.spool.discard(block); err == nil {
|
||||
err = e
|
||||
}
|
||||
}
|
||||
return n, err
|
||||
}
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
select {
|
||||
case h.demand <- h.pos:
|
||||
default:
|
||||
}
|
||||
// Read as soon as a partial response supplies these bytes, rather than
|
||||
// waiting for its entire multi-block Range request to finish.
|
||||
select {
|
||||
case <-wake:
|
||||
case <-h.ctx.Done():
|
||||
return 0, h.ctx.Err()
|
||||
}
|
||||
}
|
||||
}
|
||||
func (h *HTTPSpool) Seek(offset int64, whence int) (int64, error) {
|
||||
if !h.Seekable() {
|
||||
return 0, ErrSeek
|
||||
}
|
||||
h.readMu.Lock()
|
||||
defer h.readMu.Unlock()
|
||||
base := int64(0)
|
||||
switch whence {
|
||||
case io.SeekStart:
|
||||
case io.SeekCurrent:
|
||||
base = h.pos
|
||||
case io.SeekEnd:
|
||||
base = h.Size()
|
||||
default:
|
||||
return 0, errors.New("invalid seek origin")
|
||||
}
|
||||
next := base + offset
|
||||
if next < 0 || (offset > 0 && next < base) {
|
||||
return 0, errors.New("invalid seek offset")
|
||||
}
|
||||
h.pos = next
|
||||
return next, nil
|
||||
}
|
||||
func (h *HTTPSpool) Size() int64 { return h.source.Size() }
|
||||
func (h *HTTPSpool) Seekable() bool { return h.source.Seekable() }
|
||||
func (h *HTTPSpool) Progress() (int64, int64) { return h.disk.Progress() }
|
||||
func (h *HTTPSpool) Close() error {
|
||||
h.cancel()
|
||||
h.source.Close()
|
||||
<-h.done
|
||||
<-h.demandDone
|
||||
return h.disk.Close()
|
||||
}
|
||||
|
||||
func (h *HTTPSpool) Failures() <-chan error { return h.disk.Failures() }
|
||||
@@ -0,0 +1,198 @@
|
||||
package ffmpeg
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestHTTPSpoolRangeCrossBlocksAndRefetch(t *testing.T) {
|
||||
data := make([]byte, 4*SpoolBlockSize+57)
|
||||
for i := range data {
|
||||
data[i] = byte(i*31 + i/257)
|
||||
}
|
||||
var large atomic.Bool
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
var start, end int64
|
||||
fmt.Sscanf(r.Header.Get("Range"), "bytes=%d-%d", &start, &end)
|
||||
if end-start+1 > SpoolBlockSize {
|
||||
large.Store(true)
|
||||
}
|
||||
w.Header().Set("ETag", `"stable"`)
|
||||
http.ServeContent(w, r, "media", time.Time{}, bytes.NewReader(data))
|
||||
}))
|
||||
defer srv.Close()
|
||||
input, err := OpenSpoolingHTTP(context.Background(), srv.Client(), HTTPSource{URL: srv.URL}, t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer input.Close()
|
||||
h := input.(*HTTPSpool)
|
||||
select {
|
||||
case <-h.done:
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatal("prefetch did not finish")
|
||||
}
|
||||
if !large.Load() {
|
||||
t.Fatal("did not fetch multiple blocks in one response")
|
||||
}
|
||||
for _, offset := range []int64{3*SpoolBlockSize - 123, 0, 2*SpoolBlockSize + 41, SpoolBlockSize - 5} {
|
||||
if _, err := h.Seek(offset, io.SeekStart); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := make([]byte, min(int64(2*SpoolBlockSize), int64(len(data))-offset))
|
||||
if _, err := io.ReadFull(h, got); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !bytes.Equal(got, data[offset:offset+int64(len(got))]) {
|
||||
t.Fatal("incorrect random/cross-block read", offset)
|
||||
}
|
||||
}
|
||||
unique, received := h.Progress()
|
||||
if unique != int64(len(data)) || received <= unique {
|
||||
t.Fatal("refetch progress must not double-count stored bytes", unique, received)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHTTPSpoolSequentialAndCancellation(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
entered := make(chan struct{})
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Write([]byte("hello"))
|
||||
w.(http.Flusher).Flush()
|
||||
close(entered)
|
||||
<-r.Context().Done()
|
||||
}))
|
||||
defer srv.Close()
|
||||
in, err := OpenSpoolingHTTP(ctx, srv.Client(), HTTPSource{URL: srv.URL}, t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
<-entered
|
||||
if in.Seekable() {
|
||||
t.Fatal("200 response advertised seek")
|
||||
}
|
||||
got := make([]byte, 5)
|
||||
if _, err := io.ReadFull(in, got); err != nil || string(got) != "hello" {
|
||||
t.Fatal(string(got), err)
|
||||
}
|
||||
cancel()
|
||||
if err := in.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHTTPSpoolRejectsBadRanges(t *testing.T) {
|
||||
for _, mode := range []string{"etag", "length", "offset", "status", "truncated", "encoding"} {
|
||||
t.Run(mode, func(t *testing.T) {
|
||||
var calls atomic.Int32
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
first := calls.Add(1) == 1
|
||||
start, end := int64(0), int64(httpBlockSize-1)
|
||||
if !first {
|
||||
fmt.Sscanf(r.Header.Get("Range"), "bytes=%d-%d", &start, &end)
|
||||
}
|
||||
tag := `"stable"`
|
||||
if !first && mode == "etag" {
|
||||
tag = `"changed"`
|
||||
}
|
||||
w.Header().Set("ETag", tag)
|
||||
if !first && mode == "status" {
|
||||
w.WriteHeader(200)
|
||||
return
|
||||
}
|
||||
if !first && mode == "offset" {
|
||||
start++
|
||||
}
|
||||
if !first && mode == "encoding" {
|
||||
w.Header().Set("Content-Encoding", "gzip")
|
||||
}
|
||||
w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, 2*SpoolBlockSize))
|
||||
length := end - start + 1
|
||||
if !first && mode == "length" {
|
||||
length++
|
||||
}
|
||||
w.Header().Set("Content-Length", fmt.Sprint(length))
|
||||
w.WriteHeader(206)
|
||||
if !first && mode == "truncated" {
|
||||
length /= 2
|
||||
}
|
||||
chunk := make([]byte, 64*1024)
|
||||
for length > 0 {
|
||||
n := min(int64(len(chunk)), length)
|
||||
if _, err := w.Write(chunk[:n]); err != nil {
|
||||
return
|
||||
}
|
||||
length -= n
|
||||
}
|
||||
}))
|
||||
defer srv.Close()
|
||||
in, err := OpenSpoolingHTTP(context.Background(), srv.Client(), HTTPSource{URL: srv.URL}, t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer in.Close()
|
||||
h := in.(*HTTPSpool)
|
||||
select {
|
||||
case err := <-h.Failures():
|
||||
if err == nil {
|
||||
t.Fatal("nil failure")
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("bad range did not fail")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestHTTPSpoolReadsPartialRangeBeforeResponseFinishes(t *testing.T) {
|
||||
ready, finish := make(chan struct{}), make(chan struct{})
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
var start, end int64
|
||||
fmt.Sscanf(r.Header.Get("Range"), "bytes=%d-%d", &start, &end)
|
||||
end = min(end, 4*1024*1024-1)
|
||||
w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, 4*1024*1024))
|
||||
w.Header().Set("Content-Length", fmt.Sprint(end-start+1))
|
||||
w.WriteHeader(206)
|
||||
if start == 0 {
|
||||
w.Write(make([]byte, end+1))
|
||||
return
|
||||
}
|
||||
w.Write(bytes.Repeat([]byte{93}, 64*1024))
|
||||
w.(http.Flusher).Flush()
|
||||
close(ready)
|
||||
select {
|
||||
case <-finish:
|
||||
w.Write(make([]byte, end-start+1-64*1024))
|
||||
case <-r.Context().Done():
|
||||
}
|
||||
}))
|
||||
defer srv.Close()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
|
||||
defer cancel()
|
||||
input, err := OpenSpoolingHTTP(ctx, srv.Client(), HTTPSource{URL: srv.URL}, t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer input.Close()
|
||||
select {
|
||||
case <-ready:
|
||||
case <-ctx.Done():
|
||||
t.Fatal(ctx.Err())
|
||||
}
|
||||
if _, err := input.Seek(1024*1024, io.SeekStart); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b := make([]byte, 1)
|
||||
if _, err := io.ReadFull(input, b); err != nil || b[0] != 93 {
|
||||
t.Fatal("waited for complete response", b, err)
|
||||
}
|
||||
close(finish)
|
||||
}
|
||||
@@ -0,0 +1,252 @@
|
||||
// Package ffmpeg runs an embedded FFmpeg with host-controlled media inputs.
|
||||
package ffmpeg
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
const httpBlockSize = 1024 * 1024
|
||||
|
||||
// Input is a file visible to FFmpeg. Only verified HTTP Range sources seek.
|
||||
type Input interface {
|
||||
io.ReadCloser
|
||||
Seek(int64, int) (int64, error)
|
||||
Size() int64
|
||||
Seekable() bool
|
||||
}
|
||||
|
||||
var ErrSeek = errors.New("ffmpeg: input requires seeking but source is sequential")
|
||||
|
||||
type HTTPSource struct {
|
||||
URL string `json:"url"`
|
||||
Headers map[string]string `json:"headers"`
|
||||
}
|
||||
|
||||
type httpInput struct {
|
||||
ctx context.Context
|
||||
client *http.Client
|
||||
source HTTPSource
|
||||
body io.ReadCloser
|
||||
size, pos, blockStart int64
|
||||
block []byte
|
||||
ranged bool
|
||||
etag, modified string
|
||||
}
|
||||
|
||||
// OpenHTTP combines capability detection with the first media read. A 200
|
||||
// response remains open and is consumed sequentially without another request.
|
||||
func OpenHTTP(ctx context.Context, client *http.Client, source HTTPSource) (Input, error) {
|
||||
u, err := url.Parse(source.URL)
|
||||
if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" {
|
||||
return nil, errors.New("ffmpeg: expected an HTTP(S) input URL")
|
||||
}
|
||||
h := &httpInput{ctx: ctx, client: client, source: source, size: -1}
|
||||
resp, err := h.request(0, httpBlockSize-1)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if resp.StatusCode == http.StatusOK {
|
||||
if err = identityEncoding(resp); err != nil {
|
||||
resp.Body.Close()
|
||||
return nil, err
|
||||
}
|
||||
h.body, h.size = resp.Body, resp.ContentLength
|
||||
return h, nil
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusPartialContent {
|
||||
return nil, fmt.Errorf("ffmpeg: HTTP input returned %d", resp.StatusCode)
|
||||
}
|
||||
start, end, size, err := contentRange(resp.Header.Get("Content-Range"))
|
||||
if err != nil || start != 0 || end >= httpBlockSize {
|
||||
return nil, errors.New("ffmpeg: invalid initial Content-Range")
|
||||
}
|
||||
h.size, h.ranged = size, true
|
||||
h.etag = resp.Header.Get("ETag")
|
||||
if strings.HasPrefix(h.etag, "W/") {
|
||||
h.etag = ""
|
||||
}
|
||||
h.modified = resp.Header.Get("Last-Modified")
|
||||
h.block, err = readBlock(resp, end-start+1)
|
||||
return h, err
|
||||
}
|
||||
|
||||
func (h *httpInput) request(start, end int64) (*http.Response, error) {
|
||||
ctx, cancel := context.WithCancel(h.ctx)
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, h.source.URL, nil)
|
||||
if err != nil {
|
||||
cancel()
|
||||
return nil, err
|
||||
}
|
||||
for k, v := range h.source.Headers {
|
||||
req.Header.Set(k, v)
|
||||
}
|
||||
// Offsets must refer to the actual media bytes, not a decompressed response.
|
||||
req.Header.Set("Accept-Encoding", "identity")
|
||||
req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", start, end))
|
||||
if h.etag != "" {
|
||||
req.Header.Set("If-Range", h.etag)
|
||||
} else if h.modified != "" {
|
||||
req.Header.Set("If-Range", h.modified)
|
||||
}
|
||||
timer := time.AfterFunc(15*time.Second, cancel)
|
||||
resp, err := h.client.Do(req)
|
||||
timer.Stop()
|
||||
if err != nil {
|
||||
cancel()
|
||||
return nil, err
|
||||
}
|
||||
resp.Body = &idleMediaBody{ReadCloser: resp.Body, cancel: cancel}
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
func identityEncoding(resp *http.Response) error {
|
||||
if enc := resp.Header.Get("Content-Encoding"); enc != "" && !strings.EqualFold(enc, "identity") {
|
||||
return errors.New("ffmpeg: encoded HTTP responses cannot represent media byte offsets")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func contentRange(value string) (start, end, size int64, err error) {
|
||||
err = errors.New("invalid Content-Range")
|
||||
if !strings.HasPrefix(value, "bytes ") {
|
||||
return
|
||||
}
|
||||
parts := strings.Split(strings.TrimPrefix(value, "bytes "), "/")
|
||||
if len(parts) != 2 {
|
||||
return
|
||||
}
|
||||
bounds := strings.Split(parts[0], "-")
|
||||
if len(bounds) != 2 {
|
||||
return
|
||||
}
|
||||
var e error
|
||||
if start, e = strconv.ParseInt(bounds[0], 10, 64); e != nil {
|
||||
return
|
||||
}
|
||||
if end, e = strconv.ParseInt(bounds[1], 10, 64); e != nil {
|
||||
return
|
||||
}
|
||||
if size, e = strconv.ParseInt(parts[1], 10, 64); e != nil {
|
||||
return
|
||||
}
|
||||
if start < 0 || end < start || size <= end {
|
||||
return
|
||||
}
|
||||
err = nil
|
||||
return
|
||||
}
|
||||
|
||||
func readBlock(resp *http.Response, length int64) ([]byte, error) {
|
||||
if err := identityEncoding(resp); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if resp.ContentLength >= 0 && resp.ContentLength != length {
|
||||
return nil, errors.New("ffmpeg: Range response length mismatch")
|
||||
}
|
||||
b, err := io.ReadAll(io.LimitReader(resp.Body, length+1))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if int64(len(b)) != length {
|
||||
return nil, errors.New("ffmpeg: truncated or oversized Range response")
|
||||
}
|
||||
return b, nil
|
||||
}
|
||||
|
||||
func (h *httpInput) Read(p []byte) (int, error) {
|
||||
if len(p) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
if err := h.ctx.Err(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if !h.ranged {
|
||||
return h.body.Read(p)
|
||||
}
|
||||
if h.pos >= h.size {
|
||||
return 0, io.EOF
|
||||
}
|
||||
if h.pos < h.blockStart || h.pos >= h.blockStart+int64(len(h.block)) {
|
||||
end := min(h.pos+httpBlockSize-1, h.size-1)
|
||||
resp, err := h.request(h.pos, end)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusPartialContent {
|
||||
return 0, fmt.Errorf("ffmpeg: Range request returned %d; source changed or stopped supporting Range", resp.StatusCode)
|
||||
}
|
||||
start, actualEnd, size, err := contentRange(resp.Header.Get("Content-Range"))
|
||||
if err != nil || start != h.pos || actualEnd > end || size != h.size {
|
||||
return 0, errors.New("ffmpeg: inconsistent Content-Range")
|
||||
}
|
||||
if h.etag != "" && resp.Header.Get("ETag") != h.etag {
|
||||
return 0, errors.New("ffmpeg: HTTP input ETag changed")
|
||||
}
|
||||
if h.etag == "" && h.modified != "" && resp.Header.Get("Last-Modified") != h.modified {
|
||||
return 0, errors.New("ffmpeg: HTTP input Last-Modified changed")
|
||||
}
|
||||
h.block, err = readBlock(resp, actualEnd-start+1)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
h.blockStart = start
|
||||
}
|
||||
n := copy(p, h.block[h.pos-h.blockStart:])
|
||||
h.pos += int64(n)
|
||||
return n, nil
|
||||
}
|
||||
|
||||
func (h *httpInput) Seek(offset int64, whence int) (int64, error) {
|
||||
if !h.ranged {
|
||||
return 0, ErrSeek
|
||||
}
|
||||
base := int64(0)
|
||||
switch whence {
|
||||
case io.SeekStart:
|
||||
case io.SeekCurrent:
|
||||
base = h.pos
|
||||
case io.SeekEnd:
|
||||
base = h.size
|
||||
default:
|
||||
return 0, errors.New("invalid seek origin")
|
||||
}
|
||||
next := base + offset
|
||||
if next < 0 || (offset > 0 && next < base) {
|
||||
return 0, errors.New("invalid seek offset")
|
||||
}
|
||||
h.pos = next
|
||||
return next, nil
|
||||
}
|
||||
|
||||
func (h *httpInput) Size() int64 { return h.size }
|
||||
func (h *httpInput) Seekable() bool { return h.ranged }
|
||||
func (h *httpInput) Close() error {
|
||||
if h.body != nil {
|
||||
return h.body.Close()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Only time spent waiting on the network counts toward this idle deadline.
|
||||
// Pauses between reads (disk writes or FFmpeg scheduling) do not count.
|
||||
type idleMediaBody struct {
|
||||
io.ReadCloser
|
||||
cancel context.CancelFunc
|
||||
}
|
||||
|
||||
func (b *idleMediaBody) Read(p []byte) (int, error) {
|
||||
timer := time.AfterFunc(15*time.Second, b.cancel)
|
||||
defer timer.Stop()
|
||||
return b.ReadCloser.Read(p)
|
||||
}
|
||||
func (b *idleMediaBody) Close() error { b.cancel(); return b.ReadCloser.Close() }
|
||||
@@ -0,0 +1,131 @@
|
||||
package ffmpeg
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestHTTPRange(t *testing.T) {
|
||||
data := bytes.Repeat([]byte("0123456789"), 300000)
|
||||
var requests atomic.Int32
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
count := requests.Add(1)
|
||||
if r.Header.Get("X-Test") != "secret" {
|
||||
t.Error("missing headers")
|
||||
}
|
||||
if r.Header.Get("Accept-Encoding") != "identity" {
|
||||
t.Error("missing identity encoding")
|
||||
}
|
||||
if count > 1 && r.Header.Get("If-Range") != `"v1"` {
|
||||
t.Error("missing If-Range")
|
||||
}
|
||||
w.Header().Set("ETag", `"v1"`)
|
||||
http.ServeContent(w, r, "media", time.Time{}, bytes.NewReader(data))
|
||||
}))
|
||||
defer srv.Close()
|
||||
in, err := OpenHTTP(context.Background(), srv.Client(), HTTPSource{URL: srv.URL, Headers: map[string]string{"X-Test": "secret"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer in.Close()
|
||||
if !in.Seekable() || in.Size() != int64(len(data)) {
|
||||
t.Fatal("range not detected")
|
||||
}
|
||||
for _, off := range []int64{0, int64(len(data) - 16), 12, int64(httpBlockSize + 35)} {
|
||||
if _, err = in.Seek(off, io.SeekStart); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b := make([]byte, 16)
|
||||
if _, err = io.ReadFull(in, b); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !bytes.Equal(b, data[off:off+16]) {
|
||||
t.Fatalf("wrong bytes at %d", off)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestHTTPSequentialAndValidation(t *testing.T) {
|
||||
for _, kind := range []string{"sequential", "bad-range", "changed", "truncated", "unauthorized"} {
|
||||
t.Run(kind, func(t *testing.T) {
|
||||
var requests atomic.Int32
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
count := requests.Add(1)
|
||||
switch kind {
|
||||
case "sequential":
|
||||
w.Write([]byte("media"))
|
||||
case "unauthorized":
|
||||
w.WriteHeader(403)
|
||||
case "bad-range":
|
||||
w.Header().Set("Content-Range", "bytes 2-6/10")
|
||||
w.WriteHeader(206)
|
||||
w.Write([]byte("media"))
|
||||
case "truncated":
|
||||
w.Header().Set("Content-Range", "bytes 0-9/10")
|
||||
w.WriteHeader(206)
|
||||
w.Write([]byte("media"))
|
||||
case "changed":
|
||||
if count == 1 {
|
||||
w.Header().Set("Content-Range", "bytes 0-4/10")
|
||||
w.Header().Set("ETag", `"v1"`)
|
||||
w.WriteHeader(206)
|
||||
}
|
||||
w.Write([]byte("media"))
|
||||
}
|
||||
}))
|
||||
defer srv.Close()
|
||||
in, err := OpenHTTP(context.Background(), srv.Client(), HTTPSource{URL: srv.URL})
|
||||
if kind == "sequential" {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer in.Close()
|
||||
if in.Seekable() {
|
||||
t.Fatal("unexpected seeking")
|
||||
}
|
||||
if _, err = in.Seek(0, 0); err != ErrSeek {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b, err := io.ReadAll(in)
|
||||
if err != nil || string(b) != "media" || requests.Load() != 1 {
|
||||
t.Fatalf("%q %v count %d", b, err, requests.Load())
|
||||
}
|
||||
} else if kind == "changed" {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer in.Close()
|
||||
in.Seek(5, 0)
|
||||
if _, err = in.Read(make([]byte, 2)); err == nil {
|
||||
t.Fatal("accepted 200 after range")
|
||||
}
|
||||
} else if err == nil {
|
||||
in.Close()
|
||||
t.Fatal("accepted invalid response")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestOutputArguments(t *testing.T) {
|
||||
a, err := Arguments("", []string{"-shortest", "-metadata", "title=test"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(strings.Join(a, " "), "-f mp4 pipe:1") {
|
||||
t.Fatal(a)
|
||||
}
|
||||
for _, args := range [][]string{{"-i", "/etc/passwd"}, {"-report"}, {"-c:v", "libx264"}, {"-metadata"}, {"other.mp4"}} {
|
||||
if _, err := Arguments("mp4", args); err == nil {
|
||||
t.Fatal(fmt.Sprint(args))
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,255 @@
|
||||
package ffmpeg
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
ffembed "codeberg.org/gruf/go-ffmpreg/embed"
|
||||
"codeberg.org/gruf/go-ffmpreg/wasm"
|
||||
"github.com/tetratelabs/wazero"
|
||||
expsys "github.com/tetratelabs/wazero/experimental/sys"
|
||||
"github.com/tetratelabs/wazero/experimental/sysfs"
|
||||
"github.com/tetratelabs/wazero/sys"
|
||||
)
|
||||
|
||||
var shared struct {
|
||||
once sync.Once
|
||||
runtime wazero.Runtime
|
||||
module wazero.CompiledModule
|
||||
err error
|
||||
}
|
||||
|
||||
// Limit active instances independently of the number of extension engines.
|
||||
var slots = make(chan struct{}, 2)
|
||||
|
||||
func initialize() {
|
||||
shared.once.Do(func() {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
shared.err = fmt.Errorf("ffmpeg initialization: %v", r)
|
||||
}
|
||||
}()
|
||||
cfg := wazero.NewRuntimeConfig().WithCloseOnContextDone(true).WithMemoryLimitPages(8192)
|
||||
shared.runtime, shared.err = wasm.NewRuntime(context.Background(), cfg)
|
||||
if shared.err != nil {
|
||||
return
|
||||
}
|
||||
shared.module, shared.err = shared.runtime.CompileModule(context.Background(), ffembed.B())
|
||||
if shared.err != nil {
|
||||
shared.runtime.Close(context.Background())
|
||||
return
|
||||
}
|
||||
ffembed.Free()
|
||||
})
|
||||
}
|
||||
|
||||
// Output arguments deliberately do not accept extra inputs, output paths or
|
||||
// global filesystem/logging options. The host owns all I/O and cancellation.
|
||||
var outputOptions = map[string]int{
|
||||
"-c": 1, "-codec": 1, "-c:v": 1, "-c:a": 1, "-codec:v": 1, "-codec:a": 1,
|
||||
"-map": 1, "-map_metadata": 1, "-map_chapters": 1, "-metadata": 1, "-metadata:s:a:0": 1, "-metadata:s:v:0": 1,
|
||||
"-movflags": 1, "-frag_duration": 1, "-frag_size": 1, "-min_frag_duration": 1,
|
||||
"-bsf:v": 1, "-bsf:a": 1, "-strict": 1, "-shortest": 0, "-t": 1, "-to": 1,
|
||||
"-avoid_negative_ts": 1, "-max_interleave_delta": 1, "-flush_packets": 1,
|
||||
"-disposition:a:0": 1, "-disposition:v:0": 1,
|
||||
}
|
||||
|
||||
func Arguments(format string, extra []string) ([]string, error) {
|
||||
if format == "" {
|
||||
format = "mp4"
|
||||
}
|
||||
if format == "mkv" {
|
||||
format = "matroska"
|
||||
}
|
||||
switch format {
|
||||
case "mp4", "matroska", "webm", "mpegts":
|
||||
default:
|
||||
return nil, fmt.Errorf("ffmpeg: unsupported output format %q", format)
|
||||
}
|
||||
for i := 0; i < len(extra); {
|
||||
count, ok := outputOptions[extra[i]]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("ffmpeg: unsupported output option %q", extra[i])
|
||||
}
|
||||
if i+count >= len(extra) {
|
||||
return nil, fmt.Errorf("ffmpeg: missing value for %s", extra[i])
|
||||
}
|
||||
if strings.HasPrefix(extra[i], "-c") && extra[i+1] != "copy" {
|
||||
return nil, errors.New("ffmpeg: merge only supports stream copy")
|
||||
}
|
||||
i += count + 1
|
||||
}
|
||||
args := []string{"-hide_banner", "-nostdin", "-xerror", "-loglevel", "error", "-protocol_whitelist", "file,pipe", "-i", "/inputs/video", "-protocol_whitelist", "file,pipe", "-i", "/inputs/audio", "-map", "0:v:0", "-map", "1:a:0", "-c", "copy"}
|
||||
fragmentFlags := ""
|
||||
for i := 0; i < len(extra); {
|
||||
count := outputOptions[extra[i]]
|
||||
if format == "mp4" && extra[i] == "-movflags" {
|
||||
if strings.Contains(extra[i+1], "faststart") || strings.Contains(extra[i+1], "global_sidx") {
|
||||
return nil, errors.New("ffmpeg: these movflags require seekable output")
|
||||
}
|
||||
fragmentFlags = extra[i+1]
|
||||
} else {
|
||||
args = append(args, extra[i:i+count+1]...)
|
||||
}
|
||||
i += count + 1
|
||||
}
|
||||
if format == "mp4" {
|
||||
args = append(args, "-movflags", fragmentFlags+"+frag_keyframe+empty_moov+default_base_moof")
|
||||
}
|
||||
args = append(args, "-f", format, "pipe:1")
|
||||
return args, nil
|
||||
}
|
||||
|
||||
// Run shares compilation, but gives every merge its own memory and filesystem.
|
||||
// A clean stdout EOF is only delivered by the caller after this succeeds.
|
||||
func Run(ctx context.Context, video, audio Input, out io.Writer, format string, extra []string) error {
|
||||
_, err := Arguments(format, extra)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
release, err := Acquire(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer release()
|
||||
return RunAcquired(ctx, video, audio, out, format, extra)
|
||||
}
|
||||
|
||||
// Acquire reserves WASM execution capacity. Disk-backed input downloads may
|
||||
// already be running while this call waits.
|
||||
func Acquire(ctx context.Context) (func(), error) {
|
||||
select {
|
||||
case slots <- struct{}{}:
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
var once sync.Once
|
||||
return func() { once.Do(func() { <-slots }) }, nil
|
||||
}
|
||||
|
||||
// RunAcquired requires a reservation held until this call returns.
|
||||
func RunAcquired(ctx context.Context, video, audio Input, out io.Writer, format string, extra []string) error {
|
||||
args, err := Arguments(format, extra)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ready := make(chan struct{})
|
||||
go func() { initialize(); close(ready) }()
|
||||
select {
|
||||
case <-ready:
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
if shared.err != nil {
|
||||
return shared.err
|
||||
}
|
||||
files := &inputFS{files: map[string]Input{"video": video, "audio": audio}}
|
||||
fsconfig := wazero.NewFSConfig().(sysfs.FSConfig).WithSysFSMount(files, "/inputs")
|
||||
logs := &tailLog{}
|
||||
rc, err := wasm.Run(ctx, shared.runtime, shared.module, wasm.Args{
|
||||
Name: "ffmpeg", Args: args, Stdout: out, Stderr: logs,
|
||||
Config: func(c wazero.ModuleConfig) wazero.ModuleConfig { return c.WithFSConfig(fsconfig) },
|
||||
})
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
if files.err != nil {
|
||||
return fmt.Errorf("ffmpeg input: %w (%s)", files.err, logs.String())
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("ffmpeg runtime: %w (%s)", err, logs.String())
|
||||
}
|
||||
if rc != 0 {
|
||||
return fmt.Errorf("ffmpeg exited with code %d: %s", rc, logs.String())
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type tailLog struct{ data []byte }
|
||||
|
||||
func (l *tailLog) Write(p []byte) (int, error) {
|
||||
n := len(p)
|
||||
if len(p) > 8192 {
|
||||
p = p[len(p)-8192:]
|
||||
}
|
||||
l.data = append(l.data, p...)
|
||||
if len(l.data) > 8192 {
|
||||
l.data = l.data[len(l.data)-8192:]
|
||||
}
|
||||
return n, nil
|
||||
}
|
||||
func (l *tailLog) String() string { return strings.TrimSpace(string(l.data)) }
|
||||
|
||||
type inputFS struct {
|
||||
expsys.UnimplementedFS
|
||||
files map[string]Input
|
||||
err error
|
||||
}
|
||||
|
||||
func (f *inputFS) OpenFile(path string, flag expsys.Oflag, perm fs.FileMode) (expsys.File, expsys.Errno) {
|
||||
if flag&(expsys.O_WRONLY|expsys.O_RDWR|expsys.O_CREAT|expsys.O_TRUNC) != 0 {
|
||||
return nil, expsys.EROFS
|
||||
}
|
||||
path = strings.TrimPrefix(path, "/")
|
||||
if path == "." || path == "" {
|
||||
return &inputFile{owner: f}, 0
|
||||
}
|
||||
in := f.files[path]
|
||||
if in == nil {
|
||||
return nil, expsys.ENOENT
|
||||
}
|
||||
return &inputFile{owner: f, in: in}, 0
|
||||
}
|
||||
func (f *inputFS) Stat(path string) (sys.Stat_t, expsys.Errno) {
|
||||
file, err := f.OpenFile(path, 0, 0)
|
||||
if err != 0 {
|
||||
return sys.Stat_t{}, err
|
||||
}
|
||||
return file.Stat()
|
||||
}
|
||||
|
||||
type inputFile struct {
|
||||
expsys.UnimplementedFile
|
||||
owner *inputFS
|
||||
in Input
|
||||
}
|
||||
|
||||
func (f *inputFile) IsDir() (bool, expsys.Errno) { return f.in == nil, 0 }
|
||||
func (f *inputFile) Stat() (sys.Stat_t, expsys.Errno) {
|
||||
if f.in == nil {
|
||||
return sys.Stat_t{Mode: fs.ModeDir | 0555}, 0
|
||||
}
|
||||
mode := fs.FileMode(0444)
|
||||
if !f.in.Seekable() {
|
||||
mode |= fs.ModeNamedPipe
|
||||
}
|
||||
return sys.Stat_t{Mode: mode, Size: max(0, f.in.Size()), Nlink: 1}, 0
|
||||
}
|
||||
func (f *inputFile) Read(p []byte) (int, expsys.Errno) {
|
||||
if f.in == nil {
|
||||
return 0, expsys.EISDIR
|
||||
}
|
||||
n, err := f.in.Read(p)
|
||||
if err != nil && err != io.EOF {
|
||||
f.owner.err = err
|
||||
return n, expsys.EIO
|
||||
}
|
||||
return n, 0
|
||||
}
|
||||
func (f *inputFile) Seek(off int64, whence int) (int64, expsys.Errno) {
|
||||
if f.in == nil {
|
||||
return 0, expsys.EISDIR
|
||||
}
|
||||
n, err := f.in.Seek(off, whence)
|
||||
// Capability probes may legitimately fail; a failed seek alone is not a
|
||||
// media error. FFmpeg decides whether it can continue sequentially.
|
||||
if err != nil {
|
||||
return 0, expsys.ENOSYS
|
||||
}
|
||||
return n, 0
|
||||
}
|
||||
@@ -0,0 +1,221 @@
|
||||
package ffmpeg
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/binary"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"codeberg.org/gruf/go-ffmpreg/wasm"
|
||||
"github.com/tetratelabs/wazero"
|
||||
)
|
||||
|
||||
// Generate tiny deterministic media with the exact embedded build under test;
|
||||
// no system FFmpeg, network media or third-party fixture licensing is needed.
|
||||
func fixture(t *testing.T, audio bool) []byte {
|
||||
t.Helper()
|
||||
initialize()
|
||||
if shared.err != nil {
|
||||
t.Fatal(shared.err)
|
||||
}
|
||||
args := []string{"-hide_banner", "-loglevel", "error", "-f", "lavfi", "-i", "color=c=black:s=32x32:r=10", "-t", "0.3", "-c:v", "libx264", "-preset", "ultrafast", "-movflags", "frag_keyframe+empty_moov", "-f", "mp4", "pipe:1"}
|
||||
if audio {
|
||||
args = []string{"-hide_banner", "-loglevel", "error", "-f", "lavfi", "-i", "anullsrc=r=48000:cl=mono", "-t", "0.3", "-c:a", "aac", "-movflags", "frag_keyframe+empty_moov", "-f", "mp4", "pipe:1"}
|
||||
}
|
||||
var out, logs bytes.Buffer
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
rc, err := wasm.Run(ctx, shared.runtime, shared.module, wasm.Args{Name: "ffmpeg", Args: args, Stdout: &out, Stderr: &logs})
|
||||
if err != nil || rc != 0 {
|
||||
t.Fatalf("fixture: %d %v %s", rc, err, logs.String())
|
||||
}
|
||||
return out.Bytes()
|
||||
}
|
||||
|
||||
func TestTailMoovNeedsRange(t *testing.T) {
|
||||
initialize()
|
||||
if shared.err != nil {
|
||||
t.Fatal(shared.err)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
dir := t.TempDir()
|
||||
var logs bytes.Buffer
|
||||
rc, err := wasm.Run(ctx, shared.runtime, shared.module, wasm.Args{Name: "ffmpeg", Args: []string{
|
||||
"-v", "error", "-f", "lavfi", "-i", "color=c=black:s=32x32:r=10", "-t", "0.3", "-c:v", "libx264", "-preset", "ultrafast", "/out/video.mp4",
|
||||
}, Stderr: &logs, Config: func(c wazero.ModuleConfig) wazero.ModuleConfig {
|
||||
return c.WithFSConfig(wazero.NewFSConfig().WithDirMount(dir, "/out"))
|
||||
}})
|
||||
if rc != 0 || err != nil {
|
||||
t.Fatalf("fixture %d %v %s", rc, err, logs.String())
|
||||
}
|
||||
raw, err := os.ReadFile(filepath.Join(dir, "video.mp4"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var video []byte
|
||||
for offset := 0; offset < len(raw); {
|
||||
length := int(binary.BigEndian.Uint32(raw[offset : offset+4]))
|
||||
if string(raw[offset+4:offset+8]) == "moov" {
|
||||
free := make([]byte, 2*httpBlockSize)
|
||||
binary.BigEndian.PutUint32(free, uint32(len(free)))
|
||||
copy(free[4:], "free")
|
||||
video = append(video, free...)
|
||||
}
|
||||
video = append(video, raw[offset:offset+length]...)
|
||||
offset += length
|
||||
}
|
||||
audio := fixture(t, true)
|
||||
for _, ranged := range []bool{true, false} {
|
||||
t.Run(fmtBool(ranged), func(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
data := video
|
||||
if r.URL.Path == "/audio" {
|
||||
data = audio
|
||||
}
|
||||
if ranged {
|
||||
http.ServeContent(w, r, "media", time.Time{}, bytes.NewReader(data))
|
||||
} else {
|
||||
w.Write(data)
|
||||
}
|
||||
}))
|
||||
defer srv.Close()
|
||||
v, err := OpenHTTP(ctx, srv.Client(), HTTPSource{URL: srv.URL})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer v.Close()
|
||||
a, err := OpenHTTP(ctx, srv.Client(), HTTPSource{URL: srv.URL + "/audio"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer a.Close()
|
||||
err = Run(ctx, v, a, io.Discard, "mp4", nil)
|
||||
if ranged && err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !ranged && err == nil {
|
||||
t.Fatal("non-seekable tail-moov input unexpectedly succeeded")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func fmtBool(b bool) string {
|
||||
if b {
|
||||
return "range"
|
||||
}
|
||||
return "sequential"
|
||||
}
|
||||
|
||||
func TestWebMToMP4(t *testing.T) {
|
||||
initialize()
|
||||
if shared.err != nil {
|
||||
t.Fatal(shared.err)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
inputs := make([]*DiskInput, 2)
|
||||
for i, name := range []string{"video.webm", "audio.webm"} {
|
||||
data, err := os.ReadFile(filepath.Join("..", "testdata", "ffmpeg", name))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
inputs[i], err = NewDiskInput(ctx, t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer inputs[i].Close()
|
||||
if _, err = inputs[i].Write(data); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
inputs[i].End(nil)
|
||||
}
|
||||
|
||||
var output, probe, logs bytes.Buffer
|
||||
if err := Run(ctx, inputs[0], inputs[1], &output, "mp4", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rc, err := wasm.Run(ctx, shared.runtime, shared.module, wasm.Args{Name: "ffprobe", Args: []string{"-v", "error", "-show_entries", "stream=codec_name", "-of", "json", "pipe:0"}, Stdin: bytes.NewReader(output.Bytes()), Stdout: &probe, Stderr: &logs})
|
||||
if rc != 0 || err != nil {
|
||||
t.Fatalf("probe %d %v %s", rc, err, logs.String())
|
||||
}
|
||||
if !bytes.Contains(probe.Bytes(), []byte(`"vp9"`)) || !bytes.Contains(probe.Bytes(), []byte(`"opus"`)) {
|
||||
t.Fatalf("codecs changed: %s", probe.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestWASMMerge(t *testing.T) {
|
||||
video, audio := fixture(t, false), fixture(t, true)
|
||||
for _, ranged := range []bool{false, true} {
|
||||
t.Run(map[bool]string{false: "sequential", true: "range"}[ranged], func(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
data := video
|
||||
if r.URL.Path == "/audio" {
|
||||
data = audio
|
||||
}
|
||||
if ranged {
|
||||
http.ServeContent(w, r, "media", time.Time{}, bytes.NewReader(data))
|
||||
} else {
|
||||
w.Write(data)
|
||||
}
|
||||
}))
|
||||
defer srv.Close()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
v, err := OpenHTTP(ctx, srv.Client(), HTTPSource{URL: srv.URL + "/video"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer v.Close()
|
||||
a, err := OpenHTTP(ctx, srv.Client(), HTTPSource{URL: srv.URL + "/audio"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer a.Close()
|
||||
var output bytes.Buffer
|
||||
if err = Run(ctx, v, a, &output, "", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !bytes.Contains(output.Bytes(), []byte("moof")) {
|
||||
t.Fatal("output is not fragmented MP4")
|
||||
}
|
||||
// Probe the actual result, rather than just checking its filename/header.
|
||||
var probe, logs bytes.Buffer
|
||||
rc, err := wasm.Run(ctx, shared.runtime, shared.module, wasm.Args{Name: "ffprobe", Args: []string{"-v", "error", "-show_streams", "-of", "json", "pipe:0"}, Stdin: bytes.NewReader(output.Bytes()), Stdout: &probe, Stderr: &logs})
|
||||
if rc != 0 || err != nil {
|
||||
t.Fatalf("probe: %d %v %s", rc, err, logs.String())
|
||||
}
|
||||
var result struct {
|
||||
Streams []struct {
|
||||
CodecType string `json:"codec_type"`
|
||||
CodecName string `json:"codec_name"`
|
||||
}
|
||||
}
|
||||
if err = json.Unmarshal(probe.Bytes(), &result); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(result.Streams) != 2 || result.Streams[0].CodecName != "h264" || result.Streams[1].CodecName != "aac" {
|
||||
t.Fatalf("unexpected streams %s", probe.String())
|
||||
}
|
||||
})
|
||||
}
|
||||
// A malformed input must produce an error, not successful empty output.
|
||||
s, err := NewDiskInput(context.Background(), t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer s.Close()
|
||||
s.Write([]byte("invalid"))
|
||||
s.End(nil)
|
||||
if err := Run(context.Background(), s, s, io.Discard, "", nil); err == nil {
|
||||
t.Fatal("accepted invalid media")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,284 @@
|
||||
package ffmpeg
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
|
||||
"github.com/GopeedLab/gopeed/internal/tempfiles"
|
||||
)
|
||||
|
||||
const SpoolBlockSize int64 = 8 * 1024 * 1024
|
||||
|
||||
type byteSpan struct{ start, end int64 }
|
||||
|
||||
// blockSpool tracks valid byte intervals separately from file length: a sparse
|
||||
// file's unwritten holes are never valid media. No file handles survive a call.
|
||||
type blockSpool struct {
|
||||
owner *tempfiles.Scope
|
||||
failures chan error
|
||||
mu sync.Mutex
|
||||
dir string
|
||||
blocks map[int64][]byteSpan
|
||||
seen []byteSpan
|
||||
wake chan struct{}
|
||||
ended, closed bool
|
||||
err error
|
||||
downloaded, received int64
|
||||
}
|
||||
|
||||
func newBlockSpool(root string, owners ...*tempfiles.Scope) (*blockSpool, error) {
|
||||
if root != "" {
|
||||
if err := os.MkdirAll(root, 0700); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
dir, err := os.MkdirTemp(root, "gopeed-media-")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var owner *tempfiles.Scope
|
||||
if len(owners) > 0 {
|
||||
owner = owners[0]
|
||||
}
|
||||
if err := owner.Track(dir); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &blockSpool{owner: owner, failures: make(chan error, 1), dir: dir, blocks: make(map[int64][]byteSpan), wake: make(chan struct{})}, nil
|
||||
}
|
||||
func (s *blockSpool) notify() { close(s.wake); s.wake = make(chan struct{}) }
|
||||
func (s *blockSpool) path(index int64) string { return filepath.Join(s.dir, fmt.Sprintf("%d", index)) }
|
||||
func addSpan(spans []byteSpan, next byteSpan) ([]byteSpan, int64) {
|
||||
before := int64(0)
|
||||
for _, span := range spans {
|
||||
before += span.end - span.start
|
||||
}
|
||||
out := make([]byteSpan, 0, len(spans)+1)
|
||||
for _, span := range spans {
|
||||
if span.end < next.start {
|
||||
out = append(out, span)
|
||||
} else if next.end < span.start {
|
||||
out = append(out, next)
|
||||
next = span
|
||||
} else {
|
||||
next.start = min(next.start, span.start)
|
||||
next.end = max(next.end, span.end)
|
||||
}
|
||||
}
|
||||
out = append(out, next)
|
||||
after := int64(0)
|
||||
for _, span := range out {
|
||||
after += span.end - span.start
|
||||
}
|
||||
return out, after - before
|
||||
}
|
||||
func (s *blockSpool) WriteAt(p []byte, offset int64) (int, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if offset < 0 {
|
||||
return 0, errors.New("negative media offset")
|
||||
}
|
||||
if s.closed {
|
||||
return 0, io.ErrClosedPipe
|
||||
}
|
||||
if s.err != nil {
|
||||
return 0, s.err
|
||||
}
|
||||
written := 0
|
||||
for len(p) > 0 {
|
||||
index, local := offset/SpoolBlockSize, offset%SpoolBlockSize
|
||||
count := int(min(int64(len(p)), SpoolBlockSize-local))
|
||||
f, err := os.OpenFile(s.path(index), os.O_CREATE|os.O_WRONLY, 0600)
|
||||
if err != nil {
|
||||
s.fail(err)
|
||||
s.notify()
|
||||
return written, err
|
||||
}
|
||||
n, err := f.WriteAt(p[:count], local)
|
||||
closeErr := f.Close()
|
||||
if err == nil {
|
||||
err = closeErr
|
||||
}
|
||||
if n > 0 {
|
||||
s.blocks[index], _ = addSpan(s.blocks[index], byteSpan{local, local + int64(n)})
|
||||
var added int64
|
||||
s.seen, added = addSpan(s.seen, byteSpan{offset, offset + int64(n)})
|
||||
s.downloaded += added
|
||||
s.received += int64(n)
|
||||
written += n
|
||||
offset += int64(n)
|
||||
p = p[n:]
|
||||
}
|
||||
if err == nil && n != count {
|
||||
err = io.ErrShortWrite
|
||||
}
|
||||
if err != nil {
|
||||
s.fail(err)
|
||||
s.notify()
|
||||
return written, err
|
||||
}
|
||||
}
|
||||
s.notify()
|
||||
return written, nil
|
||||
}
|
||||
func (s *blockSpool) readAt(p []byte, offset int64) (int, error, <-chan struct{}) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.closed {
|
||||
return 0, io.ErrClosedPipe, nil
|
||||
}
|
||||
if s.err != nil {
|
||||
return 0, s.err, nil
|
||||
}
|
||||
index, local := offset/SpoolBlockSize, offset%SpoolBlockSize
|
||||
for _, span := range s.blocks[index] {
|
||||
if span.start <= local && local < span.end {
|
||||
f, err := os.Open(s.path(index))
|
||||
if err != nil {
|
||||
return 0, err, nil
|
||||
}
|
||||
n, err := f.ReadAt(p[:min(int64(len(p)), span.end-local)], local)
|
||||
f.Close()
|
||||
return n, err, nil
|
||||
}
|
||||
}
|
||||
return 0, nil, s.wake
|
||||
}
|
||||
func (s *blockSpool) discard(index int64) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if err := os.Remove(s.path(index)); err != nil && !os.IsNotExist(err) {
|
||||
return err
|
||||
}
|
||||
delete(s.blocks, index)
|
||||
return nil
|
||||
}
|
||||
func (s *blockSpool) end(err error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.ended = true
|
||||
if s.err == nil {
|
||||
s.fail(err)
|
||||
}
|
||||
s.notify()
|
||||
}
|
||||
func (s *blockSpool) Close() error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.closed = true
|
||||
clear(s.blocks)
|
||||
s.seen = nil
|
||||
s.notify()
|
||||
return s.owner.Remove(s.dir)
|
||||
}
|
||||
func (s *blockSpool) progress() (int64, int64) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return s.downloaded, s.received
|
||||
}
|
||||
|
||||
// DiskInput consumes a sequential producer without retaining its payload in RAM.
|
||||
// Writes acknowledge durable-to-the-OS chunks, allowing both SABR tracks to drain
|
||||
// even while FFmpeg is queued or currently reading only the other track.
|
||||
type DiskInput struct {
|
||||
ctx context.Context
|
||||
spool *blockSpool
|
||||
writeMu, readMu sync.Mutex
|
||||
written, read int64
|
||||
}
|
||||
|
||||
func NewDiskInput(ctx context.Context, root string, owners ...*tempfiles.Scope) (*DiskInput, error) {
|
||||
spool, err := newBlockSpool(root, owners...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &DiskInput{ctx: ctx, spool: spool}, nil
|
||||
}
|
||||
func (s *DiskInput) Write(p []byte) (int, error) {
|
||||
s.writeMu.Lock()
|
||||
defer s.writeMu.Unlock()
|
||||
if err := s.ctx.Err(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
s.spool.mu.Lock()
|
||||
ended := s.spool.ended
|
||||
s.spool.mu.Unlock()
|
||||
if ended {
|
||||
return 0, io.ErrClosedPipe
|
||||
}
|
||||
n, err := s.spool.WriteAt(p, s.written)
|
||||
s.written += int64(n)
|
||||
return n, err
|
||||
}
|
||||
func (s *DiskInput) End(err error) { s.writeMu.Lock(); defer s.writeMu.Unlock(); s.spool.end(err) }
|
||||
func (s *DiskInput) Read(p []byte) (int, error) {
|
||||
s.readMu.Lock()
|
||||
defer s.readMu.Unlock()
|
||||
if len(p) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
for {
|
||||
if err := s.ctx.Err(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
n, err, wake := s.spool.readAt(p, s.read)
|
||||
if n > 0 {
|
||||
previous := s.read / SpoolBlockSize
|
||||
s.read += int64(n)
|
||||
if s.read/SpoolBlockSize > previous {
|
||||
if e := s.spool.discard(previous); err == nil {
|
||||
err = e
|
||||
}
|
||||
}
|
||||
return n, err
|
||||
}
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
s.spool.mu.Lock()
|
||||
ended := s.spool.ended
|
||||
s.spool.mu.Unlock()
|
||||
// End can race the missing-data check; recheck after observing it.
|
||||
if ended {
|
||||
n, err, _ = s.spool.readAt(p, s.read)
|
||||
if n > 0 {
|
||||
// Use the regular consumption path, including block eviction.
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if err := s.spool.discard(s.read / SpoolBlockSize); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return 0, io.EOF
|
||||
}
|
||||
select {
|
||||
case <-wake:
|
||||
case <-s.ctx.Done():
|
||||
return 0, s.ctx.Err()
|
||||
}
|
||||
}
|
||||
}
|
||||
func (s *DiskInput) Close() error { return s.spool.Close() }
|
||||
func (*DiskInput) Seek(int64, int) (int64, error) { return 0, ErrSeek }
|
||||
func (*DiskInput) Size() int64 { return -1 }
|
||||
func (*DiskInput) Seekable() bool { return false }
|
||||
func (s *DiskInput) Progress() (int64, int64) { return s.spool.progress() }
|
||||
|
||||
// fail is called with mu held. The first error wakes a queued FFmpeg job.
|
||||
func (s *blockSpool) fail(err error) {
|
||||
if err == nil || s.err != nil {
|
||||
return
|
||||
}
|
||||
s.err = err
|
||||
select {
|
||||
case s.failures <- err:
|
||||
default:
|
||||
}
|
||||
}
|
||||
func (s *DiskInput) Failures() <-chan error { return s.spool.failures }
|
||||
@@ -0,0 +1,101 @@
|
||||
package ffmpeg
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"io"
|
||||
"os"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestSpoolSparseCrossBlock(t *testing.T) {
|
||||
s, err := newBlockSpool(t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer s.Close()
|
||||
offset := 7*SpoolBlockSize + 123
|
||||
data := bytes.Repeat([]byte{91}, int(3*SpoolBlockSize+37))
|
||||
if n, err := s.WriteAt(data, offset); err != nil || n != len(data) {
|
||||
t.Fatal(n, err)
|
||||
}
|
||||
if n, err, wake := s.readAt(make([]byte, 16), 7*SpoolBlockSize); n != 0 || err != nil || wake == nil {
|
||||
t.Fatal("sparse hole was exposed", n, err)
|
||||
}
|
||||
got := make([]byte, len(data))
|
||||
pos := 0
|
||||
for pos < len(got) {
|
||||
n, err, _ := s.readAt(got[pos:], offset+int64(pos))
|
||||
if err != nil || n == 0 {
|
||||
t.Fatal(n, err)
|
||||
}
|
||||
pos += n
|
||||
}
|
||||
if !bytes.Equal(data, got) {
|
||||
t.Fatal("cross-block contents differ")
|
||||
}
|
||||
if err := s.discard(8); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n, _, wake := s.readAt(got[:1], 8*SpoolBlockSize); n != 0 || wake == nil {
|
||||
t.Fatal("deleted block remained readable")
|
||||
}
|
||||
if _, err := s.WriteAt(data, offset); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
unique, received := s.progress()
|
||||
if unique != int64(len(data)) || received != 2*unique {
|
||||
t.Fatal(unique, received)
|
||||
}
|
||||
}
|
||||
func TestDiskInputDrainAndCleanup(t *testing.T) {
|
||||
s, err := NewDiskInput(context.Background(), t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
data := bytes.Repeat([]byte{33}, int(3*SpoolBlockSize+99))
|
||||
if _, err := s.Write(data); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
s.End(nil)
|
||||
got, err := io.ReadAll(s)
|
||||
if err != nil || !bytes.Equal(data, got) {
|
||||
t.Fatal("drain", err)
|
||||
}
|
||||
files, err := os.ReadDir(s.spool.dir)
|
||||
if err != nil || len(files) != 0 {
|
||||
t.Fatal("consumed blocks retained", len(files), err)
|
||||
}
|
||||
if err := s.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := os.Stat(s.spool.dir); !os.IsNotExist(err) {
|
||||
t.Fatal("spool not removed", err)
|
||||
}
|
||||
}
|
||||
func TestDiskInputCancelAndWriteFailure(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
s, err := NewDiskInput(ctx, t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer s.Close()
|
||||
cancel()
|
||||
if _, err := s.Read(make([]byte, 1)); err != context.Canceled {
|
||||
t.Fatal(err)
|
||||
}
|
||||
broken, err := NewDiskInput(context.Background(), t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer broken.Close()
|
||||
if err := os.RemoveAll(broken.spool.dir); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := broken.Write([]byte{1}); err == nil {
|
||||
t.Fatal("missing disk error")
|
||||
}
|
||||
if _, err := broken.Read(make([]byte, 1)); err == nil {
|
||||
t.Fatal("disk error not propagated")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,433 @@
|
||||
package engine
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"github.com/GopeedLab/gopeed/internal/production"
|
||||
media "github.com/GopeedLab/gopeed/pkg/download/engine/ffmpeg"
|
||||
"github.com/GopeedLab/gopeed/pkg/download/engine/inject/stream"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestFFmpegJSStreams(t *testing.T) {
|
||||
video, err := os.ReadFile("testdata/ffmpeg/video.mp4")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
audio, err := os.ReadFile("testdata/ffmpeg/audio.mp4")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, mode := range []string{"streams", "http", "mixed"} {
|
||||
t.Run(mode, func(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
b := video
|
||||
if r.URL.Path == "/audio" {
|
||||
b = audio
|
||||
}
|
||||
http.ServeContent(w, r, "media", time.Time{}, bytes.NewReader(b))
|
||||
}))
|
||||
defer srv.Close()
|
||||
e := NewEngine(nil)
|
||||
defer e.Close()
|
||||
// Input data is deliberately interleaved through one JS producer.
|
||||
// Native push/read operations must never block the event loop.
|
||||
e.Runtime.Set("videoBytes", e.Runtime.NewArrayBuffer(video))
|
||||
e.Runtime.Set("audioBytes", e.Runtime.NewArrayBuffer(audio))
|
||||
e.Runtime.Set("sourceURL", srv.URL)
|
||||
e.Runtime.Set("inputMode", mode)
|
||||
result, err := e.RunString(`(async()=>{
|
||||
let vc, ac, offset=0;
|
||||
const v=new Uint8Array(videoBytes), a=new Uint8Array(audioBytes);
|
||||
const video=new ReadableStream({start(c){vc=c;}});
|
||||
const audio=new ReadableStream({start(c){ac=c;}});
|
||||
const timer=setInterval(()=>{
|
||||
if(offset<v.length)vc.enqueue(v.slice(offset,offset+97));
|
||||
if(offset<a.length)ac.enqueue(a.slice(offset,offset+97));
|
||||
offset+=97;
|
||||
if(offset>=Math.max(v.length,a.length)){clearInterval(timer);vc.close();ac.close();}
|
||||
},1);
|
||||
const stream=__gopeed_ffmpeg.merge({
|
||||
inputs: () => ({ video:inputMode==='http'?{url:sourceURL+'/video'}:video,
|
||||
audio:inputMode==='streams'?audio:{url:sourceURL+'/audio'} }),
|
||||
args:['-metadata','title=gopeed-test']
|
||||
});
|
||||
const bytes=await new Response(stream).arrayBuffer();
|
||||
return new Uint8Array(bytes);
|
||||
})()`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out, ok := result.([]byte)
|
||||
if !ok {
|
||||
t.Fatalf("output %T", result)
|
||||
}
|
||||
if !bytes.Contains(out, []byte("moof")) || !bytes.Contains(out, []byte("gopeed-test")) {
|
||||
t.Fatalf("invalid output (%d bytes)", len(out))
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestFFmpegJSErrorAndCancel(t *testing.T) {
|
||||
for _, scenario := range []string{"input-error", "abort", "output-cancel", "invalid-args"} {
|
||||
t.Run(scenario, func(t *testing.T) {
|
||||
e := NewEngine(nil)
|
||||
defer e.Close()
|
||||
e.Runtime.Set("scenario", scenario)
|
||||
result, err := e.RunString(`(async()=>{
|
||||
let cancelled=0;
|
||||
const source=()=>new ReadableStream({
|
||||
pull(c){if(scenario==='input-error')c.error(new Error('upstream failed'));else return new Promise(()=>{});},
|
||||
cancel(){cancelled++;}
|
||||
});
|
||||
const controller=new AbortController();
|
||||
const output=__gopeed_ffmpeg.merge({inputs:()=>({video:source(),audio:source()}),signal:controller.signal,
|
||||
args:scenario==='invalid-args'?['-i','/outside']:[]});
|
||||
const reader=output.getReader();
|
||||
const pending=reader.read().then(()=>'',e=>e.message);
|
||||
if(scenario==='abort')setTimeout(()=>controller.abort(),20);
|
||||
if(scenario==='output-cancel')setTimeout(()=>reader.cancel(),20);
|
||||
const message=await pending;
|
||||
await new Promise(r=>setTimeout(r,10));
|
||||
return {message,cancelled};
|
||||
})()`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m := result.(map[string]any)
|
||||
if scenario != "output-cancel" && m["message"] == "" {
|
||||
t.Fatalf("missing error: %v", m)
|
||||
}
|
||||
if scenario == "abort" || scenario == "output-cancel" {
|
||||
if fmt.Sprint(m["cancelled"]) != "2" {
|
||||
t.Fatalf("inputs not cancelled: %v", m)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestFFmpegMalformedMediaFails(t *testing.T) {
|
||||
e := NewEngine(nil)
|
||||
defer e.Close()
|
||||
_, err := e.RunString(`(async()=>{
|
||||
const source=()=>new ReadableStream({start(c){c.enqueue(new Uint8Array([1,2,3]));c.close();}});
|
||||
return await new Response(__gopeed_ffmpeg.merge({inputs:()=>({video:source(),audio:source()})})).arrayBuffer();
|
||||
})()`)
|
||||
if err == nil || !strings.Contains(err.Error(), "ffmpeg") {
|
||||
t.Fatalf("expected FFmpeg error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFFmpegBlobOpener(t *testing.T) {
|
||||
var open stream.ObjectURLOpener
|
||||
e := NewEngine(&Config{StreamConfig: &stream.Config{CreateObjectURL: func(opts *stream.ObjectURLOptions, opener stream.ObjectURLOpener) (string, error) {
|
||||
if opts.Range || opts.Size != 0 {
|
||||
t.Fatal("merged output must not advertise range or a guessed size")
|
||||
}
|
||||
open = opener
|
||||
return "blob:ffmpeg-test", nil
|
||||
}}})
|
||||
defer e.Close()
|
||||
video, _ := os.ReadFile("testdata/ffmpeg/video.mp4")
|
||||
audio, _ := os.ReadFile("testdata/ffmpeg/audio.mp4")
|
||||
e.Runtime.Set("videoBytes", e.Runtime.NewArrayBuffer(video))
|
||||
e.Runtime.Set("audioBytes", e.Runtime.NewArrayBuffer(audio))
|
||||
_, err := e.RunString(`__gopeed_blob_create_object_url(()=>{
|
||||
const source=b=>new ReadableStream({start(c){c.enqueue(new Uint8Array(b));c.close();}});
|
||||
return __gopeed_ffmpeg.merge({inputs:()=>({video:source(videoBytes),audio:source(audioBytes)})});
|
||||
},{contentType:'video/mp4'})`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for i := 0; i < 2; i++ {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
|
||||
r, err := open(ctx, stream.ObjectURLOpenRequest{End: -1})
|
||||
if err != nil {
|
||||
cancel()
|
||||
t.Fatal(err)
|
||||
}
|
||||
b, err := io.ReadAll(r)
|
||||
r.Close()
|
||||
cancel()
|
||||
if err != nil || !bytes.Contains(b, []byte("moof")) {
|
||||
t.Fatalf("Blob read: %v (%d bytes)", err, len(b))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestFFmpegEngineCloseCancelsRequest(t *testing.T) {
|
||||
started, closed := make(chan struct{}), make(chan struct{})
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
close(started)
|
||||
<-r.Context().Done()
|
||||
close(closed)
|
||||
}))
|
||||
defer srv.Close()
|
||||
e := NewEngine(nil)
|
||||
defer e.Close()
|
||||
_, err := e.RunString(fmt.Sprintf(`
|
||||
globalThis.cancelCount=0;
|
||||
const audio=new ReadableStream({cancel(){cancelCount++;}});
|
||||
__gopeed_ffmpeg.merge({inputs:()=>({video:{url:%q},audio})}).getReader().read().catch(()=>{});
|
||||
"started";`, srv.URL))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
select {
|
||||
case <-started:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("HTTP did not start")
|
||||
}
|
||||
e.Close()
|
||||
select {
|
||||
case <-closed:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("HTTP leaked after engine shutdown")
|
||||
}
|
||||
if e.Runtime.Get("cancelCount").ToInteger() != 1 {
|
||||
t.Fatal("JS input not cancelled on shutdown")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFFmpegInputsStartBeforeCapacity(t *testing.T) {
|
||||
for _, mode := range []string{"factory", "http", "cancel"} {
|
||||
t.Run(mode, func(t *testing.T) {
|
||||
release1, _ := media.Acquire(context.Background())
|
||||
defer release1()
|
||||
release2, _ := media.Acquire(context.Background())
|
||||
defer release2()
|
||||
var requests atomic.Int32
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { requests.Add(1); http.Error(w, "test", 500) }))
|
||||
defer srv.Close()
|
||||
e := NewEngine(nil)
|
||||
defer e.Close()
|
||||
e.Runtime.Set("mode", mode)
|
||||
e.Runtime.Set("sourceURL", srv.URL)
|
||||
result, err := e.RunString(`(async()=>{
|
||||
globalThis.factoryCalls=0;
|
||||
globalThis.abortQueue=new AbortController();
|
||||
const options=mode==='http'?{video:{url:sourceURL},audio:{url:sourceURL}}:
|
||||
{inputs:()=>{factoryCalls++;throw new Error('factory failed');}};
|
||||
options.signal=abortQueue.signal;
|
||||
globalThis.work=__gopeed_ffmpeg.merge(options).getReader().read().then(()=>'',e=>e.message);
|
||||
await new Promise(r=>setTimeout(r,50));
|
||||
if(mode==='cancel')abortQueue.abort();
|
||||
return factoryCalls;
|
||||
})()`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if mode == "http" {
|
||||
if requests.Load() != 1 {
|
||||
t.Fatalf("queued HTTP did not start: %d", requests.Load())
|
||||
}
|
||||
} else if fmt.Sprint(result) != "1" {
|
||||
t.Fatalf("queued factory did not start: %v", result)
|
||||
}
|
||||
if mode == "cancel" {
|
||||
if _, err = e.RunString(`work`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
release1()
|
||||
if _, err = e.RunString(`work`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
calls, err := e.RunString(`factoryCalls`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if mode == "factory" && fmt.Sprint(calls) != "1" {
|
||||
t.Fatal(calls)
|
||||
}
|
||||
if mode == "cancel" && fmt.Sprint(calls) != "1" {
|
||||
t.Fatal("cancelled factory ran")
|
||||
}
|
||||
if mode == "http" && requests.Load() != 1 {
|
||||
t.Fatal("HTTP was restarted unnecessarily")
|
||||
}
|
||||
// Failure/cancellation must return the acquired slot even with the other held.
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
|
||||
defer cancel()
|
||||
release, err := media.Acquire(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("slot leaked: %v", err)
|
||||
}
|
||||
release()
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestFFmpegRequiresFactoryForStreams(t *testing.T) {
|
||||
e := NewEngine(nil)
|
||||
defer e.Close()
|
||||
result, err := e.RunString(`(()=>{
|
||||
const source=()=>new ReadableStream({});
|
||||
let rejected=0;
|
||||
for(const opts of [
|
||||
{video:source(),audio:source()},
|
||||
{video:{url:'https://example.test'},audio:source()},
|
||||
{inputs:()=>({}),video:{url:'https://example.test'}},
|
||||
{inputs:{video:source(),audio:source()}}
|
||||
]) {try{__gopeed_ffmpeg.merge(opts);}catch(e){if(e instanceof TypeError)rejected++;}}
|
||||
return rejected;
|
||||
})()`)
|
||||
if err != nil || fmt.Sprint(result) != "4" {
|
||||
t.Fatalf("validation: %v %v", result, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFFmpegCancelPendingFactoryCleansLateInputs(t *testing.T) {
|
||||
e := NewEngine(nil)
|
||||
defer e.Close()
|
||||
result, err := e.RunString(`(async()=>{
|
||||
let ready,finish,cancelled=0,signal;
|
||||
const entered=new Promise(r=>ready=r);
|
||||
const abort=new AbortController();
|
||||
const output=__gopeed_ffmpeg.merge({signal:abort.signal,inputs:async opts=>{
|
||||
signal=opts.signal;ready();await new Promise(r=>finish=r);
|
||||
const source=()=>new ReadableStream({cancel(){cancelled++;}});
|
||||
return {video:source(),audio:source()};
|
||||
}});
|
||||
const work=output.getReader().read().catch(e=>e.name);
|
||||
await entered;abort.abort();await work;finish();
|
||||
await new Promise(r=>setTimeout(r,30));
|
||||
return {cancelled,aborted:signal.aborted};
|
||||
})()`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
state := result.(map[string]any)
|
||||
if fmt.Sprint(state["cancelled"]) != "2" || state["aborted"] != true {
|
||||
t.Fatal(state)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
|
||||
defer cancel()
|
||||
first, err := media.Acquire(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer first()
|
||||
second, err := media.Acquire(ctx)
|
||||
if err != nil {
|
||||
t.Fatal("factory cancellation leaked capacity")
|
||||
}
|
||||
second()
|
||||
}
|
||||
|
||||
func TestQueuedMergePrefetchFailureAndCancel(t *testing.T) {
|
||||
for _, mode := range []string{"failure", "cancel"} {
|
||||
t.Run(mode, func(t *testing.T) {
|
||||
release1, _ := media.Acquire(context.Background())
|
||||
defer release1()
|
||||
release2, _ := media.Acquire(context.Background())
|
||||
defer release2()
|
||||
second := make(chan struct{})
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Header.Get("Range") == "bytes=0-1048575" {
|
||||
w.Header().Set("Content-Range", "bytes 0-1048575/33554432")
|
||||
w.Header().Set("Content-Length", "1048576")
|
||||
w.WriteHeader(206)
|
||||
w.Write(make([]byte, 1048576))
|
||||
return
|
||||
}
|
||||
close(second)
|
||||
if mode == "cancel" {
|
||||
<-r.Context().Done()
|
||||
return
|
||||
}
|
||||
http.Error(w, "upstream failed", 503)
|
||||
}))
|
||||
defer srv.Close()
|
||||
root := t.TempDir()
|
||||
e := NewEngine(&Config{TempDir: root})
|
||||
defer e.Close()
|
||||
e.Runtime.Set("sourceURL", srv.URL)
|
||||
_, err := e.RunString(`
|
||||
globalThis.abortJob=new AbortController();
|
||||
globalThis.result=__gopeed_ffmpeg.merge({signal:abortJob.signal,inputs:()=>({video:{url:sourceURL},audio:new ReadableStream({start(c){c.close();}})})})
|
||||
.getReader().read().then(()=>'',e=>String(e.message||e));
|
||||
'started';`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
select {
|
||||
case <-second:
|
||||
case <-time.After(3 * time.Second):
|
||||
t.Fatal("queued input did not download")
|
||||
}
|
||||
if mode == "cancel" {
|
||||
if _, err := e.RunString(`abortJob.abort()`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
result, err := e.RunString(`Promise.race([result,new Promise(r=>setTimeout(()=>r('test timeout'),3000))])`)
|
||||
if err != nil || fmt.Sprint(result) == "test timeout" || fmt.Sprint(result) == "" {
|
||||
t.Fatal(result, err)
|
||||
}
|
||||
if mode == "failure" && !strings.Contains(fmt.Sprint(result), "503") {
|
||||
t.Fatal("lost upstream failure", result)
|
||||
}
|
||||
deadline := time.Now().Add(3 * time.Second)
|
||||
for {
|
||||
entries, err := os.ReadDir(root)
|
||||
if err == nil && len(entries) == 0 {
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatal("queued input files leaked", entries, err)
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestFFmpegUnavailableTempDirReportsBlobError(t *testing.T) {
|
||||
root := filepath.Join(t.TempDir(), "blocked")
|
||||
if err := os.WriteFile(root, []byte("not a directory"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var open stream.ObjectURLOpener
|
||||
e := NewEngine(&Config{TempDir: root, StreamConfig: &stream.Config{CreateObjectURL: func(_ *stream.ObjectURLOptions, opener stream.ObjectURLOpener) (string, error) {
|
||||
open = opener
|
||||
return "blob:test", nil
|
||||
}}})
|
||||
defer e.Close()
|
||||
_, err := e.RunString(`__gopeed_blob_create_object_url(()=>__gopeed_ffmpeg.merge({inputs:()=>({video:new ReadableStream(),audio:new ReadableStream()})}),{})`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
|
||||
defer cancel()
|
||||
reader, err := open(ctx, stream.ObjectURLOpenRequest{End: -1})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer reader.Close()
|
||||
produced := reader.(production.Reader)
|
||||
if err := produced.WaitProduction(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
source := produced.Production()
|
||||
if source == nil {
|
||||
t.Fatal("missing producer")
|
||||
}
|
||||
if _, err := io.ReadAll(reader); err == nil || !strings.Contains(err.Error(), "blocked") {
|
||||
t.Fatal("missing useful disk error", err)
|
||||
}
|
||||
if downloaded, received := source.Progress(); downloaded != 0 || received != 0 {
|
||||
t.Fatal(downloaded, received)
|
||||
}
|
||||
}
|
||||
@@ -29,8 +29,9 @@ const FingerprintMagicKey = "__gopeed_fetch_fingerprint"
|
||||
var script string
|
||||
|
||||
type Config struct {
|
||||
ProxyHandler func(*http.Request) (*url.URL, error)
|
||||
RegisterCleanup func(func())
|
||||
DefaultUserAgent *string
|
||||
ProxyHandler func(*http.Request) (*url.URL, error)
|
||||
RegisterCleanup func(func())
|
||||
}
|
||||
|
||||
func Enable(runtime *goja.Runtime, loop *eventloop.EventLoop, cfg *Config) error {
|
||||
@@ -51,6 +52,9 @@ func Enable(runtime *goja.Runtime, loop *eventloop.EventLoop, cfg *Config) error
|
||||
if err != nil {
|
||||
panic(runtime.NewGoError(err))
|
||||
}
|
||||
if cfg.DefaultUserAgent != nil && !hasHeader(request.Headers, "User-Agent") {
|
||||
request.Headers = append(request.Headers, [2]string{"User-Agent", *cfg.DefaultUserAgent})
|
||||
}
|
||||
fingerprint := util.SafeGet[string](runtime, FingerprintMagicKey)
|
||||
promise, resolve, reject := runtime.NewPromise()
|
||||
go func() {
|
||||
|
||||
@@ -0,0 +1,144 @@
|
||||
(function () {
|
||||
const create = __gopeed_ffmpeg_create;
|
||||
const read = __gopeed_ffmpeg_read;
|
||||
const start = __gopeed_ffmpeg_start;
|
||||
const push = __gopeed_ffmpeg_push;
|
||||
const end = __gopeed_ffmpeg_end;
|
||||
const cancel = __gopeed_ffmpeg_cancel;
|
||||
const active = new Set();
|
||||
globalThis.__gopeed_ffmpeg_close_all = () => {
|
||||
for (const stop of Array.from(active)) stop(new Error("extension engine closed"));
|
||||
};
|
||||
|
||||
function input(value, name, allowStream) {
|
||||
if (value && typeof value.getReader === "function") {
|
||||
if (!allowStream) throw new TypeError("stream inputs must be created by the inputs callback");
|
||||
if (value.locked) throw new TypeError(name + " stream is locked");
|
||||
return { stream: value, request: null };
|
||||
}
|
||||
if (!value || typeof value.url !== "string" || !/^https?:\/\//i.test(value.url)) {
|
||||
throw new TypeError(name + " must be an HTTP request or ReadableStream");
|
||||
}
|
||||
const headers = {};
|
||||
new Headers(value.headers || {}).forEach((v, k) => { headers[k] = v; });
|
||||
return { request: { url: value.url, headers } };
|
||||
}
|
||||
|
||||
function merge(options) {
|
||||
if (!options || typeof options !== "object") throw new TypeError("merge options are required");
|
||||
const factory = options.inputs;
|
||||
if (factory !== undefined && typeof factory !== "function") throw new TypeError("inputs must be a callback");
|
||||
if (factory && (options.video !== undefined || options.audio !== undefined)) throw new TypeError("inputs cannot be combined with video/audio");
|
||||
let video = factory ? null : input(options.video, "video", false);
|
||||
let audio = factory ? null : input(options.audio, "audio", false);
|
||||
const args = options.args === undefined ? [] : options.args;
|
||||
if (!Array.isArray(args) || args.some(v => typeof v !== "string")) throw new TypeError("args must be a string array");
|
||||
const format = options.format === undefined ? "mp4" : options.format;
|
||||
if (typeof format !== "string") throw new TypeError("format must be a string");
|
||||
const signal = options.signal;
|
||||
const id = create({ format, args });
|
||||
let controller, initialized = false, stopped = false;
|
||||
const readers = [];
|
||||
const lifetime = new AbortController();
|
||||
let ownedInputs;
|
||||
function discardInputs(inputs, reason) {
|
||||
for (const source of new Set([inputs && inputs.video, inputs && inputs.audio])) {
|
||||
if (source && typeof source.cancel === "function" && !source.locked) Promise.resolve(source.cancel(reason)).catch(() => {});
|
||||
}
|
||||
}
|
||||
|
||||
function stop(reason) {
|
||||
if (stopped) return;
|
||||
stopped = true;
|
||||
active.delete(shutdown);
|
||||
lifetime.abort();
|
||||
discardInputs(ownedInputs, reason);
|
||||
if (id) cancel(id);
|
||||
if (signal) signal.removeEventListener("abort", abort);
|
||||
for (const reader of readers) {
|
||||
Promise.resolve(reader.cancel(reason)).catch(() => {}).then(() => {
|
||||
try { reader.releaseLock(); } catch (_) {}
|
||||
});
|
||||
}
|
||||
}
|
||||
function shutdown(reason) {
|
||||
controller.error(reason);
|
||||
stop(reason);
|
||||
}
|
||||
function abort() {
|
||||
const error = new Error("FFmpeg operation aborted");
|
||||
error.name = "AbortError";
|
||||
stop(error);
|
||||
controller.error(error);
|
||||
}
|
||||
async function pump(reader, index) {
|
||||
try {
|
||||
while (!stopped) {
|
||||
const result = await reader.read();
|
||||
if (stopped) return;
|
||||
if (result.done) { end(id, index, ""); return; }
|
||||
if (!(result.value instanceof Uint8Array)) throw new TypeError("media streams must contain Uint8Array chunks");
|
||||
// Keep individual crossings bounded, even if upstream emits a large chunk.
|
||||
for (let offset = 0; offset < result.value.byteLength && !stopped; offset += 256 * 1024) {
|
||||
if (!await push(id, index, result.value.subarray(offset, offset + 256 * 1024))) return;
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
if (!stopped) {
|
||||
end(id, index, String(error && error.message || error));
|
||||
controller.error(error);
|
||||
stop(error);
|
||||
}
|
||||
}
|
||||
}
|
||||
const output = new ReadableStream({
|
||||
start(c) {
|
||||
controller = c;
|
||||
active.add(shutdown);
|
||||
if (signal) {
|
||||
if (signal.aborted) abort();
|
||||
else signal.addEventListener("abort", abort);
|
||||
}
|
||||
},
|
||||
async pull(c) {
|
||||
if (stopped) return;
|
||||
try {
|
||||
if (!initialized) {
|
||||
initialized = true;
|
||||
if (stopped) return;
|
||||
if (factory) {
|
||||
const inputs = await factory({ signal: lifetime.signal });
|
||||
if (stopped) { discardInputs(inputs, new Error("FFmpeg operation cancelled")); return; }
|
||||
ownedInputs = inputs;
|
||||
video = input(inputs && inputs.video, "video", true);
|
||||
audio = input(inputs && inputs.audio, "audio", true);
|
||||
}
|
||||
if (video.stream && video.stream === audio.stream) throw new TypeError("video and audio must be distinct streams");
|
||||
for (const source of [video, audio]) {
|
||||
if (source.stream) readers.push(source.stream.getReader());
|
||||
}
|
||||
start(id, { video: video.request, audio: audio.request, format, args });
|
||||
let readerIndex = 0;
|
||||
for (const [index, source] of [video, audio].entries()) {
|
||||
if (source.stream) void pump(readers[readerIndex++], index);
|
||||
}
|
||||
}
|
||||
const chunk = await read(id);
|
||||
if (stopped) return;
|
||||
if (chunk === null) { stop(); c.close(); }
|
||||
else c.enqueue(new Uint8Array(chunk));
|
||||
} catch (error) {
|
||||
if (!stopped) {
|
||||
const failure = typeof MessageError === "function" && !(error instanceof MessageError)
|
||||
? new MessageError(String(error && error.message || error)) : error;
|
||||
c.error(failure); stop(failure);
|
||||
}
|
||||
}
|
||||
},
|
||||
cancel(reason) { stop(reason); }
|
||||
}, { highWaterMark: 0 });
|
||||
if (globalThis.__gopeed_bind_producer) globalThis.__gopeed_bind_producer(output, id);
|
||||
return output;
|
||||
}
|
||||
globalThis.__gopeed_ffmpeg = Object.freeze({ merge, supportsInputFactory: true, supportsProducerProgress: true });
|
||||
})();
|
||||
@@ -0,0 +1,337 @@
|
||||
package ffmpeg
|
||||
|
||||
import (
|
||||
"context"
|
||||
_ "embed"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
|
||||
"github.com/GopeedLab/gopeed/internal/httpclient"
|
||||
"github.com/GopeedLab/gopeed/internal/production"
|
||||
"github.com/GopeedLab/gopeed/internal/tempfiles"
|
||||
media "github.com/GopeedLab/gopeed/pkg/download/engine/ffmpeg"
|
||||
"github.com/dop251/goja"
|
||||
"github.com/dop251/goja_nodejs/eventloop"
|
||||
)
|
||||
|
||||
//go:embed ffmpeg.js
|
||||
var script string
|
||||
|
||||
type Config struct {
|
||||
TempFiles *tempfiles.Scope
|
||||
Producers *production.Registry
|
||||
TempDir string
|
||||
DefaultUserAgent *string
|
||||
ProxyHandler func(*http.Request) (*url.URL, error)
|
||||
RegisterCleanup func(func())
|
||||
}
|
||||
type options struct {
|
||||
Video *media.HTTPSource `json:"video"`
|
||||
Audio *media.HTTPSource `json:"audio"`
|
||||
Format string `json:"format"`
|
||||
Args []string `json:"args"`
|
||||
}
|
||||
type job struct {
|
||||
inputs [2]media.Input
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
output *io.PipeReader
|
||||
streams [2]*media.DiskInput
|
||||
mu sync.Mutex
|
||||
start chan options
|
||||
started bool
|
||||
}
|
||||
|
||||
func (j *job) Progress() (downloaded, received int64) {
|
||||
j.mu.Lock()
|
||||
defer j.mu.Unlock()
|
||||
for _, input := range j.inputs {
|
||||
if p, ok := input.(production.Source); ok {
|
||||
d, r := p.Progress()
|
||||
downloaded += d
|
||||
received += r
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (j *job) stop() {
|
||||
j.cancel()
|
||||
j.output.CloseWithError(context.Canceled)
|
||||
j.mu.Lock()
|
||||
defer j.mu.Unlock()
|
||||
for _, s := range j.streams {
|
||||
if s != nil {
|
||||
s.Close()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func Enable(vm *goja.Runtime, loop *eventloop.EventLoop, cfg *Config) error {
|
||||
if cfg == nil {
|
||||
cfg = &Config{}
|
||||
}
|
||||
spoolDir := cfg.TempDir
|
||||
if cfg.Producers == nil {
|
||||
cfg.Producers = &production.Registry{}
|
||||
}
|
||||
client, err := httpclient.NewClient(httpclient.Options{Transport: httpclient.TransportOptions{Proxy: cfg.ProxyHandler}})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var mu sync.Mutex
|
||||
jobs := map[string]*job{}
|
||||
var next atomic.Uint64
|
||||
get := func(id string) *job { mu.Lock(); defer mu.Unlock(); return jobs[id] }
|
||||
remove := func(id string) {
|
||||
mu.Lock()
|
||||
j := jobs[id]
|
||||
delete(jobs, id)
|
||||
cfg.Producers.Remove(id)
|
||||
mu.Unlock()
|
||||
if j != nil {
|
||||
j.stop()
|
||||
}
|
||||
}
|
||||
if cfg.RegisterCleanup != nil {
|
||||
cfg.RegisterCleanup(func() {
|
||||
done := make(chan struct{})
|
||||
if loop.RunOnLoop(func(vm *goja.Runtime) {
|
||||
defer close(done)
|
||||
if closeAll, ok := goja.AssertFunction(vm.Get("__gopeed_ffmpeg_close_all")); ok {
|
||||
_, _ = closeAll(goja.Undefined())
|
||||
}
|
||||
}) {
|
||||
<-done
|
||||
}
|
||||
mu.Lock()
|
||||
old := jobs
|
||||
jobs = map[string]*job{}
|
||||
mu.Unlock()
|
||||
for _, j := range old {
|
||||
j.stop()
|
||||
}
|
||||
client.CloseIdleConnections()
|
||||
})
|
||||
}
|
||||
if err := vm.Set("__gopeed_ffmpeg_create", func(call goja.FunctionCall) goja.Value {
|
||||
var opts options
|
||||
if err := vm.ExportTo(call.Argument(0), &opts); err != nil {
|
||||
panic(vm.NewGoError(err))
|
||||
}
|
||||
ctx, cancelCause := context.WithCancelCause(context.Background())
|
||||
cancel := func() { cancelCause(context.Canceled) }
|
||||
r, w := io.Pipe()
|
||||
j := &job{ctx: ctx, cancel: cancel, output: r, start: make(chan options, 1)}
|
||||
id := fmt.Sprintf("ffmpeg-%d", next.Add(1))
|
||||
mu.Lock()
|
||||
jobs[id] = j
|
||||
cfg.Producers.Set(id, j)
|
||||
mu.Unlock()
|
||||
go func() {
|
||||
var inputs [2]media.Input
|
||||
var runErr error
|
||||
defer func() {
|
||||
if v := recover(); v != nil {
|
||||
runErr = fmt.Errorf("ffmpeg panic: %v", v)
|
||||
}
|
||||
if errors.Is(runErr, context.Canceled) {
|
||||
runErr = context.Cause(ctx)
|
||||
}
|
||||
cancel()
|
||||
for _, in := range inputs {
|
||||
if in != nil {
|
||||
in.Close()
|
||||
}
|
||||
}
|
||||
w.CloseWithError(runErr)
|
||||
}()
|
||||
var opts options
|
||||
select {
|
||||
case opts = <-j.start:
|
||||
case <-ctx.Done():
|
||||
runErr = ctx.Err()
|
||||
return
|
||||
}
|
||||
for i, source := range []*media.HTTPSource{opts.Video, opts.Audio} {
|
||||
if source == nil {
|
||||
inputs[i] = j.streams[i]
|
||||
} else {
|
||||
inputs[i], runErr = media.OpenSpoolingHTTP(ctx, client, *source, spoolDir, cfg.TempFiles)
|
||||
if runErr != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
j.mu.Lock()
|
||||
j.inputs[i] = inputs[i]
|
||||
j.mu.Unlock()
|
||||
if input, ok := inputs[i].(interface{ Failures() <-chan error }); ok {
|
||||
go func() {
|
||||
select {
|
||||
case err := <-input.Failures():
|
||||
cancelCause(err)
|
||||
case <-ctx.Done():
|
||||
}
|
||||
}()
|
||||
}
|
||||
}
|
||||
release, err := media.Acquire(ctx)
|
||||
if err != nil {
|
||||
runErr = err
|
||||
return
|
||||
}
|
||||
defer release()
|
||||
runErr = media.RunAcquired(ctx, inputs[0], inputs[1], w, opts.Format, opts.Args)
|
||||
}()
|
||||
return vm.ToValue(id)
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := vm.Set("__gopeed_ffmpeg_start", func(call goja.FunctionCall) goja.Value {
|
||||
id := call.Argument(0).String()
|
||||
j := get(id)
|
||||
if j == nil {
|
||||
panic(vm.NewGoError(io.ErrClosedPipe))
|
||||
}
|
||||
var opts options
|
||||
if err := vm.ExportTo(call.Argument(1), &opts); err != nil {
|
||||
panic(vm.NewGoError(err))
|
||||
}
|
||||
if _, err := media.Arguments(opts.Format, opts.Args); err != nil {
|
||||
panic(vm.NewGoError(err))
|
||||
}
|
||||
for _, source := range []*media.HTTPSource{opts.Video, opts.Audio} {
|
||||
if source == nil || cfg.DefaultUserAgent == nil {
|
||||
continue
|
||||
}
|
||||
specified := false
|
||||
for key := range source.Headers {
|
||||
if strings.EqualFold(key, "User-Agent") {
|
||||
specified = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !specified {
|
||||
if source.Headers == nil {
|
||||
source.Headers = make(map[string]string)
|
||||
}
|
||||
source.Headers["User-Agent"] = *cfg.DefaultUserAgent
|
||||
}
|
||||
}
|
||||
|
||||
j.mu.Lock()
|
||||
defer j.mu.Unlock()
|
||||
if j.started || j.ctx.Err() != nil {
|
||||
panic(vm.NewGoError(io.ErrClosedPipe))
|
||||
}
|
||||
j.started = true
|
||||
for i, source := range []*media.HTTPSource{opts.Video, opts.Audio} {
|
||||
if source == nil {
|
||||
var err error
|
||||
j.streams[i], err = media.NewDiskInput(j.ctx, spoolDir, cfg.TempFiles)
|
||||
if err != nil {
|
||||
j.cancel()
|
||||
panic(vm.NewGoError(err))
|
||||
}
|
||||
j.inputs[i] = j.streams[i]
|
||||
}
|
||||
}
|
||||
j.start <- opts
|
||||
return goja.Undefined()
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := vm.Set("__gopeed_ffmpeg_read", func(id string) goja.Value {
|
||||
promise, resolve, reject := vm.NewPromise()
|
||||
go func() {
|
||||
var readErr error
|
||||
buf := make([]byte, 64*1024)
|
||||
n := 0
|
||||
j := get(id)
|
||||
if j == nil {
|
||||
readErr = io.ErrClosedPipe
|
||||
} else {
|
||||
n, readErr = j.output.Read(buf)
|
||||
}
|
||||
if !loop.RunOnLoop(func(vm *goja.Runtime) {
|
||||
if readErr != nil {
|
||||
remove(id)
|
||||
if readErr == io.EOF {
|
||||
resolve(goja.Null())
|
||||
} else {
|
||||
reject(vm.NewGoError(readErr))
|
||||
}
|
||||
return
|
||||
}
|
||||
resolve(vm.ToValue(vm.NewArrayBuffer(buf[:n])))
|
||||
}) {
|
||||
remove(id)
|
||||
}
|
||||
}()
|
||||
return vm.ToValue(promise)
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := vm.Set("__gopeed_ffmpeg_push", func(call goja.FunctionCall) goja.Value {
|
||||
id := call.Argument(0).String()
|
||||
index := int(call.Argument(1).ToInteger())
|
||||
var chunk []byte
|
||||
switch b := call.Argument(2).Export().(type) {
|
||||
case goja.ArrayBuffer:
|
||||
chunk = append([]byte(nil), b.Bytes()...)
|
||||
case []byte:
|
||||
chunk = append([]byte(nil), b...)
|
||||
default:
|
||||
panic(vm.NewTypeError("ffmpeg input requires bytes"))
|
||||
}
|
||||
promise, resolve, reject := vm.NewPromise()
|
||||
go func() {
|
||||
var err error
|
||||
j := get(id)
|
||||
if j == nil || index < 0 || index > 1 || j.streams[index] == nil {
|
||||
err = io.ErrClosedPipe
|
||||
} else {
|
||||
_, err = j.streams[index].Write(chunk)
|
||||
}
|
||||
if err != nil && j != nil && index >= 0 && index < 2 && j.streams[index] != nil {
|
||||
j.streams[index].End(err)
|
||||
}
|
||||
loop.RunOnLoop(func(vm *goja.Runtime) {
|
||||
if j != nil && j.ctx.Err() != nil {
|
||||
resolve(false)
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
reject(vm.NewGoError(err))
|
||||
} else {
|
||||
resolve(true)
|
||||
}
|
||||
})
|
||||
}()
|
||||
return vm.ToValue(promise)
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := vm.Set("__gopeed_ffmpeg_end", func(id string, index int, message string) {
|
||||
if j := get(id); j != nil && index >= 0 && index < 2 && j.streams[index] != nil {
|
||||
var err error
|
||||
if message != "" {
|
||||
err = errors.New(message)
|
||||
}
|
||||
j.streams[index].End(err)
|
||||
}
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := vm.Set("__gopeed_ffmpeg_cancel", remove); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = vm.RunString(script)
|
||||
return err
|
||||
}
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
|
||||
"github.com/GopeedLab/gopeed/internal/production"
|
||||
"github.com/dop251/goja"
|
||||
"github.com/dop251/goja_nodejs/eventloop"
|
||||
)
|
||||
@@ -17,6 +18,7 @@ import (
|
||||
var script string
|
||||
|
||||
type Config struct {
|
||||
Producers *production.Registry
|
||||
CreateObjectURL func(opts *ObjectURLOptions, open ObjectURLOpener) (string, error)
|
||||
RevokeObjectURL func(url string) error
|
||||
}
|
||||
@@ -39,6 +41,23 @@ func Enable(runtime *goja.Runtime, loop *eventloop.EventLoop, cfg *Config) error
|
||||
cfg = &Config{}
|
||||
}
|
||||
blobPipes := newBlobPipeRegistry()
|
||||
if err := runtime.Set("__gopeed_blob_pipe_bind", func(pipeID, producerID string) {
|
||||
var source production.Source
|
||||
if cfg.Producers != nil {
|
||||
source = cfg.Producers.Get(producerID)
|
||||
}
|
||||
blobPipes.mu.Lock()
|
||||
pipe := blobPipes.pipes[pipeID]
|
||||
blobPipes.mu.Unlock()
|
||||
if pipe != nil {
|
||||
pipe.mu.Lock()
|
||||
pipe.producer = source
|
||||
pipe.mu.Unlock()
|
||||
pipe.readyOnce.Do(func() { close(pipe.ready) })
|
||||
}
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := runtime.Set("__gopeed_create_blob_object_url", func(call goja.FunctionCall) goja.Value {
|
||||
if cfg.CreateObjectURL == nil {
|
||||
panic(runtime.NewGoError(fmt.Errorf("blob object url handler not configured")))
|
||||
@@ -86,18 +105,23 @@ func Enable(runtime *goja.Runtime, loop *eventloop.EventLoop, cfg *Config) error
|
||||
}
|
||||
if err := runtime.Set("__gopeed_blob_pipe_chunk", func(call goja.FunctionCall) goja.Value {
|
||||
id := call.Argument(0).String()
|
||||
ok := blobPipes.Push(id, exportBytes(call.Argument(1)))
|
||||
return runtime.ToValue(ok)
|
||||
chunk := append([]byte(nil), exportBytes(call.Argument(1))...)
|
||||
promise, resolve, _ := runtime.NewPromise()
|
||||
go func() {
|
||||
ok := blobPipes.Push(id, chunk)
|
||||
loop.RunOnLoop(func(vm *goja.Runtime) { resolve(ok) })
|
||||
}()
|
||||
return runtime.ToValue(promise)
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := runtime.Set("__gopeed_blob_pipe_close", func(id string) {
|
||||
blobPipes.Close(id, nil)
|
||||
go blobPipes.Close(id, nil)
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := runtime.Set("__gopeed_blob_pipe_error", func(id string, message string) {
|
||||
blobPipes.Close(id, errors.New(message))
|
||||
go blobPipes.Close(id, errors.New(message))
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -184,12 +208,15 @@ type blobPipeItem struct {
|
||||
}
|
||||
|
||||
type blobPipe struct {
|
||||
id string
|
||||
loop *eventloop.EventLoop
|
||||
registry *blobPipeRegistry
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
ch chan blobPipeItem
|
||||
ready chan struct{}
|
||||
readyOnce sync.Once
|
||||
producer production.Source
|
||||
id string
|
||||
loop *eventloop.EventLoop
|
||||
registry *blobPipeRegistry
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
ch chan blobPipeItem
|
||||
|
||||
mu sync.Mutex
|
||||
buf []byte
|
||||
@@ -200,6 +227,7 @@ type blobPipe struct {
|
||||
func newBlobPipe(ctx context.Context, loop *eventloop.EventLoop, registry *blobPipeRegistry) *blobPipe {
|
||||
pipeCtx, cancel := context.WithCancel(ctx)
|
||||
return &blobPipe{
|
||||
ready: make(chan struct{}),
|
||||
id: registry.NewID(),
|
||||
loop: loop,
|
||||
registry: registry,
|
||||
@@ -223,6 +251,7 @@ func (p *blobPipe) Push(chunk []byte) bool {
|
||||
}
|
||||
|
||||
func (p *blobPipe) CloseWithError(err error) {
|
||||
p.readyOnce.Do(func() { close(p.ready) })
|
||||
p.once.Do(func() {
|
||||
p.registry.Remove(p.id)
|
||||
select {
|
||||
@@ -475,3 +504,20 @@ func exportJSError(value goja.Value) error {
|
||||
}
|
||||
return errors.New(stack)
|
||||
}
|
||||
|
||||
func (p *blobPipe) Production() production.Source {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
return p.producer
|
||||
}
|
||||
|
||||
func (p *blobPipe) WaitProduction(ctx context.Context) error {
|
||||
select {
|
||||
case <-p.ready:
|
||||
return nil
|
||||
case <-p.ctx.Done():
|
||||
return p.ctx.Err()
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
}
|
||||
@@ -269,3 +269,57 @@ func newStreamTestEngine(t *testing.T, created chan<- createdObjectURL) (*engine
|
||||
})
|
||||
return e, e.Close
|
||||
}
|
||||
|
||||
// A stalled disk/network consumer must pause the producer without blocking
|
||||
// the JS event loop, so cancellation and other tracks can still run.
|
||||
func TestBlobBackpressureLeavesEventLoopResponsive(t *testing.T) {
|
||||
created := make(chan createdObjectURL, 1)
|
||||
e, cleanup := newStreamTestEngine(t, created)
|
||||
defer cleanup()
|
||||
_, err := e.RunString(`
|
||||
globalThis.produced = 0;
|
||||
globalThis.sourceCancelled = false;
|
||||
__gopeed_blob_create_object_url(() => new ReadableStream({
|
||||
pull(c) { produced++; c.enqueue(new Uint8Array(64*1024)); },
|
||||
cancel() { sourceCancelled = true; }
|
||||
}));
|
||||
`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
object := <-created
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
|
||||
defer cancel()
|
||||
reader, err := object.open(ctx, stream.ObjectURLOpenRequest{End: -1})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer reader.Close()
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
value, err := e.RunString("produced")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
before := value.(int64)
|
||||
if before > 10 {
|
||||
t.Fatalf("unbounded production: %d", before)
|
||||
}
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
after, err := e.RunString("produced")
|
||||
if err != nil || after != value {
|
||||
t.Fatalf("producer did not pause: %v -> %v (%v)", value, after, err)
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
t.Fatal("event loop only resumed after cancellation")
|
||||
}
|
||||
if _, err = reader.Read(make([]byte, 64*1024)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
time.Sleep(30 * time.Millisecond)
|
||||
after, err = e.RunString("produced")
|
||||
if err != nil || after.(int64) <= before {
|
||||
t.Fatalf("producer did not resume: %v (%v)", after, err)
|
||||
}
|
||||
reader.Close()
|
||||
waitForJSValue(t, e, "String(sourceCancelled)", "true")
|
||||
}
|
||||
@@ -105,6 +105,10 @@
|
||||
return Promise.resolve({ done: true, value: undefined });
|
||||
}
|
||||
this._maybePull();
|
||||
// pull() may synchronously error before a waiter has been registered.
|
||||
if (this._errored) {
|
||||
return Promise.reject(this._errored);
|
||||
}
|
||||
if (this._queue.length > 0) {
|
||||
return Promise.resolve({ done: false, value: this._queue.shift() });
|
||||
}
|
||||
@@ -730,6 +734,8 @@
|
||||
finishBlobPipeReader(state, true, reason).catch(function () {});
|
||||
};
|
||||
|
||||
const producers = new WeakMap();
|
||||
globalThis.__gopeed_bind_producer = (stream, id) => producers.set(stream, id);
|
||||
globalThis.__gopeed_blob_pipe_source = function (openReadable, request, pipeId) {
|
||||
if (typeof openReadable !== "function") {
|
||||
throw new TypeError("blob opener must be callable");
|
||||
@@ -748,6 +754,8 @@
|
||||
try {
|
||||
const source = await openReadable(normalizeOpenRequest(request));
|
||||
state.openSettled = true;
|
||||
const producer = producers.get(source);
|
||||
globalThis.__gopeed_blob_pipe_bind(pipeId, producer || "");
|
||||
state.reader = toReadableStreamReader(source, "gopeed.runtime.blob.createObjectURL opener function");
|
||||
if (state.cancelRequested) {
|
||||
await finishBlobPipeReader(state, true, state.cancelReason);
|
||||
@@ -760,17 +768,20 @@
|
||||
return;
|
||||
}
|
||||
const chunk = value instanceof Uint8Array ? value : new Uint8Array(value);
|
||||
const buffer = chunk.buffer.slice(chunk.byteOffset, chunk.byteOffset + chunk.byteLength);
|
||||
if (!globalThis.__gopeed_blob_pipe_chunk(pipeId, buffer)) {
|
||||
state.cancelRequested = true;
|
||||
state.cancelReason = "blob pipe closed";
|
||||
return;
|
||||
for (let offset = 0; offset < chunk.byteLength; offset += 256 * 1024) {
|
||||
const part = chunk.subarray(offset, offset + 256 * 1024);
|
||||
const buffer = part.buffer.slice(part.byteOffset, part.byteOffset + part.byteLength);
|
||||
if (!await globalThis.__gopeed_blob_pipe_chunk(pipeId, buffer)) {
|
||||
state.cancelRequested = true;
|
||||
state.cancelReason = "blob pipe closed";
|
||||
return;
|
||||
}
|
||||
}
|
||||
await yieldBlobPipeTask();
|
||||
}
|
||||
} catch (error) {
|
||||
if (!state.cancelRequested) {
|
||||
globalThis.__gopeed_blob_pipe_error(pipeId, error && error.stack ? error.stack : String(error));
|
||||
globalThis.__gopeed_blob_pipe_error(pipeId, error && (error.stack || error.message) ? (error.stack || error.message) : String(error));
|
||||
}
|
||||
} finally {
|
||||
state.openSettled = true;
|
||||
|
||||
+18
@@ -0,0 +1,18 @@
|
||||
# Synthetic media fixtures
|
||||
|
||||
These files contain 0.3 seconds of a generated 32x32 black image (H.264) and
|
||||
generated mono silence (AAC). They contain no third-party media. Generated
|
||||
with the pinned go-ffmpreg FFmpeg binary using the equivalent commands:
|
||||
|
||||
```
|
||||
ffmpeg -f lavfi -i color=c=black:s=32x32:r=10 -t 0.3 -c:v libx264 -preset ultrafast -movflags frag_keyframe+empty_moov -f mp4 video.mp4
|
||||
ffmpeg -f lavfi -i anullsrc=r=48000:cl=mono -t 0.3 -c:a aac -movflags frag_keyframe+empty_moov -f mp4 audio.mp4
|
||||
```
|
||||
|
||||
The backend tests also generate and probe media using the embedded FFmpeg and
|
||||
FFprobe at test time, without relying on a system installation.
|
||||
|
||||
`video.webm` and `audio.webm` use the same black/silent sources, encoded once
|
||||
with native FFmpeg 7.1 using `-c:v libvpx-vp9` and `-c:a libopus`. The committed
|
||||
fixtures let the tests verify WASM stream-copy remuxing without exercising its
|
||||
VP9 encoder or requiring native FFmpeg on developer/CI machines.
|
||||
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
@@ -0,0 +1,93 @@
|
||||
package webview
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// Event is a main-page notification. Data follows WebviewEventMap.
|
||||
type Event struct {
|
||||
Name string `json:"event"`
|
||||
Data map[string]any `json:"data"`
|
||||
}
|
||||
|
||||
type EventSource interface {
|
||||
On(event string, handler func(Event)) (unsubscribe func(), err error)
|
||||
}
|
||||
|
||||
func ValidEvent(event string) bool {
|
||||
switch event {
|
||||
case "url-changed", "load", "load-error", "closed":
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// EventHub delivers notifications in registration order. Handlers must only
|
||||
// enqueue work, never block or enter the browser. Closed is terminal.
|
||||
type EventHub struct {
|
||||
mu sync.Mutex
|
||||
seq uint64
|
||||
listeners []eventListener
|
||||
closed bool
|
||||
}
|
||||
type eventListener struct {
|
||||
id uint64
|
||||
name string
|
||||
handler func(Event)
|
||||
}
|
||||
|
||||
func (h *EventHub) On(name string, handler func(Event)) (func(), error) {
|
||||
if !ValidEvent(name) || handler == nil {
|
||||
return nil, fmt.Errorf("invalid webview event or handler: %s", name)
|
||||
}
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
if h.closed {
|
||||
return nil, fmt.Errorf("webview page is closed")
|
||||
}
|
||||
h.seq++
|
||||
id := h.seq
|
||||
h.listeners = append(h.listeners, eventListener{id, name, handler})
|
||||
return func() {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
for i, listener := range h.listeners {
|
||||
if listener.id == id {
|
||||
h.listeners = append(h.listeners[:i], h.listeners[i+1:]...)
|
||||
break
|
||||
}
|
||||
}
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (h *EventHub) Emit(event Event) {
|
||||
h.mu.Lock()
|
||||
if h.closed {
|
||||
h.mu.Unlock()
|
||||
return
|
||||
}
|
||||
listeners := append([]eventListener(nil), h.listeners...)
|
||||
if event.Name == "closed" {
|
||||
h.closed = true
|
||||
h.listeners = nil
|
||||
}
|
||||
h.mu.Unlock()
|
||||
for _, listener := range listeners {
|
||||
if listener.name == event.Name {
|
||||
listener.handler(event)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (p *PageHandle) On(name string, handler func(Event)) (func(), error) {
|
||||
page, err := p.page()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
source, ok := page.(EventSource)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("WebView provider does not support events")
|
||||
}
|
||||
return source.On(name, handler)
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
package webview
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"sync"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestEventHubLifecycle(t *testing.T) {
|
||||
var hub EventHub
|
||||
var got []string
|
||||
hub.Emit(Event{Name: "load"}) // Never replay pre-registration events.
|
||||
off, err := hub.On("load", func(Event) { got = append(got, "first") })
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, _ = hub.On("load", func(Event) { got = append(got, "second") })
|
||||
_, _ = hub.On("closed", func(e Event) {
|
||||
got = append(got, e.Data["reason"].(string))
|
||||
off() // Unsubscription from a callback must not deadlock.
|
||||
})
|
||||
hub.Emit(Event{Name: "load"})
|
||||
off()
|
||||
off()
|
||||
hub.Emit(Event{Name: "load"})
|
||||
hub.Emit(Event{Name: "closed", Data: map[string]any{"reason": "api"}})
|
||||
hub.Emit(Event{Name: "load"})
|
||||
hub.Emit(Event{Name: "closed"})
|
||||
if !reflect.DeepEqual(got, []string{"first", "second", "second", "api"}) {
|
||||
t.Fatal(got)
|
||||
}
|
||||
if len(hub.listeners) != 0 {
|
||||
t.Fatal("closed page retained listeners")
|
||||
}
|
||||
if _, err := hub.On("load", func(Event) {}); err == nil {
|
||||
t.Fatal("registration on closed page succeeded")
|
||||
}
|
||||
}
|
||||
|
||||
func TestEventHubConcurrentUnsubscribe(t *testing.T) {
|
||||
var hub EventHub
|
||||
off, _ := hub.On("load", func(Event) {})
|
||||
var wg sync.WaitGroup
|
||||
for i := 0; i < 20; i++ {
|
||||
wg.Add(1)
|
||||
go func() { defer wg.Done(); off(); hub.Emit(Event{Name: "load"}) }()
|
||||
}
|
||||
wg.Wait()
|
||||
if _, err := hub.On("invalid", func(Event) {}); err == nil {
|
||||
t.Fatal("accepted invalid event")
|
||||
}
|
||||
}
|
||||
@@ -5,8 +5,10 @@ import "encoding/json"
|
||||
const RPCEndpointPath = "/webview"
|
||||
|
||||
const (
|
||||
MethodProfileRemove = "profile.remove"
|
||||
MethodIsAvailable = "webview.isAvailable"
|
||||
MethodPageOpen = "page.open"
|
||||
MethodPageEvents = "page.events"
|
||||
MethodPageAddInitScript = "page.addInitScript"
|
||||
MethodPageGoto = "page.goto"
|
||||
MethodPageExecute = "page.execute"
|
||||
@@ -68,6 +70,9 @@ type IsAvailableResult struct {
|
||||
}
|
||||
|
||||
type PageOpenParams struct {
|
||||
ProfileID string `json:"profileId,omitempty"`
|
||||
DataPath string `json:"dataPath,omitempty"`
|
||||
ProxyURL string `json:"proxyUrl,omitempty"`
|
||||
Headless bool `json:"headless,omitempty"`
|
||||
Debug bool `json:"debug,omitempty"`
|
||||
Title string `json:"title,omitempty"`
|
||||
@@ -78,6 +83,9 @@ type PageOpenParams struct {
|
||||
|
||||
func NewPageOpenParams(opts OpenOptions) PageOpenParams {
|
||||
return PageOpenParams{
|
||||
ProfileID: opts.ProfileID,
|
||||
DataPath: opts.DataPath,
|
||||
ProxyURL: opts.ProxyURL,
|
||||
Headless: opts.Headless,
|
||||
Debug: opts.Debug,
|
||||
Title: opts.Title,
|
||||
@@ -141,3 +149,8 @@ type PageCloseParams struct {
|
||||
}
|
||||
|
||||
type EmptyResult struct{}
|
||||
|
||||
// ProfileRemoveParams belongs to the private host RPC.
|
||||
type ProfileRemoveParams struct {
|
||||
ProfileID string `json:"profileId"`
|
||||
}
|
||||
@@ -17,6 +17,11 @@ type Opener interface {
|
||||
Open(opts OpenOptions) (Page, error)
|
||||
}
|
||||
|
||||
// ProfileRemover is a host-only lifecycle operation, never exposed to extension JS.
|
||||
type ProfileRemover interface {
|
||||
RemoveProfile(profileID, dataPath string) error
|
||||
}
|
||||
|
||||
type Page interface {
|
||||
AddInitScript(script string) error
|
||||
Goto(url string, opts GotoOptions) error
|
||||
@@ -29,6 +34,10 @@ type Page interface {
|
||||
}
|
||||
|
||||
type OpenOptions struct {
|
||||
// Host-owned fields are never parsed from extension JavaScript options.
|
||||
ProfileID string
|
||||
DataPath string
|
||||
ProxyURL string
|
||||
Headless bool
|
||||
Debug bool
|
||||
Title string
|
||||
|
||||
@@ -336,3 +336,15 @@ func (p *fakePage) Close() error {
|
||||
p.closed = true
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestExtensionCannotSupplyHostProfileOrProxy(t *testing.T) {
|
||||
opener := &fakeOpener{page: &fakePage{}}
|
||||
runtime := NewRuntime(opener, true)
|
||||
_, err := runtime.Open(map[string]any{"profileId": "other-extension", "dataPath": "/tmp/other-profile", "proxyUrl": "http://example.com:1234", "ProfileID": "other"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if opener.opts.ProfileID != "" || opener.opts.DataPath != "" || opener.opts.ProxyURL != "" {
|
||||
t.Fatalf("extension controlled host settings: %+v", opener.opts)
|
||||
}
|
||||
}
|
||||
@@ -26,7 +26,6 @@ import (
|
||||
var (
|
||||
gitSuffix = ".git"
|
||||
|
||||
tempExtensionsDir = ".extensions"
|
||||
extensionsDir = "extensions"
|
||||
extensionIgnoreDirs = []string{gitSuffix, "node_modules"}
|
||||
|
||||
@@ -65,6 +64,7 @@ func (d *Downloader) InstallExtensionByFolder(path string, devMode bool) (*Exten
|
||||
|
||||
// if extension is not installed, add it to the list, otherwise update it
|
||||
installedExt := d.getExtension(ext.Identity)
|
||||
newInstallation := installedExt == nil
|
||||
if installedExt == nil {
|
||||
ext.CreatedAt = time.Now()
|
||||
ext.UpdatedAt = ext.CreatedAt
|
||||
@@ -76,6 +76,11 @@ func (d *Downloader) InstallExtensionByFolder(path string, devMode bool) (*Exten
|
||||
if err = d.storage.Put(bucketExtension, installedExt.Identity, installedExt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if newInstallation {
|
||||
d.webviewProfilesLock.Lock()
|
||||
delete(d.webviewProfiles, ext.Identity)
|
||||
d.webviewProfilesLock.Unlock()
|
||||
}
|
||||
return installedExt, nil
|
||||
}
|
||||
|
||||
@@ -144,6 +149,9 @@ func (d *Downloader) DeleteExtension(identity string) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := d.removeExtensionWebViewProfile(identity); err != nil {
|
||||
return fmt.Errorf("remove extension WebView data: %w", err)
|
||||
}
|
||||
// remove from disk
|
||||
if !ext.DevMode {
|
||||
if err := os.RemoveAll(d.ExtensionPath(ext)); err != nil {
|
||||
@@ -204,11 +212,14 @@ func (d *Downloader) fetchExtensionByGit(url string, handler func(tempExtPath st
|
||||
|
||||
// Use unique suffix to avoid concurrent conflicts when multiple git operations
|
||||
// target the same extension (e.g., install and update check happening simultaneously)
|
||||
tempExtDir := filepath.Join(d.cfg.StorageDir, tempExtensionsDir, fmt.Sprintf("%s_%d", projectPath, time.Now().UnixNano()))
|
||||
if err := os.MkdirAll(tempExtDir, 0755); err != nil {
|
||||
tempExtDir, err := os.MkdirTemp(d.cfg.TempDir, "gopeed-extension-")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer os.RemoveAll(tempExtDir)
|
||||
if err := d.tempFiles.Track(tempExtDir); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer d.tempFiles.Remove(tempExtDir)
|
||||
|
||||
proxyOptions := transport.ProxyOptions{}
|
||||
proxyUrl := d.cfg.DownloaderStoreConfig.Proxy.ToUrl()
|
||||
@@ -365,9 +376,9 @@ func doTrigger[T any](d *Downloader, event ActivationEvent, req *base.Request, c
|
||||
engine, session := d.newExtensionEngine()
|
||||
defer session.CloseIfIdle()
|
||||
gopeed.Runtime = &InstanceRuntime{
|
||||
WebView: d.newExtensionWebViewRuntime(session),
|
||||
WebView: d.newExtensionWebViewRuntime(session, ext.buildIdentity()),
|
||||
}
|
||||
err = injectGopeed(engine.Runtime, gopeed)
|
||||
err = injectGopeed(engine.Runtime, gopeed, engine.Post)
|
||||
if err != nil {
|
||||
gopeed.Logger.logger.Error().Err(err).Msgf("[%s] engine inject failed", ext.buildIdentity())
|
||||
return
|
||||
|
||||
@@ -30,10 +30,10 @@ func (d *Downloader) NewExtensionEngine(ext *Extension, settings map[string]any)
|
||||
identity: ext.buildIdentity(),
|
||||
},
|
||||
Runtime: &InstanceRuntime{
|
||||
WebView: d.newExtensionWebViewRuntime(session),
|
||||
WebView: d.newExtensionWebViewRuntime(session, ext.buildIdentity()),
|
||||
},
|
||||
}
|
||||
if err := injectGopeed(engine.Runtime, gopeed); err != nil {
|
||||
if err := injectGopeed(engine.Runtime, gopeed, engine.Post); err != nil {
|
||||
session.CloseIfIdle()
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -3,9 +3,11 @@ package download
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
internalblob "github.com/GopeedLab/gopeed/internal/blob"
|
||||
httpProtocol "github.com/GopeedLab/gopeed/internal/protocol/http"
|
||||
"github.com/GopeedLab/gopeed/pkg/download/engine"
|
||||
"github.com/GopeedLab/gopeed/pkg/download/engine/inject/stream"
|
||||
enginewebview "github.com/GopeedLab/gopeed/pkg/download/engine/webview"
|
||||
@@ -119,21 +121,29 @@ func (d *Downloader) newExtensionEngine() (*engine.Engine, *engineSession) {
|
||||
return d.blob.Revoke(url)
|
||||
},
|
||||
}
|
||||
httpConfig := struct {
|
||||
UserAgent string `json:"userAgent"`
|
||||
}{UserAgent: httpProtocol.DefaultUserAgent}
|
||||
d.getProtocolConfig("http", &httpConfig)
|
||||
userAgent := strings.TrimSpace(httpConfig.UserAgent)
|
||||
e := engine.NewEngine(&engine.Config{
|
||||
ProxyConfig: d.cfg.Proxy,
|
||||
StreamConfig: engineCfg,
|
||||
TempDir: d.cfg.TempDir,
|
||||
TempFiles: d.tempFiles,
|
||||
HTTPUserAgent: &userAgent,
|
||||
ProxyConfig: d.cfg.Proxy,
|
||||
StreamConfig: engineCfg,
|
||||
})
|
||||
session.SetEngine(e)
|
||||
return e, session
|
||||
}
|
||||
|
||||
func (d *Downloader) newExtensionWebViewRuntime(session *engineSession) *enginewebview.Runtime {
|
||||
func (d *Downloader) newExtensionWebViewRuntime(session *engineSession, identity string) *enginewebview.Runtime {
|
||||
var (
|
||||
opener enginewebview.Opener
|
||||
available bool
|
||||
)
|
||||
if provider := d.cfg.WebViewProvider; provider != nil && provider.IsAvailable() {
|
||||
opener = provider
|
||||
opener = &extensionWebViewOpener{downloader: d, provider: provider, identity: identity, profile: d.extensionWebViewProfile(identity)}
|
||||
available = true
|
||||
}
|
||||
runtime := enginewebview.NewRuntime(opener, available)
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
package download
|
||||
|
||||
import (
|
||||
"runtime"
|
||||
|
||||
"github.com/GopeedLab/gopeed/pkg/base"
|
||||
)
|
||||
|
||||
type InstanceHost struct {
|
||||
Env HostEnv `json:"env"`
|
||||
}
|
||||
|
||||
type HostEnv struct {
|
||||
Version string `json:"version"`
|
||||
OS string `json:"os"`
|
||||
Arch string `json:"arch"`
|
||||
}
|
||||
|
||||
func newInstanceHost() *InstanceHost {
|
||||
return &InstanceHost{Env: HostEnv{
|
||||
Version: base.Version,
|
||||
OS: runtime.GOOS,
|
||||
Arch: runtime.GOARCH,
|
||||
}}
|
||||
}
|
||||
@@ -0,0 +1,122 @@
|
||||
package download
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/GopeedLab/gopeed/pkg/base"
|
||||
"github.com/GopeedLab/gopeed/pkg/download/engine"
|
||||
wv "github.com/GopeedLab/gopeed/pkg/download/engine/webview"
|
||||
"runtime"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
type eventTestPage struct {
|
||||
fakeRuntimeWebViewPage
|
||||
wv.EventHub
|
||||
}
|
||||
|
||||
func (p *eventTestPage) Close() error {
|
||||
p.Emit(wv.Event{Name: "closed", Data: map[string]any{"reason": "api"}})
|
||||
return nil
|
||||
}
|
||||
|
||||
type eventTestOpener struct{ page wv.Page }
|
||||
|
||||
func (o eventTestOpener) Open(wv.OpenOptions) (wv.Page, error) { return o.page, nil }
|
||||
|
||||
func TestExtensionHostEnv(t *testing.T) {
|
||||
e := engine.NewEngine(nil)
|
||||
defer e.Close()
|
||||
info := NewExtensionInfo(&Extension{Name: "sample", Author: "test", Version: "1.2.3"})
|
||||
if err := injectGopeed(e.Runtime, &Instance{Info: info}, e.Post); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
result, err := e.RunString(`gopeed.host.env.version + "/" + gopeed.info.version + "/" + gopeed.host.env.os + "/" + gopeed.host.env.arch`)
|
||||
if err != nil || result != base.Version+"/1.2.3/"+runtime.GOOS+"/"+runtime.GOARCH {
|
||||
t.Fatalf("%v %v", result, err)
|
||||
}
|
||||
result, err = e.RunString(`["hostVersion", "os", "arch"].some(key => key in gopeed.info)`)
|
||||
if err != nil || result != false {
|
||||
t.Fatalf("host metadata retained in extension info: %v %v", result, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtensionWebViewEvents(t *testing.T) {
|
||||
e := engine.NewEngine(nil)
|
||||
defer e.Close()
|
||||
page := &eventTestPage{}
|
||||
runtime := wv.NewRuntime(eventTestOpener{page}, true)
|
||||
defer runtime.Close()
|
||||
if err := injectGopeed(e.Runtime, &Instance{Runtime: &InstanceRuntime{WebView: runtime}}, e.Post); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err := e.RunString(`var page = gopeed.runtime.webview.open(); var seen = []; var off;
|
||||
(async () => {
|
||||
const registered = page.on("load", () => { seen.push("load"); return new Promise(() => {}); });
|
||||
if (!(registered instanceof Promise)) throw new Error("on must return a Promise");
|
||||
off = await registered;
|
||||
await page.on("closed", data => seen.push(data.reason));
|
||||
for (const args of [["invalid", () => {}], ["load", 1]]) {
|
||||
let rejected = false;
|
||||
try { await page.on(...args); } catch (_) { rejected = true; }
|
||||
if (!rejected) throw new Error("invalid listener accepted");
|
||||
}
|
||||
})()`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
page.Emit(wv.Event{Name: "load", Data: map[string]any{"url": "https://example.test"}})
|
||||
result, err := e.RunString(`off(); off(); seen.join(",")`)
|
||||
if err != nil || result != "load" {
|
||||
t.Fatalf("%v %v", result, err)
|
||||
}
|
||||
page.Emit(wv.Event{Name: "load"})
|
||||
if _, err = e.RunString(`page.close()`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
result, err = e.RunString(`seen.join(",")`)
|
||||
if err != nil || result != "load,api" {
|
||||
t.Fatalf("%v %v", result, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Navigation must not occupy the Goja loop: a notification can execute page
|
||||
// operations even while the original goto promise is still pending.
|
||||
type navigationEventTestPage struct {
|
||||
eventTestPage
|
||||
release chan struct{}
|
||||
}
|
||||
|
||||
func (p *navigationEventTestPage) Goto(url string, _ wv.GotoOptions) error {
|
||||
p.Emit(wv.Event{Name: "load", Data: map[string]any{"url": url}})
|
||||
select {
|
||||
case <-p.release:
|
||||
return nil
|
||||
case <-time.After(2 * time.Second):
|
||||
return fmt.Errorf("event callback blocked by navigation")
|
||||
}
|
||||
}
|
||||
func (p *navigationEventTestPage) Execute(string, ...any) (any, error) {
|
||||
close(p.release)
|
||||
return nil, nil
|
||||
}
|
||||
func TestExtensionWebViewEventsDuringNavigation(t *testing.T) {
|
||||
e := engine.NewEngine(nil)
|
||||
defer e.Close()
|
||||
page := &navigationEventTestPage{release: make(chan struct{})}
|
||||
runtime := wv.NewRuntime(eventTestOpener{page}, true)
|
||||
defer runtime.Close()
|
||||
if err := injectGopeed(e.Runtime, &Instance{Runtime: &InstanceRuntime{WebView: runtime}}, e.Post); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
result, err := e.RunString(`(async () => {
|
||||
const page = await gopeed.runtime.webview.open();
|
||||
await page.on("load", () => page.execute("release"));
|
||||
if (await page.addInitScript("globalThis.ready = true") !== undefined) throw new Error("init result must be void");
|
||||
if (await page.goto("https://example.test") !== undefined) throw new Error("goto result must be void");
|
||||
return true;
|
||||
})()`)
|
||||
if err != nil || result != true {
|
||||
t.Fatalf("%v %v", result, err)
|
||||
}
|
||||
}
|
||||
@@ -8,7 +8,7 @@ import (
|
||||
"github.com/dop251/goja"
|
||||
)
|
||||
|
||||
func injectGopeed(vm *goja.Runtime, gopeed *Instance) error {
|
||||
func injectGopeed(vm *goja.Runtime, gopeed *Instance, post func(func(*goja.Runtime)) bool) error {
|
||||
gopeedObject := vm.NewObject()
|
||||
if gopeed == nil {
|
||||
return vm.Set("gopeed", gopeedObject)
|
||||
@@ -16,6 +16,9 @@ func injectGopeed(vm *goja.Runtime, gopeed *Instance) error {
|
||||
if err := gopeedObject.Set("events", newJSEventsRuntime(vm, gopeed.Events)); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := gopeedObject.Set("host", newInstanceHost()); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := gopeedObject.Set("info", gopeed.Info); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -30,11 +33,14 @@ func injectGopeed(vm *goja.Runtime, gopeed *Instance) error {
|
||||
}
|
||||
runtimeObject := vm.NewObject()
|
||||
if gopeed.Runtime != nil {
|
||||
if err := runtimeObject.Set("ffmpeg", vm.Get("__gopeed_ffmpeg")); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := runtimeObject.Set("blob", newJSBlobRuntime(vm)); err != nil {
|
||||
return err
|
||||
}
|
||||
if gopeed.Runtime.WebView != nil {
|
||||
if err := runtimeObject.Set("webview", newJSWebViewRuntime(vm, gopeed.Runtime.WebView)); err != nil {
|
||||
if err := runtimeObject.Set("webview", newJSWebViewRuntime(vm, gopeed.Runtime.WebView, post)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
@@ -100,7 +106,7 @@ func newJSEventsRuntime(vm *goja.Runtime, events InstanceEvents) *goja.Object {
|
||||
return obj
|
||||
}
|
||||
|
||||
func newJSWebViewRuntime(vm *goja.Runtime, runtime *enginewebview.Runtime) *goja.Object {
|
||||
func newJSWebViewRuntime(vm *goja.Runtime, runtime *enginewebview.Runtime, post func(func(*goja.Runtime)) bool) *goja.Object {
|
||||
obj := vm.NewObject()
|
||||
_ = obj.Set("isAvailable", func(goja.FunctionCall) goja.Value {
|
||||
return vm.ToValue(runtime.IsAvailable())
|
||||
@@ -110,35 +116,48 @@ func newJSWebViewRuntime(vm *goja.Runtime, runtime *enginewebview.Runtime) *goja
|
||||
if err != nil {
|
||||
panic(vm.ToValue(err))
|
||||
}
|
||||
return newJSWebViewPage(vm, page)
|
||||
return newJSWebViewPage(vm, page, post)
|
||||
})
|
||||
return obj
|
||||
}
|
||||
|
||||
func newJSWebViewPage(vm *goja.Runtime, page *enginewebview.PageHandle) *goja.Object {
|
||||
func newJSWebViewPage(vm *goja.Runtime, page *enginewebview.PageHandle, post func(func(*goja.Runtime)) bool) *goja.Object {
|
||||
obj := vm.NewObject()
|
||||
_ = obj.Set("on", func(call goja.FunctionCall) goja.Value {
|
||||
promise, resolve, reject := vm.NewPromise()
|
||||
handler, ok := goja.AssertFunction(call.Argument(1))
|
||||
if !ok {
|
||||
_ = reject(vm.NewTypeError("handler must be a function"))
|
||||
return vm.ToValue(promise)
|
||||
}
|
||||
unsubscribe, err := page.On(call.Argument(0).String(), func(event enginewebview.Event) {
|
||||
post(func(vm *goja.Runtime) {
|
||||
// A callback is a notification: never await its returned promise.
|
||||
_, _ = handler(goja.Undefined(), vm.ToValue(event.Data))
|
||||
})
|
||||
})
|
||||
if err != nil {
|
||||
_ = reject(vm.NewGoError(err))
|
||||
} else {
|
||||
_ = resolve(func(goja.FunctionCall) goja.Value { unsubscribe(); return goja.Undefined() })
|
||||
}
|
||||
return vm.ToValue(promise)
|
||||
})
|
||||
|
||||
_ = obj.Set("addInitScript", func(call goja.FunctionCall) goja.Value {
|
||||
script, err := requireStringArg(call, 0, "script")
|
||||
if err != nil {
|
||||
panic(vm.ToValue(err))
|
||||
}
|
||||
if err := page.AddInitScript(script); err != nil {
|
||||
panic(vm.ToValue(err))
|
||||
}
|
||||
return goja.Undefined()
|
||||
return webviewPromise(vm, post, func() (any, error) { return nil, page.AddInitScript(script) })
|
||||
})
|
||||
_ = obj.Set("goto", func(call goja.FunctionCall) goja.Value {
|
||||
url, err := requireStringArg(call, 0, "url")
|
||||
if err != nil {
|
||||
panic(vm.ToValue(err))
|
||||
}
|
||||
if err := page.Goto(
|
||||
url,
|
||||
optionalMap(call.Argument(1)),
|
||||
); err != nil {
|
||||
panic(vm.ToValue(err))
|
||||
}
|
||||
return goja.Undefined()
|
||||
opts := optionalMap(call.Argument(1))
|
||||
return webviewPromise(vm, post, func() (any, error) { return nil, page.Goto(url, opts) })
|
||||
})
|
||||
_ = obj.Set("execute", func(call goja.FunctionCall) goja.Value {
|
||||
expression, err := enginewebview.NormalizeExecutableValue(call.Argument(0))
|
||||
@@ -251,10 +270,7 @@ func newJSWebViewPage(vm *goja.Runtime, page *enginewebview.PageHandle) *goja.Ob
|
||||
return vm.ToValue(result)
|
||||
})
|
||||
_ = obj.Set("close", func(goja.FunctionCall) goja.Value {
|
||||
if err := page.Close(); err != nil {
|
||||
panic(vm.ToValue(err))
|
||||
}
|
||||
return goja.Undefined()
|
||||
return webviewPromise(vm, post, func() (any, error) { return nil, page.Close() })
|
||||
})
|
||||
return obj
|
||||
}
|
||||
@@ -287,3 +303,22 @@ func requireStringArg(call goja.FunctionCall, index int, name string) (string, e
|
||||
}
|
||||
return value.String(), nil
|
||||
}
|
||||
|
||||
// Browser operations run outside Goja, allowing notifications to be delivered
|
||||
// while navigation is pending. Only the event loop touches JavaScript values.
|
||||
func webviewPromise(vm *goja.Runtime, post func(func(*goja.Runtime)) bool, work func() (any, error)) goja.Value {
|
||||
promise, resolve, reject := vm.NewPromise()
|
||||
go func() {
|
||||
value, err := work()
|
||||
post(func(vm *goja.Runtime) {
|
||||
if err != nil {
|
||||
_ = reject(vm.NewGoError(err))
|
||||
} else if value == nil {
|
||||
_ = resolve(goja.Undefined())
|
||||
} else {
|
||||
_ = resolve(value)
|
||||
}
|
||||
})
|
||||
}()
|
||||
return vm.ToValue(promise)
|
||||
}
|
||||
Loaded 100 of 231 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user