mirror of
https://github.com/GopeedLab/gopeed.git
synced 2026-11-07 09:00:47 +00:00
Merge remote-tracking branch 'origin/main' into codex/fix-bt-file-selection
# Conflicts: # ui/flutter/test/features/tasks/domain/task_record_test.dart
This commit is contained in:
commit
a83e3a2b84
272 files changed
+17212
-1065
No files matched your search
@@ -0,0 +1,2 @@
|
||||
# Keep upstream plugin sources recognizable and collapsed in GitHub reviews.
|
||||
ui/flutter/third_party/** linguist-vendored whitespace=-blank-at-eol,-blank-at-eof
|
||||
@@ -594,7 +594,7 @@ jobs:
|
||||
overwrite: true
|
||||
build-ios:
|
||||
if: ${{ github.event.inputs.platform == 'all' || github.event.inputs.platform == 'ios' }}
|
||||
runs-on: macos-14
|
||||
runs-on: macos-26
|
||||
needs: [get-release]
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
|
||||
@@ -6,15 +6,19 @@
|
||||
// What it checks:
|
||||
// - Every ARB file uses the repository's canonical two-space JSON formatting.
|
||||
// - Every app_<locale>.arb file declares the matching @@locale value.
|
||||
// - Message values are not empty.
|
||||
// - Every locale has exactly the same message keys as app_en.arb.
|
||||
// - Every translation preserves the placeholder variables used by app_en.arb.
|
||||
// - Message values are not empty (warning only).
|
||||
// - Every locale has the same message keys as app_en.arb (warning only).
|
||||
// - Every translation preserves the placeholder variables used by app_en.arb
|
||||
// (warning only).
|
||||
//
|
||||
// Notes:
|
||||
// - This script does not judge translation quality.
|
||||
// - It does not generate localization code or fully parse ICU MessageFormat.
|
||||
// `flutter gen-l10n` runs immediately afterwards in CI to validate ICU syntax
|
||||
// and generate app_localizations*.dart.
|
||||
// - The script exits with a failure only for malformed JSON, invalid ARB
|
||||
// formatting, or a mismatched @@locale value. Translation completeness is
|
||||
// reported as warnings; generation and analysis remain the compile-time gate.
|
||||
|
||||
import 'dart:convert';
|
||||
import 'dart:io';
|
||||
@@ -49,27 +53,24 @@ void main() {
|
||||
keys.where((key) => (messages[key] as String).trim().isEmpty).toList()
|
||||
..sort();
|
||||
if (empty.isNotEmpty) {
|
||||
stderr.writeln('${file.path}: empty messages $empty');
|
||||
failed = true;
|
||||
_warning('${file.path}: empty messages $empty');
|
||||
}
|
||||
if (file.path == templateFile.path) continue;
|
||||
final missing = templateKeys.difference(keys);
|
||||
final extra = keys.difference(templateKeys);
|
||||
if (missing.isNotEmpty || extra.isNotEmpty) {
|
||||
stderr.writeln(
|
||||
_warning(
|
||||
'${file.path}: missing ${missing.toList()..sort()}, extra ${extra.toList()..sort()}',
|
||||
);
|
||||
failed = true;
|
||||
}
|
||||
|
||||
for (final key in templateKeys.intersection(keys)) {
|
||||
final expected = _placeholders(template[key] as String);
|
||||
final actual = _placeholders(messages[key] as String);
|
||||
if (!_sameSet(expected, actual)) {
|
||||
stderr.writeln(
|
||||
_warning(
|
||||
'${file.path}: $key placeholders $actual do not match $expected',
|
||||
);
|
||||
failed = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -102,3 +103,5 @@ Set<String> _placeholders(String message) => RegExp(
|
||||
/// Compares sets without depending on iteration order.
|
||||
bool _sameSet(Set<String> left, Set<String> right) =>
|
||||
left.length == right.length && left.containsAll(right);
|
||||
|
||||
void _warning(String message) => stderr.writeln('warning: $message');
|
||||
@@ -91,7 +91,8 @@ jobs:
|
||||
flutter-version: ${{ env.FLUTTER_VERSION }}
|
||||
- name: Test webview
|
||||
if: runner.os == 'Windows'
|
||||
run: go test -tags webview -v -count=1 -timeout 300s ./internal/webview/goprovider/
|
||||
# Repeat close/reopen coverage to catch browser shutdown races.
|
||||
run: go test -tags webview -v -count=3 -timeout 300s ./internal/webview/goprovider/
|
||||
env:
|
||||
CGO_ENABLED: 1
|
||||
- name: Test webview
|
||||
@@ -213,7 +214,7 @@ jobs:
|
||||
runs-on: ${{ matrix.os }}
|
||||
strategy:
|
||||
matrix:
|
||||
os: [ macos-14, ubuntu-latest ]
|
||||
os: [ macos-26, ubuntu-latest ]
|
||||
needs: [ test ]
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
|
||||
@@ -63,7 +63,7 @@ Connect Gopeed to an AI agent and manage downloads with natural language. For ex
|
||||
|
||||
Gopeed 2.0.0 is currently in public beta, introducing a redesigned interface, a native communication architecture that connects desktop and mobile clients directly to the Go core through FFI, a more consistent cross-platform experience, improved task management, more flexible API support, and MCP-based AI agent integration. Some features may still be incomplete or unstable, so please try it and report any issues you encounter.
|
||||
|
||||
- [Download Gopeed 2.0.0 Beta 1](https://github.com/GopeedLab/gopeed/releases/tag/v2.0.0-beta.1)
|
||||
- [Download Gopeed 2.0.0 Beta 2](https://github.com/GopeedLab/gopeed/releases/tag/v2.0.0-beta.2)
|
||||
|
||||
Once the features and stability meet our release standards, we will publish the official Gopeed 2.0.0 release. Beta users will be able to upgrade directly to the final release, while existing stable users will not be automatically moved onto the beta channel.
|
||||
|
||||
|
||||
+1
-1
@@ -63,7 +63,7 @@ Gopeed を AI Agent と連携すると、自然言語でダウンロードを管
|
||||
|
||||
Gopeed 2.0.0 は現在パブリックベータ版です。刷新されたインターフェース、デスクトップおよびモバイルから FFI で Go コアへ直接接続するネイティブ通信アーキテクチャ、より統一されたクロスプラットフォーム体験、強化されたタスク管理、より柔軟な API、MCP による AI Agent 連携機能を導入しています。一部の機能はまだ不完全または不安定な場合がありますので、ぜひお試しいただき、問題があればフィードバックをお寄せください。
|
||||
|
||||
- [Gopeed 2.0.0 Beta 1 をダウンロード](https://github.com/GopeedLab/gopeed/releases/tag/v2.0.0-beta.1)
|
||||
- [Gopeed 2.0.0 Beta 2 をダウンロード](https://github.com/GopeedLab/gopeed/releases/tag/v2.0.0-beta.2)
|
||||
|
||||
機能と安定性が正式リリースの基準に達した時点で、Gopeed 2.0.0 正式版を公開します。ベータ版のユーザーは正式版へ直接アップデートできます。既存の安定版ユーザーが自動的にベータチャンネルへ移行することはありません。
|
||||
|
||||
|
||||
+1
-1
@@ -63,7 +63,7 @@ Sau khi kết nối Gopeed với AI Agent, bạn có thể quản lý tải xu
|
||||
|
||||
Gopeed 2.0.0 hiện đang trong giai đoạn beta công khai, với giao diện được thiết kế lại, kiến trúc giao tiếp native kết nối trực tiếp ứng dụng desktop và di động với lõi Go qua FFI, trải nghiệm đa nền tảng nhất quán hơn, khả năng quản lý tác vụ được cải thiện, API linh hoạt hơn và khả năng tích hợp AI Agent qua MCP. Một số tính năng có thể vẫn chưa hoàn thiện hoặc chưa ổn định, vì vậy hãy dùng thử và gửi phản hồi nếu bạn gặp vấn đề.
|
||||
|
||||
- [Tải Gopeed 2.0.0 Beta 1](https://github.com/GopeedLab/gopeed/releases/tag/v2.0.0-beta.1)
|
||||
- [Tải Gopeed 2.0.0 Beta 2](https://github.com/GopeedLab/gopeed/releases/tag/v2.0.0-beta.2)
|
||||
|
||||
Khi các tính năng và độ ổn định đáp ứng tiêu chuẩn phát hành, chúng tôi sẽ phát hành Gopeed 2.0.0 chính thức. Người dùng bản beta có thể nâng cấp trực tiếp lên bản chính thức, trong khi người dùng bản ổn định hiện tại sẽ không tự động được chuyển sang kênh beta.
|
||||
|
||||
|
||||
+1
-1
@@ -63,7 +63,7 @@ Gopeed(全称 **Go Speed**,也叫“够快下载器”)是一款使用 Go
|
||||
|
||||
Gopeed 2.0.0 目前处于公开 Beta 测试阶段,引入了全新界面、通过 FFI 直连 Go 核心的原生通信架构、更一致的跨平台体验、更完善的任务管理、更灵活的 API,以及用于 AI Agent 接入的 MCP 能力。部分功能仍可能不够完善或稳定,欢迎提前体验并向我们反馈问题。
|
||||
|
||||
- [下载 Gopeed 2.0.0 Beta 1](https://github.com/GopeedLab/gopeed/releases/tag/v2.0.0-beta.1)
|
||||
- [下载 Gopeed 2.0.0 Beta 2](https://github.com/GopeedLab/gopeed/releases/tag/v2.0.0-beta.2)
|
||||
|
||||
当功能完整度和稳定性达到正式发布标准后,我们会发布 Gopeed 2.0.0 正式版。已安装 Beta 版本的用户可以直接升级到最终正式版,现有稳定版用户则不会被自动切换到 Beta 渠道。
|
||||
|
||||
|
||||
+1
-1
@@ -63,7 +63,7 @@ Gopeed(**Go Speed** 的縮寫)是一款以 Go 與 Flutter 開發的高速、
|
||||
|
||||
Gopeed 2.0.0 目前處於公開 Beta 測試階段,引入了全新介面、透過 FFI 直連 Go 核心的原生通訊架構、更一致的跨平台體驗、更完善的任務管理、更靈活的 API,以及透過 MCP 串接 AI Agent 的能力。部分功能仍可能不夠完善或穩定,歡迎搶先體驗並向我們回報問題。
|
||||
|
||||
- [下載 Gopeed 2.0.0 Beta 1](https://github.com/GopeedLab/gopeed/releases/tag/v2.0.0-beta.1)
|
||||
- [下載 Gopeed 2.0.0 Beta 2](https://github.com/GopeedLab/gopeed/releases/tag/v2.0.0-beta.2)
|
||||
|
||||
當功能完整度與穩定性達到正式發布標準後,我們會發布 Gopeed 2.0.0 正式版。已安裝 Beta 版本的使用者可以直接升級至最終正式版,現有穩定版使用者則不會被自動切換至 Beta 頻道。
|
||||
|
||||
|
||||
+38
-5
@@ -6,9 +6,15 @@ package main
|
||||
|
||||
typedef void (*TaskEventCallback)(char* payload);
|
||||
|
||||
typedef void (*InvokeResultCallback)(uint64_t request_id, int success, char* payload);
|
||||
|
||||
static void callTaskEventCallback(uintptr_t callback, char* payload) {
|
||||
((TaskEventCallback)callback)(payload);
|
||||
}
|
||||
|
||||
static void callInvokeResultCallback(uintptr_t callback, uint64_t request_id, int success, char* payload) {
|
||||
((InvokeResultCallback)callback)(request_id, success, payload);
|
||||
}
|
||||
*/
|
||||
import "C"
|
||||
import (
|
||||
@@ -16,6 +22,7 @@ import (
|
||||
"fmt"
|
||||
"unsafe"
|
||||
|
||||
nativebridge "github.com/GopeedLab/gopeed/bind/native"
|
||||
"github.com/GopeedLab/gopeed/pkg/rest"
|
||||
"github.com/GopeedLab/gopeed/pkg/rest/model"
|
||||
)
|
||||
@@ -35,12 +42,13 @@ func Start(cfg *C.char) (int, *C.char) {
|
||||
if err != nil {
|
||||
return 0, C.CString(err.Error())
|
||||
}
|
||||
nativebridge.ResumeInvokes()
|
||||
return realPort, nil
|
||||
}
|
||||
|
||||
//export Stop
|
||||
func Stop() {
|
||||
rest.Stop()
|
||||
nativebridge.Stop()
|
||||
}
|
||||
|
||||
//export GetAPIServerState
|
||||
@@ -75,14 +83,39 @@ func apiServerResult(state *model.APIServerState, err error) string {
|
||||
return string(data)
|
||||
}
|
||||
|
||||
//export Invoke
|
||||
func Invoke(method *C.char, path *C.char, query *C.char, body *C.char) *C.char {
|
||||
return C.CString(rest.Invoke(
|
||||
//export InvokeAsync
|
||||
func InvokeAsync(
|
||||
method *C.char,
|
||||
path *C.char,
|
||||
query *C.char,
|
||||
body *C.char,
|
||||
requestID C.ulonglong,
|
||||
callback C.uintptr_t,
|
||||
) {
|
||||
if callback == 0 {
|
||||
return
|
||||
}
|
||||
complete := func(result string, err error) {
|
||||
success := C.int(1)
|
||||
payload := result
|
||||
if err != nil {
|
||||
success = 0
|
||||
payload = err.Error()
|
||||
}
|
||||
C.callInvokeResultCallback(
|
||||
callback,
|
||||
C.uint64_t(requestID),
|
||||
success,
|
||||
C.CString(payload),
|
||||
)
|
||||
}
|
||||
nativebridge.InvokeAsync(
|
||||
C.GoString(method),
|
||||
C.GoString(path),
|
||||
C.GoString(query),
|
||||
C.GoString(body),
|
||||
))
|
||||
complete,
|
||||
)
|
||||
}
|
||||
|
||||
//export SubscribeTaskEvents
|
||||
|
||||
+34
-4
@@ -5,7 +5,9 @@ import "C"
|
||||
import (
|
||||
"encoding/json"
|
||||
|
||||
nativebridge "github.com/GopeedLab/gopeed/bind/native"
|
||||
"github.com/GopeedLab/gopeed/internal/webview/rpcprovider"
|
||||
goapi "github.com/GopeedLab/gopeed/pkg/api"
|
||||
"github.com/GopeedLab/gopeed/pkg/rest"
|
||||
"github.com/GopeedLab/gopeed/pkg/rest/model"
|
||||
)
|
||||
@@ -18,11 +20,15 @@ func Start(cfg string) (int, error) {
|
||||
config.ProductionMode = true
|
||||
config.NativeMode = true
|
||||
applyWebViewProvider(&config)
|
||||
return rest.Start(&config)
|
||||
port, err := rest.Start(&config)
|
||||
if err == nil {
|
||||
nativebridge.ResumeInvokes()
|
||||
}
|
||||
return port, err
|
||||
}
|
||||
|
||||
func Stop() {
|
||||
rest.Stop()
|
||||
nativebridge.Stop()
|
||||
}
|
||||
|
||||
func GetAPIServerState() string {
|
||||
@@ -50,8 +56,21 @@ func apiServerResult(state *model.APIServerState, err error) string {
|
||||
return string(data)
|
||||
}
|
||||
|
||||
func Invoke(method, path, query, body string) string {
|
||||
return rest.Invoke(method, path, query, body)
|
||||
type InvokeResultListener interface {
|
||||
OnResult(requestID int64, success bool, payload string)
|
||||
}
|
||||
|
||||
func InvokeAsync(method, path, query, body string, requestID int64, listener InvokeResultListener) {
|
||||
if listener == nil {
|
||||
return
|
||||
}
|
||||
nativebridge.InvokeAsync(method, path, query, body, func(result string, err error) {
|
||||
if err != nil {
|
||||
listener.OnResult(requestID, false, err.Error())
|
||||
return
|
||||
}
|
||||
listener.OnResult(requestID, true, result)
|
||||
})
|
||||
}
|
||||
|
||||
type TaskEventListener interface {
|
||||
@@ -72,3 +91,14 @@ func applyWebViewProvider(config *model.StartConfig) {
|
||||
}
|
||||
config.WebViewProvider = rpcprovider.New(*config.WebViewRPCConfig)
|
||||
}
|
||||
|
||||
func LiveActivityTaskEventMask() int64 {
|
||||
return int64(
|
||||
goapi.TaskEventDone |
|
||||
goapi.TaskEventError |
|
||||
goapi.TaskEventStart |
|
||||
goapi.TaskEventProgress |
|
||||
goapi.TaskEventPause |
|
||||
goapi.TaskEventDelete,
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,240 @@
|
||||
// Package nativebridge contains transport-neutral behavior shared by the
|
||||
// Desktop C FFI and gomobile bridge adapters.
|
||||
package nativebridge
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/GopeedLab/gopeed/pkg/rest"
|
||||
)
|
||||
|
||||
// StopTimeout bounds native shutdown even when an in-process request never returns.
|
||||
const StopTimeout = 3 * time.Second
|
||||
|
||||
var errNativeBridgeStopping = errors.New("native bridge is stopping")
|
||||
|
||||
type invokeRequest struct {
|
||||
method string
|
||||
path string
|
||||
query string
|
||||
body string
|
||||
}
|
||||
|
||||
type invokeTask struct {
|
||||
id uint64
|
||||
request invokeRequest
|
||||
complete func(result string, err error)
|
||||
once sync.Once
|
||||
}
|
||||
|
||||
func (t *invokeTask) finish(result string, err error) {
|
||||
t.once.Do(func() {
|
||||
completeCallback(t.complete, result, err)
|
||||
})
|
||||
}
|
||||
|
||||
// InvokeAsync starts an in-process API request and completes it through the
|
||||
// supplied callback. Desktop and mobile adapters share this executor.
|
||||
func InvokeAsync(method, path, query, body string, callback func(result string, err error)) {
|
||||
if callback == nil {
|
||||
return
|
||||
}
|
||||
task := invokeTask{
|
||||
request: invokeRequest{
|
||||
method: method,
|
||||
path: path,
|
||||
query: query,
|
||||
body: body,
|
||||
},
|
||||
complete: callback,
|
||||
}
|
||||
if !getInvokeExecutor().submit(task) {
|
||||
completeCallback(callback, "", errNativeBridgeStopping)
|
||||
}
|
||||
}
|
||||
|
||||
// ResumeInvokes enables request submission after the native runtime starts.
|
||||
func ResumeInvokes() {
|
||||
getInvokeExecutor().resume()
|
||||
}
|
||||
|
||||
// PauseInvokesAndWait prevents new submissions and gives running requests a
|
||||
// bounded opportunity to finish before a native runtime is stopped.
|
||||
// Requests still pending after StopTimeout are failed and detached so shutdown
|
||||
// can continue without invoking their callbacks a second time.
|
||||
func PauseInvokesAndWait() bool {
|
||||
return getInvokeExecutor().pauseAndWait(StopTimeout)
|
||||
}
|
||||
|
||||
// Stop pauses native requests and shuts down the shared runtime, but never
|
||||
// keeps the host application waiting longer than StopTimeout. Cleanup that
|
||||
// cannot finish within the deadline is allowed to continue in the background.
|
||||
func Stop() {
|
||||
stopWithin(getInvokeExecutor(), rest.Stop, StopTimeout)
|
||||
}
|
||||
|
||||
func stopWithin(executor *invokeExecutor, cleanup func(), timeout time.Duration) {
|
||||
deadline := time.Now().Add(timeout)
|
||||
executor.pauseAndWait(time.Until(deadline))
|
||||
|
||||
stopped := make(chan struct{})
|
||||
go func() {
|
||||
defer close(stopped)
|
||||
cleanup()
|
||||
}()
|
||||
|
||||
remaining := time.Until(deadline)
|
||||
if remaining <= 0 {
|
||||
return
|
||||
}
|
||||
timer := time.NewTimer(remaining)
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case <-stopped:
|
||||
case <-timer.C:
|
||||
}
|
||||
}
|
||||
|
||||
var (
|
||||
invokeExecutorOnce sync.Once
|
||||
sharedInvoker *invokeExecutor
|
||||
)
|
||||
|
||||
func getInvokeExecutor() *invokeExecutor {
|
||||
invokeExecutorOnce.Do(func() {
|
||||
sharedInvoker = newInvokeExecutor(func(request invokeRequest) string {
|
||||
return rest.Dispatch(request.method, request.path, request.query, request.body)
|
||||
})
|
||||
})
|
||||
return sharedInvoker
|
||||
}
|
||||
|
||||
// invokeExecutor starts every native API request in its own goroutine, matching
|
||||
// net/http's request concurrency. It only tracks them to support bounded drain
|
||||
// during shutdown; one stuck request must never prevent another from starting.
|
||||
type invokeExecutor struct {
|
||||
mu sync.Mutex
|
||||
closed bool
|
||||
accepting bool
|
||||
nextID uint64
|
||||
pending map[uint64]*invokeTask
|
||||
drained chan struct{}
|
||||
running sync.WaitGroup
|
||||
handler func(invokeRequest) string
|
||||
}
|
||||
|
||||
func newInvokeExecutor(handler func(invokeRequest) string) *invokeExecutor {
|
||||
drained := make(chan struct{})
|
||||
close(drained)
|
||||
return &invokeExecutor{
|
||||
handler: handler,
|
||||
accepting: true,
|
||||
pending: make(map[uint64]*invokeTask),
|
||||
drained: drained,
|
||||
}
|
||||
}
|
||||
|
||||
func (e *invokeExecutor) submit(task invokeTask) bool {
|
||||
e.mu.Lock()
|
||||
if e.closed || !e.accepting {
|
||||
e.mu.Unlock()
|
||||
return false
|
||||
}
|
||||
if len(e.pending) == 0 {
|
||||
e.drained = make(chan struct{})
|
||||
}
|
||||
taskID := e.nextID
|
||||
e.nextID++
|
||||
task.id = taskID
|
||||
taskPtr := &task
|
||||
e.pending[taskID] = taskPtr
|
||||
e.running.Add(1)
|
||||
e.mu.Unlock()
|
||||
|
||||
go e.run(taskPtr)
|
||||
return true
|
||||
}
|
||||
|
||||
func (e *invokeExecutor) resume() {
|
||||
e.mu.Lock()
|
||||
e.accepting = true
|
||||
e.mu.Unlock()
|
||||
}
|
||||
|
||||
func (e *invokeExecutor) pauseAndWait(timeout time.Duration) bool {
|
||||
e.mu.Lock()
|
||||
e.accepting = false
|
||||
if len(e.pending) == 0 {
|
||||
e.mu.Unlock()
|
||||
return true
|
||||
}
|
||||
drained := e.drained
|
||||
e.mu.Unlock()
|
||||
|
||||
timer := time.NewTimer(timeout)
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case <-drained:
|
||||
return true
|
||||
case <-timer.C:
|
||||
}
|
||||
|
||||
e.mu.Lock()
|
||||
if len(e.pending) == 0 {
|
||||
e.mu.Unlock()
|
||||
return true
|
||||
}
|
||||
pending := make([]*invokeTask, 0, len(e.pending))
|
||||
for _, task := range e.pending {
|
||||
pending = append(pending, task)
|
||||
}
|
||||
e.pending = make(map[uint64]*invokeTask)
|
||||
close(e.drained)
|
||||
e.mu.Unlock()
|
||||
|
||||
for _, task := range pending {
|
||||
task.finish("", errNativeBridgeStopping)
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (e *invokeExecutor) close() {
|
||||
e.mu.Lock()
|
||||
e.closed = true
|
||||
e.accepting = false
|
||||
e.mu.Unlock()
|
||||
e.running.Wait()
|
||||
}
|
||||
|
||||
func (e *invokeExecutor) run(task *invokeTask) {
|
||||
defer e.running.Done()
|
||||
result, err := e.execute(task.request)
|
||||
task.finish(result, err)
|
||||
e.mu.Lock()
|
||||
if _, exists := e.pending[task.id]; exists {
|
||||
delete(e.pending, task.id)
|
||||
if len(e.pending) == 0 {
|
||||
close(e.drained)
|
||||
}
|
||||
}
|
||||
e.mu.Unlock()
|
||||
}
|
||||
|
||||
func (e *invokeExecutor) execute(request invokeRequest) (result string, err error) {
|
||||
defer func() {
|
||||
if recovered := recover(); recovered != nil {
|
||||
err = fmt.Errorf("invoke panic: %v", recovered)
|
||||
}
|
||||
}()
|
||||
return e.handler(request), nil
|
||||
}
|
||||
|
||||
func completeCallback(callback func(string, error), result string, err error) {
|
||||
defer func() {
|
||||
_ = recover()
|
||||
}()
|
||||
callback(result, err)
|
||||
}
|
||||
@@ -0,0 +1,258 @@
|
||||
package nativebridge
|
||||
|
||||
import (
|
||||
"runtime"
|
||||
"strconv"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestInvokeExecutorStartsEveryRequestIndependently(t *testing.T) {
|
||||
const taskCount = 64
|
||||
release := make(chan struct{})
|
||||
started := make(chan struct{}, taskCount)
|
||||
executor := newInvokeExecutor(func(request invokeRequest) string {
|
||||
started <- struct{}{}
|
||||
<-release
|
||||
return request.path
|
||||
})
|
||||
|
||||
var completed sync.WaitGroup
|
||||
completed.Add(taskCount)
|
||||
for i := range taskCount {
|
||||
accepted := executor.submit(invokeTask{
|
||||
request: invokeRequest{path: strconv.Itoa(i)},
|
||||
complete: func(string, error) {
|
||||
completed.Done()
|
||||
},
|
||||
})
|
||||
if !accepted {
|
||||
t.Fatalf("request %d was rejected", i)
|
||||
}
|
||||
}
|
||||
|
||||
// Every handler must start before any of them is released. A worker pool
|
||||
// would stall here once all of its workers were occupied.
|
||||
for range taskCount {
|
||||
select {
|
||||
case <-started:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("a request waited behind other blocked requests")
|
||||
}
|
||||
}
|
||||
|
||||
close(release)
|
||||
completed.Wait()
|
||||
executor.close()
|
||||
}
|
||||
|
||||
func TestInvokeExecutorConvertsPanicsToErrors(t *testing.T) {
|
||||
executor := newInvokeExecutor(func(invokeRequest) string {
|
||||
panic("boom")
|
||||
})
|
||||
completed := make(chan error, 1)
|
||||
executor.submit(invokeTask{complete: func(_ string, err error) {
|
||||
completed <- err
|
||||
}})
|
||||
select {
|
||||
case err := <-completed:
|
||||
if err == nil || err.Error() != "invoke panic: boom" {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("panic result was not completed")
|
||||
}
|
||||
executor.close()
|
||||
}
|
||||
|
||||
func TestInvokeExecutorSurvivesCallbackPanic(t *testing.T) {
|
||||
executor := newInvokeExecutor(func(request invokeRequest) string {
|
||||
return request.path
|
||||
})
|
||||
executor.submit(invokeTask{complete: func(string, error) {
|
||||
panic("callback failed")
|
||||
}})
|
||||
|
||||
completed := make(chan string, 1)
|
||||
executor.submit(invokeTask{
|
||||
request: invokeRequest{path: "next"},
|
||||
complete: func(result string, _ error) {
|
||||
completed <- result
|
||||
},
|
||||
})
|
||||
select {
|
||||
case result := <-completed:
|
||||
if result != "next" {
|
||||
t.Fatalf("unexpected result: %q", result)
|
||||
}
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("worker stopped after callback panic")
|
||||
}
|
||||
executor.close()
|
||||
}
|
||||
|
||||
func TestInvokeExecutorPausesAndDrains(t *testing.T) {
|
||||
release := make(chan struct{})
|
||||
started := make(chan struct{})
|
||||
executor := newInvokeExecutor(func(request invokeRequest) string {
|
||||
if request.path == "running" {
|
||||
close(started)
|
||||
<-release
|
||||
}
|
||||
return request.path
|
||||
})
|
||||
completed := make(chan struct{})
|
||||
if !executor.submit(invokeTask{
|
||||
request: invokeRequest{path: "running"},
|
||||
complete: func(string, error) {
|
||||
close(completed)
|
||||
},
|
||||
}) {
|
||||
t.Fatal("initial task was rejected")
|
||||
}
|
||||
<-started
|
||||
|
||||
drained := make(chan struct{})
|
||||
go func() {
|
||||
executor.pauseAndWait(time.Second)
|
||||
close(drained)
|
||||
}()
|
||||
deadline := time.Now().Add(time.Second)
|
||||
for {
|
||||
executor.mu.Lock()
|
||||
paused := !executor.accepting
|
||||
executor.mu.Unlock()
|
||||
if paused {
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatal("executor did not pause")
|
||||
}
|
||||
runtime.Gosched()
|
||||
}
|
||||
select {
|
||||
case <-drained:
|
||||
t.Fatal("pause returned before the running request completed")
|
||||
default:
|
||||
}
|
||||
if executor.submit(invokeTask{}) {
|
||||
t.Fatal("paused executor accepted a new request")
|
||||
}
|
||||
|
||||
close(release)
|
||||
<-completed
|
||||
select {
|
||||
case <-drained:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("pause did not return after pending requests drained")
|
||||
}
|
||||
|
||||
executor.resume()
|
||||
resumed := make(chan struct{})
|
||||
if !executor.submit(invokeTask{complete: func(string, error) {
|
||||
close(resumed)
|
||||
}}) {
|
||||
t.Fatal("resumed executor rejected a request")
|
||||
}
|
||||
<-resumed
|
||||
executor.close()
|
||||
}
|
||||
|
||||
func TestInvokeExecutorPauseTimesOutAndFailsPendingRequests(t *testing.T) {
|
||||
release := make(chan struct{})
|
||||
started := make(chan struct{})
|
||||
executor := newInvokeExecutor(func(request invokeRequest) string {
|
||||
started <- struct{}{}
|
||||
<-release
|
||||
return request.path
|
||||
})
|
||||
|
||||
errors := make(chan error, 2)
|
||||
for _, path := range []string{"first", "second"} {
|
||||
if !executor.submit(invokeTask{
|
||||
request: invokeRequest{path: path},
|
||||
complete: func(_ string, err error) {
|
||||
errors <- err
|
||||
},
|
||||
}) {
|
||||
t.Fatalf("request %q was rejected", path)
|
||||
}
|
||||
}
|
||||
for range 2 {
|
||||
<-started
|
||||
}
|
||||
|
||||
const timeout = 30 * time.Millisecond
|
||||
startedAt := time.Now()
|
||||
if executor.pauseAndWait(timeout) {
|
||||
t.Fatal("pause unexpectedly reported a complete drain")
|
||||
}
|
||||
if elapsed := time.Since(startedAt); elapsed < timeout || elapsed > 10*timeout {
|
||||
t.Fatalf("pause returned after %s, want close to %s", elapsed, timeout)
|
||||
}
|
||||
for range 2 {
|
||||
select {
|
||||
case err := <-errors:
|
||||
if err == nil || err.Error() != errNativeBridgeStopping.Error() {
|
||||
t.Fatalf("unexpected timeout error: %v", err)
|
||||
}
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("pending request was not failed after the timeout")
|
||||
}
|
||||
}
|
||||
|
||||
close(release)
|
||||
executor.close()
|
||||
select {
|
||||
case err := <-errors:
|
||||
t.Fatalf("request callback ran more than once: %v", err)
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
func TestStopWithinUsesOneDeadlineForRequestsAndCleanup(t *testing.T) {
|
||||
releaseRequest := make(chan struct{})
|
||||
requestStarted := make(chan struct{})
|
||||
executor := newInvokeExecutor(func(invokeRequest) string {
|
||||
close(requestStarted)
|
||||
<-releaseRequest
|
||||
return "done"
|
||||
})
|
||||
requestResult := make(chan error, 1)
|
||||
executor.submit(invokeTask{complete: func(_ string, err error) {
|
||||
requestResult <- err
|
||||
}})
|
||||
<-requestStarted
|
||||
|
||||
releaseCleanup := make(chan struct{})
|
||||
cleanupStarted := make(chan struct{})
|
||||
cleanup := func() {
|
||||
close(cleanupStarted)
|
||||
<-releaseCleanup
|
||||
}
|
||||
|
||||
const timeout = 40 * time.Millisecond
|
||||
startedAt := time.Now()
|
||||
stopWithin(executor, cleanup, timeout)
|
||||
if elapsed := time.Since(startedAt); elapsed < timeout || elapsed > 10*timeout {
|
||||
t.Fatalf("stop returned after %s, want close to %s", elapsed, timeout)
|
||||
}
|
||||
select {
|
||||
case <-cleanupStarted:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("cleanup was not started after the request deadline")
|
||||
}
|
||||
select {
|
||||
case err := <-requestResult:
|
||||
if err == nil || err.Error() != errNativeBridgeStopping.Error() {
|
||||
t.Fatalf("unexpected request result: %v", err)
|
||||
}
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("timed-out request was not failed")
|
||||
}
|
||||
|
||||
close(releaseRequest)
|
||||
close(releaseCleanup)
|
||||
executor.close()
|
||||
}
|
||||
@@ -3,13 +3,15 @@ module github.com/GopeedLab/gopeed
|
||||
go 1.25.4
|
||||
|
||||
require (
|
||||
github.com/GopeedLab/webview_go v0.0.0-20260423085439-7a2f88b6e9b5
|
||||
codeberg.org/gruf/go-ffmpreg v0.6.23-0.20260827095416-df1836cb5be0
|
||||
github.com/GopeedLab/webview_go v0.0.0-20260915010614-2567e492369a
|
||||
github.com/anacrolix/torrent v1.60.1-0.20251217073903-486bcbe758e0
|
||||
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5
|
||||
github.com/bodgit/sevenzip v1.6.1
|
||||
github.com/dop251/goja v0.0.0-20260106131823-651366fbe6e3
|
||||
github.com/dop251/goja_nodejs v0.0.0-20240728170619-29b559befffc
|
||||
github.com/go-git/go-git/v5 v5.8.1
|
||||
github.com/google/jsonschema-go v0.4.3
|
||||
github.com/gorilla/handlers v1.5.1
|
||||
github.com/gorilla/mux v1.8.0
|
||||
github.com/imroc/req/v3 v3.52.2
|
||||
@@ -21,6 +23,7 @@ require (
|
||||
github.com/pkg/errors v0.9.1
|
||||
github.com/rs/dnscache v0.0.0-20230804202142-fc85eb664529
|
||||
github.com/rs/zerolog v1.31.0
|
||||
github.com/tetratelabs/wazero v1.12.0
|
||||
github.com/xiaoqidun/setft v0.0.0-20220310121541-be86327699ad
|
||||
go.etcd.io/bbolt v1.4.3
|
||||
golang.org/x/exp v0.0.0-20251219203646-944ab1f22d93
|
||||
@@ -38,7 +41,6 @@ require (
|
||||
github.com/felixge/fgprof v0.9.5 // indirect
|
||||
github.com/go-task/slim-sprig/v3 v3.0.0 // indirect
|
||||
github.com/google/go-cmp v0.7.0 // indirect
|
||||
github.com/google/jsonschema-go v0.4.3 // indirect
|
||||
github.com/hashicorp/errwrap v1.1.0 // indirect
|
||||
github.com/hashicorp/go-multierror v1.1.1 // indirect
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
|
||||
@@ -156,7 +158,7 @@ require (
|
||||
golang.org/x/crypto v0.48.0 // indirect
|
||||
golang.org/x/mod v0.33.0 // indirect
|
||||
golang.org/x/net v0.50.0
|
||||
golang.org/x/sys v0.41.0 // indirect
|
||||
golang.org/x/sys v0.44.0 // indirect
|
||||
golang.org/x/text v0.34.0
|
||||
golang.org/x/time v0.15.0 // indirect
|
||||
golang.org/x/tools v0.42.0 // indirect
|
||||
|
||||
@@ -14,6 +14,8 @@ cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2k
|
||||
cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw=
|
||||
cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
|
||||
cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=
|
||||
codeberg.org/gruf/go-ffmpreg v0.6.23-0.20260827095416-df1836cb5be0 h1:AqNFl60o4xhTTOxmzAvSf7pazXHKBGuV5ATw7sETIRA=
|
||||
codeberg.org/gruf/go-ffmpreg v0.6.23-0.20260827095416-df1836cb5be0/go.mod h1:XM6eW8CSgAuZRKRynm5nDiSNedm0rav30IychqygOHE=
|
||||
crawshaw.io/iox v0.0.0-20181124134642-c51c3df30797/go.mod h1:sXBiorCo8c46JlQV3oXPKINnZ8mcqnye1EkVkqsectk=
|
||||
crawshaw.io/sqlite v0.3.2/go.mod h1:igAO5JulrQ1DbdZdtVq48mnZUBAPOeFzer7VhDWNtW4=
|
||||
dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk=
|
||||
@@ -23,8 +25,8 @@ filippo.io/edwards25519 v1.0.0-rc.1 h1:m0VOOB23frXZvAOK44usCgLWvtsxIoMCTBGJZlpmG
|
||||
filippo.io/edwards25519 v1.0.0-rc.1/go.mod h1:N1IkdkCkiLB6tki+MYJoSx2JTY9NUlxZE7eHn5EwJns=
|
||||
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
|
||||
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
|
||||
github.com/GopeedLab/webview_go v0.0.0-20260423085439-7a2f88b6e9b5 h1:86LVTaNDM/j/uTAD2HedkXNO9LIQlt642VAdbU/0D/M=
|
||||
github.com/GopeedLab/webview_go v0.0.0-20260423085439-7a2f88b6e9b5/go.mod h1:FvV19lu9rQ9NcbczN5y8pbWecURRRbWa8tbfFARBDa0=
|
||||
github.com/GopeedLab/webview_go v0.0.0-20260915010614-2567e492369a h1:Vzz+8aMVX9ZPPpkxgxa7mCorrAgdXCZL5EunOpOBvao=
|
||||
github.com/GopeedLab/webview_go v0.0.0-20260915010614-2567e492369a/go.mod h1:FvV19lu9rQ9NcbczN5y8pbWecURRRbWa8tbfFARBDa0=
|
||||
github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0rYXWg0=
|
||||
github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ=
|
||||
github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY=
|
||||
@@ -552,6 +554,8 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO
|
||||
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
|
||||
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
|
||||
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
||||
github.com/tetratelabs/wazero v1.12.0 h1:DuWcpNu/FzgEXgGBDp8J1Spc+CWOvvtvVyjKlaZopYU=
|
||||
github.com/tetratelabs/wazero v1.12.0/go.mod h1:LvKtzl2RqO4gyF27BiXU+nKAjcV8f38U+kP/q2vgxh0=
|
||||
github.com/tidwall/btree v1.8.1 h1:27ehoXvm5AG/g+1VxLS1SD3vRhp/H7LuEfwNvddEdmA=
|
||||
github.com/tidwall/btree v1.8.1/go.mod h1:jBbTdUWhSZClZWoDg54VnvV7/54modSOzDN7VXftj1A=
|
||||
github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE=
|
||||
@@ -732,6 +736,8 @@ golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k=
|
||||
golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
|
||||
golang.org/x/sys v0.44.0 h1:ildZl3J4uzeKP07r2F++Op7E9B29JRUy+a27EibtBTQ=
|
||||
golang.org/x/sys v0.44.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc=
|
||||
|
||||
@@ -106,7 +106,13 @@ func (s *SchemeFilter) Match(uri string) bool {
|
||||
case FilterTypeUrl:
|
||||
return strings.HasPrefix(uriUpper, patternUpper+":")
|
||||
case FilterTypeFile:
|
||||
return strings.HasSuffix(uriUpper, "."+patternUpper)
|
||||
// Strip query and fragment before matching so URLs like
|
||||
// https://example.com/video.m3u8?sign=abc still route by extension.
|
||||
clean := uri
|
||||
if idx := strings.IndexAny(clean, "?#"); idx >= 0 {
|
||||
clean = clean[:idx]
|
||||
}
|
||||
return strings.HasSuffix(strings.ToUpper(clean), "."+patternUpper)
|
||||
case FilterTypeBase64:
|
||||
return strings.HasPrefix(uriUpper, "DATA:"+patternUpper+";BASE64,")
|
||||
}
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
package hls
|
||||
|
||||
const (
|
||||
defaultSegmentConnections = 8
|
||||
defaultMaxRetries = 5
|
||||
defaultTimeoutSeconds = 30
|
||||
|
||||
// maxPrefetchSizeSegments caps the HEAD probing done at resolve time.
|
||||
// Playlists larger than this skip size prefetch entirely.
|
||||
maxPrefetchSizeSegments = 500
|
||||
)
|
||||
|
||||
type config struct {
|
||||
SegmentConnections int `json:"segmentConnections"`
|
||||
MaxRetries int `json:"maxRetries"`
|
||||
TimeoutSeconds int `json:"timeoutSeconds"`
|
||||
PrefetchContentLength bool `json:"prefetchContentLength"`
|
||||
}
|
||||
|
||||
func (c *config) normalize() {
|
||||
if c.SegmentConnections <= 0 {
|
||||
c.SegmentConnections = defaultSegmentConnections
|
||||
}
|
||||
if c.MaxRetries < 0 {
|
||||
c.MaxRetries = defaultMaxRetries
|
||||
}
|
||||
if c.TimeoutSeconds <= 0 {
|
||||
c.TimeoutSeconds = defaultTimeoutSeconds
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
package hls
|
||||
|
||||
import (
|
||||
"crypto/aes"
|
||||
"crypto/cipher"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// decryptAES128 decrypts an AES-128-CBC encrypted segment and strips PKCS7
|
||||
// padding. data length must be a multiple of the AES block size.
|
||||
func decryptAES128(data, key, iv []byte) ([]byte, error) {
|
||||
if len(key) != 16 {
|
||||
return nil, fmt.Errorf("invalid AES-128 key size %d", len(key))
|
||||
}
|
||||
if len(iv) != 16 {
|
||||
return nil, fmt.Errorf("invalid AES-128 IV size %d", len(iv))
|
||||
}
|
||||
if len(data) == 0 || len(data)%aes.BlockSize != 0 {
|
||||
return nil, fmt.Errorf("invalid AES-128 data size %d", len(data))
|
||||
}
|
||||
block, err := aes.NewCipher(key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
plain := make([]byte, len(data))
|
||||
cipher.NewCBCDecrypter(block, iv).CryptBlocks(plain, data)
|
||||
return pkcs7Unpad(plain, aes.BlockSize)
|
||||
}
|
||||
|
||||
func pkcs7Unpad(data []byte, blockSize int) ([]byte, error) {
|
||||
if len(data) == 0 {
|
||||
return nil, fmt.Errorf("empty plaintext")
|
||||
}
|
||||
pad := int(data[len(data)-1])
|
||||
if pad == 0 || pad > blockSize || pad > len(data) {
|
||||
return nil, fmt.Errorf("invalid PKCS7 padding")
|
||||
}
|
||||
for _, b := range data[len(data)-pad:] {
|
||||
if int(b) != pad {
|
||||
return nil, fmt.Errorf("invalid PKCS7 padding")
|
||||
}
|
||||
}
|
||||
return data[:len(data)-pad], nil
|
||||
}
|
||||
|
||||
// sequenceIV derives the default per-segment IV mandated by RFC 8216: the
|
||||
// segment media sequence number as a 16-byte big-endian integer.
|
||||
func sequenceIV(sequence int64) []byte {
|
||||
iv := make([]byte, 16)
|
||||
iv[8] = byte(sequence >> 56)
|
||||
iv[9] = byte(sequence >> 48)
|
||||
iv[10] = byte(sequence >> 40)
|
||||
iv[11] = byte(sequence >> 32)
|
||||
iv[12] = byte(sequence >> 24)
|
||||
iv[13] = byte(sequence >> 16)
|
||||
iv[14] = byte(sequence >> 8)
|
||||
iv[15] = byte(sequence)
|
||||
return iv
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
package hls
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/aes"
|
||||
"crypto/cipher"
|
||||
"crypto/rand"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func pkcs7Pad(data []byte, blockSize int) []byte {
|
||||
pad := blockSize - len(data)%blockSize
|
||||
return append(data, bytes.Repeat([]byte{byte(pad)}, pad)...)
|
||||
}
|
||||
|
||||
func TestDecryptAES128Roundtrip(t *testing.T) {
|
||||
key := make([]byte, 16)
|
||||
iv := make([]byte, 16)
|
||||
if _, err := rand.Read(key); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := rand.Read(iv); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
plain := []byte("hello hls segment payload, padded to block size!")
|
||||
padded := pkcs7Pad(plain, aes.BlockSize)
|
||||
block, err := aes.NewCipher(key)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
encrypted := make([]byte, len(padded))
|
||||
cipher.NewCBCEncrypter(block, iv).CryptBlocks(encrypted, padded)
|
||||
|
||||
got, err := decryptAES128(encrypted, key, iv)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !bytes.Equal(got, plain) {
|
||||
t.Errorf("roundtrip mismatch: %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecryptAES128InvalidInput(t *testing.T) {
|
||||
if _, err := decryptAES128([]byte{1, 2, 3}, make([]byte, 16), make([]byte, 16)); err == nil {
|
||||
t.Error("non-block-aligned data should fail")
|
||||
}
|
||||
if _, err := decryptAES128(make([]byte, 16), make([]byte, 15), make([]byte, 16)); err == nil {
|
||||
t.Error("short key should fail")
|
||||
}
|
||||
if _, err := decryptAES128(make([]byte, 16), make([]byte, 16), make([]byte, 16)); err == nil {
|
||||
t.Error("all-zero block has invalid padding and should fail")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSequenceIV(t *testing.T) {
|
||||
iv := sequenceIV(1)
|
||||
if len(iv) != 16 || iv[15] != 1 {
|
||||
t.Errorf("sequence 1 IV wrong: %v", iv)
|
||||
}
|
||||
iv = sequenceIV(0x0102030405060708)
|
||||
for i, want := range []byte{1, 2, 3, 4, 5, 6, 7, 8} {
|
||||
if iv[8+i] != want {
|
||||
t.Errorf("sequence IV byte %d: want %d got %d", i, want, iv[8+i])
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
// Package hls implements native HTTP Live Streaming downloads for m3u8
|
||||
// playlists. A task produces exactly one merged, playable output file:
|
||||
// MPEG-2 TS segments are concatenated as-is, fMP4 streams are merged as
|
||||
// init section followed by the media fragments, and AES-128 encrypted
|
||||
// segments are decrypted before they are stored.
|
||||
//
|
||||
// Supported subset:
|
||||
// - VOD media playlists (EXT-X-ENDLIST present) and master playlists whose
|
||||
// selected variant carries its own audio
|
||||
// - AES-128 with the identity KEYFORMAT, explicit IV or media-sequence
|
||||
// derived IV, including key rotation mid-playlist
|
||||
// - EXT-X-BYTERANGE (including offset inheritance) and fMP4 init sections
|
||||
// via a single, constant EXT-X-MAP
|
||||
//
|
||||
// Everything the merge pipeline cannot reproduce faithfully is rejected with
|
||||
// an explicit error instead of silently producing a broken output file:
|
||||
// - live streams (no EXT-X-ENDLIST) and SAMPLE-AES / DRM encryption
|
||||
// - EXT-X-GAP (would leave a hole) and EXT-X-DISCONTINUITY (broken timeline)
|
||||
// - changing init sections (a repeated EXT-X-MAP is compared by all its
|
||||
// attributes, not just the URI) and encrypted EXT-X-MAP without an
|
||||
// explicit IV
|
||||
// - separate audio renditions referenced by the selected variant (the
|
||||
// merged output would have no audio) and non-identity KEYFORMATs
|
||||
package hls
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,100 @@
|
||||
package hls
|
||||
|
||||
import (
|
||||
"net/url"
|
||||
"path"
|
||||
"strings"
|
||||
|
||||
"github.com/GopeedLab/gopeed/internal/fetcher"
|
||||
)
|
||||
|
||||
type FetcherManager struct {
|
||||
}
|
||||
|
||||
func (fm *FetcherManager) Name() string {
|
||||
return "hls"
|
||||
}
|
||||
|
||||
// Filters registers the .m3u8 extension. The manager must be registered BEFORE
|
||||
// the http manager in the downloader config: protocol routing picks the first
|
||||
// manager whose filter matches, and the http filter claims every http(s) URL.
|
||||
func (fm *FetcherManager) Filters() []*fetcher.SchemeFilter {
|
||||
return []*fetcher.SchemeFilter{
|
||||
{
|
||||
Type: fetcher.FilterTypeFile,
|
||||
Pattern: "M3U8",
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (fm *FetcherManager) Build() fetcher.Fetcher {
|
||||
return &Fetcher{}
|
||||
}
|
||||
|
||||
func (fm *FetcherManager) ParseName(u string) string {
|
||||
parsed, err := url.Parse(u)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
name := path.Base(parsed.Path)
|
||||
name = strings.TrimSuffix(name, path.Ext(name))
|
||||
if name == "" || name == "/" || name == "." {
|
||||
name = parsed.Hostname()
|
||||
}
|
||||
return name
|
||||
}
|
||||
|
||||
func (fm *FetcherManager) AutoRename() bool {
|
||||
return true
|
||||
}
|
||||
|
||||
func (fm *FetcherManager) DefaultConfig() any {
|
||||
return &config{
|
||||
SegmentConnections: defaultSegmentConnections,
|
||||
MaxRetries: defaultMaxRetries,
|
||||
TimeoutSeconds: defaultTimeoutSeconds,
|
||||
PrefetchContentLength: true,
|
||||
}
|
||||
}
|
||||
|
||||
func (fm *FetcherManager) Store(f fetcher.Fetcher) (data any, err error) {
|
||||
_f := f.(*Fetcher)
|
||||
_f.mu.Lock()
|
||||
defer _f.mu.Unlock()
|
||||
if _f.state == nil {
|
||||
return nil, nil
|
||||
}
|
||||
// The engine's periodic checkpoint calls Store without holding the task
|
||||
// lock and serializes the returned value afterwards, while Resolve/Patch/
|
||||
// Start mutate the live state under the fetcher mutex: return a snapshot
|
||||
// copy taken under that mutex, mirroring the HTTP protocol's Store.
|
||||
return cloneState(_f.state), nil
|
||||
}
|
||||
|
||||
func cloneState(state *fetcherState) *fetcherState {
|
||||
cloned := *state
|
||||
cloned.Segments = make([]*Segment, len(state.Segments))
|
||||
for i, seg := range state.Segments {
|
||||
s := *seg
|
||||
cloned.Segments[i] = &s
|
||||
}
|
||||
return &cloned
|
||||
}
|
||||
|
||||
func (fm *FetcherManager) Restore() (v any, f func(meta *fetcher.FetcherMeta, v any) fetcher.Fetcher) {
|
||||
return &fetcherState{}, func(meta *fetcher.FetcherMeta, v any) fetcher.Fetcher {
|
||||
fb := &FetcherManager{}
|
||||
restored := fb.Build().(*Fetcher)
|
||||
restored.meta = meta
|
||||
if state, ok := v.(*fetcherState); ok && state.MediaURL != "" {
|
||||
restored.state = state
|
||||
}
|
||||
// Setup is invoked by the engine (restoreFetcher) after this builder
|
||||
// returns; ReqExtra parsing happens there as well.
|
||||
return restored
|
||||
}
|
||||
}
|
||||
|
||||
func (fm *FetcherManager) Close() error {
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,857 @@
|
||||
package hls
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/aes"
|
||||
"crypto/cipher"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/GopeedLab/gopeed/internal/controller"
|
||||
"github.com/GopeedLab/gopeed/internal/fetcher"
|
||||
"github.com/GopeedLab/gopeed/pkg/base"
|
||||
)
|
||||
|
||||
func pkcs7PadForTest(data []byte, blockSize int) []byte {
|
||||
pad := blockSize - len(data)%blockSize
|
||||
return append(data, bytes.Repeat([]byte{byte(pad)}, pad)...)
|
||||
}
|
||||
|
||||
func encryptSegment(plain, key, iv []byte) []byte {
|
||||
padded := pkcs7PadForTest(plain, aes.BlockSize)
|
||||
block, _ := aes.NewCipher(key)
|
||||
encrypted := make([]byte, len(padded))
|
||||
cipher.NewCBCEncrypter(block, iv).CryptBlocks(encrypted, padded)
|
||||
return encrypted
|
||||
}
|
||||
|
||||
// serveContentWithReferer is a helper handler that enforces an optional
|
||||
// Referer requirement and reports request counts.
|
||||
type cdnHandler struct {
|
||||
mux *http.ServeMux
|
||||
requireReferer string
|
||||
requests atomic.Int64
|
||||
}
|
||||
|
||||
func (h *cdnHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
h.requests.Add(1)
|
||||
if h.requireReferer != "" && r.Header.Get("Referer") != h.requireReferer {
|
||||
w.WriteHeader(http.StatusForbidden)
|
||||
return
|
||||
}
|
||||
h.mux.ServeHTTP(w, r)
|
||||
}
|
||||
|
||||
func newTestFetcher(t *testing.T, cfg config) *Fetcher {
|
||||
t.Helper()
|
||||
fm := &FetcherManager{}
|
||||
f := fm.Build().(*Fetcher)
|
||||
ctl := controller.NewController()
|
||||
ctl.GetConfig = func(v any) {
|
||||
if c, ok := v.(*config); ok {
|
||||
*c = cfg
|
||||
}
|
||||
}
|
||||
f.Setup(ctl)
|
||||
return f
|
||||
}
|
||||
|
||||
func testConfig() config {
|
||||
return config{
|
||||
SegmentConnections: 4,
|
||||
MaxRetries: 2,
|
||||
TimeoutSeconds: 5,
|
||||
PrefetchContentLength: true,
|
||||
}
|
||||
}
|
||||
|
||||
func mustResolve(t *testing.T, f *Fetcher, playlistURL, referer string) {
|
||||
t.Helper()
|
||||
req := &base.Request{URL: playlistURL}
|
||||
if referer != "" {
|
||||
req.Extra = &ReqExtra{Header: map[string]string{"Referer": referer}}
|
||||
}
|
||||
if err := f.Resolve(req, &base.Options{Path: t.TempDir()}); err != nil {
|
||||
t.Fatalf("resolve: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func mustRun(t *testing.T, f *Fetcher) {
|
||||
t.Helper()
|
||||
if err := f.Start(); err != nil {
|
||||
t.Fatalf("start: %v", err)
|
||||
}
|
||||
if err := f.Wait(); err != nil {
|
||||
t.Fatalf("wait: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func statsOf(t *testing.T, f *Fetcher) Stats {
|
||||
t.Helper()
|
||||
s, ok := f.Stats().Snapshot.(*Stats)
|
||||
if !ok {
|
||||
t.Fatalf("unexpected stats snapshot type %T", f.Stats().Snapshot)
|
||||
}
|
||||
return *s
|
||||
}
|
||||
|
||||
func readOutput(t *testing.T, f *Fetcher) []byte {
|
||||
t.Helper()
|
||||
data, err := os.ReadFile(f.meta.SingleFilepath())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return data
|
||||
}
|
||||
|
||||
func TestFetcherE2E_MasterTS(t *testing.T) {
|
||||
seg := func(i int) []byte {
|
||||
return []byte(fmt.Sprintf("SEGMENT-%02d-PAYLOAD", i))
|
||||
}
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/master.m3u8", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Write([]byte("#EXTM3U\n" +
|
||||
"#EXT-X-STREAM-INF:BANDWIDTH=800000\nv1/index.m3u8\n" +
|
||||
"#EXT-X-STREAM-INF:BANDWIDTH=2000000\nv2/index.m3u8\n"))
|
||||
})
|
||||
mux.HandleFunc("/v1/index.m3u8", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Write([]byte("#EXTM3U\n#EXTINF:4,\nlow-0.ts\n#EXT-X-ENDLIST\n"))
|
||||
})
|
||||
mux.HandleFunc("/v2/index.m3u8", func(w http.ResponseWriter, r *http.Request) {
|
||||
var b strings.Builder
|
||||
b.WriteString("#EXTM3U\n")
|
||||
for i := 0; i < 5; i++ {
|
||||
b.WriteString(fmt.Sprintf("#EXTINF:4,\nseg-%d.ts\n", i))
|
||||
}
|
||||
b.WriteString("#EXT-X-ENDLIST\n")
|
||||
w.Write([]byte(b.String()))
|
||||
})
|
||||
for i := 0; i < 5; i++ {
|
||||
content := seg(i)
|
||||
path := "/v2/seg-" + strconv.Itoa(i) + ".ts"
|
||||
mux.HandleFunc(path, func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Length", strconv.Itoa(len(content)))
|
||||
w.Write(content)
|
||||
})
|
||||
}
|
||||
server := httptest.NewServer(&cdnHandler{mux: mux})
|
||||
defer server.Close()
|
||||
|
||||
f := newTestFetcher(t, testConfig())
|
||||
mustResolve(t, f, server.URL+"/master.m3u8", "")
|
||||
// Master resolution picked the 2000000 variant.
|
||||
if f.state.MediaURL != server.URL+"/v2/index.m3u8" {
|
||||
t.Fatalf("wrong variant picked: %s", f.state.MediaURL)
|
||||
}
|
||||
// HEAD prefetch should have computed the exact total size.
|
||||
if f.meta.Res.Size != int64(5*len(seg(0))) {
|
||||
t.Fatalf("prefetched size wrong: %d", f.meta.Res.Size)
|
||||
}
|
||||
mustRun(t, f)
|
||||
|
||||
var want []byte
|
||||
for i := 0; i < 5; i++ {
|
||||
want = append(want, seg(i)...)
|
||||
}
|
||||
got := readOutput(t, f)
|
||||
if !bytes.Equal(got, want) {
|
||||
t.Errorf("merged output mismatch: %q", got)
|
||||
}
|
||||
// The staging folder must be cleaned up after a successful merge; only
|
||||
// the output file remains in the task directory.
|
||||
if entries, _ := os.ReadDir(filepath.Dir(f.meta.SingleFilepath())); len(entries) != 1 {
|
||||
for _, e := range entries {
|
||||
t.Logf("leftover entry: %s", e.Name())
|
||||
}
|
||||
t.Errorf("staging dir should be cleaned up after merge")
|
||||
}
|
||||
if f.meta.Res.Size != int64(len(want)) {
|
||||
t.Errorf("final size wrong: %d", f.meta.Res.Size)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetcherE2E_RefererEnforced(t *testing.T) {
|
||||
page := "https://player.example.com/watch/123"
|
||||
var playlist strings.Builder
|
||||
playlist.WriteString("#EXTM3U\n")
|
||||
segments := make(map[string][]byte)
|
||||
for i := 0; i < 3; i++ {
|
||||
name := fmt.Sprintf("seg-%d.ts", i)
|
||||
segments["/"+name] = []byte(fmt.Sprintf("DATA-%d", i))
|
||||
playlist.WriteString(fmt.Sprintf("#EXTINF:4,\n%s\n", name))
|
||||
}
|
||||
playlist.WriteString("#EXT-X-ENDLIST\n")
|
||||
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/video.m3u8", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Write([]byte(playlist.String()))
|
||||
})
|
||||
for path, content := range segments {
|
||||
c := content
|
||||
mux.HandleFunc(path, func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Write(c)
|
||||
})
|
||||
}
|
||||
server := httptest.NewServer(&cdnHandler{mux: mux, requireReferer: page})
|
||||
defer server.Close()
|
||||
|
||||
// Without Referer the resolve itself must fail (403 on the playlist).
|
||||
blocked := newTestFetcher(t, testConfig())
|
||||
req := &base.Request{URL: server.URL + "/video.m3u8"}
|
||||
if err := blocked.Resolve(req, &base.Options{Path: t.TempDir()}); err == nil {
|
||||
t.Fatal("resolve without Referer should fail")
|
||||
}
|
||||
|
||||
// With the Referer set through the task extra headers it must succeed.
|
||||
allowed := newTestFetcher(t, testConfig())
|
||||
allowedReq := &base.Request{
|
||||
URL: server.URL + "/video.m3u8",
|
||||
Extra: &ReqExtra{Header: map[string]string{"Referer": page}},
|
||||
}
|
||||
if err := allowed.Resolve(allowedReq, &base.Options{Path: t.TempDir()}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
mustRun(t, allowed)
|
||||
want := []byte("DATA-0DATA-1DATA-2")
|
||||
if !bytes.Equal(readOutput(t, allowed), want) {
|
||||
t.Error("output mismatch with Referer download")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetcherE2E_AES128KeyRotation(t *testing.T) {
|
||||
key1 := []byte("0123456789abcdef")
|
||||
key2 := []byte("fedcba9876543210")
|
||||
iv1 := []byte("aaaaaaaaaaaaaaaa")
|
||||
plain1 := []byte("ENCRYPTED-SEGMENT-ONE!!") // padded on encrypt
|
||||
plain2 := []byte("ENCRYPTED-SEGMENT-TWO!!")
|
||||
// seg-2 has no explicit IV: derived from media sequence 1.
|
||||
enc1 := encryptSegment(plain1, key1, iv1)
|
||||
enc2 := encryptSegment(plain2, key2, sequenceIV(1))
|
||||
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/video.m3u8", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Write([]byte("#EXTM3U\n" +
|
||||
"#EXT-X-MEDIA-SEQUENCE:0\n" +
|
||||
"#EXT-X-KEY:METHOD=AES-128,URI=\"key-1.bin\",IV=0x" + hexEncode(iv1) + "\n" +
|
||||
"#EXTINF:5,\nseg-1.ts\n" +
|
||||
"#EXT-X-KEY:METHOD=AES-128,URI=\"key-2.bin\"\n" +
|
||||
"#EXTINF:5,\nseg-2.ts\n" +
|
||||
"#EXT-X-ENDLIST\n"))
|
||||
})
|
||||
mux.HandleFunc("/key-1.bin", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Write(key1)
|
||||
})
|
||||
mux.HandleFunc("/key-2.bin", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Write(key2)
|
||||
})
|
||||
mux.HandleFunc("/seg-1.ts", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Write(enc1)
|
||||
})
|
||||
mux.HandleFunc("/seg-2.ts", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Write(enc2)
|
||||
})
|
||||
server := httptest.NewServer(mux)
|
||||
defer server.Close()
|
||||
|
||||
f := newTestFetcher(t, testConfig())
|
||||
mustResolve(t, f, server.URL+"/video.m3u8", "")
|
||||
// Ciphertext sizes overstate the final output after decryption, so an
|
||||
// encrypted plan must report an indeterminate size.
|
||||
if f.meta.Res.Size != 0 {
|
||||
t.Errorf("encrypted plan should report size 0, got %d", f.meta.Res.Size)
|
||||
}
|
||||
mustRun(t, f)
|
||||
|
||||
want := append(append([]byte{}, plain1...), plain2...)
|
||||
if !bytes.Equal(readOutput(t, f), want) {
|
||||
t.Errorf("decrypted output mismatch: %q", readOutput(t, f))
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetcherE2E_ByteRange(t *testing.T) {
|
||||
data := []byte("0123456789ABCDEFGHIJ") // 20 bytes
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/video.m3u8", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Write([]byte("#EXTM3U\n" +
|
||||
"#EXT-X-BYTERANGE:10@0\nblob.bin\n" +
|
||||
"#EXT-X-BYTERANGE:10\nblob.bin\n" +
|
||||
"#EXT-X-ENDLIST\n"))
|
||||
})
|
||||
mux.HandleFunc("/blob.bin", func(w http.ResponseWriter, r *http.Request) {
|
||||
rng := r.Header.Get("Range")
|
||||
if rng == "" {
|
||||
w.Write(data)
|
||||
return
|
||||
}
|
||||
// bytes=start-end
|
||||
var start, end int
|
||||
fmt.Sscanf(rng, "bytes=%d-%d", &start, &end)
|
||||
w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, len(data)))
|
||||
w.WriteHeader(http.StatusPartialContent)
|
||||
w.Write(data[start : end+1])
|
||||
})
|
||||
server := httptest.NewServer(mux)
|
||||
defer server.Close()
|
||||
|
||||
f := newTestFetcher(t, testConfig())
|
||||
mustResolve(t, f, server.URL+"/video.m3u8", "")
|
||||
if f.meta.Res.Size != 20 {
|
||||
t.Fatalf("byterange sizes should be known from the playlist, got %d", f.meta.Res.Size)
|
||||
}
|
||||
mustRun(t, f)
|
||||
if !bytes.Equal(readOutput(t, f), data) {
|
||||
t.Errorf("byterange merge mismatch: %q", readOutput(t, f))
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetcherE2E_PauseResume(t *testing.T) {
|
||||
const total = 8
|
||||
var served atomic.Int64
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/video.m3u8", func(w http.ResponseWriter, r *http.Request) {
|
||||
var b strings.Builder
|
||||
b.WriteString("#EXTM3U\n")
|
||||
for i := 0; i < total; i++ {
|
||||
b.WriteString(fmt.Sprintf("#EXTINF:4,\nseg-%d.ts\n", i))
|
||||
}
|
||||
b.WriteString("#EXT-X-ENDLIST\n")
|
||||
w.Write([]byte(b.String()))
|
||||
})
|
||||
segments := make(map[int][]byte)
|
||||
for i := 0; i < total; i++ {
|
||||
segments[i] = []byte(fmt.Sprintf("SLOW-SEGMENT-%02d", i))
|
||||
}
|
||||
for i := 0; i < total; i++ {
|
||||
idx := i
|
||||
mux.HandleFunc(fmt.Sprintf("/seg-%d.ts", i), func(w http.ResponseWriter, r *http.Request) {
|
||||
time.Sleep(150 * time.Millisecond)
|
||||
served.Add(1)
|
||||
w.Write(segments[idx])
|
||||
})
|
||||
}
|
||||
server := httptest.NewServer(mux)
|
||||
defer server.Close()
|
||||
|
||||
cfg := testConfig()
|
||||
cfg.SegmentConnections = 2
|
||||
cfg.PrefetchContentLength = false
|
||||
f := newTestFetcher(t, cfg)
|
||||
mustResolve(t, f, server.URL+"/video.m3u8", "")
|
||||
|
||||
if err := f.Start(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Let a couple of segments finish, then pause.
|
||||
time.Sleep(400 * time.Millisecond)
|
||||
if err := f.Pause(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
doneBefore := statsOf(t, f).SegmentsDone
|
||||
if doneBefore == 0 {
|
||||
t.Fatal("expected some segments to complete before pause")
|
||||
}
|
||||
if doneBefore == total {
|
||||
t.Fatal("all segments finished before pause, test is not exercising resume")
|
||||
}
|
||||
|
||||
// Resume must finish the rest and produce the full output.
|
||||
if err := f.Start(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := f.Wait(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var want []byte
|
||||
for i := 0; i < total; i++ {
|
||||
want = append(want, segments[i]...)
|
||||
}
|
||||
if !bytes.Equal(readOutput(t, f), want) {
|
||||
t.Errorf("resumed output mismatch: %q", readOutput(t, f))
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetcherE2E_StoreRestoreResume(t *testing.T) {
|
||||
const total = 6
|
||||
var served atomic.Int64
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/video.m3u8", func(w http.ResponseWriter, r *http.Request) {
|
||||
var b strings.Builder
|
||||
b.WriteString("#EXTM3U\n")
|
||||
for i := 0; i < total; i++ {
|
||||
b.WriteString(fmt.Sprintf("#EXTINF:4,\nseg-%d.ts\n", i))
|
||||
}
|
||||
b.WriteString("#EXT-X-ENDLIST\n")
|
||||
w.Write([]byte(b.String()))
|
||||
})
|
||||
segments := make(map[int][]byte)
|
||||
for i := 0; i < total; i++ {
|
||||
segments[i] = []byte(fmt.Sprintf("RESTORE-SEGMENT-%02d", i))
|
||||
}
|
||||
for i := 0; i < total; i++ {
|
||||
idx := i
|
||||
mux.HandleFunc(fmt.Sprintf("/seg-%d.ts", i), func(w http.ResponseWriter, r *http.Request) {
|
||||
time.Sleep(120 * time.Millisecond)
|
||||
served.Add(1)
|
||||
w.Write(segments[idx])
|
||||
})
|
||||
}
|
||||
server := httptest.NewServer(mux)
|
||||
defer server.Close()
|
||||
|
||||
cfg := testConfig()
|
||||
cfg.SegmentConnections = 2
|
||||
cfg.PrefetchContentLength = false
|
||||
|
||||
f := newTestFetcher(t, cfg)
|
||||
mustResolve(t, f, server.URL+"/video.m3u8", "")
|
||||
if err := f.Start(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
time.Sleep(350 * time.Millisecond)
|
||||
if err := f.Pause(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
doneBefore := statsOf(t, f).SegmentsDone
|
||||
if doneBefore == 0 {
|
||||
t.Fatal("expected some segments to finish before simulated restart")
|
||||
}
|
||||
|
||||
// Simulate an app restart: persist, then restore into a fresh fetcher.
|
||||
fm := &FetcherManager{}
|
||||
state, err := fm.Store(f)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if state == nil {
|
||||
t.Fatal("state should be persisted")
|
||||
}
|
||||
ctl := controller.NewController()
|
||||
ctl.GetConfig = func(v any) {
|
||||
if c, ok := v.(*config); ok {
|
||||
*c = cfg
|
||||
}
|
||||
}
|
||||
holder := fm.Build().(*Fetcher)
|
||||
holder.Setup(ctl)
|
||||
holder.meta = &fetcher.FetcherMeta{
|
||||
Req: f.meta.Req,
|
||||
Opts: f.meta.Opts,
|
||||
Res: f.meta.Res,
|
||||
}
|
||||
_, builder := fm.Restore()
|
||||
rf := builder(holder.meta, state).(*Fetcher)
|
||||
// The engine calls Setup on the builder result (restoreFetcher).
|
||||
rf.Setup(ctl)
|
||||
if rf.state == nil || len(rf.state.Segments) != total {
|
||||
t.Fatal("restored state missing segment plan")
|
||||
}
|
||||
|
||||
if err := rf.Start(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := rf.Wait(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if done := statsOf(t, rf).SegmentsDone; done != total {
|
||||
t.Errorf("want %d done, got %d", total, done)
|
||||
}
|
||||
var want []byte
|
||||
for i := 0; i < total; i++ {
|
||||
want = append(want, segments[i]...)
|
||||
}
|
||||
if !bytes.Equal(readOutput(t, rf), want) {
|
||||
t.Errorf("restored output mismatch: %q", readOutput(t, rf))
|
||||
}
|
||||
}
|
||||
|
||||
func hexEncode(b []byte) string {
|
||||
const digits = "0123456789abcdef"
|
||||
out := make([]byte, 0, len(b)*2)
|
||||
for _, v := range b {
|
||||
out = append(out, digits[v>>4], digits[v&0x0f])
|
||||
}
|
||||
return string(out)
|
||||
}
|
||||
|
||||
// restoreLikeEngine round-trips the persisted state and the task meta through
|
||||
// JSON the way the engine storage does, then builds a fresh fetcher. This is
|
||||
// the path that historically lost the request extra headers.
|
||||
func restoreLikeEngine(t *testing.T, f *Fetcher, cfg config) *Fetcher {
|
||||
t.Helper()
|
||||
fm := &FetcherManager{}
|
||||
stateAny, err := fm.Store(f)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rawState, err := json.Marshal(stateAny)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
storedState := &fetcherState{}
|
||||
if err := json.Unmarshal(rawState, storedState); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rawMeta, err := json.Marshal(f.meta)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
restoredMeta := &fetcher.FetcherMeta{}
|
||||
if err := json.Unmarshal(rawMeta, restoredMeta); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, ok := restoredMeta.Req.Extra.(*ReqExtra); ok {
|
||||
t.Fatal("storage round trip should yield a map-shaped extra, like the engine")
|
||||
}
|
||||
ctl := controller.NewController()
|
||||
ctl.GetConfig = func(v any) {
|
||||
if c, ok := v.(*config); ok {
|
||||
*c = cfg
|
||||
}
|
||||
}
|
||||
_, builder := fm.Restore()
|
||||
rf := builder(restoredMeta, storedState).(*Fetcher)
|
||||
// The engine calls Setup on the builder result (restoreFetcher).
|
||||
rf.Setup(ctl)
|
||||
return rf
|
||||
}
|
||||
|
||||
func TestFetcherE2E_StoreRestoreAuthenticated(t *testing.T) {
|
||||
const total = 4
|
||||
page := "https://player.example.com/watch/42"
|
||||
var served atomic.Int64
|
||||
mux := http.NewServeMux()
|
||||
var playlist strings.Builder
|
||||
playlist.WriteString("#EXTM3U\n")
|
||||
for i := 0; i < total; i++ {
|
||||
playlist.WriteString(fmt.Sprintf("#EXTINF:4,\nseg-%d.ts\n", i))
|
||||
}
|
||||
playlist.WriteString("#EXT-X-ENDLIST\n")
|
||||
mux.HandleFunc("/video.m3u8", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Write([]byte(playlist.String()))
|
||||
})
|
||||
segments := make(map[int][]byte)
|
||||
for i := 0; i < total; i++ {
|
||||
segments[i] = []byte(fmt.Sprintf("AUTH-SEGMENT-%02d", i))
|
||||
idx := i
|
||||
mux.HandleFunc(fmt.Sprintf("/seg-%d.ts", idx), func(w http.ResponseWriter, r *http.Request) {
|
||||
time.Sleep(80 * time.Millisecond)
|
||||
served.Add(1)
|
||||
w.Write(segments[idx])
|
||||
})
|
||||
}
|
||||
server := httptest.NewServer(&cdnHandler{mux: mux, requireReferer: page})
|
||||
defer server.Close()
|
||||
|
||||
cfg := testConfig()
|
||||
cfg.SegmentConnections = 2
|
||||
cfg.PrefetchContentLength = false
|
||||
|
||||
f := newTestFetcher(t, cfg)
|
||||
req := &base.Request{
|
||||
URL: server.URL + "/video.m3u8",
|
||||
Extra: &ReqExtra{Header: map[string]string{"Referer": page}},
|
||||
}
|
||||
if err := f.Resolve(req, &base.Options{Path: t.TempDir()}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := f.Start(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
time.Sleep(150 * time.Millisecond)
|
||||
if err := f.Pause(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
rf := restoreLikeEngine(t, f, cfg)
|
||||
if rf.extra == nil || rf.extra.Header["Referer"] != page {
|
||||
t.Fatalf("restored fetcher lost the request extra headers: %+v", rf.extra)
|
||||
}
|
||||
if err := rf.Start(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := rf.Wait(); err != nil {
|
||||
t.Fatalf("resumed download failed (request headers lost?): %v", err)
|
||||
}
|
||||
var want []byte
|
||||
for i := 0; i < total; i++ {
|
||||
want = append(want, segments[i]...)
|
||||
}
|
||||
if !bytes.Equal(readOutput(t, rf), want) {
|
||||
t.Errorf("restored output mismatch: %q", readOutput(t, rf))
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetcherE2E_PatchPartialUpdate(t *testing.T) {
|
||||
page := "https://player.example.com/watch/7"
|
||||
var requiredReferer atomic.Value // string
|
||||
requiredReferer.Store(page)
|
||||
var playlist strings.Builder
|
||||
playlist.WriteString("#EXTM3U\n")
|
||||
segments := make(map[string][]byte)
|
||||
for i := 0; i < 3; i++ {
|
||||
segments[fmt.Sprintf("/s%d.ts", i)] = []byte(fmt.Sprintf("PATCH-%d", i))
|
||||
playlist.WriteString(fmt.Sprintf("#EXTINF:4,\ns%d.ts\n", i))
|
||||
}
|
||||
playlist.WriteString("#EXT-X-ENDLIST\n")
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/video.m3u8", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Write([]byte(playlist.String()))
|
||||
})
|
||||
for path, content := range segments {
|
||||
c := content
|
||||
mux.HandleFunc(path, func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Write(c)
|
||||
})
|
||||
}
|
||||
// The Referer requirement can be flipped mid-test to prove that a patched
|
||||
// header is actually sent by the next run.
|
||||
dyn := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if want, _ := requiredReferer.Load().(string); want != "" && r.Header.Get("Referer") != want {
|
||||
w.WriteHeader(http.StatusForbidden)
|
||||
return
|
||||
}
|
||||
mux.ServeHTTP(w, r)
|
||||
})
|
||||
server := httptest.NewServer(dyn)
|
||||
defer server.Close()
|
||||
|
||||
f := newTestFetcher(t, testConfig())
|
||||
req := &base.Request{
|
||||
URL: server.URL + "/video.m3u8",
|
||||
Extra: &ReqExtra{Header: map[string]string{"Referer": page}},
|
||||
}
|
||||
opts := &base.Options{Path: t.TempDir()}
|
||||
if err := f.Resolve(req, opts); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// A nil/nil patch must be a no-op, not a panic.
|
||||
if err := f.Patch(nil, nil); err != nil {
|
||||
t.Fatalf("nil patch should be accepted: %v", err)
|
||||
}
|
||||
|
||||
// Header-only patch: the resolved plan and output options are retained
|
||||
// while the new header takes effect on the next run.
|
||||
newReferer := "https://player.example.com/watch/9"
|
||||
if err := f.Patch(&base.Request{
|
||||
URL: server.URL + "/video.m3u8",
|
||||
Extra: &ReqExtra{Header: map[string]string{"Referer": newReferer}},
|
||||
}, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if f.state == nil || f.meta.Res == nil {
|
||||
t.Fatal("header-only patch must keep the resolved plan")
|
||||
}
|
||||
if f.meta.Opts.Path != opts.Path {
|
||||
t.Fatalf("header-only patch must keep the output path: %s", f.meta.Opts.Path)
|
||||
}
|
||||
if f.extra.Header["Referer"] != newReferer {
|
||||
t.Fatalf("patched header not applied: %+v", f.extra.Header)
|
||||
}
|
||||
requiredReferer.Store(newReferer)
|
||||
if err := f.Start(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := f.Wait(); err != nil {
|
||||
t.Fatalf("run with patched header failed: %v", err)
|
||||
}
|
||||
if got := readOutput(t, f); string(got) != "PATCH-0PATCH-1PATCH-2" {
|
||||
t.Errorf("output mismatch after header patch: %q", got)
|
||||
}
|
||||
|
||||
// Options-only patch: renaming the task keeps the resolved plan.
|
||||
if err := f.Patch(nil, &base.Options{Name: "renamed.ts"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if f.state == nil {
|
||||
t.Fatal("options-only patch must keep the resolved plan")
|
||||
}
|
||||
if f.meta.Opts.Name != "renamed.ts" || f.meta.Opts.Path != opts.Path {
|
||||
t.Fatalf("options patch misapplied: %+v", f.meta.Opts)
|
||||
}
|
||||
|
||||
// URL-only patch: the plan is dropped for a re-resolve but the output
|
||||
// path and renamed file survive.
|
||||
if err := f.Patch(&base.Request{URL: server.URL + "/refreshed.m3u8"}, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if f.state != nil || f.meta.Res != nil {
|
||||
t.Fatal("URL patch must drop the resolved plan")
|
||||
}
|
||||
if f.meta.Opts.Path != opts.Path || f.meta.Opts.Name != "renamed.ts" {
|
||||
t.Fatal("URL patch must keep the output options")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetcherE2E_PatchWhileRunningRejected(t *testing.T) {
|
||||
mux := http.NewServeMux()
|
||||
var playlist strings.Builder
|
||||
playlist.WriteString("#EXTM3U\n")
|
||||
for i := 0; i < 6; i++ {
|
||||
playlist.WriteString(fmt.Sprintf("#EXTINF:4,\nseg-%d.ts\n", i))
|
||||
}
|
||||
playlist.WriteString("#EXT-X-ENDLIST\n")
|
||||
mux.HandleFunc("/video.m3u8", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Write([]byte(playlist.String()))
|
||||
})
|
||||
for i := 0; i < 6; i++ {
|
||||
mux.HandleFunc(fmt.Sprintf("/seg-%d.ts", i), func(w http.ResponseWriter, r *http.Request) {
|
||||
time.Sleep(150 * time.Millisecond)
|
||||
w.Write([]byte("X"))
|
||||
})
|
||||
}
|
||||
server := httptest.NewServer(mux)
|
||||
defer server.Close()
|
||||
|
||||
f := newTestFetcher(t, testConfig())
|
||||
if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: t.TempDir()}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := f.Start(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := f.Patch(&base.Request{URL: server.URL + "/other.m3u8"}, nil); err == nil {
|
||||
t.Fatal("patching a running task must be rejected")
|
||||
}
|
||||
if err := f.Pause(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// After the pause the patch goes through and only then drops the plan.
|
||||
if err := f.Patch(&base.Request{URL: server.URL + "/other.m3u8"}, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if f.state != nil {
|
||||
t.Fatal("URL patch after pause must drop the plan")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetcherE2E_RetryAfterFailure(t *testing.T) {
|
||||
const total = 4
|
||||
var healthy atomic.Bool
|
||||
mux := http.NewServeMux()
|
||||
var playlist strings.Builder
|
||||
playlist.WriteString("#EXTM3U\n")
|
||||
for i := 0; i < total; i++ {
|
||||
playlist.WriteString(fmt.Sprintf("#EXTINF:4,\nseg-%d.ts\n", i))
|
||||
}
|
||||
playlist.WriteString("#EXT-X-ENDLIST\n")
|
||||
mux.HandleFunc("/video.m3u8", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Write([]byte(playlist.String()))
|
||||
})
|
||||
segments := make(map[int][]byte)
|
||||
for i := 0; i < total; i++ {
|
||||
segments[i] = []byte(fmt.Sprintf("RETRY-SEGMENT-%02d", i))
|
||||
idx := i
|
||||
mux.HandleFunc(fmt.Sprintf("/seg-%d.ts", idx), func(w http.ResponseWriter, r *http.Request) {
|
||||
if idx == 1 && !healthy.Load() {
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
time.Sleep(60 * time.Millisecond)
|
||||
w.Write(segments[idx])
|
||||
})
|
||||
}
|
||||
server := httptest.NewServer(mux)
|
||||
defer server.Close()
|
||||
|
||||
cfg := testConfig()
|
||||
cfg.MaxRetries = 0 // fail the run on the first bad segment
|
||||
cfg.PrefetchContentLength = false
|
||||
f := newTestFetcher(t, cfg)
|
||||
if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: t.TempDir()}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := f.Start(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := f.Wait(); err == nil {
|
||||
t.Fatal("first run should fail while the segment endpoint is down")
|
||||
}
|
||||
if got := statsOf(t, f).SegmentsFailed; got == 0 {
|
||||
t.Fatal("failed segments should be reported in stats")
|
||||
}
|
||||
|
||||
// Server recovered: a fresh run must succeed and report zero failures.
|
||||
healthy.Store(true)
|
||||
if err := f.Start(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := f.Wait(); err != nil {
|
||||
t.Fatalf("retry run failed: %v", err)
|
||||
}
|
||||
stats := statsOf(t, f)
|
||||
if stats.SegmentsFailed != 0 {
|
||||
t.Errorf("failures must reset per run, got %d", stats.SegmentsFailed)
|
||||
}
|
||||
if stats.SegmentsDone != total {
|
||||
t.Errorf("want %d done, got %d", total, stats.SegmentsDone)
|
||||
}
|
||||
var want []byte
|
||||
for i := 0; i < total; i++ {
|
||||
want = append(want, segments[i]...)
|
||||
}
|
||||
if !bytes.Equal(readOutput(t, f), want) {
|
||||
t.Errorf("retry output mismatch: %q", readOutput(t, f))
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetcherE2E_StateMissingSelfHeal(t *testing.T) {
|
||||
const total = 3
|
||||
mux := http.NewServeMux()
|
||||
var playlist strings.Builder
|
||||
playlist.WriteString("#EXTM3U\n")
|
||||
for i := 0; i < total; i++ {
|
||||
playlist.WriteString(fmt.Sprintf("#EXTINF:4,\nseg-%d.ts\n", i))
|
||||
}
|
||||
playlist.WriteString("#EXT-X-ENDLIST\n")
|
||||
mux.HandleFunc("/video.m3u8", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Write([]byte(playlist.String()))
|
||||
})
|
||||
segments := make(map[int][]byte)
|
||||
for i := 0; i < total; i++ {
|
||||
segments[i] = []byte(fmt.Sprintf("SELFHEAL-%02d", i))
|
||||
idx := i
|
||||
mux.HandleFunc(fmt.Sprintf("/seg-%d.ts", idx), func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Write(segments[idx])
|
||||
})
|
||||
}
|
||||
server := httptest.NewServer(mux)
|
||||
defer server.Close()
|
||||
|
||||
f := newTestFetcher(t, testConfig())
|
||||
if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: t.TempDir()}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Simulate the crash window where the engine persisted the task with its
|
||||
// resource but the HLS plan was never checkpointed.
|
||||
f.state = nil
|
||||
outputName := f.meta.Res.Files[0].Name
|
||||
if err := f.Start(); err != nil {
|
||||
t.Fatalf("start should self-heal a missing plan: %v", err)
|
||||
}
|
||||
if err := f.Wait(); err != nil {
|
||||
t.Fatalf("self-healed run failed: %v", err)
|
||||
}
|
||||
if f.state == nil || len(f.state.Segments) != total {
|
||||
t.Fatal("self-heal should rebuild the segment plan")
|
||||
}
|
||||
if f.meta.Res.Files[0].Name != outputName {
|
||||
t.Fatalf("self-heal renamed the output: %s", f.meta.Res.Files[0].Name)
|
||||
}
|
||||
var want []byte
|
||||
for i := 0; i < total; i++ {
|
||||
want = append(want, segments[i]...)
|
||||
}
|
||||
if !bytes.Equal(readOutput(t, f), want) {
|
||||
t.Errorf("self-healed output mismatch: %q", readOutput(t, f))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,100 @@
|
||||
package hls
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/cookiejar"
|
||||
"time"
|
||||
|
||||
"github.com/GopeedLab/gopeed/internal/httpclient"
|
||||
"github.com/GopeedLab/gopeed/pkg/base"
|
||||
)
|
||||
|
||||
const defaultUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36"
|
||||
|
||||
const maxPlaylistSize = 10 << 20 // 10MB, playlists are small text files
|
||||
|
||||
func (f *Fetcher) timeout() time.Duration {
|
||||
return time.Duration(f.config.TimeoutSeconds) * time.Second
|
||||
}
|
||||
|
||||
// buildClient builds an HTTP client honoring the task proxy and TLS settings.
|
||||
// The client has no total timeout on purpose: large segments over slow links
|
||||
// must not be aborted; body stalls are handled by stallReader instead. It is
|
||||
// only called while the fetcher is quiescent (Resolve/Start), never from
|
||||
// inside running worker goroutines.
|
||||
func (f *Fetcher) buildClient() *http.Client {
|
||||
var proxy *base.RequestProxy
|
||||
skipVerify := false
|
||||
if req := f.meta.Req; req != nil {
|
||||
proxy = req.Proxy
|
||||
skipVerify = req.SkipVerifyCert
|
||||
}
|
||||
jar, _ := cookiejar.New(nil)
|
||||
client, err := httpclient.NewClient(httpclient.Options{
|
||||
Client: httpclient.ClientOptions{
|
||||
Jar: jar,
|
||||
},
|
||||
Transport: httpclient.TransportOptions{
|
||||
DialContext: (&net.Dialer{
|
||||
Timeout: f.timeout(),
|
||||
}).DialContext,
|
||||
Proxy: f.Ctl.GetProxy(proxy),
|
||||
TLSClientConfig: &tls.Config{
|
||||
InsecureSkipVerify: skipVerify,
|
||||
},
|
||||
TLSHandshakeTimeout: f.timeout(),
|
||||
},
|
||||
Impersonation: httpclient.ImpersonationOptions{
|
||||
Mode: httpclient.ImpersonationAuto,
|
||||
Session: f.impSession,
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
panic(fmt.Sprintf("build HTTP client: %v", err))
|
||||
}
|
||||
return client
|
||||
}
|
||||
|
||||
// buildRequest builds a request with task extra headers, the default user
|
||||
// agent and an optional byte range. extra may be nil.
|
||||
func buildRequest(ctx context.Context, extra *ReqExtra, method, u string, body io.Reader, byterange *ByteRange) (*http.Request, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, method, u, body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if extra != nil {
|
||||
for key, value := range extra.Header {
|
||||
req.Header.Set(key, value)
|
||||
}
|
||||
}
|
||||
if req.Header.Get("User-Agent") == "" {
|
||||
req.Header.Set("User-Agent", defaultUserAgent)
|
||||
}
|
||||
if byterange != nil {
|
||||
req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", byterange.Offset, byterange.Offset+byterange.Length-1))
|
||||
}
|
||||
return req, nil
|
||||
}
|
||||
|
||||
// stallReader aborts reads that see no data for the configured timeout.
|
||||
type stallReader struct {
|
||||
rc io.ReadCloser
|
||||
timeout time.Duration
|
||||
}
|
||||
|
||||
func (r *stallReader) Read(p []byte) (int, error) {
|
||||
timer := time.AfterFunc(r.timeout, func() {
|
||||
r.rc.Close()
|
||||
})
|
||||
defer timer.Stop()
|
||||
return r.rc.Read(p)
|
||||
}
|
||||
|
||||
func (r *stallReader) Close() error {
|
||||
return r.rc.Close()
|
||||
}
|
||||
@@ -0,0 +1,112 @@
|
||||
package hls
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"crypto/sha1"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"strconv"
|
||||
)
|
||||
|
||||
// journalFile is the persisted completed-segment ledger. It is bound to the
|
||||
// exact download plan: a journal whose fingerprint does not match the current
|
||||
// plan is discarded instead of being applied to the wrong segments.
|
||||
type journalFile struct {
|
||||
StagingID string `json:"stagingID"`
|
||||
PlanHash string `json:"planHash"`
|
||||
SegmentCount int `json:"segmentCount"`
|
||||
Sizes map[string]int64 `json:"sizes"`
|
||||
}
|
||||
|
||||
// randomStagingID returns a random identifier used to give every task its own
|
||||
// staging folder, stable across app restarts through the persisted state.
|
||||
func randomStagingID() string {
|
||||
buf := make([]byte, 6)
|
||||
if _, err := rand.Read(buf); err != nil {
|
||||
// crypto/rand failing is catastrophic anyway; fall back to address
|
||||
// entropy rather than crashing the fetch.
|
||||
return fmt.Sprintf("%x", sha1.Sum([]byte(fmt.Sprintf("%p", buf))))[:12]
|
||||
}
|
||||
return hex.EncodeToString(buf)
|
||||
}
|
||||
|
||||
// planFingerprint hashes everything that defines the download plan: the media
|
||||
// URL, the ordered segments with their sequence numbers, byte ranges, keys
|
||||
// and init markers. Two tasks resolve to different fingerprints even when
|
||||
// they point at the same playlist URL.
|
||||
func planFingerprint(state *fetcherState) string {
|
||||
h := sha1.New()
|
||||
io.WriteString(h, state.MediaURL+"\n")
|
||||
io.WriteString(h, strconv.Itoa(len(state.Segments))+"\n")
|
||||
for _, seg := range state.Segments {
|
||||
fmt.Fprintf(h, "%d|%s|", seg.Sequence, seg.URI)
|
||||
if seg.Byterange != nil {
|
||||
fmt.Fprintf(h, "%d@%d|", seg.Byterange.Offset, seg.Byterange.Length)
|
||||
}
|
||||
if seg.Key != nil {
|
||||
// The IV is part of the plan: same URI with a rotated IV would
|
||||
// silently decrypt to garbage if the journal were reused.
|
||||
fmt.Fprintf(h, "key=%s iv=%x|", seg.Key.URI, seg.Key.IV)
|
||||
}
|
||||
if seg.IsInit {
|
||||
io.WriteString(h, "init|")
|
||||
}
|
||||
io.WriteString(h, "\n")
|
||||
}
|
||||
return hex.EncodeToString(h.Sum(nil))[:16]
|
||||
}
|
||||
|
||||
// loadJournal reads the journal of a run, discarding anything that cannot be
|
||||
// proven to belong to the current plan: foreign or legacy journals, corrupted
|
||||
// files, and entries whose recorded size no longer matches the plan.
|
||||
func loadJournal(run *hlsRun) error {
|
||||
data, err := os.ReadFile(run.journalPath)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
var file journalFile
|
||||
if err := json.Unmarshal(data, &file); err != nil {
|
||||
// A broken or legacy journal must not kill a restartable download.
|
||||
return nil
|
||||
}
|
||||
if file.StagingID != run.stagingID || file.PlanHash != run.planHash || file.SegmentCount != len(run.segs) {
|
||||
return nil
|
||||
}
|
||||
for key, size := range file.Sizes {
|
||||
idx, err := strconv.ParseInt(key, 10, 64)
|
||||
if err != nil || idx < 0 || idx >= int64(len(run.segs)) || size <= 0 {
|
||||
continue
|
||||
}
|
||||
run.journal[idx] = size
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// saveJournal atomically rewrites the journal through a temp file + rename so
|
||||
// a crash never leaves a half-written ledger behind.
|
||||
func saveJournal(run *hlsRun) error {
|
||||
file := journalFile{
|
||||
StagingID: run.stagingID,
|
||||
PlanHash: run.planHash,
|
||||
SegmentCount: len(run.segs),
|
||||
Sizes: make(map[string]int64, len(run.journal)),
|
||||
}
|
||||
for idx, size := range run.journal {
|
||||
file.Sizes[strconv.FormatInt(idx, 10)] = size
|
||||
}
|
||||
data, err := json.Marshal(file)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tmp := run.journalPath + ".tmp"
|
||||
if err := os.WriteFile(tmp, data, 0666); err != nil {
|
||||
return err
|
||||
}
|
||||
return os.Rename(tmp, run.journalPath)
|
||||
}
|
||||
@@ -0,0 +1,297 @@
|
||||
package hls
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/GopeedLab/gopeed/pkg/base"
|
||||
)
|
||||
|
||||
// The validators below check real media structure, not just byte
|
||||
// concatenation: TS packets are verified on the 188 byte grid with
|
||||
// continuity-counter rules, PAT/PMT presence and PES start codes; fMP4
|
||||
// output is verified by walking the ISO-BMFF box hierarchy.
|
||||
|
||||
func be32(b []byte) int {
|
||||
return int(b[0])<<24 | int(b[1])<<16 | int(b[2])<<8 | int(b[3])
|
||||
}
|
||||
|
||||
// validateTS asserts structural validity of a merged MPEG-2 TS stream.
|
||||
func validateTS(t *testing.T, data []byte) {
|
||||
t.Helper()
|
||||
if len(data) == 0 || len(data)%188 != 0 {
|
||||
t.Fatalf("TS size %d is not a positive multiple of 188", len(data))
|
||||
}
|
||||
type pidState struct{ cc int }
|
||||
ccByPID := map[int]int{}
|
||||
patSeen, pmtSeen, pesSeen := false, false, false
|
||||
for off := 0; off < len(data); off += 188 {
|
||||
pkt := data[off : off+188]
|
||||
if pkt[0] != 0x47 {
|
||||
t.Fatalf("bad TS sync byte at offset %d", off)
|
||||
}
|
||||
pid := (int(pkt[1]&0x1F) << 8) | int(pkt[2])
|
||||
pusi := pkt[1]&0x40 != 0
|
||||
afc := (pkt[3] >> 4) & 0x3
|
||||
if afc == 0 {
|
||||
continue // reserved
|
||||
}
|
||||
idx := 4
|
||||
if afc&0x2 != 0 { // adaptation field present
|
||||
afLen := int(pkt[4])
|
||||
if afLen > 0 && pkt[5]&0x80 != 0 {
|
||||
// discontinuity_indicator: restart the CC expectation
|
||||
delete(ccByPID, pid)
|
||||
}
|
||||
idx += 1 + afLen
|
||||
if afc == 2 {
|
||||
continue // adaptation only, no payload: no CC rule
|
||||
}
|
||||
}
|
||||
if idx > 188 {
|
||||
t.Fatalf("adaptation field overruns packet at offset %d", off)
|
||||
}
|
||||
if pid == 0x1FFF {
|
||||
continue // null packet
|
||||
}
|
||||
payload := pkt[idx:]
|
||||
if pusi && len(payload) >= 3 && pid != 0 && payload[0] == 0 && payload[1] == 0 && payload[2] == 1 {
|
||||
pesSeen = true
|
||||
}
|
||||
if pid == 0 {
|
||||
if ptr := int(payload[0]); 1+ptr < len(payload) && payload[1+ptr] == 0x00 {
|
||||
patSeen = true // program_association_section table_id
|
||||
}
|
||||
} else if pusi && len(payload) >= 2 && payload[0] == 0 && payload[1] == 0x02 {
|
||||
pmtSeen = true // program_map_section table_id
|
||||
}
|
||||
cc := int(pkt[3] & 0x0F)
|
||||
prev, ok := ccByPID[pid]
|
||||
if !ok {
|
||||
ccByPID[pid] = cc
|
||||
continue
|
||||
}
|
||||
// CC increments by one per payload-bearing packet; a repeated value
|
||||
// models the single legal duplicate packet.
|
||||
if cc != (prev+1)%16 && cc != prev {
|
||||
t.Fatalf("continuity error on PID %d: %d after %d at offset %d", pid, cc, prev, off)
|
||||
}
|
||||
ccByPID[pid] = cc
|
||||
}
|
||||
if !patSeen {
|
||||
t.Error("no PAT found in merged TS")
|
||||
}
|
||||
if !pmtSeen {
|
||||
t.Error("no PMT found in merged TS")
|
||||
}
|
||||
if !pesSeen {
|
||||
t.Error("no PES start code found in merged TS")
|
||||
}
|
||||
}
|
||||
|
||||
// walkBoxes iterates the top-level ISO-BMFF boxes of data[off:end].
|
||||
func walkBoxes(t *testing.T, data []byte, off, end int) []string {
|
||||
t.Helper()
|
||||
var types []string
|
||||
for off+8 <= end {
|
||||
size := be32(data[off:])
|
||||
typ := string(data[off+4 : off+8])
|
||||
if size < 8 || off+size > end {
|
||||
t.Fatalf("invalid box %q at offset %d (size %d)", typ, off, size)
|
||||
}
|
||||
types = append(types, typ)
|
||||
off += size
|
||||
}
|
||||
return types
|
||||
}
|
||||
|
||||
// validateFMP4 asserts the merged fMP4 output is init section followed by
|
||||
// well-formed moof+mdat fragments.
|
||||
func validateFMP4(t *testing.T, merged []byte, wantFragments int) {
|
||||
t.Helper()
|
||||
types := walkBoxes(t, merged, 0, len(merged))
|
||||
hasFtyp, hasMoov := false, false
|
||||
fragments := 0
|
||||
expectFragment := false
|
||||
for i, typ := range types {
|
||||
switch typ {
|
||||
case "ftyp":
|
||||
if i != 0 {
|
||||
t.Errorf("ftyp not at stream start")
|
||||
}
|
||||
hasFtyp = true
|
||||
case "moov":
|
||||
if i != 1 {
|
||||
t.Errorf("moov must follow ftyp, found at index %d", i)
|
||||
}
|
||||
hasMoov = true
|
||||
case "styp":
|
||||
expectFragment = true
|
||||
case "moof":
|
||||
if !hasMoov {
|
||||
t.Errorf("fragment before moov")
|
||||
}
|
||||
expectFragment = true
|
||||
case "mdat":
|
||||
if !expectFragment {
|
||||
t.Errorf("mdat outside of a fragment at index %d", i)
|
||||
}
|
||||
expectFragment = false
|
||||
fragments++
|
||||
case "mfra":
|
||||
// optional trailer
|
||||
default:
|
||||
t.Errorf("unexpected top-level box %q", typ)
|
||||
}
|
||||
}
|
||||
if !hasFtyp || !hasMoov {
|
||||
t.Error("merged fMP4 missing ftyp/moov init section")
|
||||
}
|
||||
if fragments != wantFragments {
|
||||
t.Errorf("want %d fragments in merged fMP4, got %d", wantFragments, fragments)
|
||||
}
|
||||
}
|
||||
|
||||
func mediaFixture(t *testing.T, name string) []byte {
|
||||
t.Helper()
|
||||
data, err := os.ReadFile("testdata/media/" + name)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return data
|
||||
}
|
||||
|
||||
func serveFixtureServer(t *testing.T) *httptest.Server {
|
||||
t.Helper()
|
||||
mux := http.NewServeMux()
|
||||
fs := http.FileServer(http.Dir("testdata/media"))
|
||||
mux.Handle("/", fs)
|
||||
return httptest.NewServer(mux)
|
||||
}
|
||||
|
||||
func TestHLSRealMedia_TS(t *testing.T) {
|
||||
seg0 := mediaFixture(t, "sample-00.ts")
|
||||
seg1 := mediaFixture(t, "sample-01.ts")
|
||||
playlist := "#EXTM3U\n" +
|
||||
"#EXTINF:0.5,\nsample-00.ts\n" +
|
||||
"#EXTINF:0.5,\nsample-01.ts\n" +
|
||||
"#EXT-X-ENDLIST\n"
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/video.m3u8", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Write([]byte(playlist))
|
||||
})
|
||||
mux.Handle("/", http.FileServer(http.Dir("testdata/media")))
|
||||
server := httptest.NewServer(mux)
|
||||
defer server.Close()
|
||||
|
||||
f := newTestFetcher(t, testConfig())
|
||||
if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: t.TempDir()}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := f.Start(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := f.Wait(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := readOutput(t, f)
|
||||
if !bytes.Equal(got, append(append([]byte{}, seg0...), seg1...)) {
|
||||
t.Fatal("merged TS is not the exact concatenation of the segments")
|
||||
}
|
||||
validateTS(t, got)
|
||||
}
|
||||
|
||||
func TestHLSRealMedia_FMP4(t *testing.T) {
|
||||
init := mediaFixture(t, "fmp4-init.mp4")
|
||||
frag1 := mediaFixture(t, "fmp4-seg1.m4s")
|
||||
frag2 := mediaFixture(t, "fmp4-seg2.m4s")
|
||||
frag3 := mediaFixture(t, "fmp4-seg3.m4s")
|
||||
playlist := "#EXTM3U\n" +
|
||||
"#EXT-X-MAP:URI=\"fmp4-init.mp4\"\n" +
|
||||
"#EXTINF:0.5,\nfmp4-seg1.m4s\n" +
|
||||
"#EXTINF:0.5,\nfmp4-seg2.m4s\n" +
|
||||
"#EXTINF:0.5,\nfmp4-seg3.m4s\n" +
|
||||
"#EXT-X-ENDLIST\n"
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/video.m3u8", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Write([]byte(playlist))
|
||||
})
|
||||
mux.Handle("/", http.FileServer(http.Dir("testdata/media")))
|
||||
server := httptest.NewServer(mux)
|
||||
defer server.Close()
|
||||
|
||||
f := newTestFetcher(t, testConfig())
|
||||
if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: t.TempDir()}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := f.Start(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := f.Wait(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := readOutput(t, f)
|
||||
want := append(append(append(append([]byte{}, init...), frag1...), frag2...), frag3...)
|
||||
if !bytes.Equal(got, want) {
|
||||
t.Fatal("merged fMP4 is not the exact concatenation of init + fragments")
|
||||
}
|
||||
validateFMP4(t, got, 3)
|
||||
}
|
||||
|
||||
func TestHLSRealMedia_AES128(t *testing.T) {
|
||||
keys := [][]byte{[]byte("key-A-0123456789"), []byte("key-B-9876543210")}
|
||||
iv := []byte("iv-1337-01234567")
|
||||
plain := [][]byte{mediaFixture(t, "sample-00.ts"), mediaFixture(t, "sample-01.ts")}
|
||||
|
||||
playlist := "#EXTM3U\n" +
|
||||
"#EXT-X-MEDIA-SEQUENCE:0\n" +
|
||||
"#EXT-X-KEY:METHOD=AES-128,URI=\"key-0.bin\",IV=0x" + hexEncode(iv) + "\n" +
|
||||
"#EXTINF:0.5,\nenc-0.ts\n" +
|
||||
"#EXT-X-KEY:METHOD=AES-128,URI=\"key-1.bin\"\n" +
|
||||
"#EXTINF:0.5,\nenc-1.ts\n" +
|
||||
"#EXT-X-ENDLIST\n"
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/video.m3u8", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Write([]byte(playlist))
|
||||
})
|
||||
for i, key := range keys {
|
||||
k := key
|
||||
mux.HandleFunc(fmt.Sprintf("/key-%d.bin", i), func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Write(k)
|
||||
})
|
||||
// Segment 0 has an explicit IV, segment 1 derives it from its media
|
||||
// sequence number (1).
|
||||
enc := encryptSegment(plain[i], k, map[bool][]byte{true: iv, false: sequenceIV(int64(i))}[i == 0])
|
||||
mux.HandleFunc(fmt.Sprintf("/enc-%d.ts", i), func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Write(enc)
|
||||
})
|
||||
}
|
||||
server := httptest.NewServer(mux)
|
||||
defer server.Close()
|
||||
|
||||
f := newTestFetcher(t, testConfig())
|
||||
if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: t.TempDir()}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if f.meta.Res.Size != 0 {
|
||||
t.Errorf("encrypted plan must report indeterminate size, got %d", f.meta.Res.Size)
|
||||
}
|
||||
if err := f.Start(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := f.Wait(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := readOutput(t, f)
|
||||
if !bytes.Equal(got, append(append([]byte{}, plain[0]...), plain[1]...)) {
|
||||
t.Fatal("decrypted output mismatch")
|
||||
}
|
||||
validateTS(t, got)
|
||||
if p := f.Progress(); p.TotalDownloaded() != int64(len(got)) {
|
||||
t.Errorf("final progress %d != merged size %d", p.TotalDownloaded(), len(got))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,479 @@
|
||||
package hls
|
||||
|
||||
import (
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"path"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Variant is one rendition entry of a master playlist.
|
||||
type Variant struct {
|
||||
URI string
|
||||
Bandwidth int64
|
||||
AvgBandwidth int64
|
||||
Resolution string
|
||||
Codecs string
|
||||
Name string
|
||||
AudioGroup string // AUDIO attribute: group id of the audio renditions
|
||||
}
|
||||
|
||||
// MediaRendition is one EXT-X-MEDIA entry of a master playlist. Only the
|
||||
// fields needed to detect unsupported track layouts are kept.
|
||||
type MediaRendition struct {
|
||||
Type string
|
||||
GroupID string
|
||||
URI string
|
||||
}
|
||||
|
||||
// MasterPlaylist is a parsed master playlist with its rendition groups.
|
||||
type MasterPlaylist struct {
|
||||
Variants []*Variant
|
||||
Rendition map[string][]*MediaRendition // group id -> all EXT-X-MEDIA members
|
||||
}
|
||||
|
||||
// Key is the decryption key applied to the segments that follow its tag, until
|
||||
// the next EXT-X-KEY tag or the end of the playlist.
|
||||
type Key struct {
|
||||
Method string
|
||||
URI string // absolute URI
|
||||
IV []byte // explicit IV; nil means derive from the media sequence number
|
||||
}
|
||||
|
||||
// ByteRange is an EXT-X-BYTERANGE sub range of the segment resource.
|
||||
type ByteRange struct {
|
||||
Length int64
|
||||
Offset int64
|
||||
}
|
||||
|
||||
// Segment is one downloadable piece of a media playlist. The init section of
|
||||
// an fMP4 stream is represented as a segment with IsInit set.
|
||||
type Segment struct {
|
||||
URI string
|
||||
Sequence int64 // media sequence number, used to derive the default AES IV
|
||||
Byterange *ByteRange
|
||||
Key *Key // nil means unencrypted
|
||||
IsInit bool
|
||||
Gap bool
|
||||
Duration float64
|
||||
Size int64 // exact size in bytes when known (byterange / HEAD probe), 0 when unknown
|
||||
}
|
||||
|
||||
// Media is a parsed media playlist.
|
||||
type Media struct {
|
||||
Segments []*Segment
|
||||
Version int
|
||||
Live bool // no EXT-X-ENDLIST tag
|
||||
IsFMP4 bool // playlist declares an init section via EXT-X-MAP
|
||||
Discontinuity bool // playlist contains EXT-X-DISCONTINUITY
|
||||
}
|
||||
|
||||
const (
|
||||
keyMethodNone = "NONE"
|
||||
keyMethodAES128 = "AES-128"
|
||||
keyMethodSampleAES = "SAMPLE-AES"
|
||||
)
|
||||
|
||||
// IsMasterPlaylist reports whether the content looks like a master playlist.
|
||||
func IsMasterPlaylist(content string) bool {
|
||||
return strings.Contains(content, "#EXT-X-STREAM-INF")
|
||||
}
|
||||
|
||||
func validateM3U8Header(content string) error {
|
||||
for _, line := range strings.Split(content, "\n") {
|
||||
line = strings.TrimSpace(line)
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
if line != "#EXTM3U" {
|
||||
return errors.New("content is not a valid m3u8 playlist")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
return errors.New("empty m3u8 playlist")
|
||||
}
|
||||
|
||||
// ParseMaster parses a master playlist and returns all variants.
|
||||
func ParseMaster(content string, baseURL *url.URL) ([]*Variant, error) {
|
||||
master, err := ParseMasterPlaylist(content, baseURL)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return master.Variants, nil
|
||||
}
|
||||
|
||||
// ParseMasterPlaylist parses a master playlist including its EXT-X-MEDIA
|
||||
// rendition groups.
|
||||
func ParseMasterPlaylist(content string, baseURL *url.URL) (*MasterPlaylist, error) {
|
||||
if err := validateM3U8Header(content); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
master := &MasterPlaylist{Rendition: make(map[string][]*MediaRendition)}
|
||||
var pending *Variant
|
||||
for _, line := range strings.Split(content, "\n") {
|
||||
line = strings.TrimSpace(line)
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
if strings.HasPrefix(line, "#EXT-X-STREAM-INF:") {
|
||||
attrs := parseAttributes(line[len("#EXT-X-STREAM-INF:"):])
|
||||
pending = &Variant{
|
||||
Bandwidth: parseInt64(attrs["BANDWIDTH"]),
|
||||
AvgBandwidth: parseInt64(attrs["AVERAGE-BANDWIDTH"]),
|
||||
Resolution: attrs["RESOLUTION"],
|
||||
Codecs: unquote(attrs["CODECS"]),
|
||||
Name: unquote(attrs["NAME"]),
|
||||
AudioGroup: unquote(attrs["AUDIO"]),
|
||||
}
|
||||
continue
|
||||
}
|
||||
if strings.HasPrefix(line, "#EXT-X-MEDIA:") {
|
||||
attrs := parseAttributes(line[len("#EXT-X-MEDIA:"):])
|
||||
rendition := &MediaRendition{
|
||||
Type: strings.ToUpper(attrs["TYPE"]),
|
||||
GroupID: unquote(attrs["GROUP-ID"]),
|
||||
}
|
||||
// Only an explicit URI means a separate track; resolving an empty
|
||||
// reference would fabricate one out of the playlist URL.
|
||||
if uri := unquote(attrs["URI"]); uri != "" {
|
||||
rendition.URI = resolveURL(baseURL, uri)
|
||||
}
|
||||
if rendition.GroupID != "" {
|
||||
// Keep every member: a group is unsupported as a whole when
|
||||
// ANY of its members carries its own URI.
|
||||
master.Rendition[rendition.GroupID] = append(master.Rendition[rendition.GroupID], rendition)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if pending != nil {
|
||||
if line[0] != '#' {
|
||||
pending.URI = resolveURL(baseURL, line)
|
||||
master.Variants = append(master.Variants, pending)
|
||||
}
|
||||
pending = nil
|
||||
}
|
||||
}
|
||||
if len(master.Variants) == 0 {
|
||||
return nil, errors.New("master playlist has no variants")
|
||||
}
|
||||
return master, nil
|
||||
}
|
||||
|
||||
// checkAudioRendition rejects the selected variant when its AUDIO group
|
||||
// carries a separate audio track: downloading only the video variant would
|
||||
// silently produce a silent file. Groups the variant does not reference are
|
||||
// ignored (variants without an AUDIO attribute are assumed self-contained).
|
||||
func checkAudioRendition(master *MasterPlaylist, best *Variant) error {
|
||||
if best == nil || best.AudioGroup == "" {
|
||||
return nil
|
||||
}
|
||||
for _, rendition := range master.Rendition[best.AudioGroup] {
|
||||
if rendition == nil || rendition.Type != "AUDIO" || rendition.URI == "" {
|
||||
continue
|
||||
}
|
||||
return fmt.Errorf(
|
||||
"selected variant uses a separate audio rendition (group %q), which is not supported: the merged output would have no audio",
|
||||
best.AudioGroup)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// PickBestVariant selects the highest-bandwidth variant.
|
||||
func PickBestVariant(variants []*Variant) *Variant {
|
||||
var best *Variant
|
||||
for _, v := range variants {
|
||||
if best == nil || v.Bandwidth > best.Bandwidth {
|
||||
best = v
|
||||
}
|
||||
}
|
||||
return best
|
||||
}
|
||||
|
||||
// ParseMedia parses a media playlist into an ordered segment plan.
|
||||
func ParseMedia(content string, baseURL *url.URL) (*Media, error) {
|
||||
if err := validateM3U8Header(content); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
media := &Media{Live: true}
|
||||
var currentKey *Key
|
||||
var pendingByterange *ByteRange
|
||||
var pendingGap bool
|
||||
var pendingDuration float64
|
||||
var lastRangeEnd int64
|
||||
var sequence int64
|
||||
var mapIdentity string // normalized identity of the current EXT-X-MAP
|
||||
|
||||
for _, line := range strings.Split(content, "\n") {
|
||||
line = strings.TrimSpace(line)
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
if line[0] != '#' {
|
||||
// Every non-comment line in a media playlist is a segment URI.
|
||||
seg := &Segment{
|
||||
URI: resolveURL(baseURL, line),
|
||||
Sequence: sequence,
|
||||
Byterange: pendingByterange,
|
||||
Key: currentKey,
|
||||
Gap: pendingGap,
|
||||
Duration: pendingDuration,
|
||||
}
|
||||
if pendingByterange != nil {
|
||||
// Byterange segments have exact sizes straight from the playlist.
|
||||
seg.Size = pendingByterange.Length
|
||||
}
|
||||
sequence++
|
||||
pendingByterange = nil
|
||||
pendingGap = false
|
||||
pendingDuration = 0
|
||||
media.Segments = append(media.Segments, seg)
|
||||
continue
|
||||
}
|
||||
|
||||
switch {
|
||||
case strings.HasPrefix(line, "#EXTINF:"):
|
||||
value := line[len("#EXTINF:"):]
|
||||
if idx := strings.Index(value, ","); idx >= 0 {
|
||||
value = value[:idx]
|
||||
}
|
||||
pendingDuration = parseFloat(value)
|
||||
case strings.HasPrefix(line, "#EXT-X-KEY:"):
|
||||
key, err := parseKey(line[len("#EXT-X-KEY:"):], baseURL)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
currentKey = key
|
||||
case strings.HasPrefix(line, "#EXT-X-MAP:"):
|
||||
attrs := parseAttributes(line[len("#EXT-X-MAP:"):])
|
||||
uriAttr := unquote(attrs["URI"])
|
||||
if uriAttr == "" {
|
||||
return nil, errors.New("EXT-X-MAP without URI is not supported")
|
||||
}
|
||||
resolved := resolveURL(baseURL, uriAttr)
|
||||
// The init section identity covers every attribute, not just the
|
||||
// URI: a repeated tag with a different BYTERANGE addresses
|
||||
// different bytes and changes the init section. NUL keeps the
|
||||
// two parts unambiguously separated.
|
||||
identity := resolved + "\x00" + attrs["BYTERANGE"]
|
||||
if mapIdentity != "" {
|
||||
if mapIdentity != identity {
|
||||
return nil, errors.New("changing init sections are not supported")
|
||||
}
|
||||
media.IsFMP4 = true
|
||||
continue
|
||||
}
|
||||
mapIdentity = identity
|
||||
init := &Segment{
|
||||
URI: resolved,
|
||||
Sequence: sequence,
|
||||
IsInit: true,
|
||||
Key: currentKey,
|
||||
}
|
||||
if attrs["BYTERANGE"] != "" {
|
||||
init.Byterange = parseByteRange(attrs["BYTERANGE"], 0)
|
||||
if init.Byterange == nil {
|
||||
return nil, fmt.Errorf("invalid EXT-X-MAP BYTERANGE: %s", attrs["BYTERANGE"])
|
||||
}
|
||||
}
|
||||
if init.Key != nil && init.Key.IV == nil {
|
||||
// RFC 8216 only defines sequence-derived IVs for media
|
||||
// segments; guessing one for the init section produces
|
||||
// corrupt output, so require the playlist to be explicit.
|
||||
return nil, errors.New("encrypted EXT-X-MAP without an explicit IV is not supported")
|
||||
}
|
||||
// The init section is prepended to the segment plan, it takes
|
||||
// the media sequence number of the playlist for IV derivation.
|
||||
media.Segments = append(media.Segments, init)
|
||||
media.IsFMP4 = true
|
||||
case strings.HasPrefix(line, "#EXT-X-BYTERANGE:"):
|
||||
pendingByterange = parseByteRange(line[len("#EXT-X-BYTERANGE:"):], lastRangeEnd)
|
||||
if pendingByterange == nil {
|
||||
return nil, fmt.Errorf("invalid EXT-X-BYTERANGE: %s", line)
|
||||
}
|
||||
case line == "#EXT-X-GAP":
|
||||
pendingGap = true
|
||||
case line == "#EXT-X-DISCONTINUITY":
|
||||
media.Discontinuity = true
|
||||
case line == "#EXT-X-ENDLIST":
|
||||
media.Live = false
|
||||
case strings.HasPrefix(line, "#EXT-X-MEDIA-SEQUENCE:"):
|
||||
sequence = parseInt64(line[len("#EXT-X-MEDIA-SEQUENCE:"):])
|
||||
case strings.HasPrefix(line, "#EXT-X-VERSION:"):
|
||||
media.Version = int(parseInt64(line[len("#EXT-X-VERSION:"):]))
|
||||
}
|
||||
|
||||
if pendingByterange != nil {
|
||||
// Non-segment tags between BYTERANGE and its URI must not reset the
|
||||
// running offset, so it is tracked here after each parse.
|
||||
lastRangeEnd = pendingByterange.Offset + pendingByterange.Length
|
||||
}
|
||||
}
|
||||
|
||||
if len(media.Segments) == 0 {
|
||||
return nil, errors.New("media playlist has no segments")
|
||||
}
|
||||
return media, nil
|
||||
}
|
||||
|
||||
// validateSupported rejects playlist features the byte-concatenation merge
|
||||
// cannot reproduce faithfully. Skipping such features silently would report
|
||||
// success while producing a broken file.
|
||||
func validateSupported(media *Media) error {
|
||||
if media.Live {
|
||||
return errors.New("live streams are not supported yet")
|
||||
}
|
||||
if media.Discontinuity {
|
||||
return errors.New("EXT-X-DISCONTINUITY is not supported: the merged output would have a broken timeline")
|
||||
}
|
||||
for _, seg := range media.Segments {
|
||||
if seg.Gap {
|
||||
return errors.New("EXT-X-GAP is not supported: the missing segment would leave a hole in the merged output")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func parseKey(attrs string, baseURL *url.URL) (*Key, error) {
|
||||
parsed := parseAttributes(attrs)
|
||||
switch method := parsed["METHOD"]; method {
|
||||
case keyMethodNone:
|
||||
return nil, nil
|
||||
case keyMethodAES128:
|
||||
uri := unquote(parsed["URI"])
|
||||
if uri == "" {
|
||||
return nil, errors.New("AES-128 EXT-X-KEY without URI")
|
||||
}
|
||||
if format := unquote(parsed["KEYFORMAT"]); format != "" && format != "identity" {
|
||||
return nil, fmt.Errorf("unsupported EXT-X-KEY KEYFORMAT: %s", format)
|
||||
}
|
||||
key := &Key{Method: keyMethodAES128, URI: resolveURL(baseURL, uri)}
|
||||
if iv, ok := parsed["IV"]; ok {
|
||||
value := strings.TrimPrefix(strings.ToLower(iv), "0x")
|
||||
decoded, err := hex.DecodeString(value)
|
||||
if err != nil || len(decoded) != 16 {
|
||||
return nil, fmt.Errorf("invalid EXT-X-KEY IV: %s", iv)
|
||||
}
|
||||
key.IV = decoded
|
||||
}
|
||||
return key, nil
|
||||
case keyMethodSampleAES:
|
||||
return nil, errors.New("SAMPLE-AES encryption is not supported")
|
||||
default:
|
||||
return nil, fmt.Errorf("unsupported encryption method: %s", method)
|
||||
}
|
||||
}
|
||||
|
||||
func parseByteRange(value string, defaultOffset int64) *ByteRange {
|
||||
lengthPart := value
|
||||
offsetPart := ""
|
||||
if idx := strings.Index(value, "@"); idx >= 0 {
|
||||
lengthPart = value[:idx]
|
||||
offsetPart = value[idx+1:]
|
||||
}
|
||||
length := parseInt64(lengthPart)
|
||||
if length <= 0 {
|
||||
return nil
|
||||
}
|
||||
offset := defaultOffset
|
||||
if offsetPart != "" {
|
||||
offset = parseInt64(offsetPart)
|
||||
if offset < 0 {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return &ByteRange{Length: length, Offset: offset}
|
||||
}
|
||||
|
||||
// resolveURL resolves a playlist URI reference against the playlist URL
|
||||
// following RFC 3986.
|
||||
func resolveURL(baseURL *url.URL, ref string) string {
|
||||
refURL, err := url.Parse(ref)
|
||||
if err != nil {
|
||||
return ref
|
||||
}
|
||||
return baseURL.ResolveReference(refURL).String()
|
||||
}
|
||||
|
||||
// parseAttributes parses an HLS attribute list like FOO="bar",NUM=42.
|
||||
func parseAttributes(s string) map[string]string {
|
||||
attrs := make(map[string]string)
|
||||
var key, val strings.Builder
|
||||
inKey := true
|
||||
inQuote := false
|
||||
flush := func() {
|
||||
name := strings.ToUpper(strings.TrimSpace(key.String()))
|
||||
if name != "" {
|
||||
attrs[name] = strings.TrimSpace(val.String())
|
||||
}
|
||||
key.Reset()
|
||||
val.Reset()
|
||||
inKey = true
|
||||
}
|
||||
for i := 0; i < len(s); i++ {
|
||||
c := s[i]
|
||||
if inKey {
|
||||
if c == '=' {
|
||||
inKey = false
|
||||
} else {
|
||||
key.WriteByte(c)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if c == '"' {
|
||||
inQuote = !inQuote
|
||||
continue
|
||||
}
|
||||
if c == ',' && !inQuote {
|
||||
flush()
|
||||
continue
|
||||
}
|
||||
val.WriteByte(c)
|
||||
}
|
||||
flush()
|
||||
return attrs
|
||||
}
|
||||
|
||||
func unquote(s string) string {
|
||||
s = strings.TrimSpace(s)
|
||||
if len(s) >= 2 && strings.HasPrefix(s, "\"") && strings.HasSuffix(s, "\"") {
|
||||
return s[1 : len(s)-1]
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func parseInt64(s string) int64 {
|
||||
value, _ := strconv.ParseInt(strings.TrimSpace(s), 10, 64)
|
||||
return value
|
||||
}
|
||||
|
||||
func parseFloat(s string) float64 {
|
||||
value, _ := strconv.ParseFloat(strings.TrimSpace(s), 64)
|
||||
return value
|
||||
}
|
||||
|
||||
// OutputExtension returns the container file extension for the playlist.
|
||||
func (m *Media) OutputExtension() string {
|
||||
if m.IsFMP4 {
|
||||
return ".mp4"
|
||||
}
|
||||
return ".ts"
|
||||
}
|
||||
|
||||
// DeriveOutputName builds the merged output file name from a playlist URL.
|
||||
func DeriveOutputName(playlistURL string, media *Media) string {
|
||||
name := ""
|
||||
if u, err := url.Parse(playlistURL); err == nil {
|
||||
name = path.Base(u.Path)
|
||||
name = strings.TrimSuffix(name, path.Ext(name))
|
||||
}
|
||||
if name == "" || name == "/" || name == "." {
|
||||
if u, err := url.Parse(playlistURL); err == nil {
|
||||
name = u.Hostname()
|
||||
}
|
||||
}
|
||||
return name + media.OutputExtension()
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
package hls
|
||||
|
||||
import (
|
||||
_ "embed"
|
||||
"net/url"
|
||||
"testing"
|
||||
)
|
||||
|
||||
//go:embed testdata/olemovienews.m3u8
|
||||
var realWorldPlaylist string
|
||||
|
||||
// TestParseMediaRealWorld guards parsing against a real site playlist
|
||||
// (olemovienews.com VOD, 138 TS segments).
|
||||
func TestParseMediaRealWorld(t *testing.T) {
|
||||
base, err := url.Parse("https://europe.olemovienews.com/ts2/20230125/afAxfjit/mp4/afAxfjit.mp4/index-v1-a1.m3u8")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
media, err := ParseMedia(realWorldPlaylist, base)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if media.Live {
|
||||
t.Error("real VOD playlist should not be live")
|
||||
}
|
||||
if media.IsFMP4 {
|
||||
t.Error("playlist should be detected as TS")
|
||||
}
|
||||
if len(media.Segments) != 138 {
|
||||
t.Errorf("want 138 segments, got %d", len(media.Segments))
|
||||
}
|
||||
first := media.Segments[0]
|
||||
if first.URI != "https://europe.olemovienews.com/ts2/20230125/afAxfjit/mp4/afAxfjit.mp4/seg-1-v1-a1.ts" {
|
||||
t.Errorf("segment URI resolution wrong: %s", first.URI)
|
||||
}
|
||||
if media.Segments[0].Sequence != 1 {
|
||||
t.Errorf("first sequence = %d, want 1", media.Segments[0].Sequence)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,252 @@
|
||||
package hls
|
||||
|
||||
import (
|
||||
"net/url"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func mustURL(t *testing.T, raw string) *url.URL {
|
||||
t.Helper()
|
||||
u, err := url.Parse(raw)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return u
|
||||
}
|
||||
|
||||
func TestParseMediaTS(t *testing.T) {
|
||||
content := "#EXTM3U\n" +
|
||||
"#EXT-X-TARGETDURATION:20\n" +
|
||||
"#EXT-X-VERSION:3\n" +
|
||||
"#EXT-X-MEDIA-SEQUENCE:1\n" +
|
||||
"#EXTINF:20.000,\n" +
|
||||
"seg-1.ts\n" +
|
||||
"#EXTINF:20.000,\n" +
|
||||
"seg-2.ts\n" +
|
||||
"#EXT-X-ENDLIST\n"
|
||||
media, err := ParseMedia(content, mustURL(t, "https://cdn.example.com/vod/index.m3u8"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if media.Live {
|
||||
t.Error("VOD playlist should not be live")
|
||||
}
|
||||
if media.IsFMP4 {
|
||||
t.Error("TS playlist should not be fMP4")
|
||||
}
|
||||
if len(media.Segments) != 2 {
|
||||
t.Fatalf("want 2 segments, got %d", len(media.Segments))
|
||||
}
|
||||
want := "https://cdn.example.com/vod/seg-1.ts"
|
||||
if media.Segments[0].URI != want {
|
||||
t.Errorf("want uri %s, got %s", want, media.Segments[0].URI)
|
||||
}
|
||||
if media.Segments[0].Sequence != 1 {
|
||||
t.Errorf("want sequence 1, got %d", media.Segments[0].Sequence)
|
||||
}
|
||||
if media.Segments[0].Key != nil {
|
||||
t.Error("segment should be unencrypted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseMediaWithoutExtinf(t *testing.T) {
|
||||
// Some playlists list segment URIs without EXTINF lines.
|
||||
content := "#EXTM3U\nseg-1.ts\nseg-2.ts\n#EXT-X-ENDLIST\n"
|
||||
media, err := ParseMedia(content, mustURL(t, "https://cdn.example.com/a/index.m3u8"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(media.Segments) != 2 {
|
||||
t.Fatalf("want 2 segments, got %d", len(media.Segments))
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseMediaLive(t *testing.T) {
|
||||
content := "#EXTM3U\n#EXT-X-TARGETDURATION:6\n#EXTINF:6.0,\nseg-1.ts\n"
|
||||
media, err := ParseMedia(content, mustURL(t, "https://cdn.example.com/live/index.m3u8"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !media.Live {
|
||||
t.Error("playlist without ENDLIST should be live")
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseMediaFMP4(t *testing.T) {
|
||||
content := "#EXTM3U\n" +
|
||||
"#EXT-X-VERSION:6\n" +
|
||||
"#EXT-X-MAP:URI=\"init.mp4\"\n" +
|
||||
"#EXTINF:5.0,\n" +
|
||||
"seg-1.m4s\n" +
|
||||
"#EXT-X-ENDLIST\n"
|
||||
media, err := ParseMedia(content, mustURL(t, "https://cdn.example.com/vod/index.m3u8"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !media.IsFMP4 {
|
||||
t.Error("playlist with MAP should be fMP4")
|
||||
}
|
||||
if len(media.Segments) != 2 {
|
||||
t.Fatalf("want 2 segments (init + 1), got %d", len(media.Segments))
|
||||
}
|
||||
if !media.Segments[0].IsInit {
|
||||
t.Error("first segment should be the init section")
|
||||
}
|
||||
if media.Segments[0].URI != "https://cdn.example.com/vod/init.mp4" {
|
||||
t.Errorf("init uri wrong: %s", media.Segments[0].URI)
|
||||
}
|
||||
if media.OutputExtension() != ".mp4" {
|
||||
t.Errorf("want .mp4, got %s", media.OutputExtension())
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseMediaMultiMapError(t *testing.T) {
|
||||
content := "#EXTM3U\n" +
|
||||
"#EXT-X-MAP:URI=\"init1.mp4\"\n" +
|
||||
"#EXTINF:5.0,\nseg-1.m4s\n" +
|
||||
"#EXT-X-DISCONTINUITY\n" +
|
||||
"#EXT-X-MAP:URI=\"init2.mp4\"\n" +
|
||||
"#EXTINF:5.0,\nseg-2.m4s\n" +
|
||||
"#EXT-X-ENDLIST\n"
|
||||
if _, err := ParseMedia(content, mustURL(t, "https://cdn.example.com/vod/index.m3u8")); err == nil {
|
||||
t.Error("multi-period fMP4 should be rejected")
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseMediaKeyRotation(t *testing.T) {
|
||||
content := "#EXTM3U\n" +
|
||||
"#EXT-X-KEY:METHOD=AES-128,URI=\"key1.bin\",IV=0x00000000000000000000000000000001\n" +
|
||||
"#EXTINF:5.0,\nseg-1.ts\n" +
|
||||
"#EXT-X-KEY:METHOD=AES-128,URI=\"key2.bin\"\n" +
|
||||
"#EXTINF:5.0,\nseg-2.ts\n" +
|
||||
"#EXT-X-KEY:METHOD=NONE\n" +
|
||||
"#EXTINF:5.0,\nseg-3.ts\n" +
|
||||
"#EXT-X-ENDLIST\n"
|
||||
media, err := ParseMedia(content, mustURL(t, "https://cdn.example.com/vod/index.m3u8"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if media.Segments[0].Key == nil || media.Segments[0].Key.URI != "https://cdn.example.com/vod/key1.bin" {
|
||||
t.Fatalf("segment 1 key wrong: %+v", media.Segments[0].Key)
|
||||
}
|
||||
if len(media.Segments[0].Key.IV) != 16 || media.Segments[0].Key.IV[15] != 1 {
|
||||
t.Error("explicit IV not parsed")
|
||||
}
|
||||
if media.Segments[1].Key == nil || media.Segments[1].Key.URI != "https://cdn.example.com/vod/key2.bin" {
|
||||
t.Fatalf("segment 2 key wrong: %+v", media.Segments[1].Key)
|
||||
}
|
||||
if media.Segments[1].Key.IV != nil {
|
||||
t.Error("key without IV should leave IV nil (sequence-derived)")
|
||||
}
|
||||
if media.Segments[2].Key != nil {
|
||||
t.Error("segment 3 should be unencrypted after METHOD=NONE")
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseMediaSampleAESError(t *testing.T) {
|
||||
content := "#EXTM3U\n#EXT-X-KEY:METHOD=SAMPLE-AES,URI=\"skd://key\"\n#EXTINF:5.0,\nseg-1.ts\n"
|
||||
if _, err := ParseMedia(content, mustURL(t, "https://cdn.example.com/vod/index.m3u8")); err == nil {
|
||||
t.Error("SAMPLE-AES should be rejected")
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseMediaByterange(t *testing.T) {
|
||||
content := "#EXTM3U\n" +
|
||||
"#EXT-X-BYTERANGE:1000@0\nseg.ts\n" +
|
||||
"#EXT-X-BYTERANGE:2000\nseg.ts\n" +
|
||||
"#EXT-X-BYTERANGE:3000\nseg.ts\n" +
|
||||
"#EXT-X-ENDLIST\n"
|
||||
media, err := ParseMedia(content, mustURL(t, "https://cdn.example.com/vod/index.m3u8"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ranges := [][2]int64{{0, 1000}, {1000, 2000}, {3000, 3000}}
|
||||
for i, want := range ranges {
|
||||
br := media.Segments[i].Byterange
|
||||
if br == nil {
|
||||
t.Fatalf("segment %d byterange missing", i)
|
||||
}
|
||||
if br.Offset != want[0] || br.Length != want[1] {
|
||||
t.Errorf("segment %d: want offset %d length %d, got %d/%d", i, want[0], want[1], br.Offset, br.Length)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseMediaGap(t *testing.T) {
|
||||
content := "#EXTM3U\n#EXTINF:5.0,\nseg-1.ts\n#EXT-X-GAP\n#EXTINF:5.0,\nseg-2.ts\n#EXT-X-ENDLIST\n"
|
||||
media, err := ParseMedia(content, mustURL(t, "https://cdn.example.com/vod/index.m3u8"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if media.Segments[0].Gap || !media.Segments[1].Gap {
|
||||
t.Error("gap flags wrong")
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseMediaInvalidHeader(t *testing.T) {
|
||||
if _, err := ParseMedia("not a playlist", mustURL(t, "https://cdn.example.com/a.m3u8")); err == nil {
|
||||
t.Error("invalid content should be rejected")
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseMasterPickBest(t *testing.T) {
|
||||
content := "#EXTM3U\n" +
|
||||
"#EXT-X-MEDIA:TYPE=AUDIO,GROUP-ID=\"aud\",NAME=\"en\",DEFAULT=YES,URI=\"audio/index.m3u8\"\n" +
|
||||
"#EXT-X-STREAM-INF:BANDWIDTH=800000,RESOLUTION=640x360\n" +
|
||||
"360p/index.m3u8\n" +
|
||||
"#EXT-X-STREAM-INF:BANDWIDTH=2500000,RESOLUTION=1920x1080,CODECS=\"avc1.640028,mp4a.40.2\"\n" +
|
||||
"1080p/index.m3u8?sign=abc\n" +
|
||||
"#EXT-X-ENDLIST\n"
|
||||
variants, err := ParseMaster(content, mustURL(t, "https://cdn.example.com/vod/master.m3u8"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(variants) != 2 {
|
||||
t.Fatalf("want 2 variants, got %d", len(variants))
|
||||
}
|
||||
best := PickBestVariant(variants)
|
||||
if best.Bandwidth != 2500000 {
|
||||
t.Errorf("want highest bandwidth variant, got %d", best.Bandwidth)
|
||||
}
|
||||
if best.URI != "https://cdn.example.com/vod/1080p/index.m3u8?sign=abc" {
|
||||
t.Errorf("variant uri wrong: %s", best.URI)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveURLQueryNotInherited(t *testing.T) {
|
||||
base := mustURL(t, "https://cdn.example.com/vod/index.m3u8?sign=abc")
|
||||
got := resolveURL(base, "seg-1.ts")
|
||||
if got != "https://cdn.example.com/vod/seg-1.ts" {
|
||||
t.Errorf("relative resolution should follow RFC3986 (no query inheritance), got %s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeriveOutputName(t *testing.T) {
|
||||
media := &Media{}
|
||||
if got := DeriveOutputName("https://cdn.example.com/ts2/a/index-v1-a1.m3u8", media); got != "index-v1-a1.ts" {
|
||||
t.Errorf("want index-v1-a1.ts, got %s", got)
|
||||
}
|
||||
fmp4 := &Media{IsFMP4: true}
|
||||
if got := DeriveOutputName("https://cdn.example.com/ts2/a/index.m3u8", fmp4); got != "index.mp4" {
|
||||
t.Errorf("want index.mp4, got %s", got)
|
||||
}
|
||||
if got := DeriveOutputName("https://cdn.example.com/play?token=x", media); got != "play.ts" {
|
||||
t.Errorf("want play.ts, got %s", got)
|
||||
}
|
||||
if got := DeriveOutputName("https://cdn.example.com/", media); got != "cdn.example.com.ts" {
|
||||
t.Errorf("want host fallback, got %s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseAttributesQuotedComma(t *testing.T) {
|
||||
attrs := parseAttributes(`URI="a,b.ts",IV=0xABCDEF00000000000000000000000001,NAME="x"`)
|
||||
if attrs["URI"] != "a,b.ts" {
|
||||
t.Errorf("quoted comma value wrong: %q", attrs["URI"])
|
||||
}
|
||||
if attrs["IV"] != "0xABCDEF00000000000000000000000001" {
|
||||
t.Errorf("hex value wrong: %q", attrs["IV"])
|
||||
}
|
||||
if attrs["NAME"] != "x" {
|
||||
t.Errorf("quoted value wrong: %q", attrs["NAME"])
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,296 @@
|
||||
package hls
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/GopeedLab/gopeed/pkg/base"
|
||||
)
|
||||
|
||||
func TestFetcherE2E_SameURLTwoTasks(t *testing.T) {
|
||||
const total = 6
|
||||
mux := http.NewServeMux()
|
||||
var playlist strings.Builder
|
||||
playlist.WriteString("#EXTM3U\n")
|
||||
for i := 0; i < total; i++ {
|
||||
playlist.WriteString(fmt.Sprintf("#EXTINF:4,\nseg-%d.ts\n", i))
|
||||
}
|
||||
playlist.WriteString("#EXT-X-ENDLIST\n")
|
||||
mux.HandleFunc("/video.m3u8", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Write([]byte(playlist.String()))
|
||||
})
|
||||
segments := make(map[int][]byte)
|
||||
for i := 0; i < total; i++ {
|
||||
segments[i] = []byte(fmt.Sprintf("SHARED-SEGMENT-%02d", i))
|
||||
idx := i
|
||||
mux.HandleFunc(fmt.Sprintf("/seg-%d.ts", idx), func(w http.ResponseWriter, r *http.Request) {
|
||||
time.Sleep(120 * time.Millisecond)
|
||||
w.Write(segments[idx])
|
||||
})
|
||||
}
|
||||
server := httptest.NewServer(mux)
|
||||
defer server.Close()
|
||||
|
||||
cfg := testConfig()
|
||||
cfg.SegmentConnections = 2
|
||||
cfg.PrefetchContentLength = false
|
||||
dir := t.TempDir()
|
||||
|
||||
// Two tasks for the same playlist URL into the same directory, like a
|
||||
// user duplicating a task. The engine would rename the second output
|
||||
// (AutoRename); mirror that here.
|
||||
f1 := newTestFetcher(t, cfg)
|
||||
if err := f1.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: dir}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f2 := newTestFetcher(t, cfg)
|
||||
if err := f2.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: dir, Name: "video-2.ts"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if f1.state.StagingDir == "" || f1.state.StagingDir == f2.state.StagingDir {
|
||||
t.Fatalf("staging dirs must be distinct per task: %q vs %q", f1.state.StagingDir, f2.state.StagingDir)
|
||||
}
|
||||
|
||||
// Start task 1, let it stage some segments, then pause it.
|
||||
if err := f1.Start(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
time.Sleep(250 * time.Millisecond)
|
||||
if err := f1.Pause(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
staged1, err := os.ReadDir(f1.state.StagingDir)
|
||||
if err != nil || len(staged1) == 0 {
|
||||
t.Fatalf("task 1 should have staged segments, dir: %s", f1.state.StagingDir)
|
||||
}
|
||||
|
||||
// Task 2 must complete without touching task 1's staging data...
|
||||
if err := f2.Start(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := f2.Wait(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
stillThere, err := os.ReadDir(f1.state.StagingDir)
|
||||
if err != nil || len(stillThere) != len(staged1) {
|
||||
t.Fatalf("task 2 disturbed task 1 staging: had %d entries, now %d", len(staged1), len(stillThere))
|
||||
}
|
||||
|
||||
// ...and task 1 resumes to a complete, correct output of its own.
|
||||
if err := f1.Start(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := f1.Wait(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var want []byte
|
||||
for i := 0; i < total; i++ {
|
||||
want = append(want, segments[i]...)
|
||||
}
|
||||
if !bytes.Equal(readOutput(t, f1), want) || !bytes.Equal(readOutput(t, f2), want) {
|
||||
t.Error("same-URL task outputs mismatch")
|
||||
}
|
||||
entries, _ := os.ReadDir(dir)
|
||||
if len(entries) != 2 {
|
||||
names := make([]string, 0, len(entries))
|
||||
for _, e := range entries {
|
||||
names = append(names, e.Name())
|
||||
}
|
||||
t.Errorf("expected only the two outputs to remain, got %v", names)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetcherE2E_ForeignJournalDiscarded(t *testing.T) {
|
||||
const total = 3
|
||||
var served atomic.Int64
|
||||
mux := http.NewServeMux()
|
||||
var playlist strings.Builder
|
||||
playlist.WriteString("#EXTM3U\n")
|
||||
for i := 0; i < total; i++ {
|
||||
playlist.WriteString(fmt.Sprintf("#EXTINF:4,\nseg-%d.ts\n", i))
|
||||
}
|
||||
playlist.WriteString("#EXT-X-ENDLIST\n")
|
||||
mux.HandleFunc("/video.m3u8", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Write([]byte(playlist.String()))
|
||||
})
|
||||
for i := 0; i < total; i++ {
|
||||
mux.HandleFunc(fmt.Sprintf("/seg-%d.ts", i), func(w http.ResponseWriter, r *http.Request) {
|
||||
served.Add(1)
|
||||
w.Write([]byte("DATA"))
|
||||
})
|
||||
}
|
||||
server := httptest.NewServer(mux)
|
||||
defer server.Close()
|
||||
|
||||
cfg := testConfig()
|
||||
cfg.PrefetchContentLength = false
|
||||
f := newTestFetcher(t, cfg)
|
||||
if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: t.TempDir()}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Plant a journal that does not belong to this plan (wrong fingerprint).
|
||||
tempDir := f.resolveStagingDirLocked()
|
||||
if err := os.MkdirAll(tempDir, 0777); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
foreign := journalFile{
|
||||
StagingID: "some-other-task",
|
||||
PlanHash: "deadbeef",
|
||||
SegmentCount: total,
|
||||
Sizes: map[string]int64{"0": 4, "1": 4, "2": 4},
|
||||
}
|
||||
data, _ := json.Marshal(foreign)
|
||||
if err := os.WriteFile(filepath.Join(tempDir, "journal.json"), data, 0666); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if err := f.Start(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := f.Wait(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := served.Load(); got != total {
|
||||
t.Errorf("foreign journal must be discarded, want %d downloads, got %d", total, got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetcherE2E_MergeFailurePreservesStaging(t *testing.T) {
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/video.m3u8", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Write([]byte("#EXTM3U\n#EXTINF:4,\na.ts\n#EXTINF:4,\nb.ts\n#EXT-X-ENDLIST\n"))
|
||||
})
|
||||
mux.HandleFunc("/a.ts", func(w http.ResponseWriter, r *http.Request) { w.Write([]byte("AAAA")) })
|
||||
mux.HandleFunc("/b.ts", func(w http.ResponseWriter, r *http.Request) { w.Write([]byte("BBBB")) })
|
||||
server := httptest.NewServer(mux)
|
||||
defer server.Close()
|
||||
|
||||
f := newTestFetcher(t, testConfig())
|
||||
if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: t.TempDir()}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
run := &hlsRun{
|
||||
ctx: context.Background(),
|
||||
cancel: func() {},
|
||||
done: make(chan struct{}),
|
||||
segs: f.state.Segments,
|
||||
outputPath: f.meta.SingleFilepath(),
|
||||
stagingID: f.state.StagingID,
|
||||
planHash: planFingerprint(f.state),
|
||||
tempDir: f.resolveStagingDirLocked(),
|
||||
keyCache: make(map[string][]byte),
|
||||
journal: make(map[int64]int64),
|
||||
}
|
||||
run.journalPath = filepath.Join(run.tempDir, "journal.json")
|
||||
if err := os.MkdirAll(run.tempDir, 0777); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(segmentPath(run.tempDir, 0), []byte("AAAA"), 0666); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Segment 1 is missing on disk: the merge must fail without deleting the
|
||||
// staged data or leaving a partial output behind.
|
||||
if err := f.merge(run); err == nil {
|
||||
t.Fatal("merge with a missing segment should fail")
|
||||
}
|
||||
if _, err := os.Stat(segmentPath(run.tempDir, 0)); err != nil {
|
||||
t.Fatal("merge failure must preserve staged segments")
|
||||
}
|
||||
entries, _ := os.ReadDir(filepath.Dir(run.outputPath))
|
||||
for _, e := range entries {
|
||||
if strings.HasSuffix(e.Name(), ".part") {
|
||||
t.Errorf("failed merge left a partial output behind: %s", e.Name())
|
||||
}
|
||||
}
|
||||
|
||||
// Recover the segment: the retried merge succeeds.
|
||||
if err := os.WriteFile(segmentPath(run.tempDir, 1), []byte("BBBB"), 0666); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := f.merge(run); err != nil {
|
||||
t.Fatalf("retried merge failed: %v", err)
|
||||
}
|
||||
got, err := os.ReadFile(run.outputPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if string(got) != "AAAABBBB" {
|
||||
t.Errorf("retried merge output mismatch: %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetcherE2E_ReMergeOverExistingOutput(t *testing.T) {
|
||||
const total = 2
|
||||
mux := http.NewServeMux()
|
||||
var playlist strings.Builder
|
||||
playlist.WriteString("#EXTM3U\n")
|
||||
for i := 0; i < total; i++ {
|
||||
playlist.WriteString(fmt.Sprintf("#EXTINF:4,\nseg-%d.ts\n", i))
|
||||
}
|
||||
playlist.WriteString("#EXT-X-ENDLIST\n")
|
||||
mux.HandleFunc("/video.m3u8", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Write([]byte(playlist.String()))
|
||||
})
|
||||
for i := 0; i < total; i++ {
|
||||
mux.HandleFunc(fmt.Sprintf("/seg-%d.ts", i), func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Write([]byte("DATA"))
|
||||
})
|
||||
}
|
||||
server := httptest.NewServer(mux)
|
||||
defer server.Close()
|
||||
|
||||
f := newTestFetcher(t, testConfig())
|
||||
if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: t.TempDir()}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
tempDir := f.resolveStagingDirLocked()
|
||||
if err := os.MkdirAll(tempDir, 0777); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Simulate the crash window after a completed rename but before the task
|
||||
// was marked done: a stale output file, a complete journal and all
|
||||
// segments staged.
|
||||
stale := filepath.Join(filepath.Dir(f.meta.SingleFilepath()), f.state.OutputName)
|
||||
if err := os.WriteFile(stale, []byte("STALE-OUTPUT"), 0666); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
jr := &hlsRun{
|
||||
segs: f.state.Segments,
|
||||
stagingID: f.state.StagingID,
|
||||
planHash: planFingerprint(f.state),
|
||||
journalPath: filepath.Join(tempDir, "journal.json"),
|
||||
journal: map[int64]int64{0: 4, 1: 4},
|
||||
}
|
||||
if err := saveJournal(jr); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for i := 0; i < total; i++ {
|
||||
if err := os.WriteFile(segmentPath(tempDir, int64(i)), []byte("DATA"), 0666); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
if err := f.Start(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := f.Wait(); err != nil {
|
||||
t.Fatalf("re-merge over an existing output failed: %v", err)
|
||||
}
|
||||
got, err := os.ReadFile(stale)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if string(got) != "DATADATA" {
|
||||
t.Errorf("stale output was not replaced: %q", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,146 @@
|
||||
package hls
|
||||
|
||||
import (
|
||||
"net/url"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func mustParseURL(t *testing.T, raw string) *url.URL {
|
||||
t.Helper()
|
||||
u, err := url.Parse(raw)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return u
|
||||
}
|
||||
|
||||
// subsetCase is a media playlist plus the error substring the pipeline must
|
||||
// reject it with; wantOK playlists must parse AND validate cleanly.
|
||||
type subsetCase struct {
|
||||
name string
|
||||
playlist string
|
||||
wantErr string
|
||||
}
|
||||
|
||||
func TestSubsetRejections(t *testing.T) {
|
||||
cases := []subsetCase{
|
||||
{
|
||||
name: "gap",
|
||||
playlist: "#EXTM3U\n#EXTINF:4,\na.ts\n#EXT-X-GAP\n#EXTINF:4,\nb.ts\n#EXT-X-ENDLIST\n",
|
||||
wantErr: "EXT-X-GAP",
|
||||
},
|
||||
{
|
||||
name: "gap-before-first-segment",
|
||||
playlist: "#EXTM3U\n#EXT-X-GAP\n#EXTINF:4,\na.ts\n#EXT-X-ENDLIST\n",
|
||||
wantErr: "EXT-X-GAP",
|
||||
},
|
||||
{
|
||||
name: "discontinuity",
|
||||
playlist: "#EXTM3U\n#EXTINF:4,\na.ts\n#EXT-X-DISCONTINUITY\n#EXTINF:4,\nb.ts\n#EXT-X-ENDLIST\n",
|
||||
wantErr: "EXT-X-DISCONTINUITY",
|
||||
},
|
||||
{
|
||||
name: "live",
|
||||
playlist: "#EXTM3U\n#EXTINF:4,\na.ts\n",
|
||||
wantErr: "live",
|
||||
},
|
||||
{
|
||||
name: "keyformat",
|
||||
playlist: "#EXTM3U\n#EXT-X-KEY:METHOD=AES-128,URI=\"k\",KEYFORMAT=\"com.apple.streamingkeydelivery\"\n#EXTINF:4,\na.ts\n#EXT-X-ENDLIST\n",
|
||||
wantErr: "KEYFORMAT",
|
||||
},
|
||||
{
|
||||
name: "encrypted-map-without-iv",
|
||||
playlist: "#EXTM3U\n#EXT-X-KEY:METHOD=AES-128,URI=\"k\"\n#EXT-X-MAP:URI=\"init.mp4\"\n#EXTINF:4,\na.m4s\n#EXT-X-ENDLIST\n",
|
||||
wantErr: "explicit IV",
|
||||
},
|
||||
{
|
||||
name: "map-change-byterange",
|
||||
playlist: "#EXTM3U\n#EXT-X-MAP:URI=\"init.mp4\",BYTERANGE=\"100@0\"\n#EXTINF:4,\na.m4s\n#EXT-X-MAP:URI=\"init.mp4\",BYTERANGE=\"100@100\"\n#EXTINF:4,\nb.m4s\n#EXT-X-ENDLIST\n",
|
||||
wantErr: "changing init sections",
|
||||
},
|
||||
{
|
||||
name: "map-change-uri",
|
||||
playlist: "#EXTM3U\n#EXT-X-MAP:URI=\"init1.mp4\"\n#EXTINF:4,\na.m4s\n#EXT-X-MAP:URI=\"init2.mp4\"\n#EXTINF:4,\nb.m4s\n#EXT-X-ENDLIST\n",
|
||||
wantErr: "changing init sections",
|
||||
},
|
||||
}
|
||||
baseURL := mustParseURL(t, "https://cdn.example.com/vod/index.m3u8")
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
media, parseErr := ParseMedia(tc.playlist, baseURL)
|
||||
if parseErr != nil {
|
||||
if !strings.Contains(parseErr.Error(), tc.wantErr) {
|
||||
t.Fatalf("parse error %q does not mention %q", parseErr, tc.wantErr)
|
||||
}
|
||||
return
|
||||
}
|
||||
err := validateSupported(media)
|
||||
if err == nil {
|
||||
t.Fatalf("playlist should be rejected with %q", tc.wantErr)
|
||||
}
|
||||
if !strings.Contains(err.Error(), tc.wantErr) {
|
||||
t.Fatalf("error %q does not mention %q", err, tc.wantErr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSubsetRepeatedSameMapAllowed(t *testing.T) {
|
||||
playlist := "#EXTM3U\n" +
|
||||
"#EXT-X-MAP:URI=\"init.mp4\",BYTERANGE=\"100@0\"\n" +
|
||||
"#EXTINF:4,\na.m4s\n" +
|
||||
"#EXT-X-MAP:URI=\"init.mp4\",BYTERANGE=\"100@0\"\n" +
|
||||
"#EXTINF:4,\nb.m4s\n" +
|
||||
"#EXT-X-ENDLIST\n"
|
||||
media, err := ParseMedia(playlist, mustParseURL(t, "https://cdn.example.com/vod/index.m3u8"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := validateSupported(media); err != nil {
|
||||
t.Fatalf("identical repeated MAP must be allowed: %v", err)
|
||||
}
|
||||
if len(media.Segments) != 3 { // init + 2 segments, no duplicate init
|
||||
t.Fatalf("want 3 segments (init + 2), got %d", len(media.Segments))
|
||||
}
|
||||
}
|
||||
|
||||
func TestMasterAudioRenditionDetection(t *testing.T) {
|
||||
masterURL := mustParseURL(t, "https://cdn.example.com/master.m3u8")
|
||||
|
||||
separateAudio := "#EXTM3U\n" +
|
||||
"#EXT-X-MEDIA:TYPE=AUDIO,GROUP-ID=\"aud\",NAME=\"en\",DEFAULT=YES,URI=\"audio/index.m3u8\"\n" +
|
||||
"#EXT-X-STREAM-INF:BANDWIDTH=1000000,AUDIO=\"aud\"\nvideo/index.m3u8\n"
|
||||
master, err := ParseMasterPlaylist(separateAudio, masterURL)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := checkAudioRendition(master, PickBestVariant(master.Variants)); err == nil {
|
||||
t.Fatal("selected variant with a separate audio rendition must be rejected")
|
||||
}
|
||||
|
||||
// A group the variant does not reference must not cause a false positive.
|
||||
unrelatedGroup := "#EXTM3U\n" +
|
||||
"#EXT-X-MEDIA:TYPE=AUDIO,GROUP-ID=\"aud\",NAME=\"en\",DEFAULT=YES,URI=\"audio/index.m3u8\"\n" +
|
||||
"#EXT-X-STREAM-INF:BANDWIDTH=1000000\nvideo/index.m3u8\n"
|
||||
master, err = ParseMasterPlaylist(unrelatedGroup, masterURL)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := checkAudioRendition(master, PickBestVariant(master.Variants)); err != nil {
|
||||
t.Fatalf("unreferenced audio group must not be rejected: %v", err)
|
||||
}
|
||||
|
||||
// A rendition group without URIs (muxed tracks) is fine.
|
||||
groupWithoutURI := "#EXTM3U\n" +
|
||||
"#EXT-X-MEDIA:TYPE=AUDIO,GROUP-ID=\"aud\",NAME=\"en\",DEFAULT=YES\n" +
|
||||
"#EXT-X-STREAM-INF:BANDWIDTH=1000000,AUDIO=\"aud\"\nvideo/index.m3u8\n"
|
||||
master, err = ParseMasterPlaylist(groupWithoutURI, masterURL)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := checkAudioRendition(master, PickBestVariant(master.Variants)); err != nil {
|
||||
t.Fatalf("audio group without URI must not be rejected: %v", err)
|
||||
}
|
||||
}
|
||||
+10
@@ -0,0 +1,10 @@
|
||||
Media fixtures generated locally with FFmpeg 7.0.2 (testsrc + sine sources,
|
||||
no third-party content):
|
||||
|
||||
- sample-00.ts / sample-01.ts: one continuous MPEG-2 TS encode split at a
|
||||
packet boundary, so continuity counters and PCR are consistent across the
|
||||
segment join exactly like segments cut from a single real-time stream.
|
||||
- fmp4-init.mp4: ftyp+moov init section of a fragmented MP4.
|
||||
- fmp4-seg1.m4s .. fmp4-seg3.m4s: moof+mdat media fragments (styp omitted by
|
||||
the muxer, which is legal) split at moof boundaries from one fragmented MP4
|
||||
(-movflags +frag_keyframe+empty_moov+default_base_moof -frag_duration 500000).
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
+283
@@ -0,0 +1,283 @@
|
||||
#EXTM3U
|
||||
#EXT-X-TARGETDURATION:20
|
||||
#EXT-X-ALLOW-CACHE:YES
|
||||
#EXT-X-PLAYLIST-TYPE:VOD
|
||||
#EXT-X-VERSION:3
|
||||
#EXT-X-MEDIA-SEQUENCE:1
|
||||
#EXTINF:20.000,
|
||||
seg-1-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-2-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-3-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-4-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-5-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-6-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-7-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-8-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-9-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-10-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-11-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-12-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-13-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-14-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-15-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-16-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-17-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-18-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-19-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-20-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-21-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-22-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-23-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-24-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-25-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-26-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-27-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-28-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-29-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-30-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-31-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-32-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-33-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-34-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-35-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-36-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-37-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-38-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-39-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-40-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-41-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-42-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-43-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-44-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-45-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-46-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-47-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-48-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-49-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-50-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-51-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-52-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-53-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-54-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-55-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-56-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-57-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-58-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-59-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-60-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-61-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-62-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-63-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-64-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-65-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-66-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-67-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-68-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-69-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-70-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-71-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-72-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-73-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-74-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-75-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-76-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-77-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-78-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-79-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-80-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-81-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-82-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-83-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-84-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-85-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-86-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-87-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-88-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-89-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-90-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-91-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-92-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-93-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-94-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-95-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-96-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-97-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-98-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-99-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-100-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-101-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-102-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-103-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-104-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-105-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-106-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-107-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-108-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-109-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-110-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-111-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-112-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-113-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-114-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-115-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-116-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-117-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-118-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-119-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-120-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-121-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-122-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-123-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-124-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-125-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-126-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-127-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-128-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-129-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-130-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-131-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-132-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-133-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-134-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-135-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-136-v1-a1.ts
|
||||
#EXTINF:20.000,
|
||||
seg-137-v1-a1.ts
|
||||
#EXTINF:4.111,
|
||||
seg-138-v1-a1.ts
|
||||
#EXT-X-ENDLIST
|
||||
@@ -1,5 +1,8 @@
|
||||
package http
|
||||
|
||||
// DefaultUserAgent is shared by HTTP downloads and extension HTTP requests.
|
||||
const DefaultUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36"
|
||||
|
||||
type config struct {
|
||||
UserAgent string `json:"userAgent"`
|
||||
Connections int `json:"connections"`
|
||||
|
||||
@@ -448,6 +448,7 @@ func (f *Fetcher) Resolve(req *base.Request, opts *base.Options) error {
|
||||
if file.Name == "" || file.Name == "/" || file.Name == "." {
|
||||
file.Name = httpReq.URL.Hostname()
|
||||
}
|
||||
file.Name = appendFilenameExtension(file.Name, resp.Header.Get("Content-Type"))
|
||||
|
||||
res.Files = append(res.Files, file)
|
||||
f.meta.Res = res
|
||||
|
||||
@@ -72,7 +72,7 @@ func (fm *FetcherManager) AutoRename() bool {
|
||||
|
||||
func (fm *FetcherManager) DefaultConfig() any {
|
||||
return &config{
|
||||
UserAgent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36",
|
||||
UserAgent: DefaultUserAgent,
|
||||
Connections: 16,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -221,6 +221,30 @@ func TestFetcher_Resolve(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
func TestFetcher_ResolveFilenameFromContentType(t *testing.T) {
|
||||
server := httptest.NewServer(gohttp.HandlerFunc(func(w gohttp.ResponseWriter, r *gohttp.Request) {
|
||||
w.Header().Set("Content-Type", "image/webp")
|
||||
w.WriteHeader(gohttp.StatusOK)
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
fetcher := buildFetcher()
|
||||
defer fetcher.Pause()
|
||||
|
||||
const opaqueName = "OIP-C.u5nEXTXND3PJF0gNTgw5YgHaFP"
|
||||
err := fetcher.Resolve(&base.Request{URL: server.URL + "/" + opaqueName}, &base.Options{
|
||||
Name: test.DownloadName,
|
||||
Path: test.Dir,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if got := fetcher.meta.Res.Files[0].Name; got != opaqueName+".webp" {
|
||||
t.Fatalf("resolved filename = %q, want %q", got, opaqueName+".webp")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetcher_ResolveWithHostHeader(t *testing.T) {
|
||||
listener := test.StartTestHostHeaderServer()
|
||||
defer listener.Close()
|
||||
|
||||
@@ -136,3 +136,52 @@ func TestUnescapeHTMLEntities(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestAppendFilenameExtension(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
filename string
|
||||
contentType string
|
||||
want string
|
||||
}{
|
||||
{
|
||||
name: "adds extension from content type",
|
||||
filename: "download",
|
||||
contentType: "image/png",
|
||||
want: "download.png",
|
||||
},
|
||||
{
|
||||
name: "parses MIME parameters",
|
||||
filename: "download",
|
||||
contentType: "image/png; charset=binary",
|
||||
want: "download.png",
|
||||
},
|
||||
{
|
||||
name: "preserves existing extension",
|
||||
filename: "download.zip",
|
||||
contentType: "application/zip",
|
||||
want: "download.zip",
|
||||
},
|
||||
{
|
||||
name: "ignores invalid content type",
|
||||
filename: "download",
|
||||
contentType: "not a content type",
|
||||
want: "download",
|
||||
},
|
||||
{
|
||||
name: "ignores unknown content type",
|
||||
filename: "download",
|
||||
contentType: "application/x-gopeed-unknown",
|
||||
want: "download",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := appendFilenameExtension(tt.filename, tt.contentType)
|
||||
if got != tt.want {
|
||||
t.Errorf("appendFilenameExtension() = %q, want %q", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
"net/http"
|
||||
"net/http/cookiejar"
|
||||
"net/url"
|
||||
"path"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
@@ -314,6 +315,28 @@ func parseFilename(contentDisposition string) string {
|
||||
return parseFilenameFallback(contentDisposition)
|
||||
}
|
||||
|
||||
// appendFilenameExtension adds a MIME-derived extension when the resolved
|
||||
// filename has no usable extension. Some image hosts use opaque path segments
|
||||
// that look like extensions, even though the response Content-Type is the
|
||||
// only reliable source for the file type.
|
||||
func appendFilenameExtension(filename, contentType string) string {
|
||||
mediaType, _, err := mime.ParseMediaType(contentType)
|
||||
if err != nil {
|
||||
return filename
|
||||
}
|
||||
|
||||
extensions, err := mime.ExtensionsByType(mediaType)
|
||||
if err != nil || len(extensions) == 0 {
|
||||
return filename
|
||||
}
|
||||
|
||||
ext := path.Ext(filename)
|
||||
if len(ext) > 1 && len(ext) <= 20 {
|
||||
return filename
|
||||
}
|
||||
return filename + extensions[0]
|
||||
}
|
||||
|
||||
// parseFilenameExtended handles RFC 5987 extended notation (filename*=)
|
||||
// Format: filename*=charset'language'value (e.g., UTF-8”%E6%B5%8B%E8%AF%95.zip)
|
||||
func parseFilenameExtended(cd string) string {
|
||||
|
||||
@@ -0,0 +1,62 @@
|
||||
# Extension browser host configuration
|
||||
|
||||
The downloader wraps each provider with the extension's stable identity before
|
||||
constructing the runtime. `OpenOptions.ProfileID`, `DataPath`, and `ProxyURL` are
|
||||
host-only fields. `parseOpenOptions` never reads them from extension JavaScript.
|
||||
The extension API is unchanged. The Flutter host uses the fork's public
|
||||
`WebViewProfile`, `InAppWebViewSettings.profileId`, and profile-scoped
|
||||
`CookieManager` APIs; the library has no Gopeed-specific method-channel bridge.
|
||||
Only the Gopeed extension JavaScript boundary keeps these options host-only.
|
||||
|
||||
Profiles are persistent and shared across invocations/pages of the same extension.
|
||||
Different identities use different native stores. Go providers use a SHA-256-based
|
||||
path under `<StorageDir>/webview`; Apple RPC hosts use a stable UUID data store;
|
||||
Android uses a named WebView profile. Cookie operations select that same store,
|
||||
including `clearCookies`. Closing an execution closes its pages, not its profile.
|
||||
Uninstall closes the extension's pages, prevents its old runtimes from opening
|
||||
new pages, and removes its native profile before deleting the extension record.
|
||||
Cleanup failures abort uninstall and can be retried. Reinstallation starts with
|
||||
empty browser data; an ordinary extension update keeps the existing profile.
|
||||
Windows/Linux delete the profile directory; Apple removes the named WebKit store.
|
||||
Android cannot delete a profile already loaded by the current process: when
|
||||
DELETE_BROWSING_DATA is supported it clears all site data immediately, and records
|
||||
an empty-profile deletion in SharedPreferences for the next process start. A
|
||||
reinstallation may reuse that cleared profile and cancels the pending deletion.
|
||||
If the full-data clearing API is unavailable, uninstall fails with a restart
|
||||
instruction; the pending profile is deleted at startup before it can be loaded.
|
||||
No removal operation is exposed through extension JavaScript.
|
||||
|
||||
Existing shared browser data is not copied into every extension's new profile.
|
||||
|
||||
The downloader lazily owns one loopback forward proxy with HTTP and SOCKS5
|
||||
listeners. Windows/Linux/Android use HTTP; Apple uses SOCKS5 because its HTTP
|
||||
CONNECT configuration may bypass plain HTTP. The forwarder reads the current
|
||||
Gopeed proxy config for each new request/tunnel and supports direct, system,
|
||||
HTTP(S) upstream proxies with Basic authentication, and authenticated SOCKS5.
|
||||
Credentials remain in Go and are never sent over the WebView RPC.
|
||||
|
||||
Changes affect new requests/connections. Existing CONNECT/SOCKS tunnels keep their
|
||||
route until closed; opening another page in the same profile does not reset it.
|
||||
SOCKS carries host/port rather than a full URL: system proxy/PAC selection for
|
||||
Apple is host-based, with port 80 treated as HTTP and other ports as HTTPS.
|
||||
Path-dependent PAC rules cannot be reproduced from a SOCKS connection.
|
||||
|
||||
Apple requires macOS 14 / iOS 17. Android checks MULTI_PROFILE and PROXY_OVERRIDE.
|
||||
Unsupported hosts fail explicitly instead of silently sharing browser storage or
|
||||
ignoring the proxy.
|
||||
|
||||
## Validation
|
||||
|
||||
- `go test -race ./internal/webview/proxy ./pkg/download/engine/webview`
|
||||
- `go test ./pkg/download -run 'TestWebViewProfileBound|TestDownloader_ExtensionRuntimeWebView'`
|
||||
- Windows/Linux: `go test -tags webview -v ./internal/webview/goprovider` (Linux: run under `xvfb-run -a`).
|
||||
- macOS: start the normal Flutter app, then run `go test -tags webview -v ./internal/webview/rpcprovider` with its socket address. See `rpcprovider/README.md`.
|
||||
- In webview_go: `go test -tags webview_integration -v -count=1 .` (Linux: run under `xvfb-run -a`).
|
||||
|
||||
The shared provider contract covers HTTP/HTTPS proxy routing, concurrent pages,
|
||||
HttpOnly cookies, LocalStorage, IndexedDB, profile-scoped clearing, reopening,
|
||||
and removal without affecting another profile. HTTPS routing is checked with a
|
||||
rejecting upstream tunnel; no test-only certificate bypass is added to the app.
|
||||
Go proxy unit tests separately cover successful TLS forwarding and authentication.
|
||||
These cases run in the existing `test.yml` WebView jobs; existing mobile build jobs
|
||||
compile the Android/iOS bridge. No separate Flutter test application is required.
|
||||
@@ -0,0 +1,47 @@
|
||||
//go:build linux && cgo
|
||||
|
||||
package goprovider
|
||||
|
||||
/*
|
||||
#cgo pkg-config: gtk+-3.0
|
||||
#include <gtk/gtk.h>
|
||||
static void pumpGTK(void) {
|
||||
// Bound each batch so Go creation/removal requests cannot be starved.
|
||||
for (int i = 0; i < 64 && g_main_context_pending(NULL); ++i)
|
||||
g_main_context_iteration(NULL, FALSE);
|
||||
}
|
||||
*/
|
||||
import "C"
|
||||
|
||||
import (
|
||||
"runtime"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
var gtkThreadOnce sync.Once
|
||||
var gtkThreadTasks = make(chan func())
|
||||
|
||||
func RunMainThreadLoop(run func() int) int { return run() }
|
||||
|
||||
func postMainThreadTask(task func()) bool {
|
||||
gtkThreadOnce.Do(func() {
|
||||
go func() {
|
||||
runtime.LockOSThread()
|
||||
// WebKit must always use its original UI thread, including after
|
||||
// the last page closes. A page cannot stop this shared event loop.
|
||||
ticker := time.NewTicker(10 * time.Millisecond)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case task := <-gtkThreadTasks:
|
||||
task()
|
||||
case <-ticker.C:
|
||||
}
|
||||
C.pumpGTK()
|
||||
}
|
||||
}()
|
||||
})
|
||||
gtkThreadTasks <- task
|
||||
return true
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
//go:build !darwin
|
||||
//go:build !darwin && (!linux || !cgo)
|
||||
|
||||
package goprovider
|
||||
|
||||
|
||||
@@ -105,7 +105,15 @@ func (p *pageWrapper) start() error {
|
||||
defer runtimepkg.UnlockOSThread()
|
||||
|
||||
var w webview.WebView
|
||||
if p.opts.Headless {
|
||||
if p.opts.DataPath != "" {
|
||||
var err error
|
||||
w, err = webview.NewWithOptions(webview.Options{Debug: p.opts.Debug, Headless: p.opts.Headless, DataPath: p.opts.DataPath, ProxyURL: p.opts.ProxyURL})
|
||||
if err != nil {
|
||||
p.ready <- err
|
||||
close(p.done)
|
||||
return
|
||||
}
|
||||
} else if p.opts.Headless {
|
||||
w = webview.NewHeadless(p.opts.Debug)
|
||||
} else {
|
||||
w = webview.New(p.opts.Debug)
|
||||
@@ -127,12 +135,18 @@ func (p *pageWrapper) start() error {
|
||||
if err := w.Bind(p.callbackName, func(payload string) error {
|
||||
return p.handleCallback(payload)
|
||||
}); err != nil {
|
||||
w.Destroy()
|
||||
p.ready <- err
|
||||
close(p.done)
|
||||
return
|
||||
}
|
||||
|
||||
applyWindowOptions(w, p.opts)
|
||||
w.SetHtml(buildBootstrapHTML(p.callbackName, p.readyID))
|
||||
if runtimepkg.GOOS == "linux" {
|
||||
// All GTK pages share the process-wide event loop below.
|
||||
return
|
||||
}
|
||||
w.Run()
|
||||
w.Destroy()
|
||||
close(p.done)
|
||||
@@ -318,7 +332,12 @@ func (p *pageWrapper) Close() error {
|
||||
// the quit message lands on the correct queue.
|
||||
terminateDone := make(chan struct{})
|
||||
w.Dispatch(func() {
|
||||
w.Terminate()
|
||||
if runtimepkg.GOOS == "linux" {
|
||||
w.Destroy()
|
||||
close(p.done)
|
||||
} else {
|
||||
w.Terminate()
|
||||
}
|
||||
close(terminateDone)
|
||||
})
|
||||
<-terminateDone
|
||||
@@ -610,3 +629,12 @@ func firstNonEmpty(values ...string) string {
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (p *Provider) RemoveProfile(profileID, dataPath string) error {
|
||||
done := make(chan error, 1)
|
||||
run := func() { done <- webview.RemoveProfile(dataPath) }
|
||||
if !postMainThreadTask(run) {
|
||||
run()
|
||||
}
|
||||
return <-done
|
||||
}
|
||||
@@ -13,3 +13,5 @@ func TestProviderContract(t *testing.T) {
|
||||
CookieDomainMode: integrationtest.CookieDomainModeRequired,
|
||||
})
|
||||
}
|
||||
|
||||
func TestProviderProfiles(t *testing.T) { integrationtest.RunProfileContract(t, New()) }
|
||||
@@ -1,10 +1,14 @@
|
||||
package integrationtest
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"fmt"
|
||||
webviewproxy "github.com/GopeedLab/gopeed/internal/webview/proxy"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -219,7 +223,19 @@ func openTestPage(t *testing.T, provider enginewebview.Provider, opts enginewebv
|
||||
}))
|
||||
t.Cleanup(server.Close)
|
||||
|
||||
runtime := enginewebview.NewRuntime(provider, true)
|
||||
proxy, err := webviewproxy.New(nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { proxy.Close() })
|
||||
path := filepath.Join(t.TempDir(), "profile")
|
||||
sum := sha256.Sum256([]byte(path))
|
||||
profileID := fmt.Sprintf("%x-%x-%x-%x-%x", sum[:4], sum[4:6], sum[6:8], sum[8:10], sum[10:16])
|
||||
proxyURL := proxy.URL()
|
||||
if runtime.GOOS == "darwin" {
|
||||
proxyURL = proxy.SOCKSURL()
|
||||
}
|
||||
runtime := enginewebview.NewRuntime(&configuredOpener{provider: provider, profileID: profileID, dataPath: path, proxyURL: proxyURL}, true)
|
||||
page, err := runtime.Open(map[string]any{
|
||||
"headless": opts.Headless,
|
||||
"debug": opts.Debug,
|
||||
@@ -232,7 +248,14 @@ func openTestPage(t *testing.T, provider enginewebview.Provider, opts enginewebv
|
||||
t.Fatalf("failed to open webview: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
_ = runtime.Close()
|
||||
if err := runtime.Close(); err != nil {
|
||||
t.Errorf("close contract page: %v", err)
|
||||
}
|
||||
if remover, ok := provider.(enginewebview.ProfileRemover); ok {
|
||||
if err := remover.RemoveProfile(profileID, path); err != nil {
|
||||
t.Errorf("remove contract profile: %v", err)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
return page, server, server.URL
|
||||
@@ -380,3 +403,15 @@ func hasCookie(cookies []enginewebview.Cookie, name string, value string) bool {
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
type configuredOpener struct {
|
||||
provider enginewebview.Provider
|
||||
profileID, dataPath, proxyURL string
|
||||
}
|
||||
|
||||
func (o *configuredOpener) Open(opts enginewebview.OpenOptions) (enginewebview.Page, error) {
|
||||
opts.ProfileID = o.profileID
|
||||
opts.DataPath = o.dataPath
|
||||
opts.ProxyURL = o.proxyURL
|
||||
return o.provider.Open(opts)
|
||||
}
|
||||
@@ -0,0 +1,247 @@
|
||||
package integrationtest
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"runtime"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
webviewproxy "github.com/GopeedLab/gopeed/internal/webview/proxy"
|
||||
webview "github.com/GopeedLab/gopeed/pkg/download/engine/webview"
|
||||
)
|
||||
|
||||
// RunProfileContract exercises the same host API through both native and RPC
|
||||
// providers. It never bypasses the host bridge with direct Flutter plugin calls.
|
||||
func RunProfileContract(t *testing.T, provider webview.Provider) {
|
||||
t.Helper()
|
||||
waitForProviderAvailable(t, provider, defaultAvailabilityTimeout)
|
||||
remover, ok := provider.(webview.ProfileRemover)
|
||||
if !ok {
|
||||
t.Fatal("provider does not implement host profile removal")
|
||||
}
|
||||
proxyURL, secureHits := profileProxy(t)
|
||||
a := newContractProfile(t, provider, remover, proxyURL)
|
||||
b := newContractProfile(t, provider, remover, proxyURL)
|
||||
first := a.open(t)
|
||||
assertProfileValue(t, first, `async () => { localStorage.setItem('owner','a'); return 'a'; }`, "a")
|
||||
assertProfileValue(t, first, profileDatabaseScript(true), "saved")
|
||||
if err := first.SetCookie(webview.Cookie{Name: "session", Value: "a", Domain: "profiles.invalid", Path: "/", HTTPOnly: true, Expires: time.Now().Add(time.Hour)}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
second := b.open(t)
|
||||
assertProfileValue(t, second, `() => localStorage.getItem('owner')`, nil)
|
||||
assertProfileValue(t, second, profileDatabaseScript(false), nil)
|
||||
assertProfileCookies(t, second, "", "new profile B")
|
||||
assertProfileValue(t, second, `() => { localStorage.setItem('owner','b'); return 'b'; }`, "b")
|
||||
if err := second.SetCookie(webview.Cookie{Name: "session", Value: "b", Domain: "profiles.invalid", Path: "/", HTTPOnly: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
shared := a.open(t)
|
||||
assertProfileValue(t, shared, `() => localStorage.getItem('owner')`, "a")
|
||||
assertProfileValue(t, shared, profileDatabaseScript(false), "saved")
|
||||
assertProfileCookies(t, shared, "a", "shared profile A")
|
||||
storedCookies, err := shared.GetCookies()
|
||||
if err != nil || len(storedCookies) != 1 || storedCookies[0].Expires.Before(time.Now()) {
|
||||
t.Fatalf("persistent cookie lost its expiration: %#v, error %v", storedCookies, err)
|
||||
}
|
||||
assertProfileValue(t, shared, `() => document.cookie`, "") // HttpOnly stays hidden.
|
||||
if err := second.ClearCookies(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
assertProfileCookies(t, first, "a", "clearing B leaves A intact")
|
||||
if err := second.SetCookie(webview.Cookie{Name: "session", Value: "b", Domain: "profiles.invalid", Path: "/", HTTPOnly: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a.close(t)
|
||||
reopened := a.open(t)
|
||||
assertProfileValue(t, reopened, `() => localStorage.getItem('owner')`, "a")
|
||||
assertProfileValue(t, reopened, profileDatabaseScript(false), "saved")
|
||||
assertProfileCookies(t, reopened, "a", "reopened profile A")
|
||||
// Include a session cookie immediately before deletion to catch stale
|
||||
// in-process CookieStore caches when the same profile is recreated.
|
||||
if err := reopened.SetCookie(webview.Cookie{Name: "session", Value: "remove-me", Domain: "profiles.invalid", Path: "/", HTTPOnly: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a.close(t)
|
||||
a.remove(t)
|
||||
a.remove(t) // Retrying a successful uninstall is harmless.
|
||||
reinstalled := a.open(t)
|
||||
assertProfileValue(t, reinstalled, `() => localStorage.getItem('owner')`, nil)
|
||||
assertProfileValue(t, reinstalled, profileDatabaseScript(false), nil)
|
||||
assertProfileCookies(t, reinstalled, "", "reinstalled profile A")
|
||||
assertProfileValue(t, second, `() => localStorage.getItem('owner')`, "b")
|
||||
assertProfileCookies(t, second, "b", "B after removing A")
|
||||
// Refuse the HTTPS tunnel deliberately. Seeing it at our proxy proves HTTPS
|
||||
// routing without adding a certificate-validation bypass to the test host.
|
||||
// WebView2 may commit its built-in error page and report navigation as
|
||||
// complete. The invariant here is that the HTTPS tunnel reaches the proxy.
|
||||
_ = reinstalled.Goto("https://profiles.invalid/", webview.GotoOptions{TimeoutMS: 5000})
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
for secureHits.Load() == 0 && time.Now().Before(deadline) {
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
if secureHits.Load() == 0 {
|
||||
t.Fatal("HTTPS bypassed the host proxy")
|
||||
}
|
||||
}
|
||||
|
||||
type contractProfile struct {
|
||||
provider webview.Provider
|
||||
remover webview.ProfileRemover
|
||||
options webview.OpenOptions
|
||||
pages []webview.Page
|
||||
}
|
||||
|
||||
func newContractProfile(t *testing.T, provider webview.Provider, remover webview.ProfileRemover, proxyURL string) *contractProfile {
|
||||
t.Helper()
|
||||
path := filepath.Join(t.TempDir(), "profile")
|
||||
hash := sha256.Sum256([]byte(path))
|
||||
id := fmt.Sprintf("%x-%x-%x-%x-%x", hash[:4], hash[4:6], hash[6:8], hash[8:10], hash[10:16])
|
||||
p := &contractProfile{provider: provider, remover: remover, options: webview.OpenOptions{ProfileID: id, DataPath: path, ProxyURL: proxyURL, Headless: true, Title: "Gopeed profile contract"}}
|
||||
t.Cleanup(func() { p.close(t); p.remove(t) })
|
||||
return p
|
||||
}
|
||||
func (p *contractProfile) open(t *testing.T) webview.Page {
|
||||
t.Helper()
|
||||
page, err := p.provider.Open(p.options)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
p.pages = append(p.pages, page)
|
||||
if err := page.Goto("http://profiles.invalid/", webview.GotoOptions{TimeoutMS: 10000}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return page
|
||||
}
|
||||
func (p *contractProfile) close(t *testing.T) {
|
||||
t.Helper()
|
||||
for _, page := range p.pages {
|
||||
if err := page.Close(); err != nil {
|
||||
t.Errorf("close profile page: %v", err)
|
||||
}
|
||||
}
|
||||
p.pages = nil
|
||||
}
|
||||
func (p *contractProfile) remove(t *testing.T) {
|
||||
t.Helper()
|
||||
if err := p.remover.RemoveProfile(p.options.ProfileID, p.options.DataPath); err != nil {
|
||||
t.Errorf("remove profile: %v", err)
|
||||
}
|
||||
}
|
||||
func assertProfileValue(t *testing.T, page webview.Page, script string, want any) {
|
||||
t.Helper()
|
||||
got, err := page.Execute(script)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("script %s: got %#v, want %#v", script, got, want)
|
||||
}
|
||||
}
|
||||
func assertProfileCookies(t *testing.T, page webview.Page, want, stage string) {
|
||||
t.Helper()
|
||||
// Native cookie managers propagate updates across browser processes
|
||||
// asynchronously. Wait for the expected state rather than assuming the
|
||||
// first IPC read already includes the preceding write.
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
for {
|
||||
cookies, err := page.GetCookies()
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", stage, err)
|
||||
}
|
||||
matches := len(cookies) == 0 && want == "" || len(cookies) == 1 && want != "" && cookies[0].Name == "session" && cookies[0].Value == want && cookies[0].HTTPOnly
|
||||
if matches {
|
||||
return
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("%s: cookies %#v, want %q", stage, cookies, want)
|
||||
}
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
func profileDatabaseScript(write bool) string {
|
||||
operation := "store.get('owner')"
|
||||
if write {
|
||||
operation = "store.put('saved','owner')"
|
||||
}
|
||||
return fmt.Sprintf(`async () => await new Promise((resolve,reject)=>{
|
||||
const request=indexedDB.open('profile-contract',1);
|
||||
request.onupgradeneeded=()=>request.result.createObjectStore('state');
|
||||
request.onerror=()=>reject(String(request.error));
|
||||
request.onsuccess=()=>{
|
||||
const db=request.result, tx=db.transaction('state','readwrite'), store=tx.objectStore('state');
|
||||
const op=%s; let value;
|
||||
op.onsuccess=()=>value=%s;
|
||||
tx.oncomplete=()=>{db.close();resolve(value===undefined?null:value)};
|
||||
tx.onerror=()=>{db.close();reject(String(tx.error))};
|
||||
};
|
||||
})`, operation, map[bool]string{true: "'saved'", false: "op.result"}[write])
|
||||
}
|
||||
|
||||
func profileProxy(t *testing.T) (string, *atomic.Int32) {
|
||||
t.Helper()
|
||||
origin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "text/html")
|
||||
w.Header().Set("Cache-Control", "no-store")
|
||||
fmt.Fprint(w, "<!doctype html><title>Profile contract</title>")
|
||||
}))
|
||||
t.Cleanup(origin.Close)
|
||||
secureHits := &atomic.Int32{}
|
||||
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodConnect {
|
||||
origin.Config.Handler.ServeHTTP(w, r)
|
||||
return
|
||||
}
|
||||
if r.Host == "profiles.invalid:443" {
|
||||
secureHits.Add(1)
|
||||
http.Error(w, "HTTPS routing fixture rejects tunnels", http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
if r.Host != "profiles.invalid:80" {
|
||||
http.Error(w, "unexpected target", http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
remote, err := net.DialTimeout("tcp", origin.Listener.Addr().String(), 5*time.Second)
|
||||
if err != nil {
|
||||
http.Error(w, "origin unavailable", http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
defer remote.Close()
|
||||
client, buffer, err := w.(http.Hijacker).Hijack()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer client.Close()
|
||||
if _, err := buffer.WriteString("HTTP/1.1 200 Connection Established\r\n\r\n"); err != nil {
|
||||
return
|
||||
}
|
||||
if err := buffer.Flush(); err != nil {
|
||||
return
|
||||
}
|
||||
go func() { io.Copy(remote, buffer); remote.Close() }()
|
||||
io.Copy(client, remote)
|
||||
}))
|
||||
t.Cleanup(upstream.Close)
|
||||
upstreamURL, err := url.Parse(upstream.URL)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
proxy, err := webviewproxy.New(func(*http.Request) (*url.URL, error) { return upstreamURL, nil })
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { proxy.Close() })
|
||||
if runtime.GOOS == "darwin" {
|
||||
return proxy.SOCKSURL(), secureHits
|
||||
}
|
||||
return proxy.URL(), secureHits
|
||||
}
|
||||
@@ -0,0 +1,341 @@
|
||||
// Package proxy provides the host-owned WebView forward proxy. Browser engines
|
||||
// always use this endpoint; Gopeed decides the upstream route for each request.
|
||||
package proxy
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/armon/go-socks5"
|
||||
xproxy "golang.org/x/net/proxy"
|
||||
)
|
||||
|
||||
type Selector func(*http.Request) (*url.URL, error)
|
||||
|
||||
type Server struct {
|
||||
listener net.Listener
|
||||
socksListener net.Listener
|
||||
server *http.Server
|
||||
transport *http.Transport
|
||||
selectProxy Selector
|
||||
mu sync.Mutex
|
||||
connections map[io.Closer]struct{}
|
||||
closed bool
|
||||
}
|
||||
|
||||
func New(selectProxy Selector) (*Server, error) {
|
||||
if selectProxy == nil {
|
||||
selectProxy = func(*http.Request) (*url.URL, error) { return nil, nil }
|
||||
}
|
||||
listener, err := net.Listen("tcp4", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
s := &Server{listener: listener, selectProxy: selectProxy, connections: make(map[io.Closer]struct{})}
|
||||
s.transport = &http.Transport{Proxy: selectProxy, DisableKeepAlives: true, TLSHandshakeTimeout: 15 * time.Second, ResponseHeaderTimeout: 30 * time.Second}
|
||||
s.server = &http.Server{Handler: s, ReadHeaderTimeout: 15 * time.Second}
|
||||
s.socksListener, err = net.Listen("tcp4", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
listener.Close()
|
||||
return nil, err
|
||||
}
|
||||
socks, err := socks5.New(&socks5.Config{Logger: log.New(io.Discard, "", 0), Resolver: unresolvedDNS{}, Dial: func(ctx context.Context, network, address string) (net.Conn, error) {
|
||||
_, port, _ := net.SplitHostPort(address)
|
||||
scheme := "https"
|
||||
if port == "80" {
|
||||
scheme = "http"
|
||||
}
|
||||
request := &http.Request{URL: &url.URL{Scheme: scheme, Host: address}, Header: make(http.Header)}
|
||||
upstream, err := selectProxy(request)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
timed, cancel := context.WithTimeout(ctx, 30*time.Second)
|
||||
defer cancel()
|
||||
conn, err := dialTunnel(timed, address, upstream)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return s.track(conn)
|
||||
}})
|
||||
if err != nil {
|
||||
listener.Close()
|
||||
s.socksListener.Close()
|
||||
return nil, err
|
||||
}
|
||||
go s.server.Serve(listener)
|
||||
go socks.Serve(&trackedListener{Listener: s.socksListener, server: s})
|
||||
return s, nil
|
||||
}
|
||||
|
||||
func (s *Server) URL() string { return "http://" + s.listener.Addr().String() }
|
||||
|
||||
func (s *Server) SOCKSURL() string { return "socks5://" + s.socksListener.Addr().String() }
|
||||
|
||||
func (s *Server) Close() error {
|
||||
s.mu.Lock()
|
||||
s.closed = true
|
||||
connections := make([]io.Closer, 0, len(s.connections))
|
||||
for c := range s.connections {
|
||||
connections = append(connections, c)
|
||||
}
|
||||
s.mu.Unlock()
|
||||
for _, c := range connections {
|
||||
c.Close()
|
||||
}
|
||||
s.socksListener.Close()
|
||||
s.transport.CloseIdleConnections()
|
||||
return s.server.Close()
|
||||
}
|
||||
|
||||
func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method == http.MethodConnect {
|
||||
s.connect(w, r)
|
||||
return
|
||||
}
|
||||
if !r.URL.IsAbs() || (r.URL.Scheme != "http" && r.URL.Scheme != "https") {
|
||||
http.Error(w, "absolute HTTP URL required", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
out := r.Clone(r.Context())
|
||||
out.RequestURI = ""
|
||||
upgrade := out.Header.Get("Upgrade")
|
||||
stripHopHeaders(out.Header)
|
||||
if upgrade != "" {
|
||||
out.Header.Set("Connection", "Upgrade")
|
||||
out.Header.Set("Upgrade", upgrade)
|
||||
}
|
||||
resp, err := s.transport.RoundTrip(out)
|
||||
if err != nil {
|
||||
http.Error(w, "upstream request failed", http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode == http.StatusSwitchingProtocols {
|
||||
remote, ok := resp.Body.(io.ReadWriteCloser)
|
||||
if !ok {
|
||||
http.Error(w, "invalid upstream upgrade", 502)
|
||||
return
|
||||
}
|
||||
client, buffer, err := w.(http.Hijacker).Hijack()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer client.Close()
|
||||
resp.Body = nil
|
||||
if err := resp.Write(buffer); err != nil {
|
||||
return
|
||||
}
|
||||
if err := buffer.Flush(); err != nil {
|
||||
return
|
||||
}
|
||||
s.bridge(client, buffer, remote)
|
||||
return
|
||||
}
|
||||
stripHopHeaders(resp.Header)
|
||||
for k, v := range resp.Header {
|
||||
w.Header()[k] = v
|
||||
}
|
||||
w.WriteHeader(resp.StatusCode)
|
||||
io.Copy(w, resp.Body)
|
||||
}
|
||||
|
||||
func stripHopHeaders(h http.Header) {
|
||||
for _, key := range strings.Split(h.Get("Connection"), ",") {
|
||||
h.Del(strings.TrimSpace(key))
|
||||
}
|
||||
for _, key := range []string{"Connection", "Proxy-Connection", "Proxy-Authorization", "Proxy-Authenticate", "Keep-Alive", "TE", "Trailer", "Transfer-Encoding", "Upgrade"} {
|
||||
h.Del(key)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Server) connect(w http.ResponseWriter, r *http.Request) {
|
||||
if _, _, err := net.SplitHostPort(r.Host); err != nil {
|
||||
http.Error(w, "invalid tunnel destination", 400)
|
||||
return
|
||||
}
|
||||
target := r.Clone(r.Context())
|
||||
target.URL = &url.URL{Scheme: "https", Host: r.Host}
|
||||
upstream, err := s.selectProxy(target)
|
||||
if err != nil {
|
||||
http.Error(w, "proxy selection failed", 502)
|
||||
return
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(r.Context(), 30*time.Second)
|
||||
defer cancel()
|
||||
remote, err := dialTunnel(ctx, r.Host, upstream)
|
||||
if err != nil {
|
||||
http.Error(w, "upstream connection failed", 502)
|
||||
return
|
||||
}
|
||||
defer remote.Close()
|
||||
client, buffer, err := w.(http.Hijacker).Hijack()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer client.Close()
|
||||
if _, err := buffer.WriteString("HTTP/1.1 200 Connection Established\r\n\r\n"); err != nil {
|
||||
return
|
||||
}
|
||||
if err := buffer.Flush(); err != nil {
|
||||
return
|
||||
}
|
||||
s.bridge(client, buffer, remote)
|
||||
}
|
||||
|
||||
func (s *Server) bridge(client net.Conn, buffer *bufio.ReadWriter, remote io.ReadWriteCloser) {
|
||||
s.mu.Lock()
|
||||
if s.closed {
|
||||
s.mu.Unlock()
|
||||
return
|
||||
}
|
||||
s.connections[client] = struct{}{}
|
||||
s.connections[remote] = struct{}{}
|
||||
s.mu.Unlock()
|
||||
defer func() { s.mu.Lock(); delete(s.connections, client); delete(s.connections, remote); s.mu.Unlock() }()
|
||||
|
||||
done := make(chan struct{})
|
||||
go func() { io.Copy(remote, buffer); remote.Close(); close(done) }()
|
||||
io.Copy(client, remote)
|
||||
client.Close()
|
||||
<-done
|
||||
}
|
||||
|
||||
type bufferedConn struct {
|
||||
net.Conn
|
||||
reader *bufio.Reader
|
||||
}
|
||||
|
||||
func (c *bufferedConn) Read(p []byte) (int, error) { return c.reader.Read(p) }
|
||||
|
||||
func dialTunnel(ctx context.Context, target string, upstream *url.URL) (net.Conn, error) {
|
||||
dialer := &net.Dialer{}
|
||||
if upstream == nil {
|
||||
return dialer.DialContext(ctx, "tcp", target)
|
||||
}
|
||||
if upstream.Scheme == "socks5" || upstream.Scheme == "socks5h" {
|
||||
u := *upstream
|
||||
u.Scheme = "socks5"
|
||||
d, err := xproxy.FromURL(&u, dialer)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return d.(xproxy.ContextDialer).DialContext(ctx, "tcp", target)
|
||||
}
|
||||
if upstream.Scheme != "http" && upstream.Scheme != "https" {
|
||||
return nil, errors.New("unsupported proxy scheme")
|
||||
}
|
||||
host := upstream.Host
|
||||
if upstream.Port() == "" {
|
||||
port := "80"
|
||||
if upstream.Scheme == "https" {
|
||||
port = "443"
|
||||
}
|
||||
host = net.JoinHostPort(upstream.Hostname(), port)
|
||||
}
|
||||
conn, err := dialer.DialContext(ctx, "tcp", host)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ok := false
|
||||
defer func() {
|
||||
if !ok {
|
||||
conn.Close()
|
||||
}
|
||||
}()
|
||||
if deadline, exists := ctx.Deadline(); exists {
|
||||
conn.SetDeadline(deadline)
|
||||
}
|
||||
if upstream.Scheme == "https" {
|
||||
secure := tls.Client(conn, &tls.Config{ServerName: upstream.Hostname(), MinVersion: tls.VersionTLS12})
|
||||
if err := secure.HandshakeContext(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
conn = secure
|
||||
}
|
||||
request := &http.Request{Method: http.MethodConnect, URL: &url.URL{Opaque: target}, Host: target, Header: make(http.Header)}
|
||||
if upstream.User != nil {
|
||||
password, _ := upstream.User.Password()
|
||||
request.Header.Set("Proxy-Authorization", "Basic "+base64.StdEncoding.EncodeToString([]byte(upstream.User.Username()+":"+password)))
|
||||
}
|
||||
if err := request.Write(conn); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
reader := bufio.NewReader(conn)
|
||||
response, err := http.ReadResponse(reader, request)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if response.StatusCode != 200 {
|
||||
response.Body.Close()
|
||||
return nil, fmt.Errorf("upstream CONNECT status %d", response.StatusCode)
|
||||
}
|
||||
conn.SetDeadline(time.Time{})
|
||||
ok = true
|
||||
return &bufferedConn{Conn: conn, reader: reader}, nil
|
||||
}
|
||||
|
||||
// Leave hostname resolution to the selected upstream (SOCKS remote DNS / CONNECT).
|
||||
type unresolvedDNS struct{}
|
||||
|
||||
func (unresolvedDNS) Resolve(ctx context.Context, name string) (context.Context, net.IP, error) {
|
||||
return ctx, nil, nil
|
||||
}
|
||||
|
||||
type trackedListener struct {
|
||||
net.Listener
|
||||
server *Server
|
||||
}
|
||||
|
||||
func (l *trackedListener) Accept() (net.Conn, error) {
|
||||
c, err := l.Listener.Accept()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return l.server.track(c)
|
||||
}
|
||||
|
||||
type trackedConn struct {
|
||||
net.Conn
|
||||
server *Server
|
||||
}
|
||||
|
||||
func (c *trackedConn) Close() error {
|
||||
err := c.Conn.Close()
|
||||
c.server.mu.Lock()
|
||||
delete(c.server.connections, c)
|
||||
c.server.mu.Unlock()
|
||||
return err
|
||||
}
|
||||
func (s *Server) track(c net.Conn) (net.Conn, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.closed {
|
||||
c.Close()
|
||||
return nil, net.ErrClosed
|
||||
}
|
||||
tracked := &trackedConn{Conn: c, server: s}
|
||||
s.connections[tracked] = struct{}{}
|
||||
return tracked, nil
|
||||
}
|
||||
|
||||
func (c *trackedConn) CloseWrite() error { return closeWrite(c.Conn) }
|
||||
func (c *bufferedConn) CloseWrite() error { return closeWrite(c.Conn) }
|
||||
func closeWrite(c net.Conn) error {
|
||||
if writer, ok := c.(interface{ CloseWrite() error }); ok {
|
||||
return writer.CloseWrite()
|
||||
}
|
||||
return c.Close()
|
||||
}
|
||||
@@ -0,0 +1,315 @@
|
||||
package proxy
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"errors"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/armon/go-socks5"
|
||||
)
|
||||
|
||||
func TestRejectedProxyRequests(t *testing.T) {
|
||||
secret := "upstream-password"
|
||||
for _, tc := range []struct {
|
||||
name, method, target string
|
||||
status int
|
||||
}{
|
||||
{"relative URL", "GET", "/relative", 400},
|
||||
{"unsupported URL", "GET", "ftp://example.com/file", 400},
|
||||
{"invalid CONNECT", "CONNECT", "example.com", 400},
|
||||
{"HTTP selection failure", "GET", "http://example.com/", 502},
|
||||
{"CONNECT selection failure", "CONNECT", "example.com:443", 502},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
s := &Server{selectProxy: func(*http.Request) (*url.URL, error) { return nil, errors.New(secret) }}
|
||||
s.transport = &http.Transport{Proxy: s.selectProxy}
|
||||
defer s.transport.CloseIdleConnections()
|
||||
response := httptest.NewRecorder()
|
||||
s.ServeHTTP(response, httptest.NewRequest(tc.method, tc.target, nil))
|
||||
if response.Code != tc.status || strings.Contains(response.Body.String(), secret) {
|
||||
t.Fatalf("unexpected proxy rejection: %d %q", response.Code, response.Body.String())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestTunnelRejectsUnsafeUpstreams(t *testing.T) {
|
||||
rejected := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
http.Error(w, "authentication required", http.StatusProxyAuthRequired)
|
||||
}))
|
||||
defer rejected.Close()
|
||||
secure := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
t.Error("untrusted TLS proxy received a request")
|
||||
}))
|
||||
defer secure.Close()
|
||||
for _, upstream := range []string{"ftp://127.0.0.1", rejected.URL, secure.URL} {
|
||||
t.Run(upstream, func(t *testing.T) {
|
||||
selected, err := url.Parse(upstream)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
|
||||
defer cancel()
|
||||
conn, err := dialTunnel(ctx, "example.com:443", selected)
|
||||
if conn != nil {
|
||||
conn.Close()
|
||||
}
|
||||
if err == nil {
|
||||
t.Fatal("unsafe upstream unexpectedly accepted")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRoutesAndAuthentication(t *testing.T) {
|
||||
origin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Header.Get("Proxy-Authorization") != "" {
|
||||
t.Error("proxy credentials leaked to origin")
|
||||
}
|
||||
w.Write([]byte("origin"))
|
||||
}))
|
||||
defer origin.Close()
|
||||
secure := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Write([]byte("secure")) }))
|
||||
defer secure.Close()
|
||||
upstream, err := New(func(*http.Request) (*url.URL, error) { return nil, nil })
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer upstream.Close()
|
||||
var authenticated atomic.Int32
|
||||
authProxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Header.Get("Proxy-Authorization") != "Basic dXNlcjpwYXNz" {
|
||||
http.Error(w, "authentication required", 407)
|
||||
return
|
||||
}
|
||||
authenticated.Add(1)
|
||||
upstream.ServeHTTP(w, r)
|
||||
}))
|
||||
defer authProxy.Close()
|
||||
socks, err := socks5.New(&socks5.Config{Credentials: socks5.StaticCredentials{"user": "pass"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
listener, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer listener.Close()
|
||||
go socks.Serve(listener)
|
||||
for _, route := range []string{"", authProxy.URL, "socks5://" + listener.Addr().String()} {
|
||||
t.Run(route, func(t *testing.T) {
|
||||
var selected *url.URL
|
||||
if route != "" {
|
||||
selected, _ = url.Parse(route)
|
||||
selected.User = url.UserPassword("user", "pass")
|
||||
}
|
||||
local, err := New(func(r *http.Request) (*url.URL, error) { return selected, nil })
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer local.Close()
|
||||
proxyURL, _ := url.Parse(local.URL())
|
||||
transport := &http.Transport{Proxy: http.ProxyURL(proxyURL), TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}
|
||||
defer transport.CloseIdleConnections()
|
||||
client := &http.Client{Transport: transport, Timeout: 5 * time.Second}
|
||||
for _, test := range []struct{ url, want string }{{origin.URL, "origin"}, {secure.URL, "secure"}} {
|
||||
response, err := client.Get(test.url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body, _ := io.ReadAll(response.Body)
|
||||
response.Body.Close()
|
||||
if string(body) != test.want {
|
||||
t.Fatalf("got status %d, body %q", response.StatusCode, body)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
if authenticated.Load() != 2 {
|
||||
t.Fatalf("expected authenticated HTTP and CONNECT, got %d", authenticated.Load())
|
||||
}
|
||||
}
|
||||
|
||||
func TestRoutingChangesAndFailureDoesNotBypassProxy(t *testing.T) {
|
||||
var hits atomic.Int32
|
||||
origin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { hits.Add(1); w.Write([]byte("direct")) }))
|
||||
defer origin.Close()
|
||||
var useProxy atomic.Bool
|
||||
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Write([]byte("proxied")) }))
|
||||
upstreamURL, _ := url.Parse(upstream.URL)
|
||||
local, err := New(func(*http.Request) (*url.URL, error) {
|
||||
if useProxy.Load() {
|
||||
return upstreamURL, nil
|
||||
}
|
||||
return nil, nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer local.Close()
|
||||
endpoint, _ := url.Parse(local.URL())
|
||||
transport := &http.Transport{Proxy: http.ProxyURL(endpoint)}
|
||||
defer transport.CloseIdleConnections()
|
||||
client := &http.Client{Transport: transport, Timeout: 3 * time.Second}
|
||||
for _, want := range []string{"direct", "proxied", "upstream request failed\n"} {
|
||||
resp, err := client.Get(origin.URL)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
resp.Body.Close()
|
||||
if string(body) != want {
|
||||
t.Fatalf("got %q want %q", body, want)
|
||||
}
|
||||
if want == "direct" {
|
||||
useProxy.Store(true)
|
||||
} else if want == "proxied" {
|
||||
upstream.Close()
|
||||
}
|
||||
}
|
||||
if hits.Load() != 1 {
|
||||
t.Fatal("failed proxy must not fall back to direct")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCloseTerminatesTunnels(t *testing.T) {
|
||||
echo, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer echo.Close()
|
||||
go func() {
|
||||
c, e := echo.Accept()
|
||||
if e == nil {
|
||||
defer c.Close()
|
||||
io.Copy(c, c)
|
||||
}
|
||||
}()
|
||||
s, err := New(func(*http.Request) (*url.URL, error) { return nil, nil })
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
endpoint, _ := url.Parse(s.URL())
|
||||
c, err := dialTunnel(context.Background(), echo.Addr().String(), endpoint)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer c.Close()
|
||||
c.SetDeadline(time.Now().Add(3 * time.Second))
|
||||
c.Write([]byte("hello\n"))
|
||||
line, err := bufio.NewReader(c).ReadString('\n')
|
||||
if err != nil || line != "hello\n" {
|
||||
t.Fatalf("echo %q %v", line, err)
|
||||
}
|
||||
s.Close()
|
||||
if _, err := c.Read(make([]byte, 1)); err == nil {
|
||||
t.Fatal("tunnel stayed open")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSOCKSIngressPreservesRemoteDNS(t *testing.T) {
|
||||
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != "CONNECT" || r.Host != "unresolved.invalid:80" {
|
||||
t.Errorf("lost destination: %s %s", r.Method, r.Host)
|
||||
}
|
||||
c, b, err := w.(http.Hijacker).Hijack()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer c.Close()
|
||||
b.WriteString("HTTP/1.1 200 Connection Established\r\n\r\n")
|
||||
b.Flush()
|
||||
request, err := http.ReadRequest(b.Reader)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
return
|
||||
}
|
||||
request.Body.Close()
|
||||
b.WriteString("HTTP/1.1 200 OK\r\nContent-Length: 7\r\nConnection: close\r\n\r\nproxied")
|
||||
b.Flush()
|
||||
}))
|
||||
defer upstream.Close()
|
||||
target, _ := url.Parse(upstream.URL)
|
||||
local, err := New(func(r *http.Request) (*url.URL, error) {
|
||||
if r.URL.Host != "unresolved.invalid:80" {
|
||||
t.Errorf("selector lost hostname: %s", r.URL)
|
||||
}
|
||||
return target, nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer local.Close()
|
||||
endpoint, _ := url.Parse(local.SOCKSURL())
|
||||
transport := &http.Transport{Proxy: http.ProxyURL(endpoint)}
|
||||
defer transport.CloseIdleConnections()
|
||||
client := &http.Client{Transport: transport, Timeout: 5 * time.Second}
|
||||
response, err := client.Get("http://unresolved.invalid/")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer response.Body.Close()
|
||||
body, _ := io.ReadAll(response.Body)
|
||||
if string(body) != "proxied" {
|
||||
t.Fatalf("unexpected body %q", body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHTTPUpgradeIsForwarded(t *testing.T) {
|
||||
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Header.Get("Upgrade") != "fixture" {
|
||||
http.Error(w, "upgrade lost", 400)
|
||||
return
|
||||
}
|
||||
c, b, err := w.(http.Hijacker).Hijack()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer c.Close()
|
||||
b.WriteString("HTTP/1.1 101 Switching Protocols\r\nConnection: Upgrade\r\nUpgrade: fixture\r\n\r\n")
|
||||
b.Flush()
|
||||
line, err := b.ReadString('\n')
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
b.WriteString(line)
|
||||
b.Flush()
|
||||
}))
|
||||
defer upstream.Close()
|
||||
local, err := New(nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer local.Close()
|
||||
endpoint, _ := url.Parse(local.URL())
|
||||
c, err := net.Dial("tcp", endpoint.Host)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer c.Close()
|
||||
c.SetDeadline(time.Now().Add(3 * time.Second))
|
||||
io.WriteString(c, "GET "+upstream.URL+"/ HTTP/1.1\r\nHost: fixture\r\nConnection: Upgrade\r\nUpgrade: fixture\r\n\r\n")
|
||||
reader := bufio.NewReader(c)
|
||||
response, err := http.ReadResponse(reader, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if response.StatusCode != 101 {
|
||||
t.Fatalf("upgrade failed: %s", response.Status)
|
||||
}
|
||||
io.WriteString(c, "hello\n")
|
||||
line, err := reader.ReadString('\n')
|
||||
if err != nil || line != "hello\n" {
|
||||
t.Fatalf("upgraded echo %q %v", line, err)
|
||||
}
|
||||
}
|
||||
@@ -1,5 +1,11 @@
|
||||
# rpcprovider
|
||||
|
||||
The private Flutter native profile bridge is maintained in the
|
||||
[GopeedLab flutter_inappwebview fork](https://github.com/GopeedLab/flutter_inappwebview/tree/codex/gopeed-profiles).
|
||||
All three platform overrides pin one commit in `ui/flutter/pubspec.yaml`;
|
||||
see the fork’s [GOPEED.md](https://github.com/GopeedLab/flutter_inappwebview/blob/codex/gopeed-profiles/GOPEED.md)
|
||||
for patch scope and upstream synchronization.
|
||||
|
||||
`rpcprovider` is the RPC-backed implementation of Gopeed's `webview.Provider` interface.
|
||||
|
||||
It is intended for environments where WebView capability is owned by another process, most notably mobile hosts. This package does not implement a WebView itself. Instead, it translates the `pkg/download/engine/webview` page API into single-request `POST + JSON` calls sent to a local host-side RPC service.
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
@@ -109,9 +110,14 @@ func (p *page) ClearCookies() error {
|
||||
}
|
||||
|
||||
func (p *page) Close() error {
|
||||
return p.client.Call(enginewebview.MethodPageClose, enginewebview.PageCloseParams{
|
||||
err := p.client.Call(enginewebview.MethodPageClose, enginewebview.PageCloseParams{
|
||||
PageID: p.id,
|
||||
}, nil)
|
||||
var rpcErr *enginewebview.RPCError
|
||||
if errors.As(err, &rpcErr) && rpcErr.Code == enginewebview.ErrorCodePageNotFound {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
type Client struct {
|
||||
@@ -193,3 +199,7 @@ func (c *Client) Call(method string, params any, result any) error {
|
||||
}
|
||||
return json.Unmarshal(body.Result, result)
|
||||
}
|
||||
|
||||
func (p *Provider) RemoveProfile(profileID, dataPath string) error {
|
||||
return p.client.Call(enginewebview.MethodProfileRemove, enginewebview.ProfileRemoveParams{ProfileID: profileID}, nil)
|
||||
}
|
||||
@@ -37,3 +37,8 @@ func defaultWebViewRPCAddress() string {
|
||||
}
|
||||
return filepath.Join(homeDir, "Library", "Application Support", "com.gopeed.gopeed", "gopeed_webview.sock")
|
||||
}
|
||||
|
||||
func TestProviderProfiles(t *testing.T) {
|
||||
provider := New(enginewebview.RPCConfig{Network: *webViewRPCNetwork, Address: *webViewRPCAddress, Token: *webViewRPCToken})
|
||||
integrationtest.RunProfileContract(t, provider)
|
||||
}
|
||||
@@ -184,3 +184,29 @@ func TestPageSetCookieParamsOmitsZeroExpires(t *testing.T) {
|
||||
t.Fatalf("expected expires in payload: %s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemoveProfileUsesPrivateHostRPC(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
var request struct {
|
||||
Method string `json:"method"`
|
||||
Params map[string]string `json:"params"`
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&request); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
if request.Method != enginewebview.MethodProfileRemove || request.Params["profileId"] != "extension-profile" || len(request.Params) != 1 {
|
||||
t.Errorf("unexpected profile removal request: %+v", request)
|
||||
}
|
||||
if r.Header.Get("Authorization") != "Bearer secret" {
|
||||
t.Error("missing host authentication")
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.Write([]byte(`{"error":{"code":"PROFILE_REMOVE_FAILED","message":"store busy"}}`))
|
||||
}))
|
||||
defer server.Close()
|
||||
provider := New(enginewebview.RPCConfig{Network: "tcp", Address: strings.TrimPrefix(server.URL, "http://"), Token: "secret"})
|
||||
err := provider.(enginewebview.ProfileRemover).RemoveProfile("extension-profile", "/host-only/path")
|
||||
if err == nil || err.Error() != "store busy" {
|
||||
t.Fatalf("cleanup failure was lost: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -10,8 +10,15 @@ import (
|
||||
type TaskEventMask uint64
|
||||
|
||||
const (
|
||||
// Keep these first two unchanged because Flutter already uses
|
||||
// bit 0 for done and bit 1 for error.
|
||||
TaskEventDone TaskEventMask = 1 << iota
|
||||
TaskEventError
|
||||
|
||||
TaskEventStart
|
||||
TaskEventProgress
|
||||
TaskEventPause
|
||||
TaskEventDelete
|
||||
)
|
||||
|
||||
type TaskEvent struct {
|
||||
@@ -42,11 +49,26 @@ func (s *Service) emitTaskEvent(event *download.Event) {
|
||||
}
|
||||
|
||||
var mask TaskEventMask
|
||||
|
||||
switch event.Key {
|
||||
case download.EventKeyDone:
|
||||
mask = TaskEventDone
|
||||
|
||||
case download.EventKeyError:
|
||||
mask = TaskEventError
|
||||
|
||||
case download.EventKeyStart:
|
||||
mask = TaskEventStart
|
||||
|
||||
case download.EventKeyProgress:
|
||||
mask = TaskEventProgress
|
||||
|
||||
case download.EventKeyPause:
|
||||
mask = TaskEventPause
|
||||
|
||||
case download.EventKeyDelete:
|
||||
mask = TaskEventDelete
|
||||
|
||||
default:
|
||||
return
|
||||
}
|
||||
|
||||
@@ -94,9 +94,6 @@ type Resource struct {
|
||||
}
|
||||
|
||||
func (r *Resource) Validate() error {
|
||||
if r.Name == "" {
|
||||
return fmt.Errorf("invalid resource name")
|
||||
}
|
||||
if len(r.Files) == 0 {
|
||||
return fmt.Errorf("invalid resource files")
|
||||
}
|
||||
|
||||
@@ -21,6 +21,7 @@ import (
|
||||
"github.com/GopeedLab/gopeed/internal/controller"
|
||||
"github.com/GopeedLab/gopeed/internal/fetcher"
|
||||
"github.com/GopeedLab/gopeed/internal/logger"
|
||||
webviewproxy "github.com/GopeedLab/gopeed/internal/webview/proxy"
|
||||
"github.com/GopeedLab/gopeed/pkg/base"
|
||||
"github.com/GopeedLab/gopeed/pkg/protocol/http"
|
||||
"github.com/GopeedLab/gopeed/pkg/util"
|
||||
@@ -97,8 +98,12 @@ type Progress struct {
|
||||
}
|
||||
|
||||
type Downloader struct {
|
||||
Logger *logger.Logger
|
||||
ExtensionLogger *logger.Logger
|
||||
webviewProfilesLock sync.Mutex
|
||||
webviewProfiles map[string]*extensionWebViewProfile
|
||||
webviewProxyLock sync.Mutex
|
||||
webviewProxy *webviewproxy.Server
|
||||
Logger *logger.Logger
|
||||
ExtensionLogger *logger.Logger
|
||||
|
||||
cfg *DownloaderConfig
|
||||
fetcherCache map[string]fetcher.Fetcher
|
||||
@@ -1138,6 +1143,11 @@ func (d *Downloader) doDelete(task *Task, force bool) (err error) {
|
||||
|
||||
func (d *Downloader) Close() error {
|
||||
d.closed.Store(true)
|
||||
d.webviewProxyLock.Lock()
|
||||
if d.webviewProxy != nil {
|
||||
_ = d.webviewProxy.Close()
|
||||
}
|
||||
d.webviewProxyLock.Unlock()
|
||||
|
||||
closeArr := []func() error{
|
||||
d.pauseAll,
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"github.com/GopeedLab/gopeed/pkg/base"
|
||||
gojaerror "github.com/GopeedLab/gopeed/pkg/download/engine/inject/error"
|
||||
fetchapi "github.com/GopeedLab/gopeed/pkg/download/engine/inject/fetch"
|
||||
ffmpegapi "github.com/GopeedLab/gopeed/pkg/download/engine/inject/ffmpeg"
|
||||
"github.com/GopeedLab/gopeed/pkg/download/engine/inject/file"
|
||||
"github.com/GopeedLab/gopeed/pkg/download/engine/inject/stream"
|
||||
"github.com/GopeedLab/gopeed/pkg/download/engine/inject/vm"
|
||||
@@ -159,8 +160,10 @@ func (e *Engine) addCleanup(cleanup func()) {
|
||||
}
|
||||
|
||||
type Config struct {
|
||||
ProxyConfig *base.DownloaderProxyConfig
|
||||
StreamConfig *stream.Config
|
||||
// HTTPUserAgent is a snapshot of the downloader HTTP default. Nil leaves standalone engines unchanged.
|
||||
HTTPUserAgent *string
|
||||
ProxyConfig *base.DownloaderProxyConfig
|
||||
StreamConfig *stream.Config
|
||||
}
|
||||
|
||||
func NewEngine(cfg *Config) *Engine {
|
||||
@@ -203,9 +206,17 @@ func NewEngine(cfg *Config) *Engine {
|
||||
if err := stream.Enable(runtime, loop, cfg.StreamConfig); err != nil {
|
||||
return
|
||||
}
|
||||
if err := ffmpegapi.Enable(runtime, loop, &ffmpegapi.Config{
|
||||
ProxyHandler: cfg.ProxyConfig.ToHandler(),
|
||||
DefaultUserAgent: cfg.HTTPUserAgent,
|
||||
RegisterCleanup: engine.addCleanup,
|
||||
}); err != nil {
|
||||
return
|
||||
}
|
||||
if err := fetchapi.Enable(runtime, loop, &fetchapi.Config{
|
||||
ProxyHandler: cfg.ProxyConfig.ToHandler(),
|
||||
RegisterCleanup: engine.addCleanup,
|
||||
ProxyHandler: cfg.ProxyConfig.ToHandler(),
|
||||
DefaultUserAgent: cfg.HTTPUserAgent,
|
||||
RegisterCleanup: engine.addCleanup,
|
||||
}); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
@@ -0,0 +1,128 @@
|
||||
# Extension FFmpeg runtime
|
||||
|
||||
`gopeed.runtime.ffmpeg.merge(options)` returns a `ReadableStream<Uint8Array>`.
|
||||
It merges one video track and one audio track without re-encoding, using
|
||||
[go-ffmpreg](https://codeberg.org/gruf/go-ffmpreg)'s embedded FFmpeg 5.1.10 and
|
||||
wazero. No system FFmpeg installation is required. go-ffmpreg and its embedded
|
||||
GPL-enabled FFmpeg are distributed under GPLv3; see the pinned dependency's
|
||||
LICENSE, README and build scripts for its source and included components.
|
||||
|
||||
## HTTP inputs and one downloadable output
|
||||
|
||||
```js
|
||||
gopeed.events.onResolve(async (ctx) => {
|
||||
// Resolve these URLs and headers using the site's API in your extension.
|
||||
const video = { url: videoURL, headers: { Referer: pageURL } };
|
||||
const audio = { url: audioURL, headers: { Referer: pageURL } };
|
||||
const url = await gopeed.runtime.blob.createObjectURL(
|
||||
() => gopeed.runtime.ffmpeg.merge({ video, audio }),
|
||||
{ contentType: "video/mp4" }
|
||||
);
|
||||
ctx.res = {
|
||||
name: "video",
|
||||
files: [{ name: "video.mp4", req: { url } }]
|
||||
};
|
||||
});
|
||||
```
|
||||
|
||||
The opener creates a new merge on each open. Output size is unknown and range
|
||||
support must remain disabled. Do not declare the sum of the input sizes as the
|
||||
output size. The downloader reads the merged output as one file; FFmpeg errors
|
||||
propagate through the stream instead of completing a partial download.
|
||||
|
||||
## JS inputs, including SABR
|
||||
|
||||
Stream inputs **must** be created by an `inputs({ signal })` callback. It returns
|
||||
`{ video, audio }`, each an HTTP descriptor or `ReadableStream<Uint8Array>`,
|
||||
including mixed inputs. Do not start producers before this callback. SABR/UMP decoding belongs
|
||||
to the extension: pass the resulting media streams, with initialization data,
|
||||
not the SABR protocol bytes. Keep each selected track's codec/configuration
|
||||
stable for the duration of a merge.
|
||||
|
||||
```js
|
||||
const output = gopeed.runtime.ffmpeg.merge({
|
||||
inputs: async ({ signal }) => {
|
||||
const session = await prepareSession({ signal });
|
||||
const { videoStream, audioStream, abort } = await session.openStreams();
|
||||
if (signal.aborted) { abort(); throw new Error("Cancelled"); }
|
||||
signal.addEventListener("abort", abort, { once: true });
|
||||
return { video: videoStream, audio: audioStream };
|
||||
},
|
||||
signal: abortController.signal,
|
||||
args: ["-shortest", "-metadata", "title=Example"]
|
||||
});
|
||||
```
|
||||
|
||||
Creating the output does not start requests or acquire input readers until the
|
||||
output is pulled and a global execution slot is acquired. Queued jobs do not
|
||||
open HTTP inputs, invoke input factories, or allocate media ring buffers.
|
||||
The callback runs once per output; its signal aborts on cancellation, engine
|
||||
shutdown, failure, and completion. A callback must cooperate with cancellation;
|
||||
returned streams are cancelled even if the callback finishes after cancellation.
|
||||
A stream input is single-use. To retry a Blob-backed task,
|
||||
its opener must create a new SABR session and fresh input streams. Arbitrary
|
||||
output byte ranges / resumable merged output are not supported.
|
||||
|
||||
`gopeed.runtime.ffmpeg.supportsInputFactory === true` identifies this API.
|
||||
Older builds accept eager streams and must be upgraded before using callbacks.
|
||||
|
||||
## Options
|
||||
|
||||
- `inputs`: callback returning `{ video, audio }`, synchronously or asynchronously.
|
||||
Required for stream inputs. Cannot be combined with top-level `video`/`audio`.
|
||||
- `video`, `audio`: HTTP descriptors only when supplied at the top level; both
|
||||
are required unless `inputs` is supplied. HTTP descriptors have `url` and optional
|
||||
`headers` (standard `HeadersInit`). HTTP requests use Gopeed's configured
|
||||
proxy. Supply required cookies/authorization explicitly; browser or `fetch`
|
||||
cookie jars are not implicitly shared. When `User-Agent` is absent, requests
|
||||
use the downloader HTTP protocol's configured UA, captured when the extension
|
||||
engine is created, matching fetch and XHR. Explicit UA values (including an
|
||||
empty string) take precedence.
|
||||
- `format`: defaults to `"mp4"`. MP4 output always enables fragmentation
|
||||
(`frag_keyframe+empty_moov+default_base_moof`) for non-seekable output. The
|
||||
other accepted muxers are `"mkv"`/`"matroska"`, `"webm"`, and `"mpegts"`.
|
||||
Codec/container incompatibility is an error, not an automatic conversion.
|
||||
Choose the matching filename and Blob content type in the extension.
|
||||
- `args`: optional array of **output options**, not a shell command or a full
|
||||
FFmpeg invocation. Defaults select `0:v:0`, `1:a:0`, and `-c copy`. Additional
|
||||
`-map` options are additive. Allowed options are listed in `outputOptions` in
|
||||
`runtime.go`: metadata, dispositions, bitstream filters, duration/shortest,
|
||||
fragmentation, timestamp/interleaving options and `-strict`. Codec options
|
||||
must remain `copy`. Extra inputs, output paths, reports and arbitrary global
|
||||
options are rejected. Required MP4 fragmentation is enforced by the host.
|
||||
- `signal`: optional `AbortSignal`. Aborting or cancelling the output stops
|
||||
network requests and native execution, cancels input readers, and removes
|
||||
temporary buffers. Await/consume the output inside a live engine or retain
|
||||
it through a Blob URL; unowned work ends with the extension engine.
|
||||
|
||||
The backend defaults to strict input errors (`-xerror`). Native errors include
|
||||
the FFmpeg exit code and a bounded stderr tail. No host directories are exposed
|
||||
to the WASM module; it can access only its two input files and standard output.
|
||||
|
||||
## Input behavior and resource limits
|
||||
|
||||
HTTP inputs start with a 1 MiB Range GET, reusing the bytes for media reads.
|
||||
A validated 206 response with a known total size enables seeking; a 200
|
||||
response is consumed as-is, sequentially. Subsequent ranges verify positions,
|
||||
lengths, total size and available ETag/Last-Modified validators. Without a
|
||||
server validator, same-size remote changes cannot be reliably detected.
|
||||
Content encoding is forced to identity. Authentication/status failures and
|
||||
malformed range responses are errors. There is no full-file seek fallback.
|
||||
|
||||
Sequential JS inputs are drained independently into memory ring buffers that
|
||||
grow on demand up to **32 MiB per input**, independent of total media size.
|
||||
No temporary files are used. This does not enable seeking. Full buffers apply
|
||||
async backpressure until space is available or the operation is cancelled.
|
||||
Extensions must propagate backpressure through their upstream queues and
|
||||
network reads. Independent upstream track requests avoid blocking one track
|
||||
behind another in a shared response. Buffers are released on close.
|
||||
Blob output writes also wait asynchronously for bounded downstream capacity,
|
||||
so a slow consumer does not block the extension event loop.
|
||||
|
||||
At most two WASM merges execute concurrently per process, with a 512 MiB
|
||||
linear-memory limit per instance. Compilation is shared and lazy; the upstream
|
||||
embedded package still decompresses its WASM at process initialization. Native
|
||||
compilation memory, host buffers and JS allocations are additional to the
|
||||
linear-memory limit. Playback support depends on the selected codecs, not just
|
||||
the `.mp4` suffix. Platform execution and performance must be tested on target
|
||||
devices, especially mobile/32-bit devices.
|
||||
@@ -0,0 +1,227 @@
|
||||
// Package ffmpeg runs an embedded FFmpeg with host-controlled media inputs.
|
||||
package ffmpeg
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const httpBlockSize = 1024 * 1024
|
||||
|
||||
// Input is a file visible to FFmpeg. Only verified HTTP Range sources seek.
|
||||
type Input interface {
|
||||
io.ReadCloser
|
||||
Seek(int64, int) (int64, error)
|
||||
Size() int64
|
||||
Seekable() bool
|
||||
}
|
||||
|
||||
var ErrSeek = errors.New("ffmpeg: input requires seeking but source is sequential")
|
||||
|
||||
type HTTPSource struct {
|
||||
URL string `json:"url"`
|
||||
Headers map[string]string `json:"headers"`
|
||||
}
|
||||
|
||||
type httpInput struct {
|
||||
ctx context.Context
|
||||
client *http.Client
|
||||
source HTTPSource
|
||||
body io.ReadCloser
|
||||
size, pos, blockStart int64
|
||||
block []byte
|
||||
ranged bool
|
||||
etag, modified string
|
||||
}
|
||||
|
||||
// OpenHTTP combines capability detection with the first media read. A 200
|
||||
// response remains open and is consumed sequentially without another request.
|
||||
func OpenHTTP(ctx context.Context, client *http.Client, source HTTPSource) (Input, error) {
|
||||
u, err := url.Parse(source.URL)
|
||||
if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" {
|
||||
return nil, errors.New("ffmpeg: expected an HTTP(S) input URL")
|
||||
}
|
||||
h := &httpInput{ctx: ctx, client: client, source: source, size: -1}
|
||||
resp, err := h.request(0, httpBlockSize-1)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if resp.StatusCode == http.StatusOK {
|
||||
if err = identityEncoding(resp); err != nil {
|
||||
resp.Body.Close()
|
||||
return nil, err
|
||||
}
|
||||
h.body, h.size = resp.Body, resp.ContentLength
|
||||
return h, nil
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusPartialContent {
|
||||
return nil, fmt.Errorf("ffmpeg: HTTP input returned %d", resp.StatusCode)
|
||||
}
|
||||
start, end, size, err := contentRange(resp.Header.Get("Content-Range"))
|
||||
if err != nil || start != 0 || end >= httpBlockSize {
|
||||
return nil, errors.New("ffmpeg: invalid initial Content-Range")
|
||||
}
|
||||
h.size, h.ranged = size, true
|
||||
h.etag = resp.Header.Get("ETag")
|
||||
if strings.HasPrefix(h.etag, "W/") {
|
||||
h.etag = ""
|
||||
}
|
||||
h.modified = resp.Header.Get("Last-Modified")
|
||||
h.block, err = readBlock(resp, end-start+1)
|
||||
return h, err
|
||||
}
|
||||
|
||||
func (h *httpInput) request(start, end int64) (*http.Response, error) {
|
||||
req, err := http.NewRequestWithContext(h.ctx, http.MethodGet, h.source.URL, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for k, v := range h.source.Headers {
|
||||
req.Header.Set(k, v)
|
||||
}
|
||||
// Offsets must refer to the actual media bytes, not a decompressed response.
|
||||
req.Header.Set("Accept-Encoding", "identity")
|
||||
req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", start, end))
|
||||
if h.etag != "" {
|
||||
req.Header.Set("If-Range", h.etag)
|
||||
} else if h.modified != "" {
|
||||
req.Header.Set("If-Range", h.modified)
|
||||
}
|
||||
return h.client.Do(req)
|
||||
}
|
||||
|
||||
func identityEncoding(resp *http.Response) error {
|
||||
if enc := resp.Header.Get("Content-Encoding"); enc != "" && !strings.EqualFold(enc, "identity") {
|
||||
return errors.New("ffmpeg: encoded HTTP responses cannot represent media byte offsets")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func contentRange(value string) (start, end, size int64, err error) {
|
||||
err = errors.New("invalid Content-Range")
|
||||
if !strings.HasPrefix(value, "bytes ") {
|
||||
return
|
||||
}
|
||||
parts := strings.Split(strings.TrimPrefix(value, "bytes "), "/")
|
||||
if len(parts) != 2 {
|
||||
return
|
||||
}
|
||||
bounds := strings.Split(parts[0], "-")
|
||||
if len(bounds) != 2 {
|
||||
return
|
||||
}
|
||||
var e error
|
||||
if start, e = strconv.ParseInt(bounds[0], 10, 64); e != nil {
|
||||
return
|
||||
}
|
||||
if end, e = strconv.ParseInt(bounds[1], 10, 64); e != nil {
|
||||
return
|
||||
}
|
||||
if size, e = strconv.ParseInt(parts[1], 10, 64); e != nil {
|
||||
return
|
||||
}
|
||||
if start < 0 || end < start || size <= end {
|
||||
return
|
||||
}
|
||||
err = nil
|
||||
return
|
||||
}
|
||||
|
||||
func readBlock(resp *http.Response, length int64) ([]byte, error) {
|
||||
if err := identityEncoding(resp); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if resp.ContentLength >= 0 && resp.ContentLength != length {
|
||||
return nil, errors.New("ffmpeg: Range response length mismatch")
|
||||
}
|
||||
b, err := io.ReadAll(io.LimitReader(resp.Body, length+1))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if int64(len(b)) != length {
|
||||
return nil, errors.New("ffmpeg: truncated or oversized Range response")
|
||||
}
|
||||
return b, nil
|
||||
}
|
||||
|
||||
func (h *httpInput) Read(p []byte) (int, error) {
|
||||
if len(p) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
if err := h.ctx.Err(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if !h.ranged {
|
||||
return h.body.Read(p)
|
||||
}
|
||||
if h.pos >= h.size {
|
||||
return 0, io.EOF
|
||||
}
|
||||
if h.pos < h.blockStart || h.pos >= h.blockStart+int64(len(h.block)) {
|
||||
end := min(h.pos+httpBlockSize-1, h.size-1)
|
||||
resp, err := h.request(h.pos, end)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusPartialContent {
|
||||
return 0, fmt.Errorf("ffmpeg: Range request returned %d; source changed or stopped supporting Range", resp.StatusCode)
|
||||
}
|
||||
start, actualEnd, size, err := contentRange(resp.Header.Get("Content-Range"))
|
||||
if err != nil || start != h.pos || actualEnd > end || size != h.size {
|
||||
return 0, errors.New("ffmpeg: inconsistent Content-Range")
|
||||
}
|
||||
if h.etag != "" && resp.Header.Get("ETag") != h.etag {
|
||||
return 0, errors.New("ffmpeg: HTTP input ETag changed")
|
||||
}
|
||||
if h.etag == "" && h.modified != "" && resp.Header.Get("Last-Modified") != h.modified {
|
||||
return 0, errors.New("ffmpeg: HTTP input Last-Modified changed")
|
||||
}
|
||||
h.block, err = readBlock(resp, actualEnd-start+1)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
h.blockStart = start
|
||||
}
|
||||
n := copy(p, h.block[h.pos-h.blockStart:])
|
||||
h.pos += int64(n)
|
||||
return n, nil
|
||||
}
|
||||
|
||||
func (h *httpInput) Seek(offset int64, whence int) (int64, error) {
|
||||
if !h.ranged {
|
||||
return 0, ErrSeek
|
||||
}
|
||||
base := int64(0)
|
||||
switch whence {
|
||||
case io.SeekStart:
|
||||
case io.SeekCurrent:
|
||||
base = h.pos
|
||||
case io.SeekEnd:
|
||||
base = h.size
|
||||
default:
|
||||
return 0, errors.New("invalid seek origin")
|
||||
}
|
||||
next := base + offset
|
||||
if next < 0 || (offset > 0 && next < base) {
|
||||
return 0, errors.New("invalid seek offset")
|
||||
}
|
||||
h.pos = next
|
||||
return next, nil
|
||||
}
|
||||
|
||||
func (h *httpInput) Size() int64 { return h.size }
|
||||
func (h *httpInput) Seekable() bool { return h.ranged }
|
||||
func (h *httpInput) Close() error {
|
||||
if h.body != nil {
|
||||
return h.body.Close()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,222 @@
|
||||
package ffmpeg
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestHTTPRange(t *testing.T) {
|
||||
data := bytes.Repeat([]byte("0123456789"), 300000)
|
||||
var requests atomic.Int32
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
count := requests.Add(1)
|
||||
if r.Header.Get("X-Test") != "secret" {
|
||||
t.Error("missing headers")
|
||||
}
|
||||
if r.Header.Get("Accept-Encoding") != "identity" {
|
||||
t.Error("missing identity encoding")
|
||||
}
|
||||
if count > 1 && r.Header.Get("If-Range") != `"v1"` {
|
||||
t.Error("missing If-Range")
|
||||
}
|
||||
w.Header().Set("ETag", `"v1"`)
|
||||
http.ServeContent(w, r, "media", time.Time{}, bytes.NewReader(data))
|
||||
}))
|
||||
defer srv.Close()
|
||||
in, err := OpenHTTP(context.Background(), srv.Client(), HTTPSource{URL: srv.URL, Headers: map[string]string{"X-Test": "secret"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer in.Close()
|
||||
if !in.Seekable() || in.Size() != int64(len(data)) {
|
||||
t.Fatal("range not detected")
|
||||
}
|
||||
for _, off := range []int64{0, int64(len(data) - 16), 12, int64(httpBlockSize + 35)} {
|
||||
if _, err = in.Seek(off, io.SeekStart); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b := make([]byte, 16)
|
||||
if _, err = io.ReadFull(in, b); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !bytes.Equal(b, data[off:off+16]) {
|
||||
t.Fatalf("wrong bytes at %d", off)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestHTTPSequentialAndValidation(t *testing.T) {
|
||||
for _, kind := range []string{"sequential", "bad-range", "changed", "truncated", "unauthorized"} {
|
||||
t.Run(kind, func(t *testing.T) {
|
||||
var requests atomic.Int32
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
count := requests.Add(1)
|
||||
switch kind {
|
||||
case "sequential":
|
||||
w.Write([]byte("media"))
|
||||
case "unauthorized":
|
||||
w.WriteHeader(403)
|
||||
case "bad-range":
|
||||
w.Header().Set("Content-Range", "bytes 2-6/10")
|
||||
w.WriteHeader(206)
|
||||
w.Write([]byte("media"))
|
||||
case "truncated":
|
||||
w.Header().Set("Content-Range", "bytes 0-9/10")
|
||||
w.WriteHeader(206)
|
||||
w.Write([]byte("media"))
|
||||
case "changed":
|
||||
if count == 1 {
|
||||
w.Header().Set("Content-Range", "bytes 0-4/10")
|
||||
w.Header().Set("ETag", `"v1"`)
|
||||
w.WriteHeader(206)
|
||||
}
|
||||
w.Write([]byte("media"))
|
||||
}
|
||||
}))
|
||||
defer srv.Close()
|
||||
in, err := OpenHTTP(context.Background(), srv.Client(), HTTPSource{URL: srv.URL})
|
||||
if kind == "sequential" {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer in.Close()
|
||||
if in.Seekable() {
|
||||
t.Fatal("unexpected seeking")
|
||||
}
|
||||
if _, err = in.Seek(0, 0); err != ErrSeek {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b, err := io.ReadAll(in)
|
||||
if err != nil || string(b) != "media" || requests.Load() != 1 {
|
||||
t.Fatalf("%q %v count %d", b, err, requests.Load())
|
||||
}
|
||||
} else if kind == "changed" {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer in.Close()
|
||||
in.Seek(5, 0)
|
||||
if _, err = in.Read(make([]byte, 2)); err == nil {
|
||||
t.Fatal("accepted 200 after range")
|
||||
}
|
||||
} else if err == nil {
|
||||
in.Close()
|
||||
t.Fatal("accepted invalid response")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestStreamRing(t *testing.T) {
|
||||
s := NewStreamInput(context.Background())
|
||||
defer s.Close()
|
||||
// Start near the boundary to exercise wrap-around without a huge fixture.
|
||||
s.buffer = make([]byte, 256*1024)
|
||||
s.read = int64(len(s.buffer)) - 3
|
||||
s.written = s.read
|
||||
if _, err := s.Write([]byte("abcdefgh")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
s.End(nil)
|
||||
b, err := io.ReadAll(s)
|
||||
if err != nil || string(b) != "abcdefgh" {
|
||||
t.Fatalf("%q %v", b, err)
|
||||
}
|
||||
if err = s.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if s.buffer != nil {
|
||||
t.Fatal("buffer retained after close")
|
||||
}
|
||||
}
|
||||
|
||||
func TestOutputArguments(t *testing.T) {
|
||||
a, err := Arguments("", []string{"-shortest", "-metadata", "title=test"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(strings.Join(a, " "), "-f mp4 pipe:1") {
|
||||
t.Fatal(a)
|
||||
}
|
||||
for _, args := range [][]string{{"-i", "/etc/passwd"}, {"-report"}, {"-c:v", "libx264"}, {"-metadata"}, {"other.mp4"}} {
|
||||
if _, err := Arguments("mp4", args); err == nil {
|
||||
t.Fatal(fmt.Sprint(args))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestStreamBackpressureAndCancel(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
s := NewStreamInput(ctx)
|
||||
defer s.Close()
|
||||
s.buffer = make([]byte, StreamBufferLimit)
|
||||
s.written = StreamBufferLimit
|
||||
finished := make(chan error, 1)
|
||||
go func() { _, err := s.Write([]byte{42}); finished <- err }()
|
||||
select {
|
||||
case err := <-finished:
|
||||
t.Fatalf("did not apply backpressure: %v", err)
|
||||
case <-time.After(10 * time.Millisecond):
|
||||
}
|
||||
if _, err := s.Read(make([]byte, 1)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
select {
|
||||
case err := <-finished:
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("writer did not resume")
|
||||
}
|
||||
go func() { _, err := s.Write([]byte{43}); finished <- err }()
|
||||
cancel()
|
||||
select {
|
||||
case err := <-finished:
|
||||
if err == nil {
|
||||
t.Fatal("cancelled write succeeded")
|
||||
}
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("writer did not cancel")
|
||||
}
|
||||
}
|
||||
|
||||
func TestStreamMemoryGrowthAndWrap(t *testing.T) {
|
||||
s := NewStreamInput(context.Background())
|
||||
defer s.Close()
|
||||
if len(s.buffer) != 0 {
|
||||
t.Fatal("buffer allocated before first write")
|
||||
}
|
||||
a := bytes.Repeat([]byte{1}, 200*1024)
|
||||
b := bytes.Repeat([]byte{2}, 180*1024)
|
||||
s.Write(a)
|
||||
consumed := make([]byte, 180*1024)
|
||||
if _, err := io.ReadFull(s, consumed); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
s.Write(b) // Wrap in the initial 256 KiB allocation.
|
||||
c := bytes.Repeat([]byte{3}, 300*1024)
|
||||
s.Write(c) // Grow while unread data wraps around the old ring.
|
||||
s.End(nil)
|
||||
got, err := io.ReadAll(s)
|
||||
want := append(append(append([]byte{}, a[len(consumed):]...), b...), c...)
|
||||
if err != nil || !bytes.Equal(got, want) {
|
||||
t.Fatalf("growth corrupted data: %v", err)
|
||||
}
|
||||
if len(s.buffer) > StreamBufferLimit {
|
||||
t.Fatal("buffer exceeded limit")
|
||||
}
|
||||
s.Close()
|
||||
if s.buffer != nil {
|
||||
t.Fatal("buffer retained on close")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,254 @@
|
||||
package ffmpeg
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
ffembed "codeberg.org/gruf/go-ffmpreg/embed"
|
||||
"codeberg.org/gruf/go-ffmpreg/wasm"
|
||||
"github.com/tetratelabs/wazero"
|
||||
expsys "github.com/tetratelabs/wazero/experimental/sys"
|
||||
"github.com/tetratelabs/wazero/experimental/sysfs"
|
||||
"github.com/tetratelabs/wazero/sys"
|
||||
)
|
||||
|
||||
var shared struct {
|
||||
once sync.Once
|
||||
runtime wazero.Runtime
|
||||
module wazero.CompiledModule
|
||||
err error
|
||||
}
|
||||
|
||||
// Limit active instances independently of the number of extension engines.
|
||||
var slots = make(chan struct{}, 2)
|
||||
|
||||
func initialize() {
|
||||
shared.once.Do(func() {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
shared.err = fmt.Errorf("ffmpeg initialization: %v", r)
|
||||
}
|
||||
}()
|
||||
cfg := wazero.NewRuntimeConfig().WithCloseOnContextDone(true).WithMemoryLimitPages(8192)
|
||||
shared.runtime, shared.err = wasm.NewRuntime(context.Background(), cfg)
|
||||
if shared.err != nil {
|
||||
return
|
||||
}
|
||||
shared.module, shared.err = shared.runtime.CompileModule(context.Background(), ffembed.B())
|
||||
if shared.err != nil {
|
||||
shared.runtime.Close(context.Background())
|
||||
return
|
||||
}
|
||||
ffembed.Free()
|
||||
})
|
||||
}
|
||||
|
||||
// Output arguments deliberately do not accept extra inputs, output paths or
|
||||
// global filesystem/logging options. The host owns all I/O and cancellation.
|
||||
var outputOptions = map[string]int{
|
||||
"-c": 1, "-codec": 1, "-c:v": 1, "-c:a": 1, "-codec:v": 1, "-codec:a": 1,
|
||||
"-map": 1, "-map_metadata": 1, "-map_chapters": 1, "-metadata": 1, "-metadata:s:a:0": 1, "-metadata:s:v:0": 1,
|
||||
"-movflags": 1, "-frag_duration": 1, "-frag_size": 1, "-min_frag_duration": 1,
|
||||
"-bsf:v": 1, "-bsf:a": 1, "-strict": 1, "-shortest": 0, "-t": 1, "-to": 1,
|
||||
"-avoid_negative_ts": 1, "-max_interleave_delta": 1, "-flush_packets": 1,
|
||||
"-disposition:a:0": 1, "-disposition:v:0": 1,
|
||||
}
|
||||
|
||||
func Arguments(format string, extra []string) ([]string, error) {
|
||||
if format == "" {
|
||||
format = "mp4"
|
||||
}
|
||||
if format == "mkv" {
|
||||
format = "matroska"
|
||||
}
|
||||
switch format {
|
||||
case "mp4", "matroska", "webm", "mpegts":
|
||||
default:
|
||||
return nil, fmt.Errorf("ffmpeg: unsupported output format %q", format)
|
||||
}
|
||||
for i := 0; i < len(extra); {
|
||||
count, ok := outputOptions[extra[i]]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("ffmpeg: unsupported output option %q", extra[i])
|
||||
}
|
||||
if i+count >= len(extra) {
|
||||
return nil, fmt.Errorf("ffmpeg: missing value for %s", extra[i])
|
||||
}
|
||||
if strings.HasPrefix(extra[i], "-c") && extra[i+1] != "copy" {
|
||||
return nil, errors.New("ffmpeg: merge only supports stream copy")
|
||||
}
|
||||
i += count + 1
|
||||
}
|
||||
args := []string{"-hide_banner", "-nostdin", "-xerror", "-loglevel", "error", "-protocol_whitelist", "file,pipe", "-i", "/inputs/video", "-protocol_whitelist", "file,pipe", "-i", "/inputs/audio", "-map", "0:v:0", "-map", "1:a:0", "-c", "copy"}
|
||||
fragmentFlags := ""
|
||||
for i := 0; i < len(extra); {
|
||||
count := outputOptions[extra[i]]
|
||||
if format == "mp4" && extra[i] == "-movflags" {
|
||||
if strings.Contains(extra[i+1], "faststart") || strings.Contains(extra[i+1], "global_sidx") {
|
||||
return nil, errors.New("ffmpeg: these movflags require seekable output")
|
||||
}
|
||||
fragmentFlags = extra[i+1]
|
||||
} else {
|
||||
args = append(args, extra[i:i+count+1]...)
|
||||
}
|
||||
i += count + 1
|
||||
}
|
||||
if format == "mp4" {
|
||||
args = append(args, "-movflags", fragmentFlags+"+frag_keyframe+empty_moov+default_base_moof")
|
||||
}
|
||||
args = append(args, "-f", format, "pipe:1")
|
||||
return args, nil
|
||||
}
|
||||
|
||||
// Run shares compilation, but gives every merge its own memory and filesystem.
|
||||
// A clean stdout EOF is only delivered by the caller after this succeeds.
|
||||
func Run(ctx context.Context, video, audio Input, out io.Writer, format string, extra []string) error {
|
||||
_, err := Arguments(format, extra)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
release, err := Acquire(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer release()
|
||||
return RunAcquired(ctx, video, audio, out, format, extra)
|
||||
}
|
||||
|
||||
// Acquire reserves execution capacity before input factories or HTTP requests start.
|
||||
func Acquire(ctx context.Context) (func(), error) {
|
||||
select {
|
||||
case slots <- struct{}{}:
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
var once sync.Once
|
||||
return func() { once.Do(func() { <-slots }) }, nil
|
||||
}
|
||||
|
||||
// RunAcquired requires a reservation held until this call returns.
|
||||
func RunAcquired(ctx context.Context, video, audio Input, out io.Writer, format string, extra []string) error {
|
||||
args, err := Arguments(format, extra)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ready := make(chan struct{})
|
||||
go func() { initialize(); close(ready) }()
|
||||
select {
|
||||
case <-ready:
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
if shared.err != nil {
|
||||
return shared.err
|
||||
}
|
||||
files := &inputFS{files: map[string]Input{"video": video, "audio": audio}}
|
||||
fsconfig := wazero.NewFSConfig().(sysfs.FSConfig).WithSysFSMount(files, "/inputs")
|
||||
logs := &tailLog{}
|
||||
rc, err := wasm.Run(ctx, shared.runtime, shared.module, wasm.Args{
|
||||
Name: "ffmpeg", Args: args, Stdout: out, Stderr: logs,
|
||||
Config: func(c wazero.ModuleConfig) wazero.ModuleConfig { return c.WithFSConfig(fsconfig) },
|
||||
})
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
if files.err != nil {
|
||||
return fmt.Errorf("ffmpeg input: %w (%s)", files.err, logs.String())
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("ffmpeg runtime: %w (%s)", err, logs.String())
|
||||
}
|
||||
if rc != 0 {
|
||||
return fmt.Errorf("ffmpeg exited with code %d: %s", rc, logs.String())
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type tailLog struct{ data []byte }
|
||||
|
||||
func (l *tailLog) Write(p []byte) (int, error) {
|
||||
n := len(p)
|
||||
if len(p) > 8192 {
|
||||
p = p[len(p)-8192:]
|
||||
}
|
||||
l.data = append(l.data, p...)
|
||||
if len(l.data) > 8192 {
|
||||
l.data = l.data[len(l.data)-8192:]
|
||||
}
|
||||
return n, nil
|
||||
}
|
||||
func (l *tailLog) String() string { return strings.TrimSpace(string(l.data)) }
|
||||
|
||||
type inputFS struct {
|
||||
expsys.UnimplementedFS
|
||||
files map[string]Input
|
||||
err error
|
||||
}
|
||||
|
||||
func (f *inputFS) OpenFile(path string, flag expsys.Oflag, perm fs.FileMode) (expsys.File, expsys.Errno) {
|
||||
if flag&(expsys.O_WRONLY|expsys.O_RDWR|expsys.O_CREAT|expsys.O_TRUNC) != 0 {
|
||||
return nil, expsys.EROFS
|
||||
}
|
||||
path = strings.TrimPrefix(path, "/")
|
||||
if path == "." || path == "" {
|
||||
return &inputFile{owner: f}, 0
|
||||
}
|
||||
in := f.files[path]
|
||||
if in == nil {
|
||||
return nil, expsys.ENOENT
|
||||
}
|
||||
return &inputFile{owner: f, in: in}, 0
|
||||
}
|
||||
func (f *inputFS) Stat(path string) (sys.Stat_t, expsys.Errno) {
|
||||
file, err := f.OpenFile(path, 0, 0)
|
||||
if err != 0 {
|
||||
return sys.Stat_t{}, err
|
||||
}
|
||||
return file.Stat()
|
||||
}
|
||||
|
||||
type inputFile struct {
|
||||
expsys.UnimplementedFile
|
||||
owner *inputFS
|
||||
in Input
|
||||
}
|
||||
|
||||
func (f *inputFile) IsDir() (bool, expsys.Errno) { return f.in == nil, 0 }
|
||||
func (f *inputFile) Stat() (sys.Stat_t, expsys.Errno) {
|
||||
if f.in == nil {
|
||||
return sys.Stat_t{Mode: fs.ModeDir | 0555}, 0
|
||||
}
|
||||
mode := fs.FileMode(0444)
|
||||
if !f.in.Seekable() {
|
||||
mode |= fs.ModeNamedPipe
|
||||
}
|
||||
return sys.Stat_t{Mode: mode, Size: max(0, f.in.Size()), Nlink: 1}, 0
|
||||
}
|
||||
func (f *inputFile) Read(p []byte) (int, expsys.Errno) {
|
||||
if f.in == nil {
|
||||
return 0, expsys.EISDIR
|
||||
}
|
||||
n, err := f.in.Read(p)
|
||||
if err != nil && err != io.EOF {
|
||||
f.owner.err = err
|
||||
return n, expsys.EIO
|
||||
}
|
||||
return n, 0
|
||||
}
|
||||
func (f *inputFile) Seek(off int64, whence int) (int64, expsys.Errno) {
|
||||
if f.in == nil {
|
||||
return 0, expsys.EISDIR
|
||||
}
|
||||
n, err := f.in.Seek(off, whence)
|
||||
// Capability probes may legitimately fail; a failed seek alone is not a
|
||||
// media error. FFmpeg decides whether it can continue sequentially.
|
||||
if err != nil {
|
||||
return 0, expsys.ENOSYS
|
||||
}
|
||||
return n, 0
|
||||
}
|
||||
@@ -0,0 +1,215 @@
|
||||
package ffmpeg
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/binary"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"codeberg.org/gruf/go-ffmpreg/wasm"
|
||||
"github.com/tetratelabs/wazero"
|
||||
)
|
||||
|
||||
// Generate tiny deterministic media with the exact embedded build under test;
|
||||
// no system FFmpeg, network media or third-party fixture licensing is needed.
|
||||
func fixture(t *testing.T, audio bool) []byte {
|
||||
t.Helper()
|
||||
initialize()
|
||||
if shared.err != nil {
|
||||
t.Fatal(shared.err)
|
||||
}
|
||||
args := []string{"-hide_banner", "-loglevel", "error", "-f", "lavfi", "-i", "color=c=black:s=32x32:r=10", "-t", "0.3", "-c:v", "libx264", "-preset", "ultrafast", "-movflags", "frag_keyframe+empty_moov", "-f", "mp4", "pipe:1"}
|
||||
if audio {
|
||||
args = []string{"-hide_banner", "-loglevel", "error", "-f", "lavfi", "-i", "anullsrc=r=48000:cl=mono", "-t", "0.3", "-c:a", "aac", "-movflags", "frag_keyframe+empty_moov", "-f", "mp4", "pipe:1"}
|
||||
}
|
||||
var out, logs bytes.Buffer
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
rc, err := wasm.Run(ctx, shared.runtime, shared.module, wasm.Args{Name: "ffmpeg", Args: args, Stdout: &out, Stderr: &logs})
|
||||
if err != nil || rc != 0 {
|
||||
t.Fatalf("fixture: %d %v %s", rc, err, logs.String())
|
||||
}
|
||||
return out.Bytes()
|
||||
}
|
||||
|
||||
func TestTailMoovNeedsRange(t *testing.T) {
|
||||
initialize()
|
||||
if shared.err != nil {
|
||||
t.Fatal(shared.err)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
dir := t.TempDir()
|
||||
var logs bytes.Buffer
|
||||
rc, err := wasm.Run(ctx, shared.runtime, shared.module, wasm.Args{Name: "ffmpeg", Args: []string{
|
||||
"-v", "error", "-f", "lavfi", "-i", "color=c=black:s=32x32:r=10", "-t", "0.3", "-c:v", "libx264", "-preset", "ultrafast", "/out/video.mp4",
|
||||
}, Stderr: &logs, Config: func(c wazero.ModuleConfig) wazero.ModuleConfig {
|
||||
return c.WithFSConfig(wazero.NewFSConfig().WithDirMount(dir, "/out"))
|
||||
}})
|
||||
if rc != 0 || err != nil {
|
||||
t.Fatalf("fixture %d %v %s", rc, err, logs.String())
|
||||
}
|
||||
raw, err := os.ReadFile(filepath.Join(dir, "video.mp4"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var video []byte
|
||||
for offset := 0; offset < len(raw); {
|
||||
length := int(binary.BigEndian.Uint32(raw[offset : offset+4]))
|
||||
if string(raw[offset+4:offset+8]) == "moov" {
|
||||
free := make([]byte, 2*httpBlockSize)
|
||||
binary.BigEndian.PutUint32(free, uint32(len(free)))
|
||||
copy(free[4:], "free")
|
||||
video = append(video, free...)
|
||||
}
|
||||
video = append(video, raw[offset:offset+length]...)
|
||||
offset += length
|
||||
}
|
||||
audio := fixture(t, true)
|
||||
for _, ranged := range []bool{true, false} {
|
||||
t.Run(fmtBool(ranged), func(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
data := video
|
||||
if r.URL.Path == "/audio" {
|
||||
data = audio
|
||||
}
|
||||
if ranged {
|
||||
http.ServeContent(w, r, "media", time.Time{}, bytes.NewReader(data))
|
||||
} else {
|
||||
w.Write(data)
|
||||
}
|
||||
}))
|
||||
defer srv.Close()
|
||||
v, err := OpenHTTP(ctx, srv.Client(), HTTPSource{URL: srv.URL})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer v.Close()
|
||||
a, err := OpenHTTP(ctx, srv.Client(), HTTPSource{URL: srv.URL + "/audio"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer a.Close()
|
||||
err = Run(ctx, v, a, io.Discard, "mp4", nil)
|
||||
if ranged && err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !ranged && err == nil {
|
||||
t.Fatal("non-seekable tail-moov input unexpectedly succeeded")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func fmtBool(b bool) string {
|
||||
if b {
|
||||
return "range"
|
||||
}
|
||||
return "sequential"
|
||||
}
|
||||
|
||||
func TestWebMToMP4(t *testing.T) {
|
||||
initialize()
|
||||
if shared.err != nil {
|
||||
t.Fatal(shared.err)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
inputs := make([]*StreamInput, 2)
|
||||
for i, name := range []string{"video.webm", "audio.webm"} {
|
||||
data, err := os.ReadFile(filepath.Join("..", "testdata", "ffmpeg", name))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
inputs[i] = NewStreamInput(ctx)
|
||||
defer inputs[i].Close()
|
||||
if _, err = inputs[i].Write(data); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
inputs[i].End(nil)
|
||||
}
|
||||
|
||||
var output, probe, logs bytes.Buffer
|
||||
if err := Run(ctx, inputs[0], inputs[1], &output, "mp4", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rc, err := wasm.Run(ctx, shared.runtime, shared.module, wasm.Args{Name: "ffprobe", Args: []string{"-v", "error", "-show_entries", "stream=codec_name", "-of", "json", "pipe:0"}, Stdin: bytes.NewReader(output.Bytes()), Stdout: &probe, Stderr: &logs})
|
||||
if rc != 0 || err != nil {
|
||||
t.Fatalf("probe %d %v %s", rc, err, logs.String())
|
||||
}
|
||||
if !bytes.Contains(probe.Bytes(), []byte(`"vp9"`)) || !bytes.Contains(probe.Bytes(), []byte(`"opus"`)) {
|
||||
t.Fatalf("codecs changed: %s", probe.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestWASMMerge(t *testing.T) {
|
||||
video, audio := fixture(t, false), fixture(t, true)
|
||||
for _, ranged := range []bool{false, true} {
|
||||
t.Run(map[bool]string{false: "sequential", true: "range"}[ranged], func(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
data := video
|
||||
if r.URL.Path == "/audio" {
|
||||
data = audio
|
||||
}
|
||||
if ranged {
|
||||
http.ServeContent(w, r, "media", time.Time{}, bytes.NewReader(data))
|
||||
} else {
|
||||
w.Write(data)
|
||||
}
|
||||
}))
|
||||
defer srv.Close()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
v, err := OpenHTTP(ctx, srv.Client(), HTTPSource{URL: srv.URL + "/video"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer v.Close()
|
||||
a, err := OpenHTTP(ctx, srv.Client(), HTTPSource{URL: srv.URL + "/audio"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer a.Close()
|
||||
var output bytes.Buffer
|
||||
if err = Run(ctx, v, a, &output, "", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !bytes.Contains(output.Bytes(), []byte("moof")) {
|
||||
t.Fatal("output is not fragmented MP4")
|
||||
}
|
||||
// Probe the actual result, rather than just checking its filename/header.
|
||||
var probe, logs bytes.Buffer
|
||||
rc, err := wasm.Run(ctx, shared.runtime, shared.module, wasm.Args{Name: "ffprobe", Args: []string{"-v", "error", "-show_streams", "-of", "json", "pipe:0"}, Stdin: bytes.NewReader(output.Bytes()), Stdout: &probe, Stderr: &logs})
|
||||
if rc != 0 || err != nil {
|
||||
t.Fatalf("probe: %d %v %s", rc, err, logs.String())
|
||||
}
|
||||
var result struct {
|
||||
Streams []struct {
|
||||
CodecType string `json:"codec_type"`
|
||||
CodecName string `json:"codec_name"`
|
||||
}
|
||||
}
|
||||
if err = json.Unmarshal(probe.Bytes(), &result); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(result.Streams) != 2 || result.Streams[0].CodecName != "h264" || result.Streams[1].CodecName != "aac" {
|
||||
t.Fatalf("unexpected streams %s", probe.String())
|
||||
}
|
||||
})
|
||||
}
|
||||
// A malformed input must produce an error, not successful empty output.
|
||||
s := NewStreamInput(context.Background())
|
||||
defer s.Close()
|
||||
s.Write([]byte("invalid"))
|
||||
s.End(nil)
|
||||
if err := Run(context.Background(), s, s, io.Discard, "", nil); err == nil {
|
||||
t.Fatal("accepted invalid media")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,163 @@
|
||||
package ffmpeg
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// A bounded memory ring lets both JS inputs drain independently, including when
|
||||
// they share a SABR producer. It does not make a sequential input seekable.
|
||||
// A full ring waits for consumption or cancellation. Producers must propagate
|
||||
// backpressure and allow each track to make progress independently.
|
||||
const StreamBufferLimit = 32 * 1024 * 1024
|
||||
|
||||
type StreamInput struct {
|
||||
ctx context.Context
|
||||
mu sync.Mutex
|
||||
buffer []byte
|
||||
read, written int64
|
||||
ended, closed bool
|
||||
err error
|
||||
wake chan struct{}
|
||||
space chan struct{}
|
||||
}
|
||||
|
||||
func NewStreamInput(ctx context.Context) *StreamInput {
|
||||
return &StreamInput{ctx: ctx, wake: make(chan struct{}, 1), space: make(chan struct{}, 1)}
|
||||
}
|
||||
|
||||
func (s *StreamInput) Write(p []byte) (int, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if len(p) > StreamBufferLimit {
|
||||
return 0, errors.New("ffmpeg: input chunk exceeds buffer capacity")
|
||||
}
|
||||
for int64(len(p)) > StreamBufferLimit-(s.written-s.read) && !s.closed && !s.ended {
|
||||
s.mu.Unlock()
|
||||
var err error
|
||||
select {
|
||||
case <-s.space:
|
||||
case <-s.ctx.Done():
|
||||
err = s.ctx.Err()
|
||||
}
|
||||
s.mu.Lock()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
if s.closed || s.ended {
|
||||
return 0, io.ErrClosedPipe
|
||||
}
|
||||
if err := s.ctx.Err(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if len(p) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
// Grow only as needed. Rebase unread bytes when the ring grows.
|
||||
need := int(s.written-s.read) + len(p)
|
||||
if need > len(s.buffer) {
|
||||
capacity := min(StreamBufferLimit, max(256*1024, max(need, len(s.buffer)*2)))
|
||||
buffer := make([]byte, capacity)
|
||||
unread := int(s.written - s.read)
|
||||
if unread > 0 {
|
||||
off := int(s.read % int64(len(s.buffer)))
|
||||
first := min(unread, len(s.buffer)-off)
|
||||
copy(buffer, s.buffer[off:off+first])
|
||||
copy(buffer[first:], s.buffer[:unread-first])
|
||||
}
|
||||
s.buffer = buffer
|
||||
s.read, s.written = 0, int64(unread)
|
||||
}
|
||||
off := int(s.written % int64(len(s.buffer)))
|
||||
first := min(len(p), len(s.buffer)-off)
|
||||
copy(s.buffer[off:], p[:first])
|
||||
copy(s.buffer, p[first:])
|
||||
s.written += int64(len(p))
|
||||
select {
|
||||
case s.wake <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
return len(p), nil
|
||||
}
|
||||
|
||||
func (s *StreamInput) End(err error) {
|
||||
s.mu.Lock()
|
||||
s.ended = true
|
||||
if s.err == nil {
|
||||
s.err = err
|
||||
}
|
||||
s.mu.Unlock()
|
||||
select {
|
||||
case s.space <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
select {
|
||||
case s.wake <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
func (s *StreamInput) Read(p []byte) (int, error) {
|
||||
if len(p) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
for {
|
||||
if err := s.ctx.Err(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
s.mu.Lock()
|
||||
if s.closed {
|
||||
s.mu.Unlock()
|
||||
return 0, io.ErrClosedPipe
|
||||
}
|
||||
if s.err != nil {
|
||||
err := s.err
|
||||
s.mu.Unlock()
|
||||
return 0, err
|
||||
}
|
||||
if s.read < s.written {
|
||||
off := s.read % int64(len(s.buffer))
|
||||
n := min(int64(len(p)), s.written-s.read, int64(len(s.buffer))-off)
|
||||
count := copy(p, s.buffer[off:off+n])
|
||||
s.read += int64(count)
|
||||
s.mu.Unlock()
|
||||
select {
|
||||
case s.space <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
return count, nil
|
||||
}
|
||||
ended := s.ended
|
||||
s.mu.Unlock()
|
||||
if ended {
|
||||
return 0, io.EOF
|
||||
}
|
||||
select {
|
||||
case <-s.wake:
|
||||
case <-s.ctx.Done():
|
||||
return 0, s.ctx.Err()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *StreamInput) Close() error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.closed = true
|
||||
select {
|
||||
case s.space <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
select {
|
||||
case s.wake <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
s.buffer = nil
|
||||
return nil
|
||||
}
|
||||
func (*StreamInput) Seek(int64, int) (int64, error) { return 0, ErrSeek }
|
||||
func (*StreamInput) Size() int64 { return -1 }
|
||||
func (*StreamInput) Seekable() bool { return false }
|
||||
@@ -0,0 +1,321 @@
|
||||
package engine
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
media "github.com/GopeedLab/gopeed/pkg/download/engine/ffmpeg"
|
||||
"github.com/GopeedLab/gopeed/pkg/download/engine/inject/stream"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestFFmpegJSStreams(t *testing.T) {
|
||||
video, err := os.ReadFile("testdata/ffmpeg/video.mp4")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
audio, err := os.ReadFile("testdata/ffmpeg/audio.mp4")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, mode := range []string{"streams", "http", "mixed"} {
|
||||
t.Run(mode, func(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
b := video
|
||||
if r.URL.Path == "/audio" {
|
||||
b = audio
|
||||
}
|
||||
http.ServeContent(w, r, "media", time.Time{}, bytes.NewReader(b))
|
||||
}))
|
||||
defer srv.Close()
|
||||
e := NewEngine(nil)
|
||||
defer e.Close()
|
||||
// Input data is deliberately interleaved through one JS producer.
|
||||
// Native push/read operations must never block the event loop.
|
||||
e.Runtime.Set("videoBytes", e.Runtime.NewArrayBuffer(video))
|
||||
e.Runtime.Set("audioBytes", e.Runtime.NewArrayBuffer(audio))
|
||||
e.Runtime.Set("sourceURL", srv.URL)
|
||||
e.Runtime.Set("inputMode", mode)
|
||||
result, err := e.RunString(`(async()=>{
|
||||
let vc, ac, offset=0;
|
||||
const v=new Uint8Array(videoBytes), a=new Uint8Array(audioBytes);
|
||||
const video=new ReadableStream({start(c){vc=c;}});
|
||||
const audio=new ReadableStream({start(c){ac=c;}});
|
||||
const timer=setInterval(()=>{
|
||||
if(offset<v.length)vc.enqueue(v.slice(offset,offset+97));
|
||||
if(offset<a.length)ac.enqueue(a.slice(offset,offset+97));
|
||||
offset+=97;
|
||||
if(offset>=Math.max(v.length,a.length)){clearInterval(timer);vc.close();ac.close();}
|
||||
},1);
|
||||
const stream=__gopeed_ffmpeg.merge({
|
||||
inputs: () => ({ video:inputMode==='http'?{url:sourceURL+'/video'}:video,
|
||||
audio:inputMode==='streams'?audio:{url:sourceURL+'/audio'} }),
|
||||
args:['-metadata','title=gopeed-test']
|
||||
});
|
||||
const bytes=await new Response(stream).arrayBuffer();
|
||||
return new Uint8Array(bytes);
|
||||
})()`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out, ok := result.([]byte)
|
||||
if !ok {
|
||||
t.Fatalf("output %T", result)
|
||||
}
|
||||
if !bytes.Contains(out, []byte("moof")) || !bytes.Contains(out, []byte("gopeed-test")) {
|
||||
t.Fatalf("invalid output (%d bytes)", len(out))
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestFFmpegJSErrorAndCancel(t *testing.T) {
|
||||
for _, scenario := range []string{"input-error", "abort", "output-cancel", "invalid-args"} {
|
||||
t.Run(scenario, func(t *testing.T) {
|
||||
e := NewEngine(nil)
|
||||
defer e.Close()
|
||||
e.Runtime.Set("scenario", scenario)
|
||||
result, err := e.RunString(`(async()=>{
|
||||
let cancelled=0;
|
||||
const source=()=>new ReadableStream({
|
||||
pull(c){if(scenario==='input-error')c.error(new Error('upstream failed'));else return new Promise(()=>{});},
|
||||
cancel(){cancelled++;}
|
||||
});
|
||||
const controller=new AbortController();
|
||||
const output=__gopeed_ffmpeg.merge({inputs:()=>({video:source(),audio:source()}),signal:controller.signal,
|
||||
args:scenario==='invalid-args'?['-i','/outside']:[]});
|
||||
const reader=output.getReader();
|
||||
const pending=reader.read().then(()=>'',e=>e.message);
|
||||
if(scenario==='abort')setTimeout(()=>controller.abort(),20);
|
||||
if(scenario==='output-cancel')setTimeout(()=>reader.cancel(),20);
|
||||
const message=await pending;
|
||||
await new Promise(r=>setTimeout(r,10));
|
||||
return {message,cancelled};
|
||||
})()`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m := result.(map[string]any)
|
||||
if scenario != "output-cancel" && m["message"] == "" {
|
||||
t.Fatalf("missing error: %v", m)
|
||||
}
|
||||
if scenario == "abort" || scenario == "output-cancel" {
|
||||
if fmt.Sprint(m["cancelled"]) != "2" {
|
||||
t.Fatalf("inputs not cancelled: %v", m)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestFFmpegMalformedMediaFails(t *testing.T) {
|
||||
e := NewEngine(nil)
|
||||
defer e.Close()
|
||||
_, err := e.RunString(`(async()=>{
|
||||
const source=()=>new ReadableStream({start(c){c.enqueue(new Uint8Array([1,2,3]));c.close();}});
|
||||
return await new Response(__gopeed_ffmpeg.merge({inputs:()=>({video:source(),audio:source()})})).arrayBuffer();
|
||||
})()`)
|
||||
if err == nil || !strings.Contains(err.Error(), "ffmpeg") {
|
||||
t.Fatalf("expected FFmpeg error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFFmpegBlobOpener(t *testing.T) {
|
||||
var open stream.ObjectURLOpener
|
||||
e := NewEngine(&Config{StreamConfig: &stream.Config{CreateObjectURL: func(opts *stream.ObjectURLOptions, opener stream.ObjectURLOpener) (string, error) {
|
||||
if opts.Range || opts.Size != 0 {
|
||||
t.Fatal("merged output must not advertise range or a guessed size")
|
||||
}
|
||||
open = opener
|
||||
return "blob:ffmpeg-test", nil
|
||||
}}})
|
||||
defer e.Close()
|
||||
video, _ := os.ReadFile("testdata/ffmpeg/video.mp4")
|
||||
audio, _ := os.ReadFile("testdata/ffmpeg/audio.mp4")
|
||||
e.Runtime.Set("videoBytes", e.Runtime.NewArrayBuffer(video))
|
||||
e.Runtime.Set("audioBytes", e.Runtime.NewArrayBuffer(audio))
|
||||
_, err := e.RunString(`__gopeed_blob_create_object_url(()=>{
|
||||
const source=b=>new ReadableStream({start(c){c.enqueue(new Uint8Array(b));c.close();}});
|
||||
return __gopeed_ffmpeg.merge({inputs:()=>({video:source(videoBytes),audio:source(audioBytes)})});
|
||||
},{contentType:'video/mp4'})`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for i := 0; i < 2; i++ {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
|
||||
r, err := open(ctx, stream.ObjectURLOpenRequest{End: -1})
|
||||
if err != nil {
|
||||
cancel()
|
||||
t.Fatal(err)
|
||||
}
|
||||
b, err := io.ReadAll(r)
|
||||
r.Close()
|
||||
cancel()
|
||||
if err != nil || !bytes.Contains(b, []byte("moof")) {
|
||||
t.Fatalf("Blob read: %v (%d bytes)", err, len(b))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestFFmpegEngineCloseCancelsRequest(t *testing.T) {
|
||||
started, closed := make(chan struct{}), make(chan struct{})
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
close(started)
|
||||
<-r.Context().Done()
|
||||
close(closed)
|
||||
}))
|
||||
defer srv.Close()
|
||||
e := NewEngine(nil)
|
||||
defer e.Close()
|
||||
_, err := e.RunString(fmt.Sprintf(`
|
||||
globalThis.cancelCount=0;
|
||||
const audio=new ReadableStream({cancel(){cancelCount++;}});
|
||||
__gopeed_ffmpeg.merge({inputs:()=>({video:{url:%q},audio})}).getReader().read().catch(()=>{});
|
||||
"started";`, srv.URL))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
select {
|
||||
case <-started:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("HTTP did not start")
|
||||
}
|
||||
e.Close()
|
||||
select {
|
||||
case <-closed:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("HTTP leaked after engine shutdown")
|
||||
}
|
||||
if e.Runtime.Get("cancelCount").ToInteger() != 1 {
|
||||
t.Fatal("JS input not cancelled on shutdown")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFFmpegInputFactoriesWaitForCapacity(t *testing.T) {
|
||||
for _, mode := range []string{"factory", "http", "cancel"} {
|
||||
t.Run(mode, func(t *testing.T) {
|
||||
release1, _ := media.Acquire(context.Background())
|
||||
defer release1()
|
||||
release2, _ := media.Acquire(context.Background())
|
||||
defer release2()
|
||||
var requests atomic.Int32
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { requests.Add(1); http.Error(w, "test", 500) }))
|
||||
defer srv.Close()
|
||||
e := NewEngine(nil)
|
||||
defer e.Close()
|
||||
e.Runtime.Set("mode", mode)
|
||||
e.Runtime.Set("sourceURL", srv.URL)
|
||||
result, err := e.RunString(`(async()=>{
|
||||
globalThis.factoryCalls=0;
|
||||
globalThis.abortQueue=new AbortController();
|
||||
const options=mode==='http'?{video:{url:sourceURL},audio:{url:sourceURL}}:
|
||||
{inputs:()=>{factoryCalls++;throw new Error('factory failed');}};
|
||||
options.signal=abortQueue.signal;
|
||||
globalThis.work=__gopeed_ffmpeg.merge(options).getReader().read().then(()=>'',e=>e.message);
|
||||
await new Promise(r=>setTimeout(r,50));
|
||||
if(mode==='cancel')abortQueue.abort();
|
||||
return factoryCalls;
|
||||
})()`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if fmt.Sprint(result) != "0" || requests.Load() != 0 {
|
||||
t.Fatalf("queued inputs started: %v, requests=%d", result, requests.Load())
|
||||
}
|
||||
if mode == "cancel" {
|
||||
if _, err = e.RunString(`work`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
release1()
|
||||
if _, err = e.RunString(`work`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
calls, err := e.RunString(`factoryCalls`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if mode == "factory" && fmt.Sprint(calls) != "1" {
|
||||
t.Fatal(calls)
|
||||
}
|
||||
if mode == "cancel" && fmt.Sprint(calls) != "0" {
|
||||
t.Fatal("cancelled factory ran")
|
||||
}
|
||||
if mode == "http" && requests.Load() != 1 {
|
||||
t.Fatal("HTTP did not start after admission")
|
||||
}
|
||||
// Failure/cancellation must return the acquired slot even with the other held.
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
|
||||
defer cancel()
|
||||
release, err := media.Acquire(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("slot leaked: %v", err)
|
||||
}
|
||||
release()
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestFFmpegRequiresFactoryForStreams(t *testing.T) {
|
||||
e := NewEngine(nil)
|
||||
defer e.Close()
|
||||
result, err := e.RunString(`(()=>{
|
||||
const source=()=>new ReadableStream({});
|
||||
let rejected=0;
|
||||
for(const opts of [
|
||||
{video:source(),audio:source()},
|
||||
{video:{url:'https://example.test'},audio:source()},
|
||||
{inputs:()=>({}),video:{url:'https://example.test'}},
|
||||
{inputs:{video:source(),audio:source()}}
|
||||
]) {try{__gopeed_ffmpeg.merge(opts);}catch(e){if(e instanceof TypeError)rejected++;}}
|
||||
return rejected;
|
||||
})()`)
|
||||
if err != nil || fmt.Sprint(result) != "4" {
|
||||
t.Fatalf("validation: %v %v", result, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFFmpegCancelPendingFactoryCleansLateInputs(t *testing.T) {
|
||||
e := NewEngine(nil)
|
||||
defer e.Close()
|
||||
result, err := e.RunString(`(async()=>{
|
||||
let ready,finish,cancelled=0,signal;
|
||||
const entered=new Promise(r=>ready=r);
|
||||
const abort=new AbortController();
|
||||
const output=__gopeed_ffmpeg.merge({signal:abort.signal,inputs:async opts=>{
|
||||
signal=opts.signal;ready();await new Promise(r=>finish=r);
|
||||
const source=()=>new ReadableStream({cancel(){cancelled++;}});
|
||||
return {video:source(),audio:source()};
|
||||
}});
|
||||
const work=output.getReader().read().catch(e=>e.name);
|
||||
await entered;abort.abort();await work;finish();
|
||||
await new Promise(r=>setTimeout(r,30));
|
||||
return {cancelled,aborted:signal.aborted};
|
||||
})()`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
state := result.(map[string]any)
|
||||
if fmt.Sprint(state["cancelled"]) != "2" || state["aborted"] != true {
|
||||
t.Fatal(state)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
|
||||
defer cancel()
|
||||
first, err := media.Acquire(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer first()
|
||||
second, err := media.Acquire(ctx)
|
||||
if err != nil {
|
||||
t.Fatal("factory cancellation leaked capacity")
|
||||
}
|
||||
second()
|
||||
}
|
||||
@@ -29,8 +29,9 @@ const FingerprintMagicKey = "__gopeed_fetch_fingerprint"
|
||||
var script string
|
||||
|
||||
type Config struct {
|
||||
ProxyHandler func(*http.Request) (*url.URL, error)
|
||||
RegisterCleanup func(func())
|
||||
DefaultUserAgent *string
|
||||
ProxyHandler func(*http.Request) (*url.URL, error)
|
||||
RegisterCleanup func(func())
|
||||
}
|
||||
|
||||
func Enable(runtime *goja.Runtime, loop *eventloop.EventLoop, cfg *Config) error {
|
||||
@@ -51,6 +52,9 @@ func Enable(runtime *goja.Runtime, loop *eventloop.EventLoop, cfg *Config) error
|
||||
if err != nil {
|
||||
panic(runtime.NewGoError(err))
|
||||
}
|
||||
if cfg.DefaultUserAgent != nil && !hasHeader(request.Headers, "User-Agent") {
|
||||
request.Headers = append(request.Headers, [2]string{"User-Agent", *cfg.DefaultUserAgent})
|
||||
}
|
||||
fingerprint := util.SafeGet[string](runtime, FingerprintMagicKey)
|
||||
promise, resolve, reject := runtime.NewPromise()
|
||||
go func() {
|
||||
|
||||
@@ -0,0 +1,139 @@
|
||||
(function () {
|
||||
const create = __gopeed_ffmpeg_create;
|
||||
const read = __gopeed_ffmpeg_read;
|
||||
const acquire = __gopeed_ffmpeg_acquire;
|
||||
const start = __gopeed_ffmpeg_start;
|
||||
const push = __gopeed_ffmpeg_push;
|
||||
const end = __gopeed_ffmpeg_end;
|
||||
const cancel = __gopeed_ffmpeg_cancel;
|
||||
const active = new Set();
|
||||
globalThis.__gopeed_ffmpeg_close_all = () => {
|
||||
for (const stop of Array.from(active)) stop(new Error("extension engine closed"));
|
||||
};
|
||||
|
||||
function input(value, name, allowStream) {
|
||||
if (value && typeof value.getReader === "function") {
|
||||
if (!allowStream) throw new TypeError("stream inputs must be created by the inputs callback");
|
||||
if (value.locked) throw new TypeError(name + " stream is locked");
|
||||
return { stream: value, request: null };
|
||||
}
|
||||
if (!value || typeof value.url !== "string" || !/^https?:\/\//i.test(value.url)) {
|
||||
throw new TypeError(name + " must be an HTTP request or ReadableStream");
|
||||
}
|
||||
const headers = {};
|
||||
new Headers(value.headers || {}).forEach((v, k) => { headers[k] = v; });
|
||||
return { request: { url: value.url, headers } };
|
||||
}
|
||||
|
||||
function merge(options) {
|
||||
if (!options || typeof options !== "object") throw new TypeError("merge options are required");
|
||||
const factory = options.inputs;
|
||||
if (factory !== undefined && typeof factory !== "function") throw new TypeError("inputs must be a callback");
|
||||
if (factory && (options.video !== undefined || options.audio !== undefined)) throw new TypeError("inputs cannot be combined with video/audio");
|
||||
let video = factory ? null : input(options.video, "video", false);
|
||||
let audio = factory ? null : input(options.audio, "audio", false);
|
||||
const args = options.args === undefined ? [] : options.args;
|
||||
if (!Array.isArray(args) || args.some(v => typeof v !== "string")) throw new TypeError("args must be a string array");
|
||||
const format = options.format === undefined ? "mp4" : options.format;
|
||||
if (typeof format !== "string") throw new TypeError("format must be a string");
|
||||
const signal = options.signal;
|
||||
let id, controller, stopped = false;
|
||||
const readers = [];
|
||||
const lifetime = new AbortController();
|
||||
let ownedInputs;
|
||||
function discardInputs(inputs, reason) {
|
||||
for (const source of new Set([inputs && inputs.video, inputs && inputs.audio])) {
|
||||
if (source && typeof source.cancel === "function" && !source.locked) Promise.resolve(source.cancel(reason)).catch(() => {});
|
||||
}
|
||||
}
|
||||
|
||||
function stop(reason) {
|
||||
if (stopped) return;
|
||||
stopped = true;
|
||||
active.delete(shutdown);
|
||||
lifetime.abort();
|
||||
discardInputs(ownedInputs, reason);
|
||||
if (id) cancel(id);
|
||||
if (signal) signal.removeEventListener("abort", abort);
|
||||
for (const reader of readers) {
|
||||
Promise.resolve(reader.cancel(reason)).catch(() => {}).then(() => {
|
||||
try { reader.releaseLock(); } catch (_) {}
|
||||
});
|
||||
}
|
||||
}
|
||||
function shutdown(reason) {
|
||||
controller.error(reason);
|
||||
stop(reason);
|
||||
}
|
||||
function abort() {
|
||||
const error = new Error("FFmpeg operation aborted");
|
||||
error.name = "AbortError";
|
||||
stop(error);
|
||||
controller.error(error);
|
||||
}
|
||||
async function pump(reader, index) {
|
||||
try {
|
||||
while (!stopped) {
|
||||
const result = await reader.read();
|
||||
if (stopped) return;
|
||||
if (result.done) { end(id, index, ""); return; }
|
||||
if (!(result.value instanceof Uint8Array)) throw new TypeError("media streams must contain Uint8Array chunks");
|
||||
// Keep individual crossings bounded, even if upstream emits a large chunk.
|
||||
for (let offset = 0; offset < result.value.byteLength && !stopped; offset += 256 * 1024) {
|
||||
if (!await push(id, index, result.value.subarray(offset, offset + 256 * 1024))) return;
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
if (!stopped) {
|
||||
end(id, index, String(error && error.message || error));
|
||||
controller.error(error);
|
||||
stop(error);
|
||||
}
|
||||
}
|
||||
}
|
||||
return new ReadableStream({
|
||||
start(c) {
|
||||
controller = c;
|
||||
active.add(shutdown);
|
||||
if (signal) {
|
||||
if (signal.aborted) abort();
|
||||
else signal.addEventListener("abort", abort);
|
||||
}
|
||||
},
|
||||
async pull(c) {
|
||||
if (stopped) return;
|
||||
try {
|
||||
if (!id) {
|
||||
id = create({ format, args });
|
||||
await acquire(id);
|
||||
if (stopped) return;
|
||||
if (factory) {
|
||||
const inputs = await factory({ signal: lifetime.signal });
|
||||
if (stopped) { discardInputs(inputs, new Error("FFmpeg operation cancelled")); return; }
|
||||
ownedInputs = inputs;
|
||||
video = input(inputs && inputs.video, "video", true);
|
||||
audio = input(inputs && inputs.audio, "audio", true);
|
||||
}
|
||||
if (video.stream && video.stream === audio.stream) throw new TypeError("video and audio must be distinct streams");
|
||||
for (const source of [video, audio]) {
|
||||
if (source.stream) readers.push(source.stream.getReader());
|
||||
}
|
||||
start(id, { video: video.request, audio: audio.request, format, args });
|
||||
let readerIndex = 0;
|
||||
for (const [index, source] of [video, audio].entries()) {
|
||||
if (source.stream) void pump(readers[readerIndex++], index);
|
||||
}
|
||||
}
|
||||
const chunk = await read(id);
|
||||
if (stopped) return;
|
||||
if (chunk === null) { stop(); c.close(); }
|
||||
else c.enqueue(new Uint8Array(chunk));
|
||||
} catch (error) {
|
||||
if (!stopped) { c.error(error); stop(error); }
|
||||
}
|
||||
},
|
||||
cancel(reason) { stop(reason); }
|
||||
});
|
||||
}
|
||||
globalThis.__gopeed_ffmpeg = Object.freeze({ merge, supportsInputFactory: true });
|
||||
})();
|
||||
@@ -0,0 +1,324 @@
|
||||
package ffmpeg
|
||||
|
||||
import (
|
||||
"context"
|
||||
_ "embed"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
|
||||
"github.com/GopeedLab/gopeed/internal/httpclient"
|
||||
media "github.com/GopeedLab/gopeed/pkg/download/engine/ffmpeg"
|
||||
"github.com/dop251/goja"
|
||||
"github.com/dop251/goja_nodejs/eventloop"
|
||||
)
|
||||
|
||||
//go:embed ffmpeg.js
|
||||
var script string
|
||||
|
||||
type Config struct {
|
||||
DefaultUserAgent *string
|
||||
ProxyHandler func(*http.Request) (*url.URL, error)
|
||||
RegisterCleanup func(func())
|
||||
}
|
||||
type options struct {
|
||||
Video *media.HTTPSource `json:"video"`
|
||||
Audio *media.HTTPSource `json:"audio"`
|
||||
Format string `json:"format"`
|
||||
Args []string `json:"args"`
|
||||
}
|
||||
type job struct {
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
output *io.PipeReader
|
||||
streams [2]*media.StreamInput
|
||||
mu sync.Mutex
|
||||
ready chan struct{}
|
||||
start chan options
|
||||
started bool
|
||||
}
|
||||
|
||||
func (j *job) stop() {
|
||||
j.cancel()
|
||||
j.output.CloseWithError(context.Canceled)
|
||||
j.mu.Lock()
|
||||
defer j.mu.Unlock()
|
||||
for _, s := range j.streams {
|
||||
if s != nil {
|
||||
s.Close()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func Enable(vm *goja.Runtime, loop *eventloop.EventLoop, cfg *Config) error {
|
||||
if cfg == nil {
|
||||
cfg = &Config{}
|
||||
}
|
||||
client, err := httpclient.NewClient(httpclient.Options{Transport: httpclient.TransportOptions{Proxy: cfg.ProxyHandler}})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var mu sync.Mutex
|
||||
jobs := map[string]*job{}
|
||||
var next atomic.Uint64
|
||||
get := func(id string) *job { mu.Lock(); defer mu.Unlock(); return jobs[id] }
|
||||
remove := func(id string) {
|
||||
mu.Lock()
|
||||
j := jobs[id]
|
||||
delete(jobs, id)
|
||||
mu.Unlock()
|
||||
if j != nil {
|
||||
j.stop()
|
||||
}
|
||||
}
|
||||
if cfg.RegisterCleanup != nil {
|
||||
cfg.RegisterCleanup(func() {
|
||||
done := make(chan struct{})
|
||||
if loop.RunOnLoop(func(vm *goja.Runtime) {
|
||||
defer close(done)
|
||||
if closeAll, ok := goja.AssertFunction(vm.Get("__gopeed_ffmpeg_close_all")); ok {
|
||||
_, _ = closeAll(goja.Undefined())
|
||||
}
|
||||
}) {
|
||||
<-done
|
||||
}
|
||||
mu.Lock()
|
||||
old := jobs
|
||||
jobs = map[string]*job{}
|
||||
mu.Unlock()
|
||||
for _, j := range old {
|
||||
j.stop()
|
||||
}
|
||||
client.CloseIdleConnections()
|
||||
})
|
||||
}
|
||||
if err := vm.Set("__gopeed_ffmpeg_create", func(call goja.FunctionCall) goja.Value {
|
||||
var opts options
|
||||
if err := vm.ExportTo(call.Argument(0), &opts); err != nil {
|
||||
panic(vm.NewGoError(err))
|
||||
}
|
||||
if _, err := media.Arguments(opts.Format, opts.Args); err != nil {
|
||||
panic(vm.NewGoError(err))
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
r, w := io.Pipe()
|
||||
j := &job{ctx: ctx, cancel: cancel, output: r, ready: make(chan struct{}), start: make(chan options, 1)}
|
||||
id := fmt.Sprintf("ffmpeg-%d", next.Add(1))
|
||||
mu.Lock()
|
||||
jobs[id] = j
|
||||
mu.Unlock()
|
||||
go func() {
|
||||
var inputs [2]media.Input
|
||||
var runErr error
|
||||
defer func() {
|
||||
if v := recover(); v != nil {
|
||||
runErr = fmt.Errorf("ffmpeg panic: %v", v)
|
||||
}
|
||||
cancel()
|
||||
for _, in := range inputs {
|
||||
if in != nil {
|
||||
in.Close()
|
||||
}
|
||||
}
|
||||
w.CloseWithError(runErr)
|
||||
}()
|
||||
release, err := media.Acquire(ctx)
|
||||
if err != nil {
|
||||
runErr = err
|
||||
return
|
||||
}
|
||||
defer release()
|
||||
close(j.ready)
|
||||
var opts options
|
||||
select {
|
||||
case opts = <-j.start:
|
||||
case <-ctx.Done():
|
||||
runErr = ctx.Err()
|
||||
return
|
||||
}
|
||||
for i, source := range []*media.HTTPSource{opts.Video, opts.Audio} {
|
||||
if source == nil {
|
||||
inputs[i] = j.streams[i]
|
||||
} else {
|
||||
inputs[i], runErr = media.OpenHTTP(ctx, client, *source)
|
||||
if runErr != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
runErr = media.RunAcquired(ctx, inputs[0], inputs[1], w, opts.Format, opts.Args)
|
||||
}()
|
||||
return vm.ToValue(id)
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := vm.Set("__gopeed_ffmpeg_acquire", func(id string) goja.Value {
|
||||
promise, resolve, reject := vm.NewPromise()
|
||||
j := get(id)
|
||||
go func() {
|
||||
var err error
|
||||
if j == nil {
|
||||
err = io.ErrClosedPipe
|
||||
} else {
|
||||
select {
|
||||
case <-j.ready:
|
||||
case <-j.ctx.Done():
|
||||
err = j.ctx.Err()
|
||||
}
|
||||
if j.ctx.Err() != nil {
|
||||
err = j.ctx.Err()
|
||||
}
|
||||
}
|
||||
loop.RunOnLoop(func(vm *goja.Runtime) {
|
||||
if err != nil {
|
||||
reject(vm.NewGoError(err))
|
||||
} else {
|
||||
resolve(true)
|
||||
}
|
||||
})
|
||||
}()
|
||||
return vm.ToValue(promise)
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := vm.Set("__gopeed_ffmpeg_start", func(call goja.FunctionCall) goja.Value {
|
||||
id := call.Argument(0).String()
|
||||
j := get(id)
|
||||
if j == nil {
|
||||
panic(vm.NewGoError(io.ErrClosedPipe))
|
||||
}
|
||||
var opts options
|
||||
if err := vm.ExportTo(call.Argument(1), &opts); err != nil {
|
||||
panic(vm.NewGoError(err))
|
||||
}
|
||||
if _, err := media.Arguments(opts.Format, opts.Args); err != nil {
|
||||
panic(vm.NewGoError(err))
|
||||
}
|
||||
for _, source := range []*media.HTTPSource{opts.Video, opts.Audio} {
|
||||
if source == nil || cfg.DefaultUserAgent == nil {
|
||||
continue
|
||||
}
|
||||
specified := false
|
||||
for key := range source.Headers {
|
||||
if strings.EqualFold(key, "User-Agent") {
|
||||
specified = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !specified {
|
||||
if source.Headers == nil {
|
||||
source.Headers = make(map[string]string)
|
||||
}
|
||||
source.Headers["User-Agent"] = *cfg.DefaultUserAgent
|
||||
}
|
||||
}
|
||||
|
||||
j.mu.Lock()
|
||||
defer j.mu.Unlock()
|
||||
if j.started || j.ctx.Err() != nil {
|
||||
panic(vm.NewGoError(io.ErrClosedPipe))
|
||||
}
|
||||
j.started = true
|
||||
for i, source := range []*media.HTTPSource{opts.Video, opts.Audio} {
|
||||
if source == nil {
|
||||
j.streams[i] = media.NewStreamInput(j.ctx)
|
||||
}
|
||||
}
|
||||
j.start <- opts
|
||||
return goja.Undefined()
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := vm.Set("__gopeed_ffmpeg_read", func(id string) goja.Value {
|
||||
promise, resolve, reject := vm.NewPromise()
|
||||
go func() {
|
||||
var readErr error
|
||||
buf := make([]byte, 64*1024)
|
||||
n := 0
|
||||
j := get(id)
|
||||
if j == nil {
|
||||
readErr = io.ErrClosedPipe
|
||||
} else {
|
||||
n, readErr = j.output.Read(buf)
|
||||
}
|
||||
if !loop.RunOnLoop(func(vm *goja.Runtime) {
|
||||
if readErr != nil {
|
||||
remove(id)
|
||||
if readErr == io.EOF {
|
||||
resolve(goja.Null())
|
||||
} else {
|
||||
reject(vm.NewGoError(readErr))
|
||||
}
|
||||
return
|
||||
}
|
||||
resolve(vm.ToValue(vm.NewArrayBuffer(buf[:n])))
|
||||
}) {
|
||||
remove(id)
|
||||
}
|
||||
}()
|
||||
return vm.ToValue(promise)
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := vm.Set("__gopeed_ffmpeg_push", func(call goja.FunctionCall) goja.Value {
|
||||
id := call.Argument(0).String()
|
||||
index := int(call.Argument(1).ToInteger())
|
||||
var chunk []byte
|
||||
switch b := call.Argument(2).Export().(type) {
|
||||
case goja.ArrayBuffer:
|
||||
chunk = append([]byte(nil), b.Bytes()...)
|
||||
case []byte:
|
||||
chunk = append([]byte(nil), b...)
|
||||
default:
|
||||
panic(vm.NewTypeError("ffmpeg input requires bytes"))
|
||||
}
|
||||
promise, resolve, reject := vm.NewPromise()
|
||||
go func() {
|
||||
var err error
|
||||
j := get(id)
|
||||
if j == nil || index < 0 || index > 1 || j.streams[index] == nil {
|
||||
err = io.ErrClosedPipe
|
||||
} else {
|
||||
_, err = j.streams[index].Write(chunk)
|
||||
}
|
||||
if err != nil && j != nil && index >= 0 && index < 2 && j.streams[index] != nil {
|
||||
j.streams[index].End(err)
|
||||
}
|
||||
loop.RunOnLoop(func(vm *goja.Runtime) {
|
||||
if j != nil && j.ctx.Err() != nil {
|
||||
resolve(false)
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
reject(vm.NewGoError(err))
|
||||
} else {
|
||||
resolve(true)
|
||||
}
|
||||
})
|
||||
}()
|
||||
return vm.ToValue(promise)
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := vm.Set("__gopeed_ffmpeg_end", func(id string, index int, message string) {
|
||||
if j := get(id); j != nil && index >= 0 && index < 2 && j.streams[index] != nil {
|
||||
var err error
|
||||
if message != "" {
|
||||
err = errors.New(message)
|
||||
}
|
||||
j.streams[index].End(err)
|
||||
}
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := vm.Set("__gopeed_ffmpeg_cancel", remove); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = vm.RunString(script)
|
||||
return err
|
||||
}
|
||||
@@ -86,18 +86,23 @@ func Enable(runtime *goja.Runtime, loop *eventloop.EventLoop, cfg *Config) error
|
||||
}
|
||||
if err := runtime.Set("__gopeed_blob_pipe_chunk", func(call goja.FunctionCall) goja.Value {
|
||||
id := call.Argument(0).String()
|
||||
ok := blobPipes.Push(id, exportBytes(call.Argument(1)))
|
||||
return runtime.ToValue(ok)
|
||||
chunk := append([]byte(nil), exportBytes(call.Argument(1))...)
|
||||
promise, resolve, _ := runtime.NewPromise()
|
||||
go func() {
|
||||
ok := blobPipes.Push(id, chunk)
|
||||
loop.RunOnLoop(func(vm *goja.Runtime) { resolve(ok) })
|
||||
}()
|
||||
return runtime.ToValue(promise)
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := runtime.Set("__gopeed_blob_pipe_close", func(id string) {
|
||||
blobPipes.Close(id, nil)
|
||||
go blobPipes.Close(id, nil)
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := runtime.Set("__gopeed_blob_pipe_error", func(id string, message string) {
|
||||
blobPipes.Close(id, errors.New(message))
|
||||
go blobPipes.Close(id, errors.New(message))
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -269,3 +269,57 @@ func newStreamTestEngine(t *testing.T, created chan<- createdObjectURL) (*engine
|
||||
})
|
||||
return e, e.Close
|
||||
}
|
||||
|
||||
// A stalled disk/network consumer must pause the producer without blocking
|
||||
// the JS event loop, so cancellation and other tracks can still run.
|
||||
func TestBlobBackpressureLeavesEventLoopResponsive(t *testing.T) {
|
||||
created := make(chan createdObjectURL, 1)
|
||||
e, cleanup := newStreamTestEngine(t, created)
|
||||
defer cleanup()
|
||||
_, err := e.RunString(`
|
||||
globalThis.produced = 0;
|
||||
globalThis.sourceCancelled = false;
|
||||
__gopeed_blob_create_object_url(() => new ReadableStream({
|
||||
pull(c) { produced++; c.enqueue(new Uint8Array(64*1024)); },
|
||||
cancel() { sourceCancelled = true; }
|
||||
}));
|
||||
`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
object := <-created
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
|
||||
defer cancel()
|
||||
reader, err := object.open(ctx, stream.ObjectURLOpenRequest{End: -1})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer reader.Close()
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
value, err := e.RunString("produced")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
before := value.(int64)
|
||||
if before > 10 {
|
||||
t.Fatalf("unbounded production: %d", before)
|
||||
}
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
after, err := e.RunString("produced")
|
||||
if err != nil || after != value {
|
||||
t.Fatalf("producer did not pause: %v -> %v (%v)", value, after, err)
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
t.Fatal("event loop only resumed after cancellation")
|
||||
}
|
||||
if _, err = reader.Read(make([]byte, 64*1024)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
time.Sleep(30 * time.Millisecond)
|
||||
after, err = e.RunString("produced")
|
||||
if err != nil || after.(int64) <= before {
|
||||
t.Fatalf("producer did not resume: %v (%v)", after, err)
|
||||
}
|
||||
reader.Close()
|
||||
waitForJSValue(t, e, "String(sourceCancelled)", "true")
|
||||
}
|
||||
@@ -105,6 +105,10 @@
|
||||
return Promise.resolve({ done: true, value: undefined });
|
||||
}
|
||||
this._maybePull();
|
||||
// pull() may synchronously error before a waiter has been registered.
|
||||
if (this._errored) {
|
||||
return Promise.reject(this._errored);
|
||||
}
|
||||
if (this._queue.length > 0) {
|
||||
return Promise.resolve({ done: false, value: this._queue.shift() });
|
||||
}
|
||||
@@ -760,11 +764,14 @@
|
||||
return;
|
||||
}
|
||||
const chunk = value instanceof Uint8Array ? value : new Uint8Array(value);
|
||||
const buffer = chunk.buffer.slice(chunk.byteOffset, chunk.byteOffset + chunk.byteLength);
|
||||
if (!globalThis.__gopeed_blob_pipe_chunk(pipeId, buffer)) {
|
||||
state.cancelRequested = true;
|
||||
state.cancelReason = "blob pipe closed";
|
||||
return;
|
||||
for (let offset = 0; offset < chunk.byteLength; offset += 256 * 1024) {
|
||||
const part = chunk.subarray(offset, offset + 256 * 1024);
|
||||
const buffer = part.buffer.slice(part.byteOffset, part.byteOffset + part.byteLength);
|
||||
if (!await globalThis.__gopeed_blob_pipe_chunk(pipeId, buffer)) {
|
||||
state.cancelRequested = true;
|
||||
state.cancelReason = "blob pipe closed";
|
||||
return;
|
||||
}
|
||||
}
|
||||
await yieldBlobPipeTask();
|
||||
}
|
||||
|
||||
+18
@@ -0,0 +1,18 @@
|
||||
# Synthetic media fixtures
|
||||
|
||||
These files contain 0.3 seconds of a generated 32x32 black image (H.264) and
|
||||
generated mono silence (AAC). They contain no third-party media. Generated
|
||||
with the pinned go-ffmpreg FFmpeg binary using the equivalent commands:
|
||||
|
||||
```
|
||||
ffmpeg -f lavfi -i color=c=black:s=32x32:r=10 -t 0.3 -c:v libx264 -preset ultrafast -movflags frag_keyframe+empty_moov -f mp4 video.mp4
|
||||
ffmpeg -f lavfi -i anullsrc=r=48000:cl=mono -t 0.3 -c:a aac -movflags frag_keyframe+empty_moov -f mp4 audio.mp4
|
||||
```
|
||||
|
||||
The backend tests also generate and probe media using the embedded FFmpeg and
|
||||
FFprobe at test time, without relying on a system installation.
|
||||
|
||||
`video.webm` and `audio.webm` use the same black/silent sources, encoded once
|
||||
with native FFmpeg 7.1 using `-c:v libvpx-vp9` and `-c:a libopus`. The committed
|
||||
fixtures let the tests verify WASM stream-copy remuxing without exercising its
|
||||
VP9 encoder or requiring native FFmpeg on developer/CI machines.
|
||||
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
@@ -5,6 +5,7 @@ import "encoding/json"
|
||||
const RPCEndpointPath = "/webview"
|
||||
|
||||
const (
|
||||
MethodProfileRemove = "profile.remove"
|
||||
MethodIsAvailable = "webview.isAvailable"
|
||||
MethodPageOpen = "page.open"
|
||||
MethodPageAddInitScript = "page.addInitScript"
|
||||
@@ -68,6 +69,9 @@ type IsAvailableResult struct {
|
||||
}
|
||||
|
||||
type PageOpenParams struct {
|
||||
ProfileID string `json:"profileId,omitempty"`
|
||||
DataPath string `json:"dataPath,omitempty"`
|
||||
ProxyURL string `json:"proxyUrl,omitempty"`
|
||||
Headless bool `json:"headless,omitempty"`
|
||||
Debug bool `json:"debug,omitempty"`
|
||||
Title string `json:"title,omitempty"`
|
||||
@@ -78,6 +82,9 @@ type PageOpenParams struct {
|
||||
|
||||
func NewPageOpenParams(opts OpenOptions) PageOpenParams {
|
||||
return PageOpenParams{
|
||||
ProfileID: opts.ProfileID,
|
||||
DataPath: opts.DataPath,
|
||||
ProxyURL: opts.ProxyURL,
|
||||
Headless: opts.Headless,
|
||||
Debug: opts.Debug,
|
||||
Title: opts.Title,
|
||||
@@ -141,3 +148,8 @@ type PageCloseParams struct {
|
||||
}
|
||||
|
||||
type EmptyResult struct{}
|
||||
|
||||
// ProfileRemoveParams belongs to the private host RPC.
|
||||
type ProfileRemoveParams struct {
|
||||
ProfileID string `json:"profileId"`
|
||||
}
|
||||
@@ -17,6 +17,11 @@ type Opener interface {
|
||||
Open(opts OpenOptions) (Page, error)
|
||||
}
|
||||
|
||||
// ProfileRemover is a host-only lifecycle operation, never exposed to extension JS.
|
||||
type ProfileRemover interface {
|
||||
RemoveProfile(profileID, dataPath string) error
|
||||
}
|
||||
|
||||
type Page interface {
|
||||
AddInitScript(script string) error
|
||||
Goto(url string, opts GotoOptions) error
|
||||
@@ -29,6 +34,10 @@ type Page interface {
|
||||
}
|
||||
|
||||
type OpenOptions struct {
|
||||
// Host-owned fields are never parsed from extension JavaScript options.
|
||||
ProfileID string
|
||||
DataPath string
|
||||
ProxyURL string
|
||||
Headless bool
|
||||
Debug bool
|
||||
Title string
|
||||
|
||||
@@ -336,3 +336,15 @@ func (p *fakePage) Close() error {
|
||||
p.closed = true
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestExtensionCannotSupplyHostProfileOrProxy(t *testing.T) {
|
||||
opener := &fakeOpener{page: &fakePage{}}
|
||||
runtime := NewRuntime(opener, true)
|
||||
_, err := runtime.Open(map[string]any{"profileId": "other-extension", "dataPath": "/tmp/other-profile", "proxyUrl": "http://example.com:1234", "ProfileID": "other"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if opener.opts.ProfileID != "" || opener.opts.DataPath != "" || opener.opts.ProxyURL != "" {
|
||||
t.Fatalf("extension controlled host settings: %+v", opener.opts)
|
||||
}
|
||||
}
|
||||
@@ -65,6 +65,7 @@ func (d *Downloader) InstallExtensionByFolder(path string, devMode bool) (*Exten
|
||||
|
||||
// if extension is not installed, add it to the list, otherwise update it
|
||||
installedExt := d.getExtension(ext.Identity)
|
||||
newInstallation := installedExt == nil
|
||||
if installedExt == nil {
|
||||
ext.CreatedAt = time.Now()
|
||||
ext.UpdatedAt = ext.CreatedAt
|
||||
@@ -76,6 +77,11 @@ func (d *Downloader) InstallExtensionByFolder(path string, devMode bool) (*Exten
|
||||
if err = d.storage.Put(bucketExtension, installedExt.Identity, installedExt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if newInstallation {
|
||||
d.webviewProfilesLock.Lock()
|
||||
delete(d.webviewProfiles, ext.Identity)
|
||||
d.webviewProfilesLock.Unlock()
|
||||
}
|
||||
return installedExt, nil
|
||||
}
|
||||
|
||||
@@ -144,6 +150,9 @@ func (d *Downloader) DeleteExtension(identity string) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := d.removeExtensionWebViewProfile(identity); err != nil {
|
||||
return fmt.Errorf("remove extension WebView data: %w", err)
|
||||
}
|
||||
// remove from disk
|
||||
if !ext.DevMode {
|
||||
if err := os.RemoveAll(d.ExtensionPath(ext)); err != nil {
|
||||
@@ -365,7 +374,7 @@ func doTrigger[T any](d *Downloader, event ActivationEvent, req *base.Request, c
|
||||
engine, session := d.newExtensionEngine()
|
||||
defer session.CloseIfIdle()
|
||||
gopeed.Runtime = &InstanceRuntime{
|
||||
WebView: d.newExtensionWebViewRuntime(session),
|
||||
WebView: d.newExtensionWebViewRuntime(session, ext.buildIdentity()),
|
||||
}
|
||||
err = injectGopeed(engine.Runtime, gopeed)
|
||||
if err != nil {
|
||||
|
||||
@@ -30,7 +30,7 @@ func (d *Downloader) NewExtensionEngine(ext *Extension, settings map[string]any)
|
||||
identity: ext.buildIdentity(),
|
||||
},
|
||||
Runtime: &InstanceRuntime{
|
||||
WebView: d.newExtensionWebViewRuntime(session),
|
||||
WebView: d.newExtensionWebViewRuntime(session, ext.buildIdentity()),
|
||||
},
|
||||
}
|
||||
if err := injectGopeed(engine.Runtime, gopeed); err != nil {
|
||||
|
||||
@@ -3,9 +3,11 @@ package download
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
internalblob "github.com/GopeedLab/gopeed/internal/blob"
|
||||
httpProtocol "github.com/GopeedLab/gopeed/internal/protocol/http"
|
||||
"github.com/GopeedLab/gopeed/pkg/download/engine"
|
||||
"github.com/GopeedLab/gopeed/pkg/download/engine/inject/stream"
|
||||
enginewebview "github.com/GopeedLab/gopeed/pkg/download/engine/webview"
|
||||
@@ -119,21 +121,27 @@ func (d *Downloader) newExtensionEngine() (*engine.Engine, *engineSession) {
|
||||
return d.blob.Revoke(url)
|
||||
},
|
||||
}
|
||||
httpConfig := struct {
|
||||
UserAgent string `json:"userAgent"`
|
||||
}{UserAgent: httpProtocol.DefaultUserAgent}
|
||||
d.getProtocolConfig("http", &httpConfig)
|
||||
userAgent := strings.TrimSpace(httpConfig.UserAgent)
|
||||
e := engine.NewEngine(&engine.Config{
|
||||
ProxyConfig: d.cfg.Proxy,
|
||||
StreamConfig: engineCfg,
|
||||
HTTPUserAgent: &userAgent,
|
||||
ProxyConfig: d.cfg.Proxy,
|
||||
StreamConfig: engineCfg,
|
||||
})
|
||||
session.SetEngine(e)
|
||||
return e, session
|
||||
}
|
||||
|
||||
func (d *Downloader) newExtensionWebViewRuntime(session *engineSession) *enginewebview.Runtime {
|
||||
func (d *Downloader) newExtensionWebViewRuntime(session *engineSession, identity string) *enginewebview.Runtime {
|
||||
var (
|
||||
opener enginewebview.Opener
|
||||
available bool
|
||||
)
|
||||
if provider := d.cfg.WebViewProvider; provider != nil && provider.IsAvailable() {
|
||||
opener = provider
|
||||
opener = &extensionWebViewOpener{downloader: d, provider: provider, identity: identity, profile: d.extensionWebViewProfile(identity)}
|
||||
available = true
|
||||
}
|
||||
runtime := enginewebview.NewRuntime(opener, available)
|
||||
|
||||
@@ -30,6 +30,9 @@ func injectGopeed(vm *goja.Runtime, gopeed *Instance) error {
|
||||
}
|
||||
runtimeObject := vm.NewObject()
|
||||
if gopeed.Runtime != nil {
|
||||
if err := runtimeObject.Set("ffmpeg", vm.Get("__gopeed_ffmpeg")); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := runtimeObject.Set("blob", newJSBlobRuntime(vm)); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -1484,6 +1484,7 @@ func TestDownloader_ExtensionRuntimeWebViewInjected(t *testing.T) {
|
||||
value, err := runtime.Eval(`(async () => ({
|
||||
hasRuntime: !!gopeed.runtime,
|
||||
hasBlob: !!(gopeed.runtime && gopeed.runtime.blob),
|
||||
hasFFmpegMerge: typeof gopeed.runtime.ffmpeg.merge,
|
||||
hasCreateObjectURL: typeof gopeed.runtime.blob.createObjectURL,
|
||||
hasRevokeObjectURL: typeof gopeed.runtime.blob.revokeObjectURL,
|
||||
hasWebView: !!(gopeed.runtime && gopeed.runtime.webview),
|
||||
@@ -1502,6 +1503,9 @@ func TestDownloader_ExtensionRuntimeWebViewInjected(t *testing.T) {
|
||||
if result["hasRuntime"] != true {
|
||||
t.Fatalf("expected runtime injection, got %#v", result)
|
||||
}
|
||||
if result["hasFFmpegMerge"] != "function" {
|
||||
t.Fatalf("expected ffmpeg injection, got %#v", result)
|
||||
}
|
||||
if result["hasBlob"] != true {
|
||||
t.Fatalf("expected blob injection, got %#v", result)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
package download
|
||||
|
||||
import (
|
||||
"github.com/GopeedLab/gopeed/pkg/base"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestExtensionUnnamedResource(t *testing.T) {
|
||||
d, cleanup, err := newTestExtensionEngineDownloader()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer cleanup()
|
||||
dir := t.TempDir()
|
||||
manifest := `{"name":"unnamed","title":"Unnamed test","author":"test","version":"1.0.0","scripts":[{"event":"onResolve","match":{"urls":["https://example.com/unnamed"]},"entry":"index.js"}]}`
|
||||
script := `gopeed.events.onResolve(async(ctx)=>{ctx.res={range:false,files:[{name:'video.mp4',req:{url:await gopeed.runtime.blob.createObjectURL(()=>new ReadableStream({start(c){c.enqueue(new Uint8Array([1,2,3]));c.close();}}))}}]};});`
|
||||
for name, data := range map[string]string{"manifest.json": manifest, "index.js": script} {
|
||||
if err = os.WriteFile(filepath.Join(dir, name), []byte(data), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if _, err = d.InstallExtensionByFolder(dir, true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
res, err := d.Resolve(&base.Request{URL: "https://example.com/unnamed"}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if res.Res.Name != "" || len(res.Res.Files) != 1 || res.Res.Files[0].Name != "video.mp4" {
|
||||
t.Fatalf("unexpected resource: %+v", res.Res)
|
||||
}
|
||||
if !d.blob.IsURL(res.Res.Files[0].Req.URL) {
|
||||
t.Fatal("extension result was discarded")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
package download
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
httpProtocol "github.com/GopeedLab/gopeed/internal/protocol/http"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestExtensionHTTPUserAgent(t *testing.T) {
|
||||
video, err := os.ReadFile("engine/testdata/ffmpeg/video.mp4")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
audio, err := os.ReadFile("engine/testdata/ffmpeg/audio.mp4")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, configured := range []string{" Configured-UA/1.0 ", "", "<default>"} {
|
||||
for _, header := range []string{"default", "custom", "empty"} {
|
||||
t.Run(fmt.Sprintf("configured=%q/header=%s", configured, header), func(t *testing.T) {
|
||||
expected := strings.TrimSpace(configured)
|
||||
if configured == "<default>" {
|
||||
expected = httpProtocol.DefaultUserAgent
|
||||
}
|
||||
headers := `{}`
|
||||
if header == "custom" {
|
||||
expected = "Explicit-UA/2.0"
|
||||
headers = `{"uSeR-aGeNt":"Explicit-UA/2.0"}`
|
||||
}
|
||||
if header == "empty" {
|
||||
expected = ""
|
||||
headers = `{"user-agent":""}`
|
||||
}
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.UserAgent() != expected {
|
||||
t.Errorf("%s UA=%q, want %q", r.URL.Path, r.UserAgent(), expected)
|
||||
}
|
||||
if strings.HasPrefix(r.URL.Path, "/redirect/") {
|
||||
http.Redirect(w, r, strings.TrimPrefix(r.URL.Path, "/redirect"), http.StatusFound)
|
||||
return
|
||||
}
|
||||
data := video
|
||||
if strings.HasSuffix(r.URL.Path, "audio") {
|
||||
data = audio
|
||||
}
|
||||
http.ServeContent(w, r, "media", time.Time{}, bytes.NewReader(data))
|
||||
}))
|
||||
defer server.Close()
|
||||
d, cleanup, err := newTestExtensionEngineDownloader()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer cleanup()
|
||||
cfg, err := d.GetConfig()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cfg.ProtocolConfig["http"] = map[string]any{"userAgent": configured}
|
||||
if configured == "<default>" {
|
||||
cfg.ProtocolConfig["http"] = map[string]any{"connections": 4}
|
||||
}
|
||||
if err = d.PutConfig(cfg); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
runtime, err := newTestExtensionEngine(t, d)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer runtime.Close()
|
||||
// The existing engine keeps its UA snapshot when the downloader config changes.
|
||||
cfg, err = d.GetConfig()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cfg.ProtocolConfig["http"] = map[string]any{"userAgent": "Next-engine-UA"}
|
||||
if err = d.PutConfig(cfg); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
result, err := runtime.Eval(fmt.Sprintf(`(async()=>{
|
||||
const headers=%s, url=%q;
|
||||
await (await fetch(url+'/redirect/fetch',{headers})).arrayBuffer();
|
||||
await new Promise((resolve,reject)=>{
|
||||
const xhr=new XMLHttpRequest();xhr.open('GET',url+'/redirect/xhr');
|
||||
for(const key of Object.keys(headers))xhr.setRequestHeader(key,headers[key]);
|
||||
xhr.onload=resolve;xhr.onerror=()=>reject(new Error('XHR failed'));xhr.send();
|
||||
});
|
||||
// Profile defaults must not replace the configured or explicitly empty UA.
|
||||
__gopeed_setFingerprint('chrome');
|
||||
await (await fetch(url+'/fingerprint',{headers})).arrayBuffer();
|
||||
const output=gopeed.runtime.ffmpeg.merge({video:{url:url+'/redirect/video',headers},audio:{url:url+'/redirect/audio',headers}});
|
||||
return (await new Response(output).arrayBuffer()).byteLength > 0;
|
||||
})()`, headers, server.URL))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if result != true {
|
||||
t.Fatalf("merge result=%v", result)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,183 @@
|
||||
package download
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"sync"
|
||||
|
||||
"github.com/GopeedLab/gopeed/pkg/base"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
|
||||
webviewproxy "github.com/GopeedLab/gopeed/internal/webview/proxy"
|
||||
enginewebview "github.com/GopeedLab/gopeed/pkg/download/engine/webview"
|
||||
)
|
||||
|
||||
// extensionWebViewOpener binds browser state to the owning extension. Neither
|
||||
// identity, filesystem paths nor proxy settings are accepted from JavaScript.
|
||||
type extensionWebViewOpener struct {
|
||||
downloader *Downloader
|
||||
provider enginewebview.Provider
|
||||
identity string
|
||||
profile *extensionWebViewProfile
|
||||
}
|
||||
|
||||
func (o *extensionWebViewOpener) Open(opts enginewebview.OpenOptions) (enginewebview.Page, error) {
|
||||
if o.identity == "" {
|
||||
return nil, fmt.Errorf("webview extension identity is required")
|
||||
}
|
||||
profile := o.profile
|
||||
if profile == nil {
|
||||
profile = o.downloader.extensionWebViewProfile(o.identity)
|
||||
}
|
||||
profile.mu.Lock()
|
||||
defer profile.mu.Unlock()
|
||||
if profile.revoked {
|
||||
return nil, fmt.Errorf("extension WebView profile has been removed")
|
||||
}
|
||||
id, path, err := o.downloader.extensionWebViewLocation(o.identity)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
opts.ProfileID, opts.DataPath = id, path
|
||||
opts.ProxyURL, err = o.downloader.webviewProxyURL()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
page, err := o.provider.Open(opts)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
tracked := &extensionWebViewPage{Page: page, profile: profile}
|
||||
profile.pages[tracked] = struct{}{}
|
||||
return tracked, nil
|
||||
}
|
||||
|
||||
func (d *Downloader) webviewProxyURL() (string, error) {
|
||||
d.webviewProxyLock.Lock()
|
||||
defer d.webviewProxyLock.Unlock()
|
||||
if d.closed.Load() {
|
||||
return "", fmt.Errorf("downloader is closed")
|
||||
}
|
||||
if d.webviewProxy == nil {
|
||||
server, err := webviewproxy.New(func(r *http.Request) (*url.URL, error) {
|
||||
if d.blob != nil && d.blob.IsURL(r.URL.String()) {
|
||||
return nil, nil
|
||||
}
|
||||
d.configLock.RLock()
|
||||
var cfg base.DownloaderProxyConfig
|
||||
if d.cfg.Proxy != nil {
|
||||
cfg = *d.cfg.Proxy
|
||||
}
|
||||
d.configLock.RUnlock()
|
||||
handler := cfg.ToHandler()
|
||||
if handler == nil {
|
||||
return nil, nil
|
||||
}
|
||||
return handler(r)
|
||||
})
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
d.webviewProxy = server
|
||||
}
|
||||
if runtime.GOOS == "darwin" || runtime.GOOS == "ios" {
|
||||
return d.webviewProxy.SOCKSURL(), nil
|
||||
}
|
||||
return d.webviewProxy.URL(), nil
|
||||
}
|
||||
|
||||
// A revoked profile is retained until a fresh installation. Existing runtimes
|
||||
// retain this object, so reinstalling cannot let an old runtime recreate data.
|
||||
type extensionWebViewProfile struct {
|
||||
mu sync.Mutex
|
||||
deleteMu sync.Mutex
|
||||
revoked bool
|
||||
pages map[*extensionWebViewPage]struct{}
|
||||
}
|
||||
|
||||
type extensionWebViewPage struct {
|
||||
enginewebview.Page
|
||||
profile *extensionWebViewProfile
|
||||
mu sync.Mutex
|
||||
closed bool
|
||||
}
|
||||
|
||||
func (p *extensionWebViewPage) Close() error {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
if p.closed {
|
||||
return nil
|
||||
}
|
||||
if err := p.Page.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
p.closed = true
|
||||
p.profile.mu.Lock()
|
||||
delete(p.profile.pages, p)
|
||||
p.profile.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *Downloader) extensionWebViewProfile(identity string) *extensionWebViewProfile {
|
||||
d.webviewProfilesLock.Lock()
|
||||
defer d.webviewProfilesLock.Unlock()
|
||||
if d.webviewProfiles == nil {
|
||||
d.webviewProfiles = make(map[string]*extensionWebViewProfile)
|
||||
}
|
||||
profile := d.webviewProfiles[identity]
|
||||
if profile == nil {
|
||||
profile = &extensionWebViewProfile{pages: make(map[*extensionWebViewPage]struct{})}
|
||||
d.webviewProfiles[identity] = profile
|
||||
}
|
||||
return profile
|
||||
}
|
||||
|
||||
func (d *Downloader) extensionWebViewLocation(identity string) (string, string, error) {
|
||||
if identity == "" {
|
||||
return "", "", fmt.Errorf("webview extension identity is required")
|
||||
}
|
||||
sum := sha256.Sum256([]byte(identity))
|
||||
id := fmt.Sprintf("%x-%x-%x-%x-%x", sum[:4], sum[4:6], sum[6:8], sum[8:10], sum[10:16])
|
||||
path, err := filepath.Abs(filepath.Join(d.cfg.StorageDir, "webview", fmt.Sprintf("%x", sum[:])))
|
||||
return id, path, err
|
||||
}
|
||||
|
||||
func (d *Downloader) removeExtensionWebViewProfile(identity string) error {
|
||||
profile := d.extensionWebViewProfile(identity)
|
||||
profile.deleteMu.Lock()
|
||||
defer profile.deleteMu.Unlock()
|
||||
profile.mu.Lock()
|
||||
profile.revoked = true
|
||||
pages := make([]*extensionWebViewPage, 0, len(profile.pages))
|
||||
for page := range profile.pages {
|
||||
pages = append(pages, page)
|
||||
}
|
||||
profile.mu.Unlock()
|
||||
for _, page := range pages {
|
||||
if err := page.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
id, path, err := d.extensionWebViewLocation(identity)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if provider := d.cfg.WebViewProvider; provider != nil {
|
||||
remover, ok := provider.(enginewebview.ProfileRemover)
|
||||
if !ok {
|
||||
// An unavailable backend cannot have created a profile this run.
|
||||
// Still fail if a persisted native profile directory exists.
|
||||
_, statErr := os.Stat(path)
|
||||
if provider.IsAvailable() || len(pages) != 0 || !os.IsNotExist(statErr) {
|
||||
return fmt.Errorf("WebView provider does not support profile removal")
|
||||
}
|
||||
} else if err := remover.RemoveProfile(id, path); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return os.RemoveAll(path)
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
package download
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/GopeedLab/gopeed/pkg/base"
|
||||
enginewebview "github.com/GopeedLab/gopeed/pkg/download/engine/webview"
|
||||
)
|
||||
|
||||
type profileCaptureProvider struct{ options []enginewebview.OpenOptions }
|
||||
|
||||
func (*profileCaptureProvider) IsAvailable() bool { return true }
|
||||
func (p *profileCaptureProvider) Open(opts enginewebview.OpenOptions) (enginewebview.Page, error) {
|
||||
p.options = append(p.options, opts)
|
||||
return fakeRuntimeWebViewPage{}, nil
|
||||
}
|
||||
|
||||
func TestWebViewProfileBoundToExtension(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
d := &Downloader{cfg: &DownloaderConfig{StorageDir: root, DownloaderStoreConfig: &base.DownloaderStoreConfig{}}, configLock: &sync.RWMutex{}}
|
||||
t.Cleanup(func() {
|
||||
if d.webviewProxy != nil {
|
||||
d.webviewProxy.Close()
|
||||
}
|
||||
})
|
||||
provider := &profileCaptureProvider{}
|
||||
for _, id := range []string{"extension-a", "extension-b", "extension-a", "../../escape"} {
|
||||
opener := &extensionWebViewOpener{downloader: d, provider: provider, identity: id}
|
||||
_, err := opener.Open(enginewebview.OpenOptions{ProfileID: "forged", DataPath: "/tmp/forged", ProxyURL: "http://wrong:80"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
a, b, reopen := provider.options[0], provider.options[1], provider.options[2]
|
||||
if a.ProfileID == b.ProfileID || a.DataPath == b.DataPath {
|
||||
t.Fatal("different extensions shared browser storage")
|
||||
}
|
||||
if a.ProfileID != reopen.ProfileID || a.DataPath != reopen.DataPath {
|
||||
t.Fatal("same extension lost profile identity")
|
||||
}
|
||||
for _, opts := range provider.options {
|
||||
if filepath.Dir(opts.DataPath) != filepath.Join(root, "webview") {
|
||||
t.Fatalf("profile escaped storage root: %s", opts.DataPath)
|
||||
}
|
||||
if opts.ProxyURL != d.webviewProxy.URL() && opts.ProxyURL != d.webviewProxy.SOCKSURL() {
|
||||
t.Fatal("host proxy was overridden")
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,189 @@
|
||||
package download
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/GopeedLab/gopeed/pkg/base"
|
||||
webview "github.com/GopeedLab/gopeed/pkg/download/engine/webview"
|
||||
)
|
||||
|
||||
type removalProvider struct {
|
||||
pages map[string][]*removalPage
|
||||
removed []string
|
||||
failure error
|
||||
entered chan struct{}
|
||||
resume chan struct{}
|
||||
}
|
||||
type removalPage struct {
|
||||
fakeRuntimeWebViewPage
|
||||
closed bool
|
||||
}
|
||||
|
||||
func (p *removalPage) Close() error { p.closed = true; return nil }
|
||||
func (p *removalProvider) IsAvailable() bool { return true }
|
||||
func (p *removalProvider) Open(opts webview.OpenOptions) (webview.Page, error) {
|
||||
if p.entered != nil {
|
||||
close(p.entered)
|
||||
<-p.resume
|
||||
}
|
||||
if p.pages == nil {
|
||||
p.pages = make(map[string][]*removalPage)
|
||||
}
|
||||
page := &removalPage{}
|
||||
p.pages[opts.ProfileID] = append(p.pages[opts.ProfileID], page)
|
||||
return page, nil
|
||||
}
|
||||
func (p *removalProvider) RemoveProfile(id, path string) error {
|
||||
for _, page := range p.pages[id] {
|
||||
if !page.closed {
|
||||
return errors.New("profile removed before closing pages")
|
||||
}
|
||||
}
|
||||
if p.failure != nil {
|
||||
return p.failure
|
||||
}
|
||||
p.removed = append(p.removed, id)
|
||||
return nil
|
||||
}
|
||||
func removalDownloader(t *testing.T, provider *removalProvider) *Downloader {
|
||||
t.Helper()
|
||||
storage := NewMemStorage()
|
||||
if err := storage.Setup([]string{bucketExtension, bucketExtensionStorage}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d := &Downloader{cfg: &DownloaderConfig{StorageDir: t.TempDir(), DownloaderStoreConfig: &base.DownloaderStoreConfig{}, WebViewProvider: provider}, storage: storage, configLock: &sync.RWMutex{}}
|
||||
t.Cleanup(func() {
|
||||
if d.webviewProxy != nil {
|
||||
d.webviewProxy.Close()
|
||||
}
|
||||
})
|
||||
return d
|
||||
}
|
||||
func profileSentinel(t *testing.T, d *Downloader, id string) string {
|
||||
t.Helper()
|
||||
_, path, err := d.extensionWebViewLocation(id)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.MkdirAll(path, 0700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sentinel := filepath.Join(path, "cookies.sqlite")
|
||||
if err := os.WriteFile(sentinel, []byte("private browser state"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return sentinel
|
||||
}
|
||||
func TestDeleteExtensionRemovesWebViewData(t *testing.T) {
|
||||
provider := &removalProvider{}
|
||||
d := removalDownloader(t, provider)
|
||||
ext, err := d.InstallExtensionByFolder("testdata/extensions/basic", true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
identity := ext.Identity
|
||||
opener := &extensionWebViewOpener{downloader: d, provider: provider, identity: identity, profile: d.extensionWebViewProfile(identity)}
|
||||
for i := 0; i < 2; i++ {
|
||||
if _, err := opener.Open(webview.OpenOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
sentinel := profileSentinel(t, d, identity)
|
||||
other := profileSentinel(t, d, "other-extension")
|
||||
// Updating the installation keeps both its profile object and stored data.
|
||||
if _, err := d.InstallExtensionByFolder("testdata/extensions/basic", true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := os.Stat(sentinel); err != nil {
|
||||
t.Fatal("upgrade removed browser data", err)
|
||||
}
|
||||
if err := d.DeleteExtension(identity); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := os.Stat(sentinel); !os.IsNotExist(err) {
|
||||
t.Fatalf("browser data retained: %v", err)
|
||||
}
|
||||
if _, err := os.Stat(other); err != nil {
|
||||
t.Fatal("other extension data was removed", err)
|
||||
}
|
||||
if _, err := os.Stat(ext.DevPath); err != nil {
|
||||
t.Fatal("dev source was removed", err)
|
||||
}
|
||||
if len(provider.removed) != 1 {
|
||||
t.Fatal("native profile was not removed")
|
||||
}
|
||||
if _, err := opener.Open(webview.OpenOptions{}); err == nil {
|
||||
t.Fatal("old runtime recreated removed profile")
|
||||
}
|
||||
if _, err := d.InstallExtensionByFolder("testdata/extensions/basic", true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fresh := &extensionWebViewOpener{downloader: d, provider: provider, identity: identity, profile: d.extensionWebViewProfile(identity)}
|
||||
if _, err := fresh.Open(webview.OpenOptions{}); err != nil {
|
||||
t.Fatal("reinstall could not open profile", err)
|
||||
}
|
||||
if _, err := opener.Open(webview.OpenOptions{}); err == nil {
|
||||
t.Fatal("reinstall reactivated old runtime")
|
||||
}
|
||||
}
|
||||
func TestDeleteExtensionProfileFailureIsRetryable(t *testing.T) {
|
||||
provider := &removalProvider{failure: errors.New("native store busy")}
|
||||
d := removalDownloader(t, provider)
|
||||
ext, err := d.InstallExtensionByFolder("testdata/extensions/basic", true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sentinel := profileSentinel(t, d, ext.Identity)
|
||||
if err := d.DeleteExtension(ext.Identity); err == nil {
|
||||
t.Fatal("cleanup failure was swallowed")
|
||||
}
|
||||
if _, err := d.GetExtension(ext.Identity); err != nil {
|
||||
t.Fatal("extension removed before cleanup completed")
|
||||
}
|
||||
if _, err := os.Stat(sentinel); err != nil {
|
||||
t.Fatal("profile path removed before native cleanup completed")
|
||||
}
|
||||
provider.failure = nil
|
||||
if err := d.DeleteExtension(ext.Identity); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := d.GetExtension(ext.Identity); !errors.Is(err, ErrExtensionNotFound) {
|
||||
t.Fatal("extension was not uninstalled")
|
||||
}
|
||||
}
|
||||
func TestRemoveProfileWaitsForOpeningPage(t *testing.T) {
|
||||
provider := &removalProvider{entered: make(chan struct{}), resume: make(chan struct{})}
|
||||
d := removalDownloader(t, provider)
|
||||
opener := &extensionWebViewOpener{downloader: d, provider: provider, identity: "race", profile: d.extensionWebViewProfile("race")}
|
||||
opened := make(chan error, 1)
|
||||
go func() { _, err := opener.Open(webview.OpenOptions{}); opened <- err }()
|
||||
<-provider.entered
|
||||
removed := make(chan error, 1)
|
||||
go func() { removed <- d.removeExtensionWebViewProfile("race") }()
|
||||
close(provider.resume)
|
||||
if err := <-opened; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := <-removed; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(provider.removed) != 1 {
|
||||
t.Fatal("profile not removed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestUninstallWithoutAvailableWebView(t *testing.T) {
|
||||
d := removalDownloader(t, &removalProvider{})
|
||||
d.cfg.WebViewProvider = webview.NewUnavailableProvider()
|
||||
ext, err := d.InstallExtensionByFolder("testdata/extensions/basic", true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := d.DeleteExtension(ext.Identity); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,97 @@
|
||||
package download
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/GopeedLab/gopeed/pkg/base"
|
||||
)
|
||||
|
||||
// TestDownloader_HlsLiveSite is a manual acceptance gate against a real site.
|
||||
// It is skipped unless HLS_LIVE_URL is set, e.g.:
|
||||
//
|
||||
// HLS_LIVE_URL="https://host/path/index.m3u8" go test ./pkg/download/ -run TestDownloader_HlsLiveSite -v
|
||||
func TestDownloader_HlsLiveSite(t *testing.T) {
|
||||
playlistURL := os.Getenv("HLS_LIVE_URL")
|
||||
if playlistURL == "" {
|
||||
t.Skip("set HLS_LIVE_URL to run the live-site acceptance test")
|
||||
}
|
||||
referer := os.Getenv("HLS_LIVE_REFERER")
|
||||
|
||||
downloader := NewDownloader(&DownloaderConfig{Storage: NewMemStorage(), StorageDir: t.TempDir()})
|
||||
if err := downloader.Setup(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer downloader.Clear()
|
||||
downloadDir := t.TempDir()
|
||||
cfg, err := downloader.GetConfig()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cfg.DownloadDir = downloadDir
|
||||
if err := downloader.PutConfig(cfg); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
req := &base.Request{URL: playlistURL}
|
||||
if referer != "" {
|
||||
req.Extra = map[string]any{"header": map[string]string{"Referer": referer}}
|
||||
}
|
||||
rr, err := downloader.Resolve(req, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("resolve: %v", err)
|
||||
}
|
||||
name := rr.Res.Files[0].Name
|
||||
size := rr.Res.Size
|
||||
t.Logf("resolved: name=%s size=%d", name, size)
|
||||
|
||||
taskID, err := downloader.Create(rr.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
deadline := time.Now().Add(30 * time.Minute)
|
||||
var lastLogged int64
|
||||
for {
|
||||
task := downloader.GetTask(taskID)
|
||||
if task == nil {
|
||||
t.Fatal("task not found")
|
||||
}
|
||||
if task.Status == base.DownloadStatusDone {
|
||||
break
|
||||
}
|
||||
if task.Status == base.DownloadStatusError {
|
||||
t.Fatalf("task entered error state after %d bytes", task.Progress.Downloaded)
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("timeout, downloaded %d bytes", task.Progress.Downloaded)
|
||||
}
|
||||
if tick := task.Progress.Downloaded; tick-lastLogged > 10<<20 {
|
||||
lastLogged = tick
|
||||
t.Logf("progress: %d bytes", tick)
|
||||
}
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
}
|
||||
|
||||
data, err := os.ReadFile(filepath.Join(downloadDir, name))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if int64(len(data)) == 0 {
|
||||
t.Fatal("merged file is empty")
|
||||
}
|
||||
t.Logf("merged: %s (%d bytes)", name, len(data))
|
||||
if filepath.Ext(name) == ".ts" {
|
||||
// MPEG-TS packets are 188 bytes and start with the 0x47 sync byte.
|
||||
for i := 0; i < 188*4 && i < len(data); i += 188 {
|
||||
if data[i] != 0x47 {
|
||||
t.Fatalf("invalid TS sync byte at offset %d: %#x", i, data[i])
|
||||
}
|
||||
}
|
||||
} else if string(data[4:8]) != "ftyp" {
|
||||
t.Fatalf("invalid fMP4 header: %q", data[:8])
|
||||
}
|
||||
fmt.Printf("LIVE ACCEPTANCE OK: %s (%d bytes)\n", filepath.Join(downloadDir, name), len(data))
|
||||
}
|
||||
@@ -0,0 +1,172 @@
|
||||
package download
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/GopeedLab/gopeed/pkg/base"
|
||||
)
|
||||
|
||||
// hlsTestCdn spins up a master-playlist HLS site whose segments require a
|
||||
// Referer header (anti-hotlink) and returns the server plus expected bytes.
|
||||
func hlsTestCdn(t *testing.T, segmentDelay time.Duration) (*httptest.Server, []byte) {
|
||||
t.Helper()
|
||||
const total = 6
|
||||
segments := make(map[int][]byte)
|
||||
for i := 0; i < total; i++ {
|
||||
segments[i] = []byte(fmt.Sprintf("ENGINE-SEGMENT-%02d", i))
|
||||
}
|
||||
var want []byte
|
||||
for i := 0; i < total; i++ {
|
||||
want = append(want, segments[i]...)
|
||||
}
|
||||
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/master.m3u8", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Write([]byte("#EXTM3U\n" +
|
||||
"#EXT-X-STREAM-INF:BANDWIDTH=500000\nlow/index.m3u8\n" +
|
||||
"#EXT-X-STREAM-INF:BANDWIDTH=2000000\nhigh/index.m3u8\n"))
|
||||
})
|
||||
mux.HandleFunc("/low/index.m3u8", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Write([]byte("#EXTM3U\n#EXTINF:4,\nseg-0.ts\n#EXT-X-ENDLIST\n"))
|
||||
})
|
||||
mux.HandleFunc("/high/index.m3u8", func(w http.ResponseWriter, r *http.Request) {
|
||||
var b strings.Builder
|
||||
b.WriteString("#EXTM3U\n")
|
||||
for i := 0; i < total; i++ {
|
||||
b.WriteString(fmt.Sprintf("#EXTINF:4,\nseg-%d.ts\n", i))
|
||||
}
|
||||
b.WriteString("#EXT-X-ENDLIST\n")
|
||||
w.Write([]byte(b.String()))
|
||||
})
|
||||
for i := 0; i < total; i++ {
|
||||
idx := i
|
||||
mux.HandleFunc(fmt.Sprintf("/high/seg-%d.ts", i), func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Header.Get("Referer") == "" {
|
||||
w.WriteHeader(http.StatusForbidden)
|
||||
return
|
||||
}
|
||||
if segmentDelay > 0 {
|
||||
time.Sleep(segmentDelay)
|
||||
}
|
||||
w.Write(segments[idx])
|
||||
})
|
||||
}
|
||||
server := httptest.NewServer(mux)
|
||||
t.Cleanup(server.Close)
|
||||
return server, want
|
||||
}
|
||||
|
||||
func hlsResolveReferer() *base.Request {
|
||||
return &base.Request{
|
||||
URL: "",
|
||||
Extra: map[string]any{"header": map[string]string{"Referer": "https://player.example.com/page"}},
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloader_HlsProtocolEndToEnd(t *testing.T) {
|
||||
server, want := hlsTestCdn(t, 0)
|
||||
downloader := NewDownloader(&DownloaderConfig{Storage: NewMemStorage(), StorageDir: t.TempDir()})
|
||||
if err := downloader.Setup(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer downloader.Clear()
|
||||
downloadDir := t.TempDir()
|
||||
cfg, err := downloader.GetConfig()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cfg.DownloadDir = downloadDir
|
||||
if err := downloader.PutConfig(cfg); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
req := hlsResolveReferer()
|
||||
req.URL = server.URL + "/master.m3u8"
|
||||
rr, err := downloader.Resolve(req, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := rr.Res.Files[0].Name; got != "index.ts" {
|
||||
t.Errorf("resolved name = %s, want index.ts", got)
|
||||
}
|
||||
taskID, err := downloader.Create(rr.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waitForTaskStatus(t, downloader, taskID, base.DownloadStatusDone, 10*time.Second)
|
||||
|
||||
task := downloader.GetTask(taskID)
|
||||
if task.Protocol != "hls" {
|
||||
t.Errorf("task protocol = %s, want hls", task.Protocol)
|
||||
}
|
||||
data, err := os.ReadFile(filepath.Join(downloadDir, "index.ts"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if string(data) != string(want) {
|
||||
t.Errorf("merged file mismatch: %q", data)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloader_HlsProtocolPauseContinue(t *testing.T) {
|
||||
server, want := hlsTestCdn(t, 120*time.Millisecond)
|
||||
downloader := NewDownloader(&DownloaderConfig{Storage: NewMemStorage(), StorageDir: t.TempDir()})
|
||||
if err := downloader.Setup(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer downloader.Clear()
|
||||
downloadDir := t.TempDir()
|
||||
cfg, err := downloader.GetConfig()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cfg.DownloadDir = downloadDir
|
||||
// Throttle the protocol so the pause lands mid-download.
|
||||
cfg.ProtocolConfig["hls"] = map[string]any{
|
||||
"segmentConnections": 2,
|
||||
"maxRetries": 3,
|
||||
"timeoutSeconds": 5,
|
||||
"prefetchContentLength": false,
|
||||
}
|
||||
if err := downloader.PutConfig(cfg); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
req := hlsResolveReferer()
|
||||
req.URL = server.URL + "/master.m3u8"
|
||||
req.Extra = map[string]any{"header": map[string]string{"Referer": "https://player.example.com/page"}}
|
||||
// CreateDirect: engine resolves lazily on start, like a direct task.
|
||||
taskID, err := downloader.CreateDirect(req, &base.Options{Name: "movie.ts"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waitForTaskStatus(t, downloader, taskID, base.DownloadStatusRunning, 10*time.Second)
|
||||
if err := downloader.Pause(&TaskFilter{IDs: []string{taskID}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waitForTaskStatus(t, downloader, taskID, base.DownloadStatusPause, 5*time.Second)
|
||||
|
||||
if err := downloader.Continue(&TaskFilter{IDs: []string{taskID}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waitForTaskStatus(t, downloader, taskID, base.DownloadStatusDone, 30*time.Second)
|
||||
|
||||
task := downloader.GetTask(taskID)
|
||||
if task.Meta.Res.Size != int64(len(want)) {
|
||||
t.Errorf("final size = %d, want %d", task.Meta.Res.Size, len(want))
|
||||
}
|
||||
data, err := os.ReadFile(filepath.Join(downloadDir, "movie.ts"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if string(data) != string(want) {
|
||||
t.Errorf("merged file mismatch after pause/continue: %q", data)
|
||||
}
|
||||
}
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"github.com/GopeedLab/gopeed/internal/fetcher"
|
||||
"github.com/GopeedLab/gopeed/internal/protocol/bt"
|
||||
"github.com/GopeedLab/gopeed/internal/protocol/ed2k"
|
||||
"github.com/GopeedLab/gopeed/internal/protocol/hls"
|
||||
"github.com/GopeedLab/gopeed/internal/protocol/http"
|
||||
"github.com/GopeedLab/gopeed/pkg/base"
|
||||
enginewebview "github.com/GopeedLab/gopeed/pkg/download/engine/webview"
|
||||
@@ -197,6 +198,7 @@ func (cfg *DownloaderConfig) Init() *DownloaderConfig {
|
||||
}
|
||||
if len(cfg.FetchManagers) == 0 {
|
||||
cfg.FetchManagers = []fetcher.FetcherManager{
|
||||
new(hls.FetcherManager),
|
||||
new(http.FetcherManager),
|
||||
new(bt.FetcherManager),
|
||||
new(ed2k.FetcherManager),
|
||||
|
||||
+8
-3
@@ -13,10 +13,13 @@ const (
|
||||
TaskEventError = uint64(goapi.TaskEventError)
|
||||
)
|
||||
|
||||
// Invoke dispatches a Core API request without using an HTTP listener. It is
|
||||
// the shared entry point for Desktop FFI and gomobile bindings.
|
||||
func Invoke(method, path, rawQuery, body string) string {
|
||||
// Dispatch executes one in-process API request. Native scheduling and callback
|
||||
// behavior belong to bind/native; this function only bridges into the shared
|
||||
// API Service without opening an HTTP listener.
|
||||
func Dispatch(method, path, rawQuery, body string) string {
|
||||
runtimeMu.RLock()
|
||||
service := APIService
|
||||
runtimeMu.RUnlock()
|
||||
if service == nil {
|
||||
return marshalInvokeResult(model.NewErrorResult("service not started"))
|
||||
}
|
||||
@@ -29,6 +32,8 @@ func Invoke(method, path, rawQuery, body string) string {
|
||||
}
|
||||
|
||||
func SubscribeTaskEvents(mask uint64, listener func(payload string)) {
|
||||
runtimeMu.RLock()
|
||||
defer runtimeMu.RUnlock()
|
||||
if APIService == nil {
|
||||
return
|
||||
}
|
||||
|
||||
+76
-4
@@ -4,9 +4,12 @@ import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/GopeedLab/gopeed/pkg/base"
|
||||
"github.com/GopeedLab/gopeed/pkg/rest/model"
|
||||
@@ -27,7 +30,7 @@ func TestNativeInvokeWithoutRESTListener(t *testing.T) {
|
||||
}
|
||||
|
||||
var result model.Result[json.RawMessage]
|
||||
if err := json.Unmarshal([]byte(Invoke(http.MethodGet, "/api/v1/info", "", "")), &result); err != nil {
|
||||
if err := json.Unmarshal([]byte(Dispatch(http.MethodGet, "/api/v1/info", "", "")), &result); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if result.Code != model.CodeOk {
|
||||
@@ -35,6 +38,75 @@ func TestNativeInvokeWithoutRESTListener(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestStartAPIServerIsNotBlockedByPendingDispatch(t *testing.T) {
|
||||
requestStarted := make(chan struct{})
|
||||
releaseRequest := make(chan struct{})
|
||||
var releaseOnce sync.Once
|
||||
release := func() { releaseOnce.Do(func() { close(releaseRequest) }) }
|
||||
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
close(requestStarted)
|
||||
<-releaseRequest
|
||||
w.Header().Set("Content-Length", "0")
|
||||
}))
|
||||
t.Cleanup(func() {
|
||||
release()
|
||||
upstream.Close()
|
||||
Stop()
|
||||
})
|
||||
|
||||
if _, err := Start(&model.StartConfig{NativeMode: true, Storage: model.StorageMem}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
requestBody, err := json.Marshal(&model.ResolveTask{Req: &base.Request{URL: upstream.URL}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
dispatchDone := make(chan struct{})
|
||||
go func() {
|
||||
defer close(dispatchDone)
|
||||
Dispatch(http.MethodPost, "/api/v1/resolve", "", string(requestBody))
|
||||
}()
|
||||
select {
|
||||
case <-requestStarted:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("resolve request did not start")
|
||||
}
|
||||
|
||||
config, err := Downloader.GetConfig()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
config.API = &base.APIServerConfig{Enable: true, Network: "tcp", Address: "127.0.0.1:0"}
|
||||
if err := Downloader.PutConfig(config); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
started := make(chan *model.APIServerState, 1)
|
||||
startErrors := make(chan error, 1)
|
||||
go func() {
|
||||
state, err := StartAPIServer()
|
||||
started <- state
|
||||
startErrors <- err
|
||||
}()
|
||||
select {
|
||||
case state := <-started:
|
||||
if err := <-startErrors; err != nil {
|
||||
t.Fatalf("start API server failed: %v", err)
|
||||
}
|
||||
if !state.Running || state.RunningPort == 0 {
|
||||
t.Fatalf("API server did not start: %#v", state)
|
||||
}
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("pending Resolve blocked API server startup")
|
||||
}
|
||||
|
||||
release()
|
||||
select {
|
||||
case <-dispatchDone:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("resolve request did not finish after release")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAPIServerLifecycleUsesPersistedConfigWithoutRestartingCore(t *testing.T) {
|
||||
port, err := Start(&model.StartConfig{NativeMode: true, Storage: model.StorageMem})
|
||||
if err != nil {
|
||||
@@ -106,7 +178,7 @@ func TestAPIServerLifecycleUsesPersistedConfigWithoutRestartingCore(t *testing.T
|
||||
}
|
||||
|
||||
var result model.Result[json.RawMessage]
|
||||
if err := json.Unmarshal([]byte(Invoke(http.MethodGet, "/api/v1/info", "", "")), &result); err != nil {
|
||||
if err := json.Unmarshal([]byte(Dispatch(http.MethodGet, "/api/v1/info", "", "")), &result); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if result.Code != model.CodeOk {
|
||||
@@ -153,7 +225,7 @@ func TestRestartAPIServerFailsFastAndLeavesListenerStopped(t *testing.T) {
|
||||
}
|
||||
|
||||
var result model.Result[json.RawMessage]
|
||||
if err := json.Unmarshal([]byte(Invoke(http.MethodGet, "/api/v1/info", "", "")), &result); err != nil {
|
||||
if err := json.Unmarshal([]byte(Dispatch(http.MethodGet, "/api/v1/info", "", "")), &result); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if result.Code != model.CodeOk {
|
||||
@@ -229,7 +301,7 @@ func TestNativeAPIServerAutoStartFailureKeepsCoreAvailable(t *testing.T) {
|
||||
}
|
||||
|
||||
var result model.Result[json.RawMessage]
|
||||
if err := json.Unmarshal([]byte(Invoke(http.MethodGet, "/api/v1/info", "", "")), &result); err != nil {
|
||||
if err := json.Unmarshal([]byte(Dispatch(http.MethodGet, "/api/v1/info", "", "")), &result); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if result.Code != model.CodeOk {
|
||||
|
||||
+1
-1
@@ -29,7 +29,7 @@ import (
|
||||
)
|
||||
|
||||
var (
|
||||
runtimeMu sync.Mutex
|
||||
runtimeMu sync.RWMutex
|
||||
|
||||
Downloader *download.Downloader
|
||||
APIService *goapi.Service
|
||||
|
||||
File renamed without changes.
Loaded 100 of 272 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user