feat: replace gblob with blob http transport (#1407)

This commit is contained in:
Levi authored and GitHub committed 2026-07-14 20:05:33 +08:00
1 parent e2ac8ce64b
commit 9fb3758203
34 files changed
+5303 -3057

No files matched your search

+378 -123
View File
File diff suppressed because it is too large. Load diff
File diff suppressed because it is too large. Load diff
-15
View File
@@ -47,21 +47,6 @@ func TestError(t *testing.T) {
}
}
func TestURLCreateObjectURLRejectsUnsupportedType(t *testing.T) {
engine := NewEngine(nil)
defer engine.Close()
_, err := engine.RunString(`
URL.createObjectURL({});
`)
if err == nil {
t.Fatal("expected unsupported type error")
}
if !strings.Contains(err.Error(), "Unsupported object type for URL.createObjectURL: Object") {
t.Fatalf("unexpected unsupported type error: %v", err)
}
}
func TestCallFunction_DetachedAsyncWorkDoesNotBlockReturn(t *testing.T) {
engine := NewEngine(nil)
defer engine.Close()
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,278 @@
package stream_test
import (
"context"
"errors"
"fmt"
"io"
"net/http"
"net/http/httptest"
"sync/atomic"
"testing"
"time"
"github.com/GopeedLab/gopeed/pkg/download/engine"
"github.com/GopeedLab/gopeed/pkg/download/engine/inject/stream"
)
type createdObjectURL struct {
opts *stream.ObjectURLOptions
open stream.ObjectURLOpener
}
func TestResponseBlobObjectURLReadsResponseStream(t *testing.T) {
const payload = "response blob payload"
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.Header().Set("Content-Type", "text/plain")
_, _ = io.WriteString(w, payload[:8])
w.(http.Flusher).Flush() // force an unknown-length/chunked response
_, _ = io.WriteString(w, payload[8:])
}))
defer server.Close()
created := make(chan createdObjectURL, 1)
engine, cleanup := newStreamTestEngine(t, created)
defer func() {
cleanup()
engine.Close()
}()
value, err := engine.RunString(fmt.Sprintf(`
(async () => {
const response = await fetch(%q);
const blob = await response.blob();
return __gopeed_blob_create_object_url(blob);
})()
`, server.URL))
if err != nil {
t.Fatal(err)
}
if value != "blob:test-1" {
t.Fatalf("unexpected object URL: %#v", value)
}
object := <-created
if object.opts.Size != int64(len(payload)) || object.opts.ContentType != "text/plain" {
t.Fatalf("unexpected object URL options: %#v", object.opts)
}
if !object.opts.Range {
t.Fatal("materialized response Blob should advertise range support")
}
for i := 0; i < 2; i++ {
reader, err := object.open(context.Background(), stream.ObjectURLOpenRequest{Offset: 0, End: -1})
if err != nil {
t.Fatal(err)
}
data, readErr := io.ReadAll(reader)
_ = reader.Close()
if readErr != nil {
t.Fatal(readErr)
}
if got := string(data); got != payload {
t.Fatalf("unexpected response Blob data on open %d: %q", i+1, got)
}
}
rangeReader, err := object.open(context.Background(), stream.ObjectURLOpenRequest{Offset: 9, End: 12})
if err != nil {
t.Fatal(err)
}
rangeData, readErr := io.ReadAll(rangeReader)
_ = rangeReader.Close()
if readErr != nil {
t.Fatal(readErr)
}
if got, want := string(rangeData), payload[9:13]; got != want {
t.Fatalf("unexpected materialized Blob range: got %q want %q", got, want)
}
}
func TestResponseBlobRejectsTruncatedResponse(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.Header().Set("Content-Length", "64")
_, _ = io.WriteString(w, "partial")
}))
defer server.Close()
created := make(chan createdObjectURL, 1)
engine, cleanup := newStreamTestEngine(t, created)
defer func() {
cleanup()
engine.Close()
}()
value, err := engine.RunString(fmt.Sprintf(`
(async () => {
const response = await fetch(%q);
try {
await response.blob();
return "resolved";
} catch (_) {
return "rejected";
}
})()
`, server.URL))
if err != nil {
t.Fatal(err)
}
if value != "rejected" {
t.Fatalf("truncated response unexpectedly materialized a Blob: %#v", value)
}
}
func TestEmptyBlobObjectURL(t *testing.T) {
created := make(chan createdObjectURL, 1)
engine, cleanup := newStreamTestEngine(t, created)
defer func() {
cleanup()
engine.Close()
}()
value, err := engine.RunString(`
__gopeed_blob_create_object_url(new Blob([], { type: "application/empty" }))
`)
if err != nil {
t.Fatal(err)
}
if value != "blob:test-1" {
t.Fatalf("unexpected object URL: %#v", value)
}
object := <-created
if object.opts.Size != 0 || object.opts.Range {
t.Fatalf("unexpected empty Blob options: %#v", object.opts)
}
reader, err := object.open(context.Background(), stream.ObjectURLOpenRequest{Offset: 0, End: -1})
if err != nil {
t.Fatal(err)
}
defer reader.Close()
data, err := io.ReadAll(reader)
if err != nil {
t.Fatal(err)
}
if len(data) != 0 {
t.Fatalf("expected empty Blob, got %q", data)
}
}
func TestBlobObjectURLReaderCloseCancelsPendingJSReader(t *testing.T) {
created := make(chan createdObjectURL, 1)
engine, cleanup := newStreamTestEngine(t, created)
defer func() {
cleanup()
engine.Close()
}()
value, err := engine.RunString(`
globalThis.__blobPipeCancelState = {
readCalls: 0,
cancelCalls: 0,
releaseCalls: 0,
cancelReason: "",
resolveRead: null,
};
const pendingSource = {
getReader() {
return {
read() {
__blobPipeCancelState.readCalls++;
return new Promise((resolve) => {
__blobPipeCancelState.resolveRead = resolve;
});
},
cancel(reason) {
__blobPipeCancelState.cancelCalls++;
__blobPipeCancelState.cancelReason = String(reason);
if (__blobPipeCancelState.resolveRead) {
__blobPipeCancelState.resolveRead({ done: true, value: undefined });
}
return Promise.resolve();
},
releaseLock() {
__blobPipeCancelState.releaseCalls++;
},
};
},
};
__gopeed_blob_create_object_url(async () => pendingSource);
`)
if err != nil {
t.Fatal(err)
}
if value != "blob:test-1" {
t.Fatalf("unexpected object URL: %#v", value)
}
object := <-created
reader, err := object.open(context.Background(), stream.ObjectURLOpenRequest{Offset: 0, End: -1})
if err != nil {
t.Fatal(err)
}
readDone := make(chan error, 1)
go func() {
_, readErr := reader.Read(make([]byte, 1))
readDone <- readErr
}()
waitForJSValue(t, engine, `String(__blobPipeCancelState.readCalls)`, "1")
if err := reader.Close(); err != nil {
t.Fatal(err)
}
if err := reader.Close(); err != nil {
t.Fatal(err)
}
select {
case readErr := <-readDone:
if !errors.Is(readErr, context.Canceled) {
t.Fatalf("unexpected pending read result: %v", readErr)
}
case <-time.After(2 * time.Second):
t.Fatal("pending Go read was not released by Close")
}
waitForJSValue(t, engine, `JSON.stringify({
cancelCalls: __blobPipeCancelState.cancelCalls,
releaseCalls: __blobPipeCancelState.releaseCalls,
cancelReason: __blobPipeCancelState.cancelReason,
})`, `{"cancelCalls":1,"releaseCalls":1,"cancelReason":"blob request closed"}`)
}
func waitForJSValue(t *testing.T, engine *engine.Engine, expression, want string) {
t.Helper()
deadline := time.Now().Add(2 * time.Second)
var got any
for time.Now().Before(deadline) {
var err error
got, err = engine.RunString(expression)
if err != nil {
t.Fatal(err)
}
if got == want {
return
}
time.Sleep(10 * time.Millisecond)
}
t.Fatalf("timed out waiting for JavaScript value %q, got %#v", want, got)
}
func newStreamTestEngine(t *testing.T, created chan<- createdObjectURL) (*engine.Engine, func()) {
t.Helper()
var next atomic.Int64
var cleanup func()
e := engine.NewEngine(&engine.Config{
StreamConfig: &stream.Config{
CreateObjectURL: func(opts *stream.ObjectURLOptions, open stream.ObjectURLOpener) (string, error) {
created <- createdObjectURL{opts: opts, open: open}
return fmt.Sprintf("blob:test-%d", next.Add(1)), nil
},
RevokeObjectURL: func(string) error { return nil },
RegisterCleanup: func(fn func()) {
cleanup = fn
},
},
})
if cleanup == nil {
t.Fatal("stream cleanup was not registered")
}
return e, cleanup
}
@@ -0,0 +1,191 @@
package stream
import (
"context"
"io"
"strings"
"sync/atomic"
"testing"
"github.com/GopeedLab/gopeed/pkg/download/engine/inject/file"
"github.com/GopeedLab/gopeed/pkg/download/engine/inject/formdata"
"github.com/dop251/goja"
)
func TestExportFetchRequestSnapshotsArrayBuffer(t *testing.T) {
runtime := goja.New()
buffer := runtime.NewArrayBuffer([]byte("before"))
request := runtime.NewObject()
setFetchRequestDefaults(t, request)
if err := request.Set("body", buffer); err != nil {
t.Fatal(err)
}
meta, err := exportFetchRequest(runtime, request)
if err != nil {
t.Fatal(err)
}
buffer.Bytes()[0] = 'X'
body, ok := meta.Body.([]byte)
if !ok {
t.Fatalf("expected byte snapshot, got %T", meta.Body)
}
if got := string(body); got != "before" {
t.Fatalf("ArrayBuffer snapshot changed after export: %q", got)
}
}
func TestExportFetchRequestSnapshotsFormData(t *testing.T) {
runtime := goja.New()
runtime.SetFieldNameMapper(goja.TagFieldNameMapper("json", true))
if err := formdata.Enable(runtime); err != nil {
t.Fatal(err)
}
value, err := runtime.RunString(`
globalThis.form = new FormData();
form.append("field", "before");
form;
`)
if err != nil {
t.Fatal(err)
}
request := runtime.NewObject()
setFetchRequestDefaults(t, request)
if err := request.Set("body", value); err != nil {
t.Fatal(err)
}
meta, err := exportFetchRequest(runtime, request)
if err != nil {
t.Fatal(err)
}
if _, err := runtime.RunString(`form.set("field", "after")`); err != nil {
t.Fatal(err)
}
body, ok := meta.Body.(*fetchFormDataSnapshot)
if !ok {
t.Fatalf("expected FormData snapshot, got %T", meta.Body)
}
if len(body.entries) != 1 {
t.Fatalf("expected one FormData entry, got %d", len(body.entries))
}
if got := body.entries[0]; got.name != "field" || got.value != "before" {
t.Fatalf("unexpected FormData snapshot: %#v", got)
}
}
func setFetchRequestDefaults(t *testing.T, request *goja.Object) {
t.Helper()
for key, value := range map[string]any{
"url": "https://example.test",
"method": "POST",
"redirect": "follow",
"credentials": "same-origin",
} {
if err := request.Set(key, value); err != nil {
t.Fatal(err)
}
}
}
func TestSnapshotFetchBodyCopiesFileMetadata(t *testing.T) {
source := &file.File{
Reader: strings.NewReader("payload"),
Name: "before.txt",
Size: 7,
}
value, err := snapshotFetchBody(source)
if err != nil {
t.Fatal(err)
}
snapshot, ok := value.(*file.File)
if !ok {
t.Fatalf("expected file snapshot, got %T", value)
}
source.Name = "after.txt"
source.Size = 99
if snapshot.Name != "before.txt" || snapshot.Size != 7 {
t.Fatalf("file metadata was not snapshotted: %#v", snapshot)
}
}
func TestFetchRegistryCloseAll(t *testing.T) {
registry := newFetchRegistry()
streamCtx, cancel := context.WithCancel(context.Background())
body := &trackingReadCloser{}
registry.streams["active"] = &fetchStream{
body: body,
cancel: cancel,
ctx: streamCtx,
ch: make(chan fetchChunk),
}
registry.CloseAll()
registry.CloseAll()
select {
case <-streamCtx.Done():
default:
t.Fatal("active fetch context was not canceled")
}
if got := body.closeCalls.Load(); got != 1 {
t.Fatalf("expected body to close once, got %d", got)
}
registry.mu.Lock()
closed := registry.closed
remaining := len(registry.streams)
registry.mu.Unlock()
if !closed || remaining != 0 {
t.Fatalf("registry not fully closed: closed=%v remaining=%d", closed, remaining)
}
if _, err := registry.Open("", nil, &fetchRequest{URL: "https://example.test"}); err == nil {
t.Fatal("expected a closed registry to reject new fetches")
}
}
func TestFetchRegistryReadPreservesChunkOrder(t *testing.T) {
registry := newFetchRegistry()
ctx, cancel := context.WithCancel(context.Background())
stream := &fetchStream{
body: io.NopCloser(strings.NewReader("")),
cancel: cancel,
ctx: ctx,
ch: make(chan fetchChunk, 2),
}
stream.ch <- fetchChunk{data: []byte("abc")}
stream.ch <- fetchChunk{data: []byte("def")}
registry.streams["ordered"] = stream
defer registry.CloseAll()
var got strings.Builder
for range 4 {
chunk, done, err := registry.Read("ordered", 2)
if err != nil {
t.Fatal(err)
}
if done {
t.Fatal("stream ended before buffered chunks were read")
}
got.Write(chunk)
}
if got.String() != "abcdef" {
t.Fatalf("fetch chunks were reordered: %q", got.String())
}
}
type trackingReadCloser struct {
closeCalls atomic.Int32
}
func (r *trackingReadCloser) Read([]byte) (int, error) {
return 0, io.EOF
}
func (r *trackingReadCloser) Close() error {
r.closeCalls.Add(1)
return nil
}
File diff suppressed because it is too large. Load diff
+19 -29
View File
@@ -271,7 +271,6 @@ func (d *Downloader) triggerOnResolve(req *base.Request) (res *base.Resource, er
}
ensureResourceRequestRawURLs(req, ctx.Res)
ctx.Res.CalcSize(nil)
d.applyGBlobResourceMetadata(ctx.Res)
}
res = ctx.Res
},
@@ -279,25 +278,12 @@ func (d *Downloader) triggerOnResolve(req *base.Request) (res *base.Resource, er
return
}
func (d *Downloader) applyGBlobResourceMetadata(res *base.Resource) {
if d.gblob == nil || res == nil {
return
}
for _, file := range res.Files {
if file == nil || file.Req == nil {
continue
}
_ = d.gblob.SetSize(file.Req.URL, file.Size)
_ = d.gblob.SetRange(file.Req.URL, res.Range)
}
}
func (d *Downloader) triggerOnStart(task *Task) {
doTrigger(d,
EventOnStart,
task.Meta.Req,
&OnStartContext{
Task: NewExtensionTask(d, task),
Task: newOnStartExtensionTask(task),
},
func(ext *Extension, gopeed *Instance, ctx *OnStartContext) {
// Validate request structure
@@ -309,7 +295,6 @@ func (d *Downloader) triggerOnStart(task *Task) {
}
},
)
return
}
func (d *Downloader) triggerOnError(task *Task, err error) {
@@ -317,7 +302,7 @@ func (d *Downloader) triggerOnError(task *Task, err error) {
EventOnError,
task.Meta.Req,
&OnErrorContext{
Task: NewExtensionTask(d, task),
Task: newOnErrorExtensionTask(d, task),
Error: err,
},
nil,
@@ -328,8 +313,8 @@ func (d *Downloader) triggerOnDone(task *Task) {
doTrigger(d,
EventOnDone,
task.Meta.Req,
&OnErrorContext{
Task: NewExtensionTask(d, task),
&OnDoneContext{
Task: newOnDoneExtensionTask(task),
},
nil,
)
@@ -720,7 +705,7 @@ type OnResolveContext struct {
}
type OnStartContext struct {
Task *ExtensionTask `json:"task"`
Task *Task `json:"task"`
}
type OnErrorContext struct {
@@ -741,28 +726,33 @@ type ExtensionTask struct {
*Task
}
func NewExtensionTask(download *Downloader, task *Task) *ExtensionTask {
// restricts extension scripts to only modify request info
func cloneExtensionTask(task *Task) *Task {
newTask := task.clone()
newTask.Meta.Req = task.Meta.Req
return newTask
}
func newOnStartExtensionTask(task *Task) *Task {
return cloneExtensionTask(task)
}
func newOnErrorExtensionTask(download *Downloader, task *Task) *ExtensionTask {
return &ExtensionTask{
download: download,
Task: newTask,
Task: cloneExtensionTask(task),
}
}
func newOnDoneExtensionTask(task *Task) *Task {
return cloneExtensionTask(task)
}
func (t *ExtensionTask) Continue() error {
return t.download.Continue(&TaskFilter{
IDs: []string{t.ID},
})
}
func (t *ExtensionTask) Pause() error {
return t.download.Pause(&TaskFilter{
IDs: []string{t.ID},
})
}
func parseSettings(settings []*Setting) map[string]any {
m := make(map[string]any)
for _, s := range settings {
+19 -46
View File
@@ -1,14 +1,14 @@
package download
import (
"fmt"
"context"
"io"
"sync"
"github.com/GopeedLab/gopeed/internal/protocol/gblob"
internalblob "github.com/GopeedLab/gopeed/internal/blob"
"github.com/GopeedLab/gopeed/pkg/download/engine"
"github.com/GopeedLab/gopeed/pkg/download/engine/inject/stream"
enginewebview "github.com/GopeedLab/gopeed/pkg/download/engine/webview"
"github.com/dop251/goja"
)
type engineSession struct {
@@ -96,57 +96,30 @@ func (s *engineSession) runClosers() {
}
}
func normalizeStreamChunk(data any) ([]byte, error) {
switch v := data.(type) {
case nil:
return nil, nil
case string:
return []byte(v), nil
case []byte:
return v, nil
case goja.ArrayBuffer:
return v.Bytes(), nil
default:
return nil, fmt.Errorf("unsupported stream chunk type: %T", data)
}
}
func (d *Downloader) newExtensionEngine() (*engine.Engine, *engineSession) {
session := newEngineSession(nil)
engineCfg := &stream.Config{
CreateBlobObjectURL: func(data []byte, contentType string) (string, error) {
return d.gblob.CreateBlob(data, contentType)
},
CreateWritableStreamObjectURL: func(opts *stream.WritableStreamObjectURLOptions) (string, error) {
reopenable := false
CreateObjectURL: func(opts *stream.ObjectURLOptions, open stream.ObjectURLOpener) (string, error) {
createOpts := &internalblob.CreateOptions{
Session: session,
}
if opts != nil {
reopenable = opts.Reopenable
createOpts.ContentType = opts.ContentType
createOpts.Size = opts.Size
createOpts.Range = opts.Range
}
return d.gblob.CreateWritableStream(&gblob.CreateWritableStreamOptions{
Session: session,
Reopenable: reopenable,
})
},
RegisterWritableStreamResume: func(url string, reopen func(offset int64) error) error {
return d.gblob.SetResumeOpener(url, reopen)
},
WriteWritableStreamObjectURL: func(url string, data any) error {
chunk, err := normalizeStreamChunk(data)
if err != nil {
return err
}
return d.gblob.Write(url, chunk)
},
CloseWritableStreamObjectURL: func(url string) error {
return d.gblob.CloseSource(url)
},
AbortWritableStreamObjectURL: func(url string, reason string) error {
return d.gblob.AbortSource(url, fmt.Errorf("%s", reason))
return d.blob.CreateOpener(func(ctx context.Context, req internalblob.OpenRequest) (io.ReadCloser, error) {
return open(ctx, stream.ObjectURLOpenRequest{
Offset: req.Offset,
End: req.End,
})
}, createOpts)
},
RevokeObjectURL: func(url string) error {
return d.gblob.Revoke(url)
return d.blob.Revoke(url)
},
ProxyHandler: d.cfg.Proxy.ToHandler(),
ProxyHandler: d.cfg.Proxy.ToHandler(),
RegisterCleanup: session.OnClose,
}
e := engine.NewEngine(&engine.Config{
ProxyConfig: d.cfg.Proxy,
+33 -2
View File
@@ -29,10 +29,15 @@ func injectGopeed(vm *goja.Runtime, gopeed *Instance) error {
return err
}
runtimeObject := vm.NewObject()
if gopeed.Runtime != nil && gopeed.Runtime.WebView != nil {
if err := runtimeObject.Set("webview", newJSWebViewRuntime(vm, gopeed.Runtime.WebView)); err != nil {
if gopeed.Runtime != nil {
if err := runtimeObject.Set("blob", newJSBlobRuntime(vm)); err != nil {
return err
}
if gopeed.Runtime.WebView != nil {
if err := runtimeObject.Set("webview", newJSWebViewRuntime(vm, gopeed.Runtime.WebView)); err != nil {
return err
}
}
}
if err := gopeedObject.Set("runtime", runtimeObject); err != nil {
return err
@@ -40,6 +45,32 @@ func injectGopeed(vm *goja.Runtime, gopeed *Instance) error {
return vm.Set("gopeed", gopeedObject)
}
func newJSBlobRuntime(vm *goja.Runtime) *goja.Object {
obj := vm.NewObject()
_ = obj.Set("createObjectURL", func(call goja.FunctionCall) goja.Value {
fn, ok := goja.AssertFunction(vm.Get("__gopeed_blob_create_object_url"))
if !ok {
panic(vm.ToValue(fmt.Errorf("blob runtime is not available")))
}
value, err := fn(goja.Undefined(), call.Argument(0), call.Argument(1))
if err != nil {
panic(vm.ToValue(err))
}
return value
})
_ = obj.Set("revokeObjectURL", func(call goja.FunctionCall) goja.Value {
fn, ok := goja.AssertFunction(vm.Get("__gopeed_blob_revoke_object_url"))
if !ok {
panic(vm.ToValue(fmt.Errorf("blob runtime is not available")))
}
if _, err := fn(goja.Undefined(), call.Argument(0)); err != nil {
panic(vm.ToValue(err))
}
return goja.Undefined()
})
return obj
}
func newJSEventsRuntime(vm *goja.Runtime, events InstanceEvents) *goja.Object {
obj := vm.NewObject()
register := func(event ActivationEvent, call goja.FunctionCall) goja.Value {
File diff suppressed because it is too large. Load diff
+3 -2
View File
@@ -9,7 +9,6 @@ import (
"github.com/GopeedLab/gopeed/internal/fetcher"
"github.com/GopeedLab/gopeed/internal/protocol/bt"
"github.com/GopeedLab/gopeed/internal/protocol/ed2k"
"github.com/GopeedLab/gopeed/internal/protocol/gblob"
"github.com/GopeedLab/gopeed/internal/protocol/http"
"github.com/GopeedLab/gopeed/pkg/base"
enginewebview "github.com/GopeedLab/gopeed/pkg/download/engine/webview"
@@ -37,6 +36,9 @@ type Task struct {
timer *util.Timer
statusLock *sync.Mutex
lock *sync.Mutex
blobRefLock *sync.Mutex
blobURL string
runGeneration uint64
speedArr []int64
uploadSpeedArr []int64
}
@@ -166,7 +168,6 @@ func (cfg *DownloaderConfig) Init() *DownloaderConfig {
}
if len(cfg.FetchManagers) == 0 {
cfg.FetchManagers = []fetcher.FetcherManager{
new(gblob.FetcherManager),
new(http.FetcherManager),
new(bt.FetcherManager),
new(ed2k.FetcherManager),
+111
View File
@@ -0,0 +1,111 @@
gopeed.events.onResolve(async function (ctx) {
const encoder = new TextEncoder();
if (ctx.req.url.endsWith("/blob")) {
const blob = new Blob(["hello world"], {type: "text/plain"});
const url = gopeed.runtime.blob.createObjectURL(blob);
ctx.res = {
name: "blob-blob",
files: [
{
name: "hello.txt",
size: 11,
req: {
url,
}
}
]
};
return;
}
if (ctx.req.url.endsWith("/opener-range")) {
const payload = encoder.encode("line 1\nline 2\nline 3\n");
const firstChunkLength = encoder.encode("line 1\n").byteLength;
let firstAttempt = true;
const open = async ({ offset = 0, end = -1 }) => new ReadableStream({
async start(controller) {
try {
if (offset === 0 && firstAttempt) {
firstAttempt = false;
controller.enqueue(payload.slice(0, firstChunkLength));
await new Promise((resolve) => setTimeout(resolve, 120));
controller.error(new Error("resume required"));
return;
}
controller.enqueue(payload.slice(offset, end >= 0 ? end + 1 : undefined));
controller.close();
} catch (err) {
controller.error(err);
}
}
});
const url = gopeed.runtime.blob.createObjectURL(open, { size: payload.byteLength, range: true });
ctx.res = {
name: "blob-opener-range",
range: true,
files: [
{
name: "opener-range.txt",
size: payload.byteLength,
req: {
url,
}
}
]
};
return;
}
if (ctx.req.url.endsWith("/opener-unknown")) {
const url = gopeed.runtime.blob.createObjectURL(async ({ offset = 0 }) => new ReadableStream({
async start(controller) {
if (offset > 0) {
controller.close();
return;
}
controller.enqueue(encoder.encode("line 1\n"));
await new Promise((resolve) => setTimeout(resolve, 1500));
controller.enqueue(encoder.encode("line 2\n"));
controller.close();
},
}));
ctx.res = {
name: "blob-opener-unknown",
files: [
{
name: "opener-unknown.txt",
req: {
url,
}
}
]
};
return;
}
const url = gopeed.runtime.blob.createObjectURL(async ({ offset = 0, end = -1 }) => new ReadableStream({
start(controller) {
const payload = encoder.encode("line 1\nline 2\n");
controller.enqueue(payload.slice(offset, end >= 0 ? end + 1 : undefined));
controller.close();
},
}), { size: 14 });
ctx.res = {
name: "blob-opener",
files: [
{
name: "opener.txt",
size: 14,
req: {
url,
}
}
]
};
});
@@ -1,6 +1,6 @@
{
"name": "gblob",
"title": "gopeed extension gblob test",
"name": "blob",
"title": "gopeed extension blob test",
"version": "0.0.1",
"scripts": [
{
@@ -0,0 +1,140 @@
const payloadSize = 262144;
const chunkSize = 4096;
const payloadText = 'x'.repeat(payloadSize);
function delayedPayloadStream(offset = 0, end = -1) {
const limit = end >= 0 ? Math.min(end + 1, payloadSize) : payloadSize;
let pos = offset;
return new ReadableStream({
async pull(controller) {
if (pos >= limit) {
controller.close();
return;
}
const size = Math.min(chunkSize, limit - pos);
const chunk = new Uint8Array(size);
chunk.fill(120);
pos += size;
controller.enqueue(chunk);
await new Promise((resolve) => setTimeout(resolve, 20));
},
});
}
function fastPayloadStream(offset = 0, end = -1) {
const limit = end >= 0 ? Math.min(end + 1, payloadSize) : payloadSize;
let pos = offset;
return new ReadableStream({
pull(controller) {
if (pos >= limit) {
controller.close();
return;
}
const size = Math.min(chunkSize, limit - pos);
const chunk = new Uint8Array(size);
chunk.fill(120);
pos += size;
controller.enqueue(chunk);
},
});
}
function createBlobSourceURL(delayed) {
const blob = new Blob([payloadText], { type: 'application/octet-stream' });
const originalSlice = blob.slice.bind(blob);
blob.slice = function (offset = 0, end) {
if (!delayed) {
return originalSlice(offset, end);
}
const sliceEnd = Number.isFinite(Number(end)) ? Number(end) - 1 : -1;
return {
stream() {
return delayedPayloadStream(Number(offset) || 0, sliceEnd);
},
};
};
return gopeed.runtime.blob.createObjectURL(blob);
}
function createOpenerSourceURL(delayed) {
const open = async () => delayed ? delayedPayloadStream(0, -1) : fastPayloadStream(0, -1);
return gopeed.runtime.blob.createObjectURL(open, {
size: payloadSize,
contentType: 'application/octet-stream',
});
}
function createRangeSourceURL(delayed) {
const open = async ({ offset = 0, end = -1 }) => delayed ? delayedPayloadStream(offset, end) : fastPayloadStream(offset, end);
return gopeed.runtime.blob.createObjectURL(open, {
size: payloadSize,
range: true,
contentType: 'application/octet-stream',
});
}
function modeFromURL(url) {
if (url.includes('/pause-rebuild-blob')) {
return 'blob';
}
if (url.includes('/pause-rebuild-opener')) {
return 'opener';
}
if (url.includes('/pause-rebuild-range')) {
return 'range';
}
return '';
}
function createPayloadURL(mode, delayed) {
switch (mode) {
case 'blob':
return createBlobSourceURL(delayed);
case 'opener':
return createOpenerSourceURL(delayed);
case 'range':
return createRangeSourceURL(delayed);
default:
throw new Error('unsupported pause rebuild mode: ' + mode);
}
}
gopeed.events.onResolve(async function (ctx) {
const mode = modeFromURL(ctx.req.url);
if (!mode) {
return;
}
ctx.res = {
name: 'blob-pause-rebuild-' + mode,
range: mode !== 'opener',
files: [
{
name: 'pause-rebuild-' + mode + '.bin',
size: payloadSize,
req: {
url: createPayloadURL(mode, true),
rawUrl: ctx.req.rawUrl || ctx.req.url,
labels: {
mode: 'pause-rebuild',
source: mode,
},
},
},
],
};
});
gopeed.events.onError(async function (ctx) {
const req = ctx.task?.meta?.req;
const source = req?.labels?.source || modeFromURL(req?.rawUrl || '');
if (!req || req.labels?.mode !== 'pause-rebuild' || req.labels?.rebuilt === 'true' || !source) {
return;
}
req.url = createPayloadURL(source, false);
req.labels = req.labels || {};
req.labels.started = 'true';
req.labels.rebuilt = 'true';
ctx.task.continue();
});
@@ -0,0 +1,28 @@
{
"name": "blob-pause-rebuild",
"title": "gopeed extension blob pause rebuild test",
"version": "0.0.1",
"scripts": [
{
"event": "onResolve",
"match": {
"urls": [
"*://example.com/pause-rebuild*"
]
},
"entry": "index.js"
},
{
"event": "onError",
"match": {
"labels": [
"mode"
],
"urls": [
"*://example.com/pause-rebuild*"
]
},
"entry": "index.js"
}
]
}
+46
View File
@@ -0,0 +1,46 @@
gopeed.events.onResolve(async function (ctx) {
const isRange = ctx.req.url.endsWith("/recover-range");
if (!isRange && !ctx.req.url.endsWith("/recover")) {
return;
}
const url = gopeed.runtime.blob.createObjectURL(async ({ offset = 0 }) => new ReadableStream({
async start(controller) {
if (offset > 0) {
controller.close();
return;
}
controller.enqueue(new TextEncoder().encode("stale-prefix-that-must-be-removed\n"));
await new Promise((resolve) => setTimeout(resolve, 50));
controller.error(new Error("expired"));
},
}), { size: 64, range: isRange });
ctx.res = {
name: "blob-recover",
files: [
{
name: "recover.txt",
size: 64,
req: {
url,
}
}
]
};
});
gopeed.events.onError(async function (ctx) {
const req = ctx.task.meta.req;
if (!req.rawUrl || (!req.rawUrl.endsWith("/recover") && !req.rawUrl.endsWith("/recover-range"))) {
return;
}
req.labels = req.labels || {};
if (req.labels.recovered === "true") {
return;
}
req.labels.recovered = "true";
req.url = gopeed.runtime.blob.createObjectURL(new Blob(["ok\n"], { type: "text/plain" }));
ctx.task.continue();
});
@@ -1,6 +1,6 @@
{
"name": "gblob-recover",
"title": "gopeed extension gblob recover test",
"name": "blob-recover",
"title": "gopeed extension blob recover test",
"version": "0.0.1",
"scripts": [
{
@@ -1,15 +1,15 @@
function createRangeGBlobUrl(target) {
return URL.createObjectURL(async (offset) => {
function createRangeBlobUrl(target) {
return gopeed.runtime.blob.createObjectURL(async ({ offset = 0, end = -1 }) => {
const headers = {};
if (offset > 0) {
headers.Range = `bytes=${offset}-`;
if (offset > 0 || end >= 0) {
headers.Range = `bytes=${offset}-${end >= 0 ? end : ''}`;
}
const response = await fetch(target, { headers });
if (!response.body) {
throw new Error('empty response body');
}
return response.body;
});
}, { size: 262144, range: true });
}
gopeed.events.onResolve(async function (ctx) {
@@ -18,14 +18,14 @@ gopeed.events.onResolve(async function (ctx) {
}
ctx.res = {
name: 'gblob-restart',
name: 'blob-restart',
range: true,
files: [
{
name: 'restart.bin',
size: 262144,
req: {
url: createRangeGBlobUrl(ctx.req.rawUrl || ctx.req.url),
url: createRangeBlobUrl(ctx.req.rawUrl || ctx.req.url),
rawUrl: ctx.req.rawUrl || ctx.req.url,
labels: {
mode: 'restart',
@@ -38,7 +38,7 @@ gopeed.events.onResolve(async function (ctx) {
gopeed.events.onError(async function (ctx) {
const req = ctx.task?.meta?.req;
if (!req || !req.rawUrl || !req.url || !req.url.startsWith('gblob:')) {
if (!req || !req.rawUrl || !req.url || !req.url.includes('/__blob/')) {
return;
}
req.labels = req.labels || {};
@@ -47,7 +47,7 @@ gopeed.events.onError(async function (ctx) {
}
try {
req.url = createRangeGBlobUrl(req.rawUrl);
req.url = createRangeBlobUrl(req.rawUrl);
req.labels.started = 'true';
req.labels.rebuilt = 'true';
ctx.task.continue();
@@ -1,6 +1,6 @@
{
"name": "gblob-restart",
"title": "gopeed extension gblob restart resume test",
"name": "blob-restart",
"title": "gopeed extension blob restart resume test",
"version": "0.0.1",
"scripts": [
{
@@ -0,0 +1,198 @@
const encoder = new TextEncoder();
const decoder = new TextDecoder();
function byteLength(value) {
return encoder.encode(value).byteLength;
}
function sourceURL(rawURL) {
const url = new URL(rawURL);
url.searchParams.set("source", "1");
return url.toString();
}
function createFetchDrainedStream(url) {
return new ReadableStream({
async start(controller) {
try {
const response = await fetch(url);
const reader = response.body.getReader();
while (true) {
const { done, value } = await reader.read();
if (done) {
controller.close();
return;
}
controller.enqueue(value);
}
} catch (error) {
controller.error(error);
}
},
});
}
function createFetchDrainedRangeStream(url, offset, end) {
return new ReadableStream({
async start(controller) {
try {
const headers = {};
if (offset > 0 || end >= 0) {
headers.Range = `bytes=${offset}-${end >= 0 ? end : ""}`;
}
const response = await fetch(url, { headers });
const reader = response.body.getReader();
while (true) {
const { done, value } = await reader.read();
if (done) {
controller.close();
return;
}
controller.enqueue(value);
}
} catch (error) {
controller.error(error);
}
},
});
}
function createMultiplexStreams(url) {
let videoController;
let audioController;
let started = false;
async function pump() {
if (started || !videoController || !audioController) {
return;
}
started = true;
try {
const response = await fetch(url);
const reader = response.body.getReader();
let pending = "";
while (true) {
const { done, value } = await reader.read();
if (done) {
break;
}
pending += decoder.decode(value, { stream: true });
let newlineIndex;
while ((newlineIndex = pending.indexOf("\n")) >= 0) {
const line = pending.slice(0, newlineIndex);
pending = pending.slice(newlineIndex + 1);
if (line.startsWith("v:")) {
videoController.enqueue(encoder.encode(line.slice(2)));
} else if (line.startsWith("a:")) {
audioController.enqueue(encoder.encode(line.slice(2)));
}
}
}
if (pending.startsWith("v:")) {
videoController.enqueue(encoder.encode(pending.slice(2)));
} else if (pending.startsWith("a:")) {
audioController.enqueue(encoder.encode(pending.slice(2)));
}
videoController.close();
audioController.close();
} catch (error) {
videoController.error(error);
audioController.error(error);
}
}
return {
video: new ReadableStream({
start(controller) {
videoController = controller;
pump();
},
}),
audio: new ReadableStream({
start(controller) {
audioController = controller;
pump();
},
}),
};
}
gopeed.events.onResolve(async function (ctx) {
if (ctx.req.url.includes("/single-fetch-stream")) {
const payloadSize = Number(new URL(ctx.req.url).searchParams.get("size")) || 0;
const url = gopeed.runtime.blob.createObjectURL(async () => createFetchDrainedStream(sourceURL(ctx.req.url)), {
size: payloadSize,
});
ctx.res = {
name: "single-fetch-stream",
files: [
{
name: "single.bin",
size: payloadSize,
req: { url },
},
],
};
return;
}
if (ctx.req.url.includes("/range-fetch-stream")) {
const payloadSize = Number(new URL(ctx.req.url).searchParams.get("size")) || 0;
const url = gopeed.runtime.blob.createObjectURL(
async ({ offset = 0, end = -1 }) => createFetchDrainedRangeStream(sourceURL(ctx.req.url), offset, end),
{
size: payloadSize,
range: true,
}
);
ctx.res = {
name: "range-fetch-stream",
range: true,
files: [
{
name: "range.bin",
size: payloadSize,
req: { url },
},
],
};
return;
}
if (ctx.req.url.includes("/multiplex-fetch-stream")) {
const videoChunks = ["video-0", "video-1", "video-2", "video-3"];
const audioChunks = ["audio-0", "audio-1", "audio-2", "audio-3"];
let streams;
const openStreams = () => {
if (!streams) {
streams = createMultiplexStreams(sourceURL(ctx.req.url));
}
return streams;
};
const videoUrl = gopeed.runtime.blob.createObjectURL(async () => openStreams().video, {
size: videoChunks.reduce((sum, chunk) => sum + byteLength(chunk), 0),
});
const audioUrl = gopeed.runtime.blob.createObjectURL(async () => openStreams().audio, {
size: audioChunks.reduce((sum, chunk) => sum + byteLength(chunk), 0),
});
ctx.res = {
name: "multiplex-fetch-stream",
files: [
{
name: "video.bin",
size: videoChunks.reduce((sum, chunk) => sum + byteLength(chunk), 0),
req: { url: videoUrl },
},
{
name: "audio.bin",
size: audioChunks.reduce((sum, chunk) => sum + byteLength(chunk), 0),
req: { url: audioUrl },
},
],
};
}
});
@@ -0,0 +1,18 @@
{
"name": "blob-stream-bridge",
"title": "gopeed extension blob stream bridge test",
"version": "0.0.1",
"scripts": [
{
"event": "onResolve",
"match": {
"urls": [
"*://*/single-fetch-stream*",
"*://*/range-fetch-stream*",
"*://*/multiplex-fetch-stream*"
]
},
"entry": "index.js"
}
]
}
-183
View File
@@ -1,183 +0,0 @@
gopeed.events.onResolve(async function (ctx) {
const encoder = new TextEncoder();
if (ctx.req.url.endsWith("/blob")) {
const blob = new Blob(["hello world"], {type: "text/plain"});
const url = URL.createObjectURL(blob);
ctx.res = {
name: "gblob-blob",
files: [
{
name: "hello.txt",
size: 11,
req: {
url,
}
}
]
};
return;
}
if (ctx.req.url.includes("/http-stream")) {
const source = new URL(ctx.req.url);
const target = source.searchParams.get("target");
const pair = source.searchParams.get("pair") === "1";
const resumable = ctx.req.url.includes("/http-stream-range");
const fileName = source.searchParams.get("name") || "http-stream.bin";
const headResponse = await fetch(target, {method: "HEAD"});
const headLength = parseInt(headResponse.headers.get("content-length") || "", 10);
const createOpen = () => async (offset) => {
const headers = {};
if (offset > 0) {
headers.Range = `bytes=${offset}-`;
}
const response = await fetch(target, {headers});
if (!response.body) {
throw new Error("empty response body");
}
return response.body;
};
ctx.res = {
name: "gblob-http-stream",
range: resumable,
files: pair ? [
{
name: "pair-video.bin",
size: Number.isFinite(headLength) ? headLength : 0,
req: {
url: URL.createObjectURL(createOpen()),
}
},
{
name: "pair-audio.bin",
size: Number.isFinite(headLength) ? headLength : 0,
req: {
url: URL.createObjectURL(createOpen()),
}
}
] : [
{
name: fileName,
size: Number.isFinite(headLength) ? headLength : 0,
req: {
url: URL.createObjectURL(createOpen()),
}
}
]
};
return;
}
if (ctx.req.url.includes("/http-blob")) {
const source = new URL(ctx.req.url);
const target = source.searchParams.get("target");
const fileName = source.searchParams.get("name") || "http-blob.bin";
const headResponse = await fetch(target, {method: "HEAD"});
const headLength = parseInt(headResponse.headers.get("content-length") || "", 10);
const response = await fetch(target);
const blob = await response.blob();
ctx.res = {
name: "gblob-http-blob",
files: [
{
name: fileName,
size: Number.isFinite(headLength) ? headLength : 0,
req: {
url: URL.createObjectURL(blob),
}
}
]
};
return;
}
if (ctx.req.url.endsWith("/stream-range")) {
const payload = encoder.encode("line 1\nline 2\nline 3\n");
const firstChunkLength = encoder.encode("line 1\n").byteLength;
let firstAttempt = true;
const open = async (offset) => new ReadableStream({
async start(controller) {
try {
if (offset === 0 && firstAttempt) {
firstAttempt = false;
controller.enqueue(payload.slice(0, firstChunkLength));
await new Promise((resolve) => setTimeout(resolve, 120));
controller.error(new Error("resume required"));
return;
}
controller.enqueue(payload.slice(offset));
controller.close();
} catch (err) {
controller.error(err);
}
}
});
const url = URL.createObjectURL(open);
ctx.res = {
name: "gblob-stream-range",
range: true,
files: [
{
name: "stream-range.txt",
size: payload.byteLength,
req: {
url,
}
}
]
};
return;
}
if (ctx.req.url.endsWith("/stream-unknown")) {
const url = URL.createObjectURL(new ReadableStream({
async start(controller) {
controller.enqueue(encoder.encode("line 1\n"));
await new Promise((resolve) => setTimeout(resolve, 350));
controller.enqueue(encoder.encode("line 2\n"));
controller.close();
},
}));
ctx.res = {
name: "gblob-stream-unknown",
files: [
{
name: "stream-unknown.txt",
req: {
url,
}
}
]
};
return;
}
const url = URL.createObjectURL(new ReadableStream({
start(controller) {
controller.enqueue(encoder.encode("line 1\n"));
controller.enqueue(encoder.encode("line 2\n"));
controller.close();
},
}));
ctx.res = {
name: "gblob-stream",
files: [
{
name: "stream.txt",
size: 14,
req: {
url,
}
}
]
};
});
-40
View File
@@ -1,40 +0,0 @@
gopeed.events.onResolve(async function (ctx) {
if (!ctx.req.url.endsWith("/recover")) {
return;
}
const url = URL.createObjectURL(new ReadableStream({
async start(controller) {
controller.enqueue(new TextEncoder().encode("stale\n"));
await new Promise((resolve) => setTimeout(resolve, 50));
controller.error(new Error("expired"));
},
}));
ctx.res = {
name: "gblob-recover",
files: [
{
name: "recover.txt",
req: {
url,
}
}
]
};
});
gopeed.events.onError(async function (ctx) {
const req = ctx.task.meta.req;
if (!req.rawUrl || !req.rawUrl.endsWith("/recover")) {
return;
}
req.labels = req.labels || {};
if (req.labels.recovered === "true") {
return;
}
req.labels.recovered = "true";
req.url = URL.createObjectURL(new Blob(["recovered\n"], { type: "text/plain" }));
ctx.task.continue();
});
+20 -4
View File
@@ -67,6 +67,10 @@ var (
}
)
func currentTestDownloadFile() string {
return filepath.Join(createOpts.Path, createOpts.Name)
}
func TestInfo(t *testing.T) {
matchKeys := []string{"version", "runtime", "os", "arch", "inDocker"}
doTest(func() {
@@ -109,7 +113,7 @@ func TestCreateTask(t *testing.T) {
wg.Wait()
want := test.FileMd5(test.BuildFile)
got := test.FileMd5(test.DownloadFile)
got := test.FileMd5(currentTestDownloadFile())
if want != got {
t.Errorf("CreateTask() got = %v, want %v", got, want)
}
@@ -133,7 +137,7 @@ func TestCreateDirectTask(t *testing.T) {
wg.Wait()
want := test.FileMd5(test.BuildFile)
got := test.FileMd5(test.DownloadFile)
got := test.FileMd5(currentTestDownloadFile())
if want != got {
t.Errorf("CreateDirectTask() got = %v, want %v", got, want)
}
@@ -226,7 +230,7 @@ func TestPauseAndContinueTask(t *testing.T) {
wg.Wait()
want := test.FileMd5(test.BuildFile)
got := test.FileMd5(test.DownloadFile)
got := test.FileMd5(currentTestDownloadFile())
if !reflect.DeepEqual(got, want) {
t.Errorf("PauseAndContinueTask() got = %v, want %v", got, want)
}
@@ -344,7 +348,7 @@ func TestDeleteTaskForce(t *testing.T) {
httpRequestCheckOk[any](http.MethodDelete, "/api/v1/tasks/"+taskId+"?force=true", nil)
code, _ := httpRequest[*download.Task](http.MethodGet, "/api/v1/tasks/"+taskId, nil)
checkCode(code, model.CodeTaskNotFound)
if _, err := os.Stat(test.DownloadFile); !errors.Is(err, os.ErrNotExist) {
if _, err := os.Stat(currentTestDownloadFile()); !errors.Is(err, os.ErrNotExist) {
t.Errorf("DeleteTaskForce() got = %v, want %v", err, os.ErrNotExist)
}
})
@@ -935,10 +939,22 @@ func doTest(handler func()) {
func doTest0(onStart func(cfg *model.StartConfig), handler func()) {
testFunc := func(storage model.Storage) {
downloadDir, err := os.MkdirTemp("", "gopeed-rest-test-download-")
if err != nil {
panic(err)
}
defer os.RemoveAll(downloadDir)
oldCreatePath := createOpts.Path
createOpts.Path = downloadDir
defer func() {
createOpts.Path = oldCreatePath
}()
var cfg = &model.StartConfig{}
cfg.Init()
cfg.Storage = storage
cfg.StorageDir = ".test_storage"
cfg.DownloadConfig = (&base.DownloaderStoreConfig{DownloadDir: downloadDir}).Init()
cfg.WebEnable = true
if onStart != nil {
onStart(cfg)