mirror of
https://github.com/GopeedLab/gopeed.git
synced 2026-11-06 08:30:29 +00:00
feat: add gblob scheme support for extension (#1358)
This commit is contained in:
18 files changed
+2664
-147
No files matched your search
+3
-2
@@ -14,8 +14,9 @@ import (
|
||||
|
||||
// Request download request
|
||||
type Request struct {
|
||||
URL string `json:"url"`
|
||||
Extra any `json:"extra"`
|
||||
RawURL string `json:"rawUrl"`
|
||||
URL string `json:"url"`
|
||||
Extra any `json:"extra"`
|
||||
// Labels is used to mark the download task
|
||||
Labels map[string]string `json:"labels"`
|
||||
// Proxy is special proxy config for request
|
||||
|
||||
+146
-76
@@ -18,6 +18,7 @@ import (
|
||||
"github.com/GopeedLab/gopeed/internal/controller"
|
||||
"github.com/GopeedLab/gopeed/internal/fetcher"
|
||||
"github.com/GopeedLab/gopeed/internal/logger"
|
||||
protogblob "github.com/GopeedLab/gopeed/internal/protocol/gblob"
|
||||
"github.com/GopeedLab/gopeed/pkg/base"
|
||||
"github.com/GopeedLab/gopeed/pkg/protocol/http"
|
||||
"github.com/GopeedLab/gopeed/pkg/util"
|
||||
@@ -111,6 +112,7 @@ type Downloader struct {
|
||||
claimedExtractions sync.Map
|
||||
|
||||
extensions []*Extension
|
||||
gblob *protogblob.Registry
|
||||
}
|
||||
|
||||
func NewDownloader(cfg *DownloaderConfig) *Downloader {
|
||||
@@ -142,6 +144,8 @@ func NewDownloader(cfg *DownloaderConfig) *Downloader {
|
||||
}
|
||||
|
||||
func (d *Downloader) Setup() error {
|
||||
d.gblob = protogblob.NewRegistry(filepath.Join(d.cfg.StorageDir, protogblob.Scheme))
|
||||
|
||||
// setup storage
|
||||
if err := d.storage.Setup([]string{bucketTask, bucketSave, bucketProtocolState, bucketConfig, bucketExtension, bucketExtensionStorage}); err != nil {
|
||||
return err
|
||||
@@ -167,6 +171,9 @@ func (d *Downloader) Setup() error {
|
||||
if _, ok := d.cfg.DownloaderStoreConfig.ProtocolConfig[protocol]; !ok {
|
||||
d.cfg.DownloaderStoreConfig.ProtocolConfig[protocol] = fm.DefaultConfig()
|
||||
}
|
||||
if gfm, ok := fm.(*protogblob.FetcherManager); ok {
|
||||
gfm.SetRegistry(d.gblob)
|
||||
}
|
||||
if sfm, ok := fm.(fetcher.StatefulFetcherManager); ok {
|
||||
sfm.SetStateStore(&protocolStateStore{
|
||||
storage: d.storage,
|
||||
@@ -192,6 +199,10 @@ func (d *Downloader) Setup() error {
|
||||
}
|
||||
d.assignFetcherManager(task)
|
||||
initTask(task)
|
||||
if task.Protocol == protogblob.Scheme && task.Status != base.DownloadStatusDone && task.Status != base.DownloadStatusError {
|
||||
task.Status = base.DownloadStatusError
|
||||
continue
|
||||
}
|
||||
if task.Status != base.DownloadStatusDone && task.Status != base.DownloadStatusError {
|
||||
task.Status = base.DownloadStatusPause
|
||||
}
|
||||
@@ -384,11 +395,36 @@ func (d *Downloader) saveTask(task *Task) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func ensureRequestRawURL(req *base.Request) {
|
||||
if req == nil {
|
||||
return
|
||||
}
|
||||
if req.RawURL == "" {
|
||||
req.RawURL = req.URL
|
||||
}
|
||||
}
|
||||
|
||||
func ensureResourceRequestRawURLs(parentReq *base.Request, res *base.Resource) {
|
||||
if res == nil {
|
||||
return
|
||||
}
|
||||
for _, file := range res.Files {
|
||||
if file == nil || file.Req == nil {
|
||||
continue
|
||||
}
|
||||
ensureRequestRawURL(file.Req)
|
||||
if parentReq != nil && parentReq.RawURL != "" && file.Req.RawURL == file.Req.URL {
|
||||
file.Req.RawURL = parentReq.RawURL
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (d *Downloader) Resolve(req *base.Request, opts *base.Options) (rr *ResolveResult, err error) {
|
||||
rrId, err := gonanoid.New()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
ensureRequestRawURL(req)
|
||||
|
||||
res, err := d.triggerOnResolve(req)
|
||||
if err != nil {
|
||||
@@ -451,6 +487,7 @@ func (d *Downloader) remainRunningCount() int {
|
||||
}
|
||||
|
||||
func (d *Downloader) CreateDirect(req *base.Request, opts *base.Options) (taskId string, err error) {
|
||||
ensureRequestRawURL(req)
|
||||
var fetcher fetcher.Fetcher
|
||||
fetcher, err = d.buildFetcher(req.URL)
|
||||
if err != nil {
|
||||
@@ -765,6 +802,7 @@ func (d *Downloader) Stats(id string) (sr any, err error) {
|
||||
}
|
||||
|
||||
func (d *Downloader) doDelete(task *Task, force bool) (err error) {
|
||||
defer d.unpinGBlobTask(task)
|
||||
err = func() error {
|
||||
if err := d.storage.Delete(bucketTask, task.ID); err != nil {
|
||||
return err
|
||||
@@ -809,6 +847,9 @@ func (d *Downloader) Close() error {
|
||||
for _, fm := range d.cfg.FetchManagers {
|
||||
closeArr = append(closeArr, fm.Close)
|
||||
}
|
||||
if d.gblob != nil {
|
||||
closeArr = append(closeArr, d.gblob.Close)
|
||||
}
|
||||
closeArr = append(closeArr, d.storage.Close)
|
||||
// Make sure all resources are released, if had error, return the last error
|
||||
var lastErr error
|
||||
@@ -1001,98 +1042,106 @@ func (d *Downloader) watch(task *Task) {
|
||||
return
|
||||
}
|
||||
|
||||
err := task.fetcher.Wait()
|
||||
if err != nil {
|
||||
d.doOnError(task, err)
|
||||
return
|
||||
}
|
||||
|
||||
// When delete a not resolved task, need check if the task resource is nil
|
||||
if task.Meta.Res == nil || d.GetTask(task.ID) == nil {
|
||||
return
|
||||
}
|
||||
|
||||
task.Progress.Used = task.timer.Used()
|
||||
if task.Meta.Res.Size == 0 {
|
||||
task.Meta.Res.Size = task.fetcher.Progress().TotalDownloaded()
|
||||
}
|
||||
used := task.Progress.Used / int64(time.Second)
|
||||
if used == 0 {
|
||||
used = 1
|
||||
}
|
||||
totalSize := task.Meta.Res.Size
|
||||
task.Progress.Speed = totalSize / used
|
||||
task.Progress.Downloaded = totalSize
|
||||
task.updateStatus(base.DownloadStatusDone)
|
||||
d.storage.Put(bucketTask, task.ID, task.clone())
|
||||
d.emit(EventKeyDone, task)
|
||||
d.emit(EventKeyFinally, task, err)
|
||||
d.notifyRunning()
|
||||
d.triggerOnDone(task)
|
||||
d.triggerWebhooks(WebhookEventDownloadDone, task, nil)
|
||||
d.triggerScripts(ScriptEventDownloadDone, task, nil)
|
||||
|
||||
if e, ok := task.Meta.Opts.Extra.(*http.OptsExtra); ok {
|
||||
downloadFilePath := task.Meta.SingleFilepath()
|
||||
|
||||
cfg, _ := d.GetConfig()
|
||||
|
||||
// Determine if auto-torrent is enabled (use global config if not explicitly set)
|
||||
autoTorrentEnabled := false
|
||||
if e.AutoTorrent != nil {
|
||||
autoTorrentEnabled = *e.AutoTorrent
|
||||
} else if cfg != nil && cfg.AutoTorrent != nil {
|
||||
autoTorrentEnabled = cfg.AutoTorrent.Enable
|
||||
for {
|
||||
err := task.fetcher.Wait()
|
||||
if err != nil {
|
||||
d.doOnError(task, err)
|
||||
if task.Status == base.DownloadStatusRunning {
|
||||
continue
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
if autoTorrentEnabled && strings.HasSuffix(downloadFilePath, ".torrent") {
|
||||
// Determine if should delete torrent file after creating BT task
|
||||
shouldDelete := false
|
||||
if e.DeleteTorrentAfterDownload != nil {
|
||||
shouldDelete = *e.DeleteTorrentAfterDownload
|
||||
// When delete a not resolved task, need check if the task resource is nil
|
||||
if task.Meta.Res == nil || d.GetTask(task.ID) == nil {
|
||||
return
|
||||
}
|
||||
|
||||
task.Progress.Used = task.timer.Used()
|
||||
if task.Meta.Res.Size == 0 {
|
||||
task.Meta.Res.Size = task.fetcher.Progress().TotalDownloaded()
|
||||
}
|
||||
used := task.Progress.Used / int64(time.Second)
|
||||
if used == 0 {
|
||||
used = 1
|
||||
}
|
||||
totalSize := task.Meta.Res.Size
|
||||
task.Progress.Speed = totalSize / used
|
||||
task.Progress.Downloaded = totalSize
|
||||
task.updateStatus(base.DownloadStatusDone)
|
||||
d.storage.Put(bucketTask, task.ID, task.clone())
|
||||
d.emit(EventKeyDone, task)
|
||||
d.emit(EventKeyFinally, task, err)
|
||||
d.notifyRunning()
|
||||
d.unpinGBlobTask(task)
|
||||
d.triggerOnDone(task)
|
||||
d.triggerWebhooks(WebhookEventDownloadDone, task, nil)
|
||||
d.triggerScripts(ScriptEventDownloadDone, task, nil)
|
||||
|
||||
if e, ok := task.Meta.Opts.Extra.(*http.OptsExtra); ok {
|
||||
downloadFilePath := task.Meta.SingleFilepath()
|
||||
|
||||
cfg, _ := d.GetConfig()
|
||||
|
||||
// Determine if auto-torrent is enabled (use global config if not explicitly set)
|
||||
autoTorrentEnabled := false
|
||||
if e.AutoTorrent != nil {
|
||||
autoTorrentEnabled = *e.AutoTorrent
|
||||
} else if cfg != nil && cfg.AutoTorrent != nil {
|
||||
shouldDelete = cfg.AutoTorrent.DeleteAfterDownload
|
||||
autoTorrentEnabled = cfg.AutoTorrent.Enable
|
||||
}
|
||||
|
||||
go func() {
|
||||
_, err2 := d.CreateDirect(
|
||||
&base.Request{
|
||||
URL: downloadFilePath,
|
||||
},
|
||||
&base.Options{
|
||||
Path: task.Meta.Opts.Path,
|
||||
SelectFiles: make([]int, 0),
|
||||
})
|
||||
if err2 != nil {
|
||||
d.Logger.Error().Err(err2).Msgf("auto create torrent task failed, task id: %s", task.ID)
|
||||
return
|
||||
if autoTorrentEnabled && strings.HasSuffix(downloadFilePath, ".torrent") {
|
||||
// Determine if should delete torrent file after creating BT task
|
||||
shouldDelete := false
|
||||
if e.DeleteTorrentAfterDownload != nil {
|
||||
shouldDelete = *e.DeleteTorrentAfterDownload
|
||||
} else if cfg != nil && cfg.AutoTorrent != nil {
|
||||
shouldDelete = cfg.AutoTorrent.DeleteAfterDownload
|
||||
}
|
||||
|
||||
if shouldDelete {
|
||||
d.Delete(&TaskFilter{IDs: []string{task.ID}}, true)
|
||||
}
|
||||
}()
|
||||
}
|
||||
go func() {
|
||||
_, err2 := d.CreateDirect(
|
||||
&base.Request{
|
||||
URL: downloadFilePath,
|
||||
},
|
||||
&base.Options{
|
||||
Path: task.Meta.Opts.Path,
|
||||
SelectFiles: make([]int, 0),
|
||||
})
|
||||
if err2 != nil {
|
||||
d.Logger.Error().Err(err2).Msgf("auto create torrent task failed, task id: %s", task.ID)
|
||||
return
|
||||
}
|
||||
|
||||
// Determine if auto-extract is enabled (use global config if not explicitly set)
|
||||
autoExtractEnabled := false
|
||||
if e.AutoExtract != nil {
|
||||
autoExtractEnabled = *e.AutoExtract
|
||||
} else if cfg != nil && cfg.Archive != nil {
|
||||
autoExtractEnabled = cfg.Archive.AutoExtract
|
||||
}
|
||||
if shouldDelete {
|
||||
d.Delete(&TaskFilter{IDs: []string{task.ID}}, true)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// Auto-extract archive files using the extraction queue
|
||||
// This ensures only one extraction runs at a time to prevent resource exhaustion
|
||||
if autoExtractEnabled && isArchiveFile(downloadFilePath) {
|
||||
d.enqueueExtraction(task, downloadFilePath, e)
|
||||
// Determine if auto-extract is enabled (use global config if not explicitly set)
|
||||
autoExtractEnabled := false
|
||||
if e.AutoExtract != nil {
|
||||
autoExtractEnabled = *e.AutoExtract
|
||||
} else if cfg != nil && cfg.Archive != nil {
|
||||
autoExtractEnabled = cfg.Archive.AutoExtract
|
||||
}
|
||||
|
||||
// Auto-extract archive files using the extraction queue
|
||||
// This ensures only one extraction runs at a time to prevent resource exhaustion
|
||||
if autoExtractEnabled && isArchiveFile(downloadFilePath) {
|
||||
d.enqueueExtraction(task, downloadFilePath, e)
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
func (d *Downloader) doOnError(task *Task, err error) {
|
||||
d.Logger.Warn().Err(err).Msgf("task download failed, task id: %s", task.ID)
|
||||
task.updateStatus(base.DownloadStatusError)
|
||||
d.unpinGBlobTask(task)
|
||||
d.triggerOnError(task, err)
|
||||
if task.Status == base.DownloadStatusError {
|
||||
d.emit(EventKeyError, task, err)
|
||||
@@ -1141,11 +1190,22 @@ func (d *Downloader) doCreate(f fetcher.Fetcher, opts *base.Options) (taskId str
|
||||
if f.Meta().Opts == nil {
|
||||
f.Meta().Opts = opts
|
||||
}
|
||||
ensureRequestRawURL(f.Meta().Req)
|
||||
|
||||
fm, err := d.parseFm(f.Meta().Req.URL)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if fm.Name() == protogblob.Scheme && d.gblob != nil {
|
||||
if err = d.gblob.Pin(f.Meta().Req.URL); err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer func() {
|
||||
if err != nil {
|
||||
d.gblob.Unpin(f.Meta().Req.URL)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
task := NewTask()
|
||||
task.fetcherManager = fm
|
||||
@@ -1772,6 +1832,16 @@ func initTask(task *Task) {
|
||||
task.uploadSpeedArr = make([]int64, 0)
|
||||
}
|
||||
|
||||
func (d *Downloader) unpinGBlobTask(task *Task) {
|
||||
if d.gblob == nil || task == nil || task.Meta == nil || task.Meta.Req == nil || task.Meta.Req.URL == "" {
|
||||
return
|
||||
}
|
||||
if task.Protocol != protogblob.Scheme {
|
||||
return
|
||||
}
|
||||
d.gblob.Unpin(task.Meta.Req.URL)
|
||||
}
|
||||
|
||||
var defaultDownloader = NewDownloader(nil)
|
||||
|
||||
type boot struct {
|
||||
|
||||
@@ -3,10 +3,12 @@ package engine
|
||||
import (
|
||||
_ "embed"
|
||||
"errors"
|
||||
|
||||
"github.com/GopeedLab/gopeed/pkg/base"
|
||||
gojaerror "github.com/GopeedLab/gopeed/pkg/download/engine/inject/error"
|
||||
"github.com/GopeedLab/gopeed/pkg/download/engine/inject/file"
|
||||
"github.com/GopeedLab/gopeed/pkg/download/engine/inject/formdata"
|
||||
"github.com/GopeedLab/gopeed/pkg/download/engine/inject/stream"
|
||||
"github.com/GopeedLab/gopeed/pkg/download/engine/inject/vm"
|
||||
"github.com/GopeedLab/gopeed/pkg/download/engine/inject/xhr"
|
||||
"github.com/dop251/goja"
|
||||
@@ -27,90 +29,62 @@ type Engine struct {
|
||||
// RunString executes the script and returns the go type value
|
||||
// if script result is promise, it will be resolved
|
||||
func (e *Engine) RunString(script string) (value any, err error) {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
err = r.(error)
|
||||
}
|
||||
}()
|
||||
|
||||
var result goja.Value
|
||||
e.loop.Run(func(runtime *goja.Runtime) {
|
||||
result, err = runtime.RunString(script)
|
||||
if err == nil {
|
||||
go e.await(result)
|
||||
}
|
||||
return e.runOnLoop(func(runtime *goja.Runtime) (goja.Value, error) {
|
||||
return runtime.RunString(script)
|
||||
})
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
return resolveResult(result)
|
||||
}
|
||||
|
||||
// CallFunction calls the function and returns the go type value
|
||||
// if function result is promise, it will be resolved
|
||||
func (e *Engine) CallFunction(fn goja.Callable, args ...any) (value any, err error) {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
err = r.(error)
|
||||
}
|
||||
}()
|
||||
|
||||
var result goja.Value
|
||||
e.loop.Run(func(runtime *goja.Runtime) {
|
||||
return e.runOnLoop(func(runtime *goja.Runtime) (goja.Value, error) {
|
||||
if args == nil {
|
||||
result, err = fn(nil)
|
||||
} else {
|
||||
var jsArgs []goja.Value
|
||||
for _, arg := range args {
|
||||
jsArgs = append(jsArgs, runtime.ToValue(arg))
|
||||
}
|
||||
result, err = fn(nil, jsArgs...)
|
||||
return fn(nil)
|
||||
}
|
||||
if err == nil {
|
||||
go e.await(result)
|
||||
var jsArgs []goja.Value
|
||||
for _, arg := range args {
|
||||
jsArgs = append(jsArgs, runtime.ToValue(arg))
|
||||
}
|
||||
return fn(nil, jsArgs...)
|
||||
})
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
return resolveResult(result)
|
||||
}
|
||||
|
||||
// loop.Run will hang if the script result has a non-stop code, such as setInterval.
|
||||
// This method will stop the event loop when the promise result is resolved.
|
||||
func (e *Engine) await(value any) {
|
||||
if value == nil {
|
||||
return
|
||||
func (e *Engine) runOnLoop(fn func(runtime *goja.Runtime) (goja.Value, error)) (any, error) {
|
||||
type result struct {
|
||||
value goja.Value
|
||||
err error
|
||||
}
|
||||
|
||||
if v, ok := value.(goja.Value); ok {
|
||||
// if result is promise, wait for it to be resolved
|
||||
if p, ok := v.Export().(*goja.Promise); ok {
|
||||
if p.State() != goja.PromiseStatePending {
|
||||
return
|
||||
}
|
||||
|
||||
// check promise state every 100 milliseconds, until it is resolved
|
||||
for {
|
||||
time.Sleep(time.Millisecond * 100)
|
||||
if p.State() == goja.PromiseStatePending {
|
||||
continue
|
||||
ch := make(chan result, 1)
|
||||
ok := e.loop.RunOnLoop(func(runtime *goja.Runtime) {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
if err, ok := r.(error); ok {
|
||||
ch <- result{err: err}
|
||||
return
|
||||
}
|
||||
break
|
||||
ch <- result{err: errors.New("panic")}
|
||||
}
|
||||
|
||||
// stop the event loop
|
||||
e.loop.StopNoWait()
|
||||
}
|
||||
}()
|
||||
value, err := fn(runtime)
|
||||
ch <- result{value: value, err: err}
|
||||
})
|
||||
if !ok {
|
||||
return nil, errors.New("engine loop terminated")
|
||||
}
|
||||
res := <-ch
|
||||
if res.err != nil {
|
||||
return nil, res.err
|
||||
}
|
||||
return resolveResult(res.value)
|
||||
}
|
||||
|
||||
func (e *Engine) Close() {
|
||||
e.loop.StopNoWait()
|
||||
e.loop.Terminate()
|
||||
}
|
||||
|
||||
type Config struct {
|
||||
ProxyConfig *base.DownloaderProxyConfig
|
||||
ProxyConfig *base.DownloaderProxyConfig
|
||||
StreamConfig *stream.Config
|
||||
}
|
||||
|
||||
func NewEngine(cfg *Config) *Engine {
|
||||
@@ -121,7 +95,10 @@ func NewEngine(cfg *Config) *Engine {
|
||||
engine := &Engine{
|
||||
loop: loop,
|
||||
}
|
||||
loop.Run(func(runtime *goja.Runtime) {
|
||||
loop.Start()
|
||||
done := make(chan struct{})
|
||||
loop.RunOnLoop(func(runtime *goja.Runtime) {
|
||||
defer close(done)
|
||||
engine.Runtime = runtime
|
||||
runtime.SetFieldNameMapper(goja.TagFieldNameMapper("json", true))
|
||||
vm.Enable(runtime)
|
||||
@@ -153,8 +130,12 @@ func NewEngine(cfg *Config) *Engine {
|
||||
if _, err := runtime.RunString("global.location = new URL('http://localhost');"); err != nil {
|
||||
return
|
||||
}
|
||||
if err := stream.Enable(runtime, cfg.StreamConfig); err != nil {
|
||||
return
|
||||
}
|
||||
return
|
||||
})
|
||||
<-done
|
||||
return engine
|
||||
}
|
||||
|
||||
@@ -169,6 +150,9 @@ func resolveResult(value goja.Value) (any, error) {
|
||||
switch export.(type) {
|
||||
case *goja.Promise:
|
||||
p := export.(*goja.Promise)
|
||||
for p.State() == goja.PromiseStatePending {
|
||||
time.Sleep(time.Millisecond * 10)
|
||||
}
|
||||
switch p.State() {
|
||||
case goja.PromiseStatePending:
|
||||
return nil, nil
|
||||
|
||||
@@ -0,0 +1,92 @@
|
||||
package stream
|
||||
|
||||
import (
|
||||
_ "embed"
|
||||
"fmt"
|
||||
|
||||
"github.com/dop251/goja"
|
||||
)
|
||||
|
||||
//go:embed stream.js
|
||||
var script string
|
||||
|
||||
type Config struct {
|
||||
CreateBlobObjectURL func(data []byte, contentType string) (string, error)
|
||||
CreateWritableStreamObjectURL func() (string, error)
|
||||
WriteWritableStreamObjectURL func(url string, data any) error
|
||||
CloseWritableStreamObjectURL func(url string) error
|
||||
AbortWritableStreamObjectURL func(url string, reason string) error
|
||||
RevokeObjectURL func(url string) error
|
||||
}
|
||||
|
||||
func Enable(runtime *goja.Runtime, cfg *Config) error {
|
||||
if cfg == nil {
|
||||
cfg = &Config{}
|
||||
}
|
||||
if err := runtime.Set("__gopeed_create_blob_object_url", func(data []byte, contentType string) string {
|
||||
if cfg.CreateBlobObjectURL == nil {
|
||||
panic(fmt.Errorf("gblob blob object url handler not configured"))
|
||||
}
|
||||
url, err := cfg.CreateBlobObjectURL(data, contentType)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return url
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := runtime.Set("__gopeed_create_writable_stream_object_url", func() string {
|
||||
if cfg.CreateWritableStreamObjectURL == nil {
|
||||
panic(fmt.Errorf("gblob writable stream object url handler not configured"))
|
||||
}
|
||||
url, err := cfg.CreateWritableStreamObjectURL()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return url
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := runtime.Set("__gopeed_write_writable_stream_object_url", func(url string, data any) {
|
||||
if cfg.WriteWritableStreamObjectURL == nil {
|
||||
panic(fmt.Errorf("gblob writable stream write handler not configured"))
|
||||
}
|
||||
if err := cfg.WriteWritableStreamObjectURL(url, data); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := runtime.Set("__gopeed_close_writable_stream_object_url", func(url string) {
|
||||
if cfg.CloseWritableStreamObjectURL == nil {
|
||||
panic(fmt.Errorf("gblob writable stream close handler not configured"))
|
||||
}
|
||||
if err := cfg.CloseWritableStreamObjectURL(url); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := runtime.Set("__gopeed_abort_writable_stream_object_url", func(url string, reason string) {
|
||||
if cfg.AbortWritableStreamObjectURL == nil {
|
||||
panic(fmt.Errorf("gblob writable stream abort handler not configured"))
|
||||
}
|
||||
if err := cfg.AbortWritableStreamObjectURL(url, reason); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := runtime.Set("__gopeed_revoke_object_url", func(url string) {
|
||||
if cfg.RevokeObjectURL == nil {
|
||||
panic(fmt.Errorf("gblob revoke handler not configured"))
|
||||
}
|
||||
if err := cfg.RevokeObjectURL(url); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err := runtime.RunString(script)
|
||||
return err
|
||||
}
|
||||
@@ -0,0 +1,341 @@
|
||||
(function () {
|
||||
const createBlobObjectURL = globalThis.__gopeed_create_blob_object_url;
|
||||
const createWritableObjectURL = globalThis.__gopeed_create_writable_stream_object_url;
|
||||
const writeWritableObjectURL = globalThis.__gopeed_write_writable_stream_object_url;
|
||||
const closeWritableObjectURL = globalThis.__gopeed_close_writable_stream_object_url;
|
||||
const abortWritableObjectURL = globalThis.__gopeed_abort_writable_stream_object_url;
|
||||
const revokeObjectURL = globalThis.__gopeed_revoke_object_url;
|
||||
|
||||
if (typeof globalThis.ReadableStream === "undefined") {
|
||||
class ReadableStreamDefaultController {
|
||||
constructor(stream) {
|
||||
this._stream = stream;
|
||||
}
|
||||
|
||||
enqueue(chunk) {
|
||||
this._stream._enqueue(chunk);
|
||||
}
|
||||
|
||||
close() {
|
||||
this._stream._close();
|
||||
}
|
||||
|
||||
error(err) {
|
||||
this._stream._error(err);
|
||||
}
|
||||
}
|
||||
|
||||
class ReadableStreamDefaultReader {
|
||||
constructor(stream) {
|
||||
this._stream = stream;
|
||||
}
|
||||
|
||||
read() {
|
||||
return this._stream._read();
|
||||
}
|
||||
|
||||
releaseLock() {
|
||||
if (this._stream) {
|
||||
this._stream._reader = null;
|
||||
this._stream.locked = false;
|
||||
this._stream = null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
class ReadableStream {
|
||||
constructor(source = {}) {
|
||||
this._queue = [];
|
||||
this._waiters = [];
|
||||
this._closed = false;
|
||||
this._errored = null;
|
||||
this._reader = null;
|
||||
this.locked = false;
|
||||
const controller = new ReadableStreamDefaultController(this);
|
||||
if (typeof source.start === "function") {
|
||||
source.start(controller);
|
||||
}
|
||||
}
|
||||
|
||||
_enqueue(chunk) {
|
||||
if (this._closed || this._errored) {
|
||||
return;
|
||||
}
|
||||
if (this._waiters.length > 0) {
|
||||
const waiter = this._waiters.shift();
|
||||
waiter.resolve({ done: false, value: chunk });
|
||||
return;
|
||||
}
|
||||
this._queue.push(chunk);
|
||||
}
|
||||
|
||||
_close() {
|
||||
this._closed = true;
|
||||
while (this._waiters.length > 0) {
|
||||
this._waiters.shift().resolve({ done: true, value: undefined });
|
||||
}
|
||||
}
|
||||
|
||||
_error(err) {
|
||||
this._errored = err || new Error("ReadableStream error");
|
||||
while (this._waiters.length > 0) {
|
||||
this._waiters.shift().reject(this._errored);
|
||||
}
|
||||
}
|
||||
|
||||
_read() {
|
||||
if (this._errored) {
|
||||
return Promise.reject(this._errored);
|
||||
}
|
||||
if (this._queue.length > 0) {
|
||||
return Promise.resolve({ done: false, value: this._queue.shift() });
|
||||
}
|
||||
if (this._closed) {
|
||||
return Promise.resolve({ done: true, value: undefined });
|
||||
}
|
||||
return new Promise((resolve, reject) => {
|
||||
this._waiters.push({ resolve, reject });
|
||||
});
|
||||
}
|
||||
|
||||
getReader() {
|
||||
if (this.locked) {
|
||||
throw new TypeError("ReadableStream is locked");
|
||||
}
|
||||
this.locked = true;
|
||||
this._reader = new ReadableStreamDefaultReader(this);
|
||||
return this._reader;
|
||||
}
|
||||
|
||||
async pipeTo(dest) {
|
||||
const reader = this.getReader();
|
||||
const writer = dest.getWriter();
|
||||
try {
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) {
|
||||
await writer.close();
|
||||
break;
|
||||
}
|
||||
await writer.write(value);
|
||||
}
|
||||
} catch (err) {
|
||||
await writer.abort(err);
|
||||
throw err;
|
||||
} finally {
|
||||
reader.releaseLock();
|
||||
writer.releaseLock();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
globalThis.ReadableStream = ReadableStream;
|
||||
}
|
||||
|
||||
if (typeof globalThis.WritableStream === "undefined") {
|
||||
class WritableStreamDefaultWriter {
|
||||
constructor(stream) {
|
||||
if (stream.locked) {
|
||||
throw new TypeError("WritableStream is locked");
|
||||
}
|
||||
this._stream = stream;
|
||||
this._stream.locked = true;
|
||||
}
|
||||
|
||||
write(chunk) {
|
||||
return this._stream._write(chunk);
|
||||
}
|
||||
|
||||
close() {
|
||||
return this._stream._close();
|
||||
}
|
||||
|
||||
abort(reason) {
|
||||
return this._stream._abort(reason);
|
||||
}
|
||||
|
||||
releaseLock() {
|
||||
if (this._stream) {
|
||||
this._stream.locked = false;
|
||||
this._stream = null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
class WritableStream {
|
||||
constructor(sink = {}) {
|
||||
this._sink = sink;
|
||||
this._observers = [];
|
||||
this._state = "writable";
|
||||
this.locked = false;
|
||||
}
|
||||
|
||||
_targets() {
|
||||
return [this._sink, ...this._observers];
|
||||
}
|
||||
|
||||
_write(chunk) {
|
||||
if (this._state !== "writable") {
|
||||
return Promise.reject(new TypeError("WritableStream is not writable"));
|
||||
}
|
||||
let chain = Promise.resolve();
|
||||
for (const target of this._targets()) {
|
||||
if (target && typeof target.write === "function") {
|
||||
chain = chain.then(() => target.write(chunk));
|
||||
}
|
||||
}
|
||||
return chain;
|
||||
}
|
||||
|
||||
_close() {
|
||||
if (this._state !== "writable") {
|
||||
return Promise.reject(new TypeError("WritableStream is not writable"));
|
||||
}
|
||||
this._state = "closed";
|
||||
let chain = Promise.resolve();
|
||||
for (const target of this._targets()) {
|
||||
if (target && typeof target.close === "function") {
|
||||
chain = chain.then(() => target.close());
|
||||
}
|
||||
}
|
||||
return chain;
|
||||
}
|
||||
|
||||
_abort(reason) {
|
||||
if (this._state === "errored") {
|
||||
return Promise.resolve();
|
||||
}
|
||||
this._state = "errored";
|
||||
let chain = Promise.resolve();
|
||||
for (const target of this._targets()) {
|
||||
if (target && typeof target.abort === "function") {
|
||||
chain = chain.then(() => target.abort(reason));
|
||||
}
|
||||
}
|
||||
return chain;
|
||||
}
|
||||
|
||||
_addObserver(observer) {
|
||||
this._observers.push(observer);
|
||||
}
|
||||
|
||||
getWriter() {
|
||||
return new WritableStreamDefaultWriter(this);
|
||||
}
|
||||
}
|
||||
|
||||
globalThis.WritableStream = WritableStream;
|
||||
}
|
||||
|
||||
if (typeof globalThis.TransformStream === "undefined") {
|
||||
class TransformStream {
|
||||
constructor(transformer = {}) {
|
||||
let readableController;
|
||||
this.readable = new ReadableStream({
|
||||
start(controller) {
|
||||
readableController = controller;
|
||||
}
|
||||
});
|
||||
this.writable = new WritableStream({
|
||||
async write(chunk) {
|
||||
if (typeof transformer.transform === "function") {
|
||||
await transformer.transform(chunk, readableController);
|
||||
return;
|
||||
}
|
||||
readableController.enqueue(chunk);
|
||||
},
|
||||
async close() {
|
||||
if (typeof transformer.flush === "function") {
|
||||
await transformer.flush(readableController);
|
||||
}
|
||||
readableController.close();
|
||||
},
|
||||
async abort(reason) {
|
||||
if (typeof transformer.abort === "function") {
|
||||
await transformer.abort(reason);
|
||||
}
|
||||
readableController.error(reason || new Error("TransformStream aborted"));
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
globalThis.TransformStream = TransformStream;
|
||||
}
|
||||
|
||||
if (typeof globalThis.ByteLengthQueuingStrategy === "undefined") {
|
||||
globalThis.ByteLengthQueuingStrategy = class ByteLengthQueuingStrategy {
|
||||
constructor({ highWaterMark }) {
|
||||
this.highWaterMark = highWaterMark;
|
||||
}
|
||||
|
||||
size(chunk) {
|
||||
if (typeof chunk === "string") {
|
||||
return chunk.length;
|
||||
}
|
||||
if (chunk && typeof chunk.byteLength === "number") {
|
||||
return chunk.byteLength;
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
if (typeof globalThis.CountQueuingStrategy === "undefined") {
|
||||
globalThis.CountQueuingStrategy = class CountQueuingStrategy {
|
||||
constructor({ highWaterMark }) {
|
||||
this.highWaterMark = highWaterMark;
|
||||
}
|
||||
|
||||
size() {
|
||||
return 1;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
const originalCreateObjectURL = typeof URL.createObjectURL === "function"
|
||||
? URL.createObjectURL.bind(URL)
|
||||
: null;
|
||||
const originalRevokeObjectURL = typeof URL.revokeObjectURL === "function"
|
||||
? URL.revokeObjectURL.bind(URL)
|
||||
: null;
|
||||
|
||||
URL.createObjectURL = function (value) {
|
||||
if (value instanceof Blob) {
|
||||
return createBlobObjectURL(value._buffer, value.type || "");
|
||||
}
|
||||
if (value instanceof WritableStream) {
|
||||
if (value.__gopeedObjectURL) {
|
||||
return value.__gopeedObjectURL;
|
||||
}
|
||||
const url = createWritableObjectURL();
|
||||
value._addObserver({
|
||||
write(chunk) {
|
||||
return Promise.resolve(writeWritableObjectURL(url, chunk));
|
||||
},
|
||||
close() {
|
||||
return Promise.resolve(closeWritableObjectURL(url));
|
||||
},
|
||||
abort(reason) {
|
||||
return Promise.resolve(abortWritableObjectURL(url, reason == null ? "" : String(reason)));
|
||||
}
|
||||
});
|
||||
value.__gopeedObjectURL = url;
|
||||
return url;
|
||||
}
|
||||
if (originalCreateObjectURL) {
|
||||
return originalCreateObjectURL(value);
|
||||
}
|
||||
throw new TypeError("Unsupported object type");
|
||||
};
|
||||
|
||||
URL.revokeObjectURL = function (url) {
|
||||
if (typeof url === "string" && url.indexOf("gblob:") === 0) {
|
||||
revokeObjectURL(url);
|
||||
return;
|
||||
}
|
||||
if (originalRevokeObjectURL) {
|
||||
originalRevokeObjectURL(url);
|
||||
}
|
||||
};
|
||||
})();
|
||||
@@ -13,7 +13,6 @@ import (
|
||||
|
||||
"github.com/GopeedLab/gopeed/internal/logger"
|
||||
"github.com/GopeedLab/gopeed/pkg/base"
|
||||
"github.com/GopeedLab/gopeed/pkg/download/engine"
|
||||
gojaerror "github.com/GopeedLab/gopeed/pkg/download/engine/inject/error"
|
||||
gojautil "github.com/GopeedLab/gopeed/pkg/download/engine/util"
|
||||
"github.com/GopeedLab/gopeed/pkg/util"
|
||||
@@ -268,6 +267,7 @@ func (d *Downloader) triggerOnResolve(req *base.Request) (res *base.Resource, er
|
||||
for _, file := range ctx.Res.Files {
|
||||
file.Name = util.SafeFilename(file.Name)
|
||||
}
|
||||
ensureResourceRequestRawURLs(req, ctx.Res)
|
||||
ctx.Res.CalcSize(nil)
|
||||
}
|
||||
res = ctx.Res
|
||||
@@ -361,10 +361,8 @@ func doTrigger[T any](d *Downloader, event ActivationEvent, req *base.Request, c
|
||||
if req.Labels == nil {
|
||||
req.Labels = make(map[string]string)
|
||||
}
|
||||
engine := engine.NewEngine(&engine.Config{
|
||||
ProxyConfig: d.cfg.Proxy,
|
||||
})
|
||||
defer engine.Close()
|
||||
engine, session := d.newExtensionEngine()
|
||||
defer session.CloseIfIdle()
|
||||
err = engine.Runtime.Set("gopeed", gopeed)
|
||||
if err != nil {
|
||||
gopeed.Logger.logger.Error().Err(err).Msgf("[%s] engine inject failed", ext.buildIdentity())
|
||||
@@ -551,8 +549,12 @@ func (s *Script) match(event ActivationEvent, req *base.Request) bool {
|
||||
}
|
||||
|
||||
// match url
|
||||
targetURL := req.RawURL
|
||||
if targetURL == "" {
|
||||
targetURL = req.URL
|
||||
}
|
||||
for _, url := range s.Match.Urls {
|
||||
if util.Match(url, req.URL) {
|
||||
if util.Match(url, targetURL) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,117 @@
|
||||
package download
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sync"
|
||||
|
||||
"github.com/GopeedLab/gopeed/pkg/download/engine"
|
||||
"github.com/GopeedLab/gopeed/pkg/download/engine/inject/stream"
|
||||
"github.com/dop251/goja"
|
||||
)
|
||||
|
||||
type engineSession struct {
|
||||
engine *engine.Engine
|
||||
|
||||
mu sync.Mutex
|
||||
refs int
|
||||
closed bool
|
||||
}
|
||||
|
||||
func newEngineSession(e *engine.Engine) *engineSession {
|
||||
return &engineSession{engine: e}
|
||||
}
|
||||
|
||||
func (s *engineSession) SetEngine(e *engine.Engine) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.engine = e
|
||||
}
|
||||
|
||||
func (s *engineSession) Retain() {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.closed {
|
||||
return
|
||||
}
|
||||
s.refs++
|
||||
}
|
||||
|
||||
func (s *engineSession) Release() {
|
||||
s.mu.Lock()
|
||||
if s.refs > 0 {
|
||||
s.refs--
|
||||
}
|
||||
shouldClose := s.refs == 0 && !s.closed
|
||||
if shouldClose {
|
||||
s.closed = true
|
||||
}
|
||||
s.mu.Unlock()
|
||||
if shouldClose {
|
||||
if s.engine != nil {
|
||||
go s.engine.Close()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *engineSession) CloseIfIdle() {
|
||||
s.mu.Lock()
|
||||
shouldClose := s.refs == 0 && !s.closed
|
||||
if shouldClose {
|
||||
s.closed = true
|
||||
}
|
||||
s.mu.Unlock()
|
||||
if shouldClose {
|
||||
if s.engine != nil {
|
||||
s.engine.Close()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func normalizeStreamChunk(data any) ([]byte, error) {
|
||||
switch v := data.(type) {
|
||||
case nil:
|
||||
return nil, nil
|
||||
case string:
|
||||
return []byte(v), nil
|
||||
case []byte:
|
||||
return v, nil
|
||||
case goja.ArrayBuffer:
|
||||
return v.Bytes(), nil
|
||||
default:
|
||||
return nil, fmt.Errorf("unsupported stream chunk type: %T", data)
|
||||
}
|
||||
}
|
||||
|
||||
func (d *Downloader) newExtensionEngine() (*engine.Engine, *engineSession) {
|
||||
session := newEngineSession(nil)
|
||||
engineCfg := &stream.Config{
|
||||
CreateBlobObjectURL: func(data []byte, contentType string) (string, error) {
|
||||
return d.gblob.CreateBlob(data, contentType)
|
||||
},
|
||||
CreateWritableStreamObjectURL: func() (string, error) {
|
||||
return d.gblob.CreateWritableStream(session)
|
||||
},
|
||||
WriteWritableStreamObjectURL: func(url string, data any) error {
|
||||
chunk, err := normalizeStreamChunk(data)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return d.gblob.Write(url, chunk)
|
||||
},
|
||||
CloseWritableStreamObjectURL: func(url string) error {
|
||||
return d.gblob.CloseSource(url)
|
||||
},
|
||||
AbortWritableStreamObjectURL: func(url string, reason string) error {
|
||||
return d.gblob.AbortSource(url, fmt.Errorf("%s", reason))
|
||||
},
|
||||
RevokeObjectURL: func(url string) error {
|
||||
return d.gblob.Revoke(url)
|
||||
},
|
||||
}
|
||||
e := engine.NewEngine(&engine.Config{
|
||||
ProxyConfig: d.cfg.Proxy,
|
||||
StreamConfig: engineCfg,
|
||||
})
|
||||
session.SetEngine(e)
|
||||
return e, session
|
||||
}
|
||||
@@ -3,6 +3,7 @@ package download
|
||||
import (
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -46,6 +47,320 @@ func TestDownloader_InstallExtensionByFolderDevMode(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
func TestDownloader_Extension_GBlobBlob(t *testing.T) {
|
||||
setupDownloader(func(downloader *Downloader) {
|
||||
if _, err := downloader.InstallExtensionByFolder("./testdata/extensions/gblob", false); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
rr, err := downloader.Resolve(&base.Request{
|
||||
URL: "https://example.com/blob",
|
||||
}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if rr.ID != "" {
|
||||
t.Fatalf("expected empty resolve id for extension resource, got %q", rr.ID)
|
||||
}
|
||||
|
||||
doneCh := make(chan error, 1)
|
||||
downloader.Listener(func(event *Event) {
|
||||
if event.Key == EventKeyDone || event.Key == EventKeyError {
|
||||
doneCh <- event.Err
|
||||
}
|
||||
})
|
||||
|
||||
dir := t.TempDir()
|
||||
if _, err := downloader.CreateDirect(rr.Res.Files[0].Req, &base.Options{
|
||||
Path: dir,
|
||||
Name: rr.Res.Files[0].Name,
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
select {
|
||||
case err := <-doneCh:
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("timeout waiting for gblob blob download")
|
||||
}
|
||||
|
||||
data, err := os.ReadFile(filepath.Join(dir, "hello.txt"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if string(data) != "hello world" {
|
||||
t.Fatalf("unexpected blob download content: %q", string(data))
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestDownloader_Extension_GBlobWritableStream(t *testing.T) {
|
||||
setupDownloader(func(downloader *Downloader) {
|
||||
if _, err := downloader.InstallExtensionByFolder("./testdata/extensions/gblob", false); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
rr, err := downloader.Resolve(&base.Request{
|
||||
URL: "https://example.com/stream",
|
||||
}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if rr.ID != "" {
|
||||
t.Fatalf("expected empty resolve id for extension resource, got %q", rr.ID)
|
||||
}
|
||||
|
||||
doneCh := make(chan error, 1)
|
||||
downloader.Listener(func(event *Event) {
|
||||
if event.Key == EventKeyDone || event.Key == EventKeyError {
|
||||
doneCh <- event.Err
|
||||
}
|
||||
})
|
||||
|
||||
dir := t.TempDir()
|
||||
if _, err := downloader.CreateDirect(rr.Res.Files[0].Req, &base.Options{
|
||||
Path: dir,
|
||||
Name: rr.Res.Files[0].Name,
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
select {
|
||||
case err := <-doneCh:
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("timeout waiting for gblob writable stream download")
|
||||
}
|
||||
|
||||
data, err := os.ReadFile(filepath.Join(dir, "stream.txt"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if string(data) != "line 1\nline 2\n" {
|
||||
t.Fatalf("unexpected stream download content: %q", string(data))
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestDownloader_Extension_GBlobWritableStreamUnknownSize(t *testing.T) {
|
||||
setupDownloader(func(downloader *Downloader) {
|
||||
if _, err := downloader.InstallExtensionByFolder("./testdata/extensions/gblob", false); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
rr, err := downloader.Resolve(&base.Request{
|
||||
URL: "https://example.com/stream-unknown",
|
||||
}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if rr.ID != "" {
|
||||
t.Fatalf("expected empty resolve id for extension resource, got %q", rr.ID)
|
||||
}
|
||||
if got := rr.Res.Files[0].Size; got != 0 {
|
||||
t.Fatalf("expected unknown size in resolve result, got %d", got)
|
||||
}
|
||||
|
||||
doneCh := make(chan error, 1)
|
||||
downloader.Listener(func(event *Event) {
|
||||
if event.Key == EventKeyDone || event.Key == EventKeyError {
|
||||
doneCh <- event.Err
|
||||
}
|
||||
})
|
||||
|
||||
dir := t.TempDir()
|
||||
id, err := downloader.CreateDirect(rr.Res.Files[0].Req, &base.Options{
|
||||
Path: dir,
|
||||
Name: rr.Res.Files[0].Name,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
time.Sleep(60 * time.Millisecond)
|
||||
task := downloader.GetTask(id)
|
||||
if task == nil {
|
||||
t.Fatal("task not found")
|
||||
}
|
||||
if task.Status == base.DownloadStatusDone {
|
||||
t.Fatal("task finished before writer.close()")
|
||||
}
|
||||
|
||||
select {
|
||||
case err := <-doneCh:
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("timeout waiting for unknown-size writable stream download")
|
||||
}
|
||||
|
||||
data, err := os.ReadFile(filepath.Join(dir, "stream-unknown.txt"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if string(data) != "line 1\nline 2\n" {
|
||||
t.Fatalf("unexpected unknown-size stream content: %q", string(data))
|
||||
}
|
||||
|
||||
task = downloader.GetTask(id)
|
||||
if task == nil {
|
||||
t.Fatal("task not found after completion")
|
||||
}
|
||||
if got := task.Meta.Res.Size; got != int64(len(data)) {
|
||||
t.Fatalf("unexpected final task size: got %d want %d", got, len(data))
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestDownloader_Extension_GBlobWritableStreamPauseAndContinue(t *testing.T) {
|
||||
setupDownloader(func(downloader *Downloader) {
|
||||
if _, err := downloader.InstallExtensionByFolder("./testdata/extensions/gblob", false); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
rr, err := downloader.Resolve(&base.Request{
|
||||
URL: "https://example.com/stream-unknown",
|
||||
}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
doneCh := make(chan error, 1)
|
||||
downloader.Listener(func(event *Event) {
|
||||
if event.Key == EventKeyDone || event.Key == EventKeyError {
|
||||
doneCh <- event.Err
|
||||
}
|
||||
})
|
||||
|
||||
dir := t.TempDir()
|
||||
id, err := downloader.CreateDirect(rr.Res.Files[0].Req, &base.Options{
|
||||
Path: dir,
|
||||
Name: rr.Res.Files[0].Name,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
filePath := filepath.Join(dir, "stream-unknown.txt")
|
||||
waitForFileSizeAtLeast(t, filePath, int64(len("line 1\n")), 2*time.Second)
|
||||
|
||||
if err := downloader.Pause(&TaskFilter{IDs: []string{id}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
task := downloader.GetTask(id)
|
||||
if task == nil {
|
||||
t.Fatal("task not found after pause")
|
||||
}
|
||||
if task.Status != base.DownloadStatusPause {
|
||||
t.Fatalf("expected paused task, got %s", task.Status)
|
||||
}
|
||||
|
||||
stat, err := os.Stat(filePath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pausedSize := stat.Size()
|
||||
|
||||
time.Sleep(250 * time.Millisecond)
|
||||
|
||||
stat, err = os.Stat(filePath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if stat.Size() != pausedSize {
|
||||
t.Fatalf("expected paused file size to remain %d, got %d", pausedSize, stat.Size())
|
||||
}
|
||||
|
||||
if err := downloader.Continue(&TaskFilter{IDs: []string{id}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
select {
|
||||
case err := <-doneCh:
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("timeout waiting for pause/continue writable stream download")
|
||||
}
|
||||
|
||||
data, err := os.ReadFile(filePath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if string(data) != "line 1\nline 2\n" {
|
||||
t.Fatalf("unexpected pause/continue stream content: %q", string(data))
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestDownloader_Extension_GBlobRecoverOnError(t *testing.T) {
|
||||
setupDownloader(func(downloader *Downloader) {
|
||||
if _, err := downloader.InstallExtensionByFolder("./testdata/extensions/gblob_recover", false); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
rr, err := downloader.Resolve(&base.Request{
|
||||
URL: "https://example.com/recover",
|
||||
}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
doneCh := make(chan error, 1)
|
||||
downloader.Listener(func(event *Event) {
|
||||
if event.Key == EventKeyDone || event.Key == EventKeyError {
|
||||
doneCh <- event.Err
|
||||
}
|
||||
})
|
||||
|
||||
dir := t.TempDir()
|
||||
id, err := downloader.CreateDirect(rr.Res.Files[0].Req, &base.Options{
|
||||
Path: dir,
|
||||
Name: rr.Res.Files[0].Name,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
select {
|
||||
case err := <-doneCh:
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("timeout waiting for recovered gblob download")
|
||||
}
|
||||
|
||||
filePath := filepath.Join(dir, "recover.txt")
|
||||
data, err := os.ReadFile(filePath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if string(data) != "recovered\n" {
|
||||
t.Fatalf("unexpected recovered file content: %q", string(data))
|
||||
}
|
||||
|
||||
task := downloader.GetTask(id)
|
||||
if task == nil {
|
||||
t.Fatal("task not found after recovery")
|
||||
}
|
||||
if task.Status != base.DownloadStatusDone {
|
||||
t.Fatalf("expected recovered task done, got %s", task.Status)
|
||||
}
|
||||
if task.Meta.Req.RawURL != "https://example.com/recover" {
|
||||
t.Fatalf("unexpected raw url: %q", task.Meta.Req.RawURL)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestDownloader_InstallExtensionByGit(t *testing.T) {
|
||||
setupDownloader(func(downloader *Downloader) {
|
||||
if _, err := downloader.InstallExtensionByGit("https://github.com/GopeedLab/gopeed-extension-samples#github-release-sample"); err != nil {
|
||||
@@ -458,3 +773,16 @@ func setupDownloader(fn func(downloader *Downloader)) {
|
||||
}()
|
||||
fn(defaultDownloader)
|
||||
}
|
||||
|
||||
func waitForFileSizeAtLeast(t *testing.T, path string, size int64, timeout time.Duration) {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(timeout)
|
||||
for time.Now().Before(deadline) {
|
||||
info, err := os.Stat(path)
|
||||
if err == nil && info.Size() >= size {
|
||||
return
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
t.Fatalf("timeout waiting for file %s size >= %d", path, size)
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
package download
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
)
|
||||
|
||||
func NewExtMockDownloader() (*Downloader, func(), error) {
|
||||
d := NewDownloader(&DownloaderConfig{
|
||||
Storage: NewMemStorage(),
|
||||
})
|
||||
if err := d.Setup(); err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
cleanup := func() {
|
||||
_ = d.Clear()
|
||||
}
|
||||
return d, cleanup, nil
|
||||
}
|
||||
|
||||
type ExtMockRuntime struct {
|
||||
engine engineWrapper
|
||||
session *engineSession
|
||||
}
|
||||
|
||||
type engineWrapper interface {
|
||||
Close()
|
||||
RunString(script string) (any, error)
|
||||
}
|
||||
|
||||
func (d *Downloader) NewExtMockRuntime() (*ExtMockRuntime, error) {
|
||||
engine, session := d.newExtensionEngine()
|
||||
ext := &Extension{
|
||||
Name: "extmock",
|
||||
Author: "gopeed",
|
||||
Title: "Gopeed ExtMock Runtime",
|
||||
Version: "0.0.0",
|
||||
DevMode: true,
|
||||
}
|
||||
gopeed := &Instance{
|
||||
Events: make(InstanceEvents),
|
||||
Info: NewExtensionInfo(ext),
|
||||
Logger: newInstanceLogger(ext, d.ExtensionLogger),
|
||||
Settings: map[string]any{},
|
||||
Storage: &ContextStorage{
|
||||
storage: d.storage,
|
||||
identity: ext.buildIdentity(),
|
||||
},
|
||||
}
|
||||
if err := engine.Runtime.Set("gopeed", gopeed); err != nil {
|
||||
engine.Close()
|
||||
return nil, err
|
||||
}
|
||||
return &ExtMockRuntime{
|
||||
engine: engine,
|
||||
session: session,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (r *ExtMockRuntime) Eval(script string) (any, error) {
|
||||
if r == nil || r.engine == nil {
|
||||
return nil, fmt.Errorf("extmock runtime not initialized")
|
||||
}
|
||||
return r.engine.RunString(script)
|
||||
}
|
||||
|
||||
func (r *ExtMockRuntime) EvalFile(path string) (any, error) {
|
||||
if path == "" {
|
||||
return nil, fmt.Errorf("script path is empty")
|
||||
}
|
||||
buf, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return r.Eval(string(buf))
|
||||
}
|
||||
|
||||
func (r *ExtMockRuntime) Close() {
|
||||
if r == nil {
|
||||
return
|
||||
}
|
||||
if r.engine != nil {
|
||||
r.engine.Close()
|
||||
}
|
||||
r.engine = nil
|
||||
r.session = nil
|
||||
}
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"github.com/GopeedLab/gopeed/internal/fetcher"
|
||||
"github.com/GopeedLab/gopeed/internal/protocol/bt"
|
||||
"github.com/GopeedLab/gopeed/internal/protocol/ed2k"
|
||||
"github.com/GopeedLab/gopeed/internal/protocol/gblob"
|
||||
"github.com/GopeedLab/gopeed/internal/protocol/http"
|
||||
"github.com/GopeedLab/gopeed/pkg/base"
|
||||
"github.com/GopeedLab/gopeed/pkg/util"
|
||||
@@ -165,6 +166,7 @@ func (cfg *DownloaderConfig) Init() *DownloaderConfig {
|
||||
}
|
||||
if len(cfg.FetchManagers) == 0 {
|
||||
cfg.FetchManagers = []fetcher.FetcherManager{
|
||||
new(gblob.FetcherManager),
|
||||
new(http.FetcherManager),
|
||||
new(bt.FetcherManager),
|
||||
new(ed2k.FetcherManager),
|
||||
|
||||
+62
@@ -0,0 +1,62 @@
|
||||
gopeed.events.onResolve(async function (ctx) {
|
||||
if (ctx.req.url.endsWith("/blob")) {
|
||||
const blob = new Blob(["hello world"], {type: "text/plain"});
|
||||
const url = URL.createObjectURL(blob);
|
||||
ctx.res = {
|
||||
name: "gblob-blob",
|
||||
files: [
|
||||
{
|
||||
name: "hello.txt",
|
||||
size: 11,
|
||||
req: {
|
||||
url,
|
||||
}
|
||||
}
|
||||
]
|
||||
};
|
||||
return;
|
||||
}
|
||||
|
||||
const transform = new TransformStream();
|
||||
const writer = transform.writable.getWriter();
|
||||
const url = URL.createObjectURL(transform.writable);
|
||||
|
||||
if (ctx.req.url.endsWith("/stream-unknown")) {
|
||||
(async () => {
|
||||
await writer.write(new TextEncoder().encode("line 1\n"));
|
||||
await new Promise((resolve) => setTimeout(resolve, 120));
|
||||
await writer.write(new TextEncoder().encode("line 2\n"));
|
||||
await writer.close();
|
||||
})();
|
||||
|
||||
ctx.res = {
|
||||
name: "gblob-stream-unknown",
|
||||
files: [
|
||||
{
|
||||
name: "stream-unknown.txt",
|
||||
req: {
|
||||
url,
|
||||
}
|
||||
}
|
||||
]
|
||||
};
|
||||
return;
|
||||
}
|
||||
|
||||
await writer.write(new TextEncoder().encode("line 1\n"));
|
||||
await writer.write(new TextEncoder().encode("line 2\n"));
|
||||
await writer.close();
|
||||
|
||||
ctx.res = {
|
||||
name: "gblob-stream",
|
||||
files: [
|
||||
{
|
||||
name: "stream.txt",
|
||||
size: 14,
|
||||
req: {
|
||||
url,
|
||||
}
|
||||
}
|
||||
]
|
||||
};
|
||||
});
|
||||
@@ -0,0 +1,16 @@
|
||||
{
|
||||
"name": "gblob",
|
||||
"title": "gopeed extension gblob test",
|
||||
"version": "0.0.1",
|
||||
"scripts": [
|
||||
{
|
||||
"event": "onResolve",
|
||||
"match": {
|
||||
"urls": [
|
||||
"*://example.com/*"
|
||||
]
|
||||
},
|
||||
"entry": "index.js"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
gopeed.events.onResolve(async function (ctx) {
|
||||
if (!ctx.req.url.endsWith("/recover")) {
|
||||
return;
|
||||
}
|
||||
|
||||
const transform = new TransformStream();
|
||||
const writer = transform.writable.getWriter();
|
||||
const url = URL.createObjectURL(transform.writable);
|
||||
|
||||
(async () => {
|
||||
await writer.write(new TextEncoder().encode("stale\n"));
|
||||
await new Promise((resolve) => setTimeout(resolve, 50));
|
||||
await writer.abort("expired");
|
||||
})();
|
||||
|
||||
ctx.res = {
|
||||
name: "gblob-recover",
|
||||
files: [
|
||||
{
|
||||
name: "recover.txt",
|
||||
req: {
|
||||
url,
|
||||
}
|
||||
}
|
||||
]
|
||||
};
|
||||
});
|
||||
|
||||
gopeed.events.onError(async function (ctx) {
|
||||
const req = ctx.task.meta.req;
|
||||
if (!req.rawUrl || !req.rawUrl.endsWith("/recover")) {
|
||||
return;
|
||||
}
|
||||
req.labels = req.labels || {};
|
||||
if (req.labels.recovered === "true") {
|
||||
return;
|
||||
}
|
||||
|
||||
req.labels.recovered = "true";
|
||||
req.url = URL.createObjectURL(new Blob(["recovered\n"], { type: "text/plain" }));
|
||||
ctx.task.continue();
|
||||
});
|
||||
@@ -0,0 +1,25 @@
|
||||
{
|
||||
"name": "gblob-recover",
|
||||
"title": "gopeed extension gblob recover test",
|
||||
"version": "0.0.1",
|
||||
"scripts": [
|
||||
{
|
||||
"event": "onResolve",
|
||||
"match": {
|
||||
"urls": [
|
||||
"*://example.com/*"
|
||||
]
|
||||
},
|
||||
"entry": "index.js"
|
||||
},
|
||||
{
|
||||
"event": "onError",
|
||||
"match": {
|
||||
"urls": [
|
||||
"*://example.com/*"
|
||||
]
|
||||
},
|
||||
"entry": "index.js"
|
||||
}
|
||||
]
|
||||
}
|
||||
Reference in new issue
Block a user