mirror of
https://github.com/GopeedLab/gopeed.git
synced 2026-11-07 00:50:27 +00:00
feat: extension support (#78)
This commit is contained in:
118 files changed
+6971
-780
No files matched your search
+48
-2
@@ -1,6 +1,7 @@
|
||||
package base
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/GopeedLab/gopeed/pkg/util"
|
||||
)
|
||||
|
||||
@@ -12,20 +13,46 @@ type Request struct {
|
||||
|
||||
// Resource download resource
|
||||
type Resource struct {
|
||||
// if name is not empty, the resource is a folder and the name is the folder name
|
||||
Name string `json:"name"`
|
||||
Size int64 `json:"size"`
|
||||
// is support range download
|
||||
Range bool `json:"range"`
|
||||
RootDir string `json:"rootDir"`
|
||||
Range bool `json:"range"`
|
||||
// file list
|
||||
Files []*FileInfo `json:"files"`
|
||||
Hash string `json:"hash"`
|
||||
}
|
||||
|
||||
func (r *Resource) Validate() error {
|
||||
if r.Name == "" {
|
||||
return fmt.Errorf("invalid resource name")
|
||||
}
|
||||
if r.Files == nil || len(r.Files) == 0 {
|
||||
return fmt.Errorf("invalid resource files")
|
||||
}
|
||||
for _, file := range r.Files {
|
||||
if file.Name == "" {
|
||||
return fmt.Errorf("invalid resource file name")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
|
||||
}
|
||||
|
||||
func (r *Resource) CalcSize() {
|
||||
var size int64
|
||||
for _, file := range r.Files {
|
||||
size += file.Size
|
||||
}
|
||||
r.Size = size
|
||||
}
|
||||
|
||||
type FileInfo struct {
|
||||
Name string `json:"name"`
|
||||
Path string `json:"path"`
|
||||
Size int64 `json:"size"`
|
||||
|
||||
Req *Request `json:"req"`
|
||||
}
|
||||
|
||||
// Options for download
|
||||
@@ -40,6 +67,25 @@ type Options struct {
|
||||
Extra any `json:"extra"`
|
||||
}
|
||||
|
||||
func (o *Options) InitSelectFiles(fileSize int) {
|
||||
// if selectFiles is empty, select all files
|
||||
if len(o.SelectFiles) == 0 {
|
||||
o.SelectFiles = make([]int, fileSize)
|
||||
for i := 0; i < fileSize; i++ {
|
||||
o.SelectFiles[i] = i
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (o *Options) Clone() *Options {
|
||||
return &Options{
|
||||
Name: o.Name,
|
||||
Path: o.Path,
|
||||
SelectFiles: o.SelectFiles,
|
||||
Extra: o.Extra,
|
||||
}
|
||||
}
|
||||
|
||||
func ParseReqExtra[E any](req *Request) error {
|
||||
if req.Extra == nil {
|
||||
return nil
|
||||
|
||||
+236
-163
@@ -4,13 +4,13 @@ import (
|
||||
"errors"
|
||||
"github.com/GopeedLab/gopeed/internal/controller"
|
||||
"github.com/GopeedLab/gopeed/internal/fetcher"
|
||||
"github.com/GopeedLab/gopeed/internal/logger"
|
||||
"github.com/GopeedLab/gopeed/pkg/base"
|
||||
"github.com/GopeedLab/gopeed/pkg/util"
|
||||
gonanoid "github.com/matoous/go-nanoid/v2"
|
||||
"github.com/virtuald/go-paniclog"
|
||||
"math"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
@@ -26,6 +26,12 @@ const (
|
||||
bucketSave = "save"
|
||||
// downloader config bucket
|
||||
bucketConfig = "config"
|
||||
// downloader extension bucket
|
||||
bucketExtension = "extension"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrTaskNotFound = errors.New("task not found")
|
||||
)
|
||||
|
||||
type Listener func(event *Event)
|
||||
@@ -40,16 +46,21 @@ type Progress struct {
|
||||
}
|
||||
|
||||
type Downloader struct {
|
||||
cfg *DownloaderConfig
|
||||
fetchBuilders map[string]fetcher.FetcherBuilder
|
||||
fetcherMap map[string]fetcher.Fetcher
|
||||
storage Storage
|
||||
tasks []*Task
|
||||
listener Listener
|
||||
Logger *logger.Logger
|
||||
ExtensionLogger *logger.Logger
|
||||
|
||||
cfg *DownloaderConfig
|
||||
fetcherBuilders map[string]fetcher.FetcherBuilder
|
||||
fetcherCache map[string]fetcher.Fetcher
|
||||
storage Storage
|
||||
tasks []*Task
|
||||
listener Listener
|
||||
|
||||
lock *sync.Mutex
|
||||
fetcherMapLock *sync.RWMutex
|
||||
closed atomic.Bool
|
||||
|
||||
extensions []*Extension
|
||||
}
|
||||
|
||||
func NewDownloader(cfg *DownloaderConfig) *Downloader {
|
||||
@@ -61,26 +72,32 @@ func NewDownloader(cfg *DownloaderConfig) *Downloader {
|
||||
d := &Downloader{
|
||||
cfg: cfg,
|
||||
|
||||
fetchBuilders: make(map[string]fetcher.FetcherBuilder),
|
||||
fetcherMap: make(map[string]fetcher.Fetcher),
|
||||
fetcherBuilders: make(map[string]fetcher.FetcherBuilder),
|
||||
fetcherCache: make(map[string]fetcher.Fetcher),
|
||||
|
||||
storage: cfg.Storage,
|
||||
lock: &sync.Mutex{},
|
||||
fetcherMapLock: &sync.RWMutex{},
|
||||
|
||||
extensions: make([]*Extension, 0),
|
||||
}
|
||||
for _, f := range cfg.FetchBuilders {
|
||||
for _, p := range f.Schemes() {
|
||||
d.fetchBuilders[strings.ToUpper(p)] = f
|
||||
d.fetcherBuilders[strings.ToUpper(p)] = f
|
||||
}
|
||||
}
|
||||
|
||||
logPanic(filepath.Join(cfg.StorageDir, "logs"))
|
||||
d.Logger = logger.NewLogger(cfg.ProductionMode, filepath.Join(cfg.StorageDir, "logs", "core.log"))
|
||||
d.ExtensionLogger = logger.NewLogger(cfg.ProductionMode, filepath.Join(cfg.StorageDir, "logs", "extension.log"))
|
||||
if cfg.ProductionMode {
|
||||
logPanic(filepath.Join(cfg.StorageDir, "logs"))
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
||||
func (d *Downloader) Setup() error {
|
||||
// setup storage
|
||||
if err := d.storage.Setup([]string{bucketTask, bucketSave, bucketConfig}); err != nil {
|
||||
if err := d.storage.Setup([]string{bucketTask, bucketSave, bucketConfig, bucketExtension}); err != nil {
|
||||
return err
|
||||
}
|
||||
// load config from storage
|
||||
@@ -100,7 +117,7 @@ func (d *Downloader) Setup() error {
|
||||
d.cfg.DownloaderStoreConfig.Init()
|
||||
// load tasks from storage
|
||||
var tasks []*Task
|
||||
if err := d.storage.List(bucketTask, &tasks); err != nil {
|
||||
if err = d.storage.List(bucketTask, &tasks); err != nil {
|
||||
return err
|
||||
}
|
||||
if tasks == nil {
|
||||
@@ -118,16 +135,25 @@ func (d *Downloader) Setup() error {
|
||||
task.Status = base.DownloadStatusPause
|
||||
}
|
||||
}
|
||||
// sort by create time
|
||||
sort.Slice(tasks, func(i, j int) bool {
|
||||
return tasks[i].CreatedAt.Before(tasks[j].CreatedAt)
|
||||
})
|
||||
}
|
||||
d.tasks = tasks
|
||||
// sort by create time
|
||||
sort.Slice(d.tasks, func(i, j int) bool {
|
||||
return d.tasks[i].CreatedAt.Before(d.tasks[j].CreatedAt)
|
||||
})
|
||||
|
||||
// 每个tick统计一次下载速度
|
||||
// load extensions from storage
|
||||
var extensions []*Extension
|
||||
if err = d.storage.List(bucketExtension, &extensions); err != nil {
|
||||
return err
|
||||
}
|
||||
if extensions == nil {
|
||||
extensions = make([]*Extension, 0)
|
||||
}
|
||||
d.extensions = extensions
|
||||
|
||||
// calculate download speed every tick
|
||||
go func() {
|
||||
|
||||
for !d.closed.Load() {
|
||||
if len(d.tasks) > 0 {
|
||||
for _, task := range d.tasks {
|
||||
@@ -151,13 +177,16 @@ func (d *Downloader) Setup() error {
|
||||
// store fetcher progress
|
||||
data, err := task.fetcherBuilder.Store(task.fetcher)
|
||||
if err != nil {
|
||||
return // TODO log
|
||||
d.Logger.Error().Stack().Err(err).Msgf("serialize fetcher failed: %s", task.ID)
|
||||
return
|
||||
}
|
||||
if err := d.storage.Put(bucketSave, task.ID, data); err != nil {
|
||||
return // TODO log
|
||||
d.Logger.Error().Stack().Err(err).Msgf("persist fetcher failed: %s", task.ID)
|
||||
return
|
||||
}
|
||||
if err := d.storage.Put(bucketTask, task.ID, task); err != nil {
|
||||
return // TODO log
|
||||
d.Logger.Error().Stack().Err(err).Msgf("persist task failed: %s", task.ID)
|
||||
return
|
||||
}
|
||||
}()
|
||||
}
|
||||
@@ -170,7 +199,7 @@ func (d *Downloader) Setup() error {
|
||||
|
||||
func (d *Downloader) parseFb(url string) (fetcher.FetcherBuilder, error) {
|
||||
schema := util.ParseSchema(url)
|
||||
fetchBuilder, ok := d.fetchBuilders[schema]
|
||||
fetchBuilder, ok := d.fetcherBuilders[schema]
|
||||
if ok {
|
||||
return fetchBuilder, nil
|
||||
}
|
||||
@@ -186,22 +215,29 @@ func (d *Downloader) setupFetcher(fetcher fetcher.Fetcher) {
|
||||
}
|
||||
|
||||
func (d *Downloader) Resolve(req *base.Request) (rr *ResolveResult, err error) {
|
||||
fb, err := d.parseFb(req.URL)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
fetcher := fb.Build()
|
||||
d.setupFetcher(fetcher)
|
||||
err = fetcher.Resolve(req)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
rrId, err := gonanoid.New()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
res := d.triggerOnResolve(req)
|
||||
if res != nil && len(res.Files) > 0 {
|
||||
rr = &ResolveResult{
|
||||
Res: res,
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
fetcher, err := d.buildFetcher(req.URL)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = fetcher.Resolve(req)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
d.fetcherMapLock.Lock()
|
||||
d.fetcherMap[rrId] = fetcher
|
||||
d.fetcherCache[rrId] = fetcher
|
||||
d.fetcherMapLock.Unlock()
|
||||
rr = &ResolveResult{
|
||||
ID: rrId,
|
||||
@@ -211,11 +247,13 @@ func (d *Downloader) Resolve(req *base.Request) (rr *ResolveResult, err error) {
|
||||
}
|
||||
|
||||
func (d *Downloader) CreateDirect(req *base.Request, opts *base.Options) (taskId string, err error) {
|
||||
rr, err := d.Resolve(req)
|
||||
var fetcher fetcher.Fetcher
|
||||
fetcher, err = d.buildFetcher(req.URL)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
return d.Create(rr.ID, opts)
|
||||
fetcher.Meta().Req = req
|
||||
return d.doCreate(fetcher, opts)
|
||||
}
|
||||
|
||||
func (d *Downloader) Create(rrId string, opts *base.Options) (taskId string, err error) {
|
||||
@@ -223,95 +261,23 @@ func (d *Downloader) Create(rrId string, opts *base.Options) (taskId string, err
|
||||
opts = &base.Options{}
|
||||
}
|
||||
d.fetcherMapLock.RLock()
|
||||
fetcher, ok := d.fetcherMap[rrId]
|
||||
fetcher, ok := d.fetcherCache[rrId]
|
||||
d.fetcherMapLock.RUnlock()
|
||||
if !ok {
|
||||
return "", errors.New("invalid resource id")
|
||||
}
|
||||
defer func() {
|
||||
d.fetcherMapLock.Lock()
|
||||
delete(d.fetcherMap, rrId)
|
||||
delete(d.fetcherCache, rrId)
|
||||
d.fetcherMapLock.Unlock()
|
||||
}()
|
||||
|
||||
meta := fetcher.Meta()
|
||||
meta.Opts = opts
|
||||
res := meta.Res
|
||||
if len(opts.SelectFiles) == 0 {
|
||||
opts.SelectFiles = make([]int, 0)
|
||||
for i := range res.Files {
|
||||
opts.SelectFiles = append(opts.SelectFiles, i)
|
||||
}
|
||||
}
|
||||
if opts.Path == "" {
|
||||
storeConfig, err := d.GetConfig()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
opts.Path = storeConfig.DownloadDir
|
||||
}
|
||||
|
||||
fb, err := d.parseFb(fetcher.Meta().Req.URL)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
// check if the download file is duplicated and rename it automatically.
|
||||
files := res.Files
|
||||
if res.RootDir != "" {
|
||||
fullDirPath := path.Join(opts.Path, res.RootDir)
|
||||
newName, err := util.CheckDuplicateAndRename(fullDirPath)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
res.RootDir = newName
|
||||
} else if len(files) == 1 {
|
||||
fullPath := meta.Filepath(files[0])
|
||||
newName, err := util.CheckDuplicateAndRename(fullPath)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
opts.Name = newName
|
||||
}
|
||||
|
||||
err = fetcher.Create(opts)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
task := NewTask()
|
||||
task.fetcherBuilder = fb
|
||||
task.fetcher = fetcher
|
||||
task.Meta = fetcher.Meta()
|
||||
task.Progress = &Progress{}
|
||||
task.Status = base.DownloadStatusReady
|
||||
// calculate select files size
|
||||
for _, selectIndex := range opts.SelectFiles {
|
||||
task.Size += res.Files[selectIndex].Size
|
||||
}
|
||||
initTask(task)
|
||||
err = func() error {
|
||||
d.lock.Lock()
|
||||
defer d.lock.Unlock()
|
||||
d.tasks = append(d.tasks, task)
|
||||
return d.storage.Put(bucketTask, task.ID, task.clone())
|
||||
}()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
taskId = task.ID
|
||||
d.tryRunning(func(remain int) {
|
||||
if remain > 0 {
|
||||
err = d.start(task)
|
||||
}
|
||||
})
|
||||
return
|
||||
return d.doCreate(fetcher, opts)
|
||||
}
|
||||
|
||||
func (d *Downloader) Pause(id string) (err error) {
|
||||
task := d.GetTask(id)
|
||||
if task == nil {
|
||||
return
|
||||
return ErrTaskNotFound
|
||||
}
|
||||
if err = d.doPause(task, base.DownloadStatusPause); err != nil {
|
||||
return err
|
||||
@@ -322,7 +288,7 @@ func (d *Downloader) Pause(id string) (err error) {
|
||||
func (d *Downloader) Continue(id string) (err error) {
|
||||
task := d.GetTask(id)
|
||||
if task == nil {
|
||||
return
|
||||
return ErrTaskNotFound
|
||||
}
|
||||
d.tryRunning(func(remain int) {
|
||||
if remain == 0 {
|
||||
@@ -347,14 +313,7 @@ func (d *Downloader) Continue(id string) (err error) {
|
||||
func (d *Downloader) PauseAll() (err error) {
|
||||
d.lock.Lock()
|
||||
defer d.lock.Unlock()
|
||||
for _, task := range d.tasks {
|
||||
if task.Status == base.DownloadStatusRunning {
|
||||
if err = d.doPause(task, base.DownloadStatusPause); err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
return
|
||||
return d.doPauseAll()
|
||||
}
|
||||
|
||||
func (d *Downloader) ContinueAll() (err error) {
|
||||
@@ -384,7 +343,7 @@ func (d *Downloader) Delete(id string, force bool) (err error) {
|
||||
defer d.lock.Unlock()
|
||||
task := d.GetTask(id)
|
||||
if task == nil {
|
||||
return
|
||||
return ErrTaskNotFound
|
||||
}
|
||||
task.lock.Lock()
|
||||
defer task.lock.Unlock()
|
||||
@@ -408,16 +367,14 @@ func (d *Downloader) Delete(id string, force bool) (err error) {
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if force {
|
||||
if task.Meta.Res.RootDir != "" {
|
||||
if err = os.RemoveAll(path.Join(task.Meta.Opts.Path, task.Meta.Res.RootDir)); err != nil {
|
||||
if force && task.Meta.Res != nil {
|
||||
if task.Meta.Res.Name != "" {
|
||||
if err = os.RemoveAll(task.Meta.FolderPath()); err != nil {
|
||||
return
|
||||
}
|
||||
} else {
|
||||
for _, file := range task.Meta.Res.Files {
|
||||
if err = util.SafeRemove(task.Meta.Filepath(file)); err != nil {
|
||||
return err
|
||||
}
|
||||
if err = util.SafeRemove(task.Meta.SingleFilepath()); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -435,16 +392,20 @@ func (d *Downloader) Close() error {
|
||||
d.lock.Lock()
|
||||
defer d.lock.Unlock()
|
||||
d.closed.Store(true)
|
||||
for _, task := range d.tasks {
|
||||
d.doPause(task, base.DownloadStatusPause)
|
||||
if err := d.doPauseAll(); err != nil {
|
||||
return err
|
||||
}
|
||||
return d.storage.Close()
|
||||
}
|
||||
|
||||
func (d *Downloader) Clear() error {
|
||||
if err := d.Close(); err != nil {
|
||||
return err
|
||||
if !d.closed.Load() {
|
||||
if err := d.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
d.tasks = make([]*Task, 0)
|
||||
d.extensions = make([]*Extension, 0)
|
||||
if err := d.storage.Clear(); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -479,7 +440,12 @@ func (d *Downloader) GetTask(id string) *Task {
|
||||
}
|
||||
|
||||
func (d *Downloader) GetTasks() []*Task {
|
||||
return d.tasks
|
||||
// reverse tasks, so that the latest task is in the front
|
||||
tasks := make([]*Task, len(d.tasks))
|
||||
for i := 0; i < len(d.tasks); i++ {
|
||||
tasks[i] = d.tasks[len(d.tasks)-i-1]
|
||||
}
|
||||
return tasks
|
||||
}
|
||||
|
||||
func (d *Downloader) GetConfig() (*DownloaderStoreConfig, error) {
|
||||
@@ -552,21 +518,71 @@ func (d *Downloader) restoreFetcher(task *Task) error {
|
||||
return nil
|
||||
}()
|
||||
if err != nil {
|
||||
// TODO log
|
||||
d.Logger.Error().Stack().Err(err).Msgf("deserialize fetcher failed, task id: %s", task.ID)
|
||||
}
|
||||
if task.fetcher == nil {
|
||||
task.fetcher = fb.Build()
|
||||
}
|
||||
d.setupFetcher(task.fetcher)
|
||||
task.fetcher.Create(task.Meta.Opts)
|
||||
go d.watch(task)
|
||||
}
|
||||
task.fetcher.Create(task.Meta.Opts)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *Downloader) tryRunning(cb func(remain int)) {
|
||||
d.lock.Lock()
|
||||
defer d.lock.Unlock()
|
||||
func (d *Downloader) doCreate(fetcher fetcher.Fetcher, opts *base.Options) (taskId string, err error) {
|
||||
if opts == nil {
|
||||
opts = &base.Options{}
|
||||
}
|
||||
|
||||
meta := fetcher.Meta()
|
||||
meta.Opts = opts
|
||||
if opts.Path == "" {
|
||||
storeConfig, err := d.GetConfig()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
opts.Path = storeConfig.DownloadDir
|
||||
}
|
||||
|
||||
fb, err := d.parseFb(fetcher.Meta().Req.URL)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
task := NewTask()
|
||||
task.fetcherBuilder = fb
|
||||
task.fetcher = fetcher
|
||||
task.Meta = fetcher.Meta()
|
||||
task.Progress = &Progress{}
|
||||
task.Status = base.DownloadStatusReady
|
||||
initTask(task)
|
||||
if err = fetcher.Create(opts); err != nil {
|
||||
return
|
||||
}
|
||||
if err = d.storage.Put(bucketTask, task.ID, task.clone()); err != nil {
|
||||
return
|
||||
}
|
||||
taskId = task.ID
|
||||
|
||||
func() {
|
||||
d.lock.Lock()
|
||||
defer d.lock.Unlock()
|
||||
d.tasks = append(d.tasks, task)
|
||||
}()
|
||||
go d.watch(task)
|
||||
go d.tryRunning(func(remain int) {
|
||||
if remain > 0 {
|
||||
if err = d.start(task); err != nil {
|
||||
d.Logger.Error().Stack().Err(err).Msgf("start task failed, task id: %s", task.ID)
|
||||
return
|
||||
}
|
||||
}
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
func (d *Downloader) tryRunning(cb func(remain int)) {
|
||||
runningCount := 0
|
||||
for _, task := range d.tasks {
|
||||
if task.Status == base.DownloadStatusRunning {
|
||||
@@ -592,39 +608,75 @@ func (d *Downloader) doNotifyRunning() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *Downloader) doPauseAll() (err error) {
|
||||
for _, task := range d.tasks {
|
||||
if err = d.doPause(task, base.DownloadStatusPause); err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (d *Downloader) start(task *Task) error {
|
||||
var event EventKey
|
||||
if task.Status == base.DownloadStatusReady {
|
||||
event = EventKeyStart
|
||||
} else {
|
||||
event = EventKeyContinue
|
||||
task.lock.Lock()
|
||||
defer task.lock.Unlock()
|
||||
isCreate := task.Status == base.DownloadStatusReady
|
||||
task.Status = base.DownloadStatusRunning
|
||||
if task.Meta.Res == nil {
|
||||
err := task.fetcher.Resolve(task.Meta.Req)
|
||||
if err != nil {
|
||||
task.Status = base.DownloadStatusError
|
||||
d.emit(EventKeyError, task, err)
|
||||
return err
|
||||
}
|
||||
isCreate = true
|
||||
}
|
||||
|
||||
task.Status = base.DownloadStatusRunning
|
||||
if isCreate {
|
||||
d.lock.Lock()
|
||||
defer d.lock.Unlock()
|
||||
// check if the download file is duplicated and rename it automatically.
|
||||
if task.Meta.Res.Name != "" {
|
||||
fullDirPath := task.Meta.FolderPath()
|
||||
newName, err := util.CheckDuplicateAndRename(fullDirPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
task.Meta.Opts.Name = newName
|
||||
} else {
|
||||
fullFilePath := task.Meta.SingleFilepath()
|
||||
newName, err := util.CheckDuplicateAndRename(fullFilePath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
task.Meta.Opts.Name = newName
|
||||
}
|
||||
}
|
||||
if err := d.doStart(task); err != nil {
|
||||
task.Status = base.DownloadStatusError
|
||||
d.emit(EventKeyError, task, err)
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *Downloader) doStart(task *Task) error {
|
||||
task.Progress.Speed = 0
|
||||
task.timer.Start()
|
||||
if err := d.storage.Put(bucketTask, task.ID, task.clone()); err != nil {
|
||||
return err
|
||||
}
|
||||
var err error
|
||||
if event == EventKeyStart {
|
||||
err = task.fetcher.Start()
|
||||
}
|
||||
if event == EventKeyContinue {
|
||||
err = task.fetcher.Continue()
|
||||
}
|
||||
if err != nil {
|
||||
if err := task.fetcher.Start(); err != nil {
|
||||
return err
|
||||
}
|
||||
go d.watch(task)
|
||||
d.emit(event, task)
|
||||
d.emit(EventKeyStart, task)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *Downloader) doPause(task *Task, status base.Status) (err error) {
|
||||
task.lock.Lock()
|
||||
defer task.lock.Unlock()
|
||||
if task.Status == base.DownloadStatusRunning {
|
||||
if task.Status != base.DownloadStatusDone {
|
||||
task.Status = status
|
||||
task.timer.Pause()
|
||||
task.fetcher.Pause()
|
||||
@@ -635,15 +687,26 @@ func (d *Downloader) doPause(task *Task, status base.Status) (err error) {
|
||||
}
|
||||
|
||||
func (d *Downloader) doContinue(task *Task) (err error) {
|
||||
task.lock.Lock()
|
||||
defer task.lock.Unlock()
|
||||
if task.Status != base.DownloadStatusRunning && task.Status != base.DownloadStatusDone {
|
||||
err = d.restoreFetcher(task)
|
||||
if err != nil {
|
||||
err = func() error {
|
||||
task.lock.Lock()
|
||||
defer task.lock.Unlock()
|
||||
if task.Status != base.DownloadStatusRunning && task.Status != base.DownloadStatusDone {
|
||||
err := d.restoreFetcher(task)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
go func() {
|
||||
if err := d.start(task); err != nil {
|
||||
d.Logger.Error().Stack().Err(err).Msgf("start task failed, task id: %s", task.ID)
|
||||
return
|
||||
}
|
||||
err = d.start(task)
|
||||
}
|
||||
}()
|
||||
return
|
||||
}
|
||||
|
||||
@@ -659,6 +722,16 @@ func logPanic(logDir string) {
|
||||
paniclog.RedirectStderr(f)
|
||||
}
|
||||
|
||||
func (d *Downloader) buildFetcher(url string) (fetcher.Fetcher, error) {
|
||||
fb, err := d.parseFb(url)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
fetcher := fb.Build()
|
||||
d.setupFetcher(fetcher)
|
||||
return fetcher, nil
|
||||
}
|
||||
|
||||
func initTask(task *Task) {
|
||||
task.timer = util.NewTimer(task.Progress.Used)
|
||||
task.lock = &sync.Mutex{}
|
||||
|
||||
@@ -27,7 +27,6 @@ func TestDownloader_Resolve(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := &base.Resource{
|
||||
Name: test.BuildName,
|
||||
Size: test.BuildSize,
|
||||
Range: true,
|
||||
Files: []*base.FileInfo{
|
||||
@@ -85,6 +84,51 @@ func TestDownloader_Create(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloader_CreateRename(t *testing.T) {
|
||||
listener := test.StartTestFileServer()
|
||||
defer listener.Close()
|
||||
|
||||
downloader := NewDownloader(nil)
|
||||
if err := downloader.Setup(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer downloader.Clear()
|
||||
|
||||
req := &base.Request{
|
||||
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
|
||||
}
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(2)
|
||||
downloader.Listener(func(event *Event) {
|
||||
if event.Key == EventKeyDone {
|
||||
wg.Done()
|
||||
}
|
||||
})
|
||||
for i := 0; i < 2; i++ {
|
||||
_, err := downloader.CreateDirect(req, &base.Options{
|
||||
Path: test.Dir,
|
||||
Name: test.DownloadName,
|
||||
Extra: http.OptsExtra{
|
||||
Connections: 4,
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
want := test.FileMd5(test.BuildFile)
|
||||
got := test.FileMd5(test.DownloadFile)
|
||||
if want != got {
|
||||
t.Errorf("Downloader_CreateRename() got = %v, want %v", got, want)
|
||||
}
|
||||
got = test.FileMd5(test.DownloadRenameFile)
|
||||
if want != got {
|
||||
t.Errorf("Downloader_CreateRename() got = %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloader_StoreAndRestore(t *testing.T) {
|
||||
listener := test.StartTestSlowFileServer(time.Millisecond * 2000)
|
||||
defer listener.Close()
|
||||
|
||||
@@ -0,0 +1,125 @@
|
||||
package engine
|
||||
|
||||
import (
|
||||
_ "embed"
|
||||
"errors"
|
||||
"fmt"
|
||||
"github.com/GopeedLab/gopeed/pkg/download/engine/inject/file"
|
||||
"github.com/GopeedLab/gopeed/pkg/download/engine/inject/formdata"
|
||||
"github.com/GopeedLab/gopeed/pkg/download/engine/inject/xhr"
|
||||
"github.com/dop251/goja"
|
||||
"github.com/dop251/goja_nodejs/eventloop"
|
||||
"github.com/dop251/goja_nodejs/process"
|
||||
"github.com/dop251/goja_nodejs/url"
|
||||
)
|
||||
|
||||
//go:embed polyfill/out/index.js
|
||||
var polyfillScript string
|
||||
|
||||
type Engine struct {
|
||||
loop *eventloop.EventLoop
|
||||
|
||||
Runtime *goja.Runtime
|
||||
}
|
||||
|
||||
// RunString executes the script and returns the go type value
|
||||
// if script result is promise, it will be resolved
|
||||
func (e *Engine) RunString(script string) (value any, err error) {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
err = fmt.Errorf("%v", r)
|
||||
}
|
||||
}()
|
||||
var result goja.Value
|
||||
e.loop.Run(func(runtime *goja.Runtime) {
|
||||
result, err = runtime.RunString(script)
|
||||
})
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
return resolveResult(result)
|
||||
}
|
||||
|
||||
// CallFunction calls the function and returns the go type value
|
||||
// if function result is promise, it will be resolved
|
||||
func (e *Engine) CallFunction(fn goja.Callable, args ...any) (value any, err error) {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
err = fmt.Errorf("%v", r)
|
||||
}
|
||||
}()
|
||||
var result goja.Value
|
||||
e.loop.Run(func(runtime *goja.Runtime) {
|
||||
if args == nil {
|
||||
result, err = fn(nil)
|
||||
} else {
|
||||
var jsArgs []goja.Value
|
||||
for _, arg := range args {
|
||||
jsArgs = append(jsArgs, runtime.ToValue(arg))
|
||||
}
|
||||
result, err = fn(nil, jsArgs...)
|
||||
}
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return resolveResult(result)
|
||||
}
|
||||
|
||||
func (e *Engine) Close() {
|
||||
e.loop.Stop()
|
||||
}
|
||||
|
||||
func NewEngine() *Engine {
|
||||
loop := eventloop.NewEventLoop()
|
||||
engine := &Engine{
|
||||
loop: loop,
|
||||
}
|
||||
loop.Run(func(runtime *goja.Runtime) {
|
||||
engine.Runtime = runtime
|
||||
runtime.SetFieldNameMapper(goja.TagFieldNameMapper("json", true))
|
||||
url.Enable(runtime)
|
||||
process.Enable(runtime)
|
||||
if err := file.Enable(runtime); err != nil {
|
||||
return
|
||||
}
|
||||
if err := formdata.Enable(runtime); err != nil {
|
||||
return
|
||||
}
|
||||
if err := xhr.Enable(runtime); err != nil {
|
||||
return
|
||||
}
|
||||
if _, err := runtime.RunString(polyfillScript); err != nil {
|
||||
return
|
||||
}
|
||||
// polyfill global
|
||||
if err := runtime.Set("global", runtime.GlobalObject()); err != nil {
|
||||
return
|
||||
}
|
||||
return
|
||||
})
|
||||
return engine
|
||||
}
|
||||
|
||||
func Run(script string) (value any, err error) {
|
||||
engine := NewEngine()
|
||||
return engine.RunString(script)
|
||||
}
|
||||
|
||||
// if the value is Promise, it will be resolved and return the result.
|
||||
func resolveResult(value goja.Value) (any, error) {
|
||||
export := value.Export()
|
||||
switch export.(type) {
|
||||
case *goja.Promise:
|
||||
p := export.(*goja.Promise)
|
||||
switch p.State() {
|
||||
case goja.PromiseStatePending:
|
||||
return nil, nil
|
||||
case goja.PromiseStateFulfilled:
|
||||
return p.Result().Export(), nil
|
||||
case goja.PromiseStateRejected:
|
||||
return nil, errors.New(p.Result().String())
|
||||
}
|
||||
}
|
||||
return export, nil
|
||||
}
|
||||
@@ -0,0 +1,291 @@
|
||||
package engine
|
||||
|
||||
import (
|
||||
"crypto/md5"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"github.com/GopeedLab/gopeed/internal/test"
|
||||
"github.com/GopeedLab/gopeed/pkg/download/engine/inject/file"
|
||||
"github.com/dop251/goja"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestPolyfill(t *testing.T) {
|
||||
doTestPolyfill(t, "XMLHttpRequest")
|
||||
doTestPolyfill(t, "Blob")
|
||||
doTestPolyfill(t, "FormData")
|
||||
doTestPolyfill(t, "fetch")
|
||||
}
|
||||
|
||||
func TestFetch(t *testing.T) {
|
||||
server := startServer()
|
||||
defer server.Close()
|
||||
engine := NewEngine()
|
||||
if _, err := engine.RunString(fmt.Sprintf("var host = 'http://%s';", server.Addr().String())); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err := engine.RunString(`
|
||||
async function testGet(){
|
||||
const resp = await fetch(host+'/get');
|
||||
return resp.status;
|
||||
}
|
||||
|
||||
async function testText(){
|
||||
const resp = await fetch(host+'/text',{
|
||||
method: 'POST',
|
||||
body: 'test'
|
||||
});
|
||||
return await resp.text();
|
||||
}
|
||||
|
||||
async function testOctetStream(file){
|
||||
const resp = await fetch(host+'/octetStream',{
|
||||
method: 'POST',
|
||||
body: file
|
||||
});
|
||||
return await resp.text();
|
||||
}
|
||||
|
||||
async function testFormData(file){
|
||||
const formData = new FormData();
|
||||
formData.append('name', 'test');
|
||||
formData.append('f', file);
|
||||
const resp = await fetch(host+'/formData',{
|
||||
method: 'POST',
|
||||
body: formData
|
||||
});
|
||||
return await resp.json();
|
||||
}
|
||||
|
||||
function testProgress(){
|
||||
return new Promise((resolve, reject) => {
|
||||
const xhr = new XMLHttpRequest();
|
||||
xhr.open('GET', host+'/get');
|
||||
const xhrUploadPromise = new Promise((resolve, reject) => {
|
||||
xhr.upload.onprogress = function(e){
|
||||
if(e.loaded === e.total){
|
||||
resolve();
|
||||
}
|
||||
}
|
||||
});
|
||||
const xhrPromise = new Promise((resolve, reject) => {
|
||||
xhr.onprogress = function(e){
|
||||
if(e.loaded === e.total){
|
||||
resolve();
|
||||
}
|
||||
}
|
||||
});
|
||||
Promise.all([xhrUploadPromise, xhrPromise]).then(() => {
|
||||
resolve();
|
||||
});
|
||||
xhr.send();
|
||||
setTimeout(() => {
|
||||
reject('timeout');
|
||||
}, 1000);
|
||||
});
|
||||
}
|
||||
|
||||
function testAbort(){
|
||||
return new Promise((resolve, reject) => {
|
||||
const xhr = new XMLHttpRequest();
|
||||
xhr.open('GET', host+'/timeout?duration=500');
|
||||
xhr.onabort = function() {
|
||||
resolve();
|
||||
};
|
||||
xhr.send();
|
||||
setTimeout(() => {
|
||||
xhr.abort();
|
||||
}, 200);
|
||||
setTimeout(() => {
|
||||
reject('timeout');
|
||||
}, 1000);
|
||||
});
|
||||
}
|
||||
|
||||
function testTimeout(){
|
||||
return new Promise((resolve, reject) => {
|
||||
const xhr = new XMLHttpRequest();
|
||||
const t = 500;
|
||||
xhr.open('GET', host+'/timeout?duration='+t);
|
||||
xhr.timeout = t - 200;
|
||||
xhr.onload = function() {
|
||||
resolve();
|
||||
};
|
||||
xhr.ontimeout = function() {
|
||||
reject('timeout');
|
||||
};
|
||||
xhr.send();
|
||||
});
|
||||
}
|
||||
`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
result, err := callTestFun(engine, "testGet")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if result != int64(200) {
|
||||
t.Fatalf("testGet failed, want %d, got %d", 200, result)
|
||||
}
|
||||
|
||||
result, err = callTestFun(engine, "testText")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if result != "test" {
|
||||
t.Fatalf("testText failed, want %s, got %s", "test", result)
|
||||
}
|
||||
|
||||
func() {
|
||||
jsFile, _, md5 := buildFile(t, engine.Runtime)
|
||||
result, err = callTestFun(engine, "testOctetStream", jsFile)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if result != md5 {
|
||||
t.Fatalf("testOctetStream failed, want %s, got %s", md5, result)
|
||||
}
|
||||
}()
|
||||
|
||||
func() {
|
||||
jsFile, goFile, md5 := buildFile(t, engine.Runtime)
|
||||
result, err = callTestFun(engine, "testFormData", jsFile)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := map[string]any{
|
||||
"name": "test",
|
||||
"f": map[string]string{
|
||||
"filename": goFile.Name,
|
||||
"md5": md5,
|
||||
},
|
||||
}
|
||||
if !test.JsonEqual(result, want) {
|
||||
t.Fatalf("testFormData failed, want %v, got %v", want, result)
|
||||
}
|
||||
}()
|
||||
|
||||
_, err = callTestFun(engine, "testProgress")
|
||||
if err != nil {
|
||||
t.Fatal("progress test failed")
|
||||
}
|
||||
|
||||
_, err = callTestFun(engine, "testAbort")
|
||||
if err != nil {
|
||||
t.Fatal("abort test failed")
|
||||
}
|
||||
|
||||
_, err = callTestFun(engine, "testTimeout")
|
||||
if err == nil || err.Error() != "timeout" {
|
||||
t.Fatalf("timeout test failed, want %s, got %s", "timeout", err)
|
||||
}
|
||||
}
|
||||
|
||||
func doTestPolyfill(t *testing.T, module string) {
|
||||
value, err := Run(fmt.Sprintf(`
|
||||
!!globalThis['%s']
|
||||
`, module))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !value.(bool) {
|
||||
t.Fatalf("module %s not polyfilled", module)
|
||||
}
|
||||
}
|
||||
|
||||
func startServer() net.Listener {
|
||||
listener, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
server := &http.Server{}
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/get", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
w.Write([]byte("ok"))
|
||||
})
|
||||
mux.HandleFunc("/text", func(w http.ResponseWriter, r *http.Request) {
|
||||
buf, _ := io.ReadAll(r.Body)
|
||||
w.WriteHeader(http.StatusOK)
|
||||
w.Write(buf)
|
||||
})
|
||||
mux.HandleFunc("/octetStream", func(w http.ResponseWriter, r *http.Request) {
|
||||
md5 := calcMd5(r.Body)
|
||||
w.WriteHeader(http.StatusOK)
|
||||
w.Write([]byte(md5))
|
||||
})
|
||||
mux.HandleFunc("/formData", func(w http.ResponseWriter, r *http.Request) {
|
||||
err := r.ParseMultipartForm(1024 * 1024 * 30)
|
||||
if err != nil {
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
w.Write([]byte(err.Error()))
|
||||
return
|
||||
}
|
||||
result := make(map[string]any)
|
||||
for k, v := range r.MultipartForm.Value {
|
||||
result[k] = v[0]
|
||||
}
|
||||
for k, v := range r.MultipartForm.File {
|
||||
f, _ := v[0].Open()
|
||||
result[k] = map[string]string{
|
||||
"filename": v[0].Filename,
|
||||
"md5": calcMd5(f),
|
||||
}
|
||||
}
|
||||
w.WriteHeader(http.StatusOK)
|
||||
buf, _ := json.Marshal(result)
|
||||
w.Write(buf)
|
||||
})
|
||||
mux.HandleFunc("/timeout", func(w http.ResponseWriter, r *http.Request) {
|
||||
duration := r.URL.Query().Get("duration")
|
||||
t, _ := strconv.Atoi(duration)
|
||||
time.Sleep(time.Duration(t) * time.Millisecond)
|
||||
w.WriteHeader(http.StatusOK)
|
||||
w.Write([]byte("ok"))
|
||||
})
|
||||
server.Handler = mux
|
||||
go server.Serve(listener)
|
||||
return listener
|
||||
}
|
||||
|
||||
func buildFile(t *testing.T, runtime *goja.Runtime) (goja.Value, *file.File, string) {
|
||||
jsFile, err := file.NewJsFile(runtime)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f := jsFile.Export().(*file.File)
|
||||
data := "test"
|
||||
f.Reader = strings.NewReader(data)
|
||||
f.Name = "test.txt"
|
||||
f.Size = int64(len(data))
|
||||
return jsFile, f, calcMd5(strings.NewReader(data))
|
||||
}
|
||||
|
||||
func callTestFun(engine *Engine, fun string, args ...any) (any, error) {
|
||||
test, ok := goja.AssertFunction(engine.Runtime.Get(fun))
|
||||
if !ok {
|
||||
return nil, errors.New("function not found:" + fun)
|
||||
}
|
||||
return engine.CallFunction(test, args...)
|
||||
}
|
||||
|
||||
func calcMd5(reader io.Reader) string {
|
||||
// Open a new hash interface to write to
|
||||
hash := md5.New()
|
||||
|
||||
// Copy the file in the hash interface and check for any error
|
||||
if _, err := io.Copy(hash, reader); err != nil {
|
||||
return ""
|
||||
}
|
||||
return hex.EncodeToString(hash.Sum(nil))
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
package file
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"github.com/dop251/goja"
|
||||
"io"
|
||||
)
|
||||
|
||||
type File struct {
|
||||
io.Reader `json:""`
|
||||
io.Closer `json:""`
|
||||
Name string `json:"name"`
|
||||
Size int64 `json:"size"`
|
||||
}
|
||||
|
||||
func NewJsFile(runtime *goja.Runtime) (goja.Value, error) {
|
||||
fileCtor, ok := goja.AssertConstructor(runtime.Get("File"))
|
||||
if !ok {
|
||||
return nil, errors.New("file is not defined")
|
||||
}
|
||||
return fileCtor(nil)
|
||||
}
|
||||
|
||||
func Enable(runtime *goja.Runtime) error {
|
||||
file := runtime.ToValue(func(call goja.ConstructorCall) *goja.Object {
|
||||
instance := &File{}
|
||||
instanceValue := runtime.ToValue(instance).(*goja.Object)
|
||||
instanceValue.SetPrototype(call.This.Prototype())
|
||||
return instanceValue
|
||||
})
|
||||
return runtime.Set("File", file)
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
package formdata
|
||||
|
||||
import "github.com/dop251/goja"
|
||||
|
||||
type FormData struct {
|
||||
data map[string]any
|
||||
}
|
||||
|
||||
func (fd *FormData) Append(name string, value any) {
|
||||
fd.data[name] = value
|
||||
}
|
||||
|
||||
func (fd *FormData) Delete(name string) {
|
||||
delete(fd.data, name)
|
||||
}
|
||||
|
||||
func (fd *FormData) Entries() []any {
|
||||
var entries []any
|
||||
for k, v := range fd.data {
|
||||
entries = append(entries, []any{k, v})
|
||||
}
|
||||
return entries
|
||||
}
|
||||
|
||||
func (fd *FormData) Get(name string) any {
|
||||
return fd.data[name]
|
||||
}
|
||||
|
||||
func (fd *FormData) GetAll(name string) []any {
|
||||
return []any{fd.data[name]}
|
||||
}
|
||||
|
||||
func (fd *FormData) Has(name string) bool {
|
||||
_, ok := fd.data[name]
|
||||
return ok
|
||||
}
|
||||
|
||||
func (fd *FormData) Keys() []string {
|
||||
var keys []string
|
||||
for k := range fd.data {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
return keys
|
||||
}
|
||||
|
||||
func (fd *FormData) Set(name string, value any) {
|
||||
fd.data[name] = value
|
||||
}
|
||||
|
||||
func (fd *FormData) Values() []any {
|
||||
var values []any
|
||||
for _, v := range fd.data {
|
||||
values = append(values, v)
|
||||
}
|
||||
return values
|
||||
}
|
||||
|
||||
func Enable(runtime *goja.Runtime) error {
|
||||
file := runtime.ToValue(func(call goja.ConstructorCall) *goja.Object {
|
||||
instance := &FormData{
|
||||
data: make(map[string]any),
|
||||
}
|
||||
instanceValue := runtime.ToValue(instance).(*goja.Object)
|
||||
instanceValue.SetPrototype(call.This.Prototype())
|
||||
return instanceValue
|
||||
})
|
||||
return runtime.Set("FormData", file)
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
package inject
|
||||
|
||||
import (
|
||||
"github.com/dop251/goja"
|
||||
)
|
||||
|
||||
func ThrowTypeError(vm *goja.Runtime, msg string) {
|
||||
panic(vm.NewTypeError(msg))
|
||||
}
|
||||
@@ -0,0 +1,417 @@
|
||||
package xhr
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"github.com/GopeedLab/gopeed/pkg/download/engine/inject"
|
||||
"github.com/GopeedLab/gopeed/pkg/download/engine/inject/file"
|
||||
"github.com/GopeedLab/gopeed/pkg/download/engine/inject/formdata"
|
||||
"github.com/dop251/goja"
|
||||
"io"
|
||||
"mime/multipart"
|
||||
"net"
|
||||
"net/http"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
eventLoad = "load"
|
||||
eventReadystatechange = "readystatechange"
|
||||
eventProgress = "progress"
|
||||
eventAbort = "abort"
|
||||
eventError = "error"
|
||||
eventTimeout = "timeout"
|
||||
)
|
||||
|
||||
type ProgressEvent struct {
|
||||
Type string `json:"type"`
|
||||
LengthComputable bool `json:"lengthComputable"`
|
||||
Loaded int64 `json:"loaded"`
|
||||
Total int64 `json:"total"`
|
||||
}
|
||||
|
||||
type EventProp struct {
|
||||
eventListeners map[string]func(event *ProgressEvent)
|
||||
Onload func(event *ProgressEvent) `json:"onload"`
|
||||
Onprogress func(event *ProgressEvent) `json:"onprogress"`
|
||||
Onabort func(event *ProgressEvent) `json:"onabort"`
|
||||
Onerror func(event *ProgressEvent) `json:"onerror"`
|
||||
Ontimeout func(event *ProgressEvent) `json:"ontimeout"`
|
||||
}
|
||||
|
||||
func (ep *EventProp) AddEventListener(event string, cb func(event *ProgressEvent)) {
|
||||
ep.eventListeners[event] = cb
|
||||
}
|
||||
|
||||
func (ep *EventProp) RemoveEventListener(event string) {
|
||||
delete(ep.eventListeners, event)
|
||||
}
|
||||
|
||||
func (ep *EventProp) callOnload() {
|
||||
event := &ProgressEvent{
|
||||
Type: eventLoad,
|
||||
LengthComputable: false,
|
||||
}
|
||||
if ep.Onload != nil {
|
||||
ep.Onload(event)
|
||||
}
|
||||
ep.callEventListener(event)
|
||||
}
|
||||
|
||||
func (ep *EventProp) callOnprogress(loaded, total int64) {
|
||||
event := &ProgressEvent{
|
||||
Type: eventProgress,
|
||||
LengthComputable: true,
|
||||
Loaded: loaded,
|
||||
Total: total,
|
||||
}
|
||||
if ep.Onprogress != nil {
|
||||
ep.Onprogress(event)
|
||||
}
|
||||
ep.callEventListener(event)
|
||||
}
|
||||
|
||||
func (ep *EventProp) callOnabort() {
|
||||
event := &ProgressEvent{
|
||||
Type: eventAbort,
|
||||
LengthComputable: false,
|
||||
}
|
||||
if ep.Onabort != nil {
|
||||
ep.Onabort(event)
|
||||
}
|
||||
ep.callEventListener(event)
|
||||
}
|
||||
|
||||
func (ep *EventProp) callOnerror() {
|
||||
event := &ProgressEvent{
|
||||
Type: eventError,
|
||||
LengthComputable: false,
|
||||
}
|
||||
if ep.Onerror != nil {
|
||||
ep.Onerror(event)
|
||||
}
|
||||
ep.callEventListener(event)
|
||||
}
|
||||
|
||||
func (ep *EventProp) callOntimeout() {
|
||||
event := &ProgressEvent{
|
||||
Type: eventTimeout,
|
||||
LengthComputable: false,
|
||||
}
|
||||
if ep.Ontimeout != nil {
|
||||
ep.Ontimeout(event)
|
||||
}
|
||||
ep.callEventListener(event)
|
||||
}
|
||||
|
||||
func (ep *EventProp) callEventListener(event *ProgressEvent) {
|
||||
if cb, ok := ep.eventListeners[event.Type]; ok {
|
||||
cb(event)
|
||||
}
|
||||
}
|
||||
|
||||
type XMLHttpRequestUpload struct {
|
||||
*EventProp
|
||||
}
|
||||
|
||||
type XMLHttpRequest struct {
|
||||
method string
|
||||
url string
|
||||
requestHeaders map[string]string
|
||||
responseHeaders map[string]string
|
||||
aborted bool
|
||||
|
||||
Upload *XMLHttpRequestUpload `json:"upload"`
|
||||
Timeout int `json:"timeout"`
|
||||
ReadyState int `json:"readyState"`
|
||||
Status int `json:"status"`
|
||||
StatusText string `json:"statusText"`
|
||||
Response string `json:"response"`
|
||||
ResponseText string `json:"responseText"`
|
||||
*EventProp
|
||||
Onreadystatechange func(event *ProgressEvent) `json:"onreadystatechange"`
|
||||
}
|
||||
|
||||
func (xhr *XMLHttpRequest) Open(method, url string) {
|
||||
xhr.method = method
|
||||
xhr.url = url
|
||||
xhr.requestHeaders = make(map[string]string)
|
||||
xhr.responseHeaders = make(map[string]string)
|
||||
xhr.doReadystatechange(1)
|
||||
}
|
||||
|
||||
func (xhr *XMLHttpRequest) SetRequestHeader(key, value string) {
|
||||
xhr.requestHeaders[key] = value
|
||||
}
|
||||
|
||||
func (xhr *XMLHttpRequest) Send(data goja.Value) {
|
||||
var req *http.Request
|
||||
var err error
|
||||
d := xhr.parseData(data)
|
||||
var (
|
||||
contentType string
|
||||
contentLength int64
|
||||
)
|
||||
if d == nil || xhr.method == "GET" || xhr.method == "HEAD" {
|
||||
req, err = http.NewRequest(xhr.method, xhr.url, nil)
|
||||
} else {
|
||||
switch d.(type) {
|
||||
case string:
|
||||
req, err = http.NewRequest(xhr.method, xhr.url, bytes.NewBufferString(d.(string)))
|
||||
contentType = "text/plain;charset=UTF-8"
|
||||
contentLength = int64(len(d.(string)))
|
||||
case *file.File:
|
||||
req, err = http.NewRequest(xhr.method, xhr.url, d.(*file.File).Reader)
|
||||
contentType = "application/octet-stream"
|
||||
contentLength = d.(*file.File).Size
|
||||
case *formdata.FormData:
|
||||
pr, pw := io.Pipe()
|
||||
mw := NewMultipart(pw)
|
||||
for _, e := range d.(*formdata.FormData).Entries() {
|
||||
arr := e.([]any)
|
||||
k := arr[0].(string)
|
||||
v := arr[1]
|
||||
switch v.(type) {
|
||||
case string:
|
||||
mw.WriteField(k, v.(string))
|
||||
case *file.File:
|
||||
mw.WriteFile(k, v.(*file.File))
|
||||
}
|
||||
}
|
||||
go func() {
|
||||
defer pw.Close()
|
||||
defer mw.Close()
|
||||
mw.Send()
|
||||
}()
|
||||
req, err = http.NewRequest(xhr.method, xhr.url, pr)
|
||||
contentType = mw.FormDataContentType()
|
||||
contentLength = mw.Size()
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
xhr.callOnerror()
|
||||
return
|
||||
}
|
||||
if contentType != "" {
|
||||
req.Header.Set("Content-Type", contentType)
|
||||
}
|
||||
if contentLength > 0 {
|
||||
req.ContentLength = contentLength
|
||||
}
|
||||
for k, v := range xhr.requestHeaders {
|
||||
req.Header.Set(k, v)
|
||||
}
|
||||
client := &http.Client{
|
||||
Timeout: time.Duration(xhr.Timeout) * time.Millisecond,
|
||||
}
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
// handle timeout error
|
||||
if err, ok := err.(net.Error); ok && err.Timeout() {
|
||||
if xhr.Timeout > 0 {
|
||||
xhr.Upload.callOntimeout()
|
||||
xhr.callOntimeout()
|
||||
}
|
||||
return
|
||||
}
|
||||
xhr.Upload.callOnerror()
|
||||
xhr.callOnerror()
|
||||
return
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
xhr.Upload.callOnprogress(contentLength, contentLength)
|
||||
if !xhr.aborted {
|
||||
xhr.Upload.callOnload()
|
||||
}
|
||||
for k, v := range resp.Header {
|
||||
xhr.responseHeaders[k] = v[0]
|
||||
}
|
||||
xhr.Status = resp.StatusCode
|
||||
xhr.StatusText = resp.Status
|
||||
xhr.doReadystatechange(2)
|
||||
buf, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
xhr.callOnerror()
|
||||
return
|
||||
}
|
||||
xhr.doReadystatechange(3)
|
||||
xhr.Response = string(buf)
|
||||
xhr.ResponseText = xhr.Response
|
||||
xhr.doReadystatechange(4)
|
||||
respBodyLen := int64(len(buf))
|
||||
xhr.callOnprogress(respBodyLen, respBodyLen)
|
||||
if !xhr.aborted {
|
||||
xhr.callOnload()
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (xhr *XMLHttpRequest) Abort() {
|
||||
xhr.doReadystatechange(0)
|
||||
xhr.aborted = true
|
||||
xhr.Upload.callOnabort()
|
||||
xhr.callOnabort()
|
||||
}
|
||||
|
||||
func (xhr *XMLHttpRequest) GetResponseHeader(key string) string {
|
||||
return xhr.responseHeaders[key]
|
||||
}
|
||||
|
||||
func (xhr *XMLHttpRequest) GetAllResponseHeaders() string {
|
||||
var buf bytes.Buffer
|
||||
for k, v := range xhr.responseHeaders {
|
||||
buf.WriteString(k)
|
||||
buf.WriteString(": ")
|
||||
buf.WriteString(v)
|
||||
buf.WriteString("\r\n")
|
||||
}
|
||||
return buf.String()
|
||||
}
|
||||
|
||||
func (xhr *XMLHttpRequest) callOnreadystatechange() {
|
||||
event := &ProgressEvent{
|
||||
Type: eventReadystatechange,
|
||||
LengthComputable: false,
|
||||
}
|
||||
if xhr.Onreadystatechange != nil {
|
||||
xhr.Onreadystatechange(event)
|
||||
}
|
||||
xhr.callEventListener(event)
|
||||
}
|
||||
|
||||
func (xhr *XMLHttpRequest) doReadystatechange(state int) {
|
||||
if xhr.aborted {
|
||||
return
|
||||
}
|
||||
xhr.ReadyState = state
|
||||
xhr.callOnreadystatechange()
|
||||
}
|
||||
|
||||
// parse js data to go struct
|
||||
func (xhr *XMLHttpRequest) parseData(data goja.Value) any {
|
||||
// check if data is null or undefined
|
||||
if data == nil || goja.IsNull(data) || goja.IsUndefined(data) || goja.IsNaN(data) {
|
||||
return nil
|
||||
}
|
||||
// check if data is File
|
||||
f, ok := data.Export().(*file.File)
|
||||
if ok {
|
||||
return f
|
||||
}
|
||||
// check if data is FormData
|
||||
fd, ok := data.Export().(*formdata.FormData)
|
||||
if ok {
|
||||
return fd
|
||||
}
|
||||
// otherwise, return data as string
|
||||
return data.String()
|
||||
}
|
||||
|
||||
func Enable(runtime *goja.Runtime) error {
|
||||
progressEvent := runtime.ToValue(func(call goja.ConstructorCall) *goja.Object {
|
||||
if len(call.Arguments) < 1 {
|
||||
inject.ThrowTypeError(runtime, "Failed to construct 'ProgressEvent': 1 argument required, but only 0 present.")
|
||||
}
|
||||
instance := &ProgressEvent{
|
||||
Type: call.Argument(0).String(),
|
||||
}
|
||||
instanceValue := runtime.ToValue(instance).(*goja.Object)
|
||||
instanceValue.SetPrototype(call.This.Prototype())
|
||||
return instanceValue
|
||||
})
|
||||
xhr := runtime.ToValue(func(call goja.ConstructorCall) *goja.Object {
|
||||
instance := &XMLHttpRequest{
|
||||
Upload: &XMLHttpRequestUpload{
|
||||
EventProp: &EventProp{
|
||||
eventListeners: make(map[string]func(event *ProgressEvent)),
|
||||
},
|
||||
},
|
||||
EventProp: &EventProp{
|
||||
eventListeners: make(map[string]func(event *ProgressEvent)),
|
||||
},
|
||||
}
|
||||
instanceValue := runtime.ToValue(instance).(*goja.Object)
|
||||
instanceValue.SetPrototype(call.This.Prototype())
|
||||
return instanceValue
|
||||
})
|
||||
if err := runtime.Set("ProgressEvent", progressEvent); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := runtime.Set("XMLHttpRequest", xhr); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Wrap multipart.Writer and stat content length
|
||||
type multipartWrapper struct {
|
||||
statBuffer *bytes.Buffer
|
||||
statWriter *multipart.Writer
|
||||
writer *multipart.Writer
|
||||
fields map[string]any
|
||||
}
|
||||
|
||||
func NewMultipart(w io.Writer) *multipartWrapper {
|
||||
var buf bytes.Buffer
|
||||
return &multipartWrapper{
|
||||
statBuffer: &buf,
|
||||
statWriter: multipart.NewWriter(&buf),
|
||||
writer: multipart.NewWriter(w),
|
||||
fields: make(map[string]any),
|
||||
}
|
||||
}
|
||||
|
||||
func (w *multipartWrapper) WriteField(fieldname string, value string) error {
|
||||
w.fields[fieldname] = value
|
||||
return w.statWriter.WriteField(fieldname, value)
|
||||
}
|
||||
|
||||
func (w *multipartWrapper) WriteFile(fieldname string, file *file.File) error {
|
||||
w.fields[fieldname] = file
|
||||
_, err := w.statWriter.CreateFormFile(fieldname, file.Name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w *multipartWrapper) Size() int64 {
|
||||
w.statWriter.Close()
|
||||
size := int64(w.statBuffer.Len())
|
||||
for _, v := range w.fields {
|
||||
switch v.(type) {
|
||||
case *file.File:
|
||||
f := v.(*file.File)
|
||||
size += f.Size
|
||||
}
|
||||
}
|
||||
return size
|
||||
}
|
||||
|
||||
func (w *multipartWrapper) Send() error {
|
||||
for k, v := range w.fields {
|
||||
switch v.(type) {
|
||||
case string:
|
||||
if err := w.writer.WriteField(k, v.(string)); err != nil {
|
||||
return err
|
||||
}
|
||||
case *file.File:
|
||||
f := v.(*file.File)
|
||||
fw, err := w.writer.CreateFormFile(k, f.Name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err = io.Copy(fw, f); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w *multipartWrapper) FormDataContentType() string {
|
||||
return w.writer.FormDataContentType()
|
||||
}
|
||||
|
||||
func (w *multipartWrapper) Close() error {
|
||||
return w.writer.Close()
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
(()=>{var t={725:function(t,e,r){var o,n,i;i="undefined"!=typeof self&&self||"undefined"!=typeof window&&window||void 0!==r.g&&r.g||this,o=function(t){"use strict";var e=i.BlobBuilder||i.WebKitBlobBuilder||i.MSBlobBuilder||i.MozBlobBuilder,r=i.URL||i.webkitURL||function(t,e){return(e=document.createElement("a")).href=t,e},o=i.Blob,n=r.createObjectURL,a=r.revokeObjectURL,s=i.Symbol&&i.Symbol.toStringTag,u=!1,f=!1,l=e&&e.prototype.append&&e.prototype.getBlob;try{u=2===new Blob(["ä"]).size,f=2===new Blob([new Uint8Array([1,2])]).size}catch(t){}function c(t){return t.map((function(t){if(t.buffer instanceof ArrayBuffer){var e=t.buffer;if(t.byteLength!==e.byteLength){var r=new Uint8Array(t.byteLength);r.set(new Uint8Array(e,t.byteOffset,t.byteLength)),e=r.buffer}return e}return t}))}function h(t,r){r=r||{};var o=new e;return c(t).forEach((function(t){o.append(t)})),r.type?o.getBlob(r.type):o.getBlob()}function p(t,e){return new o(c(t),e||{})}i.Blob&&(h.prototype=Blob.prototype,p.prototype=Blob.prototype);var y="function"==typeof TextEncoder?TextEncoder.prototype.encode.bind(new TextEncoder):function(t){for(var e=0,r=t.length,o=i.Uint8Array||Array,n=0,a=Math.max(32,r+(r>>1)+7),s=new o(a>>3<<3);e<r;){var u=t.charCodeAt(e++);if(u>=55296&&u<=56319){if(e<r){var f=t.charCodeAt(e);56320==(64512&f)&&(++e,u=((1023&u)<<10)+(1023&f)+65536)}if(u>=55296&&u<=56319)continue}if(n+4>s.length){a+=8,a=(a*=1+e/t.length*2)>>3<<3;var l=new Uint8Array(a);l.set(s),s=l}if(0!=(4294967168&u)){if(0==(4294965248&u))s[n++]=u>>6&31|192;else if(0==(4294901760&u))s[n++]=u>>12&15|224,s[n++]=u>>6&63|128;else{if(0!=(4292870144&u))continue;s[n++]=u>>18&7|240,s[n++]=u>>12&63|128,s[n++]=u>>6&63|128}s[n++]=63&u|128}else s[n++]=u}return s.slice(0,n)},d="function"==typeof TextDecoder?TextDecoder.prototype.decode.bind(new TextDecoder):function(t){for(var e=t.length,r=[],o=0;o<e;){var n,i,a,s,u=t[o],f=null,l=u>239?4:u>223?3:u>191?2:1;if(o+l<=e)switch(l){case 1:u<128&&(f=u);break;case 2:128==(192&(n=t[o+1]))&&(s=(31&u)<<6|63&n)>127&&(f=s);break;case 3:n=t[o+1],i=t[o+2],128==(192&n)&&128==(192&i)&&(s=(15&u)<<12|(63&n)<<6|63&i)>2047&&(s<55296||s>57343)&&(f=s);break;case 4:n=t[o+1],i=t[o+2],a=t[o+3],128==(192&n)&&128==(192&i)&&128==(192&a)&&(s=(15&u)<<18|(63&n)<<12|(63&i)<<6|63&a)>65535&&s<1114112&&(f=s)}null===f?(f=65533,l=1):f>65535&&(f-=65536,r.push(f>>>10&1023|55296),f=56320|1023&f),r.push(f),o+=l}for(var c=r.length,h="",p=0;p<c;)h+=String.fromCharCode.apply(String,r.slice(p,p+=4096));return h};function b(){var e=!!i.ActiveXObject||"-ms-scroll-limit"in document.documentElement.style&&"-ms-ime-align"in document.documentElement.style,r=i.XMLHttpRequest&&i.XMLHttpRequest.prototype.send;e&&r&&(XMLHttpRequest.prototype.send=function(t){t instanceof Blob?(this.setRequestHeader("Content-Type",t.type),r.call(this,t)):r.call(this,t)});try{new File([],""),t.File=i.File,t.FileReader=i.FileReader}catch(e){try{t.File=new Function('class File extends Blob {constructor(chunks, name, opts) {opts = opts || {};super(chunks, opts || {});this.name = name.replace(/\\//g, ":");this.lastModifiedDate = opts.lastModified ? new Date(opts.lastModified) : new Date();this.lastModified = +this.lastModifiedDate;}};return new File([], ""), File')()}catch(e){t.File=function(t,e,r){var o=new Blob(t,r),n=r&&void 0!==r.lastModified?new Date(r.lastModified):new Date;return o.name=e.replace(/\//g,":"),o.lastModifiedDate=n,o.lastModified=+n,o.toString=function(){return"[object File]"},s&&(o[s]="File"),o}}}}u?(b(),t.Blob=f?i.Blob:p):l?(b(),t.Blob=h):function(){function e(t){for(var e=new Array(t.byteLength),r=new Uint8Array(t),o=e.length;o--;)e[o]=r[o];return e}function o(t){for(var e="ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=",r=[],o=0;o<t.length;o+=3){var n=t[o],i=o+1<t.length,a=i?t[o+1]:0,s=o+2<t.length,u=s?t[o+2]:0,f=n>>2,l=(3&n)<<4|a>>4,c=(15&a)<<2|u>>6,h=63&u;s||(h=64,i||(c=64)),r.push(e[f],e[l],e[c],e[h])}return r.join("")}var s=Object.create||function(t){function e(){}return e.prototype=t,new e};function u(t){return Object.prototype.toString.call(t).slice(8,-1)}function f(t,e){return"object"==typeof t&&Object.prototype.isPrototypeOf.call(t.prototype,e)}var l=["Int8Array","Uint8Array","Uint8ClampedArray","Int16Array","Uint16Array","Int32Array","Uint32Array","Float32Array","Float64Array","ArrayBuffer"];function c(t){return e=l,r=u(t),-1!==e.indexOf(r)||f(i.ArrayBuffer,t);var e,r}function h(t,r){r=null==r?{}:r;for(var o=0,n=(t=t?t.slice():[]).length;o<n;o++){var a=t[o];a instanceof h?t[o]=a._buffer:"string"==typeof a?t[o]=y(a):"DataView"===u(s=a)||f(i.DataView,s)?t[o]=e(a.buffer):c(a)?t[o]=e(a):t[o]=y(String(a))}var s;this._buffer=i.Uint8Array?function(t){for(var e=0,r=t.length;r--;)e+=t[r].length;for(var o=new Uint8Array(e),n=0,i=0;i<t.length;i++){var a=t[i];o.set(a,n),n+=a.byteLength||a.length}return o}(t):[].concat.apply([],t),this.size=this._buffer.length,this.type=r.type||"",/[^\u0020-\u007E]/.test(this.type)?this.type="":this.type=this.type.toLowerCase()}funcLine truncated
|
||||
+2236
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,22 @@
|
||||
{
|
||||
"name": "polyfill",
|
||||
"version": "1.0.0",
|
||||
"description": "",
|
||||
"main": "index.js",
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"test": "echo \"Error: no test specified\" && exit 1",
|
||||
"dev": "webpack --mode production --watch",
|
||||
"build": "webpack --mode production"
|
||||
},
|
||||
"author": "",
|
||||
"license": "ISC",
|
||||
"devDependencies": {
|
||||
"webpack": "^5.75.0",
|
||||
"webpack-cli": "^5.0.1"
|
||||
},
|
||||
"dependencies": {
|
||||
"blob-polyfill": "^7.0.20220408",
|
||||
"whatwg-fetch": "^3.6.2"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
import { Blob } from "blob-polyfill";
|
||||
|
||||
globalThis.Blob = Blob;
|
||||
@@ -0,0 +1,13 @@
|
||||
globalThis.crypto = {
|
||||
getRandomValues(arr) {
|
||||
for (let i = 0, len = arr.length; i < len; i++) {
|
||||
arr[i] = Math.floor(Math.random() * 256);
|
||||
}
|
||||
return arr;
|
||||
},
|
||||
randomUUID() {
|
||||
return '10000000-1000-4000-8000-100000000000'.replace(/[018]/g, (c) => {
|
||||
return (c ^ (this.getRandomValues(new Uint8Array(1))[0] & (15 >> (c / 4)))).toString(16)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
import 'whatwg-fetch'
|
||||
@@ -0,0 +1,3 @@
|
||||
import "./blob/index.js"
|
||||
import "./crypto/index.js"
|
||||
import "./fetch/index.js"
|
||||
@@ -0,0 +1,12 @@
|
||||
import path from "path";
|
||||
import { fileURLToPath } from "url";
|
||||
|
||||
const __dirname = fileURLToPath(import.meta.url);
|
||||
|
||||
export default {
|
||||
entry: "./src/index.js",
|
||||
output: {
|
||||
filename: "index.js",
|
||||
path: path.resolve(__dirname, "../out"),
|
||||
},
|
||||
};
|
||||
@@ -5,7 +5,6 @@ type EventKey string
|
||||
const (
|
||||
EventKeyStart = "start"
|
||||
EventKeyPause = "pause"
|
||||
EventKeyContinue = "continue"
|
||||
EventKeyProgress = "progress"
|
||||
EventKeyError = "error"
|
||||
EventKeyDelete = "delete"
|
||||
|
||||
@@ -0,0 +1,549 @@
|
||||
package download
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"github.com/GopeedLab/gopeed/internal/logger"
|
||||
"github.com/GopeedLab/gopeed/pkg/base"
|
||||
"github.com/GopeedLab/gopeed/pkg/download/engine"
|
||||
"github.com/GopeedLab/gopeed/pkg/util"
|
||||
"github.com/dop251/goja"
|
||||
"github.com/go-git/go-git/v5"
|
||||
"io"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
var (
|
||||
gitSuffix = ".git"
|
||||
|
||||
tempExtensionsDir = ".extensions"
|
||||
extensionsDir = "extensions"
|
||||
extensionIgnoreDirs = []string{gitSuffix, "node_modules"}
|
||||
|
||||
ErrExtensionNoManifest = fmt.Errorf("manifest.json not found")
|
||||
ErrExtensionNotFound = fmt.Errorf("extension not found")
|
||||
)
|
||||
|
||||
type ActivationEvent string
|
||||
|
||||
const (
|
||||
EventOnResolve ActivationEvent = "onResolve"
|
||||
//EventOnError ActivationEvent = "onError"
|
||||
//EventOnDone ActivationEvent = "onDone"
|
||||
)
|
||||
|
||||
func (d *Downloader) InstallExtensionByGit(url string) (*Extension, error) {
|
||||
return d.fetchExtensionByGit(url, d.InstallExtensionByFolder)
|
||||
}
|
||||
|
||||
func (d *Downloader) InstallExtensionByFolder(path string, devMode bool) (*Extension, error) {
|
||||
ext, err := d.parseExtensionByPath(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// if dev mode, don't copy the extension to the extensions' directory
|
||||
if devMode {
|
||||
ext.DevMode = true
|
||||
ext.DevPath, _ = filepath.Abs(path)
|
||||
} else {
|
||||
if err = util.CopyDir(path, d.ExtensionPath(ext), extensionIgnoreDirs...); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
// if extension is not installed, add it to the list, otherwise update it
|
||||
installedExt := d.getExtension(ext.Identity)
|
||||
if installedExt == nil {
|
||||
ext.CreatedAt = time.Now()
|
||||
ext.UpdatedAt = ext.CreatedAt
|
||||
installedExt = ext
|
||||
d.extensions = append(d.extensions, installedExt)
|
||||
} else {
|
||||
installedExt.update(ext)
|
||||
}
|
||||
if err = d.storage.Put(bucketExtension, installedExt.Identity, installedExt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return installedExt, nil
|
||||
}
|
||||
|
||||
// UpgradeCheckExtension Check if there is a new version for the extension.
|
||||
func (d *Downloader) UpgradeCheckExtension(identity string) (newVersion string, err error) {
|
||||
ext, err := d.GetExtension(identity)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
installUrl := ext.buildInstallUrl()
|
||||
if installUrl == "" {
|
||||
return
|
||||
}
|
||||
_, err = d.fetchExtensionByGit(installUrl, func(tempExtPath string, devMode bool) (*Extension, error) {
|
||||
tempExt, err := d.parseExtensionByPath(tempExtPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if tempExt.Version != ext.Version {
|
||||
newVersion = tempExt.Version
|
||||
}
|
||||
return tempExt, nil
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
func (d *Downloader) UpgradeExtension(identity string) error {
|
||||
ext, err := d.GetExtension(identity)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
installUrl := ext.buildInstallUrl()
|
||||
if installUrl == "" {
|
||||
return nil
|
||||
}
|
||||
if _, err := d.InstallExtensionByGit(installUrl); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *Downloader) UpdateExtensionSettings(identity string, settings map[string]any) error {
|
||||
ext, err := d.GetExtension(identity)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, setting := range ext.Settings {
|
||||
if value, ok := settings[setting.Name]; ok {
|
||||
setting.Value = tryParse(value, setting.Type)
|
||||
}
|
||||
}
|
||||
return d.storage.Put(bucketExtension, ext.Identity, ext)
|
||||
}
|
||||
|
||||
func (d *Downloader) SwitchExtension(identity string, status bool) error {
|
||||
ext, err := d.GetExtension(identity)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ext.Disabled = !status
|
||||
return d.storage.Put(bucketExtension, ext.Identity, ext)
|
||||
}
|
||||
|
||||
func (d *Downloader) DeleteExtension(identity string) error {
|
||||
ext, err := d.GetExtension(identity)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// remove from disk
|
||||
if !ext.DevMode {
|
||||
if err := os.RemoveAll(d.ExtensionPath(ext)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
// remove from extensions
|
||||
for i, e := range d.extensions {
|
||||
if e.Identity == identity {
|
||||
d.extensions = append(d.extensions[:i], d.extensions[i+1:]...)
|
||||
break
|
||||
}
|
||||
}
|
||||
return d.storage.Delete(bucketExtension, identity)
|
||||
}
|
||||
|
||||
func (d *Downloader) GetExtensions() []*Extension {
|
||||
return d.extensions
|
||||
}
|
||||
|
||||
func (d *Downloader) GetExtension(identity string) (*Extension, error) {
|
||||
extension := d.getExtension(identity)
|
||||
if extension == nil {
|
||||
return nil, ErrExtensionNotFound
|
||||
}
|
||||
return extension, nil
|
||||
}
|
||||
|
||||
func (d *Downloader) getExtension(identity string) *Extension {
|
||||
for _, ext := range d.extensions {
|
||||
if ext.Identity == identity {
|
||||
return ext
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *Downloader) fetchExtensionByGit(url string, handler func(tempExtPath string, devMode bool) (*Extension, error)) (*Extension, error) {
|
||||
if !strings.HasPrefix(url, "https://") && !strings.HasPrefix(url, "http://") {
|
||||
url = "https://" + url
|
||||
}
|
||||
|
||||
// resolve project path
|
||||
parentPath, projectPath := path.Split(url)
|
||||
// resolve project name and sub path
|
||||
pathArr := strings.SplitN(projectPath, "#", 2)
|
||||
projectPath = strings.TrimSuffix(pathArr[0], gitSuffix)
|
||||
subPath := ""
|
||||
if len(pathArr) > 1 {
|
||||
subPath = pathArr[1]
|
||||
}
|
||||
|
||||
tempExtDir := filepath.Join(d.cfg.StorageDir, tempExtensionsDir, projectPath)
|
||||
if err := util.RmAndMkDirAll(tempExtDir); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// clone project to extension temp dir
|
||||
gitUrl := parentPath + projectPath + gitSuffix
|
||||
if _, err := git.PlainClone(tempExtDir, false, &git.CloneOptions{
|
||||
URL: gitUrl,
|
||||
Depth: 1,
|
||||
}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer os.RemoveAll(tempExtDir)
|
||||
|
||||
return handler(filepath.Join(tempExtDir, subPath), false)
|
||||
}
|
||||
|
||||
func (d *Downloader) parseExtensionByPath(path string) (*Extension, error) {
|
||||
// resolve extension manifest
|
||||
manifestTempPath := filepath.Join(path, "manifest.json")
|
||||
if _, err := os.Stat(manifestTempPath); os.IsNotExist(err) {
|
||||
return nil, ErrExtensionNoManifest
|
||||
}
|
||||
file, err := os.ReadFile(manifestTempPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var ext Extension
|
||||
if err = json.Unmarshal(file, &ext); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err = ext.validate(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ext.Identity = ext.buildIdentity()
|
||||
return &ext, nil
|
||||
}
|
||||
|
||||
func (d *Downloader) triggerOnResolve(req *base.Request) (res *base.Resource) {
|
||||
// init extension global object
|
||||
gopeed := &Instance{
|
||||
Events: make(InstanceEvents),
|
||||
}
|
||||
ctx := &Context{
|
||||
Req: req,
|
||||
}
|
||||
var err error
|
||||
for _, ext := range d.extensions {
|
||||
if ext.Disabled {
|
||||
continue
|
||||
}
|
||||
for _, script := range ext.Scripts {
|
||||
if script.match(EventOnResolve, req.URL) {
|
||||
gopeed.Logger = newInstanceLogger(ext, d.ExtensionLogger)
|
||||
scriptFilePath := filepath.Join(d.ExtensionPath(ext), script.Entry)
|
||||
if _, err = os.Stat(scriptFilePath); os.IsNotExist(err) {
|
||||
continue
|
||||
}
|
||||
func() {
|
||||
var scriptFile *os.File
|
||||
scriptFile, err = os.Open(scriptFilePath)
|
||||
if err != nil {
|
||||
gopeed.Logger.logger.Error().Err(err).Msgf("[%s] open script file failed", ext.buildIdentity())
|
||||
return
|
||||
}
|
||||
defer scriptFile.Close()
|
||||
ctx.Settings = parseSettings(ext.Settings)
|
||||
var scriptBuf []byte
|
||||
scriptBuf, err = io.ReadAll(scriptFile)
|
||||
if err != nil {
|
||||
gopeed.Logger.logger.Error().Err(err).Msgf("[%s] read script file failed", ext.buildIdentity())
|
||||
return
|
||||
}
|
||||
engine := engine.NewEngine()
|
||||
defer engine.Close()
|
||||
err = engine.Runtime.Set("gopeed", gopeed)
|
||||
if err != nil {
|
||||
gopeed.Logger.logger.Error().Err(err).Msgf("[%s] engine inject failed", ext.buildIdentity())
|
||||
return
|
||||
}
|
||||
_, err = engine.RunString(string(scriptBuf))
|
||||
if err != nil {
|
||||
gopeed.Logger.logger.Error().Err(err).Msgf("[%s] run script failed", ext.buildIdentity())
|
||||
return
|
||||
}
|
||||
if fn, ok := gopeed.Events[EventOnResolve]; ok {
|
||||
_, err = engine.CallFunction(fn, ctx)
|
||||
if err != nil {
|
||||
gopeed.Logger.logger.Error().Err(err).Msgf("[%s] call function failed: %s", EventOnResolve, ext.buildIdentity())
|
||||
return
|
||||
}
|
||||
// calculate resource total size
|
||||
if ctx.Res != nil && len(ctx.Res.Files) > 0 {
|
||||
if err = ctx.Res.Validate(); err != nil {
|
||||
gopeed.Logger.logger.Warn().Err(err).Msgf("[%s] resource invalid", ext.buildIdentity())
|
||||
return
|
||||
}
|
||||
ctx.Res.CalcSize()
|
||||
}
|
||||
res = ctx.Res
|
||||
}
|
||||
}()
|
||||
}
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (d *Downloader) ExtensionPath(ext *Extension) string {
|
||||
if ext.DevMode {
|
||||
return ext.DevPath
|
||||
}
|
||||
return filepath.Join(d.cfg.StorageDir, extensionsDir, ext.Identity)
|
||||
}
|
||||
|
||||
type Extension struct {
|
||||
// Identity is global unique for an extension, it's a combination of author and name
|
||||
Identity string `json:"identity"`
|
||||
Name string `json:"name"`
|
||||
Author string `json:"author"`
|
||||
Title string `json:"title"`
|
||||
Description string `json:"description"`
|
||||
Icon string `json:"icon"`
|
||||
// Version semantic version string, like: 1.0.0
|
||||
Version string `json:"version"`
|
||||
// Homepage homepage url
|
||||
Homepage string `json:"homepage"`
|
||||
// Repository git repository info
|
||||
Repository *Repository `json:"repository"`
|
||||
Scripts []*Script `json:"scripts"`
|
||||
Settings []*Setting `json:"settings"`
|
||||
// Disabled if true, this extension will be ignored
|
||||
Disabled bool `json:"disabled"`
|
||||
|
||||
DevMode bool `json:"devMode"`
|
||||
// DevPath is the local path of extension source code
|
||||
DevPath string `json:"devPath"`
|
||||
|
||||
CreatedAt time.Time `json:"createdAt"`
|
||||
UpdatedAt time.Time `json:"updatedAt"`
|
||||
}
|
||||
|
||||
func (e *Extension) validate() error {
|
||||
if e.Name == "" {
|
||||
return fmt.Errorf("extension name is required")
|
||||
}
|
||||
if e.Title == "" {
|
||||
return fmt.Errorf("extension title is required")
|
||||
}
|
||||
if e.Version == "" {
|
||||
return fmt.Errorf("extension version is required")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (e *Extension) buildIdentity() string {
|
||||
if e.Author == "" {
|
||||
return e.Name
|
||||
}
|
||||
return e.Author + "@" + e.Name
|
||||
}
|
||||
|
||||
func (e *Extension) buildInstallUrl() string {
|
||||
if e.Repository == nil || e.Repository.Url == "" {
|
||||
return ""
|
||||
}
|
||||
repoUrl := e.Repository.Url
|
||||
if e.Repository.Directory != "" {
|
||||
if strings.HasSuffix(repoUrl, "/") {
|
||||
repoUrl = repoUrl[:len(repoUrl)-1]
|
||||
}
|
||||
dir := e.Repository.Directory
|
||||
if strings.HasPrefix(dir, "/") {
|
||||
dir = dir[1:]
|
||||
}
|
||||
repoUrl = repoUrl + "#" + dir
|
||||
}
|
||||
return repoUrl
|
||||
}
|
||||
|
||||
func (e *Extension) update(newExt *Extension) error {
|
||||
e.Title = newExt.Title
|
||||
e.Description = newExt.Description
|
||||
e.Icon = newExt.Icon
|
||||
e.Version = newExt.Version
|
||||
e.Homepage = newExt.Homepage
|
||||
e.Repository = newExt.Repository
|
||||
e.Scripts = newExt.Scripts
|
||||
// don't override settings
|
||||
//e.Settings = newExt.Settings
|
||||
e.UpdatedAt = time.Now()
|
||||
return nil
|
||||
}
|
||||
|
||||
type Repository struct {
|
||||
Url string `json:"url"`
|
||||
Directory string `json:"directory"`
|
||||
}
|
||||
|
||||
type Script struct {
|
||||
// Event active event name
|
||||
Event string `json:"event"`
|
||||
// Matches match request url pattern list
|
||||
Matches []string `json:"matches"`
|
||||
// Entry js script file path
|
||||
Entry string `json:"entry"`
|
||||
}
|
||||
|
||||
func (s *Script) match(event ActivationEvent, url string) bool {
|
||||
if s.Event == "" {
|
||||
return false
|
||||
}
|
||||
if s.Event != string(event) {
|
||||
return false
|
||||
}
|
||||
if len(s.Matches) == 0 {
|
||||
return false
|
||||
}
|
||||
for _, match := range s.Matches {
|
||||
if util.Match(match, url) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
type SettingType string
|
||||
|
||||
const (
|
||||
SettingTypeString SettingType = "string"
|
||||
SettingTypeNumber SettingType = "number"
|
||||
SettingTypeBoolean SettingType = "boolean"
|
||||
)
|
||||
|
||||
type Setting struct {
|
||||
Name string `json:"name"`
|
||||
Title string `json:"title"`
|
||||
Description string `json:"description"`
|
||||
Required bool `json:"required"`
|
||||
// setting type
|
||||
Type SettingType `json:"type"`
|
||||
// setting value
|
||||
Value any `json:"value"`
|
||||
//Multiple bool `json:"multiple"`
|
||||
Options []*Option `json:"options"`
|
||||
}
|
||||
|
||||
type Option struct {
|
||||
Label string `json:"label"`
|
||||
Value any `json:"value"`
|
||||
}
|
||||
|
||||
// Instance inject to js context
|
||||
type Instance struct {
|
||||
Events InstanceEvents `json:"events"`
|
||||
Logger *InstanceLogger `json:"logger"`
|
||||
}
|
||||
|
||||
type InstanceEvents map[ActivationEvent]goja.Callable
|
||||
|
||||
func (h InstanceEvents) register(name ActivationEvent, fn goja.Callable) {
|
||||
h[name] = fn
|
||||
}
|
||||
|
||||
func (h InstanceEvents) OnResolve(fn goja.Callable) {
|
||||
h.register(EventOnResolve, fn)
|
||||
}
|
||||
|
||||
//func (h InstanceEvents) OnError(fn goja.Callable) {
|
||||
// h.register(HookEventOnError, fn)
|
||||
//}
|
||||
//
|
||||
//func (h InstanceEvents) OnDone(fn goja.Callable) {
|
||||
// h.register(HookEventOnDone, fn)
|
||||
//}
|
||||
|
||||
type InstanceLogger struct {
|
||||
identity string
|
||||
devMode bool
|
||||
logger *logger.Logger
|
||||
}
|
||||
|
||||
func (l *InstanceLogger) Debug(msg ...any) {
|
||||
if l.devMode {
|
||||
l.logger.Debug().Msg(l.append(msg...))
|
||||
}
|
||||
}
|
||||
|
||||
func (l *InstanceLogger) Info(msg ...any) {
|
||||
l.logger.Info().Msg(l.append(msg...))
|
||||
}
|
||||
|
||||
func (l *InstanceLogger) Warn(msg ...any) {
|
||||
l.logger.Warn().Msg(l.append(msg...))
|
||||
}
|
||||
|
||||
func (l *InstanceLogger) Error(msg ...any) {
|
||||
l.logger.Error().Msg(l.append(msg...))
|
||||
}
|
||||
|
||||
func (l *InstanceLogger) append(msg ...any) string {
|
||||
strMsg := make([]string, len(msg))
|
||||
for i, m := range msg {
|
||||
strMsg[i] = fmt.Sprint(m)
|
||||
}
|
||||
return fmt.Sprintf("[%s] %s", l.identity, strings.Join(strMsg, " "))
|
||||
}
|
||||
|
||||
func newInstanceLogger(extension *Extension, logger *logger.Logger) *InstanceLogger {
|
||||
return &InstanceLogger{
|
||||
identity: extension.buildIdentity(),
|
||||
devMode: extension.DevMode,
|
||||
logger: logger,
|
||||
}
|
||||
}
|
||||
|
||||
type Context struct {
|
||||
Req *base.Request `json:"req"`
|
||||
Res *base.Resource `json:"res"`
|
||||
Settings map[string]any `json:"settings"`
|
||||
}
|
||||
|
||||
func parseSettings(settings []*Setting) map[string]any {
|
||||
m := make(map[string]any)
|
||||
for _, s := range settings {
|
||||
var val any
|
||||
if s.Value != nil {
|
||||
val = s.Value
|
||||
}
|
||||
m[s.Name] = tryParse(val, s.Type)
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
func tryParse(val any, settingType SettingType) any {
|
||||
if val == nil {
|
||||
return nil
|
||||
}
|
||||
switch settingType {
|
||||
case SettingTypeString:
|
||||
return fmt.Sprint(val)
|
||||
case SettingTypeNumber:
|
||||
vv, err := strconv.ParseFloat(fmt.Sprint(val), 64)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
return vv
|
||||
case SettingTypeBoolean:
|
||||
vv, err := strconv.ParseBool(fmt.Sprint(val))
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
return vv
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,245 @@
|
||||
package download
|
||||
|
||||
import (
|
||||
"github.com/GopeedLab/gopeed/internal/logger"
|
||||
"github.com/GopeedLab/gopeed/pkg/base"
|
||||
"os"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestDownloader_InstallExtensionByFolder(t *testing.T) {
|
||||
setupDownloader(func(downloader *Downloader) {
|
||||
if _, err := downloader.InstallExtensionByFolder("./testdata/extensions/basic", false); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rr, err := downloader.Resolve(&base.Request{
|
||||
URL: "https://github.com/test",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(rr.Res.Files) == 1 {
|
||||
t.Fatal("resolve error")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestDownloader_InstallExtensionByFolderDevMode(t *testing.T) {
|
||||
setupDownloader(func(downloader *Downloader) {
|
||||
if _, err := downloader.InstallExtensionByFolder("./testdata/extensions/basic", true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rr, err := downloader.Resolve(&base.Request{
|
||||
URL: "https://github.com/test",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(rr.Res.Files) == 1 {
|
||||
t.Fatal("resolve error")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestDownloader_InstallExtensionByGit(t *testing.T) {
|
||||
setupDownloader(func(downloader *Downloader) {
|
||||
if _, err := downloader.InstallExtensionByGit("https://github.com/GopeedLab/gopeed-extension-samples#github-release-sample"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rr, err := downloader.Resolve(&base.Request{
|
||||
URL: "https://github.com/GopeedLab/gopeed/releases",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(rr.Res.Files) == 1 {
|
||||
t.Fatal("resolve error")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestDownloader_InstallExtensionByGitSimple(t *testing.T) {
|
||||
setupDownloader(func(downloader *Downloader) {
|
||||
if _, err := downloader.InstallExtensionByGit("github.com/GopeedLab/gopeed-extension-samples#github-release-sample"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rr, err := downloader.Resolve(&base.Request{
|
||||
URL: "https://github.com/GopeedLab/gopeed/releases",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(rr.Res.Files) == 1 {
|
||||
t.Fatal("resolve error")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestDownloader_InstallExtensionByGitFull(t *testing.T) {
|
||||
setupDownloader(func(downloader *Downloader) {
|
||||
if _, err := downloader.InstallExtensionByGit("https://github.com/GopeedLab/gopeed-extension-samples.git#github-release-sample"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rr, err := downloader.Resolve(&base.Request{
|
||||
URL: "https://github.com/GopeedLab/gopeed/releases",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(rr.Res.Files) == 1 {
|
||||
t.Fatal("resolve error")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestDownloader_UpgradeExtension(t *testing.T) {
|
||||
setupDownloader(func(downloader *Downloader) {
|
||||
installedExt, err := downloader.InstallExtensionByFolder("./testdata/extensions/update", false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
extensions := downloader.GetExtensions()
|
||||
if len(extensions) == 0 {
|
||||
t.Fatal("extension not installed")
|
||||
}
|
||||
oldVersion := installedExt.Version
|
||||
// fetch new version from git
|
||||
newVersion, err := downloader.UpgradeCheckExtension(installedExt.Identity)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if newVersion == "" {
|
||||
t.Fatal("new version not found")
|
||||
}
|
||||
// update extension
|
||||
if err = downloader.UpgradeExtension(installedExt.Identity); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
upgradeExt := downloader.getExtension(installedExt.Identity)
|
||||
if upgradeExt.Version == oldVersion {
|
||||
t.Fatal("extension update fail")
|
||||
}
|
||||
|
||||
rr, err := downloader.Resolve(&base.Request{
|
||||
URL: "https://github.com/GopeedLab/gopeed/releases",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(rr.Res.Files) == 1 {
|
||||
t.Fatal("resolve error")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestDownloader_Extension_Settings(t *testing.T) {
|
||||
setupDownloader(func(downloader *Downloader) {
|
||||
if _, err := downloader.InstallExtensionByFolder("./testdata/extensions/settings_empty", false); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rr, err := downloader.Resolve(&base.Request{
|
||||
URL: "https://github.com/test",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(rr.Res.Files) == 1 {
|
||||
t.Fatal("settings parse error")
|
||||
}
|
||||
})
|
||||
|
||||
setupDownloader(func(downloader *Downloader) {
|
||||
installedExt, err := downloader.InstallExtensionByFolder("./testdata/extensions/settings_all", false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
downloader.UpdateExtensionSettings(installedExt.Identity, map[string]any{
|
||||
"stringValued": "valued",
|
||||
"numberValued": 1.1,
|
||||
"booleanValued": true,
|
||||
})
|
||||
rr, err := downloader.Resolve(&base.Request{
|
||||
URL: "https://github.com/test",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(rr.Res.Files) == 1 {
|
||||
t.Fatal("settings parse error")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestDownloader_SwitchExtension(t *testing.T) {
|
||||
setupDownloader(func(downloader *Downloader) {
|
||||
installedExt, err := downloader.InstallExtensionByFolder("./testdata/extensions/basic", false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if installedExt.Disabled == true {
|
||||
t.Fatal("extension disabled")
|
||||
}
|
||||
if err = downloader.SwitchExtension(installedExt.Identity, false); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if installedExt.Disabled == false {
|
||||
t.Fatal("extension enabled")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestDownloader_DeleteExtension(t *testing.T) {
|
||||
setupDownloader(func(downloader *Downloader) {
|
||||
installedExt, err := downloader.InstallExtensionByFolder("./testdata/extensions/settings_all", false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
extensions := downloader.GetExtensions()
|
||||
if err := downloader.DeleteExtension(installedExt.Identity); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
extensions = downloader.GetExtensions()
|
||||
if len(extensions) != 0 {
|
||||
t.Fatal("extension delete fail")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestDownloader_Extension_OnResolve(t *testing.T) {
|
||||
setupDownloader(func(downloader *Downloader) {
|
||||
installedExt, err := downloader.InstallExtensionByFolder("./testdata/extensions/settings_all", false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := downloader.DeleteExtension(installedExt.Identity); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
extensions := downloader.GetExtensions()
|
||||
if len(extensions) != 0 {
|
||||
t.Fatal("extension delete fail")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestDownloader_Extension_Logger(t *testing.T) {
|
||||
logger := logger.NewLogger(false, "")
|
||||
il := newInstanceLogger(&Extension{
|
||||
Name: "test",
|
||||
}, logger)
|
||||
il.Debug("msg1", "msg2")
|
||||
il.Info("msg1", "msg2")
|
||||
il.Warn("msg1", "msg2")
|
||||
il.Error("msg1", "msg2")
|
||||
}
|
||||
|
||||
func setupDownloader(fn func(downloader *Downloader)) {
|
||||
defaultDownloader.Setup()
|
||||
defaultDownloader.cfg.StorageDir = ".test_storage"
|
||||
defaultDownloader.cfg.DownloadDir = ".test_download"
|
||||
defer func() {
|
||||
defaultDownloader.Clear()
|
||||
os.RemoveAll(defaultDownloader.cfg.StorageDir)
|
||||
os.RemoveAll(defaultDownloader.cfg.DownloadDir)
|
||||
}()
|
||||
fn(defaultDownloader)
|
||||
}
|
||||
@@ -75,6 +75,8 @@ type DownloaderConfig struct {
|
||||
Storage Storage
|
||||
StorageDir string
|
||||
|
||||
ProductionMode bool
|
||||
|
||||
*DownloaderStoreConfig
|
||||
}
|
||||
|
||||
@@ -99,7 +101,7 @@ func (cfg *DownloaderConfig) Init() *DownloaderConfig {
|
||||
|
||||
// DownloaderStoreConfig is the config that can restore the downloader.
|
||||
type DownloaderStoreConfig struct {
|
||||
FirstLoad bool `json:"-"` // fromNoStore is the flag that the config is first time init and not from store
|
||||
FirstLoad bool `json:"-"` // FirstLoad is the flag that the config is first time init and not from store
|
||||
|
||||
DownloadDir string `json:"downloadDir"` // DownloadDir is the default directory to save the downloaded files
|
||||
MaxRunning int `json:"maxRunning"` // MaxRunning is the max running download count
|
||||
@@ -109,7 +111,7 @@ type DownloaderStoreConfig struct {
|
||||
|
||||
func (cfg *DownloaderStoreConfig) Init() *DownloaderStoreConfig {
|
||||
if cfg.MaxRunning == 0 {
|
||||
cfg.MaxRunning = 3
|
||||
cfg.MaxRunning = 5
|
||||
}
|
||||
return cfg
|
||||
}
|
||||
+20
-1
@@ -6,6 +6,7 @@ import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"sync"
|
||||
)
|
||||
|
||||
type Storage interface {
|
||||
@@ -47,10 +48,13 @@ func changeValue(p any, v any) {
|
||||
var memData = make(map[string]map[string]any)
|
||||
|
||||
type MemStorage struct {
|
||||
lock *sync.RWMutex
|
||||
}
|
||||
|
||||
func NewMemStorage() *MemStorage {
|
||||
return &MemStorage{}
|
||||
return &MemStorage{
|
||||
lock: &sync.RWMutex{},
|
||||
}
|
||||
}
|
||||
|
||||
func (n *MemStorage) Setup(buckets []string) error {
|
||||
@@ -63,11 +67,15 @@ func (n *MemStorage) Setup(buckets []string) error {
|
||||
}
|
||||
|
||||
func (n *MemStorage) Put(bucket string, key string, v any) error {
|
||||
n.lock.Lock()
|
||||
defer n.lock.Unlock()
|
||||
memData[bucket][key] = v
|
||||
return nil
|
||||
}
|
||||
|
||||
func (n *MemStorage) Get(bucket string, key string, v any) (bool, error) {
|
||||
n.lock.RLock()
|
||||
defer n.lock.RUnlock()
|
||||
if dv, ok := memData[bucket][key]; ok {
|
||||
changeValue(v, dv)
|
||||
return true, nil
|
||||
@@ -76,6 +84,8 @@ func (n *MemStorage) Get(bucket string, key string, v any) (bool, error) {
|
||||
}
|
||||
|
||||
func (n *MemStorage) List(bucket string, v any) error {
|
||||
n.lock.RLock()
|
||||
defer n.lock.RUnlock()
|
||||
data := memData[bucket]
|
||||
list := make([]any, 0)
|
||||
for _, v := range data {
|
||||
@@ -86,6 +96,8 @@ func (n *MemStorage) List(bucket string, v any) error {
|
||||
}
|
||||
|
||||
func (n *MemStorage) Pop(bucket string, key string, v any) error {
|
||||
n.lock.Lock()
|
||||
defer n.lock.Unlock()
|
||||
data := memData[bucket]
|
||||
changeValue(v, data[key])
|
||||
delete(data, key)
|
||||
@@ -93,6 +105,8 @@ func (n *MemStorage) Pop(bucket string, key string, v any) error {
|
||||
}
|
||||
|
||||
func (n *MemStorage) Delete(bucket string, key string) error {
|
||||
n.lock.Lock()
|
||||
defer n.lock.Unlock()
|
||||
delete(memData[bucket], key)
|
||||
return nil
|
||||
}
|
||||
@@ -102,6 +116,8 @@ func (n *MemStorage) Close() error {
|
||||
}
|
||||
|
||||
func (n *MemStorage) Clear() error {
|
||||
n.lock.Lock()
|
||||
defer n.lock.Unlock()
|
||||
memData = make(map[string]map[string]any)
|
||||
return nil
|
||||
}
|
||||
@@ -116,6 +132,9 @@ type BoltStorage struct {
|
||||
}
|
||||
|
||||
func NewBoltStorage(dir string) *BoltStorage {
|
||||
if err := os.MkdirAll(dir, 0755); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
path := filepath.Join(dir, dbFile)
|
||||
db, err := bbolt.Open(path, 0600, nil)
|
||||
if err != nil {
|
||||
|
||||
+13
@@ -0,0 +1,13 @@
|
||||
gopeed.events.onResolve(async function (ctx) {
|
||||
ctx.res = {
|
||||
name: "test",
|
||||
files: Array(2).fill(true).map((_, i) => ({
|
||||
name: `test-${i}.txt`,
|
||||
size: 1024,
|
||||
req: {
|
||||
url: ctx.req.url + "/" + i,
|
||||
}
|
||||
}),
|
||||
),
|
||||
};
|
||||
});
|
||||
@@ -0,0 +1,21 @@
|
||||
{
|
||||
"name": "basic",
|
||||
"title": "gopeed extension basic test",
|
||||
"version": "0.0.1",
|
||||
"scripts": [
|
||||
{
|
||||
"event": "onResolve",
|
||||
"matches": [
|
||||
"*://github.com/*"
|
||||
],
|
||||
"entry": "index.js"
|
||||
}
|
||||
],
|
||||
"settings": [
|
||||
{
|
||||
"name": "ua",
|
||||
"title": "User-Agent",
|
||||
"type": "string"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
gopeed.events.onResolve(async function (ctx) {
|
||||
if (ctx.settings.string != null) {
|
||||
throw new Error("string is not null");
|
||||
}
|
||||
if (ctx.settings.number != null) {
|
||||
throw new Error("number is not null");
|
||||
}
|
||||
if (ctx.settings.boolean != null) {
|
||||
throw new Error("boolean is not null");
|
||||
}
|
||||
|
||||
if (ctx.settings.stringDefault !== "default") {
|
||||
throw new Error("string default value is incorrect");
|
||||
}
|
||||
if (ctx.settings.numberDefault !== 1) {
|
||||
throw new Error("number default value is incorrect");
|
||||
}
|
||||
if (ctx.settings.booleanDefault !== true) {
|
||||
throw new Error("boolean default value is incorrect");
|
||||
}
|
||||
|
||||
if (ctx.settings.stringValued !== "valued") {
|
||||
throw new Error("string value is incorrect");
|
||||
}
|
||||
if (ctx.settings.numberValued !== 1.1) {
|
||||
throw new Error("number value is incorrect");
|
||||
}
|
||||
if (ctx.settings.booleanValued !== true) {
|
||||
throw new Error("boolean value is incorrect");
|
||||
}
|
||||
|
||||
ctx.res = {
|
||||
name: "test",
|
||||
files: Array(2).fill(true).map((_, i) => ({
|
||||
name: `test-${i}.txt`,
|
||||
size: 1024,
|
||||
req: {
|
||||
url: ctx.req.url + "/" + i,
|
||||
}
|
||||
}),
|
||||
),
|
||||
};
|
||||
});
|
||||
@@ -0,0 +1,64 @@
|
||||
{
|
||||
"name": "settings-all",
|
||||
"title": "gopeed extension settings all type test",
|
||||
"version": "0.0.1",
|
||||
"scripts": [
|
||||
{
|
||||
"event": "onResolve",
|
||||
"matches": [
|
||||
"*://*/*"
|
||||
],
|
||||
"entry": "index.js"
|
||||
}
|
||||
],
|
||||
"settings": [
|
||||
{
|
||||
"name": "string",
|
||||
"title": "string null test",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"name": "number",
|
||||
"title": "number null test",
|
||||
"type": "number"
|
||||
},
|
||||
{
|
||||
"name": "boolean",
|
||||
"title": "boolean null test",
|
||||
"type": "boolean"
|
||||
},
|
||||
{
|
||||
"name": "stringDefault",
|
||||
"title": "string default test",
|
||||
"type": "string",
|
||||
"value": "default"
|
||||
},
|
||||
{
|
||||
"name": "numberDefault",
|
||||
"title": "number default test",
|
||||
"type": "number",
|
||||
"value": 1
|
||||
},
|
||||
{
|
||||
"name": "booleanDefault",
|
||||
"title": "boolean default test",
|
||||
"type": "boolean",
|
||||
"value": true
|
||||
},
|
||||
{
|
||||
"name": "stringValued",
|
||||
"title": "string valued test",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"name": "numberValued",
|
||||
"title": "number valued test",
|
||||
"type": "number"
|
||||
},
|
||||
{
|
||||
"name": "booleanValued",
|
||||
"title": "boolean valued test",
|
||||
"type": "boolean"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
gopeed.events.onResolve(async function (ctx) {
|
||||
if (Object.keys(ctx.settings).length > 0){
|
||||
throw new Error("settings is not empty");
|
||||
}
|
||||
|
||||
ctx.res = {
|
||||
name: "test",
|
||||
files: Array(2).fill(true).map((_, i) => ({
|
||||
name: `test-${i}.txt`,
|
||||
size: 1024,
|
||||
req: {
|
||||
url: ctx.req.url + "/" + i,
|
||||
}
|
||||
}),
|
||||
),
|
||||
};
|
||||
});
|
||||
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"name": "settings-empty",
|
||||
"title": "gopeed extension settings empty test",
|
||||
"version": "0.0.1",
|
||||
"scripts": [
|
||||
{
|
||||
"event": "onResolve",
|
||||
"matches": [
|
||||
"*://*/*"
|
||||
],
|
||||
"entry": "index.js"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
gopeed.events.onResolve(async function (ctx) {
|
||||
// do nothing for test
|
||||
});
|
||||
@@ -0,0 +1,21 @@
|
||||
{
|
||||
"name": "github-release",
|
||||
"author": "gopeed",
|
||||
"title": "Github Release",
|
||||
"description": "Download github release assets",
|
||||
"version": "0.0.1",
|
||||
"homepage": "https://gopeed.com",
|
||||
"repository": {
|
||||
"url": "https://github.com/GopeedLab/gopeed-extension-samples",
|
||||
"directory": "github-release-sample"
|
||||
},
|
||||
"scripts": [
|
||||
{
|
||||
"event": "onResolve",
|
||||
"matches": [
|
||||
"*://github.com/*/releases/*"
|
||||
],
|
||||
"entry": "index.js"
|
||||
}
|
||||
]
|
||||
}
|
||||
+104
-27
@@ -8,7 +8,6 @@ import (
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func Resolve(w http.ResponseWriter, r *http.Request) {
|
||||
@@ -31,9 +30,9 @@ func CreateTask(w http.ResponseWriter, r *http.Request) {
|
||||
err error
|
||||
)
|
||||
if req.Rid != "" {
|
||||
taskId, err = Downloader.Create(req.Rid, req.Opts)
|
||||
taskId, err = Downloader.Create(req.Rid, req.Opt)
|
||||
} else if req.Req != nil {
|
||||
taskId, err = Downloader.CreateDirect(req.Req, req.Opts)
|
||||
taskId, err = Downloader.CreateDirect(req.Req, req.Opt)
|
||||
} else {
|
||||
WriteJson(w, model.NewErrorResult("param invalid: rid or req", model.CodeInvalidParam))
|
||||
return
|
||||
@@ -121,21 +120,16 @@ func GetTask(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
|
||||
func GetTasks(w http.ResponseWriter, r *http.Request) {
|
||||
status := r.FormValue("status")
|
||||
if status == "" {
|
||||
status = strings.Join([]string{
|
||||
string(base.DownloadStatusReady),
|
||||
string(base.DownloadStatusRunning),
|
||||
string(base.DownloadStatusPause),
|
||||
string(base.DownloadStatusError),
|
||||
string(base.DownloadStatusDone),
|
||||
}, ",")
|
||||
}
|
||||
statusArr := strings.Split(status, ",")
|
||||
r.ParseForm()
|
||||
status := r.Form["status"]
|
||||
tasks := Downloader.GetTasks()
|
||||
if len(status) == 0 {
|
||||
WriteJson(w, model.NewOkResult(tasks))
|
||||
return
|
||||
}
|
||||
result := make([]*download.Task, 0)
|
||||
for _, task := range tasks {
|
||||
for _, s := range statusArr {
|
||||
for _, s := range status {
|
||||
if task.Status == base.Status(s) {
|
||||
result = append(result, task)
|
||||
}
|
||||
@@ -159,6 +153,101 @@ func PutConfig(w http.ResponseWriter, r *http.Request) {
|
||||
WriteJson(w, model.NewNilResult())
|
||||
}
|
||||
|
||||
func InstallExtension(w http.ResponseWriter, r *http.Request) {
|
||||
var req model.InstallExtension
|
||||
if ReadJson(r, w, &req) {
|
||||
var (
|
||||
installedExt *download.Extension
|
||||
err error
|
||||
)
|
||||
if req.DevMode {
|
||||
installedExt, err = Downloader.InstallExtensionByFolder(req.URL, true)
|
||||
} else {
|
||||
installedExt, err = Downloader.InstallExtensionByGit(req.URL)
|
||||
}
|
||||
if err != nil {
|
||||
WriteJson(w, model.NewErrorResult(err.Error()))
|
||||
return
|
||||
}
|
||||
WriteJson(w, model.NewOkResult(installedExt.Identity))
|
||||
}
|
||||
}
|
||||
|
||||
func GetExtensions(w http.ResponseWriter, r *http.Request) {
|
||||
list := Downloader.GetExtensions()
|
||||
WriteJson(w, model.NewOkResult(list))
|
||||
}
|
||||
|
||||
func GetExtension(w http.ResponseWriter, r *http.Request) {
|
||||
vars := mux.Vars(r)
|
||||
identity := vars["identity"]
|
||||
ext, err := Downloader.GetExtension(identity)
|
||||
if err != nil {
|
||||
WriteJson(w, model.NewErrorResult(err.Error()))
|
||||
return
|
||||
}
|
||||
WriteJson(w, model.NewOkResult(ext))
|
||||
}
|
||||
|
||||
func UpdateExtensionSettings(w http.ResponseWriter, r *http.Request) {
|
||||
vars := mux.Vars(r)
|
||||
identity := vars["identity"]
|
||||
var req model.UpdateExtensionSettings
|
||||
if ReadJson(r, w, &req) {
|
||||
if err := Downloader.UpdateExtensionSettings(identity, req.Settings); err != nil {
|
||||
WriteJson(w, model.NewErrorResult(err.Error()))
|
||||
return
|
||||
}
|
||||
}
|
||||
WriteJson(w, model.NewNilResult())
|
||||
}
|
||||
|
||||
func SwitchExtension(w http.ResponseWriter, r *http.Request) {
|
||||
vars := mux.Vars(r)
|
||||
identity := vars["identity"]
|
||||
var switchExtension model.SwitchExtension
|
||||
if ReadJson(r, w, &switchExtension) {
|
||||
if err := Downloader.SwitchExtension(identity, switchExtension.Status); err != nil {
|
||||
WriteJson(w, model.NewErrorResult(err.Error()))
|
||||
return
|
||||
}
|
||||
}
|
||||
WriteJson(w, model.NewNilResult())
|
||||
}
|
||||
|
||||
func DeleteExtension(w http.ResponseWriter, r *http.Request) {
|
||||
vars := mux.Vars(r)
|
||||
identity := vars["identity"]
|
||||
if err := Downloader.DeleteExtension(identity); err != nil {
|
||||
WriteJson(w, model.NewErrorResult(err.Error()))
|
||||
return
|
||||
}
|
||||
WriteJson(w, model.NewNilResult())
|
||||
}
|
||||
|
||||
func UpdateCheckExtension(w http.ResponseWriter, r *http.Request) {
|
||||
vars := mux.Vars(r)
|
||||
identity := vars["identity"]
|
||||
newVersion, err := Downloader.UpgradeCheckExtension(identity)
|
||||
if err != nil {
|
||||
WriteJson(w, model.NewErrorResult(err.Error()))
|
||||
return
|
||||
}
|
||||
WriteJson(w, model.NewOkResult(&model.UpdateCheckExtensionResp{
|
||||
NewVersion: newVersion,
|
||||
}))
|
||||
}
|
||||
|
||||
func UpdateExtension(w http.ResponseWriter, r *http.Request) {
|
||||
vars := mux.Vars(r)
|
||||
identity := vars["identity"]
|
||||
if err := Downloader.UpgradeExtension(identity); err != nil {
|
||||
WriteJson(w, model.NewErrorResult(err.Error()))
|
||||
return
|
||||
}
|
||||
WriteJson(w, model.NewNilResult())
|
||||
}
|
||||
|
||||
func DoProxy(w http.ResponseWriter, r *http.Request) {
|
||||
target := r.Header.Get("X-Target-Uri")
|
||||
if target == "" {
|
||||
@@ -191,18 +280,6 @@ func DoProxy(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
w.Write(buf)
|
||||
//var reader io.ReadCloser
|
||||
//switch resp.Header.Get("Content-Encoding") {
|
||||
//case "gzip":
|
||||
// reader, err = gzip.NewReader(resp.Body)
|
||||
// defer reader.Close()
|
||||
//default:
|
||||
// reader = resp.Body
|
||||
//}
|
||||
//if _, err := io.Copy(w, resp.Body); err != nil {
|
||||
// writeError(w, err.Error())
|
||||
// return
|
||||
//}
|
||||
}
|
||||
|
||||
func writeError(w http.ResponseWriter, msg string) {
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
package model
|
||||
|
||||
type InstallExtension struct {
|
||||
DevMode bool `json:"devMode"`
|
||||
URL string `json:"url"`
|
||||
}
|
||||
|
||||
type UpdateExtensionSettings struct {
|
||||
Settings map[string]any `json:"settings"`
|
||||
}
|
||||
|
||||
type SwitchExtension struct {
|
||||
Status bool `json:"status"`
|
||||
}
|
||||
|
||||
type UpdateCheckExtensionResp struct {
|
||||
NewVersion string `json:"newVersion"`
|
||||
}
|
||||
@@ -20,6 +20,8 @@ type StartConfig struct {
|
||||
StorageDir string `json:"storageDir"`
|
||||
ApiToken string `json:"apiToken"`
|
||||
|
||||
ProductionMode bool
|
||||
|
||||
WebEnable bool
|
||||
WebFS fs.FS
|
||||
WebBasicAuth *WebBasicAuth
|
||||
|
||||
@@ -3,7 +3,7 @@ package model
|
||||
import "github.com/GopeedLab/gopeed/pkg/base"
|
||||
|
||||
type CreateTask struct {
|
||||
Rid string `json:"rid"`
|
||||
Req *base.Request `json:"req"`
|
||||
Opts *base.Options `json:"opts"`
|
||||
Rid string `json:"rid"`
|
||||
Req *base.Request `json:"req"`
|
||||
Opt *base.Options `json:"opt"`
|
||||
}
|
||||
+42
-3
@@ -10,6 +10,10 @@ import (
|
||||
"github.com/gorilla/mux"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -26,7 +30,6 @@ func Start(startCfg *model.StartConfig) (int, error) {
|
||||
|
||||
go func() {
|
||||
if err := srv.Serve(listener); err != nil && err != http.ErrServerClosed {
|
||||
// TODO log
|
||||
panic(err)
|
||||
}
|
||||
}()
|
||||
@@ -41,12 +44,12 @@ func Start(startCfg *model.StartConfig) (int, error) {
|
||||
func Stop() {
|
||||
if srv != nil {
|
||||
if err := srv.Shutdown(context.TODO()); err != nil {
|
||||
// TODO log
|
||||
Downloader.Logger.Warn().Err(err).Msg("shutdown server failed")
|
||||
}
|
||||
}
|
||||
if Downloader != nil {
|
||||
if err := Downloader.Close(); err != nil {
|
||||
// TODO log
|
||||
Downloader.Logger.Warn().Err(err).Msg("close downloader failed")
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -58,6 +61,7 @@ func BuildServer(startCfg *model.StartConfig) (*http.Server, net.Listener, error
|
||||
startCfg.Init()
|
||||
|
||||
downloadCfg := &download.DownloaderConfig{
|
||||
ProductionMode: startCfg.ProductionMode,
|
||||
RefreshInterval: startCfg.RefreshInterval,
|
||||
}
|
||||
if startCfg.Storage == model.StorageBolt {
|
||||
@@ -93,6 +97,15 @@ func BuildServer(startCfg *model.StartConfig) (*http.Server, net.Listener, error
|
||||
r.Methods(http.MethodGet).Path("/api/v1/tasks").HandlerFunc(GetTasks)
|
||||
r.Methods(http.MethodGet).Path("/api/v1/config").HandlerFunc(GetConfig)
|
||||
r.Methods(http.MethodPut).Path("/api/v1/config").HandlerFunc(PutConfig)
|
||||
r.Methods(http.MethodPost).Path("/api/v1/extensions").HandlerFunc(InstallExtension)
|
||||
r.Methods(http.MethodGet).Path("/api/v1/extensions").HandlerFunc(GetExtensions)
|
||||
r.Methods(http.MethodGet).Path("/api/v1/extensions/{identity}").HandlerFunc(GetExtension)
|
||||
r.Methods(http.MethodPut).Path("/api/v1/extensions/{identity}/settings").HandlerFunc(UpdateExtensionSettings)
|
||||
r.Methods(http.MethodPut).Path("/api/v1/extensions/{identity}/switch").HandlerFunc(SwitchExtension)
|
||||
r.Methods(http.MethodDelete).Path("/api/v1/extensions/{identity}").HandlerFunc(DeleteExtension)
|
||||
r.Methods(http.MethodGet).Path("/api/v1/extensions/{identity}/update").HandlerFunc(UpdateCheckExtension)
|
||||
r.Methods(http.MethodPost).Path("/api/v1/extensions/{identity}/update").HandlerFunc(UpdateExtension)
|
||||
r.PathPrefix("/fs/extensions").Handler(http.FileServer(new(extensionFileSystem)))
|
||||
r.Path("/api/v1/proxy").HandlerFunc(DoProxy)
|
||||
if startCfg.WebEnable {
|
||||
r.PathPrefix("/").Handler(http.FileServer(http.FS(startCfg.WebFS)))
|
||||
@@ -125,6 +138,32 @@ func BuildServer(startCfg *model.StartConfig) (*http.Server, net.Listener, error
|
||||
return srv, listener, nil
|
||||
}
|
||||
|
||||
// handle extension file resource
|
||||
type extensionFileSystem struct {
|
||||
}
|
||||
|
||||
func (e *extensionFileSystem) Open(name string) (http.File, error) {
|
||||
// remove prefix
|
||||
path := strings.TrimPrefix(name, "/fs/extensions")
|
||||
// match extension identity, eg: /fs/extensions/identity/xxx
|
||||
reg := regexp.MustCompile(`^/([^/]+)/(.*)$`)
|
||||
if !reg.MatchString(path) {
|
||||
return nil, os.ErrNotExist
|
||||
}
|
||||
matched := reg.FindStringSubmatch(path)
|
||||
if len(matched) != 3 {
|
||||
return nil, os.ErrNotExist
|
||||
}
|
||||
// get extension identity
|
||||
identity := matched[1]
|
||||
extension, err := Downloader.GetExtension(identity)
|
||||
if err != nil {
|
||||
return nil, os.ErrNotExist
|
||||
}
|
||||
extensionPath := Downloader.ExtensionPath(extension)
|
||||
return os.Open(filepath.Join(extensionPath, matched[2]))
|
||||
}
|
||||
|
||||
func ReadJson(r *http.Request, w http.ResponseWriter, v any) bool {
|
||||
if err := json.NewDecoder(r.Body).Decode(v); err != nil {
|
||||
WriteJson(w, model.NewErrorResult(err.Error()))
|
||||
|
||||
+189
-33
@@ -33,7 +33,6 @@ var (
|
||||
},
|
||||
}
|
||||
taskRes = &base.Resource{
|
||||
Name: test.BuildName,
|
||||
Size: test.BuildSize,
|
||||
Range: true,
|
||||
Files: []*base.FileInfo{
|
||||
@@ -44,15 +43,23 @@ var (
|
||||
},
|
||||
},
|
||||
}
|
||||
createOpts = &base.Options{
|
||||
Path: test.Dir,
|
||||
Name: test.DownloadName,
|
||||
Extra: map[string]any{
|
||||
"connections": 2,
|
||||
},
|
||||
}
|
||||
createReq = &model.CreateTask{
|
||||
Req: taskReq,
|
||||
Opts: &base.Options{
|
||||
Path: test.Dir,
|
||||
Name: test.DownloadName,
|
||||
Extra: map[string]any{
|
||||
"connections": 2,
|
||||
},
|
||||
},
|
||||
Opt: createOpts,
|
||||
}
|
||||
createResoledReq = &model.CreateTask{
|
||||
Req: taskReq,
|
||||
Opt: createOpts,
|
||||
}
|
||||
installExtensionReq = &model.InstallExtension{
|
||||
URL: "https://github.com/GopeedLab/gopeed-extension-samples#github-contributor-avatars-sample",
|
||||
}
|
||||
)
|
||||
|
||||
@@ -66,6 +73,35 @@ func TestResolve(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestCreateTask(t *testing.T) {
|
||||
doTest(func() {
|
||||
resp := httpRequestCheckOk[*download.ResolveResult](http.MethodPost, "/api/v1/resolve", taskReq)
|
||||
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(1)
|
||||
Downloader.Listener(func(event *download.Event) {
|
||||
if event.Key == download.EventKeyFinally {
|
||||
wg.Done()
|
||||
}
|
||||
})
|
||||
|
||||
taskId := httpRequestCheckOk[string](http.MethodPost, "/api/v1/tasks", &model.CreateTask{
|
||||
Rid: resp.ID,
|
||||
Opt: createOpts,
|
||||
})
|
||||
if taskId == "" {
|
||||
t.Fatal("create task failed")
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
want := test.FileMd5(test.BuildFile)
|
||||
got := test.FileMd5(test.DownloadFile)
|
||||
if want != got {
|
||||
t.Errorf("CreateTask() got = %v, want %v", got, want)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestCreateDirectTask(t *testing.T) {
|
||||
doTest(func() {
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(1)
|
||||
@@ -84,32 +120,41 @@ func TestCreateTask(t *testing.T) {
|
||||
want := test.FileMd5(test.BuildFile)
|
||||
got := test.FileMd5(test.DownloadFile)
|
||||
if want != got {
|
||||
t.Errorf("CreateTask() got = %v, want %v", got, want)
|
||||
t.Errorf("CreateDirectTask() got = %v, want %v", got, want)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestCreateDirectTaskWithResoled(t *testing.T) {
|
||||
doTest(func() {
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(1)
|
||||
Downloader.Listener(func(event *download.Event) {
|
||||
if event.Key == download.EventKeyFinally {
|
||||
wg.Done()
|
||||
}
|
||||
})
|
||||
|
||||
taskId := httpRequestCheckOk[string](http.MethodPost, "/api/v1/tasks", createResoledReq)
|
||||
if taskId == "" {
|
||||
t.Fatal("create task failed")
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
want := test.FileMd5(test.BuildFile)
|
||||
got := test.FileMd5(test.DownloadFile)
|
||||
if want != got {
|
||||
t.Errorf("CreateDirectTaskWithResoled() got = %v, want %v", got, want)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestPauseAndContinueTask(t *testing.T) {
|
||||
doTest(func() {
|
||||
type result struct {
|
||||
pauseCount int
|
||||
continueCount int
|
||||
md5 string
|
||||
}
|
||||
|
||||
var got = result{
|
||||
pauseCount: 0,
|
||||
continueCount: 0,
|
||||
md5: "",
|
||||
}
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(1)
|
||||
Downloader.Listener(func(event *download.Event) {
|
||||
switch event.Key {
|
||||
case download.EventKeyPause:
|
||||
got.pauseCount++
|
||||
case download.EventKeyContinue:
|
||||
got.continueCount++
|
||||
case download.EventKeyFinally:
|
||||
wg.Done()
|
||||
}
|
||||
@@ -132,12 +177,8 @@ func TestPauseAndContinueTask(t *testing.T) {
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
want := result{
|
||||
pauseCount: 1,
|
||||
continueCount: 1,
|
||||
md5: test.FileMd5(test.BuildFile),
|
||||
}
|
||||
got.md5 = test.FileMd5(test.DownloadFile)
|
||||
want := test.FileMd5(test.BuildFile)
|
||||
got := test.FileMd5(test.DownloadFile)
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Errorf("PauseAndContinueTask() got = %v, want %v", got, want)
|
||||
}
|
||||
@@ -163,13 +204,15 @@ func TestPauseAllAndContinueALLTasks(t *testing.T) {
|
||||
|
||||
// continue all
|
||||
httpRequestCheckOk[any](http.MethodPut, "/api/v1/tasks/continue", nil)
|
||||
time.Sleep(time.Millisecond * 100)
|
||||
tasks := httpRequestCheckOk[[]*download.Task](http.MethodGet, fmt.Sprintf("/api/v1/tasks?status=%s", base.DownloadStatusRunning), nil)
|
||||
if len(tasks) != cfg.MaxRunning {
|
||||
t.Errorf("ContinueAllTasks() got = %v, want %v", len(tasks), cfg.MaxRunning)
|
||||
}
|
||||
// pause all
|
||||
httpRequestCheckOk[any](http.MethodPut, "/api/v1/tasks/pause", nil)
|
||||
tasks = httpRequestCheckOk[[]*download.Task](http.MethodGet, fmt.Sprintf("/api/v1/tasks?status=%s,%s", base.DownloadStatusReady, base.DownloadStatusPause), nil)
|
||||
time.Sleep(time.Millisecond * 500)
|
||||
tasks = httpRequestCheckOk[[]*download.Task](http.MethodGet, fmt.Sprintf("/api/v1/tasks?status=%s", base.DownloadStatusPause), nil)
|
||||
if len(tasks) != total {
|
||||
t.Errorf("PauseAllTasks() got = %v, want %v", len(tasks), total)
|
||||
}
|
||||
@@ -210,7 +253,7 @@ func TestGetTasks(t *testing.T) {
|
||||
}
|
||||
})
|
||||
|
||||
httpRequestCheckOk[string](http.MethodPost, fmt.Sprintf("/api/v1/tasks?status=%s,%s",
|
||||
httpRequestCheckOk[string](http.MethodPost, fmt.Sprintf("/api/v1/tasks?status=%s&status=%s",
|
||||
base.DownloadStatusReady, base.DownloadStatusRunning), createReq)
|
||||
httpRequestCheckOk[[]*download.Task](http.MethodGet, "/api/v1/tasks", nil)
|
||||
|
||||
@@ -248,6 +291,115 @@ func TestGetAndPutConfig(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
func TestInstallExtension(t *testing.T) {
|
||||
doTest(func() {
|
||||
identity := httpRequestCheckOk[string](http.MethodPost, "/api/v1/extensions", installExtensionReq)
|
||||
if identity == "" {
|
||||
t.Errorf("InstallExtension() got = %v, want %v", identity, "not empty")
|
||||
}
|
||||
|
||||
// not a valid extension repository
|
||||
code, _ := httpRequest[string](http.MethodPost, "/api/v1/extensions", &model.InstallExtension{
|
||||
URL: "https://github.com/GopeedLab/gopeed",
|
||||
})
|
||||
checkCode(code, model.CodeError)
|
||||
|
||||
// not a git repository
|
||||
code, _ = httpRequest[string](http.MethodPost, "/api/v1/extensions", &model.InstallExtension{
|
||||
URL: "https://github.com",
|
||||
})
|
||||
checkCode(code, model.CodeError)
|
||||
})
|
||||
}
|
||||
|
||||
func TestGetExtensions(t *testing.T) {
|
||||
doTest(func() {
|
||||
httpRequestCheckOk[string](http.MethodPost, "/api/v1/extensions", installExtensionReq)
|
||||
extensions := httpRequestCheckOk[[]*download.Extension](http.MethodGet, "/api/v1/extensions", nil)
|
||||
if len(extensions) == 0 {
|
||||
t.Errorf("GetExtensions() got = %v, want %v", len(extensions), "not empty")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestUpdateExtensionSettings(t *testing.T) {
|
||||
doTest(func() {
|
||||
identity := httpRequestCheckOk[string](http.MethodPost, "/api/v1/extensions", installExtensionReq)
|
||||
|
||||
httpRequestCheckOk[any](http.MethodPut, "/api/v1/extensions/"+identity+"/settings", &model.UpdateExtensionSettings{
|
||||
Settings: map[string]any{
|
||||
"undefined": "test",
|
||||
"ua": "test",
|
||||
},
|
||||
})
|
||||
|
||||
settings := httpRequestCheckOk[*download.Extension](http.MethodGet, "/api/v1/extensions/"+identity, nil).Settings
|
||||
if len(settings) != 1 {
|
||||
t.Errorf("UpdateExtensionSettings() got = %v, want %v", len(settings), 1)
|
||||
}
|
||||
|
||||
if settings[0].Name != "ua" || settings[0].Value != "test" {
|
||||
t.Errorf("UpdateExtensionSettings() got = %v, want %v", settings[0].Value, "test")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestSwitchExtension(t *testing.T) {
|
||||
doTest(func() {
|
||||
identity := httpRequestCheckOk[string](http.MethodPost, "/api/v1/extensions", installExtensionReq)
|
||||
httpRequestCheckOk[any](http.MethodPut, "/api/v1/extensions/"+identity+"/switch", &model.SwitchExtension{
|
||||
Status: false,
|
||||
})
|
||||
extensions := httpRequestCheckOk[[]*download.Extension](http.MethodGet, "/api/v1/extensions", nil)
|
||||
if !extensions[0].Disabled {
|
||||
t.Errorf("TestSwitchExtension() got = %v, want %v", extensions[0].Disabled, true)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestDeleteExtension(t *testing.T) {
|
||||
doTest(func() {
|
||||
identity := httpRequestCheckOk[string](http.MethodPost, "/api/v1/extensions", installExtensionReq)
|
||||
httpRequestCheckOk[any](http.MethodDelete, "/api/v1/extensions/"+identity, nil)
|
||||
extensions := httpRequestCheckOk[[]*download.Extension](http.MethodGet, "/api/v1/extensions", nil)
|
||||
if len(extensions) != 0 {
|
||||
t.Errorf("TestDeleteExtension() got = %v, want %v", len(extensions), 0)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestUpdateCheckExtension(t *testing.T) {
|
||||
doTest(func() {
|
||||
identity := httpRequestCheckOk[string](http.MethodPost, "/api/v1/extensions", installExtensionReq)
|
||||
resp := httpRequestCheckOk[*model.UpdateCheckExtensionResp](http.MethodGet, "/api/v1/extensions/"+identity+"/update", nil)
|
||||
// no new version
|
||||
if resp.NewVersion != "" {
|
||||
t.Errorf("UpdateCheckExtension() got = %v, want %v", resp.NewVersion, "")
|
||||
}
|
||||
// force update
|
||||
httpRequestCheckOk[any](http.MethodPost, "/api/v1/extensions/"+identity+"/update", nil)
|
||||
})
|
||||
}
|
||||
|
||||
func TestFsExtension(t *testing.T) {
|
||||
doTest(func() {
|
||||
identity := httpRequestCheckOk[string](http.MethodPost, "/api/v1/extensions", installExtensionReq)
|
||||
statusCode, _ := doHttpRequest0(http.MethodGet, "/fs/extensions/"+identity+"/icon.png", nil, nil)
|
||||
if statusCode != http.StatusOK {
|
||||
t.Errorf("FsExtension() got = %v, want %v", statusCode, http.StatusOK)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestFsExtensionFail(t *testing.T) {
|
||||
doTest(func() {
|
||||
statusCode, _ := doHttpRequest0(http.MethodGet, "/fs/extensions/not_exist/icon.png", nil, nil)
|
||||
if statusCode != http.StatusNotFound {
|
||||
t.Errorf("TestFsExtensionFail() got = %v, want %v", statusCode, http.StatusNotFound)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestDoProxy(t *testing.T) {
|
||||
doTest(func() {
|
||||
code, respBody := doHttpRequest0(http.MethodGet, "/api/v1/proxy", map[string]string{
|
||||
@@ -341,15 +493,18 @@ func doTest(handler func()) {
|
||||
var cfg = &model.StartConfig{}
|
||||
cfg.Init()
|
||||
cfg.Storage = storage
|
||||
cfg.StorageDir = ".test_storage"
|
||||
fileListener := doStart(cfg)
|
||||
defer func() {
|
||||
if err := fileListener.Close(); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
Stop()
|
||||
Downloader.Clear()
|
||||
}()
|
||||
defer Downloader.Clear()
|
||||
defer func() {
|
||||
Downloader.PauseAll()
|
||||
|
||||
tasks := Downloader.GetTasks()
|
||||
taskIds := make([]string, len(tasks))
|
||||
for i, task := range tasks {
|
||||
@@ -358,6 +513,7 @@ func doTest(handler func()) {
|
||||
for _, id := range taskIds {
|
||||
Downloader.Delete(id, true)
|
||||
}
|
||||
os.RemoveAll(cfg.StorageDir)
|
||||
}()
|
||||
taskReq.URL = "http://" + fileListener.Addr().String() + "/" + test.BuildName
|
||||
handler()
|
||||
|
||||
@@ -0,0 +1,75 @@
|
||||
package util
|
||||
|
||||
import (
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Match url with pattern by chrome extension match pattern style
|
||||
// https://developer.chrome.com/docs/extensions/mv3/match_patterns/
|
||||
func Match(pattern string, u string) bool {
|
||||
scheme, host, path := parsePattern(pattern)
|
||||
url, err := url.Parse(u)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
if scheme != "*" && scheme != url.Scheme {
|
||||
return false
|
||||
}
|
||||
if !matchHost(host, url.Hostname()) {
|
||||
return false
|
||||
}
|
||||
if !matchPath(path, url.Path) {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func parsePattern(pattern string) (scheme string, host string, path string) {
|
||||
parts := strings.Split(pattern, "://")
|
||||
if len(parts) == 2 {
|
||||
scheme = parts[0]
|
||||
pattern = parts[1]
|
||||
} else {
|
||||
scheme = ""
|
||||
}
|
||||
parts = strings.SplitN(pattern, "/", 2)
|
||||
if len(parts) == 2 {
|
||||
host = parts[0]
|
||||
path = "/" + parts[1]
|
||||
} else {
|
||||
host = pattern
|
||||
path = "/"
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func matchHost(pattern string, host string) bool {
|
||||
if pattern == "*" {
|
||||
return true
|
||||
}
|
||||
if strings.HasPrefix(pattern, "*.") {
|
||||
return strings.HasSuffix(host, pattern[1:])
|
||||
}
|
||||
return pattern == host
|
||||
}
|
||||
|
||||
func matchPath(pattern string, path string) bool {
|
||||
if pattern == "*" {
|
||||
return true
|
||||
}
|
||||
if !strings.HasSuffix(pattern, "*") && !strings.HasSuffix(pattern, "/") {
|
||||
pattern += "/"
|
||||
}
|
||||
if !strings.HasSuffix(path, "/") {
|
||||
path += "/"
|
||||
}
|
||||
|
||||
if strings.Contains(pattern, "*") {
|
||||
pattern = strings.Replace(pattern, "*", ".*", -1)
|
||||
matched, _ := regexp.MatchString("^"+pattern+"$", path)
|
||||
return matched
|
||||
}
|
||||
return pattern == path
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
package util
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestMatch(t *testing.T) {
|
||||
tests := []struct {
|
||||
pattern string
|
||||
urls []string
|
||||
want bool
|
||||
}{
|
||||
{"*://*/*", []string{"https://www.google.com/", "http://example.org/foo/bar.html"}, true},
|
||||
{"https://*/*", []string{"https://www.google.com", "https://example.org/foo/bar.html"}, true},
|
||||
{"*://www.google.com", []string{"https://www.google.com/", "https://www.google.com"}, true},
|
||||
{"*://*.google.com/", []string{"https://a.www.google.com/", "https://c.www.google.com/", "https://www.google.com/"}, true},
|
||||
{"https://*/foo*", []string{"https://www.google.com/foo", "https://example.com/foo/bar.html"}, true},
|
||||
{"https://www.google.com/*/b/*", []string{"https://www.google.com/a/b", "https://www.google.com/a/b/c"}, true},
|
||||
{"https://*.google.com/foo*bar", []string{"https://www.google.com/foo/baz/bar", "https://docs.google.com/foobar"}, true},
|
||||
{"https://www.google.com/*abc*", []string{"https://www.google.com/abc", "https://www.google.com/123abc", "https://www.google.com/abc456", "https://www.google.com/123abc456"}, true},
|
||||
{"https://example.org/foo/bar.html", []string{"https://example.org/foo/bar.html"}, true},
|
||||
{"http://127.0.0.1/*", []string{"http://127.0.0.1/", "http://127.0.0.1/foo/bar.html"}, true},
|
||||
{"*://mail.google.com/*", []string{"http://mail.google.com/foo/baz/bar", "https://mail.google.com/foobar"}, true},
|
||||
{"https://www.google.com/", []string{"http://www.google.com/"}, false},
|
||||
{"www.google.com/", []string{"http://www.google.com/", "https://www.google.com/"}, false},
|
||||
{"www.google.com/*c", []string{"https://www.google.com/a", "https://www.google.com/b"}, false},
|
||||
{"https://*.example.org/*", []string{"https://www.google.com", "https://docs.google.com"}, false},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.pattern, func(t *testing.T) {
|
||||
for _, url := range tt.urls {
|
||||
if got := Match(tt.pattern, url); got != tt.want {
|
||||
t.Errorf("Match() = %v, want %v", got, tt.want)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
+99
-14
@@ -3,8 +3,10 @@ package util
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
syspath "path"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
@@ -64,22 +66,105 @@ func CheckDuplicateAndRename(path string) (string, error) {
|
||||
}
|
||||
}
|
||||
|
||||
// GetSingleDir get the top level single folder name,if not exist, return empty string
|
||||
func GetSingleDir(paths []string) string {
|
||||
if len(paths) == 0 {
|
||||
return ""
|
||||
// CopyDir Copy all files to the target directory, if the file already exists, it will be overwritten.
|
||||
// Remove target file if the source file is not exist.
|
||||
func CopyDir(source string, target string, excludeDir ...string) error {
|
||||
if err := os.MkdirAll(target, 0755); err != nil {
|
||||
return err
|
||||
}
|
||||
split := strings.Split(paths[0], "/")
|
||||
if len(split) == 0 || split[0] == "" {
|
||||
return ""
|
||||
}
|
||||
dir := split[0]
|
||||
for i := 1; i < len(paths); i++ {
|
||||
if !strings.HasPrefix(paths[i], dir) {
|
||||
return ""
|
||||
if err := filepath.Walk(source, func(path string, info os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if info.IsDir() {
|
||||
if len(excludeDir) > 0 {
|
||||
for _, dir := range excludeDir {
|
||||
if info.IsDir() && info.Name() == dir {
|
||||
return filepath.SkipDir
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
relPath, err := filepath.Rel(source, path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
targetPath := filepath.Join(target, relPath)
|
||||
if err := os.MkdirAll(filepath.Dir(targetPath), 0755); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := copyForce(path, targetPath); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
return dir
|
||||
|
||||
if err := filepath.Walk(target, func(path string, info os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if info.IsDir() {
|
||||
if len(excludeDir) > 0 {
|
||||
for _, dir := range excludeDir {
|
||||
if info.IsDir() && info.Name() == dir {
|
||||
return filepath.SkipDir
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
relPath, err := filepath.Rel(target, path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
targetPath := filepath.Join(target, relPath)
|
||||
sourcePath := filepath.Join(source, relPath)
|
||||
// if source file is not exist, remove target file
|
||||
if _, err := os.Stat(sourcePath); os.IsNotExist(err) {
|
||||
if err := SafeRemove(targetPath); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// RmAndMkDirAll remove and create directory, if the directory already exists, it will be overwritten.
|
||||
func RmAndMkDirAll(path string) error {
|
||||
if err := os.RemoveAll(path); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.MkdirAll(path, 0755); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// copy file, if the target file already exists, it will be overwritten.
|
||||
func copyForce(source string, target string) error {
|
||||
sourceFile, err := os.Open(source)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer sourceFile.Close()
|
||||
|
||||
targetFile, err := os.Create(target)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer targetFile.Close()
|
||||
|
||||
_, err = io.Copy(targetFile, sourceFile)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func CreateDirIfNotExist(dir string) error {
|
||||
@@ -92,7 +177,7 @@ func CreateDirIfNotExist(dir string) error {
|
||||
// IsExistsFile check file exists and is a file
|
||||
func IsExistsFile(path string) bool {
|
||||
info, err := os.Stat(path)
|
||||
// 判断路径是否存在，并且是一个文件
|
||||
// if file exists and is a file
|
||||
if err == nil && !info.IsDir() {
|
||||
return true
|
||||
}
|
||||
|
||||
+1
-62
@@ -78,7 +78,7 @@ func TestFilepath(t *testing.T) {
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if got := Filepath(tt.args.path, tt.args.originName, tt.args.customName); got != tt.want {
|
||||
t.Errorf("Filepath() = %v, want %v", got, tt.want)
|
||||
t.Errorf("SingleFilepath() = %v, want %v", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -105,67 +105,6 @@ func TestSafeRemove(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetSingleDir(t *testing.T) {
|
||||
type args struct {
|
||||
paths []string
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
args args
|
||||
want string
|
||||
}{
|
||||
{
|
||||
name: "empty",
|
||||
args: args{
|
||||
paths: []string{},
|
||||
},
|
||||
want: "",
|
||||
},
|
||||
{
|
||||
name: "blank",
|
||||
args: args{
|
||||
paths: []string{""},
|
||||
},
|
||||
want: "",
|
||||
},
|
||||
{
|
||||
name: "blank multi",
|
||||
args: args{
|
||||
paths: []string{"", "a"},
|
||||
},
|
||||
want: "",
|
||||
},
|
||||
{
|
||||
name: "multi dir not single",
|
||||
args: args{
|
||||
paths: []string{"a", "b"},
|
||||
},
|
||||
want: "",
|
||||
},
|
||||
{
|
||||
name: "single dir",
|
||||
args: args{
|
||||
paths: []string{"a"},
|
||||
},
|
||||
want: "a",
|
||||
},
|
||||
{
|
||||
name: "multi dir single",
|
||||
args: args{
|
||||
paths: []string{"a/b", "a/c"},
|
||||
},
|
||||
want: "a",
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if got := GetSingleDir(tt.args.paths); got != tt.want {
|
||||
t.Errorf("IsSingleDir() = %v, want %v", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckDuplicateAndRename(t *testing.T) {
|
||||
doCheckDuplicateAndRename(t, []string{}, "a.txt", "a.txt")
|
||||
doCheckDuplicateAndRename(t, []string{"a.txt"}, "a.txt", "a (1).txt")
|
||||
|
||||
Reference in new issue
Block a user