nprasad2077/gopeed
1
0 Fork 0
mirror of https://github.com/GopeedLab/gopeed.git synced 2026-11-15 12:53:41 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: reuse fetcher (#71)

Browse Source
This commit is contained in:
Li Wei authored and GitHub committed 2023-02-06 20:58:13 +08:00
1 parent 17cb59c606
commit 1a3c8c026b
36 files changed
+475 -297
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

pkg/base/model.go
-1
View File
@@ -12,7 +12,6 @@ type Request struct {
// 资源信息
type Resource struct {
Req *Request `json:"req"`
// 资源名称
Name string `json:"name"`
// 资源总大小
pkg/download/downloader.go
+58 -34
View File
@@ -2,6 +2,7 @@ package download
import (
"errors"
gonanoid "github.com/matoous/go-nanoid/v2"
"github.com/monkeyWie/gopeed/internal/controller"
"github.com/monkeyWie/gopeed/internal/fetcher"
"github.com/monkeyWie/gopeed/pkg/base"
@@ -31,6 +32,7 @@ type Progress struct {
type Downloader struct {
fetchBuilders map[string]fetcher.FetcherBuilder
fetcherMap map[string]fetcher.Fetcher
storage Storage
tasks []*Task
listener Listener
@@ -47,7 +49,9 @@ func NewDownloader(cfg *DownloaderConfig) *Downloader {
cfg.Init()
d := &Downloader{
fetchBuilders: make(map[string]fetcher.FetcherBuilder),
fetchBuilders: make(map[string]fetcher.FetcherBuilder),
fetcherMap: make(map[string]fetcher.Fetcher),
refreshInterval: cfg.RefreshInterval,
storage: cfg.Storage,
lock: &sync.Mutex{},
@@ -128,18 +132,16 @@ func (d *Downloader) Setup() error {
return nil
}
func (d *Downloader) buildFetcher(url string) (fetcher.FetcherBuilder, fetcher.Fetcher, error) {
func (d *Downloader) parseFb(url string) (fetcher.FetcherBuilder, error) {
schema := util.ParseSchema(url)
fetchBuilder, ok := d.fetchBuilders[schema]
if !ok {
fetchBuilder, ok = d.fetchBuilders[util.FileSchema]
}
if ok {
fetcher := fetchBuilder.Build()
d.setupFetcher(fetcher)
return fetchBuilder, fetcher, nil
return fetchBuilder, nil
}
return nil, nil, errors.New("unsupported protocol")
return nil, errors.New("unsupported protocol")
}
func (d *Downloader) setupFetcher(fetcher fetcher.Fetcher) {
@@ -150,26 +152,49 @@ func (d *Downloader) setupFetcher(fetcher fetcher.Fetcher) {
fetcher.Setup(ctl)
}
func (d *Downloader) Resolve(req *base.Request) (*base.Resource, error) {
_, fetcher, err := d.buildFetcher(req.URL)
func (d *Downloader) Resolve(req *base.Request) (rr *ResolveResult, err error) {
fb, err := d.parseFb(req.URL)
if err != nil {
return nil, err
return
}
res, err := fetcher.Resolve(req)
fetcher := fb.Build()
d.setupFetcher(fetcher)
err = fetcher.Resolve(req)
if err != nil {
return nil, err
return
}
return res, nil
rrId, err := gonanoid.New()
if err != nil {
return
}
d.fetcherMap[rrId] = fetcher
rr = &ResolveResult{
ID: rrId,
Res: fetcher.Meta().Res,
}
return
}
func (d *Downloader) Create(res *base.Resource, opts *base.Options) (taskId string, err error) {
func (d *Downloader) DirectCreate(req *base.Request, opts *base.Options) (taskId string, err error) {
rr, err := d.Resolve(req)
if err != nil {
return
}
return d.Create(rr.ID, opts)
}
func (d *Downloader) Create(rrId string, opts *base.Options) (taskId string, err error) {
if opts == nil {
opts = &base.Options{}
}
fetcher, ok := d.fetcherMap[rrId]
if !ok {
return "", errors.New("invalid resource id")
}
delete(d.fetcherMap, rrId)
if len(opts.SelectFiles) == 0 {
opts.SelectFiles = make([]int, 0)
for i := range res.Files {
for i := range fetcher.Meta().Res.Files {
opts.SelectFiles = append(opts.SelectFiles, i)
}
}
@@ -183,25 +208,24 @@ func (d *Downloader) Create(res *base.Resource, opts *base.Options) (taskId stri
}
}
fetcherBuilder, fetcher, err := d.buildFetcher(res.Req.URL)
fb, err := d.parseFb(fetcher.Meta().Req.URL)
if err != nil {
return
}
err = fetcher.Create(res, opts)
err = fetcher.Create(opts)
if err != nil {
return
}
task := NewTask()
task.fetcherBuilder = fetcherBuilder
task.fetcherBuilder = fb
task.fetcher = fetcher
task.Res = res
task.Opts = opts
task.Meta = fetcher.Meta()
task.Progress = &Progress{}
task.Status = base.DownloadStatusRunning
// calculate select files size
for _, selectIndex := range opts.SelectFiles {
task.Size += res.Files[selectIndex].Size
task.Size += fetcher.Meta().Res.Files[selectIndex].Size
}
initTask(task)
task.timer.Start()
@@ -303,10 +327,10 @@ func (d *Downloader) Delete(id string, force bool) (err error) {
}
if force {
names := make([]string, 0)
for _, file := range task.Res.Files {
names = append(names, util.Filepath(file.Path, file.Name, task.Opts.Name))
for _, file := range task.Meta.Res.Files {
names = append(names, util.Filepath(file.Path, file.Name, task.Meta.Opts.Name))
}
util.SafeRemoveAll(task.Opts.Path, names)
util.SafeRemoveAll(task.Meta.Opts.Path, names)
}
d.emit(EventKeyDelete, task)
task = nil
@@ -403,7 +427,7 @@ func (d *Downloader) watch(task *Task) {
task.Progress.Used = task.timer.Used()
if task.Size == 0 {
task.Size = task.fetcher.Progress().TotalDownloaded()
task.Res.Size = task.Size
task.Meta.Res.Size = task.Size
}
used := task.Progress.Used / int64(time.Second)
if used == 0 {
@@ -420,19 +444,19 @@ func (d *Downloader) watch(task *Task) {
func (d *Downloader) restoreFetcher(task *Task) error {
if task.fetcher == nil {
fetcherBuilder, _, err := d.buildFetcher(task.Res.Req.URL)
fb, err := d.parseFb(task.Meta.Req.URL)
if err != nil {
return err
}
task.fetcherBuilder = fetcherBuilder
task.fetcherBuilder = fb
err = func() error {
v, f := fetcherBuilder.Restore()
v, f := fb.Restore()
if v != nil {
err := d.storage.Pop(bucketSave, task.ID, v)
if err != nil {
return err
}
task.fetcher = f(task.Res, task.Opts, v)
task.fetcher = f(task.Meta, v)
}
return nil
}()
@@ -440,10 +464,10 @@ func (d *Downloader) restoreFetcher(task *Task) error {
// TODO log
}
if task.fetcher == nil {
task.fetcher = fetcherBuilder.Build()
task.fetcher = fb.Build()
}
d.setupFetcher(task.fetcher)
task.fetcher.Create(task.Res, task.Opts)
task.fetcher.Create(task.Meta.Opts)
go d.watch(task)
}
return nil
@@ -472,7 +496,7 @@ func (b *boot) Extra(extra interface{}) *boot {
return b
}
func (b *boot) Resolve() (*base.Resource, error) {
func (b *boot) Resolve() (*ResolveResult, error) {
return defaultDownloader.Resolve(&base.Request{
URL: b.url,
Extra: b.extra,
@@ -485,12 +509,12 @@ func (b *boot) Listener(listener Listener) *boot {
}
func (b *boot) Create(opts *base.Options) (string, error) {
res, err := b.Resolve()
rr, err := b.Resolve()
if err != nil {
return "", err
}
defaultDownloader.Listener(b.listener)
return defaultDownloader.Create(res, opts)
return defaultDownloader.Create(rr.ID, opts)
}
func Boot() *boot {
pkg/download/downloader_test.go
+7 -8
View File
@@ -22,13 +22,12 @@ func TestDownloader_Resolve(t *testing.T) {
req := &base.Request{
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
}
res, err := downloader.Resolve(req)
rr, err := downloader.Resolve(req)
if err != nil {
t.Fatal(err)
}
want := &base.Resource{
Name: test.BuildName,
Req: req,
Size: test.BuildSize,
Range: true,
Files: []*base.FileInfo{
@@ -39,8 +38,8 @@ func TestDownloader_Resolve(t *testing.T) {
},
},
}
if !reflect.DeepEqual(want, res) {
t.Errorf("Resolve() got = %v, want %v", res, want)
if !reflect.DeepEqual(want, rr.Res) {
t.Errorf("Resolve() got = %v, want %v", rr.Res, want)
}
}
@@ -56,7 +55,7 @@ func TestDownloader_Create(t *testing.T) {
req := &base.Request{
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
}
res, err := downloader.Resolve(req)
rr, err := downloader.Resolve(req)
if err != nil {
t.Fatal(err)
}
@@ -68,7 +67,7 @@ func TestDownloader_Create(t *testing.T) {
wg.Done()
}
})
_, err = downloader.Create(res, &base.Options{
_, err = downloader.Create(rr.ID, &base.Options{
Path: test.Dir,
Name: test.DownloadName,
Extra: http.OptsExtra{
@@ -100,12 +99,12 @@ func TestDownloader_StoreAndRestore(t *testing.T) {
req := &base.Request{
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
}
res, err := downloader.Resolve(req)
rr, err := downloader.Resolve(req)
if err != nil {
t.Fatal(err)
}
id, err := downloader.Create(res, &base.Options{
id, err := downloader.Create(rr.ID, &base.Options{
Path: test.Dir,
Name: test.DownloadName,
Extra: http.OptsExtra{
pkg/download/model.go
+47
View File
@@ -1,12 +1,59 @@
package download
import (
gonanoid "github.com/matoous/go-nanoid/v2"
"github.com/monkeyWie/gopeed/internal/controller"
"github.com/monkeyWie/gopeed/internal/fetcher"
"github.com/monkeyWie/gopeed/internal/protocol/bt"
"github.com/monkeyWie/gopeed/internal/protocol/http"
"github.com/monkeyWie/gopeed/pkg/base"
"github.com/monkeyWie/gopeed/pkg/util"
"sync"
"time"
)
type ResolveResult struct {
ID string `json:"id"`
Res *base.Resource `json:"res"`
}
type Task struct {
ID string `json:"id"`
Meta *fetcher.FetcherMeta `json:"meta"`
Status base.Status `json:"status"`
Progress *Progress `json:"progress"`
Size int64 `json:"size"`
CreatedAt time.Time `json:"createdAt"`
fetcherBuilder fetcher.FetcherBuilder
fetcher fetcher.Fetcher
timer *util.Timer
lock *sync.Mutex
}
func NewTask() *Task {
id, err := gonanoid.New()
if err != nil {
panic(err)
}
return &Task{
ID: id,
Status: base.DownloadStatusReady,
CreatedAt: time.Now(),
}
}
func (t *Task) clone() *Task {
return &Task{
ID: t.ID,
Meta: t.Meta,
Status: t.Status,
Progress: t.Progress,
Size: t.Size,
CreatedAt: t.CreatedAt,
}
}
type DownloaderConfig struct {
Controller *controller.Controller
FetchBuilders []fetcher.FetcherBuilder
pkg/download/task.go
-49
View File
@@ -1,49 +0,0 @@
package download
import (
gonanoid "github.com/matoous/go-nanoid/v2"
"github.com/monkeyWie/gopeed/internal/fetcher"
"github.com/monkeyWie/gopeed/pkg/base"
"github.com/monkeyWie/gopeed/pkg/util"
"sync"
"time"
)
type Task struct {
ID string `json:"id"`
Res *base.Resource `json:"res"`
Opts *base.Options `json:"opts"`
Status base.Status `json:"status"`
Progress *Progress `json:"progress"`
Size int64 `json:"size"`
CreatedAt time.Time `json:"createdAt"`
fetcherBuilder fetcher.FetcherBuilder
fetcher fetcher.Fetcher
timer *util.Timer
lock *sync.Mutex
}
func NewTask() *Task {
id, err := gonanoid.New()
if err != nil {
panic(err)
}
return &Task{
ID: id,
Status: base.DownloadStatusReady,
CreatedAt: time.Now(),
}
}
func (t *Task) clone() *Task {
return &Task{
ID: t.ID,
Res: t.Res,
Opts: t.Opts,
Status: t.Status,
Progress: t.Progress,
Size: t.Size,
CreatedAt: t.CreatedAt,
}
}
pkg/rest/api.go
+14 -3
View File
@@ -16,19 +16,30 @@ import (
func Resolve(w http.ResponseWriter, r *http.Request) {
var req base.Request
if util.ReadJson(w, r, &req) {
resource, err := Downloader.Resolve(&req)
rr, err := Downloader.Resolve(&req)
if err != nil {
util.WriteJson(w, http.StatusInternalServerError, model.NewResultWithMsg(err.Error()))
return
}
util.WriteJsonOk(w, model.NewResultWithData(resource))
util.WriteJsonOk(w, model.NewResultWithData(rr))
}
}
func CreateTask(w http.ResponseWriter, r *http.Request) {
var req model.CreateTask
if util.ReadJson(w, r, &req) {
taskId, err := Downloader.Create(req.Res, req.Opts)
var (
taskId string
err error
)
if req.Rid != "" {
taskId, err = Downloader.Create(req.Rid, req.Opts)
} else if req.Req != nil {
taskId, err = Downloader.DirectCreate(req.Req, req.Opts)
} else {
util.WriteJson(w, http.StatusBadRequest, model.NewResultWithMsg("param is required: rid or req"))
return
}
if err != nil {
util.WriteJson(w, http.StatusInternalServerError, model.NewResultWithMsg(err.Error()))
return
pkg/rest/model/task.go
+3 -2
View File
@@ -3,6 +3,7 @@ package model
import "github.com/monkeyWie/gopeed/pkg/base"
type CreateTask struct {
Res *base.Resource `json:"res"`
Opts *base.Options `json:"opts"`
Rid string `json:"rid"`
Req *base.Request `json:"req"`
Opts *base.Options `json:"opts"`
}
pkg/rest/server_test.go
+6 -6
View File
@@ -34,7 +34,6 @@ var (
}
taskRes = &base.Resource{
Name: test.BuildName,
Req: taskReq,
Size: test.BuildSize,
Range: true,
Files: []*base.FileInfo{
@@ -46,7 +45,7 @@ var (
},
}
createReq = &model.CreateTask{
Res: taskRes,
Req: taskReq,
Opts: &base.Options{
Path: test.Dir,
Name: test.DownloadName,
@@ -59,9 +58,9 @@ var (
func TestResolve(t *testing.T) {
doTest(func() {
resp := httpRequestCheckOk[*base.Resource](http.MethodPost, "/api/v1/resolve", taskReq)
if !test.JsonEqual(taskRes, resp) {
t.Errorf("Resolve() got = %v, want %v", test.ToJson(resp), test.ToJson(taskRes))
resp := httpRequestCheckOk[*download.ResolveResult](http.MethodPost, "/api/v1/resolve", taskReq)
if !test.JsonEqual(taskRes, resp.Res) {
t.Errorf("Resolve() got = %v, want %v", test.ToJson(resp.Res), test.ToJson(taskRes))
}
})
}
@@ -148,6 +147,7 @@ func TestPauseAndContinueTask(t *testing.T) {
func TestDeleteTask(t *testing.T) {
doTest(func() {
taskId := httpRequestCheckOk[string](http.MethodPost, "/api/v1/tasks", createReq)
time.Sleep(time.Millisecond * 200)
httpRequestCheckOk[any](http.MethodDelete, "/api/v1/tasks/"+taskId, nil)
code, _ := httpRequest[*download.Task](http.MethodGet, "/api/v1/tasks/"+taskId, nil)
checkCode(code, http.StatusNotFound)
@@ -157,7 +157,7 @@ func TestDeleteTask(t *testing.T) {
func TestDeleteTaskForce(t *testing.T) {
doTest(func() {
taskId := httpRequestCheckOk[string](http.MethodPost, "/api/v1/tasks", createReq)
time.Sleep(time.Millisecond * 500)
time.Sleep(time.Millisecond * 200)
httpRequestCheckOk[any](http.MethodDelete, "/api/v1/tasks/"+taskId+"?force=true", nil)
code, _ := httpRequest[*download.Task](http.MethodGet, "/api/v1/tasks/"+taskId, nil)
checkCode(code, http.StatusNotFound)
pkg/util/path.go
+6 -6
View File
@@ -4,11 +4,11 @@ import (
"errors"
"io"
"os"
"path/filepath"
syspath "path"
)
func Dir(path string) string {
dir := filepath.Dir(path)
dir := syspath.Dir(path)
if dir == "." {
return ""
}
@@ -19,7 +19,7 @@ func Filepath(path string, originName string, customName string) string {
if customName == "" {
customName = originName
}
return filepath.Join(path, customName)
return syspath.Join(path, customName)
}
// SafeRemove remove file safely, ignoring errors if the path does not exist.
@@ -33,7 +33,7 @@ func SafeRemove(name string) error {
// SafeRemoveAll remove file and parent directories safely
func SafeRemoveAll(path string, names []string) error {
for _, name := range names {
err := SafeRemove(filepath.Join(path, name))
err := SafeRemove(syspath.Join(path, name))
if err != nil {
return err
}
@@ -45,12 +45,12 @@ func SafeRemoveAll(path string, names []string) error {
}
func safeRemoveParent(path string, subPath string) error {
currPath := filepath.Dir(subPath)
currPath := syspath.Dir(subPath)
if currPath == "." {
return nil
}
// if directory is empty, remove it
dir := filepath.Join(path, filepath.Dir(subPath))
dir := syspath.Join(path, syspath.Dir(subPath))
empty, err := isEmpty(dir)
if err != nil {
return err
pkg/util/path_test.go
+8 -2
View File
@@ -87,10 +87,13 @@ func TestFilepath(t *testing.T) {
func TestSafeRemove(t *testing.T) {
name := "test_safe_remove.data"
_, err := os.Create(name)
file, err := os.Create(name)
if err != nil {
t.Fatal(err)
}
if err := file.Close(); err != nil {
t.Fatal(err)
}
if err := SafeRemove(name); err != nil {
t.Fatal(err)
}
@@ -185,10 +188,13 @@ func preCreate(t *testing.T, path string, names []string) {
if err := os.MkdirAll(filepath.Dir(fullPath), 0755); err != nil {
t.Fatal(err)
}
_, err := os.Create(fullPath)
file, err := os.Create(fullPath)
if err != nil {
t.Fatal(err)
}
if err := file.Close(); err != nil {
t.Fatal(err)
}
}
}