feat: reuse fetcher (#71)

This commit is contained in:
Li Wei authored and GitHub committed 2023-02-06 20:58:13 +08:00
1 parent 17cb59c606
commit 1a3c8c026b
36 files changed
+475 -297

No files matched your search

+17
View File
@@ -0,0 +1,17 @@
# Gopeed contributors guide
Firstly, thank you for your interest in contributing to Gopeed. This guide will help you better participate in the development of Gopeed.
## Branch description
This project only has one main branch, namely the `main` branch. If you want to participate in the development of Gopeed, please fork this project first, and then develop in your fork project. After development is completed, submit a PR to this project and merge it into the `main` branch.
## Local development
It is recommended to develop and debug through the web. First, start the backend service, the code is located in `cmd/api/main.go`, the default port of the service is `9999`, and then start the front-end project in `debug` mode to run.
## Translation
The internationalization files of Gopeed are located in the `ui/flutter/lib/i18n/langs` directory. You only need to add the corresponding language file in this directory.
> Tip: Please refer to `en_us.dart` for translation.
+17
View File
@@ -0,0 +1,17 @@
# Gopeed 贡献指南
首先感谢您对贡献代码感兴趣,这份指南将帮助您更好的参与到 Gopeed 的开发中来。
## 分支说明
本项目只有一个主分支,即 `main` 分支,如果您想要参与到 Gopeed 的开发中来,请先 fork 本项目,然后在您的 fork 项目中进行开发,开发完成后再向本项目提交 PR,合并到 `main` 分支。
## 本地开发
建议通过 web 端进行开发调试,首先启动后端服务,代码位于 `cmd/api/main.go` ,服务启动默认端口为 `9999`,然后以 `debug` 模式启动前端项目即可运行。
## 翻译
Gopeed 的国际化文件位于 `ui/flutter/lib/i18n/langs` 目录下,只需要在该目录下添加对应的语言文件即可。
> 请注意以 `en_us.dart` 为参照进行翻译。
+4 -2
View File
@@ -91,6 +91,10 @@ This project is divided into two parts, the front end uses `flutter`, the back e
git clone [email protected]:monkeyWie/gopeed.git git clone [email protected]:monkeyWie/gopeed.git
``` ```
### Contributing
Please refer to [CONTRIBUTING.md](/CONTRIBUTING.md)
### Build ### Build
#### Desktop #### Desktop
@@ -144,8 +148,6 @@ flutter build apk
#### Web #### Web
Web platform communicates directly with the backend http server, no additional environment is required.
command: command:
```bash ```bash
+5 -3
View File
@@ -91,6 +91,10 @@ docker-compose up -d
git clone [email protected]:monkeyWie/gopeed.git git clone [email protected]:monkeyWie/gopeed.git
``` ```
### 贡献代码
请参考[贡献指南](CONTRIBUTING_zh-CN.md)
### 编译 ### 编译
#### 桌面端 #### 桌面端
@@ -142,9 +146,7 @@ cd ui/flutter
flutter build apk flutter build apk
``` ```
#### Web 端(推荐本地调试使用) #### Web 端
Web 端直接与后端 http 服务通讯,不需要额外准备环境。
构建命令: 构建命令:
+2 -2
View File
@@ -56,10 +56,10 @@ var (
) )
func printProgress(task *download.Task, title string) { func printProgress(task *download.Task, title string) {
rate := float64(task.Progress.Downloaded) / float64(task.Res.Size) rate := float64(task.Progress.Downloaded) / float64(task.Meta.Res.Size)
completeWidth := int(progressWidth * rate) completeWidth := int(progressWidth * rate)
speed := util.ByteFmt(task.Progress.Speed) speed := util.ByteFmt(task.Progress.Speed)
totalSize := util.ByteFmt(task.Res.Size) totalSize := util.ByteFmt(task.Meta.Res.Size)
sb.WriteString(fmt.Sprintf("\r%s [", title)) sb.WriteString(fmt.Sprintf("\r%s [", title))
for i := 0; i < progressWidth; i++ { for i := 0; i < progressWidth; i++ {
if i < completeWidth { if i < completeWidth {
+19 -8
View File
@@ -6,24 +6,34 @@ import (
) )
// Fetcher defines the interface for a download protocol. // Fetcher defines the interface for a download protocol.
// One fetcher for each download task
type Fetcher interface { type Fetcher interface {
// Name return the name of the protocol. // Name return the name of the protocol.
Name() string Name() string
Setup(ctl *controller.Controller) error Setup(ctl *controller.Controller) error
// Resolve resource info from request // Resolve resource info from request
Resolve(req *base.Request) (res *base.Resource, err error) Resolve(req *base.Request) error
// Create ready to download, but not started // Create ready to download, but not started
Create(res *base.Resource, opts *base.Options) (err error) Create(opts *base.Options) error
Start() (err error) Start() error
Pause() (err error) Pause() error
Continue() (err error) Continue() error
Close() (err error) Close() error
// Meta returns the meta information of the download.
Meta() *FetcherMeta
// Progress returns the progress of the download. // Progress returns the progress of the download.
Progress() Progress Progress() Progress
// Wait for the download to complete, this method will block until the download is done. // Wait for the download to complete, this method will block until the download is done.
Wait() (err error) Wait() error
}
// FetcherMeta defines the meta information of a fetcher.
type FetcherMeta struct {
Req *base.Request `json:"req"`
Res *base.Resource `json:"res"`
Opts *base.Options `json:"opts"`
} }
// FetcherBuilder defines the interface for a fetcher builder. // FetcherBuilder defines the interface for a fetcher builder.
@@ -36,11 +46,12 @@ type FetcherBuilder interface {
// Store fetcher // Store fetcher
Store(fetcher Fetcher) (any, error) Store(fetcher Fetcher) (any, error)
// Restore fetcher // Restore fetcher
Restore() (v any, f func(res *base.Resource, opts *base.Options, v any) Fetcher) Restore() (v any, f func(meta *FetcherMeta, v any) Fetcher)
} }
type DefaultFetcher struct { type DefaultFetcher struct {
Ctl *controller.Controller Ctl *controller.Controller
Meta *FetcherMeta
DoneCh chan error DoneCh chan error
} }
+68 -63
View File
@@ -17,11 +17,10 @@ import (
var client *torrent.Client var client *torrent.Client
type Fetcher struct { type Fetcher struct {
Ctl *controller.Controller ctl *controller.Controller
torrent *torrent.Torrent torrent *torrent.Torrent
res *base.Resource meta *fetcher.FetcherMeta
opts *base.Options
ready atomic.Bool ready atomic.Bool
progress fetcher.Progress progress fetcher.Progress
@@ -34,7 +33,10 @@ func (f *Fetcher) Name() string {
} }
func (f *Fetcher) Setup(ctl *controller.Controller) (err error) { func (f *Fetcher) Setup(ctl *controller.Controller) (err error) {
f.Ctl = ctl f.ctl = ctl
if f.meta == nil {
f.meta = &fetcher.FetcherMeta{}
}
var once sync.Once var once sync.Once
once.Do(func() { once.Do(func() {
cfg := torrent.NewDefaultClientConfig() cfg := torrent.NewDefaultClientConfig()
@@ -66,46 +68,12 @@ func (f *Fetcher) Setup(ctl *controller.Controller) (err error) {
return return
} }
func (f *Fetcher) Wait() (err error) { func (f *Fetcher) Resolve(req *base.Request) error {
for { if err := f.addTorrent(req.URL); err != nil {
if f.ready.Load() { return err
done := true
for _, selectIndex := range f.opts.SelectFiles {
file := f.torrent.Files()[selectIndex]
if file.BytesCompleted() < file.Length() {
done = false
break
}
}
if done {
// remove unselected files
for i, file := range f.torrent.Files() {
selected := false
for _, selectIndex := range f.opts.SelectFiles {
if i == selectIndex {
selected = true
break
}
}
if !selected {
util.SafeRemove(filepath.Join(f.opts.Path, file.Path()))
}
}
break
}
}
time.Sleep(time.Millisecond * 500)
}
return nil
}
func (f *Fetcher) Resolve(req *base.Request) (res *base.Resource, err error) {
if err = f.addTorrent(req.URL, true); err != nil {
return
} }
defer f.torrent.Drop() defer f.torrent.Drop()
res = &base.Resource{ res := &base.Resource{
Req: req,
Name: f.torrent.Name(), Name: f.torrent.Name(),
Range: true, Range: true,
Files: make([]*base.FileInfo, len(f.torrent.Files())), Files: make([]*base.FileInfo, len(f.torrent.Files())),
@@ -119,36 +87,39 @@ func (f *Fetcher) Resolve(req *base.Request) (res *base.Resource, err error) {
} }
res.Size += file.Length() res.Size += file.Length()
} }
return f.meta.Req = req
f.meta.Res = res
return nil
} }
func (f *Fetcher) Create(res *base.Resource, opts *base.Options) (err error) { func (f *Fetcher) Create(opts *base.Options) (err error) {
f.res = res f.meta.Opts = opts
f.opts = opts if len(opts.SelectFiles) == 0 {
if len(f.opts.SelectFiles) == 0 { opts.SelectFiles = make([]int, 0)
f.opts.SelectFiles = make([]int, 0)
for i := range f.torrent.Files() { for i := range f.torrent.Files() {
f.opts.SelectFiles = append(f.opts.SelectFiles, i) opts.SelectFiles = append(opts.SelectFiles, i)
} }
} }
if f.opts.Path != "" { if opts.Path != "" {
f.torrentPaths[res.Hash] = f.opts.Path f.torrentPaths[f.meta.Res.Hash] = f.meta.Opts.Path
} }
f.progress = make(fetcher.Progress, len(f.opts.SelectFiles)) f.progress = make(fetcher.Progress, len(f.meta.Opts.SelectFiles))
f.ready.Store(false) f.ready.Store(false)
return nil return nil
} }
func (f *Fetcher) Start() (err error) { func (f *Fetcher) Start() (err error) {
if err = f.addTorrent(f.res.Req.URL, false); err != nil { if f.torrent == nil {
return if err = f.addTorrent(f.meta.Req.URL); err != nil {
return
}
} }
files := f.torrent.Files() files := f.torrent.Files()
if len(f.opts.SelectFiles) == len(files) { if len(f.meta.Opts.SelectFiles) == len(files) {
f.torrent.DownloadAll() f.torrent.DownloadAll()
} else { } else {
for _, selectIndex := range f.opts.SelectFiles { for _, selectIndex := range f.meta.Opts.SelectFiles {
file := files[selectIndex] file := files[selectIndex]
file.Download() file.Download()
} }
@@ -171,19 +142,56 @@ func (f *Fetcher) Close() (err error) {
return nil return nil
} }
func (f *Fetcher) Wait() (err error) {
for {
if f.ready.Load() {
done := true
for _, selectIndex := range f.meta.Opts.SelectFiles {
file := f.torrent.Files()[selectIndex]
if file.BytesCompleted() < file.Length() {
done = false
break
}
}
if done {
// remove unselected files
for i, file := range f.torrent.Files() {
selected := false
for _, selectIndex := range f.meta.Opts.SelectFiles {
if i == selectIndex {
selected = true
break
}
}
if !selected {
util.SafeRemove(filepath.Join(f.meta.Opts.Path, file.Path()))
}
}
break
}
}
time.Sleep(time.Millisecond * 500)
}
return nil
}
func (f *Fetcher) Meta() *fetcher.FetcherMeta {
return f.meta
}
func (f *Fetcher) Progress() fetcher.Progress { func (f *Fetcher) Progress() fetcher.Progress {
if !f.ready.Load() { if !f.ready.Load() {
return f.progress return f.progress
} }
for i := range f.progress { for i := range f.progress {
selectIndex := f.opts.SelectFiles[i] selectIndex := f.meta.Opts.SelectFiles[i]
file := f.torrent.Files()[selectIndex] file := f.torrent.Files()[selectIndex]
f.progress[i] = file.BytesCompleted() f.progress[i] = file.BytesCompleted()
} }
return f.progress return f.progress
} }
func (f *Fetcher) addTorrent(url string, resolve bool) (err error) { func (f *Fetcher) addTorrent(url string) (err error) {
schema := util.ParseSchema(url) schema := util.ParseSchema(url)
if schema == "MAGNET" { if schema == "MAGNET" {
f.torrent, err = client.AddMagnet(url) f.torrent, err = client.AddMagnet(url)
@@ -194,7 +202,7 @@ func (f *Fetcher) addTorrent(url string, resolve bool) (err error) {
return return
} }
var cfg config var cfg config
exist, err := f.Ctl.GetConfig(&cfg) exist, err := f.ctl.GetConfig(&cfg)
if err != nil { if err != nil {
return return
} }
@@ -206,9 +214,6 @@ func (f *Fetcher) addTorrent(url string, resolve bool) (err error) {
f.torrent.AddTrackers(announceList) f.torrent.AddTrackers(announceList)
} }
<-f.torrent.GotInfo() <-f.torrent.GotInfo()
if resolve {
return
}
f.ready.Store(true) f.ready.Store(true)
return return
} }
@@ -230,6 +235,6 @@ func (fb *FetcherBuilder) Store(f fetcher.Fetcher) (data any, err error) {
return nil, nil return nil, nil
} }
func (fb *FetcherBuilder) Restore() (v any, f func(res *base.Resource, opts *base.Options, v any) fetcher.Fetcher) { func (fb *FetcherBuilder) Restore() (v any, f func(meta *fetcher.FetcherMeta, v any) fetcher.Fetcher) {
return nil, nil return nil, nil
} }
+3 -4
View File
@@ -19,7 +19,7 @@ func TestFetcher_Config(t *testing.T) {
} }
func doResolve(t *testing.T, fetcher fetcher.Fetcher) { func doResolve(t *testing.T, fetcher fetcher.Fetcher) {
res, err := fetcher.Resolve(&base.Request{ err := fetcher.Resolve(&base.Request{
URL: "./testdata/ubuntu-22.04-live-server-amd64.iso.torrent", URL: "./testdata/ubuntu-22.04-live-server-amd64.iso.torrent",
}) })
if err != nil { if err != nil {
@@ -27,7 +27,6 @@ func doResolve(t *testing.T, fetcher fetcher.Fetcher) {
} }
want := &base.Resource{ want := &base.Resource{
Req: res.Req,
Name: "ubuntu-22.04-live-server-amd64.iso", Name: "ubuntu-22.04-live-server-amd64.iso",
Size: 1466714112, Size: 1466714112,
Range: true, Range: true,
@@ -39,8 +38,8 @@ func doResolve(t *testing.T, fetcher fetcher.Fetcher) {
}, },
Hash: "8a55cfbd5ca5d11507364765936c4f9e55b253ed", Hash: "8a55cfbd5ca5d11507364765936c4f9e55b253ed",
} }
if !reflect.DeepEqual(want, res) { if !reflect.DeepEqual(want, fetcher.Meta().Res) {
t.Errorf("Resolve() got = %v, want %v", res, want) t.Errorf("Resolve() got = %v, want %v", fetcher.Meta().Res, want)
} }
} }
+57 -44
View File
@@ -4,6 +4,7 @@ import (
"bytes" "bytes"
"context" "context"
"fmt" "fmt"
"github.com/monkeyWie/gopeed/internal/controller"
"github.com/monkeyWie/gopeed/internal/fetcher" "github.com/monkeyWie/gopeed/internal/fetcher"
"github.com/monkeyWie/gopeed/pkg/base" "github.com/monkeyWie/gopeed/pkg/base"
fhttp "github.com/monkeyWie/gopeed/pkg/protocol/http" fhttp "github.com/monkeyWie/gopeed/pkg/protocol/http"
@@ -16,9 +17,7 @@ import (
"net/url" "net/url"
"os" "os"
"path" "path"
"path/filepath"
"strconv" "strconv"
"strings"
"time" "time"
) )
@@ -51,10 +50,10 @@ func newChunk(begin int64, end int64) *chunk {
} }
type Fetcher struct { type Fetcher struct {
*fetcher.DefaultFetcher ctl *controller.Controller
doneCh chan error
res *base.Resource meta *fetcher.FetcherMeta
opts *base.Options
status base.Status status base.Status
chunks []*chunk chunks []*chunk
@@ -68,25 +67,34 @@ func (f *Fetcher) Name() string {
return "http" return "http"
} }
func (f *Fetcher) Resolve(req *base.Request) (*base.Resource, error) { func (f *Fetcher) Setup(ctl *controller.Controller) (err error) {
f.ctl = ctl
f.doneCh = make(chan error, 1)
f.pauseCh = make(chan interface{}, 1)
if f.meta == nil {
f.meta = &fetcher.FetcherMeta{}
}
return
}
func (f *Fetcher) Resolve(req *base.Request) error {
if err := base.ParseReqExtra[fhttp.ReqExtra](req); err != nil { if err := base.ParseReqExtra[fhttp.ReqExtra](req); err != nil {
return nil, err return err
} }
httpReq, err := buildRequest(nil, req) httpReq, err := buildRequest(nil, req)
if err != nil { if err != nil {
return nil, err return err
} }
client := buildClient() client := buildClient()
// 只访问一个字节,测试资源是否支持Range请求 // 只访问一个字节,测试资源是否支持Range请求
httpReq.Header.Set(base.HttpHeaderRange, fmt.Sprintf(base.HttpHeaderRangeFormat, 0, 0)) httpReq.Header.Set(base.HttpHeaderRange, fmt.Sprintf(base.HttpHeaderRangeFormat, 0, 0))
httpResp, err := client.Do(httpReq) httpResp, err := client.Do(httpReq)
if err != nil { if err != nil {
return nil, err return err
} }
// 拿到响应头就关闭,不用加defer // 拿到响应头就关闭,不用加defer
httpResp.Body.Close() httpResp.Body.Close()
res := &base.Resource{ res := &base.Resource{
Req: req,
Range: false, Range: false,
Files: []*base.FileInfo{}, Files: []*base.FileInfo{},
} }
@@ -98,7 +106,7 @@ func (f *Fetcher) Resolve(req *base.Request) (*base.Resource, error) {
if contentTotal != "" { if contentTotal != "" {
parse, err := strconv.ParseInt(contentTotal, 10, 64) parse, err := strconv.ParseInt(contentTotal, 10, 64)
if err != nil { if err != nil {
return nil, err return err
} }
res.Size = parse res.Size = parse
} }
@@ -108,12 +116,12 @@ func (f *Fetcher) Resolve(req *base.Request) (*base.Resource, error) {
if contentLength != "" { if contentLength != "" {
parse, err := strconv.ParseInt(contentLength, 10, 64) parse, err := strconv.ParseInt(contentLength, 10, 64)
if err != nil { if err != nil {
return nil, err return err
} }
res.Size = parse res.Size = parse
} }
} else { } else {
return nil, NewRequestError(httpResp.StatusCode, httpResp.Status) return NewRequestError(httpResp.StatusCode, httpResp.Status)
} }
file := &base.FileInfo{ file := &base.FileInfo{
Size: res.Size, Size: res.Size,
@@ -127,27 +135,28 @@ func (f *Fetcher) Resolve(req *base.Request) (*base.Resource, error) {
} }
} }
// Get file filePath by URL // Get file filePath by URL
if file.Name == "" && strings.Count(req.URL, "/") > 2 { if file.Name == "" {
file.Name = filepath.Base(req.URL) file.Name = path.Base(httpReq.URL.Path)
} }
// unknown file filePath // unknown file filePath
if file.Name == "" { if file.Name == "" || file.Name == "/" || file.Name == "." {
file.Name = "unknown" file.Name = "unknown"
} }
res.Files = append(res.Files, file) res.Files = append(res.Files, file)
res.Name = file.Name res.Name = file.Name
return res, nil f.meta.Req = req
f.meta.Res = res
return nil
} }
func (f *Fetcher) Create(res *base.Resource, opts *base.Options) error { func (f *Fetcher) Create(opts *base.Options) error {
f.res = res f.meta.Opts = opts
f.opts = opts
f.status = base.DownloadStatusReady f.status = base.DownloadStatusReady
if err := base.ParseReqExtra[fhttp.ReqExtra](res.Req); err != nil { if err := base.ParseReqExtra[fhttp.ReqExtra](f.meta.Req); err != nil {
return err return err
} }
if err := base.ParseOptsExtra[fhttp.OptsExtra](opts); err != nil { if err := base.ParseOptsExtra[fhttp.OptsExtra](f.meta.Opts); err != nil {
return err return err
} }
if opts.Extra == nil { if opts.Extra == nil {
@@ -156,7 +165,7 @@ func (f *Fetcher) Create(res *base.Resource, opts *base.Options) error {
extra := opts.Extra.(*fhttp.OptsExtra) extra := opts.Extra.(*fhttp.OptsExtra)
if extra.Connections == 0 { if extra.Connections == 0 {
var cfg config var cfg config
exist, err := f.Ctl.GetConfig(&cfg) exist, err := f.ctl.GetConfig(&cfg)
if err != nil { if err != nil {
return err return err
} }
@@ -172,7 +181,7 @@ func (f *Fetcher) Create(res *base.Resource, opts *base.Options) error {
func (f *Fetcher) Start() (err error) { func (f *Fetcher) Start() (err error) {
// 创建文件 // 创建文件
name := f.filepath() name := f.filepath()
f.file, err = f.Ctl.Touch(name, f.res.Size) f.file, err = f.ctl.Touch(name, f.meta.Res.Size)
if err != nil { if err != nil {
return err return err
} }
@@ -215,6 +224,10 @@ func (f *Fetcher) Close() (err error) {
return return
} }
func (f *Fetcher) Meta() *fetcher.FetcherMeta {
return f.meta
}
func (f *Fetcher) Progress() fetcher.Progress { func (f *Fetcher) Progress() fetcher.Progress {
p := make(fetcher.Progress, 0) p := make(fetcher.Progress, 0)
if len(f.chunks) > 0 { if len(f.chunks) > 0 {
@@ -227,14 +240,18 @@ func (f *Fetcher) Progress() fetcher.Progress {
return p return p
} }
func (f *Fetcher) Wait() (err error) {
return <-f.doneCh
}
func (f *Fetcher) filepath() string { func (f *Fetcher) filepath() string {
return util.Filepath(f.opts.Path, f.res.Files[0].Name, f.opts.Name) return util.Filepath(f.meta.Opts.Path, f.meta.Res.Files[0].Name, f.meta.Opts.Name)
} }
func (f *Fetcher) fetch() { func (f *Fetcher) fetch() {
f.ctx, f.cancel = context.WithCancel(context.Background()) f.ctx, f.cancel = context.WithCancel(context.Background())
eg, _ := errgroup.WithContext(f.ctx) eg, _ := errgroup.WithContext(f.ctx)
for i := 0; i < f.opts.Extra.(*fhttp.OptsExtra).Connections; i++ { for i := 0; i < f.meta.Opts.Extra.(*fhttp.OptsExtra).Connections; i++ {
i := i i := i
eg.Go(func() error { eg.Go(func() error {
return f.fetchChunk(i) return f.fetchChunk(i)
@@ -253,7 +270,7 @@ func (f *Fetcher) fetch() {
} else { } else {
f.status = base.DownloadStatusDone f.status = base.DownloadStatusDone
} }
f.DoneCh <- err f.doneCh <- err
} }
}() }()
} }
@@ -261,7 +278,7 @@ func (f *Fetcher) fetch() {
func (f *Fetcher) fetchChunk(index int) (err error) { func (f *Fetcher) fetchChunk(index int) (err error) {
chunk := f.chunks[index] chunk := f.chunks[index]
httpReq, err := buildRequest(f.ctx, f.res.Req) httpReq, err := buildRequest(f.ctx, f.meta.Req)
if err != nil { if err != nil {
return err return err
} }
@@ -283,7 +300,7 @@ func (f *Fetcher) fetchChunk(index int) (err error) {
resp *http.Response resp *http.Response
retry bool retry bool
) )
if f.res.Range { if f.meta.Res.Range {
httpReq.Header.Set(base.HttpHeaderRange, httpReq.Header.Set(base.HttpHeaderRange,
fmt.Sprintf(base.HttpHeaderRangeFormat, chunk.Begin+chunk.Downloaded, chunk.End)) fmt.Sprintf(base.HttpHeaderRangeFormat, chunk.Begin+chunk.Downloaded, chunk.End))
} else { } else {
@@ -337,7 +354,7 @@ func (f *Fetcher) fetchChunk(index int) (err error) {
return return
} }
isComplete := f.res.Range && chunk.Downloaded >= chunk.End-chunk.Begin+1 isComplete := f.meta.Res.Range && chunk.Downloaded >= chunk.End-chunk.Begin+1
if f.status == base.DownloadStatusPause && !isComplete { if f.status == base.DownloadStatusPause && !isComplete {
chunk.Status = base.DownloadStatusPause chunk.Status = base.DownloadStatusPause
return return
@@ -348,10 +365,10 @@ func (f *Fetcher) fetchChunk(index int) (err error) {
} }
func (f *Fetcher) splitChunk() (chunks []*chunk) { func (f *Fetcher) splitChunk() (chunks []*chunk) {
if f.res.Range { if f.meta.Res.Range {
connections := f.opts.Extra.(*fhttp.OptsExtra).Connections connections := f.meta.Opts.Extra.(*fhttp.OptsExtra).Connections
// 每个连接平均需要下载的分块大小 // 每个连接平均需要下载的分块大小
chunkSize := f.res.Size / int64(connections) chunkSize := f.meta.Res.Size / int64(connections)
chunks = make([]*chunk, connections) chunks = make([]*chunk, connections)
for i := 0; i < connections; i++ { for i := 0; i < connections; i++ {
var ( var (
@@ -360,7 +377,7 @@ func (f *Fetcher) splitChunk() (chunks []*chunk) {
) )
if i == connections-1 { if i == connections-1 {
// 最后一个分块需要保证把文件下载完 // 最后一个分块需要保证把文件下载完
end = f.res.Size - 1 end = f.meta.Res.Size - 1
} else { } else {
end = begin + chunkSize - 1 end = begin + chunkSize - 1
} }
@@ -440,10 +457,7 @@ func (fb *FetcherBuilder) Schemes() []string {
} }
func (fb *FetcherBuilder) Build() fetcher.Fetcher { func (fb *FetcherBuilder) Build() fetcher.Fetcher {
return &Fetcher{ return &Fetcher{}
DefaultFetcher: new(fetcher.DefaultFetcher),
pauseCh: make(chan interface{}),
}
} }
func (fb *FetcherBuilder) Store(f fetcher.Fetcher) (data any, err error) { func (fb *FetcherBuilder) Store(f fetcher.Fetcher) (data any, err error) {
@@ -453,16 +467,15 @@ func (fb *FetcherBuilder) Store(f fetcher.Fetcher) (data any, err error) {
}, nil }, nil
} }
func (fb *FetcherBuilder) Restore() (v any, f func(res *base.Resource, opts *base.Options, v any) fetcher.Fetcher) { func (fb *FetcherBuilder) Restore() (v any, f func(meta *fetcher.FetcherMeta, v any) fetcher.Fetcher) {
return &fetcherData{}, func(res *base.Resource, opts *base.Options, v any) fetcher.Fetcher { return &fetcherData{}, func(meta *fetcher.FetcherMeta, v any) fetcher.Fetcher {
fd := v.(*fetcherData) fd := v.(*fetcherData)
fb := &FetcherBuilder{} fb := &FetcherBuilder{}
fetcher := fb.Build().(*Fetcher) fetcher := fb.Build().(*Fetcher)
fetcher.res = res fetcher.meta = meta
fetcher.opts = opts
fetcher.status = base.DownloadStatusPause fetcher.status = base.DownloadStatusPause
base.ParseReqExtra[fhttp.ReqExtra](res.Req) base.ParseReqExtra[fhttp.ReqExtra](fetcher.meta.Req)
base.ParseOptsExtra[fhttp.OptsExtra](opts) base.ParseOptsExtra[fhttp.OptsExtra](fetcher.meta.Opts)
if len(fd.Chunks) == 0 { if len(fd.Chunks) == 0 {
fetcher.chunks = fetcher.splitChunk() fetcher.chunks = fetcher.splitChunk()
} else { } else {
+17 -19
View File
@@ -43,15 +43,14 @@ func testResolve(startTestServer func() net.Listener, want *base.Resource, t *te
listener := startTestServer() listener := startTestServer()
defer listener.Close() defer listener.Close()
fetcher := buildFetcher() fetcher := buildFetcher()
res, err := fetcher.Resolve(&base.Request{ err := fetcher.Resolve(&base.Request{
URL: "http://" + listener.Addr().String() + "/" + test.BuildName, URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
}) })
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
res.Req = nil if !reflect.DeepEqual(want, fetcher.meta.Res) {
if !reflect.DeepEqual(want, res) { t.Errorf("Resolve() got = %v, want %v", fetcher.meta.Res, want)
t.Errorf("Resolve() got = %v, want %v", res, want)
} }
} }
@@ -114,7 +113,7 @@ func TestFetcher_DownloadResume(t *testing.T) {
} }
func TestFetcher_Config(t *testing.T) { func TestFetcher_Config(t *testing.T) {
fetcher, _, _ := doDownloadReady(buildConfigFetcher(), test.StartTestFileServer(), 0, t) fetcher := doDownloadReady(buildConfigFetcher(), test.StartTestFileServer(), 0, t)
err := fetcher.Start() err := fetcher.Start()
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
@@ -130,13 +129,12 @@ func TestFetcher_Config(t *testing.T) {
} }
} }
func downloadReady(listener net.Listener, connections int, t *testing.T) (fetcher fetcher.Fetcher, res *base.Resource, opts *base.Options) { func downloadReady(listener net.Listener, connections int, t *testing.T) fetcher.Fetcher {
return doDownloadReady(buildFetcher(), listener, connections, t) return doDownloadReady(buildFetcher(), listener, connections, t)
} }
func doDownloadReady(f fetcher.Fetcher, listener net.Listener, connections int, t *testing.T) (fetcher fetcher.Fetcher, res *base.Resource, opts *base.Options) { func doDownloadReady(f fetcher.Fetcher, listener net.Listener, connections int, t *testing.T) fetcher.Fetcher {
fetcher = f err := f.Resolve(&base.Request{
res, err := fetcher.Resolve(&base.Request{
URL: "http://" + listener.Addr().String() + "/" + test.BuildName, URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
}) })
if err != nil { if err != nil {
@@ -148,20 +146,20 @@ func doDownloadReady(f fetcher.Fetcher, listener net.Listener, connections int,
Connections: connections, Connections: connections,
} }
} }
opts = &base.Options{ opts := &base.Options{
Name: test.DownloadName, Name: test.DownloadName,
Path: test.Dir, Path: test.Dir,
Extra: extra, Extra: extra,
} }
err = fetcher.Create(res, opts) err = f.Create(opts)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
return return f
} }
func downloadNormal(listener net.Listener, connections int, t *testing.T) { func downloadNormal(listener net.Listener, connections int, t *testing.T) {
fetcher, _, _ := downloadReady(listener, connections, t) fetcher := downloadReady(listener, connections, t)
err := fetcher.Start() err := fetcher.Start()
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
@@ -178,8 +176,8 @@ func downloadNormal(listener net.Listener, connections int, t *testing.T) {
} }
func downloadPost(listener net.Listener, connections int, t *testing.T) { func downloadPost(listener net.Listener, connections int, t *testing.T) {
fetcher, _, _ := downloadReady(listener, connections, t) fetcher := downloadReady(listener, connections, t)
fetcher.(*Fetcher).res.Req.Extra = &http.ReqExtra{ fetcher.Meta().Req.Extra = &http.ReqExtra{
Method: "POST", Method: "POST",
Header: map[string]string{ Header: map[string]string{
"Authorization": "Bearer 123456", "Authorization": "Bearer 123456",
@@ -202,7 +200,7 @@ func downloadPost(listener net.Listener, connections int, t *testing.T) {
} }
func downloadContinue(listener net.Listener, connections int, t *testing.T) { func downloadContinue(listener net.Listener, connections int, t *testing.T) {
fetcher, _, _ := downloadReady(listener, connections, t) fetcher := downloadReady(listener, connections, t)
err := fetcher.Start() err := fetcher.Start()
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
@@ -227,7 +225,7 @@ func downloadContinue(listener net.Listener, connections int, t *testing.T) {
} }
func downloadError(listener net.Listener, connections int, t *testing.T) { func downloadError(listener net.Listener, connections int, t *testing.T) {
fetcher, _, _ := downloadReady(listener, connections, t) fetcher := downloadReady(listener, connections, t)
err := fetcher.Start() err := fetcher.Start()
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
@@ -239,7 +237,7 @@ func downloadError(listener net.Listener, connections int, t *testing.T) {
} }
func downloadResume(listener net.Listener, connections int, t *testing.T) { func downloadResume(listener net.Listener, connections int, t *testing.T) {
fetcher, res, opts := downloadReady(listener, connections, t) fetcher := downloadReady(listener, connections, t)
err := fetcher.Start() err := fetcher.Start()
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
@@ -255,7 +253,7 @@ func downloadResume(listener net.Listener, connections int, t *testing.T) {
fetcher.Pause() fetcher.Pause()
_, f := fb.Restore() _, f := fb.Restore()
f(res, opts, data) f(fetcher.Meta(), data)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
-1
View File
@@ -12,7 +12,6 @@ type Request struct {
// 资源信息 // 资源信息
type Resource struct { type Resource struct {
Req *Request `json:"req"`
// 资源名称 // 资源名称
Name string `json:"name"` Name string `json:"name"`
// 资源总大小 // 资源总大小
+58 -34
View File
@@ -2,6 +2,7 @@ package download
import ( import (
"errors" "errors"
gonanoid "github.com/matoous/go-nanoid/v2"
"github.com/monkeyWie/gopeed/internal/controller" "github.com/monkeyWie/gopeed/internal/controller"
"github.com/monkeyWie/gopeed/internal/fetcher" "github.com/monkeyWie/gopeed/internal/fetcher"
"github.com/monkeyWie/gopeed/pkg/base" "github.com/monkeyWie/gopeed/pkg/base"
@@ -31,6 +32,7 @@ type Progress struct {
type Downloader struct { type Downloader struct {
fetchBuilders map[string]fetcher.FetcherBuilder fetchBuilders map[string]fetcher.FetcherBuilder
fetcherMap map[string]fetcher.Fetcher
storage Storage storage Storage
tasks []*Task tasks []*Task
listener Listener listener Listener
@@ -47,7 +49,9 @@ func NewDownloader(cfg *DownloaderConfig) *Downloader {
cfg.Init() cfg.Init()
d := &Downloader{ d := &Downloader{
fetchBuilders: make(map[string]fetcher.FetcherBuilder), fetchBuilders: make(map[string]fetcher.FetcherBuilder),
fetcherMap: make(map[string]fetcher.Fetcher),
refreshInterval: cfg.RefreshInterval, refreshInterval: cfg.RefreshInterval,
storage: cfg.Storage, storage: cfg.Storage,
lock: &sync.Mutex{}, lock: &sync.Mutex{},
@@ -128,18 +132,16 @@ func (d *Downloader) Setup() error {
return nil return nil
} }
func (d *Downloader) buildFetcher(url string) (fetcher.FetcherBuilder, fetcher.Fetcher, error) { func (d *Downloader) parseFb(url string) (fetcher.FetcherBuilder, error) {
schema := util.ParseSchema(url) schema := util.ParseSchema(url)
fetchBuilder, ok := d.fetchBuilders[schema] fetchBuilder, ok := d.fetchBuilders[schema]
if !ok { if !ok {
fetchBuilder, ok = d.fetchBuilders[util.FileSchema] fetchBuilder, ok = d.fetchBuilders[util.FileSchema]
} }
if ok { if ok {
fetcher := fetchBuilder.Build() return fetchBuilder, nil
d.setupFetcher(fetcher)
return fetchBuilder, fetcher, nil
} }
return nil, nil, errors.New("unsupported protocol") return nil, errors.New("unsupported protocol")
} }
func (d *Downloader) setupFetcher(fetcher fetcher.Fetcher) { func (d *Downloader) setupFetcher(fetcher fetcher.Fetcher) {
@@ -150,26 +152,49 @@ func (d *Downloader) setupFetcher(fetcher fetcher.Fetcher) {
fetcher.Setup(ctl) fetcher.Setup(ctl)
} }
func (d *Downloader) Resolve(req *base.Request) (*base.Resource, error) { func (d *Downloader) Resolve(req *base.Request) (rr *ResolveResult, err error) {
_, fetcher, err := d.buildFetcher(req.URL) fb, err := d.parseFb(req.URL)
if err != nil { if err != nil {
return nil, err return
} }
res, err := fetcher.Resolve(req) fetcher := fb.Build()
d.setupFetcher(fetcher)
err = fetcher.Resolve(req)
if err != nil { if err != nil {
return nil, err return
} }
rrId, err := gonanoid.New()
return res, nil if err != nil {
return
}
d.fetcherMap[rrId] = fetcher
rr = &ResolveResult{
ID: rrId,
Res: fetcher.Meta().Res,
}
return
} }
func (d *Downloader) Create(res *base.Resource, opts *base.Options) (taskId string, err error) { func (d *Downloader) DirectCreate(req *base.Request, opts *base.Options) (taskId string, err error) {
rr, err := d.Resolve(req)
if err != nil {
return
}
return d.Create(rr.ID, opts)
}
func (d *Downloader) Create(rrId string, opts *base.Options) (taskId string, err error) {
if opts == nil { if opts == nil {
opts = &base.Options{} opts = &base.Options{}
} }
fetcher, ok := d.fetcherMap[rrId]
if !ok {
return "", errors.New("invalid resource id")
}
delete(d.fetcherMap, rrId)
if len(opts.SelectFiles) == 0 { if len(opts.SelectFiles) == 0 {
opts.SelectFiles = make([]int, 0) opts.SelectFiles = make([]int, 0)
for i := range res.Files { for i := range fetcher.Meta().Res.Files {
opts.SelectFiles = append(opts.SelectFiles, i) opts.SelectFiles = append(opts.SelectFiles, i)
} }
} }
@@ -183,25 +208,24 @@ func (d *Downloader) Create(res *base.Resource, opts *base.Options) (taskId stri
} }
} }
fetcherBuilder, fetcher, err := d.buildFetcher(res.Req.URL) fb, err := d.parseFb(fetcher.Meta().Req.URL)
if err != nil { if err != nil {
return return
} }
err = fetcher.Create(res, opts) err = fetcher.Create(opts)
if err != nil { if err != nil {
return return
} }
task := NewTask() task := NewTask()
task.fetcherBuilder = fetcherBuilder task.fetcherBuilder = fb
task.fetcher = fetcher task.fetcher = fetcher
task.Res = res task.Meta = fetcher.Meta()
task.Opts = opts
task.Progress = &Progress{} task.Progress = &Progress{}
task.Status = base.DownloadStatusRunning task.Status = base.DownloadStatusRunning
// calculate select files size // calculate select files size
for _, selectIndex := range opts.SelectFiles { for _, selectIndex := range opts.SelectFiles {
task.Size += res.Files[selectIndex].Size task.Size += fetcher.Meta().Res.Files[selectIndex].Size
} }
initTask(task) initTask(task)
task.timer.Start() task.timer.Start()
@@ -303,10 +327,10 @@ func (d *Downloader) Delete(id string, force bool) (err error) {
} }
if force { if force {
names := make([]string, 0) names := make([]string, 0)
for _, file := range task.Res.Files { for _, file := range task.Meta.Res.Files {
names = append(names, util.Filepath(file.Path, file.Name, task.Opts.Name)) names = append(names, util.Filepath(file.Path, file.Name, task.Meta.Opts.Name))
} }
util.SafeRemoveAll(task.Opts.Path, names) util.SafeRemoveAll(task.Meta.Opts.Path, names)
} }
d.emit(EventKeyDelete, task) d.emit(EventKeyDelete, task)
task = nil task = nil
@@ -403,7 +427,7 @@ func (d *Downloader) watch(task *Task) {
task.Progress.Used = task.timer.Used() task.Progress.Used = task.timer.Used()
if task.Size == 0 { if task.Size == 0 {
task.Size = task.fetcher.Progress().TotalDownloaded() task.Size = task.fetcher.Progress().TotalDownloaded()
task.Res.Size = task.Size task.Meta.Res.Size = task.Size
} }
used := task.Progress.Used / int64(time.Second) used := task.Progress.Used / int64(time.Second)
if used == 0 { if used == 0 {
@@ -420,19 +444,19 @@ func (d *Downloader) watch(task *Task) {
func (d *Downloader) restoreFetcher(task *Task) error { func (d *Downloader) restoreFetcher(task *Task) error {
if task.fetcher == nil { if task.fetcher == nil {
fetcherBuilder, _, err := d.buildFetcher(task.Res.Req.URL) fb, err := d.parseFb(task.Meta.Req.URL)
if err != nil { if err != nil {
return err return err
} }
task.fetcherBuilder = fetcherBuilder task.fetcherBuilder = fb
err = func() error { err = func() error {
v, f := fetcherBuilder.Restore() v, f := fb.Restore()
if v != nil { if v != nil {
err := d.storage.Pop(bucketSave, task.ID, v) err := d.storage.Pop(bucketSave, task.ID, v)
if err != nil { if err != nil {
return err return err
} }
task.fetcher = f(task.Res, task.Opts, v) task.fetcher = f(task.Meta, v)
} }
return nil return nil
}() }()
@@ -440,10 +464,10 @@ func (d *Downloader) restoreFetcher(task *Task) error {
// TODO log // TODO log
} }
if task.fetcher == nil { if task.fetcher == nil {
task.fetcher = fetcherBuilder.Build() task.fetcher = fb.Build()
} }
d.setupFetcher(task.fetcher) d.setupFetcher(task.fetcher)
task.fetcher.Create(task.Res, task.Opts) task.fetcher.Create(task.Meta.Opts)
go d.watch(task) go d.watch(task)
} }
return nil return nil
@@ -472,7 +496,7 @@ func (b *boot) Extra(extra interface{}) *boot {
return b return b
} }
func (b *boot) Resolve() (*base.Resource, error) { func (b *boot) Resolve() (*ResolveResult, error) {
return defaultDownloader.Resolve(&base.Request{ return defaultDownloader.Resolve(&base.Request{
URL: b.url, URL: b.url,
Extra: b.extra, Extra: b.extra,
@@ -485,12 +509,12 @@ func (b *boot) Listener(listener Listener) *boot {
} }
func (b *boot) Create(opts *base.Options) (string, error) { func (b *boot) Create(opts *base.Options) (string, error) {
res, err := b.Resolve() rr, err := b.Resolve()
if err != nil { if err != nil {
return "", err return "", err
} }
defaultDownloader.Listener(b.listener) defaultDownloader.Listener(b.listener)
return defaultDownloader.Create(res, opts) return defaultDownloader.Create(rr.ID, opts)
} }
func Boot() *boot { func Boot() *boot {
+7 -8
View File
@@ -22,13 +22,12 @@ func TestDownloader_Resolve(t *testing.T) {
req := &base.Request{ req := &base.Request{
URL: "http://" + listener.Addr().String() + "/" + test.BuildName, URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
} }
res, err := downloader.Resolve(req) rr, err := downloader.Resolve(req)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
want := &base.Resource{ want := &base.Resource{
Name: test.BuildName, Name: test.BuildName,
Req: req,
Size: test.BuildSize, Size: test.BuildSize,
Range: true, Range: true,
Files: []*base.FileInfo{ Files: []*base.FileInfo{
@@ -39,8 +38,8 @@ func TestDownloader_Resolve(t *testing.T) {
}, },
}, },
} }
if !reflect.DeepEqual(want, res) { if !reflect.DeepEqual(want, rr.Res) {
t.Errorf("Resolve() got = %v, want %v", res, want) t.Errorf("Resolve() got = %v, want %v", rr.Res, want)
} }
} }
@@ -56,7 +55,7 @@ func TestDownloader_Create(t *testing.T) {
req := &base.Request{ req := &base.Request{
URL: "http://" + listener.Addr().String() + "/" + test.BuildName, URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
} }
res, err := downloader.Resolve(req) rr, err := downloader.Resolve(req)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -68,7 +67,7 @@ func TestDownloader_Create(t *testing.T) {
wg.Done() wg.Done()
} }
}) })
_, err = downloader.Create(res, &base.Options{ _, err = downloader.Create(rr.ID, &base.Options{
Path: test.Dir, Path: test.Dir,
Name: test.DownloadName, Name: test.DownloadName,
Extra: http.OptsExtra{ Extra: http.OptsExtra{
@@ -100,12 +99,12 @@ func TestDownloader_StoreAndRestore(t *testing.T) {
req := &base.Request{ req := &base.Request{
URL: "http://" + listener.Addr().String() + "/" + test.BuildName, URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
} }
res, err := downloader.Resolve(req) rr, err := downloader.Resolve(req)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
id, err := downloader.Create(res, &base.Options{ id, err := downloader.Create(rr.ID, &base.Options{
Path: test.Dir, Path: test.Dir,
Name: test.DownloadName, Name: test.DownloadName,
Extra: http.OptsExtra{ Extra: http.OptsExtra{
+47
View File
@@ -1,12 +1,59 @@
package download package download
import ( import (
gonanoid "github.com/matoous/go-nanoid/v2"
"github.com/monkeyWie/gopeed/internal/controller" "github.com/monkeyWie/gopeed/internal/controller"
"github.com/monkeyWie/gopeed/internal/fetcher" "github.com/monkeyWie/gopeed/internal/fetcher"
"github.com/monkeyWie/gopeed/internal/protocol/bt" "github.com/monkeyWie/gopeed/internal/protocol/bt"
"github.com/monkeyWie/gopeed/internal/protocol/http" "github.com/monkeyWie/gopeed/internal/protocol/http"
"github.com/monkeyWie/gopeed/pkg/base"
"github.com/monkeyWie/gopeed/pkg/util"
"sync"
"time"
) )
type ResolveResult struct {
ID string `json:"id"`
Res *base.Resource `json:"res"`
}
type Task struct {
ID string `json:"id"`
Meta *fetcher.FetcherMeta `json:"meta"`
Status base.Status `json:"status"`
Progress *Progress `json:"progress"`
Size int64 `json:"size"`
CreatedAt time.Time `json:"createdAt"`
fetcherBuilder fetcher.FetcherBuilder
fetcher fetcher.Fetcher
timer *util.Timer
lock *sync.Mutex
}
func NewTask() *Task {
id, err := gonanoid.New()
if err != nil {
panic(err)
}
return &Task{
ID: id,
Status: base.DownloadStatusReady,
CreatedAt: time.Now(),
}
}
func (t *Task) clone() *Task {
return &Task{
ID: t.ID,
Meta: t.Meta,
Status: t.Status,
Progress: t.Progress,
Size: t.Size,
CreatedAt: t.CreatedAt,
}
}
type DownloaderConfig struct { type DownloaderConfig struct {
Controller *controller.Controller Controller *controller.Controller
FetchBuilders []fetcher.FetcherBuilder FetchBuilders []fetcher.FetcherBuilder
-49
View File
@@ -1,49 +0,0 @@
package download
import (
gonanoid "github.com/matoous/go-nanoid/v2"
"github.com/monkeyWie/gopeed/internal/fetcher"
"github.com/monkeyWie/gopeed/pkg/base"
"github.com/monkeyWie/gopeed/pkg/util"
"sync"
"time"
)
type Task struct {
ID string `json:"id"`
Res *base.Resource `json:"res"`
Opts *base.Options `json:"opts"`
Status base.Status `json:"status"`
Progress *Progress `json:"progress"`
Size int64 `json:"size"`
CreatedAt time.Time `json:"createdAt"`
fetcherBuilder fetcher.FetcherBuilder
fetcher fetcher.Fetcher
timer *util.Timer
lock *sync.Mutex
}
func NewTask() *Task {
id, err := gonanoid.New()
if err != nil {
panic(err)
}
return &Task{
ID: id,
Status: base.DownloadStatusReady,
CreatedAt: time.Now(),
}
}
func (t *Task) clone() *Task {
return &Task{
ID: t.ID,
Res: t.Res,
Opts: t.Opts,
Status: t.Status,
Progress: t.Progress,
Size: t.Size,
CreatedAt: t.CreatedAt,
}
}
+14 -3
View File
@@ -16,19 +16,30 @@ import (
func Resolve(w http.ResponseWriter, r *http.Request) { func Resolve(w http.ResponseWriter, r *http.Request) {
var req base.Request var req base.Request
if util.ReadJson(w, r, &req) { if util.ReadJson(w, r, &req) {
resource, err := Downloader.Resolve(&req) rr, err := Downloader.Resolve(&req)
if err != nil { if err != nil {
util.WriteJson(w, http.StatusInternalServerError, model.NewResultWithMsg(err.Error())) util.WriteJson(w, http.StatusInternalServerError, model.NewResultWithMsg(err.Error()))
return return
} }
util.WriteJsonOk(w, model.NewResultWithData(resource)) util.WriteJsonOk(w, model.NewResultWithData(rr))
} }
} }
func CreateTask(w http.ResponseWriter, r *http.Request) { func CreateTask(w http.ResponseWriter, r *http.Request) {
var req model.CreateTask var req model.CreateTask
if util.ReadJson(w, r, &req) { if util.ReadJson(w, r, &req) {
taskId, err := Downloader.Create(req.Res, req.Opts) var (
taskId string
err error
)
if req.Rid != "" {
taskId, err = Downloader.Create(req.Rid, req.Opts)
} else if req.Req != nil {
taskId, err = Downloader.DirectCreate(req.Req, req.Opts)
} else {
util.WriteJson(w, http.StatusBadRequest, model.NewResultWithMsg("param is required: rid or req"))
return
}
if err != nil { if err != nil {
util.WriteJson(w, http.StatusInternalServerError, model.NewResultWithMsg(err.Error())) util.WriteJson(w, http.StatusInternalServerError, model.NewResultWithMsg(err.Error()))
return return
+3 -2
View File
@@ -3,6 +3,7 @@ package model
import "github.com/monkeyWie/gopeed/pkg/base" import "github.com/monkeyWie/gopeed/pkg/base"
type CreateTask struct { type CreateTask struct {
Res *base.Resource `json:"res"` Rid string `json:"rid"`
Opts *base.Options `json:"opts"` Req *base.Request `json:"req"`
Opts *base.Options `json:"opts"`
} }
+6 -6
View File
@@ -34,7 +34,6 @@ var (
} }
taskRes = &base.Resource{ taskRes = &base.Resource{
Name: test.BuildName, Name: test.BuildName,
Req: taskReq,
Size: test.BuildSize, Size: test.BuildSize,
Range: true, Range: true,
Files: []*base.FileInfo{ Files: []*base.FileInfo{
@@ -46,7 +45,7 @@ var (
}, },
} }
createReq = &model.CreateTask{ createReq = &model.CreateTask{
Res: taskRes, Req: taskReq,
Opts: &base.Options{ Opts: &base.Options{
Path: test.Dir, Path: test.Dir,
Name: test.DownloadName, Name: test.DownloadName,
@@ -59,9 +58,9 @@ var (
func TestResolve(t *testing.T) { func TestResolve(t *testing.T) {
doTest(func() { doTest(func() {
resp := httpRequestCheckOk[*base.Resource](http.MethodPost, "/api/v1/resolve", taskReq) resp := httpRequestCheckOk[*download.ResolveResult](http.MethodPost, "/api/v1/resolve", taskReq)
if !test.JsonEqual(taskRes, resp) { if !test.JsonEqual(taskRes, resp.Res) {
t.Errorf("Resolve() got = %v, want %v", test.ToJson(resp), test.ToJson(taskRes)) t.Errorf("Resolve() got = %v, want %v", test.ToJson(resp.Res), test.ToJson(taskRes))
} }
}) })
} }
@@ -148,6 +147,7 @@ func TestPauseAndContinueTask(t *testing.T) {
func TestDeleteTask(t *testing.T) { func TestDeleteTask(t *testing.T) {
doTest(func() { doTest(func() {
taskId := httpRequestCheckOk[string](http.MethodPost, "/api/v1/tasks", createReq) taskId := httpRequestCheckOk[string](http.MethodPost, "/api/v1/tasks", createReq)
time.Sleep(time.Millisecond * 200)
httpRequestCheckOk[any](http.MethodDelete, "/api/v1/tasks/"+taskId, nil) httpRequestCheckOk[any](http.MethodDelete, "/api/v1/tasks/"+taskId, nil)
code, _ := httpRequest[*download.Task](http.MethodGet, "/api/v1/tasks/"+taskId, nil) code, _ := httpRequest[*download.Task](http.MethodGet, "/api/v1/tasks/"+taskId, nil)
checkCode(code, http.StatusNotFound) checkCode(code, http.StatusNotFound)
@@ -157,7 +157,7 @@ func TestDeleteTask(t *testing.T) {
func TestDeleteTaskForce(t *testing.T) { func TestDeleteTaskForce(t *testing.T) {
doTest(func() { doTest(func() {
taskId := httpRequestCheckOk[string](http.MethodPost, "/api/v1/tasks", createReq) taskId := httpRequestCheckOk[string](http.MethodPost, "/api/v1/tasks", createReq)
time.Sleep(time.Millisecond * 500) time.Sleep(time.Millisecond * 200)
httpRequestCheckOk[any](http.MethodDelete, "/api/v1/tasks/"+taskId+"?force=true", nil) httpRequestCheckOk[any](http.MethodDelete, "/api/v1/tasks/"+taskId+"?force=true", nil)
code, _ := httpRequest[*download.Task](http.MethodGet, "/api/v1/tasks/"+taskId, nil) code, _ := httpRequest[*download.Task](http.MethodGet, "/api/v1/tasks/"+taskId, nil)
checkCode(code, http.StatusNotFound) checkCode(code, http.StatusNotFound)
+6 -6
View File
@@ -4,11 +4,11 @@ import (
"errors" "errors"
"io" "io"
"os" "os"
"path/filepath" syspath "path"
) )
func Dir(path string) string { func Dir(path string) string {
dir := filepath.Dir(path) dir := syspath.Dir(path)
if dir == "." { if dir == "." {
return "" return ""
} }
@@ -19,7 +19,7 @@ func Filepath(path string, originName string, customName string) string {
if customName == "" { if customName == "" {
customName = originName customName = originName
} }
return filepath.Join(path, customName) return syspath.Join(path, customName)
} }
// SafeRemove remove file safely, ignoring errors if the path does not exist. // SafeRemove remove file safely, ignoring errors if the path does not exist.
@@ -33,7 +33,7 @@ func SafeRemove(name string) error {
// SafeRemoveAll remove file and parent directories safely // SafeRemoveAll remove file and parent directories safely
func SafeRemoveAll(path string, names []string) error { func SafeRemoveAll(path string, names []string) error {
for _, name := range names { for _, name := range names {
err := SafeRemove(filepath.Join(path, name)) err := SafeRemove(syspath.Join(path, name))
if err != nil { if err != nil {
return err return err
} }
@@ -45,12 +45,12 @@ func SafeRemoveAll(path string, names []string) error {
} }
func safeRemoveParent(path string, subPath string) error { func safeRemoveParent(path string, subPath string) error {
currPath := filepath.Dir(subPath) currPath := syspath.Dir(subPath)
if currPath == "." { if currPath == "." {
return nil return nil
} }
// if directory is empty, remove it // if directory is empty, remove it
dir := filepath.Join(path, filepath.Dir(subPath)) dir := syspath.Join(path, syspath.Dir(subPath))
empty, err := isEmpty(dir) empty, err := isEmpty(dir)
if err != nil { if err != nil {
return err return err
+8 -2
View File
@@ -87,10 +87,13 @@ func TestFilepath(t *testing.T) {
func TestSafeRemove(t *testing.T) { func TestSafeRemove(t *testing.T) {
name := "test_safe_remove.data" name := "test_safe_remove.data"
_, err := os.Create(name) file, err := os.Create(name)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
if err := file.Close(); err != nil {
t.Fatal(err)
}
if err := SafeRemove(name); err != nil { if err := SafeRemove(name); err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -185,10 +188,13 @@ func preCreate(t *testing.T, path string, names []string) {
if err := os.MkdirAll(filepath.Dir(fullPath), 0755); err != nil { if err := os.MkdirAll(filepath.Dir(fullPath), 0755); err != nil {
t.Fatal(err) t.Fatal(err)
} }
_, err := os.Create(fullPath) file, err := os.Create(fullPath)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
if err := file.Close(); err != nil {
t.Fatal(err)
}
} }
} }
+4 -4
View File
@@ -3,10 +3,10 @@ import 'dart:io';
import 'package:dio/adapter.dart'; import 'package:dio/adapter.dart';
import 'package:dio/dio.dart'; import 'package:dio/dio.dart';
import 'package:flutter/foundation.dart'; import 'package:flutter/foundation.dart';
import 'model/resolve_result.dart';
import '../util/util.dart'; import '../util/util.dart';
import 'model/create_task.dart'; import 'model/create_task.dart';
import 'model/request.dart'; import 'model/request.dart';
import 'model/resource.dart';
import 'model/result.dart'; import 'model/result.dart';
import 'model/task.dart'; import 'model/task.dart';
@@ -82,10 +82,10 @@ Future<T> _parse<T>(
} }
} }
Future<Resource> resolve(Request request) async { Future<ResolveResult> resolve(Request request) async {
return _parse<Resource>( return _parse<ResolveResult>(
() => _client.dio.post("/api/v1/resolve", data: request), () => _client.dio.post("/api/v1/resolve", data: request),
(data) => Resource.fromJson(data)); (data) => ResolveResult.fromJson(data));
} }
Future<String> createTask(CreateTask createTask) async { Future<String> createTask(CreateTask createTask) async {
+5 -3
View File
@@ -1,17 +1,19 @@
import 'package:json_annotation/json_annotation.dart'; import 'package:json_annotation/json_annotation.dart';
import 'options.dart'; import 'options.dart';
import 'resource.dart'; import 'request.dart';
part 'create_task.g.dart'; part 'create_task.g.dart';
@JsonSerializable(explicitToJson: true) @JsonSerializable(explicitToJson: true)
class CreateTask { class CreateTask {
Resource res; String? rid;
Request? req;
Options? opts; Options? opts;
CreateTask({ CreateTask({
required this.res, this.rid,
this.req,
this.opts, this.opts,
}); });
+7 -4
View File
@@ -7,16 +7,17 @@ part of 'create_task.dart';
// ************************************************************************** // **************************************************************************
CreateTask _$CreateTaskFromJson(Map<String, dynamic> json) => CreateTask( CreateTask _$CreateTaskFromJson(Map<String, dynamic> json) => CreateTask(
res: Resource.fromJson(json['res'] as Map<String, dynamic>), rid: json['rid'] as String?,
req: json['req'] == null
? null
: Request.fromJson(json['req'] as Map<String, dynamic>),
opts: json['opts'] == null opts: json['opts'] == null
? null ? null
: Options.fromJson(json['opts'] as Map<String, dynamic>), : Options.fromJson(json['opts'] as Map<String, dynamic>),
); );
Map<String, dynamic> _$CreateTaskToJson(CreateTask instance) { Map<String, dynamic> _$CreateTaskToJson(CreateTask instance) {
final val = <String, dynamic>{ final val = <String, dynamic>{};
'res': instance.res.toJson(),
};
void writeNotNull(String key, dynamic value) { void writeNotNull(String key, dynamic value) {
if (value != null) { if (value != null) {
@@ -24,6 +25,8 @@ Map<String, dynamic> _$CreateTaskToJson(CreateTask instance) {
} }
} }
writeNotNull('rid', instance.rid);
writeNotNull('req', instance.req?.toJson());
writeNotNull('opts', instance.opts?.toJson()); writeNotNull('opts', instance.opts?.toJson());
return val; return val;
} }
@@ -22,8 +22,8 @@ class ProtocolConfig {
ProtocolConfig(); ProtocolConfig();
factory ProtocolConfig.fromJson(Map<String, dynamic> json) => factory ProtocolConfig.fromJson(Map<String, dynamic>? json) =>
_$ProtocolConfigFromJson(json); json == null ? ProtocolConfig() : _$ProtocolConfigFromJson(json);
Map<String, dynamic> toJson() => _$ProtocolConfigToJson(this); Map<String, dynamic> toJson() => _$ProtocolConfigToJson(this);
} }
@@ -57,8 +57,8 @@ class ExtraConfig {
ExtraConfig(); ExtraConfig();
factory ExtraConfig.fromJson(Map<String, dynamic> json) => factory ExtraConfig.fromJson(Map<String, dynamic>? json) =>
_$ExtraConfigFromJson(json); json == null ? ExtraConfig() : _$ExtraConfigFromJson(json);
Map<String, dynamic> toJson() => _$ExtraConfigToJson(this); Map<String, dynamic> toJson() => _$ExtraConfigToJson(this);
} }
@@ -10,8 +10,8 @@ DownloaderConfig _$DownloaderConfigFromJson(Map<String, dynamic> json) =>
DownloaderConfig() DownloaderConfig()
..downloadDir = json['downloadDir'] as String ..downloadDir = json['downloadDir'] as String
..protocolConfig = ProtocolConfig.fromJson( ..protocolConfig = ProtocolConfig.fromJson(
json['protocolConfig'] as Map<String, dynamic>) json['protocolConfig'] as Map<String, dynamic>?)
..extra = ExtraConfig.fromJson(json['extra'] as Map<String, dynamic>); ..extra = ExtraConfig.fromJson(json['extra'] as Map<String, dynamic>?);
Map<String, dynamic> _$DownloaderConfigToJson(DownloaderConfig instance) => Map<String, dynamic> _$DownloaderConfigToJson(DownloaderConfig instance) =>
<String, dynamic>{ <String, dynamic>{
+22
View File
@@ -0,0 +1,22 @@
import 'package:gopeed/api/model/options.dart';
import 'package:gopeed/api/model/request.dart';
import 'package:gopeed/api/model/resource.dart';
import 'package:json_annotation/json_annotation.dart';
part 'meta.g.dart';
@JsonSerializable(explicitToJson: true)
class Meta {
Request req;
Resource res;
Options opts;
Meta({
required this.req,
required this.res,
required this.opts,
});
factory Meta.fromJson(Map<String, dynamic> json) => _$MetaFromJson(json);
Map<String, dynamic> toJson() => _$MetaToJson(this);
}
+19
View File
@@ -0,0 +1,19 @@
// GENERATED CODE - DO NOT MODIFY BY HAND
part of 'meta.dart';
// **************************************************************************
// JsonSerializableGenerator
// **************************************************************************
Meta _$MetaFromJson(Map<String, dynamic> json) => Meta(
req: Request.fromJson(json['req'] as Map<String, dynamic>),
res: Resource.fromJson(json['res'] as Map<String, dynamic>),
opts: Options.fromJson(json['opts'] as Map<String, dynamic>),
);
Map<String, dynamic> _$MetaToJson(Meta instance) => <String, dynamic>{
'req': instance.req.toJson(),
'res': instance.res.toJson(),
'opts': instance.opts.toJson(),
};
+1 -1
View File
@@ -2,7 +2,7 @@ import 'package:json_annotation/json_annotation.dart';
part 'request.g.dart'; part 'request.g.dart';
@JsonSerializable(explicitToJson: true, genericArgumentFactories: true) @JsonSerializable(explicitToJson: true)
class Request { class Request {
String url; String url;
Map<String, dynamic>? extra; Map<String, dynamic>? extra;
@@ -0,0 +1,19 @@
import 'package:gopeed/api/model/resource.dart';
import 'package:json_annotation/json_annotation.dart';
part 'resolve_result.g.dart';
@JsonSerializable(explicitToJson: true)
class ResolveResult {
String id;
Resource res;
ResolveResult({
required this.id,
required this.res,
});
factory ResolveResult.fromJson(Map<String, dynamic> json) =>
_$ResolveResultFromJson(json);
Map<String, dynamic> toJson() => _$ResolveResultToJson(this);
}
@@ -0,0 +1,19 @@
// GENERATED CODE - DO NOT MODIFY BY HAND
part of 'resolve_result.dart';
// **************************************************************************
// JsonSerializableGenerator
// **************************************************************************
ResolveResult _$ResolveResultFromJson(Map<String, dynamic> json) =>
ResolveResult(
id: json['id'] as String,
res: Resource.fromJson(json['res'] as Map<String, dynamic>),
);
Map<String, dynamic> _$ResolveResultToJson(ResolveResult instance) =>
<String, dynamic>{
'id': instance.id,
'res': instance.res.toJson(),
};
+1 -5
View File
@@ -1,12 +1,9 @@
import 'package:json_annotation/json_annotation.dart'; import 'package:json_annotation/json_annotation.dart';
import 'request.dart';
part 'resource.g.dart'; part 'resource.g.dart';
@JsonSerializable(explicitToJson: true) @JsonSerializable(explicitToJson: true)
class Resource { class Resource {
Request req;
String name; String name;
int size; int size;
bool range; bool range;
@@ -14,8 +11,7 @@ class Resource {
String hash; String hash;
Resource( Resource(
{required this.req, {required this.name,
required this.name,
required this.size, required this.size,
required this.range, required this.range,
required this.files, required this.files,
-2
View File
@@ -7,7 +7,6 @@ part of 'resource.dart';
// ************************************************************************** // **************************************************************************
Resource _$ResourceFromJson(Map<String, dynamic> json) => Resource( Resource _$ResourceFromJson(Map<String, dynamic> json) => Resource(
req: Request.fromJson(json['req'] as Map<String, dynamic>),
name: json['name'] as String, name: json['name'] as String,
size: json['size'] as int, size: json['size'] as int,
range: json['range'] as bool, range: json['range'] as bool,
@@ -18,7 +17,6 @@ Resource _$ResourceFromJson(Map<String, dynamic> json) => Resource(
); );
Map<String, dynamic> _$ResourceToJson(Resource instance) => <String, dynamic>{ Map<String, dynamic> _$ResourceToJson(Resource instance) => <String, dynamic>{
'req': instance.req.toJson(),
'name': instance.name, 'name': instance.name,
'size': instance.size, 'size': instance.size,
'range': instance.range, 'range': instance.range,
+3 -4
View File
@@ -1,5 +1,6 @@
import 'package:json_annotation/json_annotation.dart'; import 'package:json_annotation/json_annotation.dart';
import 'meta.dart';
import 'options.dart'; import 'options.dart';
import 'resource.dart'; import 'resource.dart';
@@ -10,8 +11,7 @@ enum Status { ready, running, pause, error, done }
@JsonSerializable(explicitToJson: true) @JsonSerializable(explicitToJson: true)
class Task { class Task {
String id; String id;
Resource res; Meta meta;
Options opts;
Status status; Status status;
Progress progress; Progress progress;
int size; int size;
@@ -19,8 +19,7 @@ class Task {
Task({ Task({
required this.id, required this.id,
required this.res, required this.meta,
required this.opts,
required this.status, required this.status,
required this.progress, required this.progress,
required this.size, required this.size,
+2 -4
View File
@@ -8,8 +8,7 @@ part of 'task.dart';
Task _$TaskFromJson(Map<String, dynamic> json) => Task( Task _$TaskFromJson(Map<String, dynamic> json) => Task(
id: json['id'] as String, id: json['id'] as String,
res: Resource.fromJson(json['res'] as Map<String, dynamic>), meta: Meta.fromJson(json['meta'] as Map<String, dynamic>),
opts: Options.fromJson(json['opts'] as Map<String, dynamic>),
status: $enumDecode(_$StatusEnumMap, json['status']), status: $enumDecode(_$StatusEnumMap, json['status']),
progress: Progress.fromJson(json['progress'] as Map<String, dynamic>), progress: Progress.fromJson(json['progress'] as Map<String, dynamic>),
size: json['size'] as int, size: json['size'] as int,
@@ -18,8 +17,7 @@ Task _$TaskFromJson(Map<String, dynamic> json) => Task(
Map<String, dynamic> _$TaskToJson(Task instance) => <String, dynamic>{ Map<String, dynamic> _$TaskToJson(Task instance) => <String, dynamic>{
'id': instance.id, 'id': instance.id,
'res': instance.res.toJson(), 'meta': instance.meta.toJson(),
'opts': instance.opts.toJson(),
'status': _$StatusEnumMap[instance.status]!, 'status': _$StatusEnumMap[instance.status]!,
'progress': instance.progress.toJson(), 'progress': instance.progress.toJson(),
'size': instance.size, 'size': instance.size,
+6 -5
View File
@@ -1,13 +1,13 @@
import 'package:desktop_drop/desktop_drop.dart'; import 'package:desktop_drop/desktop_drop.dart';
import 'package:flutter/material.dart'; import 'package:flutter/material.dart';
import 'package:get/get.dart'; import 'package:get/get.dart';
import 'package:gopeed/api/model/resolve_result.dart';
import 'package:rounded_loading_button/rounded_loading_button.dart'; import 'package:rounded_loading_button/rounded_loading_button.dart';
import '../../api/api.dart'; import '../../api/api.dart';
import '../../api/model/create_task.dart'; import '../../api/model/create_task.dart';
import '../../api/model/options.dart'; import '../../api/model/options.dart';
import '../../api/model/request.dart'; import '../../api/model/request.dart';
import '../../api/model/resource.dart';
import '../../routes/router.dart'; import '../../routes/router.dart';
import '../../util/util.dart'; import '../../util/util.dart';
import '../../widget/directory_selector.dart'; import '../../widget/directory_selector.dart';
@@ -71,10 +71,10 @@ class CreateView extends GetView<CreateController> {
if (_resolveFormKey.currentState!.validate()) { if (_resolveFormKey.currentState!.validate()) {
_confirmController.start(); _confirmController.start();
try { try {
final res = await resolve(Request( final rr = await resolve(Request(
url: _urlController.text, url: _urlController.text,
)); ));
await _showResolveDialog(res); await _showResolveDialog(rr);
} catch (e) { } catch (e) {
Get.snackbar('error'.tr, e.toString()); Get.snackbar('error'.tr, e.toString());
} finally { } finally {
@@ -101,7 +101,8 @@ class CreateView extends GetView<CreateController> {
}); });
} }
Future<void> _showResolveDialog(Resource res) async { Future<void> _showResolveDialog(ResolveResult rr) async {
final res = rr.res;
final appController = Get.find<AppController>(); final appController = Get.find<AppController>();
final setting = appController.downloaderConfig.value; final setting = appController.downloaderConfig.value;
@@ -180,7 +181,7 @@ class CreateView extends GetView<CreateController> {
// } // }
await createTask(CreateTask( await createTask(CreateTask(
res: res, rid: rr.id,
opts: Options( opts: Options(
name: '', name: '',
path: pathController.text, path: pathController.text,
+3 -3
View File
@@ -25,7 +25,7 @@ class Item extends StatelessWidget {
mainAxisSize: MainAxisSize.min, mainAxisSize: MainAxisSize.min,
children: [ children: [
ListTile( ListTile(
title: Text(task.res.name), title: Text(task.meta.res.name),
leading: const Icon(Icons.insert_drive_file), leading: const Icon(Icons.insert_drive_file),
), ),
Row( Row(
@@ -63,8 +63,8 @@ class Item extends StatelessWidget {
list.add(IconButton( list.add(IconButton(
icon: const Icon(Icons.folder_open), icon: const Icon(Icons.folder_open),
onPressed: () async { onPressed: () async {
final file = File(Util.buildAbsPath(task.opts.path, final file = File(Util.buildAbsPath(task.meta.opts.path,
task.res.files[0].path, task.res.files[0].name)); task.meta.res.files[0].path, task.meta.res.files[0].name));
await launchUrl(file.parent.uri); await launchUrl(file.parent.uri);
}, },
)); ));