mirror of
https://github.com/GopeedLab/gopeed.git
synced 2026-11-18 22:33:42 +00:00
feat: bt improve
This commit is contained in:
1 parent
c6aa83194f
commit
0fc133e46d
28 files changed
+1537
-260
No files matched your search
+305
-69
@@ -10,9 +10,16 @@ import (
|
||||
"github.com/monkeyWie/gopeed-core/pkg/util"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
bucketTask = "task"
|
||||
bucketSave = "save"
|
||||
bucketConfig = "config"
|
||||
)
|
||||
|
||||
type Listener func(event *Event)
|
||||
|
||||
type Progress struct {
|
||||
@@ -25,46 +32,133 @@ type Progress struct {
|
||||
}
|
||||
|
||||
type Downloader struct {
|
||||
*controller.DefaultController
|
||||
controller *controller.DefaultController
|
||||
fetchBuilders map[string]fetcher.FetcherBuilder
|
||||
storage Storage
|
||||
tasks []*Task
|
||||
listener Listener
|
||||
|
||||
refreshInterval int
|
||||
lock *sync.Mutex
|
||||
closed atomic.Bool
|
||||
}
|
||||
|
||||
func NewDownloader(fbs ...fetcher.FetcherBuilder) *Downloader {
|
||||
d := &Downloader{DefaultController: controller.NewController()}
|
||||
d.fetchBuilders = make(map[string]fetcher.FetcherBuilder)
|
||||
for _, f := range fbs {
|
||||
type DownloaderConfig struct {
|
||||
Controller *controller.DefaultController
|
||||
FetchBuilders []fetcher.FetcherBuilder
|
||||
Storage Storage
|
||||
|
||||
// RefreshInterval time duration to refresh task progress(ms)
|
||||
RefreshInterval int
|
||||
}
|
||||
|
||||
func (cfg *DownloaderConfig) Init() *DownloaderConfig {
|
||||
if cfg.Controller == nil {
|
||||
cfg.Controller = controller.NewController()
|
||||
}
|
||||
if len(cfg.FetchBuilders) == 0 {
|
||||
cfg.FetchBuilders = []fetcher.FetcherBuilder{
|
||||
new(http.FetcherBuilder),
|
||||
new(bt.FetcherBuilder),
|
||||
}
|
||||
}
|
||||
if cfg.Storage == nil {
|
||||
cfg.Storage = NewMemStorage()
|
||||
}
|
||||
if cfg.RefreshInterval == 0 {
|
||||
cfg.RefreshInterval = 1000
|
||||
}
|
||||
return cfg
|
||||
}
|
||||
|
||||
func NewDownloader(cfg *DownloaderConfig) *Downloader {
|
||||
if cfg == nil {
|
||||
cfg = &DownloaderConfig{}
|
||||
}
|
||||
cfg.Init()
|
||||
|
||||
d := &Downloader{
|
||||
controller: cfg.Controller,
|
||||
fetchBuilders: make(map[string]fetcher.FetcherBuilder),
|
||||
refreshInterval: cfg.RefreshInterval,
|
||||
storage: cfg.Storage,
|
||||
lock: &sync.Mutex{},
|
||||
}
|
||||
for _, f := range cfg.FetchBuilders {
|
||||
for _, p := range f.Schemes() {
|
||||
d.fetchBuilders[strings.ToUpper(p)] = f
|
||||
}
|
||||
}
|
||||
d.tasks = make([]*Task, 0)
|
||||
|
||||
// 每秒统计一次下载速度
|
||||
go func() {
|
||||
for {
|
||||
if len(d.tasks) > 0 {
|
||||
for _, task := range d.tasks {
|
||||
if task.Status == base.DownloadStatusDone ||
|
||||
task.Status == base.DownloadStatusError ||
|
||||
task.Status == base.DownloadStatusPause {
|
||||
continue
|
||||
}
|
||||
current := task.fetcher.Progress().TotalDownloaded()
|
||||
task.Progress.Used = task.timer.Used()
|
||||
task.Progress.Speed = current - task.Progress.Downloaded
|
||||
task.Progress.Downloaded = current
|
||||
d.emit(EventKeyProgress, task)
|
||||
}
|
||||
}
|
||||
time.Sleep(time.Second)
|
||||
}
|
||||
}()
|
||||
return d
|
||||
}
|
||||
|
||||
func (d *Downloader) buildFetcher(url string) (fetcher.Fetcher, error) {
|
||||
func (d *Downloader) Setup() error {
|
||||
// setup storage
|
||||
if err := d.storage.Setup([]string{bucketTask, bucketSave, bucketConfig}); err != nil {
|
||||
return err
|
||||
}
|
||||
// load tasks from storage
|
||||
var tasks []*Task
|
||||
if err := d.storage.List(bucketTask, &tasks); err != nil {
|
||||
return err
|
||||
}
|
||||
if tasks == nil {
|
||||
tasks = make([]*Task, 0)
|
||||
} else {
|
||||
for _, task := range tasks {
|
||||
initTask(task)
|
||||
if task.Status != base.DownloadStatusDone && task.Status != base.DownloadStatusError {
|
||||
task.Status = base.DownloadStatusPause
|
||||
}
|
||||
}
|
||||
}
|
||||
d.tasks = tasks
|
||||
|
||||
// 每秒统计一次下载速度
|
||||
go func() {
|
||||
for !d.closed.Load() {
|
||||
if len(d.tasks) > 0 {
|
||||
for _, task := range d.tasks {
|
||||
func() {
|
||||
task.lock.Lock()
|
||||
defer task.lock.Unlock()
|
||||
if task.Status == base.DownloadStatusDone ||
|
||||
task.Status == base.DownloadStatusError ||
|
||||
task.Status == base.DownloadStatusPause {
|
||||
return
|
||||
}
|
||||
// check if task is deleted
|
||||
if d.GetTask(task.ID) == nil {
|
||||
return
|
||||
}
|
||||
|
||||
current := task.fetcher.Progress().TotalDownloaded()
|
||||
task.Progress.Used = task.timer.Used()
|
||||
task.Progress.Speed = current - task.Progress.Downloaded
|
||||
task.Progress.Downloaded = current
|
||||
d.emit(EventKeyProgress, task)
|
||||
|
||||
// store fetcher progress
|
||||
data, err := task.fetcherBuilder.Store(task.fetcher)
|
||||
if err != nil {
|
||||
return // TODO log
|
||||
}
|
||||
if err := d.storage.Put(bucketSave, task.ID, data); err != nil {
|
||||
return // TODO log
|
||||
}
|
||||
if err := d.storage.Put(bucketTask, task.ID, task); err != nil {
|
||||
return // TODO log
|
||||
}
|
||||
}()
|
||||
}
|
||||
}
|
||||
time.Sleep(time.Millisecond * time.Duration(d.refreshInterval))
|
||||
}
|
||||
}()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *Downloader) buildFetcher(url string) (fetcher.FetcherBuilder, fetcher.Fetcher, error) {
|
||||
schema := util.ParseSchema(url)
|
||||
fetchBuilder, ok := d.fetchBuilders[schema]
|
||||
if !ok {
|
||||
@@ -72,14 +166,14 @@ func (d *Downloader) buildFetcher(url string) (fetcher.Fetcher, error) {
|
||||
}
|
||||
if ok {
|
||||
fetcher := fetchBuilder.Build()
|
||||
fetcher.Setup(d.DefaultController)
|
||||
return fetcher, nil
|
||||
fetcher.Setup(d.controller)
|
||||
return fetchBuilder, fetcher, nil
|
||||
}
|
||||
return nil, errors.New("unsupported protocol")
|
||||
return nil, nil, errors.New("unsupported protocol")
|
||||
}
|
||||
|
||||
func (d *Downloader) Resolve(req *base.Request) (*base.Resource, error) {
|
||||
fetcher, err := d.buildFetcher(req.URL)
|
||||
_, fetcher, err := d.buildFetcher(req.URL)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -99,13 +193,13 @@ func (d *Downloader) Create(res *base.Resource, opts *base.Options) (taskId stri
|
||||
opts.Connections = 1
|
||||
}
|
||||
if len(opts.SelectFiles) == 0 {
|
||||
opts.SelectFiles = make([]int, len(res.Files))
|
||||
for i := range opts.SelectFiles {
|
||||
opts.SelectFiles[i] = i
|
||||
opts.SelectFiles = make([]int, 0)
|
||||
for i := range res.Files {
|
||||
opts.SelectFiles = append(opts.SelectFiles, i)
|
||||
}
|
||||
}
|
||||
|
||||
fetcher, err := d.buildFetcher(res.Req.URL)
|
||||
fetcherBuilder, fetcher, err := d.buildFetcher(res.Req.URL)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
@@ -115,74 +209,144 @@ func (d *Downloader) Create(res *base.Resource, opts *base.Options) (taskId stri
|
||||
}
|
||||
|
||||
task := NewTask()
|
||||
task.fetcherBuilder = fetcherBuilder
|
||||
task.fetcher = fetcher
|
||||
task.Res = res
|
||||
task.Opts = opts
|
||||
task.Progress = &Progress{}
|
||||
task.timer = &util.Timer{}
|
||||
task.locker = new(sync.Mutex)
|
||||
task.timer.Start()
|
||||
task.Status = base.DownloadStatusRunning
|
||||
d.tasks = append(d.tasks, task)
|
||||
d.emit(EventKeyStart, task)
|
||||
taskId = task.ID
|
||||
|
||||
err = fetcher.Start()
|
||||
// calculate select files size
|
||||
for _, selectIndex := range opts.SelectFiles {
|
||||
task.Size += res.Files[selectIndex].Size
|
||||
}
|
||||
initTask(task)
|
||||
task.timer.Start()
|
||||
err = func() error {
|
||||
d.lock.Lock()
|
||||
defer d.lock.Unlock()
|
||||
if err := d.storage.Put(bucketTask, task.ID, task.clone()); err != nil {
|
||||
return err
|
||||
}
|
||||
d.tasks = append(d.tasks, task)
|
||||
return nil
|
||||
}()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
d.emit(EventKeyStart, task)
|
||||
taskId = task.ID
|
||||
|
||||
go func() {
|
||||
err = fetcher.Wait()
|
||||
err := fetcher.Start()
|
||||
if err != nil {
|
||||
task.Status = base.DownloadStatusError
|
||||
d.emit(EventKeyError, task, err)
|
||||
} else {
|
||||
task.Progress.Used = task.timer.Used()
|
||||
if task.Res.Size == 0 {
|
||||
task.Res.Size = task.fetcher.Progress().TotalDownloaded()
|
||||
}
|
||||
used := task.Progress.Used / int64(time.Second)
|
||||
if used == 0 {
|
||||
used = 1
|
||||
}
|
||||
task.Progress.Speed = task.Res.Size / used
|
||||
task.Progress.Downloaded = task.Res.Size
|
||||
task.Status = base.DownloadStatusDone
|
||||
d.emit(EventKeyDone, task)
|
||||
return
|
||||
}
|
||||
d.emit(EventKeyFinally, task, err)
|
||||
d.watch(task)
|
||||
}()
|
||||
return
|
||||
}
|
||||
|
||||
func (d *Downloader) Pause(id string) {
|
||||
func (d *Downloader) Pause(id string) (err error) {
|
||||
task := d.GetTask(id)
|
||||
if task == nil {
|
||||
return
|
||||
}
|
||||
task.locker.Lock()
|
||||
defer task.locker.Unlock()
|
||||
task.lock.Lock()
|
||||
defer task.lock.Unlock()
|
||||
if task.Status == base.DownloadStatusRunning {
|
||||
task.Status = base.DownloadStatusPause
|
||||
task.timer.Pause()
|
||||
task.fetcher.Pause()
|
||||
d.storage.Put(bucketTask, task.ID, task.clone())
|
||||
d.emit(EventKeyPause, task)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (d *Downloader) Continue(id string) {
|
||||
func (d *Downloader) Continue(id string) (err error) {
|
||||
task := d.GetTask(id)
|
||||
if task == nil {
|
||||
return
|
||||
}
|
||||
task.locker.Lock()
|
||||
defer task.locker.Unlock()
|
||||
task.lock.Lock()
|
||||
defer task.lock.Unlock()
|
||||
if task.Status == base.DownloadStatusPause || task.Status == base.DownloadStatusError {
|
||||
task.Status = base.DownloadStatusRunning
|
||||
task.timer.Continue()
|
||||
task.fetcher.Continue()
|
||||
err = d.restoreFetcher(task)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
task.timer.Start()
|
||||
err = task.fetcher.Continue()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
d.storage.Put(bucketTask, task.ID, task.clone())
|
||||
d.emit(EventKeyContinue, task)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (d *Downloader) Delete(id string, force bool) (err error) {
|
||||
d.lock.Lock()
|
||||
defer d.lock.Unlock()
|
||||
task := d.GetTask(id)
|
||||
if task == nil {
|
||||
return
|
||||
}
|
||||
task.lock.Lock()
|
||||
defer task.lock.Unlock()
|
||||
if task.fetcher != nil {
|
||||
err = task.fetcher.Close()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
for i, t := range d.tasks {
|
||||
if t.ID == id {
|
||||
d.tasks = append(d.tasks[:i], d.tasks[i+1:]...)
|
||||
break
|
||||
}
|
||||
}
|
||||
err = d.storage.Delete(bucketTask, id)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = d.storage.Delete(bucketSave, id)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if force {
|
||||
names := make([]string, 0)
|
||||
for _, file := range task.Res.Files {
|
||||
names = append(names, util.Filepath(file.Path, file.Name, task.Opts.Name))
|
||||
}
|
||||
util.SafeRemoveAll(task.Opts.Path, names)
|
||||
}
|
||||
d.emit(EventKeyDelete, task)
|
||||
task = nil
|
||||
return
|
||||
}
|
||||
|
||||
func (d *Downloader) Close() error {
|
||||
d.lock.Lock()
|
||||
defer d.lock.Unlock()
|
||||
d.closed.Store(true)
|
||||
for _, task := range d.tasks {
|
||||
d.Pause(task.ID)
|
||||
}
|
||||
return d.storage.Close()
|
||||
}
|
||||
|
||||
func (d *Downloader) Clear() error {
|
||||
if err := d.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := d.storage.Clear(); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *Downloader) Listener(fn Listener) {
|
||||
@@ -216,7 +380,75 @@ func (d *Downloader) GetTasks() []*Task {
|
||||
return d.tasks
|
||||
}
|
||||
|
||||
var defaultDownloader = NewDownloader(new(http.FetcherBuilder), new(bt.FetcherBuilder))
|
||||
func (d *Downloader) GetConfig(v any) error {
|
||||
return d.storage.Get(bucketConfig, "config", v)
|
||||
}
|
||||
|
||||
func (d *Downloader) PutConfig(v any) error {
|
||||
return d.storage.Put(bucketConfig, "config", v)
|
||||
}
|
||||
|
||||
func (d *Downloader) watch(task *Task) {
|
||||
err := task.fetcher.Wait()
|
||||
if err != nil {
|
||||
task.Status = base.DownloadStatusError
|
||||
d.emit(EventKeyError, task, err)
|
||||
} else {
|
||||
task.Progress.Used = task.timer.Used()
|
||||
if task.Size == 0 {
|
||||
task.Size = task.fetcher.Progress().TotalDownloaded()
|
||||
task.Res.Size = task.Size
|
||||
}
|
||||
used := task.Progress.Used / int64(time.Second)
|
||||
if used == 0 {
|
||||
used = 1
|
||||
}
|
||||
task.Progress.Speed = task.Size / used
|
||||
task.Progress.Downloaded = task.Size
|
||||
task.Status = base.DownloadStatusDone
|
||||
d.storage.Put(bucketTask, task.ID, task.clone())
|
||||
d.emit(EventKeyDone, task)
|
||||
}
|
||||
d.emit(EventKeyFinally, task, err)
|
||||
}
|
||||
|
||||
func (d *Downloader) restoreFetcher(task *Task) error {
|
||||
if task.fetcher == nil {
|
||||
fetcherBuilder, _, err := d.buildFetcher(task.Res.Req.URL)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
task.fetcherBuilder = fetcherBuilder
|
||||
err = func() error {
|
||||
v, f := fetcherBuilder.Restore()
|
||||
if v != nil {
|
||||
err := d.storage.Pop(bucketSave, task.ID, v)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
task.fetcher = f(task.Res, task.Opts, v)
|
||||
}
|
||||
return nil
|
||||
}()
|
||||
if err != nil {
|
||||
// TODO log
|
||||
}
|
||||
if task.fetcher == nil {
|
||||
task.fetcher = fetcherBuilder.Build()
|
||||
task.fetcher.Create(task.Res, task.Opts)
|
||||
}
|
||||
task.fetcher.Setup(d.controller)
|
||||
go d.watch(task)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func initTask(task *Task) {
|
||||
task.timer = &util.Timer{}
|
||||
task.lock = &sync.Mutex{}
|
||||
}
|
||||
|
||||
var defaultDownloader = NewDownloader(nil)
|
||||
|
||||
type boot struct {
|
||||
url string
|
||||
@@ -256,5 +488,9 @@ func (b *boot) Create(opts *base.Options) (string, error) {
|
||||
}
|
||||
|
||||
func Boot() *boot {
|
||||
err := defaultDownloader.Setup()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return &boot{}
|
||||
}
|
||||
Reference in new issue
Block a user