feat: bt improve

This commit is contained in:
levi committed 2022-10-26 17:18:13 +08:00
1 parent c6aa83194f
commit 0fc133e46d
28 files changed
+1537 -260

No files matched your search

+305 -69
View File
@@ -10,9 +10,16 @@ import (
"github.com/monkeyWie/gopeed-core/pkg/util"
"strings"
"sync"
"sync/atomic"
"time"
)
const (
bucketTask = "task"
bucketSave = "save"
bucketConfig = "config"
)
type Listener func(event *Event)
type Progress struct {
@@ -25,46 +32,133 @@ type Progress struct {
}
type Downloader struct {
*controller.DefaultController
controller *controller.DefaultController
fetchBuilders map[string]fetcher.FetcherBuilder
storage Storage
tasks []*Task
listener Listener
refreshInterval int
lock *sync.Mutex
closed atomic.Bool
}
func NewDownloader(fbs ...fetcher.FetcherBuilder) *Downloader {
d := &Downloader{DefaultController: controller.NewController()}
d.fetchBuilders = make(map[string]fetcher.FetcherBuilder)
for _, f := range fbs {
type DownloaderConfig struct {
Controller *controller.DefaultController
FetchBuilders []fetcher.FetcherBuilder
Storage Storage
// RefreshInterval time duration to refresh task progress(ms)
RefreshInterval int
}
func (cfg *DownloaderConfig) Init() *DownloaderConfig {
if cfg.Controller == nil {
cfg.Controller = controller.NewController()
}
if len(cfg.FetchBuilders) == 0 {
cfg.FetchBuilders = []fetcher.FetcherBuilder{
new(http.FetcherBuilder),
new(bt.FetcherBuilder),
}
}
if cfg.Storage == nil {
cfg.Storage = NewMemStorage()
}
if cfg.RefreshInterval == 0 {
cfg.RefreshInterval = 1000
}
return cfg
}
func NewDownloader(cfg *DownloaderConfig) *Downloader {
if cfg == nil {
cfg = &DownloaderConfig{}
}
cfg.Init()
d := &Downloader{
controller: cfg.Controller,
fetchBuilders: make(map[string]fetcher.FetcherBuilder),
refreshInterval: cfg.RefreshInterval,
storage: cfg.Storage,
lock: &sync.Mutex{},
}
for _, f := range cfg.FetchBuilders {
for _, p := range f.Schemes() {
d.fetchBuilders[strings.ToUpper(p)] = f
}
}
d.tasks = make([]*Task, 0)
// 每秒统计一次下载速度
go func() {
for {
if len(d.tasks) > 0 {
for _, task := range d.tasks {
if task.Status == base.DownloadStatusDone ||
task.Status == base.DownloadStatusError ||
task.Status == base.DownloadStatusPause {
continue
}
current := task.fetcher.Progress().TotalDownloaded()
task.Progress.Used = task.timer.Used()
task.Progress.Speed = current - task.Progress.Downloaded
task.Progress.Downloaded = current
d.emit(EventKeyProgress, task)
}
}
time.Sleep(time.Second)
}
}()
return d
}
func (d *Downloader) buildFetcher(url string) (fetcher.Fetcher, error) {
func (d *Downloader) Setup() error {
// setup storage
if err := d.storage.Setup([]string{bucketTask, bucketSave, bucketConfig}); err != nil {
return err
}
// load tasks from storage
var tasks []*Task
if err := d.storage.List(bucketTask, &tasks); err != nil {
return err
}
if tasks == nil {
tasks = make([]*Task, 0)
} else {
for _, task := range tasks {
initTask(task)
if task.Status != base.DownloadStatusDone && task.Status != base.DownloadStatusError {
task.Status = base.DownloadStatusPause
}
}
}
d.tasks = tasks
// 每秒统计一次下载速度
go func() {
for !d.closed.Load() {
if len(d.tasks) > 0 {
for _, task := range d.tasks {
func() {
task.lock.Lock()
defer task.lock.Unlock()
if task.Status == base.DownloadStatusDone ||
task.Status == base.DownloadStatusError ||
task.Status == base.DownloadStatusPause {
return
}
// check if task is deleted
if d.GetTask(task.ID) == nil {
return
}
current := task.fetcher.Progress().TotalDownloaded()
task.Progress.Used = task.timer.Used()
task.Progress.Speed = current - task.Progress.Downloaded
task.Progress.Downloaded = current
d.emit(EventKeyProgress, task)
// store fetcher progress
data, err := task.fetcherBuilder.Store(task.fetcher)
if err != nil {
return // TODO log
}
if err := d.storage.Put(bucketSave, task.ID, data); err != nil {
return // TODO log
}
if err := d.storage.Put(bucketTask, task.ID, task); err != nil {
return // TODO log
}
}()
}
}
time.Sleep(time.Millisecond * time.Duration(d.refreshInterval))
}
}()
return nil
}
func (d *Downloader) buildFetcher(url string) (fetcher.FetcherBuilder, fetcher.Fetcher, error) {
schema := util.ParseSchema(url)
fetchBuilder, ok := d.fetchBuilders[schema]
if !ok {
@@ -72,14 +166,14 @@ func (d *Downloader) buildFetcher(url string) (fetcher.Fetcher, error) {
}
if ok {
fetcher := fetchBuilder.Build()
fetcher.Setup(d.DefaultController)
return fetcher, nil
fetcher.Setup(d.controller)
return fetchBuilder, fetcher, nil
}
return nil, errors.New("unsupported protocol")
return nil, nil, errors.New("unsupported protocol")
}
func (d *Downloader) Resolve(req *base.Request) (*base.Resource, error) {
fetcher, err := d.buildFetcher(req.URL)
_, fetcher, err := d.buildFetcher(req.URL)
if err != nil {
return nil, err
}
@@ -99,13 +193,13 @@ func (d *Downloader) Create(res *base.Resource, opts *base.Options) (taskId stri
opts.Connections = 1
}
if len(opts.SelectFiles) == 0 {
opts.SelectFiles = make([]int, len(res.Files))
for i := range opts.SelectFiles {
opts.SelectFiles[i] = i
opts.SelectFiles = make([]int, 0)
for i := range res.Files {
opts.SelectFiles = append(opts.SelectFiles, i)
}
}
fetcher, err := d.buildFetcher(res.Req.URL)
fetcherBuilder, fetcher, err := d.buildFetcher(res.Req.URL)
if err != nil {
return
}
@@ -115,74 +209,144 @@ func (d *Downloader) Create(res *base.Resource, opts *base.Options) (taskId stri
}
task := NewTask()
task.fetcherBuilder = fetcherBuilder
task.fetcher = fetcher
task.Res = res
task.Opts = opts
task.Progress = &Progress{}
task.timer = &util.Timer{}
task.locker = new(sync.Mutex)
task.timer.Start()
task.Status = base.DownloadStatusRunning
d.tasks = append(d.tasks, task)
d.emit(EventKeyStart, task)
taskId = task.ID
err = fetcher.Start()
// calculate select files size
for _, selectIndex := range opts.SelectFiles {
task.Size += res.Files[selectIndex].Size
}
initTask(task)
task.timer.Start()
err = func() error {
d.lock.Lock()
defer d.lock.Unlock()
if err := d.storage.Put(bucketTask, task.ID, task.clone()); err != nil {
return err
}
d.tasks = append(d.tasks, task)
return nil
}()
if err != nil {
return
}
d.emit(EventKeyStart, task)
taskId = task.ID
go func() {
err = fetcher.Wait()
err := fetcher.Start()
if err != nil {
task.Status = base.DownloadStatusError
d.emit(EventKeyError, task, err)
} else {
task.Progress.Used = task.timer.Used()
if task.Res.Size == 0 {
task.Res.Size = task.fetcher.Progress().TotalDownloaded()
}
used := task.Progress.Used / int64(time.Second)
if used == 0 {
used = 1
}
task.Progress.Speed = task.Res.Size / used
task.Progress.Downloaded = task.Res.Size
task.Status = base.DownloadStatusDone
d.emit(EventKeyDone, task)
return
}
d.emit(EventKeyFinally, task, err)
d.watch(task)
}()
return
}
func (d *Downloader) Pause(id string) {
func (d *Downloader) Pause(id string) (err error) {
task := d.GetTask(id)
if task == nil {
return
}
task.locker.Lock()
defer task.locker.Unlock()
task.lock.Lock()
defer task.lock.Unlock()
if task.Status == base.DownloadStatusRunning {
task.Status = base.DownloadStatusPause
task.timer.Pause()
task.fetcher.Pause()
d.storage.Put(bucketTask, task.ID, task.clone())
d.emit(EventKeyPause, task)
}
return
}
func (d *Downloader) Continue(id string) {
func (d *Downloader) Continue(id string) (err error) {
task := d.GetTask(id)
if task == nil {
return
}
task.locker.Lock()
defer task.locker.Unlock()
task.lock.Lock()
defer task.lock.Unlock()
if task.Status == base.DownloadStatusPause || task.Status == base.DownloadStatusError {
task.Status = base.DownloadStatusRunning
task.timer.Continue()
task.fetcher.Continue()
err = d.restoreFetcher(task)
if err != nil {
return
}
task.timer.Start()
err = task.fetcher.Continue()
if err != nil {
return
}
d.storage.Put(bucketTask, task.ID, task.clone())
d.emit(EventKeyContinue, task)
}
return
}
func (d *Downloader) Delete(id string, force bool) (err error) {
d.lock.Lock()
defer d.lock.Unlock()
task := d.GetTask(id)
if task == nil {
return
}
task.lock.Lock()
defer task.lock.Unlock()
if task.fetcher != nil {
err = task.fetcher.Close()
if err != nil {
return
}
}
for i, t := range d.tasks {
if t.ID == id {
d.tasks = append(d.tasks[:i], d.tasks[i+1:]...)
break
}
}
err = d.storage.Delete(bucketTask, id)
if err != nil {
return
}
err = d.storage.Delete(bucketSave, id)
if err != nil {
return
}
if force {
names := make([]string, 0)
for _, file := range task.Res.Files {
names = append(names, util.Filepath(file.Path, file.Name, task.Opts.Name))
}
util.SafeRemoveAll(task.Opts.Path, names)
}
d.emit(EventKeyDelete, task)
task = nil
return
}
func (d *Downloader) Close() error {
d.lock.Lock()
defer d.lock.Unlock()
d.closed.Store(true)
for _, task := range d.tasks {
d.Pause(task.ID)
}
return d.storage.Close()
}
func (d *Downloader) Clear() error {
if err := d.Close(); err != nil {
return err
}
if err := d.storage.Clear(); err != nil {
return err
}
return nil
}
func (d *Downloader) Listener(fn Listener) {
@@ -216,7 +380,75 @@ func (d *Downloader) GetTasks() []*Task {
return d.tasks
}
var defaultDownloader = NewDownloader(new(http.FetcherBuilder), new(bt.FetcherBuilder))
func (d *Downloader) GetConfig(v any) error {
return d.storage.Get(bucketConfig, "config", v)
}
func (d *Downloader) PutConfig(v any) error {
return d.storage.Put(bucketConfig, "config", v)
}
func (d *Downloader) watch(task *Task) {
err := task.fetcher.Wait()
if err != nil {
task.Status = base.DownloadStatusError
d.emit(EventKeyError, task, err)
} else {
task.Progress.Used = task.timer.Used()
if task.Size == 0 {
task.Size = task.fetcher.Progress().TotalDownloaded()
task.Res.Size = task.Size
}
used := task.Progress.Used / int64(time.Second)
if used == 0 {
used = 1
}
task.Progress.Speed = task.Size / used
task.Progress.Downloaded = task.Size
task.Status = base.DownloadStatusDone
d.storage.Put(bucketTask, task.ID, task.clone())
d.emit(EventKeyDone, task)
}
d.emit(EventKeyFinally, task, err)
}
func (d *Downloader) restoreFetcher(task *Task) error {
if task.fetcher == nil {
fetcherBuilder, _, err := d.buildFetcher(task.Res.Req.URL)
if err != nil {
return err
}
task.fetcherBuilder = fetcherBuilder
err = func() error {
v, f := fetcherBuilder.Restore()
if v != nil {
err := d.storage.Pop(bucketSave, task.ID, v)
if err != nil {
return err
}
task.fetcher = f(task.Res, task.Opts, v)
}
return nil
}()
if err != nil {
// TODO log
}
if task.fetcher == nil {
task.fetcher = fetcherBuilder.Build()
task.fetcher.Create(task.Res, task.Opts)
}
task.fetcher.Setup(d.controller)
go d.watch(task)
}
return nil
}
func initTask(task *Task) {
task.timer = &util.Timer{}
task.lock = &sync.Mutex{}
}
var defaultDownloader = NewDownloader(nil)
type boot struct {
url string
@@ -256,5 +488,9 @@ func (b *boot) Create(opts *base.Options) (string, error) {
}
func Boot() *boot {
err := defaultDownloader.Setup()
if err != nil {
panic(err)
}
return &boot{}
}