mirror of
https://github.com/GopeedLab/gopeed.git
synced 2026-11-19 23:03:43 +00:00
feat: spool media inputs to disk and report producer progress
This commit is contained in:
1 parent
e99ef6c7f4
commit
e0ced260e2
36 files changed
+1747
-366
No files matched your search
+50
-13
@@ -21,6 +21,7 @@ import (
|
||||
"github.com/GopeedLab/gopeed/internal/controller"
|
||||
"github.com/GopeedLab/gopeed/internal/fetcher"
|
||||
"github.com/GopeedLab/gopeed/internal/logger"
|
||||
"github.com/GopeedLab/gopeed/internal/tempfiles"
|
||||
"github.com/GopeedLab/gopeed/pkg/base"
|
||||
"github.com/GopeedLab/gopeed/pkg/protocol/http"
|
||||
"github.com/GopeedLab/gopeed/pkg/util"
|
||||
@@ -97,6 +98,7 @@ type Progress struct {
|
||||
}
|
||||
|
||||
type Downloader struct {
|
||||
tempFiles *tempfiles.Scope
|
||||
Logger *logger.Logger
|
||||
ExtensionLogger *logger.Logger
|
||||
|
||||
@@ -152,6 +154,9 @@ func NewDownloader(cfg *DownloaderConfig) *Downloader {
|
||||
}
|
||||
|
||||
func (d *Downloader) Setup() error {
|
||||
if err := d.setupTempDir(); err != nil {
|
||||
return err
|
||||
}
|
||||
d.blob = internalblob.NewRegistry("")
|
||||
|
||||
// setup storage
|
||||
@@ -263,24 +268,45 @@ func (d *Downloader) Setup() error {
|
||||
if d.GetTask(task.ID) == nil {
|
||||
return
|
||||
}
|
||||
// Startup can replace resource metadata while resolving a Blob.
|
||||
// Skip that short transition instead of reading half-initialized state.
|
||||
if !task.lock.TryLock() {
|
||||
return
|
||||
}
|
||||
task.statusLock.Lock()
|
||||
if task.Status != base.DownloadStatusRunning && !task.Uploading {
|
||||
task.statusLock.Unlock()
|
||||
task.lock.Unlock()
|
||||
return
|
||||
}
|
||||
if task.fetcher == nil {
|
||||
task.statusLock.Unlock()
|
||||
task.lock.Unlock()
|
||||
return
|
||||
}
|
||||
|
||||
current := task.fetcher.Progress().TotalDownloaded()
|
||||
delta := current - task.Progress.Downloaded
|
||||
if task.Meta != nil && task.Meta.Req != nil {
|
||||
if producer := d.blob.Production(task.Meta.Req.URL); producer != nil {
|
||||
downloaded, received := producer.Progress()
|
||||
if task.producer != producer {
|
||||
task.producer = producer
|
||||
task.producerReceived = 0
|
||||
task.speedArr = nil
|
||||
}
|
||||
current = downloaded
|
||||
delta = max(0, received-task.producerReceived)
|
||||
task.producerReceived = received
|
||||
}
|
||||
}
|
||||
tick := float64(d.cfg.RefreshInterval) / 1000
|
||||
downloadDataChanged := false
|
||||
uploading := task.Uploading
|
||||
if task.Status == base.DownloadStatusRunning {
|
||||
downloadDataChanged = current != task.Progress.Downloaded
|
||||
task.Progress.Used = task.timer.Used()
|
||||
task.Progress.Speed = task.updateSpeed(current-task.Progress.Downloaded, tick)
|
||||
task.Progress.Speed = task.updateSpeed(max(0, delta), tick)
|
||||
task.Progress.Downloaded = current
|
||||
}
|
||||
|
||||
@@ -299,6 +325,7 @@ func (d *Downloader) Setup() error {
|
||||
if err := d.captureTaskStats(task, true, false); err != nil {
|
||||
d.Logger.Warn().Err(err).Msgf("persist task stats failed, task id: %s", task.ID)
|
||||
}
|
||||
task.lock.Unlock()
|
||||
// Listener callbacks may Pause/Continue and acquire statusLock.
|
||||
d.emit(EventKeyProgress, task)
|
||||
|
||||
@@ -389,6 +416,9 @@ func (d *Downloader) parseFm(url string) (fetcher.FetcherManager, error) {
|
||||
|
||||
func (d *Downloader) setupFetcher(fm fetcher.FetcherManager, fetcher fetcher.Fetcher) {
|
||||
ctl := controller.NewController()
|
||||
ctl.TempDir = d.cfg.TempDir
|
||||
ctl.TempFiles = d.tempFiles
|
||||
ctl.ManagedProduction = func(raw string) bool { return d.blob.Production(raw) != nil }
|
||||
ctl.GetConfig = func(v any) {
|
||||
d.getProtocolConfig(fm.Name(), v)
|
||||
}
|
||||
@@ -1124,7 +1154,7 @@ func (d *Downloader) Close() error {
|
||||
if d.blob != nil {
|
||||
closeArr = append(closeArr, d.blob.Close)
|
||||
}
|
||||
closeArr = append(closeArr, d.storage.Close)
|
||||
closeArr = append(closeArr, d.storage.Close, d.tempFiles.Close)
|
||||
// Make sure all resources are released, if had error, return the last error
|
||||
var lastErr error
|
||||
for i, close := range closeArr {
|
||||
@@ -1399,17 +1429,6 @@ func (d *Downloader) watch(task *Task) {
|
||||
}
|
||||
}
|
||||
|
||||
task.Progress.Used = task.timer.Used()
|
||||
if task.Meta.Res.Size == 0 {
|
||||
task.Meta.Res.Size = task.fetcher.Progress().TotalDownloaded()
|
||||
}
|
||||
used := task.Progress.Used / int64(time.Second)
|
||||
if used == 0 {
|
||||
used = 1
|
||||
}
|
||||
totalSize := task.Meta.Res.Size
|
||||
task.Progress.Speed = totalSize / used
|
||||
task.Progress.Downloaded = totalSize
|
||||
if !d.markTaskDone(task) {
|
||||
return
|
||||
}
|
||||
@@ -1808,7 +1827,9 @@ func (d *Downloader) doStart(task *Task) (err error) {
|
||||
task.Meta.Res.CalcSize(task.Meta.Opts.SelectFiles)
|
||||
}
|
||||
|
||||
task.statusLock.Lock()
|
||||
task.Progress.Speed = 0
|
||||
task.statusLock.Unlock()
|
||||
if !d.taskIsRunningGeneration(task, generation) {
|
||||
return nil
|
||||
}
|
||||
@@ -1962,11 +1983,27 @@ func (d *Downloader) markTaskDone(task *Task) bool {
|
||||
if task == nil || task.statusLock == nil {
|
||||
return false
|
||||
}
|
||||
task.lock.Lock()
|
||||
defer task.lock.Unlock()
|
||||
task.statusLock.Lock()
|
||||
defer task.statusLock.Unlock()
|
||||
if task.Status != base.DownloadStatusRunning {
|
||||
return false
|
||||
}
|
||||
task.Progress.Used = task.timer.Used()
|
||||
if task.Meta.Res.Size == 0 {
|
||||
task.Meta.Res.Size = task.fetcher.Progress().TotalDownloaded()
|
||||
}
|
||||
used := task.Progress.Used / int64(time.Second)
|
||||
if used == 0 {
|
||||
used = 1
|
||||
}
|
||||
totalSize := task.Meta.Res.Size
|
||||
task.Progress.Speed = totalSize / used
|
||||
task.Progress.Downloaded = totalSize
|
||||
|
||||
task.producer = nil
|
||||
task.producerReceived = 0
|
||||
task.updateStatus(base.DownloadStatusDone)
|
||||
task.runGeneration++
|
||||
return true
|
||||
|
||||
Reference in new issue
Block a user