mirror of
https://github.com/GopeedLab/gopeed.git
synced 2026-11-10 18:40:27 +00:00
Compare commits
3
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
54fc0f7289 | ||
|
|
917fc64046 | ||
|
|
4ffb172259 |
No files matched your search
@@ -0,0 +1,259 @@
|
|||||||
|
package download
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"sync"
|
||||||
|
"sync/atomic"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/GopeedLab/gopeed/internal/fetcher"
|
||||||
|
"github.com/GopeedLab/gopeed/pkg/base"
|
||||||
|
)
|
||||||
|
|
||||||
|
type deleteResolveManager struct {
|
||||||
|
*generationTestManager
|
||||||
|
returned chan struct{}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *deleteResolveManager) Build() fetcher.Fetcher {
|
||||||
|
return &deleteResolveFetcher{generationTestFetcher: m.generationTestManager.Build().(*generationTestFetcher), returned: m.returned}
|
||||||
|
}
|
||||||
|
|
||||||
|
type deleteResolveFetcher struct {
|
||||||
|
*generationTestFetcher
|
||||||
|
returned chan struct{}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *deleteResolveFetcher) Resolve(req *base.Request) error {
|
||||||
|
defer close(f.returned)
|
||||||
|
return f.generationTestFetcher.Resolve(req)
|
||||||
|
}
|
||||||
|
func waitDeleteTest(t *testing.T, ch <-chan struct{}) {
|
||||||
|
t.Helper()
|
||||||
|
select {
|
||||||
|
case <-ch:
|
||||||
|
case <-time.After(3 * time.Second):
|
||||||
|
t.Fatal("operation did not finish")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
func TestDeleteDuringResolveCannotRestartOrPersist(t *testing.T) {
|
||||||
|
for _, force := range []bool{false, true} {
|
||||||
|
for _, resolveErr := range []error{nil, errors.New("late resolve error")} {
|
||||||
|
t.Run(fmt.Sprintf("force=%v/error=%v", force, resolveErr != nil), func(t *testing.T) {
|
||||||
|
release := make(chan struct{})
|
||||||
|
var once sync.Once
|
||||||
|
defer once.Do(func() { close(release) })
|
||||||
|
m := &deleteResolveManager{generationTestManager: &generationTestManager{resolveStarted: make(chan struct{}), resolveRelease: release, resolveErr: resolveErr}, returned: make(chan struct{})}
|
||||||
|
d := NewDownloader(&DownloaderConfig{Storage: NewMemStorage(), StorageDir: t.TempDir(), TempDir: t.TempDir(), FetchManagers: []fetcher.FetcherManager{m}})
|
||||||
|
if err := d.Setup(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer d.Close()
|
||||||
|
id, err := d.CreateDirect(&base.Request{URL: "generation://delete"}, &base.Options{Path: t.TempDir(), Name: "delete.bin"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
waitDeleteTest(t, m.resolveStarted)
|
||||||
|
task := d.GetTask(id)
|
||||||
|
if err := d.Delete(&TaskFilter{IDs: []string{id}}, force); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
once.Do(func() { close(release) })
|
||||||
|
waitDeleteTest(t, m.returned)
|
||||||
|
task.lock.Lock()
|
||||||
|
task.lock.Unlock()
|
||||||
|
// A stale error callback can run immediately after the startup handler exits.
|
||||||
|
d.doOnError(task, errors.New("late callback"))
|
||||||
|
if got := m.starts.Load(); got != 0 {
|
||||||
|
t.Errorf("deleted task started %d time(s)", got)
|
||||||
|
}
|
||||||
|
if d.GetTask(id) != nil {
|
||||||
|
t.Error("deleted task is still listed")
|
||||||
|
}
|
||||||
|
for _, bucket := range []string{bucketTask, bucketSave, bucketTaskStats} {
|
||||||
|
var value any
|
||||||
|
present, err := d.storage.Get(bucket, id, &value)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if present {
|
||||||
|
t.Errorf("deleted task resurrected in %s", bucket)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type blockedTaskStorage struct {
|
||||||
|
Storage
|
||||||
|
entered chan struct{}
|
||||||
|
release chan struct{}
|
||||||
|
blockID string
|
||||||
|
once sync.Once
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *blockedTaskStorage) Put(bucket, key string, value any) error {
|
||||||
|
if bucket == bucketTask && key == s.blockID {
|
||||||
|
s.once.Do(func() { close(s.entered); <-s.release })
|
||||||
|
}
|
||||||
|
return s.Storage.Put(bucket, key, value)
|
||||||
|
}
|
||||||
|
func TestDeleteSerializesInFlightTaskCheckpoint(t *testing.T) {
|
||||||
|
s := &blockedTaskStorage{Storage: NewMemStorage(), entered: make(chan struct{}), release: make(chan struct{})}
|
||||||
|
d := NewDownloader(&DownloaderConfig{Storage: s, StorageDir: t.TempDir(), TempDir: t.TempDir(), FetchManagers: []fetcher.FetcherManager{&generationTestManager{}}})
|
||||||
|
if err := d.Setup(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer d.Close()
|
||||||
|
d.cfg.MaxRunning = 0
|
||||||
|
id, err := d.CreateDirect(&base.Request{URL: "generation://checkpoint"}, &base.Options{Path: t.TempDir()})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
task := d.GetTask(id)
|
||||||
|
s.blockID = id
|
||||||
|
saved := make(chan struct{})
|
||||||
|
go func() { defer close(saved); _ = d.saveTask(task) }()
|
||||||
|
waitDeleteTest(t, s.entered)
|
||||||
|
deleted := make(chan struct{})
|
||||||
|
go func() { defer close(deleted); _ = d.Delete(&TaskFilter{IDs: []string{id}}, false) }()
|
||||||
|
deadline := time.Now().Add(3 * time.Second)
|
||||||
|
for d.GetTask(id) != nil && time.Now().Before(deadline) {
|
||||||
|
time.Sleep(time.Millisecond)
|
||||||
|
}
|
||||||
|
close(s.release)
|
||||||
|
waitDeleteTest(t, saved)
|
||||||
|
waitDeleteTest(t, deleted)
|
||||||
|
var value any
|
||||||
|
present, err := s.Get(bucketTask, id, &value)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if present {
|
||||||
|
t.Fatal("in-flight checkpoint restored a deleted task")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDeleteDuringPreparationCannotStartOrPersist(t *testing.T) {
|
||||||
|
s := &blockedTaskStorage{Storage: NewMemStorage(), entered: make(chan struct{}), release: make(chan struct{})}
|
||||||
|
m := &generationTestManager{}
|
||||||
|
d := NewDownloader(&DownloaderConfig{Storage: s, StorageDir: t.TempDir(), TempDir: t.TempDir(), FetchManagers: []fetcher.FetcherManager{m}})
|
||||||
|
if err := d.Setup(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer d.Close()
|
||||||
|
d.cfg.MaxRunning = 0
|
||||||
|
id, err := d.CreateDirect(&base.Request{URL: "generation://preparation"}, &base.Options{Path: t.TempDir()})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
task := d.GetTask(id)
|
||||||
|
s.blockID = id
|
||||||
|
d.cfg.MaxRunning = 1
|
||||||
|
if err := d.Continue(&TaskFilter{IDs: []string{id}}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
waitDeleteTest(t, s.entered)
|
||||||
|
done := make(chan struct{})
|
||||||
|
go func() { defer close(done); _ = d.Delete(&TaskFilter{IDs: []string{id}}, true) }()
|
||||||
|
deadline := time.Now().Add(2 * time.Second)
|
||||||
|
for d.GetTask(id) != nil && time.Now().Before(deadline) {
|
||||||
|
time.Sleep(time.Millisecond)
|
||||||
|
}
|
||||||
|
close(s.release)
|
||||||
|
waitDeleteTest(t, done)
|
||||||
|
task.lock.Lock()
|
||||||
|
task.lock.Unlock()
|
||||||
|
if m.starts.Load() != 0 {
|
||||||
|
t.Fatal("deleted preparation started downloading")
|
||||||
|
}
|
||||||
|
var saved Task
|
||||||
|
if exists, err := s.Get(bucketTask, id, &saved); err != nil || exists {
|
||||||
|
t.Fatalf("deleted preparation persisted: exists=%v err=%v", exists, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
func TestDeletedTaskRejectsLateSnapshotsAndCompletion(t *testing.T) {
|
||||||
|
d := NewDownloader(&DownloaderConfig{Storage: NewMemStorage(), StorageDir: t.TempDir(), TempDir: t.TempDir(), FetchManagers: []fetcher.FetcherManager{&generationTestManager{}}})
|
||||||
|
if err := d.Setup(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer d.Close()
|
||||||
|
d.cfg.MaxRunning = 0
|
||||||
|
id, err := d.CreateDirect(&base.Request{URL: "generation://late"}, &base.Options{Path: t.TempDir()})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
task := d.GetTask(id)
|
||||||
|
snapshot := task.clone()
|
||||||
|
task.fetcher.(*generationTestFetcher).stats = &fetcher.Stats{Snapshot: map[string]any{"bytes": 1}}
|
||||||
|
if err := d.Delete(&TaskFilter{IDs: []string{id}}, false); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := d.putTaskRecord(task, snapshot); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := d.captureTaskStats(task, false, true); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := d.saveTask(task); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if d.markTaskDone(task) {
|
||||||
|
t.Error("deleted task accepted a late completion")
|
||||||
|
}
|
||||||
|
for _, bucket := range []string{bucketTask, bucketSave, bucketTaskStats} {
|
||||||
|
var value any
|
||||||
|
if exists, err := d.storage.Get(bucket, id, &value); err != nil || exists {
|
||||||
|
t.Errorf("late callback restored %s: exists=%v err=%v", bucket, exists, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type singleFailureTaskDeleteStorage struct {
|
||||||
|
Storage
|
||||||
|
fail atomic.Bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *singleFailureTaskDeleteStorage) Delete(bucket, key string) error {
|
||||||
|
if bucket == bucketTask && s.fail.Swap(false) {
|
||||||
|
return errors.New("task deletion failed")
|
||||||
|
}
|
||||||
|
return s.Storage.Delete(bucket, key)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFailedDeleteCanBeRetriedWithoutRevivingOldRun(t *testing.T) {
|
||||||
|
s := &singleFailureTaskDeleteStorage{Storage: NewMemStorage()}
|
||||||
|
m := &generationTestManager{holdOpen: true}
|
||||||
|
d := NewDownloader(&DownloaderConfig{Storage: s, StorageDir: t.TempDir(), TempDir: t.TempDir(), FetchManagers: []fetcher.FetcherManager{m}})
|
||||||
|
if err := d.Setup(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer d.Close()
|
||||||
|
d.cfg.MaxRunning = 0
|
||||||
|
id, err := d.CreateDirect(&base.Request{URL: "generation://failed-delete"}, &base.Options{Path: t.TempDir()})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
s.fail.Store(true)
|
||||||
|
if err := d.Delete(&TaskFilter{IDs: []string{id}}, false); err == nil {
|
||||||
|
t.Fatal("expected storage deletion failure")
|
||||||
|
}
|
||||||
|
task := d.GetTask(id)
|
||||||
|
if task == nil || d.taskStatus(task) != base.DownloadStatusWait {
|
||||||
|
t.Fatal("failed deletion did not restore queued task")
|
||||||
|
}
|
||||||
|
d.cfg.MaxRunning = 1
|
||||||
|
if err := d.Continue(&TaskFilter{IDs: []string{id}}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
deadline := time.Now().Add(time.Second)
|
||||||
|
for m.starts.Load() < 1 && time.Now().Before(deadline) {
|
||||||
|
time.Sleep(time.Millisecond)
|
||||||
|
}
|
||||||
|
if m.starts.Load() != 1 {
|
||||||
|
t.Fatal("failed deletion could not be retried")
|
||||||
|
}
|
||||||
|
}
|
||||||
+73
-23
@@ -508,7 +508,31 @@ func (d *Downloader) setupFetcher(fm fetcher.FetcherManager, fetcher fetcher.Fet
|
|||||||
fetcher.Setup(ctl)
|
fetcher.Setup(ctl)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// putTaskValue serializes late progress/completion callbacks with deletion.
|
||||||
|
// A snapshot may have been captured before Delete removed the task.
|
||||||
|
func (d *Downloader) putTaskValue(task *Task, bucket string, value any) error {
|
||||||
|
if task.lifetime != nil {
|
||||||
|
task.lifetime.mu.Lock()
|
||||||
|
defer task.lifetime.mu.Unlock()
|
||||||
|
if task.lifetime.deleted.Load() {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return d.storage.Put(bucket, task.ID, value)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (d *Downloader) putTaskRecord(task *Task, snapshot *Task) error {
|
||||||
|
return d.putTaskValue(task, bucketTask, snapshot)
|
||||||
|
}
|
||||||
|
|
||||||
func (d *Downloader) saveTask(task *Task) error {
|
func (d *Downloader) saveTask(task *Task) error {
|
||||||
|
if task.lifetime != nil {
|
||||||
|
task.lifetime.mu.Lock()
|
||||||
|
defer task.lifetime.mu.Unlock()
|
||||||
|
if task.lifetime.deleted.Load() {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
}
|
||||||
data, err := task.fetcherManager.Store(task.fetcher)
|
data, err := task.fetcherManager.Store(task.fetcher)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
d.Logger.Error().Stack().Err(err).Msgf("serialize fetcher failed: %s", task.ID)
|
d.Logger.Error().Stack().Err(err).Msgf("serialize fetcher failed: %s", task.ID)
|
||||||
@@ -1220,7 +1244,7 @@ func (d *Downloader) Delete(filter *TaskFilter, force bool) error {
|
|||||||
for _, task := range tasks {
|
for _, task := range tasks {
|
||||||
if err := d.doDelete(task, force); err != nil {
|
if err := d.doDelete(task, force); err != nil {
|
||||||
// Deletion can fail after removing bucketTask. Keep recovery durable.
|
// Deletion can fail after removing bucketTask. Keep recovery durable.
|
||||||
if saveErr := d.storage.Put(bucketTask, task.ID, task.clone()); saveErr != nil {
|
if saveErr := d.putTaskRecord(task, task.clone()); saveErr != nil {
|
||||||
d.Logger.Error().Err(saveErr).Msg("restore failed task deletion failed")
|
d.Logger.Error().Err(saveErr).Msg("restore failed task deletion failed")
|
||||||
if firstErr == nil {
|
if firstErr == nil {
|
||||||
firstErr = saveErr
|
firstErr = saveErr
|
||||||
@@ -1412,7 +1436,7 @@ func (d *Downloader) captureTaskStats(task *Task, live, force bool) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := d.storage.Put(bucketTaskStats, task.ID, &storedTaskStats{Snapshot: snapshot}); err != nil {
|
if err := d.putTaskValue(task, bucketTaskStats, &storedTaskStats{Snapshot: snapshot}); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
task.statsSnapshot = append(task.statsSnapshot[:0], snapshot...)
|
task.statsSnapshot = append(task.statsSnapshot[:0], snapshot...)
|
||||||
@@ -1518,6 +1542,18 @@ func (d *Downloader) RuntimeStatus(id string) (*TaskRuntimeStatus, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (d *Downloader) doDelete(task *Task, force bool) (err error) {
|
func (d *Downloader) doDelete(task *Task, force bool) (err error) {
|
||||||
|
task.lifetime.deleted.Store(true)
|
||||||
|
task.statusLock.Lock()
|
||||||
|
task.runGeneration++
|
||||||
|
task.statusLock.Unlock()
|
||||||
|
task.lifetime.mu.Lock()
|
||||||
|
defer task.lifetime.mu.Unlock()
|
||||||
|
defer func() {
|
||||||
|
if err != nil {
|
||||||
|
// Keep failed deletions recoverable without validating the old run.
|
||||||
|
task.lifetime.deleted.Store(false)
|
||||||
|
}
|
||||||
|
}()
|
||||||
defer d.releaseBlobTask(task)
|
defer d.releaseBlobTask(task)
|
||||||
err = func() error {
|
err = func() error {
|
||||||
if err := d.storage.Delete(bucketTask, task.ID); err != nil {
|
if err := d.storage.Delete(bucketTask, task.ID); err != nil {
|
||||||
@@ -1875,7 +1911,7 @@ func (d *Downloader) watch(task *Task) {
|
|||||||
// Check if the task is deleted
|
// Check if the task is deleted
|
||||||
if d.GetTask(task.ID) != nil {
|
if d.GetTask(task.ID) != nil {
|
||||||
task.Uploading = false
|
task.Uploading = false
|
||||||
d.storage.Put(bucketTask, task.ID, task.clone())
|
d.putTaskRecord(task, task.clone())
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
}
|
}
|
||||||
@@ -1930,7 +1966,7 @@ func (d *Downloader) watch(task *Task) {
|
|||||||
task.statusLock.Lock()
|
task.statusLock.Lock()
|
||||||
snapshot := task.clone()
|
snapshot := task.clone()
|
||||||
task.statusLock.Unlock()
|
task.statusLock.Unlock()
|
||||||
d.storage.Put(bucketTask, task.ID, snapshot)
|
d.putTaskRecord(task, snapshot)
|
||||||
d.emit(EventKeyDone, task)
|
d.emit(EventKeyDone, task)
|
||||||
d.emit(EventKeyFinally, task, err)
|
d.emit(EventKeyFinally, task, err)
|
||||||
d.notifyRunning()
|
d.notifyRunning()
|
||||||
@@ -2073,7 +2109,7 @@ func (d *Downloader) handleOnError(task *Task, err error, resetFetcher bool) {
|
|||||||
d.Logger.Warn().Err(statsErr).Msgf("persist failed task stats failed, task id: %s", task.ID)
|
d.Logger.Warn().Err(statsErr).Msgf("persist failed task stats failed, task id: %s", task.ID)
|
||||||
}
|
}
|
||||||
if d.taskStatus(task) == base.DownloadStatusError {
|
if d.taskStatus(task) == base.DownloadStatusError {
|
||||||
if persistErr := d.storage.Put(bucketTask, task.ID, task.clone()); persistErr != nil {
|
if persistErr := d.putTaskRecord(task, task.clone()); persistErr != nil {
|
||||||
d.Logger.Error().Stack().Err(persistErr).Msgf("persist failed task failed, task id: %s", task.ID)
|
d.Logger.Error().Stack().Err(persistErr).Msgf("persist failed task failed, task id: %s", task.ID)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -2201,7 +2237,7 @@ func (d *Downloader) doCreate(f fetcher.Fetcher, opts *base.Options, groupID str
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if err = d.storage.Put(bucketTask, task.ID, task.clone()); err != nil {
|
if err = d.putTaskRecord(task, task.clone()); err != nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
taskId = task.ID
|
taskId = task.ID
|
||||||
@@ -2368,7 +2404,7 @@ func (d *Downloader) prepareTask(task *Task) error {
|
|||||||
func (d *Downloader) doStart(task *Task) (err error) {
|
func (d *Downloader) doStart(task *Task) (err error) {
|
||||||
var generation uint64
|
var generation uint64
|
||||||
isReturn, err := d.statusMut(task, func() (isReturn bool, err error) {
|
isReturn, err := d.statusMut(task, func() (isReturn bool, err error) {
|
||||||
if task.Status == base.DownloadStatusRunning || task.Status == base.DownloadStatusDone {
|
if task.lifetime.deleted.Load() || task.Status == base.DownloadStatusRunning || task.Status == base.DownloadStatusDone {
|
||||||
isReturn = true
|
isReturn = true
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -2423,6 +2459,9 @@ func (d *Downloader) doStart(task *Task) (err error) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
if !d.taskIsRunningGeneration(task, generation) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
task.Meta.Res = task.fetcher.Meta().Res
|
task.Meta.Res = task.fetcher.Meta().Res
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -2447,16 +2486,24 @@ func (d *Downloader) doStart(task *Task) (err error) {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
task.lifetime.mu.Lock()
|
||||||
|
if task.lifetime.deleted.Load() {
|
||||||
|
task.lifetime.mu.Unlock()
|
||||||
|
return nil
|
||||||
|
}
|
||||||
task.statusLock.Lock()
|
task.statusLock.Lock()
|
||||||
if task.Status != base.DownloadStatusRunning || task.runGeneration != generation {
|
if task.Status != base.DownloadStatusRunning || task.runGeneration != generation {
|
||||||
|
task.lifetime.mu.Unlock()
|
||||||
task.statusLock.Unlock()
|
task.statusLock.Unlock()
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
task.Progress.Speed = 0
|
task.Progress.Speed = 0
|
||||||
task.timer.Start()
|
task.timer.Start()
|
||||||
task.statusLock.Unlock()
|
task.statusLock.Unlock()
|
||||||
if err := task.fetcher.Start(); err != nil {
|
startErr := task.fetcher.Start()
|
||||||
return err
|
task.lifetime.mu.Unlock()
|
||||||
|
if startErr != nil {
|
||||||
|
return startErr
|
||||||
}
|
}
|
||||||
started = true
|
started = true
|
||||||
if err := d.captureTaskStats(task, true, true); err != nil {
|
if err := d.captureTaskStats(task, true, true); err != nil {
|
||||||
@@ -2567,7 +2614,7 @@ func (d *Downloader) runPauseHandler(task *Task, generation uint64, pauseFetcher
|
|||||||
return false, err
|
return false, err
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
if err := d.storage.Put(bucketTask, task.ID, task.clone()); err != nil {
|
if err := d.putTaskRecord(task, task.clone()); err != nil {
|
||||||
return false, err
|
return false, err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -2580,7 +2627,7 @@ func (d *Downloader) taskIsRunningGeneration(task *Task, generation uint64) bool
|
|||||||
}
|
}
|
||||||
task.statusLock.Lock()
|
task.statusLock.Lock()
|
||||||
defer task.statusLock.Unlock()
|
defer task.statusLock.Unlock()
|
||||||
return task.Status == base.DownloadStatusRunning && task.runGeneration == generation
|
return !task.lifetime.deleted.Load() && task.Status == base.DownloadStatusRunning && task.runGeneration == generation
|
||||||
}
|
}
|
||||||
|
|
||||||
func (d *Downloader) taskIsPausedGeneration(task *Task, generation uint64) bool {
|
func (d *Downloader) taskIsPausedGeneration(task *Task, generation uint64) bool {
|
||||||
@@ -2589,7 +2636,7 @@ func (d *Downloader) taskIsPausedGeneration(task *Task, generation uint64) bool
|
|||||||
}
|
}
|
||||||
task.statusLock.Lock()
|
task.statusLock.Lock()
|
||||||
defer task.statusLock.Unlock()
|
defer task.statusLock.Unlock()
|
||||||
return (task.Status == base.DownloadStatusPause || task.Status == base.DownloadStatusWait) && task.runGeneration == generation
|
return !task.lifetime.deleted.Load() && (task.Status == base.DownloadStatusPause || task.Status == base.DownloadStatusWait) && task.runGeneration == generation
|
||||||
}
|
}
|
||||||
|
|
||||||
func (d *Downloader) taskStatus(task *Task) base.Status {
|
func (d *Downloader) taskStatus(task *Task) base.Status {
|
||||||
@@ -2609,7 +2656,7 @@ func (d *Downloader) markTaskDone(task *Task) bool {
|
|||||||
defer task.lock.Unlock()
|
defer task.lock.Unlock()
|
||||||
task.statusLock.Lock()
|
task.statusLock.Lock()
|
||||||
defer task.statusLock.Unlock()
|
defer task.statusLock.Unlock()
|
||||||
if task.Status != base.DownloadStatusRunning {
|
if task.lifetime.deleted.Load() || task.Status != base.DownloadStatusRunning {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
task.Progress.Used = task.timer.Used()
|
task.Progress.Used = task.timer.Used()
|
||||||
@@ -2637,7 +2684,7 @@ func (d *Downloader) markTaskError(task *Task) bool {
|
|||||||
}
|
}
|
||||||
task.statusLock.Lock()
|
task.statusLock.Lock()
|
||||||
defer task.statusLock.Unlock()
|
defer task.statusLock.Unlock()
|
||||||
if task.Status != base.DownloadStatusRunning {
|
if task.lifetime.deleted.Load() || task.Status != base.DownloadStatusRunning {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
task.updateStatus(base.DownloadStatusError)
|
task.updateStatus(base.DownloadStatusError)
|
||||||
@@ -2698,7 +2745,7 @@ func (d *Downloader) enqueueSingleExtraction(task *Task, downloadFilePath string
|
|||||||
// Set extraction status to queued
|
// Set extraction status to queued
|
||||||
task.Progress.ExtractStatus = ExtractStatusQueued
|
task.Progress.ExtractStatus = ExtractStatusQueued
|
||||||
d.emit(EventKeyProgress, task)
|
d.emit(EventKeyProgress, task)
|
||||||
d.storage.Put(bucketTask, task.ID, task.clone())
|
d.putTaskRecord(task, task.clone())
|
||||||
d.Logger.Info().Msgf("extraction queued, task id: %s, job id: %s", task.ID, jobID)
|
d.Logger.Info().Msgf("extraction queued, task id: %s, job id: %s", task.ID, jobID)
|
||||||
|
|
||||||
// Create and enqueue the extraction job
|
// Create and enqueue the extraction job
|
||||||
@@ -2727,7 +2774,7 @@ func (d *Downloader) enqueueMultiPartExtraction(task *Task, downloadFilePath str
|
|||||||
// Not all parts are ready yet - just set status to waiting, don't queue anything
|
// Not all parts are ready yet - just set status to waiting, don't queue anything
|
||||||
task.Progress.ExtractStatus = ExtractStatusWaitingParts
|
task.Progress.ExtractStatus = ExtractStatusWaitingParts
|
||||||
d.emit(EventKeyProgress, task)
|
d.emit(EventKeyProgress, task)
|
||||||
d.storage.Put(bucketTask, task.ID, task.clone())
|
d.putTaskRecord(task, task.clone())
|
||||||
d.Logger.Info().Msgf("multi-part archive waiting for other parts, task id: %s, missing: %v", task.ID, missingParts)
|
d.Logger.Info().Msgf("multi-part archive waiting for other parts, task id: %s, missing: %v", task.ID, missingParts)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -2743,14 +2790,14 @@ func (d *Downloader) enqueueMultiPartExtraction(task *Task, downloadFilePath str
|
|||||||
task.Progress.ExtractStatus = ExtractStatusDone
|
task.Progress.ExtractStatus = ExtractStatusDone
|
||||||
task.Progress.ExtractProgress = 100
|
task.Progress.ExtractProgress = 100
|
||||||
d.emit(EventKeyProgress, task)
|
d.emit(EventKeyProgress, task)
|
||||||
d.storage.Put(bucketTask, task.ID, task.clone())
|
d.putTaskRecord(task, task.clone())
|
||||||
d.Logger.Info().Msgf("multi-part archive extraction already handled by another part, task id: %s", task.ID)
|
d.Logger.Info().Msgf("multi-part archive extraction already handled by another part, task id: %s", task.ID)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
// This task claimed the extraction - status already set to queued in tryClaimMultiPartExtraction
|
// This task claimed the extraction - status already set to queued in tryClaimMultiPartExtraction
|
||||||
d.emit(EventKeyProgress, task)
|
d.emit(EventKeyProgress, task)
|
||||||
d.storage.Put(bucketTask, task.ID, task.clone())
|
d.putTaskRecord(task, task.clone())
|
||||||
|
|
||||||
jobID := "multipart:" + fullBaseName
|
jobID := "multipart:" + fullBaseName
|
||||||
d.Logger.Info().Msgf("multi-part extraction queued, task id: %s, job id: %s", task.ID, jobID)
|
d.Logger.Info().Msgf("multi-part extraction queued, task id: %s, job id: %s", task.ID, jobID)
|
||||||
@@ -2839,7 +2886,7 @@ func (d *Downloader) performExtraction(task *Task, archivePath string, destDir s
|
|||||||
task.Progress.ExtractStatus = ExtractStatusExtracting
|
task.Progress.ExtractStatus = ExtractStatusExtracting
|
||||||
task.Progress.ExtractProgress = 0
|
task.Progress.ExtractProgress = 0
|
||||||
d.emit(EventKeyProgress, task)
|
d.emit(EventKeyProgress, task)
|
||||||
d.storage.Put(bucketTask, task.ID, task.clone())
|
d.putTaskRecord(task, task.clone())
|
||||||
|
|
||||||
// Extract the archive
|
// Extract the archive
|
||||||
extractErr := extractArchive(archivePath, destDir, opts.ArchivePassword, func(extractedFiles int, totalFiles int, progress int) {
|
extractErr := extractArchive(archivePath, destDir, opts.ArchivePassword, func(extractedFiles int, totalFiles int, progress int) {
|
||||||
@@ -2859,7 +2906,7 @@ func (d *Downloader) performMultiPartExtraction(task *Task, firstPartPath string
|
|||||||
task.Progress.ExtractStatus = ExtractStatusExtracting
|
task.Progress.ExtractStatus = ExtractStatusExtracting
|
||||||
task.Progress.ExtractProgress = 0
|
task.Progress.ExtractProgress = 0
|
||||||
d.emit(EventKeyProgress, task)
|
d.emit(EventKeyProgress, task)
|
||||||
d.storage.Put(bucketTask, task.ID, task.clone())
|
d.putTaskRecord(task, task.clone())
|
||||||
|
|
||||||
d.Logger.Info().Msgf("starting multi-part archive extraction, first part: %s, task id: %s", firstPartPath, task.ID)
|
d.Logger.Info().Msgf("starting multi-part archive extraction, first part: %s, task id: %s", firstPartPath, task.ID)
|
||||||
|
|
||||||
@@ -3003,13 +3050,13 @@ func (d *Downloader) handleExtractionResult(task *Task, extractErr error, archiv
|
|||||||
d.Logger.Error().Err(extractErr).Msgf("auto extract archive failed, task id: %s", task.ID)
|
d.Logger.Error().Err(extractErr).Msgf("auto extract archive failed, task id: %s", task.ID)
|
||||||
task.Progress.ExtractStatus = ExtractStatusError
|
task.Progress.ExtractStatus = ExtractStatusError
|
||||||
d.emit(EventKeyProgress, task)
|
d.emit(EventKeyProgress, task)
|
||||||
d.storage.Put(bucketTask, task.ID, task.clone())
|
d.putTaskRecord(task, task.clone())
|
||||||
} else {
|
} else {
|
||||||
d.Logger.Info().Msgf("auto extract archive completed, task id: %s", task.ID)
|
d.Logger.Info().Msgf("auto extract archive completed, task id: %s", task.ID)
|
||||||
task.Progress.ExtractStatus = ExtractStatusDone
|
task.Progress.ExtractStatus = ExtractStatusDone
|
||||||
task.Progress.ExtractProgress = 100
|
task.Progress.ExtractProgress = 100
|
||||||
d.emit(EventKeyProgress, task)
|
d.emit(EventKeyProgress, task)
|
||||||
d.storage.Put(bucketTask, task.ID, task.clone())
|
d.putTaskRecord(task, task.clone())
|
||||||
|
|
||||||
// Delete archive files after successful extraction if enabled
|
// Delete archive files after successful extraction if enabled
|
||||||
if deleteAfterExtract {
|
if deleteAfterExtract {
|
||||||
@@ -3049,12 +3096,15 @@ func (d *Downloader) updateMultiPartTasksStatus(sourceTask *Task, extractErr err
|
|||||||
task.Progress.ExtractStatus = status
|
task.Progress.ExtractStatus = status
|
||||||
task.Progress.ExtractProgress = progress
|
task.Progress.ExtractProgress = progress
|
||||||
d.emit(EventKeyProgress, task)
|
d.emit(EventKeyProgress, task)
|
||||||
d.storage.Put(bucketTask, task.ID, task.clone())
|
d.putTaskRecord(task, task.clone())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func initTask(task *Task) {
|
func initTask(task *Task) {
|
||||||
|
if task.lifetime == nil {
|
||||||
|
task.lifetime = &taskLifetime{}
|
||||||
|
}
|
||||||
task.timer = util.NewTimer(task.Progress.Used)
|
task.timer = util.NewTimer(task.Progress.Used)
|
||||||
|
|
||||||
task.statusLock = &sync.Mutex{}
|
task.statusLock = &sync.Mutex{}
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ package download
|
|||||||
import (
|
import (
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"sync"
|
"sync"
|
||||||
|
"sync/atomic"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/GopeedLab/gopeed/internal/controller"
|
"github.com/GopeedLab/gopeed/internal/controller"
|
||||||
@@ -46,6 +47,14 @@ type CreateTaskBatchResult struct {
|
|||||||
// Task represents a download unit.
|
// Task represents a download unit.
|
||||||
// Options flow path:
|
// Options flow path:
|
||||||
// REST Request (model.CreateTask/ResolveTask) -> pkg/download Downloader -> Task.Meta.Opts (base.Options) -> Fetcher.Resolve/Start (e.g. internal/protocol/http/fetcher.go)
|
// REST Request (model.CreateTask/ResolveTask) -> pkg/download Downloader -> Task.Meta.Opts (base.Options) -> Fetcher.Resolve/Start (e.g. internal/protocol/http/fetcher.go)
|
||||||
|
// taskLifetime is shared with preparation candidates. Its mutex serializes
|
||||||
|
// physical starts and persistence with deletion; the tombstone also invalidates
|
||||||
|
// work that is currently outside that mutex (for example Resolve).
|
||||||
|
type taskLifetime struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
deleted atomic.Bool
|
||||||
|
}
|
||||||
|
|
||||||
type Task struct {
|
type Task struct {
|
||||||
ID string `json:"id"`
|
ID string `json:"id"`
|
||||||
Protocol string `json:"protocol"`
|
Protocol string `json:"protocol"`
|
||||||
@@ -87,6 +96,7 @@ type Task struct {
|
|||||||
blobRefLock *sync.Mutex
|
blobRefLock *sync.Mutex
|
||||||
blobURL string
|
blobURL string
|
||||||
runGeneration uint64
|
runGeneration uint64
|
||||||
|
lifetime *taskLifetime
|
||||||
speedArr []int64
|
speedArr []int64
|
||||||
uploadSpeedArr []int64
|
uploadSpeedArr []int64
|
||||||
}
|
}
|
||||||
@@ -123,6 +133,7 @@ func NewTask() *Task {
|
|||||||
}
|
}
|
||||||
return &Task{
|
return &Task{
|
||||||
ID: id,
|
ID: id,
|
||||||
|
lifetime: &taskLifetime{},
|
||||||
Status: base.DownloadStatusReady,
|
Status: base.DownloadStatusReady,
|
||||||
CreatedAt: time.Now(),
|
CreatedAt: time.Now(),
|
||||||
UpdatedAt: time.Now(),
|
UpdatedAt: time.Now(),
|
||||||
|
|||||||
Reference in new issue
Block a user