mirror of
https://github.com/GopeedLab/gopeed.git
synced 2026-11-10 10:30:28 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
54fc0f7289 | ||
|
|
917fc64046 | ||
|
|
4ffb172259 |
No files matched your search
@@ -418,16 +418,6 @@ func (f *Fetcher) updateMaxConnTime(d time.Duration) {
|
||||
}
|
||||
|
||||
func (f *Fetcher) Resolve(req *base.Request) error {
|
||||
return f.ResolveContext(context.Background(), req)
|
||||
}
|
||||
|
||||
// ResolveContext cancels the initial request while waiting for headers. Once
|
||||
// Resolve has returned, the retained response belongs to the fetcher and its
|
||||
// existing Pause/Close lifecycle controls the body and prefetch.
|
||||
func (f *Fetcher) ResolveContext(ctx context.Context, req *base.Request) error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := base.ParseReqExtra[fhttp.ReqExtra](req); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -439,8 +429,6 @@ func (f *Fetcher) ResolveContext(ctx context.Context, req *base.Request) error {
|
||||
// This allows the response to be reused for downloading (important for one-time URLs)
|
||||
// The body outlives Resolve, so the read timeout gets its own cancel func.
|
||||
reqCtx, reqCancel := context.WithCancel(context.Background())
|
||||
stopCancel := context.AfterFunc(ctx, reqCancel)
|
||||
defer stopCancel()
|
||||
httpReq, err := f.buildRequest(reqCtx, req)
|
||||
if err != nil {
|
||||
reqCancel()
|
||||
|
||||
@@ -1,36 +0,0 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/GopeedLab/gopeed/pkg/base"
|
||||
"github.com/GopeedLab/gopeed/pkg/download"
|
||||
restmodel "github.com/GopeedLab/gopeed/pkg/rest/model"
|
||||
)
|
||||
|
||||
func TestCancelledTorrentPreviewCanBeResolvedAgain(t *testing.T) {
|
||||
d := download.NewDownloader(&download.DownloaderConfig{StorageDir: t.TempDir(), TempDir: t.TempDir()})
|
||||
if err := d.Setup(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer d.Close()
|
||||
service, err := NewService(d)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
path, err := filepath.Abs("../../internal/protocol/bt/testdata/test.torrent")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
req := &restmodel.ResolveTask{Req: &base.Request{URL: path}}
|
||||
first := serviceTestRequest[*download.ResolveResult](t, service, http.MethodPost, "/api/v1/resolve", req)
|
||||
serviceTestRequest[any](t, service, http.MethodDelete, "/api/v1/resolve/"+first.ID, nil)
|
||||
// Release is idempotent, including an old window disposing after Create.
|
||||
serviceTestRequest[any](t, service, http.MethodDelete, "/api/v1/resolve/"+first.ID, nil)
|
||||
second := serviceTestRequest[*download.ResolveResult](t, service, http.MethodPost, "/api/v1/resolve", req)
|
||||
if second.Res.Hash != first.Res.Hash {
|
||||
t.Fatalf("different torrent after release: %v", second)
|
||||
}
|
||||
}
|
||||
+1
-9
@@ -51,7 +51,6 @@ func NewService(downloader *download.Downloader) (*Service, error) {
|
||||
service.routes = []RouteSpec{
|
||||
newRoute(http.MethodGet, "/api/v1/info", service.info),
|
||||
newRoute(http.MethodPost, "/api/v1/resolve", service.resolve),
|
||||
newRoute(http.MethodDelete, "/api/v1/resolve/{id}", service.releaseResolve),
|
||||
newRoute(http.MethodPost, "/api/v1/tasks", service.createTask),
|
||||
newRoute(http.MethodPost, "/api/v1/tasks/batch", service.createTaskBatch),
|
||||
newRoute(http.MethodPatch, "/api/v1/tasks/{id}", service.patchTask),
|
||||
@@ -144,20 +143,13 @@ func (s *Service) resolve(ctx *Context) *Response {
|
||||
if response := decodeRequest(ctx.Body, &request); response != nil {
|
||||
return response
|
||||
}
|
||||
result, err := s.Downloader.ResolveWithID(request.ID, request.Req)
|
||||
result, err := s.Downloader.Resolve(request.Req)
|
||||
if err != nil {
|
||||
return newErrorResponse(err.Error())
|
||||
}
|
||||
return newOKResponse(result)
|
||||
}
|
||||
|
||||
func (s *Service) releaseResolve(ctx *Context) *Response {
|
||||
if err := s.Downloader.ReleaseResolve(ctx.PathParam("id")); err != nil {
|
||||
return newErrorResponse(err.Error())
|
||||
}
|
||||
return newOKResponse(nil)
|
||||
}
|
||||
|
||||
func (s *Service) createTask(ctx *Context) *Response {
|
||||
var request restmodel.CreateTask
|
||||
if response := decodeRequest(ctx.Body, &request); response != nil {
|
||||
|
||||
@@ -0,0 +1,259 @@
|
||||
package download
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/GopeedLab/gopeed/internal/fetcher"
|
||||
"github.com/GopeedLab/gopeed/pkg/base"
|
||||
)
|
||||
|
||||
type deleteResolveManager struct {
|
||||
*generationTestManager
|
||||
returned chan struct{}
|
||||
}
|
||||
|
||||
func (m *deleteResolveManager) Build() fetcher.Fetcher {
|
||||
return &deleteResolveFetcher{generationTestFetcher: m.generationTestManager.Build().(*generationTestFetcher), returned: m.returned}
|
||||
}
|
||||
|
||||
type deleteResolveFetcher struct {
|
||||
*generationTestFetcher
|
||||
returned chan struct{}
|
||||
}
|
||||
|
||||
func (f *deleteResolveFetcher) Resolve(req *base.Request) error {
|
||||
defer close(f.returned)
|
||||
return f.generationTestFetcher.Resolve(req)
|
||||
}
|
||||
func waitDeleteTest(t *testing.T, ch <-chan struct{}) {
|
||||
t.Helper()
|
||||
select {
|
||||
case <-ch:
|
||||
case <-time.After(3 * time.Second):
|
||||
t.Fatal("operation did not finish")
|
||||
}
|
||||
}
|
||||
func TestDeleteDuringResolveCannotRestartOrPersist(t *testing.T) {
|
||||
for _, force := range []bool{false, true} {
|
||||
for _, resolveErr := range []error{nil, errors.New("late resolve error")} {
|
||||
t.Run(fmt.Sprintf("force=%v/error=%v", force, resolveErr != nil), func(t *testing.T) {
|
||||
release := make(chan struct{})
|
||||
var once sync.Once
|
||||
defer once.Do(func() { close(release) })
|
||||
m := &deleteResolveManager{generationTestManager: &generationTestManager{resolveStarted: make(chan struct{}), resolveRelease: release, resolveErr: resolveErr}, returned: make(chan struct{})}
|
||||
d := NewDownloader(&DownloaderConfig{Storage: NewMemStorage(), StorageDir: t.TempDir(), TempDir: t.TempDir(), FetchManagers: []fetcher.FetcherManager{m}})
|
||||
if err := d.Setup(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer d.Close()
|
||||
id, err := d.CreateDirect(&base.Request{URL: "generation://delete"}, &base.Options{Path: t.TempDir(), Name: "delete.bin"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waitDeleteTest(t, m.resolveStarted)
|
||||
task := d.GetTask(id)
|
||||
if err := d.Delete(&TaskFilter{IDs: []string{id}}, force); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
once.Do(func() { close(release) })
|
||||
waitDeleteTest(t, m.returned)
|
||||
task.lock.Lock()
|
||||
task.lock.Unlock()
|
||||
// A stale error callback can run immediately after the startup handler exits.
|
||||
d.doOnError(task, errors.New("late callback"))
|
||||
if got := m.starts.Load(); got != 0 {
|
||||
t.Errorf("deleted task started %d time(s)", got)
|
||||
}
|
||||
if d.GetTask(id) != nil {
|
||||
t.Error("deleted task is still listed")
|
||||
}
|
||||
for _, bucket := range []string{bucketTask, bucketSave, bucketTaskStats} {
|
||||
var value any
|
||||
present, err := d.storage.Get(bucket, id, &value)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if present {
|
||||
t.Errorf("deleted task resurrected in %s", bucket)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type blockedTaskStorage struct {
|
||||
Storage
|
||||
entered chan struct{}
|
||||
release chan struct{}
|
||||
blockID string
|
||||
once sync.Once
|
||||
}
|
||||
|
||||
func (s *blockedTaskStorage) Put(bucket, key string, value any) error {
|
||||
if bucket == bucketTask && key == s.blockID {
|
||||
s.once.Do(func() { close(s.entered); <-s.release })
|
||||
}
|
||||
return s.Storage.Put(bucket, key, value)
|
||||
}
|
||||
func TestDeleteSerializesInFlightTaskCheckpoint(t *testing.T) {
|
||||
s := &blockedTaskStorage{Storage: NewMemStorage(), entered: make(chan struct{}), release: make(chan struct{})}
|
||||
d := NewDownloader(&DownloaderConfig{Storage: s, StorageDir: t.TempDir(), TempDir: t.TempDir(), FetchManagers: []fetcher.FetcherManager{&generationTestManager{}}})
|
||||
if err := d.Setup(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer d.Close()
|
||||
d.cfg.MaxRunning = 0
|
||||
id, err := d.CreateDirect(&base.Request{URL: "generation://checkpoint"}, &base.Options{Path: t.TempDir()})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
task := d.GetTask(id)
|
||||
s.blockID = id
|
||||
saved := make(chan struct{})
|
||||
go func() { defer close(saved); _ = d.saveTask(task) }()
|
||||
waitDeleteTest(t, s.entered)
|
||||
deleted := make(chan struct{})
|
||||
go func() { defer close(deleted); _ = d.Delete(&TaskFilter{IDs: []string{id}}, false) }()
|
||||
deadline := time.Now().Add(3 * time.Second)
|
||||
for d.GetTask(id) != nil && time.Now().Before(deadline) {
|
||||
time.Sleep(time.Millisecond)
|
||||
}
|
||||
close(s.release)
|
||||
waitDeleteTest(t, saved)
|
||||
waitDeleteTest(t, deleted)
|
||||
var value any
|
||||
present, err := s.Get(bucketTask, id, &value)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if present {
|
||||
t.Fatal("in-flight checkpoint restored a deleted task")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeleteDuringPreparationCannotStartOrPersist(t *testing.T) {
|
||||
s := &blockedTaskStorage{Storage: NewMemStorage(), entered: make(chan struct{}), release: make(chan struct{})}
|
||||
m := &generationTestManager{}
|
||||
d := NewDownloader(&DownloaderConfig{Storage: s, StorageDir: t.TempDir(), TempDir: t.TempDir(), FetchManagers: []fetcher.FetcherManager{m}})
|
||||
if err := d.Setup(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer d.Close()
|
||||
d.cfg.MaxRunning = 0
|
||||
id, err := d.CreateDirect(&base.Request{URL: "generation://preparation"}, &base.Options{Path: t.TempDir()})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
task := d.GetTask(id)
|
||||
s.blockID = id
|
||||
d.cfg.MaxRunning = 1
|
||||
if err := d.Continue(&TaskFilter{IDs: []string{id}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waitDeleteTest(t, s.entered)
|
||||
done := make(chan struct{})
|
||||
go func() { defer close(done); _ = d.Delete(&TaskFilter{IDs: []string{id}}, true) }()
|
||||
deadline := time.Now().Add(2 * time.Second)
|
||||
for d.GetTask(id) != nil && time.Now().Before(deadline) {
|
||||
time.Sleep(time.Millisecond)
|
||||
}
|
||||
close(s.release)
|
||||
waitDeleteTest(t, done)
|
||||
task.lock.Lock()
|
||||
task.lock.Unlock()
|
||||
if m.starts.Load() != 0 {
|
||||
t.Fatal("deleted preparation started downloading")
|
||||
}
|
||||
var saved Task
|
||||
if exists, err := s.Get(bucketTask, id, &saved); err != nil || exists {
|
||||
t.Fatalf("deleted preparation persisted: exists=%v err=%v", exists, err)
|
||||
}
|
||||
}
|
||||
func TestDeletedTaskRejectsLateSnapshotsAndCompletion(t *testing.T) {
|
||||
d := NewDownloader(&DownloaderConfig{Storage: NewMemStorage(), StorageDir: t.TempDir(), TempDir: t.TempDir(), FetchManagers: []fetcher.FetcherManager{&generationTestManager{}}})
|
||||
if err := d.Setup(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer d.Close()
|
||||
d.cfg.MaxRunning = 0
|
||||
id, err := d.CreateDirect(&base.Request{URL: "generation://late"}, &base.Options{Path: t.TempDir()})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
task := d.GetTask(id)
|
||||
snapshot := task.clone()
|
||||
task.fetcher.(*generationTestFetcher).stats = &fetcher.Stats{Snapshot: map[string]any{"bytes": 1}}
|
||||
if err := d.Delete(&TaskFilter{IDs: []string{id}}, false); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := d.putTaskRecord(task, snapshot); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := d.captureTaskStats(task, false, true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := d.saveTask(task); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if d.markTaskDone(task) {
|
||||
t.Error("deleted task accepted a late completion")
|
||||
}
|
||||
for _, bucket := range []string{bucketTask, bucketSave, bucketTaskStats} {
|
||||
var value any
|
||||
if exists, err := d.storage.Get(bucket, id, &value); err != nil || exists {
|
||||
t.Errorf("late callback restored %s: exists=%v err=%v", bucket, exists, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type singleFailureTaskDeleteStorage struct {
|
||||
Storage
|
||||
fail atomic.Bool
|
||||
}
|
||||
|
||||
func (s *singleFailureTaskDeleteStorage) Delete(bucket, key string) error {
|
||||
if bucket == bucketTask && s.fail.Swap(false) {
|
||||
return errors.New("task deletion failed")
|
||||
}
|
||||
return s.Storage.Delete(bucket, key)
|
||||
}
|
||||
|
||||
func TestFailedDeleteCanBeRetriedWithoutRevivingOldRun(t *testing.T) {
|
||||
s := &singleFailureTaskDeleteStorage{Storage: NewMemStorage()}
|
||||
m := &generationTestManager{holdOpen: true}
|
||||
d := NewDownloader(&DownloaderConfig{Storage: s, StorageDir: t.TempDir(), TempDir: t.TempDir(), FetchManagers: []fetcher.FetcherManager{m}})
|
||||
if err := d.Setup(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer d.Close()
|
||||
d.cfg.MaxRunning = 0
|
||||
id, err := d.CreateDirect(&base.Request{URL: "generation://failed-delete"}, &base.Options{Path: t.TempDir()})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
s.fail.Store(true)
|
||||
if err := d.Delete(&TaskFilter{IDs: []string{id}}, false); err == nil {
|
||||
t.Fatal("expected storage deletion failure")
|
||||
}
|
||||
task := d.GetTask(id)
|
||||
if task == nil || d.taskStatus(task) != base.DownloadStatusWait {
|
||||
t.Fatal("failed deletion did not restore queued task")
|
||||
}
|
||||
d.cfg.MaxRunning = 1
|
||||
if err := d.Continue(&TaskFilter{IDs: []string{id}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
deadline := time.Now().Add(time.Second)
|
||||
for m.starts.Load() < 1 && time.Now().Before(deadline) {
|
||||
time.Sleep(time.Millisecond)
|
||||
}
|
||||
if m.starts.Load() != 1 {
|
||||
t.Fatal("failed deletion could not be retried")
|
||||
}
|
||||
}
|
||||
+102
-185
@@ -116,7 +116,6 @@ type Downloader struct {
|
||||
|
||||
cfg *DownloaderConfig
|
||||
fetcherCache map[string]fetcher.Fetcher
|
||||
resolveStates map[string]*resolveState
|
||||
storage Storage
|
||||
tasks []*Task
|
||||
waitTasks []*Task
|
||||
@@ -154,7 +153,6 @@ func NewDownloader(cfg *DownloaderConfig) *Downloader {
|
||||
cancelBackground: cancelBackground,
|
||||
cfg: cfg,
|
||||
fetcherCache: make(map[string]fetcher.Fetcher),
|
||||
resolveStates: make(map[string]*resolveState),
|
||||
waitTasks: make([]*Task, 0),
|
||||
groups: make(map[string]*TaskGroup),
|
||||
creatingGroups: make(map[string]struct{}),
|
||||
@@ -510,7 +508,31 @@ func (d *Downloader) setupFetcher(fm fetcher.FetcherManager, fetcher fetcher.Fet
|
||||
fetcher.Setup(ctl)
|
||||
}
|
||||
|
||||
// putTaskValue serializes late progress/completion callbacks with deletion.
|
||||
// A snapshot may have been captured before Delete removed the task.
|
||||
func (d *Downloader) putTaskValue(task *Task, bucket string, value any) error {
|
||||
if task.lifetime != nil {
|
||||
task.lifetime.mu.Lock()
|
||||
defer task.lifetime.mu.Unlock()
|
||||
if task.lifetime.deleted.Load() {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return d.storage.Put(bucket, task.ID, value)
|
||||
}
|
||||
|
||||
func (d *Downloader) putTaskRecord(task *Task, snapshot *Task) error {
|
||||
return d.putTaskValue(task, bucketTask, snapshot)
|
||||
}
|
||||
|
||||
func (d *Downloader) saveTask(task *Task) error {
|
||||
if task.lifetime != nil {
|
||||
task.lifetime.mu.Lock()
|
||||
defer task.lifetime.mu.Unlock()
|
||||
if task.lifetime.deleted.Load() {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
data, err := task.fetcherManager.Store(task.fetcher)
|
||||
if err != nil {
|
||||
d.Logger.Error().Stack().Err(err).Msgf("serialize fetcher failed: %s", task.ID)
|
||||
@@ -561,156 +583,42 @@ func ensureResourceRequestRawURLs(parentReq *base.Request, res *base.Resource) {
|
||||
}
|
||||
}
|
||||
|
||||
// resolveState owns a Fetcher until Create transfers it to a task.
|
||||
// State transitions are protected by fetcherMapLock; closeOnce serializes cleanup.
|
||||
type contextResolver interface {
|
||||
ResolveContext(context.Context, *base.Request) error
|
||||
}
|
||||
|
||||
type resolveState struct {
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
fetcher fetcher.Fetcher
|
||||
cancelled bool
|
||||
resolved bool
|
||||
claimed bool
|
||||
closeOnce sync.Once
|
||||
closeErr error
|
||||
}
|
||||
|
||||
func (s *resolveState) close(f fetcher.Fetcher) error {
|
||||
if f == nil {
|
||||
return nil
|
||||
func (d *Downloader) Resolve(req *base.Request) (rr *ResolveResult, err error) {
|
||||
rrId, err := gonanoid.New()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
s.closeOnce.Do(func() { s.closeErr = f.Close() })
|
||||
return s.closeErr
|
||||
}
|
||||
|
||||
func (d *Downloader) Resolve(req *base.Request) (*ResolveResult, error) {
|
||||
return d.ResolveWithID("", req)
|
||||
}
|
||||
|
||||
// ResolveWithID lets a preview cancel before metadata is available. Clients
|
||||
// that do not supply an ID retain the server-generated resource ID behavior.
|
||||
func (d *Downloader) ResolveWithID(id string, req *base.Request) (rr *ResolveResult, err error) {
|
||||
if id == "" {
|
||||
id, err = gonanoid.New()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if req == nil {
|
||||
return nil, errors.New("resolve request is required")
|
||||
}
|
||||
resolveCtx, cancel := context.WithCancel(d.backgroundCtx)
|
||||
defer cancel()
|
||||
state := &resolveState{ctx: resolveCtx, cancel: cancel}
|
||||
d.fetcherMapLock.Lock()
|
||||
if d.closed.Load() {
|
||||
d.fetcherMapLock.Unlock()
|
||||
return nil, errors.New("downloader is closed")
|
||||
}
|
||||
if d.resolveStates[id] != nil || d.fetcherCache[id] != nil {
|
||||
d.fetcherMapLock.Unlock()
|
||||
return nil, errors.New("resource id is already in use")
|
||||
}
|
||||
d.resolveStates[id] = state
|
||||
d.fetcherMapLock.Unlock()
|
||||
defer func() {
|
||||
if err != nil || state.fetcher == nil {
|
||||
d.fetcherMapLock.Lock()
|
||||
if d.resolveStates[id] == state {
|
||||
delete(d.resolveStates, id)
|
||||
}
|
||||
d.fetcherMapLock.Unlock()
|
||||
_ = state.close(state.fetcher)
|
||||
}
|
||||
}()
|
||||
ensureRequestRawURL(req)
|
||||
|
||||
res, err := d.triggerOnResolve(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return
|
||||
}
|
||||
if res != nil && len(res.Files) > 0 {
|
||||
return &ResolveResult{Res: res, SuggestedDownloadDir: d.suggestedDownloadDir(res)}, nil
|
||||
}
|
||||
f, err := d.buildFetcher(req.URL)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
d.fetcherMapLock.Lock()
|
||||
state.fetcher = f
|
||||
cancelled := state.cancelled || d.closed.Load()
|
||||
d.fetcherMapLock.Unlock()
|
||||
if cancelled {
|
||||
return nil, errors.New("resource resolution cancelled")
|
||||
}
|
||||
if resolver, ok := f.(contextResolver); ok {
|
||||
err = resolver.ResolveContext(state.ctx, req)
|
||||
} else {
|
||||
err = f.Resolve(req)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
d.fetcherMapLock.Lock()
|
||||
defer d.fetcherMapLock.Unlock()
|
||||
if state.cancelled || d.closed.Load() {
|
||||
return nil, errors.New("resource resolution cancelled")
|
||||
}
|
||||
state.resolved = true
|
||||
d.fetcherCache[id] = f
|
||||
return &ResolveResult{ID: id, Res: f.Meta().Res, SuggestedDownloadDir: d.suggestedDownloadDir(f.Meta().Res)}, nil
|
||||
}
|
||||
|
||||
// ReleaseResolve is idempotent. A resource already transferred to a task is
|
||||
// never closed by a late preview cancellation.
|
||||
func (d *Downloader) ReleaseResolve(id string) error {
|
||||
d.fetcherMapLock.Lock()
|
||||
state := d.resolveStates[id]
|
||||
f := d.fetcherCache[id]
|
||||
delete(d.fetcherCache, id)
|
||||
delete(d.resolveStates, id)
|
||||
if state != nil {
|
||||
state.cancelled = true
|
||||
state.cancel()
|
||||
if state.claimed {
|
||||
f = nil
|
||||
} else {
|
||||
f = state.fetcher
|
||||
if _, cancellable := f.(contextResolver); cancellable && !state.resolved {
|
||||
// Resolve owns unpublished response/prefetch state. Its context
|
||||
// cancels the request; the returning resolver closes it once
|
||||
// publication has stopped, including a racing response.
|
||||
f = nil
|
||||
}
|
||||
rr = &ResolveResult{
|
||||
Res: res,
|
||||
SuggestedDownloadDir: d.suggestedDownloadDir(res),
|
||||
}
|
||||
return
|
||||
}
|
||||
d.fetcherMapLock.Unlock()
|
||||
if state != nil {
|
||||
return state.close(f)
|
||||
}
|
||||
if f != nil {
|
||||
return f.Close()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *Downloader) closeResolvedResources() {
|
||||
fetcher, err := d.buildFetcher(req.URL)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = fetcher.Resolve(req)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
d.fetcherMapLock.Lock()
|
||||
ids := make(map[string]struct{}, len(d.resolveStates)+len(d.fetcherCache))
|
||||
for id := range d.resolveStates {
|
||||
ids[id] = struct{}{}
|
||||
}
|
||||
for id := range d.fetcherCache {
|
||||
ids[id] = struct{}{}
|
||||
}
|
||||
d.fetcherCache[rrId] = fetcher
|
||||
d.fetcherMapLock.Unlock()
|
||||
for id := range ids {
|
||||
if err := d.ReleaseResolve(id); err != nil {
|
||||
d.Logger.Warn().Err(err).Msg("close resolved resource failed")
|
||||
}
|
||||
rr = &ResolveResult{
|
||||
ID: rrId,
|
||||
Res: fetcher.Meta().Res,
|
||||
SuggestedDownloadDir: d.suggestedDownloadDir(fetcher.Meta().Res),
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (d *Downloader) notifyRunning() {
|
||||
@@ -984,34 +892,18 @@ func (d *Downloader) CreateWithOptions(rrId string, opts *base.Options) (string,
|
||||
func (d *Downloader) createResolved(rrId string, opts *base.Options, groupID string) (taskId string, err error) {
|
||||
d.fetcherMapLock.Lock()
|
||||
fetcher, ok := d.fetcherCache[rrId]
|
||||
state := d.resolveStates[rrId]
|
||||
if ok && state != nil {
|
||||
state.claimed = true
|
||||
}
|
||||
delete(d.fetcherCache, rrId)
|
||||
d.fetcherMapLock.Unlock()
|
||||
if !ok {
|
||||
return "", errors.New("invalid resource id")
|
||||
}
|
||||
// Failed creation returns ownership to the preview unless it was cancelled.
|
||||
// Only one creator may use a resolved fetcher. Failed creation makes it
|
||||
// available again; a persisted task consumes it even if starting fails.
|
||||
defer func() {
|
||||
d.fetcherMapLock.Lock()
|
||||
cancelled := d.closed.Load() || (state != nil && state.cancelled)
|
||||
if taskId == "" && !cancelled {
|
||||
if taskId == "" {
|
||||
d.fetcherMapLock.Lock()
|
||||
d.fetcherCache[rrId] = fetcher
|
||||
if state != nil {
|
||||
state.claimed = false
|
||||
}
|
||||
} else if d.resolveStates[rrId] == state {
|
||||
delete(d.resolveStates, rrId)
|
||||
}
|
||||
d.fetcherMapLock.Unlock()
|
||||
if taskId == "" && cancelled {
|
||||
if state != nil {
|
||||
_ = state.close(fetcher)
|
||||
} else {
|
||||
_ = fetcher.Close()
|
||||
}
|
||||
d.fetcherMapLock.Unlock()
|
||||
}
|
||||
}()
|
||||
if opts == nil {
|
||||
@@ -1352,7 +1244,7 @@ func (d *Downloader) Delete(filter *TaskFilter, force bool) error {
|
||||
for _, task := range tasks {
|
||||
if err := d.doDelete(task, force); err != nil {
|
||||
// Deletion can fail after removing bucketTask. Keep recovery durable.
|
||||
if saveErr := d.storage.Put(bucketTask, task.ID, task.clone()); saveErr != nil {
|
||||
if saveErr := d.putTaskRecord(task, task.clone()); saveErr != nil {
|
||||
d.Logger.Error().Err(saveErr).Msg("restore failed task deletion failed")
|
||||
if firstErr == nil {
|
||||
firstErr = saveErr
|
||||
@@ -1544,7 +1436,7 @@ func (d *Downloader) captureTaskStats(task *Task, live, force bool) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := d.storage.Put(bucketTaskStats, task.ID, &storedTaskStats{Snapshot: snapshot}); err != nil {
|
||||
if err := d.putTaskValue(task, bucketTaskStats, &storedTaskStats{Snapshot: snapshot}); err != nil {
|
||||
return err
|
||||
}
|
||||
task.statsSnapshot = append(task.statsSnapshot[:0], snapshot...)
|
||||
@@ -1650,6 +1542,18 @@ func (d *Downloader) RuntimeStatus(id string) (*TaskRuntimeStatus, error) {
|
||||
}
|
||||
|
||||
func (d *Downloader) doDelete(task *Task, force bool) (err error) {
|
||||
task.lifetime.deleted.Store(true)
|
||||
task.statusLock.Lock()
|
||||
task.runGeneration++
|
||||
task.statusLock.Unlock()
|
||||
task.lifetime.mu.Lock()
|
||||
defer task.lifetime.mu.Unlock()
|
||||
defer func() {
|
||||
if err != nil {
|
||||
// Keep failed deletions recoverable without validating the old run.
|
||||
task.lifetime.deleted.Store(false)
|
||||
}
|
||||
}()
|
||||
defer d.releaseBlobTask(task)
|
||||
err = func() error {
|
||||
if err := d.storage.Delete(bucketTask, task.ID); err != nil {
|
||||
@@ -1694,7 +1598,6 @@ func (d *Downloader) Close() error {
|
||||
return nil
|
||||
}
|
||||
d.cancelBackground()
|
||||
d.closeResolvedResources()
|
||||
d.waitForExtensionGit()
|
||||
if d.protocolJobs != nil {
|
||||
d.protocolJobs.close()
|
||||
@@ -2008,7 +1911,7 @@ func (d *Downloader) watch(task *Task) {
|
||||
// Check if the task is deleted
|
||||
if d.GetTask(task.ID) != nil {
|
||||
task.Uploading = false
|
||||
d.storage.Put(bucketTask, task.ID, task.clone())
|
||||
d.putTaskRecord(task, task.clone())
|
||||
}
|
||||
}()
|
||||
}
|
||||
@@ -2063,7 +1966,7 @@ func (d *Downloader) watch(task *Task) {
|
||||
task.statusLock.Lock()
|
||||
snapshot := task.clone()
|
||||
task.statusLock.Unlock()
|
||||
d.storage.Put(bucketTask, task.ID, snapshot)
|
||||
d.putTaskRecord(task, snapshot)
|
||||
d.emit(EventKeyDone, task)
|
||||
d.emit(EventKeyFinally, task, err)
|
||||
d.notifyRunning()
|
||||
@@ -2206,7 +2109,7 @@ func (d *Downloader) handleOnError(task *Task, err error, resetFetcher bool) {
|
||||
d.Logger.Warn().Err(statsErr).Msgf("persist failed task stats failed, task id: %s", task.ID)
|
||||
}
|
||||
if d.taskStatus(task) == base.DownloadStatusError {
|
||||
if persistErr := d.storage.Put(bucketTask, task.ID, task.clone()); persistErr != nil {
|
||||
if persistErr := d.putTaskRecord(task, task.clone()); persistErr != nil {
|
||||
d.Logger.Error().Stack().Err(persistErr).Msgf("persist failed task failed, task id: %s", task.ID)
|
||||
}
|
||||
}
|
||||
@@ -2334,7 +2237,7 @@ func (d *Downloader) doCreate(f fetcher.Fetcher, opts *base.Options, groupID str
|
||||
return
|
||||
}
|
||||
}
|
||||
if err = d.storage.Put(bucketTask, task.ID, task.clone()); err != nil {
|
||||
if err = d.putTaskRecord(task, task.clone()); err != nil {
|
||||
return
|
||||
}
|
||||
taskId = task.ID
|
||||
@@ -2501,7 +2404,7 @@ func (d *Downloader) prepareTask(task *Task) error {
|
||||
func (d *Downloader) doStart(task *Task) (err error) {
|
||||
var generation uint64
|
||||
isReturn, err := d.statusMut(task, func() (isReturn bool, err error) {
|
||||
if task.Status == base.DownloadStatusRunning || task.Status == base.DownloadStatusDone {
|
||||
if task.lifetime.deleted.Load() || task.Status == base.DownloadStatusRunning || task.Status == base.DownloadStatusDone {
|
||||
isReturn = true
|
||||
return
|
||||
}
|
||||
@@ -2556,6 +2459,9 @@ func (d *Downloader) doStart(task *Task) (err error) {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !d.taskIsRunningGeneration(task, generation) {
|
||||
return nil
|
||||
}
|
||||
task.Meta.Res = task.fetcher.Meta().Res
|
||||
}
|
||||
|
||||
@@ -2580,16 +2486,24 @@ func (d *Downloader) doStart(task *Task) (err error) {
|
||||
return err
|
||||
}
|
||||
|
||||
task.lifetime.mu.Lock()
|
||||
if task.lifetime.deleted.Load() {
|
||||
task.lifetime.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
task.statusLock.Lock()
|
||||
if task.Status != base.DownloadStatusRunning || task.runGeneration != generation {
|
||||
task.lifetime.mu.Unlock()
|
||||
task.statusLock.Unlock()
|
||||
return nil
|
||||
}
|
||||
task.Progress.Speed = 0
|
||||
task.timer.Start()
|
||||
task.statusLock.Unlock()
|
||||
if err := task.fetcher.Start(); err != nil {
|
||||
return err
|
||||
startErr := task.fetcher.Start()
|
||||
task.lifetime.mu.Unlock()
|
||||
if startErr != nil {
|
||||
return startErr
|
||||
}
|
||||
started = true
|
||||
if err := d.captureTaskStats(task, true, true); err != nil {
|
||||
@@ -2700,7 +2614,7 @@ func (d *Downloader) runPauseHandler(task *Task, generation uint64, pauseFetcher
|
||||
return false, err
|
||||
}
|
||||
} else {
|
||||
if err := d.storage.Put(bucketTask, task.ID, task.clone()); err != nil {
|
||||
if err := d.putTaskRecord(task, task.clone()); err != nil {
|
||||
return false, err
|
||||
}
|
||||
}
|
||||
@@ -2713,7 +2627,7 @@ func (d *Downloader) taskIsRunningGeneration(task *Task, generation uint64) bool
|
||||
}
|
||||
task.statusLock.Lock()
|
||||
defer task.statusLock.Unlock()
|
||||
return task.Status == base.DownloadStatusRunning && task.runGeneration == generation
|
||||
return !task.lifetime.deleted.Load() && task.Status == base.DownloadStatusRunning && task.runGeneration == generation
|
||||
}
|
||||
|
||||
func (d *Downloader) taskIsPausedGeneration(task *Task, generation uint64) bool {
|
||||
@@ -2722,7 +2636,7 @@ func (d *Downloader) taskIsPausedGeneration(task *Task, generation uint64) bool
|
||||
}
|
||||
task.statusLock.Lock()
|
||||
defer task.statusLock.Unlock()
|
||||
return (task.Status == base.DownloadStatusPause || task.Status == base.DownloadStatusWait) && task.runGeneration == generation
|
||||
return !task.lifetime.deleted.Load() && (task.Status == base.DownloadStatusPause || task.Status == base.DownloadStatusWait) && task.runGeneration == generation
|
||||
}
|
||||
|
||||
func (d *Downloader) taskStatus(task *Task) base.Status {
|
||||
@@ -2742,7 +2656,7 @@ func (d *Downloader) markTaskDone(task *Task) bool {
|
||||
defer task.lock.Unlock()
|
||||
task.statusLock.Lock()
|
||||
defer task.statusLock.Unlock()
|
||||
if task.Status != base.DownloadStatusRunning {
|
||||
if task.lifetime.deleted.Load() || task.Status != base.DownloadStatusRunning {
|
||||
return false
|
||||
}
|
||||
task.Progress.Used = task.timer.Used()
|
||||
@@ -2770,7 +2684,7 @@ func (d *Downloader) markTaskError(task *Task) bool {
|
||||
}
|
||||
task.statusLock.Lock()
|
||||
defer task.statusLock.Unlock()
|
||||
if task.Status != base.DownloadStatusRunning {
|
||||
if task.lifetime.deleted.Load() || task.Status != base.DownloadStatusRunning {
|
||||
return false
|
||||
}
|
||||
task.updateStatus(base.DownloadStatusError)
|
||||
@@ -2831,7 +2745,7 @@ func (d *Downloader) enqueueSingleExtraction(task *Task, downloadFilePath string
|
||||
// Set extraction status to queued
|
||||
task.Progress.ExtractStatus = ExtractStatusQueued
|
||||
d.emit(EventKeyProgress, task)
|
||||
d.storage.Put(bucketTask, task.ID, task.clone())
|
||||
d.putTaskRecord(task, task.clone())
|
||||
d.Logger.Info().Msgf("extraction queued, task id: %s, job id: %s", task.ID, jobID)
|
||||
|
||||
// Create and enqueue the extraction job
|
||||
@@ -2860,7 +2774,7 @@ func (d *Downloader) enqueueMultiPartExtraction(task *Task, downloadFilePath str
|
||||
// Not all parts are ready yet - just set status to waiting, don't queue anything
|
||||
task.Progress.ExtractStatus = ExtractStatusWaitingParts
|
||||
d.emit(EventKeyProgress, task)
|
||||
d.storage.Put(bucketTask, task.ID, task.clone())
|
||||
d.putTaskRecord(task, task.clone())
|
||||
d.Logger.Info().Msgf("multi-part archive waiting for other parts, task id: %s, missing: %v", task.ID, missingParts)
|
||||
return
|
||||
}
|
||||
@@ -2876,14 +2790,14 @@ func (d *Downloader) enqueueMultiPartExtraction(task *Task, downloadFilePath str
|
||||
task.Progress.ExtractStatus = ExtractStatusDone
|
||||
task.Progress.ExtractProgress = 100
|
||||
d.emit(EventKeyProgress, task)
|
||||
d.storage.Put(bucketTask, task.ID, task.clone())
|
||||
d.putTaskRecord(task, task.clone())
|
||||
d.Logger.Info().Msgf("multi-part archive extraction already handled by another part, task id: %s", task.ID)
|
||||
return
|
||||
}
|
||||
|
||||
// This task claimed the extraction - status already set to queued in tryClaimMultiPartExtraction
|
||||
d.emit(EventKeyProgress, task)
|
||||
d.storage.Put(bucketTask, task.ID, task.clone())
|
||||
d.putTaskRecord(task, task.clone())
|
||||
|
||||
jobID := "multipart:" + fullBaseName
|
||||
d.Logger.Info().Msgf("multi-part extraction queued, task id: %s, job id: %s", task.ID, jobID)
|
||||
@@ -2972,7 +2886,7 @@ func (d *Downloader) performExtraction(task *Task, archivePath string, destDir s
|
||||
task.Progress.ExtractStatus = ExtractStatusExtracting
|
||||
task.Progress.ExtractProgress = 0
|
||||
d.emit(EventKeyProgress, task)
|
||||
d.storage.Put(bucketTask, task.ID, task.clone())
|
||||
d.putTaskRecord(task, task.clone())
|
||||
|
||||
// Extract the archive
|
||||
extractErr := extractArchive(archivePath, destDir, opts.ArchivePassword, func(extractedFiles int, totalFiles int, progress int) {
|
||||
@@ -2992,7 +2906,7 @@ func (d *Downloader) performMultiPartExtraction(task *Task, firstPartPath string
|
||||
task.Progress.ExtractStatus = ExtractStatusExtracting
|
||||
task.Progress.ExtractProgress = 0
|
||||
d.emit(EventKeyProgress, task)
|
||||
d.storage.Put(bucketTask, task.ID, task.clone())
|
||||
d.putTaskRecord(task, task.clone())
|
||||
|
||||
d.Logger.Info().Msgf("starting multi-part archive extraction, first part: %s, task id: %s", firstPartPath, task.ID)
|
||||
|
||||
@@ -3136,13 +3050,13 @@ func (d *Downloader) handleExtractionResult(task *Task, extractErr error, archiv
|
||||
d.Logger.Error().Err(extractErr).Msgf("auto extract archive failed, task id: %s", task.ID)
|
||||
task.Progress.ExtractStatus = ExtractStatusError
|
||||
d.emit(EventKeyProgress, task)
|
||||
d.storage.Put(bucketTask, task.ID, task.clone())
|
||||
d.putTaskRecord(task, task.clone())
|
||||
} else {
|
||||
d.Logger.Info().Msgf("auto extract archive completed, task id: %s", task.ID)
|
||||
task.Progress.ExtractStatus = ExtractStatusDone
|
||||
task.Progress.ExtractProgress = 100
|
||||
d.emit(EventKeyProgress, task)
|
||||
d.storage.Put(bucketTask, task.ID, task.clone())
|
||||
d.putTaskRecord(task, task.clone())
|
||||
|
||||
// Delete archive files after successful extraction if enabled
|
||||
if deleteAfterExtract {
|
||||
@@ -3182,12 +3096,15 @@ func (d *Downloader) updateMultiPartTasksStatus(sourceTask *Task, extractErr err
|
||||
task.Progress.ExtractStatus = status
|
||||
task.Progress.ExtractProgress = progress
|
||||
d.emit(EventKeyProgress, task)
|
||||
d.storage.Put(bucketTask, task.ID, task.clone())
|
||||
d.putTaskRecord(task, task.clone())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func initTask(task *Task) {
|
||||
if task.lifetime == nil {
|
||||
task.lifetime = &taskLifetime{}
|
||||
}
|
||||
task.timer = util.NewTimer(task.Progress.Used)
|
||||
|
||||
task.statusLock = &sync.Mutex{}
|
||||
|
||||
@@ -3,6 +3,7 @@ package download
|
||||
import (
|
||||
"encoding/json"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/GopeedLab/gopeed/internal/controller"
|
||||
@@ -46,6 +47,14 @@ type CreateTaskBatchResult struct {
|
||||
// Task represents a download unit.
|
||||
// Options flow path:
|
||||
// REST Request (model.CreateTask/ResolveTask) -> pkg/download Downloader -> Task.Meta.Opts (base.Options) -> Fetcher.Resolve/Start (e.g. internal/protocol/http/fetcher.go)
|
||||
// taskLifetime is shared with preparation candidates. Its mutex serializes
|
||||
// physical starts and persistence with deletion; the tombstone also invalidates
|
||||
// work that is currently outside that mutex (for example Resolve).
|
||||
type taskLifetime struct {
|
||||
mu sync.Mutex
|
||||
deleted atomic.Bool
|
||||
}
|
||||
|
||||
type Task struct {
|
||||
ID string `json:"id"`
|
||||
Protocol string `json:"protocol"`
|
||||
@@ -87,6 +96,7 @@ type Task struct {
|
||||
blobRefLock *sync.Mutex
|
||||
blobURL string
|
||||
runGeneration uint64
|
||||
lifetime *taskLifetime
|
||||
speedArr []int64
|
||||
uploadSpeedArr []int64
|
||||
}
|
||||
@@ -123,6 +133,7 @@ func NewTask() *Task {
|
||||
}
|
||||
return &Task{
|
||||
ID: id,
|
||||
lifetime: &taskLifetime{},
|
||||
Status: base.DownloadStatusReady,
|
||||
CreatedAt: time.Now(),
|
||||
UpdatedAt: time.Now(),
|
||||
|
||||
@@ -1,224 +0,0 @@
|
||||
package download
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"github.com/GopeedLab/gopeed/internal/fetcher"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/GopeedLab/gopeed/pkg/base"
|
||||
)
|
||||
|
||||
func TestReleaseResolveCancelsPendingHTTPHeaders(t *testing.T) {
|
||||
entered := make(chan struct{})
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { close(entered); <-r.Context().Done() }))
|
||||
defer server.Close()
|
||||
d := NewDownloader(&DownloaderConfig{Storage: NewMemStorage(), StorageDir: t.TempDir(), TempDir: t.TempDir()})
|
||||
if err := d.Setup(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer d.Close()
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
_, err := d.ResolveWithID("pending-http", &base.Request{URL: server.URL + "/file"})
|
||||
done <- err
|
||||
}()
|
||||
select {
|
||||
case <-entered:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("request did not start")
|
||||
}
|
||||
if err := d.ReleaseResolve("pending-http"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
select {
|
||||
case err := <-done:
|
||||
if err == nil {
|
||||
t.Fatal("cancelled Resolve succeeded")
|
||||
}
|
||||
case <-time.After(300 * time.Millisecond):
|
||||
server.CloseClientConnections()
|
||||
t.Fatal("ReleaseResolve did not cancel the request waiting for headers")
|
||||
}
|
||||
if len(d.fetcherCache) != 0 || len(d.resolveStates) != 0 {
|
||||
t.Fatal("cancelled operation retained resources")
|
||||
}
|
||||
}
|
||||
func TestReleaseResolveAfterCreateDoesNotCloseTask(t *testing.T) {
|
||||
d := NewDownloader(&DownloaderConfig{Storage: NewMemStorage(), StorageDir: t.TempDir(), TempDir: t.TempDir(), FetchManagers: nil})
|
||||
if err := d.Setup(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer d.Close()
|
||||
d.cfg.MaxRunning = 0
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Write([]byte("ok")) }))
|
||||
defer server.Close()
|
||||
rr, err := d.Resolve(&base.Request{URL: server.URL})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
id, err := d.CreateWithOptions(rr.ID, &base.Options{Path: t.TempDir()})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := d.ReleaseResolve(rr.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
task := d.GetTask(id)
|
||||
if task == nil {
|
||||
t.Fatal("late release removed task")
|
||||
}
|
||||
if err := d.doStart(task); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waitForTaskStatus(t, d, id, base.DownloadStatusDone, 3*time.Second)
|
||||
}
|
||||
func TestResolveRejectsClosedDownloader(t *testing.T) {
|
||||
d := NewDownloader(&DownloaderConfig{StorageDir: t.TempDir(), TempDir: t.TempDir()})
|
||||
if err := d.Setup(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d.Close()
|
||||
if _, err := d.ResolveWithID("closed", &base.Request{URL: "http://127.0.0.1/"}); err == nil {
|
||||
t.Fatalf("closed downloader accepted Resolve: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReleaseResolveCancelsPendingMagnetMetadata(t *testing.T) {
|
||||
d := NewDownloader(&DownloaderConfig{Storage: NewMemStorage(), StorageDir: t.TempDir(), TempDir: t.TempDir()})
|
||||
if err := d.Setup(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer d.Close()
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
_, err := d.ResolveWithID("pending-magnet", &base.Request{URL: "magnet:?xt=urn:btih:0123456789abcdef0123456789abcdef01234567"})
|
||||
done <- err
|
||||
}()
|
||||
deadline := time.Now().Add(2 * time.Second)
|
||||
for {
|
||||
d.fetcherMapLock.Lock()
|
||||
state := d.resolveStates["pending-magnet"]
|
||||
ready := state != nil && state.fetcher != nil
|
||||
d.fetcherMapLock.Unlock()
|
||||
if ready {
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatal("magnet resolve did not start")
|
||||
}
|
||||
time.Sleep(time.Millisecond)
|
||||
}
|
||||
if err := d.ReleaseResolve("pending-magnet"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
select {
|
||||
case err := <-done:
|
||||
if err == nil {
|
||||
t.Fatal("cancelled magnet Resolve succeeded")
|
||||
}
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("cancelled magnet remained in metadata wait")
|
||||
}
|
||||
if len(d.fetcherCache) != 0 || len(d.resolveStates) != 0 {
|
||||
t.Fatal("cancelled magnet retained its resource")
|
||||
}
|
||||
}
|
||||
func TestCloseDrainsResolvedPreviewCache(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Write([]byte("ok")) }))
|
||||
defer server.Close()
|
||||
d := NewDownloader(&DownloaderConfig{Storage: NewMemStorage(), StorageDir: t.TempDir(), TempDir: t.TempDir()})
|
||||
if err := d.Setup(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := d.Resolve(&base.Request{URL: server.URL}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := d.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(d.fetcherCache) != 0 || len(d.resolveStates) != 0 {
|
||||
t.Fatal("shutdown retained preview resources")
|
||||
}
|
||||
}
|
||||
|
||||
type cancelledCreationFetcher struct {
|
||||
fetcher.Fetcher
|
||||
entered chan struct{}
|
||||
release chan struct{}
|
||||
err error
|
||||
}
|
||||
|
||||
func (f *cancelledCreationFetcher) Patch(*base.Request, *base.Options) error {
|
||||
close(f.entered)
|
||||
<-f.release
|
||||
return f.err
|
||||
}
|
||||
func TestReleaseResolveDuringCreationTransfersOrClosesOwnership(t *testing.T) {
|
||||
for _, fail := range []bool{false, true} {
|
||||
t.Run(fmt.Sprintf("fail=%v", fail), func(t *testing.T) {
|
||||
d := NewDownloader(&DownloaderConfig{StorageDir: t.TempDir(), TempDir: t.TempDir()})
|
||||
if err := d.Setup(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer d.Close()
|
||||
d.cfg.MaxRunning = 0
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Write([]byte("ok")) }))
|
||||
defer server.Close()
|
||||
rr, err := d.Resolve(&base.Request{URL: server.URL})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f := &cancelledCreationFetcher{Fetcher: d.fetcherCache[rr.ID], entered: make(chan struct{}), release: make(chan struct{})}
|
||||
if fail {
|
||||
f.err = errors.New("creation failed")
|
||||
}
|
||||
d.fetcherMapLock.Lock()
|
||||
d.fetcherCache[rr.ID] = f
|
||||
d.fetcherMapLock.Unlock()
|
||||
done := make(chan struct{})
|
||||
var id string
|
||||
var createErr error
|
||||
go func() {
|
||||
defer close(done)
|
||||
id, createErr = d.CreateWithOptions(rr.ID, &base.Options{Path: t.TempDir(), Size: 2})
|
||||
}()
|
||||
select {
|
||||
case <-f.entered:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("creation did not claim resource")
|
||||
}
|
||||
if err := d.ReleaseResolve(rr.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
close(f.release)
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("creation did not return")
|
||||
}
|
||||
if len(d.fetcherCache) != 0 || len(d.resolveStates) != 0 {
|
||||
t.Fatal("cancelled creation returned ownership to preview cache")
|
||||
}
|
||||
if fail {
|
||||
if createErr == nil {
|
||||
t.Fatal("expected creation failure")
|
||||
}
|
||||
return
|
||||
}
|
||||
if createErr != nil {
|
||||
t.Fatal(createErr)
|
||||
}
|
||||
// The protocol manager persists its concrete Fetcher type.
|
||||
task := d.GetTask(id)
|
||||
task.fetcher = f.Fetcher
|
||||
if err := d.doStart(task); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waitForTaskStatus(t, d, id, base.DownloadStatusDone, 3*time.Second)
|
||||
})
|
||||
}
|
||||
}
|
||||
+1
-1
@@ -24,7 +24,7 @@ func Info(w http.ResponseWriter, r *http.Request) {
|
||||
func Resolve(w http.ResponseWriter, r *http.Request) {
|
||||
var req model.ResolveTask
|
||||
if ReadJson(r, w, &req) {
|
||||
rr, err := Downloader.ResolveWithID(req.ID, req.Req)
|
||||
rr, err := Downloader.Resolve(req.Req)
|
||||
if err != nil {
|
||||
WriteJson(w, model.NewErrorResult(err.Error()))
|
||||
return
|
||||
|
||||
@@ -3,7 +3,6 @@ package model
|
||||
import "github.com/GopeedLab/gopeed/pkg/base"
|
||||
|
||||
type ResolveTask struct {
|
||||
ID string `json:"id,omitempty"`
|
||||
Req *base.Request `json:"req"`
|
||||
}
|
||||
|
||||
|
||||
@@ -81,9 +81,6 @@ Future<ResolveResult> resolve(ResolveTask resolveTask) {
|
||||
);
|
||||
}
|
||||
|
||||
Future<void> releaseResolve(String id) =>
|
||||
_parse<void>(() => _request('api/v1/resolve/${Uri.encodeComponent(id)}', method: 'DELETE'), null);
|
||||
|
||||
Future<String> createTask(CreateTask createTask) {
|
||||
return _parse<String>(() => _request('api/v1/tasks', method: 'POST', data: createTask), (data) => data as String);
|
||||
}
|
||||
|
||||
@@ -8,10 +8,7 @@ part 'resolve_task.g.dart';
|
||||
class ResolveTask {
|
||||
Request? req;
|
||||
|
||||
@JsonKey(includeIfNull: false)
|
||||
String? id;
|
||||
|
||||
ResolveTask({this.req, this.id});
|
||||
ResolveTask({this.req});
|
||||
|
||||
factory ResolveTask.fromJson(Map<String, dynamic> json) => _$ResolveTaskFromJson(json);
|
||||
Map<String, dynamic> toJson() => _$ResolveTaskToJson(this);
|
||||
|
||||
@@ -10,8 +10,7 @@ ResolveTask _$ResolveTaskFromJson(Map<String, dynamic> json) => ResolveTask(
|
||||
req: json['req'] == null
|
||||
? null
|
||||
: Request.fromJson(json['req'] as Map<String, dynamic>),
|
||||
id: json['id'] as String?,
|
||||
);
|
||||
|
||||
Map<String, dynamic> _$ResolveTaskToJson(ResolveTask instance) =>
|
||||
<String, dynamic>{'req': ?instance.req, 'id': ?instance.id};
|
||||
<String, dynamic>{'req': ?instance.req};
|
||||
@@ -93,10 +93,6 @@ void _bindGopeed(CapabilityRegistry registry, _ConfigStore configStore) {
|
||||
(p) => api.getMirrorCandidates(p['provider'] as String, p['url'] as String),
|
||||
)
|
||||
..bind(GopeedMethods.resolve, api.resolve)
|
||||
..bind(GopeedMethods.releaseResolve, (id) async {
|
||||
await api.releaseResolve(id);
|
||||
return const RpcUnit();
|
||||
})
|
||||
..bind(GopeedMethods.createTask, api.createTask)
|
||||
..bind(GopeedMethods.createTaskBatch, api.createTaskBatch)
|
||||
..bind(GopeedMethods.patchTask, (params) async {
|
||||
|
||||
@@ -16,7 +16,6 @@ abstract final class GopeedMethods {
|
||||
static const triggerProtocolJob = RpcMethod<Map<String, dynamic>, RpcUnit>('gopeed.protocol.job.trigger');
|
||||
static const getMirrorCandidates = RpcMethod<Map<String, dynamic>, List<String>>('gopeed.mirrors.candidates');
|
||||
static const resolve = RpcMethod<ResolveTask, ResolveResult>('gopeed.resolve');
|
||||
static const releaseResolve = RpcMethod<String, RpcUnit>('gopeed.releaseResolve');
|
||||
static const createTask = RpcMethod<CreateTask, String>('gopeed.task.create');
|
||||
static const createTaskBatch = RpcMethod<CreateTaskBatch, CreateTaskBatchResult>('gopeed.task.createBatch');
|
||||
static const patchTask = RpcMethod<Map<String, dynamic>, RpcUnit>('gopeed.task.patch');
|
||||
@@ -51,7 +50,6 @@ class GopeedService {
|
||||
Future<List<String>> getMirrorCandidates(String provider, String url) =>
|
||||
_invoker.invoke(GopeedMethods.getMirrorCandidates, {'provider': provider, 'url': url});
|
||||
Future<ResolveResult> resolve(ResolveTask request) => _invoker.invoke(GopeedMethods.resolve, request);
|
||||
Future<void> releaseResolve(String id) async => _invoker.invoke(GopeedMethods.releaseResolve, id);
|
||||
|
||||
Future<String> createTask(CreateTask request) => _invoker.invoke(GopeedMethods.createTask, request);
|
||||
|
||||
|
||||
@@ -1,5 +1,3 @@
|
||||
import 'dart:developer' as developer;
|
||||
import 'package:uuid/uuid.dart';
|
||||
import 'package:path/path.dart' as path;
|
||||
|
||||
import '../../../api/model/create_task_batch.dart';
|
||||
@@ -18,8 +16,6 @@ class PendingDownload {
|
||||
final Request request;
|
||||
final Options options;
|
||||
ResolveResult? result;
|
||||
String? _resolveID;
|
||||
bool _cancelled = false;
|
||||
String? resolveError;
|
||||
String? createError;
|
||||
final Set<int> selected = {};
|
||||
@@ -54,16 +50,11 @@ class PendingDownload {
|
||||
bool get hasSelection => !created && result != null && selected.isNotEmpty;
|
||||
|
||||
Future<void> resolve(GopeedService service) async {
|
||||
if (_cancelled || result != null || resolving) return;
|
||||
if (result != null || resolving) return;
|
||||
resolving = true;
|
||||
resolveError = null;
|
||||
try {
|
||||
_resolveID = const Uuid().v4();
|
||||
result = await service.resolve(ResolveTask(req: request, id: _resolveID));
|
||||
if (_cancelled) {
|
||||
await _release(service, result!.id);
|
||||
return;
|
||||
}
|
||||
result = await service.resolve(ResolveTask(req: request));
|
||||
selected.addAll(List.generate(files.isEmpty ? 1 : files.length, (index) => index));
|
||||
} catch (error) {
|
||||
resolveError = error.toString();
|
||||
@@ -72,23 +63,6 @@ class PendingDownload {
|
||||
}
|
||||
}
|
||||
|
||||
/// Stop an outstanding preview and release any resource it already owns.
|
||||
/// A response arriving after cancellation is released by resolve() as well.
|
||||
Future<void> cancel(GopeedService service) async {
|
||||
if (_cancelled || created) return;
|
||||
_cancelled = true;
|
||||
await _release(service, result?.id.isNotEmpty == true ? result!.id : _resolveID);
|
||||
}
|
||||
|
||||
Future<void> _release(GopeedService service, String? id) async {
|
||||
if (id == null || id.isEmpty) return;
|
||||
try {
|
||||
await service.releaseResolve(id);
|
||||
} catch (error, stack) {
|
||||
developer.log('Failed to release resolved resource', name: 'gopeed.resolve', error: error, stackTrace: stack);
|
||||
}
|
||||
}
|
||||
|
||||
List<CreateTaskBatchItem> creationItems() {
|
||||
final indexes = selected.toList()..sort();
|
||||
if (result!.id.isNotEmpty || files.isEmpty) {
|
||||
|
||||
@@ -1,5 +1,3 @@
|
||||
import 'dart:async';
|
||||
|
||||
import 'package:shadcn_flutter/shadcn_flutter.dart';
|
||||
import 'package:flutter/widgets.dart' as flutter show Flexible;
|
||||
|
||||
@@ -51,7 +49,6 @@ class _ResolveDownloadsDialogState extends State<_ResolveDownloadsDialog> {
|
||||
@override
|
||||
void dispose() {
|
||||
_cancelled = true;
|
||||
_releasePreviews();
|
||||
super.dispose();
|
||||
}
|
||||
|
||||
@@ -67,15 +64,8 @@ class _ResolveDownloadsDialogState extends State<_ResolveDownloadsDialog> {
|
||||
}
|
||||
}
|
||||
|
||||
void _releasePreviews() {
|
||||
for (final source in widget.sources) {
|
||||
unawaited(source.cancel(widget.service));
|
||||
}
|
||||
}
|
||||
|
||||
void _cancel() {
|
||||
_cancelled = true;
|
||||
_releasePreviews();
|
||||
closeOverlay(context, false);
|
||||
}
|
||||
|
||||
|
||||
@@ -64,81 +64,6 @@ Future<void> open(
|
||||
}
|
||||
|
||||
void main() {
|
||||
testWidgets('cancel releases a ready preview resource', (tester) async {
|
||||
final released = <String>[];
|
||||
final registry = CapabilityRegistry(createAppCapabilityCodecs())
|
||||
..bind(GopeedMethods.resolve, (_) => resolved('ready-rid'))
|
||||
..bind(const RpcMethod<String, RpcUnit>('gopeed.releaseResolve'), (id) {
|
||||
released.add(id);
|
||||
return const RpcUnit();
|
||||
});
|
||||
await open(tester, [pending('torrent')], registry);
|
||||
await tester.pumpAndSettle();
|
||||
await tester.tap(find.byKey(const ValueKey('resolve-cancel-button')));
|
||||
await tester.pumpAndSettle();
|
||||
expect(released, contains('ready-rid'));
|
||||
});
|
||||
|
||||
testWidgets('cancel releases a late preview result', (tester) async {
|
||||
final response = Completer<ResolveResult>();
|
||||
final released = <String>[];
|
||||
final registry = CapabilityRegistry(createAppCapabilityCodecs())
|
||||
..bind(GopeedMethods.resolve, (_) => response.future)
|
||||
..bind(const RpcMethod<String, RpcUnit>('gopeed.releaseResolve'), (id) {
|
||||
released.add(id);
|
||||
return const RpcUnit();
|
||||
});
|
||||
await open(tester, [pending('torrent')], registry);
|
||||
await tester.tap(find.byKey(const ValueKey('resolve-close-button')));
|
||||
await tester.pumpAndSettle();
|
||||
response.complete(resolved('late-rid'));
|
||||
await tester.pumpAndSettle();
|
||||
expect(released, contains('late-rid'));
|
||||
});
|
||||
|
||||
testWidgets('cancel sends the request ID before a preview completes', (tester) async {
|
||||
final response = Completer<ResolveResult>();
|
||||
final released = <String>[];
|
||||
String? requestedID;
|
||||
final registry = CapabilityRegistry(createAppCapabilityCodecs())
|
||||
..bind(GopeedMethods.resolve, (request) {
|
||||
requestedID = request.id;
|
||||
return response.future;
|
||||
})
|
||||
..bind(GopeedMethods.releaseResolve, (id) {
|
||||
released.add(id);
|
||||
return const RpcUnit();
|
||||
});
|
||||
await open(tester, [pending('torrent')], registry);
|
||||
await tester.tap(find.byKey(const ValueKey('resolve-close-button')));
|
||||
await tester.pumpAndSettle();
|
||||
expect(requestedID, isNotEmpty);
|
||||
expect(released, contains(requestedID));
|
||||
response.complete(
|
||||
ResolveResult(
|
||||
id: requestedID!,
|
||||
res: Resource(files: [FileInfo(name: 'late.txt')]),
|
||||
),
|
||||
);
|
||||
await tester.pumpAndSettle();
|
||||
expect(released.where((id) => id == requestedID), hasLength(2));
|
||||
});
|
||||
|
||||
testWidgets('successful creation keeps task-owned resources', (tester) async {
|
||||
final released = <String>[];
|
||||
final registry = CapabilityRegistry(createAppCapabilityCodecs())
|
||||
..bind(GopeedMethods.resolve, (_) => resolved('task-owned'))
|
||||
..bind(GopeedMethods.createTaskBatch, (_) => CreateTaskBatchResult(taskIds: ['task']))
|
||||
..bind(GopeedMethods.releaseResolve, (id) {
|
||||
released.add(id);
|
||||
return const RpcUnit();
|
||||
});
|
||||
await open(tester, [pending('torrent')], registry);
|
||||
await tester.pumpAndSettle();
|
||||
await tester.tap(find.byKey(const ValueKey('resolve-create-button')));
|
||||
await tester.pumpAndSettle();
|
||||
expect(released, isEmpty);
|
||||
});
|
||||
for (final width in [390.0, 700.0, 1024.0]) {
|
||||
testWidgets('file list fits and exposes ready selections at width $width', (tester) async {
|
||||
final registry = CapabilityRegistry(createAppCapabilityCodecs())
|
||||
|
||||
Reference in new issue
Block a user