refactor(downloader): remove opts parameter from Resolve (#1595)

This commit is contained in:
Levi authored and GitHub committed 2026-10-19 21:40:57 +08:00
1 parent 5462b14705
commit de65149b29
45 files changed
+1589 -384

No files matched your search

+41 -29
View File
@@ -244,16 +244,25 @@ func (d *Downloader) Setup() error {
tasks := append([]*Task(nil), d.tasks...)
d.lock.Unlock()
for _, task := range tasks {
if task.Status == base.DownloadStatusDone && task.Uploading {
func() {
task.lock.Lock()
defer task.lock.Unlock()
if d.closed.Load() {
return
}
if d.taskStatus(task) != base.DownloadStatusDone || !task.Uploading {
return
}
if err := d.restoreTask(task); err != nil {
d.Logger.Error().Stack().Err(err).Msgf("task upload restore fetcher failed, task id: %s", task.ID)
d.Logger.Error().Err(err).Msgf("task upload restore fetcher failed, task id: %s", task.ID)
return
}
if uploader, ok := task.fetcher.(fetcher.Uploader); ok {
if err := uploader.Upload(); err != nil {
d.Logger.Error().Stack().Err(err).Msgf("task upload failed, task id: %s", task.ID)
d.Logger.Error().Err(err).Msgf("task upload failed, task id: %s", task.ID)
}
}
}
}()
}
}()
@@ -467,7 +476,10 @@ func (d *Downloader) saveTask(task *Task) error {
return err
}
}
if err := d.storage.Put(bucketTask, task.ID, task); err != nil {
task.statusLock.Lock()
snapshot := task.clone()
task.statusLock.Unlock()
if err := d.storage.Put(bucketTask, task.ID, snapshot); err != nil {
d.Logger.Error().Stack().Err(err).Msgf("persist task failed: %s", task.ID)
return err
}
@@ -498,7 +510,7 @@ func ensureResourceRequestRawURLs(parentReq *base.Request, res *base.Resource) {
}
}
func (d *Downloader) Resolve(req *base.Request, opts *base.Options) (rr *ResolveResult, err error) {
func (d *Downloader) Resolve(req *base.Request) (rr *ResolveResult, err error) {
rrId, err := gonanoid.New()
if err != nil {
return
@@ -520,11 +532,7 @@ func (d *Downloader) Resolve(req *base.Request, opts *base.Options) (rr *Resolve
if err != nil {
return
}
initOpt, err := d.initOptions(opts)
if err != nil {
return
}
err = fetcher.Resolve(req, initOpt)
err = fetcher.Resolve(req)
if err != nil {
return
}
@@ -558,7 +566,7 @@ func (d *Downloader) notifyRunning() {
func (d *Downloader) remainRunningCount() int {
runningCount := 0
for _, t := range d.tasks {
if t.Status == base.DownloadStatusRunning {
if d.taskStatus(t) == base.DownloadStatusRunning {
runningCount++
}
}
@@ -621,7 +629,7 @@ func (d *Downloader) Create(rrId string) (taskId string, err error) {
}
// CreateWithOptions creates a resolved task using the submitted options, or the
// resolve-time options when opts is nil.
// default download options when opts is nil.
func (d *Downloader) CreateWithOptions(rrId string, opts *base.Options) (taskId string, err error) {
d.fetcherMapLock.RLock()
fetcher, ok := d.fetcherCache[rrId]
@@ -629,21 +637,22 @@ func (d *Downloader) CreateWithOptions(rrId string, opts *base.Options) (taskId
if !ok {
return "", errors.New("invalid resource id")
}
if opts != nil {
opts, err = d.initOptions(opts.Clone())
if err != nil {
return "", err
}
if res := fetcher.Meta().Res; res != nil {
for _, index := range opts.SelectFiles {
if index < 0 || index >= len(res.Files) {
return "", fmt.Errorf("invalid selected file index: %d", index)
}
}
opts.InitSelectFiles(len(res.Files))
}
fetcher.Meta().Opts = opts
if opts == nil {
opts = &base.Options{}
}
opts, err = d.initOptions(opts.Clone())
if err != nil {
return "", err
}
if res := fetcher.Meta().Res; res != nil {
for _, index := range opts.SelectFiles {
if index < 0 || index >= len(res.Files) {
return "", fmt.Errorf("invalid selected file index: %d", index)
}
}
opts.InitSelectFiles(len(res.Files))
}
fetcher.Meta().Opts = opts
if res := fetcher.Meta().Res; res != nil {
res.CalcSize(fetcher.Meta().Opts.SelectFiles)
}
@@ -1469,7 +1478,10 @@ func (d *Downloader) watch(task *Task) {
if err := d.captureTaskStats(task, false, true); err != nil {
d.Logger.Error().Stack().Err(err).Msgf("persist completed task stats failed, task id: %s", task.ID)
}
d.storage.Put(bucketTask, task.ID, task.clone())
task.statusLock.Lock()
snapshot := task.clone()
task.statusLock.Unlock()
d.storage.Put(bucketTask, task.ID, snapshot)
d.emit(EventKeyDone, task)
d.emit(EventKeyFinally, task, err)
d.notifyRunning()
@@ -1826,7 +1838,7 @@ func (d *Downloader) doStart(task *Task) (err error) {
return err
}
if task.Meta.Res == nil {
err := task.fetcher.Resolve(task.Meta.Req, task.Meta.Opts)
err := task.fetcher.Resolve(task.Meta.Req)
if err != nil {
return err
}
+23 -23
View File
@@ -132,7 +132,7 @@ func (f *generationTestFetcher) Setup(*controller.Controller) {
f.done = make(chan error, 2)
}
}
func (f *generationTestFetcher) Resolve(req *base.Request, opts *base.Options) error {
func (f *generationTestFetcher) Resolve(req *base.Request) error {
if f.manager.resolveStarted != nil {
f.manager.resolveOnce.Do(func() { close(f.manager.resolveStarted) })
}
@@ -143,7 +143,7 @@ func (f *generationTestFetcher) Resolve(req *base.Request, opts *base.Options) e
return f.manager.resolveErr
}
f.meta.Req = req
f.meta.Opts = opts
f.meta.Res = &base.Resource{Files: []*base.FileInfo{{Name: "generation.bin", Size: 1}}}
return nil
}
@@ -245,7 +245,7 @@ func TestDownloader_Resolve(t *testing.T) {
req := &base.Request{
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
}
rr, err := downloader.Resolve(req, nil)
rr, err := downloader.Resolve(req)
if err != nil {
t.Fatal(err)
}
@@ -772,7 +772,7 @@ func TestDownloader_Create(t *testing.T) {
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
}
opts := newTestDownloadOpt(t)
rr, err := downloader.Resolve(req, opts)
rr, err := downloader.Resolve(req)
if err != nil {
t.Fatal(err)
}
@@ -784,7 +784,7 @@ func TestDownloader_Create(t *testing.T) {
wg.Done()
}
})
_, err = downloader.Create(rr.ID)
_, err = downloader.CreateWithOptions(rr.ID, opts)
if err != nil {
t.Fatal(err)
}
@@ -878,15 +878,15 @@ func TestDownloader_CreateNotInWhite(t *testing.T) {
req := &base.Request{
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
}
// With new fetcher design, white list check happens during Resolve (not Create)
// because Resolve now requires Options which includes the download path
_, err := downloader.Resolve(req, newTestDownloadOpt(t))
rr, err := downloader.Resolve(req)
if err != nil {
t.Fatal(err)
}
_, err = downloader.CreateWithOptions(rr.ID, newTestDownloadOpt(t))
if err == nil {
t.Error("TestDownloader_CreateNotInWhite() expected error but got nil")
}
if !strings.Contains(err.Error(), "white") {
t.Errorf("TestDownloader_CreateNotInWhite() got = %v, want error containing 'white'", err.Error())
t.Fatal("Create should reject a directory outside the allowlist")
}
}
func TestDownloader_CreateDirectBatch(t *testing.T) {
@@ -1072,7 +1072,7 @@ func doTestDownloaderCreateWithProxy(t *testing.T, auth bool, buildReqProxy func
Pwd: pwd,
})
}
rr, err := downloader.Resolve(req, nil)
rr, err := downloader.Resolve(req)
if err != nil {
if errHandler == nil {
t.Fatal(err)
@@ -1158,7 +1158,7 @@ func TestDownloader_StoreAndRestore(t *testing.T) {
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
}
opts := newTestDownloadOpt(t)
rr, err := downloader.Resolve(req, opts)
rr, err := downloader.Resolve(req)
if err != nil {
t.Fatal(err)
}
@@ -1172,7 +1172,7 @@ func TestDownloader_StoreAndRestore(t *testing.T) {
})
}
})
id, err := downloader.Create(rr.ID)
id, err := downloader.CreateWithOptions(rr.ID, opts)
if err != nil {
t.Fatal(err)
}
@@ -1434,7 +1434,7 @@ func TestDownloader_Stats(t *testing.T) {
req := &base.Request{
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
}
rr, err := downloader.Resolve(req, newTestDownloadOpt(t))
rr, err := downloader.Resolve(req)
if err != nil {
t.Fatal(err)
}
@@ -1447,7 +1447,7 @@ func TestDownloader_Stats(t *testing.T) {
}
})
taskId, err := downloader.Create(rr.ID)
taskId, err := downloader.CreateWithOptions(rr.ID, newTestDownloadOpt(t))
if err != nil {
t.Fatal(err)
}
@@ -1677,7 +1677,7 @@ func TestDownloader_CompletedHTTPStatsSurviveRestore(t *testing.T) {
t.Fatal(err)
}
req := &base.Request{URL: "http://" + listener.Addr().String() + "/" + test.BuildName}
rr, err := downloader.Resolve(req, newTestDownloadOpt(t))
rr, err := downloader.Resolve(req)
if err != nil {
t.Fatal(err)
}
@@ -1688,7 +1688,7 @@ func TestDownloader_CompletedHTTPStatsSurviveRestore(t *testing.T) {
done <- struct{}{}
}
})
taskID, err := downloader.Create(rr.ID)
taskID, err := downloader.CreateWithOptions(rr.ID, newTestDownloadOpt(t))
if err != nil {
t.Fatal(err)
}
@@ -1861,11 +1861,11 @@ func TestDownloader_PauseAndContinue(t *testing.T) {
req := &base.Request{
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
}
rr, err := downloader.Resolve(req, newTestDownloadOpt(t))
rr, err := downloader.Resolve(req)
if err != nil {
t.Fatal(err)
}
taskId, err := downloader.Create(rr.ID)
taskId, err := downloader.CreateWithOptions(rr.ID, newTestDownloadOpt(t))
if err != nil {
t.Fatal(err)
}
@@ -2045,11 +2045,11 @@ func TestDownloader_PauseAllAndContinueAll(t *testing.T) {
req := &base.Request{
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
}
rr, err := downloader.Resolve(req, newTestDownloadOpt(t))
rr, err := downloader.Resolve(req)
if err != nil {
t.Fatal(err)
}
taskId, err := downloader.Create(rr.ID)
taskId, err := downloader.CreateWithOptions(rr.ID, newTestDownloadOpt(t))
if err != nil {
t.Fatal(err)
}
+4
View File
@@ -753,6 +753,10 @@ type OnErrorExtensionTask struct {
}
func cloneExtensionTask(task *Task) *Task {
if task.statusLock != nil {
task.statusLock.Lock()
defer task.statusLock.Unlock()
}
newTask := task.clone()
newTask.Meta.Req = task.Meta.Req
return newTask
+2 -2
View File
@@ -65,11 +65,11 @@ func TestExtensionExtraMutationHTTP(t *testing.T) {
var id string
var err error
if resolved {
rr, resolveErr := d.Resolve(req, opts)
rr, resolveErr := d.Resolve(req)
if resolveErr != nil {
t.Fatal(resolveErr)
}
id, err = d.Create(rr.ID)
id, err = d.CreateWithOptions(rr.ID, opts)
} else {
id, err = d.CreateDirect(req, opts)
}
+24 -24
View File
@@ -32,7 +32,7 @@ func TestDownloader_InstallExtensionByFolder(t *testing.T) {
URL: "https://github.com/test",
Labels: map[string]string{"original": "true"},
}
rr, err := downloader.Resolve(req, nil)
rr, err := downloader.Resolve(req)
if err != nil {
t.Fatal(err)
}
@@ -58,7 +58,7 @@ func TestDownloader_InstallExtensionByFolderDevMode(t *testing.T) {
}
rr, err := downloader.Resolve(&base.Request{
URL: "https://github.com/test",
}, nil)
})
if err != nil {
t.Fatal(err)
}
@@ -76,7 +76,7 @@ func TestDownloader_Extension_BlobBlob(t *testing.T) {
rr, err := downloader.Resolve(&base.Request{
URL: "https://example.com/blob",
}, nil)
})
if err != nil {
t.Fatal(err)
}
@@ -130,7 +130,7 @@ func TestDownloader_Extension_BlobFunctionOpener(t *testing.T) {
rr, err := downloader.Resolve(&base.Request{
URL: "https://example.com/opener",
}, nil)
})
if err != nil {
t.Fatal(err)
}
@@ -166,7 +166,7 @@ func TestDownloader_Extension_BlobFunctionOpenerUnknownSize(t *testing.T) {
rr, err := downloader.Resolve(&base.Request{
URL: "https://example.com/opener-unknown",
}, nil)
})
if err != nil {
t.Fatal(err)
}
@@ -222,7 +222,7 @@ func TestDownloader_Extension_BlobSourceSizePropagatesToCreatedTask(t *testing.T
rr, err := downloader.Resolve(&base.Request{
URL: "https://example.com/opener",
}, nil)
})
if err != nil {
t.Fatal(err)
}
@@ -303,7 +303,7 @@ func TestDownloader_Extension_BlobFetchDrainedStreamReportsEarlyProgress(t *test
rr, err := downloader.Resolve(&base.Request{
URL: fmt.Sprintf("%s/single-fetch-stream?size=%d", server.URL, len(payload)),
}, nil)
})
if err != nil {
t.Fatal(err)
}
@@ -419,7 +419,7 @@ func TestDownloader_Extension_BlobRangeFetchDrainedStreamReportsEarlyProgress(t
rr, err := downloader.Resolve(&base.Request{
URL: fmt.Sprintf("%s/range-fetch-stream?size=%d", server.URL, len(payload)),
}, nil)
})
if err != nil {
t.Fatal(err)
}
@@ -494,7 +494,7 @@ func TestDownloader_Extension_BlobMultiplexFetchStreamsReportEarlyProgress(t *te
rr, err := downloader.Resolve(&base.Request{
URL: server.URL + "/multiplex-fetch-stream",
}, nil)
})
if err != nil {
t.Fatal(err)
}
@@ -550,7 +550,7 @@ func TestDownloader_Extension_BlobFunctionOpenerRangeResume(t *testing.T) {
rr, err := downloader.Resolve(&base.Request{
URL: "https://example.com/opener-range",
}, nil)
})
if err != nil {
t.Fatal(err)
}
@@ -683,7 +683,7 @@ func TestDownloader_Extension_BlobResumeAfterRestartViaOnError(t *testing.T) {
rr, err := downloader.Resolve(&base.Request{
URL: server.URL + "/restart-range",
}, nil)
})
if err != nil {
t.Fatal(err)
}
@@ -855,7 +855,7 @@ func TestDownloader_Extension_BlobPauseContinueKeepsSource(t *testing.T) {
rr, err := downloader.Resolve(&base.Request{
URL: tc.url,
}, nil)
})
if err != nil {
t.Fatal(err)
}
@@ -932,7 +932,7 @@ func TestDownloader_Extension_BlobUnknownSizePauseContinueKeepsSource(t *testing
rr, err := downloader.Resolve(&base.Request{
URL: "https://example.com/opener-unknown",
}, nil)
})
if err != nil {
t.Fatal(err)
}
@@ -1009,7 +1009,7 @@ func TestDownloader_Extension_BlobRecoverOnError(t *testing.T) {
path := path
t.Run(path, func(t *testing.T) {
rawURL := "https://example.com/" + path
rr, err := downloader.Resolve(&base.Request{URL: rawURL}, nil)
rr, err := downloader.Resolve(&base.Request{URL: rawURL})
if err != nil {
t.Fatal(err)
}
@@ -1076,7 +1076,7 @@ func TestDownloader_InstallExtensionByGit(t *testing.T) {
}
rr, err := downloader.Resolve(&base.Request{
URL: "https://github.com/GopeedLab/gopeed/releases",
}, nil)
})
if err != nil {
t.Fatal(err)
}
@@ -1093,7 +1093,7 @@ func TestDownloader_InstallExtensionByGitSimple(t *testing.T) {
}
rr, err := downloader.Resolve(&base.Request{
URL: "https://github.com/GopeedLab/gopeed/releases",
}, nil)
})
if err != nil {
t.Fatal(err)
}
@@ -1110,7 +1110,7 @@ func TestDownloader_InstallExtensionByGitFull(t *testing.T) {
}
rr, err := downloader.Resolve(&base.Request{
URL: "https://github.com/GopeedLab/gopeed/releases",
}, nil)
})
if err != nil {
t.Fatal(err)
}
@@ -1180,7 +1180,7 @@ func TestDownloader_UpgradeExtension(t *testing.T) {
rr, err := downloader.Resolve(&base.Request{
URL: "https://test.com",
}, nil)
})
if err != nil {
t.Fatal(err)
}
@@ -1355,7 +1355,7 @@ func TestDownloader_Extension_Errors(t *testing.T) {
}
rr, err := downloader.Resolve(&base.Request{
URL: "https://github.com/test",
}, nil)
})
if err != nil {
t.Fatal(err)
}
@@ -1370,7 +1370,7 @@ func TestDownloader_Extension_Errors(t *testing.T) {
}
rr, err := downloader.Resolve(&base.Request{
URL: "https://github.com/test",
}, nil)
})
if err != nil {
t.Fatal(err)
}
@@ -1385,7 +1385,7 @@ func TestDownloader_Extension_Errors(t *testing.T) {
}
_, err := downloader.Resolve(&base.Request{
URL: "https://github.com/test",
}, nil)
})
if err == nil {
t.Fatalf("except error, but got nil")
}
@@ -1407,7 +1407,7 @@ func TestDownloader_Extension_Settings(t *testing.T) {
}
rr, err := downloader.Resolve(&base.Request{
URL: "https://github.com/test",
}, nil)
})
if err != nil {
t.Fatal(err)
}
@@ -1428,7 +1428,7 @@ func TestDownloader_Extension_Settings(t *testing.T) {
})
rr, err := downloader.Resolve(&base.Request{
URL: "https://github.com/test",
}, nil)
})
if err != nil {
t.Fatal(err)
}
@@ -1445,7 +1445,7 @@ func TestDownloader_ExtensionStorage(t *testing.T) {
}
rr, err := downloader.Resolve(&base.Request{
URL: "https://github.com/test",
}, nil)
})
if err != nil {
t.Fatal(err)
}
@@ -24,7 +24,7 @@ func TestExtensionUnnamedResource(t *testing.T) {
if _, err = d.InstallExtensionByFolder(dir, true); err != nil {
t.Fatal(err)
}
res, err := d.Resolve(&base.Request{URL: "https://example.com/unnamed"}, nil)
res, err := d.Resolve(&base.Request{URL: "https://example.com/unnamed"})
if err != nil {
t.Fatal(err)
}
+1 -1
View File
@@ -40,7 +40,7 @@ func TestDownloader_HlsLiveSite(t *testing.T) {
if referer != "" {
req.Extra = map[string]any{"header": map[string]string{"Referer": referer}}
}
rr, err := downloader.Resolve(req, nil)
rr, err := downloader.Resolve(req)
if err != nil {
t.Fatalf("resolve: %v", err)
}
+1 -1
View File
@@ -89,7 +89,7 @@ func TestDownloader_HlsProtocolEndToEnd(t *testing.T) {
req := hlsResolveReferer()
req.URL = server.URL + "/master.m3u8"
rr, err := downloader.Resolve(req, nil)
rr, err := downloader.Resolve(req)
if err != nil {
t.Fatal(err)
}
+4 -4
View File
@@ -18,7 +18,7 @@ func TestCreateResolvedFileSelection(t *testing.T) {
}{
{"partial", []int{2, 0}, true, []int{2, 0}, 40},
{"default all", nil, true, []int{0, 1, 2}, 60},
{"legacy retains resolve options", nil, false, []int{1}, 20},
{"default creation options", nil, false, []int{0, 1, 2}, 60},
} {
t.Run(tc.name, func(t *testing.T) {
manager := &generationTestManager{}
@@ -28,8 +28,8 @@ func TestCreateResolvedFileSelection(t *testing.T) {
}
defer d.Clear()
d.cfg.MaxRunning = 0
original := &base.Options{Path: t.TempDir(), SelectFiles: []int{1}}
rr, err := d.Resolve(&base.Request{URL: "generation://selection"}, original)
original := &base.Options{Path: t.TempDir()}
rr, err := d.Resolve(&base.Request{URL: "generation://selection"})
if err != nil {
t.Fatal(err)
}
@@ -40,7 +40,7 @@ func TestCreateResolvedFileSelection(t *testing.T) {
if tc.submit {
id, err = d.CreateWithOptions(rr.ID, &base.Options{Path: original.Path, SelectFiles: tc.selection})
} else {
id, err = d.Create(rr.ID)
id, err = d.CreateWithOptions(rr.ID, original)
}
if err != nil {
t.Fatal(err)