feat(tasks): group extension folder downloads (#1594)

This commit is contained in:
WuFeng authored and GitHub committed 2026-11-02 15:39:08 +08:00
1 parent fc0a1e0ef9
commit 1e36599d52
84 files changed
+6690 -881

No files matched your search

+7
View File
@@ -32,6 +32,10 @@ func categoryForResource(cfg *base.DownloaderStoreConfig, res *base.Resource) *b
// runs on the result, so a category directory the white list rejects falls
// back to the base directory of this task instead of failing it.
func (d *Downloader) decideDownloadDir(opts *base.Options, res *base.Resource) (string, error) {
return d.decideDownloadDirForTask(opts, res, nil)
}
func (d *Downloader) decideDownloadDirForTask(opts *base.Options, res *base.Resource, group *TaskGroup) (string, error) {
if opts == nil {
opts = &base.Options{}
}
@@ -48,6 +52,9 @@ func (d *Downloader) decideDownloadDir(opts *base.Options, res *base.Resource) (
if base == "" {
return "", errors.New("download directory is empty")
}
if group != nil && opts.SkipAutoCategorize {
return d.initDownloadPathForTask(base, group)
}
target := base
if !opts.SkipAutoCategorize {
if category := categoryForResource(cfg, res); category != nil {
+32
View File
@@ -234,6 +234,38 @@ func TestDecideDownloadDirWhiteList(t *testing.T) {
})
}
func TestDecideGroupDownloadDirWhiteList(t *testing.T) {
parent := t.TempDir()
root := filepath.Join(parent, "folder")
nested := filepath.Join(root, "nested")
for _, test := range []struct {
name string
path string
group *TaskGroup
allowed string
wantErr bool
}{
{name: "group root", path: root, group: &TaskGroup{Path: root}, allowed: filepath.Join(parent, "*")},
{name: "nested member", path: nested, group: &TaskGroup{Path: root}, allowed: filepath.Join(parent, "*")},
{name: "ordinary nested task", path: nested, allowed: filepath.Join(parent, "*"), wantErr: true},
{name: "member outside group", path: filepath.Join(parent, "other"), group: &TaskGroup{Path: root}, allowed: filepath.Join(parent, "*"), wantErr: true},
{name: "group permission revoked", path: nested, group: &TaskGroup{Path: root}, allowed: nested, wantErr: true},
{name: "literal group name", path: filepath.Join(parent, "%year%", "nested"), group: &TaskGroup{Path: filepath.Join(parent, "%year%")}, allowed: filepath.Join(parent, "*")},
} {
t.Run(test.name, func(t *testing.T) {
d := newUnitCategoryDownloader(newTestCategoryConfig())
d.cfg.WhiteDownloadDirs = []string{test.allowed}
got, err := d.decideDownloadDirForTask(&base.Options{Path: test.path, SkipAutoCategorize: true}, singleFileResource("file.txt"), test.group)
if (err != nil) != test.wantErr {
t.Fatalf("decideDownloadDirForTask() = %q, %v; want error = %v", got, err, test.wantErr)
}
if err == nil && got != test.path {
t.Fatalf("path = %q, want %q", got, test.path)
}
})
}
}
func TestSuggestedDownloadDir(t *testing.T) {
programDir := filepath.Join("./downloads", "Program")
+490 -122
View File
File diff suppressed because it is too large. Load diff
File diff suppressed because it is too large. Load diff
+2 -1
View File
@@ -70,10 +70,11 @@ func TestExtensionResolveSizeThroughTaskCreation(t *testing.T) {
if err := json.Unmarshal(wire, &submitted); err != nil {
t.Fatal(err)
}
ids, err := d.CreateDirectBatch(&submitted)
batchResult, err := d.CreateDirectBatch(&submitted)
if err != nil {
t.Fatal(err)
}
ids := batchResult
for i, id := range ids {
task := d.GetTask(id)
if task.Meta.Res != nil {
+9 -7
View File
@@ -2044,8 +2044,13 @@ func waitForTaskStatus(t *testing.T, downloader *Downloader, id string, status b
deadline := time.Now().Add(timeout)
for time.Now().Before(deadline) {
task := downloader.GetTask(id)
if task != nil && task.Status == status {
return
if task != nil && downloader.taskStatus(task) == status {
if status != base.DownloadStatusDone {
return
}
if _, watching := downloader.watchedTasks.Load(id); !watching {
return
}
}
time.Sleep(10 * time.Millisecond)
}
@@ -2053,9 +2058,6 @@ func waitForTaskStatus(t *testing.T, downloader *Downloader, id string, status b
if task == nil {
t.Fatalf("timeout waiting for task %s status %s: task not found", id, status)
}
total := int64(0)
if task.Meta != nil && task.Meta.Res != nil {
total = task.Meta.Res.Size
}
t.Fatalf("timeout waiting for task %s status %s: got %s downloaded=%d total=%d", id, status, task.Status, task.Progress.Downloaded, total)
snapshot, err := downloader.RuntimeStatus(id)
t.Fatalf("timeout waiting for task %s status %s: got %+v, error=%v", id, status, snapshot, err)
}
+29 -9
View File
@@ -29,18 +29,34 @@ type ResolveResult struct {
SuggestedDownloadDir string `json:"suggestedDownloadDir,omitempty"`
}
// TaskGroup stores the directory shared by independently scheduled tasks.
type TaskGroup struct {
ID string `json:"id"`
Name string `json:"name"`
Path string `json:"path"`
CreatedAt time.Time `json:"createdAt"`
}
// CreateTaskBatchResult is shared by ordinary and grouped batches.
type CreateTaskBatchResult struct {
TaskIDs []string `json:"taskIds"`
ParentID string `json:"parentId,omitempty"`
}
// Task represents a download unit.
// Options flow path:
// REST Request (model.CreateTask/ResolveTask) -> pkg/download Downloader -> Task.Meta.Opts (base.Options) -> Fetcher.Resolve/Start (e.g. internal/protocol/http/fetcher.go)
type Task struct {
ID string `json:"id"`
Protocol string `json:"protocol"`
Meta *fetcher.FetcherMeta `json:"meta"`
Status base.Status `json:"status"`
Uploading bool `json:"uploading"`
Progress *Progress `json:"progress"`
CreatedAt time.Time `json:"createdAt"`
UpdatedAt time.Time `json:"updatedAt"`
ID string `json:"id"`
Protocol string `json:"protocol"`
Meta *fetcher.FetcherMeta `json:"meta"`
// ParentID identifies the independently stored group.
ParentID string `json:"parentId,omitempty"`
Status base.Status `json:"status"`
Uploading bool `json:"uploading"`
Progress *Progress `json:"progress"`
CreatedAt time.Time `json:"createdAt"`
UpdatedAt time.Time `json:"updatedAt"`
// DoneAt is the time when the task completed; nil if it has never
// finished. A done task never leaves the done state, so the value is
// written once and kept.
@@ -54,6 +70,8 @@ type Task struct {
// does not have it and keeps its directory and name.
NeedsPreparation bool `json:"needsPreparation,omitempty"`
group *TaskGroup
producer production.Source
producerReceived int64
@@ -144,10 +162,12 @@ func (t *Task) MarshalJSON() ([]byte, error) {
type rawTaskType Task
jsonTask := struct {
*rawTaskType
Name string `json:"name"`
Name string `json:"name"`
Group *TaskGroup `json:"group,omitempty"`
}{
(*rawTaskType)(t),
t.Name(),
t.group,
}
return json.Marshal(jsonTask)
}