mirror of
https://github.com/GopeedLab/gopeed.git
synced 2026-11-17 05:43:41 +00:00
feat(tasks): group extension folder downloads (#1594)
This commit is contained in:
1 parent
fc0a1e0ef9
commit
1e36599d52
84 files changed
+6690
-881
No files matched your search
@@ -32,6 +32,10 @@ func categoryForResource(cfg *base.DownloaderStoreConfig, res *base.Resource) *b
|
||||
// runs on the result, so a category directory the white list rejects falls
|
||||
// back to the base directory of this task instead of failing it.
|
||||
func (d *Downloader) decideDownloadDir(opts *base.Options, res *base.Resource) (string, error) {
|
||||
return d.decideDownloadDirForTask(opts, res, nil)
|
||||
}
|
||||
|
||||
func (d *Downloader) decideDownloadDirForTask(opts *base.Options, res *base.Resource, group *TaskGroup) (string, error) {
|
||||
if opts == nil {
|
||||
opts = &base.Options{}
|
||||
}
|
||||
@@ -48,6 +52,9 @@ func (d *Downloader) decideDownloadDir(opts *base.Options, res *base.Resource) (
|
||||
if base == "" {
|
||||
return "", errors.New("download directory is empty")
|
||||
}
|
||||
if group != nil && opts.SkipAutoCategorize {
|
||||
return d.initDownloadPathForTask(base, group)
|
||||
}
|
||||
target := base
|
||||
if !opts.SkipAutoCategorize {
|
||||
if category := categoryForResource(cfg, res); category != nil {
|
||||
|
||||
@@ -234,6 +234,38 @@ func TestDecideDownloadDirWhiteList(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
func TestDecideGroupDownloadDirWhiteList(t *testing.T) {
|
||||
parent := t.TempDir()
|
||||
root := filepath.Join(parent, "folder")
|
||||
nested := filepath.Join(root, "nested")
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
path string
|
||||
group *TaskGroup
|
||||
allowed string
|
||||
wantErr bool
|
||||
}{
|
||||
{name: "group root", path: root, group: &TaskGroup{Path: root}, allowed: filepath.Join(parent, "*")},
|
||||
{name: "nested member", path: nested, group: &TaskGroup{Path: root}, allowed: filepath.Join(parent, "*")},
|
||||
{name: "ordinary nested task", path: nested, allowed: filepath.Join(parent, "*"), wantErr: true},
|
||||
{name: "member outside group", path: filepath.Join(parent, "other"), group: &TaskGroup{Path: root}, allowed: filepath.Join(parent, "*"), wantErr: true},
|
||||
{name: "group permission revoked", path: nested, group: &TaskGroup{Path: root}, allowed: nested, wantErr: true},
|
||||
{name: "literal group name", path: filepath.Join(parent, "%year%", "nested"), group: &TaskGroup{Path: filepath.Join(parent, "%year%")}, allowed: filepath.Join(parent, "*")},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
d := newUnitCategoryDownloader(newTestCategoryConfig())
|
||||
d.cfg.WhiteDownloadDirs = []string{test.allowed}
|
||||
got, err := d.decideDownloadDirForTask(&base.Options{Path: test.path, SkipAutoCategorize: true}, singleFileResource("file.txt"), test.group)
|
||||
if (err != nil) != test.wantErr {
|
||||
t.Fatalf("decideDownloadDirForTask() = %q, %v; want error = %v", got, err, test.wantErr)
|
||||
}
|
||||
if err == nil && got != test.path {
|
||||
t.Fatalf("path = %q, want %q", got, test.path)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSuggestedDownloadDir(t *testing.T) {
|
||||
programDir := filepath.Join("./downloads", "Program")
|
||||
|
||||
|
||||
+490
-122
File diff suppressed because it is too large.
Load diff
+1099
-1
File diff suppressed because it is too large.
Load diff
@@ -70,10 +70,11 @@ func TestExtensionResolveSizeThroughTaskCreation(t *testing.T) {
|
||||
if err := json.Unmarshal(wire, &submitted); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ids, err := d.CreateDirectBatch(&submitted)
|
||||
batchResult, err := d.CreateDirectBatch(&submitted)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ids := batchResult
|
||||
for i, id := range ids {
|
||||
task := d.GetTask(id)
|
||||
if task.Meta.Res != nil {
|
||||
|
||||
@@ -2044,8 +2044,13 @@ func waitForTaskStatus(t *testing.T, downloader *Downloader, id string, status b
|
||||
deadline := time.Now().Add(timeout)
|
||||
for time.Now().Before(deadline) {
|
||||
task := downloader.GetTask(id)
|
||||
if task != nil && task.Status == status {
|
||||
return
|
||||
if task != nil && downloader.taskStatus(task) == status {
|
||||
if status != base.DownloadStatusDone {
|
||||
return
|
||||
}
|
||||
if _, watching := downloader.watchedTasks.Load(id); !watching {
|
||||
return
|
||||
}
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
@@ -2053,9 +2058,6 @@ func waitForTaskStatus(t *testing.T, downloader *Downloader, id string, status b
|
||||
if task == nil {
|
||||
t.Fatalf("timeout waiting for task %s status %s: task not found", id, status)
|
||||
}
|
||||
total := int64(0)
|
||||
if task.Meta != nil && task.Meta.Res != nil {
|
||||
total = task.Meta.Res.Size
|
||||
}
|
||||
t.Fatalf("timeout waiting for task %s status %s: got %s downloaded=%d total=%d", id, status, task.Status, task.Progress.Downloaded, total)
|
||||
snapshot, err := downloader.RuntimeStatus(id)
|
||||
t.Fatalf("timeout waiting for task %s status %s: got %+v, error=%v", id, status, snapshot, err)
|
||||
}
|
||||
+29
-9
@@ -29,18 +29,34 @@ type ResolveResult struct {
|
||||
SuggestedDownloadDir string `json:"suggestedDownloadDir,omitempty"`
|
||||
}
|
||||
|
||||
// TaskGroup stores the directory shared by independently scheduled tasks.
|
||||
type TaskGroup struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Path string `json:"path"`
|
||||
CreatedAt time.Time `json:"createdAt"`
|
||||
}
|
||||
|
||||
// CreateTaskBatchResult is shared by ordinary and grouped batches.
|
||||
type CreateTaskBatchResult struct {
|
||||
TaskIDs []string `json:"taskIds"`
|
||||
ParentID string `json:"parentId,omitempty"`
|
||||
}
|
||||
|
||||
// Task represents a download unit.
|
||||
// Options flow path:
|
||||
// REST Request (model.CreateTask/ResolveTask) -> pkg/download Downloader -> Task.Meta.Opts (base.Options) -> Fetcher.Resolve/Start (e.g. internal/protocol/http/fetcher.go)
|
||||
type Task struct {
|
||||
ID string `json:"id"`
|
||||
Protocol string `json:"protocol"`
|
||||
Meta *fetcher.FetcherMeta `json:"meta"`
|
||||
Status base.Status `json:"status"`
|
||||
Uploading bool `json:"uploading"`
|
||||
Progress *Progress `json:"progress"`
|
||||
CreatedAt time.Time `json:"createdAt"`
|
||||
UpdatedAt time.Time `json:"updatedAt"`
|
||||
ID string `json:"id"`
|
||||
Protocol string `json:"protocol"`
|
||||
Meta *fetcher.FetcherMeta `json:"meta"`
|
||||
// ParentID identifies the independently stored group.
|
||||
ParentID string `json:"parentId,omitempty"`
|
||||
Status base.Status `json:"status"`
|
||||
Uploading bool `json:"uploading"`
|
||||
Progress *Progress `json:"progress"`
|
||||
CreatedAt time.Time `json:"createdAt"`
|
||||
UpdatedAt time.Time `json:"updatedAt"`
|
||||
// DoneAt is the time when the task completed; nil if it has never
|
||||
// finished. A done task never leaves the done state, so the value is
|
||||
// written once and kept.
|
||||
@@ -54,6 +70,8 @@ type Task struct {
|
||||
// does not have it and keeps its directory and name.
|
||||
NeedsPreparation bool `json:"needsPreparation,omitempty"`
|
||||
|
||||
group *TaskGroup
|
||||
|
||||
producer production.Source
|
||||
producerReceived int64
|
||||
|
||||
@@ -144,10 +162,12 @@ func (t *Task) MarshalJSON() ([]byte, error) {
|
||||
type rawTaskType Task
|
||||
jsonTask := struct {
|
||||
*rawTaskType
|
||||
Name string `json:"name"`
|
||||
Name string `json:"name"`
|
||||
Group *TaskGroup `json:"group,omitempty"`
|
||||
}{
|
||||
(*rawTaskType)(t),
|
||||
t.Name(),
|
||||
t.group,
|
||||
}
|
||||
return json.Marshal(jsonTask)
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user