From d289708f9ec4842b26b82a71753ee4c74ca389b0 Mon Sep 17 00:00:00 2001 From: Levi Date: Sat, 5 Apr 2025 22:57:17 +0800 Subject: [PATCH] feat: use batch download api for create multitasks (#978) --- pkg/base/model.go | 10 ++++ pkg/download/downloader.go | 16 ++++-- pkg/download/downloader_test.go | 38 ++++++++----- pkg/rest/api.go | 4 +- pkg/rest/model/task.go | 5 -- pkg/rest/server_test.go | 49 ++++++++++++++++- ui/flutter/lib/api/api.dart | 15 +++-- .../lib/api/model/create_task_batch.dart | 39 +++++++++++++ .../lib/api/model/create_task_batch.g.dart | 55 +++++++++++++++++++ .../app/modules/create/views/create_view.dart | 13 +++-- 10 files changed, 207 insertions(+), 37 deletions(-) create mode 100644 ui/flutter/lib/api/model/create_task_batch.dart create mode 100644 ui/flutter/lib/api/model/create_task_batch.g.dart diff --git a/pkg/base/model.go b/pkg/base/model.go index da5cefd8..5b006edd 100644 --- a/pkg/base/model.go +++ b/pkg/base/model.go @@ -163,6 +163,16 @@ func ParseOptsExtra[E any](opts *Options) error { return nil } +type CreateTaskBatch struct { + Reqs []*CreateTaskBatchItem `json:"reqs"` + Opts *Options `json:"opts"` +} + +type CreateTaskBatchItem struct { + Req *Request `json:"req"` + Opts *Options `json:"opts"` +} + // DownloaderStoreConfig is the config that can restore the downloader. type DownloaderStoreConfig struct { FirstLoad bool `json:"-"` // FirstLoad is the flag that the config is first time init and not from store diff --git a/pkg/download/downloader.go b/pkg/download/downloader.go index 5ca70e62..84cb5ec9 100644 --- a/pkg/download/downloader.go +++ b/pkg/download/downloader.go @@ -359,10 +359,14 @@ func (d *Downloader) CreateDirect(req *base.Request, opts *base.Options) (taskId return d.doCreate(fetcher, opts) } -func (d *Downloader) CreateDirectBatch(reqs []*base.Request, opts *base.Options) (taskId []string, err error) { +func (d *Downloader) CreateDirectBatch(req *base.CreateTaskBatch) (taskId []string, err error) { taskIds := make([]string, 0) - for _, req := range reqs { - taskId, err := d.CreateDirect(req, opts.Clone()) + for _, ir := range req.Reqs { + opts := ir.Opts + if opts == nil { + opts = req.Opts + } + taskId, err := d.CreateDirect(ir.Req, opts.Clone()) if err != nil { return nil, err } @@ -574,15 +578,19 @@ func (d *Downloader) Delete(filter *TaskFilter, force bool) (err error) { } func (d *Downloader) deleteAll() (err error) { + var deleteTasksTemp []*Task func() { d.lock.Lock() defer d.lock.Unlock() + for _, task := range d.tasks { + deleteTasksTemp = append(deleteTasksTemp, task) + } d.tasks = make([]*Task, 0) d.waitTasks = make([]*Task, 0) }() - for _, task := range d.tasks { + for _, task := range deleteTasksTemp { if err = d.doDelete(task, true); err != nil { return } diff --git a/pkg/download/downloader_test.go b/pkg/download/downloader_test.go index a7cfd10c..de0ffad4 100644 --- a/pkg/download/downloader_test.go +++ b/pkg/download/downloader_test.go @@ -137,13 +137,15 @@ func TestDownloader_CreateDirectBatch(t *testing.T) { downloader.Clear() }() - reqs := make([]*base.Request, 0) + reqs := make([]*base.CreateTaskBatchItem, 0) fileNames := make([]string, 0) for i := 0; i < 5; i++ { req := &base.Request{ URL: "http://" + listener.Addr().String() + "/" + test.BuildName, } - reqs = append(reqs, req) + reqs = append(reqs, &base.CreateTaskBatchItem{ + Req: req, + }) if i == 0 { fileNames = append(fileNames, test.DownloadName) } else { @@ -160,11 +162,14 @@ func TestDownloader_CreateDirectBatch(t *testing.T) { } }) - _, err := downloader.CreateDirectBatch(reqs, &base.Options{ - Path: test.Dir, - Name: test.DownloadName, - Extra: http.OptsExtra{ - Connections: 4, + _, err := downloader.CreateDirectBatch(&base.CreateTaskBatch{ + Reqs: reqs, + Opts: &base.Options{ + Path: test.Dir, + Name: test.DownloadName, + Extra: http.OptsExtra{ + Connections: 4, + }, }, }) if err != nil { @@ -499,13 +504,15 @@ func TestDownloader_GetTasksByFilter(t *testing.T) { downloader.Clear() }() - reqs := make([]*base.Request, 0) + reqs := make([]*base.CreateTaskBatchItem, 0) fileNames := make([]string, 0) for i := 0; i < 10; i++ { req := &base.Request{ URL: "http://" + listener.Addr().String() + "/" + test.BuildName, } - reqs = append(reqs, req) + reqs = append(reqs, &base.CreateTaskBatchItem{ + Req: req, + }) if i == 0 { fileNames = append(fileNames, test.DownloadName) } else { @@ -522,11 +529,14 @@ func TestDownloader_GetTasksByFilter(t *testing.T) { } }) - taskIds, err := downloader.CreateDirectBatch(reqs, &base.Options{ - Path: test.Dir, - Name: test.DownloadName, - Extra: http.OptsExtra{ - Connections: 4, + taskIds, err := downloader.CreateDirectBatch(&base.CreateTaskBatch{ + Reqs: reqs, + Opts: &base.Options{ + Path: test.Dir, + Name: test.DownloadName, + Extra: http.OptsExtra{ + Connections: 4, + }, }, }) if err != nil { diff --git a/pkg/rest/api.go b/pkg/rest/api.go index 9e030b34..c6f4d738 100644 --- a/pkg/rest/api.go +++ b/pkg/rest/api.go @@ -58,13 +58,13 @@ func CreateTask(w http.ResponseWriter, r *http.Request) { } func CreateTaskBatch(w http.ResponseWriter, r *http.Request) { - var req model.CreateTaskBatch + var req base.CreateTaskBatch if ReadJson(r, w, &req) { if len(req.Reqs) == 0 { WriteJson(w, model.NewErrorResult("param invalid: reqs", model.CodeInvalidParam)) return } - taskIds, err := Downloader.CreateDirectBatch(req.Reqs, req.Opt) + taskIds, err := Downloader.CreateDirectBatch(&req) if err != nil { WriteJson(w, model.NewErrorResult(err.Error())) return diff --git a/pkg/rest/model/task.go b/pkg/rest/model/task.go index f93ccdb7..4cd9e656 100644 --- a/pkg/rest/model/task.go +++ b/pkg/rest/model/task.go @@ -7,8 +7,3 @@ type CreateTask struct { Req *base.Request `json:"req"` Opt *base.Options `json:"opt"` } - -type CreateTaskBatch struct { - Reqs []*base.Request `json:"reqs"` - Opt *base.Options `json:"opt"` -} diff --git a/pkg/rest/server_test.go b/pkg/rest/server_test.go index 6e2e9228..ee18f3fc 100644 --- a/pkg/rest/server_test.go +++ b/pkg/rest/server_test.go @@ -16,6 +16,7 @@ import ( "os" "path/filepath" "reflect" + "strings" "sync" "testing" "time" @@ -140,11 +141,13 @@ func TestCreateDirectTask(t *testing.T) { func TestCreateDirectTaskBatch(t *testing.T) { doTest(func() { - reqs := make([]*base.Request, 0) + reqs := make([]*base.CreateTaskBatchItem, 0) for i := 0; i < 5; i++ { - reqs = append(reqs, createReq.Req) + reqs = append(reqs, &base.CreateTaskBatchItem{ + Req: createReq.Req, + }) } - taskIds := httpRequestCheckOk[[]string](http.MethodPost, "/api/v1/tasks/batch", &model.CreateTaskBatch{ + taskIds := httpRequestCheckOk[[]string](http.MethodPost, "/api/v1/tasks/batch", &base.CreateTaskBatch{ Reqs: reqs, }) if len(taskIds) != len(reqs) { @@ -153,6 +156,45 @@ func TestCreateDirectTaskBatch(t *testing.T) { }) } +func TestCreateDirectTaskBatchWithOpt(t *testing.T) { + doTest(func() { + reqs := make([]*base.CreateTaskBatchItem, 0) + for i := 0; i < 5; i++ { + item := &base.CreateTaskBatchItem{ + Req: createReq.Req, + } + if i == 0 { + item.Opts = &base.Options{ + Name: "spe_opt.data", + } + } + reqs = append(reqs, item) + } + taskIds := httpRequestCheckOk[[]string](http.MethodPost, "/api/v1/tasks/batch", &base.CreateTaskBatch{ + Reqs: reqs, + Opts: &base.Options{ + Name: "default_opt.data", + }, + }) + if len(taskIds) != len(reqs) { + t.Errorf("CreateDirectTaskBatch() got = %v, want %v", len(taskIds), len(reqs)) + } + + for i, taskId := range taskIds { + task := httpRequestCheckOk[*download.Task](http.MethodGet, "/api/v1/tasks/"+taskId, nil) + if i == 0 { + if !strings.Contains(task.Name(), "spe_opt") { + t.Errorf("CreateDirectTaskBatch() got = %v, want %v", task.Name(), "spe_opt.data") + } + } else { + if !strings.Contains(task.Name(), "default_opt") { + t.Errorf("CreateDirectTaskBatch() got = %v, want %v", task.Name(), "default_opt.data") + } + } + } + }) +} + func TestCreateDirectTaskWithResoled(t *testing.T) { doTest(func() { var wg sync.WaitGroup @@ -666,6 +708,7 @@ func doTest0(onStart func(cfg *model.StartConfig), handler func()) { Downloader.Clear() }() defer func() { + time.Sleep(500 * time.Millisecond) Downloader.Pause(nil) Downloader.Delete(nil, true) os.RemoveAll(cfg.StorageDir) diff --git a/ui/flutter/lib/api/api.dart b/ui/flutter/lib/api/api.dart index 3d4d3d3a..5eb1238f 100644 --- a/ui/flutter/lib/api/api.dart +++ b/ui/flutter/lib/api/api.dart @@ -3,19 +3,20 @@ import 'dart:io'; import 'package:dio/dio.dart'; import 'package:dio/io.dart'; import 'package:flutter/foundation.dart'; -import 'package:gopeed/api/model/extension.dart'; -import 'package:gopeed/api/model/install_extension.dart'; -import 'package:gopeed/api/model/switch_extension.dart'; import '../util/util.dart'; import 'model/create_task.dart'; +import 'model/create_task_batch.dart'; import 'model/downloader_config.dart'; +import 'model/extension.dart'; +import 'model/install_extension.dart'; import 'model/request.dart'; import 'model/resolve_result.dart'; import 'model/result.dart'; +import 'model/switch_extension.dart'; import 'model/task.dart'; -import 'model/update_extension_settings.dart'; import 'model/update_check_extension_resp.dart'; +import 'model/update_extension_settings.dart'; class _Client { static _Client? _instance; @@ -114,6 +115,12 @@ Future createTask(CreateTask createTask) async { (data) => data as String); } +Future> createTaskBatch(CreateTaskBatch createTaskBatch) async { + return _parse>( + () => _client.dio.post("/api/v1/tasks/batch", data: createTaskBatch), + (data) => (data as List).map((e) => e as String).toList()); +} + Future> getTasks(List statuses) async { return _parse>( () => _client.dio.get( diff --git a/ui/flutter/lib/api/model/create_task_batch.dart b/ui/flutter/lib/api/model/create_task_batch.dart new file mode 100644 index 00000000..8b2f7c98 --- /dev/null +++ b/ui/flutter/lib/api/model/create_task_batch.dart @@ -0,0 +1,39 @@ +import 'package:json_annotation/json_annotation.dart'; + +import 'options.dart'; +import 'request.dart'; + +part 'create_task_batch.g.dart'; + +@JsonSerializable(explicitToJson: true) +class CreateTaskBatch { + List? reqs; + Options? opt; + + CreateTaskBatch({ + this.reqs, + this.opt, + }); + + factory CreateTaskBatch.fromJson( + Map json, + ) => + _$CreateTaskBatchFromJson(json); + + Map toJson() => _$CreateTaskBatchToJson(this); +} + +@JsonSerializable() +class CreateTaskBatchItem { + Request? req; + Options? opts; + + CreateTaskBatchItem({ + this.req, + this.opts, + }); + + factory CreateTaskBatchItem.fromJson(Map json) => + _$CreateTaskBatchItemFromJson(json); + Map toJson() => _$CreateTaskBatchItemToJson(this); +} diff --git a/ui/flutter/lib/api/model/create_task_batch.g.dart b/ui/flutter/lib/api/model/create_task_batch.g.dart new file mode 100644 index 00000000..5bc113e5 --- /dev/null +++ b/ui/flutter/lib/api/model/create_task_batch.g.dart @@ -0,0 +1,55 @@ +// GENERATED CODE - DO NOT MODIFY BY HAND + +part of 'create_task_batch.dart'; + +// ************************************************************************** +// JsonSerializableGenerator +// ************************************************************************** + +CreateTaskBatch _$CreateTaskBatchFromJson(Map json) => + CreateTaskBatch( + reqs: (json['reqs'] as List?) + ?.map((e) => CreateTaskBatchItem.fromJson(e as Map)) + .toList(), + opt: json['opt'] == null + ? null + : Options.fromJson(json['opt'] as Map), + ); + +Map _$CreateTaskBatchToJson(CreateTaskBatch instance) { + final val = {}; + + void writeNotNull(String key, dynamic value) { + if (value != null) { + val[key] = value; + } + } + + writeNotNull('reqs', instance.reqs?.map((e) => e.toJson()).toList()); + writeNotNull('opt', instance.opt?.toJson()); + return val; +} + +CreateTaskBatchItem _$CreateTaskBatchItemFromJson(Map json) => + CreateTaskBatchItem( + req: json['req'] == null + ? null + : Request.fromJson(json['req'] as Map), + opts: json['opts'] == null + ? null + : Options.fromJson(json['opts'] as Map), + ); + +Map _$CreateTaskBatchItemToJson(CreateTaskBatchItem instance) { + final val = {}; + + void writeNotNull(String key, dynamic value) { + if (value != null) { + val[key] = value; + } + } + + writeNotNull('req', instance.req); + writeNotNull('opts', instance.opts); + return val; +} diff --git a/ui/flutter/lib/app/modules/create/views/create_view.dart b/ui/flutter/lib/app/modules/create/views/create_view.dart index 27f93934..82016acb 100644 --- a/ui/flutter/lib/app/modules/create/views/create_view.dart +++ b/ui/flutter/lib/app/modules/create/views/create_view.dart @@ -11,6 +11,7 @@ import 'package:rounded_loading_button_plus/rounded_loading_button.dart'; import '../../../../api/api.dart'; import '../../../../api/model/create_task.dart'; +import '../../../../api/model/create_task_batch.dart'; import '../../../../api/model/options.dart'; import '../../../../api/model/request.dart'; import '../../../../api/model/resolve_result.dart'; @@ -869,18 +870,20 @@ class CreateView extends GetView { if (createFormKey.currentState!.validate()) { if (rr.id.isEmpty) { // from extension resolve result - await Future.wait( + final reqs = controller.selectedIndexes.map((index) { final file = rr.res.files[index]; - return createTask(CreateTask( + return CreateTaskBatchItem( req: file.req!..proxy = parseProxy(), - opt: Options( + opts: Options( name: file.name, path: path.join(_pathController.text, rr.res.name, file.path), selectFiles: [], - extra: optExtra))); - })); + extra: optExtra)); + }).toList(); + await createTaskBatch( + CreateTaskBatch(reqs: reqs)); } else { await createTask(CreateTask( rid: rr.id,