feat: use batch download api for create multitasks (#978)

This commit is contained in:
Levi authored and GitHub committed 2025-04-05 22:57:17 +08:00
1 parent cd5fd45667
commit d289708f9e
10 files changed
+207 -37

No files matched your search

+10
View File
@@ -163,6 +163,16 @@ func ParseOptsExtra[E any](opts *Options) error {
return nil return nil
} }
type CreateTaskBatch struct {
Reqs []*CreateTaskBatchItem `json:"reqs"`
Opts *Options `json:"opts"`
}
type CreateTaskBatchItem struct {
Req *Request `json:"req"`
Opts *Options `json:"opts"`
}
// DownloaderStoreConfig is the config that can restore the downloader. // DownloaderStoreConfig is the config that can restore the downloader.
type DownloaderStoreConfig struct { type DownloaderStoreConfig struct {
FirstLoad bool `json:"-"` // FirstLoad is the flag that the config is first time init and not from store FirstLoad bool `json:"-"` // FirstLoad is the flag that the config is first time init and not from store
+12 -4
View File
@@ -359,10 +359,14 @@ func (d *Downloader) CreateDirect(req *base.Request, opts *base.Options) (taskId
return d.doCreate(fetcher, opts) return d.doCreate(fetcher, opts)
} }
func (d *Downloader) CreateDirectBatch(reqs []*base.Request, opts *base.Options) (taskId []string, err error) { func (d *Downloader) CreateDirectBatch(req *base.CreateTaskBatch) (taskId []string, err error) {
taskIds := make([]string, 0) taskIds := make([]string, 0)
for _, req := range reqs { for _, ir := range req.Reqs {
taskId, err := d.CreateDirect(req, opts.Clone()) opts := ir.Opts
if opts == nil {
opts = req.Opts
}
taskId, err := d.CreateDirect(ir.Req, opts.Clone())
if err != nil { if err != nil {
return nil, err return nil, err
} }
@@ -574,15 +578,19 @@ func (d *Downloader) Delete(filter *TaskFilter, force bool) (err error) {
} }
func (d *Downloader) deleteAll() (err error) { func (d *Downloader) deleteAll() (err error) {
var deleteTasksTemp []*Task
func() { func() {
d.lock.Lock() d.lock.Lock()
defer d.lock.Unlock() defer d.lock.Unlock()
for _, task := range d.tasks {
deleteTasksTemp = append(deleteTasksTemp, task)
}
d.tasks = make([]*Task, 0) d.tasks = make([]*Task, 0)
d.waitTasks = make([]*Task, 0) d.waitTasks = make([]*Task, 0)
}() }()
for _, task := range d.tasks { for _, task := range deleteTasksTemp {
if err = d.doDelete(task, true); err != nil { if err = d.doDelete(task, true); err != nil {
return return
} }
+24 -14
View File
@@ -137,13 +137,15 @@ func TestDownloader_CreateDirectBatch(t *testing.T) {
downloader.Clear() downloader.Clear()
}() }()
reqs := make([]*base.Request, 0) reqs := make([]*base.CreateTaskBatchItem, 0)
fileNames := make([]string, 0) fileNames := make([]string, 0)
for i := 0; i < 5; i++ { for i := 0; i < 5; i++ {
req := &base.Request{ req := &base.Request{
URL: "http://" + listener.Addr().String() + "/" + test.BuildName, URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
} }
reqs = append(reqs, req) reqs = append(reqs, &base.CreateTaskBatchItem{
Req: req,
})
if i == 0 { if i == 0 {
fileNames = append(fileNames, test.DownloadName) fileNames = append(fileNames, test.DownloadName)
} else { } else {
@@ -160,11 +162,14 @@ func TestDownloader_CreateDirectBatch(t *testing.T) {
} }
}) })
_, err := downloader.CreateDirectBatch(reqs, &base.Options{ _, err := downloader.CreateDirectBatch(&base.CreateTaskBatch{
Path: test.Dir, Reqs: reqs,
Name: test.DownloadName, Opts: &base.Options{
Extra: http.OptsExtra{ Path: test.Dir,
Connections: 4, Name: test.DownloadName,
Extra: http.OptsExtra{
Connections: 4,
},
}, },
}) })
if err != nil { if err != nil {
@@ -499,13 +504,15 @@ func TestDownloader_GetTasksByFilter(t *testing.T) {
downloader.Clear() downloader.Clear()
}() }()
reqs := make([]*base.Request, 0) reqs := make([]*base.CreateTaskBatchItem, 0)
fileNames := make([]string, 0) fileNames := make([]string, 0)
for i := 0; i < 10; i++ { for i := 0; i < 10; i++ {
req := &base.Request{ req := &base.Request{
URL: "http://" + listener.Addr().String() + "/" + test.BuildName, URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
} }
reqs = append(reqs, req) reqs = append(reqs, &base.CreateTaskBatchItem{
Req: req,
})
if i == 0 { if i == 0 {
fileNames = append(fileNames, test.DownloadName) fileNames = append(fileNames, test.DownloadName)
} else { } else {
@@ -522,11 +529,14 @@ func TestDownloader_GetTasksByFilter(t *testing.T) {
} }
}) })
taskIds, err := downloader.CreateDirectBatch(reqs, &base.Options{ taskIds, err := downloader.CreateDirectBatch(&base.CreateTaskBatch{
Path: test.Dir, Reqs: reqs,
Name: test.DownloadName, Opts: &base.Options{
Extra: http.OptsExtra{ Path: test.Dir,
Connections: 4, Name: test.DownloadName,
Extra: http.OptsExtra{
Connections: 4,
},
}, },
}) })
if err != nil { if err != nil {
+2 -2
View File
@@ -58,13 +58,13 @@ func CreateTask(w http.ResponseWriter, r *http.Request) {
} }
func CreateTaskBatch(w http.ResponseWriter, r *http.Request) { func CreateTaskBatch(w http.ResponseWriter, r *http.Request) {
var req model.CreateTaskBatch var req base.CreateTaskBatch
if ReadJson(r, w, &req) { if ReadJson(r, w, &req) {
if len(req.Reqs) == 0 { if len(req.Reqs) == 0 {
WriteJson(w, model.NewErrorResult("param invalid: reqs", model.CodeInvalidParam)) WriteJson(w, model.NewErrorResult("param invalid: reqs", model.CodeInvalidParam))
return return
} }
taskIds, err := Downloader.CreateDirectBatch(req.Reqs, req.Opt) taskIds, err := Downloader.CreateDirectBatch(&req)
if err != nil { if err != nil {
WriteJson(w, model.NewErrorResult(err.Error())) WriteJson(w, model.NewErrorResult(err.Error()))
return return
-5
View File
@@ -7,8 +7,3 @@ type CreateTask struct {
Req *base.Request `json:"req"` Req *base.Request `json:"req"`
Opt *base.Options `json:"opt"` Opt *base.Options `json:"opt"`
} }
type CreateTaskBatch struct {
Reqs []*base.Request `json:"reqs"`
Opt *base.Options `json:"opt"`
}
+46 -3
View File
@@ -16,6 +16,7 @@ import (
"os" "os"
"path/filepath" "path/filepath"
"reflect" "reflect"
"strings"
"sync" "sync"
"testing" "testing"
"time" "time"
@@ -140,11 +141,13 @@ func TestCreateDirectTask(t *testing.T) {
func TestCreateDirectTaskBatch(t *testing.T) { func TestCreateDirectTaskBatch(t *testing.T) {
doTest(func() { doTest(func() {
reqs := make([]*base.Request, 0) reqs := make([]*base.CreateTaskBatchItem, 0)
for i := 0; i < 5; i++ { for i := 0; i < 5; i++ {
reqs = append(reqs, createReq.Req) reqs = append(reqs, &base.CreateTaskBatchItem{
Req: createReq.Req,
})
} }
taskIds := httpRequestCheckOk[[]string](http.MethodPost, "/api/v1/tasks/batch", &model.CreateTaskBatch{ taskIds := httpRequestCheckOk[[]string](http.MethodPost, "/api/v1/tasks/batch", &base.CreateTaskBatch{
Reqs: reqs, Reqs: reqs,
}) })
if len(taskIds) != len(reqs) { if len(taskIds) != len(reqs) {
@@ -153,6 +156,45 @@ func TestCreateDirectTaskBatch(t *testing.T) {
}) })
} }
func TestCreateDirectTaskBatchWithOpt(t *testing.T) {
doTest(func() {
reqs := make([]*base.CreateTaskBatchItem, 0)
for i := 0; i < 5; i++ {
item := &base.CreateTaskBatchItem{
Req: createReq.Req,
}
if i == 0 {
item.Opts = &base.Options{
Name: "spe_opt.data",
}
}
reqs = append(reqs, item)
}
taskIds := httpRequestCheckOk[[]string](http.MethodPost, "/api/v1/tasks/batch", &base.CreateTaskBatch{
Reqs: reqs,
Opts: &base.Options{
Name: "default_opt.data",
},
})
if len(taskIds) != len(reqs) {
t.Errorf("CreateDirectTaskBatch() got = %v, want %v", len(taskIds), len(reqs))
}
for i, taskId := range taskIds {
task := httpRequestCheckOk[*download.Task](http.MethodGet, "/api/v1/tasks/"+taskId, nil)
if i == 0 {
if !strings.Contains(task.Name(), "spe_opt") {
t.Errorf("CreateDirectTaskBatch() got = %v, want %v", task.Name(), "spe_opt.data")
}
} else {
if !strings.Contains(task.Name(), "default_opt") {
t.Errorf("CreateDirectTaskBatch() got = %v, want %v", task.Name(), "default_opt.data")
}
}
}
})
}
func TestCreateDirectTaskWithResoled(t *testing.T) { func TestCreateDirectTaskWithResoled(t *testing.T) {
doTest(func() { doTest(func() {
var wg sync.WaitGroup var wg sync.WaitGroup
@@ -666,6 +708,7 @@ func doTest0(onStart func(cfg *model.StartConfig), handler func()) {
Downloader.Clear() Downloader.Clear()
}() }()
defer func() { defer func() {
time.Sleep(500 * time.Millisecond)
Downloader.Pause(nil) Downloader.Pause(nil)
Downloader.Delete(nil, true) Downloader.Delete(nil, true)
os.RemoveAll(cfg.StorageDir) os.RemoveAll(cfg.StorageDir)
+11 -4
View File
@@ -3,19 +3,20 @@ import 'dart:io';
import 'package:dio/dio.dart'; import 'package:dio/dio.dart';
import 'package:dio/io.dart'; import 'package:dio/io.dart';
import 'package:flutter/foundation.dart'; import 'package:flutter/foundation.dart';
import 'package:gopeed/api/model/extension.dart';
import 'package:gopeed/api/model/install_extension.dart';
import 'package:gopeed/api/model/switch_extension.dart';
import '../util/util.dart'; import '../util/util.dart';
import 'model/create_task.dart'; import 'model/create_task.dart';
import 'model/create_task_batch.dart';
import 'model/downloader_config.dart'; import 'model/downloader_config.dart';
import 'model/extension.dart';
import 'model/install_extension.dart';
import 'model/request.dart'; import 'model/request.dart';
import 'model/resolve_result.dart'; import 'model/resolve_result.dart';
import 'model/result.dart'; import 'model/result.dart';
import 'model/switch_extension.dart';
import 'model/task.dart'; import 'model/task.dart';
import 'model/update_extension_settings.dart';
import 'model/update_check_extension_resp.dart'; import 'model/update_check_extension_resp.dart';
import 'model/update_extension_settings.dart';
class _Client { class _Client {
static _Client? _instance; static _Client? _instance;
@@ -114,6 +115,12 @@ Future<String> createTask(CreateTask createTask) async {
(data) => data as String); (data) => data as String);
} }
Future<List<String>> createTaskBatch(CreateTaskBatch createTaskBatch) async {
return _parse<List<String>>(
() => _client.dio.post("/api/v1/tasks/batch", data: createTaskBatch),
(data) => (data as List).map((e) => e as String).toList());
}
Future<List<Task>> getTasks(List<Status> statuses) async { Future<List<Task>> getTasks(List<Status> statuses) async {
return _parse<List<Task>>( return _parse<List<Task>>(
() => _client.dio.get( () => _client.dio.get(
@@ -0,0 +1,39 @@
import 'package:json_annotation/json_annotation.dart';
import 'options.dart';
import 'request.dart';
part 'create_task_batch.g.dart';
@JsonSerializable(explicitToJson: true)
class CreateTaskBatch {
List<CreateTaskBatchItem>? reqs;
Options? opt;
CreateTaskBatch({
this.reqs,
this.opt,
});
factory CreateTaskBatch.fromJson(
Map<String, dynamic> json,
) =>
_$CreateTaskBatchFromJson(json);
Map<String, dynamic> toJson() => _$CreateTaskBatchToJson(this);
}
@JsonSerializable()
class CreateTaskBatchItem {
Request? req;
Options? opts;
CreateTaskBatchItem({
this.req,
this.opts,
});
factory CreateTaskBatchItem.fromJson(Map<String, dynamic> json) =>
_$CreateTaskBatchItemFromJson(json);
Map<String, dynamic> toJson() => _$CreateTaskBatchItemToJson(this);
}
@@ -0,0 +1,55 @@
// GENERATED CODE - DO NOT MODIFY BY HAND
part of 'create_task_batch.dart';
// **************************************************************************
// JsonSerializableGenerator
// **************************************************************************
CreateTaskBatch _$CreateTaskBatchFromJson(Map<String, dynamic> json) =>
CreateTaskBatch(
reqs: (json['reqs'] as List<dynamic>?)
?.map((e) => CreateTaskBatchItem.fromJson(e as Map<String, dynamic>))
.toList(),
opt: json['opt'] == null
? null
: Options.fromJson(json['opt'] as Map<String, dynamic>),
);
Map<String, dynamic> _$CreateTaskBatchToJson(CreateTaskBatch instance) {
final val = <String, dynamic>{};
void writeNotNull(String key, dynamic value) {
if (value != null) {
val[key] = value;
}
}
writeNotNull('reqs', instance.reqs?.map((e) => e.toJson()).toList());
writeNotNull('opt', instance.opt?.toJson());
return val;
}
CreateTaskBatchItem _$CreateTaskBatchItemFromJson(Map<String, dynamic> json) =>
CreateTaskBatchItem(
req: json['req'] == null
? null
: Request.fromJson(json['req'] as Map<String, dynamic>),
opts: json['opts'] == null
? null
: Options.fromJson(json['opts'] as Map<String, dynamic>),
);
Map<String, dynamic> _$CreateTaskBatchItemToJson(CreateTaskBatchItem instance) {
final val = <String, dynamic>{};
void writeNotNull(String key, dynamic value) {
if (value != null) {
val[key] = value;
}
}
writeNotNull('req', instance.req);
writeNotNull('opts', instance.opts);
return val;
}
@@ -11,6 +11,7 @@ import 'package:rounded_loading_button_plus/rounded_loading_button.dart';
import '../../../../api/api.dart'; import '../../../../api/api.dart';
import '../../../../api/model/create_task.dart'; import '../../../../api/model/create_task.dart';
import '../../../../api/model/create_task_batch.dart';
import '../../../../api/model/options.dart'; import '../../../../api/model/options.dart';
import '../../../../api/model/request.dart'; import '../../../../api/model/request.dart';
import '../../../../api/model/resolve_result.dart'; import '../../../../api/model/resolve_result.dart';
@@ -869,18 +870,20 @@ class CreateView extends GetView<CreateController> {
if (createFormKey.currentState!.validate()) { if (createFormKey.currentState!.validate()) {
if (rr.id.isEmpty) { if (rr.id.isEmpty) {
// from extension resolve result // from extension resolve result
await Future.wait( final reqs =
controller.selectedIndexes.map((index) { controller.selectedIndexes.map((index) {
final file = rr.res.files[index]; final file = rr.res.files[index];
return createTask(CreateTask( return CreateTaskBatchItem(
req: file.req!..proxy = parseProxy(), req: file.req!..proxy = parseProxy(),
opt: Options( opts: Options(
name: file.name, name: file.name,
path: path.join(_pathController.text, path: path.join(_pathController.text,
rr.res.name, file.path), rr.res.name, file.path),
selectFiles: [], selectFiles: [],
extra: optExtra))); extra: optExtra));
})); }).toList();
await createTaskBatch(
CreateTaskBatch(reqs: reqs));
} else { } else {
await createTask(CreateTask( await createTask(CreateTask(
rid: rr.id, rid: rr.id,