feat: support use server time for file creation (#359)

This commit is contained in:
Levi authored and GitHub committed 2024-01-30 09:08:01 +08:00
1 parent b89c7bc608
commit 62c22bc6b4
20 files changed
+192 -75

No files matched your search

+2 -2
View File
@@ -53,6 +53,7 @@ jobs:
go build -tags nosqlite -ldflags="-w -s -X github.com/GopeedLab/gopeed/pkg/base.Version=$VERSION" -buildmode=c-shared -o ui/flutter/windows/libgopeed.dll github.com/GopeedLab/gopeed/bind/desktop
cd ui/flutter
$TAG = "v$env:VERSION"
flutter build windows
$mingw = "C:\Program Files\Git\mingw64\bin"
$system = "C:\Windows\System32"
@@ -65,7 +66,7 @@ jobs:
cp $system\vcruntime140_1.dll $release
New-Item -Path build\windows\Output -ItemType Directory
Compress-Archive -Path "$release*" -DestinationPath "build\windows\Output\Gopeed-$env:TAG-windows-amd64-portable.zip"
Compress-Archive -Path "$release*" -DestinationPath "build\windows\Output\Gopeed-$TAG-windows-amd64-portable.zip"
cd build/windows
echo @"
@@ -124,7 +125,6 @@ jobs:
Filename: "{app}\{#MyAppExeName}"; Description: "{cm:LaunchProgram,{#StringChange(MyAppName, '&', '&&')}}"; Flags: nowait postinstall skipifsilent
"@ > setup.iss
iscc.exe setup.iss
$TAG = "v$env:VERSION"
mv "Output\gopeed.exe" "Output\Gopeed-$TAG-windows-amd64.exe"
Compress-Archive -Path "Output\Gopeed-$TAG-windows-amd64.exe" -DestinationPath "Output\Gopeed-$TAG-windows-amd64.zip"
Remove-Item -Path "Output\Gopeed-$TAG-windows-amd64.exe"
+1
View File
@@ -96,6 +96,7 @@ require (
github.com/stretchr/testify v1.8.4 // indirect
github.com/tidwall/btree v1.7.0 // indirect
github.com/xanzy/ssh-agent v0.3.3 // indirect
github.com/xiaoqidun/setft v0.0.0-20220310121541-be86327699ad // indirect
go.opentelemetry.io/otel v1.22.0 // indirect
go.opentelemetry.io/otel/metric v1.22.0 // indirect
go.opentelemetry.io/otel/trace v1.22.0 // indirect
+2
View File
@@ -425,6 +425,8 @@ github.com/willf/bitset v1.1.9/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPyS
github.com/willf/bitset v1.1.10/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4=
github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM=
github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw=
github.com/xiaoqidun/setft v0.0.0-20220310121541-be86327699ad h1:QtuWk6dsrNXc/ugwCBEN0jG52q/4tXRzMvZm4fH6T9g=
github.com/xiaoqidun/setft v0.0.0-20220310121541-be86327699ad/go.mod h1:Jj8p9bgKGTPQ+M8CdUMS9p7Mmdoxa3OAcAjJQBu0CcI=
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
+3 -2
View File
@@ -1,6 +1,7 @@
package http
type config struct {
UserAgent string `json:"userAgent"`
Connections int `json:"connections"`
UserAgent string `json:"userAgent"`
Connections int `json:"connections"`
UseServerCtime bool `json:"useServerCtime"`
}
+16 -2
View File
@@ -9,6 +9,7 @@ import (
"github.com/GopeedLab/gopeed/internal/fetcher"
"github.com/GopeedLab/gopeed/pkg/base"
fhttp "github.com/GopeedLab/gopeed/pkg/protocol/http"
"github.com/xiaoqidun/setft"
"golang.org/x/sync/errgroup"
"io"
"mime"
@@ -131,8 +132,17 @@ func (f *Fetcher) Resolve(req *base.Request) error {
} else {
return NewRequestError(httpResp.StatusCode, httpResp.Status)
}
// Parse last modified time
var lastModifiedTime *time.Time
lastModified := httpResp.Header.Get(base.HttpHeaderLastModified)
if lastModified != "" {
// ignore parse error
t, _ := time.Parse(time.RFC1123, lastModified)
lastModifiedTime = &t
}
file := &base.FileInfo{
Size: res.Size,
Size: res.Size,
Ctime: lastModifiedTime,
}
contentDisposition := httpResp.Header.Get(base.HttpHeaderContentDisposition)
if contentDisposition != "" {
@@ -258,6 +268,10 @@ func (f *Fetcher) fetch() {
return
}
f.file.Close()
// Update file last modified time
if f.config.UseServerCtime && f.meta.Res.Files[0].Ctime != nil {
setft.SetFileTime(f.file.Name(), time.Now(), *f.meta.Res.Files[0].Ctime, *f.meta.Res.Files[0].Ctime)
}
f.doneCh <- err
}()
}
@@ -276,7 +290,7 @@ func (f *Fetcher) fetchChunk(index int, ctx context.Context) (err error) {
maxRetries = 3
)
// retry until all remain chunks failed
for true {
for {
// if chunk is completed, return
if f.meta.Res.Range && chunk.Downloaded >= chunk.End-chunk.Begin+1 {
return
+29 -9
View File
@@ -10,7 +10,6 @@ import (
"github.com/GopeedLab/gopeed/pkg/protocol/http"
"github.com/GopeedLab/gopeed/pkg/util"
"net"
"reflect"
"testing"
"time"
)
@@ -69,7 +68,7 @@ func testResolve(startTestServer func() net.Listener, path string, want *base.Re
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(want, fetcher.meta.Res) {
if !test.AssertResourceEqual(want, fetcher.meta.Res) {
t.Errorf("Resolve() got = %v, want %v", fetcher.meta.Res, want)
}
}
@@ -151,10 +150,34 @@ func TestFetcher_DownloadWithProxy(t *testing.T) {
downloadWithProxy(httpListener, proxyListener, t)
}
func TestFetcher_Config(t *testing.T) {
func TestFetcher_ConfigConnections(t *testing.T) {
listener := test.StartTestFileServer()
defer listener.Close()
fetcher := doDownloadReady(buildConfigFetcher(), listener, 0, t)
fetcher := doDownloadReady(buildConfigFetcher(config{
Connections: 16,
}), listener, 0, t)
err := fetcher.Start()
if err != nil {
t.Fatal(err)
}
err = fetcher.Wait()
if err != nil {
t.Fatal(err)
}
want := test.FileMd5(test.BuildFile)
got := test.FileMd5(test.DownloadFile)
if want != got {
t.Errorf("Download() got = %v, want %v", got, want)
}
}
func TestFetcher_ConfigUseServerCtime(t *testing.T) {
listener := test.StartTestFileServer()
defer listener.Close()
fetcher := doDownloadReady(buildConfigFetcher(config{
Connections: 16,
UseServerCtime: true,
}), listener, 0, t)
err := fetcher.Start()
if err != nil {
t.Fatal(err)
@@ -338,14 +361,11 @@ func buildFetcher() *Fetcher {
return fetcher.(*Fetcher)
}
func buildConfigFetcher() fetcher.Fetcher {
func buildConfigFetcher(cfg config) fetcher.Fetcher {
fetcher := new(FetcherBuilder).Build()
newController := controller.NewController()
mockCfg := config{
Connections: 16,
}
newController.GetConfig = func(v any) bool {
if err := json.Unmarshal([]byte(test.ToJson(mockCfg)), v); err != nil {
if err := json.Unmarshal([]byte(test.ToJson(cfg)), v); err != nil {
return false
}
return true
+10
View File
@@ -4,12 +4,14 @@ import (
"context"
"encoding/json"
"fmt"
"github.com/GopeedLab/gopeed/pkg/base"
"github.com/armon/go-socks5"
"io"
"math/rand"
"net"
"net/http"
"os"
"reflect"
"strconv"
"strings"
"sync/atomic"
@@ -337,3 +339,11 @@ func StartSocks5Server(usr, pwd string) net.Listener {
go server.Serve(listener)
return listener
}
func AssertResourceEqual(want, got *base.Resource) bool {
// Ignore ctime
if got != nil && len(got.Files) > 0 {
got.Files[0].Ctime = nil
}
return reflect.DeepEqual(want, got)
}
+2 -1
View File
@@ -20,7 +20,8 @@ const (
HttpHeaderContentLength = "Content-Length"
HttpHeaderContentRange = "Content-Range"
HttpHeaderContentDisposition = "Content-Disposition"
HttpHeaderUserAgent = "Usr-Agent"
HttpHeaderUserAgent = "User-Agent"
HttpHeaderLastModified = "Last-Modified"
HttpHeaderBytes = "bytes"
HttpHeaderRangeFormat = "bytes=%d-%d"
+5 -3
View File
@@ -4,6 +4,7 @@ import (
"fmt"
"github.com/GopeedLab/gopeed/pkg/util"
"golang.org/x/exp/slices"
"time"
)
// Request download request
@@ -59,9 +60,10 @@ func (r *Resource) CalcSize(selectFiles []int) {
}
type FileInfo struct {
Name string `json:"name"`
Path string `json:"path"`
Size int64 `json:"size"`
Name string `json:"name"`
Path string `json:"path"`
Size int64 `json:"size"`
Ctime *time.Time `json:"ctime"`
Req *Request `json:"req"`
}
+5
View File
@@ -124,6 +124,11 @@ func (d *Downloader) Setup() error {
}
// init default config
d.cfg.DownloaderStoreConfig.Init()
for _, fb := range d.fetcherBuilders {
f := fb.Build()
d.setupFetcher(f)
}
// load tasks from storage
var tasks []*Task
if err = d.storage.List(bucketTask, &tasks); err != nil {
+2 -3
View File
@@ -4,7 +4,6 @@ import (
"github.com/GopeedLab/gopeed/internal/test"
"github.com/GopeedLab/gopeed/pkg/base"
"github.com/GopeedLab/gopeed/pkg/protocol/http"
"reflect"
"sync"
"testing"
"time"
@@ -37,7 +36,7 @@ func TestDownloader_Resolve(t *testing.T) {
},
},
}
if !reflect.DeepEqual(want, rr.Res) {
if !test.AssertResourceEqual(want, rr.Res) {
t.Errorf("Resolve() got = %v, want %v", rr.Res, want)
}
}
@@ -155,7 +154,7 @@ func doTestDownloaderCreateWithProxy(t *testing.T, auth bool, buildProxyConfig f
},
},
}
if !reflect.DeepEqual(want, rr.Res) {
if !test.AssertResourceEqual(want, rr.Res) {
t.Errorf("Resolve() got = %v, want %v", rr.Res, want)
}
}
+1 -1
View File
@@ -66,7 +66,7 @@ var (
func TestResolve(t *testing.T) {
doTest(func() {
resp := httpRequestCheckOk[*download.ResolveResult](http.MethodPost, "/api/v1/resolve", taskReq)
if !test.JsonEqual(taskRes, resp.Res) {
if !test.AssertResourceEqual(taskRes, resp.Res) {
t.Errorf("Resolve() got = %v, want %v", test.ToJson(resp.Res), test.ToJson(taskRes))
}
})
+38 -17
View File
@@ -4,13 +4,16 @@ part 'downloader_config.g.dart';
@JsonSerializable(explicitToJson: true)
class DownloaderConfig {
String downloadDir = '';
int maxRunning = 0;
String downloadDir;
int maxRunning;
ProtocolConfig protocolConfig = ProtocolConfig();
ExtraConfig extra = ExtraConfig();
ProxyConfig proxy = ProxyConfig();
DownloaderConfig();
DownloaderConfig({
this.downloadDir = '',
this.maxRunning = 0,
});
factory DownloaderConfig.fromJson(Map<String, dynamic> json) =>
_$DownloaderConfigFromJson(json);
@@ -31,11 +34,16 @@ class ProtocolConfig {
@JsonSerializable()
class HttpConfig {
String userAgent =
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36';
int connections = 0;
String userAgent;
int connections;
bool useServerCtime;
HttpConfig();
HttpConfig({
this.userAgent =
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36',
this.connections = 16,
this.useServerCtime = false,
});
factory HttpConfig.fromJson(Map<String, dynamic> json) =>
_$HttpConfigFromJson(json);
@@ -69,13 +77,19 @@ class ExtraConfig {
@JsonSerializable()
class ProxyConfig {
bool enable = false;
String scheme = '';
String host = '';
String usr = '';
String pwd = '';
bool enable;
String scheme;
String host;
String usr;
String pwd;
ProxyConfig();
ProxyConfig({
this.enable = false,
this.scheme = '',
this.host = '',
this.usr = '',
this.pwd = '',
});
factory ProxyConfig.fromJson(Map<String, dynamic> json) =>
_$ProxyConfigFromJson(json);
@@ -85,13 +99,20 @@ class ProxyConfig {
@JsonSerializable()
class ExtraConfigBt {
List<String> trackerSubscribeUrls = [];
List<String> subscribeTrackers = [];
List<String> trackerSubscribeUrls;
List<String> subscribeTrackers;
bool autoUpdateTrackers;
DateTime? lastTrackerUpdateTime;
List<String> customTrackers = [];
List<String> customTrackers;
ExtraConfigBt();
ExtraConfigBt({
this.trackerSubscribeUrls = const [],
this.subscribeTrackers = const [],
this.autoUpdateTrackers = true,
this.lastTrackerUpdateTime,
this.customTrackers = const [],
});
factory ExtraConfigBt.fromJson(Map<String, dynamic> json) =>
_$ExtraConfigBtFromJson(json);
@@ -7,9 +7,10 @@ part of 'downloader_config.dart';
// **************************************************************************
DownloaderConfig _$DownloaderConfigFromJson(Map<String, dynamic> json) =>
DownloaderConfig()
..downloadDir = json['downloadDir'] as String
..maxRunning = json['maxRunning'] as int
DownloaderConfig(
downloadDir: json['downloadDir'] as String? ?? '',
maxRunning: json['maxRunning'] as int? ?? 0,
)
..protocolConfig = ProtocolConfig.fromJson(
json['protocolConfig'] as Map<String, dynamic>?)
..extra = ExtraConfig.fromJson(json['extra'] as Map<String, dynamic>?)
@@ -35,14 +36,18 @@ Map<String, dynamic> _$ProtocolConfigToJson(ProtocolConfig instance) =>
'bt': instance.bt.toJson(),
};
HttpConfig _$HttpConfigFromJson(Map<String, dynamic> json) => HttpConfig()
..userAgent = json['userAgent'] as String
..connections = json['connections'] as int;
HttpConfig _$HttpConfigFromJson(Map<String, dynamic> json) => HttpConfig(
userAgent: json['userAgent'] as String? ??
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36',
connections: json['connections'] as int? ?? 16,
useServerCtime: json['useServerCtime'] as bool? ?? false,
);
Map<String, dynamic> _$HttpConfigToJson(HttpConfig instance) =>
<String, dynamic>{
'userAgent': instance.userAgent,
'connections': instance.connections,
'useServerCtime': instance.useServerCtime,
};
BtConfig _$BtConfigFromJson(Map<String, dynamic> json) => BtConfig()
@@ -67,12 +72,13 @@ Map<String, dynamic> _$ExtraConfigToJson(ExtraConfig instance) =>
'bt': instance.bt.toJson(),
};
ProxyConfig _$ProxyConfigFromJson(Map<String, dynamic> json) => ProxyConfig()
..enable = json['enable'] as bool
..scheme = json['scheme'] as String
..host = json['host'] as String
..usr = json['usr'] as String
..pwd = json['pwd'] as String;
ProxyConfig _$ProxyConfigFromJson(Map<String, dynamic> json) => ProxyConfig(
enable: json['enable'] as bool? ?? false,
scheme: json['scheme'] as String? ?? '',
host: json['host'] as String? ?? '',
usr: json['usr'] as String? ?? '',
pwd: json['pwd'] as String? ?? '',
);
Map<String, dynamic> _$ProxyConfigToJson(ProxyConfig instance) =>
<String, dynamic>{
@@ -84,24 +90,30 @@ Map<String, dynamic> _$ProxyConfigToJson(ProxyConfig instance) =>
};
ExtraConfigBt _$ExtraConfigBtFromJson(Map<String, dynamic> json) =>
ExtraConfigBt()
..trackerSubscribeUrls = (json['trackerSubscribeUrls'] as List<dynamic>)
.map((e) => e as String)
.toList()
..subscribeTrackers = (json['subscribeTrackers'] as List<dynamic>)
.map((e) => e as String)
.toList()
..lastTrackerUpdateTime = json['lastTrackerUpdateTime'] == null
ExtraConfigBt(
trackerSubscribeUrls: (json['trackerSubscribeUrls'] as List<dynamic>?)
?.map((e) => e as String)
.toList() ??
const [],
subscribeTrackers: (json['subscribeTrackers'] as List<dynamic>?)
?.map((e) => e as String)
.toList() ??
const [],
autoUpdateTrackers: json['autoUpdateTrackers'] as bool? ?? true,
lastTrackerUpdateTime: json['lastTrackerUpdateTime'] == null
? null
: DateTime.parse(json['lastTrackerUpdateTime'] as String)
..customTrackers = (json['customTrackers'] as List<dynamic>)
.map((e) => e as String)
.toList();
: DateTime.parse(json['lastTrackerUpdateTime'] as String),
customTrackers: (json['customTrackers'] as List<dynamic>?)
?.map((e) => e as String)
.toList() ??
const [],
);
Map<String, dynamic> _$ExtraConfigBtToJson(ExtraConfigBt instance) {
final val = <String, dynamic>{
'trackerSubscribeUrls': instance.trackerSubscribeUrls,
'subscribeTrackers': instance.subscribeTrackers,
'autoUpdateTrackers': instance.autoUpdateTrackers,
};
void writeNotNull(String key, dynamic value) {
@@ -95,6 +95,9 @@ class AppController extends GetxController with WindowListener, TrayListener {
_initForegroundTask().onError((error, stackTrace) =>
logger.w("initForegroundTask error", error, stackTrace));
_initTrackerUpdate().onError((error, stackTrace) =>
logger.w("initTrackerUpdate error", error, stackTrace));
}
@override
@@ -357,12 +360,13 @@ class AppController extends GetxController with WindowListener, TrayListener {
btConfig.trackers.toSet().toList();
}
Future<void> trackerUpdateOnStart() async {
Future<void> _initTrackerUpdate() async {
final btExtConfig = downloaderConfig.value.extra.bt;
final lastUpdateTime = btExtConfig.lastTrackerUpdateTime;
// if last update time is null or more than 1 day, update trackers
if (lastUpdateTime == null ||
lastUpdateTime.difference(DateTime.now()).inDays < 0) {
if (btExtConfig.autoUpdateTrackers &&
(lastUpdateTime == null ||
lastUpdateTime.difference(DateTime.now()).inDays < 0)) {
try {
await trackerUpdate();
} catch (e) {
@@ -386,9 +390,6 @@ class AppController extends GetxController with WindowListener, TrayListener {
_initDownloaderConfig() async {
final config = downloaderConfig.value;
if (config.protocolConfig.http.connections == 0) {
config.protocolConfig.http.connections = 16;
}
final extra = config.extra;
if (extra.themeMode.isEmpty) {
extra.themeMode = ThemeMode.dark.name;
@@ -138,6 +138,22 @@ class SettingView extends GetView<SettingController> {
],
);
});
final buildHttpUseServerCtime = _buildConfigItem(
'useServerCtime'.tr, () => httpConfig.useServerCtime.toString(),
(Key key) {
return Container(
alignment: Alignment.centerLeft,
child: Switch(
value: httpConfig.useServerCtime,
onChanged: (bool value) {
downloaderCfg.update((val) {
val!.protocolConfig.http.useServerCtime = value;
});
debounceSave();
},
),
);
});
// bt config items start
final btConfig = downloaderCfg.value.protocolConfig.bt;
@@ -207,8 +223,13 @@ class SettingView extends GetView<SettingController> {
Expanded(
child: SwitchListTile(
controlAffinity: ListTileControlAffinity.leading,
value: true,
onChanged: (bool value) {},
value: btExtConfig.autoUpdateTrackers,
onChanged: (bool value) {
downloaderCfg.update((val) {
val!.extra.bt.autoUpdateTrackers = value;
});
debounceSave();
},
title: Text('updateDaily'.tr)),
),
],
@@ -676,6 +697,7 @@ class SettingView extends GetView<SettingController> {
children: _addDivider([
buildHttpUa(),
buildHttpConnections(),
buildHttpUseServerCtime(),
]),
)),
const Text('BitTorrent'),
+3
View File
@@ -5,6 +5,8 @@ const enUS = {
'tip': 'Tip',
'confirm': 'Confirm',
'cancel': 'Cancel',
'on': 'On',
'off': 'Off',
'selectAll': 'Select All',
'task': 'Tasks',
'downloading': 'downloading',
@@ -31,6 +33,7 @@ const enUS = {
'downloadDir': 'Download Directory',
'downloadDirValid': 'Please select the download directory',
'connections': 'Connections',
'useServerCtime': 'Use server time for file creation',
'maxRunning': 'Max Running Tasks',
'items': '@count items',
'subscribeTracker': 'Subscribe Tracker',
+3
View File
@@ -5,6 +5,8 @@ const zhCN = {
'tip': '提示',
'confirm': '确认',
'cancel': '取消',
'on': '开启',
'off': '关闭',
'selectAll': '全选',
'task': '任务',
'downloading': '下载中',
@@ -30,6 +32,7 @@ const zhCN = {
'downloadDir': '下载目录',
'downloadDirValid': '请选择下载目录',
'connections': '连接数',
'useServerCtime': '创建文件使用服务器时间',
'maxRunning': '最大下载数',
'items': '@count 项',
'subscribeTracker': '订阅 Tracker',
+3
View File
@@ -5,6 +5,8 @@ const zhTW = {
'tip': '提示',
'confirm': '確定',
'cancel': '取消',
'on': '開啟',
'off': '關閉',
'selectAll': '全選',
'task': '任務',
'downloading': '下載中',
@@ -30,6 +32,7 @@ const zhTW = {
'downloadDir': '下載目録',
'downloadDirValid': '請選擇下載目録',
'connections': '同時連接數',
'useServerCtime': '使用伺服器時間作為檔案建立時間',
'maxRunning': '最大同時下載數',
'items': '@count 項',
'subscribeTracker': '已訂閲的追蹤者列表',
-3
View File
@@ -73,9 +73,6 @@ Future<void> init() async {
}
Future<void> onStart() async {
final appController = Get.find<AppController>();
await appController.trackerUpdateOnStart();
// if is debug mode, check language message is complete,change debug locale to your comfortable language if you want
if (kDebugMode) {
final debugLang = getLocaleKey(debugLocale);