mirror of
https://github.com/GopeedLab/gopeed.git
synced 2026-11-18 22:33:42 +00:00
feat(download): auto-categorize downloads into folders by file extension (#1572)
This commit is contained in:
1 parent
d125d4ab75
commit
9aae13453a
55 files changed
+4403
-217
No files matched your search
@@ -0,0 +1,186 @@
|
||||
package download
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"path"
|
||||
"strings"
|
||||
"unicode"
|
||||
|
||||
"github.com/GopeedLab/gopeed/pkg/base"
|
||||
)
|
||||
|
||||
// categoryForResource returns the category that receives the resolved
|
||||
// resource, or nil when auto categorize does not apply. Only a single file
|
||||
// resource is routed: a folder or a multi file resource keeps its directory
|
||||
// layout, so its files are never matched one by one, not even when only a
|
||||
// single one of them is selected.
|
||||
func categoryForResource(cfg *base.DownloaderStoreConfig, res *base.Resource) *base.DownloadCategory {
|
||||
if !cfg.AutoCategorizeEnabled() || len(cfg.Categories) == 0 {
|
||||
return nil
|
||||
}
|
||||
if res == nil || res.Name != "" || len(res.Files) != 1 {
|
||||
return nil
|
||||
}
|
||||
return matchDownloadCategory(cfg, res.Files[0].Name)
|
||||
}
|
||||
|
||||
// decideDownloadDir returns the download directory of a task. The base
|
||||
// directory is the one the task asked for, or the configured download
|
||||
// directory when the task left it empty. A task that did not skip auto
|
||||
// categorize is routed to the category directory of its resolved resource
|
||||
// when one matches. Path placeholders are expanded and the white list check
|
||||
// runs on the result, so a category directory the white list rejects falls
|
||||
// back to the base directory of this task instead of failing it.
|
||||
func (d *Downloader) decideDownloadDir(opts *base.Options, res *base.Resource) (string, error) {
|
||||
if opts == nil {
|
||||
opts = &base.Options{}
|
||||
}
|
||||
cfg, err := d.GetConfig()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
base := dartTrimSpace(opts.Path)
|
||||
if base == "" {
|
||||
base = cfg.DownloadDir
|
||||
}
|
||||
// Refuse to fall through to the process working directory when neither the
|
||||
// task nor the config names a directory.
|
||||
if base == "" {
|
||||
return "", errors.New("download directory is empty")
|
||||
}
|
||||
target := base
|
||||
if !opts.SkipAutoCategorize {
|
||||
if category := categoryForResource(cfg, res); category != nil {
|
||||
target = dartTrimSpace(category.Path)
|
||||
}
|
||||
}
|
||||
dir, err := d.initDownloadPath(target)
|
||||
if err != nil && target != base {
|
||||
d.Logger.Warn().Err(err).Msgf("category directory %q rejected by white list, falling back to %q", target, base)
|
||||
dir, err = d.initDownloadPath(base)
|
||||
}
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return dir, nil
|
||||
}
|
||||
|
||||
// suggestedDownloadDir returns the category directory a task would be routed
|
||||
// to, or an empty string when auto categorize does not apply. It only reports
|
||||
// the decision of decideDownloadDir, it expands no directory and writes
|
||||
// nothing.
|
||||
func (d *Downloader) suggestedDownloadDir(res *base.Resource) string {
|
||||
cfg, err := d.GetConfig()
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
category := categoryForResource(cfg, res)
|
||||
if category == nil {
|
||||
return ""
|
||||
}
|
||||
dir, err := d.initDownloadPath(dartTrimSpace(category.Path))
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return dir
|
||||
}
|
||||
|
||||
// asciiLower lowercases the ascii letters of s and leaves the rest alone. The
|
||||
// ui lowercases the same way, because the unicode case tables of the two
|
||||
// languages disagree for some code points and would let the two sides pick a
|
||||
// different category for the same file.
|
||||
func asciiLower(s string) string {
|
||||
hasUpper := false
|
||||
for i := 0; i < len(s); i++ {
|
||||
if c := s[i]; c >= 'A' && c <= 'Z' {
|
||||
hasUpper = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !hasUpper {
|
||||
return s
|
||||
}
|
||||
b := []byte(s)
|
||||
for i, c := range b {
|
||||
if c >= 'A' && c <= 'Z' {
|
||||
b[i] = c + ('a' - 'A')
|
||||
}
|
||||
}
|
||||
return string(b)
|
||||
}
|
||||
|
||||
// matchDownloadCategory returns the first category whose extension list
|
||||
// contains the extension of the given file name, or nil when nothing matches.
|
||||
// An empty extension list never matches.
|
||||
func matchDownloadCategory(cfg *base.DownloaderStoreConfig, name string) *base.DownloadCategory {
|
||||
ext := fileExtension(name)
|
||||
if ext == "" {
|
||||
return nil
|
||||
}
|
||||
for _, category := range cfg.Categories {
|
||||
if category == nil || category.IsDeleted || dartTrimSpace(category.Path) == "" {
|
||||
continue
|
||||
}
|
||||
for _, e := range normalizeCategoryExtensions(category.Extensions) {
|
||||
if e == ext {
|
||||
return category
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// normalizeCategoryExtensions mirrors the dart side: entries are split on
|
||||
// commas and whitespace, trimmed, lowercased, stripped of leading dots and
|
||||
// deduplicated while keeping the input order, so a stored value like
|
||||
// "exe, msi" behaves the same in the backend as it does in the ui.
|
||||
func normalizeCategoryExtensions(raw []string) []string {
|
||||
var result []string
|
||||
seen := make(map[string]struct{}, len(raw))
|
||||
for _, item := range raw {
|
||||
for _, part := range strings.FieldsFunc(item, func(r rune) bool {
|
||||
return r == ',' || isECMAScriptSpace(r)
|
||||
}) {
|
||||
ext := asciiLower(strings.TrimLeft(dartTrimSpace(part), "."))
|
||||
if ext == "" {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[ext]; ok {
|
||||
continue
|
||||
}
|
||||
seen[ext] = struct{}{}
|
||||
result = append(result, ext)
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// fileExtension returns the lower case extension of a file name without the
|
||||
// leading dot, e.g. "setup.exe" -> "exe". Windows style separators are
|
||||
// treated like the ui does, so a dot inside a directory segment never counts
|
||||
// as an extension.
|
||||
func fileExtension(name string) string {
|
||||
name = strings.ReplaceAll(name, `\`, "/")
|
||||
return strings.TrimPrefix(asciiLower(path.Ext(name)), ".")
|
||||
}
|
||||
|
||||
// dartTrimSpace trims the whitespace the dart String.trim() removes, which is
|
||||
// the unicode white space set plus the byte order mark. The ui trims the
|
||||
// category paths and the extension lists with it, so the backend uses the same
|
||||
// set instead of the narrower unicode.IsSpace.
|
||||
func dartTrimSpace(s string) string {
|
||||
return strings.TrimFunc(s, func(r rune) bool {
|
||||
return unicode.IsSpace(r) || r == 0xFEFF
|
||||
})
|
||||
}
|
||||
|
||||
// isECMAScriptSpace reports whether r belongs to the whitespace class of the
|
||||
// ECMAScript regular expressions. The ui splits its extension lists with that
|
||||
// class, so the backend uses the same one instead of the wider unicode.IsSpace.
|
||||
func isECMAScriptSpace(r rune) bool {
|
||||
switch r {
|
||||
case '\t', '\n', '\v', '\f', '\r', ' ', 0x00A0, 0x1680, 0x2028, 0x2029, 0x202F, 0x205F, 0x3000, 0xFEFF:
|
||||
return true
|
||||
}
|
||||
return r >= 0x2000 && r <= 0x200A
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
+126
-46
@@ -191,6 +191,15 @@ func (d *Downloader) Setup() error {
|
||||
FirstLoad: true,
|
||||
}
|
||||
}
|
||||
// Move the legacy extra.downloadCategories list written by v2.0.0-beta
|
||||
// builds into the top level categories field and persist the cleared key,
|
||||
// before Init seeds the built-in categories. The config is written
|
||||
// directly, PutConfig would validate a config that is not initialized yet.
|
||||
if d.cfg.DownloaderStoreConfig.MigrateLegacyExtraCategories() {
|
||||
if err := d.storage.Put(bucketConfig, "config", d.cfg.DownloaderStoreConfig); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
// Initialize the current backend configuration schema and defaults.
|
||||
if err := d.initProtocolConfigs(); err != nil {
|
||||
return err
|
||||
@@ -537,7 +546,8 @@ func (d *Downloader) Resolve(req *base.Request) (rr *ResolveResult, err error) {
|
||||
}
|
||||
if res != nil && len(res.Files) > 0 {
|
||||
rr = &ResolveResult{
|
||||
Res: res,
|
||||
Res: res,
|
||||
SuggestedDownloadDir: d.suggestedDownloadDir(res),
|
||||
}
|
||||
return
|
||||
}
|
||||
@@ -554,8 +564,9 @@ func (d *Downloader) Resolve(req *base.Request) (rr *ResolveResult, err error) {
|
||||
d.fetcherCache[rrId] = fetcher
|
||||
d.fetcherMapLock.Unlock()
|
||||
rr = &ResolveResult{
|
||||
ID: rrId,
|
||||
Res: fetcher.Meta().Res,
|
||||
ID: rrId,
|
||||
Res: fetcher.Meta().Res,
|
||||
SuggestedDownloadDir: d.suggestedDownloadDir(fetcher.Meta().Res),
|
||||
}
|
||||
return
|
||||
}
|
||||
@@ -654,26 +665,32 @@ func (d *Downloader) CreateWithOptions(rrId string, opts *base.Options) (taskId
|
||||
if opts == nil {
|
||||
opts = &base.Options{}
|
||||
}
|
||||
opts, err = d.initOptions(opts.Clone(), fetcher.Meta().Req.URL)
|
||||
// The options of the caller decide the global default download directory,
|
||||
// the task options are prepared later, when the task starts.
|
||||
defaultPath, err := d.prepareDefaultPath(opts)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
taskOpts, err := d.initOptions(opts.Clone(), fetcher.Meta().Req.URL)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if res := fetcher.Meta().Res; res != nil {
|
||||
for _, index := range opts.SelectFiles {
|
||||
for _, index := range taskOpts.SelectFiles {
|
||||
if index < 0 || index >= len(res.Files) {
|
||||
return "", fmt.Errorf("invalid selected file index: %d", index)
|
||||
}
|
||||
}
|
||||
opts.InitSelectFiles(len(res.Files))
|
||||
taskOpts.InitSelectFiles(len(res.Files))
|
||||
}
|
||||
fetcher.Meta().Opts = opts
|
||||
fetcher.Meta().Opts = taskOpts
|
||||
if res := fetcher.Meta().Res; res != nil {
|
||||
res.CalcSize(fetcher.Meta().Opts.SelectFiles)
|
||||
}
|
||||
// Resolve has no creation options. Publish a submitted HTTP estimate now
|
||||
// while preserving any size already confirmed by the response.
|
||||
if opts.Size > 0 {
|
||||
if err := fetcher.Patch(nil, &base.Options{Size: opts.Size}); err != nil {
|
||||
if taskOpts.Size > 0 {
|
||||
if err := fetcher.Patch(nil, &base.Options{Size: taskOpts.Size}); err != nil {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
@@ -682,10 +699,6 @@ func (d *Downloader) CreateWithOptions(rrId string, opts *base.Options) (taskId
|
||||
delete(d.fetcherCache, rrId)
|
||||
d.fetcherMapLock.Unlock()
|
||||
}()
|
||||
defaultPath, err := d.prepareDefaultPath(fetcher.Meta().Opts)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
taskId, err = d.doCreate(fetcher, nil)
|
||||
if err == nil {
|
||||
d.persistDefaultPathBestEffort(defaultPath)
|
||||
@@ -1370,6 +1383,15 @@ func (d *Downloader) GetConfig() (*base.DownloaderStoreConfig, error) {
|
||||
return util.DeepClone(d.cfg.DownloaderStoreConfig), nil
|
||||
}
|
||||
|
||||
// IsFirstLoad reports whether Setup created the config instead of loading it
|
||||
// from storage. FirstLoad is not serialized, so the clone GetConfig returns
|
||||
// always reports false.
|
||||
func (d *Downloader) IsFirstLoad() bool {
|
||||
d.configLock.RLock()
|
||||
defer d.configLock.RUnlock()
|
||||
return d.cfg.FirstLoad
|
||||
}
|
||||
|
||||
func (d *Downloader) PutConfig(v *base.DownloaderStoreConfig) error {
|
||||
if v == nil {
|
||||
return errors.New("config is nil")
|
||||
@@ -1433,7 +1455,10 @@ func (d *Downloader) PutConfig(v *base.DownloaderStoreConfig) error {
|
||||
}
|
||||
|
||||
func (d *Downloader) prepareDefaultPath(opts *base.Options) (string, error) {
|
||||
if opts == nil || !opts.AsDefaultPath || strings.TrimSpace(opts.Path) == "" {
|
||||
// Only a directory the user named explicitly becomes the global default
|
||||
// download directory. A task that keeps the auto categorize decision owns
|
||||
// its directory, the backend must not turn it into the global default.
|
||||
if opts == nil || !opts.SkipAutoCategorize || !opts.AsDefaultPath || strings.TrimSpace(opts.Path) == "" {
|
||||
return "", nil
|
||||
}
|
||||
selectedPath := strings.TrimSpace(opts.Path)
|
||||
@@ -1780,6 +1805,9 @@ func (d *Downloader) doCreate(f fetcher.Fetcher, opts *base.Options) (taskId str
|
||||
task.Meta = f.Meta()
|
||||
task.Progress = &Progress{}
|
||||
_, task.Uploading = f.(fetcher.Uploader)
|
||||
// A new task still needs its first preparation, it decides the download
|
||||
// directory and the final name of the task.
|
||||
task.NeedsPreparation = true
|
||||
initTask(task)
|
||||
if err = d.syncBlobTaskLease(task); err != nil {
|
||||
return "", err
|
||||
@@ -1823,6 +1851,10 @@ func httpSizeOptions(opts *base.Options, protocol string) *base.Options {
|
||||
return opts
|
||||
}
|
||||
|
||||
// initOptions makes sure the option set is usable: a nil option set becomes
|
||||
// an empty one, the selection is never nil and the base download directory
|
||||
// falls back to the configured one. The directory is decided and checked
|
||||
// against the white list when the task starts.
|
||||
func (d *Downloader) initOptions(opts *base.Options, requestURL string) (*base.Options, error) {
|
||||
if opts == nil {
|
||||
opts = &base.Options{}
|
||||
@@ -1838,17 +1870,12 @@ func (d *Downloader) initOptions(opts *base.Options, requestURL string) (*base.O
|
||||
opts.SelectFiles = make([]int, 0)
|
||||
}
|
||||
if opts.Path == "" {
|
||||
storeConfig, err := d.GetConfig()
|
||||
cfg, err := d.GetConfig()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
opts.Path = storeConfig.DownloadDir
|
||||
opts.Path = cfg.DownloadDir
|
||||
}
|
||||
path, err := d.initDownloadPath(opts.Path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
opts.Path = path
|
||||
return opts, nil
|
||||
}
|
||||
|
||||
@@ -1879,8 +1906,68 @@ func (d *Downloader) statusMut(task *Task, fn func() (bool, error)) (bool, error
|
||||
return fn()
|
||||
}
|
||||
|
||||
// prepareTask runs the first preparation of a task: it decides the download
|
||||
// directory, resolves a duplicate name in it and persists both. The task keeps
|
||||
// its NeedsPreparation flag when the preparation fails, so a retry prepares it
|
||||
// again. The caller holds checkDuplicateLock when the fetcher manager renames
|
||||
// duplicates, the lock has to stay held until the fetcher created the file.
|
||||
func (d *Downloader) prepareTask(task *Task) error {
|
||||
if task.Meta.Res == nil {
|
||||
return errors.New("task resource is not resolved")
|
||||
}
|
||||
preparedMeta := *task.Meta
|
||||
preparedOpts := *task.Meta.Opts
|
||||
preparedMeta.Opts = &preparedOpts
|
||||
preparedMeta.Res = util.DeepClone(task.Meta.Res)
|
||||
dir, err := d.decideDownloadDir(preparedMeta.Opts, preparedMeta.Res)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
preparedMeta.Opts.Path = dir
|
||||
|
||||
if task.fetcherManager.AutoRename() {
|
||||
preparedMeta.Opts.Name = util.SafeFilename(preparedMeta.Opts.Name)
|
||||
// check if the download file is duplicated and rename it automatically.
|
||||
if preparedMeta.Res.Name != "" {
|
||||
preparedMeta.Res.Name = util.SafeFilename(preparedMeta.Res.Name)
|
||||
fullDirPath := preparedMeta.FolderPath()
|
||||
newName, err := util.CheckDuplicateAndRename(fullDirPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
preparedMeta.Opts.Name = newName
|
||||
} else {
|
||||
preparedMeta.Res.Files[0].Name = util.SafeFilename(preparedMeta.Res.Files[0].Name)
|
||||
fullFilePath := preparedMeta.SingleFilepath()
|
||||
newName, err := util.CheckDuplicateAndRename(fullFilePath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
preparedMeta.Opts.Name = newName
|
||||
}
|
||||
}
|
||||
|
||||
preparedMeta.Res.CalcSize(preparedMeta.Opts.SelectFiles)
|
||||
|
||||
// Save a candidate task before publishing its metadata. The fetcher keeps
|
||||
// its resolved protocol state, so a failed save cannot change the live
|
||||
// directory, name or preparation flag.
|
||||
task.statusLock.Lock()
|
||||
preparedTask := *task
|
||||
task.statusLock.Unlock()
|
||||
preparedTask.Meta = &preparedMeta
|
||||
preparedTask.NeedsPreparation = false
|
||||
if err := d.saveTask(&preparedTask); err != nil {
|
||||
return err
|
||||
}
|
||||
task.statusLock.Lock()
|
||||
*task.Meta = preparedMeta
|
||||
task.NeedsPreparation = false
|
||||
task.statusLock.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *Downloader) doStart(task *Task) (err error) {
|
||||
var isCreate bool
|
||||
var generation uint64
|
||||
isReturn, err := d.statusMut(task, func() (isReturn bool, err error) {
|
||||
if task.Status == base.DownloadStatusRunning || task.Status == base.DownloadStatusDone {
|
||||
@@ -1893,7 +1980,6 @@ func (d *Downloader) doStart(task *Task) (err error) {
|
||||
d.Logger.Error().Stack().Err(err).Msgf("restore fetcher failed, task id: %s", task.ID)
|
||||
return
|
||||
}
|
||||
isCreate = task.Status == base.DownloadStatusReady
|
||||
task.updateStatus(base.DownloadStatusRunning)
|
||||
task.runGeneration++
|
||||
generation = task.runGeneration
|
||||
@@ -1933,7 +2019,8 @@ func (d *Downloader) doStart(task *Task) (err error) {
|
||||
if err := d.syncBlobTaskLease(task); err != nil {
|
||||
return err
|
||||
}
|
||||
if task.Meta.Res == nil {
|
||||
resolvedHere := task.Meta.Res == nil
|
||||
if resolvedHere {
|
||||
err := task.fetcher.Resolve(task.Meta.Req)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -1941,32 +2028,25 @@ func (d *Downloader) doStart(task *Task) (err error) {
|
||||
task.Meta.Res = task.fetcher.Meta().Res
|
||||
}
|
||||
|
||||
if isCreate {
|
||||
if task.NeedsPreparation {
|
||||
if task.fetcherManager.AutoRename() {
|
||||
// The duplicate name is resolved from the files on disk, but
|
||||
// the file only appears when the fetcher starts, so the lock
|
||||
// has to cover the start as well.
|
||||
d.checkDuplicateLock.Lock()
|
||||
defer d.checkDuplicateLock.Unlock()
|
||||
task.Meta.Opts.Name = util.SafeFilename(task.Meta.Opts.Name)
|
||||
// check if the download file is duplicated and rename it automatically.
|
||||
if task.Meta.Res.Name != "" {
|
||||
task.Meta.Res.Name = util.SafeFilename(task.Meta.Res.Name)
|
||||
fullDirPath := task.Meta.FolderPath()
|
||||
newName, err := util.CheckDuplicateAndRename(fullDirPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
task.Meta.Opts.Name = newName
|
||||
} else {
|
||||
task.Meta.Res.Files[0].Name = util.SafeFilename(task.Meta.Res.Files[0].Name)
|
||||
fullFilePath := task.Meta.SingleFilepath()
|
||||
newName, err := util.CheckDuplicateAndRename(fullFilePath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
task.Meta.Opts.Name = newName
|
||||
}
|
||||
}
|
||||
|
||||
task.Meta.Res.CalcSize(task.Meta.Opts.SelectFiles)
|
||||
// The first preparation decides the download directory and the
|
||||
// final name of the task. It runs for every task that has not
|
||||
// been prepared yet, no matter whether the task waited in the
|
||||
// queue, was paused or failed before.
|
||||
if err := d.prepareTask(task); err != nil {
|
||||
return err
|
||||
}
|
||||
} else if _, err := d.initDownloadPath(task.Meta.Opts.Path); err != nil {
|
||||
// A prepared task keeps its directory, but the directory still has
|
||||
// to pass the white list on every start.
|
||||
return err
|
||||
}
|
||||
|
||||
task.statusLock.Lock()
|
||||
|
||||
@@ -841,6 +841,9 @@ func TestDownloader_CreateDirectUsesPathAsDefault(t *testing.T) {
|
||||
selectedRoot := t.TempDir()
|
||||
defaultPathOpts := newTestDownloadOptAt(selectedRoot)
|
||||
defaultPathOpts.AsDefaultPath = true
|
||||
// The global default download directory is only saved for a directory the
|
||||
// user named explicitly, so the task skips the auto categorize decision.
|
||||
defaultPathOpts.SkipAutoCategorize = true
|
||||
createAndWait(defaultPathOpts)
|
||||
cfg, err = downloader.GetConfig()
|
||||
if err != nil {
|
||||
@@ -882,11 +885,26 @@ func TestDownloader_CreateNotInWhite(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err = downloader.CreateWithOptions(rr.ID, newTestDownloadOpt(t))
|
||||
if err == nil {
|
||||
t.Fatal("Create should reject a directory outside the allowlist")
|
||||
// The directory is decided when the task starts, so a directory outside
|
||||
// the allowlist fails the task instead of the create call.
|
||||
taskId, err := downloader.CreateWithOptions(rr.ID, newTestDownloadOpt(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
task := downloader.GetTask(taskId)
|
||||
if task == nil {
|
||||
t.Fatal("task not found")
|
||||
}
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
if task.Status == base.DownloadStatusError {
|
||||
break
|
||||
}
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
}
|
||||
if task.Status != base.DownloadStatusError {
|
||||
t.Fatalf("task status = %q, want the task to be rejected by the allowlist", task.Status)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func TestDownloader_CreateDirectBatch(t *testing.T) {
|
||||
@@ -1142,6 +1160,33 @@ func TestDownloader_CreateRename(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloader_IsFirstLoadSurvivesTheConfigClone(t *testing.T) {
|
||||
d := NewDownloader(&DownloaderConfig{StorageDir: t.TempDir(), TempDir: t.TempDir()})
|
||||
if err := d.Setup(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer d.Close()
|
||||
|
||||
if !d.IsFirstLoad() {
|
||||
t.Fatal("a downloader that created its config reports IsFirstLoad() = false")
|
||||
}
|
||||
// GetConfig clones through json and FirstLoad is not serialized, so the cmd
|
||||
// layer cannot read the flag from the returned config.
|
||||
cfg, err := d.GetConfig()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if cfg.FirstLoad {
|
||||
t.Fatal("the cloned config must not report the first load")
|
||||
}
|
||||
if err := d.PutConfig(cfg); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if d.IsFirstLoad() {
|
||||
t.Fatal("a downloader that stored its config reports IsFirstLoad() = true")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloader_StoreAndRestore(t *testing.T) {
|
||||
listener := test.StartTestSlowFileServer(time.Millisecond * 2000)
|
||||
defer listener.Close()
|
||||
|
||||
@@ -21,6 +21,11 @@ import (
|
||||
type ResolveResult struct {
|
||||
ID string `json:"id"`
|
||||
Res *base.Resource `json:"res"`
|
||||
// SuggestedDownloadDir is the category directory the created task would be
|
||||
// routed to. It is only set when auto categorize is enabled, the resource is
|
||||
// a single file, a category matches and its directory passes the white list
|
||||
// check. It is a hint for the ui, the create call decides the final path.
|
||||
SuggestedDownloadDir string `json:"suggestedDownloadDir,omitempty"`
|
||||
}
|
||||
|
||||
// Task represents a download unit.
|
||||
@@ -39,6 +44,14 @@ type Task struct {
|
||||
// finished. A done task never leaves the done state, so the value is
|
||||
// written once and kept.
|
||||
DoneAt *time.Time `json:"doneAt"`
|
||||
// NeedsPreparation marks a task that has not run its first preparation
|
||||
// yet. The preparation decides the download directory of the task and
|
||||
// resolves the duplicate name in it. It runs once, after the resource is
|
||||
// resolved, and the flag is cleared afterwards, so a queued, paused or
|
||||
// retried task is still prepared while a prepared task is never
|
||||
// classified or renamed again. A task stored before this flag existed
|
||||
// does not have it and keeps its directory and name.
|
||||
NeedsPreparation bool `json:"needsPreparation,omitempty"`
|
||||
|
||||
producer production.Source
|
||||
producerReceived int64
|
||||
|
||||
@@ -0,0 +1,59 @@
|
||||
{
|
||||
"normalizeExtensions": [
|
||||
{
|
||||
"input": [
|
||||
"exe, msi",
|
||||
" ZIP ",
|
||||
".tar.gz",
|
||||
"",
|
||||
"exe"
|
||||
],
|
||||
"expected": [
|
||||
"exe",
|
||||
"msi",
|
||||
"zip",
|
||||
"tar.gz"
|
||||
]
|
||||
},
|
||||
{
|
||||
"input": [],
|
||||
"expected": []
|
||||
},
|
||||
{
|
||||
"input": [
|
||||
" "
|
||||
],
|
||||
"expected": []
|
||||
},
|
||||
{
|
||||
"input": [
|
||||
"video/mp4"
|
||||
],
|
||||
"expected": [
|
||||
"video/mp4"
|
||||
]
|
||||
},
|
||||
{
|
||||
"input": [
|
||||
".exe",
|
||||
"..msi"
|
||||
],
|
||||
"expected": [
|
||||
"exe",
|
||||
"msi"
|
||||
]
|
||||
},
|
||||
{
|
||||
"input": [
|
||||
"EXE",
|
||||
"ſ",
|
||||
"Ä"
|
||||
],
|
||||
"expected": [
|
||||
"exe",
|
||||
"ſ",
|
||||
"Ä"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
Reference in new issue
Block a user