feat(download): auto-categorize downloads into folders by file extension (#1572)

This commit is contained in:
WuFeng authored and GitHub committed 2026-10-23 20:38:25 +08:00
1 parent d125d4ab75
commit 9aae13453a
55 files changed
+4403 -217

No files matched your search

+186
View File
@@ -0,0 +1,186 @@
package download
import (
"errors"
"path"
"strings"
"unicode"
"github.com/GopeedLab/gopeed/pkg/base"
)
// categoryForResource returns the category that receives the resolved
// resource, or nil when auto categorize does not apply. Only a single file
// resource is routed: a folder or a multi file resource keeps its directory
// layout, so its files are never matched one by one, not even when only a
// single one of them is selected.
func categoryForResource(cfg *base.DownloaderStoreConfig, res *base.Resource) *base.DownloadCategory {
if !cfg.AutoCategorizeEnabled() || len(cfg.Categories) == 0 {
return nil
}
if res == nil || res.Name != "" || len(res.Files) != 1 {
return nil
}
return matchDownloadCategory(cfg, res.Files[0].Name)
}
// decideDownloadDir returns the download directory of a task. The base
// directory is the one the task asked for, or the configured download
// directory when the task left it empty. A task that did not skip auto
// categorize is routed to the category directory of its resolved resource
// when one matches. Path placeholders are expanded and the white list check
// runs on the result, so a category directory the white list rejects falls
// back to the base directory of this task instead of failing it.
func (d *Downloader) decideDownloadDir(opts *base.Options, res *base.Resource) (string, error) {
if opts == nil {
opts = &base.Options{}
}
cfg, err := d.GetConfig()
if err != nil {
return "", err
}
base := dartTrimSpace(opts.Path)
if base == "" {
base = cfg.DownloadDir
}
// Refuse to fall through to the process working directory when neither the
// task nor the config names a directory.
if base == "" {
return "", errors.New("download directory is empty")
}
target := base
if !opts.SkipAutoCategorize {
if category := categoryForResource(cfg, res); category != nil {
target = dartTrimSpace(category.Path)
}
}
dir, err := d.initDownloadPath(target)
if err != nil && target != base {
d.Logger.Warn().Err(err).Msgf("category directory %q rejected by white list, falling back to %q", target, base)
dir, err = d.initDownloadPath(base)
}
if err != nil {
return "", err
}
return dir, nil
}
// suggestedDownloadDir returns the category directory a task would be routed
// to, or an empty string when auto categorize does not apply. It only reports
// the decision of decideDownloadDir, it expands no directory and writes
// nothing.
func (d *Downloader) suggestedDownloadDir(res *base.Resource) string {
cfg, err := d.GetConfig()
if err != nil {
return ""
}
category := categoryForResource(cfg, res)
if category == nil {
return ""
}
dir, err := d.initDownloadPath(dartTrimSpace(category.Path))
if err != nil {
return ""
}
return dir
}
// asciiLower lowercases the ascii letters of s and leaves the rest alone. The
// ui lowercases the same way, because the unicode case tables of the two
// languages disagree for some code points and would let the two sides pick a
// different category for the same file.
func asciiLower(s string) string {
hasUpper := false
for i := 0; i < len(s); i++ {
if c := s[i]; c >= 'A' && c <= 'Z' {
hasUpper = true
break
}
}
if !hasUpper {
return s
}
b := []byte(s)
for i, c := range b {
if c >= 'A' && c <= 'Z' {
b[i] = c + ('a' - 'A')
}
}
return string(b)
}
// matchDownloadCategory returns the first category whose extension list
// contains the extension of the given file name, or nil when nothing matches.
// An empty extension list never matches.
func matchDownloadCategory(cfg *base.DownloaderStoreConfig, name string) *base.DownloadCategory {
ext := fileExtension(name)
if ext == "" {
return nil
}
for _, category := range cfg.Categories {
if category == nil || category.IsDeleted || dartTrimSpace(category.Path) == "" {
continue
}
for _, e := range normalizeCategoryExtensions(category.Extensions) {
if e == ext {
return category
}
}
}
return nil
}
// normalizeCategoryExtensions mirrors the dart side: entries are split on
// commas and whitespace, trimmed, lowercased, stripped of leading dots and
// deduplicated while keeping the input order, so a stored value like
// "exe, msi" behaves the same in the backend as it does in the ui.
func normalizeCategoryExtensions(raw []string) []string {
var result []string
seen := make(map[string]struct{}, len(raw))
for _, item := range raw {
for _, part := range strings.FieldsFunc(item, func(r rune) bool {
return r == ',' || isECMAScriptSpace(r)
}) {
ext := asciiLower(strings.TrimLeft(dartTrimSpace(part), "."))
if ext == "" {
continue
}
if _, ok := seen[ext]; ok {
continue
}
seen[ext] = struct{}{}
result = append(result, ext)
}
}
return result
}
// fileExtension returns the lower case extension of a file name without the
// leading dot, e.g. "setup.exe" -> "exe". Windows style separators are
// treated like the ui does, so a dot inside a directory segment never counts
// as an extension.
func fileExtension(name string) string {
name = strings.ReplaceAll(name, `\`, "/")
return strings.TrimPrefix(asciiLower(path.Ext(name)), ".")
}
// dartTrimSpace trims the whitespace the dart String.trim() removes, which is
// the unicode white space set plus the byte order mark. The ui trims the
// category paths and the extension lists with it, so the backend uses the same
// set instead of the narrower unicode.IsSpace.
func dartTrimSpace(s string) string {
return strings.TrimFunc(s, func(r rune) bool {
return unicode.IsSpace(r) || r == 0xFEFF
})
}
// isECMAScriptSpace reports whether r belongs to the whitespace class of the
// ECMAScript regular expressions. The ui splits its extension lists with that
// class, so the backend uses the same one instead of the wider unicode.IsSpace.
func isECMAScriptSpace(r rune) bool {
switch r {
case '\t', '\n', '\v', '\f', '\r', ' ', 0x00A0, 0x1680, 0x2028, 0x2029, 0x202F, 0x205F, 0x3000, 0xFEFF:
return true
}
return r >= 0x2000 && r <= 0x200A
}
File diff suppressed because it is too large. Load diff
File diff suppressed because it is too large. Load diff
+126 -46
View File
@@ -191,6 +191,15 @@ func (d *Downloader) Setup() error {
FirstLoad: true,
}
}
// Move the legacy extra.downloadCategories list written by v2.0.0-beta
// builds into the top level categories field and persist the cleared key,
// before Init seeds the built-in categories. The config is written
// directly, PutConfig would validate a config that is not initialized yet.
if d.cfg.DownloaderStoreConfig.MigrateLegacyExtraCategories() {
if err := d.storage.Put(bucketConfig, "config", d.cfg.DownloaderStoreConfig); err != nil {
return err
}
}
// Initialize the current backend configuration schema and defaults.
if err := d.initProtocolConfigs(); err != nil {
return err
@@ -537,7 +546,8 @@ func (d *Downloader) Resolve(req *base.Request) (rr *ResolveResult, err error) {
}
if res != nil && len(res.Files) > 0 {
rr = &ResolveResult{
Res: res,
Res: res,
SuggestedDownloadDir: d.suggestedDownloadDir(res),
}
return
}
@@ -554,8 +564,9 @@ func (d *Downloader) Resolve(req *base.Request) (rr *ResolveResult, err error) {
d.fetcherCache[rrId] = fetcher
d.fetcherMapLock.Unlock()
rr = &ResolveResult{
ID: rrId,
Res: fetcher.Meta().Res,
ID: rrId,
Res: fetcher.Meta().Res,
SuggestedDownloadDir: d.suggestedDownloadDir(fetcher.Meta().Res),
}
return
}
@@ -654,26 +665,32 @@ func (d *Downloader) CreateWithOptions(rrId string, opts *base.Options) (taskId
if opts == nil {
opts = &base.Options{}
}
opts, err = d.initOptions(opts.Clone(), fetcher.Meta().Req.URL)
// The options of the caller decide the global default download directory,
// the task options are prepared later, when the task starts.
defaultPath, err := d.prepareDefaultPath(opts)
if err != nil {
return "", err
}
taskOpts, err := d.initOptions(opts.Clone(), fetcher.Meta().Req.URL)
if err != nil {
return "", err
}
if res := fetcher.Meta().Res; res != nil {
for _, index := range opts.SelectFiles {
for _, index := range taskOpts.SelectFiles {
if index < 0 || index >= len(res.Files) {
return "", fmt.Errorf("invalid selected file index: %d", index)
}
}
opts.InitSelectFiles(len(res.Files))
taskOpts.InitSelectFiles(len(res.Files))
}
fetcher.Meta().Opts = opts
fetcher.Meta().Opts = taskOpts
if res := fetcher.Meta().Res; res != nil {
res.CalcSize(fetcher.Meta().Opts.SelectFiles)
}
// Resolve has no creation options. Publish a submitted HTTP estimate now
// while preserving any size already confirmed by the response.
if opts.Size > 0 {
if err := fetcher.Patch(nil, &base.Options{Size: opts.Size}); err != nil {
if taskOpts.Size > 0 {
if err := fetcher.Patch(nil, &base.Options{Size: taskOpts.Size}); err != nil {
return "", err
}
}
@@ -682,10 +699,6 @@ func (d *Downloader) CreateWithOptions(rrId string, opts *base.Options) (taskId
delete(d.fetcherCache, rrId)
d.fetcherMapLock.Unlock()
}()
defaultPath, err := d.prepareDefaultPath(fetcher.Meta().Opts)
if err != nil {
return "", err
}
taskId, err = d.doCreate(fetcher, nil)
if err == nil {
d.persistDefaultPathBestEffort(defaultPath)
@@ -1370,6 +1383,15 @@ func (d *Downloader) GetConfig() (*base.DownloaderStoreConfig, error) {
return util.DeepClone(d.cfg.DownloaderStoreConfig), nil
}
// IsFirstLoad reports whether Setup created the config instead of loading it
// from storage. FirstLoad is not serialized, so the clone GetConfig returns
// always reports false.
func (d *Downloader) IsFirstLoad() bool {
d.configLock.RLock()
defer d.configLock.RUnlock()
return d.cfg.FirstLoad
}
func (d *Downloader) PutConfig(v *base.DownloaderStoreConfig) error {
if v == nil {
return errors.New("config is nil")
@@ -1433,7 +1455,10 @@ func (d *Downloader) PutConfig(v *base.DownloaderStoreConfig) error {
}
func (d *Downloader) prepareDefaultPath(opts *base.Options) (string, error) {
if opts == nil || !opts.AsDefaultPath || strings.TrimSpace(opts.Path) == "" {
// Only a directory the user named explicitly becomes the global default
// download directory. A task that keeps the auto categorize decision owns
// its directory, the backend must not turn it into the global default.
if opts == nil || !opts.SkipAutoCategorize || !opts.AsDefaultPath || strings.TrimSpace(opts.Path) == "" {
return "", nil
}
selectedPath := strings.TrimSpace(opts.Path)
@@ -1780,6 +1805,9 @@ func (d *Downloader) doCreate(f fetcher.Fetcher, opts *base.Options) (taskId str
task.Meta = f.Meta()
task.Progress = &Progress{}
_, task.Uploading = f.(fetcher.Uploader)
// A new task still needs its first preparation, it decides the download
// directory and the final name of the task.
task.NeedsPreparation = true
initTask(task)
if err = d.syncBlobTaskLease(task); err != nil {
return "", err
@@ -1823,6 +1851,10 @@ func httpSizeOptions(opts *base.Options, protocol string) *base.Options {
return opts
}
// initOptions makes sure the option set is usable: a nil option set becomes
// an empty one, the selection is never nil and the base download directory
// falls back to the configured one. The directory is decided and checked
// against the white list when the task starts.
func (d *Downloader) initOptions(opts *base.Options, requestURL string) (*base.Options, error) {
if opts == nil {
opts = &base.Options{}
@@ -1838,17 +1870,12 @@ func (d *Downloader) initOptions(opts *base.Options, requestURL string) (*base.O
opts.SelectFiles = make([]int, 0)
}
if opts.Path == "" {
storeConfig, err := d.GetConfig()
cfg, err := d.GetConfig()
if err != nil {
return nil, err
}
opts.Path = storeConfig.DownloadDir
opts.Path = cfg.DownloadDir
}
path, err := d.initDownloadPath(opts.Path)
if err != nil {
return nil, err
}
opts.Path = path
return opts, nil
}
@@ -1879,8 +1906,68 @@ func (d *Downloader) statusMut(task *Task, fn func() (bool, error)) (bool, error
return fn()
}
// prepareTask runs the first preparation of a task: it decides the download
// directory, resolves a duplicate name in it and persists both. The task keeps
// its NeedsPreparation flag when the preparation fails, so a retry prepares it
// again. The caller holds checkDuplicateLock when the fetcher manager renames
// duplicates, the lock has to stay held until the fetcher created the file.
func (d *Downloader) prepareTask(task *Task) error {
if task.Meta.Res == nil {
return errors.New("task resource is not resolved")
}
preparedMeta := *task.Meta
preparedOpts := *task.Meta.Opts
preparedMeta.Opts = &preparedOpts
preparedMeta.Res = util.DeepClone(task.Meta.Res)
dir, err := d.decideDownloadDir(preparedMeta.Opts, preparedMeta.Res)
if err != nil {
return err
}
preparedMeta.Opts.Path = dir
if task.fetcherManager.AutoRename() {
preparedMeta.Opts.Name = util.SafeFilename(preparedMeta.Opts.Name)
// check if the download file is duplicated and rename it automatically.
if preparedMeta.Res.Name != "" {
preparedMeta.Res.Name = util.SafeFilename(preparedMeta.Res.Name)
fullDirPath := preparedMeta.FolderPath()
newName, err := util.CheckDuplicateAndRename(fullDirPath)
if err != nil {
return err
}
preparedMeta.Opts.Name = newName
} else {
preparedMeta.Res.Files[0].Name = util.SafeFilename(preparedMeta.Res.Files[0].Name)
fullFilePath := preparedMeta.SingleFilepath()
newName, err := util.CheckDuplicateAndRename(fullFilePath)
if err != nil {
return err
}
preparedMeta.Opts.Name = newName
}
}
preparedMeta.Res.CalcSize(preparedMeta.Opts.SelectFiles)
// Save a candidate task before publishing its metadata. The fetcher keeps
// its resolved protocol state, so a failed save cannot change the live
// directory, name or preparation flag.
task.statusLock.Lock()
preparedTask := *task
task.statusLock.Unlock()
preparedTask.Meta = &preparedMeta
preparedTask.NeedsPreparation = false
if err := d.saveTask(&preparedTask); err != nil {
return err
}
task.statusLock.Lock()
*task.Meta = preparedMeta
task.NeedsPreparation = false
task.statusLock.Unlock()
return nil
}
func (d *Downloader) doStart(task *Task) (err error) {
var isCreate bool
var generation uint64
isReturn, err := d.statusMut(task, func() (isReturn bool, err error) {
if task.Status == base.DownloadStatusRunning || task.Status == base.DownloadStatusDone {
@@ -1893,7 +1980,6 @@ func (d *Downloader) doStart(task *Task) (err error) {
d.Logger.Error().Stack().Err(err).Msgf("restore fetcher failed, task id: %s", task.ID)
return
}
isCreate = task.Status == base.DownloadStatusReady
task.updateStatus(base.DownloadStatusRunning)
task.runGeneration++
generation = task.runGeneration
@@ -1933,7 +2019,8 @@ func (d *Downloader) doStart(task *Task) (err error) {
if err := d.syncBlobTaskLease(task); err != nil {
return err
}
if task.Meta.Res == nil {
resolvedHere := task.Meta.Res == nil
if resolvedHere {
err := task.fetcher.Resolve(task.Meta.Req)
if err != nil {
return err
@@ -1941,32 +2028,25 @@ func (d *Downloader) doStart(task *Task) (err error) {
task.Meta.Res = task.fetcher.Meta().Res
}
if isCreate {
if task.NeedsPreparation {
if task.fetcherManager.AutoRename() {
// The duplicate name is resolved from the files on disk, but
// the file only appears when the fetcher starts, so the lock
// has to cover the start as well.
d.checkDuplicateLock.Lock()
defer d.checkDuplicateLock.Unlock()
task.Meta.Opts.Name = util.SafeFilename(task.Meta.Opts.Name)
// check if the download file is duplicated and rename it automatically.
if task.Meta.Res.Name != "" {
task.Meta.Res.Name = util.SafeFilename(task.Meta.Res.Name)
fullDirPath := task.Meta.FolderPath()
newName, err := util.CheckDuplicateAndRename(fullDirPath)
if err != nil {
return err
}
task.Meta.Opts.Name = newName
} else {
task.Meta.Res.Files[0].Name = util.SafeFilename(task.Meta.Res.Files[0].Name)
fullFilePath := task.Meta.SingleFilepath()
newName, err := util.CheckDuplicateAndRename(fullFilePath)
if err != nil {
return err
}
task.Meta.Opts.Name = newName
}
}
task.Meta.Res.CalcSize(task.Meta.Opts.SelectFiles)
// The first preparation decides the download directory and the
// final name of the task. It runs for every task that has not
// been prepared yet, no matter whether the task waited in the
// queue, was paused or failed before.
if err := d.prepareTask(task); err != nil {
return err
}
} else if _, err := d.initDownloadPath(task.Meta.Opts.Path); err != nil {
// A prepared task keeps its directory, but the directory still has
// to pass the white list on every start.
return err
}
task.statusLock.Lock()
+49 -4
View File
@@ -841,6 +841,9 @@ func TestDownloader_CreateDirectUsesPathAsDefault(t *testing.T) {
selectedRoot := t.TempDir()
defaultPathOpts := newTestDownloadOptAt(selectedRoot)
defaultPathOpts.AsDefaultPath = true
// The global default download directory is only saved for a directory the
// user named explicitly, so the task skips the auto categorize decision.
defaultPathOpts.SkipAutoCategorize = true
createAndWait(defaultPathOpts)
cfg, err = downloader.GetConfig()
if err != nil {
@@ -882,11 +885,26 @@ func TestDownloader_CreateNotInWhite(t *testing.T) {
if err != nil {
t.Fatal(err)
}
_, err = downloader.CreateWithOptions(rr.ID, newTestDownloadOpt(t))
if err == nil {
t.Fatal("Create should reject a directory outside the allowlist")
// The directory is decided when the task starts, so a directory outside
// the allowlist fails the task instead of the create call.
taskId, err := downloader.CreateWithOptions(rr.ID, newTestDownloadOpt(t))
if err != nil {
t.Fatal(err)
}
task := downloader.GetTask(taskId)
if task == nil {
t.Fatal("task not found")
}
deadline := time.Now().Add(5 * time.Second)
for time.Now().Before(deadline) {
if task.Status == base.DownloadStatusError {
break
}
time.Sleep(20 * time.Millisecond)
}
if task.Status != base.DownloadStatusError {
t.Fatalf("task status = %q, want the task to be rejected by the allowlist", task.Status)
}
}
func TestDownloader_CreateDirectBatch(t *testing.T) {
@@ -1142,6 +1160,33 @@ func TestDownloader_CreateRename(t *testing.T) {
}
}
func TestDownloader_IsFirstLoadSurvivesTheConfigClone(t *testing.T) {
d := NewDownloader(&DownloaderConfig{StorageDir: t.TempDir(), TempDir: t.TempDir()})
if err := d.Setup(); err != nil {
t.Fatal(err)
}
defer d.Close()
if !d.IsFirstLoad() {
t.Fatal("a downloader that created its config reports IsFirstLoad() = false")
}
// GetConfig clones through json and FirstLoad is not serialized, so the cmd
// layer cannot read the flag from the returned config.
cfg, err := d.GetConfig()
if err != nil {
t.Fatal(err)
}
if cfg.FirstLoad {
t.Fatal("the cloned config must not report the first load")
}
if err := d.PutConfig(cfg); err != nil {
t.Fatal(err)
}
if d.IsFirstLoad() {
t.Fatal("a downloader that stored its config reports IsFirstLoad() = true")
}
}
func TestDownloader_StoreAndRestore(t *testing.T) {
listener := test.StartTestSlowFileServer(time.Millisecond * 2000)
defer listener.Close()
+13
View File
@@ -21,6 +21,11 @@ import (
type ResolveResult struct {
ID string `json:"id"`
Res *base.Resource `json:"res"`
// SuggestedDownloadDir is the category directory the created task would be
// routed to. It is only set when auto categorize is enabled, the resource is
// a single file, a category matches and its directory passes the white list
// check. It is a hint for the ui, the create call decides the final path.
SuggestedDownloadDir string `json:"suggestedDownloadDir,omitempty"`
}
// Task represents a download unit.
@@ -39,6 +44,14 @@ type Task struct {
// finished. A done task never leaves the done state, so the value is
// written once and kept.
DoneAt *time.Time `json:"doneAt"`
// NeedsPreparation marks a task that has not run its first preparation
// yet. The preparation decides the download directory of the task and
// resolves the duplicate name in it. It runs once, after the resource is
// resolved, and the flag is cleared afterwards, so a queued, paused or
// retried task is still prepared while a prepared task is never
// classified or renamed again. A task stored before this flag existed
// does not have it and keeps its directory and name.
NeedsPreparation bool `json:"needsPreparation,omitempty"`
producer production.Source
producerReceived int64
+59
View File
@@ -0,0 +1,59 @@
{
"normalizeExtensions": [
{
"input": [
"exe, msi",
" ZIP ",
".tar.gz",
"",
"exe"
],
"expected": [
"exe",
"msi",
"zip",
"tar.gz"
]
},
{
"input": [],
"expected": []
},
{
"input": [
" "
],
"expected": []
},
{
"input": [
"video/mp4"
],
"expected": [
"video/mp4"
]
},
{
"input": [
".exe",
"..msi"
],
"expected": [
"exe",
"msi"
]
},
{
"input": [
"EXE",
"ſ",
"Ä"
],
"expected": [
"exe",
"ſ",
"Ä"
]
}
]
}