feat(download): auto-categorize downloads into folders by file extension (#1572)

pkg/base/model.go

@@ -1,9 +1,11 @@
 package base
 
 import (
+	"encoding/json"
 	"fmt"
 	"net/http"
 	"net/url"
+	"path/filepath"
 	"sync"
 	"time"
 
@@ -117,6 +119,9 @@ type Options struct {
 	Path string `json:"path"`
 	// Use this path as the default download directory after creation succeeds
 	AsDefaultPath bool `json:"asDefaultPath,omitempty"`
+	// SkipAutoCategorize keeps the base download directory of this task instead
+	// of routing it to the category directory of its resolved file
+	SkipAutoCategorize bool `json:"skipAutoCategorize,omitempty"`
 	// Select file indexes to download
 	SelectFiles []int `json:"selectFiles"`
 	// Extra info for specific fetcher
@@ -191,6 +196,8 @@ type DownloaderStoreConfig struct {
 	Script                     *ScriptConfig          `json:"script"`                     // Script is the script execution configuration
 	AutoTorrent                *AutoTorrentConfig     `json:"autoTorrent"`                // AutoTorrent is the auto torrent task creation configuration
 	Archive                    *ArchiveConfig         `json:"archive"`                    // Archive is the archive extraction configuration
+	Categories                 []*DownloadCategory    `json:"categories"`                 // Categories is the download directory category configuration
+	AutoCategorize             *bool                  `json:"autoCategorize,omitempty"`   // AutoCategorize routes downloads to category directories by file extension; unset defaults to disabled
 	API                        *APIServerConfig       `json:"api"`                        // API is the optional REST server configuration
 	AutoStartTasks             bool                   `json:"autoStartTasks"`             // AutoStartTasks continues all unfinished tasks when the backend starts
 	AutoDeleteMissingFileTasks bool                   `json:"autoDeleteMissingFileTasks"` // AutoDeleteMissingFileTasks enables automatic deletion of tasks with missing files
@@ -227,9 +234,85 @@ func (cfg *DownloaderStoreConfig) Init() *DownloaderStoreConfig {
 			DeleteAfterExtract: false,
 		}
 	}
+	// Seed the built-in categories for deployments that never run the flutter
+	// startup flow, e.g. headless api servers.
+	if len(cfg.Categories) == 0 && cfg.DownloadDir != "" {
+		cfg.Categories = []*DownloadCategory{
+			builtinSeedCategory("categoryMusic", "Music", cfg.DownloadDir),
+			builtinSeedCategory("categoryVideo", "Video", cfg.DownloadDir),
+			builtinSeedCategory("categoryDocument", "Document", cfg.DownloadDir),
+			builtinSeedCategory("categoryProgram", "Program", cfg.DownloadDir),
+		}
+	}
 	return cfg
 }
 
+// builtinCategoryExtensions mirrors the flutter side
+// kDefaultCategoryExtensions, keep both maps in sync.
+var builtinCategoryExtensions = map[string][]string{
+	"categoryProgram":  {"exe", "msi", "msix", "apk", "dmg", "deb", "rpm", "pkg", "appimage"},
+	"categoryVideo":    {"mp4", "mkv", "avi", "mov", "wmv", "flv", "webm", "m4v", "mpg", "mpeg", "3gp"},
+	"categoryMusic":    {"mp3", "flac", "wav", "aac", "ogg", "m4a", "wma", "ape"},
+	"categoryDocument": {"pdf", "doc", "docx", "xls", "xlsx", "ppt", "pptx", "txt", "md", "csv", "epub", "rtf"},
+}
+
+func builtinSeedCategory(nameKey, name, downloadDir string) *DownloadCategory {
+	return &DownloadCategory{
+		Name:       "",
+		Path:       filepath.Join(downloadDir, name),
+		NameKey:    nameKey,
+		IsBuiltIn:  true,
+		Extensions: append([]string(nil), builtinCategoryExtensions[nameKey]...),
+	}
+}
+
+// MigrateLegacyExtraCategories moves the extra.downloadCategories list written
+// by v2.0.0-beta builds into the top level categories field and clears the
+// legacy key. The legacy list predates extension based routing, so built-in
+// entries regain their default extension lists from their name key, while user
+// created folders keep their empty list until edited in settings. It reports
+// whether the config changed so the caller can persist the cleared key.
+func (cfg *DownloaderStoreConfig) MigrateLegacyExtraCategories() bool {
+	if len(cfg.Categories) > 0 || cfg.Extra == nil {
+		return false
+	}
+	raw, ok := cfg.Extra["downloadCategories"]
+	if !ok {
+		return false
+	}
+	data, err := json.Marshal(raw)
+	if err != nil {
+		return false
+	}
+	var legacy []*DownloadCategory
+	if err := json.Unmarshal(data, &legacy); err != nil {
+		return false
+	}
+	categories := make([]*DownloadCategory, 0, len(legacy))
+	for _, category := range legacy {
+		if category == nil {
+			continue
+		}
+		if extensions, exist := builtinCategoryExtensions[category.NameKey]; exist && len(category.Extensions) == 0 {
+			category.Extensions = append([]string(nil), extensions...)
+		}
+		categories = append(categories, category)
+	}
+	if len(categories) == 0 {
+		return false
+	}
+	cfg.Categories = categories
+	delete(cfg.Extra, "downloadCategories")
+	return true
+}
+
+// AutoCategorizeEnabled reports whether downloads are routed to category
+// directories by file extension. The switch is optional: an unset value keeps
+// the feature off, so an older config that never saved the field stays off.
+func (cfg *DownloaderStoreConfig) AutoCategorizeEnabled() bool {
+	return cfg.AutoCategorize != nil && *cfg.AutoCategorize
+}
+
 func (cfg *DownloaderStoreConfig) Merge(beforeCfg *DownloaderStoreConfig) *DownloaderStoreConfig {
 	if beforeCfg == nil {
 		return cfg
@@ -264,6 +347,12 @@ func (cfg *DownloaderStoreConfig) Merge(beforeCfg *DownloaderStoreConfig) *Downl
 	if cfg.Archive == nil {
 		cfg.Archive = beforeCfg.Archive
 	}
+	if cfg.Categories == nil {
+		cfg.Categories = beforeCfg.Categories
+	}
+	if cfg.AutoCategorize == nil {
+		cfg.AutoCategorize = beforeCfg.AutoCategorize
+	}
 	if cfg.API == nil {
 		cfg.API = beforeCfg.API
 	}
@@ -312,6 +401,16 @@ type ArchiveConfig struct {
 	DeleteAfterExtract bool `json:"deleteAfterExtract"` // DeleteAfterExtract deletes the archive after successful extraction
 }
 
+// DownloadCategory is a download directory category that files can be routed to by extension
+type DownloadCategory struct {
+	Name       string   `json:"name"`                 // Name is the category display name, empty for built-in categories that use NameKey
+	Path       string   `json:"path"`                 // Path is the download directory of the category
+	NameKey    string   `json:"nameKey,omitempty"`    // NameKey is the i18n key of the built-in category name
+	IsBuiltIn  bool     `json:"isBuiltIn,omitempty"`  // IsBuiltIn marks the category as built-in
+	IsDeleted  bool     `json:"isDeleted,omitempty"`  // IsDeleted marks the built-in category as deleted
+	Extensions []string `json:"extensions,omitempty"` // Extensions is the file extension list routed to the category, without leading dots
+}
+
 type DownloaderProxyConfig struct {
 	Enable bool `json:"enable"`
 	// System is the flag that use system proxy

pkg/base/model_test.go

@@ -1,8 +1,13 @@
 package base
 
 import (
+	"encoding/json"
+	"path/filepath"
 	"reflect"
+	"strings"
 	"testing"
+
+	"github.com/GopeedLab/gopeed/pkg/util"
 )
 
 func TestDownloaderStoreConfig_Init(t *testing.T) {
@@ -192,6 +197,202 @@ func TestDownloaderStoreConfig_Init(t *testing.T) {
 	}
 }
 
+func TestDownloaderStoreConfig_InitSeedsBuiltInCategories(t *testing.T) {
+	t.Run("fresh config with a download dir gets the built-in categories", func(t *testing.T) {
+		dir := t.TempDir()
+		cfg := &DownloaderStoreConfig{DownloadDir: dir}
+		cfg.Init()
+
+		wantKeys := []string{"categoryMusic", "categoryVideo", "categoryDocument", "categoryProgram"}
+		wantNames := []string{"Music", "Video", "Document", "Program"}
+		if len(cfg.Categories) != len(wantKeys) {
+			t.Fatalf("categories = %d, want %d", len(cfg.Categories), len(wantKeys))
+		}
+		for i, category := range cfg.Categories {
+			if category.NameKey != wantKeys[i] {
+				t.Errorf("category[%d].NameKey = %q, want %q", i, category.NameKey, wantKeys[i])
+			}
+			if !category.IsBuiltIn {
+				t.Errorf("category[%d] %q is not built-in", i, category.NameKey)
+			}
+			if category.Path != filepath.Join(dir, wantNames[i]) {
+				t.Errorf("category[%d].Path = %q, want %q", i, category.Path, filepath.Join(dir, wantNames[i]))
+			}
+			if len(category.Extensions) == 0 {
+				t.Errorf("category[%d] %q has no default extensions", i, category.NameKey)
+			}
+		}
+	})
+
+	// Init itself never reads the legacy key: Setup calls
+	// MigrateLegacyExtraCategories first, and only a config that skipped the
+	// migration ends up with the built-in seed.
+	t.Run("legacy extra downloadCategories is not read by Init alone", func(t *testing.T) {
+		cfg := &DownloaderStoreConfig{
+			DownloadDir: t.TempDir(),
+			Extra: map[string]any{
+				"downloadCategories": []any{map[string]any{"name": "Music", "path": "/old/Music"}},
+			},
+		}
+		cfg.Init()
+		if len(cfg.Categories) != 4 {
+			t.Fatalf("categories = %d, want the 4 built-ins seeded anyway", len(cfg.Categories))
+		}
+	})
+
+	t.Run("existing categories and missing download dirs are untouched", func(t *testing.T) {
+		existing := []*DownloadCategory{{Name: "Mine", Path: "/downloads/Mine"}}
+		cfg := &DownloaderStoreConfig{DownloadDir: t.TempDir(), Categories: existing}
+		cfg.Init()
+		if len(cfg.Categories) != 1 || cfg.Categories[0].Name != "Mine" {
+			t.Fatalf("categories = %v, want the existing one", cfg.Categories)
+		}
+
+		empty := &DownloaderStoreConfig{}
+		empty.Init()
+		if len(empty.Categories) != 0 {
+			t.Fatalf("categories = %d, want 0 without a download dir", len(empty.Categories))
+		}
+	})
+}
+
+func TestDownloaderStoreConfig_AutoCategorizeEnabled(t *testing.T) {
+	// A config written before the switch existed has no value, and stays off.
+	if (&DownloaderStoreConfig{}).AutoCategorizeEnabled() {
+		t.Fatal("unset auto categorize = enabled, want disabled by default")
+	}
+	if !(&DownloaderStoreConfig{AutoCategorize: util.BoolPtr(true)}).AutoCategorizeEnabled() {
+		t.Fatal("explicit on = disabled, want the opt-in preserved")
+	}
+	if (&DownloaderStoreConfig{AutoCategorize: util.BoolPtr(false)}).AutoCategorizeEnabled() {
+		t.Fatal("explicit off = enabled, want the opt-out preserved")
+	}
+}
+
+func TestDownloaderStoreConfig_MigrateLegacyExtraCategories(t *testing.T) {
+	t.Run("legacy built-ins move to the top level with their default extensions", func(t *testing.T) {
+		cfg := &DownloaderStoreConfig{
+			DownloadDir: "/downloads",
+			Extra: map[string]any{
+				"downloadCategories": []any{
+					map[string]any{"name": "", "path": "/old/Music", "isBuiltIn": true, "nameKey": "categoryMusic"},
+					map[string]any{"name": "Games", "path": "/old/Games"},
+				},
+				"themeMode": "dark",
+			},
+		}
+		if !cfg.MigrateLegacyExtraCategories() {
+			t.Fatal("migration reported no change")
+		}
+		if len(cfg.Categories) != 2 {
+			t.Fatalf("categories = %d, want the 2 legacy entries", len(cfg.Categories))
+		}
+		builtIn := cfg.Categories[0]
+		if builtIn.NameKey != "categoryMusic" || !builtIn.IsBuiltIn || builtIn.Path != "/old/Music" {
+			t.Fatalf("built-in category = %+v, want the legacy entry", builtIn)
+		}
+		if len(builtIn.Extensions) == 0 {
+			t.Fatal("migrated built-in must regain its default extensions")
+		}
+		custom := cfg.Categories[1]
+		if custom.Name != "Games" || custom.Path != "/old/Games" || len(custom.Extensions) != 0 {
+			t.Fatalf("custom category = %+v, want the legacy folder untouched", custom)
+		}
+		if _, ok := cfg.Extra["downloadCategories"]; ok {
+			t.Fatal("legacy key must be cleared")
+		}
+		if cfg.Extra["themeMode"] != "dark" {
+			t.Fatalf("unrelated extra entries must survive, got %v", cfg.Extra)
+		}
+
+		// Setup runs Init after the migration, which must not seed over it.
+		cfg.Init()
+		if len(cfg.Categories) != 2 {
+			t.Fatalf("categories after Init = %d, want the migrated list kept", len(cfg.Categories))
+		}
+	})
+
+	t.Run("existing top level categories are left alone", func(t *testing.T) {
+		cfg := &DownloaderStoreConfig{
+			Extra: map[string]any{
+				"downloadCategories": []any{map[string]any{"name": "Old", "path": "/old/Default"}},
+			},
+			Categories: []*DownloadCategory{{Name: "Mine", Path: "/downloads/Mine"}},
+		}
+		if cfg.MigrateLegacyExtraCategories() {
+			t.Fatal("migration must not overwrite existing categories")
+		}
+		if len(cfg.Categories) != 1 || cfg.Categories[0].Name != "Mine" {
+			t.Fatalf("categories = %v, want the existing one", cfg.Categories)
+		}
+		if _, ok := cfg.Extra["downloadCategories"]; !ok {
+			t.Fatal("legacy key must stay for a manual move")
+		}
+	})
+
+	t.Run("missing, empty and malformed legacy lists are a no-op", func(t *testing.T) {
+		if (&DownloaderStoreConfig{}).MigrateLegacyExtraCategories() {
+			t.Fatal("empty config reported a change")
+		}
+		unrelated := &DownloaderStoreConfig{
+			Extra: map[string]any{"themeMode": "dark"},
+		}
+		if unrelated.MigrateLegacyExtraCategories() {
+			t.Fatal("config without the legacy key reported a change")
+		}
+		empty := &DownloaderStoreConfig{
+			Extra: map[string]any{"downloadCategories": []any{}},
+		}
+		if empty.MigrateLegacyExtraCategories() {
+			t.Fatal("empty legacy list reported a change")
+		}
+		malformed := &DownloaderStoreConfig{
+			Extra: map[string]any{"downloadCategories": "not-a-list"},
+		}
+		if malformed.MigrateLegacyExtraCategories() {
+			t.Fatal("malformed legacy list reported a change")
+		}
+		if _, ok := malformed.Extra["downloadCategories"]; !ok {
+			t.Fatal("malformed legacy key must stay untouched")
+		}
+	})
+
+	t.Run("a nil entry in the legacy list is skipped", func(t *testing.T) {
+		cfg := &DownloaderStoreConfig{
+			DownloadDir: "/downloads",
+			Extra: map[string]any{
+				"downloadCategories": []any{
+					nil,
+					map[string]any{"name": "Games", "path": "/old/Games"},
+				},
+			},
+		}
+		if !cfg.MigrateLegacyExtraCategories() {
+			t.Fatal("migration reported no change")
+		}
+		if len(cfg.Categories) != 1 || cfg.Categories[0].Name != "Games" {
+			t.Fatalf("categories = %+v, want the single valid legacy entry", cfg.Categories)
+		}
+	})
+}
+
+func TestDownloaderStoreConfig_AutoCategorizeJSON(t *testing.T) {
+	unset, err := json.Marshal(&DownloaderStoreConfig{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if strings.Contains(string(unset), `"autoCategorize"`) {
+		t.Fatalf("unset autoCategorize must be omitted from the api json, got %s", unset)
+	}
+	off, err := json.Marshal(&DownloaderStoreConfig{AutoCategorize: util.BoolPtr(false)})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !strings.Contains(string(off), `"autoCategorize":false`) {
+		t.Fatalf("explicit autoCategorize=false must be emitted, got %s", off)
+	}
+}
+
 func TestDownloaderStoreConfig_Merge(t *testing.T) {
 	type args struct {
 		beforeCfg *DownloaderStoreConfig
@@ -236,6 +437,24 @@ func TestDownloaderStoreConfig_Merge(t *testing.T) {
 				DownloadDir: "after",
 			},
 		},
+		{
+			"Merge Categories and AutoCategorize Override",
+			&DownloaderStoreConfig{},
+			args{
+				beforeCfg: &DownloaderStoreConfig{
+					Categories: []*DownloadCategory{
+						{Name: "Music", Path: "/downloads/Music"},
+					},
+					AutoCategorize: util.BoolPtr(true),
+				},
+			},
+			&DownloaderStoreConfig{
+				Categories: []*DownloadCategory{
+					{Name: "Music", Path: "/downloads/Music"},
+				},
+				AutoCategorize: util.BoolPtr(true),
+			},
+		},
 		{
 			"Merge MaxRunning No Override",
 			&DownloaderStoreConfig{
@@ -449,6 +668,8 @@ func TestDownloaderStoreConfig_Merge(t *testing.T) {
 				Webhook:        tt.fields.Webhook,
 				AutoTorrent:    tt.fields.AutoTorrent,
 				Archive:        tt.fields.Archive,
+				Categories:     tt.fields.Categories,
+				AutoCategorize: tt.fields.AutoCategorize,
 			}
 			if got := cfg.Merge(tt.args.beforeCfg); !reflect.DeepEqual(got, tt.want) {
 				t.Errorf("Merge() = %v, want %v", got, tt.want)

pkg/download/category.go

@@ -0,0 +1,186 @@
+package download
+
+import (
+	"errors"
+	"path"
+	"strings"
+	"unicode"
+
+	"github.com/GopeedLab/gopeed/pkg/base"
+)
+
+// categoryForResource returns the category that receives the resolved
+// resource, or nil when auto categorize does not apply. Only a single file
+// resource is routed: a folder or a multi file resource keeps its directory
+// layout, so its files are never matched one by one, not even when only a
+// single one of them is selected.
+func categoryForResource(cfg *base.DownloaderStoreConfig, res *base.Resource) *base.DownloadCategory {
+	if !cfg.AutoCategorizeEnabled() || len(cfg.Categories) == 0 {
+		return nil
+	}
+	if res == nil || res.Name != "" || len(res.Files) != 1 {
+		return nil
+	}
+	return matchDownloadCategory(cfg, res.Files[0].Name)
+}
+
+// decideDownloadDir returns the download directory of a task. The base
+// directory is the one the task asked for, or the configured download
+// directory when the task left it empty. A task that did not skip auto
+// categorize is routed to the category directory of its resolved resource
+// when one matches. Path placeholders are expanded and the white list check
+// runs on the result, so a category directory the white list rejects falls
+// back to the base directory of this task instead of failing it.
+func (d *Downloader) decideDownloadDir(opts *base.Options, res *base.Resource) (string, error) {
+	if opts == nil {
+		opts = &base.Options{}
+	}
+	cfg, err := d.GetConfig()
+	if err != nil {
+		return "", err
+	}
+	base := dartTrimSpace(opts.Path)
+	if base == "" {
+		base = cfg.DownloadDir
+	}
+	// Refuse to fall through to the process working directory when neither the
+	// task nor the config names a directory.
+	if base == "" {
+		return "", errors.New("download directory is empty")
+	}
+	target := base
+	if !opts.SkipAutoCategorize {
+		if category := categoryForResource(cfg, res); category != nil {
+			target = dartTrimSpace(category.Path)
+		}
+	}
+	dir, err := d.initDownloadPath(target)
+	if err != nil && target != base {
+		d.Logger.Warn().Err(err).Msgf("category directory %q rejected by white list, falling back to %q", target, base)
+		dir, err = d.initDownloadPath(base)
+	}
+	if err != nil {
+		return "", err
+	}
+	return dir, nil
+}
+
+// suggestedDownloadDir returns the category directory a task would be routed
+// to, or an empty string when auto categorize does not apply. It only reports
+// the decision of decideDownloadDir, it expands no directory and writes
+// nothing.
+func (d *Downloader) suggestedDownloadDir(res *base.Resource) string {
+	cfg, err := d.GetConfig()
+	if err != nil {
+		return ""
+	}
+	category := categoryForResource(cfg, res)
+	if category == nil {
+		return ""
+	}
+	dir, err := d.initDownloadPath(dartTrimSpace(category.Path))
+	if err != nil {
+		return ""
+	}
+	return dir
+}
+
+// asciiLower lowercases the ascii letters of s and leaves the rest alone. The
+// ui lowercases the same way, because the unicode case tables of the two
+// languages disagree for some code points and would let the two sides pick a
+// different category for the same file.
+func asciiLower(s string) string {
+	hasUpper := false
+	for i := 0; i < len(s); i++ {
+		if c := s[i]; c >= 'A' && c <= 'Z' {
+			hasUpper = true
+			break
+		}
+	}
+	if !hasUpper {
+		return s
+	}
+	b := []byte(s)
+	for i, c := range b {
+		if c >= 'A' && c <= 'Z' {
+			b[i] = c + ('a' - 'A')
+		}
+	}
+	return string(b)
+}
+
+// matchDownloadCategory returns the first category whose extension list
+// contains the extension of the given file name, or nil when nothing matches.
+// An empty extension list never matches.
+func matchDownloadCategory(cfg *base.DownloaderStoreConfig, name string) *base.DownloadCategory {
+	ext := fileExtension(name)
+	if ext == "" {
+		return nil
+	}
+	for _, category := range cfg.Categories {
+		if category == nil || category.IsDeleted || dartTrimSpace(category.Path) == "" {
+			continue
+		}
+		for _, e := range normalizeCategoryExtensions(category.Extensions) {
+			if e == ext {
+				return category
+			}
+		}
+	}
+	return nil
+}
+
+// normalizeCategoryExtensions mirrors the dart side: entries are split on
+// commas and whitespace, trimmed, lowercased, stripped of leading dots and
+// deduplicated while keeping the input order, so a stored value like
+// "exe, msi" behaves the same in the backend as it does in the ui.
+func normalizeCategoryExtensions(raw []string) []string {
+	var result []string
+	seen := make(map[string]struct{}, len(raw))
+	for _, item := range raw {
+		for _, part := range strings.FieldsFunc(item, func(r rune) bool {
+			return r == ',' || isECMAScriptSpace(r)
+		}) {
+			ext := asciiLower(strings.TrimLeft(dartTrimSpace(part), "."))
+			if ext == "" {
+				continue
+			}
+			if _, ok := seen[ext]; ok {
+				continue
+			}
+			seen[ext] = struct{}{}
+			result = append(result, ext)
+		}
+	}
+	return result
+}
+
+// fileExtension returns the lower case extension of a file name without the
+// leading dot, e.g. "setup.exe" -> "exe". Windows style separators are
+// treated like the ui does, so a dot inside a directory segment never counts
+// as an extension.
+func fileExtension(name string) string {
+	name = strings.ReplaceAll(name, `\`, "/")
+	return strings.TrimPrefix(asciiLower(path.Ext(name)), ".")
+}
+
+// dartTrimSpace trims the whitespace the dart String.trim() removes, which is
+// the unicode white space set plus the byte order mark. The ui trims the
+// category paths and the extension lists with it, so the backend uses the same
+// set instead of the narrower unicode.IsSpace.
+func dartTrimSpace(s string) string {
+	return strings.TrimFunc(s, func(r rune) bool {
+		return unicode.IsSpace(r) || r == 0xFEFF
+	})
+}
+
+// isECMAScriptSpace reports whether r belongs to the whitespace class of the
+// ECMAScript regular expressions. The ui splits its extension lists with that
+// class, so the backend uses the same one instead of the wider unicode.IsSpace.
+func isECMAScriptSpace(r rune) bool {
+	switch r {
+	case '\t', '\n', '\v', '\f', '\r', ' ', 0x00A0, 0x1680, 0x2028, 0x2029, 0x202F, 0x205F, 0x3000, 0xFEFF:
+		return true
+	}
+	return r >= 0x2000 && r <= 0x200A
+}

pkg/download/downloader.go

@@ -191,6 +191,15 @@ func (d *Downloader) Setup() error {
 			FirstLoad: true,
 		}
 	}
+	// Move the legacy extra.downloadCategories list written by v2.0.0-beta
+	// builds into the top level categories field and persist the cleared key,
+	// before Init seeds the built-in categories. The config is written
+	// directly, PutConfig would validate a config that is not initialized yet.
+	if d.cfg.DownloaderStoreConfig.MigrateLegacyExtraCategories() {
+		if err := d.storage.Put(bucketConfig, "config", d.cfg.DownloaderStoreConfig); err != nil {
+			return err
+		}
+	}
 	// Initialize the current backend configuration schema and defaults.
 	if err := d.initProtocolConfigs(); err != nil {
 		return err
@@ -537,7 +546,8 @@ func (d *Downloader) Resolve(req *base.Request) (rr *ResolveResult, err error) {
 	}
 	if res != nil && len(res.Files) > 0 {
 		rr = &ResolveResult{
-			Res: res,
+			Res:                  res,
+			SuggestedDownloadDir: d.suggestedDownloadDir(res),
 		}
 		return
 	}
@@ -554,8 +564,9 @@ func (d *Downloader) Resolve(req *base.Request) (rr *ResolveResult, err error) {
 	d.fetcherCache[rrId] = fetcher
 	d.fetcherMapLock.Unlock()
 	rr = &ResolveResult{
-		ID:  rrId,
-		Res: fetcher.Meta().Res,
+		ID:                   rrId,
+		Res:                  fetcher.Meta().Res,
+		SuggestedDownloadDir: d.suggestedDownloadDir(fetcher.Meta().Res),
 	}
 	return
 }
@@ -654,26 +665,32 @@ func (d *Downloader) CreateWithOptions(rrId string, opts *base.Options) (taskId
 	if opts == nil {
 		opts = &base.Options{}
 	}
-	opts, err = d.initOptions(opts.Clone(), fetcher.Meta().Req.URL)
+	// The options of the caller decide the global default download directory,
+	// the task options are prepared later, when the task starts.
+	defaultPath, err := d.prepareDefaultPath(opts)
+	if err != nil {
+		return "", err
+	}
+	taskOpts, err := d.initOptions(opts.Clone(), fetcher.Meta().Req.URL)
 	if err != nil {
 		return "", err
 	}
 	if res := fetcher.Meta().Res; res != nil {
-		for _, index := range opts.SelectFiles {
+		for _, index := range taskOpts.SelectFiles {
 			if index < 0 || index >= len(res.Files) {
 				return "", fmt.Errorf("invalid selected file index: %d", index)
 			}
 		}
-		opts.InitSelectFiles(len(res.Files))
+		taskOpts.InitSelectFiles(len(res.Files))
 	}
-	fetcher.Meta().Opts = opts
+	fetcher.Meta().Opts = taskOpts
 	if res := fetcher.Meta().Res; res != nil {
 		res.CalcSize(fetcher.Meta().Opts.SelectFiles)
 	}
 	// Resolve has no creation options. Publish a submitted HTTP estimate now
 	// while preserving any size already confirmed by the response.
-	if opts.Size > 0 {
-		if err := fetcher.Patch(nil, &base.Options{Size: opts.Size}); err != nil {
+	if taskOpts.Size > 0 {
+		if err := fetcher.Patch(nil, &base.Options{Size: taskOpts.Size}); err != nil {
 			return "", err
 		}
 	}
@@ -682,10 +699,6 @@ func (d *Downloader) CreateWithOptions(rrId string, opts *base.Options) (taskId
 		delete(d.fetcherCache, rrId)
 		d.fetcherMapLock.Unlock()
 	}()
-	defaultPath, err := d.prepareDefaultPath(fetcher.Meta().Opts)
-	if err != nil {
-		return "", err
-	}
 	taskId, err = d.doCreate(fetcher, nil)
 	if err == nil {
 		d.persistDefaultPathBestEffort(defaultPath)
@@ -1370,6 +1383,15 @@ func (d *Downloader) GetConfig() (*base.DownloaderStoreConfig, error) {
 	return util.DeepClone(d.cfg.DownloaderStoreConfig), nil
 }
 
+// IsFirstLoad reports whether Setup created the config instead of loading it
+// from storage. FirstLoad is not serialized, so the clone GetConfig returns
+// always reports false.
+func (d *Downloader) IsFirstLoad() bool {
+	d.configLock.RLock()
+	defer d.configLock.RUnlock()
+	return d.cfg.FirstLoad
+}
+
 func (d *Downloader) PutConfig(v *base.DownloaderStoreConfig) error {
 	if v == nil {
 		return errors.New("config is nil")
@@ -1433,7 +1455,10 @@ func (d *Downloader) PutConfig(v *base.DownloaderStoreConfig) error {
 }
 
 func (d *Downloader) prepareDefaultPath(opts *base.Options) (string, error) {
-	if opts == nil || !opts.AsDefaultPath || strings.TrimSpace(opts.Path) == "" {
+	// Only a directory the user named explicitly becomes the global default
+	// download directory. A task that keeps the auto categorize decision owns
+	// its directory, the backend must not turn it into the global default.
+	if opts == nil || !opts.SkipAutoCategorize || !opts.AsDefaultPath || strings.TrimSpace(opts.Path) == "" {
 		return "", nil
 	}
 	selectedPath := strings.TrimSpace(opts.Path)
@@ -1780,6 +1805,9 @@ func (d *Downloader) doCreate(f fetcher.Fetcher, opts *base.Options) (taskId str
 	task.Meta = f.Meta()
 	task.Progress = &Progress{}
 	_, task.Uploading = f.(fetcher.Uploader)
+	// A new task still needs its first preparation, it decides the download
+	// directory and the final name of the task.
+	task.NeedsPreparation = true
 	initTask(task)
 	if err = d.syncBlobTaskLease(task); err != nil {
 		return "", err
@@ -1823,6 +1851,10 @@ func httpSizeOptions(opts *base.Options, protocol string) *base.Options {
 	return opts
 }
 
+// initOptions makes sure the option set is usable: a nil option set becomes
+// an empty one, the selection is never nil and the base download directory
+// falls back to the configured one. The directory is decided and checked
+// against the white list when the task starts.
 func (d *Downloader) initOptions(opts *base.Options, requestURL string) (*base.Options, error) {
 	if opts == nil {
 		opts = &base.Options{}
@@ -1838,17 +1870,12 @@ func (d *Downloader) initOptions(opts *base.Options, requestURL string) (*base.O
 		opts.SelectFiles = make([]int, 0)
 	}
 	if opts.Path == "" {
-		storeConfig, err := d.GetConfig()
+		cfg, err := d.GetConfig()
 		if err != nil {
 			return nil, err
 		}
-		opts.Path = storeConfig.DownloadDir
+		opts.Path = cfg.DownloadDir
 	}
-	path, err := d.initDownloadPath(opts.Path)
-	if err != nil {
-		return nil, err
-	}
-	opts.Path = path
 	return opts, nil
 }
 
@@ -1879,8 +1906,68 @@ func (d *Downloader) statusMut(task *Task, fn func() (bool, error)) (bool, error
 	return fn()
 }
 
+// prepareTask runs the first preparation of a task: it decides the download
+// directory, resolves a duplicate name in it and persists both. The task keeps
+// its NeedsPreparation flag when the preparation fails, so a retry prepares it
+// again. The caller holds checkDuplicateLock when the fetcher manager renames
+// duplicates, the lock has to stay held until the fetcher created the file.
+func (d *Downloader) prepareTask(task *Task) error {
+	if task.Meta.Res == nil {
+		return errors.New("task resource is not resolved")
+	}
+	preparedMeta := *task.Meta
+	preparedOpts := *task.Meta.Opts
+	preparedMeta.Opts = &preparedOpts
+	preparedMeta.Res = util.DeepClone(task.Meta.Res)
+	dir, err := d.decideDownloadDir(preparedMeta.Opts, preparedMeta.Res)
+	if err != nil {
+		return err
+	}
+	preparedMeta.Opts.Path = dir
+
+	if task.fetcherManager.AutoRename() {
+		preparedMeta.Opts.Name = util.SafeFilename(preparedMeta.Opts.Name)
+		// check if the download file is duplicated and rename it automatically.
+		if preparedMeta.Res.Name != "" {
+			preparedMeta.Res.Name = util.SafeFilename(preparedMeta.Res.Name)
+			fullDirPath := preparedMeta.FolderPath()
+			newName, err := util.CheckDuplicateAndRename(fullDirPath)
+			if err != nil {
+				return err
+			}
+			preparedMeta.Opts.Name = newName
+		} else {
+			preparedMeta.Res.Files[0].Name = util.SafeFilename(preparedMeta.Res.Files[0].Name)
+			fullFilePath := preparedMeta.SingleFilepath()
+			newName, err := util.CheckDuplicateAndRename(fullFilePath)
+			if err != nil {
+				return err
+			}
+			preparedMeta.Opts.Name = newName
+		}
+	}
+
+	preparedMeta.Res.CalcSize(preparedMeta.Opts.SelectFiles)
+
+	// Save a candidate task before publishing its metadata. The fetcher keeps
+	// its resolved protocol state, so a failed save cannot change the live
+	// directory, name or preparation flag.
+	task.statusLock.Lock()
+	preparedTask := *task
+	task.statusLock.Unlock()
+	preparedTask.Meta = &preparedMeta
+	preparedTask.NeedsPreparation = false
+	if err := d.saveTask(&preparedTask); err != nil {
+		return err
+	}
+	task.statusLock.Lock()
+	*task.Meta = preparedMeta
+	task.NeedsPreparation = false
+	task.statusLock.Unlock()
+	return nil
+}
+
 func (d *Downloader) doStart(task *Task) (err error) {
-	var isCreate bool
 	var generation uint64
 	isReturn, err := d.statusMut(task, func() (isReturn bool, err error) {
 		if task.Status == base.DownloadStatusRunning || task.Status == base.DownloadStatusDone {
@@ -1893,7 +1980,6 @@ func (d *Downloader) doStart(task *Task) (err error) {
 			d.Logger.Error().Stack().Err(err).Msgf("restore fetcher failed, task id: %s", task.ID)
 			return
 		}
-		isCreate = task.Status == base.DownloadStatusReady
 		task.updateStatus(base.DownloadStatusRunning)
 		task.runGeneration++
 		generation = task.runGeneration
@@ -1933,7 +2019,8 @@ func (d *Downloader) doStart(task *Task) (err error) {
 		if err := d.syncBlobTaskLease(task); err != nil {
 			return err
 		}
-		if task.Meta.Res == nil {
+		resolvedHere := task.Meta.Res == nil
+		if resolvedHere {
 			err := task.fetcher.Resolve(task.Meta.Req)
 			if err != nil {
 				return err
@@ -1941,32 +2028,25 @@ func (d *Downloader) doStart(task *Task) (err error) {
 			task.Meta.Res = task.fetcher.Meta().Res
 		}
 
-		if isCreate {
+		if task.NeedsPreparation {
 			if task.fetcherManager.AutoRename() {
+				// The duplicate name is resolved from the files on disk, but
+				// the file only appears when the fetcher starts, so the lock
+				// has to cover the start as well.
 				d.checkDuplicateLock.Lock()
 				defer d.checkDuplicateLock.Unlock()
-				task.Meta.Opts.Name = util.SafeFilename(task.Meta.Opts.Name)
-				// check if the download file is duplicated and rename it automatically.
-				if task.Meta.Res.Name != "" {
-					task.Meta.Res.Name = util.SafeFilename(task.Meta.Res.Name)
-					fullDirPath := task.Meta.FolderPath()
-					newName, err := util.CheckDuplicateAndRename(fullDirPath)
-					if err != nil {
-						return err
-					}
-					task.Meta.Opts.Name = newName
-				} else {
-					task.Meta.Res.Files[0].Name = util.SafeFilename(task.Meta.Res.Files[0].Name)
-					fullFilePath := task.Meta.SingleFilepath()
-					newName, err := util.CheckDuplicateAndRename(fullFilePath)
-					if err != nil {
-						return err
-					}
-					task.Meta.Opts.Name = newName
-				}
 			}
-
-			task.Meta.Res.CalcSize(task.Meta.Opts.SelectFiles)
+			// The first preparation decides the download directory and the
+			// final name of the task. It runs for every task that has not
+			// been prepared yet, no matter whether the task waited in the
+			// queue, was paused or failed before.
+			if err := d.prepareTask(task); err != nil {
+				return err
+			}
+		} else if _, err := d.initDownloadPath(task.Meta.Opts.Path); err != nil {
+			// A prepared task keeps its directory, but the directory still has
+			// to pass the white list on every start.
+			return err
 		}
 
 		task.statusLock.Lock()

pkg/download/downloader_test.go

@@ -841,6 +841,9 @@ func TestDownloader_CreateDirectUsesPathAsDefault(t *testing.T) {
 	selectedRoot := t.TempDir()
 	defaultPathOpts := newTestDownloadOptAt(selectedRoot)
 	defaultPathOpts.AsDefaultPath = true
+	// The global default download directory is only saved for a directory the
+	// user named explicitly, so the task skips the auto categorize decision.
+	defaultPathOpts.SkipAutoCategorize = true
 	createAndWait(defaultPathOpts)
 	cfg, err = downloader.GetConfig()
 	if err != nil {
@@ -882,11 +885,26 @@ func TestDownloader_CreateNotInWhite(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	_, err = downloader.CreateWithOptions(rr.ID, newTestDownloadOpt(t))
-	if err == nil {
-		t.Fatal("Create should reject a directory outside the allowlist")
+	// The directory is decided when the task starts, so a directory outside
+	// the allowlist fails the task instead of the create call.
+	taskId, err := downloader.CreateWithOptions(rr.ID, newTestDownloadOpt(t))
+	if err != nil {
+		t.Fatal(err)
+	}
+	task := downloader.GetTask(taskId)
+	if task == nil {
+		t.Fatal("task not found")
+	}
+	deadline := time.Now().Add(5 * time.Second)
+	for time.Now().Before(deadline) {
+		if task.Status == base.DownloadStatusError {
+			break
+		}
+		time.Sleep(20 * time.Millisecond)
+	}
+	if task.Status != base.DownloadStatusError {
+		t.Fatalf("task status = %q, want the task to be rejected by the allowlist", task.Status)
 	}
-
 }
 
 func TestDownloader_CreateDirectBatch(t *testing.T) {
@@ -1142,6 +1160,33 @@ func TestDownloader_CreateRename(t *testing.T) {
 	}
 }
 
+func TestDownloader_IsFirstLoadSurvivesTheConfigClone(t *testing.T) {
+	d := NewDownloader(&DownloaderConfig{StorageDir: t.TempDir(), TempDir: t.TempDir()})
+	if err := d.Setup(); err != nil {
+		t.Fatal(err)
+	}
+	defer d.Close()
+
+	if !d.IsFirstLoad() {
+		t.Fatal("a downloader that created its config reports IsFirstLoad() = false")
+	}
+	// GetConfig clones through json and FirstLoad is not serialized, so the cmd
+	// layer cannot read the flag from the returned config.
+	cfg, err := d.GetConfig()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if cfg.FirstLoad {
+		t.Fatal("the cloned config must not report the first load")
+	}
+	if err := d.PutConfig(cfg); err != nil {
+		t.Fatal(err)
+	}
+	if d.IsFirstLoad() {
+		t.Fatal("a downloader that stored its config reports IsFirstLoad() = true")
+	}
+}
+
 func TestDownloader_StoreAndRestore(t *testing.T) {
 	listener := test.StartTestSlowFileServer(time.Millisecond * 2000)
 	defer listener.Close()

pkg/download/model.go

@@ -21,6 +21,11 @@ import (
 type ResolveResult struct {
 	ID  string         `json:"id"`
 	Res *base.Resource `json:"res"`
+	// SuggestedDownloadDir is the category directory the created task would be
+	// routed to. It is only set when auto categorize is enabled, the resource is
+	// a single file, a category matches and its directory passes the white list
+	// check. It is a hint for the ui, the create call decides the final path.
+	SuggestedDownloadDir string `json:"suggestedDownloadDir,omitempty"`
 }
 
 // Task represents a download unit.
@@ -39,6 +44,14 @@ type Task struct {
 	// finished. A done task never leaves the done state, so the value is
 	// written once and kept.
 	DoneAt *time.Time `json:"doneAt"`
+	// NeedsPreparation marks a task that has not run its first preparation
+	// yet. The preparation decides the download directory of the task and
+	// resolves the duplicate name in it. It runs once, after the resource is
+	// resolved, and the flag is cleared afterwards, so a queued, paused or
+	// retried task is still prepared while a prepared task is never
+	// classified or renamed again. A task stored before this flag existed
+	// does not have it and keeps its directory and name.
+	NeedsPreparation bool `json:"needsPreparation,omitempty"`
 
 	producer         production.Source
 	producerReceived int64

pkg/download/testdata/category_matching_cases.json

@@ -0,0 +1,59 @@
+{
+  "normalizeExtensions": [
+    {
+      "input": [
+        "exe, msi",
+        " ZIP ",
+        ".tar.gz",
+        "",
+        "exe"
+      ],
+      "expected": [
+        "exe",
+        "msi",
+        "zip",
+        "tar.gz"
+      ]
+    },
+    {
+      "input": [],
+      "expected": []
+    },
+    {
+      "input": [
+        "  "
+      ],
+      "expected": []
+    },
+    {
+      "input": [
+        "video/mp4"
+      ],
+      "expected": [
+        "video/mp4"
+      ]
+    },
+    {
+      "input": [
+        ".exe",
+        "..msi"
+      ],
+      "expected": [
+        "exe",
+        "msi"
+      ]
+    },
+    {
+      "input": [
+        "EXE",
+        "ſ",
+        "Ä"
+      ],
+      "expected": [
+        "exe",
+        "ſ",
+        "Ä"
+      ]
+    }
+  ]
+}

pkg/mcpserver/server.go

@@ -139,11 +139,12 @@ type checksumOption struct {
 // downloadOptions is the MCP-facing form of base.Options. Only HTTP currently
 // defines protocol-specific task options, so Extra has one concrete schema.
 type downloadOptions struct {
-	Name          string         `json:"name,omitempty" jsonschema:"Optional output file or directory name"`
-	Path          string         `json:"path,omitempty" jsonschema:"Optional download directory"`
-	AsDefaultPath bool           `json:"asDefaultPath,omitempty" jsonschema:"Save path as the default download directory after task creation"`
-	SelectFiles   []int          `json:"selectFiles,omitempty" jsonschema:"Optional zero-based file indexes to download from a multi-file resource"`
-	Extra         *httpTaskExtra `json:"extra,omitempty" jsonschema:"Optional HTTP task settings"`
+	Name               string         `json:"name,omitempty" jsonschema:"Optional output file or directory name"`
+	Path               string         `json:"path,omitempty" jsonschema:"Optional base download directory"`
+	AsDefaultPath      bool           `json:"asDefaultPath,omitempty" jsonschema:"Save path as the default download directory after task creation"`
+	SkipAutoCategorize bool           `json:"skipAutoCategorize,omitempty" jsonschema:"Keep the base download directory instead of routing the task to its category directory"`
+	SelectFiles        []int          `json:"selectFiles,omitempty" jsonschema:"Optional zero-based file indexes to download from a multi-file resource"`
+	Extra              *httpTaskExtra `json:"extra,omitempty" jsonschema:"Optional HTTP task settings"`
 }
 
 func requestExtraSchema() *jsonschema.Schema {
@@ -213,11 +214,12 @@ func (o *downloadOptions) baseOptions() *base.Options {
 		}
 	}
 	return &base.Options{
-		Name:          o.Name,
-		Path:          o.Path,
-		AsDefaultPath: o.AsDefaultPath,
-		SelectFiles:   o.SelectFiles,
-		Extra:         extra,
+		Name:               o.Name,
+		Path:               o.Path,
+		AsDefaultPath:      o.AsDefaultPath,
+		SkipAutoCategorize: o.SkipAutoCategorize,
+		SelectFiles:        o.SelectFiles,
+		Extra:              extra,
 	}
 }
 
@@ -228,6 +230,9 @@ type resolveTaskInput struct {
 type resolveTaskOutput struct {
 	ID  string         `json:"id,omitempty"`
 	Res *base.Resource `json:"res"`
+	// SuggestedDownloadDir is the category directory the created task would be
+	// routed to, empty when auto categorize does not apply.
+	SuggestedDownloadDir string `json:"suggestedDownloadDir,omitempty"`
 }
 
 type createTaskInput struct {
@@ -331,7 +336,11 @@ func registerTools(server *mcp.Server, downloader *download.Downloader) {
 		if err != nil {
 			return nil, nil, err
 		}
-		return nil, &resolveTaskOutput{ID: result.ID, Res: result.Res}, nil
+		return nil, &resolveTaskOutput{
+			ID:                   result.ID,
+			Res:                  result.Res,
+			SuggestedDownloadDir: result.SuggestedDownloadDir,
+		}, nil
 	})
 
 	mcp.AddTool(server, &mcp.Tool{

pkg/mcpserver/server_test.go

@@ -112,6 +112,21 @@ func TestStreamableHTTPTools(t *testing.T) {
 		if !schemaIncludesType(optsExtra, "object") || optsExtra["additionalProperties"] != false {
 			t.Fatalf("%s opts.extra schema = %#v, want concrete HTTP task settings", tool.Name, optsExtra)
 		}
+		opts := nestedSchemaProperty(t, schema, "opts")
+		optsProperties, ok := opts["properties"].(map[string]any)
+		if !ok {
+			t.Fatalf("%s opts schema has no properties: %s", tool.Name, inputSchema)
+		}
+		if _, exists := optsProperties["skipAutoCategorize"]; !exists {
+			t.Fatalf("%s opts schema is missing skipAutoCategorize: %s", tool.Name, inputSchema)
+		}
+	}
+	resolveOutput, err := json.Marshal(resolveTaskTool.OutputSchema)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !bytes.Contains(resolveOutput, []byte(`"suggestedDownloadDir"`)) {
+		t.Fatalf("resolve_task output schema is missing suggestedDownloadDir: %s", resolveOutput)
 	}
 	outputSchema, err := json.Marshal(getTaskTool.OutputSchema)
 	if err != nil {

pkg/rest/server_test.go

@@ -1195,6 +1195,14 @@ func doTest0(onStart func(cfg *model.StartConfig), handler func()) {
 			onStart(cfg)
 		}
 		fileListener := doStart(cfg)
+		// The server process configures the default download directory on first
+		// load; a task without a base directory is rejected without it.
+		if current, err := Downloader.GetConfig(); err == nil {
+			current.DownloadDir = downloadDir
+			if err := Downloader.PutConfig(current); err != nil {
+				panic(err)
+			}
+		}
 		defer func() {
 			if err := fileListener.Close(); err != nil {
 				panic(err)