feat(download): auto-categorize downloads into folders by file extension (#1572)

pkg/base/model.go

@@ -1,9 +1,11 @@
 package base
 
 import (
+	"encoding/json"
 	"fmt"
 	"net/http"
 	"net/url"
+	"path/filepath"
 	"sync"
 	"time"
 
@@ -117,6 +119,9 @@ type Options struct {
 	Path string `json:"path"`
 	// Use this path as the default download directory after creation succeeds
 	AsDefaultPath bool `json:"asDefaultPath,omitempty"`
+	// SkipAutoCategorize keeps the base download directory of this task instead
+	// of routing it to the category directory of its resolved file
+	SkipAutoCategorize bool `json:"skipAutoCategorize,omitempty"`
 	// Select file indexes to download
 	SelectFiles []int `json:"selectFiles"`
 	// Extra info for specific fetcher
@@ -191,6 +196,8 @@ type DownloaderStoreConfig struct {
 	Script                     *ScriptConfig          `json:"script"`                     // Script is the script execution configuration
 	AutoTorrent                *AutoTorrentConfig     `json:"autoTorrent"`                // AutoTorrent is the auto torrent task creation configuration
 	Archive                    *ArchiveConfig         `json:"archive"`                    // Archive is the archive extraction configuration
+	Categories                 []*DownloadCategory    `json:"categories"`                 // Categories is the download directory category configuration
+	AutoCategorize             *bool                  `json:"autoCategorize,omitempty"`   // AutoCategorize routes downloads to category directories by file extension; unset defaults to disabled
 	API                        *APIServerConfig       `json:"api"`                        // API is the optional REST server configuration
 	AutoStartTasks             bool                   `json:"autoStartTasks"`             // AutoStartTasks continues all unfinished tasks when the backend starts
 	AutoDeleteMissingFileTasks bool                   `json:"autoDeleteMissingFileTasks"` // AutoDeleteMissingFileTasks enables automatic deletion of tasks with missing files
@@ -227,9 +234,85 @@ func (cfg *DownloaderStoreConfig) Init() *DownloaderStoreConfig {
 			DeleteAfterExtract: false,
 		}
 	}
+	// Seed the built-in categories for deployments that never run the flutter
+	// startup flow, e.g. headless api servers.
+	if len(cfg.Categories) == 0 && cfg.DownloadDir != "" {
+		cfg.Categories = []*DownloadCategory{
+			builtinSeedCategory("categoryMusic", "Music", cfg.DownloadDir),
+			builtinSeedCategory("categoryVideo", "Video", cfg.DownloadDir),
+			builtinSeedCategory("categoryDocument", "Document", cfg.DownloadDir),
+			builtinSeedCategory("categoryProgram", "Program", cfg.DownloadDir),
+		}
+	}
 	return cfg
 }
 
+// builtinCategoryExtensions mirrors the flutter side
+// kDefaultCategoryExtensions, keep both maps in sync.
+var builtinCategoryExtensions = map[string][]string{
+	"categoryProgram":  {"exe", "msi", "msix", "apk", "dmg", "deb", "rpm", "pkg", "appimage"},
+	"categoryVideo":    {"mp4", "mkv", "avi", "mov", "wmv", "flv", "webm", "m4v", "mpg", "mpeg", "3gp"},
+	"categoryMusic":    {"mp3", "flac", "wav", "aac", "ogg", "m4a", "wma", "ape"},
+	"categoryDocument": {"pdf", "doc", "docx", "xls", "xlsx", "ppt", "pptx", "txt", "md", "csv", "epub", "rtf"},
+}
+
+func builtinSeedCategory(nameKey, name, downloadDir string) *DownloadCategory {
+	return &DownloadCategory{
+		Name:       "",
+		Path:       filepath.Join(downloadDir, name),
+		NameKey:    nameKey,
+		IsBuiltIn:  true,
+		Extensions: append([]string(nil), builtinCategoryExtensions[nameKey]...),
+	}
+}
+
+// MigrateLegacyExtraCategories moves the extra.downloadCategories list written
+// by v2.0.0-beta builds into the top level categories field and clears the
+// legacy key. The legacy list predates extension based routing, so built-in
+// entries regain their default extension lists from their name key, while user
+// created folders keep their empty list until edited in settings. It reports
+// whether the config changed so the caller can persist the cleared key.
+func (cfg *DownloaderStoreConfig) MigrateLegacyExtraCategories() bool {
+	if len(cfg.Categories) > 0 || cfg.Extra == nil {
+		return false
+	}
+	raw, ok := cfg.Extra["downloadCategories"]
+	if !ok {
+		return false
+	}
+	data, err := json.Marshal(raw)
+	if err != nil {
+		return false
+	}
+	var legacy []*DownloadCategory
+	if err := json.Unmarshal(data, &legacy); err != nil {
+		return false
+	}
+	categories := make([]*DownloadCategory, 0, len(legacy))
+	for _, category := range legacy {
+		if category == nil {
+			continue
+		}
+		if extensions, exist := builtinCategoryExtensions[category.NameKey]; exist && len(category.Extensions) == 0 {
+			category.Extensions = append([]string(nil), extensions...)
+		}
+		categories = append(categories, category)
+	}
+	if len(categories) == 0 {
+		return false
+	}
+	cfg.Categories = categories
+	delete(cfg.Extra, "downloadCategories")
+	return true
+}
+
+// AutoCategorizeEnabled reports whether downloads are routed to category
+// directories by file extension. The switch is optional: an unset value keeps
+// the feature off, so an older config that never saved the field stays off.
+func (cfg *DownloaderStoreConfig) AutoCategorizeEnabled() bool {
+	return cfg.AutoCategorize != nil && *cfg.AutoCategorize
+}
+
 func (cfg *DownloaderStoreConfig) Merge(beforeCfg *DownloaderStoreConfig) *DownloaderStoreConfig {
 	if beforeCfg == nil {
 		return cfg
@@ -264,6 +347,12 @@ func (cfg *DownloaderStoreConfig) Merge(beforeCfg *DownloaderStoreConfig) *Downl
 	if cfg.Archive == nil {
 		cfg.Archive = beforeCfg.Archive
 	}
+	if cfg.Categories == nil {
+		cfg.Categories = beforeCfg.Categories
+	}
+	if cfg.AutoCategorize == nil {
+		cfg.AutoCategorize = beforeCfg.AutoCategorize
+	}
 	if cfg.API == nil {
 		cfg.API = beforeCfg.API
 	}
@@ -312,6 +401,16 @@ type ArchiveConfig struct {
 	DeleteAfterExtract bool `json:"deleteAfterExtract"` // DeleteAfterExtract deletes the archive after successful extraction
 }
 
+// DownloadCategory is a download directory category that files can be routed to by extension
+type DownloadCategory struct {
+	Name       string   `json:"name"`                 // Name is the category display name, empty for built-in categories that use NameKey
+	Path       string   `json:"path"`                 // Path is the download directory of the category
+	NameKey    string   `json:"nameKey,omitempty"`    // NameKey is the i18n key of the built-in category name
+	IsBuiltIn  bool     `json:"isBuiltIn,omitempty"`  // IsBuiltIn marks the category as built-in
+	IsDeleted  bool     `json:"isDeleted,omitempty"`  // IsDeleted marks the built-in category as deleted
+	Extensions []string `json:"extensions,omitempty"` // Extensions is the file extension list routed to the category, without leading dots
+}
+
 type DownloaderProxyConfig struct {
 	Enable bool `json:"enable"`
 	// System is the flag that use system proxy

pkg/base/model_test.go

@@ -1,8 +1,13 @@
 package base
 
 import (
+	"encoding/json"
+	"path/filepath"
 	"reflect"
+	"strings"
 	"testing"
+
+	"github.com/GopeedLab/gopeed/pkg/util"
 )
 
 func TestDownloaderStoreConfig_Init(t *testing.T) {
@@ -192,6 +197,202 @@ func TestDownloaderStoreConfig_Init(t *testing.T) {
 	}
 }
 
+func TestDownloaderStoreConfig_InitSeedsBuiltInCategories(t *testing.T) {
+	t.Run("fresh config with a download dir gets the built-in categories", func(t *testing.T) {
+		dir := t.TempDir()
+		cfg := &DownloaderStoreConfig{DownloadDir: dir}
+		cfg.Init()
+
+		wantKeys := []string{"categoryMusic", "categoryVideo", "categoryDocument", "categoryProgram"}
+		wantNames := []string{"Music", "Video", "Document", "Program"}
+		if len(cfg.Categories) != len(wantKeys) {
+			t.Fatalf("categories = %d, want %d", len(cfg.Categories), len(wantKeys))
+		}
+		for i, category := range cfg.Categories {
+			if category.NameKey != wantKeys[i] {
+				t.Errorf("category[%d].NameKey = %q, want %q", i, category.NameKey, wantKeys[i])
+			}
+			if !category.IsBuiltIn {
+				t.Errorf("category[%d] %q is not built-in", i, category.NameKey)
+			}
+			if category.Path != filepath.Join(dir, wantNames[i]) {
+				t.Errorf("category[%d].Path = %q, want %q", i, category.Path, filepath.Join(dir, wantNames[i]))
+			}
+			if len(category.Extensions) == 0 {
+				t.Errorf("category[%d] %q has no default extensions", i, category.NameKey)
+			}
+		}
+	})
+
+	// Init itself never reads the legacy key: Setup calls
+	// MigrateLegacyExtraCategories first, and only a config that skipped the
+	// migration ends up with the built-in seed.
+	t.Run("legacy extra downloadCategories is not read by Init alone", func(t *testing.T) {
+		cfg := &DownloaderStoreConfig{
+			DownloadDir: t.TempDir(),
+			Extra: map[string]any{
+				"downloadCategories": []any{map[string]any{"name": "Music", "path": "/old/Music"}},
+			},
+		}
+		cfg.Init()
+		if len(cfg.Categories) != 4 {
+			t.Fatalf("categories = %d, want the 4 built-ins seeded anyway", len(cfg.Categories))
+		}
+	})
+
+	t.Run("existing categories and missing download dirs are untouched", func(t *testing.T) {
+		existing := []*DownloadCategory{{Name: "Mine", Path: "/downloads/Mine"}}
+		cfg := &DownloaderStoreConfig{DownloadDir: t.TempDir(), Categories: existing}
+		cfg.Init()
+		if len(cfg.Categories) != 1 || cfg.Categories[0].Name != "Mine" {
+			t.Fatalf("categories = %v, want the existing one", cfg.Categories)
+		}
+
+		empty := &DownloaderStoreConfig{}
+		empty.Init()
+		if len(empty.Categories) != 0 {
+			t.Fatalf("categories = %d, want 0 without a download dir", len(empty.Categories))
+		}
+	})
+}
+
+func TestDownloaderStoreConfig_AutoCategorizeEnabled(t *testing.T) {
+	// A config written before the switch existed has no value, and stays off.
+	if (&DownloaderStoreConfig{}).AutoCategorizeEnabled() {
+		t.Fatal("unset auto categorize = enabled, want disabled by default")
+	}
+	if !(&DownloaderStoreConfig{AutoCategorize: util.BoolPtr(true)}).AutoCategorizeEnabled() {
+		t.Fatal("explicit on = disabled, want the opt-in preserved")
+	}
+	if (&DownloaderStoreConfig{AutoCategorize: util.BoolPtr(false)}).AutoCategorizeEnabled() {
+		t.Fatal("explicit off = enabled, want the opt-out preserved")
+	}
+}
+
+func TestDownloaderStoreConfig_MigrateLegacyExtraCategories(t *testing.T) {
+	t.Run("legacy built-ins move to the top level with their default extensions", func(t *testing.T) {
+		cfg := &DownloaderStoreConfig{
+			DownloadDir: "/downloads",
+			Extra: map[string]any{
+				"downloadCategories": []any{
+					map[string]any{"name": "", "path": "/old/Music", "isBuiltIn": true, "nameKey": "categoryMusic"},
+					map[string]any{"name": "Games", "path": "/old/Games"},
+				},
+				"themeMode": "dark",
+			},
+		}
+		if !cfg.MigrateLegacyExtraCategories() {
+			t.Fatal("migration reported no change")
+		}
+		if len(cfg.Categories) != 2 {
+			t.Fatalf("categories = %d, want the 2 legacy entries", len(cfg.Categories))
+		}
+		builtIn := cfg.Categories[0]
+		if builtIn.NameKey != "categoryMusic" || !builtIn.IsBuiltIn || builtIn.Path != "/old/Music" {
+			t.Fatalf("built-in category = %+v, want the legacy entry", builtIn)
+		}
+		if len(builtIn.Extensions) == 0 {
+			t.Fatal("migrated built-in must regain its default extensions")
+		}
+		custom := cfg.Categories[1]
+		if custom.Name != "Games" || custom.Path != "/old/Games" || len(custom.Extensions) != 0 {
+			t.Fatalf("custom category = %+v, want the legacy folder untouched", custom)
+		}
+		if _, ok := cfg.Extra["downloadCategories"]; ok {
+			t.Fatal("legacy key must be cleared")
+		}
+		if cfg.Extra["themeMode"] != "dark" {
+			t.Fatalf("unrelated extra entries must survive, got %v", cfg.Extra)
+		}
+
+		// Setup runs Init after the migration, which must not seed over it.
+		cfg.Init()
+		if len(cfg.Categories) != 2 {
+			t.Fatalf("categories after Init = %d, want the migrated list kept", len(cfg.Categories))
+		}
+	})
+
+	t.Run("existing top level categories are left alone", func(t *testing.T) {
+		cfg := &DownloaderStoreConfig{
+			Extra: map[string]any{
+				"downloadCategories": []any{map[string]any{"name": "Old", "path": "/old/Default"}},
+			},
+			Categories: []*DownloadCategory{{Name: "Mine", Path: "/downloads/Mine"}},
+		}
+		if cfg.MigrateLegacyExtraCategories() {
+			t.Fatal("migration must not overwrite existing categories")
+		}
+		if len(cfg.Categories) != 1 || cfg.Categories[0].Name != "Mine" {
+			t.Fatalf("categories = %v, want the existing one", cfg.Categories)
+		}
+		if _, ok := cfg.Extra["downloadCategories"]; !ok {
+			t.Fatal("legacy key must stay for a manual move")
+		}
+	})
+
+	t.Run("missing, empty and malformed legacy lists are a no-op", func(t *testing.T) {
+		if (&DownloaderStoreConfig{}).MigrateLegacyExtraCategories() {
+			t.Fatal("empty config reported a change")
+		}
+		unrelated := &DownloaderStoreConfig{
+			Extra: map[string]any{"themeMode": "dark"},
+		}
+		if unrelated.MigrateLegacyExtraCategories() {
+			t.Fatal("config without the legacy key reported a change")
+		}
+		empty := &DownloaderStoreConfig{
+			Extra: map[string]any{"downloadCategories": []any{}},
+		}
+		if empty.MigrateLegacyExtraCategories() {
+			t.Fatal("empty legacy list reported a change")
+		}
+		malformed := &DownloaderStoreConfig{
+			Extra: map[string]any{"downloadCategories": "not-a-list"},
+		}
+		if malformed.MigrateLegacyExtraCategories() {
+			t.Fatal("malformed legacy list reported a change")
+		}
+		if _, ok := malformed.Extra["downloadCategories"]; !ok {
+			t.Fatal("malformed legacy key must stay untouched")
+		}
+	})
+
+	t.Run("a nil entry in the legacy list is skipped", func(t *testing.T) {
+		cfg := &DownloaderStoreConfig{
+			DownloadDir: "/downloads",
+			Extra: map[string]any{
+				"downloadCategories": []any{
+					nil,
+					map[string]any{"name": "Games", "path": "/old/Games"},
+				},
+			},
+		}
+		if !cfg.MigrateLegacyExtraCategories() {
+			t.Fatal("migration reported no change")
+		}
+		if len(cfg.Categories) != 1 || cfg.Categories[0].Name != "Games" {
+			t.Fatalf("categories = %+v, want the single valid legacy entry", cfg.Categories)
+		}
+	})
+}
+
+func TestDownloaderStoreConfig_AutoCategorizeJSON(t *testing.T) {
+	unset, err := json.Marshal(&DownloaderStoreConfig{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if strings.Contains(string(unset), `"autoCategorize"`) {
+		t.Fatalf("unset autoCategorize must be omitted from the api json, got %s", unset)
+	}
+	off, err := json.Marshal(&DownloaderStoreConfig{AutoCategorize: util.BoolPtr(false)})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !strings.Contains(string(off), `"autoCategorize":false`) {
+		t.Fatalf("explicit autoCategorize=false must be emitted, got %s", off)
+	}
+}
+
 func TestDownloaderStoreConfig_Merge(t *testing.T) {
 	type args struct {
 		beforeCfg *DownloaderStoreConfig
@@ -236,6 +437,24 @@ func TestDownloaderStoreConfig_Merge(t *testing.T) {
 				DownloadDir: "after",
 			},
 		},
+		{
+			"Merge Categories and AutoCategorize Override",
+			&DownloaderStoreConfig{},
+			args{
+				beforeCfg: &DownloaderStoreConfig{
+					Categories: []*DownloadCategory{
+						{Name: "Music", Path: "/downloads/Music"},
+					},
+					AutoCategorize: util.BoolPtr(true),
+				},
+			},
+			&DownloaderStoreConfig{
+				Categories: []*DownloadCategory{
+					{Name: "Music", Path: "/downloads/Music"},
+				},
+				AutoCategorize: util.BoolPtr(true),
+			},
+		},
 		{
 			"Merge MaxRunning No Override",
 			&DownloaderStoreConfig{
@@ -449,6 +668,8 @@ func TestDownloaderStoreConfig_Merge(t *testing.T) {
 				Webhook:        tt.fields.Webhook,
 				AutoTorrent:    tt.fields.AutoTorrent,
 				Archive:        tt.fields.Archive,
+				Categories:     tt.fields.Categories,
+				AutoCategorize: tt.fields.AutoCategorize,
 			}
 			if got := cfg.Merge(tt.args.beforeCfg); !reflect.DeepEqual(got, tt.want) {
 				t.Errorf("Merge() = %v, want %v", got, tt.want)