mirror of
https://github.com/GopeedLab/gopeed.git
synced 2026-11-04 15:49:49 +00:00
516 lines
16 KiB
Go
516 lines
16 KiB
Go
package base
|
|
|
|
import (
|
|
"encoding/json"
|
|
"fmt"
|
|
"net/http"
|
|
"net/url"
|
|
"path/filepath"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/GopeedLab/gopeed/pkg/util"
|
|
"github.com/mattn/go-ieproxy"
|
|
"golang.org/x/exp/slices"
|
|
)
|
|
|
|
// Request download request
|
|
type Request struct {
|
|
RawURL string `json:"rawUrl"`
|
|
URL string `json:"url"`
|
|
Extra any `json:"extra"`
|
|
// Labels is used to mark the download task
|
|
Labels map[string]string `json:"labels"`
|
|
// Proxy is special proxy config for request
|
|
Proxy *RequestProxy `json:"proxy"`
|
|
// SkipVerifyCert is the flag that skip verify cert
|
|
SkipVerifyCert bool `json:"skipVerifyCert"`
|
|
}
|
|
|
|
// SetLabels replaces all labels on the request.
|
|
//
|
|
// Deprecated: In onResolve and onDone, assign request labels directly.
|
|
// In onStart and onError, use task.meta.req.setLabels instead.
|
|
func (r *Request) SetLabels(labels map[string]string) {
|
|
r.Labels = labels
|
|
}
|
|
|
|
// PutLabel sets a label on the request.
|
|
//
|
|
// Deprecated: In onResolve and onDone, assign request labels directly.
|
|
// In onStart and onError, use task.meta.req.putLabel instead.
|
|
func (r *Request) PutLabel(key, value string) {
|
|
if r.Labels == nil {
|
|
r.Labels = make(map[string]string)
|
|
}
|
|
r.Labels[key] = value
|
|
}
|
|
|
|
// DelLabel deletes a label from the request.
|
|
//
|
|
// Deprecated: In onResolve and onDone, delete request labels directly.
|
|
// In onStart and onError, use task.meta.req.delLabel instead.
|
|
func (r *Request) DelLabel(key string) {
|
|
delete(r.Labels, key)
|
|
}
|
|
|
|
func (r *Request) Validate() error {
|
|
if r.URL == "" {
|
|
return fmt.Errorf("invalid request url")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
type RequestProxyMode string
|
|
|
|
const (
|
|
// RequestProxyModeFollow follow setting proxy
|
|
RequestProxyModeFollow RequestProxyMode = "follow"
|
|
// RequestProxyModeNone not use proxy
|
|
RequestProxyModeNone RequestProxyMode = "none"
|
|
// RequestProxyModeCustom custom proxy
|
|
RequestProxyModeCustom RequestProxyMode = "custom"
|
|
)
|
|
|
|
type RequestProxy struct {
|
|
Mode RequestProxyMode `json:"mode"`
|
|
Scheme string `json:"scheme"`
|
|
Host string `json:"host"`
|
|
Usr string `json:"usr"`
|
|
Pwd string `json:"pwd"`
|
|
}
|
|
|
|
func (p *RequestProxy) ToHandler() func(r *http.Request) (*url.URL, error) {
|
|
if p == nil || p.Mode != RequestProxyModeCustom {
|
|
return nil
|
|
}
|
|
|
|
if p.Scheme == "" || p.Host == "" {
|
|
return nil
|
|
}
|
|
|
|
return http.ProxyURL(util.BuildProxyUrl(p.Scheme, p.Host, p.Usr, p.Pwd))
|
|
}
|
|
|
|
// Resource download resource
|
|
type Resource struct {
|
|
// if name is not empty, the resource is a folder and the name is the folder name
|
|
Name string `json:"name"`
|
|
Size int64 `json:"size"`
|
|
// is support range download
|
|
Range bool `json:"range"`
|
|
// file list
|
|
Files []*FileInfo `json:"files"`
|
|
Hash string `json:"hash"`
|
|
}
|
|
|
|
func (r *Resource) Validate() error {
|
|
if len(r.Files) == 0 {
|
|
return fmt.Errorf("invalid resource files")
|
|
}
|
|
for _, file := range r.Files {
|
|
if file.Name == "" {
|
|
return fmt.Errorf("invalid resource file name")
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (r *Resource) CalcSize(selectFiles []int) {
|
|
var size int64
|
|
for i, file := range r.Files {
|
|
if len(selectFiles) == 0 || slices.Contains(selectFiles, i) {
|
|
size += file.Size
|
|
}
|
|
}
|
|
r.Size = size
|
|
}
|
|
|
|
type FileInfo struct {
|
|
Name string `json:"name"`
|
|
Path string `json:"path"`
|
|
Size int64 `json:"size"`
|
|
Ctime *time.Time `json:"ctime"`
|
|
|
|
Req *Request `json:"req"`
|
|
}
|
|
|
|
// Options for download
|
|
type Options struct {
|
|
// Size is an optional HTTP download size estimate. HTTP metadata and clean EOF take precedence.
|
|
// Other protocols ignore it.
|
|
Size int64 `json:"size,omitempty"`
|
|
// Download file name
|
|
Name string `json:"name"`
|
|
// Download file path
|
|
Path string `json:"path"`
|
|
// Use this path as the default download directory after creation succeeds
|
|
AsDefaultPath bool `json:"asDefaultPath,omitempty"`
|
|
// SkipAutoCategorize keeps the base download directory of this task instead
|
|
// of routing it to the category directory of its resolved file
|
|
SkipAutoCategorize bool `json:"skipAutoCategorize,omitempty"`
|
|
// Select file indexes to download
|
|
SelectFiles []int `json:"selectFiles"`
|
|
// Extra info for specific fetcher
|
|
Extra any `json:"extra"`
|
|
}
|
|
|
|
func (o *Options) InitSelectFiles(fileSize int) {
|
|
// if selectFiles is empty, select all files
|
|
if len(o.SelectFiles) == 0 {
|
|
o.SelectFiles = make([]int, fileSize)
|
|
for i := range fileSize {
|
|
o.SelectFiles[i] = i
|
|
}
|
|
}
|
|
}
|
|
|
|
func (o *Options) Clone() *Options {
|
|
return util.DeepClone(o)
|
|
}
|
|
|
|
func ParseReqExtra[E any](req *Request) error {
|
|
if req.Extra == nil {
|
|
return nil
|
|
}
|
|
if _, ok := req.Extra.(*E); ok {
|
|
return nil
|
|
}
|
|
var t E
|
|
if err := util.MapToStruct(req.Extra, &t); err != nil {
|
|
return err
|
|
}
|
|
req.Extra = &t
|
|
return nil
|
|
}
|
|
|
|
func ParseOptExtra[E any](opts *Options) error {
|
|
if opts.Extra == nil {
|
|
return nil
|
|
}
|
|
if _, ok := opts.Extra.(*E); ok {
|
|
return nil
|
|
}
|
|
var t E
|
|
if err := util.MapToStruct(opts.Extra, &t); err != nil {
|
|
return err
|
|
}
|
|
opts.Extra = &t
|
|
return nil
|
|
}
|
|
|
|
type CreateTaskBatch struct {
|
|
Reqs []*CreateTaskBatchItem `json:"reqs"`
|
|
Opts *Options `json:"opts"`
|
|
Group bool `json:"group"`
|
|
GroupName string `json:"groupName"`
|
|
}
|
|
|
|
type CreateTaskBatchItem struct {
|
|
Rid string `json:"rid,omitempty"`
|
|
Req *Request `json:"req"`
|
|
Opts *Options `json:"opts"`
|
|
}
|
|
|
|
// DownloaderStoreConfig is the config that can restore the downloader.
|
|
type DownloaderStoreConfig struct {
|
|
FirstLoad bool `json:"-"` // FirstLoad is the flag that the config is first time init and not from store
|
|
|
|
DownloadDir string `json:"downloadDir"` // DownloadDir is the default directory to save the downloaded files
|
|
MaxRunning int `json:"maxRunning"` // MaxRunning is the max running download count
|
|
ProtocolConfig map[string]any `json:"protocolConfig"` // ProtocolConfig is special config for each protocol
|
|
GithubMirror *GithubMirrorConfig `json:"githubMirror"`
|
|
Extra map[string]any `json:"extra"`
|
|
Proxy *DownloaderProxyConfig `json:"proxy"`
|
|
Webhook *WebhookConfig `json:"webhook"` // Webhook is the webhook configuration
|
|
Script *ScriptConfig `json:"script"` // Script is the script execution configuration
|
|
AutoTorrent *AutoTorrentConfig `json:"autoTorrent"` // AutoTorrent is the auto torrent task creation configuration
|
|
Archive *ArchiveConfig `json:"archive"` // Archive is the archive extraction configuration
|
|
Categories []*DownloadCategory `json:"categories"` // Categories is the download directory category configuration
|
|
AutoCategorize *bool `json:"autoCategorize,omitempty"` // AutoCategorize routes downloads to category directories by file extension; unset defaults to disabled
|
|
API *APIServerConfig `json:"api"` // API is the optional REST server configuration
|
|
AutoStartTasks bool `json:"autoStartTasks"` // AutoStartTasks continues all unfinished tasks when the backend starts
|
|
AutoDeleteMissingFileTasks bool `json:"autoDeleteMissingFileTasks"` // AutoDeleteMissingFileTasks enables automatic deletion of tasks with missing files
|
|
}
|
|
|
|
func (cfg *DownloaderStoreConfig) Init() *DownloaderStoreConfig {
|
|
if cfg.MaxRunning == 0 {
|
|
cfg.MaxRunning = 5
|
|
}
|
|
if cfg.ProtocolConfig == nil {
|
|
cfg.ProtocolConfig = make(map[string]any)
|
|
}
|
|
if cfg.GithubMirror == nil {
|
|
cfg.GithubMirror = DefaultGithubMirrorConfig()
|
|
}
|
|
if cfg.Proxy == nil {
|
|
cfg.Proxy = &DownloaderProxyConfig{}
|
|
}
|
|
if cfg.Webhook == nil {
|
|
cfg.Webhook = &WebhookConfig{}
|
|
}
|
|
if cfg.Script == nil {
|
|
cfg.Script = &ScriptConfig{}
|
|
}
|
|
if cfg.AutoTorrent == nil {
|
|
cfg.AutoTorrent = &AutoTorrentConfig{
|
|
Enable: false,
|
|
DeleteAfterDownload: false,
|
|
}
|
|
}
|
|
if cfg.Archive == nil {
|
|
cfg.Archive = &ArchiveConfig{
|
|
AutoExtract: false,
|
|
DeleteAfterExtract: false,
|
|
}
|
|
}
|
|
// Seed the built-in categories for deployments that never run the flutter
|
|
// startup flow, e.g. headless api servers.
|
|
if len(cfg.Categories) == 0 && cfg.DownloadDir != "" {
|
|
cfg.Categories = []*DownloadCategory{
|
|
builtinSeedCategory("categoryMusic", "Music", cfg.DownloadDir),
|
|
builtinSeedCategory("categoryVideo", "Video", cfg.DownloadDir),
|
|
builtinSeedCategory("categoryDocument", "Document", cfg.DownloadDir),
|
|
builtinSeedCategory("categoryProgram", "Program", cfg.DownloadDir),
|
|
}
|
|
}
|
|
return cfg
|
|
}
|
|
|
|
// builtinCategoryExtensions mirrors the flutter side
|
|
// kDefaultCategoryExtensions, keep both maps in sync.
|
|
var builtinCategoryExtensions = map[string][]string{
|
|
"categoryProgram": {"exe", "msi", "msix", "apk", "dmg", "deb", "rpm", "pkg", "appimage"},
|
|
"categoryVideo": {"mp4", "mkv", "avi", "mov", "wmv", "flv", "webm", "m4v", "mpg", "mpeg", "3gp"},
|
|
"categoryMusic": {"mp3", "flac", "wav", "aac", "ogg", "m4a", "wma", "ape"},
|
|
"categoryDocument": {"pdf", "doc", "docx", "xls", "xlsx", "ppt", "pptx", "txt", "md", "csv", "epub", "rtf"},
|
|
}
|
|
|
|
func builtinSeedCategory(nameKey, name, downloadDir string) *DownloadCategory {
|
|
return &DownloadCategory{
|
|
Name: "",
|
|
Path: filepath.Join(downloadDir, name),
|
|
NameKey: nameKey,
|
|
IsBuiltIn: true,
|
|
Extensions: append([]string(nil), builtinCategoryExtensions[nameKey]...),
|
|
}
|
|
}
|
|
|
|
// MigrateLegacyExtraCategories moves the extra.downloadCategories list written
|
|
// by v2.0.0-beta builds into the top level categories field and clears the
|
|
// legacy key. The legacy list predates extension based routing, so built-in
|
|
// entries regain their default extension lists from their name key, while user
|
|
// created folders keep their empty list until edited in settings. It reports
|
|
// whether the config changed so the caller can persist the cleared key.
|
|
func (cfg *DownloaderStoreConfig) MigrateLegacyExtraCategories() bool {
|
|
if len(cfg.Categories) > 0 || cfg.Extra == nil {
|
|
return false
|
|
}
|
|
raw, ok := cfg.Extra["downloadCategories"]
|
|
if !ok {
|
|
return false
|
|
}
|
|
data, err := json.Marshal(raw)
|
|
if err != nil {
|
|
return false
|
|
}
|
|
var legacy []*DownloadCategory
|
|
if err := json.Unmarshal(data, &legacy); err != nil {
|
|
return false
|
|
}
|
|
categories := make([]*DownloadCategory, 0, len(legacy))
|
|
for _, category := range legacy {
|
|
if category == nil {
|
|
continue
|
|
}
|
|
if extensions, exist := builtinCategoryExtensions[category.NameKey]; exist && len(category.Extensions) == 0 {
|
|
category.Extensions = append([]string(nil), extensions...)
|
|
}
|
|
categories = append(categories, category)
|
|
}
|
|
if len(categories) == 0 {
|
|
return false
|
|
}
|
|
cfg.Categories = categories
|
|
delete(cfg.Extra, "downloadCategories")
|
|
return true
|
|
}
|
|
|
|
// AutoCategorizeEnabled reports whether downloads are routed to category
|
|
// directories by file extension. The switch is optional: an unset value keeps
|
|
// the feature off, so an older config that never saved the field stays off.
|
|
func (cfg *DownloaderStoreConfig) AutoCategorizeEnabled() bool {
|
|
return cfg.AutoCategorize != nil && *cfg.AutoCategorize
|
|
}
|
|
|
|
func (cfg *DownloaderStoreConfig) Merge(beforeCfg *DownloaderStoreConfig) *DownloaderStoreConfig {
|
|
if beforeCfg == nil {
|
|
return cfg
|
|
}
|
|
if cfg.DownloadDir == "" {
|
|
cfg.DownloadDir = beforeCfg.DownloadDir
|
|
}
|
|
if cfg.MaxRunning == 0 {
|
|
cfg.MaxRunning = beforeCfg.MaxRunning
|
|
}
|
|
if cfg.ProtocolConfig == nil {
|
|
cfg.ProtocolConfig = beforeCfg.ProtocolConfig
|
|
}
|
|
if cfg.Extra == nil {
|
|
cfg.Extra = beforeCfg.Extra
|
|
}
|
|
if cfg.GithubMirror == nil {
|
|
cfg.GithubMirror = beforeCfg.GithubMirror
|
|
}
|
|
if cfg.Proxy == nil {
|
|
cfg.Proxy = beforeCfg.Proxy
|
|
}
|
|
if cfg.Webhook == nil {
|
|
cfg.Webhook = beforeCfg.Webhook
|
|
}
|
|
if cfg.Script == nil {
|
|
cfg.Script = beforeCfg.Script
|
|
}
|
|
if cfg.AutoTorrent == nil {
|
|
cfg.AutoTorrent = beforeCfg.AutoTorrent
|
|
}
|
|
if cfg.Archive == nil {
|
|
cfg.Archive = beforeCfg.Archive
|
|
}
|
|
if cfg.Categories == nil {
|
|
cfg.Categories = beforeCfg.Categories
|
|
}
|
|
if cfg.AutoCategorize == nil {
|
|
cfg.AutoCategorize = beforeCfg.AutoCategorize
|
|
}
|
|
if cfg.API == nil {
|
|
cfg.API = beforeCfg.API
|
|
}
|
|
return cfg
|
|
}
|
|
|
|
type APIServerConfig struct {
|
|
Enable bool `json:"enable"`
|
|
MCPEnable bool `json:"mcpEnable"`
|
|
Network string `json:"network"`
|
|
Address string `json:"address"`
|
|
Token string `json:"token"`
|
|
}
|
|
|
|
func (cfg *APIServerConfig) Init() *APIServerConfig {
|
|
if cfg.Network == "" {
|
|
cfg.Network = "tcp"
|
|
}
|
|
if cfg.Address == "" {
|
|
cfg.Address = "127.0.0.1:9999"
|
|
}
|
|
return cfg
|
|
}
|
|
|
|
// WebhookConfig is the webhook configuration
|
|
type WebhookConfig struct {
|
|
Enable bool `json:"enable"` // Enable is the flag to enable/disable webhooks
|
|
URLs []string `json:"urls"` // URLs is the list of webhook URLs
|
|
}
|
|
|
|
// ScriptConfig is the script execution configuration
|
|
type ScriptConfig struct {
|
|
Enable bool `json:"enable"` // Enable is the flag to enable/disable script execution
|
|
Paths []string `json:"paths"` // Paths is the list of script paths to execute
|
|
}
|
|
|
|
// AutoTorrentConfig is the auto torrent task creation configuration
|
|
type AutoTorrentConfig struct {
|
|
Enable bool `json:"enable"` // Enable enables automatic BT task creation when downloading .torrent files
|
|
DeleteAfterDownload bool `json:"deleteAfterDownload"` // DeleteAfterDownload deletes the .torrent file after BT task creation
|
|
}
|
|
|
|
// ArchiveConfig is the archive extraction configuration
|
|
type ArchiveConfig struct {
|
|
AutoExtract bool `json:"autoExtract"` // AutoExtract enables automatic extraction of archives after download
|
|
DeleteAfterExtract bool `json:"deleteAfterExtract"` // DeleteAfterExtract deletes the archive after successful extraction
|
|
}
|
|
|
|
// DownloadCategory is a download directory category that files can be routed to by extension
|
|
type DownloadCategory struct {
|
|
Name string `json:"name"` // Name is the category display name, empty for built-in categories that use NameKey
|
|
Path string `json:"path"` // Path is the download directory of the category
|
|
NameKey string `json:"nameKey,omitempty"` // NameKey is the i18n key of the built-in category name
|
|
IsBuiltIn bool `json:"isBuiltIn,omitempty"` // IsBuiltIn marks the category as built-in
|
|
IsDeleted bool `json:"isDeleted,omitempty"` // IsDeleted marks the built-in category as deleted
|
|
Extensions []string `json:"extensions,omitempty"` // Extensions is the file extension list routed to the category, without leading dots
|
|
}
|
|
|
|
type DownloaderProxyConfig struct {
|
|
Enable bool `json:"enable"`
|
|
// System is the flag that use system proxy
|
|
System bool `json:"system"`
|
|
Scheme string `json:"scheme"`
|
|
Host string `json:"host"`
|
|
Usr string `json:"usr"`
|
|
Pwd string `json:"pwd"`
|
|
}
|
|
|
|
func (cfg *DownloaderProxyConfig) ToHandler() func(r *http.Request) (*url.URL, error) {
|
|
if cfg == nil || cfg.Enable == false {
|
|
return nil
|
|
}
|
|
if cfg.System {
|
|
safeProxyReloadConf()
|
|
return ieproxy.GetProxyFunc()
|
|
}
|
|
if cfg.Scheme == "" || cfg.Host == "" {
|
|
return nil
|
|
}
|
|
return http.ProxyURL(util.BuildProxyUrl(cfg.Scheme, cfg.Host, cfg.Usr, cfg.Pwd))
|
|
}
|
|
|
|
// ToUrl returns the proxy url, just for git clone
|
|
func (cfg *DownloaderProxyConfig) ToUrl() *url.URL {
|
|
if cfg == nil || cfg.Enable == false {
|
|
return nil
|
|
}
|
|
if cfg.System {
|
|
safeProxyReloadConf()
|
|
static := ieproxy.GetConf().Static
|
|
if static.Active && len(static.Protocols) > 0 {
|
|
// If only one protocol, use it
|
|
if len(static.Protocols) == 1 {
|
|
for _, v := range static.Protocols {
|
|
return parseUrlSafe(v)
|
|
}
|
|
}
|
|
// Check https
|
|
if v, ok := static.Protocols["https"]; ok {
|
|
return parseUrlSafe(v)
|
|
}
|
|
// Check http
|
|
if v, ok := static.Protocols["http"]; ok {
|
|
return parseUrlSafe(v)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
if cfg.Scheme == "" || cfg.Host == "" {
|
|
return nil
|
|
}
|
|
return util.BuildProxyUrl(cfg.Scheme, cfg.Host, cfg.Usr, cfg.Pwd)
|
|
}
|
|
|
|
var prcLock sync.Mutex
|
|
|
|
func safeProxyReloadConf() {
|
|
prcLock.Lock()
|
|
defer prcLock.Unlock()
|
|
|
|
ieproxy.ReloadConf()
|
|
}
|
|
|
|
func parseUrlSafe(rawUrl string) *url.URL {
|
|
u, err := url.Parse(rawUrl)
|
|
if err != nil {
|
|
return nil
|
|
}
|
|
return u
|
|
}
|