feat(protocol): add FTP, FTPS and FTPES downloads (#1569)

pkg/download/downloader.go

@@ -25,6 +25,7 @@ import (
 	"github.com/GopeedLab/gopeed/internal/tempfiles"
 	webviewproxy "github.com/GopeedLab/gopeed/internal/webview/proxy"
 	"github.com/GopeedLab/gopeed/pkg/base"
+	"github.com/GopeedLab/gopeed/pkg/protocol/ftp"
 	"github.com/GopeedLab/gopeed/pkg/protocol/http"
 	"github.com/GopeedLab/gopeed/pkg/util"
 	gonanoid "github.com/matoous/go-nanoid/v2"
@@ -459,6 +460,9 @@ func (d *Downloader) setupFetcher(fm fetcher.FetcherManager, fetcher fetcher.Fet
 	ctl.GetConfig = func(v any) {
 		d.getProtocolConfig(fm.Name(), v)
 	}
+	if d.Logger != nil {
+		ctl.Logger = &d.Logger.Logger
+	}
 	// Get proxy config, task request proxy config has higher priority, then use global proxy config
 	ctl.GetProxy = func(requestProxy *base.RequestProxy) func(*gohttp.Request) (*url.URL, error) {
 		// Registry-backed blob URLs are loopback capability URLs and must never be
@@ -1594,7 +1598,7 @@ func (d *Downloader) watch(task *Task) {
 		d.triggerWebhooks(WebhookEventDownloadDone, task, nil)
 		d.triggerScripts(ScriptEventDownloadDone, task, nil)
 
-		if e, ok := task.Meta.Opts.Extra.(*http.OptsExtra); ok {
+		if e := postDownloadExtra(task.Meta); e != nil {
 			downloadFilePath := task.Meta.SingleFilepath()
 
 			cfg, _ := d.GetConfig()
@@ -1654,6 +1658,33 @@ func (d *Downloader) watch(task *Task) {
 	}
 }
 
+// postDownloadExtra returns the auto-torrent and auto-extract options of a
+// finished single-file task, in the HTTP options shape the extraction code
+// takes. FTP options carry the same fields. An FTP folder has no single file
+// to act on, so it gets nil, as does any other protocol.
+func postDownloadExtra(meta *fetcher.FetcherMeta) *http.OptsExtra {
+	if meta == nil || meta.Opts == nil {
+		return nil
+	}
+	switch e := meta.Opts.Extra.(type) {
+	case *http.OptsExtra:
+		return e
+	case *ftp.OptsExtra:
+		if meta.Res == nil || meta.Res.Name != "" || len(meta.Res.Files) != 1 {
+			return nil
+		}
+		return &http.OptsExtra{
+			Connections:                e.Connections,
+			AutoTorrent:                e.AutoTorrent,
+			DeleteTorrentAfterDownload: e.DeleteTorrentAfterDownload,
+			AutoExtract:                e.AutoExtract,
+			ArchivePassword:            e.ArchivePassword,
+			DeleteAfterExtract:         e.DeleteAfterExtract,
+		}
+	}
+	return nil
+}
+
 func (d *Downloader) doOnError(task *Task, err error) {
 	d.handleOnError(task, err, false)
 }

pkg/download/ftp_test.go

@@ -0,0 +1,57 @@
+package download
+
+import (
+	"testing"
+
+	"github.com/GopeedLab/gopeed/internal/fetcher"
+	"github.com/GopeedLab/gopeed/pkg/base"
+	"github.com/GopeedLab/gopeed/pkg/protocol/ftp"
+	"github.com/GopeedLab/gopeed/pkg/protocol/http"
+)
+
+// TestTorrentNameIsFile checks the default manager order: an FTP URL whose
+// name looks like a torrent or a playlist is still an FTP download.
+func TestTorrentNameIsFile(t *testing.T) {
+	d := NewDownloader(nil)
+	for u, want := range map[string]string{
+		"ftp://h/x.torrent":    "ftp",
+		"ftp://h/v.m3u8":       "ftp",
+		"ftps://h/x.torrent":   "ftp",
+		"ftpes://h/a.bin":      "ftp",
+		"https://h/v.m3u8":     "hls",
+		"https://h/x.torrent":  "http",
+		"magnet:?xt=urn:btih:": "bt",
+	} {
+		fm, err := d.parseFm(u)
+		if err != nil {
+			t.Fatalf("%s: %v", u, err)
+		}
+		if fm.Name() != want {
+			t.Fatalf("%s goes to %q, want %q", u, fm.Name(), want)
+		}
+	}
+}
+
+func TestPostDownloadExtraAcceptsFTP(t *testing.T) {
+	yes := true
+	file := &base.Resource{Files: []*base.FileInfo{{Name: "a.zip"}}}
+	folder := &base.Resource{Name: "dir", Files: []*base.FileInfo{{Name: "a.zip"}}}
+
+	httpExtra := &http.OptsExtra{AutoExtract: &yes}
+	if got := postDownloadExtra(&fetcher.FetcherMeta{Res: file, Opts: &base.Options{Extra: httpExtra}}); got != httpExtra {
+		t.Fatalf("HTTP options must pass through unchanged, got %+v", got)
+	}
+
+	ftpExtra := &ftp.OptsExtra{Connections: 4, AutoTorrent: &yes, DeleteTorrentAfterDownload: &yes, AutoExtract: &yes, ArchivePassword: "pw", DeleteAfterExtract: true}
+	got := postDownloadExtra(&fetcher.FetcherMeta{Res: file, Opts: &base.Options{Extra: ftpExtra}})
+	if got == nil || got.AutoTorrent != &yes || got.DeleteTorrentAfterDownload != &yes || got.AutoExtract != &yes ||
+		got.ArchivePassword != "pw" || !got.DeleteAfterExtract {
+		t.Fatalf("FTP options were not carried over: %+v", got)
+	}
+	if got := postDownloadExtra(&fetcher.FetcherMeta{Res: folder, Opts: &base.Options{Extra: ftpExtra}}); got != nil {
+		t.Fatalf("an FTP folder has no single file to extract, got %+v", got)
+	}
+	if got := postDownloadExtra(&fetcher.FetcherMeta{Res: file, Opts: &base.Options{}}); got != nil {
+		t.Fatalf("no options, got %+v", got)
+	}
+}

pkg/download/model.go

@@ -10,6 +10,7 @@ import (
 	"github.com/GopeedLab/gopeed/internal/production"
 	"github.com/GopeedLab/gopeed/internal/protocol/bt"
 	"github.com/GopeedLab/gopeed/internal/protocol/ed2k"
+	"github.com/GopeedLab/gopeed/internal/protocol/ftp"
 	"github.com/GopeedLab/gopeed/internal/protocol/hls"
 	"github.com/GopeedLab/gopeed/internal/protocol/http"
 	"github.com/GopeedLab/gopeed/pkg/base"
@@ -229,6 +230,10 @@ func (cfg *DownloaderConfig) Init() *DownloaderConfig {
 	}
 	if len(cfg.FetchManagers) == 0 {
 		cfg.FetchManagers = []fetcher.FetcherManager{
+			// FTP matches by URL scheme only. It comes before the managers that
+			// match by file extension, so ftp://host/x.torrent or x.m3u8 is
+			// downloaded as a file over FTP.
+			new(ftp.FetcherManager),
 			new(hls.FetcherManager),
 			new(http.FetcherManager),
 			new(bt.FetcherManager),

pkg/protocol/ftp/model.go

@@ -0,0 +1,54 @@
+package ftp
+
+// TLS modes for the "tls" field of an ftp:// request. ftps:// URLs always use
+// implicit TLS and ftpes:// URLs always use explicit TLS.
+const (
+	TLSNone     = "none"
+	TLSExplicit = "explicit"
+	TLSImplicit = "implicit"
+)
+
+// ReqExtra is the extra of an FTP request. The TLS mode is part of the request,
+// because resolving the link already needs a connection.
+type ReqExtra struct {
+	// TLS upgrades a plain ftp:// URL: "none" (default), "explicit" (AUTH TLS)
+	// or "implicit". It is ignored for ftps:// and ftpes:// URLs.
+	TLS string `json:"tls"`
+}
+
+type OptsExtra struct {
+	// Connections is the maximum number of parallel logins. Servers often cap
+	// logins per user, so the fetcher lowers it when the server refuses one.
+	Connections int `json:"connections"`
+	// AutoTorrent when task download complete, and it is a .torrent file, it will be auto create a new task for the torrent file
+	// nil means use global config, true/false means explicit setting
+	AutoTorrent *bool `json:"autoTorrent"`
+	// DeleteTorrentAfterDownload when true, deletes the .torrent file after creating BT task
+	// nil means use global config, true/false means explicit setting
+	DeleteTorrentAfterDownload *bool `json:"deleteTorrentAfterDownload"`
+	// AutoExtract when task download complete, and it is an archive file, it will be auto extracted
+	// nil means use global config, true/false means explicit setting
+	AutoExtract *bool `json:"autoExtract"`
+	// ArchivePassword is the password for extracting password-protected archives
+	ArchivePassword string `json:"archivePassword"`
+	// DeleteAfterExtract when true, deletes the archive file after successful extraction
+	DeleteAfterExtract bool `json:"deleteAfterExtract"`
+}
+
+// Stats for download. It has the same shape as the HTTP protocol's stats, so
+// clients can show FTP connections the way they show HTTP connections.
+type Stats struct {
+	Connections []*StatsConnection `json:"connections"`
+}
+
+// StatsConnection describes one login that took part in the download.
+type StatsConnection struct {
+	Downloaded int64 `json:"downloaded"`
+	// Total is the resource size divided by the number of connections,
+	// rounded up to a whole byte, as in the HTTP protocol's stats. It is zero
+	// when the resource size is unknown.
+	Total      int64 `json:"total"`
+	Completed  bool  `json:"completed"`
+	Failed     bool  `json:"failed"`
+	RetryTimes int   `json:"retryTimes"`
+}