feat(protocol): add FTP, FTPS and FTPES downloads (#1569)

internal/controller/controller.go

@@ -3,6 +3,7 @@ package controller
 import (
 	"github.com/GopeedLab/gopeed/internal/tempfiles"
 	"github.com/GopeedLab/gopeed/pkg/base"
+	"github.com/rs/zerolog"
 	"net/http"
 	"net/url"
 	"os"
@@ -15,6 +16,9 @@ type Controller struct {
 	TempDir           string
 	GetConfig         func(v any)
 	GetProxy          func(requestProxy *base.RequestProxy) func(*http.Request) (*url.URL, error)
+	// Logger is the downloader's core logger. It may be nil, in which case
+	// fetchers log nothing.
+	Logger *zerolog.Logger
 	FileController
 	//ContextDialer() (proxy.Dialer, error)
 }

internal/protocol/ftp/conn.go

@@ -0,0 +1,404 @@
+package ftp
+
+import (
+	"context"
+	"crypto/tls"
+	"errors"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"net/textproto"
+	"net/url"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/jlaffaye/ftp"
+	"golang.org/x/net/proxy"
+
+	pftp "github.com/GopeedLab/gopeed/pkg/protocol/ftp"
+)
+
+const (
+	connectTimeout = 15 * time.Second
+	readTimeout    = 15 * time.Second
+	quitTimeout    = 2 * time.Second
+)
+
+// target is a parsed FTP URL. It holds the password, so it must never be
+// formatted into an error or a log line.
+type target struct {
+	host    string // host name or IP, the TLS server name
+	addr    string // host:port
+	user    string
+	pass    string
+	path    string // path relative to the login directory, "" for the directory itself
+	tlsMode string
+}
+
+// parseTarget reads an ftp, ftps or ftpes URL. The path follows RFC 1738: it
+// is relative to the login directory, and an encoded leading slash
+// (ftp://host/%2Fpub) makes it absolute.
+func parseTarget(rawURL, optTLS string) (*target, error) {
+	u, err := url.Parse(rawURL)
+	if err != nil {
+		// url.Error repeats the whole URL, password included.
+		return nil, errors.New("ftp: invalid URL")
+	}
+	t := &target{}
+	switch strings.ToLower(u.Scheme) {
+	case "ftps":
+		t.tlsMode = pftp.TLSImplicit
+	case "ftpes":
+		t.tlsMode = pftp.TLSExplicit
+	case "ftp":
+		switch strings.ToLower(optTLS) {
+		case "", pftp.TLSNone:
+			t.tlsMode = pftp.TLSNone
+		case pftp.TLSExplicit:
+			t.tlsMode = pftp.TLSExplicit
+		case pftp.TLSImplicit:
+			t.tlsMode = pftp.TLSImplicit
+		default:
+			return nil, fmt.Errorf("ftp: unknown tls mode %q", optTLS)
+		}
+	default:
+		return nil, fmt.Errorf("ftp: unsupported scheme %q", u.Scheme)
+	}
+	t.host = u.Hostname()
+	if t.host == "" {
+		return nil, errors.New("ftp: URL has no host")
+	}
+	port := u.Port()
+	if port == "" {
+		port = "21"
+		if t.tlsMode == pftp.TLSImplicit {
+			port = "990"
+		}
+	}
+	t.addr = net.JoinHostPort(t.host, port)
+	if u.User != nil {
+		// Username and Password are already percent-decoded.
+		t.user = u.User.Username()
+		t.pass, _ = u.User.Password()
+	}
+	if t.user == "" {
+		t.user, t.pass = "anonymous", "anonymous@"
+	}
+	t.path = strings.TrimPrefix(u.Path, "/")
+	return t, nil
+}
+
+type dialContextFunc func(ctx context.Context, network, addr string) (net.Conn, error)
+
+// session is one logged-in control connection. It owns the network
+// connections it opens, so abort can unblock any read or write in progress.
+type session struct {
+	*ftp.ServerConn
+
+	ctrl     net.Conn // control connection as dialed, under any TLS layer
+	dataHost string   // where data connections go
+
+	mu     sync.Mutex
+	data   net.Conn // current data connection as dialed
+	closed bool
+	stop   func() bool
+}
+
+// deadline bounds the next control-connection exchange. The control
+// connection is idle during a transfer, so every exchange sets a new one.
+func (s *session) deadline() {
+	_ = s.ctrl.SetDeadline(time.Now().Add(readTimeout))
+}
+
+// clearDeadline lets the control connection stay idle through a transfer.
+func (s *session) clearDeadline() {
+	_ = s.ctrl.SetDeadline(time.Time{})
+}
+
+// abort closes every connection at once. Pending calls fail immediately.
+func (s *session) abort() {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	s.closed = true
+	_ = s.ctrl.Close()
+	if s.data != nil {
+		_ = s.data.Close()
+	}
+}
+
+// quit logs out and waits, briefly, for the server to hang up. Servers that
+// cap logins count a session until they have seen it end, so the next login
+// would otherwise be refused.
+func (s *session) quit() {
+	if s.stop != nil {
+		s.stop()
+	}
+	_ = s.ctrl.SetDeadline(time.Now().Add(quitTimeout))
+	if s.ServerConn != nil {
+		_ = s.ServerConn.Quit()
+	}
+	s.abort()
+}
+
+func (s *session) setData(c net.Conn) error {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	if s.closed {
+		_ = c.Close()
+		return net.ErrClosed
+	}
+	s.data = c
+	return nil
+}
+
+// ctrlConn is the control connection handed to the ftp library.
+type ctrlConn struct {
+	net.Conn
+}
+
+// RemoteAddr must be a *net.TCPAddr: the library asserts it. A connection
+// through a proxy may report something else, and its address is not used
+// for data connections anyway (see dialData).
+func (c ctrlConn) RemoteAddr() net.Addr {
+	if a, ok := c.Conn.RemoteAddr().(*net.TCPAddr); ok {
+		return a
+	}
+	return &net.TCPAddr{}
+}
+
+// Close is called by the library's Quit right after it sends QUIT. Reading
+// until the server hangs up (bounded by the deadline quit set) tells us the
+// server has ended the session.
+func (c ctrlConn) Close() error {
+	_, _ = io.Copy(io.Discard, c.Conn)
+	return c.Conn.Close()
+}
+
+// dial opens a control connection, upgrades it to TLS when asked, and logs in.
+//
+// The library is given a dial function for every connection. That is how
+// data connections go through the same proxy as the control connection, but
+// it also means the library does no TLS on them: with a dial function it
+// neither wraps an implicit-TLS control connection nor any data connection.
+// We do both here. Explicit TLS (AUTH TLS) is still done by the library, and
+// so are PBSZ and PROT P after login. For the same reason the library's
+// DialWithTimeout and DialWithContext would have no effect: connectTimeout
+// and ctx are applied to our own dials instead.
+func (f *Fetcher) dial(ctx context.Context, t *target) (*session, error) {
+	netDial, proxied, err := f.netDialer(t)
+	if err != nil {
+		return nil, err
+	}
+	var tlsConf *tls.Config
+	if t.tlsMode != pftp.TLSNone {
+		tlsConf = f.tlsConfig(t.host)
+	}
+
+	dialCtx, cancel := context.WithTimeout(ctx, connectTimeout)
+	defer cancel()
+	raw, err := netDial(dialCtx, "tcp", t.addr)
+	if err != nil {
+		return nil, err
+	}
+	ctrl := raw
+	if t.tlsMode == pftp.TLSImplicit {
+		tc := tls.Client(raw, tlsConf)
+		if err := tc.HandshakeContext(dialCtx); err != nil {
+			_ = raw.Close()
+			return nil, err
+		}
+		ctrl = tc
+	}
+
+	s := &session{ctrl: raw, dataHost: t.host}
+	if a, ok := raw.RemoteAddr().(*net.TCPAddr); ok && !proxied {
+		// Same IP as the control connection, as the library itself does,
+		// so a name with several addresses cannot send us elsewhere.
+		s.dataHost = a.IP.String()
+	}
+	s.stop = context.AfterFunc(ctx, s.abort)
+	s.deadline()
+
+	handedOut := false
+	opts := []ftp.DialOption{
+		ftp.DialWithShutTimeout(readTimeout),
+		ftp.DialWithDialFunc(func(network, addr string) (net.Conn, error) {
+			if !handedOut {
+				// Dial asks for the control connection first, once.
+				handedOut = true
+				return ctrlConn{ctrl}, nil
+			}
+			return s.dialData(ctx, netDial, tlsConf, addr)
+		}),
+	}
+	switch t.tlsMode {
+	case pftp.TLSImplicit:
+		opts = append(opts, ftp.DialWithTLS(tlsConf))
+	case pftp.TLSExplicit:
+		opts = append(opts, ftp.DialWithExplicitTLS(tlsConf))
+	}
+	c, err := ftp.Dial(t.addr, opts...)
+	if err != nil {
+		s.stop()
+		s.abort()
+		return nil, err
+	}
+	s.ServerConn = c
+	s.deadline()
+	if err := c.Login(t.user, t.pass); err != nil {
+		s.stop()
+		s.abort()
+		return nil, err
+	}
+	return s, nil
+}
+
+// dialData opens a passive data connection. The library takes the host from
+// the control connection's peer, which through a proxy is the proxy, so only
+// the port is used here.
+func (s *session) dialData(ctx context.Context, netDial dialContextFunc, tlsConf *tls.Config, addr string) (net.Conn, error) {
+	_, port, err := net.SplitHostPort(addr)
+	if err != nil {
+		return nil, err
+	}
+	dialCtx, cancel := context.WithTimeout(ctx, connectTimeout)
+	defer cancel()
+	raw, err := netDial(dialCtx, "tcp", net.JoinHostPort(s.dataHost, port))
+	if err != nil {
+		return nil, err
+	}
+	if err := s.setData(raw); err != nil {
+		return nil, err
+	}
+	if tlsConf != nil {
+		// The handshake runs on the first read, as in the library, because
+		// some servers only start it once the transfer command is answered.
+		return tls.Client(raw, tlsConf), nil
+	}
+	return raw, nil
+}
+
+// tlsConfig returns the TLS settings for one control connection and its data
+// connections. Certificates are verified unless the request sets
+// skipVerifyCert, as for HTTP.
+//
+// Each call gets its own one-entry session cache, so a data connection
+// resumes the TLS session of its own control connection. vsftpd
+// (require_ssl_reuse) only checks that some session was resumed, but ProFTPD's
+// mod_tls also checks that it is the control connection's; a cache shared by
+// several workers would hand a data connection another worker's session.
+// A cache in the manager's base config is kept as given.
+func (f *Fetcher) tlsConfig(host string) *tls.Config {
+	cfg := &tls.Config{}
+	if f.manager != nil && f.manager.baseTLS != nil {
+		cfg = f.manager.baseTLS.Clone()
+	}
+	cfg.ServerName = host
+	if f.meta.Req != nil && f.meta.Req.SkipVerifyCert {
+		cfg.InsecureSkipVerify = true
+	}
+	if cfg.ClientSessionCache == nil {
+		cfg.ClientSessionCache = tls.NewLRUClientSessionCache(1)
+	}
+	return cfg
+}
+
+// netDialer picks how to reach the server, from the same proxy settings the
+// HTTP protocol reads (the task's proxy, otherwise the global one).
+func (f *Fetcher) netDialer(t *target) (dialContextFunc, bool, error) {
+	direct := (&net.Dialer{Timeout: connectTimeout}).DialContext
+	pu, err := f.proxyURL(t)
+	if err != nil {
+		return nil, false, err
+	}
+	if pu == nil {
+		return direct, false, nil
+	}
+	switch strings.ToLower(pu.Scheme) {
+	case "socks5", "socks5h":
+		d, err := proxy.FromURL(pu, &net.Dialer{Timeout: connectTimeout})
+		if err != nil {
+			return nil, false, fmt.Errorf("ftp: socks5 proxy: %w", err)
+		}
+		if cd, ok := d.(proxy.ContextDialer); ok {
+			return cd.DialContext, true, nil
+		}
+		return func(_ context.Context, network, addr string) (net.Conn, error) {
+			return d.Dial(network, addr)
+		}, true, nil
+	default:
+		// An HTTP proxy is not used. Every passive data connection goes to
+		// a port the server picks, and HTTP proxies usually allow CONNECT to
+		// port 443 only, so the transfer would fail after the login worked.
+		f.warnOnce.Do(func() {
+			if l := f.logger(); l != nil {
+				l.Warn().Msgf("ftp: the %s proxy at %s cannot carry FTP data connections (HTTP proxy), connecting directly; use a SOCKS5 proxy for FTP", strings.ToLower(pu.Scheme), pu.Host)
+			}
+		})
+		return direct, false, nil
+	}
+}
+
+func (f *Fetcher) proxyURL(t *target) (*url.URL, error) {
+	if f.ctl == nil || f.ctl.GetProxy == nil {
+		return nil, nil
+	}
+	handler := f.ctl.GetProxy(f.meta.Req.Proxy)
+	if handler == nil {
+		return nil, nil
+	}
+	// The proxy chooser reads only the URL; the credentials stay out.
+	req := &http.Request{URL: &url.URL{Scheme: "ftp", Host: t.addr, Path: "/" + t.path}, Header: http.Header{}}
+	pu, err := handler(req)
+	if err != nil {
+		return nil, errors.New("ftp: cannot read the proxy settings")
+	}
+	return pu, nil
+}
+
+// isLoginRefusal tells a server that is full apart from bad credentials.
+// 421 always means "not now". 530 means a refused login; it is only a
+// capacity limit when these credentials have worked before. The library
+// drops the code when USER itself is refused, so a "too many" message counts
+// as well.
+func isLoginRefusal(err error, credentialsWorked bool) bool {
+	var te *textproto.Error
+	if errors.As(err, &te) {
+		switch te.Code {
+		case ftp.StatusNotAvailable:
+			return true
+		case ftp.StatusNotLoggedIn:
+			return credentialsWorked || strings.Contains(strings.ToLower(te.Msg), "too many")
+		}
+		return false
+	}
+	return strings.Contains(strings.ToLower(err.Error()), "too many")
+}
+
+// isPermanent reports errors a retry cannot fix: a 5xx reply and a
+// certificate that does not verify.
+func isPermanent(err error) bool {
+	var te *textproto.Error
+	if errors.As(err, &te) {
+		return te.Code >= 500
+	}
+	var verifyErr *tls.CertificateVerificationError
+	if errors.As(err, &verifyErr) {
+		return true
+	}
+	var recordErr tls.RecordHeaderError
+	if errors.As(err, &recordErr) {
+		return true
+	}
+	return errors.Is(err, errSizeChanged)
+}
+
+func replyCode(err error) int {
+	var te *textproto.Error
+	if errors.As(err, &te) {
+		return te.Code
+	}
+	return 0
+}

internal/protocol/ftp/download_test.go

@@ -0,0 +1,146 @@
+package ftp_test
+
+import (
+	"bytes"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/rs/zerolog"
+
+	"github.com/GopeedLab/gopeed/internal/logger"
+	iftp "github.com/GopeedLab/gopeed/internal/protocol/ftp"
+	"github.com/GopeedLab/gopeed/pkg/base"
+	"github.com/GopeedLab/gopeed/pkg/download"
+	pftp "github.com/GopeedLab/gopeed/pkg/protocol/ftp"
+)
+
+// syncBuffer is a log sink that the downloader's goroutines can share.
+type syncBuffer struct {
+	mu  sync.Mutex
+	buf bytes.Buffer
+}
+
+func (b *syncBuffer) Write(p []byte) (int, error) {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	return b.buf.Write(p)
+}
+
+func (b *syncBuffer) String() string {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	return b.buf.String()
+}
+
+// taskEvents records the first done or error event of every task, so the
+// test never reads a task's status while the downloader writes it.
+type taskEvents struct {
+	mu    sync.Mutex
+	final map[string]download.EventKey
+}
+
+func (e *taskEvents) listen(ev *download.Event) {
+	if ev.Key != download.EventKeyDone && ev.Key != download.EventKeyError {
+		return
+	}
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	if _, seen := e.final[ev.Task.ID]; !seen {
+		e.final[ev.Task.ID] = ev.Key
+	}
+}
+
+func (e *taskEvents) wait(t *testing.T, id string) download.EventKey {
+	t.Helper()
+	deadline := time.Now().Add(60 * time.Second)
+	for time.Now().Before(deadline) {
+		e.mu.Lock()
+		key, ok := e.final[id]
+		e.mu.Unlock()
+		if ok {
+			return key
+		}
+		time.Sleep(20 * time.Millisecond)
+	}
+	t.Fatalf("task %s did not finish", id)
+	return ""
+}
+
+// TestPasswordNeverLogged runs FTP tasks through the downloader with its core
+// logger captured: a success, a missing file and a wrong password, all behind
+// an HTTP proxy that FTP must skip with a warning.
+func TestPasswordNeverLogged(t *testing.T) {
+	srv := iftp.StartFTPServer(t, iftp.ServerOptions{User: "u", Pass: "p@ss"})
+	want := srv.WriteFile(t, "secret.bin", iftp.RandomBytes(300000, 21))
+
+	d := download.NewDownloader(&download.DownloaderConfig{Storage: download.NewMemStorage(), StorageDir: t.TempDir()})
+	logs := &syncBuffer{}
+	d.Logger = &logger.Logger{Logger: zerolog.New(logs)}
+	events := &taskEvents{final: map[string]download.EventKey{}}
+	d.Listener(events.listen)
+	if err := d.Setup(); err != nil {
+		t.Fatal(err)
+	}
+	// Close, not Clear: Clear resets the extension list without a lock while
+	// the last task's done hooks may still read it.
+	defer d.Close()
+	cfg, err := d.GetConfig()
+	if err != nil {
+		t.Fatal(err)
+	}
+	cfg.Proxy = &base.DownloaderProxyConfig{Enable: true, Scheme: "http", Host: "127.0.0.1:1", Usr: "proxyuser", Pwd: "proxysecret"}
+	if err := d.PutConfig(cfg); err != nil {
+		t.Fatal(err)
+	}
+
+	dir := t.TempDir()
+	create := func(rawURL string) string {
+		id, err := d.CreateDirect(&base.Request{URL: rawURL}, &base.Options{Path: dir, Extra: &pftp.OptsExtra{Connections: 2}})
+		if err != nil {
+			if strings.Contains(err.Error(), "p@ss") || strings.Contains(err.Error(), "p%40ss") {
+				t.Fatalf("the password leaked into an error: %v", err)
+			}
+			t.Fatal(err)
+		}
+		return id
+	}
+	root := "ftp://u:p%40ss@" + srv.Addr + "/"
+
+	okID := create(root + "secret.bin")
+	if key := events.wait(t, okID); key != download.EventKeyDone {
+		t.Fatalf("download ended with %q, logs:\n%s", key, logs.String())
+	}
+	ok := d.GetTask(okID)
+	if ok.Protocol != "ftp" {
+		t.Fatalf("protocol %q, want ftp", ok.Protocol)
+	}
+	if got := iftp.FileSHA256(t, ok.Meta.SingleFilepath()); got != want {
+		t.Fatalf("sha256 %s, want %s", got, want)
+	}
+	missingID := create(root + "missing.bin")
+	if key := events.wait(t, missingID); key != download.EventKeyError {
+		t.Fatalf("a missing file must fail, got %q", key)
+	}
+	wrongID := create("ftp://u:wr0ng%40pw@" + srv.Addr + "/secret.bin")
+	if key := events.wait(t, wrongID); key != download.EventKeyError {
+		t.Fatalf("a wrong password must fail, got %q", key)
+	}
+	missing, wrong := d.GetTask(missingID), d.GetTask(wrongID)
+	for _, task := range []*download.Task{ok, missing, wrong} {
+		if name := task.Name(); strings.Contains(name, "p@ss") || strings.Contains(name, "p%40ss") || strings.Contains(name, "wr0ng") {
+			t.Fatalf("task name %q shows the password", name)
+		}
+	}
+
+	out := logs.String()
+	if !strings.Contains(out, "task id") || !strings.Contains(out, "HTTP proxy") {
+		t.Fatalf("the captured log misses the expected lines, so it proves nothing:\n%s", out)
+	}
+	for _, secret := range []string{"p@ss", "p%40ss", "wr0ng", "proxysecret"} {
+		if strings.Contains(out, secret) {
+			t.Fatalf("%q appears in the log:\n%s", secret, out)
+		}
+	}
+}

internal/protocol/ftp/fetcher_manager.go

@@ -0,0 +1,109 @@
+package ftp
+
+import (
+	"crypto/tls"
+	"net/url"
+	"path"
+	"strings"
+
+	"github.com/GopeedLab/gopeed/internal/fetcher"
+	"github.com/GopeedLab/gopeed/pkg/base"
+	pftp "github.com/GopeedLab/gopeed/pkg/protocol/ftp"
+)
+
+// fetcherData is what a task saves to resume later. Segments has the same
+// Begin, End and Downloaded fields every version writes; the other fields are
+// optional, so a save without them still restores.
+type fetcherData struct {
+	Segments []*segment
+	// MaxConns is the login ceiling the server enforced, 0 when none was hit.
+	MaxConns int `json:",omitempty"`
+	// NoRest records that the server refused REST, so the task downloads each
+	// file in one sequential stream.
+	NoRest bool `json:",omitempty"`
+}
+
+// FetcherManager handles ftp://, ftps:// (implicit TLS) and ftpes:// (explicit
+// TLS) URLs.
+type FetcherManager struct {
+	// baseTLS is the base for every TLS connection. Nil means the system
+	// roots; the package's tests set it to trust their own certificate. The
+	// task's skipVerifyCert still turns verification off.
+	baseTLS *tls.Config
+}
+
+func (fm *FetcherManager) Name() string {
+	return "ftp"
+}
+
+func (fm *FetcherManager) Filters() []*fetcher.SchemeFilter {
+	return []*fetcher.SchemeFilter{
+		{Type: fetcher.FilterTypeUrl, Pattern: "FTP"},
+		{Type: fetcher.FilterTypeUrl, Pattern: "FTPS"},
+		{Type: fetcher.FilterTypeUrl, Pattern: "FTPES"},
+	}
+}
+
+func (fm *FetcherManager) Build() fetcher.Fetcher {
+	return &Fetcher{manager: fm}
+}
+
+// ParseName names a task before it is resolved. It never includes the
+// userinfo of the URL.
+func (fm *FetcherManager) ParseName(u string) string {
+	parsed, err := url.Parse(u)
+	if err != nil {
+		return ""
+	}
+	name := path.Base(strings.TrimSuffix(parsed.Path, "/"))
+	if name == "" || name == "/" || name == "." {
+		name = parsed.Hostname()
+	}
+	return name
+}
+
+func (fm *FetcherManager) AutoRename() bool {
+	return true
+}
+
+func (fm *FetcherManager) DefaultConfig() any {
+	return &config{Connections: defaultConnections}
+}
+
+func (fm *FetcherManager) Store(f fetcher.Fetcher) (any, error) {
+	ff := f.(*Fetcher)
+	ff.mu.Lock()
+	defer ff.mu.Unlock()
+	segments := make([]*segment, len(ff.segments))
+	for i, s := range ff.segments {
+		c := *s
+		segments[i] = &c
+	}
+	return &fetcherData{
+		Segments: segments,
+		MaxConns: ff.maxConns,
+		NoRest:   ff.noRest,
+	}, nil
+}
+
+func (fm *FetcherManager) Restore() (v any, f func(meta *fetcher.FetcherMeta, v any) fetcher.Fetcher) {
+	return &fetcherData{}, func(meta *fetcher.FetcherMeta, v any) fetcher.Fetcher {
+		fd := v.(*fetcherData)
+		ff := fm.Build().(*Fetcher)
+		ff.meta = meta
+		if meta != nil && meta.Req != nil {
+			_ = base.ParseReqExtra[pftp.ReqExtra](meta.Req)
+		}
+		if meta != nil && meta.Opts != nil {
+			_ = base.ParseOptExtra[pftp.OptsExtra](meta.Opts)
+		}
+		ff.segments = fd.Segments
+		ff.maxConns = fd.MaxConns
+		ff.noRest = fd.NoRest
+		return ff
+	}
+}
+
+func (fm *FetcherManager) Close() error {
+	return nil
+}

internal/protocol/ftp/walk.go

@@ -0,0 +1,187 @@
+package ftp
+
+import (
+	"cmp"
+	"fmt"
+	"path"
+	"slices"
+	"strings"
+	"time"
+
+	"github.com/jlaffaye/ftp"
+
+	"github.com/GopeedLab/gopeed/pkg/base"
+)
+
+// Limits of a directory download. A tree past either one is refused with an
+// error that names the limit; it is never cut short.
+const maxWalkDepth = 8
+
+// maxWalkFiles is a variable so tests can lower it.
+var maxWalkFiles = 10000
+
+// resolveTarget describes the URL's path: one file, or a directory walked
+// into a multi-file resource whose paths are relative to it.
+func (f *Fetcher) resolveTarget(s *session, t *target) (*base.Resource, error) {
+	isDir, size, mtime, err := f.stat(s, t.path)
+	if err != nil {
+		return nil, err
+	}
+	if !isDir {
+		return &base.Resource{
+			Size:  size,
+			Range: true,
+			Files: []*base.FileInfo{{
+				Name:  path.Base(t.path),
+				Size:  size,
+				Ctime: mtime,
+			}},
+		}, nil
+	}
+
+	files, err := f.walk(s, t.path)
+	if err != nil {
+		return nil, err
+	}
+	name := path.Base(strings.TrimSuffix(t.path, "/"))
+	if name == "" || name == "." || name == "/" {
+		name = t.host
+	}
+	res := &base.Resource{Name: name, Range: true, Files: files}
+	res.CalcSize(nil)
+	return res, nil
+}
+
+// stat tells a file from a directory. MLST answers both at once. Without
+// it, SIZE answers for a file, and a directory is one we can change into.
+func (f *Fetcher) stat(s *session, p string) (isDir bool, size int64, mtime *time.Time, err error) {
+	if p == "" || strings.HasSuffix(p, "/") {
+		return true, 0, nil, nil
+	}
+	s.deadline()
+	if entry, mlstErr := s.GetEntry(p); mlstErr == nil {
+		switch entry.Type {
+		case ftp.EntryTypeFolder:
+			return true, 0, nil, nil
+		case ftp.EntryTypeFile:
+			if !entry.Time.IsZero() {
+				t := entry.Time
+				mtime = &t
+			}
+			return false, int64(entry.Size), mtime, nil
+		}
+		// A link: SIZE below follows it on the server.
+	}
+	// MLST is missing or refused the path: the older commands decide. A
+	// broken connection fails them too, with its own error.
+
+	s.deadline()
+	size, sizeErr := s.FileSize(p)
+	if sizeErr == nil {
+		if s.IsGetTimeSupported() {
+			s.deadline()
+			if t, err := s.GetTime(p); err == nil {
+				mtime = &t
+			}
+		}
+		return false, size, mtime, nil
+	}
+	if replyCode(sizeErr) == 0 {
+		return false, 0, nil, sizeErr
+	}
+
+	// SIZE failed: a directory, or a missing path.
+	s.deadline()
+	cwd, err := s.CurrentDir()
+	if err != nil {
+		return false, 0, nil, err
+	}
+	s.deadline()
+	if err := s.ChangeDir(p); err != nil {
+		return false, 0, nil, sizeErr
+	}
+	s.deadline()
+	if err := s.ChangeDir(cwd); err != nil {
+		return false, 0, nil, err
+	}
+	return true, 0, nil, nil
+}
+
+// safeEntryName reports whether a name from a listing can become a local
+// path element. A hostile server could otherwise list "../x" and write
+// outside the task directory.
+func safeEntryName(name string) bool {
+	if name == "" || name == "." || name == ".." {
+		return false
+	}
+	return !strings.ContainsAny(name, "/\\\x00")
+}
+
+// walk lists a directory tree, at most maxWalkDepth levels below root and
+// maxWalkFiles files. Files come back sorted by path, then name.
+func (f *Fetcher) walk(s *session, root string) ([]*base.FileInfo, error) {
+	type dir struct {
+		remote string // path on the server
+		rel    string // path below root, "" for root itself
+		depth  int
+	}
+	var files []*base.FileInfo
+	stack := []dir{{remote: strings.TrimSuffix(root, "/"), rel: "", depth: 0}}
+	for len(stack) > 0 {
+		d := stack[len(stack)-1]
+		stack = stack[:len(stack)-1]
+
+		s.deadline()
+		entries, err := s.List(d.remote)
+		if err != nil {
+			return nil, err
+		}
+		for _, e := range entries {
+			if e.Name == "." || e.Name == ".." {
+				continue
+			}
+			if !safeEntryName(e.Name) {
+				f.warn("ftp: skipping a listed name that is not a safe file name: %q", e.Name)
+				continue
+			}
+			remote := path.Join(d.remote, e.Name)
+			rel := path.Join(d.rel, e.Name)
+			if d.remote == "" {
+				remote = e.Name
+			}
+			entryType, size := e.Type, int64(e.Size)
+			if entryType == ftp.EntryTypeLink {
+				// A link to a file has a size; a link to a directory is
+				// not followed, so a loop cannot make the walk endless.
+				s.deadline()
+				linkSize, err := s.FileSize(remote)
+				if err != nil {
+					f.warn("ftp: skipping link %q: %v", rel, err)
+					continue
+				}
+				entryType, size = ftp.EntryTypeFile, linkSize
+			}
+			switch entryType {
+			case ftp.EntryTypeFolder:
+				if d.depth+1 > maxWalkDepth {
+					return nil, fmt.Errorf("ftp: the folder is deeper than %d levels, the limit for a folder download", maxWalkDepth)
+				}
+				stack = append(stack, dir{remote: remote, rel: rel, depth: d.depth + 1})
+			case ftp.EntryTypeFile:
+				if len(files) >= maxWalkFiles {
+					return nil, fmt.Errorf("ftp: the folder has more than %d files, the limit for a folder download", maxWalkFiles)
+				}
+				fi := &base.FileInfo{Name: e.Name, Path: d.rel, Size: size}
+				if !e.Time.IsZero() {
+					t := e.Time
+					fi.Ctime = &t
+				}
+				files = append(files, fi)
+			}
+		}
+	}
+	slices.SortFunc(files, func(a, b *base.FileInfo) int {
+		return cmp.Or(strings.Compare(a.Path, b.Path), strings.Compare(a.Name, b.Name))
+	})
+	return files, nil
+}