package ftp import ( "bytes" "context" "crypto/tls" "encoding/json" "errors" "fmt" "net" "os" "path/filepath" "slices" "strings" "sync/atomic" "testing" "time" "github.com/armon/go-socks5" "github.com/rs/zerolog" "github.com/GopeedLab/gopeed/internal/controller" "github.com/GopeedLab/gopeed/internal/fetcher" "github.com/GopeedLab/gopeed/pkg/base" pftp "github.com/GopeedLab/gopeed/pkg/protocol/ftp" ) const mib = 1 << 20 // testManager trusts the test server's certificate, the way an embedding // application could add private roots. func testManager(srv *FTPServer) *FetcherManager { fm := &FetcherManager{} if srv != nil { fm.baseTLS = &tls.Config{RootCAs: srv.CA} } return fm } func testController(fm *FetcherManager) *controller.Controller { ctl := controller.NewController() ctl.GetConfig = func(v any) { data, _ := json.Marshal(fm.DefaultConfig()) _ = json.Unmarshal(data, v) } return ctl } func buildFetcher(fm *FetcherManager) *Fetcher { f := fm.Build().(*Fetcher) f.Setup(testController(fm)) return f } func ftpURL(srv *FTPServer, scheme, userinfo, p string) string { if userinfo != "" { userinfo += "@" } return scheme + "://" + userinfo + srv.Addr + "/" + strings.TrimPrefix(p, "/") } // resolveOpts resolves req and then gives the fetcher its options, in the // order the downloader uses. func resolveOpts(f *Fetcher, req *base.Request, opts *base.Options) error { if err := f.Resolve(req); err != nil { return err } f.Meta().Opts = opts return nil } func resolveTask(t *testing.T, f *Fetcher, rawURL string, extra *pftp.OptsExtra) { t.Helper() resolveRequest(t, f, &base.Request{URL: rawURL}, extra) } func resolveRequest(t *testing.T, f *Fetcher, req *base.Request, extra *pftp.OptsExtra) { t.Helper() opts := &base.Options{Path: t.TempDir()} if extra != nil { opts.Extra = extra } if err := resolveOpts(f, req, opts); err != nil { t.Fatalf("resolve %s: %v", req.URL, err) } } func waitFetcher(t *testing.T, f fetcher.Fetcher) error { t.Helper() done := make(chan error, 1) go func() { done <- f.Wait() }() select { case err := <-done: return err case <-time.After(60 * time.Second): t.Fatal("download did not finish in 60s") return nil } } func startAndWait(t *testing.T, f *Fetcher) { t.Helper() if err := f.Start(); err != nil { t.Fatalf("start: %v", err) } if err := waitFetcher(t, f); err != nil { t.Fatalf("download: %v", err) } } func checkFile(t *testing.T, name, want string) { t.Helper() if got := FileSHA256(t, name); got != want { t.Fatalf("%s: sha256 %s, want %s", name, got, want) } } func ftpStats(t *testing.T, f *Fetcher) *pftp.Stats { t.Helper() stats, ok := f.Stats().Snapshot.(*pftp.Stats) if !ok { t.Fatalf("stats snapshot is %T, want *ftp.Stats", f.Stats().Snapshot) } return stats } func storeData(t *testing.T, fm *FetcherManager, f *Fetcher) []byte { t.Helper() data, err := fm.Store(f) if err != nil { t.Fatal(err) } blob, err := json.Marshal(data) if err != nil { t.Fatal(err) } return blob } // restoreFetcher rebuilds a fetcher the way the downloader does after a // restart: the meta and the save blob both come back from JSON. func restoreFetcher(t *testing.T, fm *FetcherManager, meta *fetcher.FetcherMeta, blob []byte) *Fetcher { t.Helper() metaJSON, err := json.Marshal(meta) if err != nil { t.Fatal(err) } var restoredMeta fetcher.FetcherMeta if err := json.Unmarshal(metaJSON, &restoredMeta); err != nil { t.Fatal(err) } v, build := fm.Restore() if err := json.Unmarshal(blob, v); err != nil { t.Fatal(err) } f := build(&restoredMeta, v).(*Fetcher) f.Setup(testController(fm)) return f } func TestResolveFile(t *testing.T) { for _, noMLST := range []bool{false, true} { t.Run(fmt.Sprintf("noMLST=%v", noMLST), func(t *testing.T) { srv := StartFTPServer(t, ServerOptions{NoMLST: noMLST}) data := RandomBytes(100*1024, 1) srv.WriteFile(t, "pub/file name.bin", data) f := buildFetcher(testManager(srv)) resolveTask(t, f, ftpURL(srv, "ftp", "", "pub/file%20name.bin"), nil) res := f.Meta().Res if res.Name != "" { t.Fatalf("a file must not be a folder resource, got name %q", res.Name) } if len(res.Files) != 1 { t.Fatalf("got %d files, want 1", len(res.Files)) } file := res.Files[0] if file.Name != "file name.bin" || file.Path != "" || file.Size != int64(len(data)) { t.Fatalf("got file %+v, want name %q, empty path, size %d", file, "file name.bin", len(data)) } if res.Size != int64(len(data)) { t.Fatalf("resource size %d, want %d", res.Size, len(data)) } if file.Ctime == nil || time.Since(*file.Ctime) > time.Hour { t.Fatalf("modification time %v is missing or wrong", file.Ctime) } }) } } func writeTree(t *testing.T, srv *FTPServer) map[string]string { t.Helper() return map[string]string{ "a.txt": srv.WriteFile(t, "pub/tree/a.txt", []byte("hello")), "sub/b.bin": srv.WriteFile(t, "pub/tree/sub/b.bin", RandomBytes(3*mib+17, 2)), "sub/deeper/c.dat": srv.WriteFile(t, "pub/tree/sub/deeper/c.dat", RandomBytes(2000, 3)), "sub/empty.txt": srv.WriteFile(t, "pub/tree/sub/empty.txt", nil), } } func TestResolveDir(t *testing.T) { for _, noMLST := range []bool{false, true} { for _, p := range []string{"pub/tree", "pub/tree/"} { t.Run(fmt.Sprintf("noMLST=%v/%s", noMLST, p), func(t *testing.T) { srv := StartFTPServer(t, ServerOptions{NoMLST: noMLST}) writeTree(t, srv) if err := os.MkdirAll(filepath.Join(srv.Root, "pub/tree/void"), 0o755); err != nil { t.Fatal(err) } f := buildFetcher(testManager(srv)) resolveTask(t, f, ftpURL(srv, "ftp", "", p), nil) res := f.Meta().Res if res.Name != "tree" { t.Fatalf("folder name %q, want tree", res.Name) } var got []string for _, file := range res.Files { got = append(got, fmt.Sprintf("%s|%s|%d", file.Path, file.Name, file.Size)) } want := []string{ "|a.txt|5", "sub|b.bin|3145745", "sub|empty.txt|0", "sub/deeper|c.dat|2000", } if !slices.Equal(got, want) { t.Fatalf("files\n got %q\nwant %q", got, want) } if res.Size != 5+3145745+2000 { t.Fatalf("resource size %d", res.Size) } }) } } } func TestResolveDirLimits(t *testing.T) { srv := StartFTPServer(t, ServerOptions{}) srv.WriteFile(t, "deep/1/2/3/4/5/6/7/8/f.txt", []byte("ok")) srv.WriteFile(t, "deep9/1/2/3/4/5/6/7/8/9/f.txt", []byte("too deep")) for i := 0; i < 4; i++ { srv.WriteFile(t, fmt.Sprintf("many/f%d", i), []byte("x")) } f := buildFetcher(testManager(srv)) resolveTask(t, f, ftpURL(srv, "ftp", "", "deep"), nil) if got := f.Meta().Res.Files[0].Path; got != "1/2/3/4/5/6/7/8" { t.Fatalf("depth 8 path %q", got) } f = buildFetcher(testManager(srv)) err := resolveOpts(f, &base.Request{URL: ftpURL(srv, "ftp", "", "deep9")}, &base.Options{Path: t.TempDir()}) if err == nil || !strings.Contains(err.Error(), "8 levels") { t.Fatalf("a tree deeper than 8 levels must fail and name the limit, got %v", err) } saved := maxWalkFiles maxWalkFiles = 3 defer func() { maxWalkFiles = saved }() f = buildFetcher(testManager(srv)) err = resolveOpts(f, &base.Request{URL: ftpURL(srv, "ftp", "", "many")}, &base.Options{Path: t.TempDir()}) if err == nil || !strings.Contains(err.Error(), "3 files") { t.Fatalf("a tree over the file cap must fail and name the limit, got %v", err) } } func TestDownloadPlain(t *testing.T) { srv := StartFTPServer(t, ServerOptions{}) data := RandomBytes(3*mib+12345, 4) want := srv.WriteFile(t, "pub/plain.bin", data) f := buildFetcher(testManager(srv)) resolveTask(t, f, ftpURL(srv, "ftp", "", "pub/plain.bin"), &pftp.OptsExtra{Connections: 4}) startAndWait(t, f) checkFile(t, f.Meta().SingleFilepath(), want) if got := f.Progress().TotalDownloaded(); got != int64(len(data)) { t.Fatalf("progress %d, want %d", got, len(data)) } stats := ftpStats(t, f) if len(stats.Connections) == 0 || len(stats.Connections) > 4 { t.Fatalf("got %d connections in stats", len(stats.Connections)) } var sum int64 for _, c := range stats.Connections { sum += c.Downloaded if !c.Completed || c.Failed { t.Fatalf("connection stats %+v after success", c) } } if sum != int64(len(data)) { t.Fatalf("connections downloaded %d bytes, want %d", sum, len(data)) } } func TestDownloadDir(t *testing.T) { srv := StartFTPServer(t, ServerOptions{}) want := writeTree(t, srv) f := buildFetcher(testManager(srv)) resolveTask(t, f, ftpURL(srv, "ftp", "", "pub/tree"), &pftp.OptsExtra{Connections: 2}) startAndWait(t, f) root := f.Meta().FolderPath() if root != filepath.Join(f.Meta().Opts.Path, "tree") { t.Fatalf("folder path %q", root) } for rel, sum := range want { checkFile(t, filepath.Join(root, filepath.FromSlash(rel)), sum) } } func testDownloadTLS(t *testing.T, mode string, schemes map[string]string) { srv := StartFTPServer(t, ServerOptions{TLS: mode}) data := RandomBytes(2*mib+999, 5) want := srv.WriteFile(t, "secure.bin", data) for scheme, reqTLS := range schemes { t.Run(scheme+"+"+reqTLS, func(t *testing.T) { f := buildFetcher(testManager(srv)) req := &base.Request{URL: ftpURL(srv, scheme, "", "secure.bin"), Extra: &pftp.ReqExtra{TLS: reqTLS}} resolveRequest(t, f, req, &pftp.OptsExtra{Connections: 2}) startAndWait(t, f) checkFile(t, f.Meta().SingleFilepath(), want) }) } } func TestDownloadExplicitTLS(t *testing.T) { testDownloadTLS(t, pftp.TLSExplicit, map[string]string{ "ftpes": "", "ftp": pftp.TLSExplicit, }) } func TestDownloadImplicitTLS(t *testing.T) { testDownloadTLS(t, pftp.TLSImplicit, map[string]string{ "ftps": "", "ftp": pftp.TLSImplicit, }) } // The TLS mode of a plain ftp:// link lives on the request, so a task that is // saved and restored from JSON still logs in over TLS. func TestRestoredRequestKeepsTLSMode(t *testing.T) { srv := StartFTPServer(t, ServerOptions{TLS: pftp.TLSExplicit}) want := srv.WriteFile(t, "keep.bin", RandomBytes(2*mib+5, 12)) fm := testManager(srv) f := buildFetcher(fm) req := &base.Request{URL: ftpURL(srv, "ftp", "", "keep.bin"), Extra: &pftp.ReqExtra{TLS: pftp.TLSExplicit}} resolveRequest(t, f, req, &pftp.OptsExtra{Connections: 2}) restored := restoreFetcher(t, fm, f.Meta(), storeData(t, fm, f)) if _, ok := restored.Meta().Req.Extra.(*pftp.ReqExtra); !ok { t.Fatalf("restored request extra is %T, want *ftp.ReqExtra", restored.Meta().Req.Extra) } // Explicit mode refuses plain logins, so the download only finishes over TLS. startAndWait(t, restored) checkFile(t, restored.Meta().SingleFilepath(), want) } func TestBadCertRejectedByDefault(t *testing.T) { for mode, scheme := range map[string]string{pftp.TLSExplicit: "ftpes", pftp.TLSImplicit: "ftps"} { t.Run(mode, func(t *testing.T) { srv := StartFTPServer(t, ServerOptions{TLS: mode}) want := srv.WriteFile(t, "f.bin", RandomBytes(4096, 6)) rawURL := ftpURL(srv, scheme, "", "f.bin") // No injected roots: the self-signed certificate is unknown. f := buildFetcher(testManager(nil)) err := resolveOpts(f, &base.Request{URL: rawURL}, &base.Options{Path: t.TempDir()}) var verifyErr *tls.CertificateVerificationError if !errors.As(err, &verifyErr) { t.Fatalf("want a certificate verification error, got %v", err) } // The request's existing skipVerifyCert switch accepts it. f = buildFetcher(testManager(nil)) if err := resolveOpts(f, &base.Request{URL: rawURL, SkipVerifyCert: true}, &base.Options{Path: t.TempDir()}); err != nil { t.Fatalf("skipVerifyCert resolve: %v", err) } startAndWait(t, f) checkFile(t, f.Meta().SingleFilepath(), want) }) } } func TestSegmentsUseRest(t *testing.T) { srv := StartFTPServer(t, ServerOptions{}) srv.ReadDelay.Store(int64(time.Millisecond)) size := 8 * mib want := srv.WriteFile(t, "big.bin", RandomBytes(size, 7)) f := buildFetcher(testManager(srv)) resolveTask(t, f, ftpURL(srv, "ftp", "", "big.bin"), &pftp.OptsExtra{Connections: 4}) startAndWait(t, f) checkFile(t, f.Meta().SingleFilepath(), want) rests := srv.RestOffsets() for _, off := range []int64{2 * mib, 4 * mib, 6 * mib} { if !slices.Contains(rests, off) { t.Fatalf("REST offsets %v miss the segment start %d", rests, off) } } for _, off := range rests { if off <= 0 || off >= int64(size) { t.Fatalf("REST offset %d out of range", off) } } if got := len(ftpStats(t, f).Connections); got != 4 { t.Fatalf("stats report %d connections, want 4", got) } } func TestLoginCapFallsBack(t *testing.T) { srv := StartFTPServer(t, ServerOptions{MaxLogins: 1}) want := srv.WriteFile(t, "capped.bin", RandomBytes(4*mib+5, 8)) fm := testManager(srv) f := buildFetcher(fm) resolveTask(t, f, ftpURL(srv, "ftp", "", "capped.bin"), &pftp.OptsExtra{Connections: 4}) startAndWait(t, f) checkFile(t, f.Meta().SingleFilepath(), want) if srv.Refused() == 0 { t.Fatal("the server never refused a login, so the cap was not exercised") } if got := len(ftpStats(t, f).Connections); got != 1 { t.Fatalf("stats report %d connections, want 1", got) } var saved fetcherData if err := json.Unmarshal(storeData(t, fm, f), &saved); err != nil { t.Fatal(err) } if saved.MaxConns != 1 { t.Fatalf("saved login ceiling %d, want 1", saved.MaxConns) } } func TestNoRestSequential(t *testing.T) { srv := StartFTPServer(t, ServerOptions{NoRest: true}) srv.ReadDelay.Store(int64(time.Millisecond)) want := srv.WriteFile(t, "norest.bin", RandomBytes(4*mib+3, 9)) fm := testManager(srv) f := buildFetcher(fm) resolveTask(t, f, ftpURL(srv, "ftp", "", "norest.bin"), &pftp.OptsExtra{Connections: 4}) startAndWait(t, f) checkFile(t, f.Meta().SingleFilepath(), want) if rests := srv.RestOffsets(); len(rests) != 0 { t.Fatalf("the server cannot have seen a REST, got %v", rests) } var saved fetcherData if err := json.Unmarshal(storeData(t, fm, f), &saved); err != nil { t.Fatal(err) } if !saved.NoRest { t.Fatal("the missing REST support was not recorded") } } // TestResumeAfterKill stops the server's transfers at a fixed byte count, // waits until the client has written exactly those bytes, then pauses, saves, // restores and finishes. No timing decides where the pause lands. func TestResumeAfterKill(t *testing.T) { srv := StartFTPServer(t, ServerOptions{}) size := int64(4*mib + 777) want := srv.WriteFile(t, "resume.bin", RandomBytes(int(size), 10)) stallAt := size / 2 srv.StallAfter(stallAt) fm := testManager(srv) f := buildFetcher(fm) resolveTask(t, f, ftpURL(srv, "ftp", "", "resume.bin"), &pftp.OptsExtra{Connections: 2}) if err := f.Start(); err != nil { t.Fatal(err) } select { case <-srv.Stalled(): case <-time.After(30 * time.Second): t.Fatal("the server never reached the stall point") } deadline := time.Now().Add(30 * time.Second) for f.Progress().TotalDownloaded() != stallAt { if time.Now().After(deadline) { t.Fatalf("progress %d, want the %d bytes the server sent", f.Progress().TotalDownloaded(), stallAt) } time.Sleep(5 * time.Millisecond) } if err := f.Pause(); err != nil { t.Fatal(err) } srv.Release() blob := storeData(t, fm, f) var saved fetcherData if err := json.Unmarshal(blob, &saved); err != nil { t.Fatal(err) } var savedTotal int64 resumeAt := map[int64]bool{} for _, seg := range saved.Segments { savedTotal += seg.Downloaded if seg.Begin+seg.Downloaded <= seg.End { resumeAt[seg.Begin+seg.Downloaded] = true } } if savedTotal != stallAt { t.Fatalf("saved progress %d, want %d", savedTotal, stallAt) } before := len(srv.RetrOffsets()) restored := restoreFetcher(t, fm, f.Meta(), blob) if got := restored.Progress().TotalDownloaded(); got != savedTotal { t.Fatalf("restored progress %d, want %d", got, savedTotal) } startAndWait(t, restored) checkFile(t, restored.Meta().SingleFilepath(), want) resumed := srv.RetrOffsets()[before:] if len(resumed) == 0 { t.Fatal("no transfer after resume") } for _, off := range resumed { if !resumeAt[off] { t.Fatalf("resumed RETR at %d, want one of the saved offsets %v", off, resumeAt) } } } // TestLateRefusalAfterQueueDrained: worker B holds the second file's segment // while the server takes its time to refuse B's login. Worker A finishes the // first file meanwhile and finds the queue empty. B must not leave too, or // nobody fetches the second file. func TestLateRefusalAfterQueueDrained(t *testing.T) { srv := StartFTPServer(t, ServerOptions{MaxLogins: 1, RefuseDelay: 500 * time.Millisecond}) want := map[string]string{ "a.bin": srv.WriteFile(t, "dir/a.bin", RandomBytes(1000, 16)), "b.bin": srv.WriteFile(t, "dir/b.bin", RandomBytes(1000, 17)), } f := buildFetcher(testManager(srv)) resolveTask(t, f, ftpURL(srv, "ftp", "", "dir"), &pftp.OptsExtra{Connections: 2}) startAndWait(t, f) for name, sum := range want { checkFile(t, filepath.Join(f.Meta().FolderPath(), name), sum) } if srv.Refused() == 0 { t.Fatal("the server never refused a login, so the late refusal was not exercised") } } // TestLoginCapLoweredMidway: both workers have logged in when the server's // cap drops to one, as when another client takes a slot. The refused worker // leaves, the ceiling drops to one, and the task finishes. func TestLoginCapLoweredMidway(t *testing.T) { srv := StartFTPServer(t, ServerOptions{}) srv.ReadDelay.Store(int64(time.Millisecond)) want := map[string]string{} for i := 0; i < 8; i++ { name := fmt.Sprintf("f%d.bin", i) want[name] = srv.WriteFile(t, "dir/"+name, RandomBytes(512<<10, byte(30+i))) } fm := testManager(srv) f := buildFetcher(fm) resolveTask(t, f, ftpURL(srv, "ftp", "", "dir"), &pftp.OptsExtra{Connections: 2}) if err := f.Start(); err != nil { t.Fatal(err) } deadline := time.Now().Add(30 * time.Second) for len(ftpStats(t, f).Connections) < 2 || f.Progress().TotalDownloaded() < 512<<10 { if time.Now().After(deadline) { t.Fatal("the two workers never got going") } time.Sleep(2 * time.Millisecond) } srv.SetMaxLogins(1) if err := waitFetcher(t, f); err != nil { t.Fatalf("download: %v", err) } for name, sum := range want { checkFile(t, filepath.Join(f.Meta().FolderPath(), name), sum) } if srv.Refused() == 0 { t.Fatal("the server never refused a login, so the lowered cap was not exercised") } for _, c := range ftpStats(t, f).Connections { if c.Failed { t.Fatalf("a refused worker failed: %+v", c) } } var saved fetcherData if err := json.Unmarshal(storeData(t, fm, f), &saved); err != nil { t.Fatal(err) } if saved.MaxConns != 1 { t.Fatalf("saved login ceiling %d, want 1", saved.MaxConns) } } // TestDataConnectionsResumeOwnSession runs four FTPS workers against a server // that, like ProFTPD, accepts a data connection only when it resumes the TLS // session of its own control connection. func TestDataConnectionsResumeOwnSession(t *testing.T) { srv := StartFTPServer(t, ServerOptions{TLS: pftp.TLSImplicit, RequireSessionReuse: true}) srv.ReadDelay.Store(int64(200 * time.Microsecond)) want := srv.WriteFile(t, "reuse.bin", RandomBytes(8*mib, 18)) f := buildFetcher(testManager(srv)) resolveTask(t, f, ftpURL(srv, "ftps", "", "reuse.bin"), &pftp.OptsExtra{Connections: 4}) startAndWait(t, f) checkFile(t, f.Meta().SingleFilepath(), want) ok, bad := srv.SessionReuse() if ok < 4 || bad != 0 { t.Fatalf("data connections: %d resumed their own session, %d did not", ok, bad) } for _, c := range ftpStats(t, f).Connections { if c.RetryTimes != 0 { t.Fatalf("a worker needed retries: %+v", c) } } } func TestRetryAfterDroppedConnection(t *testing.T) { srv := StartFTPServer(t, ServerOptions{}) want := srv.WriteFile(t, "drop.bin", RandomBytes(3*mib, 11)) srv.DropAfter.Store(mib + 100) f := buildFetcher(testManager(srv)) resolveTask(t, f, ftpURL(srv, "ftp", "", "drop.bin"), &pftp.OptsExtra{Connections: 1}) startAndWait(t, f) checkFile(t, f.Meta().SingleFilepath(), want) offsets := srv.RetrOffsets() if len(offsets) != 2 || offsets[0] != 0 || offsets[1] <= 0 || offsets[1] > mib+100 { t.Fatalf("RETR offsets %v, want a restart at the dropped offset", offsets) } if got := ftpStats(t, f).Connections[0].RetryTimes; got != 1 { t.Fatalf("retry count %d, want 1", got) } } func TestPercentEncodedPassword(t *testing.T) { srv := StartFTPServer(t, ServerOptions{User: "u", Pass: "p@ss"}) want := srv.WriteFile(t, "secret.bin", RandomBytes(5000, 12)) f := buildFetcher(testManager(srv)) resolveTask(t, f, ftpURL(srv, "ftp", "u:p%40ss", "secret.bin"), nil) startAndWait(t, f) checkFile(t, f.Meta().SingleFilepath(), want) f = buildFetcher(testManager(srv)) err := resolveOpts(f, &base.Request{URL: ftpURL(srv, "ftp", "u:wrong", "secret.bin")}, &base.Options{Path: t.TempDir()}) if err == nil || !strings.Contains(err.Error(), "530") { t.Fatalf("a wrong password must fail with 530, got %v", err) } } func TestOldSaveFormatResumes(t *testing.T) { srv := StartFTPServer(t, ServerOptions{}) data := RandomBytes(2*mib+1, 13) want := srv.WriteFile(t, "old.bin", data) fm := testManager(srv) f := buildFetcher(fm) resolveTask(t, f, ftpURL(srv, "ftp", "", "old.bin"), &pftp.OptsExtra{Connections: 1}) // A save blob that has only the segment fields every version writes. const done = 700000 blob := []byte(fmt.Sprintf(`{"Segments":[{"Begin":0,"End":%d,"Downloaded":%d}]}`, len(data)-1, done)) name := f.Meta().SingleFilepath() if err := os.WriteFile(name, append(data[:done:done], make([]byte, len(data)-done)...), 0o644); err != nil { t.Fatal(err) } restored := restoreFetcher(t, fm, f.Meta(), blob) if got := restored.Progress().TotalDownloaded(); got != done { t.Fatalf("restored progress %d, want %d", got, done) } startAndWait(t, restored) checkFile(t, name, want) if got := srv.RetrOffsets(); !slices.Equal(got, []int64{done}) { t.Fatalf("RETR offsets %v, want [%d]", got, done) } } func TestDownloadThroughSOCKS5(t *testing.T) { srv := StartFTPServer(t, ServerOptions{TLS: pftp.TLSExplicit}) want := srv.WriteFile(t, "proxied.bin", RandomBytes(2*mib+1, 14)) var dials atomic.Int32 proxySrv, err := socks5.New(&socks5.Config{ Dial: func(ctx context.Context, network, addr string) (net.Conn, error) { dials.Add(1) var d net.Dialer return d.DialContext(ctx, network, addr) }, }) if err != nil { t.Fatal(err) } ln, err := net.Listen("tcp", "127.0.0.1:0") if err != nil { t.Fatal(err) } defer ln.Close() go proxySrv.Serve(ln) f := buildFetcher(testManager(srv)) req := &base.Request{ URL: ftpURL(srv, "ftpes", "", "proxied.bin"), Proxy: &base.RequestProxy{ Mode: base.RequestProxyModeCustom, Scheme: "socks5", Host: ln.Addr().String(), }, } if err := resolveOpts(f, req, &base.Options{Path: t.TempDir(), Extra: &pftp.OptsExtra{Connections: 2}}); err != nil { t.Fatal(err) } startAndWait(t, f) checkFile(t, f.Meta().SingleFilepath(), want) // Resolve's control and listing, then per worker one control and one // data connection at least: every one of them went through the proxy. if got := dials.Load(); got < 4 { t.Fatalf("the proxy saw %d connections, want every control and data connection", got) } } func TestHTTPProxyFallsBackToDirect(t *testing.T) { srv := StartFTPServer(t, ServerOptions{}) want := srv.WriteFile(t, "direct.bin", RandomBytes(100000, 15)) var logs bytes.Buffer logger := zerolog.New(&logs) fm := testManager(srv) f := fm.Build().(*Fetcher) ctl := testController(fm) ctl.Logger = &logger f.Setup(ctl) req := &base.Request{ URL: ftpURL(srv, "ftp", "", "direct.bin"), Proxy: &base.RequestProxy{ Mode: base.RequestProxyModeCustom, Scheme: "http", Host: "127.0.0.1:1", Usr: "proxyuser", Pwd: "proxysecret", }, } if err := resolveOpts(f, req, &base.Options{Path: t.TempDir(), Extra: &pftp.OptsExtra{Connections: 2}}); err != nil { t.Fatal(err) } startAndWait(t, f) checkFile(t, f.Meta().SingleFilepath(), want) out := logs.String() if n := strings.Count(out, "HTTP proxy"); n != 1 { t.Fatalf("want exactly one HTTP proxy warning, got %d in %q", n, out) } if strings.Contains(out, "proxysecret") { t.Fatalf("the proxy password leaked into the log: %q", out) } } func TestTorrentNameIsFile(t *testing.T) { fm := testManager(nil) for _, u := range []string{"ftp://h/x.torrent", "ftps://h/x.torrent", "ftpes://h/x.torrent", "FTP://h/v.m3u8"} { matched := false for _, filter := range fm.Filters() { if filter.Match(u) { matched = true } } if !matched { t.Fatalf("%s is not matched by the FTP filters", u) } } if fm.Name() != "ftp" { t.Fatalf("manager name %q", fm.Name()) } srv := StartFTPServer(t, ServerOptions{}) want := srv.WriteFile(t, "x.torrent", []byte("d8:announce0:e")) f := buildFetcher(testManager(srv)) resolveTask(t, f, ftpURL(srv, "ftp", "", "x.torrent"), nil) res := f.Meta().Res if res.Name != "" || len(res.Files) != 1 || res.Files[0].Name != "x.torrent" { t.Fatalf("x.torrent must resolve to one FTP file, got name %q and %d files", res.Name, len(res.Files)) } startAndWait(t, f) checkFile(t, f.Meta().SingleFilepath(), want) } func TestParseNameHidesCredentials(t *testing.T) { fm := testManager(nil) for u, want := range map[string]string{ "ftp://u:p%40ss@example.com/pub/a.iso": "a.iso", "ftp://u:p%40ss@example.com/": "example.com", "ftps://u:p%40ss@example.com": "example.com", "ftp://u:p%40ss@example.com/pub/dir/": "dir", } { if got := fm.ParseName(u); got != want { t.Fatalf("ParseName(%q) = %q, want %q", u, got, want) } } } func TestUnsafeEntryNames(t *testing.T) { for name, ok := range map[string]bool{ "a.txt": true, "..a": true, "a b": true, "": false, ".": false, "..": false, "a/b": false, `a\b`: false, "../x": false, "x\x00y": false, } { if got := safeEntryName(name); got != ok { t.Fatalf("safeEntryName(%q) = %v, want %v", name, got, ok) } } }