Files
gopeed/internal/protocol/http/fetcher_test.go
T

1604 lines
43 KiB
Go

package http
import (
"encoding/json"
"fmt"
"net"
gohttp "net/http"
"net/url"
"os"
"strings"
"testing"
"time"
"github.com/GopeedLab/gopeed/internal/controller"
"github.com/GopeedLab/gopeed/internal/fetcher"
"github.com/GopeedLab/gopeed/internal/test"
"github.com/GopeedLab/gopeed/pkg/base"
"github.com/GopeedLab/gopeed/pkg/protocol/http"
)
func TestFetcher_Resolve(t *testing.T) {
testResolve(test.StartTestFileServer, test.BuildName, t, func(err error) (*base.Resource, error) {
return &base.Resource{
Size: test.BuildSize,
Range: true,
Files: []*base.FileInfo{
{
Name: test.BuildName,
Size: test.BuildSize,
},
},
}, nil
})
testResolve(test.StartTestCustomServer, "disposition", t, func(err error) (*base.Resource, error) {
return &base.Resource{
Size: test.BuildSize,
Range: false,
Files: []*base.FileInfo{
{
Name: test.BuildName,
Size: test.BuildSize,
},
},
}, nil
})
testResolve(test.StartTestCustomServer, "encoded-word", t, func(err error) (*base.Resource, error) {
return &base.Resource{
Size: test.BuildSize,
Range: false,
Files: []*base.FileInfo{
{
Name: test.TestChineseFileName,
Size: test.BuildSize,
},
},
}, nil
})
testResolve(test.StartTestCustomServer, "no-encode", t, func(err error) (*base.Resource, error) {
return &base.Resource{
Size: test.BuildSize,
Range: false,
Files: []*base.FileInfo{
{
Name: test.TestChineseFileName,
Size: test.BuildSize,
},
},
}, nil
})
testResolve(test.StartTestCustomServer, "%E6%B5%8B%E8%AF%95.zip", t, func(err error) (*base.Resource, error) {
return &base.Resource{
Size: 0,
Range: false,
Files: []*base.FileInfo{
{
Name: test.TestChineseFileName,
Size: 0,
},
},
}, nil
})
testResolve(test.StartTestCustomServer, test.BuildName, t, func(err error) (*base.Resource, error) {
return &base.Resource{
Size: 0,
Range: false,
Files: []*base.FileInfo{
{
Name: test.BuildName,
Size: 0,
},
},
}, nil
})
// Test mixed encoding Content-Disposition where mime.ParseMediaType fails
// due to invalid characters, but filename*= contains the correct UTF-8 encoded name
testResolve(test.StartTestCustomServer, "mixed-encoding", t, func(err error) (*base.Resource, error) {
return &base.Resource{
Size: test.BuildSize,
Range: false,
Files: []*base.FileInfo{
{
Name: test.TestChineseFileName,
Size: test.BuildSize,
},
},
}, nil
})
// Test filename*= only (RFC 5987 format)
testResolve(test.StartTestCustomServer, "filename-star", t, func(err error) (*base.Resource, error) {
return &base.Resource{
Size: test.BuildSize,
Range: false,
Files: []*base.FileInfo{
{
Name: test.TestChineseFileName,
Size: test.BuildSize,
},
},
}, nil
})
// Test GBK-encoded filename (common on Chinese Windows servers)
// Before fix: "测试.zip" sent as GBK bytes -> parsed as "²âÊÔ.zip" (garbled)
// After fix: correctly decoded back to "测试.zip"
testResolve(test.StartTestCustomServer, "gbk-encoded", t, func(err error) (*base.Resource, error) {
return &base.Resource{
Size: test.BuildSize,
Range: false,
Files: []*base.FileInfo{
{
Name: test.TestChineseFileName,
Size: test.BuildSize,
},
},
}, nil
})
// Test filename with plus signs (e.g., C++ Primer)
// Before fix: %2B decoded to space -> "C++ Primer" became "C Primer"
// After fix: %2B correctly decoded to + -> "C++ Primer Plus.mobi"
testResolve(test.StartTestCustomServer, "plus-sign-encoded", t, func(err error) (*base.Resource, error) {
return &base.Resource{
Size: test.BuildSize,
Range: false,
Files: []*base.FileInfo{
{
Name: "C++ Primer Plus.mobi",
Size: test.BuildSize,
},
},
}, nil
})
// Test filename with plus sign in URL path
// Before fix: %2B decoded to space
// After fix: %2B correctly decoded to +
testResolve(test.StartTestCustomServer, "C%2B%2B%20Primer.txt", t, func(err error) (*base.Resource, error) {
return &base.Resource{
Size: 0,
Range: false,
Files: []*base.FileInfo{
{
Name: "C++ Primer.txt",
Size: 0,
},
},
}, nil
})
// Test filename with HTML-encoded ampersand (fixes issue with & being truncated)
// Before fix: "查询处理&优化.pptx" -> "查询处理&amp" (truncated at semicolon)
// After fix: correctly decoded to "查询处理&优化.pptx"
testResolve(test.StartTestCustomServer, "ampersand-encoded", t, func(err error) (*base.Resource, error) {
return &base.Resource{
Size: test.BuildSize,
Range: false,
Files: []*base.FileInfo{
{
Name: "查询处理&优化.pptx",
Size: test.BuildSize,
},
},
}, nil
})
// Test unquoted filename with HTML-encoded ampersand
testResolve(test.StartTestCustomServer, "ampersand-unquoted", t, func(err error) (*base.Resource, error) {
return &base.Resource{
Size: test.BuildSize,
Range: false,
Files: []*base.FileInfo{
{
Name: "test&file.txt",
Size: test.BuildSize,
},
},
}, nil
})
// Test URL without file path - should use domain/host as filename
testResolve(test.StartTestCustomServer, "", t, func(err error) (*base.Resource, error) {
return &base.Resource{
Size: 0,
Range: false,
Files: []*base.FileInfo{
{
Name: "127.0.0.1",
Size: 0,
},
},
}, nil
})
// Test 403 Forbidden response handling
testResolve(test.StartTestCustomServer, "forbidden", t, func(err error) (*base.Resource, error) {
requestError := extractRequestError(err)
if requestError != nil && requestError.Code == 403 {
return nil, nil
}
return nil, err
})
}
func TestFetcher_ResolveWithHostHeader(t *testing.T) {
listener := test.StartTestHostHeaderServer()
defer listener.Close()
fetcher := buildFetcher()
err := fetcher.Resolve(&base.Request{
URL: "http://" + listener.Addr().String() + "/",
Extra: &http.ReqExtra{
Header: map[string]string{
"Host": "test",
},
},
}, &base.Options{
Name: test.DownloadName,
Path: test.Dir,
})
// The server should return 400 for invalid Host header
if err == nil || !strings.Contains(err.Error(), "400") {
t.Errorf("Resolve() got = %v, want error containing 400", err)
}
}
func TestFetcher_ResolveWithInvalidHeader(t *testing.T) {
listener := test.StartTestCustomServer()
defer listener.Close()
fetcher := buildFetcher()
defer fetcher.Pause() // Close the resolve response to allow server shutdown
err := fetcher.Resolve(&base.Request{
URL: "http://" + listener.Addr().String() + "/",
Extra: &http.ReqExtra{
Header: map[string]string{
"Referer": "\rtest",
},
},
}, &base.Options{
Name: test.DownloadName,
Path: test.Dir,
})
// Invalid header with \r should be sanitized by Go's http client, allowing the request to succeed
if err != nil {
t.Errorf("Resolve() got = %v, want nil (invalid headers should be sanitized)", err)
}
}
func testResolve(startTestServer func() net.Listener, path string, t *testing.T, wantFn func(error) (*base.Resource, error)) {
listener := startTestServer()
defer listener.Close()
fetcher := buildFetcher()
defer fetcher.Pause() // Close the resolve response to allow server shutdown
err := fetcher.Resolve(&base.Request{
URL: "http://" + listener.Addr().String() + "/" + path,
}, &base.Options{
Name: test.DownloadName,
Path: test.Dir,
})
want, err := wantFn(err)
if err != nil {
t.Fatal(err)
}
if want != nil && !test.AssertResourceEqual(want, fetcher.meta.Res) {
t.Errorf("Resolve() got = %+v, want %+v", fetcher.meta.Res, want)
}
}
func TestFetcher_DownloadNormal(t *testing.T) {
listener := test.StartTestFileServer()
defer listener.Close()
downloadNormal(listener, 1, t)
downloadNormal(listener, 5, t)
downloadNormal(listener, 8, t)
downloadNormal(listener, 16, t)
}
func TestFetcher_DownloadContinue(t *testing.T) {
listener := test.StartTestFileServer()
defer listener.Close()
downloadContinue(listener, 1, t)
downloadContinue(listener, 5, t)
downloadContinue(listener, 8, t)
downloadContinue(listener, 16, t)
}
func TestFetcher_DownloadContinue_NoRangeRestart(t *testing.T) {
listener := test.StartTestNoRangeSlowServer(time.Millisecond)
defer listener.Close()
fetcher := downloadReady(listener, 4, t)
if err := fetcher.Start(); err != nil {
t.Fatal(err)
}
time.Sleep(20 * time.Millisecond)
stats := fetcher.Stats().(*http.Stats)
if len(stats.Connections) != 1 {
t.Fatalf("expected a single non-range connection, got %d", len(stats.Connections))
}
if stats.Connections[0].Downloaded <= 0 || stats.Connections[0].Downloaded >= test.BuildSize {
t.Fatalf("expected partial download before pause, got %d", stats.Connections[0].Downloaded)
}
if err := fetcher.Pause(); err != nil {
t.Fatal(err)
}
if err := fetcher.Start(); err != nil {
t.Fatal(err)
}
if err := fetcher.Wait(); err != nil {
t.Fatal(err)
}
finalStats := fetcher.Stats().(*http.Stats)
if len(finalStats.Connections) != 1 {
t.Fatalf("expected a single non-range connection after resume, got %d", len(finalStats.Connections))
}
if finalStats.Connections[0].Downloaded != test.BuildSize {
t.Fatalf("downloaded bytes should restart cleanly: got %d, want %d", finalStats.Connections[0].Downloaded, test.BuildSize)
}
if total := fetcher.Progress().TotalDownloaded(); total != test.BuildSize {
t.Fatalf("progress total = %d, want %d", total, test.BuildSize)
}
want := test.FileMd5(test.BuildFile)
got := test.FileMd5(test.DownloadFile)
if want != got {
t.Errorf("Download() got = %v, want %v", got, want)
}
}
func TestFetcher_DownloadChunked(t *testing.T) {
listener := test.StartTestCustomServer()
defer listener.Close()
downloadNormal(listener, 1, t)
downloadNormal(listener, 2, t)
}
func TestFetcher_DownloadPost(t *testing.T) {
listener := test.StartTestPostServer()
defer listener.Close()
downloadPost(listener, 1, t)
}
func TestFetcher_DownloadRetry(t *testing.T) {
listener := test.StartTestRetryServer()
defer listener.Close()
downloadNormal(listener, 1, t)
}
func TestFetcher_DownloadError(t *testing.T) {
listener := test.StartTestErrorServer()
defer listener.Close()
downloadError(listener, 1, t)
}
func TestFetcher_DownloadLimit(t *testing.T) {
listener := test.StartTestConLimitServer(4)
defer listener.Close()
downloadNormal(listener, 1, t)
downloadNormal(listener, 2, t)
downloadNormal(listener, 8, t)
}
func TestFetcher_DownloadResponseBodyReadTimeout(t *testing.T) {
// Server will timeout once (first request delays longer than readTimeout),
// then subsequent requests work normally
listener := test.StartTestTimeoutOnceServer(readTimeout.Milliseconds() + 5000)
defer listener.Close()
for _, connections := range []int{1, 4} {
os.Remove(test.DownloadFile)
fetcher := downloadReady(listener, connections, t)
if err := fetcher.Start(); err != nil {
t.Fatal(err)
}
if err := fetcher.Wait(); err != nil {
t.Fatal(err)
}
stats := fetcher.Stats().(*http.Stats)
if len(stats.Connections) == 0 {
t.Fatalf("expected connections stats for timeout test")
}
// Verify successful download after timeout recovery
want := test.FileMd5(test.BuildFile)
got := test.FileMd5(test.DownloadFile)
if want != got {
t.Errorf("Download() got = %v, want %v", got, want)
}
// Verify timeouts don't count as failures (retryTimes should be 0)
for _, conn := range stats.Connections {
if conn.Failed {
t.Fatalf("expected no counted failures after timeout recovery, got retries=%d", conn.RetryTimes)
}
if conn.RetryTimes != 0 {
t.Fatalf("expected retryTimes to stay zero for non-counted timeouts, got %d", conn.RetryTimes)
}
}
}
}
func TestFetcher_Download500Recovery(t *testing.T) {
// Server returns 500 for 15 seconds, then recovers
listener := test.StartTestTemporary500Server(15 * time.Second)
defer listener.Close()
os.Remove(test.DownloadFile)
fetcher := downloadReady(listener, 4, t)
if err := fetcher.Start(); err != nil {
t.Fatal(err)
}
if err := fetcher.Wait(); err != nil {
t.Fatal(err)
}
// Verify successful download after 500 errors
want := test.FileMd5(test.BuildFile)
got := test.FileMd5(test.DownloadFile)
if want != got {
t.Errorf("Download() got = %v, want %v", got, want)
}
// Verify 500 errors don't count as failures (retryTimes should be 0)
stats := fetcher.Stats().(*http.Stats)
for _, conn := range stats.Connections {
if conn.RetryTimes != 0 {
t.Errorf("Expected retryTimes to be 0 for 500 errors (exempt), got %d", conn.RetryTimes)
}
}
}
func TestFetcher_DownloadOnBugFileServer(t *testing.T) {
listener := test.StartTestRangeBugServer()
defer listener.Close()
downloadNormal(listener, 1, t)
downloadNormal(listener, 4, t)
}
func TestFetcher_DownloadResume(t *testing.T) {
listener := test.StartTestFileServer()
defer listener.Close()
downloadResume(listener, 1, t)
downloadResume(listener, 5, t)
downloadResume(listener, 8, t)
downloadResume(listener, 16, t)
}
func TestFetcher_DownloadWithProxy(t *testing.T) {
httpListener := test.StartTestFileServer()
defer httpListener.Close()
proxyListener := test.StartSocks5Server("", "")
defer proxyListener.Close()
downloadWithProxy(httpListener, proxyListener, t)
}
func TestFetcher_ConfigConnections(t *testing.T) {
listener := test.StartTestFileServer()
defer listener.Close()
fetcher := doDownloadReady(buildConfigFetcher(config{
Connections: 16,
}), listener, 0, t)
err := fetcher.Start()
if err != nil {
t.Fatal(err)
}
err = fetcher.Wait()
if err != nil {
t.Fatal(err)
}
want := test.FileMd5(test.BuildFile)
got := test.FileMd5(test.DownloadFile)
if want != got {
t.Errorf("Download() got = %v, want %v", got, want)
}
}
func TestFetcher_ConfigUseServerCtime(t *testing.T) {
listener := test.StartTestFileServer()
defer listener.Close()
fetcher := doDownloadReady(buildConfigFetcher(config{
Connections: 16,
UseServerCtime: true,
}), listener, 0, t)
err := fetcher.Start()
if err != nil {
t.Fatal(err)
}
err = fetcher.Wait()
if err != nil {
t.Fatal(err)
}
want := test.FileMd5(test.BuildFile)
got := test.FileMd5(test.DownloadFile)
if want != got {
t.Errorf("Download() got = %v, want %v", got, want)
}
}
func TestFetcher_Stats(t *testing.T) {
listener := test.StartTestFileServer()
defer listener.Close()
fetcher := doDownloadReady(buildConfigFetcher(config{
Connections: 16,
}), listener, 0, t)
err := fetcher.Start()
if err != nil {
t.Fatal(err)
}
err = fetcher.Wait()
if err != nil {
t.Fatal(err)
}
stats := fetcher.Stats().(*http.Stats)
// With slow-start strategy, connection count may be less than max if download is fast
// Just verify we have at least 1 connection and no more than max
if len(stats.Connections) < 1 || len(stats.Connections) > 16 {
t.Errorf("Stats() connection count got = %v, want between 1 and 16", len(stats.Connections))
}
totalDownloaded := int64(0)
for i, conn := range stats.Connections {
t.Logf("Connection %d: Downloaded=%d, Completed=%v", i, conn.Downloaded, conn.Completed)
totalDownloaded += conn.Downloaded
}
if totalDownloaded != test.BuildSize {
t.Errorf("Stats() got = %v, want %v", totalDownloaded, test.BuildSize)
}
}
// TestFetcher_DownloadOneTimeURL tests downloading from a URL that can only be accessed once
// This simulates signed URLs or one-time download links that expire after first use
func TestFetcher_DownloadOneTimeURL(t *testing.T) {
listener := test.StartTestOneTimeServer()
defer listener.Close()
fetcher := buildFetcher()
err := fetcher.Resolve(&base.Request{
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
}, &base.Options{
Name: test.DownloadName,
Path: test.Dir,
Extra: &http.OptsExtra{
Connections: 4, // Try to use multiple connections, but only first should work
},
})
if err != nil {
t.Fatal(err)
}
err = fetcher.Start()
if err != nil {
t.Fatal(err)
}
err = fetcher.Wait()
if err != nil {
t.Fatal(err)
}
// Verify file content
want := test.FileMd5(test.BuildFile)
got := test.FileMd5(test.DownloadFile)
if want != got {
t.Errorf("Download() got = %v, want %v", got, want)
}
}
// TestFetcher_SlowStartExpansion tests slow-start connection expansion edge cases
// Tests that slow-start expansion reaches exactly maxConns
// Expansion pattern: 1 -> 2 -> 4 -> 8 -> 16...
// For max=5: 1 -> 2 -> 4 -> 5 (capped)
// For max=9: 1 -> 2 -> 4 -> 8 -> 9 (capped)
func TestFetcher_SlowStartExpansion(t *testing.T) {
testCases := []struct {
name string
maxConns int
}{
{"MaxConns5", 5}, // 1->2->4->5
{"MaxConns9", 9}, // 1->2->4->8->9
{"MaxConns8", 8}, // 1->2->4->8
}
for _, tc := range testCases {
tc := tc // capture range variable
t.Run(tc.name, func(t *testing.T) {
// Clean up any leftover files from previous tests
os.Remove(test.DownloadFile)
// Use 100ns delay per byte for faster test (~10MB/s theoretical)
listener := test.StartTestLowSpeedServer(100 * time.Nanosecond)
// Ensure cleanup happens before next subtest
cleanup := func() {
listener.Close()
os.Remove(test.DownloadFile)
// Wait for server to fully stop
time.Sleep(50 * time.Millisecond)
}
fetcher := buildConfigFetcher(config{
Connections: tc.maxConns,
})
err := fetcher.Resolve(&base.Request{
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
}, &base.Options{
Name: test.DownloadName,
Path: test.Dir,
Extra: &http.OptsExtra{
Connections: tc.maxConns,
},
})
if err != nil {
cleanup()
t.Fatal(err)
}
err = fetcher.Start()
if err != nil {
cleanup()
t.Fatal(err)
}
err = fetcher.Wait()
if err != nil {
t.Logf("Wait() returned error: %v", err)
cleanup()
t.Fatal(err)
}
// Check final connection count equals maxConns exactly
stats := fetcher.Stats().(*http.Stats)
finalConns := len(stats.Connections)
// Debug: show connection details and metadata
httpFetcher := fetcher.(*Fetcher)
t.Logf("Resource: Size=%d, Range=%v", httpFetcher.Meta().Res.Size, httpFetcher.Meta().Res.Range)
for i, conn := range stats.Connections {
t.Logf("Connection %d: Downloaded=%d, Completed=%v", i, conn.Downloaded, conn.Completed)
}
if finalConns != tc.maxConns {
t.Errorf("Expected exactly %d connections, got %d", tc.maxConns, finalConns)
}
// Verify file content before cleanup
want := test.FileMd5(test.BuildFile)
got := test.FileMd5(test.DownloadFile)
if want != got {
t.Errorf("Download() got = %v, want %v", got, want)
}
cleanup()
})
}
}
// TestFetcher_AsyncPrefetch tests the async prefetch functionality
// where data is downloaded in background during resolve phase and reused in start
func TestFetcher_AsyncPrefetch(t *testing.T) {
// Test 1: Prefetch completes entire file before Start is called
t.Run("PrefetchComplete", func(t *testing.T) {
listener := test.StartTestFileServer()
defer listener.Close()
fetcher := buildFetcher()
err := fetcher.Resolve(&base.Request{
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
}, &base.Options{
Name: test.DownloadName,
Path: test.Dir,
Extra: &http.OptsExtra{
Connections: 4,
},
})
if err != nil {
t.Fatal(err)
}
// Poll until prefetch completes the entire file (with timeout)
timeout := time.After(30 * time.Second)
ticker := time.NewTicker(100 * time.Millisecond)
defer ticker.Stop()
pollLoop:
for {
select {
case <-timeout:
t.Fatal("Timeout waiting for prefetch to complete")
case <-ticker.C:
if fetcher.prefetchSize.Load() >= test.BuildSize {
break pollLoop
}
}
}
prefetchedBefore := fetcher.prefetchSize.Load()
t.Logf("Prefetched bytes before Start: %d (%.2f MB)", prefetchedBefore, float64(prefetchedBefore)/(1024*1024))
// Should have prefetched the entire file
if prefetchedBefore != test.BuildSize {
t.Errorf("Prefetch should complete entire file, got %d, want %d", prefetchedBefore, test.BuildSize)
}
// Now start the download
err = fetcher.Start()
if err != nil {
t.Fatal(err)
}
// Wait for download to complete
err = fetcher.Wait()
if err != nil {
t.Fatal(err)
}
// Check how much was utilized from prefetch
prefetchedUsed := fetcher.resolveDataPos.Load()
t.Logf("Prefetched bytes used: %d (%.2f MB)", prefetchedUsed, float64(prefetchedUsed)/(1024*1024))
// Verify file is correct
want := test.FileMd5(test.BuildFile)
got := test.FileMd5(test.DownloadFile)
if want != got {
t.Errorf("Download() got = %v, want %v", got, want)
}
os.Remove(test.DownloadFile)
})
// Test 2: Prefetch only downloads partial data before Start is called
t.Run("PrefetchPartial", func(t *testing.T) {
// Use slow server with 100 nanosecond delay per byte
// This means ~10MB/s speed, so 100ms should download ~1MB
listener := test.StartTestLowSpeedServer(100 * time.Nanosecond)
defer listener.Close()
fetcher := buildFetcher()
err := fetcher.Resolve(&base.Request{
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
}, &base.Options{
Name: test.DownloadName,
Path: test.Dir,
Extra: &http.OptsExtra{
Connections: 4,
},
})
if err != nil {
t.Fatal(err)
}
// Wait only 100ms - should only prefetch a small portion
time.Sleep(100 * time.Millisecond)
prefetchedBefore := fetcher.prefetchSize.Load()
t.Logf("Prefetched bytes before Start: %d (%.2f KB)", prefetchedBefore, float64(prefetchedBefore)/1024)
// Verify we have partial data (not zero, but not complete)
if prefetchedBefore == 0 {
t.Log("Warning: No data prefetched, may be too slow")
}
if prefetchedBefore >= test.BuildSize {
t.Log("Warning: Prefetch completed entire file, test may not be valid")
}
// Now start the download
err = fetcher.Start()
if err != nil {
t.Fatal(err)
}
// Wait for download to complete
err = fetcher.Wait()
if err != nil {
t.Fatal(err)
}
// Check stats - should have connections that downloaded remaining data
stats := fetcher.Stats().(*http.Stats)
t.Logf("Final connections: %d", len(stats.Connections))
prefetchedUsed := fetcher.resolveDataPos.Load()
t.Logf("Prefetched bytes used: %d (%.2f KB)", prefetchedUsed, float64(prefetchedUsed)/1024)
// Verify connections picked up where prefetch left off
if len(stats.Connections) > 0 {
firstConn := stats.Connections[0]
t.Logf("First connection downloaded: %d bytes", firstConn.Downloaded)
}
// Verify file is correct
want := test.FileMd5(test.BuildFile)
got := test.FileMd5(test.DownloadFile)
if want != got {
t.Errorf("Download() got = %v, want %v", got, want)
}
os.Remove(test.DownloadFile)
})
}
// TestFetcher_DownloadExpiringRedirectURL tests that the fetcher correctly handles
// expiring redirect URLs by falling back to the original URL and getting a new redirect.
func TestFetcher_DownloadExpiringRedirectURL(t *testing.T) {
// Test with redirect expiring after 2 requests
// This means:
// - Request 1: Resolve (original URL redirects to temp URL v1)
// - Request 2: First download request to temp URL v1 succeeds
// - Request 3: Second download request to temp URL v1 returns 403
// - Fetcher should then retry with original URL, get temp URL v2
// - Continue downloading from temp URL v2
t.Run("RedirectExpiresAfter2Requests", func(t *testing.T) {
os.Remove(test.DownloadFile)
// Create server with redirect expiring after 2 requests, with slow transfer to ensure
// multiple connection attempts are needed
listener := test.StartTestExpiringRedirectServer(2, 100*time.Nanosecond)
defer listener.Close()
fetcher := buildFetcher()
err := fetcher.Resolve(&base.Request{
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
}, &base.Options{
Name: test.DownloadName,
Path: test.Dir,
Extra: &http.OptsExtra{
Connections: 4, // Use multiple connections to trigger redirect expiration
},
})
if err != nil {
t.Fatal(err)
}
err = fetcher.Start()
if err != nil {
t.Fatal(err)
}
err = fetcher.Wait()
if err != nil {
t.Fatal(err)
}
// Verify file content is correct despite redirect expiration
want := test.FileMd5(test.BuildFile)
got := test.FileMd5(test.DownloadFile)
if want != got {
t.Errorf("Download() got = %v, want %v", got, want)
}
os.Remove(test.DownloadFile)
})
// Test with redirect expiring after 5 requests (more room for initial connections)
t.Run("RedirectExpiresAfter5Requests", func(t *testing.T) {
os.Remove(test.DownloadFile)
listener := test.StartTestExpiringRedirectServer(5, 100*time.Nanosecond)
defer listener.Close()
fetcher := buildFetcher()
err := fetcher.Resolve(&base.Request{
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
}, &base.Options{
Name: test.DownloadName,
Path: test.Dir,
Extra: &http.OptsExtra{
Connections: 8,
},
})
if err != nil {
t.Fatal(err)
}
err = fetcher.Start()
if err != nil {
t.Fatal(err)
}
err = fetcher.Wait()
if err != nil {
t.Fatal(err)
}
// Verify file content
want := test.FileMd5(test.BuildFile)
got := test.FileMd5(test.DownloadFile)
if want != got {
t.Errorf("Download() got = %v, want %v", got, want)
}
os.Remove(test.DownloadFile)
})
}
// TestFetcher_RetryAfterError tests that the fetcher can retry downloading
// after a previous download attempt failed by calling Start() again.
func TestFetcher_RetryAfterError(t *testing.T) {
os.Remove(test.DownloadFile)
// Server fails first 3 requests (after resolve), then recovers
// With 1 connection and 3 retries:
// - First Start(): requests 2, 3, 4 → all fail (3 retries exhausted) → returns error
// - Second Start(): request 5 → succeeds (server recovered after 3 failures)
listener := test.StartTestFailThenRecoverServer(3)
defer listener.Close()
fetcher := buildFetcher()
err := fetcher.Resolve(&base.Request{
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
}, &base.Options{
Name: test.DownloadName,
Path: test.Dir,
Extra: &http.OptsExtra{
Connections: 1, // Use single connection to simplify test
},
})
if err != nil {
t.Fatal(err)
}
// First download attempt - should fail because server returns 416 after resolve
err = fetcher.Start()
if err != nil {
t.Fatal(err)
}
err = fetcher.Wait()
// First attempt should fail with 416 error
if err == nil {
t.Fatal("Expected first download attempt to fail, but it succeeded")
}
t.Logf("First attempt failed as expected: %v", err)
// Check that fetcher is in error state
state := fetcher.getState()
if state != stateError {
t.Errorf("Expected fetcher to be in stateError, got %v", state)
}
// Verify that we can call Start() again after error
// This tests the stateError handling in Start()
err = fetcher.Start()
if err != nil {
t.Fatalf("Start() after error failed: %v", err)
}
// Wait for second attempt - should succeed now that server has recovered
err = fetcher.Wait()
if err != nil {
t.Fatalf("Retry failed: %v", err)
}
// Verify file content
want := test.FileMd5(test.BuildFile)
got := test.FileMd5(test.DownloadFile)
if want != got {
t.Errorf("Download() got = %v, want %v", got, want)
}
os.Remove(test.DownloadFile)
}
func TestFetcherManager_ParseName(t *testing.T) {
type args struct {
u string
}
tests := []struct {
name string
args args
want string
}{
{
name: "broken url",
args: args{
u: "https://!@#%github.com",
},
want: "",
},
{
name: "file path",
args: args{
u: "https://github.com/index.html",
},
want: "index.html",
},
{
name: "file path with query and hash",
args: args{
u: "https://github.com/a/b/index.html/#list?name=1",
},
want: "index.html",
},
{
name: "no file path",
args: args{
u: "https://github.com",
},
want: "github.com",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
fm := &FetcherManager{}
if got := fm.ParseName(tt.args.u); got != tt.want {
t.Errorf("ParseName() = %v, want %v", got, tt.want)
}
})
}
}
func downloadReady(listener net.Listener, connections int, t *testing.T) fetcher.Fetcher {
return doDownloadReady(buildFetcher(), listener, connections, t)
}
func doDownloadReady(f fetcher.Fetcher, listener net.Listener, connections int, t *testing.T) fetcher.Fetcher {
var extra any = nil
if connections > 0 {
extra = &http.OptsExtra{
Connections: connections,
}
}
opts := &base.Options{
Name: test.DownloadName,
Path: test.Dir,
Extra: extra,
}
err := f.Resolve(&base.Request{
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
}, opts)
if err != nil {
t.Fatal(err)
}
return f
}
func downloadNormal(listener net.Listener, connections int, t *testing.T) {
fetcher := downloadReady(listener, connections, t)
err := fetcher.Start()
if err != nil {
t.Fatal(err)
}
err = fetcher.Wait()
if err != nil {
t.Fatal(err)
}
want := test.FileMd5(test.BuildFile)
got := test.FileMd5(test.DownloadFile)
if want != got {
t.Errorf("Download() got = %v, want %v", got, want)
}
}
func downloadPost(listener net.Listener, connections int, t *testing.T) {
// POST parameters must be set before Resolve since the new design
// starts downloading during Resolve phase
f := buildFetcher()
var extra any = nil
if connections > 0 {
extra = &http.OptsExtra{
Connections: connections,
}
}
opts := &base.Options{
Name: test.DownloadName,
Path: test.Dir,
Extra: extra,
}
req := &base.Request{
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
Extra: &http.ReqExtra{
Method: "POST",
Header: map[string]string{
"Authorization": "Bearer 123456",
},
Body: fmt.Sprintf(`{"name":"%s"}`, test.BuildName),
},
}
err := f.Resolve(req, opts)
if err != nil {
t.Fatal(err)
}
err = f.Start()
if err != nil {
t.Fatal(err)
}
err = f.Wait()
if err != nil {
t.Fatal(err)
}
want := test.FileMd5(test.BuildFile)
got := test.FileMd5(test.DownloadFile)
if want != got {
t.Errorf("Download() got = %v, want %v", got, want)
}
}
func downloadContinue(listener net.Listener, connections int, t *testing.T) {
fetcher := downloadReady(listener, connections, t)
err := fetcher.Start()
if err != nil {
t.Fatal(err)
}
time.Sleep(time.Millisecond * 50)
if err := fetcher.Pause(); err != nil {
t.Fatal(err)
}
time.Sleep(time.Millisecond * 50)
if err := fetcher.Start(); err != nil {
t.Fatal(err)
}
err = fetcher.Wait()
if err != nil {
t.Fatal(err)
}
want := test.FileMd5(test.BuildFile)
got := test.FileMd5(test.DownloadFile)
if want != got {
t.Errorf("Download() got = %v, want %v", got, want)
}
}
func downloadError(listener net.Listener, connections int, t *testing.T) {
fetcher := buildFetcher()
err := fetcher.Resolve(&base.Request{
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
}, &base.Options{
Name: test.DownloadName,
Path: test.Dir,
})
// With the new async design, Resolve may succeed (HTTP response received)
// but errors occur during async download or Start/Wait
if err != nil {
// Error detected in Resolve - this is fine
return
}
// Resolve succeeded, error should occur during Start/Wait
err = fetcher.Start()
if err != nil {
// Error detected in Start - this is fine
return
}
err = fetcher.Wait()
if err == nil {
t.Errorf("Expected error during download, but got none")
}
}
func downloadResume(listener net.Listener, connections int, t *testing.T) {
fetcher := downloadReady(listener, connections, t)
err := fetcher.Start()
if err != nil {
t.Fatal(err)
}
fb := new(FetcherManager)
time.Sleep(time.Millisecond * 50)
data, err := fb.Store(fetcher)
if err != nil {
t.Fatal(err)
}
time.Sleep(time.Millisecond * 50)
fetcher.Pause()
_, f := fb.Restore()
f(fetcher.Meta(), data)
if err != nil {
t.Fatal(err)
}
fetcher.Setup(controller.NewController())
fetcher.Start()
err = fetcher.Wait()
if err != nil {
t.Fatal(err)
}
want := test.FileMd5(test.BuildFile)
got := test.FileMd5(test.DownloadFile)
if want != got {
t.Errorf("Download() got = %v, want %v", got, want)
}
}
func downloadWithProxy(httpListener net.Listener, proxyListener net.Listener, t *testing.T) {
fetcher := downloadReady(httpListener, 4, t)
ctl := controller.NewController()
ctl.GetProxy = func(requestProxy *base.RequestProxy) func(*gohttp.Request) (*url.URL, error) {
return (&base.DownloaderProxyConfig{
Enable: true,
Scheme: "socks5",
Host: proxyListener.Addr().String(),
}).ToHandler()
}
fetcher.Setup(ctl)
err := fetcher.Start()
if err != nil {
t.Fatal(err)
}
err = fetcher.Wait()
if err != nil {
t.Fatal(err)
}
want := test.FileMd5(test.BuildFile)
got := test.FileMd5(test.DownloadFile)
if want != got {
t.Errorf("Download() got = %v, want %v", got, want)
}
}
func buildFetcher() *Fetcher {
fm := new(FetcherManager)
fetcher := fm.Build()
newController := controller.NewController()
newController.GetConfig = func(v any) {
json.Unmarshal([]byte(test.ToJson(fm.DefaultConfig())), v)
}
fetcher.Setup(newController)
return fetcher.(*Fetcher)
}
func buildConfigFetcher(cfg config) fetcher.Fetcher {
fetcher := new(FetcherManager).Build()
newController := controller.NewController()
newController.GetConfig = func(v any) {
json.Unmarshal([]byte(test.ToJson(cfg)), v)
}
fetcher.Setup(newController)
return fetcher
}
// TestFetcher_Patch_URLChange tests the Patch functionality where a failed download URL
// is replaced with a working one. This simulates:
// 1. Initial download attempt with a bad URL (returns 404)
// 2. Patching the task with a new working URL
// 3. Successful download after URL modification
func TestFetcher_Patch_URLChange(t *testing.T) {
listener := test.StartTestPatchURLServer()
defer listener.Close()
f := buildFetcher()
badURL := "http://" + listener.Addr().String() + "/bad-url"
goodURL := "http://" + listener.Addr().String() + "/good-url"
opts := &base.Options{
Name: test.DownloadName,
Path: test.Dir,
Extra: &http.OptsExtra{Connections: 1},
}
// Step 1: Try to resolve with bad URL - should fail with error
err := f.Resolve(&base.Request{URL: badURL}, opts)
if err == nil {
t.Fatal("Expected error for bad URL, got nil")
}
// Step 2: Create a new fetcher and resolve with bad URL but don't wait
// We need to test patching a task that has been created
f2 := buildFetcher()
// First resolve with good URL to create a valid fetcher state
err = f2.Resolve(&base.Request{URL: goodURL}, opts)
if err != nil {
t.Fatal(err)
}
// Verify initial URL
if f2.meta.Req.URL != goodURL {
t.Errorf("Initial URL = %v, want %v", f2.meta.Req.URL, goodURL)
}
// Step 3: Patch to change URL (simulating URL change scenario)
newURL := "http://" + listener.Addr().String() + "/good-url"
err = f2.Patch(&base.Request{URL: newURL}, nil)
if err != nil {
t.Fatal(err)
}
// Verify URL was patched
if f2.meta.Req.URL != newURL {
t.Errorf("Patched URL = %v, want %v", f2.meta.Req.URL, newURL)
}
// Step 4: Start download and verify success
err = f2.Start()
if err != nil {
t.Fatal(err)
}
err = f2.Wait()
if err != nil {
t.Fatal(err)
}
// Verify file was downloaded correctly
want := test.FileMd5(test.BuildFile)
got := test.FileMd5(test.DownloadFile)
if want != got {
t.Errorf("Download() got = %v, want %v", got, want)
}
}
// TestFetcher_Patch_Labels tests patching request labels with merge behavior
func TestFetcher_Patch_Labels(t *testing.T) {
listener := test.StartTestFileServer()
defer listener.Close()
f := buildFetcher()
opts := &base.Options{
Name: test.DownloadName,
Path: test.Dir,
Extra: &http.OptsExtra{Connections: 1},
}
err := f.Resolve(&base.Request{
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
Labels: map[string]string{
"key1": "value1",
"key3": "value3",
},
}, opts)
if err != nil {
t.Fatal(err)
}
// Verify initial labels
if f.meta.Req.Labels["key1"] != "value1" {
t.Errorf("Initial label key1 = %v, want value1", f.meta.Req.Labels["key1"])
}
if f.meta.Req.Labels["key3"] != "value3" {
t.Errorf("Initial label key3 = %v, want value3", f.meta.Req.Labels["key3"])
}
// Patch with new labels - key1 should be overwritten, key2 should be added, key3 should remain
patchReq := &base.Request{
Labels: map[string]string{
"key1": "modified",
"key2": "newValue",
},
}
err = f.Patch(patchReq, nil)
if err != nil {
t.Fatal(err)
}
// Verify labels were merged correctly
if f.meta.Req.Labels["key1"] != "modified" {
t.Errorf("Patched label key1 = %v, want modified", f.meta.Req.Labels["key1"])
}
if f.meta.Req.Labels["key2"] != "newValue" {
t.Errorf("Patched label key2 = %v, want newValue", f.meta.Req.Labels["key2"])
}
// key3 should remain unchanged
if f.meta.Req.Labels["key3"] != "value3" {
t.Errorf("Label key3 = %v, want value3 (should remain unchanged)", f.meta.Req.Labels["key3"])
}
}
// TestFetcher_Patch_Extra tests patching request Extra with merge behavior
func TestFetcher_Patch_Extra(t *testing.T) {
listener := test.StartTestFileServer()
defer listener.Close()
f := buildFetcher()
opts := &base.Options{
Name: test.DownloadName,
Path: test.Dir,
Extra: &http.OptsExtra{Connections: 1},
}
// Resolve with initial Extra
err := f.Resolve(&base.Request{
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
Extra: &http.ReqExtra{
Method: "GET",
Body: "initial body",
Header: map[string]string{
"Authorization": "Bearer token123",
"X-Custom": "original",
},
},
}, opts)
if err != nil {
t.Fatal(err)
}
// Verify initial Extra
initialExtra := f.meta.Req.Extra.(*http.ReqExtra)
if initialExtra.Method != "GET" {
t.Errorf("Initial Method = %v, want GET", initialExtra.Method)
}
if initialExtra.Body != "initial body" {
t.Errorf("Initial Body = %v, want 'initial body'", initialExtra.Body)
}
if initialExtra.Header["Authorization"] != "Bearer token123" {
t.Errorf("Initial Authorization header = %v, want 'Bearer token123'", initialExtra.Header["Authorization"])
}
if initialExtra.Header["X-Custom"] != "original" {
t.Errorf("Initial X-Custom header = %v, want 'original'", initialExtra.Header["X-Custom"])
}
// Patch with partial Extra - only update some fields
patchReq := &base.Request{
Extra: &http.ReqExtra{
Method: "POST", // Update method
// Body is empty, should NOT update
Header: map[string]string{
"X-Custom": "modified", // Overwrite existing
"X-New": "added", // Add new
// Authorization is not in patch, should remain
},
},
}
err = f.Patch(patchReq, nil)
if err != nil {
t.Fatal(err)
}
// Verify Extra was merged correctly
patchedExtra := f.meta.Req.Extra.(*http.ReqExtra)
// Method should be updated
if patchedExtra.Method != "POST" {
t.Errorf("Patched Method = %v, want POST", patchedExtra.Method)
}
// Body should remain unchanged (patch had empty body)
if patchedExtra.Body != "initial body" {
t.Errorf("Patched Body = %v, want 'initial body' (should remain unchanged)", patchedExtra.Body)
}
// Authorization header should remain unchanged
if patchedExtra.Header["Authorization"] != "Bearer token123" {
t.Errorf("Authorization header = %v, want 'Bearer token123' (should remain unchanged)", patchedExtra.Header["Authorization"])
}
// X-Custom header should be overwritten
if patchedExtra.Header["X-Custom"] != "modified" {
t.Errorf("X-Custom header = %v, want 'modified'", patchedExtra.Header["X-Custom"])
}
// X-New header should be added
if patchedExtra.Header["X-New"] != "added" {
t.Errorf("X-New header = %v, want 'added'", patchedExtra.Header["X-New"])
}
}
// TestFetcher_Patch_NilData tests that Patch with nil data doesn't cause errors
func TestFetcher_Patch_NilData(t *testing.T) {
listener := test.StartTestFileServer()
defer listener.Close()
f := buildFetcher()
opts := &base.Options{
Name: test.DownloadName,
Path: test.Dir,
Extra: &http.OptsExtra{Connections: 1},
}
err := f.Resolve(&base.Request{
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
}, opts)
if err != nil {
t.Fatal(err)
}
originalURL := f.meta.Req.URL
// Patch with nil data - should not cause error
err = f.Patch(nil, nil)
if err != nil {
t.Fatal(err)
}
// Verify URL unchanged
if f.meta.Req.URL != originalURL {
t.Errorf("URL changed after nil patch: got %v, want %v", f.meta.Req.URL, originalURL)
}
// Patch with empty request - should not cause error
err = f.Patch(&base.Request{}, nil)
if err != nil {
t.Fatal(err)
}
// Verify URL still unchanged
if f.meta.Req.URL != originalURL {
t.Errorf("URL changed after empty patch: got %v, want %v", f.meta.Req.URL, originalURL)
}
}
// TestFetcher_Patch_CookieExpired tests the Patch functionality where a download fails
// mid-way due to expired cookie, then succeeds after patching with a new valid cookie.
// This simulates:
// 1. Initial resolve with valid cookie succeeds
// 2. Download starts but fails because cookie expires mid-download (server returns 401)
// 3. User patches the task with a new valid cookie
// 4. Download resumes and completes successfully
func TestFetcher_Patch_CookieExpired(t *testing.T) {
listener := test.StartTestCookieExpiringServer()
defer listener.Close()
downloadURL := "http://" + listener.Addr().String() + "/" + test.BuildName
opts := &base.Options{
Name: test.DownloadName,
Path: test.Dir,
Extra: &http.OptsExtra{Connections: 1},
}
// Step 1: Resolve with old_token - should succeed (first request accepts old_token)
f := buildFetcher()
err := f.Resolve(&base.Request{
URL: downloadURL,
Extra: &http.ReqExtra{
Header: map[string]string{
"Cookie": "session=old_token",
},
},
}, opts)
if err != nil {
t.Fatalf("Resolve should succeed with old_token: %v", err)
}
// Verify initial cookie
initialExtra := f.meta.Req.Extra.(*http.ReqExtra)
if initialExtra.Header["Cookie"] != "session=old_token" {
t.Errorf("Initial Cookie = %v, want session=old_token", initialExtra.Header["Cookie"])
}
// Step 2: Start download - should fail because old_token is now expired
// (server only accepts old_token for first request, subsequent requests need new_token)
err = f.Start()
if err != nil {
t.Fatalf("Start failed: %v", err)
}
err = f.Wait()
// Download should fail with 401 error
if err == nil {
t.Fatal("Expected download to fail with expired cookie, but it succeeded")
}
t.Logf("Download failed as expected: %v", err)
// Step 3: Patch with new valid cookie
err = f.Patch(&base.Request{
Extra: &http.ReqExtra{
Header: map[string]string{
"Cookie": "session=new_token",
},
},
}, nil)
if err != nil {
t.Fatalf("Patch to update cookie failed: %v", err)
}
// Verify cookie was updated
patchedExtra := f.meta.Req.Extra.(*http.ReqExtra)
if patchedExtra.Header["Cookie"] != "session=new_token" {
t.Errorf("Cookie should be updated: got %v, want session=new_token", patchedExtra.Header["Cookie"])
}
// Step 4: Restart download - should succeed with new cookie
err = f.Start()
if err != nil {
t.Fatalf("Restart failed: %v", err)
}
err = f.Wait()
if err != nil {
t.Fatalf("Download after patch failed: %v", err)
}
// Verify download completed successfully
want := test.FileMd5(test.BuildFile)
got := test.FileMd5(test.DownloadFile)
if want != got {
t.Errorf("File MD5 mismatch: got %v, want %v", got, want)
}
}