package http import ( "encoding/json" "fmt" "net" gohttp "net/http" "net/url" "os" "strings" "testing" "time" "github.com/GopeedLab/gopeed/internal/controller" "github.com/GopeedLab/gopeed/internal/fetcher" "github.com/GopeedLab/gopeed/internal/test" "github.com/GopeedLab/gopeed/pkg/base" "github.com/GopeedLab/gopeed/pkg/protocol/http" ) func TestFetcher_Resolve(t *testing.T) { testResolve(test.StartTestFileServer, test.BuildName, t, func(err error) (*base.Resource, error) { return &base.Resource{ Size: test.BuildSize, Range: true, Files: []*base.FileInfo{ { Name: test.BuildName, Size: test.BuildSize, }, }, }, nil }) testResolve(test.StartTestCustomServer, "disposition", t, func(err error) (*base.Resource, error) { return &base.Resource{ Size: test.BuildSize, Range: false, Files: []*base.FileInfo{ { Name: test.BuildName, Size: test.BuildSize, }, }, }, nil }) testResolve(test.StartTestCustomServer, "encoded-word", t, func(err error) (*base.Resource, error) { return &base.Resource{ Size: test.BuildSize, Range: false, Files: []*base.FileInfo{ { Name: test.TestChineseFileName, Size: test.BuildSize, }, }, }, nil }) testResolve(test.StartTestCustomServer, "no-encode", t, func(err error) (*base.Resource, error) { return &base.Resource{ Size: test.BuildSize, Range: false, Files: []*base.FileInfo{ { Name: test.TestChineseFileName, Size: test.BuildSize, }, }, }, nil }) testResolve(test.StartTestCustomServer, "%E6%B5%8B%E8%AF%95.zip", t, func(err error) (*base.Resource, error) { return &base.Resource{ Size: 0, Range: false, Files: []*base.FileInfo{ { Name: test.TestChineseFileName, Size: 0, }, }, }, nil }) testResolve(test.StartTestCustomServer, test.BuildName, t, func(err error) (*base.Resource, error) { return &base.Resource{ Size: 0, Range: false, Files: []*base.FileInfo{ { Name: test.BuildName, Size: 0, }, }, }, nil }) // Test mixed encoding Content-Disposition where mime.ParseMediaType fails // due to invalid characters, but filename*= contains the correct UTF-8 encoded name testResolve(test.StartTestCustomServer, "mixed-encoding", t, func(err error) (*base.Resource, error) { return &base.Resource{ Size: test.BuildSize, Range: false, Files: []*base.FileInfo{ { Name: test.TestChineseFileName, Size: test.BuildSize, }, }, }, nil }) // Test filename*= only (RFC 5987 format) testResolve(test.StartTestCustomServer, "filename-star", t, func(err error) (*base.Resource, error) { return &base.Resource{ Size: test.BuildSize, Range: false, Files: []*base.FileInfo{ { Name: test.TestChineseFileName, Size: test.BuildSize, }, }, }, nil }) // Test GBK-encoded filename (common on Chinese Windows servers) // Before fix: "测试.zip" sent as GBK bytes -> parsed as "²âÊÔ.zip" (garbled) // After fix: correctly decoded back to "测试.zip" testResolve(test.StartTestCustomServer, "gbk-encoded", t, func(err error) (*base.Resource, error) { return &base.Resource{ Size: test.BuildSize, Range: false, Files: []*base.FileInfo{ { Name: test.TestChineseFileName, Size: test.BuildSize, }, }, }, nil }) // Test filename with plus signs (e.g., C++ Primer) // Before fix: %2B decoded to space -> "C++ Primer" became "C Primer" // After fix: %2B correctly decoded to + -> "C++ Primer Plus.mobi" testResolve(test.StartTestCustomServer, "plus-sign-encoded", t, func(err error) (*base.Resource, error) { return &base.Resource{ Size: test.BuildSize, Range: false, Files: []*base.FileInfo{ { Name: "C++ Primer Plus.mobi", Size: test.BuildSize, }, }, }, nil }) // Test filename with plus sign in URL path // Before fix: %2B decoded to space // After fix: %2B correctly decoded to + testResolve(test.StartTestCustomServer, "C%2B%2B%20Primer.txt", t, func(err error) (*base.Resource, error) { return &base.Resource{ Size: 0, Range: false, Files: []*base.FileInfo{ { Name: "C++ Primer.txt", Size: 0, }, }, }, nil }) // Test filename with HTML-encoded ampersand (fixes issue with & being truncated) // Before fix: "查询处理&优化.pptx" -> "查询处理&" (truncated at semicolon) // After fix: correctly decoded to "查询处理&优化.pptx" testResolve(test.StartTestCustomServer, "ampersand-encoded", t, func(err error) (*base.Resource, error) { return &base.Resource{ Size: test.BuildSize, Range: false, Files: []*base.FileInfo{ { Name: "查询处理&优化.pptx", Size: test.BuildSize, }, }, }, nil }) // Test unquoted filename with HTML-encoded ampersand testResolve(test.StartTestCustomServer, "ampersand-unquoted", t, func(err error) (*base.Resource, error) { return &base.Resource{ Size: test.BuildSize, Range: false, Files: []*base.FileInfo{ { Name: "test&file.txt", Size: test.BuildSize, }, }, }, nil }) // Test URL without file path - should use domain/host as filename testResolve(test.StartTestCustomServer, "", t, func(err error) (*base.Resource, error) { return &base.Resource{ Size: 0, Range: false, Files: []*base.FileInfo{ { Name: "127.0.0.1", Size: 0, }, }, }, nil }) // Test 403 Forbidden response handling testResolve(test.StartTestCustomServer, "forbidden", t, func(err error) (*base.Resource, error) { requestError := extractRequestError(err) if requestError != nil && requestError.Code == 403 { return nil, nil } return nil, err }) } func TestFetcher_ResolveWithHostHeader(t *testing.T) { listener := test.StartTestHostHeaderServer() defer listener.Close() fetcher := buildFetcher() err := fetcher.Resolve(&base.Request{ URL: "http://" + listener.Addr().String() + "/", Extra: &http.ReqExtra{ Header: map[string]string{ "Host": "test", }, }, }, &base.Options{ Name: test.DownloadName, Path: test.Dir, }) // The server should return 400 for invalid Host header if err == nil || !strings.Contains(err.Error(), "400") { t.Errorf("Resolve() got = %v, want error containing 400", err) } } func TestFetcher_ResolveWithInvalidHeader(t *testing.T) { listener := test.StartTestCustomServer() defer listener.Close() fetcher := buildFetcher() defer fetcher.Pause() // Close the resolve response to allow server shutdown err := fetcher.Resolve(&base.Request{ URL: "http://" + listener.Addr().String() + "/", Extra: &http.ReqExtra{ Header: map[string]string{ "Referer": "\rtest", }, }, }, &base.Options{ Name: test.DownloadName, Path: test.Dir, }) // Invalid header with \r should be sanitized by Go's http client, allowing the request to succeed if err != nil { t.Errorf("Resolve() got = %v, want nil (invalid headers should be sanitized)", err) } } func testResolve(startTestServer func() net.Listener, path string, t *testing.T, wantFn func(error) (*base.Resource, error)) { listener := startTestServer() defer listener.Close() fetcher := buildFetcher() defer fetcher.Pause() // Close the resolve response to allow server shutdown err := fetcher.Resolve(&base.Request{ URL: "http://" + listener.Addr().String() + "/" + path, }, &base.Options{ Name: test.DownloadName, Path: test.Dir, }) want, err := wantFn(err) if err != nil { t.Fatal(err) } if want != nil && !test.AssertResourceEqual(want, fetcher.meta.Res) { t.Errorf("Resolve() got = %+v, want %+v", fetcher.meta.Res, want) } } func TestFetcher_DownloadNormal(t *testing.T) { listener := test.StartTestFileServer() defer listener.Close() downloadNormal(listener, 1, t) downloadNormal(listener, 5, t) downloadNormal(listener, 8, t) downloadNormal(listener, 16, t) } func TestFetcher_DownloadContinue(t *testing.T) { listener := test.StartTestFileServer() defer listener.Close() downloadContinue(listener, 1, t) downloadContinue(listener, 5, t) downloadContinue(listener, 8, t) downloadContinue(listener, 16, t) } func TestFetcher_DownloadContinue_NoRangeRestart(t *testing.T) { listener := test.StartTestNoRangeSlowServer(time.Millisecond) defer listener.Close() fetcher := downloadReady(listener, 4, t) if err := fetcher.Start(); err != nil { t.Fatal(err) } time.Sleep(20 * time.Millisecond) stats := fetcher.Stats().(*http.Stats) if len(stats.Connections) != 1 { t.Fatalf("expected a single non-range connection, got %d", len(stats.Connections)) } if stats.Connections[0].Downloaded <= 0 || stats.Connections[0].Downloaded >= test.BuildSize { t.Fatalf("expected partial download before pause, got %d", stats.Connections[0].Downloaded) } if err := fetcher.Pause(); err != nil { t.Fatal(err) } if err := fetcher.Start(); err != nil { t.Fatal(err) } if err := fetcher.Wait(); err != nil { t.Fatal(err) } finalStats := fetcher.Stats().(*http.Stats) if len(finalStats.Connections) != 1 { t.Fatalf("expected a single non-range connection after resume, got %d", len(finalStats.Connections)) } if finalStats.Connections[0].Downloaded != test.BuildSize { t.Fatalf("downloaded bytes should restart cleanly: got %d, want %d", finalStats.Connections[0].Downloaded, test.BuildSize) } if total := fetcher.Progress().TotalDownloaded(); total != test.BuildSize { t.Fatalf("progress total = %d, want %d", total, test.BuildSize) } want := test.FileMd5(test.BuildFile) got := test.FileMd5(test.DownloadFile) if want != got { t.Errorf("Download() got = %v, want %v", got, want) } } func TestFetcher_DownloadChunked(t *testing.T) { listener := test.StartTestCustomServer() defer listener.Close() downloadNormal(listener, 1, t) downloadNormal(listener, 2, t) } func TestFetcher_DownloadPost(t *testing.T) { listener := test.StartTestPostServer() defer listener.Close() downloadPost(listener, 1, t) } func TestFetcher_DownloadRetry(t *testing.T) { listener := test.StartTestRetryServer() defer listener.Close() downloadNormal(listener, 1, t) } func TestFetcher_DownloadError(t *testing.T) { listener := test.StartTestErrorServer() defer listener.Close() downloadError(listener, 1, t) } func TestFetcher_DownloadLimit(t *testing.T) { listener := test.StartTestConLimitServer(4) defer listener.Close() downloadNormal(listener, 1, t) downloadNormal(listener, 2, t) downloadNormal(listener, 8, t) } func TestFetcher_DownloadResponseBodyReadTimeout(t *testing.T) { // Server will timeout once (first request delays longer than readTimeout), // then subsequent requests work normally listener := test.StartTestTimeoutOnceServer(readTimeout.Milliseconds() + 5000) defer listener.Close() for _, connections := range []int{1, 4} { os.Remove(test.DownloadFile) fetcher := downloadReady(listener, connections, t) if err := fetcher.Start(); err != nil { t.Fatal(err) } if err := fetcher.Wait(); err != nil { t.Fatal(err) } stats := fetcher.Stats().(*http.Stats) if len(stats.Connections) == 0 { t.Fatalf("expected connections stats for timeout test") } // Verify successful download after timeout recovery want := test.FileMd5(test.BuildFile) got := test.FileMd5(test.DownloadFile) if want != got { t.Errorf("Download() got = %v, want %v", got, want) } // Verify timeouts don't count as failures (retryTimes should be 0) for _, conn := range stats.Connections { if conn.Failed { t.Fatalf("expected no counted failures after timeout recovery, got retries=%d", conn.RetryTimes) } if conn.RetryTimes != 0 { t.Fatalf("expected retryTimes to stay zero for non-counted timeouts, got %d", conn.RetryTimes) } } } } func TestFetcher_Download500Recovery(t *testing.T) { // Server returns 500 for 15 seconds, then recovers listener := test.StartTestTemporary500Server(15 * time.Second) defer listener.Close() os.Remove(test.DownloadFile) fetcher := downloadReady(listener, 4, t) if err := fetcher.Start(); err != nil { t.Fatal(err) } if err := fetcher.Wait(); err != nil { t.Fatal(err) } // Verify successful download after 500 errors want := test.FileMd5(test.BuildFile) got := test.FileMd5(test.DownloadFile) if want != got { t.Errorf("Download() got = %v, want %v", got, want) } // Verify 500 errors don't count as failures (retryTimes should be 0) stats := fetcher.Stats().(*http.Stats) for _, conn := range stats.Connections { if conn.RetryTimes != 0 { t.Errorf("Expected retryTimes to be 0 for 500 errors (exempt), got %d", conn.RetryTimes) } } } func TestFetcher_DownloadOnBugFileServer(t *testing.T) { listener := test.StartTestRangeBugServer() defer listener.Close() downloadNormal(listener, 1, t) downloadNormal(listener, 4, t) } func TestFetcher_DownloadResume(t *testing.T) { listener := test.StartTestFileServer() defer listener.Close() downloadResume(listener, 1, t) downloadResume(listener, 5, t) downloadResume(listener, 8, t) downloadResume(listener, 16, t) } func TestFetcher_DownloadWithProxy(t *testing.T) { httpListener := test.StartTestFileServer() defer httpListener.Close() proxyListener := test.StartSocks5Server("", "") defer proxyListener.Close() downloadWithProxy(httpListener, proxyListener, t) } func TestFetcher_ConfigConnections(t *testing.T) { listener := test.StartTestFileServer() defer listener.Close() fetcher := doDownloadReady(buildConfigFetcher(config{ Connections: 16, }), listener, 0, t) err := fetcher.Start() if err != nil { t.Fatal(err) } err = fetcher.Wait() if err != nil { t.Fatal(err) } want := test.FileMd5(test.BuildFile) got := test.FileMd5(test.DownloadFile) if want != got { t.Errorf("Download() got = %v, want %v", got, want) } } func TestFetcher_ConfigUseServerCtime(t *testing.T) { listener := test.StartTestFileServer() defer listener.Close() fetcher := doDownloadReady(buildConfigFetcher(config{ Connections: 16, UseServerCtime: true, }), listener, 0, t) err := fetcher.Start() if err != nil { t.Fatal(err) } err = fetcher.Wait() if err != nil { t.Fatal(err) } want := test.FileMd5(test.BuildFile) got := test.FileMd5(test.DownloadFile) if want != got { t.Errorf("Download() got = %v, want %v", got, want) } } func TestFetcher_Stats(t *testing.T) { listener := test.StartTestFileServer() defer listener.Close() fetcher := doDownloadReady(buildConfigFetcher(config{ Connections: 16, }), listener, 0, t) err := fetcher.Start() if err != nil { t.Fatal(err) } err = fetcher.Wait() if err != nil { t.Fatal(err) } stats := fetcher.Stats().(*http.Stats) // With slow-start strategy, connection count may be less than max if download is fast // Just verify we have at least 1 connection and no more than max if len(stats.Connections) < 1 || len(stats.Connections) > 16 { t.Errorf("Stats() connection count got = %v, want between 1 and 16", len(stats.Connections)) } totalDownloaded := int64(0) for i, conn := range stats.Connections { t.Logf("Connection %d: Downloaded=%d, Completed=%v", i, conn.Downloaded, conn.Completed) totalDownloaded += conn.Downloaded } if totalDownloaded != test.BuildSize { t.Errorf("Stats() got = %v, want %v", totalDownloaded, test.BuildSize) } } // TestFetcher_DownloadOneTimeURL tests downloading from a URL that can only be accessed once // This simulates signed URLs or one-time download links that expire after first use func TestFetcher_DownloadOneTimeURL(t *testing.T) { listener := test.StartTestOneTimeServer() defer listener.Close() fetcher := buildFetcher() err := fetcher.Resolve(&base.Request{ URL: "http://" + listener.Addr().String() + "/" + test.BuildName, }, &base.Options{ Name: test.DownloadName, Path: test.Dir, Extra: &http.OptsExtra{ Connections: 4, // Try to use multiple connections, but only first should work }, }) if err != nil { t.Fatal(err) } err = fetcher.Start() if err != nil { t.Fatal(err) } err = fetcher.Wait() if err != nil { t.Fatal(err) } // Verify file content want := test.FileMd5(test.BuildFile) got := test.FileMd5(test.DownloadFile) if want != got { t.Errorf("Download() got = %v, want %v", got, want) } } // TestFetcher_SlowStartExpansion tests slow-start connection expansion edge cases // Tests that slow-start expansion reaches exactly maxConns // Expansion pattern: 1 -> 2 -> 4 -> 8 -> 16... // For max=5: 1 -> 2 -> 4 -> 5 (capped) // For max=9: 1 -> 2 -> 4 -> 8 -> 9 (capped) func TestFetcher_SlowStartExpansion(t *testing.T) { testCases := []struct { name string maxConns int }{ {"MaxConns5", 5}, // 1->2->4->5 {"MaxConns9", 9}, // 1->2->4->8->9 {"MaxConns8", 8}, // 1->2->4->8 } for _, tc := range testCases { tc := tc // capture range variable t.Run(tc.name, func(t *testing.T) { // Clean up any leftover files from previous tests os.Remove(test.DownloadFile) // Use 100ns delay per byte for faster test (~10MB/s theoretical) listener := test.StartTestLowSpeedServer(100 * time.Nanosecond) // Ensure cleanup happens before next subtest cleanup := func() { listener.Close() os.Remove(test.DownloadFile) // Wait for server to fully stop time.Sleep(50 * time.Millisecond) } fetcher := buildConfigFetcher(config{ Connections: tc.maxConns, }) err := fetcher.Resolve(&base.Request{ URL: "http://" + listener.Addr().String() + "/" + test.BuildName, }, &base.Options{ Name: test.DownloadName, Path: test.Dir, Extra: &http.OptsExtra{ Connections: tc.maxConns, }, }) if err != nil { cleanup() t.Fatal(err) } err = fetcher.Start() if err != nil { cleanup() t.Fatal(err) } err = fetcher.Wait() if err != nil { t.Logf("Wait() returned error: %v", err) cleanup() t.Fatal(err) } // Check final connection count equals maxConns exactly stats := fetcher.Stats().(*http.Stats) finalConns := len(stats.Connections) // Debug: show connection details and metadata httpFetcher := fetcher.(*Fetcher) t.Logf("Resource: Size=%d, Range=%v", httpFetcher.Meta().Res.Size, httpFetcher.Meta().Res.Range) for i, conn := range stats.Connections { t.Logf("Connection %d: Downloaded=%d, Completed=%v", i, conn.Downloaded, conn.Completed) } if finalConns != tc.maxConns { t.Errorf("Expected exactly %d connections, got %d", tc.maxConns, finalConns) } // Verify file content before cleanup want := test.FileMd5(test.BuildFile) got := test.FileMd5(test.DownloadFile) if want != got { t.Errorf("Download() got = %v, want %v", got, want) } cleanup() }) } } // TestFetcher_AsyncPrefetch tests the async prefetch functionality // where data is downloaded in background during resolve phase and reused in start func TestFetcher_AsyncPrefetch(t *testing.T) { // Test 1: Prefetch completes entire file before Start is called t.Run("PrefetchComplete", func(t *testing.T) { listener := test.StartTestFileServer() defer listener.Close() fetcher := buildFetcher() err := fetcher.Resolve(&base.Request{ URL: "http://" + listener.Addr().String() + "/" + test.BuildName, }, &base.Options{ Name: test.DownloadName, Path: test.Dir, Extra: &http.OptsExtra{ Connections: 4, }, }) if err != nil { t.Fatal(err) } // Poll until prefetch completes the entire file (with timeout) timeout := time.After(30 * time.Second) ticker := time.NewTicker(100 * time.Millisecond) defer ticker.Stop() pollLoop: for { select { case <-timeout: t.Fatal("Timeout waiting for prefetch to complete") case <-ticker.C: if fetcher.prefetchSize.Load() >= test.BuildSize { break pollLoop } } } prefetchedBefore := fetcher.prefetchSize.Load() t.Logf("Prefetched bytes before Start: %d (%.2f MB)", prefetchedBefore, float64(prefetchedBefore)/(1024*1024)) // Should have prefetched the entire file if prefetchedBefore != test.BuildSize { t.Errorf("Prefetch should complete entire file, got %d, want %d", prefetchedBefore, test.BuildSize) } // Now start the download err = fetcher.Start() if err != nil { t.Fatal(err) } // Wait for download to complete err = fetcher.Wait() if err != nil { t.Fatal(err) } // Check how much was utilized from prefetch prefetchedUsed := fetcher.resolveDataPos.Load() t.Logf("Prefetched bytes used: %d (%.2f MB)", prefetchedUsed, float64(prefetchedUsed)/(1024*1024)) // Verify file is correct want := test.FileMd5(test.BuildFile) got := test.FileMd5(test.DownloadFile) if want != got { t.Errorf("Download() got = %v, want %v", got, want) } os.Remove(test.DownloadFile) }) // Test 2: Prefetch only downloads partial data before Start is called t.Run("PrefetchPartial", func(t *testing.T) { // Use slow server with 100 nanosecond delay per byte // This means ~10MB/s speed, so 100ms should download ~1MB listener := test.StartTestLowSpeedServer(100 * time.Nanosecond) defer listener.Close() fetcher := buildFetcher() err := fetcher.Resolve(&base.Request{ URL: "http://" + listener.Addr().String() + "/" + test.BuildName, }, &base.Options{ Name: test.DownloadName, Path: test.Dir, Extra: &http.OptsExtra{ Connections: 4, }, }) if err != nil { t.Fatal(err) } // Wait only 100ms - should only prefetch a small portion time.Sleep(100 * time.Millisecond) prefetchedBefore := fetcher.prefetchSize.Load() t.Logf("Prefetched bytes before Start: %d (%.2f KB)", prefetchedBefore, float64(prefetchedBefore)/1024) // Verify we have partial data (not zero, but not complete) if prefetchedBefore == 0 { t.Log("Warning: No data prefetched, may be too slow") } if prefetchedBefore >= test.BuildSize { t.Log("Warning: Prefetch completed entire file, test may not be valid") } // Now start the download err = fetcher.Start() if err != nil { t.Fatal(err) } // Wait for download to complete err = fetcher.Wait() if err != nil { t.Fatal(err) } // Check stats - should have connections that downloaded remaining data stats := fetcher.Stats().(*http.Stats) t.Logf("Final connections: %d", len(stats.Connections)) prefetchedUsed := fetcher.resolveDataPos.Load() t.Logf("Prefetched bytes used: %d (%.2f KB)", prefetchedUsed, float64(prefetchedUsed)/1024) // Verify connections picked up where prefetch left off if len(stats.Connections) > 0 { firstConn := stats.Connections[0] t.Logf("First connection downloaded: %d bytes", firstConn.Downloaded) } // Verify file is correct want := test.FileMd5(test.BuildFile) got := test.FileMd5(test.DownloadFile) if want != got { t.Errorf("Download() got = %v, want %v", got, want) } os.Remove(test.DownloadFile) }) } // TestFetcher_DownloadExpiringRedirectURL tests that the fetcher correctly handles // expiring redirect URLs by falling back to the original URL and getting a new redirect. func TestFetcher_DownloadExpiringRedirectURL(t *testing.T) { // Test with redirect expiring after 2 requests // This means: // - Request 1: Resolve (original URL redirects to temp URL v1) // - Request 2: First download request to temp URL v1 succeeds // - Request 3: Second download request to temp URL v1 returns 403 // - Fetcher should then retry with original URL, get temp URL v2 // - Continue downloading from temp URL v2 t.Run("RedirectExpiresAfter2Requests", func(t *testing.T) { os.Remove(test.DownloadFile) // Create server with redirect expiring after 2 requests, with slow transfer to ensure // multiple connection attempts are needed listener := test.StartTestExpiringRedirectServer(2, 100*time.Nanosecond) defer listener.Close() fetcher := buildFetcher() err := fetcher.Resolve(&base.Request{ URL: "http://" + listener.Addr().String() + "/" + test.BuildName, }, &base.Options{ Name: test.DownloadName, Path: test.Dir, Extra: &http.OptsExtra{ Connections: 4, // Use multiple connections to trigger redirect expiration }, }) if err != nil { t.Fatal(err) } err = fetcher.Start() if err != nil { t.Fatal(err) } err = fetcher.Wait() if err != nil { t.Fatal(err) } // Verify file content is correct despite redirect expiration want := test.FileMd5(test.BuildFile) got := test.FileMd5(test.DownloadFile) if want != got { t.Errorf("Download() got = %v, want %v", got, want) } os.Remove(test.DownloadFile) }) // Test with redirect expiring after 5 requests (more room for initial connections) t.Run("RedirectExpiresAfter5Requests", func(t *testing.T) { os.Remove(test.DownloadFile) listener := test.StartTestExpiringRedirectServer(5, 100*time.Nanosecond) defer listener.Close() fetcher := buildFetcher() err := fetcher.Resolve(&base.Request{ URL: "http://" + listener.Addr().String() + "/" + test.BuildName, }, &base.Options{ Name: test.DownloadName, Path: test.Dir, Extra: &http.OptsExtra{ Connections: 8, }, }) if err != nil { t.Fatal(err) } err = fetcher.Start() if err != nil { t.Fatal(err) } err = fetcher.Wait() if err != nil { t.Fatal(err) } // Verify file content want := test.FileMd5(test.BuildFile) got := test.FileMd5(test.DownloadFile) if want != got { t.Errorf("Download() got = %v, want %v", got, want) } os.Remove(test.DownloadFile) }) } // TestFetcher_RetryAfterError tests that the fetcher can retry downloading // after a previous download attempt failed by calling Start() again. func TestFetcher_RetryAfterError(t *testing.T) { os.Remove(test.DownloadFile) // Server fails first 3 requests (after resolve), then recovers // With 1 connection and 3 retries: // - First Start(): requests 2, 3, 4 → all fail (3 retries exhausted) → returns error // - Second Start(): request 5 → succeeds (server recovered after 3 failures) listener := test.StartTestFailThenRecoverServer(3) defer listener.Close() fetcher := buildFetcher() err := fetcher.Resolve(&base.Request{ URL: "http://" + listener.Addr().String() + "/" + test.BuildName, }, &base.Options{ Name: test.DownloadName, Path: test.Dir, Extra: &http.OptsExtra{ Connections: 1, // Use single connection to simplify test }, }) if err != nil { t.Fatal(err) } // First download attempt - should fail because server returns 416 after resolve err = fetcher.Start() if err != nil { t.Fatal(err) } err = fetcher.Wait() // First attempt should fail with 416 error if err == nil { t.Fatal("Expected first download attempt to fail, but it succeeded") } t.Logf("First attempt failed as expected: %v", err) // Check that fetcher is in error state state := fetcher.getState() if state != stateError { t.Errorf("Expected fetcher to be in stateError, got %v", state) } // Verify that we can call Start() again after error // This tests the stateError handling in Start() err = fetcher.Start() if err != nil { t.Fatalf("Start() after error failed: %v", err) } // Wait for second attempt - should succeed now that server has recovered err = fetcher.Wait() if err != nil { t.Fatalf("Retry failed: %v", err) } // Verify file content want := test.FileMd5(test.BuildFile) got := test.FileMd5(test.DownloadFile) if want != got { t.Errorf("Download() got = %v, want %v", got, want) } os.Remove(test.DownloadFile) } func TestFetcherManager_ParseName(t *testing.T) { type args struct { u string } tests := []struct { name string args args want string }{ { name: "broken url", args: args{ u: "https://!@#%github.com", }, want: "", }, { name: "file path", args: args{ u: "https://github.com/index.html", }, want: "index.html", }, { name: "file path with query and hash", args: args{ u: "https://github.com/a/b/index.html/#list?name=1", }, want: "index.html", }, { name: "no file path", args: args{ u: "https://github.com", }, want: "github.com", }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { fm := &FetcherManager{} if got := fm.ParseName(tt.args.u); got != tt.want { t.Errorf("ParseName() = %v, want %v", got, tt.want) } }) } } func downloadReady(listener net.Listener, connections int, t *testing.T) fetcher.Fetcher { return doDownloadReady(buildFetcher(), listener, connections, t) } func doDownloadReady(f fetcher.Fetcher, listener net.Listener, connections int, t *testing.T) fetcher.Fetcher { var extra any = nil if connections > 0 { extra = &http.OptsExtra{ Connections: connections, } } opts := &base.Options{ Name: test.DownloadName, Path: test.Dir, Extra: extra, } err := f.Resolve(&base.Request{ URL: "http://" + listener.Addr().String() + "/" + test.BuildName, }, opts) if err != nil { t.Fatal(err) } return f } func downloadNormal(listener net.Listener, connections int, t *testing.T) { fetcher := downloadReady(listener, connections, t) err := fetcher.Start() if err != nil { t.Fatal(err) } err = fetcher.Wait() if err != nil { t.Fatal(err) } want := test.FileMd5(test.BuildFile) got := test.FileMd5(test.DownloadFile) if want != got { t.Errorf("Download() got = %v, want %v", got, want) } } func downloadPost(listener net.Listener, connections int, t *testing.T) { // POST parameters must be set before Resolve since the new design // starts downloading during Resolve phase f := buildFetcher() var extra any = nil if connections > 0 { extra = &http.OptsExtra{ Connections: connections, } } opts := &base.Options{ Name: test.DownloadName, Path: test.Dir, Extra: extra, } req := &base.Request{ URL: "http://" + listener.Addr().String() + "/" + test.BuildName, Extra: &http.ReqExtra{ Method: "POST", Header: map[string]string{ "Authorization": "Bearer 123456", }, Body: fmt.Sprintf(`{"name":"%s"}`, test.BuildName), }, } err := f.Resolve(req, opts) if err != nil { t.Fatal(err) } err = f.Start() if err != nil { t.Fatal(err) } err = f.Wait() if err != nil { t.Fatal(err) } want := test.FileMd5(test.BuildFile) got := test.FileMd5(test.DownloadFile) if want != got { t.Errorf("Download() got = %v, want %v", got, want) } } func downloadContinue(listener net.Listener, connections int, t *testing.T) { fetcher := downloadReady(listener, connections, t) err := fetcher.Start() if err != nil { t.Fatal(err) } time.Sleep(time.Millisecond * 50) if err := fetcher.Pause(); err != nil { t.Fatal(err) } time.Sleep(time.Millisecond * 50) if err := fetcher.Start(); err != nil { t.Fatal(err) } err = fetcher.Wait() if err != nil { t.Fatal(err) } want := test.FileMd5(test.BuildFile) got := test.FileMd5(test.DownloadFile) if want != got { t.Errorf("Download() got = %v, want %v", got, want) } } func downloadError(listener net.Listener, connections int, t *testing.T) { fetcher := buildFetcher() err := fetcher.Resolve(&base.Request{ URL: "http://" + listener.Addr().String() + "/" + test.BuildName, }, &base.Options{ Name: test.DownloadName, Path: test.Dir, }) // With the new async design, Resolve may succeed (HTTP response received) // but errors occur during async download or Start/Wait if err != nil { // Error detected in Resolve - this is fine return } // Resolve succeeded, error should occur during Start/Wait err = fetcher.Start() if err != nil { // Error detected in Start - this is fine return } err = fetcher.Wait() if err == nil { t.Errorf("Expected error during download, but got none") } } func downloadResume(listener net.Listener, connections int, t *testing.T) { fetcher := downloadReady(listener, connections, t) err := fetcher.Start() if err != nil { t.Fatal(err) } fb := new(FetcherManager) time.Sleep(time.Millisecond * 50) data, err := fb.Store(fetcher) if err != nil { t.Fatal(err) } time.Sleep(time.Millisecond * 50) fetcher.Pause() _, f := fb.Restore() f(fetcher.Meta(), data) if err != nil { t.Fatal(err) } fetcher.Setup(controller.NewController()) fetcher.Start() err = fetcher.Wait() if err != nil { t.Fatal(err) } want := test.FileMd5(test.BuildFile) got := test.FileMd5(test.DownloadFile) if want != got { t.Errorf("Download() got = %v, want %v", got, want) } } func downloadWithProxy(httpListener net.Listener, proxyListener net.Listener, t *testing.T) { fetcher := downloadReady(httpListener, 4, t) ctl := controller.NewController() ctl.GetProxy = func(requestProxy *base.RequestProxy) func(*gohttp.Request) (*url.URL, error) { return (&base.DownloaderProxyConfig{ Enable: true, Scheme: "socks5", Host: proxyListener.Addr().String(), }).ToHandler() } fetcher.Setup(ctl) err := fetcher.Start() if err != nil { t.Fatal(err) } err = fetcher.Wait() if err != nil { t.Fatal(err) } want := test.FileMd5(test.BuildFile) got := test.FileMd5(test.DownloadFile) if want != got { t.Errorf("Download() got = %v, want %v", got, want) } } func buildFetcher() *Fetcher { fm := new(FetcherManager) fetcher := fm.Build() newController := controller.NewController() newController.GetConfig = func(v any) { json.Unmarshal([]byte(test.ToJson(fm.DefaultConfig())), v) } fetcher.Setup(newController) return fetcher.(*Fetcher) } func buildConfigFetcher(cfg config) fetcher.Fetcher { fetcher := new(FetcherManager).Build() newController := controller.NewController() newController.GetConfig = func(v any) { json.Unmarshal([]byte(test.ToJson(cfg)), v) } fetcher.Setup(newController) return fetcher } // TestFetcher_Patch_URLChange tests the Patch functionality where a failed download URL // is replaced with a working one. This simulates: // 1. Initial download attempt with a bad URL (returns 404) // 2. Patching the task with a new working URL // 3. Successful download after URL modification func TestFetcher_Patch_URLChange(t *testing.T) { listener := test.StartTestPatchURLServer() defer listener.Close() f := buildFetcher() badURL := "http://" + listener.Addr().String() + "/bad-url" goodURL := "http://" + listener.Addr().String() + "/good-url" opts := &base.Options{ Name: test.DownloadName, Path: test.Dir, Extra: &http.OptsExtra{Connections: 1}, } // Step 1: Try to resolve with bad URL - should fail with error err := f.Resolve(&base.Request{URL: badURL}, opts) if err == nil { t.Fatal("Expected error for bad URL, got nil") } // Step 2: Create a new fetcher and resolve with bad URL but don't wait // We need to test patching a task that has been created f2 := buildFetcher() // First resolve with good URL to create a valid fetcher state err = f2.Resolve(&base.Request{URL: goodURL}, opts) if err != nil { t.Fatal(err) } // Verify initial URL if f2.meta.Req.URL != goodURL { t.Errorf("Initial URL = %v, want %v", f2.meta.Req.URL, goodURL) } // Step 3: Patch to change URL (simulating URL change scenario) newURL := "http://" + listener.Addr().String() + "/good-url" err = f2.Patch(&base.Request{URL: newURL}, nil) if err != nil { t.Fatal(err) } // Verify URL was patched if f2.meta.Req.URL != newURL { t.Errorf("Patched URL = %v, want %v", f2.meta.Req.URL, newURL) } // Step 4: Start download and verify success err = f2.Start() if err != nil { t.Fatal(err) } err = f2.Wait() if err != nil { t.Fatal(err) } // Verify file was downloaded correctly want := test.FileMd5(test.BuildFile) got := test.FileMd5(test.DownloadFile) if want != got { t.Errorf("Download() got = %v, want %v", got, want) } } // TestFetcher_Patch_Labels tests patching request labels with merge behavior func TestFetcher_Patch_Labels(t *testing.T) { listener := test.StartTestFileServer() defer listener.Close() f := buildFetcher() opts := &base.Options{ Name: test.DownloadName, Path: test.Dir, Extra: &http.OptsExtra{Connections: 1}, } err := f.Resolve(&base.Request{ URL: "http://" + listener.Addr().String() + "/" + test.BuildName, Labels: map[string]string{ "key1": "value1", "key3": "value3", }, }, opts) if err != nil { t.Fatal(err) } // Verify initial labels if f.meta.Req.Labels["key1"] != "value1" { t.Errorf("Initial label key1 = %v, want value1", f.meta.Req.Labels["key1"]) } if f.meta.Req.Labels["key3"] != "value3" { t.Errorf("Initial label key3 = %v, want value3", f.meta.Req.Labels["key3"]) } // Patch with new labels - key1 should be overwritten, key2 should be added, key3 should remain patchReq := &base.Request{ Labels: map[string]string{ "key1": "modified", "key2": "newValue", }, } err = f.Patch(patchReq, nil) if err != nil { t.Fatal(err) } // Verify labels were merged correctly if f.meta.Req.Labels["key1"] != "modified" { t.Errorf("Patched label key1 = %v, want modified", f.meta.Req.Labels["key1"]) } if f.meta.Req.Labels["key2"] != "newValue" { t.Errorf("Patched label key2 = %v, want newValue", f.meta.Req.Labels["key2"]) } // key3 should remain unchanged if f.meta.Req.Labels["key3"] != "value3" { t.Errorf("Label key3 = %v, want value3 (should remain unchanged)", f.meta.Req.Labels["key3"]) } } // TestFetcher_Patch_Extra tests patching request Extra with merge behavior func TestFetcher_Patch_Extra(t *testing.T) { listener := test.StartTestFileServer() defer listener.Close() f := buildFetcher() opts := &base.Options{ Name: test.DownloadName, Path: test.Dir, Extra: &http.OptsExtra{Connections: 1}, } // Resolve with initial Extra err := f.Resolve(&base.Request{ URL: "http://" + listener.Addr().String() + "/" + test.BuildName, Extra: &http.ReqExtra{ Method: "GET", Body: "initial body", Header: map[string]string{ "Authorization": "Bearer token123", "X-Custom": "original", }, }, }, opts) if err != nil { t.Fatal(err) } // Verify initial Extra initialExtra := f.meta.Req.Extra.(*http.ReqExtra) if initialExtra.Method != "GET" { t.Errorf("Initial Method = %v, want GET", initialExtra.Method) } if initialExtra.Body != "initial body" { t.Errorf("Initial Body = %v, want 'initial body'", initialExtra.Body) } if initialExtra.Header["Authorization"] != "Bearer token123" { t.Errorf("Initial Authorization header = %v, want 'Bearer token123'", initialExtra.Header["Authorization"]) } if initialExtra.Header["X-Custom"] != "original" { t.Errorf("Initial X-Custom header = %v, want 'original'", initialExtra.Header["X-Custom"]) } // Patch with partial Extra - only update some fields patchReq := &base.Request{ Extra: &http.ReqExtra{ Method: "POST", // Update method // Body is empty, should NOT update Header: map[string]string{ "X-Custom": "modified", // Overwrite existing "X-New": "added", // Add new // Authorization is not in patch, should remain }, }, } err = f.Patch(patchReq, nil) if err != nil { t.Fatal(err) } // Verify Extra was merged correctly patchedExtra := f.meta.Req.Extra.(*http.ReqExtra) // Method should be updated if patchedExtra.Method != "POST" { t.Errorf("Patched Method = %v, want POST", patchedExtra.Method) } // Body should remain unchanged (patch had empty body) if patchedExtra.Body != "initial body" { t.Errorf("Patched Body = %v, want 'initial body' (should remain unchanged)", patchedExtra.Body) } // Authorization header should remain unchanged if patchedExtra.Header["Authorization"] != "Bearer token123" { t.Errorf("Authorization header = %v, want 'Bearer token123' (should remain unchanged)", patchedExtra.Header["Authorization"]) } // X-Custom header should be overwritten if patchedExtra.Header["X-Custom"] != "modified" { t.Errorf("X-Custom header = %v, want 'modified'", patchedExtra.Header["X-Custom"]) } // X-New header should be added if patchedExtra.Header["X-New"] != "added" { t.Errorf("X-New header = %v, want 'added'", patchedExtra.Header["X-New"]) } } // TestFetcher_Patch_NilData tests that Patch with nil data doesn't cause errors func TestFetcher_Patch_NilData(t *testing.T) { listener := test.StartTestFileServer() defer listener.Close() f := buildFetcher() opts := &base.Options{ Name: test.DownloadName, Path: test.Dir, Extra: &http.OptsExtra{Connections: 1}, } err := f.Resolve(&base.Request{ URL: "http://" + listener.Addr().String() + "/" + test.BuildName, }, opts) if err != nil { t.Fatal(err) } originalURL := f.meta.Req.URL // Patch with nil data - should not cause error err = f.Patch(nil, nil) if err != nil { t.Fatal(err) } // Verify URL unchanged if f.meta.Req.URL != originalURL { t.Errorf("URL changed after nil patch: got %v, want %v", f.meta.Req.URL, originalURL) } // Patch with empty request - should not cause error err = f.Patch(&base.Request{}, nil) if err != nil { t.Fatal(err) } // Verify URL still unchanged if f.meta.Req.URL != originalURL { t.Errorf("URL changed after empty patch: got %v, want %v", f.meta.Req.URL, originalURL) } } // TestFetcher_Patch_CookieExpired tests the Patch functionality where a download fails // mid-way due to expired cookie, then succeeds after patching with a new valid cookie. // This simulates: // 1. Initial resolve with valid cookie succeeds // 2. Download starts but fails because cookie expires mid-download (server returns 401) // 3. User patches the task with a new valid cookie // 4. Download resumes and completes successfully func TestFetcher_Patch_CookieExpired(t *testing.T) { listener := test.StartTestCookieExpiringServer() defer listener.Close() downloadURL := "http://" + listener.Addr().String() + "/" + test.BuildName opts := &base.Options{ Name: test.DownloadName, Path: test.Dir, Extra: &http.OptsExtra{Connections: 1}, } // Step 1: Resolve with old_token - should succeed (first request accepts old_token) f := buildFetcher() err := f.Resolve(&base.Request{ URL: downloadURL, Extra: &http.ReqExtra{ Header: map[string]string{ "Cookie": "session=old_token", }, }, }, opts) if err != nil { t.Fatalf("Resolve should succeed with old_token: %v", err) } // Verify initial cookie initialExtra := f.meta.Req.Extra.(*http.ReqExtra) if initialExtra.Header["Cookie"] != "session=old_token" { t.Errorf("Initial Cookie = %v, want session=old_token", initialExtra.Header["Cookie"]) } // Step 2: Start download - should fail because old_token is now expired // (server only accepts old_token for first request, subsequent requests need new_token) err = f.Start() if err != nil { t.Fatalf("Start failed: %v", err) } err = f.Wait() // Download should fail with 401 error if err == nil { t.Fatal("Expected download to fail with expired cookie, but it succeeded") } t.Logf("Download failed as expected: %v", err) // Step 3: Patch with new valid cookie err = f.Patch(&base.Request{ Extra: &http.ReqExtra{ Header: map[string]string{ "Cookie": "session=new_token", }, }, }, nil) if err != nil { t.Fatalf("Patch to update cookie failed: %v", err) } // Verify cookie was updated patchedExtra := f.meta.Req.Extra.(*http.ReqExtra) if patchedExtra.Header["Cookie"] != "session=new_token" { t.Errorf("Cookie should be updated: got %v, want session=new_token", patchedExtra.Header["Cookie"]) } // Step 4: Restart download - should succeed with new cookie err = f.Start() if err != nil { t.Fatalf("Restart failed: %v", err) } err = f.Wait() if err != nil { t.Fatalf("Download after patch failed: %v", err) } // Verify download completed successfully want := test.FileMd5(test.BuildFile) got := test.FileMd5(test.DownloadFile) if want != got { t.Errorf("File MD5 mismatch: got %v, want %v", got, want) } }