refactor: fethcer interface & download flow (#1229)

internal/protocol/http/fetcher_manager.go

@@ -0,0 +1,96 @@
+package http
+
+import (
+	"net/url"
+	"path"
+
+	"github.com/GopeedLab/gopeed/internal/fetcher"
+	"github.com/GopeedLab/gopeed/pkg/base"
+	fhttp "github.com/GopeedLab/gopeed/pkg/protocol/http"
+)
+
+// ============================================================================
+// Fetcher Data (for Store/Restore)
+// ============================================================================
+
+type fetcherData struct {
+	Connections []*connection
+}
+
+// ============================================================================
+// Fetcher Manager
+// ============================================================================
+
+type FetcherManager struct {
+}
+
+func (fm *FetcherManager) Name() string {
+	return "http"
+}
+
+func (fm *FetcherManager) Filters() []*fetcher.SchemeFilter {
+	return []*fetcher.SchemeFilter{
+		{
+			Type:    fetcher.FilterTypeUrl,
+			Pattern: "HTTP",
+		},
+		{
+			Type:    fetcher.FilterTypeUrl,
+			Pattern: "HTTPS",
+		},
+	}
+}
+
+func (fm *FetcherManager) Build() fetcher.Fetcher {
+	return &Fetcher{}
+}
+
+func (fm *FetcherManager) ParseName(u string) string {
+	var name string
+	url, err := url.Parse(u)
+	if err != nil {
+		return ""
+	}
+	name = path.Base(url.Path)
+	if name == "" || name == "/" || name == "." {
+		name = url.Hostname()
+	}
+	return name
+}
+
+func (fm *FetcherManager) AutoRename() bool {
+	return true
+}
+
+func (fm *FetcherManager) DefaultConfig() any {
+	return &config{
+		UserAgent:   "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36",
+		Connections: 16,
+	}
+}
+
+func (fm *FetcherManager) Store(f fetcher.Fetcher) (data any, err error) {
+	_f := f.(*Fetcher)
+	return &fetcherData{
+		Connections: _f.connections,
+	}, nil
+}
+
+func (fm *FetcherManager) Restore() (v any, f func(meta *fetcher.FetcherMeta, v any) fetcher.Fetcher) {
+	return &fetcherData{}, func(meta *fetcher.FetcherMeta, v any) fetcher.Fetcher {
+		fd := v.(*fetcherData)
+		fb := &FetcherManager{}
+		fetcher := fb.Build().(*Fetcher)
+		fetcher.meta = meta
+		base.ParseReqExtra[fhttp.ReqExtra](fetcher.meta.Req)
+		base.ParseOptExtra[fhttp.OptsExtra](fetcher.meta.Opts)
+		if len(fd.Connections) > 0 {
+			fetcher.connections = fd.Connections
+		}
+		return fetcher
+	}
+}
+
+func (fm *FetcherManager) Close() error {
+	return nil
+}

internal/protocol/http/fetcher_test.go

@@ -3,188 +3,221 @@ package http
 import (
 	"encoding/json"
 	"fmt"
+	"net"
+	gohttp "net/http"
+	"net/url"
+	"os"
+	"strings"
+	"testing"
+	"time"
+
 	"github.com/GopeedLab/gopeed/internal/controller"
 	"github.com/GopeedLab/gopeed/internal/fetcher"
 	"github.com/GopeedLab/gopeed/internal/test"
 	"github.com/GopeedLab/gopeed/pkg/base"
 	"github.com/GopeedLab/gopeed/pkg/protocol/http"
-	"net"
-	gohttp "net/http"
-	"net/url"
-	"strings"
-	"testing"
-	"time"
 )
 
 func TestFetcher_Resolve(t *testing.T) {
-	testResolve(test.StartTestFileServer, test.BuildName, &base.Resource{
-		Size:  test.BuildSize,
-		Range: true,
-		Files: []*base.FileInfo{
-			{
-				Name: test.BuildName,
-				Size: test.BuildSize,
+	testResolve(test.StartTestFileServer, test.BuildName, t, func(err error) (*base.Resource, error) {
+		return &base.Resource{
+			Size:  test.BuildSize,
+			Range: true,
+			Files: []*base.FileInfo{
+				{
+					Name: test.BuildName,
+					Size: test.BuildSize,
+				},
 			},
-		},
-	}, t)
-	testResolve(test.StartTestCustomServer, "disposition", &base.Resource{
-		Size:  test.BuildSize,
-		Range: false,
-		Files: []*base.FileInfo{
-			{
-				Name: test.BuildName,
-				Size: test.BuildSize,
+		}, nil
+	})
+	testResolve(test.StartTestCustomServer, "disposition", t, func(err error) (*base.Resource, error) {
+		return &base.Resource{
+			Size:  test.BuildSize,
+			Range: false,
+			Files: []*base.FileInfo{
+				{
+					Name: test.BuildName,
+					Size: test.BuildSize,
+				},
 			},
-		},
-	}, t)
-	testResolve(test.StartTestCustomServer, "encoded-word", &base.Resource{
-		Size:  test.BuildSize,
-		Range: false,
-		Files: []*base.FileInfo{
-			{
-				Name: test.TestChineseFileName,
-				Size: test.BuildSize,
+		}, nil
+	})
+	testResolve(test.StartTestCustomServer, "encoded-word", t, func(err error) (*base.Resource, error) {
+		return &base.Resource{
+			Size:  test.BuildSize,
+			Range: false,
+			Files: []*base.FileInfo{
+				{
+					Name: test.TestChineseFileName,
+					Size: test.BuildSize,
+				},
 			},
-		},
-	}, t)
-	testResolve(test.StartTestCustomServer, "no-encode", &base.Resource{
-		Size:  test.BuildSize,
-		Range: false,
-		Files: []*base.FileInfo{
-			{
-				Name: test.TestChineseFileName,
-				Size: test.BuildSize,
+		}, nil
+	})
+	testResolve(test.StartTestCustomServer, "no-encode", t, func(err error) (*base.Resource, error) {
+		return &base.Resource{
+			Size:  test.BuildSize,
+			Range: false,
+			Files: []*base.FileInfo{
+				{
+					Name: test.TestChineseFileName,
+					Size: test.BuildSize,
+				},
 			},
-		},
-	}, t)
-	testResolve(test.StartTestCustomServer, "%E6%B5%8B%E8%AF%95.zip", &base.Resource{
-		Size:  0,
-		Range: false,
-		Files: []*base.FileInfo{
-			{
-				Name: test.TestChineseFileName,
-				Size: 0,
+		}, nil
+	})
+	testResolve(test.StartTestCustomServer, "%E6%B5%8B%E8%AF%95.zip", t, func(err error) (*base.Resource, error) {
+		return &base.Resource{
+			Size:  0,
+			Range: false,
+			Files: []*base.FileInfo{
+				{
+					Name: test.TestChineseFileName,
+					Size: 0,
+				},
 			},
-		},
-	}, t)
-	testResolve(test.StartTestCustomServer, test.BuildName, &base.Resource{
-		Size:  0,
-		Range: false,
-		Files: []*base.FileInfo{
-			{
-				Name: test.BuildName,
-				Size: 0,
+		}, nil
+	})
+	testResolve(test.StartTestCustomServer, test.BuildName, t, func(err error) (*base.Resource, error) {
+		return &base.Resource{
+			Size:  0,
+			Range: false,
+			Files: []*base.FileInfo{
+				{
+					Name: test.BuildName,
+					Size: 0,
+				},
 			},
-		},
-	}, t)
+		}, nil
+	})
 	// Test mixed encoding Content-Disposition where mime.ParseMediaType fails
 	// due to invalid characters, but filename*= contains the correct UTF-8 encoded name
-	testResolve(test.StartTestCustomServer, "mixed-encoding", &base.Resource{
-		Size:  test.BuildSize,
-		Range: false,
-		Files: []*base.FileInfo{
-			{
-				Name: test.TestChineseFileName,
-				Size: test.BuildSize,
+	testResolve(test.StartTestCustomServer, "mixed-encoding", t, func(err error) (*base.Resource, error) {
+		return &base.Resource{
+			Size:  test.BuildSize,
+			Range: false,
+			Files: []*base.FileInfo{
+				{
+					Name: test.TestChineseFileName,
+					Size: test.BuildSize,
+				},
 			},
-		},
-	}, t)
+		}, nil
+	})
 	// Test filename*= only (RFC 5987 format)
-	testResolve(test.StartTestCustomServer, "filename-star", &base.Resource{
-		Size:  test.BuildSize,
-		Range: false,
-		Files: []*base.FileInfo{
-			{
-				Name: test.TestChineseFileName,
-				Size: test.BuildSize,
+	testResolve(test.StartTestCustomServer, "filename-star", t, func(err error) (*base.Resource, error) {
+		return &base.Resource{
+			Size:  test.BuildSize,
+			Range: false,
+			Files: []*base.FileInfo{
+				{
+					Name: test.TestChineseFileName,
+					Size: test.BuildSize,
+				},
 			},
-		},
-	}, t)
+		}, nil
+	})
 	// Test GBK-encoded filename (common on Chinese Windows servers)
 	// Before fix: "测试.zip" sent as GBK bytes -> parsed as "²âÊÔ.zip" (garbled)
 	// After fix: correctly decoded back to "测试.zip"
-	testResolve(test.StartTestCustomServer, "gbk-encoded", &base.Resource{
-		Size:  test.BuildSize,
-		Range: false,
-		Files: []*base.FileInfo{
-			{
-				Name: test.TestChineseFileName,
-				Size: test.BuildSize,
+	testResolve(test.StartTestCustomServer, "gbk-encoded", t, func(err error) (*base.Resource, error) {
+		return &base.Resource{
+			Size:  test.BuildSize,
+			Range: false,
+			Files: []*base.FileInfo{
+				{
+					Name: test.TestChineseFileName,
+					Size: test.BuildSize,
+				},
 			},
-		},
-	}, t)
+		}, nil
+	})
 	// Test filename with plus signs (e.g., C++ Primer)
 	// Before fix: %2B decoded to space -> "C++ Primer" became "C  Primer"
 	// After fix: %2B correctly decoded to + -> "C++  Primer  Plus.mobi"
-	testResolve(test.StartTestCustomServer, "plus-sign-encoded", &base.Resource{
-		Size:  test.BuildSize,
-		Range: false,
-		Files: []*base.FileInfo{
-			{
-				Name: "C++  Primer  Plus.mobi",
-				Size: test.BuildSize,
+	testResolve(test.StartTestCustomServer, "plus-sign-encoded", t, func(err error) (*base.Resource, error) {
+		return &base.Resource{
+			Size:  test.BuildSize,
+			Range: false,
+			Files: []*base.FileInfo{
+				{
+					Name: "C++  Primer  Plus.mobi",
+					Size: test.BuildSize,
+				},
 			},
-		},
-	}, t)
+		}, nil
+	})
 	// Test filename with plus sign in URL path
 	// Before fix: %2B decoded to space
 	// After fix: %2B correctly decoded to +
-	testResolve(test.StartTestCustomServer, "C%2B%2B%20Primer.txt", &base.Resource{
-		Size:  0,
-		Range: false,
-		Files: []*base.FileInfo{
-			{
-				Name: "C++ Primer.txt",
-				Size: 0,
+	testResolve(test.StartTestCustomServer, "C%2B%2B%20Primer.txt", t, func(err error) (*base.Resource, error) {
+		return &base.Resource{
+			Size:  0,
+			Range: false,
+			Files: []*base.FileInfo{
+				{
+					Name: "C++ Primer.txt",
+					Size: 0,
+				},
 			},
-		},
-	}, t)
+		}, nil
+	})
 	// Test filename with HTML-encoded ampersand (fixes issue with & being truncated)
 	// Before fix: "查询处理&优化.pptx" -> "查询处理&amp" (truncated at semicolon)
 	// After fix: correctly decoded to "查询处理&优化.pptx"
-	testResolve(test.StartTestCustomServer, "ampersand-encoded", &base.Resource{
-		Size:  test.BuildSize,
-		Range: false,
-		Files: []*base.FileInfo{
-			{
-				Name: "查询处理&优化.pptx",
-				Size: test.BuildSize,
+	testResolve(test.StartTestCustomServer, "ampersand-encoded", t, func(err error) (*base.Resource, error) {
+		return &base.Resource{
+			Size:  test.BuildSize,
+			Range: false,
+			Files: []*base.FileInfo{
+				{
+					Name: "查询处理&优化.pptx",
+					Size: test.BuildSize,
+				},
 			},
-		},
-	}, t)
+		}, nil
+	})
 	// Test unquoted filename with HTML-encoded ampersand
-	testResolve(test.StartTestCustomServer, "ampersand-unquoted", &base.Resource{
-		Size:  test.BuildSize,
-		Range: false,
-		Files: []*base.FileInfo{
-			{
-				Name: "test&file.txt",
-				Size: test.BuildSize,
+	testResolve(test.StartTestCustomServer, "ampersand-unquoted", t, func(err error) (*base.Resource, error) {
+		return &base.Resource{
+			Size:  test.BuildSize,
+			Range: false,
+			Files: []*base.FileInfo{
+				{
+					Name: "test&file.txt",
+					Size: test.BuildSize,
+				},
 			},
-		},
-	}, t)
-
+		}, nil
+	})
 	// Test URL without file path - should use domain/host as filename
-	listener := test.StartTestRootServer()
-	defer listener.Close()
-	fetcher := buildFetcher()
-	err := fetcher.Resolve(&base.Request{
-		URL: "http://" + listener.Addr().String() + "/",
+	testResolve(test.StartTestCustomServer, "", t, func(err error) (*base.Resource, error) {
+		return &base.Resource{
+			Size:  0,
+			Range: false,
+			Files: []*base.FileInfo{
+				{
+					Name: "127.0.0.1",
+					Size: 0,
+				},
+			},
+		}, nil
+	})
+	// Test 403 Forbidden response handling
+	testResolve(test.StartTestCustomServer, "forbidden", t, func(err error) (*base.Resource, error) {
+		requestError := extractRequestError(err)
+		if requestError != nil && requestError.Code == 403 {
+			return nil, nil
+		}
+		return nil, err
 	})
-	if err != nil {
-		t.Fatal(err)
-	}
-	// When no filename is provided, it should use the hostname (without port) as the name
-	expectedName := "127.0.0.1"
-	if fetcher.Meta().Res.Files[0].Name != expectedName {
-		t.Errorf("Resolve() got name = %v, want %v", fetcher.Meta().Res.Files[0].Name, expectedName)
-	}
 }
 
 func TestFetcher_ResolveWithHostHeader(t *testing.T) {
 	listener := test.StartTestHostHeaderServer()
 	defer listener.Close()
-	
+
 	fetcher := buildFetcher()
 	err := fetcher.Resolve(&base.Request{
 		URL: "http://" + listener.Addr().String() + "/",
@@ -193,6 +226,9 @@ func TestFetcher_ResolveWithHostHeader(t *testing.T) {
 				"Host": "test",
 			},
 		},
+	}, &base.Options{
+		Name: test.DownloadName,
+		Path: test.Dir,
 	})
 	// The server should return 400 for invalid Host header
 	if err == nil || !strings.Contains(err.Error(), "400") {
@@ -201,10 +237,11 @@ func TestFetcher_ResolveWithHostHeader(t *testing.T) {
 }
 
 func TestFetcher_ResolveWithInvalidHeader(t *testing.T) {
-	listener := test.StartTestRootServer()
+	listener := test.StartTestCustomServer()
 	defer listener.Close()
-	
+
 	fetcher := buildFetcher()
+	defer fetcher.Pause() // Close the resolve response to allow server shutdown
 	err := fetcher.Resolve(&base.Request{
 		URL: "http://" + listener.Addr().String() + "/",
 		Extra: &http.ReqExtra{
@@ -212,6 +249,9 @@ func TestFetcher_ResolveWithInvalidHeader(t *testing.T) {
 				"Referer": "\rtest",
 			},
 		},
+	}, &base.Options{
+		Name: test.DownloadName,
+		Path: test.Dir,
 	})
 	// Invalid header with \r should be sanitized by Go's http client, allowing the request to succeed
 	if err != nil {
@@ -219,18 +259,23 @@ func TestFetcher_ResolveWithInvalidHeader(t *testing.T) {
 	}
 }
 
-func testResolve(startTestServer func() net.Listener, path string, want *base.Resource, t *testing.T) {
+func testResolve(startTestServer func() net.Listener, path string, t *testing.T, wantFn func(error) (*base.Resource, error)) {
 	listener := startTestServer()
 	defer listener.Close()
 	fetcher := buildFetcher()
+	defer fetcher.Pause() // Close the resolve response to allow server shutdown
 	err := fetcher.Resolve(&base.Request{
 		URL: "http://" + listener.Addr().String() + "/" + path,
+	}, &base.Options{
+		Name: test.DownloadName,
+		Path: test.Dir,
 	})
+	want, err := wantFn(err)
 	if err != nil {
 		t.Fatal(err)
 	}
-	if !test.AssertResourceEqual(want, fetcher.meta.Res) {
-		t.Errorf("Resolve() got = %v, want %v", fetcher.meta.Res, want)
+	if want != nil && !test.AssertResourceEqual(want, fetcher.meta.Res) {
+		t.Errorf("Resolve() got = %+v, want %+v", fetcher.meta.Res, want)
 	}
 }
 
@@ -293,11 +338,74 @@ func TestFetcher_DownloadLimit(t *testing.T) {
 }
 
 func TestFetcher_DownloadResponseBodyReadTimeout(t *testing.T) {
-	listener := test.StartTestLimitServer(16, readTimeout.Milliseconds()+5000)
+	// Server will timeout once (first request delays longer than readTimeout),
+	// then subsequent requests work normally
+	listener := test.StartTestTimeoutOnceServer(readTimeout.Milliseconds() + 5000)
 	defer listener.Close()
 
-	downloadError(listener, 1, t)
-	downloadError(listener, 4, t)
+	for _, connections := range []int{1, 4} {
+		os.Remove(test.DownloadFile)
+
+		fetcher := downloadReady(listener, connections, t)
+		if err := fetcher.Start(); err != nil {
+			t.Fatal(err)
+		}
+		if err := fetcher.Wait(); err != nil {
+			t.Fatal(err)
+		}
+
+		stats := fetcher.Stats().(*http.Stats)
+		if len(stats.Connections) == 0 {
+			t.Fatalf("expected connections stats for timeout test")
+		}
+
+		// Verify successful download after timeout recovery
+		want := test.FileMd5(test.BuildFile)
+		got := test.FileMd5(test.DownloadFile)
+		if want != got {
+			t.Errorf("Download() got = %v, want %v", got, want)
+		}
+
+		// Verify timeouts don't count as failures (retryTimes should be 0)
+		for _, conn := range stats.Connections {
+			if conn.Failed {
+				t.Fatalf("expected no counted failures after timeout recovery, got retries=%d", conn.RetryTimes)
+			}
+			if conn.RetryTimes != 0 {
+				t.Fatalf("expected retryTimes to stay zero for non-counted timeouts, got %d", conn.RetryTimes)
+			}
+		}
+	}
+}
+
+func TestFetcher_Download500Recovery(t *testing.T) {
+	// Server returns 500 for 15 seconds, then recovers
+	listener := test.StartTestTemporary500Server(15 * time.Second)
+	defer listener.Close()
+
+	os.Remove(test.DownloadFile)
+	fetcher := downloadReady(listener, 4, t)
+	if err := fetcher.Start(); err != nil {
+		t.Fatal(err)
+	}
+	if err := fetcher.Wait(); err != nil {
+		t.Fatal(err)
+	}
+
+	// Verify successful download after 500 errors
+	want := test.FileMd5(test.BuildFile)
+	got := test.FileMd5(test.DownloadFile)
+	if want != got {
+		t.Errorf("Download() got = %v, want %v", got, want)
+	}
+
+	// Verify 500 errors don't count as failures (retryTimes should be 0)
+	stats := fetcher.Stats().(*http.Stats)
+	for _, conn := range stats.Connections {
+		if conn.RetryTimes != 0 {
+			t.Errorf("Expected retryTimes to be 0 for 500 errors (exempt), got %d", conn.RetryTimes)
+		}
+	}
 }
 
 func TestFetcher_DownloadOnBugFileServer(t *testing.T) {
@@ -385,11 +493,14 @@ func TestFetcher_Stats(t *testing.T) {
 		t.Fatal(err)
 	}
 	stats := fetcher.Stats().(*http.Stats)
-	if len(stats.Connections) != 16 {
-		t.Errorf("Stats() got = %v, want %v", len(stats.Connections), 16)
+	// With slow-start strategy, connection count may be less than max if download is fast
+	// Just verify we have at least 1 connection and no more than max
+	if len(stats.Connections) < 1 || len(stats.Connections) > 16 {
+		t.Errorf("Stats() connection count got = %v, want between 1 and 16", len(stats.Connections))
 	}
 	totalDownloaded := int64(0)
-	for _, conn := range stats.Connections {
+	for i, conn := range stats.Connections {
+		t.Logf("Connection %d: Downloaded=%d, Completed=%v", i, conn.Downloaded, conn.Completed)
 		totalDownloaded += conn.Downloaded
 	}
 	if totalDownloaded != test.BuildSize {
@@ -397,6 +508,276 @@ func TestFetcher_Stats(t *testing.T) {
 	}
 }
 
+// TestFetcher_DownloadOneTimeURL tests downloading from a URL that can only be accessed once
+// This simulates signed URLs or one-time download links that expire after first use
+func TestFetcher_DownloadOneTimeURL(t *testing.T) {
+	listener := test.StartTestOneTimeServer()
+	defer listener.Close()
+
+	fetcher := buildFetcher()
+	err := fetcher.Resolve(&base.Request{
+		URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
+	}, &base.Options{
+		Name: test.DownloadName,
+		Path: test.Dir,
+		Extra: &http.OptsExtra{
+			Connections: 4, // Try to use multiple connections, but only first should work
+		},
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = fetcher.Start()
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = fetcher.Wait()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Verify file content
+	want := test.FileMd5(test.BuildFile)
+	got := test.FileMd5(test.DownloadFile)
+	if want != got {
+		t.Errorf("Download() got = %v, want %v", got, want)
+	}
+}
+
+// TestFetcher_SlowStartExpansion tests slow-start connection expansion edge cases
+// Tests that slow-start expansion reaches exactly maxConns
+// Expansion pattern: 1 -> 2 -> 4 -> 8 -> 16...
+// For max=5: 1 -> 2 -> 4 -> 5 (capped)
+// For max=9: 1 -> 2 -> 4 -> 8 -> 9 (capped)
+func TestFetcher_SlowStartExpansion(t *testing.T) {
+	testCases := []struct {
+		name     string
+		maxConns int
+	}{
+		{"MaxConns5", 5}, // 1->2->4->5
+		{"MaxConns9", 9}, // 1->2->4->8->9
+		{"MaxConns8", 8}, // 1->2->4->8
+	}
+
+	for _, tc := range testCases {
+		tc := tc // capture range variable
+		t.Run(tc.name, func(t *testing.T) {
+			// Clean up any leftover files from previous tests
+			os.Remove(test.DownloadFile)
+
+			// Use 100ns delay per byte for faster test (~10MB/s theoretical)
+			listener := test.StartTestSlowStartServer(100 * time.Nanosecond)
+
+			// Ensure cleanup happens before next subtest
+			cleanup := func() {
+				listener.Close()
+				os.Remove(test.DownloadFile)
+				// Wait for server to fully stop
+				time.Sleep(50 * time.Millisecond)
+			}
+
+			fetcher := buildConfigFetcher(config{
+				Connections: tc.maxConns,
+			})
+
+			err := fetcher.Resolve(&base.Request{
+				URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
+			}, &base.Options{
+				Name: test.DownloadName,
+				Path: test.Dir,
+				Extra: &http.OptsExtra{
+					Connections: tc.maxConns,
+				},
+			})
+			if err != nil {
+				cleanup()
+				t.Fatal(err)
+			}
+
+			err = fetcher.Start()
+			if err != nil {
+				cleanup()
+				t.Fatal(err)
+			}
+
+			err = fetcher.Wait()
+			if err != nil {
+				t.Logf("Wait() returned error: %v", err)
+				cleanup()
+				t.Fatal(err)
+			}
+
+			// Check final connection count equals maxConns exactly
+			stats := fetcher.Stats().(*http.Stats)
+			finalConns := len(stats.Connections)
+
+			// Debug: show connection details and metadata
+			httpFetcher := fetcher.(*Fetcher)
+			t.Logf("Resource: Size=%d, Range=%v", httpFetcher.Meta().Res.Size, httpFetcher.Meta().Res.Range)
+			for i, conn := range stats.Connections {
+				t.Logf("Connection %d: Downloaded=%d, Completed=%v", i, conn.Downloaded, conn.Completed)
+			}
+
+			if finalConns != tc.maxConns {
+				t.Errorf("Expected exactly %d connections, got %d", tc.maxConns, finalConns)
+			}
+
+			// Verify file content before cleanup
+			want := test.FileMd5(test.BuildFile)
+			got := test.FileMd5(test.DownloadFile)
+			if want != got {
+				t.Errorf("Download() got = %v, want %v", got, want)
+			}
+
+			cleanup()
+		})
+	}
+}
+
+// TestFetcher_AsyncPrefetch tests the async prefetch functionality
+// where data is downloaded in background during resolve phase and reused in start
+func TestFetcher_AsyncPrefetch(t *testing.T) {
+	// Test 1: Prefetch completes entire file before Start is called
+	t.Run("PrefetchComplete", func(t *testing.T) {
+		listener := test.StartTestFileServer()
+		defer listener.Close()
+
+		fetcher := buildFetcher()
+		err := fetcher.Resolve(&base.Request{
+			URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
+		}, &base.Options{
+			Name: test.DownloadName,
+			Path: test.Dir,
+			Extra: &http.OptsExtra{
+				Connections: 4,
+			},
+		})
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		// Poll until prefetch completes the entire file (with timeout)
+		timeout := time.After(30 * time.Second)
+		ticker := time.NewTicker(100 * time.Millisecond)
+		defer ticker.Stop()
+	pollLoop:
+		for {
+			select {
+			case <-timeout:
+				t.Fatal("Timeout waiting for prefetch to complete")
+			case <-ticker.C:
+				if fetcher.prefetchSize.Load() >= test.BuildSize {
+					break pollLoop
+				}
+			}
+		}
+
+		prefetchedBefore := fetcher.prefetchSize.Load()
+		t.Logf("Prefetched bytes before Start: %d (%.2f MB)", prefetchedBefore, float64(prefetchedBefore)/(1024*1024))
+
+		// Should have prefetched the entire file
+		if prefetchedBefore != test.BuildSize {
+			t.Errorf("Prefetch should complete entire file, got %d, want %d", prefetchedBefore, test.BuildSize)
+		}
+
+		// Now start the download
+		err = fetcher.Start()
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		// Wait for download to complete
+		err = fetcher.Wait()
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		// Check how much was utilized from prefetch
+		prefetchedUsed := fetcher.resolveDataPos.Load()
+		t.Logf("Prefetched bytes used: %d (%.2f MB)", prefetchedUsed, float64(prefetchedUsed)/(1024*1024))
+
+		// Verify file is correct
+		want := test.FileMd5(test.BuildFile)
+		got := test.FileMd5(test.DownloadFile)
+		if want != got {
+			t.Errorf("Download() got = %v, want %v", got, want)
+		}
+
+		os.Remove(test.DownloadFile)
+	})
+
+	// Test 2: Prefetch only downloads partial data before Start is called
+	t.Run("PrefetchPartial", func(t *testing.T) {
+		// Use slow server with 100 nanosecond delay per byte
+		// This means ~10MB/s speed, so 100ms should download ~1MB
+		listener := test.StartTestSlowStartServer(100 * time.Nanosecond)
+		defer listener.Close()
+
+		fetcher := buildFetcher()
+		err := fetcher.Resolve(&base.Request{
+			URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
+		}, &base.Options{
+			Name: test.DownloadName,
+			Path: test.Dir,
+			Extra: &http.OptsExtra{
+				Connections: 4,
+			},
+		})
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		// Wait only 100ms - should only prefetch a small portion
+		time.Sleep(100 * time.Millisecond)
+
+		prefetchedBefore := fetcher.prefetchSize.Load()
+		t.Logf("Prefetched bytes before Start: %d (%.2f KB)", prefetchedBefore, float64(prefetchedBefore)/1024)
+
+		// Verify we have partial data (not zero, but not complete)
+		if prefetchedBefore == 0 {
+			t.Log("Warning: No data prefetched, may be too slow")
+		}
+		if prefetchedBefore >= test.BuildSize {
+			t.Log("Warning: Prefetch completed entire file, test may not be valid")
+		}
+
+		// Now start the download
+		err = fetcher.Start()
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		// Wait for download to complete
+		err = fetcher.Wait()
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		// Check stats - should have connections that downloaded remaining data
+		stats := fetcher.Stats().(*http.Stats)
+		t.Logf("Final connections: %d", len(stats.Connections))
+
+		prefetchedUsed := fetcher.resolveDataPos.Load()
+		t.Logf("Prefetched bytes used: %d (%.2f KB)", prefetchedUsed, float64(prefetchedUsed)/1024)
+
+		// Verify connections picked up where prefetch left off
+		if len(stats.Connections) > 0 {
+			firstConn := stats.Connections[0]
+			t.Logf("First connection downloaded: %d bytes", firstConn.Downloaded)
+		}
+
+		// Verify file is correct
+		want := test.FileMd5(test.BuildFile)
+		got := test.FileMd5(test.DownloadFile)
+		if want != got {
+			t.Errorf("Download() got = %v, want %v", got, want)
+		}
+
+		os.Remove(test.DownloadFile)
+	})
+}
+
 func TestFetcherManager_ParseName(t *testing.T) {
 	type args struct {
 		u string
@@ -450,15 +831,9 @@ func downloadReady(listener net.Listener, connections int, t *testing.T) fetcher
 }
 
 func doDownloadReady(f fetcher.Fetcher, listener net.Listener, connections int, t *testing.T) fetcher.Fetcher {
-	err := f.Resolve(&base.Request{
-		URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
-	})
-	if err != nil {
-		t.Fatal(err)
-	}
 	var extra any = nil
 	if connections > 0 {
-		extra = http.OptsExtra{
+		extra = &http.OptsExtra{
 			Connections: connections,
 		}
 	}
@@ -467,7 +842,9 @@ func doDownloadReady(f fetcher.Fetcher, listener net.Listener, connections int,
 		Path:  test.Dir,
 		Extra: extra,
 	}
-	err = f.Create(opts)
+	err := f.Resolve(&base.Request{
+		URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
+	}, opts)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -492,19 +869,39 @@ func downloadNormal(listener net.Listener, connections int, t *testing.T) {
 }
 
 func downloadPost(listener net.Listener, connections int, t *testing.T) {
-	fetcher := downloadReady(listener, connections, t)
-	fetcher.Meta().Req.Extra = &http.ReqExtra{
-		Method: "POST",
-		Header: map[string]string{
-			"Authorization": "Bearer 123456",
-		},
-		Body: fmt.Sprintf(`{"name":"%s"}`, test.BuildName),
+	// POST parameters must be set before Resolve since the new design
+	// starts downloading during Resolve phase
+	f := buildFetcher()
+	var extra any = nil
+	if connections > 0 {
+		extra = &http.OptsExtra{
+			Connections: connections,
+		}
 	}
-	err := fetcher.Start()
+	opts := &base.Options{
+		Name:  test.DownloadName,
+		Path:  test.Dir,
+		Extra: extra,
+	}
+	req := &base.Request{
+		URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
+		Extra: &http.ReqExtra{
+			Method: "POST",
+			Header: map[string]string{
+				"Authorization": "Bearer 123456",
+			},
+			Body: fmt.Sprintf(`{"name":"%s"}`, test.BuildName),
+		},
+	}
+	err := f.Resolve(req, opts)
 	if err != nil {
 		t.Fatal(err)
 	}
-	err = fetcher.Wait()
+	err = f.Start()
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = f.Wait()
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -541,14 +938,30 @@ func downloadContinue(listener net.Listener, connections int, t *testing.T) {
 }
 
 func downloadError(listener net.Listener, connections int, t *testing.T) {
-	fetcher := downloadReady(listener, connections, t)
-	err := fetcher.Start()
+	fetcher := buildFetcher()
+	err := fetcher.Resolve(&base.Request{
+		URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
+	}, &base.Options{
+		Name: test.DownloadName,
+		Path: test.Dir,
+	})
+	// With the new async design, Resolve may succeed (HTTP response received)
+	// but errors occur during async download or Start/Wait
 	if err != nil {
-		t.Fatal(err)
+		// Error detected in Resolve - this is fine
+		return
 	}
+
+	// Resolve succeeded, error should occur during Start/Wait
+	err = fetcher.Start()
+	if err != nil {
+		// Error detected in Start - this is fine
+		return
+	}
+
 	err = fetcher.Wait()
 	if err == nil {
-		t.Errorf("Download() got = %v, want %v", err, nil)
+		t.Errorf("Expected error during download, but got none")
 	}
 }
 

internal/protocol/http/helper.go

@@ -0,0 +1,378 @@
+package http
+
+import (
+	"bytes"
+	"context"
+	"crypto/tls"
+	"errors"
+	"fmt"
+	"io"
+	"mime"
+	"net"
+	"net/http"
+	"net/http/cookiejar"
+	"net/url"
+	"strings"
+	"unicode/utf8"
+
+	"github.com/GopeedLab/gopeed/pkg/base"
+	fhttp "github.com/GopeedLab/gopeed/pkg/protocol/http"
+	"github.com/GopeedLab/gopeed/pkg/util"
+	"golang.org/x/text/encoding/simplifiedchinese"
+)
+
+type RequestError struct {
+	Code int
+}
+
+func NewRequestError(code int) *RequestError {
+	return &RequestError{Code: code}
+}
+
+func (re *RequestError) Error() string {
+	return fmt.Sprintf("http request fail, code:%d", re.Code)
+}
+
+func isFailureExemptHTTPCode(code int) bool {
+	if code >= 500 && code <= 599 {
+		return true
+	}
+
+	switch code {
+	case 429, 408, 440, 499:
+		return true
+	default:
+		return false
+	}
+}
+
+func shouldCountHTTPFailure(err error) bool {
+	var re *RequestError
+	if !errors.As(err, &re) {
+		return false
+	}
+
+	return !isFailureExemptHTTPCode(re.Code)
+}
+
+func extractRequestError(err error) *RequestError {
+	var re *RequestError
+	if errors.As(err, &re) {
+		return re
+	}
+
+	return nil
+}
+
+// ============================================================================
+// HTTP Request/Client Building
+// ============================================================================
+
+func (f *Fetcher) buildRequest(ctx context.Context, req *base.Request) (httpReq *http.Request, err error) {
+	var reqUrl string
+	f.redirectLock.Lock()
+	if f.redirectURL != "" {
+		reqUrl = f.redirectURL
+	} else {
+		reqUrl = req.URL
+	}
+	f.redirectLock.Unlock()
+
+	var (
+		method string
+		body   io.Reader
+	)
+	headers := http.Header{}
+	if req.Extra == nil {
+		method = http.MethodGet
+	} else {
+		extra := req.Extra.(*fhttp.ReqExtra)
+		if extra.Method != "" {
+			method = extra.Method
+		} else {
+			method = http.MethodGet
+		}
+		if len(extra.Header) > 0 {
+			for k, v := range extra.Header {
+				headers.Set(k, strings.TrimSpace(v))
+			}
+		}
+		if extra.Body != "" {
+			body = bytes.NewBufferString(extra.Body)
+		}
+	}
+	if _, ok := headers[base.HttpHeaderUserAgent]; !ok {
+		headers.Set(base.HttpHeaderUserAgent, strings.TrimSpace(f.config.UserAgent))
+	}
+
+	if ctx != nil {
+		httpReq, err = http.NewRequestWithContext(ctx, method, reqUrl, body)
+	} else {
+		httpReq, err = http.NewRequest(method, reqUrl, body)
+	}
+	if err != nil {
+		return
+	}
+	httpReq.Header = headers
+	if host := headers.Get(base.HttpHeaderHost); host != "" {
+		httpReq.Host = host
+	}
+	return httpReq, nil
+}
+
+func (f *Fetcher) buildClient() *http.Client {
+	transport := &http.Transport{
+		DialContext: (&net.Dialer{
+			Timeout: connectTimeout,
+		}).DialContext,
+		Proxy: f.ctl.GetProxy(f.meta.Req.Proxy),
+		TLSClientConfig: &tls.Config{
+			InsecureSkipVerify: f.meta.Req.SkipVerifyCert,
+		},
+	}
+	jar, _ := cookiejar.New(nil)
+	return &http.Client{
+		Transport: transport,
+		Jar:       jar,
+	}
+}
+
+// ============================================================================
+// Filename Parsing
+// ============================================================================
+
+// parseFilename extracts filename from Content-Disposition header
+func parseFilename(contentDisposition string) string {
+	// Try RFC 5987 extended notation first (filename*=)
+	if filename := parseFilenameExtended(contentDisposition); filename != "" {
+		return filename
+	}
+
+	// Try standard MIME parsing
+	_, params, err := mime.ParseMediaType(contentDisposition)
+	if err == nil {
+		if filename := params["filename"]; filename != "" {
+			return decodeFilenameParam(filename)
+		}
+	}
+
+	// Fallback to manual parsing
+	return parseFilenameFallback(contentDisposition)
+}
+
+// parseFilenameExtended handles RFC 5987 extended notation (filename*=)
+// Format: filename*=charset'language'value (e.g., UTF-8”%E6%B5%8B%E8%AF%95.zip)
+func parseFilenameExtended(cd string) string {
+	lower := strings.ToLower(cd)
+	idx := strings.Index(lower, "filename*=")
+	if idx == -1 {
+		return ""
+	}
+
+	value := cd[idx+len("filename*="):]
+
+	// Find the end of the value using proper quote handling
+	endIdx := findParamValueEnd(value)
+	if endIdx != -1 {
+		value = value[:endIdx]
+	}
+	value = strings.TrimSpace(value)
+
+	// Try charset''encoded format (e.g., UTF-8''%E4%B8%AD%E6%96%87.txt)
+	parts := strings.SplitN(value, "''", 2)
+	if len(parts) == 2 {
+		// Use PathUnescape to handle %2B correctly (should decode to +, not space)
+		decoded, err := url.PathUnescape(parts[1])
+		if err == nil {
+			return decoded
+		}
+	}
+
+	// Try charset'language'encoded format
+	parts = strings.SplitN(value, "'", 3)
+	if len(parts) >= 3 {
+		// Use PathUnescape to handle %2B correctly (should decode to +, not space)
+		decoded, err := url.PathUnescape(parts[2])
+		if err == nil {
+			return decoded
+		}
+	}
+
+	return ""
+}
+
+// decodeFilenameParam decodes filename parameter value
+// Handles HTML entities, MIME encoded-word, URL encoding, and GBK encoding fallback
+func decodeFilenameParam(filename string) string {
+	// First, unescape HTML entities (e.g., &amp; -> &, &lt; -> <, &gt; -> >)
+	// This must be done before other decoding to handle cases where servers
+	// HTML-encode special characters in filenames
+	filename = unescapeHTMLEntities(filename)
+
+	// Handle RFC 2047 encoded word (=?charset?encoding?text?=)
+	if strings.HasPrefix(filename, "=?") {
+		decoder := new(mime.WordDecoder)
+		normalizedFilename := strings.Replace(filename, "UTF8", "UTF-8", 1)
+		if decoded, err := decoder.Decode(normalizedFilename); err == nil {
+			return decoded
+		}
+	}
+
+	// Try URL decoding - use PathUnescape for filenames to handle %2B correctly
+	decoded := util.TryUrlPathUnescape(filename)
+
+	// If not valid UTF-8, try GBK decoding (common for Chinese websites)
+	if !utf8.ValidString(decoded) {
+		if gbkDecoded := tryDecodeGBK(decoded); gbkDecoded != "" {
+			return gbkDecoded
+		}
+	}
+
+	return decoded
+}
+
+// unescapeHTMLEntities unescapes common HTML entities in filenames
+// This handles cases where servers HTML-encode special characters like & to &amp;
+func unescapeHTMLEntities(s string) string {
+	// Common HTML entities that might appear in filenames
+	replacements := map[string]string{
+		"&amp;":  "&",
+		"&lt;":   "<",
+		"&gt;":   ">",
+		"&quot;": "\"",
+		"&#39;":  "'",
+		"&apos;": "'",
+	}
+
+	result := s
+	for entity, char := range replacements {
+		result = strings.ReplaceAll(result, entity, char)
+	}
+	return result
+}
+
+// tryDecodeGBK attempts to decode string as GBK encoding
+func tryDecodeGBK(s string) string {
+	if len(s) == 0 {
+		return ""
+	}
+
+	decoder := simplifiedchinese.GBK.NewDecoder()
+	decoded, err := decoder.Bytes([]byte(s))
+	if err != nil {
+		return ""
+	}
+	result := string(decoded)
+	if utf8.ValidString(result) {
+		return result
+	}
+	return ""
+}
+
+// parseFilenameFallback is a fallback parser for non-standard Content-Disposition
+func parseFilenameFallback(cd string) string {
+	lower := strings.ToLower(cd)
+	idx := strings.Index(lower, "filename=")
+	if idx == -1 {
+		return ""
+	}
+
+	value := cd[idx+len("filename="):]
+
+	// Find the end of the value using proper quote handling
+	endIdx := findParamValueEnd(value)
+	if endIdx != -1 {
+		value = value[:endIdx]
+	}
+	value = strings.TrimSpace(value)
+
+	// Remove surrounding quotes
+	if len(value) >= 2 {
+		if (value[0] == '"' && value[len(value)-1] == '"') ||
+			(value[0] == '\'' && value[len(value)-1] == '\'') {
+			value = value[1 : len(value)-1]
+		}
+	}
+
+	return decodeFilenameParam(value)
+}
+
+// findParamValueEnd finds the end position of a parameter value in a Content-Disposition header.
+// It correctly handles quoted values where semicolons inside quotes should not be treated as delimiters.
+// It also handles HTML entities in unquoted values (e.g., &amp; should not be split at the semicolon).
+// Returns the end index (exclusive) of the value, or -1 if it extends to the end of the string.
+func findParamValueEnd(value string) int {
+	value = strings.TrimSpace(value)
+	if len(value) == 0 {
+		return 0
+	}
+
+	// If the value starts with a quote, find the matching closing quote
+	if value[0] == '"' || value[0] == '\'' {
+		quote := value[0]
+		// Find the closing quote, handling escaped quotes
+		for i := 1; i < len(value); i++ {
+			if value[i] == quote {
+				// Check if it's escaped
+				if i > 0 && value[i-1] == '\\' {
+					continue
+				}
+				// Found closing quote, now look for ; after it
+				remaining := value[i+1:]
+				if semiIdx := strings.Index(remaining, ";"); semiIdx != -1 {
+					return i + 1 + semiIdx
+				}
+				return -1 // No semicolon after closing quote
+			}
+		}
+		// No closing quote found, treat rest of string as value
+		return -1
+	}
+
+	// Unquoted value - find the next semicolon that's not part of an HTML entity
+	// HTML entities have the pattern &...;  (e.g., &amp; &lt; &gt; &quot; &#39;)
+	for i := 0; i < len(value); i++ {
+		if value[i] == ';' {
+			// Check if this semicolon is part of an HTML entity
+			// Look backwards for an & character
+			isEntity := false
+			if i > 0 {
+				// Look for & before this semicolon (within reasonable distance, max 10 chars)
+				for j := i - 1; j >= 0 && j >= i-10; j-- {
+					if value[j] == '&' {
+						// Found &, this semicolon might be part of an HTML entity
+						// Check if there are only alphanumeric or # between & and ;
+						entityChars := value[j+1 : i]
+						if len(entityChars) > 0 && isValidHTMLEntityChars(entityChars) {
+							isEntity = true
+						}
+						break
+					}
+					// If we hit whitespace or another special char, stop looking
+					if value[j] == ' ' || value[j] == '"' || value[j] == '\'' {
+						break
+					}
+				}
+			}
+
+			if !isEntity {
+				return i
+			}
+		}
+	}
+	return -1 // No semicolon, extends to end
+}
+
+// isValidHTMLEntityChars checks if a string contains only valid HTML entity characters
+// (alphanumeric and #, typically for entities like &amp; &lt; &#39; etc.)
+func isValidHTMLEntityChars(s string) bool {
+	if len(s) == 0 {
+		return false
+	}
+	for _, c := range s {
+		if !((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '#') {
+			return false
+		}
+	}
+	return true
+}