feat: HTTP add TLS fingerprint fallback (#1444)

internal/httpclient/client.go

@@ -0,0 +1,346 @@
+package httpclient
+
+import (
+	"context"
+	"crypto/tls"
+	"fmt"
+	"net"
+	"net/http"
+	"net/url"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/imroc/req/v3"
+)
+
+type ImpersonationMode string
+
+const (
+	ImpersonationNative ImpersonationMode = "native"
+	ImpersonationFixed  ImpersonationMode = "fixed"
+	ImpersonationAuto   ImpersonationMode = "auto"
+)
+
+type Browser string
+
+const (
+	BrowserChrome  Browser = "chrome"
+	BrowserFirefox Browser = "firefox"
+	BrowserSafari  Browser = "safari"
+)
+
+type Options struct {
+	Client        ClientOptions
+	Transport     TransportOptions
+	Impersonation ImpersonationOptions
+}
+
+type ClientOptions struct {
+	Jar           http.CookieJar
+	CheckRedirect func(req *http.Request, via []*http.Request) error
+	Timeout       time.Duration
+}
+
+type TransportOptions struct {
+	Proxy               func(*http.Request) (*url.URL, error)
+	DialContext         func(ctx context.Context, network, address string) (net.Conn, error)
+	TLSClientConfig     *tls.Config
+	TLSHandshakeTimeout time.Duration
+}
+
+type ImpersonationOptions struct {
+	Mode    ImpersonationMode
+	Browser Browser
+	Session *ImpersonationSession
+}
+
+type ImpersonationSession struct {
+	mu         sync.RWMutex
+	selections map[string]impersonationSelection
+}
+
+type impersonationSelection struct {
+	browser          Browser
+	fallbackDisabled bool
+}
+
+func NewImpersonationSession() *ImpersonationSession {
+	return &ImpersonationSession{
+		selections: make(map[string]impersonationSelection),
+	}
+}
+
+func (s *ImpersonationSession) load(url string) (impersonationSelection, bool) {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+	selection, ok := s.selections[url]
+	return selection, ok
+}
+
+func (s *ImpersonationSession) store(url string, browser Browser) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	s.selections[url] = impersonationSelection{browser: browser}
+}
+
+func (s *ImpersonationSession) disableFallback(url string) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	s.selections[url] = impersonationSelection{fallbackDisabled: true}
+}
+
+func (s *ImpersonationSession) Clear() {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	clear(s.selections)
+}
+
+func NewClient(options Options) (*http.Client, error) {
+	var transport http.RoundTripper
+	switch options.Impersonation.Mode {
+	case "", ImpersonationNative:
+		transport = newNativeTransport(options.Transport)
+	case ImpersonationFixed:
+		browserTransport, err := newBrowserTransport(options.Transport, options.Impersonation.Browser)
+		if err != nil {
+			return nil, err
+		}
+		transport = browserTransport
+	case ImpersonationAuto:
+		browserTransports := make(map[Browser]http.RoundTripper, 3)
+		for _, browser := range []Browser{BrowserChrome, BrowserFirefox, BrowserSafari} {
+			browserTransport, err := newBrowserTransport(options.Transport, browser)
+			if err != nil {
+				return nil, err
+			}
+			browserTransports[browser] = browserTransport
+		}
+		transport = &adaptiveTransport{
+			native:   newNativeTransport(options.Transport),
+			browsers: browserTransports,
+			jar:      options.Client.Jar,
+			session:  options.Impersonation.Session,
+		}
+	default:
+		return nil, fmt.Errorf("unsupported impersonation mode %q", options.Impersonation.Mode)
+	}
+
+	return &http.Client{
+		Transport:     transport,
+		Jar:           options.Client.Jar,
+		CheckRedirect: options.Client.CheckRedirect,
+		Timeout:       options.Client.Timeout,
+	}, nil
+}
+
+func newNativeTransport(options TransportOptions) *http.Transport {
+	var tlsConfig *tls.Config
+	if options.TLSClientConfig != nil {
+		tlsConfig = options.TLSClientConfig.Clone()
+	}
+	return &http.Transport{
+		Proxy:               options.Proxy,
+		DialContext:         options.DialContext,
+		TLSClientConfig:     tlsConfig,
+		TLSHandshakeTimeout: options.TLSHandshakeTimeout,
+		ForceAttemptHTTP2:   true,
+	}
+}
+
+func newBrowserTransport(options TransportOptions, browser Browser) (http.RoundTripper, error) {
+	client := req.NewClient()
+	client.SetProxy(options.Proxy)
+	if options.DialContext != nil {
+		client.SetDial(options.DialContext)
+	}
+	client.SetTLSHandshakeTimeout(options.TLSHandshakeTimeout)
+	if options.TLSClientConfig != nil {
+		tlsConfig := options.TLSClientConfig.Clone()
+		if len(tlsConfig.NextProtos) == 0 {
+			tlsConfig.NextProtos = []string{"h2", "http/1.1"}
+		}
+		client.SetTLSClientConfig(tlsConfig)
+	}
+
+	switch browser {
+	case BrowserChrome:
+		client.ImpersonateChrome()
+	case BrowserFirefox:
+		client.ImpersonateFirefox()
+	case BrowserSafari:
+		client.ImpersonateSafari()
+	default:
+		return nil, fmt.Errorf("unsupported browser %q", browser)
+	}
+
+	return &defaultHeaderTransport{
+		next:     client.GetClient().Transport,
+		defaults: client.Headers.Clone(),
+	}, nil
+}
+
+type defaultHeaderTransport struct {
+	next     http.RoundTripper
+	defaults http.Header
+}
+
+func (t *defaultHeaderTransport) RoundTrip(request *http.Request) (*http.Response, error) {
+	request = request.Clone(request.Context())
+	request.Header = request.Header.Clone()
+	for key, values := range t.defaults {
+		if len(request.Header.Values(key)) == 0 {
+			request.Header[key] = append([]string(nil), values...)
+		}
+	}
+	return t.next.RoundTrip(request)
+}
+
+func (t *defaultHeaderTransport) CloseIdleConnections() {
+	closeIdleConnections(t.next)
+}
+
+type adaptiveTransport struct {
+	native   http.RoundTripper
+	browsers map[Browser]http.RoundTripper
+	jar      http.CookieJar
+	session  *ImpersonationSession
+}
+
+func (t *adaptiveTransport) RoundTrip(request *http.Request) (*http.Response, error) {
+	cacheKey := requestCacheKey(request.URL)
+	if t.session != nil {
+		if selected, ok := t.session.load(cacheKey); ok {
+			if selected.fallbackDisabled {
+				return t.native.RoundTrip(request)
+			}
+			return t.browsers[selected.browser].RoundTrip(request)
+		}
+	}
+
+	response, err := t.native.RoundTrip(request)
+	if !shouldFallback(response, err) {
+		return response, err
+	}
+
+	retry, ok := cloneRequestForRetry(request)
+	if !ok {
+		return response, err
+	}
+	if response != nil {
+		applyResponseCookies(t.jar, request, retry, response)
+		_ = response.Body.Close()
+	}
+
+	browser := browserFromUserAgent(request.UserAgent())
+	response, err = t.browsers[browser].RoundTrip(retry)
+	if t.session != nil {
+		if isSuccessfulFallback(response, err) {
+			t.session.store(cacheKey, browser)
+		} else if err == nil && response != nil {
+			t.session.disableFallback(cacheKey)
+		}
+	}
+	return response, err
+}
+
+func requestCacheKey(requestURL *url.URL) string {
+	cacheURL := *requestURL
+	cacheURL.Fragment = ""
+	cacheURL.RawFragment = ""
+	return cacheURL.String()
+}
+
+func applyResponseCookies(jar http.CookieJar, request, retry *http.Request, response *http.Response) {
+	if jar == nil {
+		return
+	}
+	jar.SetCookies(request.URL, response.Cookies())
+	jarCookies := jar.Cookies(request.URL)
+	jarCookieNames := make(map[string]struct{}, len(jarCookies))
+	for _, cookie := range jarCookies {
+		jarCookieNames[cookie.Name] = struct{}{}
+	}
+	existingCookies := retry.Cookies()
+	retry.Header.Del("Cookie")
+	for _, cookie := range existingCookies {
+		if _, replaced := jarCookieNames[cookie.Name]; !replaced {
+			retry.AddCookie(cookie)
+		}
+	}
+	for _, cookie := range jarCookies {
+		retry.AddCookie(cookie)
+	}
+}
+
+func (t *adaptiveTransport) CloseIdleConnections() {
+	closeIdleConnections(t.native)
+	for _, transport := range t.browsers {
+		closeIdleConnections(transport)
+	}
+}
+
+func closeIdleConnections(transport http.RoundTripper) {
+	if closer, ok := transport.(interface{ CloseIdleConnections() }); ok {
+		closer.CloseIdleConnections()
+	}
+}
+
+func shouldFallback(response *http.Response, err error) bool {
+	if err != nil {
+		return true
+	}
+	if response == nil {
+		return false
+	}
+	return response.StatusCode == http.StatusForbidden ||
+		strings.EqualFold(response.Header.Get("Cf-Mitigated"), "challenge")
+}
+
+func isSuccessfulFallback(response *http.Response, err error) bool {
+	return err == nil && response != nil &&
+		response.StatusCode >= http.StatusOK &&
+		response.StatusCode < http.StatusBadRequest
+}
+
+func cloneRequestForRetry(request *http.Request) (*http.Request, bool) {
+	if !isIdempotent(request) {
+		return nil, false
+	}
+	retry := request.Clone(request.Context())
+	if request.Body == nil || request.Body == http.NoBody {
+		return retry, true
+	}
+	if request.GetBody == nil {
+		return nil, false
+	}
+	body, err := request.GetBody()
+	if err != nil {
+		return nil, false
+	}
+	retry.Body = body
+	return retry, true
+}
+
+func isIdempotent(request *http.Request) bool {
+	switch request.Method {
+	case http.MethodGet, http.MethodHead, http.MethodOptions, http.MethodTrace:
+		return true
+	}
+	return request.Header.Get("Idempotency-Key") != "" ||
+		request.Header.Get("X-Idempotency-Key") != ""
+}
+
+func browserFromUserAgent(userAgent string) Browser {
+	userAgent = strings.ToLower(userAgent)
+	switch {
+	case strings.Contains(userAgent, "firefox"):
+		return BrowserFirefox
+	case strings.Contains(userAgent, "safari") &&
+		!strings.Contains(userAgent, "chrome") &&
+		!strings.Contains(userAgent, "chromium"):
+		return BrowserSafari
+	default:
+		return BrowserChrome
+	}
+}

internal/httpclient/client_test.go

@@ -0,0 +1,962 @@
+package httpclient_test
+
+import (
+	"context"
+	"crypto/tls"
+	"errors"
+	"io"
+	"net"
+	"net/http"
+	"net/http/cookiejar"
+	"net/http/httptest"
+	"net/url"
+	"slices"
+	"strings"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/GopeedLab/gopeed/internal/httpclient"
+	"github.com/GopeedLab/gopeed/internal/httpclient/httpclienttest"
+)
+
+func TestFixedChromeUsesBrowserTLSFingerprintAndHTTP2(t *testing.T) {
+	server := httpclienttest.NewFingerprintServer(t, httpclienttest.FingerprintServerOptions{
+		RequiredProfile: httpclienttest.ProfileAnyBrowser,
+	})
+
+	client, err := httpclient.NewClient(httpclient.Options{
+		Transport: httpclient.TransportOptions{
+			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+		},
+		Impersonation: httpclient.ImpersonationOptions{
+			Mode:    httpclient.ImpersonationFixed,
+			Browser: httpclient.BrowserChrome,
+		},
+	})
+	if err != nil {
+		t.Fatalf("NewClient() error = %v", err)
+	}
+
+	response, err := client.Get(server.URL)
+	if err != nil {
+		t.Fatalf("Get() error = %v", err)
+	}
+	defer response.Body.Close()
+
+	if response.StatusCode != http.StatusOK {
+		t.Fatalf("status = %d, want %d", response.StatusCode, http.StatusOK)
+	}
+	if response.ProtoMajor != 2 {
+		t.Fatalf("protocol = %q, want HTTP/2", response.Proto)
+	}
+}
+
+func TestAutoFallsBackFromNativeToBrowserFingerprint(t *testing.T) {
+	server := httpclienttest.NewFingerprintServer(t, httpclienttest.FingerprintServerOptions{
+		RequiredProfile: httpclienttest.ProfileAnyBrowser,
+	})
+
+	client, err := httpclient.NewClient(httpclient.Options{
+		Transport: httpclient.TransportOptions{
+			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+		},
+		Impersonation: httpclient.ImpersonationOptions{
+			Mode: httpclient.ImpersonationAuto,
+		},
+	})
+	if err != nil {
+		t.Fatalf("NewClient() error = %v", err)
+	}
+
+	request, err := http.NewRequest(http.MethodGet, server.URL, nil)
+	if err != nil {
+		t.Fatalf("NewRequest() error = %v", err)
+	}
+	request.Header.Set("User-Agent", "Mozilla/5.0 Chrome/120.0.0.0 Safari/537.36")
+
+	response, err := client.Do(request)
+	if err != nil {
+		t.Fatalf("Do() error = %v", err)
+	}
+	defer response.Body.Close()
+
+	if response.ProtoMajor != 2 {
+		t.Fatalf("protocol = %q, want HTTP/2", response.Proto)
+	}
+	if got, want := server.Profiles(), []httpclienttest.Profile{
+		httpclienttest.ProfileNative,
+		httpclienttest.ProfileChrome,
+	}; !slices.Equal(got, want) {
+		t.Fatalf("ClientHello profiles = %v, want %v", got, want)
+	}
+}
+
+func TestAutoChoosesBrowserFingerprintFromUserAgent(t *testing.T) {
+	tests := []struct {
+		name        string
+		userAgent   string
+		wantBrowser httpclient.Browser
+	}{
+		{
+			name:        "chrome",
+			userAgent:   "Mozilla/5.0 Chrome/120.0.0.0 Safari/537.36",
+			wantBrowser: httpclient.BrowserChrome,
+		},
+		{
+			name:        "firefox",
+			userAgent:   "Mozilla/5.0 Firefox/120.0",
+			wantBrowser: httpclient.BrowserFirefox,
+		},
+		{
+			name:        "safari",
+			userAgent:   "Mozilla/5.0 Version/16.6 Safari/605.1.15",
+			wantBrowser: httpclient.BrowserSafari,
+		},
+		{
+			name:        "unknown defaults to chrome",
+			userAgent:   "Gopeed/1.0",
+			wantBrowser: httpclient.BrowserChrome,
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			server := httpclienttest.NewFingerprintServer(t, httpclienttest.FingerprintServerOptions{
+				RequiredProfile: httpclienttest.Profile(test.wantBrowser),
+			})
+			client, err := httpclient.NewClient(httpclient.Options{
+				Transport: httpclient.TransportOptions{
+					TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+				},
+				Impersonation: httpclient.ImpersonationOptions{Mode: httpclient.ImpersonationAuto},
+			})
+			if err != nil {
+				t.Fatalf("NewClient() error = %v", err)
+			}
+
+			request, err := http.NewRequest(http.MethodGet, server.URL, nil)
+			if err != nil {
+				t.Fatalf("NewRequest() error = %v", err)
+			}
+			request.Header.Set("User-Agent", test.userAgent)
+			response, err := client.Do(request)
+			if err != nil {
+				t.Fatalf("Do() error = %v", err)
+			}
+			response.Body.Close()
+
+			if response.ProtoMajor != 2 {
+				t.Fatalf("protocol = %q, want HTTP/2", response.Proto)
+			}
+			if got, want := server.Profiles(), []httpclienttest.Profile{
+				httpclienttest.ProfileNative,
+				httpclienttest.Profile(test.wantBrowser),
+			}; !slices.Equal(got, want) {
+				t.Fatalf("profiles = %v, want %v", got, want)
+			}
+		})
+	}
+}
+
+func TestAutoFallsBackOnForbiddenAndPinsBrowserPerURL(t *testing.T) {
+	var requests atomic.Int32
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) {
+		requests.Add(1)
+		if request.Header.Get("Sec-Ch-Ua") == "" {
+			http.Error(w, "browser required", http.StatusForbidden)
+			return
+		}
+		w.WriteHeader(http.StatusOK)
+	}))
+	t.Cleanup(server.Close)
+
+	client, err := httpclient.NewClient(httpclient.Options{
+		Impersonation: httpclient.ImpersonationOptions{
+			Mode:    httpclient.ImpersonationAuto,
+			Session: httpclient.NewImpersonationSession(),
+		},
+	})
+	if err != nil {
+		t.Fatalf("NewClient() error = %v", err)
+	}
+
+	for range 2 {
+		request, err := http.NewRequest(http.MethodGet, server.URL, nil)
+		if err != nil {
+			t.Fatalf("NewRequest() error = %v", err)
+		}
+		request.Header.Set("User-Agent", "Mozilla/5.0 Chrome/120.0.0.0 Safari/537.36")
+		response, err := client.Do(request)
+		if err != nil {
+			t.Fatalf("Do() error = %v", err)
+		}
+		response.Body.Close()
+		if response.StatusCode != http.StatusOK {
+			t.Fatalf("status = %d, want %d", response.StatusCode, http.StatusOK)
+		}
+	}
+
+	if requests.Load() != 3 {
+		t.Fatalf("requests = %d, want 3 (native + fallback + pinned)", requests.Load())
+	}
+}
+
+func TestAutoDisablesFallbackWhenBrowserReturnsHTTPFailure(t *testing.T) {
+	var requests atomic.Int32
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) {
+		requests.Add(1)
+		if request.Header.Get("Sec-Ch-Ua") == "" {
+			w.WriteHeader(http.StatusForbidden)
+			return
+		}
+		w.WriteHeader(http.StatusNotFound)
+	}))
+	t.Cleanup(server.Close)
+
+	client, err := httpclient.NewClient(httpclient.Options{
+		Impersonation: httpclient.ImpersonationOptions{
+			Mode:    httpclient.ImpersonationAuto,
+			Session: httpclient.NewImpersonationSession(),
+		},
+	})
+	if err != nil {
+		t.Fatalf("NewClient() error = %v", err)
+	}
+
+	wantStatuses := []int{http.StatusNotFound, http.StatusForbidden}
+	for _, wantStatus := range wantStatuses {
+		response, err := client.Get(server.URL)
+		if err != nil {
+			t.Fatalf("Get() error = %v", err)
+		}
+		response.Body.Close()
+		if response.StatusCode != wantStatus {
+			t.Fatalf("status = %d, want %d", response.StatusCode, wantStatus)
+		}
+	}
+
+	if requests.Load() != 3 {
+		t.Fatalf("requests = %d, want 3 (native + one fallback + native)", requests.Load())
+	}
+}
+
+func TestAutoDoesNotDisableFallbackAfterTransportFailure(t *testing.T) {
+	server := httpclienttest.NewFingerprintServer(t, httpclienttest.FingerprintServerOptions{
+		RequiredProfile: httpclienttest.Profile("unsupported"),
+	})
+	client, err := httpclient.NewClient(httpclient.Options{
+		Transport: httpclient.TransportOptions{
+			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+		},
+		Impersonation: httpclient.ImpersonationOptions{
+			Mode:    httpclient.ImpersonationAuto,
+			Session: httpclient.NewImpersonationSession(),
+		},
+	})
+	if err != nil {
+		t.Fatalf("NewClient() error = %v", err)
+	}
+
+	for range 2 {
+		if _, err := client.Get(server.URL); err == nil {
+			t.Fatal("Get() error = nil, want TLS failure")
+		}
+	}
+	if got, want := server.Profiles(), []httpclienttest.Profile{
+		httpclienttest.ProfileNative,
+		httpclienttest.ProfileChrome,
+		httpclienttest.ProfileNative,
+		httpclienttest.ProfileChrome,
+	}; !slices.Equal(got, want) {
+		t.Fatalf("ClientHello profiles = %v, want %v", got, want)
+	}
+}
+
+func TestAutoWithoutSessionDoesNotCacheSelectedBrowser(t *testing.T) {
+	var requests atomic.Int32
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) {
+		requests.Add(1)
+		if request.Header.Get("Sec-Ch-Ua") == "" {
+			w.WriteHeader(http.StatusForbidden)
+			return
+		}
+		w.WriteHeader(http.StatusOK)
+	}))
+	t.Cleanup(server.Close)
+
+	client, err := httpclient.NewClient(httpclient.Options{
+		Impersonation: httpclient.ImpersonationOptions{Mode: httpclient.ImpersonationAuto},
+	})
+	if err != nil {
+		t.Fatalf("NewClient() error = %v", err)
+	}
+	for range 2 {
+		response, err := client.Get(server.URL)
+		if err != nil {
+			t.Fatalf("Get() error = %v", err)
+		}
+		response.Body.Close()
+	}
+
+	if requests.Load() != 4 {
+		t.Fatalf("requests = %d, want 4 (native + fallback without caching)", requests.Load())
+	}
+}
+
+func TestAutoSharesSelectedBrowserAcrossClientsInSession(t *testing.T) {
+	var requests atomic.Int32
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) {
+		requests.Add(1)
+		if request.Header.Get("Sec-Ch-Ua") == "" {
+			w.WriteHeader(http.StatusForbidden)
+			return
+		}
+		w.WriteHeader(http.StatusOK)
+	}))
+	t.Cleanup(server.Close)
+	session := httpclient.NewImpersonationSession()
+
+	for range 2 {
+		client, err := httpclient.NewClient(httpclient.Options{
+			Impersonation: httpclient.ImpersonationOptions{
+				Mode:    httpclient.ImpersonationAuto,
+				Session: session,
+			},
+		})
+		if err != nil {
+			t.Fatalf("NewClient() error = %v", err)
+		}
+		request, err := http.NewRequest(http.MethodGet, server.URL, nil)
+		if err != nil {
+			t.Fatalf("NewRequest() error = %v", err)
+		}
+		request.Header.Set("User-Agent", "Mozilla/5.0 Chrome/120.0.0.0 Safari/537.36")
+		response, err := client.Do(request)
+		if err != nil {
+			t.Fatalf("Do() error = %v", err)
+		}
+		response.Body.Close()
+	}
+
+	if requests.Load() != 3 {
+		t.Fatalf("requests = %d, want 3 (native + fallback + session pinned)", requests.Load())
+	}
+}
+
+func TestAutoDoesNotShareSelectedBrowserAcrossSessions(t *testing.T) {
+	var requests atomic.Int32
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) {
+		requests.Add(1)
+		if request.Header.Get("Sec-Ch-Ua") == "" {
+			w.WriteHeader(http.StatusForbidden)
+			return
+		}
+		w.WriteHeader(http.StatusOK)
+	}))
+	t.Cleanup(server.Close)
+
+	for range 2 {
+		client, err := httpclient.NewClient(httpclient.Options{
+			Impersonation: httpclient.ImpersonationOptions{
+				Mode:    httpclient.ImpersonationAuto,
+				Session: httpclient.NewImpersonationSession(),
+			},
+		})
+		if err != nil {
+			t.Fatalf("NewClient() error = %v", err)
+		}
+		response, err := client.Get(server.URL)
+		if err != nil {
+			t.Fatalf("Get() error = %v", err)
+		}
+		response.Body.Close()
+	}
+
+	if requests.Load() != 4 {
+		t.Fatalf("requests = %d, want 4 (native + fallback per session)", requests.Load())
+	}
+}
+
+func TestAutoCachesSelectedBrowserByFullURL(t *testing.T) {
+	var publicBrowserRequests atomic.Int32
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) {
+		browserRequest := request.Header.Get("Sec-Ch-Ua") != ""
+		switch request.URL.Query().Get("access") {
+		case "protected":
+			if !browserRequest {
+				w.WriteHeader(http.StatusForbidden)
+				return
+			}
+		case "public":
+			if browserRequest {
+				publicBrowserRequests.Add(1)
+			}
+		}
+		w.WriteHeader(http.StatusOK)
+	}))
+	t.Cleanup(server.Close)
+
+	client, err := httpclient.NewClient(httpclient.Options{
+		Impersonation: httpclient.ImpersonationOptions{
+			Mode:    httpclient.ImpersonationAuto,
+			Session: httpclient.NewImpersonationSession(),
+		},
+	})
+	if err != nil {
+		t.Fatalf("NewClient() error = %v", err)
+	}
+
+	for range 2 {
+		response, err := client.Get(server.URL + "/file?access=protected")
+		if err != nil {
+			t.Fatalf("protected Get() error = %v", err)
+		}
+		response.Body.Close()
+	}
+	response, err := client.Get(server.URL + "/file?access=public")
+	if err != nil {
+		t.Fatalf("public Get() error = %v", err)
+	}
+	response.Body.Close()
+
+	if publicBrowserRequests.Load() != 0 {
+		t.Fatalf("public browser requests = %d, want 0", publicBrowserRequests.Load())
+	}
+}
+
+func TestImpersonationSessionClearRemovesSelections(t *testing.T) {
+	var requests atomic.Int32
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) {
+		requests.Add(1)
+		if request.Header.Get("Sec-Ch-Ua") == "" {
+			w.WriteHeader(http.StatusForbidden)
+			return
+		}
+		w.WriteHeader(http.StatusOK)
+	}))
+	t.Cleanup(server.Close)
+
+	session := httpclient.NewImpersonationSession()
+	client, err := httpclient.NewClient(httpclient.Options{
+		Impersonation: httpclient.ImpersonationOptions{
+			Mode:    httpclient.ImpersonationAuto,
+			Session: session,
+		},
+	})
+	if err != nil {
+		t.Fatalf("NewClient() error = %v", err)
+	}
+	for range 2 {
+		response, err := client.Get(server.URL)
+		if err != nil {
+			t.Fatalf("Get() error = %v", err)
+		}
+		response.Body.Close()
+		session.Clear()
+	}
+
+	if requests.Load() != 4 {
+		t.Fatalf("requests = %d, want 4 (native + fallback after each clear)", requests.Load())
+	}
+}
+
+func TestGlobalAutoSelectionDoesNotAffectNativeMode(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) {
+		if request.Header.Get("Sec-Ch-Ua") == "" {
+			w.WriteHeader(http.StatusForbidden)
+			return
+		}
+		w.WriteHeader(http.StatusOK)
+	}))
+	t.Cleanup(server.Close)
+
+	autoClient, err := httpclient.NewClient(httpclient.Options{
+		Impersonation: httpclient.ImpersonationOptions{Mode: httpclient.ImpersonationAuto},
+	})
+	if err != nil {
+		t.Fatalf("NewClient(auto) error = %v", err)
+	}
+	autoResponse, err := autoClient.Get(server.URL)
+	if err != nil {
+		t.Fatalf("auto Get() error = %v", err)
+	}
+	autoResponse.Body.Close()
+	if autoResponse.StatusCode != http.StatusOK {
+		t.Fatalf("auto status = %d, want %d", autoResponse.StatusCode, http.StatusOK)
+	}
+
+	nativeClient, err := httpclient.NewClient(httpclient.Options{
+		Impersonation: httpclient.ImpersonationOptions{Mode: httpclient.ImpersonationNative},
+	})
+	if err != nil {
+		t.Fatalf("NewClient(native) error = %v", err)
+	}
+	nativeResponse, err := nativeClient.Get(server.URL)
+	if err != nil {
+		t.Fatalf("native Get() error = %v", err)
+	}
+	nativeResponse.Body.Close()
+	if nativeResponse.StatusCode != http.StatusForbidden {
+		t.Fatalf("native status = %d, want %d", nativeResponse.StatusCode, http.StatusForbidden)
+	}
+}
+
+func TestAutoAppliesFallbackResponseCookiesBeforeRetry(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) {
+		if request.Header.Get("Sec-Ch-Ua") == "" {
+			http.SetCookie(w, &http.Cookie{Name: "challenge", Value: "passed", Path: "/"})
+			w.WriteHeader(http.StatusForbidden)
+			return
+		}
+		cookie, err := request.Cookie("challenge")
+		if err != nil || cookie.Value != "passed" {
+			w.WriteHeader(http.StatusForbidden)
+			return
+		}
+		w.WriteHeader(http.StatusOK)
+	}))
+	t.Cleanup(server.Close)
+
+	jar, err := cookiejar.New(nil)
+	if err != nil {
+		t.Fatalf("cookiejar.New() error = %v", err)
+	}
+	client, err := httpclient.NewClient(httpclient.Options{
+		Client:        httpclient.ClientOptions{Jar: jar},
+		Impersonation: httpclient.ImpersonationOptions{Mode: httpclient.ImpersonationAuto},
+	})
+	if err != nil {
+		t.Fatalf("NewClient() error = %v", err)
+	}
+	response, err := client.Get(server.URL)
+	if err != nil {
+		t.Fatalf("Get() error = %v", err)
+	}
+	response.Body.Close()
+	if response.StatusCode != http.StatusOK {
+		t.Fatalf("status = %d, want %d", response.StatusCode, http.StatusOK)
+	}
+}
+
+func TestAutoDoesNotRetryRequestWithNonReplayableBody(t *testing.T) {
+	var requests atomic.Int32
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		requests.Add(1)
+		w.WriteHeader(http.StatusForbidden)
+	}))
+	t.Cleanup(server.Close)
+
+	client, err := httpclient.NewClient(httpclient.Options{
+		Impersonation: httpclient.ImpersonationOptions{Mode: httpclient.ImpersonationAuto},
+	})
+	if err != nil {
+		t.Fatalf("NewClient() error = %v", err)
+	}
+	bodyReader, bodyWriter := io.Pipe()
+	t.Cleanup(func() {
+		bodyReader.Close()
+		bodyWriter.Close()
+	})
+	go func() {
+		_, _ = bodyWriter.Write([]byte("payload"))
+		_ = bodyWriter.Close()
+	}()
+	request, err := http.NewRequest(http.MethodPost, server.URL, bodyReader)
+	if err != nil {
+		t.Fatalf("NewRequest() error = %v", err)
+	}
+
+	response, err := client.Do(request)
+	if err != nil {
+		t.Fatalf("Do() error = %v", err)
+	}
+	response.Body.Close()
+	if requests.Load() != 1 {
+		t.Fatalf("requests = %d, want 1", requests.Load())
+	}
+}
+
+func TestAutoDoesNotRetryReplayablePost(t *testing.T) {
+	var requests atomic.Int32
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		requests.Add(1)
+		w.WriteHeader(http.StatusForbidden)
+	}))
+	t.Cleanup(server.Close)
+
+	client, err := httpclient.NewClient(httpclient.Options{
+		Impersonation: httpclient.ImpersonationOptions{Mode: httpclient.ImpersonationAuto},
+	})
+	if err != nil {
+		t.Fatalf("NewClient() error = %v", err)
+	}
+	request, err := http.NewRequest(http.MethodPost, server.URL, strings.NewReader("payload"))
+	if err != nil {
+		t.Fatalf("NewRequest() error = %v", err)
+	}
+
+	response, err := client.Do(request)
+	if err != nil {
+		t.Fatalf("Do() error = %v", err)
+	}
+	response.Body.Close()
+	if requests.Load() != 1 {
+		t.Fatalf("requests = %d, want 1", requests.Load())
+	}
+}
+
+func TestNativeUsesStandardTransportWithHTTP2(t *testing.T) {
+	server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		w.WriteHeader(http.StatusOK)
+	}))
+	server.EnableHTTP2 = true
+	server.StartTLS()
+	t.Cleanup(server.Close)
+
+	client, err := httpclient.NewClient(httpclient.Options{
+		Transport: httpclient.TransportOptions{
+			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+		},
+		Impersonation: httpclient.ImpersonationOptions{
+			Mode: httpclient.ImpersonationNative,
+		},
+	})
+	if err != nil {
+		t.Fatalf("NewClient() error = %v", err)
+	}
+
+	response, err := client.Get(server.URL)
+	if err != nil {
+		t.Fatalf("Get() error = %v", err)
+	}
+	defer response.Body.Close()
+
+	if response.ProtoMajor != 2 {
+		t.Fatalf("protocol = %q, want HTTP/2", response.Proto)
+	}
+}
+
+func TestClientOptionsPreserveCookiesAcrossRedirects(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) {
+		switch request.URL.Path {
+		case "/start":
+			http.SetCookie(w, &http.Cookie{Name: "session", Value: "gopeed", Path: "/"})
+			http.Redirect(w, request, "/download", http.StatusFound)
+		case "/download":
+			cookie, err := request.Cookie("session")
+			if err != nil || cookie.Value != "gopeed" {
+				http.Error(w, "cookie missing", http.StatusBadRequest)
+				return
+			}
+			w.WriteHeader(http.StatusOK)
+		default:
+			http.NotFound(w, request)
+		}
+	}))
+	t.Cleanup(server.Close)
+
+	jar, err := cookiejar.New(nil)
+	if err != nil {
+		t.Fatalf("cookiejar.New() error = %v", err)
+	}
+	var redirects atomic.Int32
+	client, err := httpclient.NewClient(httpclient.Options{
+		Client: httpclient.ClientOptions{
+			Jar: jar,
+			CheckRedirect: func(_ *http.Request, _ []*http.Request) error {
+				redirects.Add(1)
+				return nil
+			},
+		},
+		Impersonation: httpclient.ImpersonationOptions{Mode: httpclient.ImpersonationNative},
+	})
+	if err != nil {
+		t.Fatalf("NewClient() error = %v", err)
+	}
+
+	response, err := client.Get(server.URL + "/start")
+	if err != nil {
+		t.Fatalf("Get() error = %v", err)
+	}
+	defer response.Body.Close()
+
+	if response.StatusCode != http.StatusOK {
+		t.Fatalf("status = %d, want %d", response.StatusCode, http.StatusOK)
+	}
+	if redirects.Load() != 1 {
+		t.Fatalf("redirect calls = %d, want 1", redirects.Load())
+	}
+}
+
+func TestClientTimeoutCancelsRequest(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, request *http.Request) {
+		<-request.Context().Done()
+	}))
+	t.Cleanup(server.Close)
+
+	client, err := httpclient.NewClient(httpclient.Options{
+		Client: httpclient.ClientOptions{Timeout: 50 * time.Millisecond},
+		Impersonation: httpclient.ImpersonationOptions{
+			Mode: httpclient.ImpersonationNative,
+		},
+	})
+	if err != nil {
+		t.Fatalf("NewClient() error = %v", err)
+	}
+
+	_, err = client.Get(server.URL)
+	if err == nil {
+		t.Fatal("Get() error = nil, want timeout")
+	}
+	var networkError net.Error
+	if !errors.As(err, &networkError) || !networkError.Timeout() {
+		t.Fatalf("Get() error = %v, want net.Error timeout", err)
+	}
+}
+
+func TestTransportOptionsUseProxyInNativeAndFixedModes(t *testing.T) {
+	proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) {
+		if request.URL.Host != "download.example" {
+			http.Error(w, "unexpected target", http.StatusBadRequest)
+			return
+		}
+		w.WriteHeader(http.StatusOK)
+	}))
+	t.Cleanup(proxy.Close)
+	proxyURL, err := url.Parse(proxy.URL)
+	if err != nil {
+		t.Fatalf("url.Parse() error = %v", err)
+	}
+
+	tests := []struct {
+		name          string
+		impersonation httpclient.ImpersonationOptions
+	}{
+		{
+			name:          "native",
+			impersonation: httpclient.ImpersonationOptions{Mode: httpclient.ImpersonationNative},
+		},
+		{
+			name: "fixed chrome",
+			impersonation: httpclient.ImpersonationOptions{
+				Mode:    httpclient.ImpersonationFixed,
+				Browser: httpclient.BrowserChrome,
+			},
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			client, err := httpclient.NewClient(httpclient.Options{
+				Transport:     httpclient.TransportOptions{Proxy: http.ProxyURL(proxyURL)},
+				Impersonation: test.impersonation,
+			})
+			if err != nil {
+				t.Fatalf("NewClient() error = %v", err)
+			}
+
+			response, err := client.Get("http://download.example/file")
+			if err != nil {
+				t.Fatalf("Get() error = %v", err)
+			}
+			defer response.Body.Close()
+			if response.StatusCode != http.StatusOK {
+				t.Fatalf("status = %d, want %d", response.StatusCode, http.StatusOK)
+			}
+		})
+	}
+}
+
+func TestTransportOptionsUseDialContextInNativeAndFixedModes(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		w.WriteHeader(http.StatusOK)
+	}))
+	t.Cleanup(server.Close)
+
+	tests := []struct {
+		name          string
+		impersonation httpclient.ImpersonationOptions
+	}{
+		{
+			name:          "native",
+			impersonation: httpclient.ImpersonationOptions{Mode: httpclient.ImpersonationNative},
+		},
+		{
+			name: "fixed chrome",
+			impersonation: httpclient.ImpersonationOptions{
+				Mode:    httpclient.ImpersonationFixed,
+				Browser: httpclient.BrowserChrome,
+			},
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			var calls atomic.Int32
+			dialer := &net.Dialer{}
+			client, err := httpclient.NewClient(httpclient.Options{
+				Transport: httpclient.TransportOptions{
+					DialContext: func(ctx context.Context, network, address string) (net.Conn, error) {
+						calls.Add(1)
+						return dialer.DialContext(ctx, network, address)
+					},
+				},
+				Impersonation: test.impersonation,
+			})
+			if err != nil {
+				t.Fatalf("NewClient() error = %v", err)
+			}
+
+			response, err := client.Get(server.URL)
+			if err != nil {
+				t.Fatalf("Get() error = %v", err)
+			}
+			response.Body.Close()
+			if calls.Load() == 0 {
+				t.Fatal("DialContext() was not called")
+			}
+		})
+	}
+}
+
+func TestTransportOptionsApplyTLSHandshakeTimeoutInNativeAndFixedModes(t *testing.T) {
+	listener, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatalf("net.Listen() error = %v", err)
+	}
+	closed := make(chan struct{})
+	t.Cleanup(func() {
+		close(closed)
+		listener.Close()
+	})
+	go func() {
+		for {
+			connection, err := listener.Accept()
+			if err != nil {
+				return
+			}
+			go func() {
+				<-closed
+				connection.Close()
+			}()
+		}
+	}()
+
+	tests := []struct {
+		name          string
+		impersonation httpclient.ImpersonationOptions
+	}{
+		{
+			name:          "native",
+			impersonation: httpclient.ImpersonationOptions{Mode: httpclient.ImpersonationNative},
+		},
+		{
+			name: "fixed chrome",
+			impersonation: httpclient.ImpersonationOptions{
+				Mode:    httpclient.ImpersonationFixed,
+				Browser: httpclient.BrowserChrome,
+			},
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			client, err := httpclient.NewClient(httpclient.Options{
+				Client: httpclient.ClientOptions{Timeout: time.Second},
+				Transport: httpclient.TransportOptions{
+					TLSHandshakeTimeout: 50 * time.Millisecond,
+				},
+				Impersonation: test.impersonation,
+			})
+			if err != nil {
+				t.Fatalf("NewClient() error = %v", err)
+			}
+
+			started := time.Now()
+			_, err = client.Get("https://" + listener.Addr().String())
+			if err == nil {
+				t.Fatal("Get() error = nil, want TLS handshake timeout")
+			}
+			if elapsed := time.Since(started); elapsed >= 500*time.Millisecond {
+				t.Fatalf("TLS handshake took %s, want less than 500ms", elapsed)
+			}
+		})
+	}
+}
+
+func TestFixedBrowserAddsProfileHeadersWithoutOverwritingExplicitHeaders(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) {
+		w.Header().Set("Seen-User-Agent", request.UserAgent())
+		w.WriteHeader(http.StatusOK)
+	}))
+	t.Cleanup(server.Close)
+
+	client, err := httpclient.NewClient(httpclient.Options{
+		Impersonation: httpclient.ImpersonationOptions{
+			Mode:    httpclient.ImpersonationFixed,
+			Browser: httpclient.BrowserChrome,
+		},
+	})
+	if err != nil {
+		t.Fatalf("NewClient() error = %v", err)
+	}
+
+	response, err := client.Get(server.URL)
+	if err != nil {
+		t.Fatalf("Get() error = %v", err)
+	}
+	response.Body.Close()
+	if userAgent := response.Header.Get("Seen-User-Agent"); !strings.Contains(userAgent, "Chrome/120") {
+		t.Fatalf("profile User-Agent = %q, want Chrome/120", userAgent)
+	}
+
+	request, err := http.NewRequest(http.MethodGet, server.URL, nil)
+	if err != nil {
+		t.Fatalf("NewRequest() error = %v", err)
+	}
+	request.Header.Set("User-Agent", "Gopeed-Custom/1.0")
+	response, err = client.Do(request)
+	if err != nil {
+		t.Fatalf("Do() error = %v", err)
+	}
+	response.Body.Close()
+	if userAgent := response.Header.Get("Seen-User-Agent"); userAgent != "Gopeed-Custom/1.0" {
+		t.Fatalf("explicit User-Agent = %q, want Gopeed-Custom/1.0", userAgent)
+	}
+}
+
+func TestFixedClientCloseIdleConnectionsReachesBrowserTransport(t *testing.T) {
+	var connections atomic.Int32
+	server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		w.WriteHeader(http.StatusOK)
+	}))
+	server.Config.ConnState = func(_ net.Conn, state http.ConnState) {
+		if state == http.StateNew {
+			connections.Add(1)
+		}
+	}
+	server.Start()
+	t.Cleanup(server.Close)
+
+	client, err := httpclient.NewClient(httpclient.Options{
+		Impersonation: httpclient.ImpersonationOptions{
+			Mode:    httpclient.ImpersonationFixed,
+			Browser: httpclient.BrowserChrome,
+		},
+	})
+	if err != nil {
+		t.Fatalf("NewClient() error = %v", err)
+	}
+	for attempt := range 2 {
+		response, err := client.Get(server.URL)
+		if err != nil {
+			t.Fatalf("Get() error = %v", err)
+		}
+		_, _ = io.Copy(io.Discard, response.Body)
+		response.Body.Close()
+		if attempt == 0 {
+			client.CloseIdleConnections()
+		}
+	}
+
+	if connections.Load() != 2 {
+		t.Fatalf("connections = %d, want 2", connections.Load())
+	}
+}

internal/httpclient/httpclienttest/fingerprint_server.go

@@ -0,0 +1,100 @@
+package httpclienttest
+
+import (
+	"crypto/tls"
+	"errors"
+	"io"
+	"log"
+	"net/http"
+	"net/http/httptest"
+	"sync"
+	"testing"
+)
+
+type Profile string
+
+const (
+	ProfileNative     Profile = "native"
+	ProfileChrome     Profile = "chrome"
+	ProfileFirefox    Profile = "firefox"
+	ProfileSafari     Profile = "safari"
+	ProfileAnyBrowser Profile = "browser"
+)
+
+type FingerprintServerOptions struct {
+	RequiredProfile Profile
+	Handler         http.Handler
+}
+
+type FingerprintServer struct {
+	*httptest.Server
+
+	mu       sync.Mutex
+	profiles []Profile
+}
+
+func NewFingerprintServer(t testing.TB, options FingerprintServerOptions) *FingerprintServer {
+	t.Helper()
+
+	handler := options.Handler
+	if handler == nil {
+		handler = http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+			w.WriteHeader(http.StatusOK)
+		})
+	}
+	server := &FingerprintServer{
+		Server: httptest.NewUnstartedServer(handler),
+	}
+	server.EnableHTTP2 = true
+	server.Config.ErrorLog = log.New(io.Discard, "", 0)
+	server.TLS = &tls.Config{
+		GetConfigForClient: func(hello *tls.ClientHelloInfo) (*tls.Config, error) {
+			profile := classifyProfile(hello.CipherSuites)
+			server.mu.Lock()
+			server.profiles = append(server.profiles, profile)
+			server.mu.Unlock()
+			if profileMatches(profile, options.RequiredProfile) {
+				return nil, nil
+			}
+			return nil, errors.New("required TLS fingerprint was not presented")
+		},
+	}
+	server.StartTLS()
+	t.Cleanup(server.Close)
+	return server
+}
+
+func (s *FingerprintServer) Profiles() []Profile {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	return append([]Profile(nil), s.profiles...)
+}
+
+func profileMatches(actual, required Profile) bool {
+	if required == ProfileAnyBrowser {
+		return actual != ProfileNative
+	}
+	return actual == required
+}
+
+func classifyProfile(cipherSuites []uint16) Profile {
+	if len(cipherSuites) >= 5 && isGREASE(cipherSuites[0]) {
+		switch cipherSuites[4] {
+		case tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256:
+			return ProfileChrome
+		case tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384:
+			return ProfileSafari
+		}
+	}
+	if len(cipherSuites) >= 3 &&
+		cipherSuites[0] == tls.TLS_AES_128_GCM_SHA256 &&
+		cipherSuites[1] == tls.TLS_CHACHA20_POLY1305_SHA256 &&
+		cipherSuites[2] == tls.TLS_AES_256_GCM_SHA384 {
+		return ProfileFirefox
+	}
+	return ProfileNative
+}
+
+func isGREASE(value uint16) bool {
+	return value&0x0f0f == 0x0a0a && byte(value) == byte(value>>8)
+}