feat: HTTP add TLS fingerprint fallback (#1444)

internal/protocol/http/fetcher.go

@@ -17,6 +17,7 @@ import (
 
 	"github.com/GopeedLab/gopeed/internal/controller"
 	"github.com/GopeedLab/gopeed/internal/fetcher"
+	"github.com/GopeedLab/gopeed/internal/httpclient"
 	"github.com/GopeedLab/gopeed/pkg/base"
 	fhttp "github.com/GopeedLab/gopeed/pkg/protocol/http"
 	"github.com/xiaoqidun/setft"
@@ -213,6 +214,8 @@ type Fetcher struct {
 	config *config
 	doneCh chan error
 
+	impersonationSession *httpclient.ImpersonationSession
+
 	meta *fetcher.FetcherMeta
 
 	// State machine
@@ -277,6 +280,9 @@ func (f *Fetcher) Setup(ctl *controller.Controller) {
 		f.meta = &fetcher.FetcherMeta{}
 	}
 	f.ctl.GetConfig(&f.config)
+	if f.impersonationSession == nil {
+		f.impersonationSession = httpclient.NewImpersonationSession()
+	}
 	f.resolvedCh = make(chan struct{})
 	f.primaryReadyCh = make(chan struct{})
 
@@ -1713,7 +1719,11 @@ func (f *Fetcher) Pause() error {
 }
 
 func (f *Fetcher) Close() error {
-	return f.Pause()
+	err := f.Pause()
+	if f.impersonationSession != nil {
+		f.impersonationSession.Clear()
+	}
+	return err
 }
 
 func (f *Fetcher) Meta() *fetcher.FetcherMeta {

internal/protocol/http/fetcher_test.go

@@ -5,14 +5,17 @@ import (
 	"fmt"
 	"net"
 	gohttp "net/http"
+	"net/http/httptest"
 	"net/url"
 	"os"
 	"strings"
+	"sync/atomic"
 	"testing"
 	"time"
 
 	"github.com/GopeedLab/gopeed/internal/controller"
 	"github.com/GopeedLab/gopeed/internal/fetcher"
+	"github.com/GopeedLab/gopeed/internal/httpclient/httpclienttest"
 	"github.com/GopeedLab/gopeed/internal/test"
 	"github.com/GopeedLab/gopeed/pkg/base"
 	"github.com/GopeedLab/gopeed/pkg/protocol/http"
@@ -259,6 +262,79 @@ func TestFetcher_ResolveWithInvalidHeader(t *testing.T) {
 	}
 }
 
+func TestFetcher_ResolveAutomaticallyFallsBackToBrowserFingerprint(t *testing.T) {
+	server := httpclienttest.NewFingerprintServer(t, httpclienttest.FingerprintServerOptions{
+		RequiredProfile: httpclienttest.ProfileAnyBrowser,
+		Handler: gohttp.HandlerFunc(func(w gohttp.ResponseWriter, _ *gohttp.Request) {
+			w.Header().Set(base.HttpHeaderContentLength, "4")
+			w.Header().Set(base.HttpHeaderContentDisposition, `attachment; filename="file.bin"`)
+			_, _ = w.Write([]byte("data"))
+		}),
+	})
+
+	fetcher := buildFetcher()
+	defer fetcher.Pause()
+	err := fetcher.Resolve(&base.Request{
+		URL:            server.URL + "/file.bin",
+		SkipVerifyCert: true,
+	}, &base.Options{
+		Name: test.DownloadName,
+		Path: test.Dir,
+	})
+	if err != nil {
+		t.Fatalf("Resolve() error = %v", err)
+	}
+	if fetcher.meta.Res == nil || len(fetcher.meta.Res.Files) != 1 {
+		t.Fatalf("resource = %+v, want one file", fetcher.meta.Res)
+	}
+	if fetcher.meta.Res.Files[0].Name != "file.bin" {
+		t.Fatalf("file name = %q, want file.bin", fetcher.meta.Res.Files[0].Name)
+	}
+}
+
+func TestFetcherSharesAndClearsImpersonationSession(t *testing.T) {
+	var requests atomic.Int32
+	server := httptest.NewServer(gohttp.HandlerFunc(func(w gohttp.ResponseWriter, request *gohttp.Request) {
+		requests.Add(1)
+		if request.Header.Get("Sec-Ch-Ua") == "" {
+			w.WriteHeader(gohttp.StatusForbidden)
+			return
+		}
+		w.WriteHeader(gohttp.StatusOK)
+	}))
+	defer server.Close()
+
+	fetcher := buildFetcher()
+	fetcher.meta.Req = &base.Request{URL: server.URL}
+
+	for range 2 {
+		client := fetcher.buildClient()
+		response, err := client.Get(server.URL)
+		if err != nil {
+			t.Fatalf("Get() error = %v", err)
+		}
+		response.Body.Close()
+		client.CloseIdleConnections()
+	}
+	if requests.Load() != 3 {
+		t.Fatalf("requests before completion = %d, want 3", requests.Load())
+	}
+
+	if err := fetcher.Close(); err != nil {
+		t.Fatalf("Close() error = %v", err)
+	}
+	client := fetcher.buildClient()
+	response, err := client.Get(server.URL)
+	if err != nil {
+		t.Fatalf("Get() after close error = %v", err)
+	}
+	response.Body.Close()
+	client.CloseIdleConnections()
+	if requests.Load() != 5 {
+		t.Fatalf("requests after close = %d, want 5", requests.Load())
+	}
+}
+
 func testResolve(startTestServer func() net.Listener, path string, t *testing.T, wantFn func(error) (*base.Resource, error)) {
 	listener := startTestServer()
 	defer listener.Close()

internal/protocol/http/helper.go

@@ -16,6 +16,7 @@ import (
 	"time"
 	"unicode/utf8"
 
+	"github.com/GopeedLab/gopeed/internal/httpclient"
 	"github.com/GopeedLab/gopeed/pkg/base"
 	fhttp "github.com/GopeedLab/gopeed/pkg/protocol/http"
 	"github.com/GopeedLab/gopeed/pkg/util"
@@ -216,21 +217,30 @@ func (f *Fetcher) buildFastFailClient() *http.Client {
 
 // buildClientWithTimeout creates an HTTP client with the specified connection timeout.
 func (f *Fetcher) buildClientWithTimeout(timeout time.Duration) *http.Client {
-	transport := &http.Transport{
-		DialContext: (&net.Dialer{
-			Timeout: timeout,
-		}).DialContext,
-		Proxy: f.ctl.GetProxy(f.meta.Req.Proxy),
-		TLSClientConfig: &tls.Config{
-			InsecureSkipVerify: f.meta.Req.SkipVerifyCert,
-		},
-		TLSHandshakeTimeout: timeout,
-	}
 	jar, _ := cookiejar.New(nil)
-	return &http.Client{
-		Transport: transport,
-		Jar:       jar,
+	client, err := httpclient.NewClient(httpclient.Options{
+		Client: httpclient.ClientOptions{
+			Jar: jar,
+		},
+		Transport: httpclient.TransportOptions{
+			DialContext: (&net.Dialer{
+				Timeout: timeout,
+			}).DialContext,
+			Proxy: f.ctl.GetProxy(f.meta.Req.Proxy),
+			TLSClientConfig: &tls.Config{
+				InsecureSkipVerify: f.meta.Req.SkipVerifyCert,
+			},
+			TLSHandshakeTimeout: timeout,
+		},
+		Impersonation: httpclient.ImpersonationOptions{
+			Mode:    httpclient.ImpersonationAuto,
+			Session: f.impersonationSession,
+		},
+	})
+	if err != nil {
+		panic(fmt.Sprintf("build HTTP client: %v", err))
 	}
+	return client
 }
 
 // ============================================================================