feat: HTTP add TLS fingerprint fallback (#1444)

This commit is contained in:
Levi authored and GitHub committed 2026-08-20 20:21:02 +08:00
1 parent 99924296ef
commit cfdf87c3b8
68 files changed
+7243 -1604

No files matched your search

+6 -473
View File
@@ -1,482 +1,15 @@
package xhr
import (
"bytes"
"errors"
"io"
"mime/multipart"
"net"
"net/http"
"net/url"
"strings"
"time"
_ "embed"
"github.com/GopeedLab/gopeed/pkg/download/engine/inject/file"
"github.com/GopeedLab/gopeed/pkg/download/engine/inject/formdata"
"github.com/GopeedLab/gopeed/pkg/download/engine/util"
"github.com/dop251/goja"
"github.com/imroc/req/v3"
)
const (
eventLoad = "load"
eventReadystatechange = "readystatechange"
eventProgress = "progress"
eventAbort = "abort"
eventError = "error"
eventTimeout = "timeout"
)
//go:embed xhr.js
var script string
const (
redirectError = "error"
redirectFollow = "follow"
redirectManual = "manual"
)
type ProgressEvent struct {
Type string `json:"type"`
LengthComputable bool `json:"lengthComputable"`
Loaded int64 `json:"loaded"`
Total int64 `json:"total"`
}
type EventProp struct {
eventListeners map[string]func(event *ProgressEvent)
Onload func(event *ProgressEvent) `json:"onload"`
Onprogress func(event *ProgressEvent) `json:"onprogress"`
Onabort func(event *ProgressEvent) `json:"onabort"`
Onerror func(event *ProgressEvent) `json:"onerror"`
Ontimeout func(event *ProgressEvent) `json:"ontimeout"`
}
func (ep *EventProp) AddEventListener(event string, cb func(event *ProgressEvent)) {
ep.eventListeners[event] = cb
}
func (ep *EventProp) RemoveEventListener(event string) {
delete(ep.eventListeners, event)
}
func (ep *EventProp) callOnload() {
event := &ProgressEvent{
Type: eventLoad,
LengthComputable: false,
}
if ep.Onload != nil {
ep.Onload(event)
}
ep.callEventListener(event)
}
func (ep *EventProp) callOnprogress(loaded, total int64) {
event := &ProgressEvent{
Type: eventProgress,
LengthComputable: true,
Loaded: loaded,
Total: total,
}
if ep.Onprogress != nil {
ep.Onprogress(event)
}
ep.callEventListener(event)
}
func (ep *EventProp) callOnabort() {
event := &ProgressEvent{
Type: eventAbort,
LengthComputable: false,
}
if ep.Onabort != nil {
ep.Onabort(event)
}
ep.callEventListener(event)
}
func (ep *EventProp) callOnerror() {
event := &ProgressEvent{
Type: eventError,
LengthComputable: false,
}
if ep.Onerror != nil {
ep.Onerror(event)
}
ep.callEventListener(event)
}
func (ep *EventProp) callOntimeout() {
event := &ProgressEvent{
Type: eventTimeout,
LengthComputable: false,
}
if ep.Ontimeout != nil {
ep.Ontimeout(event)
}
ep.callEventListener(event)
}
func (ep *EventProp) callEventListener(event *ProgressEvent) {
if cb, ok := ep.eventListeners[event.Type]; ok {
cb(event)
}
}
type XMLHttpRequestUpload struct {
*EventProp
}
type XMLHttpRequest struct {
method string
url string
requestHeaders http.Header
responseHeaders http.Header
aborted bool
client *req.Client
fingerprint string
WithCredentials bool `json:"withCredentials"`
Upload *XMLHttpRequestUpload `json:"upload"`
Timeout int `json:"timeout"`
ReadyState int `json:"readyState"`
Status int `json:"status"`
StatusText string `json:"statusText"`
Response string `json:"response"`
ResponseText string `json:"responseText"`
// https://developer.mozilla.org/zh-CN/docs/Web/API/XMLHttpRequest/responseURL
// https://xhr.spec.whatwg.org/#the-responseurl-attribute
ResponseUrl string `json:"responseURL"`
// extend fetch redirect
// https://developer.mozilla.org/en-US/docs/Web/API/RequestInit#redirect
// https://fetch.spec.whatwg.org/#concept-request-redirect-mode
Redirect string `json:"redirect"`
*EventProp
Onreadystatechange func(event *ProgressEvent) `json:"onreadystatechange"`
}
func (xhr *XMLHttpRequest) Open(method, url string) {
xhr.method = method
xhr.url = url
xhr.requestHeaders = make(http.Header)
xhr.responseHeaders = make(http.Header)
xhr.doReadystatechange(1)
}
func (xhr *XMLHttpRequest) SetRequestHeader(key, value string) {
xhr.requestHeaders.Add(key, value)
}
func (xhr *XMLHttpRequest) Send(data goja.Value) {
setFingerprint(xhr.client, xhr.fingerprint)
d := xhr.parseData(data)
var (
contentType string
contentLength int64
isStringBody bool
)
// Create request using req library
reqBuilder := xhr.client.R()
// Set headers first
if xhr.requestHeaders != nil {
for key, values := range xhr.requestHeaders {
if len(values) > 0 {
// Merge multiple values with comma separator (HTTP standard)
mergedValue := strings.Join(values, ", ")
reqBuilder.SetHeader(key, mergedValue)
}
}
}
// Handle request body
if d != nil && xhr.method != "GET" && xhr.method != "HEAD" {
switch v := d.(type) {
case string:
reqBuilder.SetBody(v)
contentType = "text/plain;charset=UTF-8"
contentLength = int64(len(v))
isStringBody = true
case *file.File:
reqBuilder.SetBody(v.Reader)
contentType = "application/octet-stream"
contentLength = v.Size
case *formdata.FormData:
pr, pw := io.Pipe()
mw := NewMultipart(pw)
for _, e := range v.Entries() {
arr := e.([]any)
k := arr[0].(string)
v := arr[1]
switch v := v.(type) {
case string:
mw.WriteField(k, v)
case *file.File:
mw.WriteFile(k, v)
}
}
go func() {
defer pw.Close()
defer mw.Close()
mw.Send()
}()
reqBuilder.SetBody(pr)
contentType = mw.FormDataContentType()
contentLength = mw.Size()
}
}
// Only string body can specify Content-Type header by user
if contentType != "" && (!isStringBody || xhr.requestHeaders.Get("Content-Type") == "") {
reqBuilder.SetHeader("Content-Type", contentType)
}
// Set timeout
if xhr.Timeout > 0 {
xhr.client.SetTimeout(time.Duration(xhr.Timeout) * time.Millisecond)
}
// Configure redirect behavior
xhr.client.SetRedirectPolicy(func(req *http.Request, via []*http.Request) error {
if xhr.Redirect == redirectManual {
return http.ErrUseLastResponse
}
if xhr.Redirect == redirectError {
return errors.New("redirect failed")
}
if len(via) > 20 {
return errors.New("too many redirects")
}
return nil
})
// Execute request
resp, err := reqBuilder.Send(xhr.method, xhr.url)
if err != nil {
// handle timeout error
var ne net.Error
if errors.As(err, &ne) && ne.Timeout() {
if xhr.Timeout > 0 {
xhr.Upload.callOntimeout()
xhr.callOntimeout()
}
return
}
xhr.Upload.callOnerror()
xhr.callOnerror()
return
}
xhr.Upload.callOnprogress(contentLength, contentLength)
if !xhr.aborted {
xhr.Upload.callOnload()
}
// Set response URL (final URL after redirects)
if resp.Response != nil && resp.Response.Request != nil && resp.Response.Request.URL != nil {
responseUrl := resp.Response.Request.URL
responseUrl.Fragment = ""
xhr.ResponseUrl = responseUrl.String()
} else {
xhr.ResponseUrl = xhr.url
}
// Set response headers
xhr.responseHeaders = resp.Header
xhr.Status = resp.StatusCode
xhr.StatusText = resp.Status
xhr.doReadystatechange(2)
bodyBytes := resp.Bytes()
xhr.doReadystatechange(3)
xhr.Response = string(bodyBytes)
xhr.ResponseText = xhr.Response
xhr.doReadystatechange(4)
respBodyLen := int64(len(bodyBytes))
xhr.callOnprogress(respBodyLen, respBodyLen)
if !xhr.aborted {
xhr.callOnload()
}
}
func (xhr *XMLHttpRequest) Abort() {
xhr.doReadystatechange(0)
xhr.aborted = true
xhr.Upload.callOnabort()
xhr.callOnabort()
}
func (xhr *XMLHttpRequest) GetResponseHeader(key string) string {
if xhr.responseHeaders == nil {
return ""
}
return strings.Join(xhr.responseHeaders.Values(key), ", ")
}
func (xhr *XMLHttpRequest) GetAllResponseHeaders() string {
var buf bytes.Buffer
for k, v := range xhr.responseHeaders {
buf.WriteString(k)
buf.WriteString(": ")
buf.WriteString(strings.Join(v, ", "))
buf.WriteString("\r\n")
}
return buf.String()
}
func (xhr *XMLHttpRequest) callOnreadystatechange() {
event := &ProgressEvent{
Type: eventReadystatechange,
LengthComputable: false,
}
if xhr.Onreadystatechange != nil {
xhr.Onreadystatechange(event)
}
xhr.callEventListener(event)
}
func (xhr *XMLHttpRequest) doReadystatechange(state int) {
if xhr.aborted {
return
}
xhr.ReadyState = state
xhr.callOnreadystatechange()
}
// parse js data to go struct
func (xhr *XMLHttpRequest) parseData(data goja.Value) any {
// check if data is null or undefined
if data == nil || goja.IsNull(data) || goja.IsUndefined(data) || goja.IsNaN(data) {
return nil
}
// check if data is File
f, ok := data.Export().(*file.File)
if ok {
return f
}
// check if data is FormData
fd, ok := data.Export().(*formdata.FormData)
if ok {
return fd
}
// otherwise, return data as string
return data.String()
}
func Enable(runtime *goja.Runtime, proxyHandler func(r *http.Request) (*url.URL, error)) error {
progressEvent := runtime.ToValue(func(call goja.ConstructorCall) *goja.Object {
if len(call.Arguments) < 1 {
util.ThrowTypeError(runtime, "Failed to construct 'ProgressEvent': 1 argument required, but only 0 present.")
}
instance := &ProgressEvent{
Type: call.Argument(0).String(),
}
instanceValue := runtime.ToValue(instance).(*goja.Object)
instanceValue.SetPrototype(call.This.Prototype())
return instanceValue
})
xhr := runtime.ToValue(func(call goja.ConstructorCall) *goja.Object {
// Create req client with proxy support
client := req.C()
if proxyHandler != nil {
client.SetProxy(proxyHandler)
}
instance := &XMLHttpRequest{
client: client,
fingerprint: util.SafeGet[string](runtime, FingerprintMagicKey),
Upload: &XMLHttpRequestUpload{
EventProp: &EventProp{
eventListeners: make(map[string]func(event *ProgressEvent)),
},
},
EventProp: &EventProp{
eventListeners: make(map[string]func(event *ProgressEvent)),
},
}
instanceValue := runtime.ToValue(instance).(*goja.Object)
instanceValue.SetPrototype(call.This.Prototype())
return instanceValue
})
runtime.Set("__gopeed_setFingerprint", func(fingerprint string) {
runtime.Set(FingerprintMagicKey, fingerprint)
})
if err := runtime.Set("ProgressEvent", progressEvent); err != nil {
return err
}
if err := runtime.Set("XMLHttpRequest", xhr); err != nil {
return err
}
return nil
}
// Wrap multipart.Writer and stat content length
type multipartWrapper struct {
statBuffer *bytes.Buffer
statWriter *multipart.Writer
writer *multipart.Writer
fields map[string]any
}
func NewMultipart(w io.Writer) *multipartWrapper {
var buf bytes.Buffer
return &multipartWrapper{
statBuffer: &buf,
statWriter: multipart.NewWriter(&buf),
writer: multipart.NewWriter(w),
fields: make(map[string]any),
}
}
func (w *multipartWrapper) WriteField(fieldname string, value string) error {
w.fields[fieldname] = value
return w.statWriter.WriteField(fieldname, value)
}
func (w *multipartWrapper) WriteFile(fieldname string, file *file.File) error {
w.fields[fieldname] = file
_, err := w.statWriter.CreateFormFile(fieldname, file.Name)
if err != nil {
return err
}
return nil
}
func (w *multipartWrapper) Size() int64 {
w.statWriter.Close()
size := int64(w.statBuffer.Len())
for _, v := range w.fields {
switch v := v.(type) {
case *file.File:
size += v.Size
}
}
return size
}
func (w *multipartWrapper) Send() error {
for k, v := range w.fields {
switch v := v.(type) {
case string:
if err := w.writer.WriteField(k, v); err != nil {
return err
}
case *file.File:
fw, err := w.writer.CreateFormFile(k, v.Name)
if err != nil {
return err
}
if _, err = io.Copy(fw, v); err != nil {
return err
}
}
}
return nil
}
func (w *multipartWrapper) FormDataContentType() string {
return w.writer.FormDataContentType()
}
func (w *multipartWrapper) Close() error {
return w.writer.Close()
func Enable(runtime *goja.Runtime) error {
_, err := runtime.RunString(script)
return err
}
@@ -1,24 +0,0 @@
package xhr
import "github.com/imroc/req/v3"
type Fingerprint string
const (
FingerprintMagicKey = "__gopeed_xhr_fingerprint"
fingerprintChrome = "chrome"
fingerprintFirefox = "firefox"
fingerprintSafari = "safari"
)
func setFingerprint(client *req.Client, fingerprint string) {
switch fingerprint {
case fingerprintChrome:
client.ImpersonateChrome()
case fingerprintFirefox:
client.ImpersonateFirefox()
case fingerprintSafari:
client.ImpersonateSafari()
}
}
+302
View File
@@ -0,0 +1,302 @@
(function () {
if (typeof globalThis.ProgressEvent === "undefined") {
globalThis.ProgressEvent = class ProgressEvent extends Event {
constructor(type, init) {
super(type);
init = init || {};
this.lengthComputable = Boolean(init.lengthComputable);
this.loaded = Number(init.loaded) || 0;
this.total = Number(init.total) || 0;
}
};
}
function createProgressEvent(type, loaded, total) {
const lengthComputable = Number.isFinite(total) && total >= 0;
return new ProgressEvent(type, {
lengthComputable,
loaded: loaded || 0,
total: lengthComputable ? total : 0,
});
}
function canonicalHeaderName(name) {
return String(name).toLowerCase().split("-").map(function (part) {
return part ? part.charAt(0).toUpperCase() + part.slice(1) : part;
}).join("-");
}
function bodySize(body) {
if (body == null) return 0;
if (typeof body === "string") return new TextEncoder().encode(body).byteLength;
if (body instanceof ArrayBuffer) return body.byteLength;
if (typeof ArrayBuffer !== "undefined" && ArrayBuffer.isView && ArrayBuffer.isView(body)) {
return body.byteLength;
}
if (typeof Blob !== "undefined" && body instanceof Blob) return Number(body.size) || 0;
if (body && Number.isFinite(Number(body.size))) return Number(body.size);
return 0;
}
class XMLHttpRequestUpload extends EventTarget {
constructor() {
super();
this.onloadstart = null;
this.onprogress = null;
this.onabort = null;
this.onerror = null;
this.onload = null;
this.ontimeout = null;
this.onloadend = null;
}
_emit(type, loaded, total) {
const event = createProgressEvent(type, loaded, total);
const handler = this["on" + type];
if (typeof handler === "function") handler.call(this, event);
this.dispatchEvent(event);
}
}
class XMLHttpRequest extends EventTarget {
constructor() {
super();
this.readyState = XMLHttpRequest.UNSENT;
this.response = "";
this.responseText = "";
this.responseType = "";
this.responseURL = "";
this.responseXML = null;
this.status = 0;
this.statusText = "";
this.timeout = 0;
this.withCredentials = false;
this.upload = new XMLHttpRequestUpload();
this.redirect = "follow";
this.onreadystatechange = null;
this.onloadstart = null;
this.onprogress = null;
this.onabort = null;
this.onerror = null;
this.onload = null;
this.ontimeout = null;
this.onloadend = null;
this._method = "GET";
this._url = "";
this._requestHeaders = new Headers();
this._responseHeaders = null;
this._controller = null;
this._sendStarted = false;
this._aborted = false;
this._timedOut = false;
this._timeoutID = null;
}
open(method, url, async) {
if (arguments.length < 2) {
throw new TypeError("XMLHttpRequest.open requires at least 2 arguments");
}
if (async === false) {
throw new DOMException("Synchronous XMLHttpRequest is not supported", "NotSupportedError");
}
this._method = String(method).toUpperCase();
this._url = String(url);
this._requestHeaders = new Headers();
this._responseHeaders = null;
this._sendStarted = false;
this._aborted = false;
this._timedOut = false;
this.response = "";
this.responseText = "";
this.responseURL = "";
this.status = 0;
this.statusText = "";
this._setReadyState(XMLHttpRequest.OPENED);
}
setRequestHeader(name, value) {
if (this.readyState !== XMLHttpRequest.OPENED || this._sendStarted) {
throw new DOMException("XMLHttpRequest is not opened", "InvalidStateError");
}
this._requestHeaders.append(name, value);
}
getResponseHeader(name) {
if (!this._responseHeaders || this.readyState < XMLHttpRequest.HEADERS_RECEIVED) return null;
return this._responseHeaders.get(name);
}
getAllResponseHeaders() {
if (!this._responseHeaders || this.readyState < XMLHttpRequest.HEADERS_RECEIVED) return "";
let result = "";
this._responseHeaders.forEach(function (value, name) {
result += canonicalHeaderName(name) + ": " + value + "\r\n";
});
return result;
}
overrideMimeType() {
// Kept as a compatibility no-op until MIME decoding is configurable.
}
send(body) {
if (this.readyState !== XMLHttpRequest.OPENED || this._sendStarted) {
throw new DOMException("XMLHttpRequest is not opened", "InvalidStateError");
}
this._sendStarted = true;
this._aborted = false;
this._timedOut = false;
this._controller = new AbortController();
const uploadTotal = bodySize(body);
this._emit("loadstart", 0, -1);
this.upload._emit("loadstart", 0, uploadTotal);
if (this.timeout > 0) {
this._timeoutID = setTimeout(() => {
this._timedOut = true;
this._controller.abort();
}, this.timeout);
}
const options = {
method: this._method,
headers: this._requestHeaders,
redirect: this.redirect || "follow",
credentials: this.withCredentials ? "include" : "same-origin",
signal: this._controller.signal,
};
if (this._method !== "GET" && this._method !== "HEAD" && body != null) {
options.body = body;
}
Promise.resolve().then(() => globalThis.fetch(this._url, options)).then(async (response) => {
if (this._aborted || this._timedOut) return;
this.upload._emit("progress", uploadTotal, uploadTotal);
this.upload._emit("load", uploadTotal, uploadTotal);
this.upload._emit("loadend", uploadTotal, uploadTotal);
this.status = response.status;
this.statusText = response.statusText;
this.responseURL = response.url || this._url;
this._responseHeaders = response.headers;
this._setReadyState(XMLHttpRequest.HEADERS_RECEIVED);
const contentLengthValue = response.headers.get("content-length");
const contentLength = contentLengthValue == null ? -1 : Number(contentLengthValue);
const chunks = [];
let loaded = 0;
if (response.body) {
const reader = response.body.getReader();
try {
while (true) {
const item = await reader.read();
if (item.done) break;
const chunk = item.value instanceof Uint8Array ? item.value : new Uint8Array(item.value);
chunks.push(chunk);
loaded += chunk.byteLength;
this._setReadyState(XMLHttpRequest.LOADING);
this._emit("progress", loaded, contentLength);
}
} finally {
if (typeof reader.releaseLock === "function") reader.releaseLock();
}
}
const bytes = new Uint8Array(loaded);
let offset = 0;
for (const chunk of chunks) {
bytes.set(chunk, offset);
offset += chunk.byteLength;
}
const text = new TextDecoder().decode(bytes);
switch (this.responseType) {
case "arraybuffer":
this.response = bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength);
break;
case "blob":
this.response = new Blob([bytes], { type: response.headers.get("content-type") || "" });
break;
case "json":
try {
this.response = text === "" ? null : JSON.parse(text);
} catch (_) {
this.response = null;
}
break;
case "":
case "text":
default:
this.response = text;
this.responseText = text;
break;
}
this._clearTimeout();
this._setReadyState(XMLHttpRequest.DONE);
this._emit("load", loaded, contentLength);
this._emit("loadend", loaded, contentLength);
}).catch((error) => {
this._clearTimeout();
if (this._timedOut) {
this.status = 0;
this._setReadyState(XMLHttpRequest.DONE);
this.upload._emit("timeout", 0, uploadTotal);
this._emit("timeout", 0, -1);
this._emit("loadend", 0, -1);
return;
}
if (this._aborted || (error && error.name === "AbortError")) {
this.status = 0;
this._setReadyState(XMLHttpRequest.DONE);
this.upload._emit("abort", 0, uploadTotal);
this._emit("abort", 0, -1);
this._emit("loadend", 0, -1);
return;
}
this.status = 0;
this._setReadyState(XMLHttpRequest.DONE);
this.upload._emit("error", 0, uploadTotal);
this._emit("error", 0, -1);
this._emit("loadend", 0, -1);
});
}
abort() {
if (!this._sendStarted || this.readyState === XMLHttpRequest.DONE) return;
this._aborted = true;
this._clearTimeout();
if (this._controller) this._controller.abort();
}
_clearTimeout() {
if (this._timeoutID != null) {
clearTimeout(this._timeoutID);
this._timeoutID = null;
}
}
_setReadyState(state) {
this.readyState = state;
this._emit("readystatechange", 0, -1);
}
_emit(type, loaded, total) {
const event = createProgressEvent(type, loaded, total);
const handler = this["on" + type];
if (typeof handler === "function") handler.call(this, event);
this.dispatchEvent(event);
}
}
XMLHttpRequest.UNSENT = 0;
XMLHttpRequest.OPENED = 1;
XMLHttpRequest.HEADERS_RECEIVED = 2;
XMLHttpRequest.LOADING = 3;
XMLHttpRequest.DONE = 4;
XMLHttpRequest.prototype.UNSENT = 0;
XMLHttpRequest.prototype.OPENED = 1;
XMLHttpRequest.prototype.HEADERS_RECEIVED = 2;
XMLHttpRequest.prototype.LOADING = 3;
XMLHttpRequest.prototype.DONE = 4;
globalThis.XMLHttpRequest = XMLHttpRequest;
})();