mirror of
https://github.com/GopeedLab/gopeed.git
synced 2026-11-09 10:00:27 +00:00
feat: HTTP add TLS fingerprint fallback (#1444)
This commit is contained in:
68 files changed
+7243
-1604
No files matched your search
@@ -6,21 +6,11 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/GopeedLab/gopeed/pkg/download/engine/inject/file"
|
||||
"github.com/GopeedLab/gopeed/pkg/download/engine/inject/formdata"
|
||||
"github.com/GopeedLab/gopeed/pkg/download/engine/inject/xhr"
|
||||
"github.com/GopeedLab/gopeed/pkg/download/engine/util"
|
||||
"github.com/dop251/goja"
|
||||
"github.com/dop251/goja_nodejs/eventloop"
|
||||
"github.com/imroc/req/v3"
|
||||
)
|
||||
|
||||
//go:embed stream.js
|
||||
@@ -29,8 +19,6 @@ var script string
|
||||
type Config struct {
|
||||
CreateObjectURL func(opts *ObjectURLOptions, open ObjectURLOpener) (string, error)
|
||||
RevokeObjectURL func(url string) error
|
||||
ProxyHandler func(r *http.Request) (*url.URL, error)
|
||||
RegisterCleanup func(cleanup func())
|
||||
}
|
||||
|
||||
type ObjectURLOptions struct {
|
||||
@@ -96,69 +84,6 @@ func Enable(runtime *goja.Runtime, loop *eventloop.EventLoop, cfg *Config) error
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
fetchRegistry := newFetchRegistry()
|
||||
if cfg.RegisterCleanup != nil {
|
||||
cfg.RegisterCleanup(fetchRegistry.CloseAll)
|
||||
}
|
||||
if err := runtime.Set("__gopeed_fetch_open", func(call goja.FunctionCall) goja.Value {
|
||||
reqMeta, err := exportFetchRequest(runtime, call.Argument(0))
|
||||
if err != nil {
|
||||
panic(runtime.NewGoError(err))
|
||||
}
|
||||
fingerprint := util.SafeGet[string](runtime, xhr.FingerprintMagicKey)
|
||||
promise, resolve, reject := runtime.NewPromise()
|
||||
go func() {
|
||||
meta, err := fetchRegistry.Open(fingerprint, cfg.ProxyHandler, reqMeta)
|
||||
ok := loop.RunOnLoop(func(runtime *goja.Runtime) {
|
||||
if err != nil {
|
||||
reject(runtime.NewGoError(err))
|
||||
return
|
||||
}
|
||||
resolve(runtime.ToValue(meta))
|
||||
})
|
||||
if !ok && meta != nil {
|
||||
fetchRegistry.Close(meta.ID)
|
||||
}
|
||||
}()
|
||||
return runtime.ToValue(promise)
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := runtime.Set("__gopeed_fetch_read", func(call goja.FunctionCall) goja.Value {
|
||||
id := call.Argument(0).String()
|
||||
chunkSize := int(call.Argument(1).ToInteger())
|
||||
promise, resolve, reject := runtime.NewPromise()
|
||||
go func() {
|
||||
chunk, done, err := fetchRegistry.Read(id, chunkSize)
|
||||
ok := loop.RunOnLoop(func(runtime *goja.Runtime) {
|
||||
if err != nil {
|
||||
reject(runtime.NewGoError(err))
|
||||
return
|
||||
}
|
||||
if done {
|
||||
resolve(goja.Null())
|
||||
return
|
||||
}
|
||||
resolve(runtime.ToValue(runtime.NewArrayBuffer(chunk)))
|
||||
})
|
||||
if !ok {
|
||||
fetchRegistry.Close(id)
|
||||
}
|
||||
}()
|
||||
return runtime.ToValue(promise)
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := runtime.Set("__gopeed_fetch_close", func(id string) {
|
||||
fetchRegistry.Close(id)
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := runtime.Set("__gopeed_fetch_abort", func(id string, reason string) {
|
||||
fetchRegistry.Abort(id, reason)
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := runtime.Set("__gopeed_blob_pipe_chunk", func(call goja.FunctionCall) goja.Value {
|
||||
id := call.Argument(0).String()
|
||||
ok := blobPipes.Push(id, exportBytes(call.Argument(1)))
|
||||
@@ -550,441 +475,3 @@ func exportJSError(value goja.Value) error {
|
||||
}
|
||||
return errors.New(stack)
|
||||
}
|
||||
|
||||
type fetchRegistry struct {
|
||||
mu sync.Mutex
|
||||
streams map[string]*fetchStream
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
closed bool
|
||||
}
|
||||
|
||||
type fetchStream struct {
|
||||
body io.ReadCloser
|
||||
cancel context.CancelFunc
|
||||
closeOnce sync.Once
|
||||
ctx context.Context
|
||||
ch chan fetchChunk
|
||||
readMu sync.Mutex
|
||||
pending []byte
|
||||
}
|
||||
|
||||
type fetchChunk struct {
|
||||
data []byte
|
||||
err error
|
||||
}
|
||||
|
||||
type fetchRequest struct {
|
||||
URL string
|
||||
Method string
|
||||
Headers [][2]string
|
||||
Body any
|
||||
Redirect string
|
||||
Credentials string
|
||||
}
|
||||
|
||||
type fetchFormDataEntry struct {
|
||||
name string
|
||||
value any
|
||||
}
|
||||
|
||||
type fetchFormDataSnapshot struct {
|
||||
entries []fetchFormDataEntry
|
||||
}
|
||||
|
||||
type fetchOpenMeta struct {
|
||||
ID string `json:"id"`
|
||||
Status int `json:"status"`
|
||||
StatusText string `json:"statusText"`
|
||||
URL string `json:"url"`
|
||||
Headers [][2]string `json:"headers"`
|
||||
}
|
||||
|
||||
func newFetchRegistry() *fetchRegistry {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
return &fetchRegistry{
|
||||
streams: make(map[string]*fetchStream),
|
||||
ctx: ctx,
|
||||
cancel: cancel,
|
||||
}
|
||||
}
|
||||
|
||||
func exportFetchRequest(runtime *goja.Runtime, value goja.Value) (*fetchRequest, error) {
|
||||
obj := value.ToObject(runtime)
|
||||
if obj == nil {
|
||||
return nil, fmt.Errorf("invalid fetch request")
|
||||
}
|
||||
meta := &fetchRequest{
|
||||
URL: obj.Get("url").String(),
|
||||
Method: obj.Get("method").String(),
|
||||
Redirect: obj.Get("redirect").String(),
|
||||
Credentials: obj.Get("credentials").String(),
|
||||
}
|
||||
if meta.Method == "" {
|
||||
meta.Method = http.MethodGet
|
||||
}
|
||||
if headersVal := obj.Get("headers"); headersVal != nil && !goja.IsUndefined(headersVal) && !goja.IsNull(headersVal) {
|
||||
if exported, ok := headersVal.Export().([]any); ok {
|
||||
for _, item := range exported {
|
||||
pair, ok := item.([]any)
|
||||
if !ok || len(pair) != 2 {
|
||||
continue
|
||||
}
|
||||
meta.Headers = append(meta.Headers, [2]string{fmt.Sprint(pair[0]), fmt.Sprint(pair[1])})
|
||||
}
|
||||
}
|
||||
}
|
||||
bodyVal := obj.Get("body")
|
||||
if bodyVal != nil && !goja.IsUndefined(bodyVal) && !goja.IsNull(bodyVal) {
|
||||
body, err := snapshotFetchBody(bodyVal.Export())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
meta.Body = body
|
||||
}
|
||||
return meta, nil
|
||||
}
|
||||
|
||||
// snapshotFetchBody runs on the JavaScript event loop before fetch work is
|
||||
// handed to a background goroutine. Values exported by goja may still share
|
||||
// mutable storage with JavaScript (notably ArrayBuffer and FormData), so the
|
||||
// background request must only retain immutable Go-owned data.
|
||||
func snapshotFetchBody(body any) (any, error) {
|
||||
switch v := body.(type) {
|
||||
case nil, string:
|
||||
return v, nil
|
||||
case []byte:
|
||||
return append([]byte(nil), v...), nil
|
||||
case goja.ArrayBuffer:
|
||||
return append([]byte(nil), v.Bytes()...), nil
|
||||
case *file.File:
|
||||
return snapshotFetchFile(v), nil
|
||||
case *formdata.FormData:
|
||||
entries := v.Entries()
|
||||
snapshot := &fetchFormDataSnapshot{
|
||||
entries: make([]fetchFormDataEntry, 0, len(entries)),
|
||||
}
|
||||
for _, entry := range entries {
|
||||
pair, ok := entry.([]any)
|
||||
if !ok || len(pair) != 2 {
|
||||
return nil, fmt.Errorf("invalid FormData entry %T", entry)
|
||||
}
|
||||
name, ok := pair[0].(string)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("invalid FormData field name %T", pair[0])
|
||||
}
|
||||
value := pair[1]
|
||||
if formFile, ok := value.(*file.File); ok {
|
||||
value = snapshotFetchFile(formFile)
|
||||
}
|
||||
snapshot.entries = append(snapshot.entries, fetchFormDataEntry{
|
||||
name: name,
|
||||
value: value,
|
||||
})
|
||||
}
|
||||
return snapshot, nil
|
||||
default:
|
||||
if typed, ok := v.(interface{ Bytes() []byte }); ok {
|
||||
return append([]byte(nil), typed.Bytes()...), nil
|
||||
}
|
||||
return v, nil
|
||||
}
|
||||
}
|
||||
|
||||
func snapshotFetchFile(src *file.File) *file.File {
|
||||
if src == nil {
|
||||
return nil
|
||||
}
|
||||
return &file.File{
|
||||
Reader: src.Reader,
|
||||
Closer: src.Closer,
|
||||
Name: src.Name,
|
||||
Size: src.Size,
|
||||
}
|
||||
}
|
||||
|
||||
func (r *fetchRegistry) Open(fingerprint string, proxyHandler func(r *http.Request) (*url.URL, error), reqMeta *fetchRequest) (*fetchOpenMeta, error) {
|
||||
r.mu.Lock()
|
||||
if r.closed {
|
||||
r.mu.Unlock()
|
||||
return nil, context.Canceled
|
||||
}
|
||||
registryCtx := r.ctx
|
||||
r.mu.Unlock()
|
||||
|
||||
client := req.C()
|
||||
if proxyHandler != nil {
|
||||
client.SetProxy(proxyHandler)
|
||||
}
|
||||
setFetchFingerprint(client, fingerprint)
|
||||
contentType, body, err := buildFetchBody(reqMeta.Body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ctx, cancel := context.WithCancel(registryCtx)
|
||||
keepCancel := false
|
||||
defer func() {
|
||||
if !keepCancel {
|
||||
cancel()
|
||||
}
|
||||
}()
|
||||
reqBuilder := client.R()
|
||||
reqBuilder.SetContext(ctx)
|
||||
reqBuilder.DisableAutoReadResponse()
|
||||
for _, header := range reqMeta.Headers {
|
||||
reqBuilder.SetHeader(header[0], header[1])
|
||||
}
|
||||
if body != nil && reqMeta.Method != http.MethodGet && reqMeta.Method != http.MethodHead {
|
||||
reqBuilder.SetBody(body)
|
||||
if contentType != "" && !hasHeader(reqMeta.Headers, "Content-Type") {
|
||||
reqBuilder.SetHeader("Content-Type", contentType)
|
||||
}
|
||||
}
|
||||
client.SetRedirectPolicy(func(req *http.Request, via []*http.Request) error {
|
||||
switch reqMeta.Redirect {
|
||||
case "manual":
|
||||
return http.ErrUseLastResponse
|
||||
case "error":
|
||||
return fmt.Errorf("redirect failed")
|
||||
default:
|
||||
if len(via) > 20 {
|
||||
return fmt.Errorf("too many redirects")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
})
|
||||
resp, err := reqBuilder.Send(reqMeta.Method, reqMeta.URL)
|
||||
if err != nil {
|
||||
var ne net.Error
|
||||
if errorsAsTimeout(err, &ne) {
|
||||
return nil, fmt.Errorf("Network request timed out")
|
||||
}
|
||||
return nil, fmt.Errorf("Network request failed: %w", err)
|
||||
}
|
||||
id := fmt.Sprintf("%d", time.Now().UnixNano())
|
||||
meta := &fetchOpenMeta{
|
||||
ID: id,
|
||||
Status: resp.StatusCode,
|
||||
StatusText: resp.Status,
|
||||
URL: reqMeta.URL,
|
||||
}
|
||||
if resp.Response != nil && resp.Response.Request != nil && resp.Response.Request.URL != nil {
|
||||
responseURL := *resp.Response.Request.URL
|
||||
responseURL.Fragment = ""
|
||||
meta.URL = responseURL.String()
|
||||
}
|
||||
for key, values := range resp.Header {
|
||||
meta.Headers = append(meta.Headers, [2]string{key, strings.Join(values, ", ")})
|
||||
}
|
||||
bodyCloser := io.NopCloser(strings.NewReader(""))
|
||||
if resp.Response != nil && resp.Response.Body != nil {
|
||||
bodyCloser = resp.Response.Body
|
||||
}
|
||||
stream := &fetchStream{
|
||||
body: bodyCloser,
|
||||
cancel: cancel,
|
||||
ctx: ctx,
|
||||
ch: make(chan fetchChunk, 8),
|
||||
}
|
||||
r.mu.Lock()
|
||||
if r.closed {
|
||||
r.mu.Unlock()
|
||||
closeFetchStream(stream)
|
||||
return nil, context.Canceled
|
||||
}
|
||||
r.streams[id] = stream
|
||||
r.mu.Unlock()
|
||||
keepCancel = true
|
||||
go stream.readLoop()
|
||||
return meta, nil
|
||||
}
|
||||
|
||||
func (r *fetchRegistry) Read(id string, chunkSize int) ([]byte, bool, error) {
|
||||
stream := r.get(id)
|
||||
if stream == nil {
|
||||
return nil, true, nil
|
||||
}
|
||||
stream.readMu.Lock()
|
||||
defer stream.readMu.Unlock()
|
||||
if len(stream.pending) > 0 {
|
||||
n := len(stream.pending)
|
||||
if chunkSize > 0 && n > chunkSize {
|
||||
n = chunkSize
|
||||
}
|
||||
chunk := append([]byte(nil), stream.pending[:n]...)
|
||||
stream.pending = stream.pending[n:]
|
||||
return chunk, false, nil
|
||||
}
|
||||
select {
|
||||
case item, ok := <-stream.ch:
|
||||
if !ok {
|
||||
r.Close(id)
|
||||
return nil, true, nil
|
||||
}
|
||||
if item.err != nil {
|
||||
r.Close(id)
|
||||
return nil, false, item.err
|
||||
}
|
||||
if chunkSize > 0 && len(item.data) > chunkSize {
|
||||
head := append([]byte(nil), item.data[:chunkSize]...)
|
||||
stream.pending = append(stream.pending[:0], item.data[chunkSize:]...)
|
||||
return head, false, nil
|
||||
}
|
||||
return item.data, false, nil
|
||||
case <-stream.ctx.Done():
|
||||
return nil, false, stream.ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
func (s *fetchStream) readLoop() {
|
||||
defer close(s.ch)
|
||||
buf := make([]byte, 64*1024)
|
||||
for {
|
||||
n, err := s.body.Read(buf)
|
||||
if n > 0 {
|
||||
chunk := append([]byte(nil), buf[:n]...)
|
||||
select {
|
||||
case s.ch <- fetchChunk{data: chunk}:
|
||||
case <-s.ctx.Done():
|
||||
return
|
||||
}
|
||||
}
|
||||
if err == io.EOF {
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
select {
|
||||
case s.ch <- fetchChunk{err: err}:
|
||||
case <-s.ctx.Done():
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (r *fetchRegistry) Close(id string) {
|
||||
r.mu.Lock()
|
||||
stream := r.streams[id]
|
||||
delete(r.streams, id)
|
||||
r.mu.Unlock()
|
||||
closeFetchStream(stream)
|
||||
}
|
||||
|
||||
// CloseAll permanently closes the registry and cancels both active and
|
||||
// in-flight fetches. It is safe to call more than once.
|
||||
func (r *fetchRegistry) CloseAll() {
|
||||
if r == nil {
|
||||
return
|
||||
}
|
||||
r.mu.Lock()
|
||||
if r.closed {
|
||||
r.mu.Unlock()
|
||||
return
|
||||
}
|
||||
r.closed = true
|
||||
if r.cancel != nil {
|
||||
r.cancel()
|
||||
}
|
||||
streams := r.streams
|
||||
r.streams = make(map[string]*fetchStream)
|
||||
r.mu.Unlock()
|
||||
|
||||
for _, stream := range streams {
|
||||
closeFetchStream(stream)
|
||||
}
|
||||
}
|
||||
|
||||
func closeFetchStream(stream *fetchStream) {
|
||||
if stream == nil {
|
||||
return
|
||||
}
|
||||
stream.closeOnce.Do(func() {
|
||||
if stream.cancel != nil {
|
||||
stream.cancel()
|
||||
}
|
||||
_ = stream.body.Close()
|
||||
})
|
||||
}
|
||||
|
||||
func (r *fetchRegistry) Abort(id string, _ string) {
|
||||
r.Close(id)
|
||||
}
|
||||
|
||||
func (r *fetchRegistry) get(id string) *fetchStream {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
return r.streams[id]
|
||||
}
|
||||
|
||||
func buildFetchBody(body any) (string, any, error) {
|
||||
switch v := body.(type) {
|
||||
case nil:
|
||||
return "", nil, nil
|
||||
case string:
|
||||
return "text/plain;charset=UTF-8", v, nil
|
||||
case []byte:
|
||||
return "application/octet-stream", v, nil
|
||||
case goja.ArrayBuffer:
|
||||
return "application/octet-stream", v.Bytes(), nil
|
||||
case *file.File:
|
||||
return "application/octet-stream", v.Reader, nil
|
||||
case *fetchFormDataSnapshot:
|
||||
pr, pw := io.Pipe()
|
||||
mw := xhr.NewMultipart(pw)
|
||||
for _, entry := range v.entries {
|
||||
switch vv := entry.value.(type) {
|
||||
case string:
|
||||
if err := mw.WriteField(entry.name, vv); err != nil {
|
||||
_ = pw.CloseWithError(err)
|
||||
return "", nil, err
|
||||
}
|
||||
case *file.File:
|
||||
if err := mw.WriteFile(entry.name, vv); err != nil {
|
||||
_ = pw.CloseWithError(err)
|
||||
return "", nil, err
|
||||
}
|
||||
}
|
||||
}
|
||||
go func() {
|
||||
if err := mw.Send(); err != nil {
|
||||
_ = pw.CloseWithError(err)
|
||||
return
|
||||
}
|
||||
if err := mw.Close(); err != nil {
|
||||
_ = pw.CloseWithError(err)
|
||||
return
|
||||
}
|
||||
_ = pw.Close()
|
||||
}()
|
||||
return mw.FormDataContentType(), pr, nil
|
||||
default:
|
||||
if typed, ok := v.(interface{ Bytes() []byte }); ok {
|
||||
return "application/octet-stream", typed.Bytes(), nil
|
||||
}
|
||||
return "", fmt.Sprint(v), nil
|
||||
}
|
||||
}
|
||||
|
||||
func hasHeader(headers [][2]string, key string) bool {
|
||||
for _, header := range headers {
|
||||
if strings.EqualFold(header[0], key) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func setFetchFingerprint(client *req.Client, fingerprint string) {
|
||||
switch fingerprint {
|
||||
case "chrome":
|
||||
client.ImpersonateChrome()
|
||||
case "firefox":
|
||||
client.ImpersonateFirefox()
|
||||
case "safari":
|
||||
client.ImpersonateSafari()
|
||||
}
|
||||
}
|
||||
|
||||
func errorsAsTimeout(err error, target *net.Error) bool {
|
||||
return errors.As(err, target) && (*target).Timeout()
|
||||
}
|
||||
Reference in new issue
Block a user