feat: support proxy (#284)

pkg/base/constants.go

@@ -19,7 +19,7 @@ const (
 	HttpHeaderContentLength      = "Content-Length"
 	HttpHeaderContentRange       = "Content-Range"
 	HttpHeaderContentDisposition = "Content-Disposition"
-	HttpHeaderUserAgent          = "User-Agent"
+	HttpHeaderUserAgent          = "Usr-Agent"
 
 	HttpHeaderRangeFormat = "bytes=%d-%d"
 )

pkg/download/downloader.go

@@ -215,6 +215,7 @@ func (d *Downloader) setupFetcher(fetcher fetcher.Fetcher) {
 	ctl.GetConfig = func(v any) bool {
 		return d.getProtocolConfig(fetcher.Name(), v)
 	}
+	ctl.ProxyUrl = d.cfg.ProxyUrl()
 	fetcher.Setup(ctl)
 }
 

pkg/download/downloader_test.go

@@ -84,6 +84,82 @@ func TestDownloader_Create(t *testing.T) {
 	}
 }
 
+func TestDownloader_CreateWithProxy(t *testing.T) {
+	// No proxy
+	doTestDownloaderCreateWithProxy(t, false, func(proxyCfg *DownloaderProxyConfig) *DownloaderProxyConfig {
+		return nil
+	})
+	// Disable proxy
+	doTestDownloaderCreateWithProxy(t, false, func(proxyCfg *DownloaderProxyConfig) *DownloaderProxyConfig {
+		proxyCfg.Enable = false
+		return proxyCfg
+	})
+	// Invalid proxy scheme
+	doTestDownloaderCreateWithProxy(t, false, func(proxyCfg *DownloaderProxyConfig) *DownloaderProxyConfig {
+		proxyCfg.Scheme = ""
+		return proxyCfg
+	})
+	// Invalid proxy host
+	doTestDownloaderCreateWithProxy(t, false, func(proxyCfg *DownloaderProxyConfig) *DownloaderProxyConfig {
+		proxyCfg.Host = ""
+		return proxyCfg
+	})
+	// Use proxy without auth
+	doTestDownloaderCreateWithProxy(t, false, func(proxyCfg *DownloaderProxyConfig) *DownloaderProxyConfig {
+		return proxyCfg
+	})
+	// Use proxy with auth
+	doTestDownloaderCreateWithProxy(t, true, func(proxyCfg *DownloaderProxyConfig) *DownloaderProxyConfig {
+		return proxyCfg
+	})
+}
+
+func doTestDownloaderCreateWithProxy(t *testing.T, auth bool, buildProxyConfig func(proxyCfg *DownloaderProxyConfig) *DownloaderProxyConfig) {
+	httpListener := test.StartTestFileServer()
+	defer httpListener.Close()
+	usr, pwd := "", ""
+	if auth {
+		usr, pwd = "admin", "123"
+	}
+	proxyListener := test.StartSocks5Server(usr, pwd)
+	defer proxyListener.Close()
+
+	downloader := NewDownloader(nil)
+	if err := downloader.Setup(); err != nil {
+		t.Fatal(err)
+	}
+	defer downloader.Clear()
+	downloader.cfg.DownloaderStoreConfig.Proxy = buildProxyConfig(&DownloaderProxyConfig{
+		Enable: true,
+		Scheme: "socks5",
+		Host:   proxyListener.Addr().String(),
+		Usr:    usr,
+		Pwd:    pwd,
+	})
+
+	req := &base.Request{
+		URL: "http://" + httpListener.Addr().String() + "/" + test.BuildName,
+	}
+	rr, err := downloader.Resolve(req)
+	if err != nil {
+		t.Fatal(err)
+	}
+	want := &base.Resource{
+		Size:  test.BuildSize,
+		Range: true,
+		Files: []*base.FileInfo{
+			{
+				Name: test.BuildName,
+				Path: "",
+				Size: test.BuildSize,
+			},
+		},
+	}
+	if !reflect.DeepEqual(want, rr.Res) {
+		t.Errorf("Resolve() got = %v, want %v", rr.Res, want)
+	}
+}
+
 func TestDownloader_CreateRename(t *testing.T) {
 	listener := test.StartTestFileServer()
 	defer listener.Close()

pkg/download/engine/engine.go

@@ -9,7 +9,8 @@ import (
 	"github.com/GopeedLab/gopeed/pkg/download/engine/inject/xhr"
 	"github.com/dop251/goja"
 	"github.com/dop251/goja_nodejs/eventloop"
-	"github.com/dop251/goja_nodejs/url"
+	gojaurl "github.com/dop251/goja_nodejs/url"
+	"net/url"
 	"time"
 )
 
@@ -107,7 +108,14 @@ func (e *Engine) Close() {
 	e.loop.Stop()
 }
 
-func NewEngine() *Engine {
+type Config struct {
+	ProxyURL *url.URL
+}
+
+func NewEngine(cfg *Config) *Engine {
+	if cfg == nil {
+		cfg = &Config{}
+	}
 	loop := eventloop.NewEventLoop()
 	engine := &Engine{
 		loop: loop,
@@ -116,14 +124,14 @@ func NewEngine() *Engine {
 		engine.Runtime = runtime
 		runtime.SetFieldNameMapper(goja.TagFieldNameMapper("json", true))
 		vm.Enable(runtime)
-		url.Enable(runtime)
+		gojaurl.Enable(runtime)
 		if err := file.Enable(runtime); err != nil {
 			return
 		}
 		if err := formdata.Enable(runtime); err != nil {
 			return
 		}
-		if err := xhr.Enable(runtime); err != nil {
+		if err := xhr.Enable(runtime, cfg.ProxyURL); err != nil {
 			return
 		}
 		if _, err := runtime.RunString(polyfillScript); err != nil {
@@ -143,7 +151,7 @@ func NewEngine() *Engine {
 }
 
 func Run(script string) (value any, err error) {
-	engine := NewEngine()
+	engine := NewEngine(nil)
 	return engine.RunString(script)
 }
 

pkg/download/engine/engine_test.go

@@ -8,6 +8,7 @@ import (
 	"fmt"
 	"github.com/GopeedLab/gopeed/internal/test"
 	"github.com/GopeedLab/gopeed/pkg/download/engine/inject/file"
+	"github.com/GopeedLab/gopeed/pkg/util"
 	"github.com/dop251/goja"
 	"io"
 	"net"
@@ -29,7 +30,7 @@ func TestPolyfill(t *testing.T) {
 func TestFetch(t *testing.T) {
 	server := startServer()
 	defer server.Close()
-	engine := NewEngine()
+	engine := NewEngine(nil)
 	if _, err := engine.RunString(fmt.Sprintf("var host = 'http://%s';", server.Addr().String())); err != nil {
 		t.Fatal(err)
 	}
@@ -192,8 +193,40 @@ function testTimeout(){
 	}
 }
 
+func TestFetchWithProxy(t *testing.T) {
+	doTestFetchWithProxy(t, "", "")
+	doTestFetchWithProxy(t, "admin", "123")
+}
+
+func doTestFetchWithProxy(t *testing.T, usr, pwd string) {
+	httpListener := startServer()
+	defer httpListener.Close()
+
+	proxyListener := test.StartSocks5Server(usr, pwd)
+	defer proxyListener.Close()
+
+	engine := NewEngine(&Config{ProxyURL: util.BuildProxyUrl("socks5", proxyListener.Addr().String(), usr, pwd)})
+
+	if _, err := engine.RunString(fmt.Sprintf("var host = 'http://%s';", httpListener.Addr().String())); err != nil {
+		t.Fatal(err)
+	}
+
+	respCode, err := engine.RunString(`
+(async function(){
+	const resp = await fetch(host+'/get');
+	return resp.status;
+})()
+`)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if respCode != int64(200) {
+		t.Fatalf("fetch with proxy failed, want %d, got %d", 200, respCode)
+	}
+}
+
 func TestVm(t *testing.T) {
-	engine := NewEngine()
+	engine := NewEngine(nil)
 
 	value, err := engine.RunString(`
 const vm = __gopeed_create_vm()
@@ -221,7 +254,7 @@ out
 }
 
 func TestNonStopLoop(t *testing.T) {
-	engine := NewEngine()
+	engine := NewEngine(nil)
 
 	_, err := engine.RunString(`
 function leak(){

pkg/download/engine/inject/xhr/module.go

@@ -10,6 +10,7 @@ import (
 	"mime/multipart"
 	"net"
 	"net/http"
+	"net/url"
 	"time"
 )
 
@@ -119,6 +120,7 @@ type XMLHttpRequest struct {
 	requestHeaders  map[string]string
 	responseHeaders map[string]string
 	aborted         bool
+	proxyUrl        *url.URL
 
 	Upload       *XMLHttpRequestUpload `json:"upload"`
 	Timeout      int                   `json:"timeout"`
@@ -200,8 +202,13 @@ func (xhr *XMLHttpRequest) Send(data goja.Value) {
 	for k, v := range xhr.requestHeaders {
 		req.Header.Set(k, v)
 	}
+	transport := &http.Transport{}
+	if xhr.proxyUrl != nil {
+		transport.Proxy = http.ProxyURL(xhr.proxyUrl)
+	}
 	client := &http.Client{
-		Timeout: time.Duration(xhr.Timeout) * time.Millisecond,
+		Transport: transport,
+		Timeout:   time.Duration(xhr.Timeout) * time.Millisecond,
 	}
 	resp, err := client.Do(req)
 	if err != nil {
@@ -306,7 +313,7 @@ func (xhr *XMLHttpRequest) parseData(data goja.Value) any {
 	return data.String()
 }
 
-func Enable(runtime *goja.Runtime) error {
+func Enable(runtime *goja.Runtime, proxyUrl *url.URL) error {
 	progressEvent := runtime.ToValue(func(call goja.ConstructorCall) *goja.Object {
 		if len(call.Arguments) < 1 {
 			inject.ThrowTypeError(runtime, "Failed to construct 'ProgressEvent': 1 argument required, but only 0 present.")
@@ -320,6 +327,7 @@ func Enable(runtime *goja.Runtime) error {
 	})
 	xhr := runtime.ToValue(func(call goja.ConstructorCall) *goja.Object {
 		instance := &XMLHttpRequest{
+			proxyUrl: proxyUrl,
 			Upload: &XMLHttpRequestUpload{
 				EventProp: &EventProp{
 					eventListeners: make(map[string]func(event *ProgressEvent)),

pkg/download/extension.go

@@ -9,6 +9,7 @@ import (
 	"github.com/GopeedLab/gopeed/pkg/util"
 	"github.com/dop251/goja"
 	"github.com/go-git/go-git/v5"
+	"github.com/go-git/go-git/v5/plumbing/transport"
 	"io"
 	"os"
 	"path"
@@ -195,11 +196,19 @@ func (d *Downloader) fetchExtensionByGit(url string, handler func(tempExtPath st
 	if err := util.RmAndMkDirAll(tempExtDir); err != nil {
 		return nil, err
 	}
+	proxyOptions := transport.ProxyOptions{}
+	proxyUrl := d.cfg.ProxyUrl()
+	if proxyUrl != nil {
+		proxyOptions.URL = proxyUrl.Scheme + "://" + proxyUrl.Host
+		proxyOptions.Username = proxyUrl.User.Username()
+		proxyOptions.Password, _ = proxyUrl.User.Password()
+	}
 	// clone project to extension temp dir
 	gitUrl := parentPath + projectPath + gitSuffix
 	if _, err := git.PlainClone(tempExtDir, false, &git.CloneOptions{
-		URL:   gitUrl,
-		Depth: 1,
+		URL:          gitUrl,
+		Depth:        1,
+		ProxyOptions: proxyOptions,
 	}); err != nil {
 		return nil, err
 	}
@@ -314,7 +323,9 @@ func doTrigger[T any](d *Downloader, event ActivationEvent, req *base.Request, c
 					if req.Labels == nil {
 						req.Labels = make(map[string]string)
 					}
-					engine := engine.NewEngine()
+					engine := engine.NewEngine(&engine.Config{
+						ProxyURL: d.cfg.ProxyUrl(),
+					})
 					defer engine.Close()
 					err = engine.Runtime.Set("gopeed", gopeed)
 					if err != nil {

pkg/download/model.go

@@ -8,6 +8,7 @@ import (
 	"github.com/GopeedLab/gopeed/pkg/base"
 	"github.com/GopeedLab/gopeed/pkg/util"
 	gonanoid "github.com/matoous/go-nanoid/v2"
+	"net/url"
 	"sync"
 	"time"
 )
@@ -101,15 +102,37 @@ func (cfg *DownloaderConfig) Init() *DownloaderConfig {
 type DownloaderStoreConfig struct {
 	FirstLoad bool `json:"-"` // FirstLoad is the flag that the config is first time init and not from store
 
-	DownloadDir    string         `json:"downloadDir"`    // DownloadDir is the default directory to save the downloaded files
-	MaxRunning     int            `json:"maxRunning"`     // MaxRunning is the max running download count
-	ProtocolConfig map[string]any `json:"protocolConfig"` // ProtocolConfig is special config for each protocol
-	Extra          map[string]any `json:"extra"`          // Extra is the extra config
+	DownloadDir    string                 `json:"downloadDir"`    // DownloadDir is the default directory to save the downloaded files
+	MaxRunning     int                    `json:"maxRunning"`     // MaxRunning is the max running download count
+	ProtocolConfig map[string]any         `json:"protocolConfig"` // ProtocolConfig is special config for each protocol
+	Extra          map[string]any         `json:"extra"`
+	Proxy          *DownloaderProxyConfig `json:"proxy"`
 }
 
 func (cfg *DownloaderStoreConfig) Init() *DownloaderStoreConfig {
 	if cfg.MaxRunning == 0 {
 		cfg.MaxRunning = 5
 	}
+	if cfg.Proxy == nil {
+		cfg.Proxy = &DownloaderProxyConfig{}
+	}
 	return cfg
 }
+
+func (cfg *DownloaderStoreConfig) ProxyUrl() *url.URL {
+	if cfg.Proxy == nil {
+		return nil
+	}
+	if cfg.Proxy.Enable == false || cfg.Proxy.Scheme == "" || cfg.Proxy.Host == "" {
+		return nil
+	}
+	return util.BuildProxyUrl(cfg.Proxy.Scheme, cfg.Proxy.Host, cfg.Proxy.Usr, cfg.Proxy.Pwd)
+}
+
+type DownloaderProxyConfig struct {
+	Enable bool   `json:"enable"`
+	Scheme string `json:"scheme"`
+	Host   string `json:"host"`
+	Usr    string `json:"usr"`
+	Pwd    string `json:"pwd"`
+}

pkg/rest/server_test.go

@@ -27,7 +27,7 @@ var (
 		Extra: map[string]any{
 			"method": "",
 			"header": map[string]string{
-				"User-Agent": "gopeed",
+				"Usr-Agent": "gopeed",
 			},
 			"body": "",
 		},

pkg/util/url.go

@@ -2,6 +2,7 @@ package util
 
 import (
 	"encoding/base64"
+	"net/url"
 	"regexp"
 	"strings"
 )
@@ -37,3 +38,16 @@ func ParseDataUri(uri string) (string, []byte) {
 	}
 	return mime, data
 }
+
+// BuildProxyUrl builds a proxy url with given host, username and password.
+func BuildProxyUrl(scheme, host, usr, pwd string) *url.URL {
+	var user *url.Userinfo
+	if usr != "" && pwd != "" {
+		user = url.UserPassword(usr, pwd)
+	}
+	return &url.URL{
+		Scheme: scheme,
+		User:   user,
+		Host:   host,
+	}
+}