feat: support system proxy (#515)

pkg/base/model.go

@@ -3,7 +3,10 @@ package base
 import (
 	"fmt"
 	"github.com/GopeedLab/gopeed/pkg/util"
+	"github.com/mattn/go-ieproxy"
 	"golang.org/x/exp/slices"
+	"net/http"
+	"net/url"
 	"time"
 )
 
@@ -128,3 +131,88 @@ func ParseOptsExtra[E any](opts *Options) error {
 	opts.Extra = &t
 	return nil
 }
+
+// DownloaderStoreConfig is the config that can restore the downloader.
+type DownloaderStoreConfig struct {
+	FirstLoad bool `json:"-"` // FirstLoad is the flag that the config is first time init and not from store
+
+	DownloadDir    string                 `json:"downloadDir"`    // DownloadDir is the default directory to save the downloaded files
+	MaxRunning     int                    `json:"maxRunning"`     // MaxRunning is the max running download count
+	ProtocolConfig map[string]any         `json:"protocolConfig"` // ProtocolConfig is special config for each protocol
+	Extra          map[string]any         `json:"extra"`
+	Proxy          *DownloaderProxyConfig `json:"proxy"`
+}
+
+func (cfg *DownloaderStoreConfig) Init() *DownloaderStoreConfig {
+	if cfg.MaxRunning == 0 {
+		cfg.MaxRunning = 5
+	}
+	if cfg.Proxy == nil {
+		cfg.Proxy = &DownloaderProxyConfig{}
+	}
+	return cfg
+}
+
+type DownloaderProxyConfig struct {
+	Enable bool `json:"enable"`
+	// System is the flag that use system proxy
+	System bool   `json:"system"`
+	Scheme string `json:"scheme"`
+	Host   string `json:"host"`
+	Usr    string `json:"usr"`
+	Pwd    string `json:"pwd"`
+}
+
+func (cfg *DownloaderProxyConfig) ToHandler() func(r *http.Request) (*url.URL, error) {
+	if cfg == nil || cfg.Enable == false {
+		return nil
+	}
+	if cfg.System {
+		ieproxy.ReloadConf()
+		return ieproxy.GetProxyFunc()
+	}
+	if cfg.Scheme == "" || cfg.Host == "" {
+		return nil
+	}
+	return http.ProxyURL(util.BuildProxyUrl(cfg.Scheme, cfg.Host, cfg.Usr, cfg.Pwd))
+}
+
+// ToUrl returns the proxy url, just for git clone
+func (cfg *DownloaderProxyConfig) ToUrl() *url.URL {
+	if cfg == nil || cfg.Enable == false {
+		return nil
+	}
+	if cfg.System {
+		ieproxy.ReloadConf()
+		static := ieproxy.GetConf().Static
+		if static.Active && len(static.Protocols) > 0 {
+			// If only one protocol, use it
+			if len(static.Protocols) == 1 {
+				for _, v := range static.Protocols {
+					return parseUrlSafe(v)
+				}
+			}
+			// Check https
+			if v, ok := static.Protocols["https"]; ok {
+				return parseUrlSafe(v)
+			}
+			// Check http
+			if v, ok := static.Protocols["http"]; ok {
+				return parseUrlSafe(v)
+			}
+		}
+		return nil
+	}
+	if cfg.Scheme == "" || cfg.Host == "" {
+		return nil
+	}
+	return util.BuildProxyUrl(cfg.Scheme, cfg.Host, cfg.Usr, cfg.Pwd)
+}
+
+func parseUrlSafe(rawUrl string) *url.URL {
+	u, err := url.Parse(rawUrl)
+	if err != nil {
+		return nil
+	}
+	return u
+}

pkg/download/downloader.go

@@ -110,7 +110,7 @@ func (d *Downloader) Setup() error {
 		return err
 	}
 	// load config from storage
-	var cfg DownloaderStoreConfig
+	var cfg base.DownloaderStoreConfig
 	exist, err := d.storage.Get(bucketConfig, "config", &cfg)
 	if err != nil {
 		return err
@@ -118,7 +118,7 @@ func (d *Downloader) Setup() error {
 	if exist {
 		d.cfg.DownloaderStoreConfig = &cfg
 	} else {
-		d.cfg.DownloaderStoreConfig = &DownloaderStoreConfig{
+		d.cfg.DownloaderStoreConfig = &base.DownloaderStoreConfig{
 			FirstLoad: true,
 		}
 	}
@@ -225,7 +225,7 @@ func (d *Downloader) setupFetcher(fetcher fetcher.Fetcher) {
 	ctl.GetConfig = func(v any) bool {
 		return d.getProtocolConfig(fetcher.Name(), v)
 	}
-	ctl.ProxyUrl = d.cfg.ProxyUrl()
+	ctl.ProxyConfig = d.cfg.Proxy
 	fetcher.Setup(ctl)
 }
 
@@ -603,11 +603,11 @@ func (d *Downloader) GetTasksByStatues(statues []base.Status) []*Task {
 	return tasks
 }
 
-func (d *Downloader) GetConfig() (*DownloaderStoreConfig, error) {
+func (d *Downloader) GetConfig() (*base.DownloaderStoreConfig, error) {
 	return d.cfg.DownloaderStoreConfig, nil
 }
 
-func (d *Downloader) PutConfig(v *DownloaderStoreConfig) error {
+func (d *Downloader) PutConfig(v *base.DownloaderStoreConfig) error {
 	d.cfg.DownloaderStoreConfig = v
 	return d.storage.Put(bucketConfig, "config", v)
 }

pkg/download/downloader_test.go

@@ -4,6 +4,7 @@ import (
 	"github.com/GopeedLab/gopeed/internal/test"
 	"github.com/GopeedLab/gopeed/pkg/base"
 	"github.com/GopeedLab/gopeed/pkg/protocol/http"
+	"os"
 	"sync"
 	"testing"
 	"time"
@@ -85,37 +86,58 @@ func TestDownloader_Create(t *testing.T) {
 
 func TestDownloader_CreateWithProxy(t *testing.T) {
 	// No proxy
-	doTestDownloaderCreateWithProxy(t, false, func(proxyCfg *DownloaderProxyConfig) *DownloaderProxyConfig {
+	doTestDownloaderCreateWithProxy(t, false, func(proxyCfg *base.DownloaderProxyConfig) *base.DownloaderProxyConfig {
 		return nil
-	})
+	}, nil)
 	// Disable proxy
-	doTestDownloaderCreateWithProxy(t, false, func(proxyCfg *DownloaderProxyConfig) *DownloaderProxyConfig {
+	doTestDownloaderCreateWithProxy(t, false, func(proxyCfg *base.DownloaderProxyConfig) *base.DownloaderProxyConfig {
 		proxyCfg.Enable = false
 		return proxyCfg
+	}, nil)
+	// Enable system proxy but not set proxy environment variable
+	doTestDownloaderCreateWithProxy(t, false, func(proxyCfg *base.DownloaderProxyConfig) *base.DownloaderProxyConfig {
+		proxyCfg.System = true
+		return proxyCfg
+	}, nil)
+	// Enable proxy but error proxy environment variable
+	doTestDownloaderCreateWithProxy(t, false, func(proxyCfg *base.DownloaderProxyConfig) *base.DownloaderProxyConfig {
+		os.Setenv("HTTP_PROXY", "http://127.0.0.1:1234")
+		os.Setenv("HTTPS_PROXY", "http://127.0.0.1:1234")
+		proxyCfg.System = true
+		return proxyCfg
+	}, func(err error) {
+		if err == nil {
+			t.Fatal("doTestDownloaderCreateWithProxy() got = nil, want error")
+		}
 	})
+	// Enable system proxy and set proxy environment variable
+	doTestDownloaderCreateWithProxy(t, false, func(proxyCfg *base.DownloaderProxyConfig) *base.DownloaderProxyConfig {
+		os.Setenv("HTTP_PROXY", proxyCfg.ToUrl().String())
+		os.Setenv("HTTPS_PROXY", proxyCfg.ToUrl().String())
+		proxyCfg.System = true
+		return proxyCfg
+	}, nil)
 	// Invalid proxy scheme
-	doTestDownloaderCreateWithProxy(t, false, func(proxyCfg *DownloaderProxyConfig) *DownloaderProxyConfig {
+	doTestDownloaderCreateWithProxy(t, false, func(proxyCfg *base.DownloaderProxyConfig) *base.DownloaderProxyConfig {
 		proxyCfg.Scheme = ""
 		return proxyCfg
-	})
+	}, nil)
 	// Invalid proxy host
-	doTestDownloaderCreateWithProxy(t, false, func(proxyCfg *DownloaderProxyConfig) *DownloaderProxyConfig {
+	doTestDownloaderCreateWithProxy(t, false, func(proxyCfg *base.DownloaderProxyConfig) *base.DownloaderProxyConfig {
 		proxyCfg.Host = ""
 		return proxyCfg
-	})
+	}, nil)
 	// Use proxy without auth
-	doTestDownloaderCreateWithProxy(t, false, func(proxyCfg *DownloaderProxyConfig) *DownloaderProxyConfig {
+	doTestDownloaderCreateWithProxy(t, false, func(proxyCfg *base.DownloaderProxyConfig) *base.DownloaderProxyConfig {
 		return proxyCfg
-	})
+	}, nil)
 	// Use proxy with auth
-	doTestDownloaderCreateWithProxy(t, true, func(proxyCfg *DownloaderProxyConfig) *DownloaderProxyConfig {
+	doTestDownloaderCreateWithProxy(t, true, func(proxyCfg *base.DownloaderProxyConfig) *base.DownloaderProxyConfig {
 		return proxyCfg
-	})
+	}, nil)
 }
 
-func doTestDownloaderCreateWithProxy(t *testing.T, auth bool, buildProxyConfig func(proxyCfg *DownloaderProxyConfig) *DownloaderProxyConfig) {
-	httpListener := test.StartTestFileServer()
-	defer httpListener.Close()
+func doTestDownloaderCreateWithProxy(t *testing.T, auth bool, buildProxyConfig func(proxyCfg *base.DownloaderProxyConfig) *base.DownloaderProxyConfig, errHandler func(err error)) {
 	usr, pwd := "", ""
 	if auth {
 		usr, pwd = "admin", "123"
@@ -128,7 +150,7 @@ func doTestDownloaderCreateWithProxy(t *testing.T, auth bool, buildProxyConfig f
 		t.Fatal(err)
 	}
 	defer downloader.Clear()
-	downloader.cfg.DownloaderStoreConfig.Proxy = buildProxyConfig(&DownloaderProxyConfig{
+	downloader.cfg.DownloaderStoreConfig.Proxy = buildProxyConfig(&base.DownloaderProxyConfig{
 		Enable: true,
 		Scheme: "socks5",
 		Host:   proxyListener.Addr().String(),
@@ -137,20 +159,24 @@ func doTestDownloaderCreateWithProxy(t *testing.T, auth bool, buildProxyConfig f
 	})
 
 	req := &base.Request{
-		URL: "http://" + httpListener.Addr().String() + "/" + test.BuildName,
+		URL: test.ExternalDownloadUrl,
 	}
 	rr, err := downloader.Resolve(req)
 	if err != nil {
-		t.Fatal(err)
+		if errHandler == nil {
+			t.Fatal(err)
+		}
+		errHandler(err)
+		return
 	}
 	want := &base.Resource{
-		Size:  test.BuildSize,
+		Size:  test.ExternalDownloadSize,
 		Range: true,
 		Files: []*base.FileInfo{
 			{
-				Name: test.BuildName,
+				Name: test.ExternalDownloadName,
 				Path: "",
-				Size: test.BuildSize,
+				Size: test.ExternalDownloadSize,
 			},
 		},
 	}
@@ -285,7 +311,7 @@ func TestDownloader_Protocol_Config(t *testing.T) {
 		t.Errorf("getProtocolConfig() got = %v, want %v", exits, false)
 	}
 
-	storeCfg := &DownloaderStoreConfig{
+	storeCfg := &base.DownloaderStoreConfig{
 		DownloadDir: "./downloads",
 		ProtocolConfig: map[string]any{
 			"http": map[string]any{

pkg/download/engine/engine.go

@@ -3,6 +3,7 @@ package engine
 import (
 	_ "embed"
 	"errors"
+	"github.com/GopeedLab/gopeed/pkg/base"
 	gojaerror "github.com/GopeedLab/gopeed/pkg/download/engine/inject/error"
 	"github.com/GopeedLab/gopeed/pkg/download/engine/inject/file"
 	"github.com/GopeedLab/gopeed/pkg/download/engine/inject/formdata"
@@ -11,7 +12,6 @@ import (
 	"github.com/dop251/goja"
 	"github.com/dop251/goja_nodejs/eventloop"
 	gojaurl "github.com/dop251/goja_nodejs/url"
-	"net/url"
 	"time"
 )
 
@@ -110,7 +110,7 @@ func (e *Engine) Close() {
 }
 
 type Config struct {
-	ProxyURL *url.URL
+	ProxyConfig *base.DownloaderProxyConfig
 }
 
 func NewEngine(cfg *Config) *Engine {
@@ -135,7 +135,7 @@ func NewEngine(cfg *Config) *Engine {
 		if err := formdata.Enable(runtime); err != nil {
 			return
 		}
-		if err := xhr.Enable(runtime, cfg.ProxyURL); err != nil {
+		if err := xhr.Enable(runtime, cfg.ProxyConfig.ToHandler()); err != nil {
 			return
 		}
 		if _, err := runtime.RunString(polyfillScript); err != nil {

pkg/download/engine/engine_test.go

@@ -7,10 +7,10 @@ import (
 	"errors"
 	"fmt"
 	"github.com/GopeedLab/gopeed/internal/test"
+	"github.com/GopeedLab/gopeed/pkg/base"
 	gojaerror "github.com/GopeedLab/gopeed/pkg/download/engine/inject/error"
 	"github.com/GopeedLab/gopeed/pkg/download/engine/inject/file"
 	gojautil "github.com/GopeedLab/gopeed/pkg/download/engine/util"
-	"github.com/GopeedLab/gopeed/pkg/util"
 	"github.com/dop251/goja"
 	"io"
 	"net"
@@ -217,8 +217,16 @@ func doTestFetchWithProxy(t *testing.T, usr, pwd string) {
 
 	proxyListener := test.StartSocks5Server(usr, pwd)
 	defer proxyListener.Close()
-
-	engine := NewEngine(&Config{ProxyURL: util.BuildProxyUrl("socks5", proxyListener.Addr().String(), usr, pwd)})
+	engine := NewEngine(&Config{
+		ProxyConfig: &base.DownloaderProxyConfig{
+			Enable: true,
+			System: false,
+			Scheme: "socks5",
+			Host:   proxyListener.Addr().String(),
+			Usr:    usr,
+			Pwd:    pwd,
+		},
+	})
 
 	if _, err := engine.RunString(fmt.Sprintf("var host = 'http://%s';", httpListener.Addr().String())); err != nil {
 		t.Fatal(err)

pkg/download/engine/inject/xhr/module.go

@@ -2,6 +2,7 @@ package xhr
 
 import (
 	"bytes"
+	"errors"
 	"github.com/GopeedLab/gopeed/pkg/download/engine/inject/file"
 	"github.com/GopeedLab/gopeed/pkg/download/engine/inject/formdata"
 	"github.com/GopeedLab/gopeed/pkg/download/engine/util"
@@ -120,7 +121,7 @@ type XMLHttpRequest struct {
 	requestHeaders  map[string]string
 	responseHeaders map[string]string
 	aborted         bool
-	proxyUrl        *url.URL
+	proxyHandler    func(r *http.Request) (*url.URL, error)
 
 	Upload       *XMLHttpRequestUpload `json:"upload"`
 	Timeout      int                   `json:"timeout"`
@@ -202,9 +203,8 @@ func (xhr *XMLHttpRequest) Send(data goja.Value) {
 	for k, v := range xhr.requestHeaders {
 		req.Header.Set(k, v)
 	}
-	transport := &http.Transport{}
-	if xhr.proxyUrl != nil {
-		transport.Proxy = http.ProxyURL(xhr.proxyUrl)
+	transport := &http.Transport{
+		Proxy: xhr.proxyHandler,
 	}
 	client := &http.Client{
 		Transport: transport,
@@ -213,7 +213,8 @@ func (xhr *XMLHttpRequest) Send(data goja.Value) {
 	resp, err := client.Do(req)
 	if err != nil {
 		// handle timeout error
-		if err, ok := err.(net.Error); ok && err.Timeout() {
+		var ne net.Error
+		if errors.As(err, &ne) && ne.Timeout() {
 			if xhr.Timeout > 0 {
 				xhr.Upload.callOntimeout()
 				xhr.callOntimeout()
@@ -313,7 +314,7 @@ func (xhr *XMLHttpRequest) parseData(data goja.Value) any {
 	return data.String()
 }
 
-func Enable(runtime *goja.Runtime, proxyUrl *url.URL) error {
+func Enable(runtime *goja.Runtime, proxyHandler func(r *http.Request) (*url.URL, error)) error {
 	progressEvent := runtime.ToValue(func(call goja.ConstructorCall) *goja.Object {
 		if len(call.Arguments) < 1 {
 			util.ThrowTypeError(runtime, "Failed to construct 'ProgressEvent': 1 argument required, but only 0 present.")
@@ -327,7 +328,7 @@ func Enable(runtime *goja.Runtime, proxyUrl *url.URL) error {
 	})
 	xhr := runtime.ToValue(func(call goja.ConstructorCall) *goja.Object {
 		instance := &XMLHttpRequest{
-			proxyUrl: proxyUrl,
+			proxyHandler: proxyHandler,
 			Upload: &XMLHttpRequestUpload{
 				EventProp: &EventProp{
 					eventListeners: make(map[string]func(event *ProgressEvent)),

pkg/download/extension.go

@@ -199,7 +199,7 @@ func (d *Downloader) fetchExtensionByGit(url string, handler func(tempExtPath st
 		return nil, err
 	}
 	proxyOptions := transport.ProxyOptions{}
-	proxyUrl := d.cfg.ProxyUrl()
+	proxyUrl := d.cfg.DownloaderStoreConfig.Proxy.ToUrl()
 	if proxyUrl != nil {
 		proxyOptions.URL = proxyUrl.Scheme + "://" + proxyUrl.Host
 		proxyOptions.Username = proxyUrl.User.Username()
@@ -341,7 +341,7 @@ func doTrigger[T any](d *Downloader, event ActivationEvent, req *base.Request, c
 						req.Labels = make(map[string]string)
 					}
 					engine := engine.NewEngine(&engine.Config{
-						ProxyURL: d.cfg.ProxyUrl(),
+						ProxyConfig: d.cfg.Proxy,
 					})
 					defer engine.Close()
 					err = engine.Runtime.Set("gopeed", gopeed)

pkg/download/model.go

@@ -7,9 +7,7 @@ import (
 	"github.com/GopeedLab/gopeed/internal/protocol/http"
 	"github.com/GopeedLab/gopeed/pkg/base"
 	"github.com/GopeedLab/gopeed/pkg/util"
-
 	gonanoid "github.com/matoous/go-nanoid/v2"
-	"net/url"
 	"sync"
 	"time"
 )
@@ -87,7 +85,7 @@ type DownloaderConfig struct {
 
 	ProductionMode bool
 
-	*DownloaderStoreConfig
+	*base.DownloaderStoreConfig
 }
 
 func (cfg *DownloaderConfig) Init() *DownloaderConfig {
@@ -108,42 +106,3 @@ func (cfg *DownloaderConfig) Init() *DownloaderConfig {
 	}
 	return cfg
 }
-
-// DownloaderStoreConfig is the config that can restore the downloader.
-type DownloaderStoreConfig struct {
-	FirstLoad bool `json:"-"` // FirstLoad is the flag that the config is first time init and not from store
-
-	DownloadDir    string                 `json:"downloadDir"`    // DownloadDir is the default directory to save the downloaded files
-	MaxRunning     int                    `json:"maxRunning"`     // MaxRunning is the max running download count
-	ProtocolConfig map[string]any         `json:"protocolConfig"` // ProtocolConfig is special config for each protocol
-	Extra          map[string]any         `json:"extra"`
-	Proxy          *DownloaderProxyConfig `json:"proxy"`
-}
-
-func (cfg *DownloaderStoreConfig) Init() *DownloaderStoreConfig {
-	if cfg.MaxRunning == 0 {
-		cfg.MaxRunning = 5
-	}
-	if cfg.Proxy == nil {
-		cfg.Proxy = &DownloaderProxyConfig{}
-	}
-	return cfg
-}
-
-func (cfg *DownloaderStoreConfig) ProxyUrl() *url.URL {
-	if cfg.Proxy == nil {
-		return nil
-	}
-	if cfg.Proxy.Enable == false || cfg.Proxy.Scheme == "" || cfg.Proxy.Host == "" {
-		return nil
-	}
-	return util.BuildProxyUrl(cfg.Proxy.Scheme, cfg.Proxy.Host, cfg.Proxy.Usr, cfg.Proxy.Pwd)
-}
-
-type DownloaderProxyConfig struct {
-	Enable bool   `json:"enable"`
-	Scheme string `json:"scheme"`
-	Host   string `json:"host"`
-	Usr    string `json:"usr"`
-	Pwd    string `json:"pwd"`
-}

pkg/rest/api.go

@@ -142,7 +142,7 @@ func GetConfig(w http.ResponseWriter, r *http.Request) {
 }
 
 func PutConfig(w http.ResponseWriter, r *http.Request) {
-	var cfg download.DownloaderStoreConfig
+	var cfg base.DownloaderStoreConfig
 	if ReadJson(r, w, &cfg) {
 		if err := Downloader.PutConfig(&cfg); err != nil {
 			WriteJson(w, model.NewErrorResult(err.Error()))
@@ -309,7 +309,7 @@ func writeError(w http.ResponseWriter, msg string) {
 	w.Write([]byte(msg))
 }
 
-func getServerConfig() *download.DownloaderStoreConfig {
+func getServerConfig() *base.DownloaderStoreConfig {
 	cfg, _ := Downloader.GetConfig()
 	return cfg
 }

pkg/rest/server_test.go

@@ -324,7 +324,7 @@ func TestGetTasks(t *testing.T) {
 
 func TestGetAndPutConfig(t *testing.T) {
 	doTest(func() {
-		cfg := httpRequestCheckOk[*download.DownloaderStoreConfig](http.MethodGet, "/api/v1/config", nil)
+		cfg := httpRequestCheckOk[*base.DownloaderStoreConfig](http.MethodGet, "/api/v1/config", nil)
 		cfg.DownloadDir = "./download"
 		cfg.Extra = map[string]any{
 			"serverConfig": &Config{
@@ -335,7 +335,7 @@ func TestGetAndPutConfig(t *testing.T) {
 		}
 		httpRequestCheckOk[any](http.MethodPut, "/api/v1/config", cfg)
 
-		newCfg := httpRequestCheckOk[*download.DownloaderStoreConfig](http.MethodGet, "/api/v1/config", nil)
+		newCfg := httpRequestCheckOk[*base.DownloaderStoreConfig](http.MethodGet, "/api/v1/config", nil)
 		if !test.JsonEqual(cfg, newCfg) {
 			t.Errorf("GetAndPutConfig() got = %v, want %v", test.ToJson(newCfg), test.ToJson(cfg))
 		}

pkg/util/url.go

@@ -2,6 +2,7 @@ package util
 
 import (
 	"encoding/base64"
+	"net/http"
 	"net/url"
 	"regexp"
 	"strings"
@@ -51,3 +52,14 @@ func BuildProxyUrl(scheme, host, usr, pwd string) *url.URL {
 		Host:   host,
 	}
 }
+
+// ProxyUrlToHandler gets the proxy handler from the proxy url.
+func ProxyUrlToHandler(proxyUrl *url.URL) func(*http.Request) (*url.URL, error) {
+	if proxyUrl == nil {
+		return nil
+	}
+	if proxyUrl.Scheme == "system" {
+		return http.ProxyFromEnvironment
+	}
+	return http.ProxyURL(proxyUrl)
+}