mirror of
https://github.com/GopeedLab/gopeed.git
synced 2026-11-18 06:13:42 +00:00
feat: support system proxy (#515)
This commit is contained in:
30 files changed
+324
-145
No files matched your search
@@ -110,7 +110,7 @@ func (d *Downloader) Setup() error {
|
||||
return err
|
||||
}
|
||||
// load config from storage
|
||||
var cfg DownloaderStoreConfig
|
||||
var cfg base.DownloaderStoreConfig
|
||||
exist, err := d.storage.Get(bucketConfig, "config", &cfg)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -118,7 +118,7 @@ func (d *Downloader) Setup() error {
|
||||
if exist {
|
||||
d.cfg.DownloaderStoreConfig = &cfg
|
||||
} else {
|
||||
d.cfg.DownloaderStoreConfig = &DownloaderStoreConfig{
|
||||
d.cfg.DownloaderStoreConfig = &base.DownloaderStoreConfig{
|
||||
FirstLoad: true,
|
||||
}
|
||||
}
|
||||
@@ -225,7 +225,7 @@ func (d *Downloader) setupFetcher(fetcher fetcher.Fetcher) {
|
||||
ctl.GetConfig = func(v any) bool {
|
||||
return d.getProtocolConfig(fetcher.Name(), v)
|
||||
}
|
||||
ctl.ProxyUrl = d.cfg.ProxyUrl()
|
||||
ctl.ProxyConfig = d.cfg.Proxy
|
||||
fetcher.Setup(ctl)
|
||||
}
|
||||
|
||||
@@ -603,11 +603,11 @@ func (d *Downloader) GetTasksByStatues(statues []base.Status) []*Task {
|
||||
return tasks
|
||||
}
|
||||
|
||||
func (d *Downloader) GetConfig() (*DownloaderStoreConfig, error) {
|
||||
func (d *Downloader) GetConfig() (*base.DownloaderStoreConfig, error) {
|
||||
return d.cfg.DownloaderStoreConfig, nil
|
||||
}
|
||||
|
||||
func (d *Downloader) PutConfig(v *DownloaderStoreConfig) error {
|
||||
func (d *Downloader) PutConfig(v *base.DownloaderStoreConfig) error {
|
||||
d.cfg.DownloaderStoreConfig = v
|
||||
return d.storage.Put(bucketConfig, "config", v)
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"github.com/GopeedLab/gopeed/internal/test"
|
||||
"github.com/GopeedLab/gopeed/pkg/base"
|
||||
"github.com/GopeedLab/gopeed/pkg/protocol/http"
|
||||
"os"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -85,37 +86,58 @@ func TestDownloader_Create(t *testing.T) {
|
||||
|
||||
func TestDownloader_CreateWithProxy(t *testing.T) {
|
||||
// No proxy
|
||||
doTestDownloaderCreateWithProxy(t, false, func(proxyCfg *DownloaderProxyConfig) *DownloaderProxyConfig {
|
||||
doTestDownloaderCreateWithProxy(t, false, func(proxyCfg *base.DownloaderProxyConfig) *base.DownloaderProxyConfig {
|
||||
return nil
|
||||
})
|
||||
}, nil)
|
||||
// Disable proxy
|
||||
doTestDownloaderCreateWithProxy(t, false, func(proxyCfg *DownloaderProxyConfig) *DownloaderProxyConfig {
|
||||
doTestDownloaderCreateWithProxy(t, false, func(proxyCfg *base.DownloaderProxyConfig) *base.DownloaderProxyConfig {
|
||||
proxyCfg.Enable = false
|
||||
return proxyCfg
|
||||
}, nil)
|
||||
// Enable system proxy but not set proxy environment variable
|
||||
doTestDownloaderCreateWithProxy(t, false, func(proxyCfg *base.DownloaderProxyConfig) *base.DownloaderProxyConfig {
|
||||
proxyCfg.System = true
|
||||
return proxyCfg
|
||||
}, nil)
|
||||
// Enable proxy but error proxy environment variable
|
||||
doTestDownloaderCreateWithProxy(t, false, func(proxyCfg *base.DownloaderProxyConfig) *base.DownloaderProxyConfig {
|
||||
os.Setenv("HTTP_PROXY", "http://127.0.0.1:1234")
|
||||
os.Setenv("HTTPS_PROXY", "http://127.0.0.1:1234")
|
||||
proxyCfg.System = true
|
||||
return proxyCfg
|
||||
}, func(err error) {
|
||||
if err == nil {
|
||||
t.Fatal("doTestDownloaderCreateWithProxy() got = nil, want error")
|
||||
}
|
||||
})
|
||||
// Enable system proxy and set proxy environment variable
|
||||
doTestDownloaderCreateWithProxy(t, false, func(proxyCfg *base.DownloaderProxyConfig) *base.DownloaderProxyConfig {
|
||||
os.Setenv("HTTP_PROXY", proxyCfg.ToUrl().String())
|
||||
os.Setenv("HTTPS_PROXY", proxyCfg.ToUrl().String())
|
||||
proxyCfg.System = true
|
||||
return proxyCfg
|
||||
}, nil)
|
||||
// Invalid proxy scheme
|
||||
doTestDownloaderCreateWithProxy(t, false, func(proxyCfg *DownloaderProxyConfig) *DownloaderProxyConfig {
|
||||
doTestDownloaderCreateWithProxy(t, false, func(proxyCfg *base.DownloaderProxyConfig) *base.DownloaderProxyConfig {
|
||||
proxyCfg.Scheme = ""
|
||||
return proxyCfg
|
||||
})
|
||||
}, nil)
|
||||
// Invalid proxy host
|
||||
doTestDownloaderCreateWithProxy(t, false, func(proxyCfg *DownloaderProxyConfig) *DownloaderProxyConfig {
|
||||
doTestDownloaderCreateWithProxy(t, false, func(proxyCfg *base.DownloaderProxyConfig) *base.DownloaderProxyConfig {
|
||||
proxyCfg.Host = ""
|
||||
return proxyCfg
|
||||
})
|
||||
}, nil)
|
||||
// Use proxy without auth
|
||||
doTestDownloaderCreateWithProxy(t, false, func(proxyCfg *DownloaderProxyConfig) *DownloaderProxyConfig {
|
||||
doTestDownloaderCreateWithProxy(t, false, func(proxyCfg *base.DownloaderProxyConfig) *base.DownloaderProxyConfig {
|
||||
return proxyCfg
|
||||
})
|
||||
}, nil)
|
||||
// Use proxy with auth
|
||||
doTestDownloaderCreateWithProxy(t, true, func(proxyCfg *DownloaderProxyConfig) *DownloaderProxyConfig {
|
||||
doTestDownloaderCreateWithProxy(t, true, func(proxyCfg *base.DownloaderProxyConfig) *base.DownloaderProxyConfig {
|
||||
return proxyCfg
|
||||
})
|
||||
}, nil)
|
||||
}
|
||||
|
||||
func doTestDownloaderCreateWithProxy(t *testing.T, auth bool, buildProxyConfig func(proxyCfg *DownloaderProxyConfig) *DownloaderProxyConfig) {
|
||||
httpListener := test.StartTestFileServer()
|
||||
defer httpListener.Close()
|
||||
func doTestDownloaderCreateWithProxy(t *testing.T, auth bool, buildProxyConfig func(proxyCfg *base.DownloaderProxyConfig) *base.DownloaderProxyConfig, errHandler func(err error)) {
|
||||
usr, pwd := "", ""
|
||||
if auth {
|
||||
usr, pwd = "admin", "123"
|
||||
@@ -128,7 +150,7 @@ func doTestDownloaderCreateWithProxy(t *testing.T, auth bool, buildProxyConfig f
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer downloader.Clear()
|
||||
downloader.cfg.DownloaderStoreConfig.Proxy = buildProxyConfig(&DownloaderProxyConfig{
|
||||
downloader.cfg.DownloaderStoreConfig.Proxy = buildProxyConfig(&base.DownloaderProxyConfig{
|
||||
Enable: true,
|
||||
Scheme: "socks5",
|
||||
Host: proxyListener.Addr().String(),
|
||||
@@ -137,20 +159,24 @@ func doTestDownloaderCreateWithProxy(t *testing.T, auth bool, buildProxyConfig f
|
||||
})
|
||||
|
||||
req := &base.Request{
|
||||
URL: "http://" + httpListener.Addr().String() + "/" + test.BuildName,
|
||||
URL: test.ExternalDownloadUrl,
|
||||
}
|
||||
rr, err := downloader.Resolve(req)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
if errHandler == nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
errHandler(err)
|
||||
return
|
||||
}
|
||||
want := &base.Resource{
|
||||
Size: test.BuildSize,
|
||||
Size: test.ExternalDownloadSize,
|
||||
Range: true,
|
||||
Files: []*base.FileInfo{
|
||||
{
|
||||
Name: test.BuildName,
|
||||
Name: test.ExternalDownloadName,
|
||||
Path: "",
|
||||
Size: test.BuildSize,
|
||||
Size: test.ExternalDownloadSize,
|
||||
},
|
||||
},
|
||||
}
|
||||
@@ -285,7 +311,7 @@ func TestDownloader_Protocol_Config(t *testing.T) {
|
||||
t.Errorf("getProtocolConfig() got = %v, want %v", exits, false)
|
||||
}
|
||||
|
||||
storeCfg := &DownloaderStoreConfig{
|
||||
storeCfg := &base.DownloaderStoreConfig{
|
||||
DownloadDir: "./downloads",
|
||||
ProtocolConfig: map[string]any{
|
||||
"http": map[string]any{
|
||||
|
||||
@@ -3,6 +3,7 @@ package engine
|
||||
import (
|
||||
_ "embed"
|
||||
"errors"
|
||||
"github.com/GopeedLab/gopeed/pkg/base"
|
||||
gojaerror "github.com/GopeedLab/gopeed/pkg/download/engine/inject/error"
|
||||
"github.com/GopeedLab/gopeed/pkg/download/engine/inject/file"
|
||||
"github.com/GopeedLab/gopeed/pkg/download/engine/inject/formdata"
|
||||
@@ -11,7 +12,6 @@ import (
|
||||
"github.com/dop251/goja"
|
||||
"github.com/dop251/goja_nodejs/eventloop"
|
||||
gojaurl "github.com/dop251/goja_nodejs/url"
|
||||
"net/url"
|
||||
"time"
|
||||
)
|
||||
|
||||
@@ -110,7 +110,7 @@ func (e *Engine) Close() {
|
||||
}
|
||||
|
||||
type Config struct {
|
||||
ProxyURL *url.URL
|
||||
ProxyConfig *base.DownloaderProxyConfig
|
||||
}
|
||||
|
||||
func NewEngine(cfg *Config) *Engine {
|
||||
@@ -135,7 +135,7 @@ func NewEngine(cfg *Config) *Engine {
|
||||
if err := formdata.Enable(runtime); err != nil {
|
||||
return
|
||||
}
|
||||
if err := xhr.Enable(runtime, cfg.ProxyURL); err != nil {
|
||||
if err := xhr.Enable(runtime, cfg.ProxyConfig.ToHandler()); err != nil {
|
||||
return
|
||||
}
|
||||
if _, err := runtime.RunString(polyfillScript); err != nil {
|
||||
|
||||
@@ -7,10 +7,10 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"github.com/GopeedLab/gopeed/internal/test"
|
||||
"github.com/GopeedLab/gopeed/pkg/base"
|
||||
gojaerror "github.com/GopeedLab/gopeed/pkg/download/engine/inject/error"
|
||||
"github.com/GopeedLab/gopeed/pkg/download/engine/inject/file"
|
||||
gojautil "github.com/GopeedLab/gopeed/pkg/download/engine/util"
|
||||
"github.com/GopeedLab/gopeed/pkg/util"
|
||||
"github.com/dop251/goja"
|
||||
"io"
|
||||
"net"
|
||||
@@ -217,8 +217,16 @@ func doTestFetchWithProxy(t *testing.T, usr, pwd string) {
|
||||
|
||||
proxyListener := test.StartSocks5Server(usr, pwd)
|
||||
defer proxyListener.Close()
|
||||
|
||||
engine := NewEngine(&Config{ProxyURL: util.BuildProxyUrl("socks5", proxyListener.Addr().String(), usr, pwd)})
|
||||
engine := NewEngine(&Config{
|
||||
ProxyConfig: &base.DownloaderProxyConfig{
|
||||
Enable: true,
|
||||
System: false,
|
||||
Scheme: "socks5",
|
||||
Host: proxyListener.Addr().String(),
|
||||
Usr: usr,
|
||||
Pwd: pwd,
|
||||
},
|
||||
})
|
||||
|
||||
if _, err := engine.RunString(fmt.Sprintf("var host = 'http://%s';", httpListener.Addr().String())); err != nil {
|
||||
t.Fatal(err)
|
||||
|
||||
@@ -2,6 +2,7 @@ package xhr
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"github.com/GopeedLab/gopeed/pkg/download/engine/inject/file"
|
||||
"github.com/GopeedLab/gopeed/pkg/download/engine/inject/formdata"
|
||||
"github.com/GopeedLab/gopeed/pkg/download/engine/util"
|
||||
@@ -120,7 +121,7 @@ type XMLHttpRequest struct {
|
||||
requestHeaders map[string]string
|
||||
responseHeaders map[string]string
|
||||
aborted bool
|
||||
proxyUrl *url.URL
|
||||
proxyHandler func(r *http.Request) (*url.URL, error)
|
||||
|
||||
Upload *XMLHttpRequestUpload `json:"upload"`
|
||||
Timeout int `json:"timeout"`
|
||||
@@ -202,9 +203,8 @@ func (xhr *XMLHttpRequest) Send(data goja.Value) {
|
||||
for k, v := range xhr.requestHeaders {
|
||||
req.Header.Set(k, v)
|
||||
}
|
||||
transport := &http.Transport{}
|
||||
if xhr.proxyUrl != nil {
|
||||
transport.Proxy = http.ProxyURL(xhr.proxyUrl)
|
||||
transport := &http.Transport{
|
||||
Proxy: xhr.proxyHandler,
|
||||
}
|
||||
client := &http.Client{
|
||||
Transport: transport,
|
||||
@@ -213,7 +213,8 @@ func (xhr *XMLHttpRequest) Send(data goja.Value) {
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
// handle timeout error
|
||||
if err, ok := err.(net.Error); ok && err.Timeout() {
|
||||
var ne net.Error
|
||||
if errors.As(err, &ne) && ne.Timeout() {
|
||||
if xhr.Timeout > 0 {
|
||||
xhr.Upload.callOntimeout()
|
||||
xhr.callOntimeout()
|
||||
@@ -313,7 +314,7 @@ func (xhr *XMLHttpRequest) parseData(data goja.Value) any {
|
||||
return data.String()
|
||||
}
|
||||
|
||||
func Enable(runtime *goja.Runtime, proxyUrl *url.URL) error {
|
||||
func Enable(runtime *goja.Runtime, proxyHandler func(r *http.Request) (*url.URL, error)) error {
|
||||
progressEvent := runtime.ToValue(func(call goja.ConstructorCall) *goja.Object {
|
||||
if len(call.Arguments) < 1 {
|
||||
util.ThrowTypeError(runtime, "Failed to construct 'ProgressEvent': 1 argument required, but only 0 present.")
|
||||
@@ -327,7 +328,7 @@ func Enable(runtime *goja.Runtime, proxyUrl *url.URL) error {
|
||||
})
|
||||
xhr := runtime.ToValue(func(call goja.ConstructorCall) *goja.Object {
|
||||
instance := &XMLHttpRequest{
|
||||
proxyUrl: proxyUrl,
|
||||
proxyHandler: proxyHandler,
|
||||
Upload: &XMLHttpRequestUpload{
|
||||
EventProp: &EventProp{
|
||||
eventListeners: make(map[string]func(event *ProgressEvent)),
|
||||
|
||||
@@ -199,7 +199,7 @@ func (d *Downloader) fetchExtensionByGit(url string, handler func(tempExtPath st
|
||||
return nil, err
|
||||
}
|
||||
proxyOptions := transport.ProxyOptions{}
|
||||
proxyUrl := d.cfg.ProxyUrl()
|
||||
proxyUrl := d.cfg.DownloaderStoreConfig.Proxy.ToUrl()
|
||||
if proxyUrl != nil {
|
||||
proxyOptions.URL = proxyUrl.Scheme + "://" + proxyUrl.Host
|
||||
proxyOptions.Username = proxyUrl.User.Username()
|
||||
@@ -341,7 +341,7 @@ func doTrigger[T any](d *Downloader, event ActivationEvent, req *base.Request, c
|
||||
req.Labels = make(map[string]string)
|
||||
}
|
||||
engine := engine.NewEngine(&engine.Config{
|
||||
ProxyURL: d.cfg.ProxyUrl(),
|
||||
ProxyConfig: d.cfg.Proxy,
|
||||
})
|
||||
defer engine.Close()
|
||||
err = engine.Runtime.Set("gopeed", gopeed)
|
||||
|
||||
+1
-42
@@ -7,9 +7,7 @@ import (
|
||||
"github.com/GopeedLab/gopeed/internal/protocol/http"
|
||||
"github.com/GopeedLab/gopeed/pkg/base"
|
||||
"github.com/GopeedLab/gopeed/pkg/util"
|
||||
|
||||
gonanoid "github.com/matoous/go-nanoid/v2"
|
||||
"net/url"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
@@ -87,7 +85,7 @@ type DownloaderConfig struct {
|
||||
|
||||
ProductionMode bool
|
||||
|
||||
*DownloaderStoreConfig
|
||||
*base.DownloaderStoreConfig
|
||||
}
|
||||
|
||||
func (cfg *DownloaderConfig) Init() *DownloaderConfig {
|
||||
@@ -108,42 +106,3 @@ func (cfg *DownloaderConfig) Init() *DownloaderConfig {
|
||||
}
|
||||
return cfg
|
||||
}
|
||||
|
||||
// DownloaderStoreConfig is the config that can restore the downloader.
|
||||
type DownloaderStoreConfig struct {
|
||||
FirstLoad bool `json:"-"` // FirstLoad is the flag that the config is first time init and not from store
|
||||
|
||||
DownloadDir string `json:"downloadDir"` // DownloadDir is the default directory to save the downloaded files
|
||||
MaxRunning int `json:"maxRunning"` // MaxRunning is the max running download count
|
||||
ProtocolConfig map[string]any `json:"protocolConfig"` // ProtocolConfig is special config for each protocol
|
||||
Extra map[string]any `json:"extra"`
|
||||
Proxy *DownloaderProxyConfig `json:"proxy"`
|
||||
}
|
||||
|
||||
func (cfg *DownloaderStoreConfig) Init() *DownloaderStoreConfig {
|
||||
if cfg.MaxRunning == 0 {
|
||||
cfg.MaxRunning = 5
|
||||
}
|
||||
if cfg.Proxy == nil {
|
||||
cfg.Proxy = &DownloaderProxyConfig{}
|
||||
}
|
||||
return cfg
|
||||
}
|
||||
|
||||
func (cfg *DownloaderStoreConfig) ProxyUrl() *url.URL {
|
||||
if cfg.Proxy == nil {
|
||||
return nil
|
||||
}
|
||||
if cfg.Proxy.Enable == false || cfg.Proxy.Scheme == "" || cfg.Proxy.Host == "" {
|
||||
return nil
|
||||
}
|
||||
return util.BuildProxyUrl(cfg.Proxy.Scheme, cfg.Proxy.Host, cfg.Proxy.Usr, cfg.Proxy.Pwd)
|
||||
}
|
||||
|
||||
type DownloaderProxyConfig struct {
|
||||
Enable bool `json:"enable"`
|
||||
Scheme string `json:"scheme"`
|
||||
Host string `json:"host"`
|
||||
Usr string `json:"usr"`
|
||||
Pwd string `json:"pwd"`
|
||||
}
|
||||
Reference in new issue
Block a user