feat: task separate proxy (#727)

This commit is contained in:
Levi authored and GitHub committed 2024-09-06 18:35:23 +08:00
1 parent 6538b29b62
commit 0c17093b1a
22 files changed
+447 -67

No files matched your search

+16 -1
View File
@@ -12,6 +12,8 @@ import (
"github.com/rs/zerolog"
"github.com/rs/zerolog/pkgerrors"
"github.com/virtuald/go-paniclog"
gohttp "net/http"
"net/url"
"os"
"path/filepath"
"sort"
@@ -253,7 +255,20 @@ func (d *Downloader) setupFetcher(fm fetcher.FetcherManager, fetcher fetcher.Fet
ctl.GetConfig = func(v any) {
d.getProtocolConfig(fm.Name(), v)
}
ctl.ProxyConfig = d.cfg.Proxy
// Get proxy config, task request proxy config has higher priority, then use global proxy config
ctl.GetProxy = func(requestProxy *base.RequestProxy) func(*gohttp.Request) (*url.URL, error) {
if requestProxy == nil {
return d.cfg.Proxy.ToHandler()
}
switch requestProxy.Mode {
case base.RequestProxyModeNone:
return nil
case base.RequestProxyModeCustom:
return requestProxy.ToHandler()
default:
return d.cfg.Proxy.ToHandler()
}
}
fetcher.Setup(ctl)
}
+41 -12
View File
@@ -159,21 +159,21 @@ func TestDownloader_CreateDirectBatch(t *testing.T) {
func TestDownloader_CreateWithProxy(t *testing.T) {
// No proxy
doTestDownloaderCreateWithProxy(t, false, func(proxyCfg *base.DownloaderProxyConfig) *base.DownloaderProxyConfig {
doTestDownloaderCreateWithProxy(t, false, nil, func(proxyCfg *base.DownloaderProxyConfig) *base.DownloaderProxyConfig {
return nil
}, nil)
// Disable proxy
doTestDownloaderCreateWithProxy(t, false, func(proxyCfg *base.DownloaderProxyConfig) *base.DownloaderProxyConfig {
doTestDownloaderCreateWithProxy(t, false, nil, func(proxyCfg *base.DownloaderProxyConfig) *base.DownloaderProxyConfig {
proxyCfg.Enable = false
return proxyCfg
}, nil)
// Enable system proxy but not set proxy environment variable
doTestDownloaderCreateWithProxy(t, false, func(proxyCfg *base.DownloaderProxyConfig) *base.DownloaderProxyConfig {
doTestDownloaderCreateWithProxy(t, false, nil, func(proxyCfg *base.DownloaderProxyConfig) *base.DownloaderProxyConfig {
proxyCfg.System = true
return proxyCfg
}, nil)
// Enable proxy but error proxy environment variable
doTestDownloaderCreateWithProxy(t, false, func(proxyCfg *base.DownloaderProxyConfig) *base.DownloaderProxyConfig {
doTestDownloaderCreateWithProxy(t, false, nil, func(proxyCfg *base.DownloaderProxyConfig) *base.DownloaderProxyConfig {
os.Setenv("HTTP_PROXY", "http://127.0.0.1:1234")
os.Setenv("HTTPS_PROXY", "http://127.0.0.1:1234")
proxyCfg.System = true
@@ -184,33 +184,50 @@ func TestDownloader_CreateWithProxy(t *testing.T) {
}
})
// Enable system proxy and set proxy environment variable
doTestDownloaderCreateWithProxy(t, false, func(proxyCfg *base.DownloaderProxyConfig) *base.DownloaderProxyConfig {
doTestDownloaderCreateWithProxy(t, false, nil, func(proxyCfg *base.DownloaderProxyConfig) *base.DownloaderProxyConfig {
os.Setenv("HTTP_PROXY", proxyCfg.ToUrl().String())
os.Setenv("HTTPS_PROXY", proxyCfg.ToUrl().String())
proxyCfg.System = true
return proxyCfg
}, nil)
// Invalid proxy scheme
doTestDownloaderCreateWithProxy(t, false, func(proxyCfg *base.DownloaderProxyConfig) *base.DownloaderProxyConfig {
doTestDownloaderCreateWithProxy(t, false, nil, func(proxyCfg *base.DownloaderProxyConfig) *base.DownloaderProxyConfig {
proxyCfg.Scheme = ""
return proxyCfg
}, nil)
// Invalid proxy host
doTestDownloaderCreateWithProxy(t, false, func(proxyCfg *base.DownloaderProxyConfig) *base.DownloaderProxyConfig {
doTestDownloaderCreateWithProxy(t, false, nil, func(proxyCfg *base.DownloaderProxyConfig) *base.DownloaderProxyConfig {
proxyCfg.Host = ""
return proxyCfg
}, nil)
// Use proxy without auth
doTestDownloaderCreateWithProxy(t, false, func(proxyCfg *base.DownloaderProxyConfig) *base.DownloaderProxyConfig {
doTestDownloaderCreateWithProxy(t, false, nil, func(proxyCfg *base.DownloaderProxyConfig) *base.DownloaderProxyConfig {
return proxyCfg
}, nil)
// Use proxy with auth
doTestDownloaderCreateWithProxy(t, true, func(proxyCfg *base.DownloaderProxyConfig) *base.DownloaderProxyConfig {
doTestDownloaderCreateWithProxy(t, true, nil, func(proxyCfg *base.DownloaderProxyConfig) *base.DownloaderProxyConfig {
return proxyCfg
}, nil)
// Request proxy mode follow
doTestDownloaderCreateWithProxy(t, false, func(reqProxy *base.RequestProxy) *base.RequestProxy {
reqProxy.Mode = base.RequestProxyModeFollow
return reqProxy
}, nil, nil)
// Request proxy mode none
doTestDownloaderCreateWithProxy(t, false, func(reqProxy *base.RequestProxy) *base.RequestProxy {
reqProxy.Mode = base.RequestProxyModeNone
return reqProxy
}, nil, nil)
// Request proxy mode custom
doTestDownloaderCreateWithProxy(t, false, func(reqProxy *base.RequestProxy) *base.RequestProxy {
return reqProxy
}, nil, nil)
}
func doTestDownloaderCreateWithProxy(t *testing.T, auth bool, buildProxyConfig func(proxyCfg *base.DownloaderProxyConfig) *base.DownloaderProxyConfig, errHandler func(err error)) {
func doTestDownloaderCreateWithProxy(t *testing.T, auth bool, buildReqProxy func(reqProxy *base.RequestProxy) *base.RequestProxy, buildProxyConfig func(proxyCfg *base.DownloaderProxyConfig) *base.DownloaderProxyConfig, errHandler func(err error)) {
usr, pwd := "", ""
if auth {
usr, pwd = "admin", "123"
@@ -223,17 +240,29 @@ func doTestDownloaderCreateWithProxy(t *testing.T, auth bool, buildProxyConfig f
t.Fatal(err)
}
defer downloader.Clear()
downloader.cfg.DownloaderStoreConfig.Proxy = buildProxyConfig(&base.DownloaderProxyConfig{
globalProxyCfg := &base.DownloaderProxyConfig{
Enable: true,
Scheme: "socks5",
Host: proxyListener.Addr().String(),
Usr: usr,
Pwd: pwd,
})
}
if buildProxyConfig != nil {
globalProxyCfg = buildProxyConfig(globalProxyCfg)
}
downloader.cfg.DownloaderStoreConfig.Proxy = globalProxyCfg
req := &base.Request{
URL: test.ExternalDownloadUrl,
}
if buildReqProxy != nil {
req.Proxy = buildReqProxy(&base.RequestProxy{
Scheme: "socks5",
Host: proxyListener.Addr().String(),
Usr: usr,
Pwd: pwd,
})
}
rr, err := downloader.Resolve(req)
if err != nil {
if errHandler == nil {
+1 -1
View File
@@ -106,7 +106,7 @@ func (e *Engine) await(value any) {
}
func (e *Engine) Close() {
e.loop.Stop()
e.loop.StopNoWait()
}
type Config struct {
+5 -1
View File
@@ -694,8 +694,12 @@ type ExtensionTask struct {
}
func NewExtensionTask(download *Downloader, task *Task) *ExtensionTask {
newTask := task.clone()
// Assign the pointer of the properties that the extension supports modification
newTask.Meta = task.Meta
newTask.Status = task.Status
return &ExtensionTask{
Task: task.clone(),
Task: newTask,
download: download,
}
}
+1 -10
View File
@@ -97,16 +97,7 @@ func (t *Task) updateStatus(status base.Status) {
}
func (t *Task) clone() *Task {
return &Task{
ID: t.ID,
Protocol: t.Protocol,
Meta: t.Meta,
Status: t.Status,
Uploading: t.Uploading,
Progress: t.Progress,
CreatedAt: t.CreatedAt,
UpdatedAt: t.UpdatedAt,
}
return util.DeepClone(t)
}
func (t *Task) calcSpeed(speedArr []int64, downloaded int64, usedTime float64) int64 {