feat: task separate proxy (#727)

pkg/base/model.go

@@ -16,6 +16,8 @@ type Request struct {
 	Extra any    `json:"extra"`
 	// Labels is used to mark the download task
 	Labels map[string]string `json:"labels"`
+	// Proxy is special proxy config for request
+	Proxy *RequestProxy `json:"proxy"`
 }
 
 func (r *Request) Validate() error {
@@ -25,6 +27,37 @@ func (r *Request) Validate() error {
 	return nil
 }
 
+type RequestProxyMode string
+
+const (
+	// RequestProxyModeFollow follow setting proxy
+	RequestProxyModeFollow RequestProxyMode = "follow"
+	// RequestProxyModeNone not use proxy
+	RequestProxyModeNone RequestProxyMode = "none"
+	// RequestProxyModeCustom custom proxy
+	RequestProxyModeCustom RequestProxyMode = "custom"
+)
+
+type RequestProxy struct {
+	Mode   RequestProxyMode `json:"mode"`
+	Scheme string           `json:"scheme"`
+	Host   string           `json:"host"`
+	Usr    string           `json:"usr"`
+	Pwd    string           `json:"pwd"`
+}
+
+func (p *RequestProxy) ToHandler() func(r *http.Request) (*url.URL, error) {
+	if p == nil || p.Mode != RequestProxyModeCustom {
+		return nil
+	}
+
+	if p.Scheme == "" || p.Host == "" {
+		return nil
+	}
+
+	return http.ProxyURL(util.BuildProxyUrl(p.Scheme, p.Host, p.Usr, p.Pwd))
+}
+
 // Resource download resource
 type Resource struct {
 	// if name is not empty, the resource is a folder and the name is the folder name
@@ -94,15 +127,7 @@ func (o *Options) InitSelectFiles(fileSize int) {
 }
 
 func (o *Options) Clone() *Options {
-	if o == nil {
-		return nil
-	}
-	return &Options{
-		Name:        o.Name,
-		Path:        o.Path,
-		SelectFiles: o.SelectFiles,
-		Extra:       o.Extra,
-	}
+	return util.DeepClone(o)
 }
 
 func ParseReqExtra[E any](req *Request) error {

pkg/download/downloader.go

@@ -12,6 +12,8 @@ import (
 	"github.com/rs/zerolog"
 	"github.com/rs/zerolog/pkgerrors"
 	"github.com/virtuald/go-paniclog"
+	gohttp "net/http"
+	"net/url"
 	"os"
 	"path/filepath"
 	"sort"
@@ -253,7 +255,20 @@ func (d *Downloader) setupFetcher(fm fetcher.FetcherManager, fetcher fetcher.Fet
 	ctl.GetConfig = func(v any) {
 		d.getProtocolConfig(fm.Name(), v)
 	}
-	ctl.ProxyConfig = d.cfg.Proxy
+	// Get proxy config, task request proxy config has higher priority, then use global proxy config
+	ctl.GetProxy = func(requestProxy *base.RequestProxy) func(*gohttp.Request) (*url.URL, error) {
+		if requestProxy == nil {
+			return d.cfg.Proxy.ToHandler()
+		}
+		switch requestProxy.Mode {
+		case base.RequestProxyModeNone:
+			return nil
+		case base.RequestProxyModeCustom:
+			return requestProxy.ToHandler()
+		default:
+			return d.cfg.Proxy.ToHandler()
+		}
+	}
 	fetcher.Setup(ctl)
 }
 

pkg/download/downloader_test.go

@@ -159,21 +159,21 @@ func TestDownloader_CreateDirectBatch(t *testing.T) {
 
 func TestDownloader_CreateWithProxy(t *testing.T) {
 	// No proxy
-	doTestDownloaderCreateWithProxy(t, false, func(proxyCfg *base.DownloaderProxyConfig) *base.DownloaderProxyConfig {
+	doTestDownloaderCreateWithProxy(t, false, nil, func(proxyCfg *base.DownloaderProxyConfig) *base.DownloaderProxyConfig {
 		return nil
 	}, nil)
 	// Disable proxy
-	doTestDownloaderCreateWithProxy(t, false, func(proxyCfg *base.DownloaderProxyConfig) *base.DownloaderProxyConfig {
+	doTestDownloaderCreateWithProxy(t, false, nil, func(proxyCfg *base.DownloaderProxyConfig) *base.DownloaderProxyConfig {
 		proxyCfg.Enable = false
 		return proxyCfg
 	}, nil)
 	// Enable system proxy but not set proxy environment variable
-	doTestDownloaderCreateWithProxy(t, false, func(proxyCfg *base.DownloaderProxyConfig) *base.DownloaderProxyConfig {
+	doTestDownloaderCreateWithProxy(t, false, nil, func(proxyCfg *base.DownloaderProxyConfig) *base.DownloaderProxyConfig {
 		proxyCfg.System = true
 		return proxyCfg
 	}, nil)
 	// Enable proxy but error proxy environment variable
-	doTestDownloaderCreateWithProxy(t, false, func(proxyCfg *base.DownloaderProxyConfig) *base.DownloaderProxyConfig {
+	doTestDownloaderCreateWithProxy(t, false, nil, func(proxyCfg *base.DownloaderProxyConfig) *base.DownloaderProxyConfig {
 		os.Setenv("HTTP_PROXY", "http://127.0.0.1:1234")
 		os.Setenv("HTTPS_PROXY", "http://127.0.0.1:1234")
 		proxyCfg.System = true
@@ -184,33 +184,50 @@ func TestDownloader_CreateWithProxy(t *testing.T) {
 		}
 	})
 	// Enable system proxy and set proxy environment variable
-	doTestDownloaderCreateWithProxy(t, false, func(proxyCfg *base.DownloaderProxyConfig) *base.DownloaderProxyConfig {
+	doTestDownloaderCreateWithProxy(t, false, nil, func(proxyCfg *base.DownloaderProxyConfig) *base.DownloaderProxyConfig {
 		os.Setenv("HTTP_PROXY", proxyCfg.ToUrl().String())
 		os.Setenv("HTTPS_PROXY", proxyCfg.ToUrl().String())
 		proxyCfg.System = true
 		return proxyCfg
 	}, nil)
 	// Invalid proxy scheme
-	doTestDownloaderCreateWithProxy(t, false, func(proxyCfg *base.DownloaderProxyConfig) *base.DownloaderProxyConfig {
+	doTestDownloaderCreateWithProxy(t, false, nil, func(proxyCfg *base.DownloaderProxyConfig) *base.DownloaderProxyConfig {
 		proxyCfg.Scheme = ""
 		return proxyCfg
 	}, nil)
 	// Invalid proxy host
-	doTestDownloaderCreateWithProxy(t, false, func(proxyCfg *base.DownloaderProxyConfig) *base.DownloaderProxyConfig {
+	doTestDownloaderCreateWithProxy(t, false, nil, func(proxyCfg *base.DownloaderProxyConfig) *base.DownloaderProxyConfig {
 		proxyCfg.Host = ""
 		return proxyCfg
 	}, nil)
 	// Use proxy without auth
-	doTestDownloaderCreateWithProxy(t, false, func(proxyCfg *base.DownloaderProxyConfig) *base.DownloaderProxyConfig {
+	doTestDownloaderCreateWithProxy(t, false, nil, func(proxyCfg *base.DownloaderProxyConfig) *base.DownloaderProxyConfig {
 		return proxyCfg
 	}, nil)
 	// Use proxy with auth
-	doTestDownloaderCreateWithProxy(t, true, func(proxyCfg *base.DownloaderProxyConfig) *base.DownloaderProxyConfig {
+	doTestDownloaderCreateWithProxy(t, true, nil, func(proxyCfg *base.DownloaderProxyConfig) *base.DownloaderProxyConfig {
 		return proxyCfg
 	}, nil)
+
+	// Request proxy mode follow
+	doTestDownloaderCreateWithProxy(t, false, func(reqProxy *base.RequestProxy) *base.RequestProxy {
+		reqProxy.Mode = base.RequestProxyModeFollow
+		return reqProxy
+	}, nil, nil)
+
+	// Request proxy mode none
+	doTestDownloaderCreateWithProxy(t, false, func(reqProxy *base.RequestProxy) *base.RequestProxy {
+		reqProxy.Mode = base.RequestProxyModeNone
+		return reqProxy
+	}, nil, nil)
+
+	// Request proxy mode custom
+	doTestDownloaderCreateWithProxy(t, false, func(reqProxy *base.RequestProxy) *base.RequestProxy {
+		return reqProxy
+	}, nil, nil)
 }
 
-func doTestDownloaderCreateWithProxy(t *testing.T, auth bool, buildProxyConfig func(proxyCfg *base.DownloaderProxyConfig) *base.DownloaderProxyConfig, errHandler func(err error)) {
+func doTestDownloaderCreateWithProxy(t *testing.T, auth bool, buildReqProxy func(reqProxy *base.RequestProxy) *base.RequestProxy, buildProxyConfig func(proxyCfg *base.DownloaderProxyConfig) *base.DownloaderProxyConfig, errHandler func(err error)) {
 	usr, pwd := "", ""
 	if auth {
 		usr, pwd = "admin", "123"
@@ -223,17 +240,29 @@ func doTestDownloaderCreateWithProxy(t *testing.T, auth bool, buildProxyConfig f
 		t.Fatal(err)
 	}
 	defer downloader.Clear()
-	downloader.cfg.DownloaderStoreConfig.Proxy = buildProxyConfig(&base.DownloaderProxyConfig{
+	globalProxyCfg := &base.DownloaderProxyConfig{
 		Enable: true,
 		Scheme: "socks5",
 		Host:   proxyListener.Addr().String(),
 		Usr:    usr,
 		Pwd:    pwd,
-	})
+	}
+	if buildProxyConfig != nil {
+		globalProxyCfg = buildProxyConfig(globalProxyCfg)
+	}
+	downloader.cfg.DownloaderStoreConfig.Proxy = globalProxyCfg
 
 	req := &base.Request{
 		URL: test.ExternalDownloadUrl,
 	}
+	if buildReqProxy != nil {
+		req.Proxy = buildReqProxy(&base.RequestProxy{
+			Scheme: "socks5",
+			Host:   proxyListener.Addr().String(),
+			Usr:    usr,
+			Pwd:    pwd,
+		})
+	}
 	rr, err := downloader.Resolve(req)
 	if err != nil {
 		if errHandler == nil {

pkg/download/engine/engine.go

@@ -106,7 +106,7 @@ func (e *Engine) await(value any) {
 }
 
 func (e *Engine) Close() {
-	e.loop.Stop()
+	e.loop.StopNoWait()
 }
 
 type Config struct {

pkg/download/extension.go

@@ -694,8 +694,12 @@ type ExtensionTask struct {
 }
 
 func NewExtensionTask(download *Downloader, task *Task) *ExtensionTask {
+	newTask := task.clone()
+	// Assign the pointer of the properties that the extension supports modification
+	newTask.Meta = task.Meta
+	newTask.Status = task.Status
 	return &ExtensionTask{
-		Task:     task.clone(),
+		Task:     newTask,
 		download: download,
 	}
 }

pkg/download/model.go

@@ -97,16 +97,7 @@ func (t *Task) updateStatus(status base.Status) {
 }
 
 func (t *Task) clone() *Task {
-	return &Task{
-		ID:        t.ID,
-		Protocol:  t.Protocol,
-		Meta:      t.Meta,
-		Status:    t.Status,
-		Uploading: t.Uploading,
-		Progress:  t.Progress,
-		CreatedAt: t.CreatedAt,
-		UpdatedAt: t.UpdatedAt,
-	}
+	return util.DeepClone(t)
 }
 
 func (t *Task) calcSpeed(speedArr []int64, downloaded int64, usedTime float64) int64 {

pkg/util/json.go

@@ -12,3 +12,17 @@ func MapToStruct(s any, v any) error {
 	}
 	return json.Unmarshal(b, v)
 }
+
+func DeepClone[T any](v *T) *T {
+	if v == nil {
+		return nil
+	}
+
+	var t T
+	b, err := json.Marshal(v)
+	if err != nil {
+		return &t
+	}
+	json.Unmarshal(b, &t)
+	return &t
+}

pkg/util/json_test.go

@@ -0,0 +1,60 @@
+package util
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestDeepClone(t *testing.T) {
+	type user struct {
+		Name string `json:"name"`
+		Age  int    `json:"age"`
+
+		v int
+	}
+
+	type args[T any] struct {
+		v *T
+	}
+	type testCase[T any] struct {
+		name string
+		args args[T]
+		want *T
+	}
+	tests := []testCase[user]{
+		{
+			name: "case 1",
+			args: args[user]{
+				v: &user{
+					Name: "test",
+					Age:  10,
+				},
+			},
+			want: &user{
+				Name: "test",
+				Age:  10,
+			},
+		},
+		{
+			name: "case 2",
+			args: args[user]{
+				v: &user{
+					Name: "test",
+					Age:  10,
+					v:    1,
+				},
+			},
+			want: &user{
+				Name: "test",
+				Age:  10,
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := DeepClone(tt.args.v); !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("DeepClone() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}