feat: bt improve

pkg/base/model.go

@@ -11,12 +11,16 @@ type Request struct {
 // 资源信息
 type Resource struct {
 	Req *Request `json:"req"`
+	// 资源名称
+	Name string `json:"name"`
 	// 资源总大小
 	Size int64 `json:"size"`
 	// 是否支持断点下载
 	Range bool `json:"range"`
 	// 资源所包含的文件列表
 	Files []*FileInfo `json:"files"`
+	// 附加信息
+	Extra interface{} `json:"extra"`
 }
 
 type FileInfo struct {
@@ -34,5 +38,5 @@ type Options struct {
 	// 并发连接数
 	Connections int `json:"connections"`
 	// 选择下载的文件下标列表
-	SelectFiles []int `json:"select_files"`
+	SelectFiles []int `json:"selectFiles"`
 }

pkg/download/downloader.go

@@ -10,9 +10,16 @@ import (
 	"github.com/monkeyWie/gopeed-core/pkg/util"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"time"
 )
 
+const (
+	bucketTask   = "task"
+	bucketSave   = "save"
+	bucketConfig = "config"
+)
+
 type Listener func(event *Event)
 
 type Progress struct {
@@ -25,46 +32,133 @@ type Progress struct {
 }
 
 type Downloader struct {
-	*controller.DefaultController
+	controller    *controller.DefaultController
 	fetchBuilders map[string]fetcher.FetcherBuilder
+	storage       Storage
 	tasks         []*Task
 	listener      Listener
+
+	refreshInterval int
+	lock            *sync.Mutex
+	closed          atomic.Bool
 }
 
-func NewDownloader(fbs ...fetcher.FetcherBuilder) *Downloader {
-	d := &Downloader{DefaultController: controller.NewController()}
-	d.fetchBuilders = make(map[string]fetcher.FetcherBuilder)
-	for _, f := range fbs {
+type DownloaderConfig struct {
+	Controller    *controller.DefaultController
+	FetchBuilders []fetcher.FetcherBuilder
+	Storage       Storage
+
+	// RefreshInterval time duration to refresh task progress(ms)
+	RefreshInterval int
+}
+
+func (cfg *DownloaderConfig) Init() *DownloaderConfig {
+	if cfg.Controller == nil {
+		cfg.Controller = controller.NewController()
+	}
+	if len(cfg.FetchBuilders) == 0 {
+		cfg.FetchBuilders = []fetcher.FetcherBuilder{
+			new(http.FetcherBuilder),
+			new(bt.FetcherBuilder),
+		}
+	}
+	if cfg.Storage == nil {
+		cfg.Storage = NewMemStorage()
+	}
+	if cfg.RefreshInterval == 0 {
+		cfg.RefreshInterval = 1000
+	}
+	return cfg
+}
+
+func NewDownloader(cfg *DownloaderConfig) *Downloader {
+	if cfg == nil {
+		cfg = &DownloaderConfig{}
+	}
+	cfg.Init()
+
+	d := &Downloader{
+		controller:      cfg.Controller,
+		fetchBuilders:   make(map[string]fetcher.FetcherBuilder),
+		refreshInterval: cfg.RefreshInterval,
+		storage:         cfg.Storage,
+		lock:            &sync.Mutex{},
+	}
+	for _, f := range cfg.FetchBuilders {
 		for _, p := range f.Schemes() {
 			d.fetchBuilders[strings.ToUpper(p)] = f
 		}
 	}
-	d.tasks = make([]*Task, 0)
-
-	// 每秒统计一次下载速度
-	go func() {
-		for {
-			if len(d.tasks) > 0 {
-				for _, task := range d.tasks {
-					if task.Status == base.DownloadStatusDone ||
-						task.Status == base.DownloadStatusError ||
-						task.Status == base.DownloadStatusPause {
-						continue
-					}
-					current := task.fetcher.Progress().TotalDownloaded()
-					task.Progress.Used = task.timer.Used()
-					task.Progress.Speed = current - task.Progress.Downloaded
-					task.Progress.Downloaded = current
-					d.emit(EventKeyProgress, task)
-				}
-			}
-			time.Sleep(time.Second)
-		}
-	}()
 	return d
 }
 
-func (d *Downloader) buildFetcher(url string) (fetcher.Fetcher, error) {
+func (d *Downloader) Setup() error {
+	// setup storage
+	if err := d.storage.Setup([]string{bucketTask, bucketSave, bucketConfig}); err != nil {
+		return err
+	}
+	// load tasks from storage
+	var tasks []*Task
+	if err := d.storage.List(bucketTask, &tasks); err != nil {
+		return err
+	}
+	if tasks == nil {
+		tasks = make([]*Task, 0)
+	} else {
+		for _, task := range tasks {
+			initTask(task)
+			if task.Status != base.DownloadStatusDone && task.Status != base.DownloadStatusError {
+				task.Status = base.DownloadStatusPause
+			}
+		}
+	}
+	d.tasks = tasks
+
+	// 每秒统计一次下载速度
+	go func() {
+		for !d.closed.Load() {
+			if len(d.tasks) > 0 {
+				for _, task := range d.tasks {
+					func() {
+						task.lock.Lock()
+						defer task.lock.Unlock()
+						if task.Status == base.DownloadStatusDone ||
+							task.Status == base.DownloadStatusError ||
+							task.Status == base.DownloadStatusPause {
+							return
+						}
+						// check if task is deleted
+						if d.GetTask(task.ID) == nil {
+							return
+						}
+
+						current := task.fetcher.Progress().TotalDownloaded()
+						task.Progress.Used = task.timer.Used()
+						task.Progress.Speed = current - task.Progress.Downloaded
+						task.Progress.Downloaded = current
+						d.emit(EventKeyProgress, task)
+
+						// store fetcher progress
+						data, err := task.fetcherBuilder.Store(task.fetcher)
+						if err != nil {
+							return // TODO log
+						}
+						if err := d.storage.Put(bucketSave, task.ID, data); err != nil {
+							return // TODO log
+						}
+						if err := d.storage.Put(bucketTask, task.ID, task); err != nil {
+							return // TODO log
+						}
+					}()
+				}
+			}
+			time.Sleep(time.Millisecond * time.Duration(d.refreshInterval))
+		}
+	}()
+	return nil
+}
+
+func (d *Downloader) buildFetcher(url string) (fetcher.FetcherBuilder, fetcher.Fetcher, error) {
 	schema := util.ParseSchema(url)
 	fetchBuilder, ok := d.fetchBuilders[schema]
 	if !ok {
@@ -72,14 +166,14 @@ func (d *Downloader) buildFetcher(url string) (fetcher.Fetcher, error) {
 	}
 	if ok {
 		fetcher := fetchBuilder.Build()
-		fetcher.Setup(d.DefaultController)
-		return fetcher, nil
+		fetcher.Setup(d.controller)
+		return fetchBuilder, fetcher, nil
 	}
-	return nil, errors.New("unsupported protocol")
+	return nil, nil, errors.New("unsupported protocol")
 }
 
 func (d *Downloader) Resolve(req *base.Request) (*base.Resource, error) {
-	fetcher, err := d.buildFetcher(req.URL)
+	_, fetcher, err := d.buildFetcher(req.URL)
 	if err != nil {
 		return nil, err
 	}
@@ -99,13 +193,13 @@ func (d *Downloader) Create(res *base.Resource, opts *base.Options) (taskId stri
 		opts.Connections = 1
 	}
 	if len(opts.SelectFiles) == 0 {
-		opts.SelectFiles = make([]int, len(res.Files))
-		for i := range opts.SelectFiles {
-			opts.SelectFiles[i] = i
+		opts.SelectFiles = make([]int, 0)
+		for i := range res.Files {
+			opts.SelectFiles = append(opts.SelectFiles, i)
 		}
 	}
 
-	fetcher, err := d.buildFetcher(res.Req.URL)
+	fetcherBuilder, fetcher, err := d.buildFetcher(res.Req.URL)
 	if err != nil {
 		return
 	}
@@ -115,74 +209,144 @@ func (d *Downloader) Create(res *base.Resource, opts *base.Options) (taskId stri
 	}
 
 	task := NewTask()
+	task.fetcherBuilder = fetcherBuilder
 	task.fetcher = fetcher
 	task.Res = res
 	task.Opts = opts
 	task.Progress = &Progress{}
-	task.timer = &util.Timer{}
-	task.locker = new(sync.Mutex)
-	task.timer.Start()
 	task.Status = base.DownloadStatusRunning
-	d.tasks = append(d.tasks, task)
-	d.emit(EventKeyStart, task)
-	taskId = task.ID
-
-	err = fetcher.Start()
+	// calculate select files size
+	for _, selectIndex := range opts.SelectFiles {
+		task.Size += res.Files[selectIndex].Size
+	}
+	initTask(task)
+	task.timer.Start()
+	err = func() error {
+		d.lock.Lock()
+		defer d.lock.Unlock()
+		if err := d.storage.Put(bucketTask, task.ID, task.clone()); err != nil {
+			return err
+		}
+		d.tasks = append(d.tasks, task)
+		return nil
+	}()
 	if err != nil {
 		return
 	}
+	d.emit(EventKeyStart, task)
+	taskId = task.ID
+
 	go func() {
-		err = fetcher.Wait()
+		err := fetcher.Start()
 		if err != nil {
-			task.Status = base.DownloadStatusError
 			d.emit(EventKeyError, task, err)
-		} else {
-			task.Progress.Used = task.timer.Used()
-			if task.Res.Size == 0 {
-				task.Res.Size = task.fetcher.Progress().TotalDownloaded()
-			}
-			used := task.Progress.Used / int64(time.Second)
-			if used == 0 {
-				used = 1
-			}
-			task.Progress.Speed = task.Res.Size / used
-			task.Progress.Downloaded = task.Res.Size
-			task.Status = base.DownloadStatusDone
-			d.emit(EventKeyDone, task)
+			return
 		}
-		d.emit(EventKeyFinally, task, err)
+		d.watch(task)
 	}()
 	return
 }
 
-func (d *Downloader) Pause(id string) {
+func (d *Downloader) Pause(id string) (err error) {
 	task := d.GetTask(id)
 	if task == nil {
 		return
 	}
-	task.locker.Lock()
-	defer task.locker.Unlock()
+	task.lock.Lock()
+	defer task.lock.Unlock()
 	if task.Status == base.DownloadStatusRunning {
 		task.Status = base.DownloadStatusPause
 		task.timer.Pause()
 		task.fetcher.Pause()
+		d.storage.Put(bucketTask, task.ID, task.clone())
 		d.emit(EventKeyPause, task)
 	}
+	return
 }
 
-func (d *Downloader) Continue(id string) {
+func (d *Downloader) Continue(id string) (err error) {
 	task := d.GetTask(id)
 	if task == nil {
 		return
 	}
-	task.locker.Lock()
-	defer task.locker.Unlock()
+	task.lock.Lock()
+	defer task.lock.Unlock()
 	if task.Status == base.DownloadStatusPause || task.Status == base.DownloadStatusError {
 		task.Status = base.DownloadStatusRunning
-		task.timer.Continue()
-		task.fetcher.Continue()
+		err = d.restoreFetcher(task)
+		if err != nil {
+			return
+		}
+		task.timer.Start()
+		err = task.fetcher.Continue()
+		if err != nil {
+			return
+		}
+		d.storage.Put(bucketTask, task.ID, task.clone())
 		d.emit(EventKeyContinue, task)
 	}
+	return
+}
+
+func (d *Downloader) Delete(id string, force bool) (err error) {
+	d.lock.Lock()
+	defer d.lock.Unlock()
+	task := d.GetTask(id)
+	if task == nil {
+		return
+	}
+	task.lock.Lock()
+	defer task.lock.Unlock()
+	if task.fetcher != nil {
+		err = task.fetcher.Close()
+		if err != nil {
+			return
+		}
+	}
+	for i, t := range d.tasks {
+		if t.ID == id {
+			d.tasks = append(d.tasks[:i], d.tasks[i+1:]...)
+			break
+		}
+	}
+	err = d.storage.Delete(bucketTask, id)
+	if err != nil {
+		return
+	}
+	err = d.storage.Delete(bucketSave, id)
+	if err != nil {
+		return
+	}
+	if force {
+		names := make([]string, 0)
+		for _, file := range task.Res.Files {
+			names = append(names, util.Filepath(file.Path, file.Name, task.Opts.Name))
+		}
+		util.SafeRemoveAll(task.Opts.Path, names)
+	}
+	d.emit(EventKeyDelete, task)
+	task = nil
+	return
+}
+
+func (d *Downloader) Close() error {
+	d.lock.Lock()
+	defer d.lock.Unlock()
+	d.closed.Store(true)
+	for _, task := range d.tasks {
+		d.Pause(task.ID)
+	}
+	return d.storage.Close()
+}
+
+func (d *Downloader) Clear() error {
+	if err := d.Close(); err != nil {
+		return err
+	}
+	if err := d.storage.Clear(); err != nil {
+		return err
+	}
+	return nil
 }
 
 func (d *Downloader) Listener(fn Listener) {
@@ -216,7 +380,75 @@ func (d *Downloader) GetTasks() []*Task {
 	return d.tasks
 }
 
-var defaultDownloader = NewDownloader(new(http.FetcherBuilder), new(bt.FetcherBuilder))
+func (d *Downloader) GetConfig(v any) error {
+	return d.storage.Get(bucketConfig, "config", v)
+}
+
+func (d *Downloader) PutConfig(v any) error {
+	return d.storage.Put(bucketConfig, "config", v)
+}
+
+func (d *Downloader) watch(task *Task) {
+	err := task.fetcher.Wait()
+	if err != nil {
+		task.Status = base.DownloadStatusError
+		d.emit(EventKeyError, task, err)
+	} else {
+		task.Progress.Used = task.timer.Used()
+		if task.Size == 0 {
+			task.Size = task.fetcher.Progress().TotalDownloaded()
+			task.Res.Size = task.Size
+		}
+		used := task.Progress.Used / int64(time.Second)
+		if used == 0 {
+			used = 1
+		}
+		task.Progress.Speed = task.Size / used
+		task.Progress.Downloaded = task.Size
+		task.Status = base.DownloadStatusDone
+		d.storage.Put(bucketTask, task.ID, task.clone())
+		d.emit(EventKeyDone, task)
+	}
+	d.emit(EventKeyFinally, task, err)
+}
+
+func (d *Downloader) restoreFetcher(task *Task) error {
+	if task.fetcher == nil {
+		fetcherBuilder, _, err := d.buildFetcher(task.Res.Req.URL)
+		if err != nil {
+			return err
+		}
+		task.fetcherBuilder = fetcherBuilder
+		err = func() error {
+			v, f := fetcherBuilder.Restore()
+			if v != nil {
+				err := d.storage.Pop(bucketSave, task.ID, v)
+				if err != nil {
+					return err
+				}
+				task.fetcher = f(task.Res, task.Opts, v)
+			}
+			return nil
+		}()
+		if err != nil {
+			// TODO log
+		}
+		if task.fetcher == nil {
+			task.fetcher = fetcherBuilder.Build()
+			task.fetcher.Create(task.Res, task.Opts)
+		}
+		task.fetcher.Setup(d.controller)
+		go d.watch(task)
+	}
+	return nil
+}
+
+func initTask(task *Task) {
+	task.timer = &util.Timer{}
+	task.lock = &sync.Mutex{}
+}
+
+var defaultDownloader = NewDownloader(nil)
 
 type boot struct {
 	url      string
@@ -256,5 +488,9 @@ func (b *boot) Create(opts *base.Options) (string, error) {
 }
 
 func Boot() *boot {
+	err := defaultDownloader.Setup()
+	if err != nil {
+		panic(err)
+	}
 	return &boot{}
 }

pkg/download/downloader_test.go

@@ -1,19 +1,23 @@
 package download
 
 import (
-	"github.com/monkeyWie/gopeed-core/internal/protocol/http"
+	"github.com/monkeyWie/gopeed-core/internal/test"
 	"github.com/monkeyWie/gopeed-core/pkg/base"
-	"github.com/monkeyWie/gopeed-core/pkg/test"
 	"reflect"
 	"sync"
 	"testing"
+	"time"
 )
 
 func TestDownloader_Resolve(t *testing.T) {
 	listener := test.StartTestFileServer()
 	defer listener.Close()
 
-	downloader := NewDownloader(new(http.FetcherBuilder))
+	downloader := NewDownloader(nil)
+	if err := downloader.Setup(); err != nil {
+		t.Fatal(err)
+	}
+	defer downloader.Clear()
 	req := &base.Request{
 		URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
 	}
@@ -22,6 +26,7 @@ func TestDownloader_Resolve(t *testing.T) {
 		t.Fatal(err)
 	}
 	want := &base.Resource{
+		Name:  test.BuildName,
 		Req:   req,
 		Size:  test.BuildSize,
 		Range: true,
@@ -42,7 +47,11 @@ func TestDownloader_Create(t *testing.T) {
 	listener := test.StartTestFileServer()
 	defer listener.Close()
 
-	downloader := NewDownloader(new(http.FetcherBuilder))
+	downloader := NewDownloader(nil)
+	if err := downloader.Setup(); err != nil {
+		t.Fatal(err)
+	}
+	defer downloader.Clear()
 	req := &base.Request{
 		URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
 	}
@@ -73,3 +82,69 @@ func TestDownloader_Create(t *testing.T) {
 		t.Errorf("Download() got = %v, want %v", got, want)
 	}
 }
+
+func TestDownloader_StoreAndRestore(t *testing.T) {
+	listener := test.StartTestSlowFileServer(time.Millisecond * 2000)
+	defer listener.Close()
+
+	downloader := NewDownloader(&DownloaderConfig{
+		Storage: NewBoltStorage("./"),
+	})
+	if err := downloader.Setup(); err != nil {
+		t.Fatal(err)
+	}
+	defer downloader.Clear()
+	req := &base.Request{
+		URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
+	}
+	res, err := downloader.Resolve(req)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	id, err := downloader.Create(res, &base.Options{
+		Path:        test.Dir,
+		Name:        test.DownloadName,
+		Connections: 4,
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	time.Sleep(time.Millisecond * 1001)
+	err = downloader.Pause(id)
+	if err != nil {
+		t.Fatal(err)
+	}
+	downloader.Close()
+
+	downloader = NewDownloader(&DownloaderConfig{
+		Storage: NewBoltStorage("./"),
+	})
+	if err := downloader.Setup(); err != nil {
+		t.Fatal(err)
+	}
+	task := downloader.GetTask(id)
+
+	if task == nil {
+		t.Fatal("task is nil")
+	}
+	var wg sync.WaitGroup
+	wg.Add(1)
+	downloader.Listener(func(event *Event) {
+		if event.Key == EventKeyDone {
+			wg.Done()
+		}
+	})
+	err = downloader.Continue(id)
+	wg.Wait()
+	if err != nil {
+		t.Fatal(err)
+	}
+	want := test.FileMd5(test.BuildFile)
+	got := test.FileMd5(test.DownloadFile)
+	if want != got {
+		t.Errorf("StoreAndResume() got = %v, want %v", got, want)
+	}
+
+	downloader.Clear()
+}

pkg/download/event.go

@@ -8,6 +8,7 @@ const (
 	EventKeyContinue = "continue"
 	EventKeyProgress = "progress"
 	EventKeyError    = "error"
+	EventKeyDelete   = "delete"
 	EventKeyDone     = "done"
 	EventKeyFinally  = "finally"
 )

pkg/download/storage.go

@@ -0,0 +1,219 @@
+package download
+
+import (
+	"encoding/json"
+	"go.etcd.io/bbolt"
+	"os"
+	"path/filepath"
+	"reflect"
+)
+
+type Storage interface {
+	Setup(buckets []string) error
+	Put(bucket string, key string, v any) error
+	Get(bucket string, key string, v any) error
+	List(bucket string, v any) error
+	Pop(bucket string, key string, v any) error
+	Delete(bucket string, key string) error
+
+	Close() error
+	Clear() error
+}
+
+func changeValue(p any, v any) {
+	if v == nil {
+		return
+	}
+	rp := reflect.ValueOf(p)
+	rv := reflect.ValueOf(v)
+	if rv.Kind() == reflect.Slice {
+		if rv.Len() == 0 {
+			return
+		}
+		// get underlying type
+		tp := reflect.TypeOf(p).Elem().Elem()
+		for i := 0; i < rv.Len(); i++ {
+			// convert to underlying type
+			vv := rv.Index(i).Elem().Convert(tp)
+			rp.Elem().Set(reflect.Append(rp.Elem(), vv))
+		}
+	} else if rv.Kind() == reflect.Ptr {
+		rp.Elem().Set(rv.Elem())
+	} else {
+		rp.Elem().Set(rv)
+	}
+}
+
+var memData = make(map[string]map[string]any)
+
+type MemStorage struct {
+}
+
+func NewMemStorage() *MemStorage {
+	return &MemStorage{}
+}
+
+func (n *MemStorage) Setup(buckets []string) error {
+	for _, bucket := range buckets {
+		if _, ok := memData[bucket]; !ok {
+			memData[bucket] = make(map[string]any)
+		}
+	}
+	return nil
+}
+
+func (n *MemStorage) Put(bucket string, key string, v any) error {
+	memData[bucket][key] = v
+	return nil
+}
+
+func (n *MemStorage) Get(bucket string, key string, v any) error {
+	changeValue(v, memData[bucket][key])
+	return nil
+}
+
+func (n *MemStorage) List(bucket string, v any) error {
+	data := memData[bucket]
+	list := make([]any, 0)
+	for _, v := range data {
+		list = append(list, v)
+	}
+	changeValue(v, list)
+	return nil
+}
+
+func (n *MemStorage) Pop(bucket string, key string, v any) error {
+	data := memData[bucket]
+	changeValue(v, data[key])
+	delete(data, key)
+	return nil
+}
+
+func (n *MemStorage) Delete(bucket string, key string) error {
+	delete(memData[bucket], key)
+	return nil
+}
+
+func (n *MemStorage) Close() error {
+	return nil
+}
+
+func (n *MemStorage) Clear() error {
+	memData = make(map[string]map[string]any)
+	return nil
+}
+
+const (
+	dbFile = "gopeed.db"
+)
+
+type BoltStorage struct {
+	db   *bbolt.DB
+	path string
+}
+
+func NewBoltStorage(dir string) *BoltStorage {
+	path := filepath.Join(dir, dbFile)
+	db, err := bbolt.Open(path, 0600, nil)
+	if err != nil {
+		panic(err)
+	}
+	return &BoltStorage{
+		db:   db,
+		path: path,
+	}
+}
+
+func (b *BoltStorage) Setup(buckets []string) error {
+	return b.db.Update(func(tx *bbolt.Tx) error {
+		for _, bucket := range buckets {
+			_, err := tx.CreateBucketIfNotExists([]byte(bucket))
+			if err != nil {
+				return err
+			}
+		}
+		return nil
+	})
+}
+
+func (b *BoltStorage) Put(bucket string, key string, v any) error {
+	buf, err := json.Marshal(v)
+	if err != nil {
+		return err
+	}
+	return b.db.Update(func(tx *bbolt.Tx) error {
+		b := tx.Bucket([]byte(bucket))
+		return b.Put([]byte(key), buf)
+	})
+}
+
+func (b *BoltStorage) Get(bucket string, key string, v any) error {
+	var data []byte
+	err := b.db.View(func(tx *bbolt.Tx) error {
+		b := tx.Bucket([]byte(bucket))
+		data = b.Get([]byte(key))
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+	return json.Unmarshal(data, v)
+}
+
+func (b *BoltStorage) List(bucket string, v any) error {
+	list := make([]any, 0)
+	tv := reflect.TypeOf(v).Elem().Elem()
+	if err := b.db.View(func(tx *bbolt.Tx) error {
+		b := tx.Bucket([]byte(bucket))
+		if err := b.ForEach(func(k, v []byte) error {
+			data := reflect.New(tv.Elem()).Interface()
+			if err := json.Unmarshal(v, &data); err != nil {
+				return err
+			}
+			list = append(list, data)
+			return nil
+		}); err != nil {
+			return err
+		}
+		return nil
+	}); err != nil {
+		return err
+	}
+	changeValue(v, list)
+	return nil
+}
+
+func (b *BoltStorage) Pop(bucket string, key string, v any) error {
+	var data []byte
+	err := b.db.Update(func(tx *bbolt.Tx) error {
+		b := tx.Bucket([]byte(bucket))
+		kb := []byte(key)
+		data = b.Get(kb)
+		return b.Delete(kb)
+	})
+	if err != nil {
+		return err
+	}
+	return json.Unmarshal(data, v)
+}
+
+func (b *BoltStorage) Delete(bucket string, key string) error {
+	return b.db.Update(func(tx *bbolt.Tx) error {
+		b := tx.Bucket([]byte(bucket))
+		return b.Delete([]byte(key))
+	})
+}
+
+func (b *BoltStorage) Close() error {
+	return b.db.Close()
+}
+
+func (b *BoltStorage) Clear() error {
+	if err := b.Close(); err != nil {
+		return err
+	}
+	if err := os.Remove(b.path); err != nil {
+		return err
+	}
+	return nil
+}

pkg/download/task.go

@@ -1,7 +1,7 @@
 package download
 
 import (
-	"github.com/google/uuid"
+	gonanoid "github.com/matoous/go-nanoid/v2"
 	"github.com/monkeyWie/gopeed-core/internal/fetcher"
 	"github.com/monkeyWie/gopeed-core/pkg/base"
 	"github.com/monkeyWie/gopeed-core/pkg/util"
@@ -10,23 +10,40 @@ import (
 )
 
 type Task struct {
-	ID         string                    `json:"id"`
-	Res        *base.Resource            `json:"res"`
-	Opts       *base.Options             `json:"opts"`
-	Status     base.Status               `json:"status"`
-	Files      map[string]*base.FileInfo `json:"files"`
-	Progress   *Progress                 `json:"progress"`
-	CreateTime time.Time                 `json:"create_time"`
+	ID        string         `json:"id"`
+	Res       *base.Resource `json:"res"`
+	Opts      *base.Options  `json:"opts"`
+	Status    base.Status    `json:"status"`
+	Progress  *Progress      `json:"progress"`
+	Size      int64          `json:"size"`
+	CreatedAt time.Time      `json:"createdAt"`
 
-	fetcher fetcher.Fetcher
-	timer   *util.Timer
-	locker  *sync.Mutex
+	fetcherBuilder fetcher.FetcherBuilder
+	fetcher        fetcher.Fetcher
+	timer          *util.Timer
+	lock           *sync.Mutex
 }
 
 func NewTask() *Task {
+	id, err := gonanoid.New()
+	if err != nil {
+		panic(err)
+	}
 	return &Task{
-		ID:         uuid.New().String(),
-		Status:     base.DownloadStatusReady,
-		CreateTime: time.Now(),
+		ID:        id,
+		Status:    base.DownloadStatusReady,
+		CreatedAt: time.Now(),
+	}
+}
+
+func (t *Task) clone() *Task {
+	return &Task{
+		ID:        t.ID,
+		Res:       t.Res,
+		Opts:      t.Opts,
+		Status:    t.Status,
+		Progress:  t.Progress,
+		Size:      t.Size,
+		CreatedAt: t.CreatedAt,
 	}
 }

pkg/rest/api.go

@@ -15,7 +15,7 @@ func Resolve(w http.ResponseWriter, r *http.Request) {
 	if util.ReadJson(w, r, &req) {
 		resource, err := Downloader.Resolve(&req)
 		if err != nil {
-			util.WriteJson(w, 500, model.NewResultWithMsg(err.Error()))
+			util.WriteJson(w, http.StatusInternalServerError, model.NewResultWithMsg(err.Error()))
 			return
 		}
 		util.WriteJsonOk(w, model.NewResultWithData(resource))
@@ -23,11 +23,17 @@ func Resolve(w http.ResponseWriter, r *http.Request) {
 }
 
 func CreateTask(w http.ResponseWriter, r *http.Request) {
-	var req model.CreateTaskReq
+	var req model.CreateTask
 	if util.ReadJson(w, r, &req) {
-		taskId, err := Downloader.Create(req.Resource, req.Options)
+		if req.Opts.Connections == 0 {
+			req.Opts.Connections = getServerConfig().Connections
+		}
+		if req.Opts.Path == "" {
+			req.Opts.Path = getServerConfig().DownloadDir
+		}
+		taskId, err := Downloader.Create(req.Res, req.Opts)
 		if err != nil {
-			util.WriteJson(w, 500, model.NewResultWithMsg(err.Error()))
+			util.WriteJson(w, http.StatusInternalServerError, model.NewResultWithMsg(err.Error()))
 			return
 		}
 		util.WriteJsonOk(w, model.NewResultWithData(taskId))
@@ -37,14 +43,31 @@ func CreateTask(w http.ResponseWriter, r *http.Request) {
 func PauseTask(w http.ResponseWriter, r *http.Request) {
 	vars := mux.Vars(r)
 	taskId := vars["id"]
-	Downloader.Pause(taskId)
+	if err := Downloader.Pause(taskId); err != nil {
+		util.WriteJson(w, http.StatusInternalServerError, model.NewResultWithMsg(err.Error()))
+		return
+	}
 	util.WriteJsonOk(w, nil)
 }
 
 func ContinueTask(w http.ResponseWriter, r *http.Request) {
 	vars := mux.Vars(r)
 	taskId := vars["id"]
-	Downloader.Continue(taskId)
+	if err := Downloader.Continue(taskId); err != nil {
+		util.WriteJson(w, http.StatusInternalServerError, model.NewResultWithMsg(err.Error()))
+		return
+	}
+	util.WriteJsonOk(w, nil)
+}
+
+func DeleteTask(w http.ResponseWriter, r *http.Request) {
+	vars := mux.Vars(r)
+	taskId := vars["id"]
+	force := r.FormValue("force")
+	if err := Downloader.Delete(taskId, force == "true"); err != nil {
+		util.WriteJson(w, http.StatusInternalServerError, model.NewResultWithMsg(err.Error()))
+		return
+	}
 	util.WriteJsonOk(w, nil)
 }
 
@@ -53,7 +76,7 @@ func GetTask(w http.ResponseWriter, r *http.Request) {
 	taskId := vars["id"]
 	task := Downloader.GetTask(taskId)
 	if task == nil {
-		util.WriteJson(w, 404, model.NewResultWithMsg("task not found"))
+		util.WriteJson(w, http.StatusNotFound, model.NewResultWithMsg("task not found"))
 		return
 	}
 	util.WriteJsonOk(w, model.NewResultWithData(task))
@@ -62,12 +85,12 @@ func GetTask(w http.ResponseWriter, r *http.Request) {
 func GetTasks(w http.ResponseWriter, r *http.Request) {
 	status := r.FormValue("status")
 	if status == "" {
-		util.WriteJson(w, 400, model.NewResultWithMsg("param is required: status"))
+		util.WriteJson(w, http.StatusBadRequest, model.NewResultWithMsg("param is required: status"))
 		return
 	}
 	statusArr := strings.Split(status, ",")
 	tasks := Downloader.GetTasks()
-	var result []*download.Task
+	result := make([]*download.Task, 0)
 	for _, task := range tasks {
 		for _, s := range statusArr {
 			if task.Status == base.Status(s) {
@@ -75,5 +98,26 @@ func GetTasks(w http.ResponseWriter, r *http.Request) {
 			}
 		}
 	}
-	util.WriteJsonOk(w, model.NewResultWithData(tasks))
+	util.WriteJsonOk(w, model.NewResultWithData(result))
+}
+
+func GetConfig(w http.ResponseWriter, r *http.Request) {
+	util.WriteJsonOk(w, model.NewResultWithData(getServerConfig()))
+}
+
+func PutConfig(w http.ResponseWriter, r *http.Request) {
+	var cfg model.ServerConfig
+	if util.ReadJson(w, r, &cfg) {
+		if err := Downloader.PutConfig(&cfg); err != nil {
+			util.WriteJson(w, http.StatusInternalServerError, model.NewResultWithMsg(err.Error()))
+			return
+		}
+	}
+	util.WriteJsonOk(w, nil)
+}
+
+func getServerConfig() *model.ServerConfig {
+	var cfg model.ServerConfig
+	Downloader.GetConfig(&cfg)
+	return &cfg
 }

pkg/rest/model/server.go

@@ -0,0 +1,53 @@
+package model
+
+type Storage string
+
+const (
+	StorageMem  Storage = "mem"
+	StorageBolt Storage = "bolt"
+)
+
+type StartConfig struct {
+	Network         string  `json:"network"`
+	Address         string  `json:"address"`
+	Storage         Storage `json:"storage"`
+	StorageDir      string  `json:"storageDir"`
+	RefreshInterval int     `json:"refreshInterval"`
+}
+
+func (cfg *StartConfig) Init() *StartConfig {
+	if cfg.Network == "" {
+		cfg.Network = "tcp"
+	}
+	if cfg.Storage == "" {
+		cfg.Storage = StorageBolt
+	}
+	if cfg.StorageDir == "" {
+		cfg.StorageDir = "./"
+	}
+	return cfg
+}
+
+// ServerConfig is present in the database
+type ServerConfig struct {
+	Host string `json:"host"`
+	Port int    `json:"port"`
+
+	Connections int    `json:"connections"`
+	DownloadDir string `json:"downloadDir"`
+
+	Extra map[string]any `json:"extra"`
+}
+
+func (cfg *ServerConfig) Init() *ServerConfig {
+	if cfg.Host == "" {
+		cfg.Host = "127.0.0.1"
+	}
+	if cfg.Port < 0 {
+		cfg.Port = 0
+	}
+	if cfg.Connections <= 0 {
+		cfg.Connections = 16
+	}
+	return cfg
+}

pkg/rest/model/task.go

@@ -2,7 +2,7 @@ package model
 
 import "github.com/monkeyWie/gopeed-core/pkg/base"
 
-type CreateTaskReq struct {
-	Resource *base.Resource `json:"resource"`
-	Options  *base.Options  `json:"options"`
+type CreateTask struct {
+	Res  *base.Resource `json:"res"`
+	Opts *base.Options  `json:"opts"`
 }

pkg/rest/server.go

@@ -2,23 +2,49 @@ package rest
 
 import (
 	"context"
-	"fmt"
 	"github.com/gorilla/mux"
-	"github.com/monkeyWie/gopeed-core/internal/protocol/bt"
-	fhttp "github.com/monkeyWie/gopeed-core/internal/protocol/http"
 	"github.com/monkeyWie/gopeed-core/pkg/download"
+	"github.com/monkeyWie/gopeed-core/pkg/rest/model"
+	"github.com/monkeyWie/gopeed-core/pkg/util"
 	"net"
 	"net/http"
+	"strconv"
 )
 
 var (
 	srv *http.Server
 
-	Downloader = download.NewDownloader(new(fhttp.FetcherBuilder), new(bt.FetcherBuilder))
+	Downloader *download.Downloader
 )
 
-func Start(addr string, port int) (int, error) {
-	listener, err := net.Listen("tcp", fmt.Sprintf("%s:%d", addr, port))
+func Start(startCfg *model.StartConfig) (int, error) {
+	if startCfg == nil {
+		startCfg = &model.StartConfig{}
+	}
+	startCfg.Init()
+
+	downloadCfg := &download.DownloaderConfig{}
+	if startCfg.Storage == model.StorageBolt {
+		downloadCfg.Storage = download.NewBoltStorage(startCfg.StorageDir)
+	} else {
+		downloadCfg.Storage = download.NewMemStorage()
+	}
+	downloadCfg.RefreshInterval = startCfg.RefreshInterval
+	Downloader = download.NewDownloader(downloadCfg.Init())
+	if err := Downloader.Setup(); err != nil {
+		return 0, err
+	}
+
+	serverConfig := getAndPutServerConfig(startCfg)
+	var address string
+	if startCfg.Network == "unix" {
+		address = startCfg.Address
+		util.SafeRemove(address)
+	} else {
+		address = net.JoinHostPort(serverConfig.Host, strconv.Itoa(serverConfig.Port))
+	}
+
+	listener, err := net.Listen(startCfg.Network, address)
 	if err != nil {
 		return 0, err
 	}
@@ -28,8 +54,11 @@ func Start(addr string, port int) (int, error) {
 	r.Methods(http.MethodPost).Path("/api/v1/tasks").HandlerFunc(CreateTask)
 	r.Methods(http.MethodPut).Path("/api/v1/tasks/{id}/pause").HandlerFunc(PauseTask)
 	r.Methods(http.MethodPut).Path("/api/v1/tasks/{id}/continue").HandlerFunc(ContinueTask)
+	r.Methods(http.MethodDelete).Path("/api/v1/tasks/{id}").HandlerFunc(DeleteTask)
 	r.Methods(http.MethodGet).Path("/api/v1/tasks/{id}").HandlerFunc(GetTask)
 	r.Methods(http.MethodGet).Path("/api/v1/tasks").HandlerFunc(GetTasks)
+	r.Methods(http.MethodGet).Path("/api/v1/config").HandlerFunc(GetConfig)
+	r.Methods(http.MethodPut).Path("/api/v1/config").HandlerFunc(PutConfig)
 
 	srv = &http.Server{Handler: r}
 	go func() {
@@ -38,12 +67,37 @@ func Start(addr string, port int) (int, error) {
 			panic(err)
 		}
 	}()
-	return listener.Addr().(*net.TCPAddr).Port, nil
+	if addr, ok := listener.Addr().(*net.TCPAddr); ok {
+		return addr.Port, nil
+	}
+	return 0, nil
 }
 
-func Stop() error {
+func Stop() {
 	if srv != nil {
-		return srv.Shutdown(context.TODO())
+		if err := srv.Shutdown(context.TODO()); err != nil {
+			// TODO log
+		}
+	}
+	if Downloader != nil {
+		if err := Downloader.Close(); err != nil {
+			// TODO log
+		}
 	}
-	return nil
+}
+
+func getAndPutServerConfig(startCfg *model.StartConfig) *model.ServerConfig {
+	var serverConfig model.ServerConfig
+	err := Downloader.GetConfig(&serverConfig)
+	// first start
+	if err != nil {
+		if startCfg.Network == "tcp" {
+			host, port, _ := net.SplitHostPort(startCfg.Address)
+			serverConfig.Host = host
+			serverConfig.Port, _ = strconv.Atoi(port)
+		}
+		serverConfig.Init()
+		Downloader.PutConfig(serverConfig)
+	}
+	return &serverConfig
 }

pkg/rest/server_test.go

@@ -3,17 +3,20 @@ package rest
 import (
 	"bytes"
 	"encoding/json"
+	"errors"
 	"fmt"
+	"github.com/monkeyWie/gopeed-core/internal/test"
 	"github.com/monkeyWie/gopeed-core/pkg/base"
 	"github.com/monkeyWie/gopeed-core/pkg/download"
 	"github.com/monkeyWie/gopeed-core/pkg/rest/model"
-	"github.com/monkeyWie/gopeed-core/pkg/test"
 	"io"
 	"net"
 	"net/http"
+	"os"
 	"reflect"
 	"sync"
 	"testing"
+	"time"
 )
 
 var (
@@ -21,6 +24,7 @@ var (
 
 	taskReq = &base.Request{}
 	taskRes = &base.Resource{
+		Name:  test.BuildName,
 		Req:   taskReq,
 		Size:  test.BuildSize,
 		Range: true,
@@ -32,9 +36,9 @@ var (
 			},
 		},
 	}
-	createReq = &model.CreateTaskReq{
-		Resource: taskRes,
-		Options: &base.Options{
+	createReq = &model.CreateTask{
+		Res: taskRes,
+		Opts: &base.Options{
 			Path:        test.Dir,
 			Name:        test.DownloadName,
 			Connections: 2,
@@ -130,6 +134,28 @@ func TestPauseAndContinueTask(t *testing.T) {
 	})
 }
 
+func TestDeleteTask(t *testing.T) {
+	doTest(func() {
+		taskId := httpRequestCheckOk[string](http.MethodPost, "/api/v1/tasks", createReq)
+		httpRequestCheckOk[any](http.MethodDelete, "/api/v1/tasks/"+taskId, nil)
+		code, _ := httpRequest[*download.Task](http.MethodGet, "/api/v1/tasks/"+taskId, nil)
+		checkCode(code, http.StatusNotFound)
+	})
+}
+
+func TestDeleteTaskForce(t *testing.T) {
+	doTest(func() {
+		taskId := httpRequestCheckOk[string](http.MethodPost, "/api/v1/tasks", createReq)
+		time.Sleep(time.Millisecond * 500)
+		httpRequestCheckOk[any](http.MethodDelete, "/api/v1/tasks/"+taskId+"?force=true", nil)
+		code, _ := httpRequest[*download.Task](http.MethodGet, "/api/v1/tasks/"+taskId, nil)
+		checkCode(code, http.StatusNotFound)
+		if _, err := os.Stat(test.DownloadFile); !errors.Is(err, os.ErrNotExist) {
+			t.Errorf("DeleteTaskForce() got = %v, want %v", err, os.ErrNotExist)
+		}
+	})
+}
+
 func TestGetTasks(t *testing.T) {
 	doTest(func() {
 		var wg sync.WaitGroup
@@ -140,30 +166,65 @@ func TestGetTasks(t *testing.T) {
 			}
 		})
 
-		httpRequestCheckOk[string](http.MethodPost, "/api/v1/tasks", createReq)
+		httpRequestCheckOk[string](http.MethodPost, fmt.Sprintf("/api/v1/tasks?status=%s,%s",
+			base.DownloadStatusReady, base.DownloadStatusRunning), createReq)
 		code, _ := httpRequest[[]*download.Task](http.MethodGet, "/api/v1/tasks", nil)
-		checkCode(code, 400)
+		checkCode(code, http.StatusBadRequest)
 
 		wg.Wait()
-		r := httpRequestCheckOk[[]*download.Task](http.MethodGet, "/api/v1/tasks?status=done", nil)
+		r := httpRequestCheckOk[[]*download.Task](http.MethodGet, fmt.Sprintf("/api/v1/tasks?status=%s",
+			base.DownloadStatusDone), nil)
 		if r[0].Status != base.DownloadStatusDone {
 			t.Errorf("GetTasks() got = %v, want %v", r[0].Status, base.DownloadStatusDone)
 		}
+		r = httpRequestCheckOk[[]*download.Task](http.MethodGet, fmt.Sprintf("/api/v1/tasks?status=%s,%s",
+			base.DownloadStatusReady, base.DownloadStatusRunning), nil)
+		if len(r) > 0 {
+			t.Errorf("GetTasks() got = %v, want %v", len(r), 0)
+		}
+	})
+}
+
+func TestGetAndPutConfig(t *testing.T) {
+	doTest(func() {
+		cfg := httpRequestCheckOk[*model.ServerConfig](http.MethodGet, "/api/v1/config", nil)
+
+		cfg.Port = 8888
+		cfg.Connections = 32
+		cfg.DownloadDir = "./download"
+		cfg.Extra = map[string]any{
+			"theme": "dark",
+		}
+		httpRequestCheckOk[any](http.MethodPut, "/api/v1/config", cfg)
+
+		newCfg := httpRequestCheckOk[*model.ServerConfig](http.MethodGet, "/api/v1/config", nil)
+		if !reflect.DeepEqual(newCfg, cfg) {
+			t.Errorf("GetAndPutConfig() got = %v, want %v", newCfg, cfg)
+		}
 	})
 }
 
 func doTest(handler func()) {
-	fileListener := doStart()
-	defer func() {
-		fileListener.Close()
-		Stop()
-	}()
-	taskReq.URL = "http://" + fileListener.Addr().String() + "/" + test.BuildName
-	handler()
+	testFunc := func(storage model.Storage) {
+		fileListener := doStart(storage)
+		defer func() {
+			if err := fileListener.Close(); err != nil {
+				panic(err)
+			}
+			Stop()
+		}()
+		defer Downloader.Clear()
+		taskReq.URL = "http://" + fileListener.Addr().String() + "/" + test.BuildName
+		handler()
+	}
+	testFunc(model.StorageMem)
+	testFunc(model.StorageBolt)
 }
 
-func doStart() net.Listener {
-	port, err := Start("127.0.0.1", 0)
+func doStart(storage model.Storage) net.Listener {
+	port, err := Start((&model.StartConfig{
+		Storage: storage,
+	}).Init())
 	if err != nil {
 		panic(err)
 	}

pkg/test/httptest.go

@@ -1,161 +0,0 @@
-package test
-
-import (
-	"encoding/json"
-	"fmt"
-	"io"
-	"math/rand"
-	"net"
-	"net/http"
-	"os"
-)
-
-const (
-	BuildName = "build.data"
-	BuildSize = 200 * 1024 * 1024
-	Dir       = "./"
-	BuildFile = Dir + BuildName
-
-	DownloadName = "download.data"
-	DownloadFile = Dir + DownloadName
-)
-
-func StartTestFileServer() net.Listener {
-	return startTestServer(func() http.Handler {
-		return http.FileServer(http.Dir(Dir))
-	})
-}
-
-func StartTestChunkedServer() net.Listener {
-	return startTestServer(func() http.Handler {
-		mux := http.NewServeMux()
-		mux.HandleFunc("/"+BuildName, func(writer http.ResponseWriter, request *http.Request) {
-			file, err := os.Open(BuildFile)
-			if err != nil {
-				panic(err)
-			}
-			defer file.Close()
-			io.Copy(writer, file)
-		})
-		return mux
-	})
-}
-
-func StartTestRetryServer() net.Listener {
-	counter := 0
-	return startTestServer(func() http.Handler {
-		mux := http.NewServeMux()
-		mux.HandleFunc("/"+BuildName, func(writer http.ResponseWriter, request *http.Request) {
-			counter++
-			if counter != 1 && counter < 5 {
-				writer.WriteHeader(500)
-				return
-			}
-			file, err := os.Open(BuildFile)
-			if err != nil {
-				panic(err)
-			}
-			defer file.Close()
-			io.Copy(writer, file)
-		})
-		return mux
-	})
-}
-
-func StartTestPostServer() net.Listener {
-	return startTestServer(func() http.Handler {
-		mux := http.NewServeMux()
-		mux.HandleFunc("/"+BuildName, func(writer http.ResponseWriter, request *http.Request) {
-			if request.Method == "POST" && request.Header.Get("Authorization") != "" {
-				var data map[string]interface{}
-				if err := json.NewDecoder(request.Body).Decode(&data); err != nil {
-					panic(err)
-				}
-				if data["name"] == BuildName {
-					file, err := os.Open(BuildFile)
-					if err != nil {
-						panic(err)
-					}
-					defer file.Close()
-					io.Copy(writer, file)
-				}
-			}
-		})
-		return mux
-	})
-}
-
-func StartTestErrorServer() net.Listener {
-	counter := 0
-	return startTestServer(func() http.Handler {
-		mux := http.NewServeMux()
-		mux.HandleFunc("/"+BuildName, func(writer http.ResponseWriter, request *http.Request) {
-			counter++
-			if counter != 1 {
-				writer.WriteHeader(500)
-				return
-			}
-		})
-		return mux
-	})
-}
-
-func startTestServer(serverHandle func() http.Handler) net.Listener {
-	listener, err := net.Listen("tcp", "127.0.0.1:0")
-	if err != nil {
-		panic(err)
-	}
-	file, err := os.Create(BuildFile)
-	if err != nil {
-		panic(err)
-	}
-	// 随机生成一个文件
-	l := int64(8192)
-	buf := make([]byte, l)
-	size := int64(0)
-	for {
-		_, err := rand.Read(buf)
-		if err != nil {
-			panic(err)
-		}
-		if size+l >= BuildSize {
-			file.WriteAt(buf[0:BuildSize-size], size)
-			break
-		}
-		file.WriteAt(buf, size)
-		size += l
-	}
-	server := &http.Server{}
-	server.Handler = serverHandle()
-	go server.Serve(listener)
-
-	return &delFileListener{
-		File:     file,
-		Listener: listener,
-	}
-}
-
-type delFileListener struct {
-	*os.File
-	net.Listener
-}
-
-func (c *delFileListener) Close() error {
-	defer func() {
-		c.File.Close()
-		if err := ifExistAndRemove(c.File.Name()); err != nil {
-			fmt.Println(err)
-		}
-		if err := ifExistAndRemove(DownloadFile); err != nil {
-			fmt.Println(err)
-		}
-	}()
-	return c.Listener.Close()
-}
-
-func ifExistAndRemove(name string) error {
-	if _, err := os.Stat(name); !os.IsNotExist(err) {
-		return os.Remove(name)
-	}
-	return nil
-}

pkg/test/util.go

@@ -1,32 +0,0 @@
-package test
-
-import (
-	"crypto/md5"
-	"encoding/hex"
-	"encoding/json"
-	"io"
-	"os"
-)
-
-func FileMd5(filePath string) string {
-	file, err := os.Open(filePath)
-	if err != nil {
-		panic(err)
-	}
-	// Tell the program to call the following function when the current function returns
-	defer file.Close()
-
-	// Open a new hash interface to write to
-	hash := md5.New()
-
-	// Copy the file in the hash interface and check for any error
-	if _, err := io.Copy(hash, file); err != nil {
-		return ""
-	}
-	return hex.EncodeToString(hash.Sum(nil))
-}
-
-func ToJson(v interface{}) string {
-	buf, _ := json.Marshal(v)
-	return string(buf)
-}

pkg/util/path.go

@@ -0,0 +1,81 @@
+package util
+
+import (
+	"errors"
+	"io"
+	"os"
+	"path/filepath"
+)
+
+func Dir(path string) string {
+	dir := filepath.Dir(path)
+	if dir == "." {
+		return ""
+	}
+	return dir
+}
+
+func Filepath(path string, originName string, customName string) string {
+	if customName == "" {
+		customName = originName
+	}
+	return filepath.Join(path, customName)
+}
+
+// SafeRemove remove file safely, ignoring errors if the path does not exist.
+func SafeRemove(name string) error {
+	if err := os.Remove(name); err != nil && !errors.Is(err, os.ErrNotExist) {
+		return err
+	}
+	return nil
+}
+
+// SafeRemoveAll remove file and parent directories safely
+func SafeRemoveAll(path string, names []string) error {
+	for _, name := range names {
+		err := SafeRemove(filepath.Join(path, name))
+		if err != nil {
+			return err
+		}
+		if err := safeRemoveParent(path, name); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func safeRemoveParent(path string, subPath string) error {
+	currPath := filepath.Dir(subPath)
+	if currPath == "." {
+		return nil
+	}
+	// if directory is empty, remove it
+	dir := filepath.Join(path, filepath.Dir(subPath))
+	empty, err := isEmpty(dir)
+	if err != nil {
+		return err
+	}
+	if empty {
+		err = SafeRemove(dir)
+		if err != nil {
+			return err
+		}
+		return safeRemoveParent(path, currPath)
+	}
+	return nil
+}
+
+// check directory is empty
+func isEmpty(name string) (bool, error) {
+	f, err := os.Open(name)
+	if err != nil {
+		return false, err
+	}
+	defer f.Close()
+
+	_, err = f.Readdirnames(1) // Or f.Readdir(1)
+	if err == io.EOF {
+		return true, nil
+	}
+	return false, err // Either not empty or error, suits both cases
+}

pkg/util/path_test.go

@@ -0,0 +1,198 @@
+package util
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+func TestDir(t *testing.T) {
+	type args struct {
+		path string
+	}
+	tests := []struct {
+		name string
+		args args
+		want string
+	}{
+		{
+			name: "empty path",
+			args: args{
+				path: ".",
+			},
+			want: "",
+		},
+		{
+			name: "normal path case 1",
+			args: args{
+				path: "./a/b/c/1.txt",
+			},
+			want: "a/b/c",
+		},
+		{
+			name: "normal path case 2",
+			args: args{
+				path: "a/b/c/1.txt",
+			},
+			want: "a/b/c",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := Dir(tt.args.path); got != tt.want {
+				t.Errorf("Dir() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestFilepath(t *testing.T) {
+	type args struct {
+		path       string
+		originName string
+		customName string
+	}
+	tests := []struct {
+		name string
+		args args
+		want string
+	}{
+		{
+			name: "origin name",
+			args: args{
+				path:       "/Downloads",
+				originName: "1.txt",
+				customName: "",
+			},
+			want: "/Downloads/1.txt",
+		},
+		{
+			name: "origin name",
+			args: args{
+				path:       "/Downloads",
+				originName: "1.txt",
+				customName: "2.txt",
+			},
+			want: "/Downloads/2.txt",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := Filepath(tt.args.path, tt.args.originName, tt.args.customName); got != tt.want {
+				t.Errorf("Filepath() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestSafeRemove(t *testing.T) {
+	name := "test_safe_remove.data"
+	_, err := os.Create(name)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := SafeRemove(name); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := os.Stat(name); err == nil {
+		t.Fatal(err)
+	}
+
+	if err := SafeRemove("test_safe_remove_not_exist.data"); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestSafeRemoveAll(t *testing.T) {
+	testDir := "test_dir"
+	err := os.Mkdir(testDir, 0755)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(testDir)
+
+	doSafeRemoveAll(t, testDir, []string{
+		"1.txt",
+	}, []string{})
+
+	doSafeRemoveAll(t, testDir, []string{
+		"1.txt",
+	}, []string{
+		"2.txt",
+	}, "2.txt")
+
+	doSafeRemoveAll(t, testDir, []string{
+		"1.txt",
+		"a/b/c/1.txt",
+		"a/b/1.txt",
+		"a/1.txt",
+	}, []string{})
+
+	doSafeRemoveAll(t, testDir, []string{
+		"1.txt",
+		"a/b/c/1.txt",
+		"a/b/1.txt",
+		"a/1.txt",
+	}, []string{
+		"a/b/2.txt",
+	}, "a/b/2.txt")
+
+	doSafeRemoveAll(t, testDir, []string{
+		"1.txt",
+		"a/b/c/1.txt",
+		"a/b/1.txt",
+		"a/1.txt",
+	}, []string{
+		"a/2.txt",
+	}, "a/2.txt")
+}
+
+func doSafeRemoveAll(t *testing.T, path string, downloadNames []string, otherNames []string, exist ...string) {
+	preCreate(t, path, downloadNames)
+	preCreate(t, path, otherNames)
+
+	if err := SafeRemoveAll(path, downloadNames); err != nil {
+		t.Fatal(err)
+	}
+
+	if len(exist) == 0 {
+		for _, name := range downloadNames {
+			filePath := filepath.Join(path, name)
+			if isExist(filePath) {
+				t.Fatalf("file %s should not exist", filePath)
+			}
+			subDirPath := filepath.Dir(name)
+			if subDirPath != "." {
+				dirPath := filepath.Join(path, subDirPath)
+				if isExist(dirPath) {
+					t.Fatalf("dir %s should not exist", dirPath)
+				}
+			}
+		}
+		return
+	}
+
+	for _, name := range exist {
+		if !isExist(filepath.Join(path, name)) {
+			t.Fatalf("file %s should exist", name)
+		}
+	}
+}
+
+func preCreate(t *testing.T, path string, names []string) {
+	for _, name := range names {
+		fullPath := filepath.Join(path, name)
+		if err := os.MkdirAll(filepath.Dir(fullPath), 0755); err != nil {
+			t.Fatal(err)
+		}
+		_, err := os.Create(fullPath)
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+}
+
+func isExist(path string) bool {
+	_, err := os.Stat(path)
+	return err == nil || os.IsExist(err)
+}

pkg/util/timer.go

@@ -16,10 +16,6 @@ func (t *Timer) Pause() {
 	t.used += time.Now().UnixNano() - t.t
 }
 
-func (t *Timer) Continue() {
-	t.t = time.Now().UnixNano()
-}
-
 func (t *Timer) Used() int64 {
 	return (time.Now().UnixNano() - t.t) + t.used
 }