feat: http downloader

This commit is contained in:
liwei committed 2020-08-03 18:35:50 +08:00
1 parent 15e5d2af3b
commit 788376eaa9
22 files changed
+813 -64

No files matched your search

+23
View File
@@ -0,0 +1,23 @@
package common
type Status int
const (
DownloadStatusReady Status = iota
DownloadStatusStart
DownloadStatusPause
DownloadStatusError
DownloadStatusDone
)
const (
HttpCodeOK = 200
HttpCodePartialContent = 206
HttpHeaderRange = "Range"
HttpHeaderContentLength = "Content-Length"
HttpHeaderContentRange = "Content-Range"
HttpHeaderContentDisposition = "Content-Disposition"
HttpHeaderRangeFormat = "bytes=%d-%d"
)
+129
View File
@@ -0,0 +1,129 @@
package common
import (
"golang.org/x/net/proxy"
"net"
"os"
"time"
)
// 下载请求
type Request struct {
// 下载链接
URL string
// 附加信息
Extra interface{}
}
// 资源信息
type Resource struct {
Req *Request
// 资源总大小
Size int64
// 是否支持断点下载
Range bool
// 资源所包含的文件列表
Files []*FileInfo
}
type Options struct {
// 保存文件名
Name string
// 保存目录
Path string
// 并发连接数
Connections int
}
type FileInfo struct {
Name string
Path string
Size int64
Downloaded int64
}
type Process interface {
Start() error
Pause() error
Continue() error
Delete() error
}
type Controller struct {
Files map[string]*os.File
}
func (c *Controller) Touch(name string, size int64) error {
if size > 0 {
return os.Truncate(name, size)
}
_, err := os.Create(name)
return err
}
func (c *Controller) Write(name string, offset int64, buf []byte) (int, error) {
file, ok := c.Files[name]
if !ok {
var err error
file, err = os.OpenFile(name, os.O_RDWR, 0666)
if err != nil {
return 0, err
}
c.Files[name] = file
}
return file.WriteAt(buf, offset)
}
func (c *Controller) ContextDialer() (proxy.Dialer, error) {
// return proxy.SOCKS5("tpc", "127.0.0.1:9999", nil, nil)
var dialer proxy.Dialer
return &DialerWarp{dialer: dialer}, nil
}
type DialerWarp struct {
dialer proxy.Dialer
}
type ConnWarp struct {
conn net.Conn
}
func (c *ConnWarp) Read(b []byte) (n int, err error) {
return c.conn.Read(b)
}
func (c *ConnWarp) Write(b []byte) (n int, err error) {
return c.conn.Write(b)
}
func (c *ConnWarp) Close() error {
return c.conn.Close()
}
func (c *ConnWarp) LocalAddr() net.Addr {
return c.conn.LocalAddr()
}
func (c *ConnWarp) RemoteAddr() net.Addr {
return c.conn.RemoteAddr()
}
func (c *ConnWarp) SetDeadline(t time.Time) error {
return c.conn.SetDeadline(t)
}
func (c *ConnWarp) SetReadDeadline(t time.Time) error {
return c.conn.SetReadDeadline(t)
}
func (c *ConnWarp) SetWriteDeadline(t time.Time) error {
return c.conn.SetWriteDeadline(t)
}
func (d *DialerWarp) Dial(network, addr string) (c net.Conn, err error) {
conn, err := d.dialer.Dial(network, addr)
if err != nil {
return nil, err
}
return &ConnWarp{conn: conn}, nil
}
+29
View File
@@ -0,0 +1,29 @@
package common
// 对应协议的下载支持
type Fetcher interface {
BaseFetch
// 支持的协议列表
Protocols() []string
// 解析请求
Resolve(req *Request) (*Resource, error)
// 创建任务
Create(res *Resource, opts *Options) (Process, error)
}
type BaseFetch interface {
InitCtl(ctl *Controller)
GetCtl() *Controller
}
type BaseFetcher struct {
ctl *Controller
}
func (b *BaseFetcher) InitCtl(ctl *Controller) {
b.ctl = ctl
}
func (b *BaseFetcher) GetCtl() *Controller {
return b.ctl
}
+82
View File
@@ -0,0 +1,82 @@
package download
import (
"errors"
"github.com/google/uuid"
"github.com/monkeyWie/gopeed/download/common"
"net/url"
"os"
"strings"
)
type TaskInfo struct {
ID string
Res *common.Resource
Opts *common.Options
Status common.Status
Files map[string]*common.FileInfo
process common.Process
}
type Downloader struct {
fetches map[string]common.Fetcher
tasks map[string]*TaskInfo
*common.Controller
}
func NewDownloader(fetchers ...common.Fetcher) *Downloader {
d := &Downloader{Controller: &common.Controller{Files: make(map[string]*os.File)}}
d.fetches = make(map[string]common.Fetcher)
for _, f := range fetchers {
for _, p := range f.Protocols() {
d.fetches[strings.ToUpper(p)] = f
}
f.InitCtl(d.Controller)
}
d.tasks = make(map[string]*TaskInfo)
return d
}
func (d *Downloader) getFetcher(URL string) (common.Fetcher, error) {
url, err := url.Parse(URL)
if err != nil {
return nil, err
}
if fetch, ok := d.fetches[strings.ToUpper(url.Scheme)]; ok {
return fetch, nil
}
return nil, errors.New("unsupported protocol")
}
func (d *Downloader) Resolve(req *common.Request) (*common.Resource, error) {
fetcher, err := d.getFetcher(req.URL)
if err != nil {
return nil, err
}
return fetcher.Resolve(req)
}
func (d *Downloader) Create(res *common.Resource, opts *common.Options) error {
fetcher, err := d.getFetcher(res.Req.URL)
if err != nil {
return err
}
process, err := fetcher.Create(res, opts)
if err != nil {
return err
}
id := uuid.New().String()
d.tasks[id] = &TaskInfo{
ID: id,
Res: res,
Opts: opts,
Status: common.DownloadStatusReady,
process: process,
}
return process.Start()
}
func (d *Downloader) Pause(id string) {
d.tasks[id].process.Pause()
}
+40
View File
@@ -0,0 +1,40 @@
package download
import (
"fmt"
"github.com/monkeyWie/gopeed/download/common"
"github.com/monkeyWie/gopeed/download/http"
"testing"
)
func TestDownloader_Resolve(t *testing.T) {
downloader := buildDownloader()
res, err := downloader.Resolve(&common.Request{
URL: "http://192.168.200.163:8088/docker-compose",
})
if err != nil {
t.Fatal(err)
}
fmt.Println(res)
}
func TestDownloader_Create(t *testing.T) {
downloader := buildDownloader()
res, err := downloader.Resolve(&common.Request{
URL: "http://192.168.200.163:8088/docker-compose",
})
if err != nil {
t.Fatal(err)
}
err = downloader.Create(res, &common.Options{
Path: "E:\\test\\gopeed\\http",
Connections: 4,
})
if err != nil {
t.Fatal(err)
}
}
func buildDownloader() *Downloader {
return NewDownloader(&http.Fetcher{BaseFetcher: &common.BaseFetcher{}})
}
+148
View File
@@ -0,0 +1,148 @@
package http
import (
"bytes"
"errors"
"fmt"
"github.com/monkeyWie/gopeed/download/common"
"github.com/monkeyWie/gopeed/download/http/model"
"io/ioutil"
"mime"
"net/http"
"net/http/cookiejar"
"net/url"
"strconv"
"strings"
"time"
)
type Fetcher struct {
*common.BaseFetcher
}
func (h *Fetcher) Protocols() []string {
return []string{"HTTP", "HTTPS"}
}
func (h *Fetcher) Resolve(req *common.Request) (*common.Resource, error) {
httpReq, err := BuildRequest(req)
if err != nil {
return nil, err
}
client := BuildClient()
// 只访问一个字节,测试资源是否支持Range请求
httpReq.Header.Set(common.HttpHeaderRange, fmt.Sprintf(common.HttpHeaderRangeFormat, 0, 0))
httpResp, err := client.Do(httpReq)
if err != nil {
return nil, err
}
defer httpResp.Body.Close()
res := &common.Resource{
Req: req,
Range: false,
Files: []*common.FileInfo{},
}
if common.HttpCodePartialContent == httpResp.StatusCode {
// 返回206响应码表示支持断点下载
res.Range = true
// 解析资源大小: bytes 0-1000/1001 => 1001
contentTotal := subLastSlash(httpResp.Header.Get(common.HttpHeaderContentRange))
if contentTotal != "" {
parse, err := strconv.ParseInt(contentTotal, 10, 64)
if err != nil {
return nil, err
}
res.Size = parse
}
} else if common.HttpCodeOK == httpResp.StatusCode {
// 返回200响应码,不支持断点下载,通过Content-Length头获取文件大小,获取不到的话可能是chunked编码
contentLength := httpResp.Header.Get(common.HttpHeaderContentLength)
parse, err := strconv.ParseInt(contentLength, 10, 64)
if err != nil {
return nil, err
}
res.Size = parse
} else {
return nil, errors.New("http request fail")
}
file := &common.FileInfo{
Size: res.Size,
}
contentDisposition := httpResp.Header.Get(common.HttpHeaderContentDisposition)
if contentDisposition != "" {
_, params, _ := mime.ParseMediaType(contentDisposition)
filename := params["filename"]
if filename != "" {
file.Name = filename
}
}
// Get file name by URL
if file.Name == "" {
parse, err := url.Parse(req.URL)
if err == nil {
// e.g. /files/test.txt => test.txt
file.Name = subLastSlash(parse.Path)
}
}
// unknown file name
if file.Name == "" {
file.Name = "unknown"
}
res.Files = append(res.Files, file)
return res, nil
}
func (h *Fetcher) Create(res *common.Resource, opts *common.Options) (common.Process, error) {
if opts.Connections != 1 && !res.Range {
opts.Connections = 1
}
return NewProcess(h, res, opts), nil
}
func subLastSlash(str string) string {
if str == "" {
return ""
}
index := strings.LastIndex(str, "/")
if index != -1 {
return str[index+1:]
}
return ""
}
func BuildClient() *http.Client {
// Cookie handle
jar, _ := cookiejar.New(nil)
return &http.Client{
Jar: jar,
Timeout: time.Second * 10,
}
}
func BuildRequest(req *common.Request) (*http.Request, error) {
url, err := url.Parse(req.URL)
if err != nil {
return nil, err
}
httpReq := &http.Request{
URL: url,
Header: map[string][]string{},
}
if req.Extra != nil {
if extra, ok := req.Extra.(model.Extra); ok {
if extra.Method != "" {
httpReq.Method = extra.Method
} else {
httpReq.Method = http.MethodGet
}
if len(extra.Header) > 0 {
for k, v := range extra.Header {
httpReq.Header[k] = []string{v}
}
}
httpReq.Body = ioutil.NopCloser(bytes.NewBufferString(extra.Body))
}
}
return httpReq, nil
}
+95
View File
@@ -0,0 +1,95 @@
package http
import (
"encoding/json"
"fmt"
"github.com/monkeyWie/gopeed/download/common"
"golang.org/x/net/proxy"
"io"
"io/ioutil"
"net/http"
"os"
"testing"
"time"
)
func TestFetcher_Resolve(t *testing.T) {
fetcher := &Fetcher{}
resolve, err := fetcher.Resolve(&common.Request{
// URL: "https://github.com/monkeyWie/github-actions-demo/releases/download/v1.0.6/simple-v1.0.6-windows-x64.zip",
URL: "http://192.168.200.163:8088/go1.12.7.linux-amd64.tar.gz",
})
if err != nil {
t.Fatal(err)
}
buf, err := json.Marshal(resolve)
if err != nil {
t.Fatal(err)
}
fmt.Printf("%s\n", string(buf))
}
func Test2(t *testing.T) {
socks5, err := proxy.SOCKS5("tcp", "127.0.0.1:1080", nil, nil)
if err != nil {
t.Fatal(err)
}
conn, err := socks5.Dial("tcp", "www.baidu.com:80")
if err != nil {
t.Fatal(err)
}
conn.Write([]byte("GET"))
}
func Test3(t *testing.T) {
var u User
fmt.Println((&u).Say1())
fmt.Println(u.Say2())
}
func Test4(t *testing.T) {
resp, err := http.Get("http://192.168.200.163:8088/go1.12.7.linux-amd64.tar.gz")
if err != nil {
t.Fatal(err)
}
defer resp.Body.Close()
go func() {
time.Sleep(time.Second * 3)
err2 := resp.Body.Close()
if err2 != nil {
fmt.Println("close err2:" + err2.Error())
}
}()
_, err = io.Copy(ioutil.Discard, resp.Body)
if err != nil {
t.Fatal(err)
}
}
func Test5(t *testing.T) {
name := "E:\\test\\gopeed\\http\\test.txt"
err := os.Truncate(name, 10)
if err != nil {
t.Fatal(err)
}
file, err := os.OpenFile(name, os.O_RDWR, 0666)
if err != nil {
t.Fatal(err)
}
_, err = file.WriteAt([]byte("test"), 5)
if err != nil {
t.Fatal(err)
}
}
type User struct {
name string
}
func (u *User) Say1() string {
return "hello1 " + u.name
}
func (u User) Say2() string {
return "hello2 " + u.name
}
+20
View File
@@ -0,0 +1,20 @@
package model
import (
common2 "github.com/monkeyWie/gopeed/download/common"
)
type Chunk struct {
Status common2.Status
Begin int64
End int64
Downloaded int64
}
func NewChunk(begin int64, end int64) *Chunk {
return &Chunk{
Status: common2.DownloadStatusReady,
Begin: begin,
End: end,
}
}
+7
View File
@@ -0,0 +1,7 @@
package model
type Extra struct {
Method string
Header map[string]string
Body string
}
+156
View File
@@ -0,0 +1,156 @@
package http
import (
"fmt"
"github.com/monkeyWie/gopeed/download/common"
"github.com/monkeyWie/gopeed/download/http/model"
"io"
"net/http"
"path/filepath"
)
type Process struct {
fetcher *Fetcher
res *common.Resource
opts *common.Options
status common.Status
clients []*http.Response
chunks []*model.Chunk
}
func NewProcess(fetcher *Fetcher, res *common.Resource, opts *common.Options) *Process {
return &Process{
fetcher: fetcher,
res: res,
opts: opts,
status: common.DownloadStatusReady,
}
}
func (p *Process) Start() error {
ctl := p.fetcher.GetCtl()
// 创建文件
var filename = p.opts.Name
if filename == "" {
filename = p.res.Files[0].Name
}
name := filepath.Join(p.opts.Path, filename)
err := ctl.Touch(name, p.res.Size)
if err != nil {
return err
}
if p.res.Range && p.res.Size > 0 {
// 每个连接平均需要下载的分块大小
var (
chunkSize = p.res.Size / int64(p.opts.Connections)
errCh = make(chan error)
)
defer close(errCh)
p.chunks = make([]*model.Chunk, p.opts.Connections)
p.clients = make([]*http.Response, p.opts.Connections)
for i := 0; i < p.opts.Connections; i++ {
begin := chunkSize * int64(i)
end := begin + chunkSize
if i == p.opts.Connections-1 {
// 最后一个连接下载的分块大小需要特殊计算,保证把文件下载完
chunkSize = end + p.res.Size%int64(p.opts.Connections)
}
p.chunks[i] = model.NewChunk(begin, end)
go func(i int) {
errCh <- p.doFetch(i, name, begin, end)
}(i)
}
var failErr error
for i := 0; i < p.opts.Connections; i++ {
err := <-errCh
if failErr == nil {
// 有一个连接下载失败,立即停止下载
failErr = err
if failErr != nil {
p.Pause()
}
}
}
if failErr != nil {
return failErr
}
}
return nil
}
func (p *Process) Pause() error {
if len(p.clients) > 0 {
for _, client := range p.clients {
client.Body.Close()
}
}
return nil
}
func (p *Process) Continue() error {
return nil
}
func (p *Process) Delete() error {
panic("implement me")
}
func (p *Process) doFetch(index int, filename string, begin, end int64) (err error) {
httpReq, err := BuildRequest(p.res.Req)
if err != nil {
return err
}
httpReq.Header.Set(common.HttpHeaderRange, fmt.Sprintf(common.HttpHeaderRangeFormat, begin, end))
client := BuildClient()
// 重试5次
for i := 0; i < 5; i++ {
var (
resp *http.Response
retry bool
)
resp, err = client.Do(httpReq)
if err != nil {
// 连接失败,直接重试
continue
}
p.chunks[index].Downloaded = 0
p.clients[index] = resp
retry, err = func() (bool, error) {
defer resp.Body.Close()
var (
buf = make([]byte, 8192)
offset = begin
)
for {
n, err := resp.Body.Read(buf)
if n > 0 {
len, err := p.fetcher.GetCtl().Write(filename, offset, buf[:n])
if err != nil {
return false, err
}
offset += int64(len)
p.chunks[index].Downloaded = p.chunks[i].Downloaded + int64(len)
}
if err != nil {
if err != io.EOF {
return true, err
}
break
}
}
return false, nil
}()
if err == nil || !retry {
// 下载成功,跳出重试
break
}
}
if err != nil {
p.chunks[index].Status = common.DownloadStatusError
} else {
p.chunks[index].Status = common.DownloadStatusDone
}
return
}