feat: http downloader

This commit is contained in:
liwei committed 2020-08-03 18:35:50 +08:00
1 parent 15e5d2af3b
commit 788376eaa9
22 files changed
+813 -64

No files matched your search

+148
View File
@@ -0,0 +1,148 @@
package http
import (
"bytes"
"errors"
"fmt"
"github.com/monkeyWie/gopeed/download/common"
"github.com/monkeyWie/gopeed/download/http/model"
"io/ioutil"
"mime"
"net/http"
"net/http/cookiejar"
"net/url"
"strconv"
"strings"
"time"
)
type Fetcher struct {
*common.BaseFetcher
}
func (h *Fetcher) Protocols() []string {
return []string{"HTTP", "HTTPS"}
}
func (h *Fetcher) Resolve(req *common.Request) (*common.Resource, error) {
httpReq, err := BuildRequest(req)
if err != nil {
return nil, err
}
client := BuildClient()
// 只访问一个字节,测试资源是否支持Range请求
httpReq.Header.Set(common.HttpHeaderRange, fmt.Sprintf(common.HttpHeaderRangeFormat, 0, 0))
httpResp, err := client.Do(httpReq)
if err != nil {
return nil, err
}
defer httpResp.Body.Close()
res := &common.Resource{
Req: req,
Range: false,
Files: []*common.FileInfo{},
}
if common.HttpCodePartialContent == httpResp.StatusCode {
// 返回206响应码表示支持断点下载
res.Range = true
// 解析资源大小: bytes 0-1000/1001 => 1001
contentTotal := subLastSlash(httpResp.Header.Get(common.HttpHeaderContentRange))
if contentTotal != "" {
parse, err := strconv.ParseInt(contentTotal, 10, 64)
if err != nil {
return nil, err
}
res.Size = parse
}
} else if common.HttpCodeOK == httpResp.StatusCode {
// 返回200响应码,不支持断点下载,通过Content-Length头获取文件大小,获取不到的话可能是chunked编码
contentLength := httpResp.Header.Get(common.HttpHeaderContentLength)
parse, err := strconv.ParseInt(contentLength, 10, 64)
if err != nil {
return nil, err
}
res.Size = parse
} else {
return nil, errors.New("http request fail")
}
file := &common.FileInfo{
Size: res.Size,
}
contentDisposition := httpResp.Header.Get(common.HttpHeaderContentDisposition)
if contentDisposition != "" {
_, params, _ := mime.ParseMediaType(contentDisposition)
filename := params["filename"]
if filename != "" {
file.Name = filename
}
}
// Get file name by URL
if file.Name == "" {
parse, err := url.Parse(req.URL)
if err == nil {
// e.g. /files/test.txt => test.txt
file.Name = subLastSlash(parse.Path)
}
}
// unknown file name
if file.Name == "" {
file.Name = "unknown"
}
res.Files = append(res.Files, file)
return res, nil
}
func (h *Fetcher) Create(res *common.Resource, opts *common.Options) (common.Process, error) {
if opts.Connections != 1 && !res.Range {
opts.Connections = 1
}
return NewProcess(h, res, opts), nil
}
func subLastSlash(str string) string {
if str == "" {
return ""
}
index := strings.LastIndex(str, "/")
if index != -1 {
return str[index+1:]
}
return ""
}
func BuildClient() *http.Client {
// Cookie handle
jar, _ := cookiejar.New(nil)
return &http.Client{
Jar: jar,
Timeout: time.Second * 10,
}
}
func BuildRequest(req *common.Request) (*http.Request, error) {
url, err := url.Parse(req.URL)
if err != nil {
return nil, err
}
httpReq := &http.Request{
URL: url,
Header: map[string][]string{},
}
if req.Extra != nil {
if extra, ok := req.Extra.(model.Extra); ok {
if extra.Method != "" {
httpReq.Method = extra.Method
} else {
httpReq.Method = http.MethodGet
}
if len(extra.Header) > 0 {
for k, v := range extra.Header {
httpReq.Header[k] = []string{v}
}
}
httpReq.Body = ioutil.NopCloser(bytes.NewBufferString(extra.Body))
}
}
return httpReq, nil
}
+95
View File
@@ -0,0 +1,95 @@
package http
import (
"encoding/json"
"fmt"
"github.com/monkeyWie/gopeed/download/common"
"golang.org/x/net/proxy"
"io"
"io/ioutil"
"net/http"
"os"
"testing"
"time"
)
func TestFetcher_Resolve(t *testing.T) {
fetcher := &Fetcher{}
resolve, err := fetcher.Resolve(&common.Request{
// URL: "https://github.com/monkeyWie/github-actions-demo/releases/download/v1.0.6/simple-v1.0.6-windows-x64.zip",
URL: "http://192.168.200.163:8088/go1.12.7.linux-amd64.tar.gz",
})
if err != nil {
t.Fatal(err)
}
buf, err := json.Marshal(resolve)
if err != nil {
t.Fatal(err)
}
fmt.Printf("%s\n", string(buf))
}
func Test2(t *testing.T) {
socks5, err := proxy.SOCKS5("tcp", "127.0.0.1:1080", nil, nil)
if err != nil {
t.Fatal(err)
}
conn, err := socks5.Dial("tcp", "www.baidu.com:80")
if err != nil {
t.Fatal(err)
}
conn.Write([]byte("GET"))
}
func Test3(t *testing.T) {
var u User
fmt.Println((&u).Say1())
fmt.Println(u.Say2())
}
func Test4(t *testing.T) {
resp, err := http.Get("http://192.168.200.163:8088/go1.12.7.linux-amd64.tar.gz")
if err != nil {
t.Fatal(err)
}
defer resp.Body.Close()
go func() {
time.Sleep(time.Second * 3)
err2 := resp.Body.Close()
if err2 != nil {
fmt.Println("close err2:" + err2.Error())
}
}()
_, err = io.Copy(ioutil.Discard, resp.Body)
if err != nil {
t.Fatal(err)
}
}
func Test5(t *testing.T) {
name := "E:\\test\\gopeed\\http\\test.txt"
err := os.Truncate(name, 10)
if err != nil {
t.Fatal(err)
}
file, err := os.OpenFile(name, os.O_RDWR, 0666)
if err != nil {
t.Fatal(err)
}
_, err = file.WriteAt([]byte("test"), 5)
if err != nil {
t.Fatal(err)
}
}
type User struct {
name string
}
func (u *User) Say1() string {
return "hello1 " + u.name
}
func (u User) Say2() string {
return "hello2 " + u.name
}
+20
View File
@@ -0,0 +1,20 @@
package model
import (
common2 "github.com/monkeyWie/gopeed/download/common"
)
type Chunk struct {
Status common2.Status
Begin int64
End int64
Downloaded int64
}
func NewChunk(begin int64, end int64) *Chunk {
return &Chunk{
Status: common2.DownloadStatusReady,
Begin: begin,
End: end,
}
}
+7
View File
@@ -0,0 +1,7 @@
package model
type Extra struct {
Method string
Header map[string]string
Body string
}
+156
View File
@@ -0,0 +1,156 @@
package http
import (
"fmt"
"github.com/monkeyWie/gopeed/download/common"
"github.com/monkeyWie/gopeed/download/http/model"
"io"
"net/http"
"path/filepath"
)
type Process struct {
fetcher *Fetcher
res *common.Resource
opts *common.Options
status common.Status
clients []*http.Response
chunks []*model.Chunk
}
func NewProcess(fetcher *Fetcher, res *common.Resource, opts *common.Options) *Process {
return &Process{
fetcher: fetcher,
res: res,
opts: opts,
status: common.DownloadStatusReady,
}
}
func (p *Process) Start() error {
ctl := p.fetcher.GetCtl()
// 创建文件
var filename = p.opts.Name
if filename == "" {
filename = p.res.Files[0].Name
}
name := filepath.Join(p.opts.Path, filename)
err := ctl.Touch(name, p.res.Size)
if err != nil {
return err
}
if p.res.Range && p.res.Size > 0 {
// 每个连接平均需要下载的分块大小
var (
chunkSize = p.res.Size / int64(p.opts.Connections)
errCh = make(chan error)
)
defer close(errCh)
p.chunks = make([]*model.Chunk, p.opts.Connections)
p.clients = make([]*http.Response, p.opts.Connections)
for i := 0; i < p.opts.Connections; i++ {
begin := chunkSize * int64(i)
end := begin + chunkSize
if i == p.opts.Connections-1 {
// 最后一个连接下载的分块大小需要特殊计算,保证把文件下载完
chunkSize = end + p.res.Size%int64(p.opts.Connections)
}
p.chunks[i] = model.NewChunk(begin, end)
go func(i int) {
errCh <- p.doFetch(i, name, begin, end)
}(i)
}
var failErr error
for i := 0; i < p.opts.Connections; i++ {
err := <-errCh
if failErr == nil {
// 有一个连接下载失败,立即停止下载
failErr = err
if failErr != nil {
p.Pause()
}
}
}
if failErr != nil {
return failErr
}
}
return nil
}
func (p *Process) Pause() error {
if len(p.clients) > 0 {
for _, client := range p.clients {
client.Body.Close()
}
}
return nil
}
func (p *Process) Continue() error {
return nil
}
func (p *Process) Delete() error {
panic("implement me")
}
func (p *Process) doFetch(index int, filename string, begin, end int64) (err error) {
httpReq, err := BuildRequest(p.res.Req)
if err != nil {
return err
}
httpReq.Header.Set(common.HttpHeaderRange, fmt.Sprintf(common.HttpHeaderRangeFormat, begin, end))
client := BuildClient()
// 重试5次
for i := 0; i < 5; i++ {
var (
resp *http.Response
retry bool
)
resp, err = client.Do(httpReq)
if err != nil {
// 连接失败,直接重试
continue
}
p.chunks[index].Downloaded = 0
p.clients[index] = resp
retry, err = func() (bool, error) {
defer resp.Body.Close()
var (
buf = make([]byte, 8192)
offset = begin
)
for {
n, err := resp.Body.Read(buf)
if n > 0 {
len, err := p.fetcher.GetCtl().Write(filename, offset, buf[:n])
if err != nil {
return false, err
}
offset += int64(len)
p.chunks[index].Downloaded = p.chunks[i].Downloaded + int64(len)
}
if err != nil {
if err != io.EOF {
return true, err
}
break
}
}
return false, nil
}()
if err == nil || !retry {
// 下载成功,跳出重试
break
}
}
if err != nil {
p.chunks[index].Status = common.DownloadStatusError
} else {
p.chunks[index].Status = common.DownloadStatusDone
}
return
}