HTTP download test pass

2026-11-06 08:30:29 +00:00 · 2019-05-05 01:23:48 +08:00
1 parent 00468e4e4a
commit 66a4b022f2
@@ -1,2 +1,2 @@
-## go-download
+## gee-download
 A fast download client,support HTTP&P2P.
@@ -1,7 +1,6 @@
 package down

 import (
-	"bytes"
 	"fmt"
 	"io"
 	"mime"
@@ -11,27 +10,23 @@ import (
 	"os"
 	"strconv"
 	"strings"
+	"sync"
 )

 // Resolve return the file response to be downloaded
 func Resolve(request *Request) (*Response, error) {
-	// Build request
-	httpRequest, err := http.NewRequest(strings.ToUpper(request.Method), request.URL, nil)
+	httpRequest, err := BuildHTTPRequest(request)
 	if err != nil {
 		return nil, err
 	}
-	for k, v := range request.Header {
-		httpRequest.Header.Add(k, v)
-	}
 	// Use "Range" header to resolve
 	httpRequest.Header.Add("Range", "bytes=0-0")
-	// Cookie handle
-	jar, _ := cookiejar.New(nil)
-	httpClient := &http.Client{Jar: jar}
+	httpClient := BuildHTTPClient()
 	response, err := httpClient.Do(httpRequest)
 	if err != nil {
 		return nil, err
 	}
+	defer response.Body.Close()
 	if response.StatusCode != 200 && response.StatusCode != 206 {
 		return nil, fmt.Errorf("Response status error:%d", response.StatusCode)
 	}
@@ -47,7 +42,7 @@ func Resolve(request *Request) (*Response, error) {
 	}
 	// Get file name by URL
 	if ret.Name == "" {
-		parse, err := url.Parse(request.URL)
+		parse, err := url.Parse(httpRequest.URL.String())
 		if err == nil {
 			// e.g. /files/test.txt => test.txt
 			ret.Name = subLastSlash(parse.Path)
@@ -82,6 +77,45 @@ func Resolve(request *Request) (*Response, error) {
 	return ret, nil
 }

+// Down
+func Down(request *Request) error {
+	response, err := Resolve(request)
+	if err != nil {
+		return err
+	}
+	// allocate file
+	file, err := os.Create(response.Name)
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+	if err := file.Truncate(response.Size); err != nil {
+		return err
+	}
+	// support range
+	if response.Range {
+		cons := 16
+		chunkSize := response.Size / int64(cons)
+		var (
+			waitGroup = &sync.WaitGroup{}
+			fileLock  = &sync.Mutex{}
+		)
+		waitGroup.Add(cons)
+		for i := 0; i < cons; i++ {
+			start := int64(i) * chunkSize
+			end := start + chunkSize
+			if i == cons-1 {
+				end = response.Size
+			}
+			go downChunk(request, file, start, end-1, waitGroup, fileLock)
+		}
+		waitGroup.Wait()
+	} else {
+		downChunk(request, file, 0, response.Size, nil, nil)
+	}
+	return nil
+}
+
 func subLastSlash(str string) string {
 	index := strings.LastIndex(str, "/")
 	if index != -1 {
@@ -90,38 +124,61 @@ func subLastSlash(str string) string {
 	return ""
 }

-// Down 下载
-func Down(request *Request) {
-	httpRequest, err := http.NewRequest(request.Method, request.URL, bytes.NewReader(request.content))
+func BuildHTTPRequest(request *Request) (*http.Request, error) {
+	// Build request
+	httpRequest, err := http.NewRequest(strings.ToUpper(request.Method), request.URL, nil)
 	if err != nil {
-		fmt.Println("create http request error")
-		return
+		return nil, err
 	}
 	for k, v := range request.Header {
 		httpRequest.Header.Add(k, v)
 	}
-	httpClient := &http.Client{}
-	response, err := httpClient.Do(httpRequest)
+	return httpRequest, nil
+}
+
+func BuildHTTPClient() *http.Client {
+	// Cookie handle
+	jar, _ := cookiejar.New(nil)
+
+	return &http.Client{Jar: jar}
+}
+
+func downChunk(request *Request, file *os.File, start int64, end int64, waitGroup *sync.WaitGroup, fileLock *sync.Mutex) {
+	defer func() {
+		if waitGroup != nil {
+			waitGroup.Done()
+		}
+	}()
+	httpRequest, _ := BuildHTTPRequest(request)
+	httpRequest.Header.Add("Range", fmt.Sprintf("bytes=%d-%d", start, end))
+	fmt.Printf("down %d-%d\n", start, end)
+	httpClient := BuildHTTPClient()
+	httpResponse, err := httpClient.Do(httpRequest)
 	if err != nil {
-		fmt.Printf("do http request error:%s\n", err)
+		fmt.Println(err)
 		return
 	}
-	if response.StatusCode >= 200 && response.StatusCode < 300 {
-		_, params, _ := mime.ParseMediaType(response.Header.Get("Content-Disposition"))
-		filename := params["filename"]
-		if len(filename) > 0 {
-			fmt.Printf("filename:%s\n", filename)
+	defer httpResponse.Body.Close()
+	buf := make([]byte, 8192)
+	writeIndex := int64(start)
+	for {
+		len, err := httpResponse.Body.Read(buf)
+		if len > 0 {
+			fileLock.Lock()
+			writeSize, err := file.WriteAt(buf[0:len], writeIndex)
+			fileLock.Unlock()
+			if err != nil {
+				fmt.Println(err)
+				return
+			}
+			writeIndex += int64(writeSize)
+		}
+		if err != nil {
+			if err != io.EOF {
+				fmt.Println(err)
+				return
+			}
+			break
 		}
 	}
-
-	fmt.Printf("response status:%d %s\n", response.StatusCode, response.Status)
-	fmt.Printf("response heads:%v\n", response.Header)
-	file, err := os.Create("test.txt")
-	if err != nil {
-		fmt.Println("create file error")
-		return
-	}
-	defer file.Close()
-	fmt.Println("file close")
-	io.Copy(file, response.Body)
 }
@@ -1,11 +1,15 @@
 package down

 import (
+	"crypto/md5"
+	"encoding/hex"
 	"fmt"
-	"net/url"
+	"io"
+	"log"
+	"os"
 	"reflect"
-	"strings"
 	"testing"
+	"time"
 )

 func TestDown(t *testing.T) {
@@ -17,19 +21,18 @@ func TestDown(t *testing.T) {
 		args args
 	}{
 		{
-			"readme",
+			"proxyee-down.ico",
 			args{
 				&Request{
 					"get",
-					"https://raw.githubusercontent.com/proxyee-down-org/proxyee-down/master/README.md",
+					"https://raw.githubusercontent.com/proxyee-down-org/proxyee-down/master/front/public/favicon.ico",
 					map[string]string{
+						"Host":            "github.com",
+						"User-Agent":      "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Safari/537.36",
 						"Accept":          "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8",
-						"Host":            "raw.githubusercontent.com",
+						"Referer":         "http://github.com/proxyee-down-org/proxyee-down/releases",
 						"Accept-Encoding": "gzip, deflate, br",
 						"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
-						"Cache-Control":   "no-cache",
-						"Connection":      "keep-alive",
-						"User-Agent":      "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Safari/537.36",
 					},
 					nil,
 				},
@@ -39,7 +42,17 @@ func TestDown(t *testing.T) {

 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			Down(tt.args.request)
+			os.Remove("favicon.ico")
+			err := Down(tt.args.request)
+			if err != nil {
+				t.Errorf("error down= %v", err)
+				return
+			}
+			downMd5 := fileMd5("favicon.ico")
+			if "8de7a6a2e786861013d61b77b2394012" != downMd5 {
+				t.Errorf("error md5= %v", downMd5)
+				return
+			}
 		})
 	}
 }
@@ -56,11 +69,11 @@ func TestResolve(t *testing.T) {
 		wantErr bool
 	}{
 		{
-			"proxyee-down",
+			"proxyee-down.ico",
 			args{
 				&Request{
 					"get",
-					"http://github.com/proxyee-down-org/proxyee-down/releases/download/3.4/proxyee-down-main.jar",
+					"https://raw.githubusercontent.com/proxyee-down-org/proxyee-down/master/front/public/favicon.ico",
 					map[string]string{
 						"Host":            "github.com",
 						"User-Agent":      "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Safari/537.36",
@@ -72,7 +85,7 @@ func TestResolve(t *testing.T) {
 					nil,
 				},
 			},
-			&Response{"proxyee-down-main.jar", 30159703, true},
+			&Response{"favicon.ico", 116093, true},
 			false,
 		},
 	}
@@ -91,12 +104,47 @@ func TestResolve(t *testing.T) {
 }

 func TestTemp(t *testing.T) {
-	parse, _ := url.Parse("http://www.baidu.com/asda/text.txt?a=11&b=333")
-	arr := strings.Split(parse.Path, "/")
-	for i := range arr {
-		fmt.Println(arr[i])
+	f, err := os.Create("e:/data/test.data")
+	if err != nil {
+		log.Fatal(err)
 	}
-	// var begin, end, total int64
-	//fmt.Sscanf("bytes 0-1000/1001", "%s %d-%d/%d", _, &begin, &end, &total)
-	//fmt.Printf("%d-%d/%d", begin, end, total)
+	s := time.Now().UnixNano()
+	if err := f.Truncate(1024 * 1024 * 1024 * 1); err != nil {
+		log.Fatal(err)
+	}
+	fmt.Printf("use %d\n", time.Now().UnixNano()-s)
+	s = time.Now().UnixNano()
+	f.Write([]byte{1})
+	fmt.Printf("use %d\n", time.Now().UnixNano()-s)
+	s = time.Now().UnixNano()
+	f.Seek(1024*1024*512, 0)
+	fmt.Printf("use %d\n", time.Now().UnixNano()-s)
+	s = time.Now().UnixNano()
+	f.Write([]byte{2})
+	fmt.Printf("use %d\n", time.Now().UnixNano()-s)
+	s = time.Now().UnixNano()
+	f.Write([]byte{3})
+	fmt.Printf("use %d\n", time.Now().UnixNano()-s)
+	s = time.Now().UnixNano()
+}
+
+func fileMd5(filePath string) string {
+	file, _ := os.Open(filePath)
+
+	//Tell the program to call the following function when the current function returns
+	defer file.Close()
+
+	//Open a new hash interface to write to
+	hash := md5.New()
+
+	//Copy the file in the hash interface and check for any error
+	if _, err := io.Copy(hash, file); err != nil {
+		return ""
+	}
+	return hex.EncodeToString(hash.Sum(nil))
+}
+
+func TestMd5(t *testing.T) {
+	//8de7a6a2e786861013d61b77b2394012
+	fmt.Println(fileMd5("d:/favicon.ico"))
 }
@@ -4,5 +4,5 @@ type Request struct {
 	Method  string
 	URL     string
 	Header  map[string]string
-	content []byte
+	Content []byte
 }
@@ -1,17 +1,3 @@
-module pdown-core
+module gee-download

 go 1.12
-
-require (
-	github.com/getlantern/context v0.0.0-20190109183933-c447772a6520 // indirect
-	github.com/getlantern/errors v0.0.0-20190325191628-abdb3e3e36f7 // indirect
-	github.com/getlantern/golog v0.0.0-20170508214112-cca714f7feb5 // indirect
-	github.com/getlantern/hex v0.0.0-20190417191902-c6586a6fe0b7 // indirect
-	github.com/getlantern/hidden v0.0.0-20190325191715-f02dbb02be55 // indirect
-	github.com/getlantern/ops v0.0.0-20190325191751-d70cb0d6f85f // indirect
-	github.com/getlantern/systray v0.0.0-20190131073753-26d5b920200d
-	github.com/go-stack/stack v1.8.0 // indirect
-	github.com/oxtoacart/bpool v0.0.0-20190227141107-8c4636f812cc // indirect
-	github.com/stretchr/testify v1.3.0 // indirect
-	github.com/zserge/webview v0.0.0-20190123072648-16c93bcaeaeb
-)
@@ -1,27 +0,0 @@
-github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
-github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/getlantern/context v0.0.0-20190109183933-c447772a6520 h1:NRUJuo3v3WGC/g5YiyF790gut6oQr5f3FBI88Wv0dx4=
-github.com/getlantern/context v0.0.0-20190109183933-c447772a6520/go.mod h1:L+mq6/vvYHKjCX2oez0CgEAJmbq1fbb/oNJIWQkBybY=
-github.com/getlantern/errors v0.0.0-20190325191628-abdb3e3e36f7 h1:6uJ+sZ/e03gkbqZ0kUG6mfKoqDb4XMAzMIwlajq19So=
-github.com/getlantern/errors v0.0.0-20190325191628-abdb3e3e36f7/go.mod h1:l+xpFBrCtDLpK9qNjxs+cHU6+BAdlBaxHqikB6Lku3A=
-github.com/getlantern/golog v0.0.0-20170508214112-cca714f7feb5 h1:Okd7vkn9CfIgDBj1ST/vtBTCfD/kxIhYD412K+FRKPc=
-github.com/getlantern/golog v0.0.0-20170508214112-cca714f7feb5/go.mod h1:Vwx1Cg64gCdIalad44uvQsKZw6LsVczIKZrUBStEjVw=
-github.com/getlantern/hex v0.0.0-20190417191902-c6586a6fe0b7 h1:micT5vkcr9tOVk1FiH8SWKID8ultN44Z+yzd2y/Vyb0=
-github.com/getlantern/hex v0.0.0-20190417191902-c6586a6fe0b7/go.mod h1:dD3CgOrwlzca8ed61CsZouQS5h5jIzkK9ZWrTcf0s+o=
-github.com/getlantern/hidden v0.0.0-20190325191715-f02dbb02be55 h1:XYzSdCbkzOC0FDNrgJqGRo8PCMFOBFL9py72DRs7bmc=
-github.com/getlantern/hidden v0.0.0-20190325191715-f02dbb02be55/go.mod h1:6mmzY2kW1TOOrVy+r41Za2MxXM+hhqTtY3oBKd2AgFA=
-github.com/getlantern/ops v0.0.0-20190325191751-d70cb0d6f85f h1:wrYrQttPS8FHIRSlsrcuKazukx/xqO/PpLZzZXsF+EA=
-github.com/getlantern/ops v0.0.0-20190325191751-d70cb0d6f85f/go.mod h1:D5ao98qkA6pxftxoqzibIBBrLSUli+kYnJqrgBf9cIA=
-github.com/getlantern/systray v0.0.0-20190131073753-26d5b920200d h1:4P2eDMAoQcQoWIIKCNIkuVbQb+paRmpMxVXVfbs7B4U=
-github.com/getlantern/systray v0.0.0-20190131073753-26d5b920200d/go.mod h1:7Splj4WBQSps8jODnMgrIV6goKL0N1HR+mhCAEVWlA0=
-github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk=
-github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
-github.com/oxtoacart/bpool v0.0.0-20190227141107-8c4636f812cc h1:uhnyuvDwdKbjemAXHKsiEZOPagHim2nRjMcazH1g26A=
-github.com/oxtoacart/bpool v0.0.0-20190227141107-8c4636f812cc/go.mod h1:L3UMQOThbttwfYRNFOWLLVXMhk5Lkio4GGOtw5UrxS0=
-github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
-github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
-github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
-github.com/zserge/webview v0.0.0-20190123072648-16c93bcaeaeb h1:zVjnyZIM7UtkG3dNckiudIm+TUHkZqi5xlVQPd3J6/c=
-github.com/zserge/webview v0.0.0-20190123072648-16c93bcaeaeb/go.mod h1:a1CV8KR4Dd1eP2g+mEijGOp+HKczwdKHWyx0aPHKvo4=
@@ -1,26 +1,24 @@
 package main

-import "github.com/zserge/webview"
-import "github.com/getlantern/systray"
+import (
+	"fmt"
+	"gee-download/down"
+)

 func main() {
-	systray.Run(onReady,onExit)
-	// Open wikipedia in a 800x600 resizable window
-	webview.Open("Minimal webview example",
-		"https://www.baidu.com", 800, 600, true)
+	httpRequest, _ := down.BuildHTTPRequest(&down.Request{
+		Method: "get",
+		URL:    "http://github.com/proxyee-down-org/proxyee-down/releases/download/3.4/proxyee-down-main.jar",
+		Header: map[string]string{
+			"Host":            "github.com",
+			"User-Agent":      "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Safari/537.36",
+			"Accept":          "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8",
+			"Referer":         "http://github.com/proxyee-down-org/proxyee-down/releases",
+			"Accept-Encoding": "gzip, deflate, br",
+			"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
+		},
+	})
+	got, _ := down.Resolve(*httpRequest)
+	fmt.Println(got)
+	// webview.Open("Minimal webview example", "https://www.baidu.com", 800, 600, true)
 }
-
-
-func onReady() {
-	systray.SetIcon(icon.Data)
-	systray.SetTitle("Awesome App")
-	systray.SetTooltip("Pretty awesome超级棒")
-	mQuit := systray.AddMenuItem("Quit", "Quit the whole app")
-
-	// Sets the icon of a menu item. Only available on Mac.
-	mQuit.SetIcon(icon.Data)
-}
-
-func onExit() {
-	// clean up here
-}