HTTP download test pass

2026-11-14 20:33:41 +00:00 · 2019-05-05 01:23:48 +08:00
1 parent 00468e4e4a
commit 66a4b022f2
@@ -1,7 +1,6 @@
 package down

 import (
-	"bytes"
 	"fmt"
 	"io"
 	"mime"
@@ -11,27 +10,23 @@ import (
 	"os"
 	"strconv"
 	"strings"
+	"sync"
 )

 // Resolve return the file response to be downloaded
 func Resolve(request *Request) (*Response, error) {
-	// Build request
-	httpRequest, err := http.NewRequest(strings.ToUpper(request.Method), request.URL, nil)
+	httpRequest, err := BuildHTTPRequest(request)
 	if err != nil {
 		return nil, err
 	}
-	for k, v := range request.Header {
-		httpRequest.Header.Add(k, v)
-	}
 	// Use "Range" header to resolve
 	httpRequest.Header.Add("Range", "bytes=0-0")
-	// Cookie handle
-	jar, _ := cookiejar.New(nil)
-	httpClient := &http.Client{Jar: jar}
+	httpClient := BuildHTTPClient()
 	response, err := httpClient.Do(httpRequest)
 	if err != nil {
 		return nil, err
 	}
+	defer response.Body.Close()
 	if response.StatusCode != 200 && response.StatusCode != 206 {
 		return nil, fmt.Errorf("Response status error:%d", response.StatusCode)
 	}
@@ -47,7 +42,7 @@ func Resolve(request *Request) (*Response, error) {
 	}
 	// Get file name by URL
 	if ret.Name == "" {
-		parse, err := url.Parse(request.URL)
+		parse, err := url.Parse(httpRequest.URL.String())
 		if err == nil {
 			// e.g. /files/test.txt => test.txt
 			ret.Name = subLastSlash(parse.Path)
@@ -82,6 +77,45 @@ func Resolve(request *Request) (*Response, error) {
 	return ret, nil
 }

+// Down
+func Down(request *Request) error {
+	response, err := Resolve(request)
+	if err != nil {
+		return err
+	}
+	// allocate file
+	file, err := os.Create(response.Name)
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+	if err := file.Truncate(response.Size); err != nil {
+		return err
+	}
+	// support range
+	if response.Range {
+		cons := 16
+		chunkSize := response.Size / int64(cons)
+		var (
+			waitGroup = &sync.WaitGroup{}
+			fileLock  = &sync.Mutex{}
+		)
+		waitGroup.Add(cons)
+		for i := 0; i < cons; i++ {
+			start := int64(i) * chunkSize
+			end := start + chunkSize
+			if i == cons-1 {
+				end = response.Size
+			}
+			go downChunk(request, file, start, end-1, waitGroup, fileLock)
+		}
+		waitGroup.Wait()
+	} else {
+		downChunk(request, file, 0, response.Size, nil, nil)
+	}
+	return nil
+}
+
 func subLastSlash(str string) string {
 	index := strings.LastIndex(str, "/")
 	if index != -1 {
@@ -90,38 +124,61 @@ func subLastSlash(str string) string {
 	return ""
 }

-// Down 下载
-func Down(request *Request) {
-	httpRequest, err := http.NewRequest(request.Method, request.URL, bytes.NewReader(request.content))
+func BuildHTTPRequest(request *Request) (*http.Request, error) {
+	// Build request
+	httpRequest, err := http.NewRequest(strings.ToUpper(request.Method), request.URL, nil)
 	if err != nil {
-		fmt.Println("create http request error")
-		return
+		return nil, err
 	}
 	for k, v := range request.Header {
 		httpRequest.Header.Add(k, v)
 	}
-	httpClient := &http.Client{}
-	response, err := httpClient.Do(httpRequest)
+	return httpRequest, nil
+}
+
+func BuildHTTPClient() *http.Client {
+	// Cookie handle
+	jar, _ := cookiejar.New(nil)
+
+	return &http.Client{Jar: jar}
+}
+
+func downChunk(request *Request, file *os.File, start int64, end int64, waitGroup *sync.WaitGroup, fileLock *sync.Mutex) {
+	defer func() {
+		if waitGroup != nil {
+			waitGroup.Done()
+		}
+	}()
+	httpRequest, _ := BuildHTTPRequest(request)
+	httpRequest.Header.Add("Range", fmt.Sprintf("bytes=%d-%d", start, end))
+	fmt.Printf("down %d-%d\n", start, end)
+	httpClient := BuildHTTPClient()
+	httpResponse, err := httpClient.Do(httpRequest)
 	if err != nil {
-		fmt.Printf("do http request error:%s\n", err)
+		fmt.Println(err)
 		return
 	}
-	if response.StatusCode >= 200 && response.StatusCode < 300 {
-		_, params, _ := mime.ParseMediaType(response.Header.Get("Content-Disposition"))
-		filename := params["filename"]
-		if len(filename) > 0 {
-			fmt.Printf("filename:%s\n", filename)
+	defer httpResponse.Body.Close()
+	buf := make([]byte, 8192)
+	writeIndex := int64(start)
+	for {
+		len, err := httpResponse.Body.Read(buf)
+		if len > 0 {
+			fileLock.Lock()
+			writeSize, err := file.WriteAt(buf[0:len], writeIndex)
+			fileLock.Unlock()
+			if err != nil {
+				fmt.Println(err)
+				return
+			}
+			writeIndex += int64(writeSize)
+		}
+		if err != nil {
+			if err != io.EOF {
+				fmt.Println(err)
+				return
+			}
+			break
 		}
 	}
-
-	fmt.Printf("response status:%d %s\n", response.StatusCode, response.Status)
-	fmt.Printf("response heads:%v\n", response.Header)
-	file, err := os.Create("test.txt")
-	if err != nil {
-		fmt.Println("create file error")
-		return
-	}
-	defer file.Close()
-	fmt.Println("file close")
-	io.Copy(file, response.Body)
 }
@@ -1,11 +1,15 @@
 package down

 import (
+	"crypto/md5"
+	"encoding/hex"
 	"fmt"
-	"net/url"
+	"io"
+	"log"
+	"os"
 	"reflect"
-	"strings"
 	"testing"
+	"time"
 )

 func TestDown(t *testing.T) {
@@ -17,19 +21,18 @@ func TestDown(t *testing.T) {
 		args args
 	}{
 		{
-			"readme",
+			"proxyee-down.ico",
 			args{
 				&Request{
 					"get",
-					"https://raw.githubusercontent.com/proxyee-down-org/proxyee-down/master/README.md",
+					"https://raw.githubusercontent.com/proxyee-down-org/proxyee-down/master/front/public/favicon.ico",
 					map[string]string{
+						"Host":            "github.com",
+						"User-Agent":      "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Safari/537.36",
 						"Accept":          "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8",
-						"Host":            "raw.githubusercontent.com",
+						"Referer":         "http://github.com/proxyee-down-org/proxyee-down/releases",
 						"Accept-Encoding": "gzip, deflate, br",
 						"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
-						"Cache-Control":   "no-cache",
-						"Connection":      "keep-alive",
-						"User-Agent":      "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Safari/537.36",
 					},
 					nil,
 				},
@@ -39,7 +42,17 @@ func TestDown(t *testing.T) {

 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			Down(tt.args.request)
+			os.Remove("favicon.ico")
+			err := Down(tt.args.request)
+			if err != nil {
+				t.Errorf("error down= %v", err)
+				return
+			}
+			downMd5 := fileMd5("favicon.ico")
+			if "8de7a6a2e786861013d61b77b2394012" != downMd5 {
+				t.Errorf("error md5= %v", downMd5)
+				return
+			}
 		})
 	}
 }
@@ -56,11 +69,11 @@ func TestResolve(t *testing.T) {
 		wantErr bool
 	}{
 		{
-			"proxyee-down",
+			"proxyee-down.ico",
 			args{
 				&Request{
 					"get",
-					"http://github.com/proxyee-down-org/proxyee-down/releases/download/3.4/proxyee-down-main.jar",
+					"https://raw.githubusercontent.com/proxyee-down-org/proxyee-down/master/front/public/favicon.ico",
 					map[string]string{
 						"Host":            "github.com",
 						"User-Agent":      "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Safari/537.36",
@@ -72,7 +85,7 @@ func TestResolve(t *testing.T) {
 					nil,
 				},
 			},
-			&Response{"proxyee-down-main.jar", 30159703, true},
+			&Response{"favicon.ico", 116093, true},
 			false,
 		},
 	}
@@ -91,12 +104,47 @@ func TestResolve(t *testing.T) {
 }

 func TestTemp(t *testing.T) {
-	parse, _ := url.Parse("http://www.baidu.com/asda/text.txt?a=11&b=333")
-	arr := strings.Split(parse.Path, "/")
-	for i := range arr {
-		fmt.Println(arr[i])
+	f, err := os.Create("e:/data/test.data")
+	if err != nil {
+		log.Fatal(err)
 	}
-	// var begin, end, total int64
-	//fmt.Sscanf("bytes 0-1000/1001", "%s %d-%d/%d", _, &begin, &end, &total)
-	//fmt.Printf("%d-%d/%d", begin, end, total)
+	s := time.Now().UnixNano()
+	if err := f.Truncate(1024 * 1024 * 1024 * 1); err != nil {
+		log.Fatal(err)
+	}
+	fmt.Printf("use %d\n", time.Now().UnixNano()-s)
+	s = time.Now().UnixNano()
+	f.Write([]byte{1})
+	fmt.Printf("use %d\n", time.Now().UnixNano()-s)
+	s = time.Now().UnixNano()
+	f.Seek(1024*1024*512, 0)
+	fmt.Printf("use %d\n", time.Now().UnixNano()-s)
+	s = time.Now().UnixNano()
+	f.Write([]byte{2})
+	fmt.Printf("use %d\n", time.Now().UnixNano()-s)
+	s = time.Now().UnixNano()
+	f.Write([]byte{3})
+	fmt.Printf("use %d\n", time.Now().UnixNano()-s)
+	s = time.Now().UnixNano()
+}
+
+func fileMd5(filePath string) string {
+	file, _ := os.Open(filePath)
+
+	//Tell the program to call the following function when the current function returns
+	defer file.Close()
+
+	//Open a new hash interface to write to
+	hash := md5.New()
+
+	//Copy the file in the hash interface and check for any error
+	if _, err := io.Copy(hash, file); err != nil {
+		return ""
+	}
+	return hex.EncodeToString(hash.Sum(nil))
+}
+
+func TestMd5(t *testing.T) {
+	//8de7a6a2e786861013d61b77b2394012
+	fmt.Println(fileMd5("d:/favicon.ico"))
 }
@@ -4,5 +4,5 @@ type Request struct {
 	Method  string
 	URL     string
 	Header  map[string]string
-	content []byte
+	Content []byte
 }