包名重构

protocol/http/down.go

@@ -0,0 +1,182 @@
+package http
+
+import (
+	"fmt"
+	"io"
+	"mime"
+	"net/http"
+	"net/http/cookiejar"
+	"net/url"
+	"os"
+	"strconv"
+	"strings"
+	"sync"
+)
+
+type Download interface {
+	Resolve(request *http.Request) (*http.Response, error)
+	Down(request *http.Request) error
+}
+
+// Resolve return the file response to be downloaded
+func Resolve(request *Request) (*Response, error) {
+	httpRequest, err := BuildHTTPRequest(request)
+	if err != nil {
+		return nil, err
+	}
+	// Use "Range" header to resolve
+	httpRequest.Header.Add("Range", "bytes=0-0")
+	httpClient := BuildHTTPClient()
+	response, err := httpClient.Do(httpRequest)
+	if err != nil {
+		return nil, err
+	}
+	defer response.Body.Close()
+	if response.StatusCode != 200 && response.StatusCode != 206 {
+		return nil, fmt.Errorf("response status error:%d", response.StatusCode)
+	}
+	ret := &Response{}
+	// Get file name by "Content-Disposition"
+	contentDisposition := response.Header.Get("Content-Disposition")
+	if contentDisposition != "" {
+		_, params, _ := mime.ParseMediaType(contentDisposition)
+		filename := params["filename"]
+		if filename != "" {
+			ret.Name = filename
+		}
+	}
+	// Get file name by URL
+	if ret.Name == "" {
+		parse, err := url.Parse(httpRequest.URL.String())
+		if err == nil {
+			// e.g. /files/test.txt => test.txt
+			ret.Name = subLastSlash(parse.Path)
+		}
+	}
+	// Unknow file name
+	if ret.Name == "" {
+		ret.Name = "unknow"
+	}
+	// Is support range
+	ret.Range = response.StatusCode == 206
+	// Get file size
+	if ret.Range {
+		contentRange := response.Header.Get("Content-Range")
+		if contentRange != "" {
+			// e.g. bytes 0-1000/1001 => 1001
+			total := subLastSlash(contentRange)
+			if total != "" && total != "*" {
+				parse, err := strconv.ParseInt(total, 10, 64)
+				if err != nil {
+					return nil, err
+				}
+				ret.Size = parse
+			}
+		}
+	} else {
+		contentLength := response.Header.Get("Content-Length")
+		if contentLength != "" {
+			ret.Size, _ = strconv.ParseInt(contentLength, 10, 64)
+		}
+	}
+	return ret, nil
+}
+
+// Down
+func Down(request *Request) error {
+	response, err := Resolve(request)
+	if err != nil {
+		return err
+	}
+	// allocate file
+	file, err := os.Create(response.Name)
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+	if err := file.Truncate(response.Size); err != nil {
+		return err
+	}
+	// support range
+	if response.Range {
+		cons := 16
+		chunkSize := response.Size / int64(cons)
+		waitGroup := &sync.WaitGroup{}
+		waitGroup.Add(cons)
+		for i := 0; i < cons; i++ {
+			start := int64(i) * chunkSize
+			end := start + chunkSize
+			if i == cons-1 {
+				end = response.Size
+			}
+			go downChunk(request, file, start, end-1, waitGroup)
+		}
+		waitGroup.Wait()
+	} else {
+		downChunk(request, file, 0, response.Size, nil)
+	}
+	return nil
+}
+
+func subLastSlash(str string) string {
+	index := strings.LastIndex(str, "/")
+	if index != -1 {
+		return str[index+1:]
+	}
+	return ""
+}
+
+func BuildHTTPRequest(request *Request) (*http.Request, error) {
+	// Build request
+	httpRequest, err := http.NewRequest(strings.ToUpper(request.Method), request.URL, nil)
+	if err != nil {
+		return nil, err
+	}
+	for k, v := range request.Header {
+		httpRequest.Header.Add(k, v)
+	}
+	return httpRequest, nil
+}
+
+func BuildHTTPClient() *http.Client {
+	// Cookie handle
+	jar, _ := cookiejar.New(nil)
+
+	return &http.Client{Jar: jar}
+}
+
+func downChunk(request *Request, file *os.File, start int64, end int64, waitGroup *sync.WaitGroup) {
+	if waitGroup != nil {
+		defer waitGroup.Done()
+	}
+	httpRequest, _ := BuildHTTPRequest(request)
+	httpRequest.Header.Add("Range", fmt.Sprintf("bytes=%d-%d", start, end))
+	fmt.Printf("down %d-%d\n", start, end)
+	httpClient := BuildHTTPClient()
+	httpResponse, err := httpClient.Do(httpRequest)
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+	defer httpResponse.Body.Close()
+	buf := make([]byte, 8192)
+	writeIndex := start
+	for {
+		n, err := httpResponse.Body.Read(buf)
+		if n > 0 {
+			writeSize, err := file.WriteAt(buf[0:n], writeIndex)
+			if err != nil {
+				fmt.Println(err)
+				return
+			}
+			writeIndex += int64(writeSize)
+		}
+		if err != nil {
+			if err != io.EOF {
+				fmt.Println(err)
+				return
+			}
+			break
+		}
+	}
+}

protocol/http/down_test.go

@@ -0,0 +1,154 @@
+package http
+
+import (
+	"crypto/md5"
+	"encoding/hex"
+	"fmt"
+	"io"
+	"log"
+	"os"
+	"reflect"
+	"testing"
+	"time"
+)
+
+func TestDown(t *testing.T) {
+	type args struct {
+		request *Request
+	}
+	tests := []struct {
+		name string
+		args args
+	}{
+		{
+			"proxyee-down.ico",
+			args{
+				&Request{
+					"get",
+					"https://raw.githubusercontent.com/proxyee-down-org/proxyee-down/master/front/public/favicon.ico",
+					map[string]string{
+						"Host":            "github.com",
+						"User-Agent":      "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Safari/537.36",
+						"Accept":          "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8",
+						"Referer":         "http://github.com/proxyee-down-org/proxyee-down/releases",
+						"Accept-Encoding": "gzip, deflate, br",
+						"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
+					},
+					nil,
+				},
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			os.Remove("favicon.ico")
+			err := Down(tt.args.request)
+			if err != nil {
+				t.Errorf("error down= %v", err)
+				return
+			}
+			downMd5 := fileMd5("favicon.ico")
+			os.Remove("favicon.ico")
+			if "8de7a6a2e786861013d61b77b2394012" != downMd5 {
+				t.Errorf("error md5= %v", downMd5)
+				return
+			}
+		})
+	}
+}
+
+func TestResolve(t *testing.T) {
+	// os.Setenv("HTTP_PROXY", "http://127.0.0.1:8888")
+	type args struct {
+		request *Request
+	}
+	tests := []struct {
+		name    string
+		args    args
+		want    *Response
+		wantErr bool
+	}{
+		{
+			"proxyee-down.ico",
+			args{
+				&Request{
+					"get",
+					"https://raw.githubusercontent.com/proxyee-down-org/proxyee-down/master/front/public/favicon.ico",
+					map[string]string{
+						"Host":            "github.com",
+						"User-Agent":      "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Safari/537.36",
+						"Accept":          "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8",
+						"Referer":         "http://github.com/proxyee-down-org/proxyee-down/releases",
+						"Accept-Encoding": "gzip, deflate, br",
+						"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
+					},
+					nil,
+				},
+			},
+			&Response{"favicon.ico", 116093, true},
+			false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := Resolve(tt.args.request)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("Resolve() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("Resolve() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestTemp(t *testing.T) {
+	f, err := os.Create("e:/testbt/test.data")
+	if err != nil {
+		log.Fatal(err)
+	}
+	if err := f.Truncate(1024 * 1024 * 512); err != nil {
+		log.Fatal(err)
+	}
+	bts := []byte{1, 2, 3, 4, 5, 6, 7, 8, 9}
+	s := time.Now().UnixNano()
+	f.Write(bts)
+	fmt.Printf("write from 0 use: %d\n", time.Now().UnixNano()-s)
+	s = time.Now().UnixNano()
+	f.Seek(1024*1024*128, 0)
+	f.Write(bts)
+	fmt.Printf("write from 128M use %d\n", time.Now().UnixNano()-s)
+	s = time.Now().UnixNano()
+	f.Write(bts)
+	fmt.Printf("write from 128M+9b use %d\n", time.Now().UnixNano()-s)
+	s = time.Now().UnixNano()
+	f.Seek(0, 0)
+	f.Write(bts)
+	fmt.Printf("write from 0 agein use %d\n", time.Now().UnixNano()-s)
+	s = time.Now().UnixNano()
+	f.Seek(1024*1024*64, 0)
+	f.Write(bts)
+	fmt.Printf("write from agein 64M use %d\n", time.Now().UnixNano()-s)
+}
+
+func fileMd5(filePath string) string {
+	file, _ := os.Open(filePath)
+
+	// Tell the program to call the following function when the current function returns
+	defer file.Close()
+
+	// Open a new hash interface to write to
+	hash := md5.New()
+
+	// Copy the file in the hash interface and check for any error
+	if _, err := io.Copy(hash, file); err != nil {
+		return ""
+	}
+	return hex.EncodeToString(hash.Sum(nil))
+}
+
+func TestTemp2(t *testing.T) {
+	fmt.Println("123456789"[2:3])
+}

protocol/http/request.go

@@ -0,0 +1,8 @@
+package http
+
+type Request struct {
+	Method  string
+	URL     string
+	Header  map[string]string
+	Content []byte
+}

protocol/http/response.go

@@ -0,0 +1,7 @@
+package http
+
+type Response struct {
+	Name  string
+	Size  int64
+	Range bool
+}