diff --git a/README.md b/README.md index 5f3019b0..cd3a615e 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,2 @@ -## go-download +## gee-download A fast download client,support HTTP&P2P. \ No newline at end of file diff --git a/down/down.go b/down/down.go index fdf7fb6a..f9ceb2ee 100644 --- a/down/down.go +++ b/down/down.go @@ -1,7 +1,6 @@ package down import ( - "bytes" "fmt" "io" "mime" @@ -11,27 +10,23 @@ import ( "os" "strconv" "strings" + "sync" ) // Resolve return the file response to be downloaded func Resolve(request *Request) (*Response, error) { - // Build request - httpRequest, err := http.NewRequest(strings.ToUpper(request.Method), request.URL, nil) + httpRequest, err := BuildHTTPRequest(request) if err != nil { return nil, err } - for k, v := range request.Header { - httpRequest.Header.Add(k, v) - } // Use "Range" header to resolve httpRequest.Header.Add("Range", "bytes=0-0") - // Cookie handle - jar, _ := cookiejar.New(nil) - httpClient := &http.Client{Jar: jar} + httpClient := BuildHTTPClient() response, err := httpClient.Do(httpRequest) if err != nil { return nil, err } + defer response.Body.Close() if response.StatusCode != 200 && response.StatusCode != 206 { return nil, fmt.Errorf("Response status error:%d", response.StatusCode) } @@ -47,7 +42,7 @@ func Resolve(request *Request) (*Response, error) { } // Get file name by URL if ret.Name == "" { - parse, err := url.Parse(request.URL) + parse, err := url.Parse(httpRequest.URL.String()) if err == nil { // e.g. /files/test.txt => test.txt ret.Name = subLastSlash(parse.Path) @@ -82,6 +77,45 @@ func Resolve(request *Request) (*Response, error) { return ret, nil } +// Down +func Down(request *Request) error { + response, err := Resolve(request) + if err != nil { + return err + } + // allocate file + file, err := os.Create(response.Name) + if err != nil { + return err + } + defer file.Close() + if err := file.Truncate(response.Size); err != nil { + return err + } + // support range + if response.Range { + cons := 16 + chunkSize := response.Size / int64(cons) + var ( + waitGroup = &sync.WaitGroup{} + fileLock = &sync.Mutex{} + ) + waitGroup.Add(cons) + for i := 0; i < cons; i++ { + start := int64(i) * chunkSize + end := start + chunkSize + if i == cons-1 { + end = response.Size + } + go downChunk(request, file, start, end-1, waitGroup, fileLock) + } + waitGroup.Wait() + } else { + downChunk(request, file, 0, response.Size, nil, nil) + } + return nil +} + func subLastSlash(str string) string { index := strings.LastIndex(str, "/") if index != -1 { @@ -90,38 +124,61 @@ func subLastSlash(str string) string { return "" } -// Down 下载 -func Down(request *Request) { - httpRequest, err := http.NewRequest(request.Method, request.URL, bytes.NewReader(request.content)) +func BuildHTTPRequest(request *Request) (*http.Request, error) { + // Build request + httpRequest, err := http.NewRequest(strings.ToUpper(request.Method), request.URL, nil) if err != nil { - fmt.Println("create http request error") - return + return nil, err } for k, v := range request.Header { httpRequest.Header.Add(k, v) } - httpClient := &http.Client{} - response, err := httpClient.Do(httpRequest) + return httpRequest, nil +} + +func BuildHTTPClient() *http.Client { + // Cookie handle + jar, _ := cookiejar.New(nil) + + return &http.Client{Jar: jar} +} + +func downChunk(request *Request, file *os.File, start int64, end int64, waitGroup *sync.WaitGroup, fileLock *sync.Mutex) { + defer func() { + if waitGroup != nil { + waitGroup.Done() + } + }() + httpRequest, _ := BuildHTTPRequest(request) + httpRequest.Header.Add("Range", fmt.Sprintf("bytes=%d-%d", start, end)) + fmt.Printf("down %d-%d\n", start, end) + httpClient := BuildHTTPClient() + httpResponse, err := httpClient.Do(httpRequest) if err != nil { - fmt.Printf("do http request error:%s\n", err) + fmt.Println(err) return } - if response.StatusCode >= 200 && response.StatusCode < 300 { - _, params, _ := mime.ParseMediaType(response.Header.Get("Content-Disposition")) - filename := params["filename"] - if len(filename) > 0 { - fmt.Printf("filename:%s\n", filename) + defer httpResponse.Body.Close() + buf := make([]byte, 8192) + writeIndex := int64(start) + for { + len, err := httpResponse.Body.Read(buf) + if len > 0 { + fileLock.Lock() + writeSize, err := file.WriteAt(buf[0:len], writeIndex) + fileLock.Unlock() + if err != nil { + fmt.Println(err) + return + } + writeIndex += int64(writeSize) + } + if err != nil { + if err != io.EOF { + fmt.Println(err) + return + } + break } } - - fmt.Printf("response status:%d %s\n", response.StatusCode, response.Status) - fmt.Printf("response heads:%v\n", response.Header) - file, err := os.Create("test.txt") - if err != nil { - fmt.Println("create file error") - return - } - defer file.Close() - fmt.Println("file close") - io.Copy(file, response.Body) } diff --git a/down/down_test.go b/down/down_test.go index 0a2aac10..b5169018 100644 --- a/down/down_test.go +++ b/down/down_test.go @@ -1,11 +1,15 @@ package down import ( + "crypto/md5" + "encoding/hex" "fmt" - "net/url" + "io" + "log" + "os" "reflect" - "strings" "testing" + "time" ) func TestDown(t *testing.T) { @@ -17,19 +21,18 @@ func TestDown(t *testing.T) { args args }{ { - "readme", + "proxyee-down.ico", args{ &Request{ "get", - "https://raw.githubusercontent.com/proxyee-down-org/proxyee-down/master/README.md", + "https://raw.githubusercontent.com/proxyee-down-org/proxyee-down/master/front/public/favicon.ico", map[string]string{ + "Host": "github.com", + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Safari/537.36", "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8", - "Host": "raw.githubusercontent.com", + "Referer": "http://github.com/proxyee-down-org/proxyee-down/releases", "Accept-Encoding": "gzip, deflate, br", "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8", - "Cache-Control": "no-cache", - "Connection": "keep-alive", - "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Safari/537.36", }, nil, }, @@ -39,7 +42,17 @@ func TestDown(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - Down(tt.args.request) + os.Remove("favicon.ico") + err := Down(tt.args.request) + if err != nil { + t.Errorf("error down= %v", err) + return + } + downMd5 := fileMd5("favicon.ico") + if "8de7a6a2e786861013d61b77b2394012" != downMd5 { + t.Errorf("error md5= %v", downMd5) + return + } }) } } @@ -56,11 +69,11 @@ func TestResolve(t *testing.T) { wantErr bool }{ { - "proxyee-down", + "proxyee-down.ico", args{ &Request{ "get", - "http://github.com/proxyee-down-org/proxyee-down/releases/download/3.4/proxyee-down-main.jar", + "https://raw.githubusercontent.com/proxyee-down-org/proxyee-down/master/front/public/favicon.ico", map[string]string{ "Host": "github.com", "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Safari/537.36", @@ -72,7 +85,7 @@ func TestResolve(t *testing.T) { nil, }, }, - &Response{"proxyee-down-main.jar", 30159703, true}, + &Response{"favicon.ico", 116093, true}, false, }, } @@ -91,12 +104,47 @@ func TestResolve(t *testing.T) { } func TestTemp(t *testing.T) { - parse, _ := url.Parse("http://www.baidu.com/asda/text.txt?a=11&b=333") - arr := strings.Split(parse.Path, "/") - for i := range arr { - fmt.Println(arr[i]) + f, err := os.Create("e:/data/test.data") + if err != nil { + log.Fatal(err) } - // var begin, end, total int64 - //fmt.Sscanf("bytes 0-1000/1001", "%s %d-%d/%d", _, &begin, &end, &total) - //fmt.Printf("%d-%d/%d", begin, end, total) + s := time.Now().UnixNano() + if err := f.Truncate(1024 * 1024 * 1024 * 1); err != nil { + log.Fatal(err) + } + fmt.Printf("use %d\n", time.Now().UnixNano()-s) + s = time.Now().UnixNano() + f.Write([]byte{1}) + fmt.Printf("use %d\n", time.Now().UnixNano()-s) + s = time.Now().UnixNano() + f.Seek(1024*1024*512, 0) + fmt.Printf("use %d\n", time.Now().UnixNano()-s) + s = time.Now().UnixNano() + f.Write([]byte{2}) + fmt.Printf("use %d\n", time.Now().UnixNano()-s) + s = time.Now().UnixNano() + f.Write([]byte{3}) + fmt.Printf("use %d\n", time.Now().UnixNano()-s) + s = time.Now().UnixNano() +} + +func fileMd5(filePath string) string { + file, _ := os.Open(filePath) + + //Tell the program to call the following function when the current function returns + defer file.Close() + + //Open a new hash interface to write to + hash := md5.New() + + //Copy the file in the hash interface and check for any error + if _, err := io.Copy(hash, file); err != nil { + return "" + } + return hex.EncodeToString(hash.Sum(nil)) +} + +func TestMd5(t *testing.T) { + //8de7a6a2e786861013d61b77b2394012 + fmt.Println(fileMd5("d:/favicon.ico")) } diff --git a/down/request.go b/down/request.go index cafe03f9..98fc3042 100644 --- a/down/request.go +++ b/down/request.go @@ -4,5 +4,5 @@ type Request struct { Method string URL string Header map[string]string - content []byte + Content []byte } diff --git a/go.mod b/go.mod index efb856c6..90a5e610 100644 --- a/go.mod +++ b/go.mod @@ -1,17 +1,3 @@ -module pdown-core +module gee-download go 1.12 - -require ( - github.com/getlantern/context v0.0.0-20190109183933-c447772a6520 // indirect - github.com/getlantern/errors v0.0.0-20190325191628-abdb3e3e36f7 // indirect - github.com/getlantern/golog v0.0.0-20170508214112-cca714f7feb5 // indirect - github.com/getlantern/hex v0.0.0-20190417191902-c6586a6fe0b7 // indirect - github.com/getlantern/hidden v0.0.0-20190325191715-f02dbb02be55 // indirect - github.com/getlantern/ops v0.0.0-20190325191751-d70cb0d6f85f // indirect - github.com/getlantern/systray v0.0.0-20190131073753-26d5b920200d - github.com/go-stack/stack v1.8.0 // indirect - github.com/oxtoacart/bpool v0.0.0-20190227141107-8c4636f812cc // indirect - github.com/stretchr/testify v1.3.0 // indirect - github.com/zserge/webview v0.0.0-20190123072648-16c93bcaeaeb -) diff --git a/go.sum b/go.sum index bce638ef..e69de29b 100644 --- a/go.sum +++ b/go.sum @@ -1,27 +0,0 @@ -github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/getlantern/context v0.0.0-20190109183933-c447772a6520 h1:NRUJuo3v3WGC/g5YiyF790gut6oQr5f3FBI88Wv0dx4= -github.com/getlantern/context v0.0.0-20190109183933-c447772a6520/go.mod h1:L+mq6/vvYHKjCX2oez0CgEAJmbq1fbb/oNJIWQkBybY= -github.com/getlantern/errors v0.0.0-20190325191628-abdb3e3e36f7 h1:6uJ+sZ/e03gkbqZ0kUG6mfKoqDb4XMAzMIwlajq19So= -github.com/getlantern/errors v0.0.0-20190325191628-abdb3e3e36f7/go.mod h1:l+xpFBrCtDLpK9qNjxs+cHU6+BAdlBaxHqikB6Lku3A= -github.com/getlantern/golog v0.0.0-20170508214112-cca714f7feb5 h1:Okd7vkn9CfIgDBj1ST/vtBTCfD/kxIhYD412K+FRKPc= -github.com/getlantern/golog v0.0.0-20170508214112-cca714f7feb5/go.mod h1:Vwx1Cg64gCdIalad44uvQsKZw6LsVczIKZrUBStEjVw= -github.com/getlantern/hex v0.0.0-20190417191902-c6586a6fe0b7 h1:micT5vkcr9tOVk1FiH8SWKID8ultN44Z+yzd2y/Vyb0= -github.com/getlantern/hex v0.0.0-20190417191902-c6586a6fe0b7/go.mod h1:dD3CgOrwlzca8ed61CsZouQS5h5jIzkK9ZWrTcf0s+o= -github.com/getlantern/hidden v0.0.0-20190325191715-f02dbb02be55 h1:XYzSdCbkzOC0FDNrgJqGRo8PCMFOBFL9py72DRs7bmc= -github.com/getlantern/hidden v0.0.0-20190325191715-f02dbb02be55/go.mod h1:6mmzY2kW1TOOrVy+r41Za2MxXM+hhqTtY3oBKd2AgFA= -github.com/getlantern/ops v0.0.0-20190325191751-d70cb0d6f85f h1:wrYrQttPS8FHIRSlsrcuKazukx/xqO/PpLZzZXsF+EA= -github.com/getlantern/ops v0.0.0-20190325191751-d70cb0d6f85f/go.mod h1:D5ao98qkA6pxftxoqzibIBBrLSUli+kYnJqrgBf9cIA= -github.com/getlantern/systray v0.0.0-20190131073753-26d5b920200d h1:4P2eDMAoQcQoWIIKCNIkuVbQb+paRmpMxVXVfbs7B4U= -github.com/getlantern/systray v0.0.0-20190131073753-26d5b920200d/go.mod h1:7Splj4WBQSps8jODnMgrIV6goKL0N1HR+mhCAEVWlA0= -github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= -github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/oxtoacart/bpool v0.0.0-20190227141107-8c4636f812cc h1:uhnyuvDwdKbjemAXHKsiEZOPagHim2nRjMcazH1g26A= -github.com/oxtoacart/bpool v0.0.0-20190227141107-8c4636f812cc/go.mod h1:L3UMQOThbttwfYRNFOWLLVXMhk5Lkio4GGOtw5UrxS0= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/zserge/webview v0.0.0-20190123072648-16c93bcaeaeb h1:zVjnyZIM7UtkG3dNckiudIm+TUHkZqi5xlVQPd3J6/c= -github.com/zserge/webview v0.0.0-20190123072648-16c93bcaeaeb/go.mod h1:a1CV8KR4Dd1eP2g+mEijGOp+HKczwdKHWyx0aPHKvo4= diff --git a/main.go b/main.go index 59bd0db5..9b71ddf7 100644 --- a/main.go +++ b/main.go @@ -1,26 +1,24 @@ package main -import "github.com/zserge/webview" -import "github.com/getlantern/systray" +import ( + "fmt" + "gee-download/down" +) func main() { - systray.Run(onReady,onExit) - // Open wikipedia in a 800x600 resizable window - webview.Open("Minimal webview example", - "https://www.baidu.com", 800, 600, true) + httpRequest, _ := down.BuildHTTPRequest(&down.Request{ + Method: "get", + URL: "http://github.com/proxyee-down-org/proxyee-down/releases/download/3.4/proxyee-down-main.jar", + Header: map[string]string{ + "Host": "github.com", + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Safari/537.36", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8", + "Referer": "http://github.com/proxyee-down-org/proxyee-down/releases", + "Accept-Encoding": "gzip, deflate, br", + "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8", + }, + }) + got, _ := down.Resolve(*httpRequest) + fmt.Println(got) + // webview.Open("Minimal webview example", "https://www.baidu.com", 800, 600, true) } - - -func onReady() { - systray.SetIcon(icon.Data) - systray.SetTitle("Awesome App") - systray.SetTooltip("Pretty awesome超级棒") - mQuit := systray.AddMenuItem("Quit", "Quit the whole app") - - // Sets the icon of a menu item. Only available on Mac. - mQuit.SetIcon(icon.Data) -} - -func onExit() { - // clean up here -} \ No newline at end of file