mirror of
https://github.com/GopeedLab/gopeed.git
synced 2026-11-17 22:03:41 +00:00
fix(http): use Content-Range total size for 206 responses (#1545)
This commit is contained in:
1 parent
f0dab90e09
commit
363153fa33
2 files changed
+166
-18
No files matched your search
@@ -417,12 +417,26 @@ func (f *Fetcher) Resolve(req *base.Request, opts *base.Options) error {
|
|||||||
res.Range = true
|
res.Range = true
|
||||||
}
|
}
|
||||||
|
|
||||||
// Get content length from Content-Length header
|
// A partial response describes one segment, not the whole resource.
|
||||||
contentLength := resp.Header.Get(base.HttpHeaderContentLength)
|
reuseResolveBody := true
|
||||||
if contentLength != "" {
|
if resp.StatusCode == base.HttpCodePartialContent {
|
||||||
parse, err := strconv.ParseInt(contentLength, 10, 64)
|
start, _, total, err := parseRangeResponse(resp)
|
||||||
if err == nil {
|
if err != nil {
|
||||||
res.Size = parse
|
resp.Body.Close()
|
||||||
|
f.setState(stateError)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
res.Size = total
|
||||||
|
res.Range = true
|
||||||
|
// Prefetch is a contiguous prefix copied at offset zero. A segment
|
||||||
|
// starting elsewhere must be fetched again through the Range path.
|
||||||
|
reuseResolveBody = start == 0
|
||||||
|
} else {
|
||||||
|
contentLength := resp.Header.Get(base.HttpHeaderContentLength)
|
||||||
|
if contentLength != "" {
|
||||||
|
if size, err := strconv.ParseInt(contentLength, 10, 64); err == nil {
|
||||||
|
res.Size = size
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -467,7 +481,12 @@ func (f *Fetcher) Resolve(req *base.Request, opts *base.Options) error {
|
|||||||
// IMPORTANT: Keep the response body open for downloading in Start phase
|
// IMPORTANT: Keep the response body open for downloading in Start phase
|
||||||
// This is crucial for one-time URLs that can only be accessed once
|
// This is crucial for one-time URLs that can only be accessed once
|
||||||
f.resolveRespLock.Lock()
|
f.resolveRespLock.Lock()
|
||||||
f.resolveResp = resp
|
if reuseResolveBody {
|
||||||
|
f.resolveResp = resp
|
||||||
|
} else {
|
||||||
|
resp.Body.Close()
|
||||||
|
f.prefetchDone.Store(true)
|
||||||
|
}
|
||||||
f.resolveRespLock.Unlock()
|
f.resolveRespLock.Unlock()
|
||||||
|
|
||||||
f.setState(stateResolved)
|
f.setState(stateResolved)
|
||||||
@@ -479,7 +498,7 @@ func (f *Fetcher) Resolve(req *base.Request, opts *base.Options) error {
|
|||||||
|
|
||||||
// Start async prefetch in background (only for range-supported resources)
|
// Start async prefetch in background (only for range-supported resources)
|
||||||
// For non-range resources, the response will be used directly in Start
|
// For non-range resources, the response will be used directly in Start
|
||||||
if res.Range {
|
if res.Range && reuseResolveBody {
|
||||||
f.prefetchStopCh = make(chan struct{})
|
f.prefetchStopCh = make(chan struct{})
|
||||||
f.resolveFallback.Store(true)
|
f.resolveFallback.Store(true)
|
||||||
go f.asyncPrefetch()
|
go f.asyncPrefetch()
|
||||||
@@ -1544,27 +1563,41 @@ func (f *Fetcher) completeFromResolveFallback(conn *connection) bool {
|
|||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
|
|
||||||
func validateRangeResponse(resp *http.Response, requestedStart int64, requestedEnd *int64, totalSize int64) (int64, int64, error) {
|
// parseRangeResponse validates a single byte range with a known complete length.
|
||||||
|
// Unknown totals are rejected: a segment length cannot stand in for resource size.
|
||||||
|
func parseRangeResponse(resp *http.Response) (int64, int64, int64, error) {
|
||||||
contentRange := resp.Header.Get(base.HttpHeaderContentRange)
|
contentRange := resp.Header.Get(base.HttpHeaderContentRange)
|
||||||
fields := strings.Fields(contentRange)
|
fields := strings.Fields(contentRange)
|
||||||
if len(fields) != 2 || !strings.EqualFold(fields[0], base.HttpHeaderBytes) {
|
if len(fields) != 2 || !strings.EqualFold(fields[0], base.HttpHeaderBytes) {
|
||||||
return 0, 0, fmt.Errorf("%w: malformed Content-Range %q", errInvalidRangeResponse, contentRange)
|
return 0, 0, 0, fmt.Errorf("%w: malformed Content-Range %q", errInvalidRangeResponse, contentRange)
|
||||||
}
|
}
|
||||||
|
|
||||||
rangeAndTotal := strings.Split(fields[1], "/")
|
rangeAndTotal := strings.Split(fields[1], "/")
|
||||||
if len(rangeAndTotal) != 2 {
|
if len(rangeAndTotal) != 2 {
|
||||||
return 0, 0, fmt.Errorf("%w: malformed Content-Range %q", errInvalidRangeResponse, contentRange)
|
return 0, 0, 0, fmt.Errorf("%w: malformed Content-Range %q", errInvalidRangeResponse, contentRange)
|
||||||
}
|
}
|
||||||
bounds := strings.Split(rangeAndTotal[0], "-")
|
bounds := strings.Split(rangeAndTotal[0], "-")
|
||||||
if len(bounds) != 2 {
|
if len(bounds) != 2 {
|
||||||
return 0, 0, fmt.Errorf("%w: malformed Content-Range %q", errInvalidRangeResponse, contentRange)
|
return 0, 0, 0, fmt.Errorf("%w: malformed Content-Range %q", errInvalidRangeResponse, contentRange)
|
||||||
}
|
}
|
||||||
|
|
||||||
start, startErr := strconv.ParseInt(bounds[0], 10, 64)
|
start, startErr := strconv.ParseInt(bounds[0], 10, 64)
|
||||||
end, endErr := strconv.ParseInt(bounds[1], 10, 64)
|
end, endErr := strconv.ParseInt(bounds[1], 10, 64)
|
||||||
total, totalErr := strconv.ParseInt(rangeAndTotal[1], 10, 64)
|
total, totalErr := strconv.ParseInt(rangeAndTotal[1], 10, 64)
|
||||||
if startErr != nil || endErr != nil || totalErr != nil || start < 0 || end < start || total <= end {
|
if startErr != nil || endErr != nil || totalErr != nil || start < 0 || end < start || total <= end {
|
||||||
return 0, 0, fmt.Errorf("%w: malformed Content-Range %q", errInvalidRangeResponse, contentRange)
|
return 0, 0, 0, fmt.Errorf("%w: malformed Content-Range %q", errInvalidRangeResponse, contentRange)
|
||||||
|
}
|
||||||
|
expectedLength := end - start + 1
|
||||||
|
if resp.ContentLength >= 0 && resp.ContentLength != expectedLength {
|
||||||
|
return 0, 0, 0, fmt.Errorf("%w: Content-Length %d does not match range length %d", errInvalidRangeResponse, resp.ContentLength, expectedLength)
|
||||||
|
}
|
||||||
|
return start, end, total, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func validateRangeResponse(resp *http.Response, requestedStart int64, requestedEnd *int64, totalSize int64) (int64, int64, error) {
|
||||||
|
start, end, total, err := parseRangeResponse(resp)
|
||||||
|
if err != nil {
|
||||||
|
return 0, 0, err
|
||||||
}
|
}
|
||||||
if start != requestedStart || (requestedEnd != nil && end != *requestedEnd) || (requestedEnd == nil && end != total-1) {
|
if start != requestedStart || (requestedEnd != nil && end != *requestedEnd) || (requestedEnd == nil && end != total-1) {
|
||||||
return 0, 0, fmt.Errorf("%w: Content-Range %d-%d does not match requested range", errInvalidRangeResponse, start, end)
|
return 0, 0, fmt.Errorf("%w: Content-Range %d-%d does not match requested range", errInvalidRangeResponse, start, end)
|
||||||
@@ -1573,11 +1606,7 @@ func validateRangeResponse(resp *http.Response, requestedStart int64, requestedE
|
|||||||
return 0, 0, fmt.Errorf("%w: Content-Range total %d does not match resource size %d", errInvalidRangeResponse, total, totalSize)
|
return 0, 0, fmt.Errorf("%w: Content-Range total %d does not match resource size %d", errInvalidRangeResponse, total, totalSize)
|
||||||
}
|
}
|
||||||
|
|
||||||
expectedLength := end - start + 1
|
return end - start + 1, total, nil
|
||||||
if resp.ContentLength >= 0 && resp.ContentLength != expectedLength {
|
|
||||||
return 0, 0, fmt.Errorf("%w: Content-Length %d does not match range length %d", errInvalidRangeResponse, resp.ContentLength, expectedLength)
|
|
||||||
}
|
|
||||||
return expectedLength, total, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (f *Fetcher) resolveOpenEndedRange(conn *connection, totalSize int64) error {
|
func (f *Fetcher) resolveOpenEndedRange(conn *connection, totalSize int64) error {
|
||||||
|
|||||||
@@ -0,0 +1,119 @@
|
|||||||
|
package http
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"os"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/GopeedLab/gopeed/pkg/base"
|
||||||
|
fhttp "github.com/GopeedLab/gopeed/pkg/protocol/http"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestResolveResponseSize(t *testing.T) {
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
status int
|
||||||
|
length, contentRange string
|
||||||
|
size int64
|
||||||
|
invalid bool
|
||||||
|
}{
|
||||||
|
{"full", 200, "801571015", "", 801571015, false},
|
||||||
|
{"partial_prefix", 206, "472321339", "bytes 0-472321338/801571015", 801571015, false},
|
||||||
|
{"partial_nonzero", 206, "100000", "bytes 100000-199999/801571015", 801571015, false},
|
||||||
|
{"second_report", 206, "335134409", "bytes 0-335134408/552140234", 552140234, false},
|
||||||
|
{"length_mismatch", 206, "10", "bytes 0-19/100", 0, true},
|
||||||
|
{"unknown_total", 206, "472321339", "bytes 0-472321338/*", 0, true},
|
||||||
|
{"missing_range", 206, "10", "", 0, true},
|
||||||
|
{"malformed_range", 206, "10", "bytes invalid", 0, true},
|
||||||
|
{"invalid_bounds", 206, "10", "bytes 20-10/100", 0, true},
|
||||||
|
{"invalid_total", 206, "10", "bytes 0-9/9", 0, true},
|
||||||
|
{"overflow", 206, "10", "bytes 0-9/9223372036854775808", 0, true},
|
||||||
|
{"chunked_partial", 206, "", "bytes 0-9/100", 100, false},
|
||||||
|
{"unknown_full", 200, "", "", 0, false},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if tc.length != "" {
|
||||||
|
w.Header().Set("Content-Length", tc.length)
|
||||||
|
}
|
||||||
|
if tc.contentRange != "" {
|
||||||
|
w.Header().Set("Content-Range", tc.contentRange)
|
||||||
|
}
|
||||||
|
w.WriteHeader(tc.status)
|
||||||
|
w.(http.Flusher).Flush()
|
||||||
|
}))
|
||||||
|
defer server.Close()
|
||||||
|
f := buildFetcher()
|
||||||
|
defer f.Close()
|
||||||
|
err := f.Resolve(&base.Request{URL: server.URL + "/file"}, &base.Options{Path: t.TempDir()})
|
||||||
|
if tc.invalid {
|
||||||
|
if !errors.Is(err, errInvalidRangeResponse) {
|
||||||
|
t.Fatalf("Resolve error = %v, want invalid range", err)
|
||||||
|
}
|
||||||
|
if f.meta.Res != nil || f.resolveResp != nil || f.getState() != stateError {
|
||||||
|
t.Fatal("invalid response was retained or published")
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if f.meta.Res.Size != tc.size || f.meta.Res.Files[0].Size != tc.size {
|
||||||
|
t.Fatalf("resource/file size = %d/%d, want %d", f.meta.Res.Size, f.meta.Res.Files[0].Size, tc.size)
|
||||||
|
}
|
||||||
|
if tc.status == 206 && !f.meta.Res.Range {
|
||||||
|
t.Fatal("partial response must enable Range downloads")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDownloadPartialResolve(t *testing.T) {
|
||||||
|
payload := make([]byte, 256*1024)
|
||||||
|
for i := range payload {
|
||||||
|
payload[i] = byte(i*31 + i/251)
|
||||||
|
}
|
||||||
|
for _, start := range []int{0, 100000} {
|
||||||
|
for _, connections := range []int{1, 4} {
|
||||||
|
t.Run(fmt.Sprintf("start_%d_connections_%d", start, connections), func(t *testing.T) {
|
||||||
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
begin, end := start, start+9999
|
||||||
|
if h := r.Header.Get("Range"); h != "" {
|
||||||
|
if _, err := fmt.Sscanf(h, "bytes=%d-%d", &begin, &end); err != nil {
|
||||||
|
t.Errorf("invalid request range %q: %v", h, err)
|
||||||
|
w.WriteHeader(400)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
w.Header().Set("Content-Length", fmt.Sprint(end-begin+1))
|
||||||
|
w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", begin, end, len(payload)))
|
||||||
|
w.WriteHeader(206)
|
||||||
|
w.Write(payload[begin : end+1])
|
||||||
|
}))
|
||||||
|
defer server.Close()
|
||||||
|
f := buildFetcher()
|
||||||
|
defer f.Close()
|
||||||
|
if err := f.Resolve(&base.Request{URL: server.URL + "/file"}, &base.Options{Path: t.TempDir(), Name: "file", Extra: &fhttp.OptsExtra{Connections: connections}}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := f.Start(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := f.Wait(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
got, err := os.ReadFile(f.meta.SingleFilepath())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !bytes.Equal(got, payload) {
|
||||||
|
t.Fatalf("download content mismatch: got %d bytes, want %d", len(got), len(payload))
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in new issue
Block a user