nprasad2077/gopeed
1
0 Fork 0
mirror of https://github.com/GopeedLab/gopeed.git synced 2026-11-09 10:00:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(http): support advisory extension download sizes (#1599)

Browse Source
This commit is contained in:
Levi authored and GitHub committed 2026-10-22 23:35:01 +08:00
1 parent 4214765d39
commit d125d4ab75
17 files changed
+1047 -182
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

internal/protocol/http/fetcher.go
+91 -115
View File
@@ -316,13 +316,17 @@ type Fetcher struct {
resolveFallback atomic.Bool // Resolve prefetch is still available until the first Range request takes over
// Target file
file *os.File
fileMu sync.Mutex
redirectURL string
redirectLock sync.Mutex
ifRange string
rangeReprobeEligible bool
rangeValidatorPinned bool
file *os.File
fileMu sync.Mutex
redirectURL string
redirectLock sync.Mutex
ifRange string
rangeReprobeEligible bool
rangeValidatorPinned bool
// sizeKnown distinguishes a display estimate from an HTTP-confirmed length,
// including zero. Nil preserves the interpretation of legacy snapshots.
sizeKnown *bool
// An interrupted chunked sequential response must restart from byte zero.
sequentialSizeUnknown bool
// Lifecycle control
@@ -369,6 +373,40 @@ func (f *Fetcher) setState(s fetcherState) {
f.state.Store(int32(s))
}
// Size state is read under connMu or before download workers are started.
func (f *Fetcher) hasKnownSize() bool {
if f.sizeKnown != nil {
return *f.sizeKnown
}
return f.meta != nil && f.meta.Res != nil && f.meta.Res.Size > 0 && !f.sequentialSizeUnknown
}
func (f *Fetcher) knownSize() int64 {
if f.hasKnownSize() {
return f.meta.Res.Size
}
return 0
}
func (f *Fetcher) sizeHint() int64 {
if f.meta.Opts != nil && f.meta.Opts.Size > 0 {
return f.meta.Opts.Size
}
return 0
}
// setResourceSizeLocked publishes the value and its authority in one snapshot.
func (f *Fetcher) setResourceSizeLocked(size int64, known bool) {
if !known {
size = f.sizeHint()
}
f.sizeKnown = &known
f.meta.Res.Size = size
if len(f.meta.Res.Files) > 0 && f.meta.Res.Files[0] != nil {
f.meta.Res.Files[0].Size = size
}
}
// updateMaxConnTime updates maxConnTime if the new duration is larger
func (f *Fetcher) updateMaxConnTime(d time.Duration) {
newVal := int64(d)
@@ -447,6 +485,13 @@ func (f *Fetcher) Resolve(req *base.Request) error {
}
}
}
// A hint is only a display value, even when the origin advertises Range.
known := resp.StatusCode == base.HttpCodePartialContent || resp.ContentLength >= 0
f.sizeKnown = &known
f.sequentialSizeUnknown = !res.Range && !known
if !known {
res.Size = f.sizeHint()
}
// Parse last modified time
var lastModifiedTime *time.Time
@@ -803,7 +848,7 @@ func (f *Fetcher) doStart() error {
_, err = os.Stat(name)
if err != nil {
if os.IsNotExist(err) {
file, err = f.ctl.Touch(name, f.meta.Res.Size)
file, err = f.ctl.Touch(name, f.knownSize())
} else {
return err
}
@@ -821,7 +866,7 @@ func (f *Fetcher) doStart() error {
// For non-range resources, the response will be used directly
var prefetchedBytes int64
if f.meta.Res.Range {
resolveComplete := f.meta.Res.Size > 0 && f.prefetchDone.Load() && f.prefetchErr == nil && f.prefetchSize.Load() >= f.meta.Res.Size
resolveComplete := f.knownSize() > 0 && f.prefetchDone.Load() && f.prefetchErr == nil && f.prefetchSize.Load() >= f.knownSize()
if f.resolveFallback.Load() && !resolveComplete {
// Keep Resolve downloading until the first Range response has been
// validated. The Range request starts at a stable snapshot; a valid
@@ -993,7 +1038,7 @@ func (f *Fetcher) expandConnections() {
return
}
totalSize := f.meta.Res.Size
totalSize := f.knownSize()
f.connMu.Lock()
@@ -1284,7 +1329,7 @@ func (f *Fetcher) downloadChunkOnce(conn *connection, client *http.Client, buf [
rangeMode := f.meta.Res.Range
ifRange := ""
if resumeProbe {
end := f.meta.Res.Size - 1
end := f.knownSize() - 1
rangeEnd = &end
ifRange = f.ifRange
} else if rangeMode && (f.rangeValidatorPinned || f.resolveFallback.Load()) {
@@ -1364,7 +1409,7 @@ func (f *Fetcher) downloadChunkOnce(conn *connection, client *http.Client, buf [
}
if rangeRequested && resp.StatusCode == base.HttpCodePartialContent {
var responseTotal int64
expectedResponseLength, responseTotal, err = validateRangeResponse(resp, rangeStart, rangeEnd, f.meta.Res.Size)
expectedResponseLength, responseTotal, err = validateRangeResponse(resp, rangeStart, rangeEnd, f.knownSize())
if err != nil {
resp.Body.Close()
return err
@@ -1389,7 +1434,7 @@ func (f *Fetcher) downloadChunkOnce(conn *connection, client *http.Client, buf [
f.rangeValidatorPinned = true
f.sequentialSizeUnknown = false
conn.Chunk.Begin = 0
conn.Chunk.setEnd(f.meta.Res.Size - 1)
conn.Chunk.setEnd(f.knownSize() - 1)
f.connMu.Unlock()
}
}
@@ -1414,7 +1459,7 @@ func (f *Fetcher) downloadChunkOnce(conn *connection, client *http.Client, buf [
resp.Body.Close()
return err
}
} else if f.meta.Res.Size > 0 {
} else if f.hasKnownSize() && f.meta.Res.Size > 0 {
expectedResponseLength = f.meta.Res.Size
if resp.ContentLength >= 0 && resp.ContentLength != expectedResponseLength {
resp.Body.Close()
@@ -1550,7 +1595,7 @@ func (f *Fetcher) commitResolveTakeover(conn *connection, requestedStart int64)
}
prefetched := f.stopPrefetchAndCopyData()
if prefetched < requestedStart || prefetched > f.meta.Res.Size {
if prefetched < requestedStart || prefetched > f.knownSize() {
return 0, fmt.Errorf("resolve prefetch advanced to invalid offset %d for range start %d", prefetched, requestedStart)
}
f.resolveDataPos.Store(prefetched)
@@ -1581,7 +1626,7 @@ func (f *Fetcher) completeFromResolveFallback(conn *connection) bool {
}
time.Sleep(10 * time.Millisecond)
}
knownSize := f.meta.Res.Size > 0
knownSize := f.hasKnownSize()
if f.prefetchErr != nil || (knownSize && f.prefetchSize.Load() != f.meta.Res.Size) {
f.cleanupPrefetchFile()
return false
@@ -1595,10 +1640,7 @@ func (f *Fetcher) completeFromResolveFallback(conn *connection) bool {
f.connMu.Lock()
if !knownSize {
f.meta.Res.Size = prefetched
if len(f.meta.Res.Files) > 0 && f.meta.Res.Files[0] != nil {
f.meta.Res.Files[0].Size = prefetched
}
f.setResourceSizeLocked(prefetched, true)
}
f.meta.Res.Range = false
f.rangeReprobeEligible = false
@@ -1680,10 +1722,7 @@ func (f *Fetcher) resolveOpenEndedRange(conn *connection, totalSize int64) error
f.connMu.Unlock()
return err
}
f.meta.Res.Size = totalSize
if len(f.meta.Res.Files) > 0 && f.meta.Res.Files[0] != nil {
f.meta.Res.Files[0].Size = totalSize
}
f.setResourceSizeLocked(totalSize, true)
conn.Chunk.setEnd(totalSize - 1)
f.connMu.Unlock()
return nil
@@ -1755,10 +1794,7 @@ func (f *Fetcher) resizeSequentialResource(size int64) error {
return err
}
f.meta.Res.Size = size
if len(f.meta.Res.Files) > 0 {
f.meta.Res.Files[0].Size = size
}
f.setResourceSizeLocked(size, true)
f.sequentialSizeUnknown = false
f.connMu.Unlock()
return nil
@@ -1783,10 +1819,7 @@ func (f *Fetcher) beginUnknownSizeSequentialResource() error {
return err
}
f.meta.Res.Size = 0
if len(f.meta.Res.Files) > 0 {
f.meta.Res.Files[0].Size = 0
}
f.setResourceSizeLocked(0, false)
f.sequentialSizeUnknown = true
f.ifRange = ""
f.rangeValidatorPinned = false
@@ -1870,11 +1903,9 @@ func (f *Fetcher) runConnectionWithResolveResp(conn *connection) {
if err != nil {
if err == io.EOF {
f.connMu.Lock()
if f.meta.Res.Size == 0 {
f.meta.Res.Size = conn.Downloaded
if len(f.meta.Res.Files) > 0 && f.meta.Res.Files[0] != nil {
f.meta.Res.Files[0].Size = conn.Downloaded
}
if !f.hasKnownSize() {
f.setResourceSizeLocked(conn.Downloaded, true)
f.sequentialSizeUnknown = false
}
conn.Completed = true
conn.State = connCompleted
@@ -1913,78 +1944,15 @@ func (f *Fetcher) runConnectionFallback(conn *connection) {
}
f.connMu.Lock()
if retries > 0 {
f.resetConnectionForRestart(conn)
}
conn.State = connConnecting
f.connMu.Unlock()
err := func() error {
httpReq, err := f.buildRequest(conn.ctx, f.meta.Req)
if err != nil {
return err
}
// Record connection start time for adaptive timeout tracking
connStartTime := time.Now()
resp, err := client.Do(httpReq)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode != base.HttpCodeOK && resp.StatusCode != base.HttpCodePartialContent {
return NewRequestError(resp.StatusCode)
}
// Record successful connection time for adaptive timeout
f.updateMaxConnTime(time.Since(connStartTime))
f.connMu.Lock()
conn.State = connDownloading
conn.failed = false
f.connMu.Unlock()
f.primaryReadyOnce.Do(func() {
close(f.primaryReadyCh)
})
if f.slowStart != nil {
f.slowStart.onConnectSuccess()
}
reader := f.responseReader(resp)
for {
if conn.ctx.Err() != nil {
return conn.ctx.Err()
}
n, err := reader.Read(buf)
if n > 0 {
f.connMu.Lock()
writeOffset := conn.Chunk.Downloaded
f.connMu.Unlock()
f.fileMu.Lock()
if f.file != nil {
_, writeErr := f.file.WriteAt(buf[:n], writeOffset)
if writeErr != nil {
f.fileMu.Unlock()
return writeErr
}
}
f.fileMu.Unlock()
f.connMu.Lock()
conn.Chunk.Downloaded += int64(n)
conn.Downloaded += int64(n)
f.connMu.Unlock()
}
if err != nil {
if err == io.EOF {
return nil
}
return err
}
}
}()
// Restored tasks without a reusable Resolve response need the same
// length handling and restart semantics as other sequential requests.
err := f.downloadChunkOnce(conn, client, buf)
if err == nil {
f.connMu.Lock()
@@ -2133,14 +2101,14 @@ func (f *Fetcher) resetConnectionForRestart(conn *connection) {
// hasSequentialPrefixLocked reports whether a single sequential connection has
// a useful contiguous prefix. The caller must hold connMu.
func (f *Fetcher) hasSequentialPrefixLocked(conn *connection) bool {
if f.meta == nil || f.meta.Res == nil || f.meta.Res.Range || !f.rangeReprobeEligible || f.meta.Res.Size <= 0 {
if f.meta == nil || f.meta.Res == nil || f.meta.Res.Range || !f.rangeReprobeEligible || f.knownSize() <= 0 {
return false
}
if conn == nil || conn.ID != 0 || len(f.connections) != 1 || conn.Chunk == nil {
return false
}
downloaded := conn.Chunk.Downloaded
return downloaded > 0 && downloaded < f.meta.Res.Size
return downloaded > 0 && downloaded < f.knownSize()
}
// canProbeSequentialResumeLocked reports whether the prefix can be resumed
@@ -2226,7 +2194,7 @@ func (f *Fetcher) onDownloadComplete() {
needsMoreData := false
if f.meta.Res.Range {
needsMoreData = conn.Chunk != nil && (conn.Chunk.openEnded() || conn.Chunk.remain() > 0)
} else if f.meta.Res.Size > 0 {
} else if f.hasKnownSize() && f.meta.Res.Size > 0 {
needsMoreData = conn.Downloaded < f.meta.Res.Size
} else {
needsMoreData = !conn.Completed && conn.State != connCompleted
@@ -2236,7 +2204,7 @@ func (f *Fetcher) onDownloadComplete() {
// This connection has remaining work and isn't done
// Check if it failed with 403 (server limit) - these can be ignored if other connections completed the work
if conn.State == connFailed && conn.failed {
if re := extractRequestError(conn.lastErr); re != nil && re.Code == 403 {
if re := extractRequestError(conn.lastErr); re != nil && re.Code == 403 && f.hasKnownSize() {
// 403 is server connection limit, check if other connections will complete this chunk
continue
}
@@ -2247,7 +2215,7 @@ func (f *Fetcher) onDownloadComplete() {
}
// If total downloaded matches file size, consider it a success regardless of connection failures
downloadComplete := f.meta.Res.Size > 0 && totalDownloaded >= f.meta.Res.Size
downloadComplete := f.hasKnownSize() && f.meta.Res.Size > 0 && totalDownloaded >= f.meta.Res.Size
// Integrity errors must win over byte-count completion. A malformed response
// may be detected only after the expected bytes have already been written.
@@ -2264,7 +2232,7 @@ func (f *Fetcher) onDownloadComplete() {
for _, conn := range f.connections {
if conn.State == connFailed && conn.failed {
// Skip 403 errors (server connection limit) - these are expected when exceeding server's limit
if re := extractRequestError(conn.lastErr); re != nil && re.Code == 403 {
if re := extractRequestError(conn.lastErr); re != nil && re.Code == 403 && f.hasKnownSize() {
continue
}
if re := extractRequestError(conn.lastErr); re != nil {
@@ -2364,7 +2332,7 @@ func (f *Fetcher) checkCompletion() bool {
totalDownloaded += conn.Downloaded
}
if f.meta.Res.Size > 0 && totalDownloaded >= f.meta.Res.Size {
if f.hasKnownSize() && f.meta.Res.Size > 0 && totalDownloaded >= f.meta.Res.Size {
// Don't start a new goroutine - let the caller handle completion
return true
}
@@ -2391,6 +2359,14 @@ func (f *Fetcher) checkCompletion() bool {
// Patch modifies the HTTP request information.
func (f *Fetcher) Patch(req *base.Request, opts *base.Options) error {
if opts != nil && opts.Size > 0 {
f.connMu.Lock()
f.meta.Opts.Size = opts.Size
if f.meta.Res != nil && !f.hasKnownSize() {
f.setResourceSizeLocked(0, false)
}
f.connMu.Unlock()
}
// Patch request info
if req != nil {
if req.URL != "" {
@@ -2488,8 +2464,8 @@ func (f *Fetcher) Pause() error {
conn := f.connections[0]
if conn != nil && conn.ID == 0 && conn.Chunk != nil && !conn.Completed {
conn.Chunk.Begin = prefetched
if f.meta.Res.Size > 0 {
conn.Chunk.setEnd(f.meta.Res.Size - 1)
if f.knownSize() > 0 {
conn.Chunk.setEnd(f.knownSize() - 1)
} else {
conn.Chunk.End = nil
}
internal/protocol/http/fetcher_manager.go
+10 -3
View File
@@ -19,10 +19,11 @@ type fetcherData struct {
IfRange string // Strong ETag or Last-Modified validator for safe resume
RangeReprobeEligible bool // Origin advertised Range before falling back to sequential mode
RangeValidatorPinned bool // Recovered Range mode must keep the same If-Range validator
SequentialSizeUnknown bool // Interrupted chunked restart must retry from byte zero
SequentialSizeUnknown bool // Interrupted chunked sequential streams restart from byte zero
SizeKnown *bool // Nil for legacy snapshots; false means Size is only a display estimate
Range *bool // Authoritative Range mode; nil for records saved by older versions
ResourceSize *int64 // Authoritative resource size from the same snapshot as Connections
FileSize *int64 // Authoritative single-file size; nil for older records
ResourceSize *int64 // Display size from the same snapshot; authority is recorded by SizeKnown
FileSize *int64 // Single-file size from the same snapshot; nil for older records
}
// ============================================================================
@@ -93,6 +94,7 @@ func (fm *FetcherManager) Store(f fetcher.Fetcher) (data any, err error) {
rangeReprobeEligible := _f.rangeReprobeEligible
rangeValidatorPinned := _f.rangeValidatorPinned
sequentialSizeUnknown := _f.sequentialSizeUnknown
sizeKnown := _f.hasKnownSize()
var rangeMode *bool
var resourceSize *int64
var fileSize *int64
@@ -115,6 +117,7 @@ func (fm *FetcherManager) Store(f fetcher.Fetcher) (data any, err error) {
RangeReprobeEligible: rangeReprobeEligible,
RangeValidatorPinned: rangeValidatorPinned,
SequentialSizeUnknown: sequentialSizeUnknown,
SizeKnown: &sizeKnown,
Range: rangeMode,
ResourceSize: resourceSize,
FileSize: fileSize,
@@ -168,6 +171,10 @@ func (fm *FetcherManager) Restore() (v any, f func(meta *fetcher.FetcherMeta, v
fetcher.rangeReprobeEligible = fd.RangeReprobeEligible
fetcher.rangeValidatorPinned = fd.RangeValidatorPinned
fetcher.sequentialSizeUnknown = fd.SequentialSizeUnknown
if fd.SizeKnown != nil {
known := *fd.SizeKnown
fetcher.sizeKnown = &known
}
if fd.Range != nil && fetcher.meta.Res != nil {
// Recovery-critical resource fields live in the same persisted snapshot
// as Connections, making them authoritative over task metadata that may
internal/protocol/http/fetcher_test.go
+9 -1
View File
@@ -1287,6 +1287,12 @@ func TestFetcher_ValidRangeTakesOverResolveResponse(t *testing.T) {
}
func TestFetcher_UnknownSizeRangeUsesOpenEndedWorker(t *testing.T) {
for _, hint := range []int64{0, 3, 8 * 1024 * 1024} {
t.Run(fmt.Sprint(hint), func(t *testing.T) { testFetcherUnknownSizeRange(t, hint) })
}
}
func testFetcherUnknownSizeRange(t *testing.T, hint int64) {
const resolveETag = `"unknown-size-v1"`
payload := make([]byte, 4*1024*1024)
for i := range payload {
@@ -1352,6 +1358,7 @@ func TestFetcher_UnknownSizeRangeUsesOpenEndedWorker(t *testing.T) {
f := buildFetcher()
f.Meta().Opts = &base.Options{
Size: hint,
Path: t.TempDir(),
Name: "unknown-size-range.data",
Extra: &http.OptsExtra{
@@ -1363,7 +1370,7 @@ func TestFetcher_UnknownSizeRangeUsesOpenEndedWorker(t *testing.T) {
}
defer f.Close()
if f.meta.Res.Size != 0 || !f.meta.Res.Range {
if f.meta.Res.Size != hint || f.hasKnownSize() || !f.meta.Res.Range {
t.Fatalf("resolved resource = %+v, want unknown size with Range support", f.meta.Res)
}
if err := f.Start(); err != nil {
@@ -1595,6 +1602,7 @@ func TestFetcher_UnknownSizeRangeRemainsOpenEndedAfterPause(t *testing.T) {
f := buildFetcher()
f.Meta().Opts = &base.Options{
Size: 3,
Path: t.TempDir(),
Name: "pause-open-range.data",
Extra: &http.OptsExtra{
internal/protocol/http/size_hint_test.go
+245
View File
@@ -0,0 +1,245 @@
package http
import (
"bytes"
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"sync/atomic"
"testing"
"github.com/GopeedLab/gopeed/pkg/base"
)
func TestFetcher_OptionsSizeHintAndPatch(t *testing.T) {
for _, known := range []bool{false, true} {
for _, payload := range []string{"", "0123456789"} {
t.Run(fmt.Sprintf("known=%v/bytes=%d", known, len(payload)), func(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
if known {
w.Header().Set("Content-Length", fmt.Sprint(len(payload)))
}
w.(http.Flusher).Flush()
fmt.Fprint(w, payload)
}))
defer server.Close()
f := buildFetcher()
defer f.Close()
f.Meta().Opts = &base.Options{Size: 7, Path: t.TempDir(), Name: "output.bin"}
if err := f.Resolve(&base.Request{URL: server.URL}); err != nil {
t.Fatal(err)
}
want := int64(7)
if known {
want = int64(len(payload))
}
if f.meta.Res.Size != want || f.hasKnownSize() != known {
t.Fatalf("resolved size=%d known=%v", f.meta.Res.Size, f.hasKnownSize())
}
if err := f.Patch(nil, &base.Options{Size: 9}); err != nil {
t.Fatal(err)
}
if !known {
want = 9
}
if f.meta.Res.Size != want {
t.Fatal("hint update overrode a real size or failed to update an estimate")
}
if err := f.Start(); err != nil {
t.Fatal(err)
}
if err := f.Wait(); err != nil {
t.Fatal(err)
}
if f.meta.Res.Size != int64(len(payload)) || !f.hasKnownSize() {
t.Fatalf("final size=%d known=%v", f.meta.Res.Size, f.hasKnownSize())
}
snapshot, err := new(FetcherManager).Store(f)
if err != nil {
t.Fatal(err)
}
fd := snapshot.(*fetcherData)
if fd.SizeKnown == nil || !*fd.SizeKnown {
t.Fatal("snapshot lost confirmed size authority")
}
_, restore := new(FetcherManager).Restore()
restored := restore(f.meta, fd).(*Fetcher)
if !restored.hasKnownSize() || restored.knownSize() != int64(len(payload)) {
t.Fatal("restored size lost authority, including known zero")
}
})
}
}
}
func TestFetcher_SizeHintCannotCompleteForbiddenUnknownRange(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Header.Get("Range") != "" {
w.WriteHeader(http.StatusForbidden)
return
}
w.Header().Set("Accept-Ranges", "bytes")
w.(http.Flusher).Flush()
fmt.Fprint(w, "0123456789")
w.(http.Flusher).Flush()
<-r.Context().Done()
}))
defer server.Close()
f := buildFetcher()
defer f.Close()
f.Meta().Opts = &base.Options{Size: 3, Path: t.TempDir(), Name: "output.bin"}
if err := f.Resolve(&base.Request{URL: server.URL}); err != nil {
t.Fatal(err)
}
if err := f.Start(); err != nil {
t.Fatal(err)
}
if err := f.Wait(); err == nil {
t.Fatal("a failed unknown-length Range request was completed from an estimate")
}
}
func TestFetcher_SizeHintPreservesHTTPIntegrity(t *testing.T) {
for _, chunked := range []bool{false, true} {
for _, broken := range []bool{false, true} {
t.Run(fmt.Sprintf("chunked=%v/broken=%v", chunked, broken), func(t *testing.T) {
payload := []byte("0123456789")
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
if chunked && broken {
conn, rw, err := w.(http.Hijacker).Hijack()
if err != nil {
t.Error(err)
return
}
defer conn.Close()
fmt.Fprint(rw, "HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\nConnection: close\r\n\r\na\r\n0123456789\r\n")
rw.Flush() // Omit the final chunk to simulate a connection interruption.
return
}
if !chunked {
length := len(payload)
if broken {
length++
}
w.Header().Set("Content-Length", fmt.Sprint(length))
}
w.(http.Flusher).Flush()
w.Write(payload)
}))
defer server.Close()
f := buildFetcher()
defer f.Close()
f.Meta().Opts = &base.Options{Size: 3, Path: t.TempDir(), Name: "output.bin"}
if err := f.Resolve(&base.Request{URL: server.URL}); err != nil {
t.Fatal(err)
}
if !chunked {
length := int64(len(payload))
if broken {
length++
}
if f.meta.Res.Size != length || f.sequentialSizeUnknown {
t.Fatal("HTTP length did not override size hint")
}
} else if f.meta.Res.Size != 3 || !f.sequentialSizeUnknown {
t.Fatal("unknown HTTP length did not retain the display hint")
}
if err := f.Start(); err != nil {
t.Fatal(err)
}
err := f.Wait()
if broken {
if err == nil {
t.Fatal("truncated HTTP response succeeded after exceeding the size hint")
}
return
}
if err != nil {
t.Fatal(err)
}
if f.meta.Res.Size != int64(len(payload)) || f.sequentialSizeUnknown {
t.Fatalf("completed size=%d unknown=%v", f.meta.Res.Size, f.sequentialSizeUnknown)
}
})
}
}
}
func TestFetcher_SizeHintSurvivesRestore(t *testing.T) {
for _, partial := range []bool{false, true} {
for _, fixedLength := range []bool{false, true} {
t.Run(fmt.Sprintf("partial=%v/fixedLength=%v", partial, fixedLength), func(t *testing.T) {
payload := []byte("0123456789")
var requests atomic.Int64
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
request := requests.Add(1)
if got := r.Header.Get("Range"); got != "" {
t.Errorf("estimated sizes must not be used for resume ranges: %s", got)
}
if fixedLength && request > 1 {
w.Header().Set("Content-Length", fmt.Sprint(len(payload)))
}
w.(http.Flusher).Flush()
if fixedLength && request == 2 {
w.Write(payload[:4]) // Retry must discard this interrupted attempt.
return
}
w.Write(payload)
}))
defer server.Close()
f := buildFetcher()
opts := &base.Options{Size: 3, Path: t.TempDir(), Name: "output.bin"}
f.Meta().Opts = opts
if err := f.Resolve(&base.Request{URL: server.URL}); err != nil {
t.Fatal(err)
}
fm := new(FetcherManager)
snapshot, err := fm.Store(f)
if err != nil {
t.Fatal(err)
}
data, err := json.Marshal(snapshot)
if err != nil {
t.Fatal(err)
}
var saved fetcherData
if err := json.Unmarshal(data, &saved); err != nil {
t.Fatal(err)
}
if !saved.SequentialSizeUnknown || saved.SizeKnown == nil || *saved.SizeKnown || *saved.ResourceSize != 3 {
t.Fatal("snapshot lost the size hint or its advisory state")
}
f.Close()
// An interrupted earlier attempt left a tail longer than the new output.
if err := os.WriteFile(filepath.Join(opts.Path, opts.Name), bytes.Repeat([]byte("old"), 100), 0600); err != nil {
t.Fatal(err)
}
if partial {
chunk := newChunk(0, 0)
chunk.Downloaded = 7 // Already exceeds the display estimate.
saved.Connections = []*connection{{ID: 0, Role: rolePrimary, State: connNotStarted, Chunk: chunk, Downloaded: 7}}
}
_, restore := fm.Restore()
restored := restore(f.meta, &saved).(*Fetcher)
restored.Setup(f.ctl)
defer restored.Close()
if err := restored.Start(); err != nil {
t.Fatal(err)
}
if err := restored.Wait(); err != nil {
t.Fatal(err)
}
body, err := os.ReadFile(restored.meta.SingleFilepath())
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(body, payload) || restored.meta.Res.Size != int64(len(payload)) || restored.Progress().TotalDownloaded() != int64(len(payload)) {
t.Fatalf("restored stream retained an estimate or stale bytes: body=%q size=%d progress=%d", body, restored.meta.Res.Size, restored.Progress().TotalDownloaded())
}
})
}
}
}