mirror of
https://github.com/GopeedLab/gopeed.git
synced 2026-11-19 06:43:42 +00:00
fix(http): re-probe range support after sequential retry (#1464)
This commit is contained in:
5 files changed
+1245
-41
No files matched your search
@@ -43,6 +43,10 @@ func isRangeIntegrityError(err error) bool {
|
|||||||
return errors.Is(err, errRangeRequestIgnored) || errors.Is(err, errInvalidRangeResponse)
|
return errors.Is(err, errRangeRequestIgnored) || errors.Is(err, errInvalidRangeResponse)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func isTerminalRangeError(err error) bool {
|
||||||
|
return isRangeIntegrityError(err)
|
||||||
|
}
|
||||||
|
|
||||||
// ============================================================================
|
// ============================================================================
|
||||||
// State Machine
|
// State Machine
|
||||||
// ============================================================================
|
// ============================================================================
|
||||||
@@ -108,6 +112,7 @@ type connection struct {
|
|||||||
Completed bool
|
Completed bool
|
||||||
|
|
||||||
failed bool
|
failed bool
|
||||||
|
exited bool
|
||||||
retryTimes int
|
retryTimes int
|
||||||
lastErr error
|
lastErr error
|
||||||
|
|
||||||
@@ -214,6 +219,13 @@ func (s *slowStartController) commitBatch(count int) {
|
|||||||
s.batchReady = 0
|
s.batchReady = 0
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (s *slowStartController) signalExpansion() {
|
||||||
|
select {
|
||||||
|
case s.expansionCh <- struct{}{}:
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// ============================================================================
|
// ============================================================================
|
||||||
// Fetcher
|
// Fetcher
|
||||||
// ============================================================================
|
// ============================================================================
|
||||||
@@ -264,10 +276,14 @@ type Fetcher struct {
|
|||||||
prefetchStopCh chan struct{} // Signal to stop prefetch
|
prefetchStopCh chan struct{} // Signal to stop prefetch
|
||||||
|
|
||||||
// Target file
|
// Target file
|
||||||
file *os.File
|
file *os.File
|
||||||
fileMu sync.Mutex
|
fileMu sync.Mutex
|
||||||
redirectURL string
|
redirectURL string
|
||||||
redirectLock sync.Mutex
|
redirectLock sync.Mutex
|
||||||
|
ifRange string
|
||||||
|
rangeReprobeEligible bool
|
||||||
|
rangeValidatorPinned bool
|
||||||
|
sequentialSizeUnknown bool
|
||||||
|
|
||||||
// Lifecycle control
|
// Lifecycle control
|
||||||
ctx context.Context
|
ctx context.Context
|
||||||
@@ -401,9 +417,11 @@ func (f *Fetcher) Resolve(req *base.Request, opts *base.Options) error {
|
|||||||
var lastModifiedTime *time.Time
|
var lastModifiedTime *time.Time
|
||||||
lastModified := resp.Header.Get(base.HttpHeaderLastModified)
|
lastModified := resp.Header.Get(base.HttpHeaderLastModified)
|
||||||
if lastModified != "" {
|
if lastModified != "" {
|
||||||
t, _ := time.Parse(time.RFC1123, lastModified)
|
if t, err := http.ParseTime(lastModified); err == nil {
|
||||||
lastModifiedTime = &t
|
lastModifiedTime = &t
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
f.ifRange = extractIfRangeValidator(resp.Header)
|
||||||
|
|
||||||
file := &base.FileInfo{
|
file := &base.FileInfo{
|
||||||
Size: res.Size,
|
Size: res.Size,
|
||||||
@@ -641,7 +659,9 @@ func (f *Fetcher) doStart() error {
|
|||||||
for _, conn := range f.connections {
|
for _, conn := range f.connections {
|
||||||
// Reset connections that can be retried
|
// Reset connections that can be retried
|
||||||
if !conn.Completed && conn.State != connCompleted {
|
if !conn.Completed && conn.State != connCompleted {
|
||||||
f.resetConnectionForRestart(conn)
|
if !f.hasSequentialPrefixLocked(conn) {
|
||||||
|
f.resetConnectionForRestart(conn)
|
||||||
|
}
|
||||||
conn.State = connNotStarted
|
conn.State = connNotStarted
|
||||||
conn.failed = false
|
conn.failed = false
|
||||||
conn.retryTimes = 0
|
conn.retryTimes = 0
|
||||||
@@ -743,10 +763,18 @@ func (f *Fetcher) downloadLoop() {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
// Resume: restart existing connections
|
// A retained sequential prefix starts as the first slow-start batch. If
|
||||||
|
// its guarded request restores Range support, the normal expansion loop
|
||||||
|
// can split the remaining suffix across the configured connections.
|
||||||
|
canRecoverRange := f.hasSequentialRecoveryCandidate()
|
||||||
|
if canRecoverRange {
|
||||||
|
f.slowStart.commitBatch(1)
|
||||||
|
}
|
||||||
f.resumeConnections()
|
f.resumeConnections()
|
||||||
f.waitForCompletion(ctx)
|
if !canRecoverRange {
|
||||||
return
|
f.waitForCompletion(ctx)
|
||||||
|
return
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Slow start loop
|
// Slow start loop
|
||||||
@@ -756,6 +784,19 @@ func (f *Fetcher) downloadLoop() {
|
|||||||
// Paused or cancelled
|
// Paused or cancelled
|
||||||
return
|
return
|
||||||
case <-f.slowStart.expansionCh:
|
case <-f.slowStart.expansionCh:
|
||||||
|
f.connMu.Lock()
|
||||||
|
rangeMode := f.meta.Res.Range
|
||||||
|
sequentialExited := len(f.connections) == 1 && f.connections[0].exited
|
||||||
|
f.connMu.Unlock()
|
||||||
|
if !rangeMode {
|
||||||
|
// The primary response is still being consumed sequentially. A
|
||||||
|
// successful re-probe or final connection exit will signal again.
|
||||||
|
if sequentialExited {
|
||||||
|
f.waitForCompletion(ctx)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
// Batch completed, try to expand
|
// Batch completed, try to expand
|
||||||
if f.checkCompletion() {
|
if f.checkCompletion() {
|
||||||
// All work is done, wait for connections to finish
|
// All work is done, wait for connections to finish
|
||||||
@@ -924,9 +965,22 @@ func (f *Fetcher) expandConnections() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (f *Fetcher) runConnection(conn *connection) {
|
func (f *Fetcher) runConnection(conn *connection) {
|
||||||
defer f.wg.Done()
|
defer func() {
|
||||||
|
// A range-advertising origin can fall back to one sequential response.
|
||||||
|
// Wake the download loop after that connection exits because its earlier
|
||||||
|
// response-ready signal was intentionally not used to expand.
|
||||||
|
f.connMu.Lock()
|
||||||
|
conn.exited = true
|
||||||
|
sequentialRecovery := !f.meta.Res.Range && f.rangeReprobeEligible && len(f.connections) == 1
|
||||||
|
f.connMu.Unlock()
|
||||||
|
f.wg.Done()
|
||||||
|
if sequentialRecovery && f.slowStart != nil {
|
||||||
|
f.slowStart.signalExpansion()
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
f.connMu.Lock()
|
f.connMu.Lock()
|
||||||
|
conn.exited = false
|
||||||
conn.State = connConnecting
|
conn.State = connConnecting
|
||||||
f.connMu.Unlock()
|
f.connMu.Unlock()
|
||||||
|
|
||||||
@@ -942,11 +996,13 @@ func (f *Fetcher) runConnection(conn *connection) {
|
|||||||
if retries > 0 {
|
if retries > 0 {
|
||||||
client = f.buildFastFailClient()
|
client = f.buildFastFailClient()
|
||||||
|
|
||||||
// A sequential response always starts at byte zero. If a read failed
|
// Preserve an eligible sequential prefix long enough to probe whether
|
||||||
// after falling back from Range mode, overwrite from the beginning on
|
// the origin has recovered byte-range support. downloadChunkOnce resets
|
||||||
// the next request instead of appending duplicate prefix bytes.
|
// it only when no safe validator exists or If-Range returns a full 200.
|
||||||
f.connMu.Lock()
|
f.connMu.Lock()
|
||||||
f.resetConnectionForRestart(conn)
|
if !f.hasSequentialPrefixLocked(conn) {
|
||||||
|
f.resetConnectionForRestart(conn)
|
||||||
|
}
|
||||||
f.connMu.Unlock()
|
f.connMu.Unlock()
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -969,7 +1025,7 @@ func (f *Fetcher) runConnection(conn *connection) {
|
|||||||
if errors.Is(err, context.Canceled) {
|
if errors.Is(err, context.Canceled) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
if isRangeIntegrityError(err) {
|
if isTerminalRangeError(err) {
|
||||||
f.connMu.Lock()
|
f.connMu.Lock()
|
||||||
conn.lastErr = err
|
conn.lastErr = err
|
||||||
conn.State = connFailed
|
conn.State = connFailed
|
||||||
@@ -1033,15 +1089,34 @@ func (f *Fetcher) downloadChunkOnce(conn *connection, client *http.Client, buf [
|
|||||||
return conn.ctx.Err()
|
return conn.ctx.Err()
|
||||||
}
|
}
|
||||||
|
|
||||||
// Read chunk boundaries under lock to get a consistent snapshot
|
// Read chunk boundaries under lock to get a consistent snapshot. A prefix
|
||||||
// This protects against concurrent modification by helpOtherConnection
|
// without a safe validator cannot be resumed, so restart it before building
|
||||||
|
// the request; the returned full response will overwrite from byte zero.
|
||||||
f.connMu.Lock()
|
f.connMu.Lock()
|
||||||
|
sequentialSizeUnknown := f.sequentialSizeUnknown
|
||||||
|
hasSequentialPrefix := f.hasSequentialPrefixLocked(conn)
|
||||||
|
resumeProbe := f.canProbeSequentialResumeLocked(conn)
|
||||||
|
intentionalRestart := sequentialSizeUnknown
|
||||||
|
if sequentialSizeUnknown || (hasSequentialPrefix && !resumeProbe) {
|
||||||
|
f.resetConnectionForRestart(conn)
|
||||||
|
f.resolveDataPos.Store(0)
|
||||||
|
f.rangeValidatorPinned = false
|
||||||
|
intentionalRestart = true
|
||||||
|
}
|
||||||
if f.meta.Res.Range && conn.Chunk.remain() <= 0 {
|
if f.meta.Res.Range && conn.Chunk.remain() <= 0 {
|
||||||
f.connMu.Unlock()
|
f.connMu.Unlock()
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
rangeStart := conn.Chunk.Begin + conn.Chunk.Downloaded
|
rangeStart := conn.Chunk.Begin + conn.Chunk.Downloaded
|
||||||
rangeEnd := conn.Chunk.End
|
rangeEnd := conn.Chunk.End
|
||||||
|
rangeMode := f.meta.Res.Range
|
||||||
|
ifRange := ""
|
||||||
|
if resumeProbe {
|
||||||
|
rangeEnd = f.meta.Res.Size - 1
|
||||||
|
ifRange = f.ifRange
|
||||||
|
} else if rangeMode && f.rangeValidatorPinned {
|
||||||
|
ifRange = f.ifRange
|
||||||
|
}
|
||||||
f.connMu.Unlock()
|
f.connMu.Unlock()
|
||||||
|
|
||||||
httpReq, err := f.buildRequest(conn.ctx, f.meta.Req)
|
httpReq, err := f.buildRequest(conn.ctx, f.meta.Req)
|
||||||
@@ -1049,9 +1124,12 @@ func (f *Fetcher) downloadChunkOnce(conn *connection, client *http.Client, buf [
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
if f.meta.Res.Range {
|
if rangeMode || resumeProbe {
|
||||||
httpReq.Header.Set(base.HttpHeaderRange,
|
httpReq.Header.Set(base.HttpHeaderRange,
|
||||||
fmt.Sprintf(base.HttpHeaderRangeFormat, rangeStart, rangeEnd))
|
fmt.Sprintf(base.HttpHeaderRangeFormat, rangeStart, rangeEnd))
|
||||||
|
if ifRange != "" {
|
||||||
|
httpReq.Header.Set(base.HttpHeaderIfRange, ifRange)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
rangeRequested := httpReq.Header.Get(base.HttpHeaderRange) != ""
|
rangeRequested := httpReq.Header.Get(base.HttpHeaderRange) != ""
|
||||||
|
|
||||||
@@ -1070,7 +1148,7 @@ func (f *Fetcher) downloadChunkOnce(conn *connection, client *http.Client, buf [
|
|||||||
// Check if this might be a redirect URL expiration error
|
// Check if this might be a redirect URL expiration error
|
||||||
// If so, try falling back to the original URL
|
// If so, try falling back to the original URL
|
||||||
if f.hasRedirectURL() && isRedirectExpiredError(originalErr) {
|
if f.hasRedirectURL() && isRedirectExpiredError(originalErr) {
|
||||||
fallbackResp, fallbackErr := f.tryFallbackToOriginalURL(conn.ctx, client, rangeStart, rangeEnd)
|
fallbackResp, fallbackErr := f.tryFallbackToOriginalURL(conn.ctx, client, rangeStart, rangeEnd, rangeRequested, ifRange)
|
||||||
if fallbackErr == nil && fallbackResp != nil {
|
if fallbackErr == nil && fallbackResp != nil {
|
||||||
// Fallback succeeded, use this response instead
|
// Fallback succeeded, use this response instead
|
||||||
resp = fallbackResp
|
resp = fallbackResp
|
||||||
@@ -1092,9 +1170,17 @@ func (f *Fetcher) downloadChunkOnce(conn *connection, client *http.Client, buf [
|
|||||||
|
|
||||||
expectedResponseLength := int64(-1)
|
expectedResponseLength := int64(-1)
|
||||||
if rangeRequested && resp.StatusCode == base.HttpCodeOK {
|
if rangeRequested && resp.StatusCode == base.HttpCodeOK {
|
||||||
if err := f.fallbackToSequentialDownload(conn); err != nil {
|
if resumeProbe {
|
||||||
resp.Body.Close()
|
// If-Range returning 200 provides the complete current representation.
|
||||||
return fmt.Errorf("%w: expected 206 Partial Content, got 200 OK", err)
|
// Discard the old prefix counters and consume this response from byte 0.
|
||||||
|
f.restartSequentialDownload(conn, extractIfRangeValidator(resp.Header))
|
||||||
|
intentionalRestart = true
|
||||||
|
} else {
|
||||||
|
if err := f.fallbackToSequentialDownload(conn, extractIfRangeValidator(resp.Header)); err != nil {
|
||||||
|
resp.Body.Close()
|
||||||
|
return fmt.Errorf("%w: expected 206 Partial Content, got 200 OK", err)
|
||||||
|
}
|
||||||
|
intentionalRestart = true
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if rangeRequested && resp.StatusCode == base.HttpCodePartialContent {
|
if rangeRequested && resp.StatusCode == base.HttpCodePartialContent {
|
||||||
@@ -1103,12 +1189,46 @@ func (f *Fetcher) downloadChunkOnce(conn *connection, client *http.Client, buf [
|
|||||||
resp.Body.Close()
|
resp.Body.Close()
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
if resumeProbe {
|
||||||
|
// The origin recovered Range support. Convert the preserved sequential
|
||||||
|
// prefix into a normal ranged chunk before writing the suffix.
|
||||||
|
f.connMu.Lock()
|
||||||
|
f.meta.Res.Range = true
|
||||||
|
f.rangeReprobeEligible = false
|
||||||
|
f.rangeValidatorPinned = true
|
||||||
|
f.sequentialSizeUnknown = false
|
||||||
|
conn.Chunk.Begin = 0
|
||||||
|
conn.Chunk.End = f.meta.Res.Size - 1
|
||||||
|
f.connMu.Unlock()
|
||||||
|
}
|
||||||
}
|
}
|
||||||
if !f.meta.Res.Range && resp.StatusCode == base.HttpCodeOK && f.meta.Res.Size > 0 {
|
if !rangeRequested && resp.StatusCode == base.HttpCodeOK {
|
||||||
expectedResponseLength = f.meta.Res.Size
|
f.connMu.Lock()
|
||||||
if resp.ContentLength >= 0 && resp.ContentLength != expectedResponseLength {
|
if f.rangeReprobeEligible {
|
||||||
resp.Body.Close()
|
// This full response might itself become the retained prefix if its
|
||||||
return fmt.Errorf("%w: Content-Length %d does not match resource size %d", errInvalidRangeResponse, resp.ContentLength, expectedResponseLength)
|
// body is interrupted, so bind any later If-Range probe to it.
|
||||||
|
f.ifRange = extractIfRangeValidator(resp.Header)
|
||||||
|
}
|
||||||
|
f.connMu.Unlock()
|
||||||
|
}
|
||||||
|
if !f.meta.Res.Range && resp.StatusCode == base.HttpCodeOK {
|
||||||
|
if intentionalRestart {
|
||||||
|
if resp.ContentLength >= 0 {
|
||||||
|
if err := f.resizeSequentialResource(resp.ContentLength); err != nil {
|
||||||
|
resp.Body.Close()
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
expectedResponseLength = resp.ContentLength
|
||||||
|
} else if err := f.beginUnknownSizeSequentialResource(); err != nil {
|
||||||
|
resp.Body.Close()
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
} else if f.meta.Res.Size > 0 {
|
||||||
|
expectedResponseLength = f.meta.Res.Size
|
||||||
|
if resp.ContentLength >= 0 && resp.ContentLength != expectedResponseLength {
|
||||||
|
resp.Body.Close()
|
||||||
|
return fmt.Errorf("%w: Content-Length %d does not match resource size %d", errInvalidRangeResponse, resp.ContentLength, expectedResponseLength)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
defer resp.Body.Close()
|
defer resp.Body.Close()
|
||||||
@@ -1202,6 +1322,11 @@ func (f *Fetcher) downloadChunkOnce(conn *connection, client *http.Client, buf [
|
|||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if err == io.EOF {
|
if err == io.EOF {
|
||||||
|
if intentionalRestart && resp.ContentLength < 0 {
|
||||||
|
if err := f.resizeSequentialResource(responseBytesRead); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
if expectedResponseLength >= 0 && responseBytesRead != expectedResponseLength {
|
if expectedResponseLength >= 0 && responseBytesRead != expectedResponseLength {
|
||||||
return fmt.Errorf("%w: response body length %d, expected %d", errInvalidRangeResponse, responseBytesRead, expectedResponseLength)
|
return fmt.Errorf("%w: response body length %d, expected %d", errInvalidRangeResponse, responseBytesRead, expectedResponseLength)
|
||||||
}
|
}
|
||||||
@@ -1254,7 +1379,7 @@ func validateRangeResponse(resp *http.Response, requestedStart, requestedEnd, to
|
|||||||
// reset the prefetch offset and consume that response from byte zero. If the
|
// reset the prefetch offset and consume that response from byte zero. If the
|
||||||
// server changes behavior after parallel connections have started, fail rather
|
// server changes behavior after parallel connections have started, fail rather
|
||||||
// than risk assembling a corrupt file.
|
// than risk assembling a corrupt file.
|
||||||
func (f *Fetcher) fallbackToSequentialDownload(conn *connection) error {
|
func (f *Fetcher) fallbackToSequentialDownload(conn *connection, ifRange string) error {
|
||||||
f.connMu.Lock()
|
f.connMu.Lock()
|
||||||
defer f.connMu.Unlock()
|
defer f.connMu.Unlock()
|
||||||
|
|
||||||
@@ -1266,11 +1391,93 @@ func (f *Fetcher) fallbackToSequentialDownload(conn *connection) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
f.meta.Res.Range = false
|
f.meta.Res.Range = false
|
||||||
|
f.rangeReprobeEligible = true
|
||||||
|
f.rangeValidatorPinned = false
|
||||||
|
f.ifRange = ifRange
|
||||||
f.resetConnectionForRestart(conn)
|
f.resetConnectionForRestart(conn)
|
||||||
f.resolveDataPos.Store(0)
|
f.resolveDataPos.Store(0)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// restartSequentialDownload consumes an If-Range 200 response as the complete
|
||||||
|
// current representation. The old prefix is no longer trusted, but this body
|
||||||
|
// can safely overwrite it from byte zero.
|
||||||
|
func (f *Fetcher) restartSequentialDownload(conn *connection, ifRange string) {
|
||||||
|
f.connMu.Lock()
|
||||||
|
defer f.connMu.Unlock()
|
||||||
|
|
||||||
|
f.meta.Res.Range = false
|
||||||
|
f.rangeReprobeEligible = true
|
||||||
|
f.rangeValidatorPinned = false
|
||||||
|
f.ifRange = ifRange
|
||||||
|
f.resetConnectionForRestart(conn)
|
||||||
|
f.resolveDataPos.Store(0)
|
||||||
|
}
|
||||||
|
|
||||||
|
// resizeSequentialResource makes a full restart response authoritative for
|
||||||
|
// both progress accounting and the target file. Truncation removes stale tail
|
||||||
|
// bytes when the new representation is shorter and preallocates when longer.
|
||||||
|
func (f *Fetcher) resizeSequentialResource(size int64) error {
|
||||||
|
if size < 0 {
|
||||||
|
return fmt.Errorf("invalid sequential response size %d", size)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Keep the file mutation and its recovery metadata in one connMu critical
|
||||||
|
// section so Store cannot persist one side of the transition without the
|
||||||
|
// other. Other download paths never hold fileMu while acquiring connMu.
|
||||||
|
f.connMu.Lock()
|
||||||
|
f.fileMu.Lock()
|
||||||
|
if f.file == nil {
|
||||||
|
f.fileMu.Unlock()
|
||||||
|
f.connMu.Unlock()
|
||||||
|
return errors.New("target file is not open")
|
||||||
|
}
|
||||||
|
err := f.file.Truncate(size)
|
||||||
|
f.fileMu.Unlock()
|
||||||
|
if err != nil {
|
||||||
|
f.connMu.Unlock()
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
f.meta.Res.Size = size
|
||||||
|
if len(f.meta.Res.Files) > 0 {
|
||||||
|
f.meta.Res.Files[0].Size = size
|
||||||
|
}
|
||||||
|
f.sequentialSizeUnknown = false
|
||||||
|
f.connMu.Unlock()
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// beginUnknownSizeSequentialResource discards all stale bytes and records that
|
||||||
|
// a restarted chunked response has no authoritative total size yet. If this
|
||||||
|
// response is interrupted, the persisted flag forces the next attempt to start
|
||||||
|
// from byte zero instead of validating a Range response against an old size.
|
||||||
|
func (f *Fetcher) beginUnknownSizeSequentialResource() error {
|
||||||
|
f.connMu.Lock()
|
||||||
|
f.fileMu.Lock()
|
||||||
|
if f.file == nil {
|
||||||
|
f.fileMu.Unlock()
|
||||||
|
f.connMu.Unlock()
|
||||||
|
return errors.New("target file is not open")
|
||||||
|
}
|
||||||
|
err := f.file.Truncate(0)
|
||||||
|
f.fileMu.Unlock()
|
||||||
|
if err != nil {
|
||||||
|
f.connMu.Unlock()
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
f.meta.Res.Size = 0
|
||||||
|
if len(f.meta.Res.Files) > 0 {
|
||||||
|
f.meta.Res.Files[0].Size = 0
|
||||||
|
}
|
||||||
|
f.sequentialSizeUnknown = true
|
||||||
|
f.ifRange = ""
|
||||||
|
f.rangeValidatorPinned = false
|
||||||
|
f.connMu.Unlock()
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
// runConnectionWithResolveResp uses the response body from Resolve phase
|
// runConnectionWithResolveResp uses the response body from Resolve phase
|
||||||
// This is crucial for one-time URLs that can only be accessed once
|
// This is crucial for one-time URLs that can only be accessed once
|
||||||
func (f *Fetcher) runConnectionWithResolveResp(conn *connection) {
|
func (f *Fetcher) runConnectionWithResolveResp(conn *connection) {
|
||||||
@@ -1590,6 +1797,34 @@ func (f *Fetcher) resetConnectionForRestart(conn *connection) {
|
|||||||
conn.lastSpeedDownload = 0
|
conn.lastSpeedDownload = 0
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// hasSequentialPrefixLocked reports whether a single sequential connection has
|
||||||
|
// a useful contiguous prefix. The caller must hold connMu.
|
||||||
|
func (f *Fetcher) hasSequentialPrefixLocked(conn *connection) bool {
|
||||||
|
if f.meta == nil || f.meta.Res == nil || f.meta.Res.Range || !f.rangeReprobeEligible || f.meta.Res.Size <= 0 {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if conn == nil || conn.ID != 0 || len(f.connections) != 1 || conn.Chunk == nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
downloaded := conn.Chunk.Downloaded
|
||||||
|
return downloaded > 0 && downloaded < f.meta.Res.Size
|
||||||
|
}
|
||||||
|
|
||||||
|
// canProbeSequentialResumeLocked reports whether the prefix can be resumed
|
||||||
|
// safely with If-Range. The caller must hold connMu.
|
||||||
|
func (f *Fetcher) canProbeSequentialResumeLocked(conn *connection) bool {
|
||||||
|
return !f.sequentialSizeUnknown && f.ifRange != "" && f.hasSequentialPrefixLocked(conn)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *Fetcher) hasSequentialRecoveryCandidate() bool {
|
||||||
|
f.connMu.Lock()
|
||||||
|
defer f.connMu.Unlock()
|
||||||
|
if len(f.connections) != 1 {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return f.hasSequentialPrefixLocked(f.connections[0])
|
||||||
|
}
|
||||||
|
|
||||||
func (f *Fetcher) resumeConnections() {
|
func (f *Fetcher) resumeConnections() {
|
||||||
// Collect connections to resume while holding the lock
|
// Collect connections to resume while holding the lock
|
||||||
var toResume []*connection
|
var toResume []*connection
|
||||||
@@ -1613,7 +1848,9 @@ func (f *Fetcher) resumeConnections() {
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
f.resetConnectionForRestart(conn)
|
if !f.hasSequentialPrefixLocked(conn) {
|
||||||
|
f.resetConnectionForRestart(conn)
|
||||||
|
}
|
||||||
// Reset the connection state for resume
|
// Reset the connection state for resume
|
||||||
conn.ctx, conn.cancel = context.WithCancel(f.ctx)
|
conn.ctx, conn.cancel = context.WithCancel(f.ctx)
|
||||||
conn.State = connNotStarted
|
conn.State = connNotStarted
|
||||||
@@ -1683,7 +1920,7 @@ func (f *Fetcher) onDownloadComplete() {
|
|||||||
// may be detected only after the expected bytes have already been written.
|
// may be detected only after the expected bytes have already been written.
|
||||||
var finalErr error
|
var finalErr error
|
||||||
for _, conn := range f.connections {
|
for _, conn := range f.connections {
|
||||||
if conn.State == connFailed && conn.failed && isRangeIntegrityError(conn.lastErr) {
|
if conn.State == connFailed && conn.failed && isTerminalRangeError(conn.lastErr) {
|
||||||
finalErr = fmt.Errorf("connection %d failed: retries=%d, err=%w", conn.ID, conn.retryTimes, conn.lastErr)
|
finalErr = fmt.Errorf("connection %d failed: retries=%d, err=%w", conn.ID, conn.retryTimes, conn.lastErr)
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -14,8 +14,15 @@ import (
|
|||||||
// ============================================================================
|
// ============================================================================
|
||||||
|
|
||||||
type fetcherData struct {
|
type fetcherData struct {
|
||||||
Connections []*connection
|
Connections []*connection
|
||||||
RedirectURL string // Saved redirect URL for resume
|
RedirectURL string // Saved redirect URL for resume
|
||||||
|
IfRange string // Strong ETag or Last-Modified validator for safe resume
|
||||||
|
RangeReprobeEligible bool // Origin advertised Range before falling back to sequential mode
|
||||||
|
RangeValidatorPinned bool // Recovered Range mode must keep the same If-Range validator
|
||||||
|
SequentialSizeUnknown bool // Interrupted chunked restart must retry from byte zero
|
||||||
|
Range *bool // Authoritative Range mode; nil for records saved by older versions
|
||||||
|
ResourceSize *int64 // Authoritative resource size from the same snapshot as Connections
|
||||||
|
FileSize *int64 // Authoritative single-file size; nil for older records
|
||||||
}
|
}
|
||||||
|
|
||||||
// ============================================================================
|
// ============================================================================
|
||||||
@@ -75,12 +82,69 @@ func (fm *FetcherManager) Store(f fetcher.Fetcher) (data any, err error) {
|
|||||||
_f.redirectLock.Lock()
|
_f.redirectLock.Lock()
|
||||||
redirectURL := _f.redirectURL
|
redirectURL := _f.redirectURL
|
||||||
_f.redirectLock.Unlock()
|
_f.redirectLock.Unlock()
|
||||||
|
|
||||||
|
// Build an immutable snapshot while holding the same lock used by download
|
||||||
|
// workers to update Range mode and connection progress. The storage layer
|
||||||
|
// marshals this value after Store returns, so returning live pointers would
|
||||||
|
// otherwise race with the active download and could persist a mixed state.
|
||||||
|
_f.connMu.Lock()
|
||||||
|
connections := snapshotConnectionsLocked(_f.connections)
|
||||||
|
ifRange := _f.ifRange
|
||||||
|
rangeReprobeEligible := _f.rangeReprobeEligible
|
||||||
|
rangeValidatorPinned := _f.rangeValidatorPinned
|
||||||
|
sequentialSizeUnknown := _f.sequentialSizeUnknown
|
||||||
|
var rangeMode *bool
|
||||||
|
var resourceSize *int64
|
||||||
|
var fileSize *int64
|
||||||
|
if _f.meta != nil && _f.meta.Res != nil {
|
||||||
|
value := _f.meta.Res.Range
|
||||||
|
rangeMode = &value
|
||||||
|
size := _f.meta.Res.Size
|
||||||
|
resourceSize = &size
|
||||||
|
if len(_f.meta.Res.Files) > 0 && _f.meta.Res.Files[0] != nil {
|
||||||
|
size := _f.meta.Res.Files[0].Size
|
||||||
|
fileSize = &size
|
||||||
|
}
|
||||||
|
}
|
||||||
|
_f.connMu.Unlock()
|
||||||
|
|
||||||
return &fetcherData{
|
return &fetcherData{
|
||||||
Connections: _f.connections,
|
Connections: connections,
|
||||||
RedirectURL: redirectURL,
|
RedirectURL: redirectURL,
|
||||||
|
IfRange: ifRange,
|
||||||
|
RangeReprobeEligible: rangeReprobeEligible,
|
||||||
|
RangeValidatorPinned: rangeValidatorPinned,
|
||||||
|
SequentialSizeUnknown: sequentialSizeUnknown,
|
||||||
|
Range: rangeMode,
|
||||||
|
ResourceSize: resourceSize,
|
||||||
|
FileSize: fileSize,
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// snapshotConnectionsLocked copies only the serialized connection state and
|
||||||
|
// deep-copies chunks. The caller must hold the fetcher's connMu.
|
||||||
|
func snapshotConnectionsLocked(connections []*connection) []*connection {
|
||||||
|
snapshot := make([]*connection, len(connections))
|
||||||
|
for i, conn := range connections {
|
||||||
|
if conn == nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
copyConn := &connection{
|
||||||
|
ID: conn.ID,
|
||||||
|
Role: conn.Role,
|
||||||
|
State: conn.State,
|
||||||
|
Downloaded: conn.Downloaded,
|
||||||
|
Completed: conn.Completed,
|
||||||
|
}
|
||||||
|
if conn.Chunk != nil {
|
||||||
|
copyChunk := *conn.Chunk
|
||||||
|
copyConn.Chunk = ©Chunk
|
||||||
|
}
|
||||||
|
snapshot[i] = copyConn
|
||||||
|
}
|
||||||
|
return snapshot
|
||||||
|
}
|
||||||
|
|
||||||
func (fm *FetcherManager) Restore() (v any, f func(meta *fetcher.FetcherMeta, v any) fetcher.Fetcher) {
|
func (fm *FetcherManager) Restore() (v any, f func(meta *fetcher.FetcherMeta, v any) fetcher.Fetcher) {
|
||||||
return &fetcherData{}, func(meta *fetcher.FetcherMeta, v any) fetcher.Fetcher {
|
return &fetcherData{}, func(meta *fetcher.FetcherMeta, v any) fetcher.Fetcher {
|
||||||
fd := v.(*fetcherData)
|
fd := v.(*fetcherData)
|
||||||
@@ -96,6 +160,22 @@ func (fm *FetcherManager) Restore() (v any, f func(meta *fetcher.FetcherMeta, v
|
|||||||
if fd.RedirectURL != "" {
|
if fd.RedirectURL != "" {
|
||||||
fetcher.redirectURL = fd.RedirectURL
|
fetcher.redirectURL = fd.RedirectURL
|
||||||
}
|
}
|
||||||
|
fetcher.ifRange = fd.IfRange
|
||||||
|
fetcher.rangeReprobeEligible = fd.RangeReprobeEligible
|
||||||
|
fetcher.rangeValidatorPinned = fd.RangeValidatorPinned
|
||||||
|
fetcher.sequentialSizeUnknown = fd.SequentialSizeUnknown
|
||||||
|
if fd.Range != nil && fetcher.meta.Res != nil {
|
||||||
|
// Recovery-critical resource fields live in the same persisted snapshot
|
||||||
|
// as Connections, making them authoritative over task metadata that may
|
||||||
|
// have been saved just before or after this record.
|
||||||
|
fetcher.meta.Res.Range = *fd.Range
|
||||||
|
}
|
||||||
|
if fd.ResourceSize != nil && fetcher.meta.Res != nil {
|
||||||
|
fetcher.meta.Res.Size = *fd.ResourceSize
|
||||||
|
}
|
||||||
|
if fd.FileSize != nil && fetcher.meta.Res != nil && len(fetcher.meta.Res.Files) > 0 && fetcher.meta.Res.Files[0] != nil {
|
||||||
|
fetcher.meta.Res.Files[0].Size = *fd.FileSize
|
||||||
|
}
|
||||||
return fetcher
|
return fetcher
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
File diff suppressed because it is too large.
Load diff
@@ -27,6 +27,45 @@ type RequestError struct {
|
|||||||
Code int
|
Code int
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// extractIfRangeValidator returns only validators that are strong enough for
|
||||||
|
// If-Range. Any entity-tag header blocks Last-Modified fallback: a weak or
|
||||||
|
// malformed ETag does not make the date validator strong.
|
||||||
|
func extractIfRangeValidator(header http.Header) string {
|
||||||
|
etag := strings.TrimSpace(header.Get(base.HttpHeaderETag))
|
||||||
|
if etag != "" {
|
||||||
|
if isStrongETag(etag) {
|
||||||
|
return etag
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
lastModified := strings.TrimSpace(header.Get(base.HttpHeaderLastModified))
|
||||||
|
date := strings.TrimSpace(header.Get("Date"))
|
||||||
|
if lastModified == "" || date == "" {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
modifiedAt, modifiedErr := http.ParseTime(lastModified)
|
||||||
|
responseAt, responseErr := http.ParseTime(date)
|
||||||
|
if modifiedErr != nil || responseErr != nil || responseAt.Sub(modifiedAt) < time.Second {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return lastModified
|
||||||
|
}
|
||||||
|
|
||||||
|
func isStrongETag(value string) bool {
|
||||||
|
if len(value) < 2 || value[0] != '"' || value[len(value)-1] != '"' {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
for i := 1; i < len(value)-1; i++ {
|
||||||
|
b := value[i]
|
||||||
|
if b == 0x21 || (b >= 0x23 && b <= 0x7e) || b >= 0x80 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
func NewRequestError(code int) *RequestError {
|
func NewRequestError(code int) *RequestError {
|
||||||
return &RequestError{Code: code}
|
return &RequestError{Code: code}
|
||||||
}
|
}
|
||||||
@@ -166,15 +205,18 @@ func isRedirectExpiredError(err error) bool {
|
|||||||
|
|
||||||
// tryFallbackToOriginalURL attempts to make a request using the original URL
|
// tryFallbackToOriginalURL attempts to make a request using the original URL
|
||||||
// when the redirect URL has expired. Returns the response if successful.
|
// when the redirect URL has expired. Returns the response if successful.
|
||||||
func (f *Fetcher) tryFallbackToOriginalURL(ctx context.Context, client *http.Client, rangeStart, rangeEnd int64) (*http.Response, error) {
|
func (f *Fetcher) tryFallbackToOriginalURL(ctx context.Context, client *http.Client, rangeStart, rangeEnd int64, rangeRequested bool, ifRange string) (*http.Response, error) {
|
||||||
httpReq, err := f.buildRequestWithOriginalURL(ctx, f.meta.Req)
|
httpReq, err := f.buildRequestWithOriginalURL(ctx, f.meta.Req)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
if f.meta.Res.Range && rangeEnd > 0 {
|
if rangeRequested {
|
||||||
httpReq.Header.Set(base.HttpHeaderRange,
|
httpReq.Header.Set(base.HttpHeaderRange,
|
||||||
fmt.Sprintf(base.HttpHeaderRangeFormat, rangeStart, rangeEnd))
|
fmt.Sprintf(base.HttpHeaderRangeFormat, rangeStart, rangeEnd))
|
||||||
|
if ifRange != "" {
|
||||||
|
httpReq.Header.Set(base.HttpHeaderIfRange, ifRange)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
resp, err := client.Do(httpReq)
|
resp, err := client.Do(httpReq)
|
||||||
|
|||||||
@@ -23,6 +23,8 @@ const (
|
|||||||
HttpHeaderContentDisposition = "Content-Disposition"
|
HttpHeaderContentDisposition = "Content-Disposition"
|
||||||
HttpHeaderUserAgent = "User-Agent"
|
HttpHeaderUserAgent = "User-Agent"
|
||||||
HttpHeaderLastModified = "Last-Modified"
|
HttpHeaderLastModified = "Last-Modified"
|
||||||
|
HttpHeaderETag = "ETag"
|
||||||
|
HttpHeaderIfRange = "If-Range"
|
||||||
|
|
||||||
HttpHeaderBytes = "bytes"
|
HttpHeaderBytes = "bytes"
|
||||||
HttpHeaderRangeFormat = "bytes=%d-%d"
|
HttpHeaderRangeFormat = "bytes=%d-%d"
|
||||||
|
|||||||
Reference in new issue
Block a user