mirror of
https://github.com/GopeedLab/gopeed.git
synced 2026-11-18 06:13:42 +00:00
feat(http): support advisory extension download sizes (#1599)
This commit is contained in:
17 files changed
+1047
-182
No files matched your search
@@ -316,13 +316,17 @@ type Fetcher struct {
|
|||||||
resolveFallback atomic.Bool // Resolve prefetch is still available until the first Range request takes over
|
resolveFallback atomic.Bool // Resolve prefetch is still available until the first Range request takes over
|
||||||
|
|
||||||
// Target file
|
// Target file
|
||||||
file *os.File
|
file *os.File
|
||||||
fileMu sync.Mutex
|
fileMu sync.Mutex
|
||||||
redirectURL string
|
redirectURL string
|
||||||
redirectLock sync.Mutex
|
redirectLock sync.Mutex
|
||||||
ifRange string
|
ifRange string
|
||||||
rangeReprobeEligible bool
|
rangeReprobeEligible bool
|
||||||
rangeValidatorPinned bool
|
rangeValidatorPinned bool
|
||||||
|
// sizeKnown distinguishes a display estimate from an HTTP-confirmed length,
|
||||||
|
// including zero. Nil preserves the interpretation of legacy snapshots.
|
||||||
|
sizeKnown *bool
|
||||||
|
// An interrupted chunked sequential response must restart from byte zero.
|
||||||
sequentialSizeUnknown bool
|
sequentialSizeUnknown bool
|
||||||
|
|
||||||
// Lifecycle control
|
// Lifecycle control
|
||||||
@@ -369,6 +373,40 @@ func (f *Fetcher) setState(s fetcherState) {
|
|||||||
f.state.Store(int32(s))
|
f.state.Store(int32(s))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Size state is read under connMu or before download workers are started.
|
||||||
|
func (f *Fetcher) hasKnownSize() bool {
|
||||||
|
if f.sizeKnown != nil {
|
||||||
|
return *f.sizeKnown
|
||||||
|
}
|
||||||
|
return f.meta != nil && f.meta.Res != nil && f.meta.Res.Size > 0 && !f.sequentialSizeUnknown
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *Fetcher) knownSize() int64 {
|
||||||
|
if f.hasKnownSize() {
|
||||||
|
return f.meta.Res.Size
|
||||||
|
}
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *Fetcher) sizeHint() int64 {
|
||||||
|
if f.meta.Opts != nil && f.meta.Opts.Size > 0 {
|
||||||
|
return f.meta.Opts.Size
|
||||||
|
}
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
// setResourceSizeLocked publishes the value and its authority in one snapshot.
|
||||||
|
func (f *Fetcher) setResourceSizeLocked(size int64, known bool) {
|
||||||
|
if !known {
|
||||||
|
size = f.sizeHint()
|
||||||
|
}
|
||||||
|
f.sizeKnown = &known
|
||||||
|
f.meta.Res.Size = size
|
||||||
|
if len(f.meta.Res.Files) > 0 && f.meta.Res.Files[0] != nil {
|
||||||
|
f.meta.Res.Files[0].Size = size
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// updateMaxConnTime updates maxConnTime if the new duration is larger
|
// updateMaxConnTime updates maxConnTime if the new duration is larger
|
||||||
func (f *Fetcher) updateMaxConnTime(d time.Duration) {
|
func (f *Fetcher) updateMaxConnTime(d time.Duration) {
|
||||||
newVal := int64(d)
|
newVal := int64(d)
|
||||||
@@ -447,6 +485,13 @@ func (f *Fetcher) Resolve(req *base.Request) error {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
// A hint is only a display value, even when the origin advertises Range.
|
||||||
|
known := resp.StatusCode == base.HttpCodePartialContent || resp.ContentLength >= 0
|
||||||
|
f.sizeKnown = &known
|
||||||
|
f.sequentialSizeUnknown = !res.Range && !known
|
||||||
|
if !known {
|
||||||
|
res.Size = f.sizeHint()
|
||||||
|
}
|
||||||
|
|
||||||
// Parse last modified time
|
// Parse last modified time
|
||||||
var lastModifiedTime *time.Time
|
var lastModifiedTime *time.Time
|
||||||
@@ -803,7 +848,7 @@ func (f *Fetcher) doStart() error {
|
|||||||
_, err = os.Stat(name)
|
_, err = os.Stat(name)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if os.IsNotExist(err) {
|
if os.IsNotExist(err) {
|
||||||
file, err = f.ctl.Touch(name, f.meta.Res.Size)
|
file, err = f.ctl.Touch(name, f.knownSize())
|
||||||
} else {
|
} else {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -821,7 +866,7 @@ func (f *Fetcher) doStart() error {
|
|||||||
// For non-range resources, the response will be used directly
|
// For non-range resources, the response will be used directly
|
||||||
var prefetchedBytes int64
|
var prefetchedBytes int64
|
||||||
if f.meta.Res.Range {
|
if f.meta.Res.Range {
|
||||||
resolveComplete := f.meta.Res.Size > 0 && f.prefetchDone.Load() && f.prefetchErr == nil && f.prefetchSize.Load() >= f.meta.Res.Size
|
resolveComplete := f.knownSize() > 0 && f.prefetchDone.Load() && f.prefetchErr == nil && f.prefetchSize.Load() >= f.knownSize()
|
||||||
if f.resolveFallback.Load() && !resolveComplete {
|
if f.resolveFallback.Load() && !resolveComplete {
|
||||||
// Keep Resolve downloading until the first Range response has been
|
// Keep Resolve downloading until the first Range response has been
|
||||||
// validated. The Range request starts at a stable snapshot; a valid
|
// validated. The Range request starts at a stable snapshot; a valid
|
||||||
@@ -993,7 +1038,7 @@ func (f *Fetcher) expandConnections() {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
totalSize := f.meta.Res.Size
|
totalSize := f.knownSize()
|
||||||
|
|
||||||
f.connMu.Lock()
|
f.connMu.Lock()
|
||||||
|
|
||||||
@@ -1284,7 +1329,7 @@ func (f *Fetcher) downloadChunkOnce(conn *connection, client *http.Client, buf [
|
|||||||
rangeMode := f.meta.Res.Range
|
rangeMode := f.meta.Res.Range
|
||||||
ifRange := ""
|
ifRange := ""
|
||||||
if resumeProbe {
|
if resumeProbe {
|
||||||
end := f.meta.Res.Size - 1
|
end := f.knownSize() - 1
|
||||||
rangeEnd = &end
|
rangeEnd = &end
|
||||||
ifRange = f.ifRange
|
ifRange = f.ifRange
|
||||||
} else if rangeMode && (f.rangeValidatorPinned || f.resolveFallback.Load()) {
|
} else if rangeMode && (f.rangeValidatorPinned || f.resolveFallback.Load()) {
|
||||||
@@ -1364,7 +1409,7 @@ func (f *Fetcher) downloadChunkOnce(conn *connection, client *http.Client, buf [
|
|||||||
}
|
}
|
||||||
if rangeRequested && resp.StatusCode == base.HttpCodePartialContent {
|
if rangeRequested && resp.StatusCode == base.HttpCodePartialContent {
|
||||||
var responseTotal int64
|
var responseTotal int64
|
||||||
expectedResponseLength, responseTotal, err = validateRangeResponse(resp, rangeStart, rangeEnd, f.meta.Res.Size)
|
expectedResponseLength, responseTotal, err = validateRangeResponse(resp, rangeStart, rangeEnd, f.knownSize())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
resp.Body.Close()
|
resp.Body.Close()
|
||||||
return err
|
return err
|
||||||
@@ -1389,7 +1434,7 @@ func (f *Fetcher) downloadChunkOnce(conn *connection, client *http.Client, buf [
|
|||||||
f.rangeValidatorPinned = true
|
f.rangeValidatorPinned = true
|
||||||
f.sequentialSizeUnknown = false
|
f.sequentialSizeUnknown = false
|
||||||
conn.Chunk.Begin = 0
|
conn.Chunk.Begin = 0
|
||||||
conn.Chunk.setEnd(f.meta.Res.Size - 1)
|
conn.Chunk.setEnd(f.knownSize() - 1)
|
||||||
f.connMu.Unlock()
|
f.connMu.Unlock()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1414,7 +1459,7 @@ func (f *Fetcher) downloadChunkOnce(conn *connection, client *http.Client, buf [
|
|||||||
resp.Body.Close()
|
resp.Body.Close()
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
} else if f.meta.Res.Size > 0 {
|
} else if f.hasKnownSize() && f.meta.Res.Size > 0 {
|
||||||
expectedResponseLength = f.meta.Res.Size
|
expectedResponseLength = f.meta.Res.Size
|
||||||
if resp.ContentLength >= 0 && resp.ContentLength != expectedResponseLength {
|
if resp.ContentLength >= 0 && resp.ContentLength != expectedResponseLength {
|
||||||
resp.Body.Close()
|
resp.Body.Close()
|
||||||
@@ -1550,7 +1595,7 @@ func (f *Fetcher) commitResolveTakeover(conn *connection, requestedStart int64)
|
|||||||
}
|
}
|
||||||
|
|
||||||
prefetched := f.stopPrefetchAndCopyData()
|
prefetched := f.stopPrefetchAndCopyData()
|
||||||
if prefetched < requestedStart || prefetched > f.meta.Res.Size {
|
if prefetched < requestedStart || prefetched > f.knownSize() {
|
||||||
return 0, fmt.Errorf("resolve prefetch advanced to invalid offset %d for range start %d", prefetched, requestedStart)
|
return 0, fmt.Errorf("resolve prefetch advanced to invalid offset %d for range start %d", prefetched, requestedStart)
|
||||||
}
|
}
|
||||||
f.resolveDataPos.Store(prefetched)
|
f.resolveDataPos.Store(prefetched)
|
||||||
@@ -1581,7 +1626,7 @@ func (f *Fetcher) completeFromResolveFallback(conn *connection) bool {
|
|||||||
}
|
}
|
||||||
time.Sleep(10 * time.Millisecond)
|
time.Sleep(10 * time.Millisecond)
|
||||||
}
|
}
|
||||||
knownSize := f.meta.Res.Size > 0
|
knownSize := f.hasKnownSize()
|
||||||
if f.prefetchErr != nil || (knownSize && f.prefetchSize.Load() != f.meta.Res.Size) {
|
if f.prefetchErr != nil || (knownSize && f.prefetchSize.Load() != f.meta.Res.Size) {
|
||||||
f.cleanupPrefetchFile()
|
f.cleanupPrefetchFile()
|
||||||
return false
|
return false
|
||||||
@@ -1595,10 +1640,7 @@ func (f *Fetcher) completeFromResolveFallback(conn *connection) bool {
|
|||||||
|
|
||||||
f.connMu.Lock()
|
f.connMu.Lock()
|
||||||
if !knownSize {
|
if !knownSize {
|
||||||
f.meta.Res.Size = prefetched
|
f.setResourceSizeLocked(prefetched, true)
|
||||||
if len(f.meta.Res.Files) > 0 && f.meta.Res.Files[0] != nil {
|
|
||||||
f.meta.Res.Files[0].Size = prefetched
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
f.meta.Res.Range = false
|
f.meta.Res.Range = false
|
||||||
f.rangeReprobeEligible = false
|
f.rangeReprobeEligible = false
|
||||||
@@ -1680,10 +1722,7 @@ func (f *Fetcher) resolveOpenEndedRange(conn *connection, totalSize int64) error
|
|||||||
f.connMu.Unlock()
|
f.connMu.Unlock()
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
f.meta.Res.Size = totalSize
|
f.setResourceSizeLocked(totalSize, true)
|
||||||
if len(f.meta.Res.Files) > 0 && f.meta.Res.Files[0] != nil {
|
|
||||||
f.meta.Res.Files[0].Size = totalSize
|
|
||||||
}
|
|
||||||
conn.Chunk.setEnd(totalSize - 1)
|
conn.Chunk.setEnd(totalSize - 1)
|
||||||
f.connMu.Unlock()
|
f.connMu.Unlock()
|
||||||
return nil
|
return nil
|
||||||
@@ -1755,10 +1794,7 @@ func (f *Fetcher) resizeSequentialResource(size int64) error {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
f.meta.Res.Size = size
|
f.setResourceSizeLocked(size, true)
|
||||||
if len(f.meta.Res.Files) > 0 {
|
|
||||||
f.meta.Res.Files[0].Size = size
|
|
||||||
}
|
|
||||||
f.sequentialSizeUnknown = false
|
f.sequentialSizeUnknown = false
|
||||||
f.connMu.Unlock()
|
f.connMu.Unlock()
|
||||||
return nil
|
return nil
|
||||||
@@ -1783,10 +1819,7 @@ func (f *Fetcher) beginUnknownSizeSequentialResource() error {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
f.meta.Res.Size = 0
|
f.setResourceSizeLocked(0, false)
|
||||||
if len(f.meta.Res.Files) > 0 {
|
|
||||||
f.meta.Res.Files[0].Size = 0
|
|
||||||
}
|
|
||||||
f.sequentialSizeUnknown = true
|
f.sequentialSizeUnknown = true
|
||||||
f.ifRange = ""
|
f.ifRange = ""
|
||||||
f.rangeValidatorPinned = false
|
f.rangeValidatorPinned = false
|
||||||
@@ -1870,11 +1903,9 @@ func (f *Fetcher) runConnectionWithResolveResp(conn *connection) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
if err == io.EOF {
|
if err == io.EOF {
|
||||||
f.connMu.Lock()
|
f.connMu.Lock()
|
||||||
if f.meta.Res.Size == 0 {
|
if !f.hasKnownSize() {
|
||||||
f.meta.Res.Size = conn.Downloaded
|
f.setResourceSizeLocked(conn.Downloaded, true)
|
||||||
if len(f.meta.Res.Files) > 0 && f.meta.Res.Files[0] != nil {
|
f.sequentialSizeUnknown = false
|
||||||
f.meta.Res.Files[0].Size = conn.Downloaded
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
conn.Completed = true
|
conn.Completed = true
|
||||||
conn.State = connCompleted
|
conn.State = connCompleted
|
||||||
@@ -1913,78 +1944,15 @@ func (f *Fetcher) runConnectionFallback(conn *connection) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
f.connMu.Lock()
|
f.connMu.Lock()
|
||||||
|
if retries > 0 {
|
||||||
|
f.resetConnectionForRestart(conn)
|
||||||
|
}
|
||||||
conn.State = connConnecting
|
conn.State = connConnecting
|
||||||
f.connMu.Unlock()
|
f.connMu.Unlock()
|
||||||
|
|
||||||
err := func() error {
|
// Restored tasks without a reusable Resolve response need the same
|
||||||
httpReq, err := f.buildRequest(conn.ctx, f.meta.Req)
|
// length handling and restart semantics as other sequential requests.
|
||||||
if err != nil {
|
err := f.downloadChunkOnce(conn, client, buf)
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
// Record connection start time for adaptive timeout tracking
|
|
||||||
connStartTime := time.Now()
|
|
||||||
|
|
||||||
resp, err := client.Do(httpReq)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
defer resp.Body.Close()
|
|
||||||
|
|
||||||
if resp.StatusCode != base.HttpCodeOK && resp.StatusCode != base.HttpCodePartialContent {
|
|
||||||
return NewRequestError(resp.StatusCode)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Record successful connection time for adaptive timeout
|
|
||||||
f.updateMaxConnTime(time.Since(connStartTime))
|
|
||||||
|
|
||||||
f.connMu.Lock()
|
|
||||||
conn.State = connDownloading
|
|
||||||
conn.failed = false
|
|
||||||
f.connMu.Unlock()
|
|
||||||
|
|
||||||
f.primaryReadyOnce.Do(func() {
|
|
||||||
close(f.primaryReadyCh)
|
|
||||||
})
|
|
||||||
if f.slowStart != nil {
|
|
||||||
f.slowStart.onConnectSuccess()
|
|
||||||
}
|
|
||||||
|
|
||||||
reader := f.responseReader(resp)
|
|
||||||
for {
|
|
||||||
if conn.ctx.Err() != nil {
|
|
||||||
return conn.ctx.Err()
|
|
||||||
}
|
|
||||||
|
|
||||||
n, err := reader.Read(buf)
|
|
||||||
if n > 0 {
|
|
||||||
f.connMu.Lock()
|
|
||||||
writeOffset := conn.Chunk.Downloaded
|
|
||||||
f.connMu.Unlock()
|
|
||||||
|
|
||||||
f.fileMu.Lock()
|
|
||||||
if f.file != nil {
|
|
||||||
_, writeErr := f.file.WriteAt(buf[:n], writeOffset)
|
|
||||||
if writeErr != nil {
|
|
||||||
f.fileMu.Unlock()
|
|
||||||
return writeErr
|
|
||||||
}
|
|
||||||
}
|
|
||||||
f.fileMu.Unlock()
|
|
||||||
|
|
||||||
f.connMu.Lock()
|
|
||||||
conn.Chunk.Downloaded += int64(n)
|
|
||||||
conn.Downloaded += int64(n)
|
|
||||||
f.connMu.Unlock()
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
if err == io.EOF {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
|
|
||||||
if err == nil {
|
if err == nil {
|
||||||
f.connMu.Lock()
|
f.connMu.Lock()
|
||||||
@@ -2133,14 +2101,14 @@ func (f *Fetcher) resetConnectionForRestart(conn *connection) {
|
|||||||
// hasSequentialPrefixLocked reports whether a single sequential connection has
|
// hasSequentialPrefixLocked reports whether a single sequential connection has
|
||||||
// a useful contiguous prefix. The caller must hold connMu.
|
// a useful contiguous prefix. The caller must hold connMu.
|
||||||
func (f *Fetcher) hasSequentialPrefixLocked(conn *connection) bool {
|
func (f *Fetcher) hasSequentialPrefixLocked(conn *connection) bool {
|
||||||
if f.meta == nil || f.meta.Res == nil || f.meta.Res.Range || !f.rangeReprobeEligible || f.meta.Res.Size <= 0 {
|
if f.meta == nil || f.meta.Res == nil || f.meta.Res.Range || !f.rangeReprobeEligible || f.knownSize() <= 0 {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
if conn == nil || conn.ID != 0 || len(f.connections) != 1 || conn.Chunk == nil {
|
if conn == nil || conn.ID != 0 || len(f.connections) != 1 || conn.Chunk == nil {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
downloaded := conn.Chunk.Downloaded
|
downloaded := conn.Chunk.Downloaded
|
||||||
return downloaded > 0 && downloaded < f.meta.Res.Size
|
return downloaded > 0 && downloaded < f.knownSize()
|
||||||
}
|
}
|
||||||
|
|
||||||
// canProbeSequentialResumeLocked reports whether the prefix can be resumed
|
// canProbeSequentialResumeLocked reports whether the prefix can be resumed
|
||||||
@@ -2226,7 +2194,7 @@ func (f *Fetcher) onDownloadComplete() {
|
|||||||
needsMoreData := false
|
needsMoreData := false
|
||||||
if f.meta.Res.Range {
|
if f.meta.Res.Range {
|
||||||
needsMoreData = conn.Chunk != nil && (conn.Chunk.openEnded() || conn.Chunk.remain() > 0)
|
needsMoreData = conn.Chunk != nil && (conn.Chunk.openEnded() || conn.Chunk.remain() > 0)
|
||||||
} else if f.meta.Res.Size > 0 {
|
} else if f.hasKnownSize() && f.meta.Res.Size > 0 {
|
||||||
needsMoreData = conn.Downloaded < f.meta.Res.Size
|
needsMoreData = conn.Downloaded < f.meta.Res.Size
|
||||||
} else {
|
} else {
|
||||||
needsMoreData = !conn.Completed && conn.State != connCompleted
|
needsMoreData = !conn.Completed && conn.State != connCompleted
|
||||||
@@ -2236,7 +2204,7 @@ func (f *Fetcher) onDownloadComplete() {
|
|||||||
// This connection has remaining work and isn't done
|
// This connection has remaining work and isn't done
|
||||||
// Check if it failed with 403 (server limit) - these can be ignored if other connections completed the work
|
// Check if it failed with 403 (server limit) - these can be ignored if other connections completed the work
|
||||||
if conn.State == connFailed && conn.failed {
|
if conn.State == connFailed && conn.failed {
|
||||||
if re := extractRequestError(conn.lastErr); re != nil && re.Code == 403 {
|
if re := extractRequestError(conn.lastErr); re != nil && re.Code == 403 && f.hasKnownSize() {
|
||||||
// 403 is server connection limit, check if other connections will complete this chunk
|
// 403 is server connection limit, check if other connections will complete this chunk
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
@@ -2247,7 +2215,7 @@ func (f *Fetcher) onDownloadComplete() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// If total downloaded matches file size, consider it a success regardless of connection failures
|
// If total downloaded matches file size, consider it a success regardless of connection failures
|
||||||
downloadComplete := f.meta.Res.Size > 0 && totalDownloaded >= f.meta.Res.Size
|
downloadComplete := f.hasKnownSize() && f.meta.Res.Size > 0 && totalDownloaded >= f.meta.Res.Size
|
||||||
|
|
||||||
// Integrity errors must win over byte-count completion. A malformed response
|
// Integrity errors must win over byte-count completion. A malformed response
|
||||||
// may be detected only after the expected bytes have already been written.
|
// may be detected only after the expected bytes have already been written.
|
||||||
@@ -2264,7 +2232,7 @@ func (f *Fetcher) onDownloadComplete() {
|
|||||||
for _, conn := range f.connections {
|
for _, conn := range f.connections {
|
||||||
if conn.State == connFailed && conn.failed {
|
if conn.State == connFailed && conn.failed {
|
||||||
// Skip 403 errors (server connection limit) - these are expected when exceeding server's limit
|
// Skip 403 errors (server connection limit) - these are expected when exceeding server's limit
|
||||||
if re := extractRequestError(conn.lastErr); re != nil && re.Code == 403 {
|
if re := extractRequestError(conn.lastErr); re != nil && re.Code == 403 && f.hasKnownSize() {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if re := extractRequestError(conn.lastErr); re != nil {
|
if re := extractRequestError(conn.lastErr); re != nil {
|
||||||
@@ -2364,7 +2332,7 @@ func (f *Fetcher) checkCompletion() bool {
|
|||||||
totalDownloaded += conn.Downloaded
|
totalDownloaded += conn.Downloaded
|
||||||
}
|
}
|
||||||
|
|
||||||
if f.meta.Res.Size > 0 && totalDownloaded >= f.meta.Res.Size {
|
if f.hasKnownSize() && f.meta.Res.Size > 0 && totalDownloaded >= f.meta.Res.Size {
|
||||||
// Don't start a new goroutine - let the caller handle completion
|
// Don't start a new goroutine - let the caller handle completion
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
@@ -2391,6 +2359,14 @@ func (f *Fetcher) checkCompletion() bool {
|
|||||||
|
|
||||||
// Patch modifies the HTTP request information.
|
// Patch modifies the HTTP request information.
|
||||||
func (f *Fetcher) Patch(req *base.Request, opts *base.Options) error {
|
func (f *Fetcher) Patch(req *base.Request, opts *base.Options) error {
|
||||||
|
if opts != nil && opts.Size > 0 {
|
||||||
|
f.connMu.Lock()
|
||||||
|
f.meta.Opts.Size = opts.Size
|
||||||
|
if f.meta.Res != nil && !f.hasKnownSize() {
|
||||||
|
f.setResourceSizeLocked(0, false)
|
||||||
|
}
|
||||||
|
f.connMu.Unlock()
|
||||||
|
}
|
||||||
// Patch request info
|
// Patch request info
|
||||||
if req != nil {
|
if req != nil {
|
||||||
if req.URL != "" {
|
if req.URL != "" {
|
||||||
@@ -2488,8 +2464,8 @@ func (f *Fetcher) Pause() error {
|
|||||||
conn := f.connections[0]
|
conn := f.connections[0]
|
||||||
if conn != nil && conn.ID == 0 && conn.Chunk != nil && !conn.Completed {
|
if conn != nil && conn.ID == 0 && conn.Chunk != nil && !conn.Completed {
|
||||||
conn.Chunk.Begin = prefetched
|
conn.Chunk.Begin = prefetched
|
||||||
if f.meta.Res.Size > 0 {
|
if f.knownSize() > 0 {
|
||||||
conn.Chunk.setEnd(f.meta.Res.Size - 1)
|
conn.Chunk.setEnd(f.knownSize() - 1)
|
||||||
} else {
|
} else {
|
||||||
conn.Chunk.End = nil
|
conn.Chunk.End = nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -19,10 +19,11 @@ type fetcherData struct {
|
|||||||
IfRange string // Strong ETag or Last-Modified validator for safe resume
|
IfRange string // Strong ETag or Last-Modified validator for safe resume
|
||||||
RangeReprobeEligible bool // Origin advertised Range before falling back to sequential mode
|
RangeReprobeEligible bool // Origin advertised Range before falling back to sequential mode
|
||||||
RangeValidatorPinned bool // Recovered Range mode must keep the same If-Range validator
|
RangeValidatorPinned bool // Recovered Range mode must keep the same If-Range validator
|
||||||
SequentialSizeUnknown bool // Interrupted chunked restart must retry from byte zero
|
SequentialSizeUnknown bool // Interrupted chunked sequential streams restart from byte zero
|
||||||
|
SizeKnown *bool // Nil for legacy snapshots; false means Size is only a display estimate
|
||||||
Range *bool // Authoritative Range mode; nil for records saved by older versions
|
Range *bool // Authoritative Range mode; nil for records saved by older versions
|
||||||
ResourceSize *int64 // Authoritative resource size from the same snapshot as Connections
|
ResourceSize *int64 // Display size from the same snapshot; authority is recorded by SizeKnown
|
||||||
FileSize *int64 // Authoritative single-file size; nil for older records
|
FileSize *int64 // Single-file size from the same snapshot; nil for older records
|
||||||
}
|
}
|
||||||
|
|
||||||
// ============================================================================
|
// ============================================================================
|
||||||
@@ -93,6 +94,7 @@ func (fm *FetcherManager) Store(f fetcher.Fetcher) (data any, err error) {
|
|||||||
rangeReprobeEligible := _f.rangeReprobeEligible
|
rangeReprobeEligible := _f.rangeReprobeEligible
|
||||||
rangeValidatorPinned := _f.rangeValidatorPinned
|
rangeValidatorPinned := _f.rangeValidatorPinned
|
||||||
sequentialSizeUnknown := _f.sequentialSizeUnknown
|
sequentialSizeUnknown := _f.sequentialSizeUnknown
|
||||||
|
sizeKnown := _f.hasKnownSize()
|
||||||
var rangeMode *bool
|
var rangeMode *bool
|
||||||
var resourceSize *int64
|
var resourceSize *int64
|
||||||
var fileSize *int64
|
var fileSize *int64
|
||||||
@@ -115,6 +117,7 @@ func (fm *FetcherManager) Store(f fetcher.Fetcher) (data any, err error) {
|
|||||||
RangeReprobeEligible: rangeReprobeEligible,
|
RangeReprobeEligible: rangeReprobeEligible,
|
||||||
RangeValidatorPinned: rangeValidatorPinned,
|
RangeValidatorPinned: rangeValidatorPinned,
|
||||||
SequentialSizeUnknown: sequentialSizeUnknown,
|
SequentialSizeUnknown: sequentialSizeUnknown,
|
||||||
|
SizeKnown: &sizeKnown,
|
||||||
Range: rangeMode,
|
Range: rangeMode,
|
||||||
ResourceSize: resourceSize,
|
ResourceSize: resourceSize,
|
||||||
FileSize: fileSize,
|
FileSize: fileSize,
|
||||||
@@ -168,6 +171,10 @@ func (fm *FetcherManager) Restore() (v any, f func(meta *fetcher.FetcherMeta, v
|
|||||||
fetcher.rangeReprobeEligible = fd.RangeReprobeEligible
|
fetcher.rangeReprobeEligible = fd.RangeReprobeEligible
|
||||||
fetcher.rangeValidatorPinned = fd.RangeValidatorPinned
|
fetcher.rangeValidatorPinned = fd.RangeValidatorPinned
|
||||||
fetcher.sequentialSizeUnknown = fd.SequentialSizeUnknown
|
fetcher.sequentialSizeUnknown = fd.SequentialSizeUnknown
|
||||||
|
if fd.SizeKnown != nil {
|
||||||
|
known := *fd.SizeKnown
|
||||||
|
fetcher.sizeKnown = &known
|
||||||
|
}
|
||||||
if fd.Range != nil && fetcher.meta.Res != nil {
|
if fd.Range != nil && fetcher.meta.Res != nil {
|
||||||
// Recovery-critical resource fields live in the same persisted snapshot
|
// Recovery-critical resource fields live in the same persisted snapshot
|
||||||
// as Connections, making them authoritative over task metadata that may
|
// as Connections, making them authoritative over task metadata that may
|
||||||
|
|||||||
@@ -1287,6 +1287,12 @@ func TestFetcher_ValidRangeTakesOverResolveResponse(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestFetcher_UnknownSizeRangeUsesOpenEndedWorker(t *testing.T) {
|
func TestFetcher_UnknownSizeRangeUsesOpenEndedWorker(t *testing.T) {
|
||||||
|
for _, hint := range []int64{0, 3, 8 * 1024 * 1024} {
|
||||||
|
t.Run(fmt.Sprint(hint), func(t *testing.T) { testFetcherUnknownSizeRange(t, hint) })
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func testFetcherUnknownSizeRange(t *testing.T, hint int64) {
|
||||||
const resolveETag = `"unknown-size-v1"`
|
const resolveETag = `"unknown-size-v1"`
|
||||||
payload := make([]byte, 4*1024*1024)
|
payload := make([]byte, 4*1024*1024)
|
||||||
for i := range payload {
|
for i := range payload {
|
||||||
@@ -1352,6 +1358,7 @@ func TestFetcher_UnknownSizeRangeUsesOpenEndedWorker(t *testing.T) {
|
|||||||
|
|
||||||
f := buildFetcher()
|
f := buildFetcher()
|
||||||
f.Meta().Opts = &base.Options{
|
f.Meta().Opts = &base.Options{
|
||||||
|
Size: hint,
|
||||||
Path: t.TempDir(),
|
Path: t.TempDir(),
|
||||||
Name: "unknown-size-range.data",
|
Name: "unknown-size-range.data",
|
||||||
Extra: &http.OptsExtra{
|
Extra: &http.OptsExtra{
|
||||||
@@ -1363,7 +1370,7 @@ func TestFetcher_UnknownSizeRangeUsesOpenEndedWorker(t *testing.T) {
|
|||||||
}
|
}
|
||||||
defer f.Close()
|
defer f.Close()
|
||||||
|
|
||||||
if f.meta.Res.Size != 0 || !f.meta.Res.Range {
|
if f.meta.Res.Size != hint || f.hasKnownSize() || !f.meta.Res.Range {
|
||||||
t.Fatalf("resolved resource = %+v, want unknown size with Range support", f.meta.Res)
|
t.Fatalf("resolved resource = %+v, want unknown size with Range support", f.meta.Res)
|
||||||
}
|
}
|
||||||
if err := f.Start(); err != nil {
|
if err := f.Start(); err != nil {
|
||||||
@@ -1595,6 +1602,7 @@ func TestFetcher_UnknownSizeRangeRemainsOpenEndedAfterPause(t *testing.T) {
|
|||||||
|
|
||||||
f := buildFetcher()
|
f := buildFetcher()
|
||||||
f.Meta().Opts = &base.Options{
|
f.Meta().Opts = &base.Options{
|
||||||
|
Size: 3,
|
||||||
Path: t.TempDir(),
|
Path: t.TempDir(),
|
||||||
Name: "pause-open-range.data",
|
Name: "pause-open-range.data",
|
||||||
Extra: &http.OptsExtra{
|
Extra: &http.OptsExtra{
|
||||||
|
|||||||
@@ -0,0 +1,245 @@
|
|||||||
|
package http
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"sync/atomic"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/GopeedLab/gopeed/pkg/base"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestFetcher_OptionsSizeHintAndPatch(t *testing.T) {
|
||||||
|
for _, known := range []bool{false, true} {
|
||||||
|
for _, payload := range []string{"", "0123456789"} {
|
||||||
|
t.Run(fmt.Sprintf("known=%v/bytes=%d", known, len(payload)), func(t *testing.T) {
|
||||||
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||||
|
if known {
|
||||||
|
w.Header().Set("Content-Length", fmt.Sprint(len(payload)))
|
||||||
|
}
|
||||||
|
w.(http.Flusher).Flush()
|
||||||
|
fmt.Fprint(w, payload)
|
||||||
|
}))
|
||||||
|
defer server.Close()
|
||||||
|
f := buildFetcher()
|
||||||
|
defer f.Close()
|
||||||
|
f.Meta().Opts = &base.Options{Size: 7, Path: t.TempDir(), Name: "output.bin"}
|
||||||
|
if err := f.Resolve(&base.Request{URL: server.URL}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
want := int64(7)
|
||||||
|
if known {
|
||||||
|
want = int64(len(payload))
|
||||||
|
}
|
||||||
|
if f.meta.Res.Size != want || f.hasKnownSize() != known {
|
||||||
|
t.Fatalf("resolved size=%d known=%v", f.meta.Res.Size, f.hasKnownSize())
|
||||||
|
}
|
||||||
|
if err := f.Patch(nil, &base.Options{Size: 9}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !known {
|
||||||
|
want = 9
|
||||||
|
}
|
||||||
|
if f.meta.Res.Size != want {
|
||||||
|
t.Fatal("hint update overrode a real size or failed to update an estimate")
|
||||||
|
}
|
||||||
|
if err := f.Start(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := f.Wait(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if f.meta.Res.Size != int64(len(payload)) || !f.hasKnownSize() {
|
||||||
|
t.Fatalf("final size=%d known=%v", f.meta.Res.Size, f.hasKnownSize())
|
||||||
|
}
|
||||||
|
snapshot, err := new(FetcherManager).Store(f)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
fd := snapshot.(*fetcherData)
|
||||||
|
if fd.SizeKnown == nil || !*fd.SizeKnown {
|
||||||
|
t.Fatal("snapshot lost confirmed size authority")
|
||||||
|
}
|
||||||
|
_, restore := new(FetcherManager).Restore()
|
||||||
|
restored := restore(f.meta, fd).(*Fetcher)
|
||||||
|
if !restored.hasKnownSize() || restored.knownSize() != int64(len(payload)) {
|
||||||
|
t.Fatal("restored size lost authority, including known zero")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFetcher_SizeHintCannotCompleteForbiddenUnknownRange(t *testing.T) {
|
||||||
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if r.Header.Get("Range") != "" {
|
||||||
|
w.WriteHeader(http.StatusForbidden)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
w.Header().Set("Accept-Ranges", "bytes")
|
||||||
|
w.(http.Flusher).Flush()
|
||||||
|
fmt.Fprint(w, "0123456789")
|
||||||
|
w.(http.Flusher).Flush()
|
||||||
|
<-r.Context().Done()
|
||||||
|
}))
|
||||||
|
defer server.Close()
|
||||||
|
f := buildFetcher()
|
||||||
|
defer f.Close()
|
||||||
|
f.Meta().Opts = &base.Options{Size: 3, Path: t.TempDir(), Name: "output.bin"}
|
||||||
|
if err := f.Resolve(&base.Request{URL: server.URL}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := f.Start(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := f.Wait(); err == nil {
|
||||||
|
t.Fatal("a failed unknown-length Range request was completed from an estimate")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFetcher_SizeHintPreservesHTTPIntegrity(t *testing.T) {
|
||||||
|
for _, chunked := range []bool{false, true} {
|
||||||
|
for _, broken := range []bool{false, true} {
|
||||||
|
t.Run(fmt.Sprintf("chunked=%v/broken=%v", chunked, broken), func(t *testing.T) {
|
||||||
|
payload := []byte("0123456789")
|
||||||
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||||
|
if chunked && broken {
|
||||||
|
conn, rw, err := w.(http.Hijacker).Hijack()
|
||||||
|
if err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
defer conn.Close()
|
||||||
|
fmt.Fprint(rw, "HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\nConnection: close\r\n\r\na\r\n0123456789\r\n")
|
||||||
|
rw.Flush() // Omit the final chunk to simulate a connection interruption.
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if !chunked {
|
||||||
|
length := len(payload)
|
||||||
|
if broken {
|
||||||
|
length++
|
||||||
|
}
|
||||||
|
w.Header().Set("Content-Length", fmt.Sprint(length))
|
||||||
|
}
|
||||||
|
w.(http.Flusher).Flush()
|
||||||
|
w.Write(payload)
|
||||||
|
}))
|
||||||
|
defer server.Close()
|
||||||
|
f := buildFetcher()
|
||||||
|
defer f.Close()
|
||||||
|
f.Meta().Opts = &base.Options{Size: 3, Path: t.TempDir(), Name: "output.bin"}
|
||||||
|
if err := f.Resolve(&base.Request{URL: server.URL}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !chunked {
|
||||||
|
length := int64(len(payload))
|
||||||
|
if broken {
|
||||||
|
length++
|
||||||
|
}
|
||||||
|
if f.meta.Res.Size != length || f.sequentialSizeUnknown {
|
||||||
|
t.Fatal("HTTP length did not override size hint")
|
||||||
|
}
|
||||||
|
} else if f.meta.Res.Size != 3 || !f.sequentialSizeUnknown {
|
||||||
|
t.Fatal("unknown HTTP length did not retain the display hint")
|
||||||
|
}
|
||||||
|
if err := f.Start(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
err := f.Wait()
|
||||||
|
if broken {
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("truncated HTTP response succeeded after exceeding the size hint")
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if f.meta.Res.Size != int64(len(payload)) || f.sequentialSizeUnknown {
|
||||||
|
t.Fatalf("completed size=%d unknown=%v", f.meta.Res.Size, f.sequentialSizeUnknown)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFetcher_SizeHintSurvivesRestore(t *testing.T) {
|
||||||
|
for _, partial := range []bool{false, true} {
|
||||||
|
for _, fixedLength := range []bool{false, true} {
|
||||||
|
t.Run(fmt.Sprintf("partial=%v/fixedLength=%v", partial, fixedLength), func(t *testing.T) {
|
||||||
|
payload := []byte("0123456789")
|
||||||
|
var requests atomic.Int64
|
||||||
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
request := requests.Add(1)
|
||||||
|
if got := r.Header.Get("Range"); got != "" {
|
||||||
|
t.Errorf("estimated sizes must not be used for resume ranges: %s", got)
|
||||||
|
}
|
||||||
|
if fixedLength && request > 1 {
|
||||||
|
w.Header().Set("Content-Length", fmt.Sprint(len(payload)))
|
||||||
|
}
|
||||||
|
w.(http.Flusher).Flush()
|
||||||
|
if fixedLength && request == 2 {
|
||||||
|
w.Write(payload[:4]) // Retry must discard this interrupted attempt.
|
||||||
|
return
|
||||||
|
}
|
||||||
|
w.Write(payload)
|
||||||
|
}))
|
||||||
|
defer server.Close()
|
||||||
|
f := buildFetcher()
|
||||||
|
opts := &base.Options{Size: 3, Path: t.TempDir(), Name: "output.bin"}
|
||||||
|
f.Meta().Opts = opts
|
||||||
|
if err := f.Resolve(&base.Request{URL: server.URL}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
fm := new(FetcherManager)
|
||||||
|
snapshot, err := fm.Store(f)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
data, err := json.Marshal(snapshot)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var saved fetcherData
|
||||||
|
if err := json.Unmarshal(data, &saved); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !saved.SequentialSizeUnknown || saved.SizeKnown == nil || *saved.SizeKnown || *saved.ResourceSize != 3 {
|
||||||
|
t.Fatal("snapshot lost the size hint or its advisory state")
|
||||||
|
}
|
||||||
|
f.Close()
|
||||||
|
// An interrupted earlier attempt left a tail longer than the new output.
|
||||||
|
if err := os.WriteFile(filepath.Join(opts.Path, opts.Name), bytes.Repeat([]byte("old"), 100), 0600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if partial {
|
||||||
|
chunk := newChunk(0, 0)
|
||||||
|
chunk.Downloaded = 7 // Already exceeds the display estimate.
|
||||||
|
saved.Connections = []*connection{{ID: 0, Role: rolePrimary, State: connNotStarted, Chunk: chunk, Downloaded: 7}}
|
||||||
|
}
|
||||||
|
_, restore := fm.Restore()
|
||||||
|
restored := restore(f.meta, &saved).(*Fetcher)
|
||||||
|
restored.Setup(f.ctl)
|
||||||
|
defer restored.Close()
|
||||||
|
if err := restored.Start(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := restored.Wait(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
body, err := os.ReadFile(restored.meta.SingleFilepath())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !bytes.Equal(body, payload) || restored.meta.Res.Size != int64(len(payload)) || restored.Progress().TotalDownloaded() != int64(len(payload)) {
|
||||||
|
t.Fatalf("restored stream retained an estimate or stale bytes: body=%q size=%d progress=%d", body, restored.meta.Res.Size, restored.Progress().TotalDownloaded())
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -108,6 +108,9 @@ type FileInfo struct {
|
|||||||
|
|
||||||
// Options for download
|
// Options for download
|
||||||
type Options struct {
|
type Options struct {
|
||||||
|
// Size is an optional HTTP download size estimate. HTTP metadata and clean EOF take precedence.
|
||||||
|
// Other protocols ignore it.
|
||||||
|
Size int64 `json:"size,omitempty"`
|
||||||
// Download file name
|
// Download file name
|
||||||
Name string `json:"name"`
|
Name string `json:"name"`
|
||||||
// Download file path
|
// Download file path
|
||||||
|
|||||||
@@ -607,7 +607,7 @@ func (d *Downloader) createDirect(req *base.Request, opts *base.Options) (taskId
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
fetcher.Meta().Req = req
|
fetcher.Meta().Req = req
|
||||||
initOpt, err := d.initOptions(opts)
|
initOpt, err := d.initOptions(opts, req.URL)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -654,7 +654,7 @@ func (d *Downloader) CreateWithOptions(rrId string, opts *base.Options) (taskId
|
|||||||
if opts == nil {
|
if opts == nil {
|
||||||
opts = &base.Options{}
|
opts = &base.Options{}
|
||||||
}
|
}
|
||||||
opts, err = d.initOptions(opts.Clone())
|
opts, err = d.initOptions(opts.Clone(), fetcher.Meta().Req.URL)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
@@ -670,6 +670,13 @@ func (d *Downloader) CreateWithOptions(rrId string, opts *base.Options) (taskId
|
|||||||
if res := fetcher.Meta().Res; res != nil {
|
if res := fetcher.Meta().Res; res != nil {
|
||||||
res.CalcSize(fetcher.Meta().Opts.SelectFiles)
|
res.CalcSize(fetcher.Meta().Opts.SelectFiles)
|
||||||
}
|
}
|
||||||
|
// Resolve has no creation options. Publish a submitted HTTP estimate now
|
||||||
|
// while preserving any size already confirmed by the response.
|
||||||
|
if opts.Size > 0 {
|
||||||
|
if err := fetcher.Patch(nil, &base.Options{Size: opts.Size}); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
}
|
||||||
defer func() {
|
defer func() {
|
||||||
d.fetcherMapLock.Lock()
|
d.fetcherMapLock.Lock()
|
||||||
delete(d.fetcherCache, rrId)
|
delete(d.fetcherCache, rrId)
|
||||||
@@ -712,7 +719,7 @@ func (d *Downloader) Patch(id string, req *base.Request, opts *base.Options) err
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Call the fetcher's Patch method
|
// Call the fetcher's Patch method
|
||||||
if err := task.fetcher.Patch(req, opts); err != nil {
|
if err := task.fetcher.Patch(req, httpSizeOptions(opts, task.Protocol)); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1123,6 +1130,9 @@ func (d *Downloader) RuntimeStatus(id string) (*TaskRuntimeStatus, error) {
|
|||||||
result.ExtractProgress = task.Progress.ExtractProgress
|
result.ExtractProgress = task.Progress.ExtractProgress
|
||||||
}
|
}
|
||||||
if task.Meta == nil || task.Meta.Res == nil {
|
if task.Meta == nil || task.Meta.Res == nil {
|
||||||
|
if task.Meta != nil && task.Meta.Opts != nil && task.Protocol == "http" {
|
||||||
|
result.Total = max(0, task.Meta.Opts.Size)
|
||||||
|
}
|
||||||
return result, nil
|
return result, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1803,10 +1813,27 @@ func (d *Downloader) doCreate(f fetcher.Fetcher, opts *base.Options) (taskId str
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
func (d *Downloader) initOptions(opts *base.Options) (*base.Options, error) {
|
// httpSizeOptions keeps estimates out of other protocols, including HLS URLs
|
||||||
|
// served over HTTP. Clone before filtering because batch tasks may share options.
|
||||||
|
func httpSizeOptions(opts *base.Options, protocol string) *base.Options {
|
||||||
|
if opts != nil && opts.Size != 0 && (protocol != "http" || opts.Size < 0) {
|
||||||
|
opts = opts.Clone()
|
||||||
|
opts.Size = 0
|
||||||
|
}
|
||||||
|
return opts
|
||||||
|
}
|
||||||
|
|
||||||
|
func (d *Downloader) initOptions(opts *base.Options, requestURL string) (*base.Options, error) {
|
||||||
if opts == nil {
|
if opts == nil {
|
||||||
opts = &base.Options{}
|
opts = &base.Options{}
|
||||||
}
|
}
|
||||||
|
if opts.Size != 0 {
|
||||||
|
fm, err := d.parseFm(requestURL)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
opts = httpSizeOptions(opts, fm.Name())
|
||||||
|
}
|
||||||
if opts.SelectFiles == nil {
|
if opts.SelectFiles == nil {
|
||||||
opts.SelectFiles = make([]int, 0)
|
opts.SelectFiles = make([]int, 0)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -30,6 +30,21 @@ support must remain disabled. Do not declare the sum of the input sizes as the
|
|||||||
output size. The downloader reads the merged output as one file; FFmpeg errors
|
output size. The downloader reads the merged output as one file; FFmpeg errors
|
||||||
propagate through the stream instead of completing a partial download.
|
propagate through the stream instead of completing a partial download.
|
||||||
|
|
||||||
|
Extension resolve results can set `files[].size` to an estimated output size,
|
||||||
|
for example the sum of the selected video and audio input lengths. Clients forward
|
||||||
|
this value as the optional task creation `opts.size` for HTTP downloads only
|
||||||
|
(including Blob URLs served over HTTP). Other protocols, including HLS, ignore
|
||||||
|
this option. The estimate is persisted
|
||||||
|
and displayed while queued and downloading. A valid HTTP total length (including
|
||||||
|
zero) always overrides it; otherwise a clean EOF records the actual output length.
|
||||||
|
Estimates never determine byte ranges, file allocation, integrity checks, or task
|
||||||
|
completion. Size authority is persisted alongside HTTP connection state for
|
||||||
|
pause/retry and process restart.
|
||||||
|
|
||||||
|
Blob options are unchanged: `createObjectURL`'s `size` is an exact source length,
|
||||||
|
not an estimate. Leave it unset for generated merge output and pass any estimate
|
||||||
|
through the extension resolve file instead.
|
||||||
|
|
||||||
## JS inputs, including SABR
|
## JS inputs, including SABR
|
||||||
|
|
||||||
Stream inputs **must** be created by an `inputs({ signal })` callback. It returns
|
Stream inputs **must** be created by an `inputs({ signal })` callback. It returns
|
||||||
@@ -130,7 +145,8 @@ associates its input progress with that Blob. No public progress callback or
|
|||||||
extra `createObjectURL` argument is required. Return the merge stream itself;
|
extra `createObjectURL` argument is required. Return the merge stream itself;
|
||||||
wrapping it in another stream loses this association. Downloaded bytes count
|
wrapping it in another stream loses this association. Downloaded bytes count
|
||||||
unique input offsets; speed counts received input bytes, including re-fetches.
|
unique input offsets; speed counts received input bytes, including re-fetches.
|
||||||
The task stays downloading with unknown total size while queued or merging;
|
The task stays downloading while queued or merging, displaying the optional
|
||||||
|
`files[].size` estimate (or an unknown total when omitted);
|
||||||
once input traffic stops the displayed speed settles to zero. Completion uses
|
once input traffic stops the displayed speed settles to zero. Completion uses
|
||||||
the actual merged output size. Generated output waits are governed by producer
|
the actual merged output size. Generated output waits are governed by producer
|
||||||
cancellation/errors rather than the HTTP body idle timeout; HTTP media inputs
|
cancellation/errors rather than the HTTP body idle timeout; HTTP media inputs
|
||||||
|
|||||||
@@ -0,0 +1,158 @@
|
|||||||
|
package download
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/GopeedLab/gopeed/pkg/base"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestExtensionResolveSizeThroughTaskCreation(t *testing.T) {
|
||||||
|
for _, mode := range []string{"http", "blob"} {
|
||||||
|
for _, known := range []bool{false, true} {
|
||||||
|
t.Run(fmt.Sprintf("%s/known=%v", mode, known), func(t *testing.T) {
|
||||||
|
payload := []byte("0123456789")
|
||||||
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
body := payload
|
||||||
|
if r.URL.Path == "/empty" {
|
||||||
|
body = nil
|
||||||
|
}
|
||||||
|
if known {
|
||||||
|
w.Header().Set("Content-Length", fmt.Sprint(len(body)))
|
||||||
|
}
|
||||||
|
w.(http.Flusher).Flush()
|
||||||
|
w.Write(body)
|
||||||
|
}))
|
||||||
|
defer server.Close()
|
||||||
|
d := NewDownloader(&DownloaderConfig{Storage: NewMemStorage(), StorageDir: t.TempDir(), TempDir: t.TempDir()})
|
||||||
|
if err := d.Setup(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer d.Clear()
|
||||||
|
d.cfg.MaxRunning = 0 // Inspect persisted queued tasks before HTTP resolution.
|
||||||
|
extDir := t.TempDir()
|
||||||
|
manifest := `{"name":"size-hint","author":"test","title":"Size hint","version":"1.0.0","scripts":[{"event":"onResolve","match":{"urls":["https://example.com/size-hint"]},"entry":"index.js"}]}`
|
||||||
|
script := fmt.Sprintf(`gopeed.events.onResolve(async ctx=>{
|
||||||
|
const mode=%q, origin=%q;
|
||||||
|
const file=async(name,size,empty)=>({name,size,req:{url:mode==='http'?origin+(empty?'/empty':'/data'):
|
||||||
|
await gopeed.runtime.blob.createObjectURL(()=>new ReadableStream({start(c){if(!empty)c.enqueue(new Uint8Array([48,49,50,51,52,53,54,55,56,57]));c.close();}}))}});
|
||||||
|
ctx.res={files:[await file('under.bin',3,false),await file('over.bin',1048576,false),await file('empty.bin',99,true)]};
|
||||||
|
});`, mode, server.URL)
|
||||||
|
for name, contents := range map[string]string{"manifest.json": manifest, "index.js": script} {
|
||||||
|
if err := os.WriteFile(filepath.Join(extDir, name), []byte(contents), 0600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if _, err := d.InstallExtensionByFolder(extDir, true); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
rr, err := d.Resolve(&base.Request{URL: "https://example.com/size-hint"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
output := t.TempDir()
|
||||||
|
batch := &base.CreateTaskBatch{}
|
||||||
|
for _, file := range rr.Res.Files {
|
||||||
|
batch.Reqs = append(batch.Reqs, &base.CreateTaskBatchItem{Req: file.Req, Opts: &base.Options{Size: file.Size, Name: file.Name, Path: output}})
|
||||||
|
}
|
||||||
|
wire, err := json.Marshal(batch)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var submitted base.CreateTaskBatch
|
||||||
|
if err := json.Unmarshal(wire, &submitted); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
ids, err := d.CreateDirectBatch(&submitted)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for i, id := range ids {
|
||||||
|
task := d.GetTask(id)
|
||||||
|
if task.Meta.Res != nil {
|
||||||
|
t.Fatal("a size estimate must not mark the task resolved")
|
||||||
|
}
|
||||||
|
status, err := d.RuntimeStatus(id)
|
||||||
|
if err != nil || status.Total != rr.Res.Files[i].Size {
|
||||||
|
t.Fatalf("queued size lost: status=%+v err=%v", status, err)
|
||||||
|
}
|
||||||
|
var saved Task
|
||||||
|
if _, err := d.storage.Get(bucketTask, id, &saved); err != nil || saved.Meta.Opts.Size != status.Total {
|
||||||
|
t.Fatal("queued estimate was not persisted", err)
|
||||||
|
}
|
||||||
|
if err := d.doStart(task); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
waitForTaskTerminal(t, d, id, 5*time.Second)
|
||||||
|
body, err := os.ReadFile(filepath.Join(output, rr.Res.Files[i].Name))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
want := payload
|
||||||
|
if i == 2 {
|
||||||
|
want = nil
|
||||||
|
}
|
||||||
|
if !bytes.Equal(body, want) {
|
||||||
|
t.Fatalf("estimate changed output: %q", body)
|
||||||
|
}
|
||||||
|
status, err = d.RuntimeStatus(id)
|
||||||
|
if err != nil || status.Total != int64(len(want)) || status.Downloaded != int64(len(want)) || task.Meta.Res.Files[0].Size != int64(len(want)) {
|
||||||
|
t.Fatalf("final size not updated: %+v err=%v", status, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResolvedHTTPSizeEstimateAtCreation(t *testing.T) {
|
||||||
|
for _, known := range []bool{false, true} {
|
||||||
|
t.Run(fmt.Sprintf("known=%v", known), func(t *testing.T) {
|
||||||
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||||
|
if known {
|
||||||
|
w.Header().Set("Content-Length", "10")
|
||||||
|
}
|
||||||
|
w.(http.Flusher).Flush()
|
||||||
|
w.Write([]byte("0123456789"))
|
||||||
|
}))
|
||||||
|
defer server.Close()
|
||||||
|
d := NewDownloader(&DownloaderConfig{StorageDir: t.TempDir(), TempDir: t.TempDir()})
|
||||||
|
if err := d.Setup(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer d.Clear()
|
||||||
|
d.cfg.MaxRunning = 0
|
||||||
|
rr, err := d.Resolve(&base.Request{URL: server.URL})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
id, err := d.CreateWithOptions(rr.ID, &base.Options{Size: 3, Path: t.TempDir(), Name: "output.bin"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
want := int64(3)
|
||||||
|
if known {
|
||||||
|
want = 10
|
||||||
|
}
|
||||||
|
status, err := d.RuntimeStatus(id)
|
||||||
|
if err != nil || status.Total != want || d.GetTask(id).Meta.Res.Files[0].Size != want {
|
||||||
|
t.Fatalf("resolve-before-create size=%+v want=%d err=%v", status, want, err)
|
||||||
|
}
|
||||||
|
if err := d.doStart(d.GetTask(id)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
waitForTaskTerminal(t, d, id, 5*time.Second)
|
||||||
|
task := d.GetTask(id)
|
||||||
|
if task.Status != base.DownloadStatusDone || task.Meta.Res.Size != 10 {
|
||||||
|
t.Fatalf("estimate changed completion: status=%s size=%d", task.Status, task.Meta.Res.Size)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,212 @@
|
|||||||
|
package download
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/GopeedLab/gopeed/pkg/base"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestExtensionFileSizeIsAdvisory(t *testing.T) {
|
||||||
|
video, err := os.ReadFile("engine/testdata/ffmpeg/video.mp4")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
audio, err := os.ReadFile("engine/testdata/ffmpeg/audio.mp4")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, mode := range []string{"plain", "merge", "wrapped-merge"} {
|
||||||
|
var expected []byte
|
||||||
|
for _, size := range []int64{0, 3, 1 << 20} {
|
||||||
|
t.Run(fmt.Sprintf("%s/size=%d", mode, size), func(t *testing.T) {
|
||||||
|
d := NewDownloader(&DownloaderConfig{Storage: NewMemStorage(), StorageDir: t.TempDir(), TempDir: t.TempDir(), RefreshInterval: 50})
|
||||||
|
if err := d.Setup(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer d.Clear()
|
||||||
|
e, _ := d.newExtensionEngine()
|
||||||
|
defer e.Close()
|
||||||
|
e.Runtime.Set("videoBytes", e.Runtime.NewArrayBuffer(video))
|
||||||
|
e.Runtime.Set("audioBytes", e.Runtime.NewArrayBuffer(audio))
|
||||||
|
e.Runtime.Set("mode", mode)
|
||||||
|
result, err := e.RunString(`__gopeed_blob_create_object_url(()=>{
|
||||||
|
const source=b=>new ReadableStream({start(c){c.enqueue(new Uint8Array(b));c.close();}});
|
||||||
|
if (mode==='plain') return source([0,1,2,3,4,5,6,7,8,9]);
|
||||||
|
const output=__gopeed_ffmpeg.merge({inputs:()=>({video:source(videoBytes),audio:source(audioBytes)})});
|
||||||
|
if (mode==='merge') return output;
|
||||||
|
const reader=output.getReader();
|
||||||
|
return new ReadableStream({async pull(c){const {done,value}=await reader.read();if(done)c.close();else c.enqueue(value);},cancel(r){return reader.cancel(r);}});
|
||||||
|
},{contentType:'video/mp4',range:false})`)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
output := t.TempDir()
|
||||||
|
id, err := d.CreateDirect(&base.Request{URL: fmt.Sprint(result)}, &base.Options{Size: size, Path: output, Name: "output.bin"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
waitForTaskTerminal(t, d, id, 30*time.Second)
|
||||||
|
body, err := os.ReadFile(filepath.Join(output, "output.bin"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if size == 0 {
|
||||||
|
expected = body
|
||||||
|
if mode == "plain" && !bytes.Equal(body, []byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}) {
|
||||||
|
t.Fatalf("unexpected stream output: %v", body)
|
||||||
|
}
|
||||||
|
if mode != "plain" && !bytes.Contains(body, []byte("moof")) {
|
||||||
|
t.Fatal("invalid merged output")
|
||||||
|
}
|
||||||
|
} else if !bytes.Equal(body, expected) {
|
||||||
|
t.Fatalf("size hint changed output: got %d bytes, want %d", len(body), len(expected))
|
||||||
|
}
|
||||||
|
task := d.GetTask(id)
|
||||||
|
task.lock.Lock()
|
||||||
|
defer task.lock.Unlock()
|
||||||
|
if task.Meta.Res.Size != int64(len(body)) || task.Meta.Res.Files[0].Size != int64(len(body)) {
|
||||||
|
t.Fatalf("final metadata must use actual size: %#v", task.Meta.Res)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExtensionFileSizeYieldsToBlobContentLength(t *testing.T) {
|
||||||
|
for _, materialized := range []bool{false, true} {
|
||||||
|
for _, size := range []int64{3, 1 << 20} {
|
||||||
|
t.Run(fmt.Sprintf("materialized=%v/estimate=%d", materialized, size), func(t *testing.T) {
|
||||||
|
d := NewDownloader(&DownloaderConfig{Storage: NewMemStorage(), StorageDir: t.TempDir(), TempDir: t.TempDir()})
|
||||||
|
if err := d.Setup(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer d.Clear()
|
||||||
|
e, _ := d.newExtensionEngine()
|
||||||
|
defer e.Close()
|
||||||
|
e.Runtime.Set("materialized", materialized)
|
||||||
|
result, err := e.RunString(`(()=>{
|
||||||
|
const payload=new Uint8Array([0,1,2,3,4,5,6,7,8,9]);
|
||||||
|
const source=materialized?new Blob([payload]):()=>new ReadableStream({start(c){c.enqueue(payload);c.close();}});
|
||||||
|
return __gopeed_blob_create_object_url(source,{size:10});
|
||||||
|
})()`)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
output := t.TempDir()
|
||||||
|
id, err := d.CreateDirect(&base.Request{URL: fmt.Sprint(result)}, &base.Options{Size: size, Path: output, Name: "output.bin"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
waitForTaskTerminal(t, d, id, 5*time.Second)
|
||||||
|
body, err := os.ReadFile(filepath.Join(output, "output.bin"))
|
||||||
|
if err != nil || !bytes.Equal(body, []byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}) {
|
||||||
|
t.Fatalf("unexpected body=%v err=%v", body, err)
|
||||||
|
}
|
||||||
|
task := d.GetTask(id)
|
||||||
|
if task.Meta.Res.Size != 10 || task.Meta.Res.Files[0].Size != 10 {
|
||||||
|
t.Fatalf("Blob Content-Length did not override estimate: %+v", task.Meta.Res)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExtensionFileSizeHintWaitsForEOF(t *testing.T) {
|
||||||
|
for _, fail := range []bool{false, true} {
|
||||||
|
for _, size := range []int64{3, 1 << 20} {
|
||||||
|
t.Run(fmt.Sprintf("fail=%v/size=%d", fail, size), func(t *testing.T) {
|
||||||
|
d := NewDownloader(&DownloaderConfig{Storage: NewMemStorage(), StorageDir: t.TempDir(), TempDir: t.TempDir(), RefreshInterval: 50})
|
||||||
|
if err := d.Setup(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer d.Clear()
|
||||||
|
e, _ := d.newExtensionEngine()
|
||||||
|
defer e.Close()
|
||||||
|
e.Runtime.Set("fail", fail)
|
||||||
|
result, err := e.RunString(`__gopeed_blob_create_object_url(()=>new ReadableStream({
|
||||||
|
start(c){c.enqueue(new Uint8Array([0,1,2,3,4,5,6,7,8,9]));globalThis.finish=()=>{if(fail)c.error(new Error('upstream failed'));else c.close();};}
|
||||||
|
}),{})`)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
output := t.TempDir()
|
||||||
|
id, err := d.CreateDirect(&base.Request{URL: fmt.Sprint(result)}, &base.Options{Size: size, Path: output, Name: "output.bin"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
waitForTaskDownloadedBetween(t, d, id, 10, 10, 5*time.Second)
|
||||||
|
task := d.GetTask(id)
|
||||||
|
task.lock.Lock()
|
||||||
|
if task.Meta.Res.Size != size || d.taskStatus(task) != base.DownloadStatusRunning {
|
||||||
|
t.Errorf("hint must remain visible until EOF: size=%d status=%s", task.Meta.Res.Size, d.taskStatus(task))
|
||||||
|
}
|
||||||
|
task.lock.Unlock()
|
||||||
|
stat, err := os.Stat(filepath.Join(output, "output.bin"))
|
||||||
|
if err != nil || stat.Size() != 10 {
|
||||||
|
t.Fatalf("hint must not preallocate or truncate output: stat=%v err=%v", stat, err)
|
||||||
|
}
|
||||||
|
if _, err := e.RunString(`finish()`); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if fail {
|
||||||
|
waitForTaskStatus(t, d, id, base.DownloadStatusError, 5*time.Second)
|
||||||
|
} else {
|
||||||
|
waitForTaskTerminal(t, d, id, 5*time.Second)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExtensionFileSizeHintPauseContinue(t *testing.T) {
|
||||||
|
for _, size := range []int64{3, 1 << 20} {
|
||||||
|
t.Run(fmt.Sprint(size), func(t *testing.T) {
|
||||||
|
d := NewDownloader(&DownloaderConfig{Storage: NewMemStorage(), StorageDir: t.TempDir(), TempDir: t.TempDir(), RefreshInterval: 50})
|
||||||
|
if err := d.Setup(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer d.Clear()
|
||||||
|
e, _ := d.newExtensionEngine()
|
||||||
|
defer e.Close()
|
||||||
|
result, err := e.RunString(`(()=>{
|
||||||
|
let opens=0;
|
||||||
|
return __gopeed_blob_create_object_url(()=>new ReadableStream({start(c){
|
||||||
|
if (++opens===1) c.enqueue(new Uint8Array([0,1,2,3,4,5,6,7,8,9]));
|
||||||
|
else {c.enqueue(new Uint8Array([5,6]));c.close();}
|
||||||
|
}}),{});
|
||||||
|
})()`)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
output := t.TempDir()
|
||||||
|
id, err := d.CreateDirect(&base.Request{URL: fmt.Sprint(result)}, &base.Options{Size: size, Path: output, Name: "output.bin"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
waitForTaskDownloadedBetween(t, d, id, 10, 10, 5*time.Second)
|
||||||
|
filter := &TaskFilter{IDs: []string{id}}
|
||||||
|
if err := d.Pause(filter); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := d.Continue(filter); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
waitForTaskTerminal(t, d, id, 5*time.Second)
|
||||||
|
body, err := os.ReadFile(filepath.Join(output, "output.bin"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
task := d.GetTask(id)
|
||||||
|
task.lock.Lock()
|
||||||
|
defer task.lock.Unlock()
|
||||||
|
if !bytes.Equal(body, []byte{5, 6}) || task.Meta.Res.Size != 2 {
|
||||||
|
t.Fatalf("resumed stream kept an estimate or stale bytes: body=%v size=%d", body, task.Meta.Res.Size)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -4,13 +4,22 @@ part 'options.g.dart';
|
|||||||
|
|
||||||
@JsonSerializable(explicitToJson: true)
|
@JsonSerializable(explicitToJson: true)
|
||||||
class Options {
|
class Options {
|
||||||
|
/// HTTP-only size estimate, superseded by HTTP metadata or completion.
|
||||||
|
int size;
|
||||||
String name;
|
String name;
|
||||||
String path;
|
String path;
|
||||||
bool asDefaultPath;
|
bool asDefaultPath;
|
||||||
List<int> selectFiles;
|
List<int> selectFiles;
|
||||||
Object? extra;
|
Object? extra;
|
||||||
|
|
||||||
Options({this.name = '', this.path = '', this.asDefaultPath = false, this.selectFiles = const [], this.extra});
|
Options({
|
||||||
|
this.size = 0,
|
||||||
|
this.name = '',
|
||||||
|
this.path = '',
|
||||||
|
this.asDefaultPath = false,
|
||||||
|
this.selectFiles = const [],
|
||||||
|
this.extra,
|
||||||
|
});
|
||||||
|
|
||||||
factory Options.fromJson(Map<String, dynamic> json) => _$OptionsFromJson(json);
|
factory Options.fromJson(Map<String, dynamic> json) => _$OptionsFromJson(json);
|
||||||
|
|
||||||
@@ -22,17 +31,14 @@ class ChecksumOption {
|
|||||||
String algorithm;
|
String algorithm;
|
||||||
String expected;
|
String expected;
|
||||||
|
|
||||||
ChecksumOption({
|
ChecksumOption({this.algorithm = '', this.expected = ''});
|
||||||
this.algorithm = '',
|
|
||||||
this.expected = '',
|
|
||||||
});
|
|
||||||
|
|
||||||
factory ChecksumOption.fromJson(Map<String, dynamic> json) => _$ChecksumOptionFromJson(json);
|
factory ChecksumOption.fromJson(Map<String, dynamic> json) => _$ChecksumOptionFromJson(json);
|
||||||
|
|
||||||
Map<String, dynamic> toJson() => _$ChecksumOptionToJson(this);
|
Map<String, dynamic> toJson() => _$ChecksumOptionToJson(this);
|
||||||
}
|
}
|
||||||
|
|
||||||
@JsonSerializable()
|
@JsonSerializable(explicitToJson: true)
|
||||||
class OptsExtraHttp {
|
class OptsExtraHttp {
|
||||||
int connections;
|
int connections;
|
||||||
bool? autoTorrent;
|
bool? autoTorrent;
|
||||||
|
|||||||
@@ -7,14 +7,20 @@ part of 'options.dart';
|
|||||||
// **************************************************************************
|
// **************************************************************************
|
||||||
|
|
||||||
Options _$OptionsFromJson(Map<String, dynamic> json) => Options(
|
Options _$OptionsFromJson(Map<String, dynamic> json) => Options(
|
||||||
|
size: (json['size'] as num?)?.toInt() ?? 0,
|
||||||
name: json['name'] as String? ?? '',
|
name: json['name'] as String? ?? '',
|
||||||
path: json['path'] as String? ?? '',
|
path: json['path'] as String? ?? '',
|
||||||
asDefaultPath: json['asDefaultPath'] as bool? ?? false,
|
asDefaultPath: json['asDefaultPath'] as bool? ?? false,
|
||||||
selectFiles: (json['selectFiles'] as List<dynamic>?)?.map((e) => (e as num).toInt()).toList() ?? const [],
|
selectFiles:
|
||||||
|
(json['selectFiles'] as List<dynamic>?)
|
||||||
|
?.map((e) => (e as num).toInt())
|
||||||
|
.toList() ??
|
||||||
|
const [],
|
||||||
extra: json['extra'],
|
extra: json['extra'],
|
||||||
);
|
);
|
||||||
|
|
||||||
Map<String, dynamic> _$OptionsToJson(Options instance) => <String, dynamic>{
|
Map<String, dynamic> _$OptionsToJson(Options instance) => <String, dynamic>{
|
||||||
|
'size': instance.size,
|
||||||
'name': instance.name,
|
'name': instance.name,
|
||||||
'path': instance.path,
|
'path': instance.path,
|
||||||
'asDefaultPath': instance.asDefaultPath,
|
'asDefaultPath': instance.asDefaultPath,
|
||||||
@@ -22,34 +28,38 @@ Map<String, dynamic> _$OptionsToJson(Options instance) => <String, dynamic>{
|
|||||||
'extra': ?instance.extra,
|
'extra': ?instance.extra,
|
||||||
};
|
};
|
||||||
|
|
||||||
ChecksumOption _$ChecksumOptionFromJson(Map<String, dynamic> json) => ChecksumOption(
|
ChecksumOption _$ChecksumOptionFromJson(Map<String, dynamic> json) =>
|
||||||
algorithm: json['algorithm'] as String? ?? '',
|
ChecksumOption(
|
||||||
expected: json['expected'] as String? ?? '',
|
algorithm: json['algorithm'] as String? ?? '',
|
||||||
);
|
expected: json['expected'] as String? ?? '',
|
||||||
|
);
|
||||||
|
|
||||||
Map<String, dynamic> _$ChecksumOptionToJson(ChecksumOption instance) => <String, dynamic>{
|
Map<String, dynamic> _$ChecksumOptionToJson(ChecksumOption instance) =>
|
||||||
'algorithm': instance.algorithm,
|
<String, dynamic>{
|
||||||
'expected': instance.expected,
|
'algorithm': instance.algorithm,
|
||||||
};
|
'expected': instance.expected,
|
||||||
|
};
|
||||||
|
|
||||||
OptsExtraHttp _$OptsExtraHttpFromJson(Map<String, dynamic> json) => OptsExtraHttp(
|
OptsExtraHttp _$OptsExtraHttpFromJson(Map<String, dynamic> json) =>
|
||||||
connections: (json['connections'] as num?)?.toInt() ?? 0,
|
OptsExtraHttp(
|
||||||
autoTorrent: json['autoTorrent'] as bool?,
|
connections: (json['connections'] as num?)?.toInt() ?? 0,
|
||||||
deleteTorrentAfterDownload: json['deleteTorrentAfterDownload'] as bool?,
|
autoTorrent: json['autoTorrent'] as bool?,
|
||||||
autoExtract: json['autoExtract'] as bool?,
|
deleteTorrentAfterDownload: json['deleteTorrentAfterDownload'] as bool?,
|
||||||
archivePassword: json['archivePassword'] as String? ?? '',
|
autoExtract: json['autoExtract'] as bool?,
|
||||||
deleteAfterExtract: json['deleteAfterExtract'] as bool? ?? false,
|
archivePassword: json['archivePassword'] as String? ?? '',
|
||||||
checksum: json['checksum'] == null
|
deleteAfterExtract: json['deleteAfterExtract'] as bool? ?? false,
|
||||||
? null
|
checksum: json['checksum'] == null
|
||||||
: ChecksumOption.fromJson(json['checksum'] as Map<String, dynamic>),
|
? null
|
||||||
);
|
: ChecksumOption.fromJson(json['checksum'] as Map<String, dynamic>),
|
||||||
|
);
|
||||||
|
|
||||||
Map<String, dynamic> _$OptsExtraHttpToJson(OptsExtraHttp instance) => <String, dynamic>{
|
Map<String, dynamic> _$OptsExtraHttpToJson(OptsExtraHttp instance) =>
|
||||||
'connections': instance.connections,
|
<String, dynamic>{
|
||||||
'autoTorrent': ?instance.autoTorrent,
|
'connections': instance.connections,
|
||||||
'deleteTorrentAfterDownload': ?instance.deleteTorrentAfterDownload,
|
'autoTorrent': ?instance.autoTorrent,
|
||||||
'autoExtract': ?instance.autoExtract,
|
'deleteTorrentAfterDownload': ?instance.deleteTorrentAfterDownload,
|
||||||
'archivePassword': instance.archivePassword,
|
'autoExtract': ?instance.autoExtract,
|
||||||
'deleteAfterExtract': instance.deleteAfterExtract,
|
'archivePassword': instance.archivePassword,
|
||||||
'checksum': ?instance.checksum?.toJson(),
|
'deleteAfterExtract': instance.deleteAfterExtract,
|
||||||
};
|
'checksum': ?instance.checksum?.toJson(),
|
||||||
|
};
|
||||||
@@ -126,7 +126,7 @@ class TaskRecord {
|
|||||||
IconData get icon => taskFileTypeIcon(name, isFolder: isFolder, protocol: protocol);
|
IconData get icon => taskFileTypeIcon(name, isFolder: isFolder, protocol: protocol);
|
||||||
|
|
||||||
factory TaskRecord.fromApi(api_task.Task task) {
|
factory TaskRecord.fromApi(api_task.Task task) {
|
||||||
final totalBytes = task.meta.res?.size ?? 0;
|
final totalBytes = task.meta.res?.size ?? task.meta.opts.size;
|
||||||
final downloadedBytes = task.progress.downloaded;
|
final downloadedBytes = task.progress.downloaded;
|
||||||
final progress = totalBytes > 0 ? (downloadedBytes / totalBytes).clamp(0.0, 1.0) : null;
|
final progress = totalBytes > 0 ? (downloadedBytes / totalBytes).clamp(0.0, 1.0) : null;
|
||||||
final status = _statusFromApi(task.status);
|
final status = _statusFromApi(task.status);
|
||||||
|
|||||||
@@ -72,6 +72,7 @@ class _CreateTaskWindowPageState extends ConsumerState<CreateTaskWindowPage> {
|
|||||||
String _httpMethod = 'GET';
|
String _httpMethod = 'GET';
|
||||||
String _httpBody = '';
|
String _httpBody = '';
|
||||||
String? _initialRawUrl;
|
String? _initialRawUrl;
|
||||||
|
int _initialSize = 0;
|
||||||
Map<String, String>? _initialLabels;
|
Map<String, String>? _initialLabels;
|
||||||
RequestProxyMode _proxyMode = RequestProxyMode.follow;
|
RequestProxyMode _proxyMode = RequestProxyMode.follow;
|
||||||
String _proxyScheme = 'http';
|
String _proxyScheme = 'http';
|
||||||
@@ -182,6 +183,7 @@ class _CreateTaskWindowPageState extends ConsumerState<CreateTaskWindowPage> {
|
|||||||
void _handleUrlChanged() {
|
void _handleUrlChanged() {
|
||||||
final urlText = _urlController.text;
|
final urlText = _urlController.text;
|
||||||
if (urlText != _lastUrlText) {
|
if (urlText != _lastUrlText) {
|
||||||
|
_initialSize = 0;
|
||||||
_lastUrlText = urlText;
|
_lastUrlText = urlText;
|
||||||
_renameController.clear();
|
_renameController.clear();
|
||||||
if (!_programmaticUrlChange && _fileDataUri.isNotEmpty) {
|
if (!_programmaticUrlChange && _fileDataUri.isNotEmpty) {
|
||||||
@@ -200,6 +202,7 @@ class _CreateTaskWindowPageState extends ConsumerState<CreateTaskWindowPage> {
|
|||||||
final req = task.req;
|
final req = task.req;
|
||||||
final opts = task.opts;
|
final opts = task.opts;
|
||||||
setState(() {
|
setState(() {
|
||||||
|
_initialSize = 0;
|
||||||
_checksumAlgorithm = 'md5';
|
_checksumAlgorithm = 'md5';
|
||||||
_checksumHashController.clear();
|
_checksumHashController.clear();
|
||||||
if (req?.url.isNotEmpty == true) {
|
if (req?.url.isNotEmpty == true) {
|
||||||
@@ -239,6 +242,7 @@ class _CreateTaskWindowPageState extends ConsumerState<CreateTaskWindowPage> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (opts != null) {
|
if (opts != null) {
|
||||||
|
_initialSize = opts.size;
|
||||||
if (opts.path.isNotEmpty) {
|
if (opts.path.isNotEmpty) {
|
||||||
_directoryController.text = opts.path;
|
_directoryController.text = opts.path;
|
||||||
}
|
}
|
||||||
@@ -1084,15 +1088,14 @@ class _CreateTaskWindowPageState extends ConsumerState<CreateTaskWindowPage> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
api_options.Options _buildOptions({String? name, List<int> selectFiles = const [], bool omitChecksum = false}) {
|
api_options.Options _buildOptions({String? name, List<int> selectFiles = const [], bool omitChecksum = false}) {
|
||||||
|
final urls = _inputUrls();
|
||||||
final connections = int.tryParse(_connectionsController.text.trim()) ?? 0;
|
final connections = int.tryParse(_connectionsController.text.trim()) ?? 0;
|
||||||
final checksumExpected = _checksumHashController.text.trim();
|
final checksumExpected = _checksumHashController.text.trim();
|
||||||
final checksum = (!omitChecksum && _inputUrls().length <= 1 && checksumExpected.isNotEmpty)
|
final checksum = (!omitChecksum && _inputUrls().length <= 1 && checksumExpected.isNotEmpty)
|
||||||
? api_options.ChecksumOption(
|
? api_options.ChecksumOption(algorithm: _checksumAlgorithm, expected: checksumExpected)
|
||||||
algorithm: _checksumAlgorithm,
|
|
||||||
expected: checksumExpected,
|
|
||||||
)
|
|
||||||
: null;
|
: null;
|
||||||
return api_options.Options(
|
return api_options.Options(
|
||||||
|
size: urls.length == 1 && _parseProtocol(_protocolUrlFor(urls.first)) == _TaskProtocol.http ? _initialSize : 0,
|
||||||
name: name ?? _renameController.text.trim(),
|
name: name ?? _renameController.text.trim(),
|
||||||
path: _directoryController.text.trim(),
|
path: _directoryController.text.trim(),
|
||||||
asDefaultPath: _asDefaultPath,
|
asDefaultPath: _asDefaultPath,
|
||||||
@@ -1135,16 +1138,25 @@ class _CreateTaskWindowPageState extends ConsumerState<CreateTaskWindowPage> {
|
|||||||
if (result.res.files.isNotEmpty) {
|
if (result.res.files.isNotEmpty) {
|
||||||
final reqs = selected.map((index) {
|
final reqs = selected.map((index) {
|
||||||
final file = result.res.files[index];
|
final file = result.res.files[index];
|
||||||
|
final fileRequest = file.req ?? request;
|
||||||
return CreateTaskBatchItem(
|
return CreateTaskBatchItem(
|
||||||
req: file.req ?? request,
|
req: fileRequest,
|
||||||
opts: api_options.Options(
|
opts: api_options.Options(
|
||||||
|
size: _parseProtocol(fileRequest.url) == _TaskProtocol.http ? file.size : 0,
|
||||||
name: file.name,
|
name: file.name,
|
||||||
path: path.join(_directoryController.text.trim(), result.res.name, file.path),
|
path: path.join(_directoryController.text.trim(), result.res.name, file.path),
|
||||||
extra: _buildOptions(omitChecksum: isMultiFile).extra,
|
extra: _buildOptions(omitChecksum: isMultiFile).extra,
|
||||||
),
|
),
|
||||||
);
|
);
|
||||||
}).toList();
|
}).toList();
|
||||||
await ref.read(gopeedServiceProvider).createTaskBatch(CreateTaskBatch(reqs: reqs, opts: _buildOptions(omitChecksum: isMultiFile)));
|
await ref
|
||||||
|
.read(gopeedServiceProvider)
|
||||||
|
.createTaskBatch(
|
||||||
|
CreateTaskBatch(
|
||||||
|
reqs: reqs,
|
||||||
|
opts: _buildOptions(omitChecksum: isMultiFile),
|
||||||
|
),
|
||||||
|
);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -236,7 +236,6 @@ class _FileDownloadProgress extends StatelessWidget {
|
|||||||
final sizeKnown = selectedFiles.every((file) => file.size > 0);
|
final sizeKnown = selectedFiles.every((file) => file.size > 0);
|
||||||
final size = sizeKnown ? selectedFiles.fold<int>(0, (total, file) => total + file.size) : 0;
|
final size = sizeKnown ? selectedFiles.fold<int>(0, (total, file) => total + file.size) : 0;
|
||||||
final progress = size > 0 ? (downloaded / size).clamp(0.0, 1.0) : null;
|
final progress = size > 0 ? (downloaded / size).clamp(0.0, 1.0) : null;
|
||||||
final completed = progress != null && progress >= 1;
|
|
||||||
final progressLabel = progress == null ? Util.fmtByte(downloaded) : '${(progress * 100).toStringAsFixed(0)}%';
|
final progressLabel = progress == null ? Util.fmtByte(downloaded) : '${(progress * 100).toStringAsFixed(0)}%';
|
||||||
|
|
||||||
return SizedBox(
|
return SizedBox(
|
||||||
@@ -244,16 +243,14 @@ class _FileDownloadProgress extends StatelessWidget {
|
|||||||
width: 116,
|
width: 116,
|
||||||
child: _TaskFileColumns(
|
child: _TaskFileColumns(
|
||||||
progress: Semantics(
|
progress: Semantics(
|
||||||
label: completed ? context.l10n.completed : progressLabel,
|
label: progressLabel,
|
||||||
child: completed
|
child: shad.CircularProgressIndicator(
|
||||||
? Icon(shad.BootstrapIcons.checkCircleFill, size: 16, color: palette.brandProgress)
|
value: progress ?? (downloaded > 0 ? null : 0),
|
||||||
: shad.CircularProgressIndicator(
|
size: 16,
|
||||||
value: progress ?? (downloaded > 0 ? null : 0),
|
strokeWidth: 2,
|
||||||
size: 16,
|
color: palette.brandProgress,
|
||||||
strokeWidth: 2,
|
backgroundColor: palette.progressTrack,
|
||||||
color: palette.brandProgress,
|
),
|
||||||
backgroundColor: palette.progressTrack,
|
|
||||||
),
|
|
||||||
),
|
),
|
||||||
size: _formatFileSize(node.size),
|
size: _formatFileSize(node.size),
|
||||||
),
|
),
|
||||||
|
|||||||
@@ -44,6 +44,30 @@ void main() {
|
|||||||
expect(record.totalBytes, 60);
|
expect(record.totalBytes, 60);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('underestimated HTTP size caps progress without completing the task', () {
|
||||||
|
final task = buildTask([], 3)..protocol = api.Protocol.http;
|
||||||
|
task.progress.speed = 2;
|
||||||
|
final record = TaskRecord.fromApi(task);
|
||||||
|
expect(record.progress, 1);
|
||||||
|
expect(record.status, TaskStatus.downloading);
|
||||||
|
expect(record.downloadedBytes, 10);
|
||||||
|
expect(record.totalBytes, 3);
|
||||||
|
expect(record.remainingSeconds, isNull);
|
||||||
|
expect(const TaskFileNode(path: '', name: 'video.mp4', sizeBytes: 3, downloadedBytes: 10).progress, 1);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('queued task uses the creation estimate until a resolved size is available', () {
|
||||||
|
final task = buildTask([], 0)..protocol = api.Protocol.http;
|
||||||
|
task.meta.opts.size = 1024;
|
||||||
|
task.meta.res = null;
|
||||||
|
task.status = api.Status.wait;
|
||||||
|
expect(TaskRecord.fromApi(task).totalBytes, 1024);
|
||||||
|
task.meta.res = buildTask([], 0).meta.res;
|
||||||
|
expect(TaskRecord.fromApi(task).totalBytes, isNull); // A confirmed zero must not fall back to 1024.
|
||||||
|
task.meta.res!.size = 512;
|
||||||
|
expect(TaskRecord.fromApi(task).totalBytes, 512);
|
||||||
|
});
|
||||||
|
|
||||||
test('completion time maps from api and stays null for legacy tasks', () {
|
test('completion time maps from api and stays null for legacy tasks', () {
|
||||||
final completed = TaskRecord.fromApi(buildTask([], 60, doneAt: '2026-01-02T03:04:05Z'));
|
final completed = TaskRecord.fromApi(buildTask([], 60, doneAt: '2026-01-02T03:04:05Z'));
|
||||||
expect(completed.doneAt, DateTime.parse('2026-01-02T03:04:05Z'));
|
expect(completed.doneAt, DateTime.parse('2026-01-02T03:04:05Z'));
|
||||||
|
|||||||
@@ -1,3 +1,5 @@
|
|||||||
|
import 'dart:convert';
|
||||||
|
|
||||||
import 'package:flutter/widgets.dart';
|
import 'package:flutter/widgets.dart';
|
||||||
import 'package:flutter/services.dart';
|
import 'package:flutter/services.dart';
|
||||||
import 'package:desktop_multi_window/desktop_multi_window.dart';
|
import 'package:desktop_multi_window/desktop_multi_window.dart';
|
||||||
@@ -8,6 +10,7 @@ import 'package:flutter_riverpod/flutter_riverpod.dart';
|
|||||||
import 'package:flutter_test/flutter_test.dart';
|
import 'package:flutter_test/flutter_test.dart';
|
||||||
import 'package:go_router/go_router.dart';
|
import 'package:go_router/go_router.dart';
|
||||||
import 'package:gopeed/api/model/create_task.dart';
|
import 'package:gopeed/api/model/create_task.dart';
|
||||||
|
import 'package:gopeed/api/model/create_task_batch.dart';
|
||||||
import 'package:gopeed/api/model/downloader_config.dart';
|
import 'package:gopeed/api/model/downloader_config.dart';
|
||||||
import 'package:gopeed/api/model/options.dart';
|
import 'package:gopeed/api/model/options.dart';
|
||||||
import 'package:gopeed/api/model/request.dart';
|
import 'package:gopeed/api/model/request.dart';
|
||||||
@@ -30,6 +33,73 @@ import 'package:gopeed/l10n/app_localizations.dart';
|
|||||||
import 'package:shadcn_flutter/shadcn_flutter.dart' as shad;
|
import 'package:shadcn_flutter/shadcn_flutter.dart' as shad;
|
||||||
|
|
||||||
void main() {
|
void main() {
|
||||||
|
testWidgets('extension file estimates survive batch submission and window serialization', (tester) async {
|
||||||
|
tester.view.physicalSize = const Size(900, 1000);
|
||||||
|
tester.view.devicePixelRatio = 1;
|
||||||
|
addTearDown(tester.view.resetPhysicalSize);
|
||||||
|
addTearDown(tester.view.resetDevicePixelRatio);
|
||||||
|
CreateTaskBatch? submitted;
|
||||||
|
final registry = CapabilityRegistry(createAppCapabilityCodecs())
|
||||||
|
..bind(GopeedMethods.getConfig, (_) => DownloaderConfig(downloadDir: '/downloads'))
|
||||||
|
..bind(
|
||||||
|
GopeedMethods.resolve,
|
||||||
|
(_) => ResolveResult(
|
||||||
|
id: '',
|
||||||
|
res: Resource(
|
||||||
|
files: [
|
||||||
|
FileInfo(
|
||||||
|
name: 'under.mp4',
|
||||||
|
size: 3,
|
||||||
|
req: Request(url: 'https://example.com/video'),
|
||||||
|
),
|
||||||
|
FileInfo(
|
||||||
|
name: 'over.mp4',
|
||||||
|
size: 1048576,
|
||||||
|
req: Request(url: 'https://example.com/audio'),
|
||||||
|
),
|
||||||
|
],
|
||||||
|
),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
..bind(GopeedMethods.createTaskBatch, (batch) {
|
||||||
|
submitted = CreateTaskBatch.fromJson(jsonDecode(jsonEncode(batch)) as Map<String, dynamic>);
|
||||||
|
return ['first', 'second'];
|
||||||
|
})
|
||||||
|
..bind(StorageMethods.saveCreateHistory, (_) => const RpcUnit());
|
||||||
|
final router = GoRouter(
|
||||||
|
initialLocation: '/create',
|
||||||
|
routes: [
|
||||||
|
GoRoute(path: '/', builder: (_, _) => const SizedBox()),
|
||||||
|
GoRoute(path: '/create', builder: (_, _) => const CreateTaskWindowPage()),
|
||||||
|
],
|
||||||
|
);
|
||||||
|
addTearDown(router.dispose);
|
||||||
|
await tester.pumpWidget(
|
||||||
|
ProviderScope(
|
||||||
|
overrides: [appCapabilitiesProvider.overrideWithValue(AppCapabilities(LocalCapabilityInvoker(registry)))],
|
||||||
|
child: shad.ShadcnApp.router(
|
||||||
|
theme: AppTheme.light(),
|
||||||
|
materialTheme: AppTheme.materialLight(),
|
||||||
|
routerConfig: router,
|
||||||
|
builder: (_, child) => AppComponentThemes(child: child!),
|
||||||
|
),
|
||||||
|
),
|
||||||
|
);
|
||||||
|
await tester.pumpAndSettle();
|
||||||
|
await tester.enterText(find.byKey(const ValueKey('create-task-url-input')), 'https://example.com/watch');
|
||||||
|
await tester.tap(find.byKey(const ValueKey('create-task-confirm-button')));
|
||||||
|
await tester.pump();
|
||||||
|
await tester.pump(const Duration(milliseconds: 300));
|
||||||
|
await tester.tap(find.byKey(const ValueKey('resolve-create-button')));
|
||||||
|
await tester.pumpAndSettle();
|
||||||
|
expect(submitted?.reqs?.map((item) => item.opts?.size), [3, 1048576]);
|
||||||
|
expect(submitted?.reqs?.map((item) => item.req?.url), [
|
||||||
|
'https://example.com/video',
|
||||||
|
'https://example.com/audio',
|
||||||
|
]);
|
||||||
|
expect(tester.takeException(), isNull);
|
||||||
|
});
|
||||||
|
|
||||||
for (final navigationFails in [false, true]) {
|
for (final navigationFails in [false, true]) {
|
||||||
testWidgets('child creation selects downloads before closing (navigation fails: $navigationFails)', (tester) async {
|
testWidgets('child creation selects downloads before closing (navigation fails: $navigationFails)', (tester) async {
|
||||||
tester.view.physicalSize = const Size(1024, 720);
|
tester.view.physicalSize = const Size(1024, 720);
|
||||||
@@ -264,6 +334,7 @@ void main() {
|
|||||||
skipVerifyCert: true,
|
skipVerifyCert: true,
|
||||||
),
|
),
|
||||||
opts: Options(
|
opts: Options(
|
||||||
|
size: 1234,
|
||||||
name: 'archive.zip',
|
name: 'archive.zip',
|
||||||
path: 'D:/Downloads',
|
path: 'D:/Downloads',
|
||||||
extra: OptsExtraHttp(
|
extra: OptsExtraHttp(
|
||||||
@@ -457,6 +528,7 @@ void main() {
|
|||||||
expect(request?.skipVerifyCert, isTrue);
|
expect(request?.skipVerifyCert, isTrue);
|
||||||
expect(ReqExtraHttp.fromJson(request?.extra! as Map<String, dynamic>).toJson(), initialTask.req?.extra);
|
expect(ReqExtraHttp.fromJson(request?.extra! as Map<String, dynamic>).toJson(), initialTask.req?.extra);
|
||||||
expect(submitted?.opts?.path, 'G:/Manually selected');
|
expect(submitted?.opts?.path, 'G:/Manually selected');
|
||||||
|
expect(submitted?.opts?.size, 1234);
|
||||||
final options = OptsExtraHttp.fromJson(submitted?.opts?.extra! as Map<String, dynamic>);
|
final options = OptsExtraHttp.fromJson(submitted?.opts?.extra! as Map<String, dynamic>);
|
||||||
expect(options.connections, 12);
|
expect(options.connections, 12);
|
||||||
expect(options.autoTorrent, isTrue);
|
expect(options.autoTorrent, isTrue);
|
||||||
@@ -676,7 +748,10 @@ void main() {
|
|||||||
// Expand advanced
|
// Expand advanced
|
||||||
await tester.tap(find.text('Advanced'));
|
await tester.tap(find.text('Advanced'));
|
||||||
await tester.pumpAndSettle();
|
await tester.pumpAndSettle();
|
||||||
expect(_fieldText(tester, 'create-task-checksum-expected'), 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855');
|
expect(
|
||||||
|
_fieldText(tester, 'create-task-checksum-expected'),
|
||||||
|
'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855',
|
||||||
|
);
|
||||||
|
|
||||||
// Task B form 1: null opts
|
// Task B form 1: null opts
|
||||||
pending.set(CreateTask(req: Request(url: 'https://example.com/taskB1.zip')));
|
pending.set(CreateTask(req: Request(url: 'https://example.com/taskB1.zip')));
|
||||||
@@ -685,7 +760,12 @@ void main() {
|
|||||||
expect(_fieldText(tester, 'create-task-checksum-expected'), '');
|
expect(_fieldText(tester, 'create-task-checksum-expected'), '');
|
||||||
|
|
||||||
// Task B form 2: empty Options
|
// Task B form 2: empty Options
|
||||||
pending.set(CreateTask(req: Request(url: 'https://example.com/taskB2.zip'), opts: Options()));
|
pending.set(
|
||||||
|
CreateTask(
|
||||||
|
req: Request(url: 'https://example.com/taskB2.zip'),
|
||||||
|
opts: Options(),
|
||||||
|
),
|
||||||
|
);
|
||||||
router.go('/create');
|
router.go('/create');
|
||||||
await tester.pumpAndSettle();
|
await tester.pumpAndSettle();
|
||||||
expect(_fieldText(tester, 'create-task-checksum-expected'), '');
|
expect(_fieldText(tester, 'create-task-checksum-expected'), '');
|
||||||
@@ -804,7 +884,10 @@ void main() {
|
|||||||
submitted = null;
|
submitted = null;
|
||||||
await tester.tap(find.text('MD5'));
|
await tester.tap(find.text('MD5'));
|
||||||
await tester.pumpAndSettle();
|
await tester.pumpAndSettle();
|
||||||
await tester.enterText(find.byKey(const ValueKey('create-task-checksum-expected')), '0123456789ABCDEF0123456789ABCDEF');
|
await tester.enterText(
|
||||||
|
find.byKey(const ValueKey('create-task-checksum-expected')),
|
||||||
|
'0123456789ABCDEF0123456789ABCDEF',
|
||||||
|
);
|
||||||
await tester.tap(find.text('Confirm'));
|
await tester.tap(find.text('Confirm'));
|
||||||
await tester.pumpAndSettle();
|
await tester.pumpAndSettle();
|
||||||
expect(submitted, isNotNull);
|
expect(submitted, isNotNull);
|
||||||
@@ -821,7 +904,10 @@ void main() {
|
|||||||
await tester.pumpAndSettle();
|
await tester.pumpAndSettle();
|
||||||
await tester.tap(find.text('SHA-1'));
|
await tester.tap(find.text('SHA-1'));
|
||||||
await tester.pumpAndSettle();
|
await tester.pumpAndSettle();
|
||||||
await tester.enterText(find.byKey(const ValueKey('create-task-checksum-expected')), 'da39a3ee5e6b4b0d3255bfef95601890afd80709');
|
await tester.enterText(
|
||||||
|
find.byKey(const ValueKey('create-task-checksum-expected')),
|
||||||
|
'da39a3ee5e6b4b0d3255bfef95601890afd80709',
|
||||||
|
);
|
||||||
await tester.tap(find.text('Confirm'));
|
await tester.tap(find.text('Confirm'));
|
||||||
await tester.pumpAndSettle();
|
await tester.pumpAndSettle();
|
||||||
expect(submitted, isNotNull);
|
expect(submitted, isNotNull);
|
||||||
@@ -838,7 +924,10 @@ void main() {
|
|||||||
await tester.pumpAndSettle();
|
await tester.pumpAndSettle();
|
||||||
await tester.tap(find.text('SHA-256'));
|
await tester.tap(find.text('SHA-256'));
|
||||||
await tester.pumpAndSettle();
|
await tester.pumpAndSettle();
|
||||||
await tester.enterText(find.byKey(const ValueKey('create-task-checksum-expected')), 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855');
|
await tester.enterText(
|
||||||
|
find.byKey(const ValueKey('create-task-checksum-expected')),
|
||||||
|
'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855',
|
||||||
|
);
|
||||||
await tester.tap(find.text('Confirm'));
|
await tester.tap(find.text('Confirm'));
|
||||||
await tester.pumpAndSettle();
|
await tester.pumpAndSettle();
|
||||||
expect(submitted, isNotNull);
|
expect(submitted, isNotNull);
|
||||||
|
|||||||
@@ -0,0 +1,75 @@
|
|||||||
|
import 'package:flutter/widgets.dart';
|
||||||
|
import 'package:flutter_test/flutter_test.dart';
|
||||||
|
import 'package:gopeed/api/model/task.dart' as api;
|
||||||
|
import 'package:gopeed/features/tasks/domain/task_record.dart';
|
||||||
|
import 'package:gopeed/features/tasks/presentation/widgets/task_file_tree.dart';
|
||||||
|
import 'package:gopeed/features/tasks/presentation/widgets/task_progress_bar.dart';
|
||||||
|
import 'package:gopeed/shared/theme/app_theme.dart';
|
||||||
|
import 'package:shadcn_flutter/shadcn_flutter.dart' as shad;
|
||||||
|
|
||||||
|
void main() {
|
||||||
|
for (final scenario in [
|
||||||
|
(protocol: api.Protocol.http, completed: false),
|
||||||
|
(protocol: api.Protocol.http, completed: true),
|
||||||
|
(protocol: api.Protocol.bt, completed: false),
|
||||||
|
]) {
|
||||||
|
testWidgets('file progress stays bounded and respects completion state: $scenario', (tester) async {
|
||||||
|
tester.view.physicalSize = const Size(520, 420);
|
||||||
|
tester.view.devicePixelRatio = 1;
|
||||||
|
addTearDown(tester.view.resetPhysicalSize);
|
||||||
|
addTearDown(tester.view.resetDevicePixelRatio);
|
||||||
|
final total = scenario.completed ? 10 : 3;
|
||||||
|
final task = TaskRecord(
|
||||||
|
id: 'estimated-size',
|
||||||
|
name: 'video.mp4',
|
||||||
|
protocol: scenario.protocol,
|
||||||
|
status: scenario.completed ? TaskStatus.completed : TaskStatus.downloading,
|
||||||
|
downloaded: '10 B',
|
||||||
|
total: '$total B',
|
||||||
|
progress: 1,
|
||||||
|
url: 'https://example.com/video',
|
||||||
|
storagePath: '/downloads/video.mp4',
|
||||||
|
files: [TaskFileNode(path: '', name: 'video.mp4', sizeBytes: total, resourceIndex: 0)],
|
||||||
|
uploading: false,
|
||||||
|
);
|
||||||
|
final runtime = api.TaskRuntimeStatus(
|
||||||
|
status: scenario.completed ? api.Status.done : api.Status.running,
|
||||||
|
used: 0,
|
||||||
|
speed: 2,
|
||||||
|
downloaded: 10,
|
||||||
|
total: total,
|
||||||
|
uploadSpeed: 0,
|
||||||
|
uploaded: 0,
|
||||||
|
files: [api.FileRuntimeStatus(index: 0, size: total, downloaded: 10)],
|
||||||
|
);
|
||||||
|
await tester.pumpWidget(
|
||||||
|
shad.ShadcnApp(
|
||||||
|
theme: AppTheme.light(),
|
||||||
|
materialTheme: AppTheme.materialLight(),
|
||||||
|
home: Padding(
|
||||||
|
padding: const EdgeInsets.all(24),
|
||||||
|
child: TaskFileTree(task: task, runtimeStatus: runtime),
|
||||||
|
),
|
||||||
|
),
|
||||||
|
);
|
||||||
|
await tester.pump();
|
||||||
|
final progress = tester.widget<TaskProgressBar>(find.byKey(const ValueKey('task-files-total-progress')));
|
||||||
|
expect(progress.value, 1);
|
||||||
|
expect(find.text('100%'), findsOneWidget);
|
||||||
|
expect(find.text('333%'), findsNothing);
|
||||||
|
final file = find.byKey(const ValueKey('task-file-progress-file:0'));
|
||||||
|
final check = find.descendant(of: file, matching: find.byIcon(shad.BootstrapIcons.checkCircleFill));
|
||||||
|
final ring = find.descendant(of: file, matching: find.byType(shad.CircularProgressIndicator));
|
||||||
|
if (!scenario.completed) {
|
||||||
|
expect(check, findsNothing);
|
||||||
|
expect(ring, findsOneWidget);
|
||||||
|
expect(tester.widget<shad.CircularProgressIndicator>(ring).value, 1);
|
||||||
|
} else {
|
||||||
|
expect(check, findsOneWidget);
|
||||||
|
expect(ring, findsNothing);
|
||||||
|
}
|
||||||
|
expect(tester.takeException(), isNull);
|
||||||
|
await tester.pumpWidget(const SizedBox());
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in new issue
Block a user