mirror of
https://github.com/GopeedLab/gopeed.git
synced 2026-11-10 18:40:27 +00:00
Compare commits
5
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
0bd306773d | ||
|
|
8b02844ec3 | ||
|
|
b8e517b35b | ||
|
|
2daa9c45df | ||
|
|
27c0f81a1e |
No files matched your search
@@ -872,9 +872,9 @@ func (f *Fetcher) expandConnections() {
|
|||||||
f.connMu.Unlock()
|
f.connMu.Unlock()
|
||||||
|
|
||||||
if len(newConns) == 0 {
|
if len(newConns) == 0 {
|
||||||
// No new connections could be created, stop expansion
|
// No new connections could be created, stop expansion.
|
||||||
|
// The downloadLoop will check stateSteady and call waitForCompletion.
|
||||||
f.setState(stateSteady)
|
f.setState(stateSteady)
|
||||||
go f.waitForCompletion()
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -987,6 +987,14 @@ func (f *Fetcher) downloadChunkOnce(conn *connection, client *http.Client, buf [
|
|||||||
f.connMu.Unlock()
|
f.connMu.Unlock()
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
// For non-range downloads, reset chunk state since server will send from beginning.
|
||||||
|
// Without Range support, each new request restarts the entire transfer.
|
||||||
|
if !f.meta.Res.Range {
|
||||||
|
conn.Chunk.Begin = 0
|
||||||
|
conn.Chunk.End = 0
|
||||||
|
conn.Chunk.Downloaded = 0
|
||||||
|
conn.Downloaded = 0
|
||||||
|
}
|
||||||
rangeStart := conn.Chunk.Begin + conn.Chunk.Downloaded
|
rangeStart := conn.Chunk.Begin + conn.Chunk.Downloaded
|
||||||
rangeEnd := conn.Chunk.End
|
rangeEnd := conn.Chunk.End
|
||||||
f.connMu.Unlock()
|
f.connMu.Unlock()
|
||||||
@@ -1080,6 +1088,17 @@ func (f *Fetcher) downloadChunkOnce(conn *connection, client *http.Client, buf [
|
|||||||
n = int(remain)
|
n = int(remain)
|
||||||
finished = true
|
finished = true
|
||||||
}
|
}
|
||||||
|
} else if f.meta.Res.Size > 0 {
|
||||||
|
// For non-range downloads with known size, enforce size limit
|
||||||
|
remaining := f.meta.Res.Size - conn.Downloaded
|
||||||
|
if remaining <= 0 {
|
||||||
|
f.connMu.Unlock()
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if remaining < int64(n) {
|
||||||
|
n = int(remaining)
|
||||||
|
finished = true
|
||||||
|
}
|
||||||
}
|
}
|
||||||
writeOffset = conn.Chunk.Begin + conn.Chunk.Downloaded
|
writeOffset = conn.Chunk.Begin + conn.Chunk.Downloaded
|
||||||
f.connMu.Unlock()
|
f.connMu.Unlock()
|
||||||
@@ -1174,6 +1193,22 @@ func (f *Fetcher) runConnectionWithResolveResp(conn *connection) {
|
|||||||
|
|
||||||
n, err := reader.Read(buf)
|
n, err := reader.Read(buf)
|
||||||
if n > 0 {
|
if n > 0 {
|
||||||
|
// For non-range downloads with known size, enforce size limit
|
||||||
|
if !f.meta.Res.Range && f.meta.Res.Size > 0 {
|
||||||
|
f.connMu.Lock()
|
||||||
|
remaining := f.meta.Res.Size - conn.Downloaded
|
||||||
|
if remaining <= 0 {
|
||||||
|
conn.Completed = true
|
||||||
|
conn.State = connCompleted
|
||||||
|
f.connMu.Unlock()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if remaining < int64(n) {
|
||||||
|
n = int(remaining)
|
||||||
|
}
|
||||||
|
f.connMu.Unlock()
|
||||||
|
}
|
||||||
|
|
||||||
f.fileMu.Lock()
|
f.fileMu.Lock()
|
||||||
if f.file != nil {
|
if f.file != nil {
|
||||||
_, writeErr := f.file.WriteAt(buf[:n], conn.Chunk.Downloaded)
|
_, writeErr := f.file.WriteAt(buf[:n], conn.Chunk.Downloaded)
|
||||||
@@ -1191,8 +1226,18 @@ func (f *Fetcher) runConnectionWithResolveResp(conn *connection) {
|
|||||||
}
|
}
|
||||||
f.fileMu.Unlock()
|
f.fileMu.Unlock()
|
||||||
|
|
||||||
|
f.connMu.Lock()
|
||||||
conn.Chunk.Downloaded += int64(n)
|
conn.Chunk.Downloaded += int64(n)
|
||||||
conn.Downloaded += int64(n)
|
conn.Downloaded += int64(n)
|
||||||
|
|
||||||
|
// Check if size limit reached after updating counters
|
||||||
|
if !f.meta.Res.Range && f.meta.Res.Size > 0 && conn.Downloaded >= f.meta.Res.Size {
|
||||||
|
conn.Completed = true
|
||||||
|
conn.State = connCompleted
|
||||||
|
f.connMu.Unlock()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
f.connMu.Unlock()
|
||||||
}
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if err == io.EOF {
|
if err == io.EOF {
|
||||||
@@ -1232,6 +1277,14 @@ func (f *Fetcher) runConnectionFallback(conn *connection) {
|
|||||||
|
|
||||||
f.connMu.Lock()
|
f.connMu.Lock()
|
||||||
conn.State = connConnecting
|
conn.State = connConnecting
|
||||||
|
// For non-range downloads, reset chunk state since server will send from beginning.
|
||||||
|
// Without Range support, each new request restarts the entire transfer.
|
||||||
|
if !f.meta.Res.Range {
|
||||||
|
conn.Chunk.Begin = 0
|
||||||
|
conn.Chunk.End = 0
|
||||||
|
conn.Chunk.Downloaded = 0
|
||||||
|
conn.Downloaded = 0
|
||||||
|
}
|
||||||
f.connMu.Unlock()
|
f.connMu.Unlock()
|
||||||
|
|
||||||
err := func() error {
|
err := func() error {
|
||||||
@@ -1276,6 +1329,17 @@ func (f *Fetcher) runConnectionFallback(conn *connection) {
|
|||||||
|
|
||||||
n, err := reader.Read(buf)
|
n, err := reader.Read(buf)
|
||||||
if n > 0 {
|
if n > 0 {
|
||||||
|
// For non-range downloads with known size, enforce size limit
|
||||||
|
if !f.meta.Res.Range && f.meta.Res.Size > 0 {
|
||||||
|
remaining := f.meta.Res.Size - conn.Downloaded
|
||||||
|
if remaining <= 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if remaining < int64(n) {
|
||||||
|
n = int(remaining)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
f.fileMu.Lock()
|
f.fileMu.Lock()
|
||||||
if f.file != nil {
|
if f.file != nil {
|
||||||
_, writeErr := f.file.WriteAt(buf[:n], conn.Chunk.Downloaded)
|
_, writeErr := f.file.WriteAt(buf[:n], conn.Chunk.Downloaded)
|
||||||
@@ -1475,17 +1539,31 @@ func (f *Fetcher) onDownloadComplete() {
|
|||||||
// Check if all chunks are complete (no remaining bytes)
|
// Check if all chunks are complete (no remaining bytes)
|
||||||
allChunksComplete := true
|
allChunksComplete := true
|
||||||
for _, conn := range f.connections {
|
for _, conn := range f.connections {
|
||||||
if conn.Chunk != nil && conn.Chunk.remain() > 0 && !conn.Completed && conn.State != connCompleted {
|
if !conn.Completed && conn.State != connCompleted {
|
||||||
// This connection has remaining work and isn't done
|
if f.meta.Res.Range {
|
||||||
// Check if it failed with 403 (server limit) - these can be ignored if other connections completed the work
|
// For range downloads, use chunk remain() to check completion
|
||||||
if conn.State == connFailed && conn.failed {
|
if conn.Chunk != nil && conn.Chunk.remain() > 0 {
|
||||||
if re := extractRequestError(conn.lastErr); re != nil && re.Code == 403 {
|
// This connection has remaining work and isn't done
|
||||||
// 403 is server connection limit, check if other connections will complete this chunk
|
// Check if it failed with 403 (server limit) - these can be ignored if other connections completed the work
|
||||||
continue
|
if conn.State == connFailed && conn.failed {
|
||||||
|
if re := extractRequestError(conn.lastErr); re != nil && re.Code == 403 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
}
|
||||||
|
allChunksComplete = false
|
||||||
|
break
|
||||||
}
|
}
|
||||||
|
} else {
|
||||||
|
// For non-range downloads, completion is determined solely by EOF (conn.Completed).
|
||||||
|
// chunk.remain() is unreliable for non-range chunks (Begin=0, End=0).
|
||||||
|
if conn.State == connFailed && conn.failed {
|
||||||
|
if re := extractRequestError(conn.lastErr); re != nil && re.Code == 403 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
}
|
||||||
|
allChunksComplete = false
|
||||||
|
break
|
||||||
}
|
}
|
||||||
allChunksComplete = false
|
|
||||||
break
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -307,6 +307,58 @@ func TestFetcher_DownloadChunked(t *testing.T) {
|
|||||||
downloadNormal(listener, 2, t)
|
downloadNormal(listener, 2, t)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestFetcher_DownloadNoRangePauseResume targets a specific bug where pausing
|
||||||
|
// and resuming a download from a server that doesn't return Content-Length
|
||||||
|
// (and doesn't support Range) causes the downloaded file to grow beyond the
|
||||||
|
// actual file size. Root cause: on resume the server resends from byte 0, but
|
||||||
|
// the old code wrote at the previously accumulated offset, appending a second
|
||||||
|
// copy instead of overwriting. The fix resets the write offset on each non-range
|
||||||
|
// retry so data is always written from the beginning.
|
||||||
|
func TestFetcher_DownloadNoRangePauseResume(t *testing.T) {
|
||||||
|
listener := test.StartTestCustomServer()
|
||||||
|
defer listener.Close()
|
||||||
|
|
||||||
|
fetcher := downloadReady(listener, 1, t)
|
||||||
|
err := fetcher.Start()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
// Let it download some data
|
||||||
|
time.Sleep(time.Millisecond * 50)
|
||||||
|
if err := fetcher.Pause(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
time.Sleep(time.Millisecond * 50)
|
||||||
|
if err := fetcher.Start(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
err = fetcher.Wait()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The critical assertion: the downloaded file must not be larger than
|
||||||
|
// the source file. Before the fix, each resume appended extra data,
|
||||||
|
// causing the file to exceed the actual size (the bug shown in the issue).
|
||||||
|
downloadInfo, err := os.Stat(test.DownloadFile)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
sourceInfo, err := os.Stat(test.BuildFile)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if downloadInfo.Size() > sourceInfo.Size() {
|
||||||
|
t.Errorf("Downloaded file size %d exceeds source file size %d", downloadInfo.Size(), sourceInfo.Size())
|
||||||
|
}
|
||||||
|
|
||||||
|
want := test.FileMd5(test.BuildFile)
|
||||||
|
got := test.FileMd5(test.DownloadFile)
|
||||||
|
if want != got {
|
||||||
|
t.Errorf("Download() got = %v, want %v", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestFetcher_DownloadPost(t *testing.T) {
|
func TestFetcher_DownloadPost(t *testing.T) {
|
||||||
listener := test.StartTestPostServer()
|
listener := test.StartTestPostServer()
|
||||||
defer listener.Close()
|
defer listener.Close()
|
||||||
|
|||||||
Reference in new issue
Block a user