feat: improve http download retry strategy (#951)

This commit is contained in:
Levi authored and GitHub committed 2025-03-07 12:01:52 +08:00
1 parent 046a3ac9f9
commit 03e26dbea8
1 file changed
+21 -19
+21 -19
View File
@@ -47,6 +47,7 @@ type chunk struct {
End int64 End int64
Downloaded int64 Downloaded int64
failed bool
retryTimes int retryTimes int
} }
@@ -265,10 +266,6 @@ func (f *Fetcher) Wait() (err error) {
return <-f.doneCh return <-f.doneCh
} }
type fetchResult struct {
err error
}
func (f *Fetcher) fetch() { func (f *Fetcher) fetch() {
var ctx context.Context var ctx context.Context
ctx, f.cancel = context.WithCancel(context.Background()) ctx, f.cancel = context.WithCancel(context.Background())
@@ -312,12 +309,12 @@ func (f *Fetcher) fetch() {
func (f *Fetcher) fetchChunk(index int, ctx context.Context) (err error) { func (f *Fetcher) fetchChunk(index int, ctx context.Context) (err error) {
chunk := f.chunks[index] chunk := f.chunks[index]
chunk.failed = false
chunk.retryTimes = 0 chunk.retryTimes = 0
var ( var (
client = f.buildClient() client = f.buildClient()
buf = make([]byte, 8192) buf = make([]byte, 8192)
maxRetries = 3
) )
// retry until all remain chunks failed // retry until all remain chunks failed
for { for {
@@ -325,9 +322,22 @@ func (f *Fetcher) fetchChunk(index int, ctx context.Context) (err error) {
if f.meta.Res.Range && chunk.Downloaded >= chunk.End-chunk.Begin+1 { if f.meta.Res.Range && chunk.Downloaded >= chunk.End-chunk.Begin+1 {
return nil return nil
} }
// if all chunks failed, return
if chunk.retryTimes >= maxRetries { if chunk.failed {
return allFailed := true
for _, c := range f.chunks {
if !c.failed {
allFailed = false
break
}
}
if allFailed {
if chunk.retryTimes >= 3 {
return
} else {
chunk.retryTimes++
}
}
} }
var ( var (
@@ -378,18 +388,10 @@ func (f *Fetcher) fetchChunk(index int, ctx context.Context) (err error) {
return return
} }
// retry request after 1 second // retry request after 1 second
chunk.retryTimes = chunk.retryTimes + 1 chunk.failed = true
time.Sleep(time.Second) time.Sleep(time.Second)
continue continue
} }
// if chunk is completed, reset other not complete chunks retry times
if err == nil {
for _, c := range f.chunks {
if c != chunk && c.retryTimes > 0 {
c.retryTimes = 0
}
}
}
break break
} }
return return