mirror of
https://github.com/GopeedLab/gopeed.git
synced 2026-11-18 14:23:42 +00:00
feat: new ui (#1455)
This commit is contained in:
742 files changed
+59951
-5934
No files matched your search
@@ -857,7 +857,9 @@ func (f *Fetcher) startResolveDownload() {
|
||||
Chunk: newChunk(0, 0), // For non-range, end doesn't matter
|
||||
}
|
||||
conn.ctx, conn.cancel = context.WithCancel(f.ctx)
|
||||
f.connMu.Lock()
|
||||
f.connections = append(f.connections, conn)
|
||||
f.connMu.Unlock()
|
||||
|
||||
f.wg.Add(1)
|
||||
// Use the resolve response directly
|
||||
@@ -1029,7 +1031,9 @@ func (f *Fetcher) runConnection(conn *connection) {
|
||||
buf := make([]byte, 8192)
|
||||
|
||||
retries := 0
|
||||
f.connMu.Lock()
|
||||
conn.retryTimes = 0
|
||||
f.connMu.Unlock()
|
||||
|
||||
for {
|
||||
// Rebuild client with updated fast-fail timeout on retries
|
||||
@@ -1058,7 +1062,9 @@ func (f *Fetcher) runConnection(conn *connection) {
|
||||
|
||||
// Reset counters after a successful help switch
|
||||
retries = 0
|
||||
f.connMu.Lock()
|
||||
conn.retryTimes = 0
|
||||
f.connMu.Unlock()
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -1082,11 +1088,13 @@ func (f *Fetcher) runConnection(conn *connection) {
|
||||
return
|
||||
}
|
||||
|
||||
f.connMu.Lock()
|
||||
if re := extractRequestError(err); re != nil {
|
||||
conn.lastErr = re
|
||||
} else {
|
||||
conn.lastErr = err
|
||||
}
|
||||
f.connMu.Unlock()
|
||||
|
||||
if shouldCountHTTPFailure(err) {
|
||||
if re := extractRequestError(err); re != nil && re.Code == 403 {
|
||||
@@ -1099,14 +1107,15 @@ func (f *Fetcher) runConnection(conn *connection) {
|
||||
}
|
||||
return
|
||||
}
|
||||
conn.retryTimes++
|
||||
f.connMu.Lock()
|
||||
conn.retryTimes++
|
||||
retryTimes := conn.retryTimes
|
||||
conn.failed = true
|
||||
f.connMu.Unlock()
|
||||
if f.slowStart != nil {
|
||||
f.slowStart.onConnectFailed()
|
||||
}
|
||||
if conn.retryTimes >= 3 {
|
||||
if retryTimes >= 3 {
|
||||
f.connMu.Lock()
|
||||
conn.State = connFailed
|
||||
f.connMu.Unlock()
|
||||
@@ -1866,11 +1875,13 @@ func (f *Fetcher) runConnectionFallback(conn *connection) {
|
||||
return
|
||||
}
|
||||
|
||||
f.connMu.Lock()
|
||||
if re := extractRequestError(err); re != nil {
|
||||
conn.lastErr = re
|
||||
} else {
|
||||
conn.lastErr = err
|
||||
}
|
||||
f.connMu.Unlock()
|
||||
|
||||
if shouldCountHTTPFailure(err) {
|
||||
// Immediate fail for server connection limit (403)
|
||||
@@ -1884,7 +1895,9 @@ func (f *Fetcher) runConnectionFallback(conn *connection) {
|
||||
}
|
||||
return
|
||||
}
|
||||
f.connMu.Lock()
|
||||
conn.retryTimes++
|
||||
f.connMu.Unlock()
|
||||
countedRetries++
|
||||
if countedRetries >= 3 {
|
||||
f.connMu.Lock()
|
||||
@@ -2348,22 +2361,72 @@ func (f *Fetcher) Meta() *fetcher.FetcherMeta {
|
||||
return f.meta
|
||||
}
|
||||
|
||||
func (f *Fetcher) Stats() any {
|
||||
func (f *Fetcher) Stats() *fetcher.Stats {
|
||||
f.connMu.Lock()
|
||||
defer f.connMu.Unlock()
|
||||
|
||||
terminal := f.getState() == stateDone
|
||||
prefetched := int64(0)
|
||||
connectionCount := len(f.connections)
|
||||
if connectionCount == 0 {
|
||||
prefetched = f.resolveDataPos.Load()
|
||||
if terminal && f.meta != nil && f.meta.Res != nil && f.meta.Res.Size > prefetched {
|
||||
// A small range response can finish entirely during resolve prefetch,
|
||||
// before a formal connection is created. The fetcher owns this fact, so
|
||||
// expose one aggregate connection here instead of making Downloader
|
||||
// infer protocol state from task-level progress.
|
||||
prefetched = f.meta.Res.Size
|
||||
}
|
||||
if prefetched > 0 {
|
||||
connectionCount = 1
|
||||
}
|
||||
}
|
||||
connectionTotal := f.averageConnectionTotal(connectionCount)
|
||||
statsConnections := make([]*fhttp.StatsConnection, 0)
|
||||
for _, connection := range f.connections {
|
||||
completed := connection.Completed || connection.State == connCompleted
|
||||
// A successful fetcher has no active downloads left. Work stealing and
|
||||
// completion-by-total-size can leave a helper connection's last internal
|
||||
// state non-terminal even though the whole HTTP transfer is finished.
|
||||
if terminal && !connection.failed {
|
||||
completed = true
|
||||
}
|
||||
statsConnections = append(statsConnections, &fhttp.StatsConnection{
|
||||
Downloaded: connection.Downloaded,
|
||||
Completed: connection.Completed,
|
||||
Total: connectionTotal,
|
||||
Completed: completed,
|
||||
Failed: connection.failed,
|
||||
RetryTimes: connection.retryTimes,
|
||||
})
|
||||
}
|
||||
return &fhttp.Stats{
|
||||
Connections: statsConnections,
|
||||
if len(statsConnections) == 0 && prefetched > 0 {
|
||||
statsConnections = append(statsConnections, &fhttp.StatsConnection{
|
||||
Downloaded: prefetched,
|
||||
Total: connectionTotal,
|
||||
Completed: terminal,
|
||||
})
|
||||
}
|
||||
return &fetcher.Stats{
|
||||
Snapshot: &fhttp.Stats{
|
||||
Connections: statsConnections,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// averageConnectionTotal returns the nominal equal share used to compare how
|
||||
// much work each currently visible connection has contributed. HTTP slow start
|
||||
// can add connections over time, so all lanes are recalculated against the
|
||||
// current connection count whenever stats are requested. A remainder is
|
||||
// rounded up to avoid reporting 100% before a connection reaches its share.
|
||||
func (f *Fetcher) averageConnectionTotal(connectionCount int) int64 {
|
||||
if f.meta == nil || f.meta.Res == nil || f.meta.Res.Size <= 0 || connectionCount <= 0 {
|
||||
return 0
|
||||
}
|
||||
total := f.meta.Res.Size / int64(connectionCount)
|
||||
if f.meta.Res.Size%int64(connectionCount) != 0 {
|
||||
total++
|
||||
}
|
||||
return total
|
||||
}
|
||||
|
||||
func (f *Fetcher) Progress() fetcher.Progress {
|
||||
|
||||
@@ -443,7 +443,7 @@ func TestFetcher_DownloadContinue_NoRangeRestart(t *testing.T) {
|
||||
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
|
||||
stats := fetcher.Stats().(*http.Stats)
|
||||
stats := fetcher.Stats().Snapshot.(*http.Stats)
|
||||
if len(stats.Connections) != 1 {
|
||||
t.Fatalf("expected a single non-range connection, got %d", len(stats.Connections))
|
||||
}
|
||||
@@ -461,7 +461,7 @@ func TestFetcher_DownloadContinue_NoRangeRestart(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
finalStats := fetcher.Stats().(*http.Stats)
|
||||
finalStats := fetcher.Stats().Snapshot.(*http.Stats)
|
||||
if len(finalStats.Connections) != 1 {
|
||||
t.Fatalf("expected a single non-range connection after resume, got %d", len(finalStats.Connections))
|
||||
}
|
||||
@@ -518,7 +518,7 @@ func TestFetcher_DownloadIgnoredRangeFallsBackToSequential(t *testing.T) {
|
||||
if f.meta.Res.Range {
|
||||
t.Fatal("expected ignored Range response to disable range mode")
|
||||
}
|
||||
stats := f.Stats().(*http.Stats)
|
||||
stats := f.Stats().Snapshot.(*http.Stats)
|
||||
if len(stats.Connections) != 1 {
|
||||
t.Fatalf("expected one sequential connection, got %d", len(stats.Connections))
|
||||
}
|
||||
@@ -783,7 +783,7 @@ func TestFetcher_SequentialRetryReprobesRange(t *testing.T) {
|
||||
t.Fatalf("Range mode = %v, want %v", f.meta.Res.Range, tt.wantRangeMode)
|
||||
}
|
||||
if tt.wantParallel {
|
||||
stats := f.Stats().(*http.Stats)
|
||||
stats := f.Stats().Snapshot.(*http.Stats)
|
||||
if len(stats.Connections) <= 1 {
|
||||
f.slowStart.mu.Lock()
|
||||
totalLaunched := f.slowStart.totalLaunched
|
||||
@@ -1999,7 +1999,7 @@ func TestFetcher_DownloadResponseBodyReadTimeout(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
stats := fetcher.Stats().(*http.Stats)
|
||||
stats := fetcher.Stats().Snapshot.(*http.Stats)
|
||||
if len(stats.Connections) == 0 {
|
||||
t.Fatalf("expected connections stats for timeout test")
|
||||
}
|
||||
@@ -2045,7 +2045,7 @@ func TestFetcher_Download500Recovery(t *testing.T) {
|
||||
}
|
||||
|
||||
// Verify 500 errors don't count as failures (retryTimes should be 0)
|
||||
stats := fetcher.Stats().(*http.Stats)
|
||||
stats := fetcher.Stats().Snapshot.(*http.Stats)
|
||||
for _, conn := range stats.Connections {
|
||||
if conn.RetryTimes != 0 {
|
||||
t.Errorf("Expected retryTimes to be 0 for 500 errors (exempt), got %d", conn.RetryTimes)
|
||||
@@ -2144,22 +2144,87 @@ func TestFetcher_Stats(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
stats := fetcher.Stats().(*http.Stats)
|
||||
stats := fetcher.Stats().Snapshot.(*http.Stats)
|
||||
// With slow-start strategy, connection count may be less than max if download is fast
|
||||
// Just verify we have at least 1 connection and no more than max
|
||||
if len(stats.Connections) < 1 || len(stats.Connections) > 16 {
|
||||
t.Errorf("Stats() connection count got = %v, want between 1 and 16", len(stats.Connections))
|
||||
}
|
||||
totalDownloaded := int64(0)
|
||||
expectedConnectionTotal := test.BuildSize / int64(len(stats.Connections))
|
||||
if test.BuildSize%int64(len(stats.Connections)) != 0 {
|
||||
expectedConnectionTotal++
|
||||
}
|
||||
for i, conn := range stats.Connections {
|
||||
t.Logf("Connection %d: Downloaded=%d, Completed=%v", i, conn.Downloaded, conn.Completed)
|
||||
totalDownloaded += conn.Downloaded
|
||||
if conn.Total != expectedConnectionTotal {
|
||||
t.Errorf("connection %d total = %d, want shared total %d", i, conn.Total, expectedConnectionTotal)
|
||||
}
|
||||
}
|
||||
if totalDownloaded != test.BuildSize {
|
||||
t.Errorf("Stats() got = %v, want %v", totalDownloaded, test.BuildSize)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetcher_StatsMarksTerminalNonFailedConnectionsComplete(t *testing.T) {
|
||||
fetcher := &Fetcher{
|
||||
meta: &fetcher.FetcherMeta{Res: &base.Resource{Size: 4096, Range: true}},
|
||||
connections: []*connection{
|
||||
{State: connDownloading, Chunk: newChunk(0, 2047), Downloaded: 1024},
|
||||
{State: connCompleted, Chunk: newChunk(2048, 3071), Downloaded: 2048},
|
||||
{State: connFailed, Chunk: newChunk(3072, 4095), Downloaded: 512, failed: true, retryTimes: 3},
|
||||
},
|
||||
}
|
||||
fetcher.setState(stateDone)
|
||||
|
||||
stats := fetcher.Stats().Snapshot.(*http.Stats)
|
||||
if !stats.Connections[0].Completed {
|
||||
t.Fatal("terminal non-failed connection was reported as downloading")
|
||||
}
|
||||
if !stats.Connections[1].Completed {
|
||||
t.Fatal("completed connection was not reported as completed")
|
||||
}
|
||||
if stats.Connections[2].Completed || !stats.Connections[2].Failed {
|
||||
t.Fatal("failed connection lost its failure state")
|
||||
}
|
||||
const averageTotal = int64(1366) // ceil(4096 / 3)
|
||||
for index, connection := range stats.Connections {
|
||||
if connection.Total != averageTotal {
|
||||
t.Fatalf("connection %d total = %d, want shared total %d", index, connection.Total, averageTotal)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetcher_StatsUsesUnknownTotalWithoutResourceSize(t *testing.T) {
|
||||
fetcher := &Fetcher{
|
||||
meta: &fetcher.FetcherMeta{Res: &base.Resource{}},
|
||||
connections: []*connection{
|
||||
{State: connDownloading, Chunk: newChunk(0, 0), Downloaded: 1024},
|
||||
},
|
||||
}
|
||||
|
||||
stats := fetcher.Stats().Snapshot.(*http.Stats)
|
||||
if stats.Connections[0].Total != 0 {
|
||||
t.Fatalf("unknown resource total = %d, want 0", stats.Connections[0].Total)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetcher_StatsExposesCompletedResolvePrefetch(t *testing.T) {
|
||||
fetcher := &Fetcher{meta: &fetcher.FetcherMeta{Res: &base.Resource{Size: 100}}}
|
||||
fetcher.resolveDataPos.Store(100)
|
||||
fetcher.setState(stateDone)
|
||||
|
||||
stats := fetcher.Stats().Snapshot.(*http.Stats)
|
||||
if len(stats.Connections) != 1 {
|
||||
t.Fatalf("connection count = %d, want 1", len(stats.Connections))
|
||||
}
|
||||
connection := stats.Connections[0]
|
||||
if connection.Downloaded != 100 || connection.Total != 100 || !connection.Completed {
|
||||
t.Fatalf("unexpected prefetch connection: %+v", connection)
|
||||
}
|
||||
}
|
||||
|
||||
// TestFetcher_DownloadOneTimeURL tests downloading from a URL that can only be accessed once
|
||||
// This simulates signed URLs or one-time download links that expire after first use
|
||||
func TestFetcher_DownloadOneTimeURL(t *testing.T) {
|
||||
@@ -2261,7 +2326,7 @@ func TestFetcher_SlowStartExpansion(t *testing.T) {
|
||||
}
|
||||
|
||||
// Check final connection count equals maxConns exactly
|
||||
stats := fetcher.Stats().(*http.Stats)
|
||||
stats := fetcher.Stats().Snapshot.(*http.Stats)
|
||||
finalConns := len(stats.Connections)
|
||||
|
||||
// Debug: show connection details and metadata
|
||||
@@ -2407,7 +2472,7 @@ func TestFetcher_AsyncPrefetch(t *testing.T) {
|
||||
}
|
||||
|
||||
// Check stats - should have connections that downloaded remaining data
|
||||
stats := fetcher.Stats().(*http.Stats)
|
||||
stats := fetcher.Stats().Snapshot.(*http.Stats)
|
||||
t.Logf("Final connections: %d", len(stats.Connections))
|
||||
|
||||
prefetchedUsed := fetcher.resolveDataPos.Load()
|
||||
|
||||
Reference in new issue
Block a user