mirror of
https://github.com/GopeedLab/gopeed.git
synced 2026-11-09 18:10:29 +00:00
* fix(http): wait for the prefetch goroutine to exit before reusing its file stopPrefetchAndCopyData waited at most 10 s for the Resolve prefetch, but a stalled body read only times out after readTimeout (15 s). The caller then closed and removed the temporary file while the prefetch goroutine could still write to it. Stopping the prefetch now closes the Resolve body, which interrupts a pending read, and waits on a channel closed when the goroutine exits. Pause uses the same path. The stop channel is closed under resolveRespLock so a concurrent Pause and first-connection takeover cannot close it twice. The proxy test helper now calls Setup before Resolve, as the downloader does, so the Resolve request also goes through the proxy. Fixes #1563 * refactor(http): wait on prefetchExitCh in completeFromResolveFallback Replaces the 10 ms polling loop with the exit channel added in the previous commit, so every place that waits for the prefetch uses it. --------- Co-authored-by: Levi <[email protected]>
3668 lines
105 KiB
Go
3668 lines
105 KiB
Go
package http
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"crypto/md5"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"net"
|
|
gohttp "net/http"
|
|
"net/http/httptest"
|
|
"net/url"
|
|
"os"
|
|
"strings"
|
|
"sync"
|
|
"sync/atomic"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/GopeedLab/gopeed/internal/controller"
|
|
"github.com/GopeedLab/gopeed/internal/fetcher"
|
|
"github.com/GopeedLab/gopeed/internal/httpclient/httpclienttest"
|
|
"github.com/GopeedLab/gopeed/internal/test"
|
|
"github.com/GopeedLab/gopeed/pkg/base"
|
|
"github.com/GopeedLab/gopeed/pkg/protocol/http"
|
|
)
|
|
|
|
func TestFetcher_Resolve(t *testing.T) {
|
|
testResolve(test.StartTestFileServer, test.BuildName, t, func(err error) (*base.Resource, error) {
|
|
return &base.Resource{
|
|
Size: test.BuildSize,
|
|
Range: true,
|
|
Files: []*base.FileInfo{
|
|
{
|
|
Name: test.BuildName,
|
|
Size: test.BuildSize,
|
|
},
|
|
},
|
|
}, nil
|
|
})
|
|
testResolve(test.StartTestCustomServer, "disposition", t, func(err error) (*base.Resource, error) {
|
|
return &base.Resource{
|
|
Size: test.BuildSize,
|
|
Range: false,
|
|
Files: []*base.FileInfo{
|
|
{
|
|
Name: test.BuildName,
|
|
Size: test.BuildSize,
|
|
},
|
|
},
|
|
}, nil
|
|
})
|
|
testResolve(test.StartTestCustomServer, "encoded-word", t, func(err error) (*base.Resource, error) {
|
|
return &base.Resource{
|
|
Size: test.BuildSize,
|
|
Range: false,
|
|
Files: []*base.FileInfo{
|
|
{
|
|
Name: test.TestChineseFileName,
|
|
Size: test.BuildSize,
|
|
},
|
|
},
|
|
}, nil
|
|
})
|
|
testResolve(test.StartTestCustomServer, "no-encode", t, func(err error) (*base.Resource, error) {
|
|
return &base.Resource{
|
|
Size: test.BuildSize,
|
|
Range: false,
|
|
Files: []*base.FileInfo{
|
|
{
|
|
Name: test.TestChineseFileName,
|
|
Size: test.BuildSize,
|
|
},
|
|
},
|
|
}, nil
|
|
})
|
|
testResolve(test.StartTestCustomServer, "%E6%B5%8B%E8%AF%95.zip", t, func(err error) (*base.Resource, error) {
|
|
return &base.Resource{
|
|
Size: 0,
|
|
Range: false,
|
|
Files: []*base.FileInfo{
|
|
{
|
|
Name: test.TestChineseFileName,
|
|
Size: 0,
|
|
},
|
|
},
|
|
}, nil
|
|
})
|
|
testResolve(test.StartTestCustomServer, test.BuildName, t, func(err error) (*base.Resource, error) {
|
|
return &base.Resource{
|
|
Size: 0,
|
|
Range: false,
|
|
Files: []*base.FileInfo{
|
|
{
|
|
Name: test.BuildName,
|
|
Size: 0,
|
|
},
|
|
},
|
|
}, nil
|
|
})
|
|
// Test mixed encoding Content-Disposition where mime.ParseMediaType fails
|
|
// due to invalid characters, but filename*= contains the correct UTF-8 encoded name
|
|
testResolve(test.StartTestCustomServer, "mixed-encoding", t, func(err error) (*base.Resource, error) {
|
|
return &base.Resource{
|
|
Size: test.BuildSize,
|
|
Range: false,
|
|
Files: []*base.FileInfo{
|
|
{
|
|
Name: test.TestChineseFileName,
|
|
Size: test.BuildSize,
|
|
},
|
|
},
|
|
}, nil
|
|
})
|
|
// Test filename*= only (RFC 5987 format)
|
|
testResolve(test.StartTestCustomServer, "filename-star", t, func(err error) (*base.Resource, error) {
|
|
return &base.Resource{
|
|
Size: test.BuildSize,
|
|
Range: false,
|
|
Files: []*base.FileInfo{
|
|
{
|
|
Name: test.TestChineseFileName,
|
|
Size: test.BuildSize,
|
|
},
|
|
},
|
|
}, nil
|
|
})
|
|
// Test GBK-encoded filename (common on Chinese Windows servers)
|
|
// Before fix: "测试.zip" sent as GBK bytes -> parsed as "²âÊÔ.zip" (garbled)
|
|
// After fix: correctly decoded back to "测试.zip"
|
|
testResolve(test.StartTestCustomServer, "gbk-encoded", t, func(err error) (*base.Resource, error) {
|
|
return &base.Resource{
|
|
Size: test.BuildSize,
|
|
Range: false,
|
|
Files: []*base.FileInfo{
|
|
{
|
|
Name: test.TestChineseFileName,
|
|
Size: test.BuildSize,
|
|
},
|
|
},
|
|
}, nil
|
|
})
|
|
// Test filename with plus signs (e.g., C++ Primer)
|
|
// Before fix: %2B decoded to space -> "C++ Primer" became "C Primer"
|
|
// After fix: %2B correctly decoded to + -> "C++ Primer Plus.mobi"
|
|
testResolve(test.StartTestCustomServer, "plus-sign-encoded", t, func(err error) (*base.Resource, error) {
|
|
return &base.Resource{
|
|
Size: test.BuildSize,
|
|
Range: false,
|
|
Files: []*base.FileInfo{
|
|
{
|
|
Name: "C++ Primer Plus.mobi",
|
|
Size: test.BuildSize,
|
|
},
|
|
},
|
|
}, nil
|
|
})
|
|
// Test filename with plus sign in URL path
|
|
// Before fix: %2B decoded to space
|
|
// After fix: %2B correctly decoded to +
|
|
testResolve(test.StartTestCustomServer, "C%2B%2B%20Primer.txt", t, func(err error) (*base.Resource, error) {
|
|
return &base.Resource{
|
|
Size: 0,
|
|
Range: false,
|
|
Files: []*base.FileInfo{
|
|
{
|
|
Name: "C++ Primer.txt",
|
|
Size: 0,
|
|
},
|
|
},
|
|
}, nil
|
|
})
|
|
// Test filename with HTML-encoded ampersand (fixes issue with & being truncated)
|
|
// Before fix: "查询处理&优化.pptx" -> "查询处理&" (truncated at semicolon)
|
|
// After fix: correctly decoded to "查询处理&优化.pptx"
|
|
testResolve(test.StartTestCustomServer, "ampersand-encoded", t, func(err error) (*base.Resource, error) {
|
|
return &base.Resource{
|
|
Size: test.BuildSize,
|
|
Range: false,
|
|
Files: []*base.FileInfo{
|
|
{
|
|
Name: "查询处理&优化.pptx",
|
|
Size: test.BuildSize,
|
|
},
|
|
},
|
|
}, nil
|
|
})
|
|
// Test unquoted filename with HTML-encoded ampersand
|
|
testResolve(test.StartTestCustomServer, "ampersand-unquoted", t, func(err error) (*base.Resource, error) {
|
|
return &base.Resource{
|
|
Size: test.BuildSize,
|
|
Range: false,
|
|
Files: []*base.FileInfo{
|
|
{
|
|
Name: "test&file.txt",
|
|
Size: test.BuildSize,
|
|
},
|
|
},
|
|
}, nil
|
|
})
|
|
// Test URL without file path - should use domain/host as filename
|
|
testResolve(test.StartTestCustomServer, "", t, func(err error) (*base.Resource, error) {
|
|
return &base.Resource{
|
|
Size: 0,
|
|
Range: false,
|
|
Files: []*base.FileInfo{
|
|
{
|
|
Name: "127.0.0.1",
|
|
Size: 0,
|
|
},
|
|
},
|
|
}, nil
|
|
})
|
|
// Test 403 Forbidden response handling
|
|
testResolve(test.StartTestCustomServer, "forbidden", t, func(err error) (*base.Resource, error) {
|
|
requestError := extractRequestError(err)
|
|
if requestError != nil && requestError.Code == 403 {
|
|
return nil, nil
|
|
}
|
|
return nil, err
|
|
})
|
|
}
|
|
|
|
func TestFetcher_ResolveFilenameFromContentType(t *testing.T) {
|
|
server := httptest.NewServer(gohttp.HandlerFunc(func(w gohttp.ResponseWriter, r *gohttp.Request) {
|
|
w.Header().Set("Content-Type", "image/webp")
|
|
w.WriteHeader(gohttp.StatusOK)
|
|
}))
|
|
defer server.Close()
|
|
|
|
fetcher := buildFetcher()
|
|
defer fetcher.Pause()
|
|
|
|
const opaqueName = "OIP-C.u5nEXTXND3PJF0gNTgw5YgHaFP"
|
|
fetcher.Meta().Opts = &base.Options{
|
|
Name: test.DownloadName,
|
|
Path: test.Dir,
|
|
}
|
|
err := fetcher.Resolve(&base.Request{URL: server.URL + "/" + opaqueName})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
if got := fetcher.meta.Res.Files[0].Name; got != opaqueName+".webp" {
|
|
t.Fatalf("resolved filename = %q, want %q", got, opaqueName+".webp")
|
|
}
|
|
}
|
|
|
|
func TestFetcher_ResolveWithHostHeader(t *testing.T) {
|
|
listener := test.StartTestHostHeaderServer()
|
|
defer listener.Close()
|
|
|
|
fetcher := buildFetcher()
|
|
fetcher.Meta().Opts = &base.Options{
|
|
Name: test.DownloadName,
|
|
Path: test.Dir,
|
|
}
|
|
err := fetcher.Resolve(&base.Request{
|
|
URL: "http://" + listener.Addr().String() + "/",
|
|
Extra: &http.ReqExtra{
|
|
Header: map[string]string{
|
|
"Host": "test",
|
|
},
|
|
},
|
|
})
|
|
// The server should return 400 for invalid Host header
|
|
if err == nil || !strings.Contains(err.Error(), "400") {
|
|
t.Errorf("Resolve() got = %v, want error containing 400", err)
|
|
}
|
|
}
|
|
|
|
func TestFetcher_ResolveWithInvalidHeader(t *testing.T) {
|
|
listener := test.StartTestCustomServer()
|
|
defer listener.Close()
|
|
|
|
fetcher := buildFetcher()
|
|
defer fetcher.Pause() // Close the resolve response to allow server shutdown
|
|
fetcher.Meta().Opts = &base.Options{
|
|
Name: test.DownloadName,
|
|
Path: test.Dir,
|
|
}
|
|
err := fetcher.Resolve(&base.Request{
|
|
URL: "http://" + listener.Addr().String() + "/",
|
|
Extra: &http.ReqExtra{
|
|
Header: map[string]string{
|
|
"Referer": "\rtest",
|
|
},
|
|
},
|
|
})
|
|
// Invalid header with \r should be sanitized by Go's http client, allowing the request to succeed
|
|
if err != nil {
|
|
t.Errorf("Resolve() got = %v, want nil (invalid headers should be sanitized)", err)
|
|
}
|
|
}
|
|
|
|
func TestFetcher_ResolveAutomaticallyFallsBackToBrowserFingerprint(t *testing.T) {
|
|
server := httpclienttest.NewFingerprintServer(t, httpclienttest.FingerprintServerOptions{
|
|
RequiredProfile: httpclienttest.ProfileAnyBrowser,
|
|
Handler: gohttp.HandlerFunc(func(w gohttp.ResponseWriter, _ *gohttp.Request) {
|
|
w.Header().Set(base.HttpHeaderContentLength, "4")
|
|
w.Header().Set(base.HttpHeaderContentDisposition, `attachment; filename="file.bin"`)
|
|
_, _ = w.Write([]byte("data"))
|
|
}),
|
|
})
|
|
|
|
fetcher := buildFetcher()
|
|
defer fetcher.Pause()
|
|
fetcher.Meta().Opts = &base.Options{
|
|
Name: test.DownloadName,
|
|
Path: test.Dir,
|
|
}
|
|
err := fetcher.Resolve(&base.Request{
|
|
URL: server.URL + "/file.bin",
|
|
SkipVerifyCert: true,
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("Resolve() error = %v", err)
|
|
}
|
|
if fetcher.meta.Res == nil || len(fetcher.meta.Res.Files) != 1 {
|
|
t.Fatalf("resource = %+v, want one file", fetcher.meta.Res)
|
|
}
|
|
if fetcher.meta.Res.Files[0].Name != "file.bin" {
|
|
t.Fatalf("file name = %q, want file.bin", fetcher.meta.Res.Files[0].Name)
|
|
}
|
|
}
|
|
|
|
func TestFetcherSharesAndClearsImpersonationSession(t *testing.T) {
|
|
var requests atomic.Int32
|
|
server := httptest.NewServer(gohttp.HandlerFunc(func(w gohttp.ResponseWriter, request *gohttp.Request) {
|
|
requests.Add(1)
|
|
if request.Header.Get("Sec-Ch-Ua") == "" {
|
|
w.WriteHeader(gohttp.StatusForbidden)
|
|
return
|
|
}
|
|
w.WriteHeader(gohttp.StatusOK)
|
|
}))
|
|
defer server.Close()
|
|
|
|
fetcher := buildFetcher()
|
|
fetcher.meta.Req = &base.Request{URL: server.URL}
|
|
|
|
for range 2 {
|
|
client := fetcher.buildClient()
|
|
response, err := client.Get(server.URL)
|
|
if err != nil {
|
|
t.Fatalf("Get() error = %v", err)
|
|
}
|
|
response.Body.Close()
|
|
client.CloseIdleConnections()
|
|
}
|
|
if requests.Load() != 3 {
|
|
t.Fatalf("requests before completion = %d, want 3", requests.Load())
|
|
}
|
|
|
|
if err := fetcher.Close(); err != nil {
|
|
t.Fatalf("Close() error = %v", err)
|
|
}
|
|
client := fetcher.buildClient()
|
|
response, err := client.Get(server.URL)
|
|
if err != nil {
|
|
t.Fatalf("Get() after close error = %v", err)
|
|
}
|
|
response.Body.Close()
|
|
client.CloseIdleConnections()
|
|
if requests.Load() != 5 {
|
|
t.Fatalf("requests after close = %d, want 5", requests.Load())
|
|
}
|
|
}
|
|
|
|
func testResolve(startTestServer func() net.Listener, path string, t *testing.T, wantFn func(error) (*base.Resource, error)) {
|
|
listener := startTestServer()
|
|
defer listener.Close()
|
|
fetcher := buildFetcher()
|
|
defer fetcher.Pause() // Close the resolve response to allow server shutdown
|
|
fetcher.Meta().Opts = &base.Options{
|
|
Name: test.DownloadName,
|
|
Path: test.Dir,
|
|
}
|
|
err := fetcher.Resolve(&base.Request{
|
|
URL: "http://" + listener.Addr().String() + "/" + path,
|
|
})
|
|
want, err := wantFn(err)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if want != nil && !test.AssertResourceEqual(want, fetcher.meta.Res) {
|
|
t.Errorf("Resolve() got = %+v, want %+v", fetcher.meta.Res, want)
|
|
}
|
|
}
|
|
|
|
func TestFetcher_DownloadNormal(t *testing.T) {
|
|
listener := test.StartTestFileServer()
|
|
defer listener.Close()
|
|
|
|
downloadNormal(listener, 1, t)
|
|
downloadNormal(listener, 5, t)
|
|
downloadNormal(listener, 8, t)
|
|
downloadNormal(listener, 16, t)
|
|
}
|
|
|
|
func TestFetcher_DownloadContinue(t *testing.T) {
|
|
listener := test.StartTestFileServer()
|
|
defer listener.Close()
|
|
|
|
downloadContinue(listener, 1, t)
|
|
downloadContinue(listener, 5, t)
|
|
downloadContinue(listener, 8, t)
|
|
downloadContinue(listener, 16, t)
|
|
}
|
|
|
|
func TestFetcher_PauseBeforeStartDiscardsPrefetchProgress(t *testing.T) {
|
|
payload := make([]byte, 32*1024*1024)
|
|
for i := range payload {
|
|
payload[i] = byte(i % 251)
|
|
}
|
|
prefetchServed := make(chan struct{}, 1)
|
|
server := httptest.NewServer(gohttp.HandlerFunc(func(w gohttp.ResponseWriter, r *gohttp.Request) {
|
|
if r.Header.Get(base.HttpHeaderRange) == "" {
|
|
w.Header().Set(base.HttpHeaderAcceptRanges, base.HttpHeaderBytes)
|
|
w.Header().Set(base.HttpHeaderContentLength, fmt.Sprintf("%d", len(payload)))
|
|
w.WriteHeader(gohttp.StatusOK)
|
|
if _, err := w.Write(payload); err == nil {
|
|
prefetchServed <- struct{}{}
|
|
}
|
|
return
|
|
}
|
|
gohttp.ServeContent(w, r, test.BuildName, time.Unix(0, 0), bytes.NewReader(payload))
|
|
}))
|
|
defer server.Close()
|
|
|
|
fetcher := buildFetcher()
|
|
fetcher.Meta().Opts = &base.Options{
|
|
Path: t.TempDir(),
|
|
Name: test.DownloadName,
|
|
Extra: &http.OptsExtra{
|
|
Connections: 2,
|
|
},
|
|
}
|
|
if err := fetcher.Resolve(&base.Request{URL: server.URL + "/" + test.BuildName}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
select {
|
|
case <-prefetchServed:
|
|
case <-time.After(5 * time.Second):
|
|
t.Fatal("timed out waiting for resolve prefetch response")
|
|
}
|
|
|
|
if err := fetcher.Pause(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := fetcher.Start(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := fetcher.Wait(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
want := fmt.Sprintf("%x", md5.Sum(payload))
|
|
got := test.FileMd5(fetcher.Meta().SingleFilepath())
|
|
if want != got {
|
|
t.Fatalf("download after pause before start got MD5 %s, want %s", got, want)
|
|
}
|
|
}
|
|
|
|
func TestFetcher_DownloadContinue_NoRangeRestart(t *testing.T) {
|
|
listener := test.StartTestNoRangeSlowServer(time.Millisecond)
|
|
defer listener.Close()
|
|
|
|
fetcher := downloadReady(listener, 4, t)
|
|
if err := fetcher.Start(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
time.Sleep(20 * time.Millisecond)
|
|
|
|
stats := fetcher.Stats().Snapshot.(*http.Stats)
|
|
if len(stats.Connections) != 1 {
|
|
t.Fatalf("expected a single non-range connection, got %d", len(stats.Connections))
|
|
}
|
|
if stats.Connections[0].Downloaded <= 0 || stats.Connections[0].Downloaded >= test.BuildSize {
|
|
t.Fatalf("expected partial download before pause, got %d", stats.Connections[0].Downloaded)
|
|
}
|
|
|
|
if err := fetcher.Pause(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := fetcher.Start(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := fetcher.Wait(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
finalStats := fetcher.Stats().Snapshot.(*http.Stats)
|
|
if len(finalStats.Connections) != 1 {
|
|
t.Fatalf("expected a single non-range connection after resume, got %d", len(finalStats.Connections))
|
|
}
|
|
if finalStats.Connections[0].Downloaded != test.BuildSize {
|
|
t.Fatalf("downloaded bytes should restart cleanly: got %d, want %d", finalStats.Connections[0].Downloaded, test.BuildSize)
|
|
}
|
|
if total := fetcher.Progress().TotalDownloaded(); total != test.BuildSize {
|
|
t.Fatalf("progress total = %d, want %d", total, test.BuildSize)
|
|
}
|
|
|
|
want := test.FileMd5(test.BuildFile)
|
|
got := test.FileMd5(test.DownloadFile)
|
|
if want != got {
|
|
t.Errorf("Download() got = %v, want %v", got, want)
|
|
}
|
|
}
|
|
|
|
func TestFetcher_DownloadNoRangeLeavesNoStaleCompletion(t *testing.T) {
|
|
listener := test.StartTestNoRangeSlowServer(0)
|
|
defer listener.Close()
|
|
|
|
f := downloadReady(listener, 1, t).(*Fetcher)
|
|
if err := f.Start(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := f.Wait(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
select {
|
|
case err := <-f.doneCh:
|
|
t.Fatalf("stale completion signal: %v", err)
|
|
case <-time.After(100 * time.Millisecond):
|
|
}
|
|
}
|
|
|
|
func TestFetcher_DownloadIgnoredRangeFallsBackToSequential(t *testing.T) {
|
|
for _, connections := range []int{1, 4} {
|
|
t.Run(fmt.Sprintf("connections_%d", connections), func(t *testing.T) {
|
|
os.Remove(test.DownloadFile)
|
|
t.Cleanup(func() { os.Remove(test.DownloadFile) })
|
|
|
|
listener := test.StartTestIgnoredRangeServer(time.Millisecond)
|
|
defer listener.Close()
|
|
|
|
f := downloadReady(listener, connections, t).(*Fetcher)
|
|
if err := f.Start(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := f.Wait(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
if f.meta.Res.Range {
|
|
t.Fatal("expected ignored Range response to disable range mode")
|
|
}
|
|
stats := f.Stats().Snapshot.(*http.Stats)
|
|
if len(stats.Connections) != 1 {
|
|
t.Fatalf("expected one sequential connection, got %d", len(stats.Connections))
|
|
}
|
|
if got, want := test.FileMd5(test.DownloadFile), test.FileMd5(test.BuildFile); got != want {
|
|
t.Fatalf("download checksum = %s, want %s", got, want)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestFetcher_SequentialRetryReprobesRange(t *testing.T) {
|
|
payload := make([]byte, 4*1024*1024)
|
|
for i := range payload {
|
|
payload[i] = byte(i % 251)
|
|
}
|
|
shortReplacement := make([]byte, 2*1024*1024)
|
|
chunkedReplacement := make([]byte, 5*1024*1024)
|
|
for i := range shortReplacement {
|
|
shortReplacement[i] = byte((i*3 + 17) % 251)
|
|
}
|
|
for i := range chunkedReplacement {
|
|
chunkedReplacement[i] = byte((i*7 + 29) % 251)
|
|
}
|
|
|
|
const (
|
|
etagA = `"range-reprobe-a"`
|
|
etagB = `"range-reprobe-b"`
|
|
weakETag = `W/"range-reprobe-b"`
|
|
lastModified = "Wed, 21 Oct 2015 07:28:00 GMT"
|
|
laterDate = "Wed, 21 Oct 2015 07:28:02 GMT"
|
|
failedSize = 128 * 1024
|
|
)
|
|
|
|
tests := []struct {
|
|
name string
|
|
resolveETag string
|
|
ignoredETag string
|
|
lastModified string
|
|
date string
|
|
wantIfRange string
|
|
resumeStatus int
|
|
wantRangeMode bool
|
|
wantParallel bool
|
|
wantRangeRequest int32
|
|
wantFullRestart bool
|
|
replacement []byte
|
|
fullRestartReplacement []byte
|
|
chunked200 bool
|
|
}{
|
|
{
|
|
name: "ignored response replaces Resolve ETag",
|
|
resolveETag: etagA,
|
|
ignoredETag: etagB,
|
|
wantIfRange: etagB,
|
|
resumeStatus: gohttp.StatusPartialContent,
|
|
wantRangeMode: true,
|
|
wantParallel: true,
|
|
wantRangeRequest: 2,
|
|
},
|
|
{
|
|
name: "strong Last-Modified resumes with 206",
|
|
lastModified: lastModified,
|
|
date: laterDate,
|
|
wantIfRange: lastModified,
|
|
resumeStatus: gohttp.StatusPartialContent,
|
|
wantRangeMode: true,
|
|
wantParallel: true,
|
|
wantRangeRequest: 2,
|
|
},
|
|
{
|
|
name: "missing validator restarts from zero",
|
|
resumeStatus: gohttp.StatusPartialContent,
|
|
wantRangeRequest: 1,
|
|
wantFullRestart: true,
|
|
fullRestartReplacement: shortReplacement,
|
|
},
|
|
{
|
|
name: "weak ETag blocks Last-Modified fallback",
|
|
ignoredETag: weakETag,
|
|
lastModified: lastModified,
|
|
date: laterDate,
|
|
resumeStatus: gohttp.StatusPartialContent,
|
|
wantRangeRequest: 1,
|
|
wantFullRestart: true,
|
|
},
|
|
{
|
|
name: "same-second Last-Modified restarts from zero",
|
|
lastModified: lastModified,
|
|
date: lastModified,
|
|
resumeStatus: gohttp.StatusPartialContent,
|
|
wantRangeRequest: 1,
|
|
wantFullRestart: true,
|
|
},
|
|
{
|
|
name: "guarded probe returning shorter 200 replaces resource",
|
|
ignoredETag: etagB,
|
|
wantIfRange: etagB,
|
|
resumeStatus: gohttp.StatusOK,
|
|
wantRangeRequest: 2,
|
|
replacement: shortReplacement,
|
|
},
|
|
{
|
|
name: "guarded probe returning larger chunked 200 replaces resource",
|
|
ignoredETag: etagB,
|
|
wantIfRange: etagB,
|
|
resumeStatus: gohttp.StatusOK,
|
|
wantRangeRequest: 2,
|
|
replacement: chunkedReplacement,
|
|
chunked200: true,
|
|
},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
var rangeRequests atomic.Int32
|
|
var fullRequests atomic.Int32
|
|
var resumedAt atomic.Int64
|
|
var sawIfRange atomic.Bool
|
|
var workerValidatorMismatch atomic.Bool
|
|
|
|
server := httptest.NewServer(gohttp.HandlerFunc(func(w gohttp.ResponseWriter, r *gohttp.Request) {
|
|
rangeHeader := r.Header.Get(base.HttpHeaderRange)
|
|
w.Header().Set(base.HttpHeaderAcceptRanges, base.HttpHeaderBytes)
|
|
|
|
if rangeHeader == "" {
|
|
requestNumber := fullRequests.Add(1)
|
|
responsePayload := payload
|
|
if requestNumber > 1 && tt.fullRestartReplacement != nil {
|
|
responsePayload = tt.fullRestartReplacement
|
|
}
|
|
if requestNumber == 1 && tt.resolveETag != "" {
|
|
w.Header().Set(base.HttpHeaderETag, tt.resolveETag)
|
|
}
|
|
w.Header().Set(base.HttpHeaderContentLength, fmt.Sprintf("%d", len(responsePayload)))
|
|
w.WriteHeader(gohttp.StatusOK)
|
|
if requestNumber == 1 {
|
|
if flusher, ok := w.(gohttp.Flusher); ok {
|
|
flusher.Flush()
|
|
}
|
|
for offset := 0; offset < len(payload); offset += 8 * 1024 {
|
|
end := min(offset+8*1024, len(payload))
|
|
if _, err := w.Write(payload[offset:end]); err != nil {
|
|
return
|
|
}
|
|
time.Sleep(time.Millisecond)
|
|
}
|
|
return
|
|
}
|
|
_, _ = w.Write(responsePayload)
|
|
return
|
|
}
|
|
|
|
requestNumber := rangeRequests.Add(1)
|
|
if requestNumber == 1 {
|
|
// Simulate the real origin: it first ignores Range, then drops the
|
|
// sequential response after a useful prefix has been written.
|
|
if tt.ignoredETag != "" {
|
|
w.Header().Set(base.HttpHeaderETag, tt.ignoredETag)
|
|
}
|
|
if tt.lastModified != "" {
|
|
w.Header().Set(base.HttpHeaderLastModified, tt.lastModified)
|
|
}
|
|
if tt.date != "" {
|
|
w.Header().Set("Date", tt.date)
|
|
}
|
|
w.Header().Set(base.HttpHeaderContentLength, fmt.Sprintf("%d", len(payload)))
|
|
w.WriteHeader(gohttp.StatusOK)
|
|
if flusher, ok := w.(gohttp.Flusher); ok {
|
|
flusher.Flush()
|
|
}
|
|
_, _ = w.Write(payload[:failedSize])
|
|
return
|
|
}
|
|
|
|
var start, end int64
|
|
if _, err := fmt.Sscanf(rangeHeader, "bytes=%d-%d", &start, &end); err != nil {
|
|
w.WriteHeader(gohttp.StatusBadRequest)
|
|
return
|
|
}
|
|
if requestNumber == 2 {
|
|
resumedAt.Store(start)
|
|
sawIfRange.Store(r.Header.Get(base.HttpHeaderIfRange) == tt.wantIfRange)
|
|
} else if tt.wantParallel && r.Header.Get(base.HttpHeaderIfRange) != tt.wantIfRange {
|
|
workerValidatorMismatch.Store(true)
|
|
}
|
|
if requestNumber == 2 && tt.resumeStatus == gohttp.StatusOK {
|
|
responsePayload := payload
|
|
if tt.replacement != nil {
|
|
responsePayload = tt.replacement
|
|
}
|
|
w.Header().Set(base.HttpHeaderETag, `"replacement"`)
|
|
if !tt.chunked200 {
|
|
w.Header().Set(base.HttpHeaderContentLength, fmt.Sprintf("%d", len(responsePayload)))
|
|
}
|
|
w.WriteHeader(gohttp.StatusOK)
|
|
if tt.chunked200 {
|
|
w.(gohttp.Flusher).Flush()
|
|
}
|
|
for offset := 0; offset < len(responsePayload); offset += 8 * 1024 {
|
|
end := min(offset+8*1024, len(responsePayload))
|
|
if _, err := w.Write(responsePayload[offset:end]); err != nil {
|
|
return
|
|
}
|
|
}
|
|
return
|
|
}
|
|
w.Header().Set(base.HttpHeaderContentRange,
|
|
fmt.Sprintf("bytes %d-%d/%d", start, end, len(payload)))
|
|
w.Header().Set(base.HttpHeaderContentLength, fmt.Sprintf("%d", end-start+1))
|
|
w.WriteHeader(gohttp.StatusPartialContent)
|
|
if requestNumber == 2 && tt.wantParallel {
|
|
w.(gohttp.Flusher).Flush()
|
|
deadline := time.Now().Add(2 * time.Second)
|
|
for rangeRequests.Load() < 3 && time.Now().Before(deadline) {
|
|
time.Sleep(10 * time.Millisecond)
|
|
}
|
|
}
|
|
for offset := start; offset <= end; offset += 8 * 1024 {
|
|
chunkEnd := min(offset+8*1024, end+1)
|
|
if _, err := w.Write(payload[offset:chunkEnd]); err != nil {
|
|
return
|
|
}
|
|
time.Sleep(time.Millisecond)
|
|
}
|
|
}))
|
|
defer server.Close()
|
|
|
|
f := buildFetcher()
|
|
f.Meta().Opts = &base.Options{
|
|
Path: t.TempDir(),
|
|
Name: "dynamic-range.data",
|
|
Extra: &http.OptsExtra{
|
|
Connections: 4,
|
|
},
|
|
}
|
|
if err := f.Resolve(&base.Request{URL: server.URL + "/dynamic-range.data"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := f.Start(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := f.Wait(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
if got := rangeRequests.Load(); got < tt.wantRangeRequest {
|
|
t.Fatalf("Range requests = %d, want at least %d", got, tt.wantRangeRequest)
|
|
}
|
|
got, err := os.ReadFile(f.meta.SingleFilepath())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if tt.wantFullRestart && fullRequests.Load() < 2 {
|
|
t.Fatal("missing validator did not issue a full restart request")
|
|
}
|
|
if tt.wantIfRange != "" && resumedAt.Load() != failedSize {
|
|
t.Fatalf("resume Range started at %d, want %d", resumedAt.Load(), failedSize)
|
|
}
|
|
if tt.wantIfRange != "" && !sawIfRange.Load() {
|
|
t.Fatalf("resume probe did not include If-Range %q", tt.wantIfRange)
|
|
}
|
|
if f.meta.Res.Range != tt.wantRangeMode {
|
|
t.Fatalf("Range mode = %v, want %v", f.meta.Res.Range, tt.wantRangeMode)
|
|
}
|
|
if tt.wantParallel {
|
|
stats := f.Stats().Snapshot.(*http.Stats)
|
|
if len(stats.Connections) <= 1 {
|
|
f.slowStart.mu.Lock()
|
|
totalLaunched := f.slowStart.totalLaunched
|
|
batchPending := f.slowStart.batchPending
|
|
batchReady := f.slowStart.batchReady
|
|
maxConnections := f.slowStart.maxConnections
|
|
nextBatchSize := f.slowStart.nextBatchSize
|
|
paused := f.slowStart.paused
|
|
f.slowStart.mu.Unlock()
|
|
t.Fatalf("successful re-probe kept %d connection, want expansion (requests=%d state=%d launched=%d pending=%d ready=%d max=%d next=%d paused=%v)", len(stats.Connections), rangeRequests.Load(), f.getState(), totalLaunched, batchPending, batchReady, maxConnections, nextBatchSize, paused)
|
|
}
|
|
if workerValidatorMismatch.Load() {
|
|
t.Fatalf("expanded Range worker omitted pinned If-Range %q", tt.wantIfRange)
|
|
}
|
|
}
|
|
wantPayload := payload
|
|
if tt.replacement != nil {
|
|
wantPayload = tt.replacement
|
|
} else if tt.fullRestartReplacement != nil {
|
|
wantPayload = tt.fullRestartReplacement
|
|
}
|
|
if !bytes.Equal(got, wantPayload) {
|
|
t.Fatal("downloaded file differs after sequential recovery")
|
|
}
|
|
if f.meta.Res.Size != int64(len(wantPayload)) || f.meta.Res.Files[0].Size != int64(len(wantPayload)) {
|
|
t.Fatalf("resource size = %d/%d, want %d", f.meta.Res.Size, f.meta.Res.Files[0].Size, len(wantPayload))
|
|
}
|
|
if got := f.Progress().TotalDownloaded(); got != int64(len(wantPayload)) {
|
|
t.Fatalf("progress = %d, want %d", got, len(wantPayload))
|
|
}
|
|
info, err := os.Stat(f.meta.SingleFilepath())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if got := info.Size(); got != int64(len(wantPayload)) {
|
|
t.Fatalf("file size = %d, want %d", got, len(wantPayload))
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestFetcher_InitialIgnoredRangeUsesResponseSize(t *testing.T) {
|
|
resolvePayload := bytes.Repeat([]byte("resolve"), 64*1024)
|
|
replacement := bytes.Repeat([]byte("replacement"), 24*1024)
|
|
var rangeRequests atomic.Int32
|
|
rangeStarted := make(chan struct{})
|
|
releaseResolve := sync.OnceFunc(func() { close(rangeStarted) })
|
|
|
|
server := httptest.NewServer(gohttp.HandlerFunc(func(w gohttp.ResponseWriter, r *gohttp.Request) {
|
|
w.Header().Set(base.HttpHeaderAcceptRanges, base.HttpHeaderBytes)
|
|
if r.Header.Get(base.HttpHeaderRange) == "" {
|
|
w.Header().Set(base.HttpHeaderContentLength, fmt.Sprintf("%d", len(resolvePayload)))
|
|
w.WriteHeader(gohttp.StatusOK)
|
|
w.(gohttp.Flusher).Flush()
|
|
// Keep Resolve prefetch incomplete until the Range request starts.
|
|
// Otherwise Start may reuse the completed body and skip this scenario.
|
|
select {
|
|
case <-rangeStarted:
|
|
case <-r.Context().Done():
|
|
return
|
|
}
|
|
_, _ = w.Write(resolvePayload)
|
|
return
|
|
}
|
|
|
|
rangeRequests.Add(1)
|
|
releaseResolve()
|
|
w.Header().Set(base.HttpHeaderContentLength, fmt.Sprintf("%d", len(replacement)))
|
|
w.WriteHeader(gohttp.StatusOK)
|
|
_, _ = w.Write(replacement)
|
|
}))
|
|
defer server.Close()
|
|
defer releaseResolve()
|
|
|
|
f := buildFetcher()
|
|
f.Meta().Opts = &base.Options{
|
|
Path: t.TempDir(),
|
|
Name: "ignored-range.data",
|
|
Extra: &http.OptsExtra{
|
|
Connections: 4,
|
|
},
|
|
}
|
|
if err := f.Resolve(&base.Request{URL: server.URL + "/ignored-range.data"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := f.Start(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := f.Wait(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
got, err := os.ReadFile(f.meta.SingleFilepath())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !bytes.Equal(got, replacement) {
|
|
t.Fatal("initial ignored-Range response did not replace the resolved representation")
|
|
}
|
|
if rangeRequests.Load() != 1 {
|
|
t.Fatalf("Range requests = %d, want 1", rangeRequests.Load())
|
|
}
|
|
wantSize := int64(len(replacement))
|
|
if f.meta.Res.Size != wantSize || f.meta.Res.Files[0].Size != wantSize {
|
|
t.Fatalf("resource size = %d/%d, want %d", f.meta.Res.Size, f.meta.Res.Files[0].Size, wantSize)
|
|
}
|
|
if got := f.Progress().TotalDownloaded(); got != wantSize {
|
|
t.Fatalf("progress = %d, want %d", got, wantSize)
|
|
}
|
|
info, err := os.Stat(f.meta.SingleFilepath())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if got := info.Size(); got != wantSize {
|
|
t.Fatalf("file size = %d, want %d", got, wantSize)
|
|
}
|
|
}
|
|
|
|
func TestFetcher_InterruptedChunkedReplacementRestartsFromZero(t *testing.T) {
|
|
payload := bytes.Repeat([]byte("original"), 512*1024)
|
|
replacement := bytes.Repeat([]byte("replacement"), 256*1024)
|
|
const firstPrefix = 128 * 1024
|
|
const replacementPrefix = 96 * 1024
|
|
var rangeRequests atomic.Int32
|
|
var fullRequests atomic.Int32
|
|
var sawFullRestart atomic.Bool
|
|
|
|
server := httptest.NewServer(gohttp.HandlerFunc(func(w gohttp.ResponseWriter, r *gohttp.Request) {
|
|
w.Header().Set(base.HttpHeaderAcceptRanges, base.HttpHeaderBytes)
|
|
rangeHeader := r.Header.Get(base.HttpHeaderRange)
|
|
if rangeHeader == "" {
|
|
requestNumber := fullRequests.Add(1)
|
|
responsePayload := payload
|
|
if requestNumber > 1 {
|
|
responsePayload = replacement
|
|
sawFullRestart.Store(true)
|
|
}
|
|
w.Header().Set(base.HttpHeaderContentLength, fmt.Sprintf("%d", len(responsePayload)))
|
|
w.WriteHeader(gohttp.StatusOK)
|
|
_, _ = w.Write(responsePayload)
|
|
return
|
|
}
|
|
|
|
requestNumber := rangeRequests.Add(1)
|
|
switch requestNumber {
|
|
case 1:
|
|
w.Header().Set(base.HttpHeaderETag, `"prefix-v1"`)
|
|
w.Header().Set(base.HttpHeaderContentLength, fmt.Sprintf("%d", len(payload)))
|
|
w.WriteHeader(gohttp.StatusOK)
|
|
w.(gohttp.Flusher).Flush()
|
|
_, _ = w.Write(payload[:firstPrefix])
|
|
case 2:
|
|
if got := r.Header.Get(base.HttpHeaderIfRange); got != `"prefix-v1"` {
|
|
t.Errorf("If-Range = %q, want %q", got, `"prefix-v1"`)
|
|
}
|
|
conn, rw, err := w.(gohttp.Hijacker).Hijack()
|
|
if err != nil {
|
|
t.Errorf("Hijack() error = %v", err)
|
|
return
|
|
}
|
|
defer conn.Close()
|
|
_, _ = fmt.Fprintf(rw, "HTTP/1.1 200 OK\r\nETag: \"replacement-v2\"\r\nTransfer-Encoding: chunked\r\nConnection: close\r\n\r\n%x\r\n", replacementPrefix)
|
|
_, _ = rw.Write(replacement[:replacementPrefix])
|
|
_, _ = rw.WriteString("\r\n")
|
|
_ = rw.Flush()
|
|
default:
|
|
t.Errorf("unexpected Range retry after unknown-size replacement: %q", rangeHeader)
|
|
w.WriteHeader(gohttp.StatusBadRequest)
|
|
}
|
|
}))
|
|
defer server.Close()
|
|
|
|
f := buildFetcher()
|
|
f.Meta().Opts = &base.Options{
|
|
Path: t.TempDir(),
|
|
Name: "chunked-replacement.data",
|
|
Extra: &http.OptsExtra{
|
|
Connections: 4,
|
|
},
|
|
}
|
|
if err := f.Resolve(&base.Request{URL: server.URL + "/chunked-replacement.data"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := f.Start(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := f.Wait(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
got, err := os.ReadFile(f.meta.SingleFilepath())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !bytes.Equal(got, replacement) {
|
|
t.Fatal("downloaded file differs after interrupted chunked replacement")
|
|
}
|
|
if rangeRequests.Load() != 2 {
|
|
t.Fatalf("Range requests = %d, want 2", rangeRequests.Load())
|
|
}
|
|
if !sawFullRestart.Load() {
|
|
t.Fatal("interrupted chunked replacement did not restart with a full request")
|
|
}
|
|
wantSize := int64(len(replacement))
|
|
if f.meta.Res.Size != wantSize || f.meta.Res.Files[0].Size != wantSize {
|
|
t.Fatalf("resource size = %d/%d, want %d", f.meta.Res.Size, f.meta.Res.Files[0].Size, wantSize)
|
|
}
|
|
if got := f.Progress().TotalDownloaded(); got != wantSize {
|
|
t.Fatalf("progress = %d, want %d", got, wantSize)
|
|
}
|
|
info, err := os.Stat(f.meta.SingleFilepath())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if got := info.Size(); got != wantSize {
|
|
t.Fatalf("file size = %d, want %d", got, wantSize)
|
|
}
|
|
if f.sequentialSizeUnknown {
|
|
t.Fatal("completed replacement kept unknown-size recovery state")
|
|
}
|
|
}
|
|
|
|
func TestExtractIfRangeValidator(t *testing.T) {
|
|
const (
|
|
lastModified = "Wed, 21 Oct 2015 07:28:00 GMT"
|
|
laterDate = "Wed, 21 Oct 2015 07:28:02 GMT"
|
|
)
|
|
tests := []struct {
|
|
name string
|
|
etag string
|
|
lm string
|
|
date string
|
|
want string
|
|
}{
|
|
{name: "strong ETag", etag: `"v1"`, want: `"v1"`},
|
|
{name: "malformed ETag", etag: `"bad value"`},
|
|
{name: "weak ETag blocks date", etag: `W/"v1"`, lm: lastModified, date: laterDate},
|
|
{name: "strong Last-Modified", lm: lastModified, date: laterDate, want: lastModified},
|
|
{name: "same-second Last-Modified", lm: lastModified, date: lastModified},
|
|
{name: "Last-Modified without Date", lm: lastModified},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
header := make(gohttp.Header)
|
|
if tt.etag != "" {
|
|
header.Set(base.HttpHeaderETag, tt.etag)
|
|
}
|
|
if tt.lm != "" {
|
|
header.Set(base.HttpHeaderLastModified, tt.lm)
|
|
}
|
|
if tt.date != "" {
|
|
header.Set("Date", tt.date)
|
|
}
|
|
if got := extractIfRangeValidator(header); got != tt.want {
|
|
t.Fatalf("extractIfRangeValidator() = %q, want %q", got, tt.want)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestFetcher_OriginalURLFallbackPreservesResumeHeaders(t *testing.T) {
|
|
const (
|
|
start = int64(128)
|
|
end = int64(255)
|
|
ifRange = `"resume-v2"`
|
|
)
|
|
server := httptest.NewServer(gohttp.HandlerFunc(func(w gohttp.ResponseWriter, r *gohttp.Request) {
|
|
if got, want := r.Header.Get(base.HttpHeaderRange), "bytes=128-255"; got != want {
|
|
t.Errorf("Range = %q, want %q", got, want)
|
|
}
|
|
if got := r.Header.Get(base.HttpHeaderIfRange); got != ifRange {
|
|
t.Errorf("If-Range = %q, want %q", got, ifRange)
|
|
}
|
|
w.Header().Set(base.HttpHeaderContentRange, "bytes 128-255/256")
|
|
w.WriteHeader(gohttp.StatusPartialContent)
|
|
}))
|
|
defer server.Close()
|
|
|
|
f := &Fetcher{
|
|
config: &config{},
|
|
meta: &fetcher.FetcherMeta{Req: &base.Request{URL: server.URL}},
|
|
}
|
|
rangeEnd := int64(end)
|
|
resp, err := f.tryFallbackToOriginalURL(context.Background(), server.Client(), start, &rangeEnd, true, ifRange)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
resp.Body.Close()
|
|
}
|
|
|
|
func TestFetcher_UsesResolveResponseWhenRangeTakeoverFails(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
rangeHandler func(gohttp.ResponseWriter, *gohttp.Request)
|
|
}{
|
|
{
|
|
name: "forbidden",
|
|
rangeHandler: func(w gohttp.ResponseWriter, _ *gohttp.Request) {
|
|
w.WriteHeader(gohttp.StatusForbidden)
|
|
},
|
|
},
|
|
{
|
|
name: "invalid partial content",
|
|
rangeHandler: func(w gohttp.ResponseWriter, r *gohttp.Request) {
|
|
var start, end int64
|
|
if _, err := fmt.Sscanf(r.Header.Get(base.HttpHeaderRange), "bytes=%d-%d", &start, &end); err != nil {
|
|
w.WriteHeader(gohttp.StatusBadRequest)
|
|
return
|
|
}
|
|
w.Header().Set(base.HttpHeaderContentRange, fmt.Sprintf("bytes %d-%d/%d", start+1, end, 2*1024*1024))
|
|
w.Header().Set(base.HttpHeaderContentLength, fmt.Sprintf("%d", end-start))
|
|
w.WriteHeader(gohttp.StatusPartialContent)
|
|
},
|
|
},
|
|
{
|
|
name: "connection closes before response",
|
|
rangeHandler: func(w gohttp.ResponseWriter, _ *gohttp.Request) {
|
|
conn, _, err := w.(gohttp.Hijacker).Hijack()
|
|
if err == nil {
|
|
conn.Close()
|
|
}
|
|
},
|
|
},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
payload := make([]byte, 2*1024*1024)
|
|
for i := range payload {
|
|
payload[i] = byte(i % 251)
|
|
}
|
|
|
|
var requests atomic.Int32
|
|
var rangeRequests atomic.Int32
|
|
server := httptest.NewServer(gohttp.HandlerFunc(func(w gohttp.ResponseWriter, r *gohttp.Request) {
|
|
requestNumber := requests.Add(1)
|
|
if requestNumber == 1 {
|
|
w.Header().Set(base.HttpHeaderAcceptRanges, base.HttpHeaderBytes)
|
|
w.Header().Set(base.HttpHeaderContentLength, fmt.Sprintf("%d", len(payload)))
|
|
w.WriteHeader(gohttp.StatusOK)
|
|
w.(gohttp.Flusher).Flush()
|
|
for offset := 0; offset < len(payload); offset += 8 * 1024 {
|
|
end := min(offset+8*1024, len(payload))
|
|
if _, err := w.Write(payload[offset:end]); err != nil {
|
|
return
|
|
}
|
|
time.Sleep(time.Millisecond)
|
|
}
|
|
return
|
|
}
|
|
|
|
if r.Header.Get(base.HttpHeaderRange) != "" {
|
|
rangeRequests.Add(1)
|
|
}
|
|
tt.rangeHandler(w, r)
|
|
}))
|
|
defer server.Close()
|
|
|
|
f := buildFetcher()
|
|
f.Meta().Opts = &base.Options{
|
|
Path: t.TempDir(),
|
|
Name: "resolve-fallback.data",
|
|
Extra: &http.OptsExtra{
|
|
Connections: 1,
|
|
},
|
|
}
|
|
if err := f.Resolve(&base.Request{URL: server.URL + "/resolve-fallback.data"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer f.Close()
|
|
|
|
if err := f.Start(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := f.Wait(); err != nil {
|
|
t.Fatalf("download should continue from the Resolve response when Range takeover fails: %v", err)
|
|
}
|
|
if rangeRequests.Load() == 0 {
|
|
t.Fatal("download did not attempt a Range request")
|
|
}
|
|
got, err := os.ReadFile(f.meta.SingleFilepath())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !bytes.Equal(got, payload) {
|
|
t.Fatal("downloaded file differs from the Resolve response")
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestFetcher_ValidRangeTakesOverResolveResponse(t *testing.T) {
|
|
const resolveETag = `"resolve-v1"`
|
|
payload := make([]byte, 4*1024*1024)
|
|
for i := range payload {
|
|
payload[i] = byte(i % 251)
|
|
}
|
|
|
|
var rangeRequests atomic.Int32
|
|
server := httptest.NewServer(gohttp.HandlerFunc(func(w gohttp.ResponseWriter, r *gohttp.Request) {
|
|
rangeHeader := r.Header.Get(base.HttpHeaderRange)
|
|
w.Header().Set(base.HttpHeaderAcceptRanges, base.HttpHeaderBytes)
|
|
if rangeHeader == "" {
|
|
w.Header().Set(base.HttpHeaderETag, resolveETag)
|
|
w.Header().Set(base.HttpHeaderContentLength, fmt.Sprintf("%d", len(payload)))
|
|
w.WriteHeader(gohttp.StatusOK)
|
|
w.(gohttp.Flusher).Flush()
|
|
for offset := 0; offset < len(payload); offset += 8 * 1024 {
|
|
end := min(offset+8*1024, len(payload))
|
|
if _, err := w.Write(payload[offset:end]); err != nil {
|
|
return
|
|
}
|
|
time.Sleep(time.Millisecond)
|
|
}
|
|
return
|
|
}
|
|
|
|
rangeRequests.Add(1)
|
|
if got := r.Header.Get(base.HttpHeaderIfRange); got != resolveETag {
|
|
t.Errorf("If-Range = %q, want Resolve ETag %q", got, resolveETag)
|
|
}
|
|
var start, end int64
|
|
if _, err := fmt.Sscanf(rangeHeader, "bytes=%d-%d", &start, &end); err != nil {
|
|
w.WriteHeader(gohttp.StatusBadRequest)
|
|
return
|
|
}
|
|
w.Header().Set(base.HttpHeaderContentRange, fmt.Sprintf("bytes %d-%d/%d", start, end, len(payload)))
|
|
w.Header().Set(base.HttpHeaderContentLength, fmt.Sprintf("%d", end-start+1))
|
|
w.WriteHeader(gohttp.StatusPartialContent)
|
|
for offset := start; offset <= end; offset += 8 * 1024 {
|
|
chunkEnd := min(offset+8*1024, end+1)
|
|
if _, err := w.Write(payload[offset:chunkEnd]); err != nil {
|
|
return
|
|
}
|
|
}
|
|
}))
|
|
defer server.Close()
|
|
|
|
f := buildFetcher()
|
|
f.Meta().Opts = &base.Options{
|
|
Path: t.TempDir(),
|
|
Name: "resolve-takeover.data",
|
|
Extra: &http.OptsExtra{
|
|
Connections: 4,
|
|
},
|
|
}
|
|
if err := f.Resolve(&base.Request{URL: server.URL + "/resolve-takeover.data"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer f.Close()
|
|
|
|
if err := f.Start(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := f.Wait(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if rangeRequests.Load() == 0 {
|
|
t.Fatal("download did not attempt a Range takeover")
|
|
}
|
|
got, err := os.ReadFile(f.meta.SingleFilepath())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !bytes.Equal(got, payload) {
|
|
t.Fatal("downloaded file differs after valid Range takeover")
|
|
}
|
|
}
|
|
|
|
func TestFetcher_UnknownSizeRangeUsesOpenEndedWorker(t *testing.T) {
|
|
for _, hint := range []int64{0, 3, 8 * 1024 * 1024} {
|
|
t.Run(fmt.Sprint(hint), func(t *testing.T) { testFetcherUnknownSizeRange(t, hint) })
|
|
}
|
|
}
|
|
|
|
func testFetcherUnknownSizeRange(t *testing.T, hint int64) {
|
|
const resolveETag = `"unknown-size-v1"`
|
|
payload := make([]byte, 4*1024*1024)
|
|
for i := range payload {
|
|
payload[i] = byte((i*11 + 7) % 251)
|
|
}
|
|
|
|
var openEndedRequests atomic.Int32
|
|
var rangeRequests atomic.Int32
|
|
server := httptest.NewServer(gohttp.HandlerFunc(func(w gohttp.ResponseWriter, r *gohttp.Request) {
|
|
rangeHeader := r.Header.Get(base.HttpHeaderRange)
|
|
w.Header().Set(base.HttpHeaderAcceptRanges, base.HttpHeaderBytes)
|
|
w.Header().Set(base.HttpHeaderETag, resolveETag)
|
|
if rangeHeader == "" {
|
|
w.WriteHeader(gohttp.StatusOK)
|
|
w.(gohttp.Flusher).Flush()
|
|
for offset := 0; offset < len(payload); offset += 8 * 1024 {
|
|
end := min(offset+8*1024, len(payload))
|
|
if _, err := w.Write(payload[offset:end]); err != nil {
|
|
return
|
|
}
|
|
w.(gohttp.Flusher).Flush()
|
|
time.Sleep(2 * time.Millisecond)
|
|
}
|
|
return
|
|
}
|
|
|
|
rangeRequests.Add(1)
|
|
if got := r.Header.Get(base.HttpHeaderIfRange); got != resolveETag {
|
|
t.Errorf("If-Range = %q, want %q", got, resolveETag)
|
|
}
|
|
var start, end int64
|
|
if strings.HasSuffix(rangeHeader, "-") {
|
|
openEndedRequests.Add(1)
|
|
if _, err := fmt.Sscanf(rangeHeader, "bytes=%d-", &start); err != nil {
|
|
t.Errorf("parse open-ended Range %q: %v", rangeHeader, err)
|
|
w.WriteHeader(gohttp.StatusBadRequest)
|
|
return
|
|
}
|
|
end = int64(len(payload) - 1)
|
|
} else if _, err := fmt.Sscanf(rangeHeader, "bytes=%d-%d", &start, &end); err != nil {
|
|
t.Errorf("parse bounded Range %q: %v", rangeHeader, err)
|
|
w.WriteHeader(gohttp.StatusBadRequest)
|
|
return
|
|
}
|
|
if start > end {
|
|
w.Header().Set(base.HttpHeaderContentRange, fmt.Sprintf("bytes */%d", len(payload)))
|
|
w.WriteHeader(gohttp.StatusRequestedRangeNotSatisfiable)
|
|
return
|
|
}
|
|
w.Header().Set(base.HttpHeaderContentRange, fmt.Sprintf("bytes %d-%d/%d", start, end, len(payload)))
|
|
w.Header().Set(base.HttpHeaderContentLength, fmt.Sprintf("%d", end-start+1))
|
|
w.WriteHeader(gohttp.StatusPartialContent)
|
|
for offset := start; offset <= end; offset += 8 * 1024 {
|
|
chunkEnd := min(offset+8*1024, end+1)
|
|
if _, err := w.Write(payload[offset:chunkEnd]); err != nil {
|
|
return
|
|
}
|
|
w.(gohttp.Flusher).Flush()
|
|
time.Sleep(time.Millisecond)
|
|
}
|
|
}))
|
|
defer server.Close()
|
|
|
|
f := buildFetcher()
|
|
f.Meta().Opts = &base.Options{
|
|
Size: hint,
|
|
Path: t.TempDir(),
|
|
Name: "unknown-size-range.data",
|
|
Extra: &http.OptsExtra{
|
|
Connections: 4,
|
|
},
|
|
}
|
|
if err := f.Resolve(&base.Request{URL: server.URL + "/unknown-size-range.data"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer f.Close()
|
|
|
|
if f.meta.Res.Size != hint || f.hasKnownSize() || !f.meta.Res.Range {
|
|
t.Fatalf("resolved resource = %+v, want unknown size with Range support", f.meta.Res)
|
|
}
|
|
if err := f.Start(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := f.Wait(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if got := openEndedRequests.Load(); got != 1 {
|
|
t.Fatalf("open-ended Range requests = %d, want 1", got)
|
|
}
|
|
if got := rangeRequests.Load(); got < 2 {
|
|
t.Fatalf("Range requests = %d, want slow-start expansion after size discovery", got)
|
|
}
|
|
if got, want := f.meta.Res.Size, int64(len(payload)); got != want {
|
|
t.Fatalf("resource size = %d, want %d", got, want)
|
|
}
|
|
got, err := os.ReadFile(f.meta.SingleFilepath())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !bytes.Equal(got, payload) {
|
|
t.Fatal("downloaded file differs after unknown-size Range takeover")
|
|
}
|
|
}
|
|
|
|
func TestFetcher_UnknownSizeRangeRetriesShortPartialBody(t *testing.T) {
|
|
payload := make([]byte, 1024*1024)
|
|
for i := range payload {
|
|
payload[i] = byte((i*13 + 5) % 251)
|
|
}
|
|
|
|
var rangeRequests atomic.Int32
|
|
server := httptest.NewServer(gohttp.HandlerFunc(func(w gohttp.ResponseWriter, r *gohttp.Request) {
|
|
rangeHeader := r.Header.Get(base.HttpHeaderRange)
|
|
w.Header().Set(base.HttpHeaderAcceptRanges, base.HttpHeaderBytes)
|
|
w.Header().Set(base.HttpHeaderETag, `"short-range-v1"`)
|
|
if rangeHeader == "" {
|
|
w.WriteHeader(gohttp.StatusOK)
|
|
w.(gohttp.Flusher).Flush()
|
|
for offset := 0; offset < len(payload); offset += 8 * 1024 {
|
|
end := min(offset+8*1024, len(payload))
|
|
if _, err := w.Write(payload[offset:end]); err != nil {
|
|
return
|
|
}
|
|
w.(gohttp.Flusher).Flush()
|
|
time.Sleep(2 * time.Millisecond)
|
|
}
|
|
return
|
|
}
|
|
|
|
requestNumber := rangeRequests.Add(1)
|
|
var start, requestedEnd int64
|
|
openEnded := strings.HasSuffix(rangeHeader, "-")
|
|
if openEnded {
|
|
if _, err := fmt.Sscanf(rangeHeader, "bytes=%d-", &start); err != nil {
|
|
t.Errorf("parse open-ended Range %q: %v", rangeHeader, err)
|
|
w.WriteHeader(gohttp.StatusBadRequest)
|
|
return
|
|
}
|
|
requestedEnd = int64(len(payload) - 1)
|
|
} else if _, err := fmt.Sscanf(rangeHeader, "bytes=%d-%d", &start, &requestedEnd); err != nil {
|
|
t.Errorf("parse Range %q: %v", rangeHeader, err)
|
|
w.WriteHeader(gohttp.StatusBadRequest)
|
|
return
|
|
}
|
|
|
|
w.Header().Set(base.HttpHeaderContentRange, fmt.Sprintf("bytes %d-%d/%d", start, requestedEnd, len(payload)))
|
|
w.WriteHeader(gohttp.StatusPartialContent)
|
|
w.(gohttp.Flusher).Flush()
|
|
if requestNumber == 1 {
|
|
// End the chunked response cleanly after only part of the declared
|
|
// Content-Range. The worker must retry from the materialized offset.
|
|
shortEnd := min(start+64*1024, requestedEnd+1)
|
|
_, _ = w.Write(payload[start:shortEnd])
|
|
return
|
|
}
|
|
_, _ = w.Write(payload[start : requestedEnd+1])
|
|
}))
|
|
defer server.Close()
|
|
|
|
f := buildFetcher()
|
|
f.Meta().Opts = &base.Options{
|
|
Path: t.TempDir(),
|
|
Name: "short-range.data",
|
|
Extra: &http.OptsExtra{
|
|
Connections: 1,
|
|
},
|
|
}
|
|
if err := f.Resolve(&base.Request{URL: server.URL + "/short-range.data"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer f.Close()
|
|
|
|
if err := f.Start(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := f.Wait(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if got := rangeRequests.Load(); got < 2 {
|
|
t.Fatalf("Range requests = %d, want a retry after the short body", got)
|
|
}
|
|
got, err := os.ReadFile(f.meta.SingleFilepath())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !bytes.Equal(got, payload) {
|
|
t.Fatal("downloaded file differs after retrying the short Range body")
|
|
}
|
|
}
|
|
|
|
func TestFetcher_UnknownSizeRangeFallsBackToResolve(t *testing.T) {
|
|
payload := make([]byte, 512*1024)
|
|
for i := range payload {
|
|
payload[i] = byte((i*17 + 3) % 251)
|
|
}
|
|
|
|
var rangeRequests atomic.Int32
|
|
server := httptest.NewServer(gohttp.HandlerFunc(func(w gohttp.ResponseWriter, r *gohttp.Request) {
|
|
w.Header().Set(base.HttpHeaderAcceptRanges, base.HttpHeaderBytes)
|
|
if r.Header.Get(base.HttpHeaderRange) == "" {
|
|
w.WriteHeader(gohttp.StatusOK)
|
|
w.(gohttp.Flusher).Flush()
|
|
for offset := 0; offset < len(payload); offset += 8 * 1024 {
|
|
end := min(offset+8*1024, len(payload))
|
|
if _, err := w.Write(payload[offset:end]); err != nil {
|
|
return
|
|
}
|
|
w.(gohttp.Flusher).Flush()
|
|
time.Sleep(time.Millisecond)
|
|
}
|
|
return
|
|
}
|
|
|
|
rangeRequests.Add(1)
|
|
w.Header().Set(base.HttpHeaderContentRange, fmt.Sprintf("bytes 1-%d/%d", len(payload)-1, len(payload)))
|
|
w.WriteHeader(gohttp.StatusPartialContent)
|
|
}))
|
|
defer server.Close()
|
|
|
|
f := buildFetcher()
|
|
f.Meta().Opts = &base.Options{
|
|
Path: t.TempDir(),
|
|
Name: "unknown-size-fallback.data",
|
|
Extra: &http.OptsExtra{
|
|
Connections: 1,
|
|
},
|
|
}
|
|
if err := f.Resolve(&base.Request{URL: server.URL + "/unknown-size-fallback.data"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer f.Close()
|
|
|
|
if err := f.Start(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := f.Wait(); err != nil {
|
|
t.Fatalf("Wait() should use the complete Resolve response: %v", err)
|
|
}
|
|
if got := rangeRequests.Load(); got != 1 {
|
|
t.Fatalf("Range requests = %d, want 1", got)
|
|
}
|
|
if f.meta.Res.Range {
|
|
t.Fatal("Resolve fallback should switch the task to sequential mode")
|
|
}
|
|
if got, want := f.meta.Res.Size, int64(len(payload)); got != want {
|
|
t.Fatalf("resource size = %d, want %d", got, want)
|
|
}
|
|
got, err := os.ReadFile(f.meta.SingleFilepath())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !bytes.Equal(got, payload) {
|
|
t.Fatal("downloaded file differs from the unknown-size Resolve fallback")
|
|
}
|
|
}
|
|
|
|
func TestFetcher_UnknownSizeRangeRemainsOpenEndedAfterPause(t *testing.T) {
|
|
payload := make([]byte, 1024*1024)
|
|
for i := range payload {
|
|
payload[i] = byte((i*19 + 1) % 251)
|
|
}
|
|
|
|
firstRangeStarted := make(chan struct{})
|
|
var rangeRequests atomic.Int32
|
|
server := httptest.NewServer(gohttp.HandlerFunc(func(w gohttp.ResponseWriter, r *gohttp.Request) {
|
|
rangeHeader := r.Header.Get(base.HttpHeaderRange)
|
|
w.Header().Set(base.HttpHeaderAcceptRanges, base.HttpHeaderBytes)
|
|
w.Header().Set(base.HttpHeaderETag, `"pause-open-range-v1"`)
|
|
if rangeHeader == "" {
|
|
w.WriteHeader(gohttp.StatusOK)
|
|
w.(gohttp.Flusher).Flush()
|
|
for offset := 0; offset < len(payload); offset += 8 * 1024 {
|
|
end := min(offset+8*1024, len(payload))
|
|
if _, err := w.Write(payload[offset:end]); err != nil {
|
|
return
|
|
}
|
|
w.(gohttp.Flusher).Flush()
|
|
time.Sleep(2 * time.Millisecond)
|
|
}
|
|
return
|
|
}
|
|
|
|
requestNumber := rangeRequests.Add(1)
|
|
if requestNumber == 1 {
|
|
close(firstRangeStarted)
|
|
<-r.Context().Done()
|
|
return
|
|
}
|
|
if !strings.HasSuffix(rangeHeader, "-") {
|
|
t.Errorf("resumed Range = %q, want an open-ended range", rangeHeader)
|
|
w.WriteHeader(gohttp.StatusBadRequest)
|
|
return
|
|
}
|
|
var start int64
|
|
if _, err := fmt.Sscanf(rangeHeader, "bytes=%d-", &start); err != nil {
|
|
t.Errorf("parse resumed Range %q: %v", rangeHeader, err)
|
|
w.WriteHeader(gohttp.StatusBadRequest)
|
|
return
|
|
}
|
|
end := int64(len(payload) - 1)
|
|
w.Header().Set(base.HttpHeaderContentRange, fmt.Sprintf("bytes %d-%d/%d", start, end, len(payload)))
|
|
w.Header().Set(base.HttpHeaderContentLength, fmt.Sprintf("%d", end-start+1))
|
|
w.WriteHeader(gohttp.StatusPartialContent)
|
|
_, _ = w.Write(payload[start:])
|
|
}))
|
|
defer server.Close()
|
|
|
|
f := buildFetcher()
|
|
f.Meta().Opts = &base.Options{
|
|
Size: 3,
|
|
Path: t.TempDir(),
|
|
Name: "pause-open-range.data",
|
|
Extra: &http.OptsExtra{
|
|
Connections: 1,
|
|
},
|
|
}
|
|
if err := f.Resolve(&base.Request{URL: server.URL + "/pause-open-range.data"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer f.Close()
|
|
|
|
if err := f.Start(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
select {
|
|
case <-firstRangeStarted:
|
|
case <-time.After(5 * time.Second):
|
|
t.Fatal("timed out waiting for the first open-ended worker")
|
|
}
|
|
if err := f.Pause(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := f.Start(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := f.Wait(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if got := rangeRequests.Load(); got != 2 {
|
|
t.Fatalf("Range requests = %d, want an open-ended retry after pause", got)
|
|
}
|
|
got, err := os.ReadFile(f.meta.SingleFilepath())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !bytes.Equal(got, payload) {
|
|
t.Fatal("downloaded file differs after pausing an unknown-size Range")
|
|
}
|
|
}
|
|
|
|
func TestFetcher_UnknownSizeResolveReadErrorFailsTask(t *testing.T) {
|
|
server := httptest.NewServer(gohttp.HandlerFunc(func(w gohttp.ResponseWriter, _ *gohttp.Request) {
|
|
conn, rw, err := w.(gohttp.Hijacker).Hijack()
|
|
if err != nil {
|
|
t.Errorf("hijack response: %v", err)
|
|
return
|
|
}
|
|
defer conn.Close()
|
|
_, _ = fmt.Fprint(rw, "HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\nConnection: close\r\n\r\n800\r\n")
|
|
_, _ = rw.Write(make([]byte, 1024))
|
|
_ = rw.Flush()
|
|
}))
|
|
defer server.Close()
|
|
|
|
f := buildFetcher()
|
|
f.Meta().Opts = &base.Options{
|
|
Path: t.TempDir(),
|
|
Name: "broken-chunked.data",
|
|
Extra: &http.OptsExtra{
|
|
Connections: 1,
|
|
},
|
|
}
|
|
if err := f.Resolve(&base.Request{URL: server.URL + "/broken-chunked.data"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer f.Close()
|
|
|
|
if err := f.Start(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := f.Wait(); err == nil {
|
|
t.Fatal("Wait() succeeded after the unknown-size response ended unexpectedly")
|
|
}
|
|
if got := f.getState(); got != stateError {
|
|
t.Fatalf("fetcher state = %v, want %v", got, stateError)
|
|
}
|
|
}
|
|
|
|
func TestFetcher_UnknownSizeResolveCleanEOFUpdatesSize(t *testing.T) {
|
|
payload := bytes.Repeat([]byte("gopeed-unknown-size\n"), 8192)
|
|
server := httptest.NewServer(gohttp.HandlerFunc(func(w gohttp.ResponseWriter, _ *gohttp.Request) {
|
|
w.WriteHeader(gohttp.StatusOK)
|
|
w.(gohttp.Flusher).Flush()
|
|
_, _ = w.Write(payload)
|
|
}))
|
|
defer server.Close()
|
|
|
|
f := buildFetcher()
|
|
f.Meta().Opts = &base.Options{
|
|
Path: t.TempDir(),
|
|
Name: "unknown-size.data",
|
|
Extra: &http.OptsExtra{
|
|
Connections: 1,
|
|
},
|
|
}
|
|
if err := f.Resolve(&base.Request{URL: server.URL + "/unknown-size.data"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer f.Close()
|
|
|
|
if err := f.Start(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := f.Wait(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if got, want := f.meta.Res.Size, int64(len(payload)); got != want {
|
|
t.Fatalf("resource size = %d, want %d", got, want)
|
|
}
|
|
if got, want := f.meta.Res.Files[0].Size, int64(len(payload)); got != want {
|
|
t.Fatalf("file size = %d, want %d", got, want)
|
|
}
|
|
got, err := os.ReadFile(f.meta.SingleFilepath())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !bytes.Equal(got, payload) {
|
|
t.Fatal("downloaded file differs for a clean unknown-size response")
|
|
}
|
|
}
|
|
|
|
func TestFetcher_RejectsInvalidPartialContent(t *testing.T) {
|
|
const (
|
|
requestedStart = int64(1024)
|
|
requestedEnd = int64(2047)
|
|
totalSize = int64(4096)
|
|
rangeLength = requestedEnd - requestedStart + 1
|
|
)
|
|
|
|
tests := []struct {
|
|
name string
|
|
contentRange string
|
|
contentLength int64
|
|
bodyLength int
|
|
chunked bool
|
|
wantError bool
|
|
}{
|
|
{
|
|
name: "valid",
|
|
contentRange: "bytes 1024-2047/4096",
|
|
contentLength: rangeLength,
|
|
bodyLength: int(rangeLength),
|
|
},
|
|
{
|
|
name: "missing content range",
|
|
contentLength: rangeLength,
|
|
bodyLength: int(rangeLength),
|
|
wantError: true,
|
|
},
|
|
{
|
|
name: "invalid unit",
|
|
contentRange: "items 1024-2047/4096",
|
|
contentLength: rangeLength,
|
|
bodyLength: int(rangeLength),
|
|
wantError: true,
|
|
},
|
|
{
|
|
name: "malformed content range",
|
|
contentRange: "bytes 1024/4096",
|
|
contentLength: rangeLength,
|
|
bodyLength: int(rangeLength),
|
|
wantError: true,
|
|
},
|
|
{
|
|
name: "missing total separator",
|
|
contentRange: "bytes 1024-2047",
|
|
contentLength: rangeLength,
|
|
bodyLength: int(rangeLength),
|
|
wantError: true,
|
|
},
|
|
{
|
|
name: "unknown total size",
|
|
contentRange: "bytes 1024-2047/*",
|
|
contentLength: rangeLength,
|
|
bodyLength: int(rangeLength),
|
|
wantError: true,
|
|
},
|
|
{
|
|
name: "shifted content range",
|
|
contentRange: "bytes 0-1023/4096",
|
|
contentLength: rangeLength,
|
|
bodyLength: int(rangeLength),
|
|
wantError: true,
|
|
},
|
|
{
|
|
name: "wrong total size",
|
|
contentRange: "bytes 1024-2047/8192",
|
|
contentLength: rangeLength,
|
|
bodyLength: int(rangeLength),
|
|
wantError: true,
|
|
},
|
|
{
|
|
name: "content length mismatch",
|
|
contentRange: "bytes 1024-2047/4096",
|
|
contentLength: rangeLength - 1,
|
|
bodyLength: int(rangeLength - 1),
|
|
wantError: true,
|
|
},
|
|
{
|
|
name: "long chunked body",
|
|
contentRange: "bytes 1024-2047/4096",
|
|
contentLength: -1,
|
|
bodyLength: int(rangeLength + 1),
|
|
chunked: true,
|
|
wantError: true,
|
|
},
|
|
}
|
|
|
|
for _, tc := range tests {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
server := httptest.NewServer(gohttp.HandlerFunc(func(writer gohttp.ResponseWriter, request *gohttp.Request) {
|
|
if got, want := request.Header.Get(base.HttpHeaderRange), "bytes=1024-2047"; got != want {
|
|
t.Errorf("Range header = %q, want %q", got, want)
|
|
}
|
|
if tc.contentRange != "" {
|
|
writer.Header().Set(base.HttpHeaderContentRange, tc.contentRange)
|
|
}
|
|
if tc.contentLength >= 0 {
|
|
writer.Header().Set(base.HttpHeaderContentLength, fmt.Sprintf("%d", tc.contentLength))
|
|
}
|
|
writer.WriteHeader(base.HttpCodePartialContent)
|
|
if tc.chunked {
|
|
writer.(gohttp.Flusher).Flush()
|
|
}
|
|
_, _ = writer.Write([]byte(strings.Repeat("x", tc.bodyLength)))
|
|
}))
|
|
defer server.Close()
|
|
|
|
f := buildFetcher()
|
|
f.meta.Req = &base.Request{URL: server.URL}
|
|
f.meta.Res = &base.Resource{Range: true, Size: totalSize}
|
|
output, err := os.CreateTemp(t.TempDir(), "range-response-*")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer output.Close()
|
|
f.file = output
|
|
|
|
conn := &connection{
|
|
ID: 0,
|
|
Role: rolePrimary,
|
|
State: connNotStarted,
|
|
Chunk: newChunk(requestedStart, requestedEnd),
|
|
ctx: context.Background(),
|
|
}
|
|
f.connections = []*connection{conn}
|
|
f.wg.Add(1)
|
|
go f.runConnection(conn)
|
|
f.wg.Wait()
|
|
|
|
if tc.wantError {
|
|
if !conn.failed || conn.State != connFailed || conn.Completed {
|
|
t.Fatalf("connection state = %v, failed = %v, completed = %v; want permanent failure", conn.State, conn.failed, conn.Completed)
|
|
}
|
|
if !errors.Is(conn.lastErr, errInvalidRangeResponse) {
|
|
t.Fatalf("connection error = %v, want %v", conn.lastErr, errInvalidRangeResponse)
|
|
}
|
|
return
|
|
}
|
|
|
|
if conn.failed || conn.State != connCompleted || !conn.Completed {
|
|
t.Fatalf("connection state = %v, failed = %v, completed = %v; want success", conn.State, conn.failed, conn.Completed)
|
|
}
|
|
if conn.Downloaded != rangeLength {
|
|
t.Fatalf("downloaded = %d, want %d", conn.Downloaded, rangeLength)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestFetcher_InvalidResponseLengthFailsTask(t *testing.T) {
|
|
const (
|
|
totalSize = int64(4096)
|
|
requestedStart = int64(1024)
|
|
requestedEnd = totalSize - 1
|
|
rangeLength = requestedEnd - requestedStart + 1
|
|
)
|
|
|
|
tests := []struct {
|
|
name string
|
|
status int
|
|
contentRange string
|
|
contentLength int64
|
|
bodyChunks []int
|
|
resolvedDownloaded int64
|
|
wantDownloaded int64
|
|
rangeMode bool
|
|
}{
|
|
{
|
|
name: "sequential declared short body",
|
|
status: base.HttpCodeOK,
|
|
contentLength: 1024,
|
|
bodyChunks: []int{1024},
|
|
wantDownloaded: 0,
|
|
},
|
|
{
|
|
name: "sequential short chunked body",
|
|
status: base.HttpCodeOK,
|
|
contentLength: -1,
|
|
bodyChunks: []int{1024},
|
|
wantDownloaded: 1024,
|
|
},
|
|
{
|
|
name: "sequential long chunked body",
|
|
status: base.HttpCodeOK,
|
|
contentLength: -1,
|
|
bodyChunks: []int{int(totalSize), 1},
|
|
wantDownloaded: totalSize,
|
|
},
|
|
{
|
|
name: "partial content long chunked body",
|
|
status: base.HttpCodePartialContent,
|
|
contentRange: "bytes 1024-4095/4096",
|
|
contentLength: -1,
|
|
bodyChunks: []int{int(rangeLength), 1},
|
|
resolvedDownloaded: requestedStart,
|
|
wantDownloaded: rangeLength,
|
|
rangeMode: true,
|
|
},
|
|
}
|
|
|
|
for _, tc := range tests {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
server := httptest.NewServer(gohttp.HandlerFunc(func(writer gohttp.ResponseWriter, request *gohttp.Request) {
|
|
wantRange := ""
|
|
if tc.rangeMode {
|
|
wantRange = "bytes=1024-4095"
|
|
}
|
|
if got := request.Header.Get(base.HttpHeaderRange); got != wantRange {
|
|
t.Errorf("Range header = %q, want %q", got, wantRange)
|
|
}
|
|
if tc.contentRange != "" {
|
|
writer.Header().Set(base.HttpHeaderContentRange, tc.contentRange)
|
|
}
|
|
if tc.contentLength >= 0 {
|
|
writer.Header().Set(base.HttpHeaderContentLength, fmt.Sprintf("%d", tc.contentLength))
|
|
}
|
|
writer.WriteHeader(tc.status)
|
|
writer.(gohttp.Flusher).Flush()
|
|
for i, chunkSize := range tc.bodyChunks {
|
|
_, _ = writer.Write([]byte(strings.Repeat("x", chunkSize)))
|
|
writer.(gohttp.Flusher).Flush()
|
|
if i+1 < len(tc.bodyChunks) {
|
|
time.Sleep(50 * time.Millisecond)
|
|
}
|
|
}
|
|
}))
|
|
defer server.Close()
|
|
|
|
f := buildFetcher()
|
|
f.meta.Req = &base.Request{URL: server.URL}
|
|
f.meta.Res = &base.Resource{Range: tc.rangeMode, Size: totalSize}
|
|
output, err := os.CreateTemp(t.TempDir(), "range-task-*")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := output.Truncate(totalSize); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
f.file = output
|
|
|
|
connChunk := newChunk(0, totalSize-1)
|
|
if tc.rangeMode {
|
|
connChunk = newChunk(requestedStart, requestedEnd)
|
|
}
|
|
conn := &connection{
|
|
ID: 0,
|
|
Role: rolePrimary,
|
|
State: connNotStarted,
|
|
Chunk: connChunk,
|
|
ctx: context.Background(),
|
|
}
|
|
f.connections = []*connection{conn}
|
|
if tc.resolvedDownloaded > 0 {
|
|
f.resolveConn = &connection{
|
|
Role: roleResolve,
|
|
State: connCompleted,
|
|
Downloaded: tc.resolvedDownloaded,
|
|
Completed: true,
|
|
}
|
|
}
|
|
|
|
f.wg.Add(1)
|
|
go f.runConnection(conn)
|
|
f.wg.Wait()
|
|
f.onDownloadComplete()
|
|
|
|
if err := f.Wait(); !errors.Is(err, errInvalidRangeResponse) {
|
|
t.Fatalf("Wait() error = %v, want %v", err, errInvalidRangeResponse)
|
|
}
|
|
if f.getState() != stateError {
|
|
t.Fatalf("fetcher state = %v, want %v", f.getState(), stateError)
|
|
}
|
|
if conn.Completed || conn.State != connFailed || !conn.failed {
|
|
t.Fatalf("connection state = %v, failed = %v, completed = %v; want permanent failure", conn.State, conn.failed, conn.Completed)
|
|
}
|
|
if conn.Downloaded != tc.wantDownloaded {
|
|
t.Fatalf("downloaded = %d, want %d", conn.Downloaded, tc.wantDownloaded)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestFetcher_DownloadChunked(t *testing.T) {
|
|
listener := test.StartTestCustomServer()
|
|
defer listener.Close()
|
|
|
|
downloadNormal(listener, 1, t)
|
|
downloadNormal(listener, 2, t)
|
|
}
|
|
|
|
func TestFetcher_DownloadPost(t *testing.T) {
|
|
listener := test.StartTestPostServer()
|
|
defer listener.Close()
|
|
|
|
downloadPost(listener, 1, t)
|
|
}
|
|
|
|
func TestFetcher_DownloadRetry(t *testing.T) {
|
|
listener := test.StartTestRetryServer()
|
|
defer listener.Close()
|
|
|
|
downloadNormal(listener, 1, t)
|
|
}
|
|
|
|
func TestFetcher_DownloadError(t *testing.T) {
|
|
listener := test.StartTestErrorServer()
|
|
defer listener.Close()
|
|
|
|
downloadError(listener, 1, t)
|
|
}
|
|
|
|
func TestFetcher_DownloadLimit(t *testing.T) {
|
|
listener := test.StartTestConLimitServer(4)
|
|
defer listener.Close()
|
|
|
|
downloadNormal(listener, 1, t)
|
|
downloadNormal(listener, 2, t)
|
|
downloadNormal(listener, 8, t)
|
|
}
|
|
|
|
func TestFetcher_DownloadResponseBodyReadTimeout(t *testing.T) {
|
|
// Server will timeout once (first request delays longer than readTimeout),
|
|
// then subsequent requests work normally
|
|
listener := test.StartTestTimeoutOnceServer(readTimeout.Milliseconds() + 5000)
|
|
defer listener.Close()
|
|
|
|
for _, connections := range []int{1, 4} {
|
|
os.Remove(test.DownloadFile)
|
|
|
|
fetcher := downloadReady(listener, connections, t)
|
|
if err := fetcher.Start(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := fetcher.Wait(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
stats := fetcher.Stats().Snapshot.(*http.Stats)
|
|
if len(stats.Connections) == 0 {
|
|
t.Fatalf("expected connections stats for timeout test")
|
|
}
|
|
|
|
// Verify successful download after timeout recovery
|
|
want := test.FileMd5(test.BuildFile)
|
|
got := test.FileMd5(test.DownloadFile)
|
|
if want != got {
|
|
t.Errorf("Download() got = %v, want %v", got, want)
|
|
}
|
|
|
|
// Verify timeouts don't count as failures (retryTimes should be 0)
|
|
for _, conn := range stats.Connections {
|
|
if conn.Failed {
|
|
t.Fatalf("expected no counted failures after timeout recovery, got retries=%d", conn.RetryTimes)
|
|
}
|
|
if conn.RetryTimes != 0 {
|
|
t.Fatalf("expected retryTimes to stay zero for non-counted timeouts, got %d", conn.RetryTimes)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestFetcher_Download500Recovery(t *testing.T) {
|
|
// Server returns 500 for 15 seconds, then recovers
|
|
listener := test.StartTestTemporary500Server(15 * time.Second)
|
|
defer listener.Close()
|
|
|
|
os.Remove(test.DownloadFile)
|
|
fetcher := downloadReady(listener, 4, t)
|
|
if err := fetcher.Start(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := fetcher.Wait(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
// Verify successful download after 500 errors
|
|
want := test.FileMd5(test.BuildFile)
|
|
got := test.FileMd5(test.DownloadFile)
|
|
if want != got {
|
|
t.Errorf("Download() got = %v, want %v", got, want)
|
|
}
|
|
|
|
// Verify 500 errors don't count as failures (retryTimes should be 0)
|
|
stats := fetcher.Stats().Snapshot.(*http.Stats)
|
|
for _, conn := range stats.Connections {
|
|
if conn.RetryTimes != 0 {
|
|
t.Errorf("Expected retryTimes to be 0 for 500 errors (exempt), got %d", conn.RetryTimes)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestFetcher_DownloadOnBugFileServer(t *testing.T) {
|
|
listener := test.StartTestRangeBugServer()
|
|
defer listener.Close()
|
|
|
|
downloadNormal(listener, 1, t)
|
|
|
|
fetcher := downloadReady(listener, 4, t)
|
|
if err := fetcher.Start(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := fetcher.Wait(); err == nil || !strings.Contains(err.Error(), errInvalidRangeResponse.Error()) {
|
|
t.Fatalf("Wait() error = %v, want %v", err, errInvalidRangeResponse)
|
|
}
|
|
}
|
|
|
|
func TestFetcher_DownloadResume(t *testing.T) {
|
|
listener := test.StartTestFileServer()
|
|
defer listener.Close()
|
|
|
|
downloadResume(listener, 1, t)
|
|
downloadResume(listener, 5, t)
|
|
downloadResume(listener, 8, t)
|
|
downloadResume(listener, 16, t)
|
|
}
|
|
|
|
func TestFetcher_DownloadWithProxy(t *testing.T) {
|
|
httpListener := test.StartTestFileServer()
|
|
defer httpListener.Close()
|
|
proxyListener := test.StartSocks5Server("", "")
|
|
defer proxyListener.Close()
|
|
|
|
downloadWithProxy(httpListener, proxyListener, t)
|
|
}
|
|
|
|
func TestFetcher_ConfigConnections(t *testing.T) {
|
|
listener := test.StartTestFileServer()
|
|
defer listener.Close()
|
|
fetcher := doDownloadReady(buildConfigFetcher(config{
|
|
Connections: 16,
|
|
}), listener, 0, t)
|
|
err := fetcher.Start()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
err = fetcher.Wait()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
want := test.FileMd5(test.BuildFile)
|
|
got := test.FileMd5(test.DownloadFile)
|
|
if want != got {
|
|
t.Errorf("Download() got = %v, want %v", got, want)
|
|
}
|
|
}
|
|
|
|
func TestFetcher_ConfigUseServerCtime(t *testing.T) {
|
|
listener := test.StartTestFileServer()
|
|
defer listener.Close()
|
|
fetcher := doDownloadReady(buildConfigFetcher(config{
|
|
Connections: 16,
|
|
UseServerCtime: true,
|
|
}), listener, 0, t)
|
|
err := fetcher.Start()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
err = fetcher.Wait()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
want := test.FileMd5(test.BuildFile)
|
|
got := test.FileMd5(test.DownloadFile)
|
|
if want != got {
|
|
t.Errorf("Download() got = %v, want %v", got, want)
|
|
}
|
|
}
|
|
|
|
func TestFetcher_Stats(t *testing.T) {
|
|
listener := test.StartTestFileServer()
|
|
defer listener.Close()
|
|
fetcher := doDownloadReady(buildConfigFetcher(config{
|
|
Connections: 16,
|
|
}), listener, 0, t)
|
|
err := fetcher.Start()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
err = fetcher.Wait()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
stats := fetcher.Stats().Snapshot.(*http.Stats)
|
|
// With slow-start strategy, connection count may be less than max if download is fast
|
|
// Just verify we have at least 1 connection and no more than max
|
|
if len(stats.Connections) < 1 || len(stats.Connections) > 16 {
|
|
t.Errorf("Stats() connection count got = %v, want between 1 and 16", len(stats.Connections))
|
|
}
|
|
totalDownloaded := int64(0)
|
|
expectedConnectionTotal := test.BuildSize / int64(len(stats.Connections))
|
|
if test.BuildSize%int64(len(stats.Connections)) != 0 {
|
|
expectedConnectionTotal++
|
|
}
|
|
for i, conn := range stats.Connections {
|
|
t.Logf("Connection %d: Downloaded=%d, Completed=%v", i, conn.Downloaded, conn.Completed)
|
|
totalDownloaded += conn.Downloaded
|
|
if conn.Total != expectedConnectionTotal {
|
|
t.Errorf("connection %d total = %d, want shared total %d", i, conn.Total, expectedConnectionTotal)
|
|
}
|
|
}
|
|
if totalDownloaded != test.BuildSize {
|
|
t.Errorf("Stats() got = %v, want %v", totalDownloaded, test.BuildSize)
|
|
}
|
|
}
|
|
|
|
func TestFetcher_StatsMarksTerminalNonFailedConnectionsComplete(t *testing.T) {
|
|
fetcher := &Fetcher{
|
|
meta: &fetcher.FetcherMeta{Res: &base.Resource{Size: 4096, Range: true}},
|
|
connections: []*connection{
|
|
{State: connDownloading, Chunk: newChunk(0, 2047), Downloaded: 1024},
|
|
{State: connCompleted, Chunk: newChunk(2048, 3071), Downloaded: 2048},
|
|
{State: connFailed, Chunk: newChunk(3072, 4095), Downloaded: 512, failed: true, retryTimes: 3},
|
|
},
|
|
}
|
|
fetcher.setState(stateDone)
|
|
|
|
stats := fetcher.Stats().Snapshot.(*http.Stats)
|
|
if !stats.Connections[0].Completed {
|
|
t.Fatal("terminal non-failed connection was reported as downloading")
|
|
}
|
|
if !stats.Connections[1].Completed {
|
|
t.Fatal("completed connection was not reported as completed")
|
|
}
|
|
if stats.Connections[2].Completed || !stats.Connections[2].Failed {
|
|
t.Fatal("failed connection lost its failure state")
|
|
}
|
|
const averageTotal = int64(1366) // ceil(4096 / 3)
|
|
for index, connection := range stats.Connections {
|
|
if connection.Total != averageTotal {
|
|
t.Fatalf("connection %d total = %d, want shared total %d", index, connection.Total, averageTotal)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestFetcher_StatsUsesUnknownTotalWithoutResourceSize(t *testing.T) {
|
|
fetcher := &Fetcher{
|
|
meta: &fetcher.FetcherMeta{Res: &base.Resource{}},
|
|
connections: []*connection{
|
|
{State: connDownloading, Chunk: newChunk(0, 0), Downloaded: 1024},
|
|
},
|
|
}
|
|
|
|
stats := fetcher.Stats().Snapshot.(*http.Stats)
|
|
if stats.Connections[0].Total != 0 {
|
|
t.Fatalf("unknown resource total = %d, want 0", stats.Connections[0].Total)
|
|
}
|
|
}
|
|
|
|
func TestFetcher_StatsExposesCompletedResolvePrefetch(t *testing.T) {
|
|
fetcher := &Fetcher{meta: &fetcher.FetcherMeta{Res: &base.Resource{Size: 100}}}
|
|
fetcher.resolveDataPos.Store(100)
|
|
fetcher.setState(stateDone)
|
|
|
|
stats := fetcher.Stats().Snapshot.(*http.Stats)
|
|
if len(stats.Connections) != 1 {
|
|
t.Fatalf("connection count = %d, want 1", len(stats.Connections))
|
|
}
|
|
connection := stats.Connections[0]
|
|
if connection.Downloaded != 100 || connection.Total != 100 || !connection.Completed {
|
|
t.Fatalf("unexpected prefetch connection: %+v", connection)
|
|
}
|
|
}
|
|
|
|
// TestFetcher_DownloadOneTimeURL tests downloading from a URL that can only be accessed once
|
|
// This simulates signed URLs or one-time download links that expire after first use
|
|
func TestFetcher_DownloadOneTimeURL(t *testing.T) {
|
|
listener := test.StartTestOneTimeServer()
|
|
defer listener.Close()
|
|
|
|
fetcher := buildFetcher()
|
|
fetcher.Meta().Opts = &base.Options{
|
|
Name: test.DownloadName,
|
|
Path: test.Dir,
|
|
Extra: &http.OptsExtra{
|
|
Connections: 4, // Try to use multiple connections, but only first should work
|
|
},
|
|
}
|
|
err := fetcher.Resolve(&base.Request{
|
|
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
err = fetcher.Start()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
err = fetcher.Wait()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
// Verify file content
|
|
want := test.FileMd5(test.BuildFile)
|
|
got := test.FileMd5(test.DownloadFile)
|
|
if want != got {
|
|
t.Errorf("Download() got = %v, want %v", got, want)
|
|
}
|
|
}
|
|
|
|
// TestFetcher_SlowStartExpansion tests slow-start connection expansion edge cases
|
|
// Tests that slow-start expansion reaches exactly maxConns
|
|
// Expansion pattern: 1 -> 2 -> 4 -> 8 -> 16...
|
|
// For max=5: 1 -> 2 -> 4 -> 5 (capped)
|
|
// For max=9: 1 -> 2 -> 4 -> 8 -> 9 (capped)
|
|
func TestFetcher_SlowStartExpansion(t *testing.T) {
|
|
testCases := []struct {
|
|
name string
|
|
maxConns int
|
|
}{
|
|
{"MaxConns5", 5}, // 1->2->4->5
|
|
{"MaxConns9", 9}, // 1->2->4->8->9
|
|
{"MaxConns8", 8}, // 1->2->4->8
|
|
}
|
|
|
|
for _, tc := range testCases {
|
|
tc := tc // capture range variable
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
// Clean up any leftover files from previous tests
|
|
os.Remove(test.DownloadFile)
|
|
|
|
// Use 100ns delay per byte for faster test (~10MB/s theoretical)
|
|
listener := test.StartTestLowSpeedServer(100 * time.Nanosecond)
|
|
|
|
// Ensure cleanup happens before next subtest
|
|
cleanup := func() {
|
|
listener.Close()
|
|
os.Remove(test.DownloadFile)
|
|
// Wait for server to fully stop
|
|
time.Sleep(50 * time.Millisecond)
|
|
}
|
|
|
|
fetcher := buildConfigFetcher(config{
|
|
Connections: tc.maxConns,
|
|
})
|
|
|
|
fetcher.Meta().Opts = &base.Options{
|
|
Name: test.DownloadName,
|
|
Path: test.Dir,
|
|
Extra: &http.OptsExtra{
|
|
Connections: tc.maxConns,
|
|
},
|
|
}
|
|
err := fetcher.Resolve(&base.Request{
|
|
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
|
|
})
|
|
if err != nil {
|
|
cleanup()
|
|
t.Fatal(err)
|
|
}
|
|
|
|
err = fetcher.Start()
|
|
if err != nil {
|
|
cleanup()
|
|
t.Fatal(err)
|
|
}
|
|
|
|
err = fetcher.Wait()
|
|
if err != nil {
|
|
t.Logf("Wait() returned error: %v", err)
|
|
cleanup()
|
|
t.Fatal(err)
|
|
}
|
|
|
|
// Check final connection count equals maxConns exactly
|
|
stats := fetcher.Stats().Snapshot.(*http.Stats)
|
|
finalConns := len(stats.Connections)
|
|
|
|
// Debug: show connection details and metadata
|
|
httpFetcher := fetcher.(*Fetcher)
|
|
t.Logf("Resource: Size=%d, Range=%v", httpFetcher.Meta().Res.Size, httpFetcher.Meta().Res.Range)
|
|
for i, conn := range stats.Connections {
|
|
t.Logf("Connection %d: Downloaded=%d, Completed=%v", i, conn.Downloaded, conn.Completed)
|
|
}
|
|
|
|
if finalConns != tc.maxConns {
|
|
t.Errorf("Expected exactly %d connections, got %d", tc.maxConns, finalConns)
|
|
}
|
|
|
|
// Verify file content before cleanup
|
|
want := test.FileMd5(test.BuildFile)
|
|
got := test.FileMd5(test.DownloadFile)
|
|
if want != got {
|
|
t.Errorf("Download() got = %v, want %v", got, want)
|
|
}
|
|
|
|
cleanup()
|
|
})
|
|
}
|
|
}
|
|
|
|
// TestFetcher_AsyncPrefetch tests the async prefetch functionality
|
|
// where data is downloaded in background during resolve phase and reused in start
|
|
func TestFetcher_AsyncPrefetch(t *testing.T) {
|
|
// Test 1: Prefetch completes entire file before Start is called
|
|
t.Run("PrefetchComplete", func(t *testing.T) {
|
|
listener := test.StartTestFileServer()
|
|
defer listener.Close()
|
|
|
|
fetcher := buildFetcher()
|
|
fetcher.Meta().Opts = &base.Options{
|
|
Name: test.DownloadName,
|
|
Path: test.Dir,
|
|
Extra: &http.OptsExtra{
|
|
Connections: 4,
|
|
},
|
|
}
|
|
err := fetcher.Resolve(&base.Request{
|
|
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
// Poll until prefetch completes the entire file (with timeout)
|
|
timeout := time.After(30 * time.Second)
|
|
ticker := time.NewTicker(100 * time.Millisecond)
|
|
defer ticker.Stop()
|
|
pollLoop:
|
|
for {
|
|
select {
|
|
case <-timeout:
|
|
t.Fatal("Timeout waiting for prefetch to complete")
|
|
case <-ticker.C:
|
|
if fetcher.prefetchSize.Load() >= test.BuildSize {
|
|
break pollLoop
|
|
}
|
|
}
|
|
}
|
|
|
|
prefetchedBefore := fetcher.prefetchSize.Load()
|
|
t.Logf("Prefetched bytes before Start: %d (%.2f MB)", prefetchedBefore, float64(prefetchedBefore)/(1024*1024))
|
|
|
|
// Should have prefetched the entire file
|
|
if prefetchedBefore != test.BuildSize {
|
|
t.Errorf("Prefetch should complete entire file, got %d, want %d", prefetchedBefore, test.BuildSize)
|
|
}
|
|
|
|
// Now start the download
|
|
err = fetcher.Start()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
// Wait for download to complete
|
|
err = fetcher.Wait()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
// Check how much was utilized from prefetch
|
|
prefetchedUsed := fetcher.resolveDataPos.Load()
|
|
t.Logf("Prefetched bytes used: %d (%.2f MB)", prefetchedUsed, float64(prefetchedUsed)/(1024*1024))
|
|
|
|
// Verify file is correct
|
|
want := test.FileMd5(test.BuildFile)
|
|
got := test.FileMd5(test.DownloadFile)
|
|
if want != got {
|
|
t.Errorf("Download() got = %v, want %v", got, want)
|
|
}
|
|
|
|
os.Remove(test.DownloadFile)
|
|
})
|
|
|
|
// Test 2: Prefetch only downloads partial data before Start is called
|
|
t.Run("PrefetchPartial", func(t *testing.T) {
|
|
// Use slow server with 100 nanosecond delay per byte
|
|
// This means ~10MB/s speed, so 100ms should download ~1MB
|
|
listener := test.StartTestLowSpeedServer(100 * time.Nanosecond)
|
|
defer listener.Close()
|
|
|
|
fetcher := buildFetcher()
|
|
fetcher.Meta().Opts = &base.Options{
|
|
Name: test.DownloadName,
|
|
Path: test.Dir,
|
|
Extra: &http.OptsExtra{
|
|
Connections: 4,
|
|
},
|
|
}
|
|
err := fetcher.Resolve(&base.Request{
|
|
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
// Wait only 100ms - should only prefetch a small portion
|
|
time.Sleep(100 * time.Millisecond)
|
|
|
|
prefetchedBefore := fetcher.prefetchSize.Load()
|
|
t.Logf("Prefetched bytes before Start: %d (%.2f KB)", prefetchedBefore, float64(prefetchedBefore)/1024)
|
|
|
|
// Verify we have partial data (not zero, but not complete)
|
|
if prefetchedBefore == 0 {
|
|
t.Log("Warning: No data prefetched, may be too slow")
|
|
}
|
|
if prefetchedBefore >= test.BuildSize {
|
|
t.Log("Warning: Prefetch completed entire file, test may not be valid")
|
|
}
|
|
|
|
// Now start the download
|
|
err = fetcher.Start()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
// Wait for download to complete
|
|
err = fetcher.Wait()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
// Check stats - should have connections that downloaded remaining data
|
|
stats := fetcher.Stats().Snapshot.(*http.Stats)
|
|
t.Logf("Final connections: %d", len(stats.Connections))
|
|
|
|
prefetchedUsed := fetcher.resolveDataPos.Load()
|
|
t.Logf("Prefetched bytes used: %d (%.2f KB)", prefetchedUsed, float64(prefetchedUsed)/1024)
|
|
|
|
// Verify connections picked up where prefetch left off
|
|
if len(stats.Connections) > 0 {
|
|
firstConn := stats.Connections[0]
|
|
t.Logf("First connection downloaded: %d bytes", firstConn.Downloaded)
|
|
}
|
|
|
|
// Verify file is correct
|
|
want := test.FileMd5(test.BuildFile)
|
|
got := test.FileMd5(test.DownloadFile)
|
|
if want != got {
|
|
t.Errorf("Download() got = %v, want %v", got, want)
|
|
}
|
|
|
|
os.Remove(test.DownloadFile)
|
|
})
|
|
}
|
|
|
|
// TestFetcher_DownloadExpiringRedirectURL tests that the fetcher correctly handles
|
|
// expiring redirect URLs by falling back to the original URL and getting a new redirect.
|
|
func TestFetcher_DownloadExpiringRedirectURL(t *testing.T) {
|
|
// Test with redirect expiring after 2 requests
|
|
// This means:
|
|
// - Request 1: Resolve (original URL redirects to temp URL v1)
|
|
// - Request 2: First download request to temp URL v1 succeeds
|
|
// - Request 3: Second download request to temp URL v1 returns 403
|
|
// - Fetcher should then retry with original URL, get temp URL v2
|
|
// - Continue downloading from temp URL v2
|
|
t.Run("RedirectExpiresAfter2Requests", func(t *testing.T) {
|
|
os.Remove(test.DownloadFile)
|
|
|
|
// Create server with redirect expiring after 2 requests, with slow transfer to ensure
|
|
// multiple connection attempts are needed
|
|
listener := test.StartTestExpiringRedirectServer(2, 100*time.Nanosecond)
|
|
defer listener.Close()
|
|
|
|
fetcher := buildFetcher()
|
|
fetcher.Meta().Opts = &base.Options{
|
|
Name: test.DownloadName,
|
|
Path: test.Dir,
|
|
Extra: &http.OptsExtra{
|
|
Connections: 4, // Use multiple connections to trigger redirect expiration
|
|
},
|
|
}
|
|
err := fetcher.Resolve(&base.Request{
|
|
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
err = fetcher.Start()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
err = fetcher.Wait()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
// Verify file content is correct despite redirect expiration
|
|
want := test.FileMd5(test.BuildFile)
|
|
got := test.FileMd5(test.DownloadFile)
|
|
if want != got {
|
|
t.Errorf("Download() got = %v, want %v", got, want)
|
|
}
|
|
|
|
os.Remove(test.DownloadFile)
|
|
})
|
|
|
|
// Test with redirect expiring after 5 requests (more room for initial connections)
|
|
t.Run("RedirectExpiresAfter5Requests", func(t *testing.T) {
|
|
os.Remove(test.DownloadFile)
|
|
|
|
listener := test.StartTestExpiringRedirectServer(5, 100*time.Nanosecond)
|
|
defer listener.Close()
|
|
|
|
fetcher := buildFetcher()
|
|
fetcher.Meta().Opts = &base.Options{
|
|
Name: test.DownloadName,
|
|
Path: test.Dir,
|
|
Extra: &http.OptsExtra{
|
|
Connections: 8,
|
|
},
|
|
}
|
|
err := fetcher.Resolve(&base.Request{
|
|
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
err = fetcher.Start()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
err = fetcher.Wait()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
// Verify file content
|
|
want := test.FileMd5(test.BuildFile)
|
|
got := test.FileMd5(test.DownloadFile)
|
|
if want != got {
|
|
t.Errorf("Download() got = %v, want %v", got, want)
|
|
}
|
|
|
|
os.Remove(test.DownloadFile)
|
|
})
|
|
}
|
|
|
|
// TestFetcher_RetryAfterError tests that the fetcher can retry downloading
|
|
// after a previous download attempt failed by calling Start() again.
|
|
func TestFetcher_RetryAfterError(t *testing.T) {
|
|
os.Remove(test.DownloadFile)
|
|
|
|
// Server fails first 3 requests (after resolve), then recovers
|
|
// With 1 connection and 3 retries:
|
|
// - First Start(): requests 2, 3, 4 → all fail (3 retries exhausted) → returns error
|
|
// - Second Start(): request 5 → succeeds (server recovered after 3 failures)
|
|
listener := test.StartTestFailThenRecoverServer(3)
|
|
defer listener.Close()
|
|
|
|
fetcher := buildFetcher()
|
|
fetcher.Meta().Opts = &base.Options{
|
|
Name: test.DownloadName,
|
|
Path: test.Dir,
|
|
Extra: &http.OptsExtra{
|
|
Connections: 1, // Use single connection to simplify test
|
|
},
|
|
}
|
|
err := fetcher.Resolve(&base.Request{
|
|
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
// This test exercises retrying a failed ranged download. Discard the live
|
|
// Resolve response first so it cannot satisfy the task as a fallback.
|
|
if err := fetcher.Pause(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
// First download attempt - should fail because server returns 416 after resolve
|
|
err = fetcher.Start()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
err = fetcher.Wait()
|
|
// First attempt should fail with 416 error
|
|
if err == nil {
|
|
t.Fatal("Expected first download attempt to fail, but it succeeded")
|
|
}
|
|
t.Logf("First attempt failed as expected: %v", err)
|
|
|
|
// Check that fetcher is in error state
|
|
state := fetcher.getState()
|
|
if state != stateError {
|
|
t.Errorf("Expected fetcher to be in stateError, got %v", state)
|
|
}
|
|
|
|
// Verify that we can call Start() again after error
|
|
// This tests the stateError handling in Start()
|
|
err = fetcher.Start()
|
|
if err != nil {
|
|
t.Fatalf("Start() after error failed: %v", err)
|
|
}
|
|
|
|
// Wait for second attempt - should succeed now that server has recovered
|
|
err = fetcher.Wait()
|
|
if err != nil {
|
|
t.Fatalf("Retry failed: %v", err)
|
|
}
|
|
|
|
// Verify file content
|
|
want := test.FileMd5(test.BuildFile)
|
|
got := test.FileMd5(test.DownloadFile)
|
|
if want != got {
|
|
t.Errorf("Download() got = %v, want %v", got, want)
|
|
}
|
|
|
|
os.Remove(test.DownloadFile)
|
|
}
|
|
|
|
func TestFetcherManager_ParseName(t *testing.T) {
|
|
type args struct {
|
|
u string
|
|
}
|
|
tests := []struct {
|
|
name string
|
|
args args
|
|
want string
|
|
}{
|
|
{
|
|
name: "broken url",
|
|
args: args{
|
|
u: "https://!@#%github.com",
|
|
},
|
|
want: "",
|
|
},
|
|
{
|
|
name: "file path",
|
|
args: args{
|
|
u: "https://github.com/index.html",
|
|
},
|
|
want: "index.html",
|
|
},
|
|
{
|
|
name: "file path with query and hash",
|
|
args: args{
|
|
u: "https://github.com/a/b/index.html/#list?name=1",
|
|
},
|
|
want: "index.html",
|
|
},
|
|
{
|
|
name: "no file path",
|
|
args: args{
|
|
u: "https://github.com",
|
|
},
|
|
want: "github.com",
|
|
},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
fm := &FetcherManager{}
|
|
if got := fm.ParseName(tt.args.u); got != tt.want {
|
|
t.Errorf("ParseName() = %v, want %v", got, tt.want)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestFetcherManager_StoreCreatesAtomicSnapshot(t *testing.T) {
|
|
f := &Fetcher{
|
|
meta: &fetcher.FetcherMeta{
|
|
Req: &base.Request{},
|
|
Res: &base.Resource{
|
|
Size: 100,
|
|
Range: true,
|
|
Files: []*base.FileInfo{{Size: 100}},
|
|
},
|
|
Opts: &base.Options{},
|
|
},
|
|
ifRange: `"snapshot-v1"`,
|
|
rangeValidatorPinned: true,
|
|
connections: []*connection{{
|
|
ID: 0,
|
|
Role: rolePrimary,
|
|
State: connDownloading,
|
|
Chunk: newChunk(0, 99),
|
|
}},
|
|
}
|
|
fm := new(FetcherManager)
|
|
|
|
stop := make(chan struct{})
|
|
done := make(chan struct{})
|
|
go func() {
|
|
defer close(done)
|
|
sequential := false
|
|
for {
|
|
select {
|
|
case <-stop:
|
|
return
|
|
default:
|
|
}
|
|
f.connMu.Lock()
|
|
if sequential {
|
|
f.meta.Res.Range = false
|
|
f.meta.Res.Size = 0
|
|
f.meta.Res.Files[0].Size = 0
|
|
f.rangeReprobeEligible = true
|
|
f.rangeValidatorPinned = false
|
|
f.sequentialSizeUnknown = true
|
|
f.connections[0].Downloaded = 64
|
|
f.connections[0].Chunk.Downloaded = 64
|
|
} else {
|
|
f.meta.Res.Range = true
|
|
f.meta.Res.Size = 100
|
|
f.meta.Res.Files[0].Size = 100
|
|
f.rangeReprobeEligible = false
|
|
f.rangeValidatorPinned = true
|
|
f.sequentialSizeUnknown = false
|
|
f.connections[0].Downloaded = 0
|
|
f.connections[0].Chunk.Downloaded = 0
|
|
}
|
|
f.connMu.Unlock()
|
|
sequential = !sequential
|
|
}
|
|
}()
|
|
|
|
for i := 0; i < 1000; i++ {
|
|
data, err := fm.Store(f)
|
|
if err != nil {
|
|
close(stop)
|
|
<-done
|
|
t.Fatal(err)
|
|
}
|
|
fd := data.(*fetcherData)
|
|
if fd.Range == nil {
|
|
close(stop)
|
|
<-done
|
|
t.Fatal("snapshot Range mode is nil")
|
|
}
|
|
if fd.ResourceSize == nil || fd.FileSize == nil {
|
|
close(stop)
|
|
<-done
|
|
t.Fatal("snapshot sizes are nil")
|
|
}
|
|
if len(fd.Connections) != 1 || fd.Connections[0] == nil || fd.Connections[0].Chunk == nil {
|
|
close(stop)
|
|
<-done
|
|
t.Fatalf("invalid connection snapshot: %#v", fd.Connections)
|
|
}
|
|
if fd.Connections[0] == f.connections[0] || fd.Connections[0].Chunk == f.connections[0].Chunk {
|
|
close(stop)
|
|
<-done
|
|
t.Fatal("Store returned live connection state instead of a deep snapshot")
|
|
}
|
|
|
|
downloaded := fd.Connections[0].Chunk.Downloaded
|
|
if *fd.Range {
|
|
if fd.RangeReprobeEligible || !fd.RangeValidatorPinned || fd.SequentialSizeUnknown || downloaded != 0 || *fd.ResourceSize != 100 || *fd.FileSize != 100 {
|
|
close(stop)
|
|
<-done
|
|
t.Fatalf("mixed ranged snapshot: eligible=%v pinned=%v unknown=%v downloaded=%d size=%d/%d", fd.RangeReprobeEligible, fd.RangeValidatorPinned, fd.SequentialSizeUnknown, downloaded, *fd.ResourceSize, *fd.FileSize)
|
|
}
|
|
} else if !fd.RangeReprobeEligible || fd.RangeValidatorPinned || !fd.SequentialSizeUnknown || downloaded != 64 || *fd.ResourceSize != 0 || *fd.FileSize != 0 {
|
|
close(stop)
|
|
<-done
|
|
t.Fatalf("mixed sequential snapshot: eligible=%v pinned=%v unknown=%v downloaded=%d size=%d/%d", fd.RangeReprobeEligible, fd.RangeValidatorPinned, fd.SequentialSizeUnknown, downloaded, *fd.ResourceSize, *fd.FileSize)
|
|
}
|
|
}
|
|
close(stop)
|
|
<-done
|
|
}
|
|
|
|
func TestFetcherManager_RestoreUsesSnapshotRangeMode(t *testing.T) {
|
|
fm := new(FetcherManager)
|
|
rangeMode := false
|
|
resourceSize := int64(0)
|
|
fileSize := int64(0)
|
|
saved := &fetcherData{
|
|
Connections: []*connection{{
|
|
ID: 0,
|
|
Role: rolePrimary,
|
|
State: connFailed,
|
|
Chunk: newChunk(0, 99),
|
|
Downloaded: 64,
|
|
}},
|
|
IfRange: `"snapshot-v1"`,
|
|
RangeReprobeEligible: true,
|
|
SequentialSizeUnknown: true,
|
|
Range: &rangeMode,
|
|
ResourceSize: &resourceSize,
|
|
FileSize: &fileSize,
|
|
}
|
|
saved.Connections[0].Chunk.Downloaded = 64
|
|
encoded, err := json.Marshal(saved)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var persisted fetcherData
|
|
if err := json.Unmarshal(encoded, &persisted); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
staleMeta := &fetcher.FetcherMeta{
|
|
Req: &base.Request{},
|
|
Res: &base.Resource{
|
|
Size: 100,
|
|
Range: true,
|
|
Files: []*base.FileInfo{{Size: 100}},
|
|
},
|
|
Opts: &base.Options{},
|
|
}
|
|
_, restore := fm.Restore()
|
|
restored := restore(staleMeta, &persisted).(*Fetcher)
|
|
if restored.meta.Res.Range {
|
|
t.Fatal("Restore kept stale task Range mode instead of the connection snapshot mode")
|
|
}
|
|
if !restored.rangeReprobeEligible || !restored.sequentialSizeUnknown || restored.ifRange != `"snapshot-v1"` {
|
|
t.Fatalf("restored recovery state = eligible:%v unknown:%v If-Range:%q", restored.rangeReprobeEligible, restored.sequentialSizeUnknown, restored.ifRange)
|
|
}
|
|
if restored.meta.Res.Size != 0 || restored.meta.Res.Files[0].Size != 0 {
|
|
t.Fatalf("restored snapshot size = %d/%d, want 0/0", restored.meta.Res.Size, restored.meta.Res.Files[0].Size)
|
|
}
|
|
if got := restored.connections[0].Chunk.Downloaded; got != 64 {
|
|
t.Fatalf("restored prefix = %d, want 64", got)
|
|
}
|
|
restored.connMu.Lock()
|
|
canProbe := restored.canProbeSequentialResumeLocked(restored.connections[0])
|
|
restored.connMu.Unlock()
|
|
if canProbe {
|
|
t.Fatal("unknown-size restored state attempted an If-Range resume against stale size")
|
|
}
|
|
|
|
pinnedRangeMode := true
|
|
pinnedMeta := &fetcher.FetcherMeta{
|
|
Req: &base.Request{},
|
|
Res: &base.Resource{Size: 100, Range: false},
|
|
Opts: &base.Options{},
|
|
}
|
|
pinned := restore(pinnedMeta, &fetcherData{
|
|
IfRange: `"snapshot-v2"`,
|
|
RangeValidatorPinned: true,
|
|
Range: &pinnedRangeMode,
|
|
}).(*Fetcher)
|
|
if !pinned.meta.Res.Range || !pinned.rangeValidatorPinned || pinned.ifRange != `"snapshot-v2"` {
|
|
t.Fatalf("restored pinned range state = range:%v pinned:%v If-Range:%q", pinned.meta.Res.Range, pinned.rangeValidatorPinned, pinned.ifRange)
|
|
}
|
|
|
|
// Records written before the Range snapshot field existed must retain the
|
|
// mode from task metadata for backward compatibility.
|
|
legacyMeta := &fetcher.FetcherMeta{
|
|
Req: &base.Request{},
|
|
Res: &base.Resource{Size: 100, Range: true},
|
|
Opts: &base.Options{},
|
|
}
|
|
legacy := restore(legacyMeta, &fetcherData{}).(*Fetcher)
|
|
if !legacy.meta.Res.Range {
|
|
t.Fatal("legacy record unexpectedly overrode task Range mode")
|
|
}
|
|
}
|
|
|
|
func TestFetcherManager_RestoreUsesAtomicSizeSnapshotForDownload(t *testing.T) {
|
|
original := bytes.Repeat([]byte("old-representation"), 16*1024)
|
|
replacement := bytes.Repeat([]byte("new-representation"), 12*1024)
|
|
const savedPrefix = 64 * 1024
|
|
const replacementPrefix = 96 * 1024
|
|
var sawGuardedProbe atomic.Bool
|
|
|
|
server := httptest.NewServer(gohttp.HandlerFunc(func(w gohttp.ResponseWriter, r *gohttp.Request) {
|
|
wantRange := fmt.Sprintf("bytes=%d-%d", savedPrefix, len(original)-1)
|
|
if got := r.Header.Get(base.HttpHeaderRange); got != wantRange {
|
|
t.Errorf("Range = %q, want %q", got, wantRange)
|
|
}
|
|
if got := r.Header.Get(base.HttpHeaderIfRange); got != `"old-v1"` {
|
|
t.Errorf("If-Range = %q, want %q", got, `"old-v1"`)
|
|
}
|
|
sawGuardedProbe.Store(true)
|
|
w.Header().Set(base.HttpHeaderETag, `"new-v2"`)
|
|
w.Header().Set(base.HttpHeaderContentLength, fmt.Sprintf("%d", len(replacement)))
|
|
w.WriteHeader(gohttp.StatusOK)
|
|
_, _ = w.Write(replacement)
|
|
}))
|
|
defer server.Close()
|
|
|
|
tempDir := t.TempDir()
|
|
const filename = "restored-replacement.data"
|
|
filepath := tempDir + string(os.PathSeparator) + filename
|
|
if err := os.WriteFile(filepath, replacement[:replacementPrefix], 0o644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
rangeMode := false
|
|
resourceSize := int64(len(original))
|
|
fileSize := resourceSize
|
|
saved := &fetcherData{
|
|
Connections: []*connection{{
|
|
ID: 0,
|
|
Role: rolePrimary,
|
|
State: connFailed,
|
|
Chunk: newChunk(0, resourceSize-1),
|
|
Downloaded: savedPrefix,
|
|
}},
|
|
IfRange: `"old-v1"`,
|
|
RangeReprobeEligible: true,
|
|
Range: &rangeMode,
|
|
ResourceSize: &resourceSize,
|
|
FileSize: &fileSize,
|
|
}
|
|
saved.Connections[0].Chunk.Downloaded = savedPrefix
|
|
// Simulate task metadata serialized after an interrupted chunked replacement
|
|
// while the fetcher snapshot still describes the preceding representation.
|
|
staleMeta := &fetcher.FetcherMeta{
|
|
Req: &base.Request{URL: server.URL},
|
|
Res: &base.Resource{
|
|
Size: 0,
|
|
Range: false,
|
|
Files: []*base.FileInfo{{Name: filename, Size: 0}},
|
|
},
|
|
Opts: &base.Options{
|
|
Path: tempDir,
|
|
Name: filename,
|
|
Extra: &http.OptsExtra{
|
|
Connections: 1,
|
|
},
|
|
},
|
|
}
|
|
|
|
fm := new(FetcherManager)
|
|
_, restore := fm.Restore()
|
|
restored := restore(staleMeta, saved).(*Fetcher)
|
|
if restored.meta.Res.Size != resourceSize || restored.meta.Res.Files[0].Size != fileSize {
|
|
t.Fatalf("restored size = %d/%d, want %d/%d", restored.meta.Res.Size, restored.meta.Res.Files[0].Size, resourceSize, fileSize)
|
|
}
|
|
ctl := controller.NewController()
|
|
ctl.GetConfig = func(v any) {
|
|
if err := json.Unmarshal([]byte(test.ToJson(fm.DefaultConfig())), v); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
restored.Setup(ctl)
|
|
if err := restored.Start(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := restored.Wait(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
if !sawGuardedProbe.Load() {
|
|
t.Fatal("restored download did not issue the guarded Range probe")
|
|
}
|
|
got, err := os.ReadFile(filepath)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !bytes.Equal(got, replacement) {
|
|
t.Fatal("restored download retained bytes from the interrupted replacement")
|
|
}
|
|
wantSize := int64(len(replacement))
|
|
if restored.meta.Res.Size != wantSize || restored.meta.Res.Files[0].Size != wantSize {
|
|
t.Fatalf("completed size = %d/%d, want %d", restored.meta.Res.Size, restored.meta.Res.Files[0].Size, wantSize)
|
|
}
|
|
if got := restored.Progress().TotalDownloaded(); got != wantSize {
|
|
t.Fatalf("progress = %d, want %d", got, wantSize)
|
|
}
|
|
info, err := os.Stat(filepath)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if got := info.Size(); got != wantSize {
|
|
t.Fatalf("file size = %d, want %d", got, wantSize)
|
|
}
|
|
}
|
|
|
|
func downloadReady(listener net.Listener, connections int, t *testing.T) fetcher.Fetcher {
|
|
return doDownloadReady(buildFetcher(), listener, connections, t)
|
|
}
|
|
|
|
func doDownloadReady(f fetcher.Fetcher, listener net.Listener, connections int, t *testing.T) fetcher.Fetcher {
|
|
var extra any = nil
|
|
if connections > 0 {
|
|
extra = &http.OptsExtra{
|
|
Connections: connections,
|
|
}
|
|
}
|
|
opts := &base.Options{
|
|
Name: test.DownloadName,
|
|
Path: test.Dir,
|
|
Extra: extra,
|
|
}
|
|
f.Meta().Opts = opts
|
|
err := f.Resolve(&base.Request{
|
|
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return f
|
|
}
|
|
|
|
func downloadNormal(listener net.Listener, connections int, t *testing.T) {
|
|
fetcher := downloadReady(listener, connections, t)
|
|
err := fetcher.Start()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
err = fetcher.Wait()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
want := test.FileMd5(test.BuildFile)
|
|
got := test.FileMd5(test.DownloadFile)
|
|
if want != got {
|
|
t.Errorf("Download() got = %v, want %v", got, want)
|
|
}
|
|
}
|
|
|
|
func downloadPost(listener net.Listener, connections int, t *testing.T) {
|
|
// POST parameters must be set before Resolve since the new design
|
|
// starts downloading during Resolve phase
|
|
f := buildFetcher()
|
|
var extra any = nil
|
|
if connections > 0 {
|
|
extra = &http.OptsExtra{
|
|
Connections: connections,
|
|
}
|
|
}
|
|
opts := &base.Options{
|
|
Name: test.DownloadName,
|
|
Path: test.Dir,
|
|
Extra: extra,
|
|
}
|
|
req := &base.Request{
|
|
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
|
|
Extra: &http.ReqExtra{
|
|
Method: "POST",
|
|
Header: map[string]string{
|
|
"Authorization": "Bearer 123456",
|
|
},
|
|
Body: fmt.Sprintf(`{"name":"%s"}`, test.BuildName),
|
|
},
|
|
}
|
|
f.Meta().Opts = opts
|
|
err := f.Resolve(req)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
err = f.Start()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
err = f.Wait()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
want := test.FileMd5(test.BuildFile)
|
|
got := test.FileMd5(test.DownloadFile)
|
|
if want != got {
|
|
t.Errorf("Download() got = %v, want %v", got, want)
|
|
}
|
|
}
|
|
|
|
func downloadContinue(listener net.Listener, connections int, t *testing.T) {
|
|
fetcher := downloadReady(listener, connections, t)
|
|
err := fetcher.Start()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
time.Sleep(time.Millisecond * 50)
|
|
if err := fetcher.Pause(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
time.Sleep(time.Millisecond * 50)
|
|
if err := fetcher.Start(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
err = fetcher.Wait()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
want := test.FileMd5(test.BuildFile)
|
|
got := test.FileMd5(test.DownloadFile)
|
|
if want != got {
|
|
t.Errorf("Download() got = %v, want %v", got, want)
|
|
}
|
|
}
|
|
|
|
func downloadError(listener net.Listener, connections int, t *testing.T) {
|
|
fetcher := buildFetcher()
|
|
fetcher.Meta().Opts = &base.Options{
|
|
Name: test.DownloadName,
|
|
Path: test.Dir,
|
|
}
|
|
err := fetcher.Resolve(&base.Request{
|
|
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
|
|
})
|
|
// With the new async design, Resolve may succeed (HTTP response received)
|
|
// but errors occur during async download or Start/Wait
|
|
if err != nil {
|
|
// Error detected in Resolve - this is fine
|
|
return
|
|
}
|
|
|
|
// Resolve succeeded, error should occur during Start/Wait
|
|
err = fetcher.Start()
|
|
if err != nil {
|
|
// Error detected in Start - this is fine
|
|
return
|
|
}
|
|
|
|
err = fetcher.Wait()
|
|
if err == nil {
|
|
t.Errorf("Expected error during download, but got none")
|
|
}
|
|
}
|
|
|
|
func downloadResume(listener net.Listener, connections int, t *testing.T) {
|
|
fetcher := downloadReady(listener, connections, t)
|
|
err := fetcher.Start()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
fb := new(FetcherManager)
|
|
time.Sleep(time.Millisecond * 50)
|
|
data, err := fb.Store(fetcher)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
time.Sleep(time.Millisecond * 50)
|
|
fetcher.Pause()
|
|
|
|
_, f := fb.Restore()
|
|
f(fetcher.Meta(), data)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
fetcher.Setup(controller.NewController())
|
|
fetcher.Start()
|
|
|
|
err = fetcher.Wait()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
want := test.FileMd5(test.BuildFile)
|
|
got := test.FileMd5(test.DownloadFile)
|
|
if want != got {
|
|
t.Errorf("Download() got = %v, want %v", got, want)
|
|
}
|
|
}
|
|
|
|
func downloadWithProxy(httpListener net.Listener, proxyListener net.Listener, t *testing.T) {
|
|
// Setup must run before Resolve, as in the downloader, so the Resolve
|
|
// request also goes through the proxy.
|
|
f := buildFetcher()
|
|
f.ctl.GetProxy = func(requestProxy *base.RequestProxy) func(*gohttp.Request) (*url.URL, error) {
|
|
return (&base.DownloaderProxyConfig{
|
|
Enable: true,
|
|
Scheme: "socks5",
|
|
Host: proxyListener.Addr().String(),
|
|
}).ToHandler()
|
|
}
|
|
fetcher := doDownloadReady(f, httpListener, 4, t)
|
|
err := fetcher.Start()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
err = fetcher.Wait()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
want := test.FileMd5(test.BuildFile)
|
|
got := test.FileMd5(test.DownloadFile)
|
|
if want != got {
|
|
t.Errorf("Download() got = %v, want %v", got, want)
|
|
}
|
|
}
|
|
|
|
func buildFetcher() *Fetcher {
|
|
fm := new(FetcherManager)
|
|
fetcher := fm.Build()
|
|
newController := controller.NewController()
|
|
newController.GetConfig = func(v any) {
|
|
json.Unmarshal([]byte(test.ToJson(fm.DefaultConfig())), v)
|
|
}
|
|
fetcher.Setup(newController)
|
|
return fetcher.(*Fetcher)
|
|
}
|
|
|
|
func buildConfigFetcher(cfg config) fetcher.Fetcher {
|
|
fetcher := new(FetcherManager).Build()
|
|
newController := controller.NewController()
|
|
newController.GetConfig = func(v any) {
|
|
json.Unmarshal([]byte(test.ToJson(cfg)), v)
|
|
}
|
|
fetcher.Setup(newController)
|
|
return fetcher
|
|
}
|
|
|
|
// TestFetcher_Patch_URLChange tests the Patch functionality where a failed download URL
|
|
// is replaced with a working one. This simulates:
|
|
// 1. Initial download attempt with a bad URL (returns 404)
|
|
// 2. Patching the task with a new working URL
|
|
// 3. Successful download after URL modification
|
|
func TestFetcher_Patch_URLChange(t *testing.T) {
|
|
listener := test.StartTestPatchURLServer()
|
|
defer listener.Close()
|
|
|
|
f := buildFetcher()
|
|
badURL := "http://" + listener.Addr().String() + "/bad-url"
|
|
goodURL := "http://" + listener.Addr().String() + "/good-url"
|
|
|
|
opts := &base.Options{
|
|
Name: test.DownloadName,
|
|
Path: test.Dir,
|
|
Extra: &http.OptsExtra{Connections: 1},
|
|
}
|
|
|
|
// Step 1: Try to resolve with bad URL - should fail with error
|
|
f.Meta().Opts = opts
|
|
err := f.Resolve(&base.Request{URL: badURL})
|
|
if err == nil {
|
|
t.Fatal("Expected error for bad URL, got nil")
|
|
}
|
|
|
|
// Step 2: Create a new fetcher and resolve with bad URL but don't wait
|
|
// We need to test patching a task that has been created
|
|
f2 := buildFetcher()
|
|
|
|
// First resolve with good URL to create a valid fetcher state
|
|
f2.Meta().Opts = opts
|
|
err = f2.Resolve(&base.Request{URL: goodURL})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
// Verify initial URL
|
|
if f2.meta.Req.URL != goodURL {
|
|
t.Errorf("Initial URL = %v, want %v", f2.meta.Req.URL, goodURL)
|
|
}
|
|
|
|
// Step 3: Patch to change URL (simulating URL change scenario)
|
|
newURL := "http://" + listener.Addr().String() + "/good-url"
|
|
err = f2.Patch(&base.Request{URL: newURL}, nil)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
// Verify URL was patched
|
|
if f2.meta.Req.URL != newURL {
|
|
t.Errorf("Patched URL = %v, want %v", f2.meta.Req.URL, newURL)
|
|
}
|
|
|
|
// Step 4: Start download and verify success
|
|
err = f2.Start()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
err = f2.Wait()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
// Verify file was downloaded correctly
|
|
want := test.FileMd5(test.BuildFile)
|
|
got := test.FileMd5(test.DownloadFile)
|
|
if want != got {
|
|
t.Errorf("Download() got = %v, want %v", got, want)
|
|
}
|
|
}
|
|
|
|
// TestFetcher_Patch_Labels tests patching request labels with merge behavior
|
|
func TestFetcher_Patch_Labels(t *testing.T) {
|
|
listener := test.StartTestFileServer()
|
|
defer listener.Close()
|
|
|
|
f := buildFetcher()
|
|
opts := &base.Options{
|
|
Name: test.DownloadName,
|
|
Path: test.Dir,
|
|
Extra: &http.OptsExtra{Connections: 1},
|
|
}
|
|
|
|
f.Meta().Opts = opts
|
|
err := f.Resolve(&base.Request{
|
|
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
|
|
Labels: map[string]string{
|
|
"key1": "value1",
|
|
"key3": "value3",
|
|
},
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
// Verify initial labels
|
|
if f.meta.Req.Labels["key1"] != "value1" {
|
|
t.Errorf("Initial label key1 = %v, want value1", f.meta.Req.Labels["key1"])
|
|
}
|
|
if f.meta.Req.Labels["key3"] != "value3" {
|
|
t.Errorf("Initial label key3 = %v, want value3", f.meta.Req.Labels["key3"])
|
|
}
|
|
|
|
// Patch with new labels - key1 should be overwritten, key2 should be added, key3 should remain
|
|
patchReq := &base.Request{
|
|
Labels: map[string]string{
|
|
"key1": "modified",
|
|
"key2": "newValue",
|
|
},
|
|
}
|
|
err = f.Patch(patchReq, nil)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
// Verify labels were merged correctly
|
|
if f.meta.Req.Labels["key1"] != "modified" {
|
|
t.Errorf("Patched label key1 = %v, want modified", f.meta.Req.Labels["key1"])
|
|
}
|
|
if f.meta.Req.Labels["key2"] != "newValue" {
|
|
t.Errorf("Patched label key2 = %v, want newValue", f.meta.Req.Labels["key2"])
|
|
}
|
|
// key3 should remain unchanged
|
|
if f.meta.Req.Labels["key3"] != "value3" {
|
|
t.Errorf("Label key3 = %v, want value3 (should remain unchanged)", f.meta.Req.Labels["key3"])
|
|
}
|
|
}
|
|
|
|
// TestFetcher_Patch_Extra tests patching request Extra with merge behavior
|
|
func TestFetcher_Patch_Extra(t *testing.T) {
|
|
listener := test.StartTestFileServer()
|
|
defer listener.Close()
|
|
|
|
f := buildFetcher()
|
|
opts := &base.Options{
|
|
Name: test.DownloadName,
|
|
Path: test.Dir,
|
|
Extra: &http.OptsExtra{Connections: 1},
|
|
}
|
|
|
|
// Resolve with initial Extra
|
|
f.Meta().Opts = opts
|
|
err := f.Resolve(&base.Request{
|
|
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
|
|
Extra: &http.ReqExtra{
|
|
Method: "GET",
|
|
Body: "initial body",
|
|
Header: map[string]string{
|
|
"Authorization": "Bearer token123",
|
|
"X-Custom": "original",
|
|
},
|
|
},
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
// Verify initial Extra
|
|
initialExtra := f.meta.Req.Extra.(*http.ReqExtra)
|
|
if initialExtra.Method != "GET" {
|
|
t.Errorf("Initial Method = %v, want GET", initialExtra.Method)
|
|
}
|
|
if initialExtra.Body != "initial body" {
|
|
t.Errorf("Initial Body = %v, want 'initial body'", initialExtra.Body)
|
|
}
|
|
if initialExtra.Header["Authorization"] != "Bearer token123" {
|
|
t.Errorf("Initial Authorization header = %v, want 'Bearer token123'", initialExtra.Header["Authorization"])
|
|
}
|
|
if initialExtra.Header["X-Custom"] != "original" {
|
|
t.Errorf("Initial X-Custom header = %v, want 'original'", initialExtra.Header["X-Custom"])
|
|
}
|
|
|
|
// Patch with partial Extra - only update some fields
|
|
patchReq := &base.Request{
|
|
Extra: &http.ReqExtra{
|
|
Method: "POST", // Update method
|
|
// Body is empty, should NOT update
|
|
Header: map[string]string{
|
|
"X-Custom": "modified", // Overwrite existing
|
|
"X-New": "added", // Add new
|
|
// Authorization is not in patch, should remain
|
|
},
|
|
},
|
|
}
|
|
err = f.Patch(patchReq, nil)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
// Verify Extra was merged correctly
|
|
patchedExtra := f.meta.Req.Extra.(*http.ReqExtra)
|
|
|
|
// Method should be updated
|
|
if patchedExtra.Method != "POST" {
|
|
t.Errorf("Patched Method = %v, want POST", patchedExtra.Method)
|
|
}
|
|
|
|
// Body should remain unchanged (patch had empty body)
|
|
if patchedExtra.Body != "initial body" {
|
|
t.Errorf("Patched Body = %v, want 'initial body' (should remain unchanged)", patchedExtra.Body)
|
|
}
|
|
|
|
// Authorization header should remain unchanged
|
|
if patchedExtra.Header["Authorization"] != "Bearer token123" {
|
|
t.Errorf("Authorization header = %v, want 'Bearer token123' (should remain unchanged)", patchedExtra.Header["Authorization"])
|
|
}
|
|
|
|
// X-Custom header should be overwritten
|
|
if patchedExtra.Header["X-Custom"] != "modified" {
|
|
t.Errorf("X-Custom header = %v, want 'modified'", patchedExtra.Header["X-Custom"])
|
|
}
|
|
|
|
// X-New header should be added
|
|
if patchedExtra.Header["X-New"] != "added" {
|
|
t.Errorf("X-New header = %v, want 'added'", patchedExtra.Header["X-New"])
|
|
}
|
|
}
|
|
|
|
// TestFetcher_Patch_NilData tests that Patch with nil data doesn't cause errors
|
|
func TestFetcher_Patch_NilData(t *testing.T) {
|
|
listener := test.StartTestFileServer()
|
|
defer listener.Close()
|
|
|
|
f := buildFetcher()
|
|
opts := &base.Options{
|
|
Name: test.DownloadName,
|
|
Path: test.Dir,
|
|
Extra: &http.OptsExtra{Connections: 1},
|
|
}
|
|
|
|
f.Meta().Opts = opts
|
|
err := f.Resolve(&base.Request{
|
|
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
originalURL := f.meta.Req.URL
|
|
|
|
// Patch with nil data - should not cause error
|
|
err = f.Patch(nil, nil)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
// Verify URL unchanged
|
|
if f.meta.Req.URL != originalURL {
|
|
t.Errorf("URL changed after nil patch: got %v, want %v", f.meta.Req.URL, originalURL)
|
|
}
|
|
|
|
// Patch with empty request - should not cause error
|
|
err = f.Patch(&base.Request{}, nil)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
// Verify URL still unchanged
|
|
if f.meta.Req.URL != originalURL {
|
|
t.Errorf("URL changed after empty patch: got %v, want %v", f.meta.Req.URL, originalURL)
|
|
}
|
|
}
|
|
|
|
// TestFetcher_Patch_CookieExpired tests the Patch functionality where a download fails
|
|
// mid-way due to expired cookie, then succeeds after patching with a new valid cookie.
|
|
// This simulates:
|
|
// 1. Initial resolve with valid cookie succeeds
|
|
// 2. Download starts but fails because cookie expires mid-download (server returns 401)
|
|
// 3. User patches the task with a new valid cookie
|
|
// 4. Download resumes and completes successfully
|
|
func TestFetcher_Patch_CookieExpired(t *testing.T) {
|
|
listener := test.StartTestCookieExpiringServer()
|
|
defer listener.Close()
|
|
|
|
downloadURL := "http://" + listener.Addr().String() + "/" + test.BuildName
|
|
opts := &base.Options{
|
|
Name: test.DownloadName,
|
|
Path: test.Dir,
|
|
Extra: &http.OptsExtra{Connections: 1},
|
|
}
|
|
|
|
// Step 1: Resolve with old_token - should succeed (first request accepts old_token)
|
|
f := buildFetcher()
|
|
f.Meta().Opts = opts
|
|
err := f.Resolve(&base.Request{
|
|
URL: downloadURL,
|
|
Extra: &http.ReqExtra{
|
|
Header: map[string]string{
|
|
"Cookie": "session=old_token",
|
|
},
|
|
},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("Resolve should succeed with old_token: %v", err)
|
|
}
|
|
|
|
// Verify initial cookie
|
|
initialExtra := f.meta.Req.Extra.(*http.ReqExtra)
|
|
if initialExtra.Header["Cookie"] != "session=old_token" {
|
|
t.Errorf("Initial Cookie = %v, want session=old_token", initialExtra.Header["Cookie"])
|
|
}
|
|
// This test exercises patching credentials after an authenticated request
|
|
// fails. Discard the already-authorized Resolve response so the stale cookie
|
|
// is observable on the first Start.
|
|
if err := f.Pause(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
// Step 2: Start download - should fail because old_token is now expired
|
|
// (server only accepts old_token for first request, subsequent requests need new_token)
|
|
err = f.Start()
|
|
if err != nil {
|
|
t.Fatalf("Start failed: %v", err)
|
|
}
|
|
err = f.Wait()
|
|
// Download should fail with 401 error
|
|
if err == nil {
|
|
t.Fatal("Expected download to fail with expired cookie, but it succeeded")
|
|
}
|
|
t.Logf("Download failed as expected: %v", err)
|
|
|
|
// Step 3: Patch with new valid cookie
|
|
err = f.Patch(&base.Request{
|
|
Extra: &http.ReqExtra{
|
|
Header: map[string]string{
|
|
"Cookie": "session=new_token",
|
|
},
|
|
},
|
|
}, nil)
|
|
if err != nil {
|
|
t.Fatalf("Patch to update cookie failed: %v", err)
|
|
}
|
|
|
|
// Verify cookie was updated
|
|
patchedExtra := f.meta.Req.Extra.(*http.ReqExtra)
|
|
if patchedExtra.Header["Cookie"] != "session=new_token" {
|
|
t.Errorf("Cookie should be updated: got %v, want session=new_token", patchedExtra.Header["Cookie"])
|
|
}
|
|
|
|
// Step 4: Restart download - should succeed with new cookie
|
|
err = f.Start()
|
|
if err != nil {
|
|
t.Fatalf("Restart failed: %v", err)
|
|
}
|
|
err = f.Wait()
|
|
if err != nil {
|
|
t.Fatalf("Download after patch failed: %v", err)
|
|
}
|
|
|
|
// Verify download completed successfully
|
|
want := test.FileMd5(test.BuildFile)
|
|
got := test.FileMd5(test.DownloadFile)
|
|
if want != got {
|
|
t.Errorf("File MD5 mismatch: got %v, want %v", got, want)
|
|
}
|
|
}
|