mirror of
https://github.com/GopeedLab/gopeed.git
synced 2026-11-17 22:03:41 +00:00
feat(http): support advisory extension download sizes (#1599)
This commit is contained in:
17 files changed
+1047
-182
No files matched your search
@@ -0,0 +1,245 @@
|
||||
package http
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
|
||||
"github.com/GopeedLab/gopeed/pkg/base"
|
||||
)
|
||||
|
||||
func TestFetcher_OptionsSizeHintAndPatch(t *testing.T) {
|
||||
for _, known := range []bool{false, true} {
|
||||
for _, payload := range []string{"", "0123456789"} {
|
||||
t.Run(fmt.Sprintf("known=%v/bytes=%d", known, len(payload)), func(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
if known {
|
||||
w.Header().Set("Content-Length", fmt.Sprint(len(payload)))
|
||||
}
|
||||
w.(http.Flusher).Flush()
|
||||
fmt.Fprint(w, payload)
|
||||
}))
|
||||
defer server.Close()
|
||||
f := buildFetcher()
|
||||
defer f.Close()
|
||||
f.Meta().Opts = &base.Options{Size: 7, Path: t.TempDir(), Name: "output.bin"}
|
||||
if err := f.Resolve(&base.Request{URL: server.URL}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := int64(7)
|
||||
if known {
|
||||
want = int64(len(payload))
|
||||
}
|
||||
if f.meta.Res.Size != want || f.hasKnownSize() != known {
|
||||
t.Fatalf("resolved size=%d known=%v", f.meta.Res.Size, f.hasKnownSize())
|
||||
}
|
||||
if err := f.Patch(nil, &base.Options{Size: 9}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !known {
|
||||
want = 9
|
||||
}
|
||||
if f.meta.Res.Size != want {
|
||||
t.Fatal("hint update overrode a real size or failed to update an estimate")
|
||||
}
|
||||
if err := f.Start(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := f.Wait(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if f.meta.Res.Size != int64(len(payload)) || !f.hasKnownSize() {
|
||||
t.Fatalf("final size=%d known=%v", f.meta.Res.Size, f.hasKnownSize())
|
||||
}
|
||||
snapshot, err := new(FetcherManager).Store(f)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fd := snapshot.(*fetcherData)
|
||||
if fd.SizeKnown == nil || !*fd.SizeKnown {
|
||||
t.Fatal("snapshot lost confirmed size authority")
|
||||
}
|
||||
_, restore := new(FetcherManager).Restore()
|
||||
restored := restore(f.meta, fd).(*Fetcher)
|
||||
if !restored.hasKnownSize() || restored.knownSize() != int64(len(payload)) {
|
||||
t.Fatal("restored size lost authority, including known zero")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetcher_SizeHintCannotCompleteForbiddenUnknownRange(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Header.Get("Range") != "" {
|
||||
w.WriteHeader(http.StatusForbidden)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Accept-Ranges", "bytes")
|
||||
w.(http.Flusher).Flush()
|
||||
fmt.Fprint(w, "0123456789")
|
||||
w.(http.Flusher).Flush()
|
||||
<-r.Context().Done()
|
||||
}))
|
||||
defer server.Close()
|
||||
f := buildFetcher()
|
||||
defer f.Close()
|
||||
f.Meta().Opts = &base.Options{Size: 3, Path: t.TempDir(), Name: "output.bin"}
|
||||
if err := f.Resolve(&base.Request{URL: server.URL}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := f.Start(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := f.Wait(); err == nil {
|
||||
t.Fatal("a failed unknown-length Range request was completed from an estimate")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetcher_SizeHintPreservesHTTPIntegrity(t *testing.T) {
|
||||
for _, chunked := range []bool{false, true} {
|
||||
for _, broken := range []bool{false, true} {
|
||||
t.Run(fmt.Sprintf("chunked=%v/broken=%v", chunked, broken), func(t *testing.T) {
|
||||
payload := []byte("0123456789")
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
if chunked && broken {
|
||||
conn, rw, err := w.(http.Hijacker).Hijack()
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
return
|
||||
}
|
||||
defer conn.Close()
|
||||
fmt.Fprint(rw, "HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\nConnection: close\r\n\r\na\r\n0123456789\r\n")
|
||||
rw.Flush() // Omit the final chunk to simulate a connection interruption.
|
||||
return
|
||||
}
|
||||
if !chunked {
|
||||
length := len(payload)
|
||||
if broken {
|
||||
length++
|
||||
}
|
||||
w.Header().Set("Content-Length", fmt.Sprint(length))
|
||||
}
|
||||
w.(http.Flusher).Flush()
|
||||
w.Write(payload)
|
||||
}))
|
||||
defer server.Close()
|
||||
f := buildFetcher()
|
||||
defer f.Close()
|
||||
f.Meta().Opts = &base.Options{Size: 3, Path: t.TempDir(), Name: "output.bin"}
|
||||
if err := f.Resolve(&base.Request{URL: server.URL}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !chunked {
|
||||
length := int64(len(payload))
|
||||
if broken {
|
||||
length++
|
||||
}
|
||||
if f.meta.Res.Size != length || f.sequentialSizeUnknown {
|
||||
t.Fatal("HTTP length did not override size hint")
|
||||
}
|
||||
} else if f.meta.Res.Size != 3 || !f.sequentialSizeUnknown {
|
||||
t.Fatal("unknown HTTP length did not retain the display hint")
|
||||
}
|
||||
if err := f.Start(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
err := f.Wait()
|
||||
if broken {
|
||||
if err == nil {
|
||||
t.Fatal("truncated HTTP response succeeded after exceeding the size hint")
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if f.meta.Res.Size != int64(len(payload)) || f.sequentialSizeUnknown {
|
||||
t.Fatalf("completed size=%d unknown=%v", f.meta.Res.Size, f.sequentialSizeUnknown)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetcher_SizeHintSurvivesRestore(t *testing.T) {
|
||||
for _, partial := range []bool{false, true} {
|
||||
for _, fixedLength := range []bool{false, true} {
|
||||
t.Run(fmt.Sprintf("partial=%v/fixedLength=%v", partial, fixedLength), func(t *testing.T) {
|
||||
payload := []byte("0123456789")
|
||||
var requests atomic.Int64
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
request := requests.Add(1)
|
||||
if got := r.Header.Get("Range"); got != "" {
|
||||
t.Errorf("estimated sizes must not be used for resume ranges: %s", got)
|
||||
}
|
||||
if fixedLength && request > 1 {
|
||||
w.Header().Set("Content-Length", fmt.Sprint(len(payload)))
|
||||
}
|
||||
w.(http.Flusher).Flush()
|
||||
if fixedLength && request == 2 {
|
||||
w.Write(payload[:4]) // Retry must discard this interrupted attempt.
|
||||
return
|
||||
}
|
||||
w.Write(payload)
|
||||
}))
|
||||
defer server.Close()
|
||||
f := buildFetcher()
|
||||
opts := &base.Options{Size: 3, Path: t.TempDir(), Name: "output.bin"}
|
||||
f.Meta().Opts = opts
|
||||
if err := f.Resolve(&base.Request{URL: server.URL}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fm := new(FetcherManager)
|
||||
snapshot, err := fm.Store(f)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
data, err := json.Marshal(snapshot)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var saved fetcherData
|
||||
if err := json.Unmarshal(data, &saved); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !saved.SequentialSizeUnknown || saved.SizeKnown == nil || *saved.SizeKnown || *saved.ResourceSize != 3 {
|
||||
t.Fatal("snapshot lost the size hint or its advisory state")
|
||||
}
|
||||
f.Close()
|
||||
// An interrupted earlier attempt left a tail longer than the new output.
|
||||
if err := os.WriteFile(filepath.Join(opts.Path, opts.Name), bytes.Repeat([]byte("old"), 100), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if partial {
|
||||
chunk := newChunk(0, 0)
|
||||
chunk.Downloaded = 7 // Already exceeds the display estimate.
|
||||
saved.Connections = []*connection{{ID: 0, Role: rolePrimary, State: connNotStarted, Chunk: chunk, Downloaded: 7}}
|
||||
}
|
||||
_, restore := fm.Restore()
|
||||
restored := restore(f.meta, &saved).(*Fetcher)
|
||||
restored.Setup(f.ctl)
|
||||
defer restored.Close()
|
||||
if err := restored.Start(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := restored.Wait(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body, err := os.ReadFile(restored.meta.SingleFilepath())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !bytes.Equal(body, payload) || restored.meta.Res.Size != int64(len(payload)) || restored.Progress().TotalDownloaded() != int64(len(payload)) {
|
||||
t.Fatalf("restored stream retained an estimate or stale bytes: body=%q size=%d progress=%d", body, restored.meta.Res.Size, restored.Progress().TotalDownloaded())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user