mirror of
https://github.com/GopeedLab/gopeed.git
synced 2026-11-14 20:33:41 +00:00
fix(hls): isolate per-run lifecycle and use final-output byte accounting
- Give every Start its own run context, WaitGroup (including the queue producer), supervisor done channel and snapshots of the plan, request headers, HTTP client and merge target; Pause/Close wait for all run goroutines, including a merge in flight, before returning - Build the HTTP client in Resolve/Start before workers launch instead of lazily inside do(), which raced between concurrent segment downloads - Reject task edits while a run is active instead of mutating shared state - Reset counters per run and seed them from the completed-segment journal; count only stored output bytes as progress, never discarded retry bytes - Report Res.Size only when every segment is unencrypted with a known length; ciphertext sums overstate the final output, indeterminate plans rely on the engine's TotalDownloaded backfill at completion
This commit is contained in:
1 parent
ee9b16794e
commit
40cfb167f6
3 files changed
+379
-203
No files matched your search
+345
-188
File diff suppressed because it is too large.
Load diff
@@ -94,6 +94,15 @@ func mustRun(t *testing.T, f *Fetcher) {
|
||||
}
|
||||
}
|
||||
|
||||
func statsOf(t *testing.T, f *Fetcher) Stats {
|
||||
t.Helper()
|
||||
s, ok := f.Stats().Snapshot.(*Stats)
|
||||
if !ok {
|
||||
t.Fatalf("unexpected stats snapshot type %T", f.Stats().Snapshot)
|
||||
}
|
||||
return *s
|
||||
}
|
||||
|
||||
func readOutput(t *testing.T, f *Fetcher) []byte {
|
||||
t.Helper()
|
||||
data, err := os.ReadFile(f.meta.SingleFilepath())
|
||||
@@ -254,10 +263,10 @@ func TestFetcherE2E_AES128KeyRotation(t *testing.T) {
|
||||
|
||||
f := newTestFetcher(t, testConfig())
|
||||
mustResolve(t, f, server.URL+"/video.m3u8", "")
|
||||
// Ciphertext sizes overstate the final output after decryption, so an
|
||||
// encrypted plan must report an indeterminate size.
|
||||
if f.meta.Res.Size != 0 {
|
||||
// Encrypted segment sizes are probed as ciphertext length, decryption
|
||||
// shrinks them, so an all-or-nothing probe may still report a size.
|
||||
t.Logf("prefetched size: %d", f.meta.Res.Size)
|
||||
t.Errorf("encrypted plan should report size 0, got %d", f.meta.Res.Size)
|
||||
}
|
||||
mustRun(t, f)
|
||||
|
||||
@@ -345,10 +354,11 @@ func TestFetcherE2E_PauseResume(t *testing.T) {
|
||||
if err := f.Pause(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if f.doneCount.Load() == 0 {
|
||||
doneBefore := statsOf(t, f).SegmentsDone
|
||||
if doneBefore == 0 {
|
||||
t.Fatal("expected some segments to complete before pause")
|
||||
}
|
||||
if f.doneCount.Load() == total {
|
||||
if doneBefore == total {
|
||||
t.Fatal("all segments finished before pause, test is not exercising resume")
|
||||
}
|
||||
|
||||
@@ -409,7 +419,7 @@ func TestFetcherE2E_StoreRestoreResume(t *testing.T) {
|
||||
if err := f.Pause(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
doneBefore := f.doneCount.Load()
|
||||
doneBefore := statsOf(t, f).SegmentsDone
|
||||
if doneBefore == 0 {
|
||||
t.Fatal("expected some segments to finish before simulated restart")
|
||||
}
|
||||
@@ -450,8 +460,8 @@ func TestFetcherE2E_StoreRestoreResume(t *testing.T) {
|
||||
if err := rf.Wait(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if rf.doneCount.Load() != total {
|
||||
t.Errorf("want %d done, got %d", total, rf.doneCount.Load())
|
||||
if done := statsOf(t, rf).SegmentsDone; done != total {
|
||||
t.Errorf("want %d done, got %d", total, done)
|
||||
}
|
||||
var want []byte
|
||||
for i := 0; i < total; i++ {
|
||||
|
||||
@@ -13,6 +13,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/GopeedLab/gopeed/internal/httpclient"
|
||||
"github.com/GopeedLab/gopeed/pkg/base"
|
||||
)
|
||||
|
||||
const defaultUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36"
|
||||
@@ -25,8 +26,16 @@ func (f *Fetcher) timeout() time.Duration {
|
||||
|
||||
// buildClient builds an HTTP client honoring the task proxy and TLS settings.
|
||||
// The client has no total timeout on purpose: large segments over slow links
|
||||
// must not be aborted; body stalls are handled by stallReader instead.
|
||||
// must not be aborted; body stalls are handled by stallReader instead. It is
|
||||
// only called while the fetcher is quiescent (Resolve/Start), never from
|
||||
// inside running worker goroutines.
|
||||
func (f *Fetcher) buildClient() *http.Client {
|
||||
var proxy *base.RequestProxy
|
||||
skipVerify := false
|
||||
if req := f.meta.Req; req != nil {
|
||||
proxy = req.Proxy
|
||||
skipVerify = req.SkipVerifyCert
|
||||
}
|
||||
jar, _ := cookiejar.New(nil)
|
||||
client, err := httpclient.NewClient(httpclient.Options{
|
||||
Client: httpclient.ClientOptions{
|
||||
@@ -36,9 +45,9 @@ func (f *Fetcher) buildClient() *http.Client {
|
||||
DialContext: (&net.Dialer{
|
||||
Timeout: f.timeout(),
|
||||
}).DialContext,
|
||||
Proxy: f.Ctl.GetProxy(f.meta.Req.Proxy),
|
||||
Proxy: f.Ctl.GetProxy(proxy),
|
||||
TLSClientConfig: &tls.Config{
|
||||
InsecureSkipVerify: f.meta.Req.SkipVerifyCert,
|
||||
InsecureSkipVerify: skipVerify,
|
||||
},
|
||||
TLSHandshakeTimeout: f.timeout(),
|
||||
},
|
||||
@@ -54,14 +63,14 @@ func (f *Fetcher) buildClient() *http.Client {
|
||||
}
|
||||
|
||||
// buildRequest builds a request with task extra headers, the default user
|
||||
// agent and an optional byte range.
|
||||
func (f *Fetcher) buildRequest(ctx context.Context, method, u string, body io.Reader, byterange *ByteRange) (*http.Request, error) {
|
||||
// agent and an optional byte range. extra may be nil.
|
||||
func buildRequest(ctx context.Context, extra *ReqExtra, method, u string, body io.Reader, byterange *ByteRange) (*http.Request, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, method, u, body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if f.extra != nil {
|
||||
for key, value := range f.extra.Header {
|
||||
if extra != nil {
|
||||
for key, value := range extra.Header {
|
||||
req.Header.Set(key, value)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user