From 46c34f2e9237bdb40a46a2b0dc49e1eb2396fd00 Mon Sep 17 00:00:00 2001 From: sun Date: Tue, 15 Sep 2026 23:40:31 +0800 Subject: [PATCH 1/5] feat(hls): add native HLS (m3u8) download protocol (#1515) --- internal/fetcher/fetcher.go | 8 +- internal/protocol/hls/config.go | 30 + internal/protocol/hls/crypto.go | 59 + internal/protocol/hls/crypto_test.go | 66 ++ internal/protocol/hls/doc.go | 24 + internal/protocol/hls/fetcher.go | 1001 +++++++++++++++++ internal/protocol/hls/fetcher_manager.go | 100 ++ internal/protocol/hls/fetcher_test.go | 857 ++++++++++++++ internal/protocol/hls/helper.go | 100 ++ internal/protocol/hls/journal.go | 112 ++ internal/protocol/hls/media_validate_test.go | 297 +++++ internal/protocol/hls/playlist.go | 479 ++++++++ .../protocol/hls/playlist_realworld_test.go | 39 + internal/protocol/hls/playlist_test.go | 252 +++++ internal/protocol/hls/staging_test.go | 296 +++++ internal/protocol/hls/subset_test.go | 146 +++ .../protocol/hls/testdata/media/README.md | 10 + .../protocol/hls/testdata/media/fmp4-init.mp4 | Bin 0 -> 1566 bytes .../protocol/hls/testdata/media/fmp4-seg1.m4s | Bin 0 -> 11674 bytes .../protocol/hls/testdata/media/fmp4-seg2.m4s | Bin 0 -> 8299 bytes .../protocol/hls/testdata/media/fmp4-seg3.m4s | Bin 0 -> 8437 bytes .../protocol/hls/testdata/media/sample-00.ts | Bin 0 -> 27072 bytes .../protocol/hls/testdata/media/sample-01.ts | Bin 0 -> 18236 bytes .../protocol/hls/testdata/olemovienews.m3u8 | 283 +++++ pkg/download/hls_live_test.go | 97 ++ pkg/download/hls_protocol_test.go | 172 +++ pkg/download/model.go | 2 + .../lib/api/model/downloader_config.dart | 20 + .../lib/api/model/downloader_config.g.dart | 18 +- ui/flutter/lib/api/model/task.dart | 2 +- ui/flutter/lib/api/model/task.g.dart | 2 +- ui/flutter/lib/api/model/task_stats.dart | 2 +- .../presentation/pages/settings_page.dart | 44 + .../features/tasks/domain/task_record.dart | 2 +- ui/flutter/lib/l10n/app_ar.arb | 1 + ui/flutter/lib/l10n/app_az.arb | 1 + ui/flutter/lib/l10n/app_ca.arb | 1 + ui/flutter/lib/l10n/app_de.arb | 1 + ui/flutter/lib/l10n/app_en.arb | 1 + ui/flutter/lib/l10n/app_es.arb | 1 + ui/flutter/lib/l10n/app_fa.arb | 1 + ui/flutter/lib/l10n/app_fr.arb | 1 + ui/flutter/lib/l10n/app_hu.arb | 1 + ui/flutter/lib/l10n/app_id.arb | 1 + ui/flutter/lib/l10n/app_it.arb | 1 + ui/flutter/lib/l10n/app_ja.arb | 1 + ui/flutter/lib/l10n/app_ko.arb | 1 + ui/flutter/lib/l10n/app_pl.arb | 1 + ui/flutter/lib/l10n/app_pt.arb | 1 + ui/flutter/lib/l10n/app_ru.arb | 1 + ui/flutter/lib/l10n/app_ta.arb | 1 + ui/flutter/lib/l10n/app_tr.arb | 1 + ui/flutter/lib/l10n/app_uk.arb | 1 + ui/flutter/lib/l10n/app_vi.arb | 1 + ui/flutter/lib/l10n/app_zh.arb | 1 + ui/flutter/lib/l10n/app_zh_TW.arb | 1 + 56 files changed, 4536 insertions(+), 6 deletions(-) create mode 100644 internal/protocol/hls/config.go create mode 100644 internal/protocol/hls/crypto.go create mode 100644 internal/protocol/hls/crypto_test.go create mode 100644 internal/protocol/hls/doc.go create mode 100644 internal/protocol/hls/fetcher.go create mode 100644 internal/protocol/hls/fetcher_manager.go create mode 100644 internal/protocol/hls/fetcher_test.go create mode 100644 internal/protocol/hls/helper.go create mode 100644 internal/protocol/hls/journal.go create mode 100644 internal/protocol/hls/media_validate_test.go create mode 100644 internal/protocol/hls/playlist.go create mode 100644 internal/protocol/hls/playlist_realworld_test.go create mode 100644 internal/protocol/hls/playlist_test.go create mode 100644 internal/protocol/hls/staging_test.go create mode 100644 internal/protocol/hls/subset_test.go create mode 100644 internal/protocol/hls/testdata/media/README.md create mode 100644 internal/protocol/hls/testdata/media/fmp4-init.mp4 create mode 100644 internal/protocol/hls/testdata/media/fmp4-seg1.m4s create mode 100644 internal/protocol/hls/testdata/media/fmp4-seg2.m4s create mode 100644 internal/protocol/hls/testdata/media/fmp4-seg3.m4s create mode 100644 internal/protocol/hls/testdata/media/sample-00.ts create mode 100644 internal/protocol/hls/testdata/media/sample-01.ts create mode 100644 internal/protocol/hls/testdata/olemovienews.m3u8 create mode 100644 pkg/download/hls_live_test.go create mode 100644 pkg/download/hls_protocol_test.go diff --git a/internal/fetcher/fetcher.go b/internal/fetcher/fetcher.go index 4a1374c3..8d3a7821 100644 --- a/internal/fetcher/fetcher.go +++ b/internal/fetcher/fetcher.go @@ -106,7 +106,13 @@ func (s *SchemeFilter) Match(uri string) bool { case FilterTypeUrl: return strings.HasPrefix(uriUpper, patternUpper+":") case FilterTypeFile: - return strings.HasSuffix(uriUpper, "."+patternUpper) + // Strip query and fragment before matching so URLs like + // https://example.com/video.m3u8?sign=abc still route by extension. + clean := uri + if idx := strings.IndexAny(clean, "?#"); idx >= 0 { + clean = clean[:idx] + } + return strings.HasSuffix(strings.ToUpper(clean), "."+patternUpper) case FilterTypeBase64: return strings.HasPrefix(uriUpper, "DATA:"+patternUpper+";BASE64,") } diff --git a/internal/protocol/hls/config.go b/internal/protocol/hls/config.go new file mode 100644 index 00000000..2c06d5f0 --- /dev/null +++ b/internal/protocol/hls/config.go @@ -0,0 +1,30 @@ +package hls + +const ( + defaultSegmentConnections = 8 + defaultMaxRetries = 5 + defaultTimeoutSeconds = 30 + + // maxPrefetchSizeSegments caps the HEAD probing done at resolve time. + // Playlists larger than this skip size prefetch entirely. + maxPrefetchSizeSegments = 500 +) + +type config struct { + SegmentConnections int `json:"segmentConnections"` + MaxRetries int `json:"maxRetries"` + TimeoutSeconds int `json:"timeoutSeconds"` + PrefetchContentLength bool `json:"prefetchContentLength"` +} + +func (c *config) normalize() { + if c.SegmentConnections <= 0 { + c.SegmentConnections = defaultSegmentConnections + } + if c.MaxRetries < 0 { + c.MaxRetries = defaultMaxRetries + } + if c.TimeoutSeconds <= 0 { + c.TimeoutSeconds = defaultTimeoutSeconds + } +} diff --git a/internal/protocol/hls/crypto.go b/internal/protocol/hls/crypto.go new file mode 100644 index 00000000..58379a4f --- /dev/null +++ b/internal/protocol/hls/crypto.go @@ -0,0 +1,59 @@ +package hls + +import ( + "crypto/aes" + "crypto/cipher" + "fmt" +) + +// decryptAES128 decrypts an AES-128-CBC encrypted segment and strips PKCS7 +// padding. data length must be a multiple of the AES block size. +func decryptAES128(data, key, iv []byte) ([]byte, error) { + if len(key) != 16 { + return nil, fmt.Errorf("invalid AES-128 key size %d", len(key)) + } + if len(iv) != 16 { + return nil, fmt.Errorf("invalid AES-128 IV size %d", len(iv)) + } + if len(data) == 0 || len(data)%aes.BlockSize != 0 { + return nil, fmt.Errorf("invalid AES-128 data size %d", len(data)) + } + block, err := aes.NewCipher(key) + if err != nil { + return nil, err + } + plain := make([]byte, len(data)) + cipher.NewCBCDecrypter(block, iv).CryptBlocks(plain, data) + return pkcs7Unpad(plain, aes.BlockSize) +} + +func pkcs7Unpad(data []byte, blockSize int) ([]byte, error) { + if len(data) == 0 { + return nil, fmt.Errorf("empty plaintext") + } + pad := int(data[len(data)-1]) + if pad == 0 || pad > blockSize || pad > len(data) { + return nil, fmt.Errorf("invalid PKCS7 padding") + } + for _, b := range data[len(data)-pad:] { + if int(b) != pad { + return nil, fmt.Errorf("invalid PKCS7 padding") + } + } + return data[:len(data)-pad], nil +} + +// sequenceIV derives the default per-segment IV mandated by RFC 8216: the +// segment media sequence number as a 16-byte big-endian integer. +func sequenceIV(sequence int64) []byte { + iv := make([]byte, 16) + iv[8] = byte(sequence >> 56) + iv[9] = byte(sequence >> 48) + iv[10] = byte(sequence >> 40) + iv[11] = byte(sequence >> 32) + iv[12] = byte(sequence >> 24) + iv[13] = byte(sequence >> 16) + iv[14] = byte(sequence >> 8) + iv[15] = byte(sequence) + return iv +} diff --git a/internal/protocol/hls/crypto_test.go b/internal/protocol/hls/crypto_test.go new file mode 100644 index 00000000..05f18423 --- /dev/null +++ b/internal/protocol/hls/crypto_test.go @@ -0,0 +1,66 @@ +package hls + +import ( + "bytes" + "crypto/aes" + "crypto/cipher" + "crypto/rand" + "testing" +) + +func pkcs7Pad(data []byte, blockSize int) []byte { + pad := blockSize - len(data)%blockSize + return append(data, bytes.Repeat([]byte{byte(pad)}, pad)...) +} + +func TestDecryptAES128Roundtrip(t *testing.T) { + key := make([]byte, 16) + iv := make([]byte, 16) + if _, err := rand.Read(key); err != nil { + t.Fatal(err) + } + if _, err := rand.Read(iv); err != nil { + t.Fatal(err) + } + plain := []byte("hello hls segment payload, padded to block size!") + padded := pkcs7Pad(plain, aes.BlockSize) + block, err := aes.NewCipher(key) + if err != nil { + t.Fatal(err) + } + encrypted := make([]byte, len(padded)) + cipher.NewCBCEncrypter(block, iv).CryptBlocks(encrypted, padded) + + got, err := decryptAES128(encrypted, key, iv) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(got, plain) { + t.Errorf("roundtrip mismatch: %q", got) + } +} + +func TestDecryptAES128InvalidInput(t *testing.T) { + if _, err := decryptAES128([]byte{1, 2, 3}, make([]byte, 16), make([]byte, 16)); err == nil { + t.Error("non-block-aligned data should fail") + } + if _, err := decryptAES128(make([]byte, 16), make([]byte, 15), make([]byte, 16)); err == nil { + t.Error("short key should fail") + } + if _, err := decryptAES128(make([]byte, 16), make([]byte, 16), make([]byte, 16)); err == nil { + t.Error("all-zero block has invalid padding and should fail") + } +} + +func TestSequenceIV(t *testing.T) { + iv := sequenceIV(1) + if len(iv) != 16 || iv[15] != 1 { + t.Errorf("sequence 1 IV wrong: %v", iv) + } + iv = sequenceIV(0x0102030405060708) + for i, want := range []byte{1, 2, 3, 4, 5, 6, 7, 8} { + if iv[8+i] != want { + t.Errorf("sequence IV byte %d: want %d got %d", i, want, iv[8+i]) + } + } +} diff --git a/internal/protocol/hls/doc.go b/internal/protocol/hls/doc.go new file mode 100644 index 00000000..78699556 --- /dev/null +++ b/internal/protocol/hls/doc.go @@ -0,0 +1,24 @@ +// Package hls implements native HTTP Live Streaming downloads for m3u8 +// playlists. A task produces exactly one merged, playable output file: +// MPEG-2 TS segments are concatenated as-is, fMP4 streams are merged as +// init section followed by the media fragments, and AES-128 encrypted +// segments are decrypted before they are stored. +// +// Supported subset: +// - VOD media playlists (EXT-X-ENDLIST present) and master playlists whose +// selected variant carries its own audio +// - AES-128 with the identity KEYFORMAT, explicit IV or media-sequence +// derived IV, including key rotation mid-playlist +// - EXT-X-BYTERANGE (including offset inheritance) and fMP4 init sections +// via a single, constant EXT-X-MAP +// +// Everything the merge pipeline cannot reproduce faithfully is rejected with +// an explicit error instead of silently producing a broken output file: +// - live streams (no EXT-X-ENDLIST) and SAMPLE-AES / DRM encryption +// - EXT-X-GAP (would leave a hole) and EXT-X-DISCONTINUITY (broken timeline) +// - changing init sections (a repeated EXT-X-MAP is compared by all its +// attributes, not just the URI) and encrypted EXT-X-MAP without an +// explicit IV +// - separate audio renditions referenced by the selected variant (the +// merged output would have no audio) and non-identity KEYFORMATs +package hls diff --git a/internal/protocol/hls/fetcher.go b/internal/protocol/hls/fetcher.go new file mode 100644 index 00000000..7259a9b0 --- /dev/null +++ b/internal/protocol/hls/fetcher.go @@ -0,0 +1,1001 @@ +package hls + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "os" + "path/filepath" + "strconv" + "sync" + "sync/atomic" + "time" + + "github.com/GopeedLab/gopeed/internal/controller" + "github.com/GopeedLab/gopeed/internal/fetcher" + "github.com/GopeedLab/gopeed/internal/httpclient" + "github.com/GopeedLab/gopeed/pkg/base" +) + +// ReqExtra mirrors the HTTP protocol extra: the create dialog attaches HTTP +// style options to m3u8 URLs (same JSON keys), so the same shape is decoded. +type ReqExtra struct { + Method string `json:"method,omitempty"` + Header map[string]string `json:"header,omitempty"` + Body string `json:"body,omitempty"` +} + +func copyReqExtra(extra *ReqExtra) *ReqExtra { + copied := &ReqExtra{} + if extra == nil { + return copied + } + copied.Method = extra.Method + copied.Body = extra.Body + if extra.Header != nil { + copied.Header = make(map[string]string, len(extra.Header)) + for key, value := range extra.Header { + copied.Header[key] = value + } + } + return copied +} + +// fetcherState is the resolved download plan persisted for resume. StagingID +// gives the task its own staging folder (two tasks downloading the same URL +// into the same directory never share segment files) and survives restarts +// through the persisted state. StagingDir pins the absolute folder so user +// renames or moves of the task output do not orphan the staged segments. +type fetcherState struct { + MediaURL string `json:"mediaURL"` + OutputName string `json:"outputName"` + Segments []*Segment `json:"segments"` + StagingID string `json:"stagingID,omitempty"` + StagingDir string `json:"stagingDir,omitempty"` +} + +// Stats is the protocol task statistics snapshot. +type Stats struct { + SegmentsTotal int `json:"segmentsTotal"` + SegmentsDone int `json:"segmentsDone"` + SegmentsFailed int `json:"segmentsFailed"` +} + +// tempDirPrefix separates HLS staging folders from regular task files. +const tempDirPrefix = ".gopeed-hls-" + +// hlsRun holds every piece of state that belongs to one Start..Pause/complete +// cycle. Runs never share mutable state: a Pause that is immediately followed +// by a Start replaces the whole run, so workers of the previous run cannot +// race with the next one. The run also snapshots the plan, request headers, +// client and merge target so concurrent Patch calls cannot change them under +// a running download. +type hlsRun struct { + ctx context.Context + cancel context.CancelFunc + // wg covers the queue producer AND the segment workers: Pause must not + // return while the producer can still hand out segment indexes. + wg sync.WaitGroup + done chan struct{} // closed when the supervisor has fully exited + + segs []*Segment // plan snapshot, read-only for the duration of the run + extra *ReqExtra // header snapshot (deep copy) + client *http.Client + outputPath string // merge target snapshot + tempDir string + + keyCache map[string][]byte + keyCacheMu sync.Mutex + + stagingID string // identity of the staging folder, bound into the journal + planHash string // fingerprint of the download plan, validated on resume + journal map[int64]int64 + journalPath string + journalMu sync.Mutex + + // downloaded counts bytes of the final output that are safely on disk + // (completed segment files), not raw network traffic: discarded retry + // bytes must never show up as progress. + downloaded atomic.Int64 + doneCount atomic.Int64 + failCount atomic.Int64 + baseBytes int64 // stored bytes restored from the journal at startup + baseDone int // completed segments restored from the journal at startup + pending []int64 // segment indexes to download, set by prepareRun +} + +func (r *hlsRun) statsSnapshot() Stats { + return Stats{ + SegmentsTotal: len(r.segs), + SegmentsDone: r.baseDone + int(r.doneCount.Load()), + SegmentsFailed: int(r.failCount.Load()), + } +} + +type Fetcher struct { + fetcher.DefaultFetcher + meta *fetcher.FetcherMeta + config *config + + // mu guards the fields below and every lifecycle transition. Progress and + // Stats are read from the engine's checkpoint goroutine while a run is + // active; they only take mu to snapshot the active run pointer, then read + // run-owned atomics lock-free. + mu sync.Mutex + extra *ReqExtra + client *http.Client + state *fetcherState + activeRun *hlsRun + closed bool + lastStats Stats + lastDownloaded int64 + + impSession *httpclient.ImpersonationSession +} + +func (f *Fetcher) Setup(ctl *controller.Controller) { + // Initializes Ctl and DoneCh, which Wait() depends on. + f.DefaultFetcher.Setup(ctl) + if f.meta == nil { + f.meta = &fetcher.FetcherMeta{} + } + f.impSession = httpclient.NewImpersonationSession() + f.config = &config{} + f.Ctl.GetConfig(f.config) + f.config.normalize() + // Restored fetchers receive their request through Setup (the engine calls + // it after the Restore builder): parse the extra headers so resumed + // segment/key requests keep Referer/Cookie and other anti-hotlink headers. + f.mu.Lock() + f.reloadExtraLocked() + f.mu.Unlock() +} + +// reloadExtraLocked refreshes f.extra from the typed extra of meta.Req. +func (f *Fetcher) reloadExtraLocked() { + if f.meta == nil || f.meta.Req == nil { + return + } + if err := base.ParseReqExtra[ReqExtra](f.meta.Req); err != nil { + return + } + if e, ok := f.meta.Req.Extra.(*ReqExtra); ok { + f.extra = e + } +} + +func (f *Fetcher) Meta() *fetcher.FetcherMeta { + return f.meta +} + +func (f *Fetcher) Resolve(req *base.Request, opts *base.Options) (err error) { + f.mu.Lock() + defer f.mu.Unlock() + f.meta.Req = req + f.meta.Opts = opts + f.extra = &ReqExtra{} + if err = base.ParseReqExtra[ReqExtra](req); err != nil { + return + } + if e, ok := req.Extra.(*ReqExtra); ok { + f.extra = e + } + if f.client == nil { + f.client = f.buildClient() + } + + state, name, err := f.resolvePlanLocked(context.Background(), "") + if err != nil { + return err + } + state.StagingID = randomStagingID() + f.state = state + // Pin the staging folder now so the very first persisted state already + // carries it: a crash right after create still resumes into the same + // folder. + f.resolveStagingDirLocked() + size := planTotalSize(state.Segments) + f.meta.Res = &base.Resource{ + Range: false, + Files: []*base.FileInfo{ + { + Name: name, + Size: size, + }, + }, + Size: size, + } + return nil +} + +// resolvePlanLocked fetches and parses the playlist into a fresh state. It +// must be called with f.mu held while the fetcher is quiescent. When keepName +// is non-empty the output keeps that file name (used when re-resolving an +// existing task so a refreshed URL does not rename the file). +func (f *Fetcher) resolvePlanLocked(ctx context.Context, keepName string) (*fetcherState, string, error) { + content, finalURL, err := f.fetchText(ctx, f.playlistMethod(), f.meta.Req.URL) + if err != nil { + return nil, "", fmt.Errorf("fetch playlist failed: %w", err) + } + baseURL, err := url.Parse(finalURL) + if err != nil { + return nil, "", err + } + + mediaURL := finalURL + if IsMasterPlaylist(content) { + master, err := ParseMasterPlaylist(content, baseURL) + if err != nil { + return nil, "", err + } + best := PickBestVariant(master.Variants) + if err := checkAudioRendition(master, best); err != nil { + return nil, "", err + } + variantContent, variantURL, err := f.fetchText(ctx, http.MethodGet, best.URI) + if err != nil { + return nil, "", fmt.Errorf("fetch variant playlist failed: %w", err) + } + variantBase, err := url.Parse(variantURL) + if err != nil { + return nil, "", err + } + content = variantContent + baseURL = variantBase + mediaURL = variantURL + } + media, err := ParseMedia(content, baseURL) + if err != nil { + return nil, "", err + } + if err := validateSupported(media); err != nil { + return nil, "", err + } + + outputName := keepName + if outputName == "" { + outputName = DeriveOutputName(mediaURL, media) + } + state := &fetcherState{ + MediaURL: mediaURL, + OutputName: outputName, + Segments: media.Segments, + } + if f.config.PrefetchContentLength { + f.prefetchSize(media.Segments) + } + return state, outputName, nil +} + +func (f *Fetcher) Patch(req *base.Request, opts *base.Options) error { + f.mu.Lock() + defer f.mu.Unlock() + if f.activeRun != nil { + return errors.New("task is running; pause it before changing its settings") + } + if req != nil { + cur := f.meta.Req + if cur == nil { + f.meta.Req = req + cur = req + } + if req.URL != "" && req.URL != cur.URL { + cur.URL = req.URL + // The playlist URL changed (e.g. refreshed signed link): drop the + // plan so the engine re-resolves on next start (it does when Res + // is nil). Only explicitly supplied fields are applied, so a + // header-only or rename-only Patch keeps the resolved plan and + // the output path. + f.dropPlanLocked() + } + if err := mergeReqExtra(cur, req); err != nil { + return err + } + if req.Proxy != nil && cur.Proxy != req.Proxy { + cur.Proxy = req.Proxy + // Transport settings changed: rebuild the client before the + // next run. + f.client = nil + } + if len(req.Labels) > 0 { + if cur.Labels == nil { + cur.Labels = make(map[string]string, len(req.Labels)) + } + for key, value := range req.Labels { + cur.Labels[key] = value + } + } + // SkipVerifyCert is a plain bool, so a partial update cannot tell an + // absent field from an explicit false: it is intentionally not merged + // (the HTTP protocol Patch ignores it as well). + } + if opts != nil { + if f.meta.Opts == nil { + f.meta.Opts = opts + } else { + if opts.Name != "" { + f.meta.Opts.Name = opts.Name + } + if opts.Path != "" { + f.meta.Opts.Path = opts.Path + } + } + } + f.reloadExtraLocked() + return nil +} + +// mergeReqExtra merges the typed extra of incoming into cur following the +// HTTP protocol semantics: Method/Body are overridden when non-empty and +// headers are merged per key. +func mergeReqExtra(cur *base.Request, incoming *base.Request) error { + if incoming.Extra == nil { + return nil + } + if err := base.ParseReqExtra[ReqExtra](incoming); err != nil { + return err + } + inc, ok := incoming.Extra.(*ReqExtra) + if !ok { + return nil + } + if err := base.ParseReqExtra[ReqExtra](cur); err != nil { + return err + } + target, ok := cur.Extra.(*ReqExtra) + if !ok { + target = &ReqExtra{} + cur.Extra = target + } + if inc.Method != "" { + target.Method = inc.Method + } + if inc.Body != "" { + target.Body = inc.Body + } + if len(inc.Header) > 0 { + if target.Header == nil { + target.Header = make(map[string]string, len(inc.Header)) + } + for key, value := range inc.Header { + target.Header[key] = value + } + } + return nil +} + +// dropPlanLocked discards the resolved plan after a URL change. The staging +// folder of the dropped plan is removed too: this only runs while the +// fetcher is quiescent, so no run can still be writing into it. +func (f *Fetcher) dropPlanLocked() { + if f.state != nil && f.state.StagingDir != "" { + os.RemoveAll(f.state.StagingDir) + } + f.state = nil + f.meta.Res = nil +} + +// existingOutputNameLocked returns the output file name the task already +// uses, so a re-resolve does not rename an in-progress download. +func (f *Fetcher) existingOutputNameLocked() string { + if f.meta.Opts != nil && f.meta.Opts.Name != "" { + return f.meta.Opts.Name + } + if res := f.meta.Res; res != nil && len(res.Files) > 0 && res.Files[0] != nil { + return res.Files[0].Name + } + return "" +} + +func (f *Fetcher) Start() error { + f.mu.Lock() + if f.closed { + f.mu.Unlock() + return errors.New("hls fetcher is closed") + } + if f.activeRun != nil { + f.mu.Unlock() + return errors.New("hls fetcher already running") + } + if f.state == nil { + if f.meta.Req == nil || f.meta.Res == nil { + f.mu.Unlock() + return errors.New("hls fetcher is not resolved") + } + // Crash window: the engine persists the task (with its resource) + // before the first HLS state checkpoint lands, so a restored task + // can have a resource but no plan. Rebuild the plan instead of + // failing, keeping the existing output name. + if f.client == nil { + f.client = f.buildClient() + } + state, _, err := f.resolvePlanLocked(context.Background(), f.existingOutputNameLocked()) + if err != nil { + f.mu.Unlock() + return fmt.Errorf("re-resolve failed: %w", err) + } + f.state = state + size := planTotalSize(state.Segments) + if res := f.meta.Res; res != nil { + res.Size = size + if len(res.Files) > 0 && res.Files[0] != nil { + res.Files[0].Size = size + } + } + } + // The client is built here, before any worker exists, so segment + // goroutines never race on lazy initialization. + if f.client == nil { + f.client = f.buildClient() + } + if f.meta.Res == nil { + // A restored plan without a resource cannot determine its output + // path; the engine re-resolves when Res is nil. + f.mu.Unlock() + return errors.New("hls fetcher is not resolved") + } + + ctx, cancel := context.WithCancel(context.Background()) + run := &hlsRun{ + ctx: ctx, + cancel: cancel, + done: make(chan struct{}), + segs: f.state.Segments, + extra: copyReqExtra(f.extra), + client: f.client, + outputPath: f.meta.SingleFilepath(), + stagingID: f.state.StagingID, + planHash: planFingerprint(f.state), + tempDir: f.resolveStagingDirLocked(), + keyCache: make(map[string][]byte), + journal: make(map[int64]int64), + } + // Prepare staging and the completed-segment counters BEFORE publishing + // the run: Stats/Progress may observe it immediately afterwards and must + // never read half-initialized plain fields. + if err := f.prepareRun(run); err != nil { + f.mu.Unlock() + run.cancel() + close(run.done) + return err + } + f.activeRun = run + f.mu.Unlock() + + f.launchWorkers(run) + go f.supervise(run) + return nil +} + +// Pause stops all segment downloads synchronously. It waits for the queue +// producer, every worker AND the supervisor (including a merge in flight) +// before returning, so a Store right after Pause sees a consistent state. +func (f *Fetcher) Pause() error { + f.mu.Lock() + run := f.activeRun + f.mu.Unlock() + if run == nil { + return nil + } + run.cancel() + <-run.done + f.clearActiveRun(run) + return nil +} + +// Close pauses any active run and only removes the staging folder once every +// run goroutine has stopped writing to it. +func (f *Fetcher) Close() error { + f.mu.Lock() + f.closed = true + run := f.activeRun + tempDir := "" + if f.state != nil { + tempDir = f.resolveStagingDirLocked() + } + f.mu.Unlock() + + if run != nil { + run.cancel() + <-run.done + f.clearActiveRun(run) + } + if f.impSession != nil { + f.impSession.Clear() + } + if tempDir != "" { + os.RemoveAll(tempDir) + } + return nil +} + +// clearActiveRun detaches a finished run and keeps its counters as the last +// known values for idle Stats/Progress reporting. +func (f *Fetcher) clearActiveRun(run *hlsRun) { + f.mu.Lock() + if f.activeRun == run { + f.activeRun = nil + f.lastStats = run.statsSnapshot() + f.lastDownloaded = run.downloaded.Load() + } + f.mu.Unlock() +} + +// Progress reports final-output bytes stored on disk for the single file. +func (f *Fetcher) Progress() fetcher.Progress { + f.mu.Lock() + run := f.activeRun + if run != nil { + f.mu.Unlock() + return fetcher.Progress{run.downloaded.Load()} + } + last := f.lastDownloaded + f.mu.Unlock() + return fetcher.Progress{last} +} + +func (f *Fetcher) Stats() *fetcher.Stats { + f.mu.Lock() + run := f.activeRun + state := f.state + if run != nil { + f.mu.Unlock() + snap := run.statsSnapshot() + return &fetcher.Stats{Snapshot: &snap} + } + snap := f.lastStats + f.mu.Unlock() + if snap.SegmentsTotal == 0 && state != nil { + snap.SegmentsTotal = len(state.Segments) + } + return &fetcher.Stats{Snapshot: &snap} +} + +// supervise waits for the segment workers, then merges. A canceled context +// exits silently (the engine is pausing), everything else is reported through +// DoneCh exactly once per run. +func (f *Fetcher) supervise(run *hlsRun) { + run.wg.Wait() + defer close(run.done) + + if run.ctx.Err() != nil { + f.clearActiveRun(run) + return + } + var err error + if failed := run.failCount.Load(); failed > 0 { + err = fmt.Errorf("%d segment(s) failed to download", failed) + } else { + err = f.merge(run) + if err == nil { + if run.ctx.Err() != nil { + // Paused while the merge was in flight: keep the staged + // segments and exit silently; the next run re-merges. + f.clearActiveRun(run) + return + } + os.RemoveAll(run.tempDir) + } + } + f.clearActiveRun(run) + select { + case f.DoneCh <- err: + default: + } +} + +// prepareRun prepares the staging folder and loads the completed-segment +// journal into run.baseBytes/baseDone/pending. It performs no goroutine +// launches and no context-bound work, so it runs safely under f.mu before +// the run is published. +func (f *Fetcher) prepareRun(run *hlsRun) error { + if err := os.MkdirAll(run.tempDir, 0777); err != nil { + return err + } + run.journalPath = filepath.Join(run.tempDir, "journal.json") + if err := loadJournal(run); err != nil { + return err + } + for i := range run.segs { + if size, ok := run.journal[int64(i)]; ok { + if info, err := os.Stat(segmentPath(run.tempDir, int64(i))); err == nil && info.Size() == size { + run.baseBytes += size + run.baseDone++ + continue + } + } + run.pending = append(run.pending, int64(i)) + } + run.downloaded.Store(run.baseBytes) + return nil +} + +// launchWorkers starts the queue producer and the segment download workers +// for a published run. Every goroutine registers itself in run.wg so Pause +// waits for all of them. +func (f *Fetcher) launchWorkers(run *hlsRun) { + if len(run.pending) == 0 { + return + } + queue := make(chan int64) + run.wg.Add(1) + go func() { + defer run.wg.Done() + defer close(queue) + for _, idx := range run.pending { + select { + case <-run.ctx.Done(): + return + case queue <- idx: + } + } + }() + + workers := min(f.config.SegmentConnections, len(run.pending)) + for i := 0; i < workers; i++ { + run.wg.Add(1) + go func() { + defer run.wg.Done() + for { + select { + case <-run.ctx.Done(): + return + case idx, ok := <-queue: + if !ok { + return + } + if err := f.downloadSegment(run, idx); err != nil { + if errors.Is(err, context.Canceled) { + return + } + run.failCount.Add(1) + } + } + } + }() + } +} + +func (f *Fetcher) downloadSegment(run *hlsRun, idx int64) error { + seg := run.segs[idx] + data, err := f.fetchSegmentData(run, seg) + if err != nil { + return err + } + if seg.Key != nil { + key, err := f.fetchKey(run, seg.Key) + if err != nil { + return err + } + iv := seg.Key.IV + if iv == nil { + iv = sequenceIV(seg.Sequence) + } + data, err = decryptAES128(data, key, iv) + if err != nil { + return err + } + } + if err := storeSegment(run, idx, data); err != nil { + return err + } + run.doneCount.Add(1) + run.downloaded.Add(int64(len(data))) + return nil +} + +// fetchSegmentData downloads one segment with retries, returning its (still +// encrypted) content. +func (f *Fetcher) fetchSegmentData(run *hlsRun, seg *Segment) ([]byte, error) { + var lastErr error + for attempt := 0; attempt <= f.config.MaxRetries; attempt++ { + if err := run.ctx.Err(); err != nil { + return nil, err + } + if attempt > 0 { + select { + case <-run.ctx.Done(): + return nil, run.ctx.Err() + case <-time.After(time.Duration(attempt) * time.Second): + } + } + data, err := f.tryGetSegment(run, seg) + if err == nil { + return data, nil + } + if errors.Is(err, context.Canceled) { + return nil, err + } + lastErr = err + } + return nil, lastErr +} + +func (f *Fetcher) tryGetSegment(run *hlsRun, seg *Segment) ([]byte, error) { + resp, err := f.do(run.ctx, run.client, run.extra, http.MethodGet, seg.URI, seg.Byterange) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return nil, fmt.Errorf("segment %s: unexpected status %d", seg.URI, resp.StatusCode) + } + reader := &stallReader{rc: resp.Body, timeout: f.timeout()} + var buf bytes.Buffer + chunk := make([]byte, 64<<10) + for { + n, err := reader.Read(chunk) + if n > 0 { + buf.Write(chunk[:n]) + } + if err != nil { + if errors.Is(err, io.EOF) { + break + } + return nil, err + } + } + if seg.Byterange != nil && int64(buf.Len()) != seg.Byterange.Length { + return nil, fmt.Errorf("segment %s: size mismatch, got %d want %d", seg.URI, buf.Len(), seg.Byterange.Length) + } + return buf.Bytes(), nil +} + +func (f *Fetcher) fetchKey(run *hlsRun, key *Key) ([]byte, error) { + run.keyCacheMu.Lock() + if cached, ok := run.keyCache[key.URI]; ok { + run.keyCacheMu.Unlock() + return cached, nil + } + run.keyCacheMu.Unlock() + + var data []byte + var err error + for attempt := 0; attempt <= f.config.MaxRetries; attempt++ { + if err = run.ctx.Err(); err != nil { + return nil, err + } + if attempt > 0 { + select { + case <-run.ctx.Done(): + return nil, run.ctx.Err() + case <-time.After(time.Duration(attempt) * time.Second): + } + } + resp, reqErr := f.do(run.ctx, run.client, run.extra, http.MethodGet, key.URI, nil) + if reqErr != nil { + err = reqErr + continue + } + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + resp.Body.Close() + err = fmt.Errorf("key %s: unexpected status %d", key.URI, resp.StatusCode) + continue + } + data, err = io.ReadAll(io.LimitReader(resp.Body, 1<<20)) + resp.Body.Close() + if err == nil { + break + } + } + if err != nil { + return nil, err + } + if len(data) != 16 { + return nil, fmt.Errorf("key %s: invalid key size %d", key.URI, len(data)) + } + run.keyCacheMu.Lock() + run.keyCache[key.URI] = data + run.keyCacheMu.Unlock() + return data, nil +} + +// merge concatenates the downloaded segments in plan order into the final +// output file. Segments are stored decrypted, so this is pure I/O. The merge +// writes to a unique temporary file next to the target, checks write and +// close errors, and only then replaces the target: a failed merge leaves the +// staged segments intact so it can be retried, and the unique part name keeps +// concurrent tasks that share an output path from clobbering each other. The +// final size is published through Progress: the engine backfills Res.Size +// from Progress().TotalDownloaded() on completion. +func (f *Fetcher) merge(run *hlsRun) error { + target := run.outputPath + if err := os.MkdirAll(filepath.Dir(target), 0777); err != nil { + return err + } + partPath := target + tempDirPrefix + run.stagingID + ".part" + out, err := os.Create(partPath) + if err != nil { + return err + } + var total int64 + for i := range run.segs { + data, err := os.ReadFile(segmentPath(run.tempDir, int64(i))) + if err != nil { + out.Close() + os.Remove(partPath) + return fmt.Errorf("read segment %d failed: %w", i, err) + } + if _, err := out.Write(data); err != nil { + out.Close() + os.Remove(partPath) + return err + } + total += int64(len(data)) + } + if err := out.Close(); err != nil { + os.Remove(partPath) + return err + } + // Remove-then-rename keeps this idempotent: if a previous merge already + // renamed its output before a crash lost the bookkeeping, re-merging + // must still succeed. + if err := os.Remove(target); err != nil && !os.IsNotExist(err) { + os.Remove(partPath) + return err + } + if err := os.Rename(partPath, target); err != nil { + os.Remove(partPath) + return err + } + run.downloaded.Store(total) + return nil +} + +func storeSegment(run *hlsRun, idx int64, data []byte) error { + partPath := segmentPath(run.tempDir, idx) + ".part" + if err := os.WriteFile(partPath, data, 0666); err != nil { + return err + } + if err := os.Rename(partPath, segmentPath(run.tempDir, idx)); err != nil { + return err + } + run.journalMu.Lock() + defer run.journalMu.Unlock() + run.journal[idx] = int64(len(data)) + return saveJournal(run) +} + +func segmentPath(tempDir string, idx int64) string { + return filepath.Join(tempDir, "seg-"+strconv.FormatInt(idx, 10)) +} + +// resolveStagingDirLocked returns the staging folder of the current plan, +// deriving and pinning it on first use so later output renames or moves do +// not orphan the staged segments. It lives next to the task output, which is +// exactly opts.Path for a single-file task. Must be called with f.mu held. +func (f *Fetcher) resolveStagingDirLocked() string { + if f.state.StagingDir == "" { + if f.state.StagingID == "" { + // Legacy state (or a hand-crafted plan): migrate to a fresh task + // owned identity. The old URL-hash folder is deliberately not + // reused: its contents cannot be proven to belong to this plan, + // and sharing it with another same-URL task is exactly the bug + // being fixed. It is left behind as orphaned temp data. + f.state.StagingID = randomStagingID() + } + dir := "" + if f.meta.Opts != nil { + dir = f.meta.Opts.Path + } + f.state.StagingDir = filepath.Join(dir, tempDirPrefix+f.state.StagingID) + } + return f.state.StagingDir +} + +// prefetchSize probes segment Content-Length via HEAD. Any failure resets all +// sizes to zero: a partial size estimate would corrupt progress reporting. +func (f *Fetcher) prefetchSize(segments []*Segment) { + var need []*Segment + for _, seg := range segments { + if seg.Size <= 0 { + need = append(need, seg) + } + } + if len(need) == 0 || len(need) > maxPrefetchSizeSegments { + return + } + + workerCount := min(f.config.SegmentConnections, 8) + var failed atomic.Bool + queue := make(chan *Segment) + var wg sync.WaitGroup + for i := 0; i < workerCount; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for seg := range queue { + ctx, cancel := context.WithTimeout(context.Background(), f.timeout()) + seg.Size = f.headContentLength(ctx, seg.URI) + cancel() + if seg.Size <= 0 { + failed.Store(true) + } + } + }() + } + for _, seg := range need { + queue <- seg + } + close(queue) + wg.Wait() + if failed.Load() { + for _, seg := range segments { + seg.Size = 0 + } + } +} + +func (f *Fetcher) headContentLength(ctx context.Context, u string) int64 { + req, err := buildRequest(ctx, f.extra, http.MethodHead, u, nil, nil) + if err != nil { + return 0 + } + resp, err := f.client.Do(req) + if err != nil { + return 0 + } + defer resp.Body.Close() + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return 0 + } + return resp.ContentLength +} + +func (f *Fetcher) fetchText(ctx context.Context, method, u string) (string, string, error) { + resp, err := f.do(ctx, f.client, f.extra, method, u, nil) + if err != nil { + return "", "", err + } + defer resp.Body.Close() + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return "", "", fmt.Errorf("unexpected status %d for %s", resp.StatusCode, u) + } + data, err := io.ReadAll(io.LimitReader(&stallReader{rc: resp.Body, timeout: f.timeout()}, maxPlaylistSize)) + if err != nil { + return "", "", err + } + return string(data), resp.Request.URL.String(), nil +} + +func (f *Fetcher) do(ctx context.Context, client *http.Client, extra *ReqExtra, method, u string, byterange *ByteRange) (*http.Response, error) { + if method == "" { + method = http.MethodGet + } + req, err := buildRequest(ctx, extra, method, u, nil, byterange) + if err != nil { + return nil, err + } + return client.Do(req) +} + +func (f *Fetcher) playlistMethod() string { + if f.extra != nil && f.extra.Method != "" { + return f.extra.Method + } + return http.MethodGet +} + +// planTotalSize sums the known segment sizes. It returns 0 unless every +// segment is unencrypted with a known size: ciphertext lengths overstate the +// final output (CBC padding is removed on decrypt), and partial estimates +// would make progress stall below 100%. For indeterminate plans the engine +// backfills Res.Size from Progress().TotalDownloaded() when the task +// completes (downloader.go watch). +func planTotalSize(segments []*Segment) int64 { + var total int64 + for _, seg := range segments { + if seg.Key != nil { + return 0 + } + if seg.Size <= 0 { + return 0 + } + total += seg.Size + } + return total +} diff --git a/internal/protocol/hls/fetcher_manager.go b/internal/protocol/hls/fetcher_manager.go new file mode 100644 index 00000000..63941ec2 --- /dev/null +++ b/internal/protocol/hls/fetcher_manager.go @@ -0,0 +1,100 @@ +package hls + +import ( + "net/url" + "path" + "strings" + + "github.com/GopeedLab/gopeed/internal/fetcher" +) + +type FetcherManager struct { +} + +func (fm *FetcherManager) Name() string { + return "hls" +} + +// Filters registers the .m3u8 extension. The manager must be registered BEFORE +// the http manager in the downloader config: protocol routing picks the first +// manager whose filter matches, and the http filter claims every http(s) URL. +func (fm *FetcherManager) Filters() []*fetcher.SchemeFilter { + return []*fetcher.SchemeFilter{ + { + Type: fetcher.FilterTypeFile, + Pattern: "M3U8", + }, + } +} + +func (fm *FetcherManager) Build() fetcher.Fetcher { + return &Fetcher{} +} + +func (fm *FetcherManager) ParseName(u string) string { + parsed, err := url.Parse(u) + if err != nil { + return "" + } + name := path.Base(parsed.Path) + name = strings.TrimSuffix(name, path.Ext(name)) + if name == "" || name == "/" || name == "." { + name = parsed.Hostname() + } + return name +} + +func (fm *FetcherManager) AutoRename() bool { + return true +} + +func (fm *FetcherManager) DefaultConfig() any { + return &config{ + SegmentConnections: defaultSegmentConnections, + MaxRetries: defaultMaxRetries, + TimeoutSeconds: defaultTimeoutSeconds, + PrefetchContentLength: true, + } +} + +func (fm *FetcherManager) Store(f fetcher.Fetcher) (data any, err error) { + _f := f.(*Fetcher) + _f.mu.Lock() + defer _f.mu.Unlock() + if _f.state == nil { + return nil, nil + } + // The engine's periodic checkpoint calls Store without holding the task + // lock and serializes the returned value afterwards, while Resolve/Patch/ + // Start mutate the live state under the fetcher mutex: return a snapshot + // copy taken under that mutex, mirroring the HTTP protocol's Store. + return cloneState(_f.state), nil +} + +func cloneState(state *fetcherState) *fetcherState { + cloned := *state + cloned.Segments = make([]*Segment, len(state.Segments)) + for i, seg := range state.Segments { + s := *seg + cloned.Segments[i] = &s + } + return &cloned +} + +func (fm *FetcherManager) Restore() (v any, f func(meta *fetcher.FetcherMeta, v any) fetcher.Fetcher) { + return &fetcherState{}, func(meta *fetcher.FetcherMeta, v any) fetcher.Fetcher { + fb := &FetcherManager{} + restored := fb.Build().(*Fetcher) + restored.meta = meta + if state, ok := v.(*fetcherState); ok && state.MediaURL != "" { + restored.state = state + } + // Setup is invoked by the engine (restoreFetcher) after this builder + // returns; ReqExtra parsing happens there as well. + return restored + } +} + +func (fm *FetcherManager) Close() error { + return nil +} diff --git a/internal/protocol/hls/fetcher_test.go b/internal/protocol/hls/fetcher_test.go new file mode 100644 index 00000000..3dd785f0 --- /dev/null +++ b/internal/protocol/hls/fetcher_test.go @@ -0,0 +1,857 @@ +package hls + +import ( + "bytes" + "crypto/aes" + "crypto/cipher" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strconv" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/GopeedLab/gopeed/internal/controller" + "github.com/GopeedLab/gopeed/internal/fetcher" + "github.com/GopeedLab/gopeed/pkg/base" +) + +func pkcs7PadForTest(data []byte, blockSize int) []byte { + pad := blockSize - len(data)%blockSize + return append(data, bytes.Repeat([]byte{byte(pad)}, pad)...) +} + +func encryptSegment(plain, key, iv []byte) []byte { + padded := pkcs7PadForTest(plain, aes.BlockSize) + block, _ := aes.NewCipher(key) + encrypted := make([]byte, len(padded)) + cipher.NewCBCEncrypter(block, iv).CryptBlocks(encrypted, padded) + return encrypted +} + +// serveContentWithReferer is a helper handler that enforces an optional +// Referer requirement and reports request counts. +type cdnHandler struct { + mux *http.ServeMux + requireReferer string + requests atomic.Int64 +} + +func (h *cdnHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + h.requests.Add(1) + if h.requireReferer != "" && r.Header.Get("Referer") != h.requireReferer { + w.WriteHeader(http.StatusForbidden) + return + } + h.mux.ServeHTTP(w, r) +} + +func newTestFetcher(t *testing.T, cfg config) *Fetcher { + t.Helper() + fm := &FetcherManager{} + f := fm.Build().(*Fetcher) + ctl := controller.NewController() + ctl.GetConfig = func(v any) { + if c, ok := v.(*config); ok { + *c = cfg + } + } + f.Setup(ctl) + return f +} + +func testConfig() config { + return config{ + SegmentConnections: 4, + MaxRetries: 2, + TimeoutSeconds: 5, + PrefetchContentLength: true, + } +} + +func mustResolve(t *testing.T, f *Fetcher, playlistURL, referer string) { + t.Helper() + req := &base.Request{URL: playlistURL} + if referer != "" { + req.Extra = &ReqExtra{Header: map[string]string{"Referer": referer}} + } + if err := f.Resolve(req, &base.Options{Path: t.TempDir()}); err != nil { + t.Fatalf("resolve: %v", err) + } +} + +func mustRun(t *testing.T, f *Fetcher) { + t.Helper() + if err := f.Start(); err != nil { + t.Fatalf("start: %v", err) + } + if err := f.Wait(); err != nil { + t.Fatalf("wait: %v", err) + } +} + +func statsOf(t *testing.T, f *Fetcher) Stats { + t.Helper() + s, ok := f.Stats().Snapshot.(*Stats) + if !ok { + t.Fatalf("unexpected stats snapshot type %T", f.Stats().Snapshot) + } + return *s +} + +func readOutput(t *testing.T, f *Fetcher) []byte { + t.Helper() + data, err := os.ReadFile(f.meta.SingleFilepath()) + if err != nil { + t.Fatal(err) + } + return data +} + +func TestFetcherE2E_MasterTS(t *testing.T) { + seg := func(i int) []byte { + return []byte(fmt.Sprintf("SEGMENT-%02d-PAYLOAD", i)) + } + mux := http.NewServeMux() + mux.HandleFunc("/master.m3u8", func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte("#EXTM3U\n" + + "#EXT-X-STREAM-INF:BANDWIDTH=800000\nv1/index.m3u8\n" + + "#EXT-X-STREAM-INF:BANDWIDTH=2000000\nv2/index.m3u8\n")) + }) + mux.HandleFunc("/v1/index.m3u8", func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte("#EXTM3U\n#EXTINF:4,\nlow-0.ts\n#EXT-X-ENDLIST\n")) + }) + mux.HandleFunc("/v2/index.m3u8", func(w http.ResponseWriter, r *http.Request) { + var b strings.Builder + b.WriteString("#EXTM3U\n") + for i := 0; i < 5; i++ { + b.WriteString(fmt.Sprintf("#EXTINF:4,\nseg-%d.ts\n", i)) + } + b.WriteString("#EXT-X-ENDLIST\n") + w.Write([]byte(b.String())) + }) + for i := 0; i < 5; i++ { + content := seg(i) + path := "/v2/seg-" + strconv.Itoa(i) + ".ts" + mux.HandleFunc(path, func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Length", strconv.Itoa(len(content))) + w.Write(content) + }) + } + server := httptest.NewServer(&cdnHandler{mux: mux}) + defer server.Close() + + f := newTestFetcher(t, testConfig()) + mustResolve(t, f, server.URL+"/master.m3u8", "") + // Master resolution picked the 2000000 variant. + if f.state.MediaURL != server.URL+"/v2/index.m3u8" { + t.Fatalf("wrong variant picked: %s", f.state.MediaURL) + } + // HEAD prefetch should have computed the exact total size. + if f.meta.Res.Size != int64(5*len(seg(0))) { + t.Fatalf("prefetched size wrong: %d", f.meta.Res.Size) + } + mustRun(t, f) + + var want []byte + for i := 0; i < 5; i++ { + want = append(want, seg(i)...) + } + got := readOutput(t, f) + if !bytes.Equal(got, want) { + t.Errorf("merged output mismatch: %q", got) + } + // The staging folder must be cleaned up after a successful merge; only + // the output file remains in the task directory. + if entries, _ := os.ReadDir(filepath.Dir(f.meta.SingleFilepath())); len(entries) != 1 { + for _, e := range entries { + t.Logf("leftover entry: %s", e.Name()) + } + t.Errorf("staging dir should be cleaned up after merge") + } + if f.meta.Res.Size != int64(len(want)) { + t.Errorf("final size wrong: %d", f.meta.Res.Size) + } +} + +func TestFetcherE2E_RefererEnforced(t *testing.T) { + page := "https://player.example.com/watch/123" + var playlist strings.Builder + playlist.WriteString("#EXTM3U\n") + segments := make(map[string][]byte) + for i := 0; i < 3; i++ { + name := fmt.Sprintf("seg-%d.ts", i) + segments["/"+name] = []byte(fmt.Sprintf("DATA-%d", i)) + playlist.WriteString(fmt.Sprintf("#EXTINF:4,\n%s\n", name)) + } + playlist.WriteString("#EXT-X-ENDLIST\n") + + mux := http.NewServeMux() + mux.HandleFunc("/video.m3u8", func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte(playlist.String())) + }) + for path, content := range segments { + c := content + mux.HandleFunc(path, func(w http.ResponseWriter, r *http.Request) { + w.Write(c) + }) + } + server := httptest.NewServer(&cdnHandler{mux: mux, requireReferer: page}) + defer server.Close() + + // Without Referer the resolve itself must fail (403 on the playlist). + blocked := newTestFetcher(t, testConfig()) + req := &base.Request{URL: server.URL + "/video.m3u8"} + if err := blocked.Resolve(req, &base.Options{Path: t.TempDir()}); err == nil { + t.Fatal("resolve without Referer should fail") + } + + // With the Referer set through the task extra headers it must succeed. + allowed := newTestFetcher(t, testConfig()) + allowedReq := &base.Request{ + URL: server.URL + "/video.m3u8", + Extra: &ReqExtra{Header: map[string]string{"Referer": page}}, + } + if err := allowed.Resolve(allowedReq, &base.Options{Path: t.TempDir()}); err != nil { + t.Fatal(err) + } + mustRun(t, allowed) + want := []byte("DATA-0DATA-1DATA-2") + if !bytes.Equal(readOutput(t, allowed), want) { + t.Error("output mismatch with Referer download") + } +} + +func TestFetcherE2E_AES128KeyRotation(t *testing.T) { + key1 := []byte("0123456789abcdef") + key2 := []byte("fedcba9876543210") + iv1 := []byte("aaaaaaaaaaaaaaaa") + plain1 := []byte("ENCRYPTED-SEGMENT-ONE!!") // padded on encrypt + plain2 := []byte("ENCRYPTED-SEGMENT-TWO!!") + // seg-2 has no explicit IV: derived from media sequence 1. + enc1 := encryptSegment(plain1, key1, iv1) + enc2 := encryptSegment(plain2, key2, sequenceIV(1)) + + mux := http.NewServeMux() + mux.HandleFunc("/video.m3u8", func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte("#EXTM3U\n" + + "#EXT-X-MEDIA-SEQUENCE:0\n" + + "#EXT-X-KEY:METHOD=AES-128,URI=\"key-1.bin\",IV=0x" + hexEncode(iv1) + "\n" + + "#EXTINF:5,\nseg-1.ts\n" + + "#EXT-X-KEY:METHOD=AES-128,URI=\"key-2.bin\"\n" + + "#EXTINF:5,\nseg-2.ts\n" + + "#EXT-X-ENDLIST\n")) + }) + mux.HandleFunc("/key-1.bin", func(w http.ResponseWriter, r *http.Request) { + w.Write(key1) + }) + mux.HandleFunc("/key-2.bin", func(w http.ResponseWriter, r *http.Request) { + w.Write(key2) + }) + mux.HandleFunc("/seg-1.ts", func(w http.ResponseWriter, r *http.Request) { + w.Write(enc1) + }) + mux.HandleFunc("/seg-2.ts", func(w http.ResponseWriter, r *http.Request) { + w.Write(enc2) + }) + server := httptest.NewServer(mux) + defer server.Close() + + f := newTestFetcher(t, testConfig()) + mustResolve(t, f, server.URL+"/video.m3u8", "") + // Ciphertext sizes overstate the final output after decryption, so an + // encrypted plan must report an indeterminate size. + if f.meta.Res.Size != 0 { + t.Errorf("encrypted plan should report size 0, got %d", f.meta.Res.Size) + } + mustRun(t, f) + + want := append(append([]byte{}, plain1...), plain2...) + if !bytes.Equal(readOutput(t, f), want) { + t.Errorf("decrypted output mismatch: %q", readOutput(t, f)) + } +} + +func TestFetcherE2E_ByteRange(t *testing.T) { + data := []byte("0123456789ABCDEFGHIJ") // 20 bytes + mux := http.NewServeMux() + mux.HandleFunc("/video.m3u8", func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte("#EXTM3U\n" + + "#EXT-X-BYTERANGE:10@0\nblob.bin\n" + + "#EXT-X-BYTERANGE:10\nblob.bin\n" + + "#EXT-X-ENDLIST\n")) + }) + mux.HandleFunc("/blob.bin", func(w http.ResponseWriter, r *http.Request) { + rng := r.Header.Get("Range") + if rng == "" { + w.Write(data) + return + } + // bytes=start-end + var start, end int + fmt.Sscanf(rng, "bytes=%d-%d", &start, &end) + w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, len(data))) + w.WriteHeader(http.StatusPartialContent) + w.Write(data[start : end+1]) + }) + server := httptest.NewServer(mux) + defer server.Close() + + f := newTestFetcher(t, testConfig()) + mustResolve(t, f, server.URL+"/video.m3u8", "") + if f.meta.Res.Size != 20 { + t.Fatalf("byterange sizes should be known from the playlist, got %d", f.meta.Res.Size) + } + mustRun(t, f) + if !bytes.Equal(readOutput(t, f), data) { + t.Errorf("byterange merge mismatch: %q", readOutput(t, f)) + } +} + +func TestFetcherE2E_PauseResume(t *testing.T) { + const total = 8 + var served atomic.Int64 + mux := http.NewServeMux() + mux.HandleFunc("/video.m3u8", func(w http.ResponseWriter, r *http.Request) { + var b strings.Builder + b.WriteString("#EXTM3U\n") + for i := 0; i < total; i++ { + b.WriteString(fmt.Sprintf("#EXTINF:4,\nseg-%d.ts\n", i)) + } + b.WriteString("#EXT-X-ENDLIST\n") + w.Write([]byte(b.String())) + }) + segments := make(map[int][]byte) + for i := 0; i < total; i++ { + segments[i] = []byte(fmt.Sprintf("SLOW-SEGMENT-%02d", i)) + } + for i := 0; i < total; i++ { + idx := i + mux.HandleFunc(fmt.Sprintf("/seg-%d.ts", i), func(w http.ResponseWriter, r *http.Request) { + time.Sleep(150 * time.Millisecond) + served.Add(1) + w.Write(segments[idx]) + }) + } + server := httptest.NewServer(mux) + defer server.Close() + + cfg := testConfig() + cfg.SegmentConnections = 2 + cfg.PrefetchContentLength = false + f := newTestFetcher(t, cfg) + mustResolve(t, f, server.URL+"/video.m3u8", "") + + if err := f.Start(); err != nil { + t.Fatal(err) + } + // Let a couple of segments finish, then pause. + time.Sleep(400 * time.Millisecond) + if err := f.Pause(); err != nil { + t.Fatal(err) + } + doneBefore := statsOf(t, f).SegmentsDone + if doneBefore == 0 { + t.Fatal("expected some segments to complete before pause") + } + if doneBefore == total { + t.Fatal("all segments finished before pause, test is not exercising resume") + } + + // Resume must finish the rest and produce the full output. + if err := f.Start(); err != nil { + t.Fatal(err) + } + if err := f.Wait(); err != nil { + t.Fatal(err) + } + var want []byte + for i := 0; i < total; i++ { + want = append(want, segments[i]...) + } + if !bytes.Equal(readOutput(t, f), want) { + t.Errorf("resumed output mismatch: %q", readOutput(t, f)) + } +} + +func TestFetcherE2E_StoreRestoreResume(t *testing.T) { + const total = 6 + var served atomic.Int64 + mux := http.NewServeMux() + mux.HandleFunc("/video.m3u8", func(w http.ResponseWriter, r *http.Request) { + var b strings.Builder + b.WriteString("#EXTM3U\n") + for i := 0; i < total; i++ { + b.WriteString(fmt.Sprintf("#EXTINF:4,\nseg-%d.ts\n", i)) + } + b.WriteString("#EXT-X-ENDLIST\n") + w.Write([]byte(b.String())) + }) + segments := make(map[int][]byte) + for i := 0; i < total; i++ { + segments[i] = []byte(fmt.Sprintf("RESTORE-SEGMENT-%02d", i)) + } + for i := 0; i < total; i++ { + idx := i + mux.HandleFunc(fmt.Sprintf("/seg-%d.ts", i), func(w http.ResponseWriter, r *http.Request) { + time.Sleep(120 * time.Millisecond) + served.Add(1) + w.Write(segments[idx]) + }) + } + server := httptest.NewServer(mux) + defer server.Close() + + cfg := testConfig() + cfg.SegmentConnections = 2 + cfg.PrefetchContentLength = false + + f := newTestFetcher(t, cfg) + mustResolve(t, f, server.URL+"/video.m3u8", "") + if err := f.Start(); err != nil { + t.Fatal(err) + } + time.Sleep(350 * time.Millisecond) + if err := f.Pause(); err != nil { + t.Fatal(err) + } + doneBefore := statsOf(t, f).SegmentsDone + if doneBefore == 0 { + t.Fatal("expected some segments to finish before simulated restart") + } + + // Simulate an app restart: persist, then restore into a fresh fetcher. + fm := &FetcherManager{} + state, err := fm.Store(f) + if err != nil { + t.Fatal(err) + } + if state == nil { + t.Fatal("state should be persisted") + } + ctl := controller.NewController() + ctl.GetConfig = func(v any) { + if c, ok := v.(*config); ok { + *c = cfg + } + } + holder := fm.Build().(*Fetcher) + holder.Setup(ctl) + holder.meta = &fetcher.FetcherMeta{ + Req: f.meta.Req, + Opts: f.meta.Opts, + Res: f.meta.Res, + } + _, builder := fm.Restore() + rf := builder(holder.meta, state).(*Fetcher) + // The engine calls Setup on the builder result (restoreFetcher). + rf.Setup(ctl) + if rf.state == nil || len(rf.state.Segments) != total { + t.Fatal("restored state missing segment plan") + } + + if err := rf.Start(); err != nil { + t.Fatal(err) + } + if err := rf.Wait(); err != nil { + t.Fatal(err) + } + if done := statsOf(t, rf).SegmentsDone; done != total { + t.Errorf("want %d done, got %d", total, done) + } + var want []byte + for i := 0; i < total; i++ { + want = append(want, segments[i]...) + } + if !bytes.Equal(readOutput(t, rf), want) { + t.Errorf("restored output mismatch: %q", readOutput(t, rf)) + } +} + +func hexEncode(b []byte) string { + const digits = "0123456789abcdef" + out := make([]byte, 0, len(b)*2) + for _, v := range b { + out = append(out, digits[v>>4], digits[v&0x0f]) + } + return string(out) +} + +// restoreLikeEngine round-trips the persisted state and the task meta through +// JSON the way the engine storage does, then builds a fresh fetcher. This is +// the path that historically lost the request extra headers. +func restoreLikeEngine(t *testing.T, f *Fetcher, cfg config) *Fetcher { + t.Helper() + fm := &FetcherManager{} + stateAny, err := fm.Store(f) + if err != nil { + t.Fatal(err) + } + rawState, err := json.Marshal(stateAny) + if err != nil { + t.Fatal(err) + } + storedState := &fetcherState{} + if err := json.Unmarshal(rawState, storedState); err != nil { + t.Fatal(err) + } + rawMeta, err := json.Marshal(f.meta) + if err != nil { + t.Fatal(err) + } + restoredMeta := &fetcher.FetcherMeta{} + if err := json.Unmarshal(rawMeta, restoredMeta); err != nil { + t.Fatal(err) + } + if _, ok := restoredMeta.Req.Extra.(*ReqExtra); ok { + t.Fatal("storage round trip should yield a map-shaped extra, like the engine") + } + ctl := controller.NewController() + ctl.GetConfig = func(v any) { + if c, ok := v.(*config); ok { + *c = cfg + } + } + _, builder := fm.Restore() + rf := builder(restoredMeta, storedState).(*Fetcher) + // The engine calls Setup on the builder result (restoreFetcher). + rf.Setup(ctl) + return rf +} + +func TestFetcherE2E_StoreRestoreAuthenticated(t *testing.T) { + const total = 4 + page := "https://player.example.com/watch/42" + var served atomic.Int64 + mux := http.NewServeMux() + var playlist strings.Builder + playlist.WriteString("#EXTM3U\n") + for i := 0; i < total; i++ { + playlist.WriteString(fmt.Sprintf("#EXTINF:4,\nseg-%d.ts\n", i)) + } + playlist.WriteString("#EXT-X-ENDLIST\n") + mux.HandleFunc("/video.m3u8", func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte(playlist.String())) + }) + segments := make(map[int][]byte) + for i := 0; i < total; i++ { + segments[i] = []byte(fmt.Sprintf("AUTH-SEGMENT-%02d", i)) + idx := i + mux.HandleFunc(fmt.Sprintf("/seg-%d.ts", idx), func(w http.ResponseWriter, r *http.Request) { + time.Sleep(80 * time.Millisecond) + served.Add(1) + w.Write(segments[idx]) + }) + } + server := httptest.NewServer(&cdnHandler{mux: mux, requireReferer: page}) + defer server.Close() + + cfg := testConfig() + cfg.SegmentConnections = 2 + cfg.PrefetchContentLength = false + + f := newTestFetcher(t, cfg) + req := &base.Request{ + URL: server.URL + "/video.m3u8", + Extra: &ReqExtra{Header: map[string]string{"Referer": page}}, + } + if err := f.Resolve(req, &base.Options{Path: t.TempDir()}); err != nil { + t.Fatal(err) + } + if err := f.Start(); err != nil { + t.Fatal(err) + } + time.Sleep(150 * time.Millisecond) + if err := f.Pause(); err != nil { + t.Fatal(err) + } + + rf := restoreLikeEngine(t, f, cfg) + if rf.extra == nil || rf.extra.Header["Referer"] != page { + t.Fatalf("restored fetcher lost the request extra headers: %+v", rf.extra) + } + if err := rf.Start(); err != nil { + t.Fatal(err) + } + if err := rf.Wait(); err != nil { + t.Fatalf("resumed download failed (request headers lost?): %v", err) + } + var want []byte + for i := 0; i < total; i++ { + want = append(want, segments[i]...) + } + if !bytes.Equal(readOutput(t, rf), want) { + t.Errorf("restored output mismatch: %q", readOutput(t, rf)) + } +} + +func TestFetcherE2E_PatchPartialUpdate(t *testing.T) { + page := "https://player.example.com/watch/7" + var requiredReferer atomic.Value // string + requiredReferer.Store(page) + var playlist strings.Builder + playlist.WriteString("#EXTM3U\n") + segments := make(map[string][]byte) + for i := 0; i < 3; i++ { + segments[fmt.Sprintf("/s%d.ts", i)] = []byte(fmt.Sprintf("PATCH-%d", i)) + playlist.WriteString(fmt.Sprintf("#EXTINF:4,\ns%d.ts\n", i)) + } + playlist.WriteString("#EXT-X-ENDLIST\n") + mux := http.NewServeMux() + mux.HandleFunc("/video.m3u8", func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte(playlist.String())) + }) + for path, content := range segments { + c := content + mux.HandleFunc(path, func(w http.ResponseWriter, r *http.Request) { + w.Write(c) + }) + } + // The Referer requirement can be flipped mid-test to prove that a patched + // header is actually sent by the next run. + dyn := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if want, _ := requiredReferer.Load().(string); want != "" && r.Header.Get("Referer") != want { + w.WriteHeader(http.StatusForbidden) + return + } + mux.ServeHTTP(w, r) + }) + server := httptest.NewServer(dyn) + defer server.Close() + + f := newTestFetcher(t, testConfig()) + req := &base.Request{ + URL: server.URL + "/video.m3u8", + Extra: &ReqExtra{Header: map[string]string{"Referer": page}}, + } + opts := &base.Options{Path: t.TempDir()} + if err := f.Resolve(req, opts); err != nil { + t.Fatal(err) + } + + // A nil/nil patch must be a no-op, not a panic. + if err := f.Patch(nil, nil); err != nil { + t.Fatalf("nil patch should be accepted: %v", err) + } + + // Header-only patch: the resolved plan and output options are retained + // while the new header takes effect on the next run. + newReferer := "https://player.example.com/watch/9" + if err := f.Patch(&base.Request{ + URL: server.URL + "/video.m3u8", + Extra: &ReqExtra{Header: map[string]string{"Referer": newReferer}}, + }, nil); err != nil { + t.Fatal(err) + } + if f.state == nil || f.meta.Res == nil { + t.Fatal("header-only patch must keep the resolved plan") + } + if f.meta.Opts.Path != opts.Path { + t.Fatalf("header-only patch must keep the output path: %s", f.meta.Opts.Path) + } + if f.extra.Header["Referer"] != newReferer { + t.Fatalf("patched header not applied: %+v", f.extra.Header) + } + requiredReferer.Store(newReferer) + if err := f.Start(); err != nil { + t.Fatal(err) + } + if err := f.Wait(); err != nil { + t.Fatalf("run with patched header failed: %v", err) + } + if got := readOutput(t, f); string(got) != "PATCH-0PATCH-1PATCH-2" { + t.Errorf("output mismatch after header patch: %q", got) + } + + // Options-only patch: renaming the task keeps the resolved plan. + if err := f.Patch(nil, &base.Options{Name: "renamed.ts"}); err != nil { + t.Fatal(err) + } + if f.state == nil { + t.Fatal("options-only patch must keep the resolved plan") + } + if f.meta.Opts.Name != "renamed.ts" || f.meta.Opts.Path != opts.Path { + t.Fatalf("options patch misapplied: %+v", f.meta.Opts) + } + + // URL-only patch: the plan is dropped for a re-resolve but the output + // path and renamed file survive. + if err := f.Patch(&base.Request{URL: server.URL + "/refreshed.m3u8"}, nil); err != nil { + t.Fatal(err) + } + if f.state != nil || f.meta.Res != nil { + t.Fatal("URL patch must drop the resolved plan") + } + if f.meta.Opts.Path != opts.Path || f.meta.Opts.Name != "renamed.ts" { + t.Fatal("URL patch must keep the output options") + } +} + +func TestFetcherE2E_PatchWhileRunningRejected(t *testing.T) { + mux := http.NewServeMux() + var playlist strings.Builder + playlist.WriteString("#EXTM3U\n") + for i := 0; i < 6; i++ { + playlist.WriteString(fmt.Sprintf("#EXTINF:4,\nseg-%d.ts\n", i)) + } + playlist.WriteString("#EXT-X-ENDLIST\n") + mux.HandleFunc("/video.m3u8", func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte(playlist.String())) + }) + for i := 0; i < 6; i++ { + mux.HandleFunc(fmt.Sprintf("/seg-%d.ts", i), func(w http.ResponseWriter, r *http.Request) { + time.Sleep(150 * time.Millisecond) + w.Write([]byte("X")) + }) + } + server := httptest.NewServer(mux) + defer server.Close() + + f := newTestFetcher(t, testConfig()) + if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: t.TempDir()}); err != nil { + t.Fatal(err) + } + if err := f.Start(); err != nil { + t.Fatal(err) + } + if err := f.Patch(&base.Request{URL: server.URL + "/other.m3u8"}, nil); err == nil { + t.Fatal("patching a running task must be rejected") + } + if err := f.Pause(); err != nil { + t.Fatal(err) + } + // After the pause the patch goes through and only then drops the plan. + if err := f.Patch(&base.Request{URL: server.URL + "/other.m3u8"}, nil); err != nil { + t.Fatal(err) + } + if f.state != nil { + t.Fatal("URL patch after pause must drop the plan") + } +} + +func TestFetcherE2E_RetryAfterFailure(t *testing.T) { + const total = 4 + var healthy atomic.Bool + mux := http.NewServeMux() + var playlist strings.Builder + playlist.WriteString("#EXTM3U\n") + for i := 0; i < total; i++ { + playlist.WriteString(fmt.Sprintf("#EXTINF:4,\nseg-%d.ts\n", i)) + } + playlist.WriteString("#EXT-X-ENDLIST\n") + mux.HandleFunc("/video.m3u8", func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte(playlist.String())) + }) + segments := make(map[int][]byte) + for i := 0; i < total; i++ { + segments[i] = []byte(fmt.Sprintf("RETRY-SEGMENT-%02d", i)) + idx := i + mux.HandleFunc(fmt.Sprintf("/seg-%d.ts", idx), func(w http.ResponseWriter, r *http.Request) { + if idx == 1 && !healthy.Load() { + w.WriteHeader(http.StatusInternalServerError) + return + } + time.Sleep(60 * time.Millisecond) + w.Write(segments[idx]) + }) + } + server := httptest.NewServer(mux) + defer server.Close() + + cfg := testConfig() + cfg.MaxRetries = 0 // fail the run on the first bad segment + cfg.PrefetchContentLength = false + f := newTestFetcher(t, cfg) + if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: t.TempDir()}); err != nil { + t.Fatal(err) + } + if err := f.Start(); err != nil { + t.Fatal(err) + } + if err := f.Wait(); err == nil { + t.Fatal("first run should fail while the segment endpoint is down") + } + if got := statsOf(t, f).SegmentsFailed; got == 0 { + t.Fatal("failed segments should be reported in stats") + } + + // Server recovered: a fresh run must succeed and report zero failures. + healthy.Store(true) + if err := f.Start(); err != nil { + t.Fatal(err) + } + if err := f.Wait(); err != nil { + t.Fatalf("retry run failed: %v", err) + } + stats := statsOf(t, f) + if stats.SegmentsFailed != 0 { + t.Errorf("failures must reset per run, got %d", stats.SegmentsFailed) + } + if stats.SegmentsDone != total { + t.Errorf("want %d done, got %d", total, stats.SegmentsDone) + } + var want []byte + for i := 0; i < total; i++ { + want = append(want, segments[i]...) + } + if !bytes.Equal(readOutput(t, f), want) { + t.Errorf("retry output mismatch: %q", readOutput(t, f)) + } +} + +func TestFetcherE2E_StateMissingSelfHeal(t *testing.T) { + const total = 3 + mux := http.NewServeMux() + var playlist strings.Builder + playlist.WriteString("#EXTM3U\n") + for i := 0; i < total; i++ { + playlist.WriteString(fmt.Sprintf("#EXTINF:4,\nseg-%d.ts\n", i)) + } + playlist.WriteString("#EXT-X-ENDLIST\n") + mux.HandleFunc("/video.m3u8", func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte(playlist.String())) + }) + segments := make(map[int][]byte) + for i := 0; i < total; i++ { + segments[i] = []byte(fmt.Sprintf("SELFHEAL-%02d", i)) + idx := i + mux.HandleFunc(fmt.Sprintf("/seg-%d.ts", idx), func(w http.ResponseWriter, r *http.Request) { + w.Write(segments[idx]) + }) + } + server := httptest.NewServer(mux) + defer server.Close() + + f := newTestFetcher(t, testConfig()) + if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: t.TempDir()}); err != nil { + t.Fatal(err) + } + // Simulate the crash window where the engine persisted the task with its + // resource but the HLS plan was never checkpointed. + f.state = nil + outputName := f.meta.Res.Files[0].Name + if err := f.Start(); err != nil { + t.Fatalf("start should self-heal a missing plan: %v", err) + } + if err := f.Wait(); err != nil { + t.Fatalf("self-healed run failed: %v", err) + } + if f.state == nil || len(f.state.Segments) != total { + t.Fatal("self-heal should rebuild the segment plan") + } + if f.meta.Res.Files[0].Name != outputName { + t.Fatalf("self-heal renamed the output: %s", f.meta.Res.Files[0].Name) + } + var want []byte + for i := 0; i < total; i++ { + want = append(want, segments[i]...) + } + if !bytes.Equal(readOutput(t, f), want) { + t.Errorf("self-healed output mismatch: %q", readOutput(t, f)) + } +} diff --git a/internal/protocol/hls/helper.go b/internal/protocol/hls/helper.go new file mode 100644 index 00000000..ba4a4f32 --- /dev/null +++ b/internal/protocol/hls/helper.go @@ -0,0 +1,100 @@ +package hls + +import ( + "context" + "crypto/tls" + "fmt" + "io" + "net" + "net/http" + "net/http/cookiejar" + "time" + + "github.com/GopeedLab/gopeed/internal/httpclient" + "github.com/GopeedLab/gopeed/pkg/base" +) + +const defaultUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36" + +const maxPlaylistSize = 10 << 20 // 10MB, playlists are small text files + +func (f *Fetcher) timeout() time.Duration { + return time.Duration(f.config.TimeoutSeconds) * time.Second +} + +// buildClient builds an HTTP client honoring the task proxy and TLS settings. +// The client has no total timeout on purpose: large segments over slow links +// must not be aborted; body stalls are handled by stallReader instead. It is +// only called while the fetcher is quiescent (Resolve/Start), never from +// inside running worker goroutines. +func (f *Fetcher) buildClient() *http.Client { + var proxy *base.RequestProxy + skipVerify := false + if req := f.meta.Req; req != nil { + proxy = req.Proxy + skipVerify = req.SkipVerifyCert + } + jar, _ := cookiejar.New(nil) + client, err := httpclient.NewClient(httpclient.Options{ + Client: httpclient.ClientOptions{ + Jar: jar, + }, + Transport: httpclient.TransportOptions{ + DialContext: (&net.Dialer{ + Timeout: f.timeout(), + }).DialContext, + Proxy: f.Ctl.GetProxy(proxy), + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: skipVerify, + }, + TLSHandshakeTimeout: f.timeout(), + }, + Impersonation: httpclient.ImpersonationOptions{ + Mode: httpclient.ImpersonationAuto, + Session: f.impSession, + }, + }) + if err != nil { + panic(fmt.Sprintf("build HTTP client: %v", err)) + } + return client +} + +// buildRequest builds a request with task extra headers, the default user +// agent and an optional byte range. extra may be nil. +func buildRequest(ctx context.Context, extra *ReqExtra, method, u string, body io.Reader, byterange *ByteRange) (*http.Request, error) { + req, err := http.NewRequestWithContext(ctx, method, u, body) + if err != nil { + return nil, err + } + if extra != nil { + for key, value := range extra.Header { + req.Header.Set(key, value) + } + } + if req.Header.Get("User-Agent") == "" { + req.Header.Set("User-Agent", defaultUserAgent) + } + if byterange != nil { + req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", byterange.Offset, byterange.Offset+byterange.Length-1)) + } + return req, nil +} + +// stallReader aborts reads that see no data for the configured timeout. +type stallReader struct { + rc io.ReadCloser + timeout time.Duration +} + +func (r *stallReader) Read(p []byte) (int, error) { + timer := time.AfterFunc(r.timeout, func() { + r.rc.Close() + }) + defer timer.Stop() + return r.rc.Read(p) +} + +func (r *stallReader) Close() error { + return r.rc.Close() +} diff --git a/internal/protocol/hls/journal.go b/internal/protocol/hls/journal.go new file mode 100644 index 00000000..9d06a80b --- /dev/null +++ b/internal/protocol/hls/journal.go @@ -0,0 +1,112 @@ +package hls + +import ( + "crypto/rand" + "crypto/sha1" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "os" + "strconv" +) + +// journalFile is the persisted completed-segment ledger. It is bound to the +// exact download plan: a journal whose fingerprint does not match the current +// plan is discarded instead of being applied to the wrong segments. +type journalFile struct { + StagingID string `json:"stagingID"` + PlanHash string `json:"planHash"` + SegmentCount int `json:"segmentCount"` + Sizes map[string]int64 `json:"sizes"` +} + +// randomStagingID returns a random identifier used to give every task its own +// staging folder, stable across app restarts through the persisted state. +func randomStagingID() string { + buf := make([]byte, 6) + if _, err := rand.Read(buf); err != nil { + // crypto/rand failing is catastrophic anyway; fall back to address + // entropy rather than crashing the fetch. + return fmt.Sprintf("%x", sha1.Sum([]byte(fmt.Sprintf("%p", buf))))[:12] + } + return hex.EncodeToString(buf) +} + +// planFingerprint hashes everything that defines the download plan: the media +// URL, the ordered segments with their sequence numbers, byte ranges, keys +// and init markers. Two tasks resolve to different fingerprints even when +// they point at the same playlist URL. +func planFingerprint(state *fetcherState) string { + h := sha1.New() + io.WriteString(h, state.MediaURL+"\n") + io.WriteString(h, strconv.Itoa(len(state.Segments))+"\n") + for _, seg := range state.Segments { + fmt.Fprintf(h, "%d|%s|", seg.Sequence, seg.URI) + if seg.Byterange != nil { + fmt.Fprintf(h, "%d@%d|", seg.Byterange.Offset, seg.Byterange.Length) + } + if seg.Key != nil { + // The IV is part of the plan: same URI with a rotated IV would + // silently decrypt to garbage if the journal were reused. + fmt.Fprintf(h, "key=%s iv=%x|", seg.Key.URI, seg.Key.IV) + } + if seg.IsInit { + io.WriteString(h, "init|") + } + io.WriteString(h, "\n") + } + return hex.EncodeToString(h.Sum(nil))[:16] +} + +// loadJournal reads the journal of a run, discarding anything that cannot be +// proven to belong to the current plan: foreign or legacy journals, corrupted +// files, and entries whose recorded size no longer matches the plan. +func loadJournal(run *hlsRun) error { + data, err := os.ReadFile(run.journalPath) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return err + } + var file journalFile + if err := json.Unmarshal(data, &file); err != nil { + // A broken or legacy journal must not kill a restartable download. + return nil + } + if file.StagingID != run.stagingID || file.PlanHash != run.planHash || file.SegmentCount != len(run.segs) { + return nil + } + for key, size := range file.Sizes { + idx, err := strconv.ParseInt(key, 10, 64) + if err != nil || idx < 0 || idx >= int64(len(run.segs)) || size <= 0 { + continue + } + run.journal[idx] = size + } + return nil +} + +// saveJournal atomically rewrites the journal through a temp file + rename so +// a crash never leaves a half-written ledger behind. +func saveJournal(run *hlsRun) error { + file := journalFile{ + StagingID: run.stagingID, + PlanHash: run.planHash, + SegmentCount: len(run.segs), + Sizes: make(map[string]int64, len(run.journal)), + } + for idx, size := range run.journal { + file.Sizes[strconv.FormatInt(idx, 10)] = size + } + data, err := json.Marshal(file) + if err != nil { + return err + } + tmp := run.journalPath + ".tmp" + if err := os.WriteFile(tmp, data, 0666); err != nil { + return err + } + return os.Rename(tmp, run.journalPath) +} diff --git a/internal/protocol/hls/media_validate_test.go b/internal/protocol/hls/media_validate_test.go new file mode 100644 index 00000000..276d2fd0 --- /dev/null +++ b/internal/protocol/hls/media_validate_test.go @@ -0,0 +1,297 @@ +package hls + +import ( + "bytes" + "fmt" + "net/http" + "net/http/httptest" + "os" + "testing" + + "github.com/GopeedLab/gopeed/pkg/base" +) + +// The validators below check real media structure, not just byte +// concatenation: TS packets are verified on the 188 byte grid with +// continuity-counter rules, PAT/PMT presence and PES start codes; fMP4 +// output is verified by walking the ISO-BMFF box hierarchy. + +func be32(b []byte) int { + return int(b[0])<<24 | int(b[1])<<16 | int(b[2])<<8 | int(b[3]) +} + +// validateTS asserts structural validity of a merged MPEG-2 TS stream. +func validateTS(t *testing.T, data []byte) { + t.Helper() + if len(data) == 0 || len(data)%188 != 0 { + t.Fatalf("TS size %d is not a positive multiple of 188", len(data)) + } + type pidState struct{ cc int } + ccByPID := map[int]int{} + patSeen, pmtSeen, pesSeen := false, false, false + for off := 0; off < len(data); off += 188 { + pkt := data[off : off+188] + if pkt[0] != 0x47 { + t.Fatalf("bad TS sync byte at offset %d", off) + } + pid := (int(pkt[1]&0x1F) << 8) | int(pkt[2]) + pusi := pkt[1]&0x40 != 0 + afc := (pkt[3] >> 4) & 0x3 + if afc == 0 { + continue // reserved + } + idx := 4 + if afc&0x2 != 0 { // adaptation field present + afLen := int(pkt[4]) + if afLen > 0 && pkt[5]&0x80 != 0 { + // discontinuity_indicator: restart the CC expectation + delete(ccByPID, pid) + } + idx += 1 + afLen + if afc == 2 { + continue // adaptation only, no payload: no CC rule + } + } + if idx > 188 { + t.Fatalf("adaptation field overruns packet at offset %d", off) + } + if pid == 0x1FFF { + continue // null packet + } + payload := pkt[idx:] + if pusi && len(payload) >= 3 && pid != 0 && payload[0] == 0 && payload[1] == 0 && payload[2] == 1 { + pesSeen = true + } + if pid == 0 { + if ptr := int(payload[0]); 1+ptr < len(payload) && payload[1+ptr] == 0x00 { + patSeen = true // program_association_section table_id + } + } else if pusi && len(payload) >= 2 && payload[0] == 0 && payload[1] == 0x02 { + pmtSeen = true // program_map_section table_id + } + cc := int(pkt[3] & 0x0F) + prev, ok := ccByPID[pid] + if !ok { + ccByPID[pid] = cc + continue + } + // CC increments by one per payload-bearing packet; a repeated value + // models the single legal duplicate packet. + if cc != (prev+1)%16 && cc != prev { + t.Fatalf("continuity error on PID %d: %d after %d at offset %d", pid, cc, prev, off) + } + ccByPID[pid] = cc + } + if !patSeen { + t.Error("no PAT found in merged TS") + } + if !pmtSeen { + t.Error("no PMT found in merged TS") + } + if !pesSeen { + t.Error("no PES start code found in merged TS") + } +} + +// walkBoxes iterates the top-level ISO-BMFF boxes of data[off:end]. +func walkBoxes(t *testing.T, data []byte, off, end int) []string { + t.Helper() + var types []string + for off+8 <= end { + size := be32(data[off:]) + typ := string(data[off+4 : off+8]) + if size < 8 || off+size > end { + t.Fatalf("invalid box %q at offset %d (size %d)", typ, off, size) + } + types = append(types, typ) + off += size + } + return types +} + +// validateFMP4 asserts the merged fMP4 output is init section followed by +// well-formed moof+mdat fragments. +func validateFMP4(t *testing.T, merged []byte, wantFragments int) { + t.Helper() + types := walkBoxes(t, merged, 0, len(merged)) + hasFtyp, hasMoov := false, false + fragments := 0 + expectFragment := false + for i, typ := range types { + switch typ { + case "ftyp": + if i != 0 { + t.Errorf("ftyp not at stream start") + } + hasFtyp = true + case "moov": + if i != 1 { + t.Errorf("moov must follow ftyp, found at index %d", i) + } + hasMoov = true + case "styp": + expectFragment = true + case "moof": + if !hasMoov { + t.Errorf("fragment before moov") + } + expectFragment = true + case "mdat": + if !expectFragment { + t.Errorf("mdat outside of a fragment at index %d", i) + } + expectFragment = false + fragments++ + case "mfra": + // optional trailer + default: + t.Errorf("unexpected top-level box %q", typ) + } + } + if !hasFtyp || !hasMoov { + t.Error("merged fMP4 missing ftyp/moov init section") + } + if fragments != wantFragments { + t.Errorf("want %d fragments in merged fMP4, got %d", wantFragments, fragments) + } +} + +func mediaFixture(t *testing.T, name string) []byte { + t.Helper() + data, err := os.ReadFile("testdata/media/" + name) + if err != nil { + t.Fatal(err) + } + return data +} + +func serveFixtureServer(t *testing.T) *httptest.Server { + t.Helper() + mux := http.NewServeMux() + fs := http.FileServer(http.Dir("testdata/media")) + mux.Handle("/", fs) + return httptest.NewServer(mux) +} + +func TestHLSRealMedia_TS(t *testing.T) { + seg0 := mediaFixture(t, "sample-00.ts") + seg1 := mediaFixture(t, "sample-01.ts") + playlist := "#EXTM3U\n" + + "#EXTINF:0.5,\nsample-00.ts\n" + + "#EXTINF:0.5,\nsample-01.ts\n" + + "#EXT-X-ENDLIST\n" + mux := http.NewServeMux() + mux.HandleFunc("/video.m3u8", func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte(playlist)) + }) + mux.Handle("/", http.FileServer(http.Dir("testdata/media"))) + server := httptest.NewServer(mux) + defer server.Close() + + f := newTestFetcher(t, testConfig()) + if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: t.TempDir()}); err != nil { + t.Fatal(err) + } + if err := f.Start(); err != nil { + t.Fatal(err) + } + if err := f.Wait(); err != nil { + t.Fatal(err) + } + got := readOutput(t, f) + if !bytes.Equal(got, append(append([]byte{}, seg0...), seg1...)) { + t.Fatal("merged TS is not the exact concatenation of the segments") + } + validateTS(t, got) +} + +func TestHLSRealMedia_FMP4(t *testing.T) { + init := mediaFixture(t, "fmp4-init.mp4") + frag1 := mediaFixture(t, "fmp4-seg1.m4s") + frag2 := mediaFixture(t, "fmp4-seg2.m4s") + frag3 := mediaFixture(t, "fmp4-seg3.m4s") + playlist := "#EXTM3U\n" + + "#EXT-X-MAP:URI=\"fmp4-init.mp4\"\n" + + "#EXTINF:0.5,\nfmp4-seg1.m4s\n" + + "#EXTINF:0.5,\nfmp4-seg2.m4s\n" + + "#EXTINF:0.5,\nfmp4-seg3.m4s\n" + + "#EXT-X-ENDLIST\n" + mux := http.NewServeMux() + mux.HandleFunc("/video.m3u8", func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte(playlist)) + }) + mux.Handle("/", http.FileServer(http.Dir("testdata/media"))) + server := httptest.NewServer(mux) + defer server.Close() + + f := newTestFetcher(t, testConfig()) + if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: t.TempDir()}); err != nil { + t.Fatal(err) + } + if err := f.Start(); err != nil { + t.Fatal(err) + } + if err := f.Wait(); err != nil { + t.Fatal(err) + } + got := readOutput(t, f) + want := append(append(append(append([]byte{}, init...), frag1...), frag2...), frag3...) + if !bytes.Equal(got, want) { + t.Fatal("merged fMP4 is not the exact concatenation of init + fragments") + } + validateFMP4(t, got, 3) +} + +func TestHLSRealMedia_AES128(t *testing.T) { + keys := [][]byte{[]byte("key-A-0123456789"), []byte("key-B-9876543210")} + iv := []byte("iv-1337-01234567") + plain := [][]byte{mediaFixture(t, "sample-00.ts"), mediaFixture(t, "sample-01.ts")} + + playlist := "#EXTM3U\n" + + "#EXT-X-MEDIA-SEQUENCE:0\n" + + "#EXT-X-KEY:METHOD=AES-128,URI=\"key-0.bin\",IV=0x" + hexEncode(iv) + "\n" + + "#EXTINF:0.5,\nenc-0.ts\n" + + "#EXT-X-KEY:METHOD=AES-128,URI=\"key-1.bin\"\n" + + "#EXTINF:0.5,\nenc-1.ts\n" + + "#EXT-X-ENDLIST\n" + mux := http.NewServeMux() + mux.HandleFunc("/video.m3u8", func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte(playlist)) + }) + for i, key := range keys { + k := key + mux.HandleFunc(fmt.Sprintf("/key-%d.bin", i), func(w http.ResponseWriter, r *http.Request) { + w.Write(k) + }) + // Segment 0 has an explicit IV, segment 1 derives it from its media + // sequence number (1). + enc := encryptSegment(plain[i], k, map[bool][]byte{true: iv, false: sequenceIV(int64(i))}[i == 0]) + mux.HandleFunc(fmt.Sprintf("/enc-%d.ts", i), func(w http.ResponseWriter, r *http.Request) { + w.Write(enc) + }) + } + server := httptest.NewServer(mux) + defer server.Close() + + f := newTestFetcher(t, testConfig()) + if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: t.TempDir()}); err != nil { + t.Fatal(err) + } + if f.meta.Res.Size != 0 { + t.Errorf("encrypted plan must report indeterminate size, got %d", f.meta.Res.Size) + } + if err := f.Start(); err != nil { + t.Fatal(err) + } + if err := f.Wait(); err != nil { + t.Fatal(err) + } + got := readOutput(t, f) + if !bytes.Equal(got, append(append([]byte{}, plain[0]...), plain[1]...)) { + t.Fatal("decrypted output mismatch") + } + validateTS(t, got) + if p := f.Progress(); p.TotalDownloaded() != int64(len(got)) { + t.Errorf("final progress %d != merged size %d", p.TotalDownloaded(), len(got)) + } +} diff --git a/internal/protocol/hls/playlist.go b/internal/protocol/hls/playlist.go new file mode 100644 index 00000000..6dd1eeb2 --- /dev/null +++ b/internal/protocol/hls/playlist.go @@ -0,0 +1,479 @@ +package hls + +import ( + "encoding/hex" + "errors" + "fmt" + "net/url" + "path" + "strconv" + "strings" +) + +// Variant is one rendition entry of a master playlist. +type Variant struct { + URI string + Bandwidth int64 + AvgBandwidth int64 + Resolution string + Codecs string + Name string + AudioGroup string // AUDIO attribute: group id of the audio renditions +} + +// MediaRendition is one EXT-X-MEDIA entry of a master playlist. Only the +// fields needed to detect unsupported track layouts are kept. +type MediaRendition struct { + Type string + GroupID string + URI string +} + +// MasterPlaylist is a parsed master playlist with its rendition groups. +type MasterPlaylist struct { + Variants []*Variant + Rendition map[string][]*MediaRendition // group id -> all EXT-X-MEDIA members +} + +// Key is the decryption key applied to the segments that follow its tag, until +// the next EXT-X-KEY tag or the end of the playlist. +type Key struct { + Method string + URI string // absolute URI + IV []byte // explicit IV; nil means derive from the media sequence number +} + +// ByteRange is an EXT-X-BYTERANGE sub range of the segment resource. +type ByteRange struct { + Length int64 + Offset int64 +} + +// Segment is one downloadable piece of a media playlist. The init section of +// an fMP4 stream is represented as a segment with IsInit set. +type Segment struct { + URI string + Sequence int64 // media sequence number, used to derive the default AES IV + Byterange *ByteRange + Key *Key // nil means unencrypted + IsInit bool + Gap bool + Duration float64 + Size int64 // exact size in bytes when known (byterange / HEAD probe), 0 when unknown +} + +// Media is a parsed media playlist. +type Media struct { + Segments []*Segment + Version int + Live bool // no EXT-X-ENDLIST tag + IsFMP4 bool // playlist declares an init section via EXT-X-MAP + Discontinuity bool // playlist contains EXT-X-DISCONTINUITY +} + +const ( + keyMethodNone = "NONE" + keyMethodAES128 = "AES-128" + keyMethodSampleAES = "SAMPLE-AES" +) + +// IsMasterPlaylist reports whether the content looks like a master playlist. +func IsMasterPlaylist(content string) bool { + return strings.Contains(content, "#EXT-X-STREAM-INF") +} + +func validateM3U8Header(content string) error { + for _, line := range strings.Split(content, "\n") { + line = strings.TrimSpace(line) + if line == "" { + continue + } + if line != "#EXTM3U" { + return errors.New("content is not a valid m3u8 playlist") + } + return nil + } + return errors.New("empty m3u8 playlist") +} + +// ParseMaster parses a master playlist and returns all variants. +func ParseMaster(content string, baseURL *url.URL) ([]*Variant, error) { + master, err := ParseMasterPlaylist(content, baseURL) + if err != nil { + return nil, err + } + return master.Variants, nil +} + +// ParseMasterPlaylist parses a master playlist including its EXT-X-MEDIA +// rendition groups. +func ParseMasterPlaylist(content string, baseURL *url.URL) (*MasterPlaylist, error) { + if err := validateM3U8Header(content); err != nil { + return nil, err + } + master := &MasterPlaylist{Rendition: make(map[string][]*MediaRendition)} + var pending *Variant + for _, line := range strings.Split(content, "\n") { + line = strings.TrimSpace(line) + if line == "" { + continue + } + if strings.HasPrefix(line, "#EXT-X-STREAM-INF:") { + attrs := parseAttributes(line[len("#EXT-X-STREAM-INF:"):]) + pending = &Variant{ + Bandwidth: parseInt64(attrs["BANDWIDTH"]), + AvgBandwidth: parseInt64(attrs["AVERAGE-BANDWIDTH"]), + Resolution: attrs["RESOLUTION"], + Codecs: unquote(attrs["CODECS"]), + Name: unquote(attrs["NAME"]), + AudioGroup: unquote(attrs["AUDIO"]), + } + continue + } + if strings.HasPrefix(line, "#EXT-X-MEDIA:") { + attrs := parseAttributes(line[len("#EXT-X-MEDIA:"):]) + rendition := &MediaRendition{ + Type: strings.ToUpper(attrs["TYPE"]), + GroupID: unquote(attrs["GROUP-ID"]), + } + // Only an explicit URI means a separate track; resolving an empty + // reference would fabricate one out of the playlist URL. + if uri := unquote(attrs["URI"]); uri != "" { + rendition.URI = resolveURL(baseURL, uri) + } + if rendition.GroupID != "" { + // Keep every member: a group is unsupported as a whole when + // ANY of its members carries its own URI. + master.Rendition[rendition.GroupID] = append(master.Rendition[rendition.GroupID], rendition) + } + continue + } + if pending != nil { + if line[0] != '#' { + pending.URI = resolveURL(baseURL, line) + master.Variants = append(master.Variants, pending) + } + pending = nil + } + } + if len(master.Variants) == 0 { + return nil, errors.New("master playlist has no variants") + } + return master, nil +} + +// checkAudioRendition rejects the selected variant when its AUDIO group +// carries a separate audio track: downloading only the video variant would +// silently produce a silent file. Groups the variant does not reference are +// ignored (variants without an AUDIO attribute are assumed self-contained). +func checkAudioRendition(master *MasterPlaylist, best *Variant) error { + if best == nil || best.AudioGroup == "" { + return nil + } + for _, rendition := range master.Rendition[best.AudioGroup] { + if rendition == nil || rendition.Type != "AUDIO" || rendition.URI == "" { + continue + } + return fmt.Errorf( + "selected variant uses a separate audio rendition (group %q), which is not supported: the merged output would have no audio", + best.AudioGroup) + } + return nil +} + +// PickBestVariant selects the highest-bandwidth variant. +func PickBestVariant(variants []*Variant) *Variant { + var best *Variant + for _, v := range variants { + if best == nil || v.Bandwidth > best.Bandwidth { + best = v + } + } + return best +} + +// ParseMedia parses a media playlist into an ordered segment plan. +func ParseMedia(content string, baseURL *url.URL) (*Media, error) { + if err := validateM3U8Header(content); err != nil { + return nil, err + } + + media := &Media{Live: true} + var currentKey *Key + var pendingByterange *ByteRange + var pendingGap bool + var pendingDuration float64 + var lastRangeEnd int64 + var sequence int64 + var mapIdentity string // normalized identity of the current EXT-X-MAP + + for _, line := range strings.Split(content, "\n") { + line = strings.TrimSpace(line) + if line == "" { + continue + } + if line[0] != '#' { + // Every non-comment line in a media playlist is a segment URI. + seg := &Segment{ + URI: resolveURL(baseURL, line), + Sequence: sequence, + Byterange: pendingByterange, + Key: currentKey, + Gap: pendingGap, + Duration: pendingDuration, + } + if pendingByterange != nil { + // Byterange segments have exact sizes straight from the playlist. + seg.Size = pendingByterange.Length + } + sequence++ + pendingByterange = nil + pendingGap = false + pendingDuration = 0 + media.Segments = append(media.Segments, seg) + continue + } + + switch { + case strings.HasPrefix(line, "#EXTINF:"): + value := line[len("#EXTINF:"):] + if idx := strings.Index(value, ","); idx >= 0 { + value = value[:idx] + } + pendingDuration = parseFloat(value) + case strings.HasPrefix(line, "#EXT-X-KEY:"): + key, err := parseKey(line[len("#EXT-X-KEY:"):], baseURL) + if err != nil { + return nil, err + } + currentKey = key + case strings.HasPrefix(line, "#EXT-X-MAP:"): + attrs := parseAttributes(line[len("#EXT-X-MAP:"):]) + uriAttr := unquote(attrs["URI"]) + if uriAttr == "" { + return nil, errors.New("EXT-X-MAP without URI is not supported") + } + resolved := resolveURL(baseURL, uriAttr) + // The init section identity covers every attribute, not just the + // URI: a repeated tag with a different BYTERANGE addresses + // different bytes and changes the init section. NUL keeps the + // two parts unambiguously separated. + identity := resolved + "\x00" + attrs["BYTERANGE"] + if mapIdentity != "" { + if mapIdentity != identity { + return nil, errors.New("changing init sections are not supported") + } + media.IsFMP4 = true + continue + } + mapIdentity = identity + init := &Segment{ + URI: resolved, + Sequence: sequence, + IsInit: true, + Key: currentKey, + } + if attrs["BYTERANGE"] != "" { + init.Byterange = parseByteRange(attrs["BYTERANGE"], 0) + if init.Byterange == nil { + return nil, fmt.Errorf("invalid EXT-X-MAP BYTERANGE: %s", attrs["BYTERANGE"]) + } + } + if init.Key != nil && init.Key.IV == nil { + // RFC 8216 only defines sequence-derived IVs for media + // segments; guessing one for the init section produces + // corrupt output, so require the playlist to be explicit. + return nil, errors.New("encrypted EXT-X-MAP without an explicit IV is not supported") + } + // The init section is prepended to the segment plan, it takes + // the media sequence number of the playlist for IV derivation. + media.Segments = append(media.Segments, init) + media.IsFMP4 = true + case strings.HasPrefix(line, "#EXT-X-BYTERANGE:"): + pendingByterange = parseByteRange(line[len("#EXT-X-BYTERANGE:"):], lastRangeEnd) + if pendingByterange == nil { + return nil, fmt.Errorf("invalid EXT-X-BYTERANGE: %s", line) + } + case line == "#EXT-X-GAP": + pendingGap = true + case line == "#EXT-X-DISCONTINUITY": + media.Discontinuity = true + case line == "#EXT-X-ENDLIST": + media.Live = false + case strings.HasPrefix(line, "#EXT-X-MEDIA-SEQUENCE:"): + sequence = parseInt64(line[len("#EXT-X-MEDIA-SEQUENCE:"):]) + case strings.HasPrefix(line, "#EXT-X-VERSION:"): + media.Version = int(parseInt64(line[len("#EXT-X-VERSION:"):])) + } + + if pendingByterange != nil { + // Non-segment tags between BYTERANGE and its URI must not reset the + // running offset, so it is tracked here after each parse. + lastRangeEnd = pendingByterange.Offset + pendingByterange.Length + } + } + + if len(media.Segments) == 0 { + return nil, errors.New("media playlist has no segments") + } + return media, nil +} + +// validateSupported rejects playlist features the byte-concatenation merge +// cannot reproduce faithfully. Skipping such features silently would report +// success while producing a broken file. +func validateSupported(media *Media) error { + if media.Live { + return errors.New("live streams are not supported yet") + } + if media.Discontinuity { + return errors.New("EXT-X-DISCONTINUITY is not supported: the merged output would have a broken timeline") + } + for _, seg := range media.Segments { + if seg.Gap { + return errors.New("EXT-X-GAP is not supported: the missing segment would leave a hole in the merged output") + } + } + return nil +} + +func parseKey(attrs string, baseURL *url.URL) (*Key, error) { + parsed := parseAttributes(attrs) + switch method := parsed["METHOD"]; method { + case keyMethodNone: + return nil, nil + case keyMethodAES128: + uri := unquote(parsed["URI"]) + if uri == "" { + return nil, errors.New("AES-128 EXT-X-KEY without URI") + } + if format := unquote(parsed["KEYFORMAT"]); format != "" && format != "identity" { + return nil, fmt.Errorf("unsupported EXT-X-KEY KEYFORMAT: %s", format) + } + key := &Key{Method: keyMethodAES128, URI: resolveURL(baseURL, uri)} + if iv, ok := parsed["IV"]; ok { + value := strings.TrimPrefix(strings.ToLower(iv), "0x") + decoded, err := hex.DecodeString(value) + if err != nil || len(decoded) != 16 { + return nil, fmt.Errorf("invalid EXT-X-KEY IV: %s", iv) + } + key.IV = decoded + } + return key, nil + case keyMethodSampleAES: + return nil, errors.New("SAMPLE-AES encryption is not supported") + default: + return nil, fmt.Errorf("unsupported encryption method: %s", method) + } +} + +func parseByteRange(value string, defaultOffset int64) *ByteRange { + lengthPart := value + offsetPart := "" + if idx := strings.Index(value, "@"); idx >= 0 { + lengthPart = value[:idx] + offsetPart = value[idx+1:] + } + length := parseInt64(lengthPart) + if length <= 0 { + return nil + } + offset := defaultOffset + if offsetPart != "" { + offset = parseInt64(offsetPart) + if offset < 0 { + return nil + } + } + return &ByteRange{Length: length, Offset: offset} +} + +// resolveURL resolves a playlist URI reference against the playlist URL +// following RFC 3986. +func resolveURL(baseURL *url.URL, ref string) string { + refURL, err := url.Parse(ref) + if err != nil { + return ref + } + return baseURL.ResolveReference(refURL).String() +} + +// parseAttributes parses an HLS attribute list like FOO="bar",NUM=42. +func parseAttributes(s string) map[string]string { + attrs := make(map[string]string) + var key, val strings.Builder + inKey := true + inQuote := false + flush := func() { + name := strings.ToUpper(strings.TrimSpace(key.String())) + if name != "" { + attrs[name] = strings.TrimSpace(val.String()) + } + key.Reset() + val.Reset() + inKey = true + } + for i := 0; i < len(s); i++ { + c := s[i] + if inKey { + if c == '=' { + inKey = false + } else { + key.WriteByte(c) + } + continue + } + if c == '"' { + inQuote = !inQuote + continue + } + if c == ',' && !inQuote { + flush() + continue + } + val.WriteByte(c) + } + flush() + return attrs +} + +func unquote(s string) string { + s = strings.TrimSpace(s) + if len(s) >= 2 && strings.HasPrefix(s, "\"") && strings.HasSuffix(s, "\"") { + return s[1 : len(s)-1] + } + return s +} + +func parseInt64(s string) int64 { + value, _ := strconv.ParseInt(strings.TrimSpace(s), 10, 64) + return value +} + +func parseFloat(s string) float64 { + value, _ := strconv.ParseFloat(strings.TrimSpace(s), 64) + return value +} + +// OutputExtension returns the container file extension for the playlist. +func (m *Media) OutputExtension() string { + if m.IsFMP4 { + return ".mp4" + } + return ".ts" +} + +// DeriveOutputName builds the merged output file name from a playlist URL. +func DeriveOutputName(playlistURL string, media *Media) string { + name := "" + if u, err := url.Parse(playlistURL); err == nil { + name = path.Base(u.Path) + name = strings.TrimSuffix(name, path.Ext(name)) + } + if name == "" || name == "/" || name == "." { + if u, err := url.Parse(playlistURL); err == nil { + name = u.Hostname() + } + } + return name + media.OutputExtension() +} diff --git a/internal/protocol/hls/playlist_realworld_test.go b/internal/protocol/hls/playlist_realworld_test.go new file mode 100644 index 00000000..6a9de3c8 --- /dev/null +++ b/internal/protocol/hls/playlist_realworld_test.go @@ -0,0 +1,39 @@ +package hls + +import ( + _ "embed" + "net/url" + "testing" +) + +//go:embed testdata/olemovienews.m3u8 +var realWorldPlaylist string + +// TestParseMediaRealWorld guards parsing against a real site playlist +// (olemovienews.com VOD, 138 TS segments). +func TestParseMediaRealWorld(t *testing.T) { + base, err := url.Parse("https://europe.olemovienews.com/ts2/20230125/afAxfjit/mp4/afAxfjit.mp4/index-v1-a1.m3u8") + if err != nil { + t.Fatal(err) + } + media, err := ParseMedia(realWorldPlaylist, base) + if err != nil { + t.Fatal(err) + } + if media.Live { + t.Error("real VOD playlist should not be live") + } + if media.IsFMP4 { + t.Error("playlist should be detected as TS") + } + if len(media.Segments) != 138 { + t.Errorf("want 138 segments, got %d", len(media.Segments)) + } + first := media.Segments[0] + if first.URI != "https://europe.olemovienews.com/ts2/20230125/afAxfjit/mp4/afAxfjit.mp4/seg-1-v1-a1.ts" { + t.Errorf("segment URI resolution wrong: %s", first.URI) + } + if media.Segments[0].Sequence != 1 { + t.Errorf("first sequence = %d, want 1", media.Segments[0].Sequence) + } +} diff --git a/internal/protocol/hls/playlist_test.go b/internal/protocol/hls/playlist_test.go new file mode 100644 index 00000000..fb219aa4 --- /dev/null +++ b/internal/protocol/hls/playlist_test.go @@ -0,0 +1,252 @@ +package hls + +import ( + "net/url" + "testing" +) + +func mustURL(t *testing.T, raw string) *url.URL { + t.Helper() + u, err := url.Parse(raw) + if err != nil { + t.Fatal(err) + } + return u +} + +func TestParseMediaTS(t *testing.T) { + content := "#EXTM3U\n" + + "#EXT-X-TARGETDURATION:20\n" + + "#EXT-X-VERSION:3\n" + + "#EXT-X-MEDIA-SEQUENCE:1\n" + + "#EXTINF:20.000,\n" + + "seg-1.ts\n" + + "#EXTINF:20.000,\n" + + "seg-2.ts\n" + + "#EXT-X-ENDLIST\n" + media, err := ParseMedia(content, mustURL(t, "https://cdn.example.com/vod/index.m3u8")) + if err != nil { + t.Fatal(err) + } + if media.Live { + t.Error("VOD playlist should not be live") + } + if media.IsFMP4 { + t.Error("TS playlist should not be fMP4") + } + if len(media.Segments) != 2 { + t.Fatalf("want 2 segments, got %d", len(media.Segments)) + } + want := "https://cdn.example.com/vod/seg-1.ts" + if media.Segments[0].URI != want { + t.Errorf("want uri %s, got %s", want, media.Segments[0].URI) + } + if media.Segments[0].Sequence != 1 { + t.Errorf("want sequence 1, got %d", media.Segments[0].Sequence) + } + if media.Segments[0].Key != nil { + t.Error("segment should be unencrypted") + } +} + +func TestParseMediaWithoutExtinf(t *testing.T) { + // Some playlists list segment URIs without EXTINF lines. + content := "#EXTM3U\nseg-1.ts\nseg-2.ts\n#EXT-X-ENDLIST\n" + media, err := ParseMedia(content, mustURL(t, "https://cdn.example.com/a/index.m3u8")) + if err != nil { + t.Fatal(err) + } + if len(media.Segments) != 2 { + t.Fatalf("want 2 segments, got %d", len(media.Segments)) + } +} + +func TestParseMediaLive(t *testing.T) { + content := "#EXTM3U\n#EXT-X-TARGETDURATION:6\n#EXTINF:6.0,\nseg-1.ts\n" + media, err := ParseMedia(content, mustURL(t, "https://cdn.example.com/live/index.m3u8")) + if err != nil { + t.Fatal(err) + } + if !media.Live { + t.Error("playlist without ENDLIST should be live") + } +} + +func TestParseMediaFMP4(t *testing.T) { + content := "#EXTM3U\n" + + "#EXT-X-VERSION:6\n" + + "#EXT-X-MAP:URI=\"init.mp4\"\n" + + "#EXTINF:5.0,\n" + + "seg-1.m4s\n" + + "#EXT-X-ENDLIST\n" + media, err := ParseMedia(content, mustURL(t, "https://cdn.example.com/vod/index.m3u8")) + if err != nil { + t.Fatal(err) + } + if !media.IsFMP4 { + t.Error("playlist with MAP should be fMP4") + } + if len(media.Segments) != 2 { + t.Fatalf("want 2 segments (init + 1), got %d", len(media.Segments)) + } + if !media.Segments[0].IsInit { + t.Error("first segment should be the init section") + } + if media.Segments[0].URI != "https://cdn.example.com/vod/init.mp4" { + t.Errorf("init uri wrong: %s", media.Segments[0].URI) + } + if media.OutputExtension() != ".mp4" { + t.Errorf("want .mp4, got %s", media.OutputExtension()) + } +} + +func TestParseMediaMultiMapError(t *testing.T) { + content := "#EXTM3U\n" + + "#EXT-X-MAP:URI=\"init1.mp4\"\n" + + "#EXTINF:5.0,\nseg-1.m4s\n" + + "#EXT-X-DISCONTINUITY\n" + + "#EXT-X-MAP:URI=\"init2.mp4\"\n" + + "#EXTINF:5.0,\nseg-2.m4s\n" + + "#EXT-X-ENDLIST\n" + if _, err := ParseMedia(content, mustURL(t, "https://cdn.example.com/vod/index.m3u8")); err == nil { + t.Error("multi-period fMP4 should be rejected") + } +} + +func TestParseMediaKeyRotation(t *testing.T) { + content := "#EXTM3U\n" + + "#EXT-X-KEY:METHOD=AES-128,URI=\"key1.bin\",IV=0x00000000000000000000000000000001\n" + + "#EXTINF:5.0,\nseg-1.ts\n" + + "#EXT-X-KEY:METHOD=AES-128,URI=\"key2.bin\"\n" + + "#EXTINF:5.0,\nseg-2.ts\n" + + "#EXT-X-KEY:METHOD=NONE\n" + + "#EXTINF:5.0,\nseg-3.ts\n" + + "#EXT-X-ENDLIST\n" + media, err := ParseMedia(content, mustURL(t, "https://cdn.example.com/vod/index.m3u8")) + if err != nil { + t.Fatal(err) + } + if media.Segments[0].Key == nil || media.Segments[0].Key.URI != "https://cdn.example.com/vod/key1.bin" { + t.Fatalf("segment 1 key wrong: %+v", media.Segments[0].Key) + } + if len(media.Segments[0].Key.IV) != 16 || media.Segments[0].Key.IV[15] != 1 { + t.Error("explicit IV not parsed") + } + if media.Segments[1].Key == nil || media.Segments[1].Key.URI != "https://cdn.example.com/vod/key2.bin" { + t.Fatalf("segment 2 key wrong: %+v", media.Segments[1].Key) + } + if media.Segments[1].Key.IV != nil { + t.Error("key without IV should leave IV nil (sequence-derived)") + } + if media.Segments[2].Key != nil { + t.Error("segment 3 should be unencrypted after METHOD=NONE") + } +} + +func TestParseMediaSampleAESError(t *testing.T) { + content := "#EXTM3U\n#EXT-X-KEY:METHOD=SAMPLE-AES,URI=\"skd://key\"\n#EXTINF:5.0,\nseg-1.ts\n" + if _, err := ParseMedia(content, mustURL(t, "https://cdn.example.com/vod/index.m3u8")); err == nil { + t.Error("SAMPLE-AES should be rejected") + } +} + +func TestParseMediaByterange(t *testing.T) { + content := "#EXTM3U\n" + + "#EXT-X-BYTERANGE:1000@0\nseg.ts\n" + + "#EXT-X-BYTERANGE:2000\nseg.ts\n" + + "#EXT-X-BYTERANGE:3000\nseg.ts\n" + + "#EXT-X-ENDLIST\n" + media, err := ParseMedia(content, mustURL(t, "https://cdn.example.com/vod/index.m3u8")) + if err != nil { + t.Fatal(err) + } + ranges := [][2]int64{{0, 1000}, {1000, 2000}, {3000, 3000}} + for i, want := range ranges { + br := media.Segments[i].Byterange + if br == nil { + t.Fatalf("segment %d byterange missing", i) + } + if br.Offset != want[0] || br.Length != want[1] { + t.Errorf("segment %d: want offset %d length %d, got %d/%d", i, want[0], want[1], br.Offset, br.Length) + } + } +} + +func TestParseMediaGap(t *testing.T) { + content := "#EXTM3U\n#EXTINF:5.0,\nseg-1.ts\n#EXT-X-GAP\n#EXTINF:5.0,\nseg-2.ts\n#EXT-X-ENDLIST\n" + media, err := ParseMedia(content, mustURL(t, "https://cdn.example.com/vod/index.m3u8")) + if err != nil { + t.Fatal(err) + } + if media.Segments[0].Gap || !media.Segments[1].Gap { + t.Error("gap flags wrong") + } +} + +func TestParseMediaInvalidHeader(t *testing.T) { + if _, err := ParseMedia("not a playlist", mustURL(t, "https://cdn.example.com/a.m3u8")); err == nil { + t.Error("invalid content should be rejected") + } +} + +func TestParseMasterPickBest(t *testing.T) { + content := "#EXTM3U\n" + + "#EXT-X-MEDIA:TYPE=AUDIO,GROUP-ID=\"aud\",NAME=\"en\",DEFAULT=YES,URI=\"audio/index.m3u8\"\n" + + "#EXT-X-STREAM-INF:BANDWIDTH=800000,RESOLUTION=640x360\n" + + "360p/index.m3u8\n" + + "#EXT-X-STREAM-INF:BANDWIDTH=2500000,RESOLUTION=1920x1080,CODECS=\"avc1.640028,mp4a.40.2\"\n" + + "1080p/index.m3u8?sign=abc\n" + + "#EXT-X-ENDLIST\n" + variants, err := ParseMaster(content, mustURL(t, "https://cdn.example.com/vod/master.m3u8")) + if err != nil { + t.Fatal(err) + } + if len(variants) != 2 { + t.Fatalf("want 2 variants, got %d", len(variants)) + } + best := PickBestVariant(variants) + if best.Bandwidth != 2500000 { + t.Errorf("want highest bandwidth variant, got %d", best.Bandwidth) + } + if best.URI != "https://cdn.example.com/vod/1080p/index.m3u8?sign=abc" { + t.Errorf("variant uri wrong: %s", best.URI) + } +} + +func TestResolveURLQueryNotInherited(t *testing.T) { + base := mustURL(t, "https://cdn.example.com/vod/index.m3u8?sign=abc") + got := resolveURL(base, "seg-1.ts") + if got != "https://cdn.example.com/vod/seg-1.ts" { + t.Errorf("relative resolution should follow RFC3986 (no query inheritance), got %s", got) + } +} + +func TestDeriveOutputName(t *testing.T) { + media := &Media{} + if got := DeriveOutputName("https://cdn.example.com/ts2/a/index-v1-a1.m3u8", media); got != "index-v1-a1.ts" { + t.Errorf("want index-v1-a1.ts, got %s", got) + } + fmp4 := &Media{IsFMP4: true} + if got := DeriveOutputName("https://cdn.example.com/ts2/a/index.m3u8", fmp4); got != "index.mp4" { + t.Errorf("want index.mp4, got %s", got) + } + if got := DeriveOutputName("https://cdn.example.com/play?token=x", media); got != "play.ts" { + t.Errorf("want play.ts, got %s", got) + } + if got := DeriveOutputName("https://cdn.example.com/", media); got != "cdn.example.com.ts" { + t.Errorf("want host fallback, got %s", got) + } +} + +func TestParseAttributesQuotedComma(t *testing.T) { + attrs := parseAttributes(`URI="a,b.ts",IV=0xABCDEF00000000000000000000000001,NAME="x"`) + if attrs["URI"] != "a,b.ts" { + t.Errorf("quoted comma value wrong: %q", attrs["URI"]) + } + if attrs["IV"] != "0xABCDEF00000000000000000000000001" { + t.Errorf("hex value wrong: %q", attrs["IV"]) + } + if attrs["NAME"] != "x" { + t.Errorf("quoted value wrong: %q", attrs["NAME"]) + } +} diff --git a/internal/protocol/hls/staging_test.go b/internal/protocol/hls/staging_test.go new file mode 100644 index 00000000..ec25acbf --- /dev/null +++ b/internal/protocol/hls/staging_test.go @@ -0,0 +1,296 @@ +package hls + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/GopeedLab/gopeed/pkg/base" +) + +func TestFetcherE2E_SameURLTwoTasks(t *testing.T) { + const total = 6 + mux := http.NewServeMux() + var playlist strings.Builder + playlist.WriteString("#EXTM3U\n") + for i := 0; i < total; i++ { + playlist.WriteString(fmt.Sprintf("#EXTINF:4,\nseg-%d.ts\n", i)) + } + playlist.WriteString("#EXT-X-ENDLIST\n") + mux.HandleFunc("/video.m3u8", func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte(playlist.String())) + }) + segments := make(map[int][]byte) + for i := 0; i < total; i++ { + segments[i] = []byte(fmt.Sprintf("SHARED-SEGMENT-%02d", i)) + idx := i + mux.HandleFunc(fmt.Sprintf("/seg-%d.ts", idx), func(w http.ResponseWriter, r *http.Request) { + time.Sleep(120 * time.Millisecond) + w.Write(segments[idx]) + }) + } + server := httptest.NewServer(mux) + defer server.Close() + + cfg := testConfig() + cfg.SegmentConnections = 2 + cfg.PrefetchContentLength = false + dir := t.TempDir() + + // Two tasks for the same playlist URL into the same directory, like a + // user duplicating a task. The engine would rename the second output + // (AutoRename); mirror that here. + f1 := newTestFetcher(t, cfg) + if err := f1.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: dir}); err != nil { + t.Fatal(err) + } + f2 := newTestFetcher(t, cfg) + if err := f2.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: dir, Name: "video-2.ts"}); err != nil { + t.Fatal(err) + } + if f1.state.StagingDir == "" || f1.state.StagingDir == f2.state.StagingDir { + t.Fatalf("staging dirs must be distinct per task: %q vs %q", f1.state.StagingDir, f2.state.StagingDir) + } + + // Start task 1, let it stage some segments, then pause it. + if err := f1.Start(); err != nil { + t.Fatal(err) + } + time.Sleep(250 * time.Millisecond) + if err := f1.Pause(); err != nil { + t.Fatal(err) + } + staged1, err := os.ReadDir(f1.state.StagingDir) + if err != nil || len(staged1) == 0 { + t.Fatalf("task 1 should have staged segments, dir: %s", f1.state.StagingDir) + } + + // Task 2 must complete without touching task 1's staging data... + if err := f2.Start(); err != nil { + t.Fatal(err) + } + if err := f2.Wait(); err != nil { + t.Fatal(err) + } + stillThere, err := os.ReadDir(f1.state.StagingDir) + if err != nil || len(stillThere) != len(staged1) { + t.Fatalf("task 2 disturbed task 1 staging: had %d entries, now %d", len(staged1), len(stillThere)) + } + + // ...and task 1 resumes to a complete, correct output of its own. + if err := f1.Start(); err != nil { + t.Fatal(err) + } + if err := f1.Wait(); err != nil { + t.Fatal(err) + } + var want []byte + for i := 0; i < total; i++ { + want = append(want, segments[i]...) + } + if !bytes.Equal(readOutput(t, f1), want) || !bytes.Equal(readOutput(t, f2), want) { + t.Error("same-URL task outputs mismatch") + } + entries, _ := os.ReadDir(dir) + if len(entries) != 2 { + names := make([]string, 0, len(entries)) + for _, e := range entries { + names = append(names, e.Name()) + } + t.Errorf("expected only the two outputs to remain, got %v", names) + } +} + +func TestFetcherE2E_ForeignJournalDiscarded(t *testing.T) { + const total = 3 + var served atomic.Int64 + mux := http.NewServeMux() + var playlist strings.Builder + playlist.WriteString("#EXTM3U\n") + for i := 0; i < total; i++ { + playlist.WriteString(fmt.Sprintf("#EXTINF:4,\nseg-%d.ts\n", i)) + } + playlist.WriteString("#EXT-X-ENDLIST\n") + mux.HandleFunc("/video.m3u8", func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte(playlist.String())) + }) + for i := 0; i < total; i++ { + mux.HandleFunc(fmt.Sprintf("/seg-%d.ts", i), func(w http.ResponseWriter, r *http.Request) { + served.Add(1) + w.Write([]byte("DATA")) + }) + } + server := httptest.NewServer(mux) + defer server.Close() + + cfg := testConfig() + cfg.PrefetchContentLength = false + f := newTestFetcher(t, cfg) + if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: t.TempDir()}); err != nil { + t.Fatal(err) + } + // Plant a journal that does not belong to this plan (wrong fingerprint). + tempDir := f.resolveStagingDirLocked() + if err := os.MkdirAll(tempDir, 0777); err != nil { + t.Fatal(err) + } + foreign := journalFile{ + StagingID: "some-other-task", + PlanHash: "deadbeef", + SegmentCount: total, + Sizes: map[string]int64{"0": 4, "1": 4, "2": 4}, + } + data, _ := json.Marshal(foreign) + if err := os.WriteFile(filepath.Join(tempDir, "journal.json"), data, 0666); err != nil { + t.Fatal(err) + } + + if err := f.Start(); err != nil { + t.Fatal(err) + } + if err := f.Wait(); err != nil { + t.Fatal(err) + } + if got := served.Load(); got != total { + t.Errorf("foreign journal must be discarded, want %d downloads, got %d", total, got) + } +} + +func TestFetcherE2E_MergeFailurePreservesStaging(t *testing.T) { + mux := http.NewServeMux() + mux.HandleFunc("/video.m3u8", func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte("#EXTM3U\n#EXTINF:4,\na.ts\n#EXTINF:4,\nb.ts\n#EXT-X-ENDLIST\n")) + }) + mux.HandleFunc("/a.ts", func(w http.ResponseWriter, r *http.Request) { w.Write([]byte("AAAA")) }) + mux.HandleFunc("/b.ts", func(w http.ResponseWriter, r *http.Request) { w.Write([]byte("BBBB")) }) + server := httptest.NewServer(mux) + defer server.Close() + + f := newTestFetcher(t, testConfig()) + if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: t.TempDir()}); err != nil { + t.Fatal(err) + } + run := &hlsRun{ + ctx: context.Background(), + cancel: func() {}, + done: make(chan struct{}), + segs: f.state.Segments, + outputPath: f.meta.SingleFilepath(), + stagingID: f.state.StagingID, + planHash: planFingerprint(f.state), + tempDir: f.resolveStagingDirLocked(), + keyCache: make(map[string][]byte), + journal: make(map[int64]int64), + } + run.journalPath = filepath.Join(run.tempDir, "journal.json") + if err := os.MkdirAll(run.tempDir, 0777); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(segmentPath(run.tempDir, 0), []byte("AAAA"), 0666); err != nil { + t.Fatal(err) + } + // Segment 1 is missing on disk: the merge must fail without deleting the + // staged data or leaving a partial output behind. + if err := f.merge(run); err == nil { + t.Fatal("merge with a missing segment should fail") + } + if _, err := os.Stat(segmentPath(run.tempDir, 0)); err != nil { + t.Fatal("merge failure must preserve staged segments") + } + entries, _ := os.ReadDir(filepath.Dir(run.outputPath)) + for _, e := range entries { + if strings.HasSuffix(e.Name(), ".part") { + t.Errorf("failed merge left a partial output behind: %s", e.Name()) + } + } + + // Recover the segment: the retried merge succeeds. + if err := os.WriteFile(segmentPath(run.tempDir, 1), []byte("BBBB"), 0666); err != nil { + t.Fatal(err) + } + if err := f.merge(run); err != nil { + t.Fatalf("retried merge failed: %v", err) + } + got, err := os.ReadFile(run.outputPath) + if err != nil { + t.Fatal(err) + } + if string(got) != "AAAABBBB" { + t.Errorf("retried merge output mismatch: %q", got) + } +} + +func TestFetcherE2E_ReMergeOverExistingOutput(t *testing.T) { + const total = 2 + mux := http.NewServeMux() + var playlist strings.Builder + playlist.WriteString("#EXTM3U\n") + for i := 0; i < total; i++ { + playlist.WriteString(fmt.Sprintf("#EXTINF:4,\nseg-%d.ts\n", i)) + } + playlist.WriteString("#EXT-X-ENDLIST\n") + mux.HandleFunc("/video.m3u8", func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte(playlist.String())) + }) + for i := 0; i < total; i++ { + mux.HandleFunc(fmt.Sprintf("/seg-%d.ts", i), func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte("DATA")) + }) + } + server := httptest.NewServer(mux) + defer server.Close() + + f := newTestFetcher(t, testConfig()) + if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: t.TempDir()}); err != nil { + t.Fatal(err) + } + tempDir := f.resolveStagingDirLocked() + if err := os.MkdirAll(tempDir, 0777); err != nil { + t.Fatal(err) + } + // Simulate the crash window after a completed rename but before the task + // was marked done: a stale output file, a complete journal and all + // segments staged. + stale := filepath.Join(filepath.Dir(f.meta.SingleFilepath()), f.state.OutputName) + if err := os.WriteFile(stale, []byte("STALE-OUTPUT"), 0666); err != nil { + t.Fatal(err) + } + jr := &hlsRun{ + segs: f.state.Segments, + stagingID: f.state.StagingID, + planHash: planFingerprint(f.state), + journalPath: filepath.Join(tempDir, "journal.json"), + journal: map[int64]int64{0: 4, 1: 4}, + } + if err := saveJournal(jr); err != nil { + t.Fatal(err) + } + for i := 0; i < total; i++ { + if err := os.WriteFile(segmentPath(tempDir, int64(i)), []byte("DATA"), 0666); err != nil { + t.Fatal(err) + } + } + + if err := f.Start(); err != nil { + t.Fatal(err) + } + if err := f.Wait(); err != nil { + t.Fatalf("re-merge over an existing output failed: %v", err) + } + got, err := os.ReadFile(stale) + if err != nil { + t.Fatal(err) + } + if string(got) != "DATADATA" { + t.Errorf("stale output was not replaced: %q", got) + } +} diff --git a/internal/protocol/hls/subset_test.go b/internal/protocol/hls/subset_test.go new file mode 100644 index 00000000..18ef4328 --- /dev/null +++ b/internal/protocol/hls/subset_test.go @@ -0,0 +1,146 @@ +package hls + +import ( + "net/url" + "strings" + "testing" +) + +func mustParseURL(t *testing.T, raw string) *url.URL { + t.Helper() + u, err := url.Parse(raw) + if err != nil { + t.Fatal(err) + } + return u +} + +// subsetCase is a media playlist plus the error substring the pipeline must +// reject it with; wantOK playlists must parse AND validate cleanly. +type subsetCase struct { + name string + playlist string + wantErr string +} + +func TestSubsetRejections(t *testing.T) { + cases := []subsetCase{ + { + name: "gap", + playlist: "#EXTM3U\n#EXTINF:4,\na.ts\n#EXT-X-GAP\n#EXTINF:4,\nb.ts\n#EXT-X-ENDLIST\n", + wantErr: "EXT-X-GAP", + }, + { + name: "gap-before-first-segment", + playlist: "#EXTM3U\n#EXT-X-GAP\n#EXTINF:4,\na.ts\n#EXT-X-ENDLIST\n", + wantErr: "EXT-X-GAP", + }, + { + name: "discontinuity", + playlist: "#EXTM3U\n#EXTINF:4,\na.ts\n#EXT-X-DISCONTINUITY\n#EXTINF:4,\nb.ts\n#EXT-X-ENDLIST\n", + wantErr: "EXT-X-DISCONTINUITY", + }, + { + name: "live", + playlist: "#EXTM3U\n#EXTINF:4,\na.ts\n", + wantErr: "live", + }, + { + name: "keyformat", + playlist: "#EXTM3U\n#EXT-X-KEY:METHOD=AES-128,URI=\"k\",KEYFORMAT=\"com.apple.streamingkeydelivery\"\n#EXTINF:4,\na.ts\n#EXT-X-ENDLIST\n", + wantErr: "KEYFORMAT", + }, + { + name: "encrypted-map-without-iv", + playlist: "#EXTM3U\n#EXT-X-KEY:METHOD=AES-128,URI=\"k\"\n#EXT-X-MAP:URI=\"init.mp4\"\n#EXTINF:4,\na.m4s\n#EXT-X-ENDLIST\n", + wantErr: "explicit IV", + }, + { + name: "map-change-byterange", + playlist: "#EXTM3U\n#EXT-X-MAP:URI=\"init.mp4\",BYTERANGE=\"100@0\"\n#EXTINF:4,\na.m4s\n#EXT-X-MAP:URI=\"init.mp4\",BYTERANGE=\"100@100\"\n#EXTINF:4,\nb.m4s\n#EXT-X-ENDLIST\n", + wantErr: "changing init sections", + }, + { + name: "map-change-uri", + playlist: "#EXTM3U\n#EXT-X-MAP:URI=\"init1.mp4\"\n#EXTINF:4,\na.m4s\n#EXT-X-MAP:URI=\"init2.mp4\"\n#EXTINF:4,\nb.m4s\n#EXT-X-ENDLIST\n", + wantErr: "changing init sections", + }, + } + baseURL := mustParseURL(t, "https://cdn.example.com/vod/index.m3u8") + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + media, parseErr := ParseMedia(tc.playlist, baseURL) + if parseErr != nil { + if !strings.Contains(parseErr.Error(), tc.wantErr) { + t.Fatalf("parse error %q does not mention %q", parseErr, tc.wantErr) + } + return + } + err := validateSupported(media) + if err == nil { + t.Fatalf("playlist should be rejected with %q", tc.wantErr) + } + if !strings.Contains(err.Error(), tc.wantErr) { + t.Fatalf("error %q does not mention %q", err, tc.wantErr) + } + }) + } +} + +func TestSubsetRepeatedSameMapAllowed(t *testing.T) { + playlist := "#EXTM3U\n" + + "#EXT-X-MAP:URI=\"init.mp4\",BYTERANGE=\"100@0\"\n" + + "#EXTINF:4,\na.m4s\n" + + "#EXT-X-MAP:URI=\"init.mp4\",BYTERANGE=\"100@0\"\n" + + "#EXTINF:4,\nb.m4s\n" + + "#EXT-X-ENDLIST\n" + media, err := ParseMedia(playlist, mustParseURL(t, "https://cdn.example.com/vod/index.m3u8")) + if err != nil { + t.Fatal(err) + } + if err := validateSupported(media); err != nil { + t.Fatalf("identical repeated MAP must be allowed: %v", err) + } + if len(media.Segments) != 3 { // init + 2 segments, no duplicate init + t.Fatalf("want 3 segments (init + 2), got %d", len(media.Segments)) + } +} + +func TestMasterAudioRenditionDetection(t *testing.T) { + masterURL := mustParseURL(t, "https://cdn.example.com/master.m3u8") + + separateAudio := "#EXTM3U\n" + + "#EXT-X-MEDIA:TYPE=AUDIO,GROUP-ID=\"aud\",NAME=\"en\",DEFAULT=YES,URI=\"audio/index.m3u8\"\n" + + "#EXT-X-STREAM-INF:BANDWIDTH=1000000,AUDIO=\"aud\"\nvideo/index.m3u8\n" + master, err := ParseMasterPlaylist(separateAudio, masterURL) + if err != nil { + t.Fatal(err) + } + if err := checkAudioRendition(master, PickBestVariant(master.Variants)); err == nil { + t.Fatal("selected variant with a separate audio rendition must be rejected") + } + + // A group the variant does not reference must not cause a false positive. + unrelatedGroup := "#EXTM3U\n" + + "#EXT-X-MEDIA:TYPE=AUDIO,GROUP-ID=\"aud\",NAME=\"en\",DEFAULT=YES,URI=\"audio/index.m3u8\"\n" + + "#EXT-X-STREAM-INF:BANDWIDTH=1000000\nvideo/index.m3u8\n" + master, err = ParseMasterPlaylist(unrelatedGroup, masterURL) + if err != nil { + t.Fatal(err) + } + if err := checkAudioRendition(master, PickBestVariant(master.Variants)); err != nil { + t.Fatalf("unreferenced audio group must not be rejected: %v", err) + } + + // A rendition group without URIs (muxed tracks) is fine. + groupWithoutURI := "#EXTM3U\n" + + "#EXT-X-MEDIA:TYPE=AUDIO,GROUP-ID=\"aud\",NAME=\"en\",DEFAULT=YES\n" + + "#EXT-X-STREAM-INF:BANDWIDTH=1000000,AUDIO=\"aud\"\nvideo/index.m3u8\n" + master, err = ParseMasterPlaylist(groupWithoutURI, masterURL) + if err != nil { + t.Fatal(err) + } + if err := checkAudioRendition(master, PickBestVariant(master.Variants)); err != nil { + t.Fatalf("audio group without URI must not be rejected: %v", err) + } +} diff --git a/internal/protocol/hls/testdata/media/README.md b/internal/protocol/hls/testdata/media/README.md new file mode 100644 index 00000000..10afcfb9 --- /dev/null +++ b/internal/protocol/hls/testdata/media/README.md @@ -0,0 +1,10 @@ +Media fixtures generated locally with FFmpeg 7.0.2 (testsrc + sine sources, +no third-party content): + +- sample-00.ts / sample-01.ts: one continuous MPEG-2 TS encode split at a + packet boundary, so continuity counters and PCR are consistent across the + segment join exactly like segments cut from a single real-time stream. +- fmp4-init.mp4: ftyp+moov init section of a fragmented MP4. +- fmp4-seg1.m4s .. fmp4-seg3.m4s: moof+mdat media fragments (styp omitted by + the muxer, which is legal) split at moof boundaries from one fragmented MP4 + (-movflags +frag_keyframe+empty_moov+default_base_moof -frag_duration 500000). diff --git a/internal/protocol/hls/testdata/media/fmp4-init.mp4 b/internal/protocol/hls/testdata/media/fmp4-init.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0c758c0b3c683403fe6bc08570224429e252771a GIT binary patch literal 1566 zcmb_cy^9l35TDH^K@Q=PBc3UOL^!SVO!N#E3PG<(VWoH=+HCSRCfsgz*?n0~3rXdF zU}F`(iUYw`v`|D3xkAB4IBc#E#Lm)d;&0w=E_-V{ES%)G^ZDk@+j+A@RBFrHZr69t z5~YYaP_x~oDp7jFcAS8SEIa53@EGaem7)>hKcPJe%n7|diJT^-u;7REj8!;q%2ladz(bP@{C`{$9}{%HHCDh2(C@@_77^2OCN}N$KApx* z_=7Zcdi99;5_&!6La8_Y-sxw{96GJjzSHl;(_DovVx7)LozD4Tr^m7X(&f#l%Xbr9 z?qTGNDAD9d8s*p)mP$3#7wV6hp#5g+J%VYZL^+=^wgC|ufzCc%Kk{D z*3XAIr|xfs`=_|9d6d72{lFQt4m<|F0$ac%;34n{cnPq78`uCI0Plh4k^BPu3-A{B H47~ac%fZMO literal 0 HcmV?d00001 diff --git a/internal/protocol/hls/testdata/media/fmp4-seg1.m4s b/internal/protocol/hls/testdata/media/fmp4-seg1.m4s new file mode 100644 index 0000000000000000000000000000000000000000..e2735d3e209223a07330d0d83063c470da7e821e GIT binary patch literal 11674 zcmZv?bwE?!|37{fi~$2iw~mw;jYvB>1f?ZqgN;y{Nm(PMM7l(2kdhQs^u=fpMMM!$ zDG@2sd(IM4IE7XX0i1Ox?n0|2lA-o9ig-M?TE(Ws%M z{d!^Qemxv${9Dg(000geY`+Hp+*EHe_22B@^8Uk&8hSPm00aY+M?q5=!T&b>I}QLO zRD$XLm;U-Uh5uKE+x}N3K%FZ8AL~O?{T8#4P9EnK6iUG~*#nV5?>x{0F7WBiB z0!jW+VU+z6E`Xv-_9NlK&U!*QC0uY=6x1DhdW4d7l@*krF3bg?=Z^#AgXUDG$2e@4d)*ebcW;$6+He|r;H2p_w#~S z`Co++E->`(5?+1*BugvcdcF{7p%by6T!xMHrP549V&j z3~3dz9i((M$U-5yN`M9cy96aEUQ0?EFv0uy0S+xAg(X!N0YTNl;uHhZJv#iFnADL~ zas*)ZPb!!%%bbUMCm*$TSDT;4Nh$$h9MD5B(1aP09BIN>6Ts3Cfh-^GrfUFhXoEW{ z1p{elTQ>kkgdVD{z)7fO=<0?Sw7&|FU*HZ7YVHj0<$^!_Z|b@6LtF|EJ_%m!0(jkB z*1q{02mfy|rCluGqX8T*fbaW-*iwLoxTi4%!_IaGBYQ?Jsa5e0O+UR95YhE!&Zo_r zL}Ns_Ng7nsCr$Z}zLd^*cyOurnb?Qd2A=$Bs&|cL#xULp8i)x7XeOG5UI7m9p^^X` zNdP!@j4-Z2zn7U#Tmbd}3FD8W!K?umNi;R}R3-q+Xbr#w8v-yy(j!IygD(IqCkO@y zEwO+ZVn~7juwnqlq@khj)YHq{FZ{PwoL>X*KngJtO@n3%HfcY{b~6?Bhbe_+0rs`# zG>?eLkDr!l8Lw({i?wW66Ka|h*T)3|ClwzuRSW&T6eCb5C`8p#?Lz|$O^Go!29lou z&}`6hcMx<~_^`jMVk(JUh7`5>QdDntk^S3?-+*Bzsgik=P$FFzlNIw+Z;Y{2B&7R3 z+E0+|m+h-i3Kk4?Os6R{m8aaIuL>rA@2v(-O8;;n&erWE0i>{$6GvfYS*Kxz9_o|r zPH7RTyEz!>_fjtZZHpwpAYuN2$3o9$fGyAgNK{X=h8icWoyVu$=*~|TlXa#HE(8r` z23fzI&Ip|PEZ2|?vNl+NOW=lSA}<=7X>=tv_hjU-+R*y;c##fg;|OE>+lc@d zo<;*CoCfIOfT>!wIj1ACrX%WVqlS|~^F?sr#1k|UO`bF7`!*iKWr$iq_uAtD3k^wh z0(j==E6QQmU(r%N-*oJzcKAP%F40v15@GDremBM(H|Sa>oW%d1PIUC_#3j_%C_ZfC z5`4cJui^rKckh5^I03$9nrXfOam7&D*CL*aqbyx)LMGD7U@&%A&%RaM8%6J!iyd9b z!#aY*%xaTD^=5;ELgv`gV$09|Rz4fJOk3u&;lwjW!yb$P)Zr%G_MD`7!U_Hl+5d$Q}MSitA4S zpaPj1xiV`tu4I z7O`QVp%TmTO*z+rqxBC`bOT))+wDeiFp}@mM_)Vu?V)`PF#^4nAtERWkjemPb^vIs z?zE>*AdxdnTciYa4C}v%VrT%%0`(>$)-MV}W~KuIKloh`{C~B6|4A`QlK`g{tnzNp z0<=|Itc`Uf49QIvuf0~$VhuqR!Ur^PZQKc<8E7CD!H^aS<)9^%V3VbAtO2j@7b&xA zs7~LOynhu-qwnj1Une{<~0P)$K$5ap(>wU0_M$pCvqZx@|H0^-Of`27tZ~&WoKte#@c(A`p9Q0FI&q9ERoYA~eyzwMRv0aFiFy<$3w2v`|~(mAxIP7mTnEfwPY^{uM?8 zHU*gUv8Ku>ejU^LcF`06UYG?oU z0%5?9I!+6H70-~SL;zG_wOz){V`z9K=&J~11lX!kkfq8`t-{IeBL-)p0a9I_W3;I2 z>BX9R8;}S>5L+6uZ$X}b@i|++($n&PtTp=*ngIa;gS&_DjTQrXggi0po7XNZhuh(10l?jr%2D%N3bmyc##uv$O=! z!sh0FI>zdU*wQkbKd`L9fhnO1Z-EN$ecbM+=QHk(xDcP4hB1Hp8W{x1j|iWD1)FO z2&fd}LT@a9%$@|$u=IEk!+3%dFpxp8-fsw2-UJvys!K^2FcHv5d-?(_1BpCI^Z_if z@1#SL(I)?#_Wy;VH6#!m4NB!it$8wTG7sA`kY(YmJTPZQ8cYjtzGnx%k`p?}kp(GJ zXu&g;IsiS6a2Kar-hh*F0E+>PENDn&+w8bS22T<|M9`RFnq57({}zXu_7$Ma7{&2;HXbn2q`yFM z0&!Xc2mxkTb15VWWNmID5uv1EwUPiZ^d7z&3^`EvPz=Ma+;`?}vhjr!$yo&;uCdAb0y`b#TyLsr%?bx zv`3uy3if=qX`2nGfv0A|kcxp>WmFz<0XRaVxox4@)Sz>v+#JWU z16v~W&e1h6F-A{(bZ9z21%^1(Y=$EA)-Hy1Rz;%cQ zCvO1DT)>;L0aLMu5@EnL4vCyX*k4dkh6jj<<3J@1LJ$pZj1yH+8=SSQIEF)1j$bby z{Q`fdyN85g8Ws{7iUc>1I%9Mah~6k4|4j(8DdnFLpZvVN!FzxOtdnetsnbWpY1SuR zi>>Nqm=^2dc1Jrn*fl3^+EXsybeQpG`UCR6P=hU4Uq4^f}1+S zbAWwBci0KGK~AQnp$4Gq&q-#4Oyq{QKAkWaynup<0t5jKvR*CVp_5pZoCPlVE#O0~ z$w#v3m5FkrDR7}mK5O{v3RRGT{G|K&yk~wgQzp)G5ui~zYzSu5HVZqg`DDAC4kv1p z`Fg4@`l6FF9nQ~BKe)TSP^x;fen%K*htnYP`pG1>iXj1JsTnB82sEX5@4F0(c3JrRt?q z@E(sc=S4_RUl_sCCng`T5T1jFh`_*v%ZxvV=goh;C;-|Ak{ko3aIo_r@@RjD?90R5 zQ-o?D2%IW~0W>A;ytxXN0Ok+|0A31kL?Tc)54t8{toaC?`dxm)H=8t{m#g1t%%ZhN zc90cpjwDZus0B~VW0+98rmDl!VV+8 z^t#53q4ju1{&@RTWYOoK^sb>tLS$^uI45B|jexkH0E=TCB@-eb`q~uCs5LY4Gl4h!T;^-#MW}5&#_X)zJf^re#dK z9VSv*4g&tpQBbG}e@&Mnm^3~mnDjMi`-Q~o|DslBY`#On8DKvcG+O*GR|xPktS(>4 z=3cQq^+fjmbGs|0nR5-HFAh~KUolt%XiV(Y9!9g%Z{r{AZQp^lrV6%RxCu5cCHqs_ z&vf6SP5kfWQlBH$ns2>c$3hV=4-v<+!Fwa4WQGV zW|+8}#vgsAb9AtO)h)Jqh+v0a_He{kv_#%8dNo3l{LhH9Vv;=pNFC4@bzAMr;(4e7 zuy%ow!AR~`g>5(Upl>^x`bA<_rb;^#aRVwMPEGp*-l)uGW|Uvg`8r@QxH zBJLGgcab=XVvl6)Fs6;7#4=J`edaf8vWeeV9w+vG*~MjT47l*|sUKU`Z+)M=^6V1p zj}WH%Z1e7TT;lES!2txfGGPGIJm6W-AY^&DVd=$DC07o?l$W}y?|%#qc(yn-ILh&) z%;%RJEX2kN{%QGnk$J!Jq)Yd~f&;^BZ5xvg_6D3$hmZF1J=lmp0)hw$u5u?cu+o+M z@Ob#gX@r(V&ChLMme9D$Y_)##sF1tC*3Nx5J3j8Wj|(DH%S06iC5M(g-$2*<)Gn1! zpDWhhij!*1@f<;m=NazcDRC_L$UD9b5W$=p6r(=5rYE)XSh8Z+>M`Cv^u`}IOxr&L5R?PY)!7};0(#%p1yPE z&b_*N<-#_Bf*1~wP|Ma_#f^R*2ZuAxsv|qxkyK5tWA{ zC5h6fhCJ+u&&0lL5S?L-@5;Y3*C z?_T7O;~w7S6(Ef!=k*#3i>l!h4aEIPv!hWZ8216sPEDkvE5uyL2YtvW@tl(mmP1Fj zr%PeEqSn?3S%Wg-0IXQ2nO_Y*Oi}ofI_wZSC#Z))Fz)q!Y^@#LY#~6HKuIg>r(hs- zi0Ft!<+zHaQr?)OLBc63l;6eUDi3dYTuCn(I`Q1y_-R`9#4!xJAVall@9mG=wUoOe zzgNm4!|Uo4-Wk7sXo7nK6SyQ}JEF3Fp$V<;_#+}BWIXp%FGpvHYklSI6dX4fCsL;k z5#W-E12kc@eyzeq4sjBI%b^$GKVC3Zn~Dt4VQ;b5yXsg1f4n5bZnhY5D?>Nreyj&D ztRF5nER%<$$HaVv(MET_Ui4r%U7P{QV8{)%kB)|Epw26!;3mefe_=ua&Y1+~Od_lp z9KboCk@Xzo)u&kOKW0OaiIqB@ya4oL^9iTk7={c-s`MoH6PO%nr-j_2#{(|)asCGu z<AKh-CDyT_r?7%(_4N_pJ!TFnZD*E zK1wn_*89QY%R!8hhFg~i`E}b39I|d%5}z@;Bg*XH;As53@K@b+pTSXp$t*2}<=hj7 zmo`&q11QM1G{rSB%jdJcLI8RV2G4r&I-zed+*w<3Z~!F@VTd*WOF=-0!j#W&yDCa| z81s(yC!@h-ccVKyb;}h3^H(|)rtj{qU+S8W3VqZM;t8FLecBSE9$`;D4~{UBH$kdhdt*f!(m%h`)fcgs`-K_q3873nPCs zkK}U*V$sG|N1J*Rof_@7!=;J8Qv@PBTU@U^Gz?IS|mgZ7<(oV+`$yn?ua;BJhmmFxjVUUOyy8d28qQ4{pa9UQ9D3 zAhTiukJ8#s76p>7r1r?ldT-N_#q_${e0*4#itNK%P9NsCJ^j_qqsm#CoOxKwh@c-P z#yNzlk=H(vZNRgI);Ez$8NC+0?a65LF?MzW4+QktVjG#5>o|2(=zSJ2npt!1R>(p` zD74ujxD1~q5h;mm4UCSw0z@xv922jh#OuE<4P)9mru#SQU_)^qu8JQl%~m|(FT2s- zZ!4b^73zv@#c?&PviesYd!@ICwpA!!Q*_{0xna50v_xEdy%4#0+zHd{3A2_~jj?7H zN>u$U>CAS2!8RTkl+WpXz1mZs=McK%-|QKO-#@6dwo8XR{$gn8)S+|968x+OD=foy z#QydQbaZeyDRo9bRfA?ATDL6gY|+&4d+BstO+ps*e!S5NdE7Q$m5<2;_k0Y7lTCR< z%7s;@rEd%`%IxO2E*Ik3eXyv*qZ5LWyJyFXW<;zf!EjVVeR2ZwP0S3qUWsV}FbZW} zfJZ!oOUwlB0Y)>?d!L^}IBJjki=%(NlJx)r^-T1GrprEai#b| z@r4v9KytRq-FnmEhQSE48hI4m`3DLX_!nktd|ZDbTK?_Lcpo(`~xPbW5RHIo5}Lr?1~O>DC*v?Wpqq^+(7`$nlJo;LdLm z=9fJ7vQ=_9w`}iPd^#oAGMU~e*iNco<+PHRw6dSDl4Li^F;~lWIBe&jmSYa(jqU!I zw=w3nG3JAL7tE-CILwtAT$I(xBmU>;=Ueg#18X0;J=_`AzWs^jI!WH`?A@MxnBn<= zKl01#lrz|+X&F_dse86W0r_Y9jg)%qQE%5P>5&KC{wAfHeHIa%w5vItvp4w1<ry<&*O6C zgQ@Pk%BsoIS;^8RIjb2Ng<0`cviYc@1sLyl{MS4gKF7<^^Oy2JtXw%RyXD$tHGEZ5 zBDGfUlU%wHW18Qrc+w9x@ka`C#@nVxqCDpva^2!uKk1%Kd*kI;-R9ZiD7|c&^5Sg0 zokX#!)bx@nVk(iJxG=qxuBxOPNs3F})e}0=egDw*c7WdQTRlBY$)(W59a)~tm+2!q zYwd+qHVt!Xowhagj&c-S>B^ytDbZFA_%oHCb+o$bc}E!h>SjA>;Eg(doQfAR=oxRO zTFR?3T5Qf;`qNah@~6A^TkezM=ci7Ws%6z9`|lN^JjZVxT5GMncIRDMnZah%(2*1R zL+7rh{8{_px@jSatNL2)A^cQ??+}XVwTjd8+XgiIvkgmP(caVlgc^}(v}9B^v(*fD{!s% zc%Cdr*&FHh9ZJopM@fnNg_obE0*y^Gy5>S1RsL9yDPZ{%o-o{=^Wb zZG5uC{j8Jt)O2j~w!ZQ+R6ASD8Ghl4WS!{_hLv-L^+&4TxF~Fxz&RN;8y{nqga+Ck zYYe*Ss#V_#3HP%&&exVtVN*3>ImLshl@!b*so;M&bgK6&a%lvGmaB* z%fvIbQm%KO6?D`Ff(3PXd~OI6+-$sCe(z6-$L__KpI=u0Dvr2fZ)s6KVkc>Ai`1@p zm*-e4XY7NV&&z#wxNJ>U!|lWrclL$*hkfh(q_R%GqjG17R2G?Nnq1a;B)u^vYjLNb zxL#QCXy;k-?D9#O#o1?8Buy&mpR~2M1P6;hA!&7;C+m_=7kH#AUnV~{QRJx~l;Jfl zpD3*yU#%SoU$4DGmc8@z(BqSn!Try_-M^eM5bNyv%1{W#8BYt1PkH!jWhtg#W-f%B z_gwCGqXpGg^G1kO0^vibl7&1FxS~;gJpHfL-s&YI$_cAR((-a!NA|uy*!#KXf9vj+ z|JL^x%{NDIyggksw)Z^QYUr6~jX!nIKR;KRl4SH-`(sz@2L%-s+g5=ox!E*#Hfc8i z^nZanG!+&DQGSb-e1w6E{hZBGnk&2YSH4yIZ|^GSR7X-Rnj*w?PO$h}re^Dg7c`s> z#ND&B6OaG3<4=|OPP$c=GMSk>Va`)~_SVp;Ca-;5;h%e3GBA~qe4Vto*&2U^EbVFi(uCP=4%HG<4X$s*T4aq}NYi zGt1M*U+Pw<5#Jox(?cKWf7o(&xx^q~3j;rHQn7^~txtf--&zSKlhQ^mRwIBdG0) z;+GY@m(2>Eau4ysErLO$Z+5wNO`->_E`y^q$5n0>_&;@(4hHs&FL+-FpTXR=W!m62n>_dY-rb@d%c#7q zLz#Nyi0Z~BwSkSt0prnklosT+a(>4y<7szRFeq+I@SwtVHF?#blcdj~$?+6@9O)@@ z+L^t0v$)ucUrej$} z{W-Vy=kD&_yOpAb$umA~o6hH;7<8`grhRFDh^R^UZS+TqHsWA%Q1zPH&580C#~rL> zSJ#;{; zKE}aVS}7NGWv%LDRw`d~&9*Q$}?MB859)LwDnX2#u40f)0HYsg&NYKtoUZ;Hc>_(s|Mo3j!I7Vt!~GPNi7 z)?ThQWe1$mk}=8=;i@xm@lzD$c%7Uwa>@IZc=1PBpAO}%)#PibT=l6_7YtsnO?9RA zTTUqO6>Um4s>@c}xJ~#Rv6@tH5NqhSX6-bH{&=W0bi_o@(WIeAL1&vhr~a%H^Yir? z(%}!Q?brt*dY5u_ZjjLx&s?7_{|UNbYi!D}#%| z+Lm_rnu?YJqE5=lT5!VbM{KL?M|s(V6SA&J@TrMeChywc_LWL_9Y3BIuf+S{oVsW5 zDd}T{7qqqhbmkhQcy1+(Z6*Ic$C<*NG#GN>MdRAq8@a=eZ$*U0o_(cjdskmTzIeVZ zy76QD#rp1JrvoC)Zal6?mp^u8@;JQap0#&RZFODi(|P+YMcV!z{py{V5Fhunl-eZt zZSP0KUia4Fo3^*}qqxG4p5CY{u)v(;^*d|qEvoWH(YE~2tJ{P2{?yYqC+%7uI~=wj zu+6q`Fjq_Z+Qr@PrPP-3=tp#EPG%5|YTty-diH0z_O#D^Ykw~1np^qcT+XS28%jA& z81z8$5rGM5E{oh!ye^RbgN*SNZ&6XSBdZarqFD*Zj6D1KRDlG;>g2lPSaY-Kg{g<4#w&bBSTA!?Cp0sBJ2IgBRuuZUJuW19f=n?KC17b&jys! zqA7eDD0}19)wsSSC!vks(+ZFKCg6qXFCvR-b0Sc;zMqvW3oQ1(f&i+X$ zFT%oT67v-L-!u`q+4;SRl5cyuLDk#Wwz8@Z+rn&fjgZ6IsjV*8Z@tTW|NTO$C(O11 zCB?7WDXcR)8|jsJxD7%Yi@9oBp^BGtc9-*(wnW*}N6FL9?`|Q+(qy4OQ&$_Ql3I@E z+|8{QOQ|-Ovxe*6P?~14;g*VLa5asIsYjdMpUl+XbzwQS+TCrl^17*wCt(h}{_9Q{ zEqcaFss1A3Q=6ame&}8LwnyD^IN9~$Yx_eph588Rld`g?v1!Gd8?BjPttajsD~((P z9!CpHoIRYMJ$m|rFIt(supfi4A!>@*JhCZHCONZr-r{_noo68{K*UIKud%OdQPZ0I=tWDruDCrHJA2+j{f}pVQM~vv+jd! z(^*Lw86$>sah_kZJ0xLYxI4<(?LrN;-wX3z=5{36OP?w|mB4RvTN!uy-FByzNbTiQ z>lqhqC!5qiyLg3j-g{`Bg!Qn!d+*xZgRQQ98`W)%(6QSFwHSJ)dQI#%aW0Dfb znCiP3*W2>XkFX#u)BDC>*R$zfG_VFqV9%DoxF6BDNU$Y54%_?md9P`ThrXn%h64lQpwBrEH9`#-K=599~Tk7)e zqz+r=$|5A$6xMS_T=I@(OrNoN!FJx|iUM`!nUB?V^)S^!-;9#9X9W@~Rjp@u%WmdR zhNbuj^?q6N2=@`;wG5O}c4{2*;CAJfL#;$ExGBGyeN$U>?S9?KyRT11w`G+)v53+# zaB*n(TIF&;xbt1k?li?Vd+1baJ=fJ|!mBBD+#`{zQ6- J2xqd-{|9x(#V~i#HSW4NV!7#RLQDYE>BwL8FWQ52PBE?w7GM2Ju%M!BFlqLHf zMo5J0B$XwtRLc8I{eJ)Jf4$dxuIoGJJm;C2^E~(UJ)iqN000s3^7eKG0D#lW)y)O$ z=?x0X#oy0~J}=}?pN9f!;1~h`FtB4d2LVT5ItYEP)|WEgQ@^k52^@MIj9y;W1uoXHGo#*@3(7ya$?$ZeqQqA)#|S9$NTMso{gWSKQoLzq^>l! zeO#%ptQ?$FouE2cWGy`P4*8Jt!3I-d{PC96ZuOMKqS@eWvnydyDqFKwTeEg&COd=t zQ`jKY;!A*`HYlv7e+naT+3FYoOG9;XJCf`Xuu2wU3UO&D!>nY_S=EKvY(@=ECiOjJ zMmFdJWok^3QV2lLVF+(sTp5Z`5ict`7Q_)`%~mJSx{Tfl`V7dQgp!T!(8Ny7q?Nm{ zeOivInias0z9I!uHC`D@$!`i(Q-XbS48X--Xoz_x+Gw%4miO;>W0~5So%Gm z3jE4xKgy>yDVx6N5y|M^EOg39btvTS4|3;|lwfPp`geqdabcqqM zu9HoZ2DhN5V~hs(M7G~s9wAd_eU8oedfI7AA|M*_YZ7Rp|CP=TV+9t^h7|{5)vk== z`H|8M)>W4X(h{R~=NFO%A3y_$q&`UHKtT=$B{@*!`Jgw26T+w;tsv)f_xyD}d++`} z2!(+cj$+GYgua0-q4-6lv$=FHm}R0vX;6z6-T zQWTfI=TDSk{IGyHt`o?;3n1vRb2VAUULNOQt7>84 zLvX6k4&iO^nGQaow&LfbMGbG3Bc$d0Bf41`Ao19lc+LVfjszWq&Lhl-0qSsBPk0n3 zFIRGSbj_Gor3x>A0DQO*J8^qj43SvYYANZ?VawL>y>PRpB`cR zfDx#+8Z*pP35>&WZd0`O2FqZkjHJ#hUzi&fA@&;#Un-`5l~e78A)|?qdQYyJZ*r)} zOzpE(6_HLjw2SfRWd`#y>D(rD2CKYoc24rAUb*QN{?iQQHA{t=%>s+7CCrC@X4l+o ziQ&s!TdA$6s0vBOMqL1ad#vobMrlq1aefIx01L^6+3q@|6DVUpy?XDP@e;WCowT!; zJm9w!82Zm@uZoLx0#S8A|MX!*(&znx`h#JQ3D&*o;5 z-NUT{FkQ=ZJ8ETS{^dxH9s%~^Jv$%FP*hc9QJ_8mImt+Zs0qP2rRr0uqc_w^=;m9Z zhjumM9Y&3FN25hlPv3p-gk^NI7LYd=i&I1^_8fX50wu;6n`yFQruj`8m*f6Ay9doy zlqIQYE%2>3n+Y&za8h||qWHYOZ#l^hBb&)ercRT19-xa$($YHioHYjDaL;&p%YS+- z9_|9hfsNt(q0}LZ$I`T|h+9=o2#Km`J=ld&VbG@;Y^DMQ3fdHU*A+)aH4rp*E-0=2 zb+mC%Z}S;=jdjyR(DYp-BWV<3YRz_IZM6e~YS4L@cWQWqTSDGo!v1a?Z_VncFKX^% z9mf&=|1?(<<6O`a8lP+*9&22DxOjwLh~<-SXRR85C_@8?5dbR93P4XXZETxzQ&G$| zL5@CSDMx&WM#pKKJzeP-gtpxF_V(5%D(N&iJRZ=JuT#MGG4Qe(STHkbF({g2hLRP9 zGJFuaP{E_oh99}ch(>gaveCcB7VL|CR%2yMd5MuRPCSX>JR1pg&I$-;=<<6`+DVx` ztNv7vLO00xT33HK?fvYn2=pjduq918w__PWm{udt3gLDzlX7s^L&N*>=?%z{H4Xtg z&}1IkIRM>caOf}vNKSSH@YVn~`K)yKA0|^IJH!;J!UXgz8$a5!rMRrHKpKxR5ift2ZAJh>)qBk2$=;X06T!@^p{Tsu(_gJBVNj8p2_Vl>Z83EOkt zRN>FeIYeGt0hp7mOlT*F2hoz98Yf&Xn$`j0>`|MPa2e3#3E%}oB!c&LU2`DgMGOmb zL0)FGNYSQ>uqnYzGA<-*6}cSiEvjvtN|xUlIzCWDN2Nd?0?2Yt_0bryv83(v7G?2r zyI2c$t?|;-G+WtSnIKX95_doMTl)1lSOhuS`At{7;d-U_&tOtkxO9wz?Au!k*Z4Vt zKafyr<-8Da88EZR?*J0)T&B3VH(Ad2lUS~Ga~9iM*yj(GW%1)bykG9-4 zZa|zX3Z7hbzSmSLM{jC_S#PppA&97e8GMGUNaev1nKEmDD}4fVS04aXzyi6ZfH*I} zJg2KkBn|;BB0KY$kq|*xa=3|<#hWZ*WVd&>ajWQ&N0%t`RazawIiHqEZ!8<8JB-ppWD-Qs zH2to>o(q_aGXU5MKxF{NJvL4dm(F`Xkjab^DVocem#J8+wT8;dD$DqqtZT!%IKw$1HT~QMffa6rhi8F@XXw1A{~M6J$o}8XrAVbAmP#+1Qs;1Rn+DUYNu$m1p7(j6 zL*{122xs8-YZd+GDGQ_hC z9dYXe%+@3GSoj%krwILqE76?s4%jBpZ&>Xd>_57l)sDx5bi-M&nwCsh%?Lreme5Q= zObo2iO!-ZU^c=&c-wMwds1tNleC%R1!^ePzi==P@>7izSZ=T~foYwgDluD2jf$AQ; z%B;6!>Guhq`(TE>@rHf@TFhP5>@nq7j_r-%lj4_1B0t(}pc zb4PFT3x*0LRWaE3I$jaZ6l&n+Gffq}%p!A9vtn&(InE~_*|T8|gc_Fl1(@dt6~B+R z3nY~?^beR9}M z<~7<7)JhM))4C$Wn1F)QBu_z^2{AE}+CsZ&j zqTU27_ooYUL4w_;Y+6}mF;Yu*?%cajP{^gAc8)ybQJzA4w(6We3Vx7cDE(7rP8FXG|S{j{a$23W84;-Hc zIUeGcSt%bGsv+#A-cmSBlvw!yk3PeChhfP_z$tdkJmcD|AX^mAV@(BVSQRCT?i)Wa zGir!YrTPC0Ea(818=VK9H9iMg2jsRYz;3v-TS(NhX-61SSU@C2UofT|9{_n}gaagP zl${CufLn~q?7Rb(ScKzMZFL0ne6mBn#{T&&#p1goyU zX9chWoFn1%(?n~)vL5xXn+WGb^Wil%7HfO6PeYytmC8xQ_efsV99{u*L_K21w& zxb!vjKb%SnAG0fFPZ%5w*N!oYhQku|F;XD;pI$CxzOXmr2%PZFW- zxIz#|3#Kw(v6T}gK(sO#n<~>;R26(#yhFf<0a)%FN9-$EO-tuC?hLa$T_ULb^At!c zdeWt5L|uZVy2~m?G{u!OaR{VtWNcH@3cqWxJeSK*CYWJ=EJri*d(I72>P=@5am+d%$&DwpN zAJ=K8GWg)H$WMxP9BoE;V@Lgs*|j|>u~rZRfB6RBq4!EDmq(f;>T$9o^(t%|JYJS!8f z`$e5U>gZu6{HL%p2j0*Agsl@gGTwX~F@Ea%%fi?k5SecE+=?`RJeJy>mV1W(OnFyH z`RfVPOHU#)GVHXcLQ0lGjmNv{;ju05=AwAte#r&omxXdnKvtXTHkUP zdMeJ|T$L6WPALBB-u_e2Y!O*DGkt7h5!Iku(Vss7xCbToPoyeN_p-wJ67L!G&rhUk+m)Ym3PkrlQ@(%4*0vX(Ze`$W z9npIm3K1eNldj#8y1+KCb*B&^92sHfiYQCn9)HGF4{yfuSVSo&Ww_Ws4*3-S%Kw^| zBg&4Y)cd)IX<%zgH^re};*9NkXE^?bZ9t%3uK<&IyGloi4(YFv>&4W_g`#c-8vL_Q z4<5chsJ-7BifOOMug^{)1N>cxYvZt9c8}Q#+KHsZy)6omQRzJ=&RSHX$?5NrU46JL znK9Za>r-I;v6yyqM^tp}5@Mmn+ox#j`*(|yWrNmHH-ql$Ngk)y57L%YR(HGy6Dkg-Pxs6yk)v^_G(Zl^}#iXf+dH1+N(#5#JtYEZ7Rm(Rc&AEk3 zM{~B>xw~Grd~K(&)y%w4UxU2318jmX@a;62zlgJ?ChQMJ_|KUmU;2$#6kMu@V&=;A zt|Ak7hI^?K)F+mv+2gR0==s{CH)fZQ&$kPE$9R*?4aU!jv8ri~vX8?uY0%!Tun?}h zEJA&^BntS*f<7w#sV36H8$yL`ubH#lYl2W$YN_933bKloIB?g#Av~(<-+rhb=2*B9 zE+mar(mZmLVb0^R*1VG)`C_?Nzprd)$f(09E9K5~Rm;&Or_Ig}4SpAUqprR`dhOQX zWn*^!t1{oRwuvr@P$k<-Uw&O%t_myr9a$=-sUCUU(7rb5=DY6(47x=F|VGB2^X{wV6tg z>|T0{>qfp(l}*?*I}{M(oBgEeVRz;o*eT?fiypX&&8-@=8G%r&EX!)v1iVv`WqZR% z)+uyyOS9!>FL~c_P~z|-EPHY4$U>#f$&K>!G{p)tN-LSI@1AY8ZMJ^j*jHIc&08&^1HE=3;EYPYt8FWX(WL+!d+-_X?$LQJ?|6`#Uf5Rzzl$A+0 z;)Tc)l&s8RnD+sP!|K5q_39fF<>|rp64-LQo&}Fcycx?jV}Op3F|yE&TlzV~%lUG2 zz?I3D6$FhyJVhH*U#8Av=kMF&qgbldsMbH`10D>HLA|f;<+Y2bl^`Vis{)o{48oz3mVC#)^fHH z_xC9yyi1vDYw0IWUU`u72k91dQ{E0zd&ZrQG2~>WAm)*f$+rj?tTize6>Q_frSng^RHxG0&>{z3?M9pvyyKe7)B&9~j zdy6Wzzm(Z(Z}w_`d0l26akXtap_H#lXg+61NdlTw2gQWszEkrIFJ(&rvL?K1e7GVm z!2T0nEi6_X?^PSr0S$Y(w~#h z=kar)_ENaayXF={St*~zx)U$CQ1v&4q5^+~#wSXBJ(enY)4;Fi&CdYyPyY8DAAZa>500y5eC1Mf z<&cbNQeL6`Sv}S7%9ji-fj4o#NgZqBR>Yhg*K&O4!1zX|O_R98sl=?r7^}NTH?c5dHcP5CF^7u4?BxihycJ@WhEnZ5|&tr)|GU9 zMVx(83Qoyc!io!j(Pk3%XZOsnutvd3+9Ln3 zRc6!gURp&uh_uRFle&gc+Xty#dWS9zQ2aGklG?MvZkvt1L@%3tP2)wvj>Su*vF|8H z)$X=NhpSnNx+t-5oi`F{jazj@xB zc74{;pR7H2&GlQzuTd#U%NsmuXV-m!*N<8m6^$OJi5UI)tX1fR<~=OEN&7@Ob@}7a znkjQZ<^biPdcHZLs{gsG;tu#e=p9{;cwEs~!B>N$I*T#~?;Z9JN@$DtmQi7zJXP=I z3)o`Q>DPMYC2^?TQ>TACi?FVQrXRL&S2D~dt3!q`|9DwCFho*}AZKzTIzDA#}ReY;4_5I;&r)GPy|LW7O2BqyHlJw7gUZ z!`3>+zmg*;?gCMj?qts(t3-04;|D_P`GdwU2aUgWzRV9BWm=YxJY4~X%8}BR6TB1M z)yjk>uj15~5_eleNh-5kzm^-7?>sQ9{&};kB2Lh(d?8X>HC!LZGM1VjBk>ilpnICT z)6~lE0buVcq(XYjO*WK=P3Xo>%A9^ZH2$U2G0&R_+unHx|MvvVE4?ag; zH6DC#4f`2l`E`DHBj>qg*}j?OB^l`&Xx`I%)&+7z_MkV&NhwXY@4MEA(^y=}%N*|$ zs;A~}`=_7GO>5!p?VT}Rf#X$gPh5iWs&FZ_$?}=hx`{{$*zgYBDEaMEi$fGRh+IM{ zC&63+j>WBNyZ1#Mik&uQv29abw)}&)S;Tnl&R$Uy%!X!H*#tHyDP;&)bS=U*zU4NM z-PK+ynzffgTF;wkzhE!GJiYzQJm!_qWz>;FO8u{OEywL^c1s&)45}-&)YZRVb-JP6 z^cOGb1hR?FSe&ry!OzQ-`Gj)eDV|rCEmKf9bk4-NEiUr_ z^Str5$*PG=l1_Sol`lE4Br#Fju~T0s+1Z1n)U>5NFI@u(++qp-uk0 z-aU9$yMMr=miOEC$D+-Jl%gW%&y;eV4}_w@?8up#xzS7W$8oZOV;z|M$G0`U%=%aw%OOM%gwE3hF>QW>f?Iu1R_r=i^+Xa7>?&z%bM7x zQgcuJkl#r2HIeLsWf3sg=ul1{ z%92Bwdnm^bW&WWYK9pmJ^6{bUI+R6+a^X;US7JQFgLH&6$XHj zOP|D!Ep%lSxqb=fOJ-mfUYk;g01IU(rgP^j zU=&{$Vr#Wb?cyUjv|Xpy-H5;YngqB3bu2$_7QAyrSI(mD8cj0jZzKS4JfK_{VxaqP zQVEtqNc9`t4_zOXUtsG(2D>PkMk7cA(y0c@809WhAz)PJ9eOhG)no)Jcg=QoxpTp% zkpUrv@Q;1Z;ra2pjW)t0$(}n~KrX%($mH{su{F+a@LPyg)bBs#(@z2@0HPo+6>FR7 z0?7hU#MhBB!>g*yo)4y#oNxKy+z35zDJ86X2N!C6e*NV{u0fqs`!T7b@MIrOul1|+ zlUD{0d|sH0IXE~ti8Z8kL)isGEiok~)ZQL2^sy}y=#FY<0j#E@r*OQU^Dc_moGpuz zvTTfRQqI>}EF>O4-~cxQ9T$Kq0D1tp6rg-ZMQ26D3CHUJR#=3^Vq=>n>{Zh|%d%t8 zrO#S7hVOANxy4BR_vM}#|5iZ=f6lWv9?59%hfDdD@`2Bn`t}~TT6{IbqX&BjU63g9 zB=5q7qYS7~z+Zz2($zbPt z!is8~BJo}wFNlAPPJV$M>-SLm7~zFDw$M;zvdd4xlm}kepfK#NZVmZC;$7RHx+lDk> z;t@m`-D4F40v!fmB+n~G@?4Q=zOuY}XT5yS%}U(EN*-J(PXYuLWY({tQwKgo0pMs1 z5#pI%H&FL+qpmWWh)#OPGp@YdjVjfvWI>Q#juX~8yD&-`NQm+Qf3y43TT0n1!=WSj z*=~UGT{HN&uyJ-$Z`(+WgouOO6k)e#*cwK_itD+j@wq*`N7^6U?gsIJGr-_!9W#dZ z_ZMKmK?s9=>$4OB^3OP4DSurz6klNO=JKbJ!EY%sw85uDUS`qB`ovF(kUgW=jj-aj-+o0aze02%jHg5z z`$?XJ@x`Kd@^BsC$PihyZ3+v;(wm>a; z-Z*X+5CnWK0C~Qb4g)gF*1{RRneFZ}$5dRa2pLEO(PA|DpN9--ND+BCcV>BM=hbAW z8aNMp({%`){-w65jf&4HyxymjM|9MW!oU{4HQ}P{8)QU_UGc{gbIbdY^>=A7=d{UgfIt= z0F@RCxdiyNlSGZ;e~}??V5;ZVqWWilnK1VD08kogaDXTm!1y-<5)cfeA2@U)9=g$X z>M9LR-ApNweAoGW=D1&bnsg!!R&a9yTnHQmf$Kqp_aLtM`zj07C*zB?5ce2iA~SSr z`(b-@G%(-G>hWyij_+}6)e751LYQOQyw{j(oDY0#WVz$}xWTyza3BG5@sj~wi-lfJ z43OgJS6kMBb9Lti)V7JI&4aBO*n^K6%nQDiaqy1R8)7v2r4@lPno@iY0?Je;f1U;? z5Ki?p+R4lp{uE81K>pItwg(N$s3_mGXBfCf#JMDMVZAD@+->i`olLpW?cR#gE|2V3 zsE2iWV(mo7LLzJ2BwaWQ02gGp1^qgcCUx*c3YfAvxC+QIv zJoKkj+II#g$(uV~49q!Vz1N4#1?{h$9^KUzum%JERqkX0UjW#`xWA`~m@<_p8W8`M zwpg`Jx5zj{Ll{TP%f{+$&`DvXM|bMqM*~Sa1qXoUBOBPc`lkfC$BZ71AdcoVE&nHu zlw$?y0Vo*L_1YLfWEG^CZ0XMkDj+3vA?zOR=gbw-$~|&J-56`E*@~@~JVEOf8PDfh ziq?j|6=|Rhyj6&dhQpbe6r!D%7fkDqj0pT47rx3yR!duT8SjE%<8!D89C)JeM@1G% zl0E&2z>o#NB>%<=p_(a0I-~5_k>pTZ0>}PRVTj$JJf#cy&1P$hl}WA2-~PGxIg9yx z*lYyrJiwcEcV^f_yJpX_wHJHbK?y#T@{@+Cs98GSR5FH#4rD86 zaP*|dMjM|+Uo~jkx(x_ZDgt8l%*W7Hv-&*S@-4G4K6L@rF7{I6T;A-Ool=7xG?2^f zc>$v_>{uJ;zPa5kD0T}pVHOO!WZ=F73xz>9AXNWk7&1 zfqAqs5o(=&MU2vy6(z2~7@;D?Ds`kbAea{}m;EZ$RXBjKemuvO{RgwL?Gow!Eh#4z zd0LQ8h<7)xWf_+}9t(l#Y7`F`cZx1I+bd%z&hfV3X?ofndpRU}Dsp76yWJW<6F!k*2^MSBFsAcK8POvUnU zsiFAZ`>Harf-3*B6!0qw3uKT0aSi~cHifp%%e{%VO~utKAowxTvDGowfnDt-+}C1Q ze-VY(_y_WnM7EY;GP=Jl=64Btk776)yMuM@5@P+z+cY>FI_j!=FXUwogte7@a;ABhlp==4Bxf7iRey%g%0t zY0X)m+wNv1f;h`eMFZ?uklgO-+ZgEDfD6k<1`Hx;?3Q)Jp?;R)$`TV30H_6s&{N9F$S2IEAR5aGI9x#c%2`#Kr8Qk*Vy?ibqqS4=nuVj8Rh=<0TrRTD)xa>FIp#eAy|h*7!L)~SOb1t5&$Vi9ioUC__fA-_7TAP zX^d<*dGs3!(7@ga-XigzS6@dNsz?Fsf2}G6bmP8) z+hwLGe$H6MmvR|97FlwZpmWeNZsW;su;dn*&dy2p~jfzNUpXea(gI6xRQ@&*8u9Sxkl z+g-+?ysT^ZmnUBqMMieE%eMrE*t&M2PH@(36Mb#0&kk|@;%HXXNS6;{4lB%REwoMW zcK|-BV79G7h%u33m-IE*IlsNT{d@;117gt*; zheM&+7K$YC+3;jCdkX=2iT#zvjhh&;;x;aGJCbYFk4eR z_17ut+=0{cf%f!UUo92nfr_|9?qDg&5J@Ao7=sMSPOHb(Glf}U7c<(OhAZVQ)GY9L z+YIK>Jef-cgVUa4Qml-LizHscS(b&>0O67~IX=Jotn6pT>nd+=EI(tKi}7*@zMU~p z&WT&CvRk?=lJQ(xvNlPJ5}RVrYA?UNDRixJi^bx?ia2_?>BjweU{OU*PLNqtH|ljt zkP$DXgZjdGpISfnz2>`<*N3Y6SG9LMhK#GKZI*Q=wHn@Rg*;<26ZwASDDxjft0_Jm z$KkoYFCTYZlTfWJnwlD#Hp$_^8m9y_Z#QVNK$vnXIHru)3XIS~fPN&YbnOOguTG!U zj7f10y~0Rl*Va(fPTu>h6@z5XegLt1S^E}|C0VW$x}SKi>EO@zYJ0zWgxHB|zwKYD zc<<0@Dd?%Loc;MVqv`9idDg;F!s$Zu+}weaFu!ogJ>+j8RxiuBV>R4z(lX0j34J%h z%O3undVPy*-O|^w4Z4D|KdPy}Zc#V?WIJBls94wBwdO+~S02}`VCEXKYQC|&(p$dy z)wnB+8rZP$w$Aq3#)5v`AD{PIsI*RE@C$GC35jWCCGwFbc`s_$Df0*J5z7_gK!6Jl z!IfBQe$3MrDy#;7!bty|)Maxq>+D>6ut*+M;7(=~S?4?v^fF)%ZBl0=qcey;a^auJ z{YD;HuZzg^!Ot|iS%i$3+&v26W=B5X)wN$O<`xzq1|EH;lQ-t(aw~2CE0+CJN|CFiK;zOvGaAaZPB=5)FL_1+FTg> zX6(&cP)mFYs#;|9O8AG5(#92ofti-|qqQT|CHyLW^K+Gc*ok}FKHV*AfCy2|8;rJ=FhxTtg#$^sljEGM)&D zF*nbzmxXOCFgq+;_Sru~qQ(9VQ$;SX`xpD(SWd0~#AEH7A6z!pyX7Z5Vi4k29 z(Y%5jG2}bMx;?C}Y$Djr_e@5e()Am&O6p$6KgPAp*q;%mI9h3G6vU6n%9Qwht|J>Y zBr*tkC5^bJS$t?sSFI|MI9b{wnqTR(G~tNZ@vgl&Pcnu}JZUE6$M?pZxtu2`Z>3Q$ zW5XRXCpff9->|mvO|sf>jicB&&1MF;CF>qXO(iG z#Ht`OI9{;&L)AIywB&@cSH*i>xoG}8yFA+i#XfjQNKi=OPK@|WGxZ*lj$ z)-+37S3f5Xuu=18D z@=2gq%AuFYVBvhF%;g{tm&`LxE^V;SmfrC&((EtnkBpa0jQfY=dNmk6q~{2`^)g%bYl_T0e9dd7S1zr@ zN`oHrq~{{;c}Bf{I{zA$&C>|_*w{DP2?x}^jz7~k!`h|W6|T3QbEV1@#b;(`;(Q$2 zf_5rMr)0m+Af~9y%F!~#Hc9+q#czwBJ?>-rm{a%~@5S^Wf!Bby^<%*R`6|@#daZ@% z)VBZ_mZkr`tPkdO<0!{0tLa<0Yih4gh!Pm;LI}A9;f|6VCn%btGhv#kCkF;0I#9Hg zu3}7q1g70?I^|=Gx16=iRV6!{Bf($vFK}pCPy=r*gcW?6`8*ZL__6_mw@)U{CB0B& z!`7S%M@MOV6@D!!G*n`@Q&{@;HJ{BqzA+=|%^zoQ_hL3moQQpeoR^4pysJ&rrl)_x zpWNnXso0$fpx_@zuGCsYe+gur{mL!QWjgt$S;(^1?+8yG?<2X!TxZ|q@G3*&3(e_v ztZpNwW0_Va$!}&)2HOVc*V>(bB0YI!_$1L_d$zjv!&~Oqe*0Tq1&&)BRi0-mDg=gQ z=EUzJWcLU68-JeKMCfH~bx3Ai(faU}a%R=oSj@{mb3Gl~e%CL=i#n`WO(JymW)AhYwt=Acn0IsQ zB%?>%OUlX+aH}-|obzv`w~q7e@M`7qON$jp1118i1MGM523p*43M`Xh=p=zxb8)AC ztXa8+&d;zsW*K_e6Ez>WQ~@yP&JMtuJ$rvry@C zY|DiodtazqU#O3Ip9rqUU+JE>S}Bf}Dz}x9aXB#>lhYfS*-$9O(F3h!7Fm5wDKv}L z;Chd-`%(4DwAQ5)-TVps&jw5`wx~SAnBO~J?0q$@i>Z} zR5?I7Z@27Pea^{aZNcPi!-@7|o8f14k6-_ClcJw2LZ$S4KTrR0>J*w2jrP_mehaEdq zUyUid%1;l=h|sG873J2<^!Po!wfN5=#t#hDb7ba{1T{L^tB?P_(Xf#f5FN#Y(^>In z^yspE!z2sy!D68<2ChaI9!zOTzyYj}i>YBbXKiJ?jA%kRQyCJK^r>vV^!NKVO?8A~ zbjAD$(uaMlg&iku|HpN+g98o29_Frtw-TBP`jIS(v&XIF5ua`~0>w6bos^9aN550H z|NZijNrOh2rv%B!F|!y(zG-)-u3h}&y!a(CfBkOY{$iXlK^Tklx$5fXdi_N0UjFo& z_VCe4XePha<>SVsnLN2?k3BBoxVEl(-N~|F-@1Bkwa4A3MN-O^#FoiUmzdku5c%R^ zo7;-z%x}!w?YIYZ*MGH$+Ew%{-n)og3fSv@%H1iK)3uPu&Gh2~XS18eDu2Rx=OZ=C z>XXS$I|&Ac_205AYoD8Hm5}&UX=Y!lmzh(yTjvrk@8VQ9^_R2u0rg&Zd&7j5?MPFS z%vyo6MW(csDXaI&xYxkPr)+Fiki=m+IUp(8}-73>;>sd4P#&&I(#d%L~q=r_Es-G9a3>V4Mi=(4z! zs(alxJ%1u6kB6BIJui8j?3y#BIdYm+!|EqscH_;_D}h3cLfTe+tT2*hxyk7lxln`a zS9xM_?-6p}vmq!IIZ}{BE%KW4qeuITHQxCRrDEY{Mb2_=+p573CW@-^=Oxa=wHd5; zBoWmTKV**%BNPb+$#g^a?o&Vf>Bt)|9l?Jed|j`)a>_~G=1jn}DNAtZZjWwJA))Dl z&HEJBJ4f?5)6z5e0p_|7fv|F&=0WATpry*elIxr{{R#>+NvWpM#6JD-{O2 z!b@T)vA58vVRsKYp7p6tY{xOyW{J<(R&24U+L2z|t?Es$y`BGSHXN~ge^cpk;*@u- zyospBSZcnCRj7n%Z`z5BUAJhr@v}ooYQ+mN(DzI6S|}sAq~oF!w2ZPAN-upuYy0Eg z`8IVe&M&OzBfqdv=@frT?y?2mR{V*{P2aLTE;$YUat^fkdIh?I(mV*1wEyD4a47yB zA^2DGH-;@j4$lk#e6&*nU{JpG->Sa{2SAm@r+-!ddxM9^0RBG>;{P`He>4dG@%}gb Pr$O*=L!jTGuI0Y~P?a5Z literal 0 HcmV?d00001 diff --git a/internal/protocol/hls/testdata/media/sample-00.ts b/internal/protocol/hls/testdata/media/sample-00.ts new file mode 100644 index 0000000000000000000000000000000000000000..4e301f775144f65a57cb3192fe6d5a5c5392ff52 GIT binary patch literal 27072 zcmdSB2Ut_vwl+LhdP0DZP(lw9dJ6(dXHk@n0Z|cA1A@|2M8z(PB1I{oh=?cw5dl#F z5y66?NmE2b#ce}DH!3P(Ln-+d?tS*z`|fk^^L_ul&-33SkEAE>oO6tKjPZ^UoM9;% za2VGBP%{7`NCcp1MB0T85f(4r7!|UCyfh>_CTv{@$9TtH$8OR8u7A!jpaD=V{u4WZ z#({1PuJ(Ty#QzkHGZ)2FG?Q-5c&h420#KZdI^A}Xmak6OwlKM z1ib>OPwvxkIrN!Dwx>bO>LQ2(KG^RX#|cuvP?iYE2Rkl;P2e#H8eg|wzn~C*5XhHC zlV88NF|%`Io;=LOF-^@ZNbrLp^*61P@AhYa5om}kR6v+X2!g}4@jHW`dUP(vMGSQ!YS?IIaBziFC#ek4*H%1qYUU5k9 z@fh}g)Qj%h7E{cx;?7vM+SzPk->*>N&D}{aw=B761agDEImCEPeQ4VE&2I0erfP_Z z6=*{z^gahOR$4J)$V(TvFc$AWx3%&I7oHx+aFyeMEC>Qa1QwUWF%_WL8O$MFF3HhS zDjV-e3OLXWxr=vEv1)>K=L4@ze0VEY4BSYF1X*>Gj1YIy2rQ~OVQr4#+`sepmBn+v6xlqIse_)MLe5xYweoi_Rju5%{2ue>hspFC#fzbAsn*mD>A}} z{>z!qdCfmfsR1fSvCF6-7QA^t;qB*Y4B|K|U}BtS;{C94`*-J_dcXGYvo{4kN5*G9 zANf^C$7oH*!x+ql*{8cvXZ({X?LidcRj2bXEU4_d9?-^dWFdqdJE$SLv%{$98Qc`R z2a!a57c6$V?v9f@iG!KA%uhHWx%NMp%}tW(2{47&Vg!~tKis^;U0o+TVCS(`ZiM`| zr58UQa=}L7;Djk51u=q*QS7oSg%$7ujDX`eFSc{Nc(8tT3*wWJz3#{jR-W16eI)!4 zSI}R*xtZN7)vmFYv?_V+HWCN`j|nUmloCo<#HJxiSZQPGxu||dQtle1LKA^M<8jZ7 zx|`3J54TyU1z=GNcm#An5Q$|8gjlZF1UU>7hiE&BKQ2%xWHQ~`Y8$nwr*QRN!^%gt z7*%74eMq<)Xn=+YiKjswF;L)}XUYeAnQlzuJX6f29XC1e-=thevCExB*tjL%aVQ1d zp1JyE6#nz}enb7*?)#iyd+uKjxFnuF7PyG2-9;tue>?84Le+Q{2Bf9#&-;Hk?*Eaa zMzJ&hza95qnXC#*!S+#ssb*;+wwi&)2!>l=abiD{#|u{tett@}uyE=_kv8F;bHjW>q0&fh`AK(VQSFvNLH z6<;h|xK_ss!^Grme(+5IPodc5hX^1ZBg79G2d8UeIS!)vMVGnI%8=|nzJ7Xs0dnV< zxhzxk?b@|*82&*MC-dZhH5kYf%7QUd3O$JqOSY2NExw!td~ZIvQFL_2(+``bTTn~E zp_cMPnr6!{P!!4_q4GUTbby?+KSi<2#g*rqoL!huyY%4VB|YEed#~H9EwRVGh9oBu z57N*)58U-h=<%W{)}MUy#5K~NX#Vz^QNkQYDkO_EkQ5MhzFSCvsCa}&R{ctj{k*#8 z*Kd7BMNbE@C_lhbSw{WM*wJPh0aBlC=3q9h*$c5y|6l`_X6(CBBU`|ivKyfT8pH1E zH}FUN*q?{mS?s`Xb_Rkz-1f=6)-!QsW~rOctZGyC^q<4d?KO#iddpwWM`5{+qHLiP zFn#cnhX+X*AqST^g}eU4@a3neQMub6Q0$6P(C9qCB6S{l&5QfYOF<*tEH58gFoX~U z62K6w0X&FL;7P)!5C=b`rG_lWMlq4ZL?*UcSHTTRH7%ugl@7=%WF1h$tBHcuBhUIk z`BWglvM^DY5*og39*=+%(&8n*A}UEBzZ8On%7`B(GMk4##h6d?#478&q<@naO-|*K zKseCE(O94pIDr_jjkICB@1zlrK7a{442oTeXFm(DfFhI- zN;Hd2Are5y1m!fL29QNm(;*iLCKW<#U6*~6(|hJbvwfV;R3HH#=%9}eqm_=uv|4&! z*RdwBTar#rIJmmHsp<#qw|Z8-xrZ0`4Nk(t63r}3oY9QSiZ?EWs3qa&EQyU;5>1qX z6J}u8LU$3O{OtJons{O#KuoJ?z#Nho)u5&Dd?{LX8lU5Q`2MyxTNhwb6+}C}UrM=N z1%9U%6ua`j8NK{LEhu)C|EFkj2?xwTG!%sy!U8HXAMA$8$%r9jLBdokB0?O?mqUbo zIeA!mSg1x0xkB3v>*glrI5z6_eN8Er!;C`@8#jb^;)taL3#qOM3#a-H*oZ^Mp_pRC z64c=y;)G)En%e~Vgx!pT8q*6u)Z8CuBGzbF@CkR}SwS2ss4%TpQ1DbEpx9LjNvG-; zE36EC{jUDukNSvSKnKAXo;Z*YN+R3vpje;K3P*Zqb0X-2VR{=jjQwi!C9#j1?V)$% z?v?q+j{Witr~(EbQAOG~L>EkAe?#bfH=oSvwr%+rj6zxa`qbjM*lHIba2NT={K%dJ ztbH_+BLzf3Q6yVe{Upq`INzAIDn#4E)%Dqx^l6G<-R5IEeoW0g+&h38838r2*Kyd- zuuK$=M*$8P(5A*V&M&|gqu6QXdgEfQh2gXefA51Xn;z%8D<-j;J`}BvQbzQ^{-tS{ zDl#B>;QFvvcve6I1T9U24q;$S9PA0)yQm-lTN~D;N&JjZjf&NPG!p8_CP3WJ(&Xz? zvFm^fsb~UyIk{+R*PH$@Qa&E*T3p(~m+z{0?WGqi{bAC6;80ce!!=;KcoXg;cWnRL1S24T-99!?ndeF)uCD2s$*mm<1qf}+xrRntgw zoLYs~2Jy%5EhoR-{!pF-aR*Lh$FKreLIO{|bb_e?m^{$(88O45g-5u8aSBfXXVn;d z*G~$O{R&|}!6-|AMsPwx5d%mfl3hIFa|vveB9-aZj<`M0<5>71CXyd2p0bW`A&%EOHs-3Ac}+lz2Q0)4s^vnV1a}@i$KyRoAhw4^fq*26 zoi0>F^07WNs=FEz)~%iQ9iOu}F-b$d@Mz8SzUdQ3?(KwJu-2ra_hguYhfE>8osc$G ziBvT9`)#As7y(MBl5jD*0JDiz-G?5>a}ETPc9LNME+jq_dUp5qt}zIaD1vw-A7I?b zvqWh(E;ySJ?>w6nzj^jqVvarbgHVAXsRq_7d2<-C@4sU00TotF+Az|`B(hBzF#U(% z&&l5wp&>xAGXkJ3tUA6*!P3Q+Bc~dOM>{&kSrRx?KF@l zUg)85niF95`>~L*ztm562p^MC?9$7VfDL3$vVGZVeowmo9W5Xy^Z5jrnFA`svyqlnL+zz(V{Pu8{8`)CQj%&UIFIs>+e(tL z&In7454KnLcLgLR&N>=N=dyCM{qW@PT|19vFMQRMus5I@0~26h)9U_EiEIVG@LeH! zeiFi<>>js0r*xKRithdpy`2pp0e>Kng7xXwM~UbLMB#xk6;qnFhEaoJmmz%LL{hDS z@^CN?8pJKaY@QXO{!LUL0)>zIk-|sA6~Grv*1Aal;}}k%i3U}SjWjo_&kD;mMK3?` zxX@0C)MP<{G9MT>4H5tzDDd+`L#1EP{WKBfnMvHMCljF(o8y52m;Lz4qERVzl|j0=vrD z6pzNzPM{(!|HERnYHZW+j%TV((^EHi1X!N+f~FBW8VYWFgS|+q@nA5&7qy602tS`U zPT7orAZ%?M5Q`!=5Y9OYjj`4y$m>HlnQpP)_nz$9b@b^BdH${UDqt@rScwUdd0531 z&cp&R{|Fw2LM@`vg@Ywgk?NcKU%DyvMaV)GM389nMe#P4_6rnkYCJ1b%VdC_D zJBTkrHJoG**ohAaB$v6Msc)HH&LFbFqo3HcG`ko4%fn zon{!NmN5d6ng-a^Mf``uWZn24o68lz{;nqy(iJXoD&& zQHaS7YiyQ=1RM$*DwjA2Jro;ZnMMQ+@Do29Bn^*GedmWga5}ap10+JXt%g z4-;Wwb~zW{uxStpDyI^}F#`D=TMudGvyYzdgRq|)SjZ$a4vos8F4?F}=;0~S3q$W9 z^j8}e%;k~tnon4F-}hm;xC$(bha=z2pnFDUU5!@VYXX_<>bCvLqQX>I5gA33P;5n6xvY|ci_ry2x!*tdi!USmp;`xaR4wp(?>yYDMMxKQGO0x*FFu~p#nRSzi6wbKQU zEYpEyIt?sHuXXTRYR_U`@5{U$YqH3}=wiI&8$=Jc_6trxfci8MeTev3C>?L~Bc6)| zqYPN~JDQ&H6!6T=dMVR!`*iIA1u+>)6(N==b_I}D?Fr0L_avd=_rKP(1Xo$&5Ebf% znBc*)N+2Dt`USMnfizvnJo64L1b$2?7OSzB={fk7WKdh{_dZ6U>Be@al0L=bs(W=` ztPw!tYKX4As%V$uS|5^XB@`ts^afgsN5!$~R&=`Qv-~fr`KvpbXp$sUK{(G5V@z@J zDPXRj&n0KwfPxS&cdYhI>OLl>CDjkbt{9=VM|b?cs3m4tXcYjW(JH{~K1v|TGw{zM zpqgMf{YA(9@pZlH_e@V2J)e4Qbo;-me9?;JUst|k!P~w*nzb@|h$tXhL&jZdkh8Xl zDBDj}9Dc=1+}?QIr1lFiMPICq<6)QVw)za<8JOA`jAFO82LBCZv@Hh|uT{ez?k`HU znKH@nJ|wp_2~96Mp$2TW6cxv2)B3@<2?2Slaf4+_44KaEk9XJ&bnSKSU7y)k@Di=; z<5^fVzG8ULPvPXBDrODnqlrcYQi!-sH}ILO@@X!Bi&pvP`+Rb*>V0uRwNx7-4S>|z zziO$TSyFR?Dr!gotPo8h=#`6cs=v#n^xcH5TJ!1AdkW4TNjlZEgn9hy?fOS&3v=!yq)X#*q#xm(MkM;|B$1HMG zb8ohJ7X>b6Ta6vRbE@&NBxT>Dn(IZ6MwC5giFpx?rhh%(P#ut=8f$T5LC_6U z1}Ix4o)l?E-E`e|*?ZQ_cJtxkj~zRh`)6Fb_EsI?o}aig7`C)P=Lh&yqpten+uQvd zqK-yVF0=1kgk|P-7JE;9OU@^iBPe#-k^7Hh*4$EEkPHts1k&g^9a1!| zC-R`NtUvBo1oANUm$<>fJyw#BiH#SOhFQ#4Cf8*-~U#jXFc2Kq+PrEwjsq}I23I7zr|S2-F5WGQOW*2D)ho$leFp5 zc@{kE;y`zcqXvCDn@`dR^mx$$^P1i$_cU4Df%&OXf>-O;2?vEz1$TxgSMv-WeiZH4 zRC7|Mzbp27>%tG$6%)AmQGVba>XZH4^yuYGypF6`ZpJ-5JZ9ybfbBKPbP(o?K;d?dpYQ3d`A*6E#7LP^9xoJ+CS2H$|f$@Ui;OfDQ^Svx{C;cI%0 zUYHC}ZwTdpzo-AFB4PdZ%g5Bu&JL=H4-(1RrSr6Ls(I6O6&+BQj`PE3lxN0MUz!F# zOQs5BHIVT;OP;vL7Ve(EcIf!+p7MMd`DAlqAJx^5Y)<^;a&($aO4?5PM@eL~NodZM z^uczRhu%r=joFr_FdYJqMPh5c9UMC6qu8ZoS-US;6guJxd*9C2l=Xb1@Ja&G9C;_} z7vz1rZHIpgv%~p)v&62aB_XjLImpP0B^(e643TYM|A7<5Ue59ns)61dfy5{0s)V;& zoL*4YN%Agc=@QoQ#Dw}(jK>Jy&<~utwG@}l{Bji7f%1HTzLWYs0(c_vu8rz=KDKLA zm4+BNLf_14Wd#RLS?^vL8z0a<;E|Ynn*EhBZ|!_QfmR^QCV6x}$e@M|z~o#>>>Hy~`IrR3~R`%=EjL zvc2Z@6AN{p>F}*ur|eFD*U#YLtP>zE<$91qdkr)4s+ed7pr$}%P>vpB4`yf+_}(^M zw>RnBG!U?)7;OzT`k z*cKg>nrM4O+f1nKDwHN<4|v2t@TO7wMIz)(l3W73o#E&+>Vn&OXU2TpSvA!WlhQH9 zFOWHYr!#h@X6&@mE{)jOPlDW)3w?zQp_=kOoh`d^i-(Of(h@&pXI%TVi`|49)IKzH z!(@79)y8`*gbNqwJaJgE+vjp0?`>tGxE8J4$y|oQ)okH8a0Z6N=+}fZ$bb=0wU9RKkM?0afsTtXry#lfIqgRb#V^x zK+7asd+kyP5ren!`$Jt_x`ut+Y^;l3W{&t(_q8Qysn@VL1+HU|D~B*yxCAi;eK=G{ zad4#?lkFFQLM(>{d|Va4B$lHK#hKN3opXd9jC!jb&!|ALGZ9NyEg`Eo<(2tLMyCeL zC3aruCDlxeJlydC>=S^1@ALAu2=wd%6V{UDlhPs=lz{^6#JzF(!$;e;ep27xW2!Ey z-xP-Fy@zIHSP9+%{slzx~PVg}K7Jn^`lYPgg8 zba*jNqw)FCu7WZ`bM5RUFYL$|j5vVUY6kbTdYXU_)uCoYqVHzC@(LyhpqFrQxck4}BF_%x!D{8D<^-y$kkonrbjW~}Lv{>z?sU%ORA^40e0 zLwO|8=fsmZE&3~KUH`A!=X*>QO>lQE9_3l5Ke0M_GA)A}9sO+U_g)d&>?6%U>TZiM zvn7VXui$W?vJ^q*38emw=Lw|#jpqq0=W^hDzrW_dDLQBl?3smH8~{iDo&($R%m680 z9DXMHA`OKqAVgYzR$qBxFm%mTDy4@0d!|>d;NX(5M0DE=eH$$OZF7I0UF@_t52LXx zE@v)0;^>1GfT~>16yk{Y1W`W+6Y%t?H`q7?70zD5BGI>io#^w>Fgh-WG=T<8^*tP7 zpF+b6Wpm)8sJ$y%;7F#_EtA;t0w~CNNTN~`Ok(=_G`-+rj|NkdP0Y*(zw~GL+-q~P z#?x_q&^;$%g$_1#C}Vpi(y!O6CQpVDfCK^g-Gv#u(a)R42!!kTb=m8qC8(BH4OTUCx`h9&9ga9_&H5aNe>=5z=P7AgxlO*CQRMd z9D+Bz5q1Jj5SDx5G;xWjoQZvCki4k6oPwyz+TbSe=K1Jm%mtFJOHk?FEhkGGt+Bu4 zgMLkZ92IhtVgOw#2Bwv2Pd$5Ai+1qq6+i9?UQcH&ox}~vxHAx@EmsNK+ZDOJIKR`j2H($S$?&q3xnc;v#-_PW}n#s;BCH7{_ zBp~-Z#1qy-0;MA$5mTGYF0?O>XQ3xdqQ`VHVS+Sv2pV(gGl`vcCeWaSvM97(wVU(R zvf01-YD+Wft0_9@D*<@$cV7)42`7PGrF}jKtN^pWxn|V61d-@2g1!VxA5|pW7A2ek zlL~gT)hsv?{b=-l96hQ7y+GaZHQL2fIE`K>&{GjxP*)J{+R1_?5j}BC@Cc37r^xoU zMV%-Mrtx((A%39g!aN_wz1giPG>I-3jMJ>sM}09pbl+*m(5&c?^b(dNlr!K#3@(*W z=kt`a`CuX63}91GBWDwlYOsvaRjlmB|0u@4lY2=Od3+-7`0?NuW=`GvL)2o;Cn;b# zqH-XJ>ZRr9yLY?y@S8agY~j(hO1Y9jLN824yiYDWbJ|C&%YDO^=|gDZs7);P1M9pw z+U_n4bsCFF@3$^YRW~;gP2+)Nw^DYLsc6PYSZtu))xXl6a9)(a^>FhV{8D}-tz|uHG4uSC|yWmzGI3c=5`Nt#*{Jw2f89=j=#u(K9Xpnv0@I1chiFk zd-b$T67B@o#hH;<4^71`FqOnrMIagSU1z?WH6Ucf4dUjzhN*aAv*&)dkvO~*9E&)r z#ILfiI7-bCbhRwW-G&ieZ<4inhv8>~vxEol+JJkPyj(8q#<~4~lo!tl|x1_dJT* zpOw7+>B6csINLACkx{!?U(nK;5}2YXxEZ*sy`YbPD1br^mLB9+L?9uomd2bacD3i2X}{=O@jW1*Idg)R z*|Rg5yKw%Kadjw~NnuLb8J#j5+@19<8mui!@m>kYJ}_1iPQy(mX`P$v4Vz^C z|L3d!(x>C%QsKL)L)gptiiE+Z8`NE&AwR$XrWV;PlQN&l1aq!KLR|;VXXyXN`3(Kv zIG-_{i3m4QBv`HlFnXdcpOI)vWwXC{ml zP5pxj10cUphE@;`O*s00Uh#x3c?*@DP(Iu*R-Vrz=d|uxx4c6KA*2tXEN7l+?!3th zgL%Qkr@j&xA^jyhD-9NDJPeJpC>3gb$2y_*F4M-L98yKp2t}gu0*iOj)5JR@Or6AY z|3?SF%^#_`FNXSbYC>vP6sYlZGd2iSde2V2rNn*THqq`b92R|HfU6iGnvJ*;?GvA( z&ez?GwRF}f6iB8Joec5bd5;UJMIvvGGL#z!!bOvmIVvQw(_W;GkP;z(;+AmiMFy2R zqqCBR0J}rn;TM}UwT>kS2T(Tcetf4*X27Ff_)Rpi%85hCwsThn=L8AEQ(#W z`w~=kp`G!DCk;9$+Cp)keK;FPtxDF%MQF02jmXV&s?NTVZ-^G8<~ z)ISX7F5KLKYyt0ZrjgffD;a7^b8kL-Kb+^x6Mw^t^t{+J1WUkb;4JnU5yftP=6@MS z{!8Ng{m!iB)1o=2`pJ>@z+%o<+uZ)*t6~d3qP{w_5`7kc{;t3OOiTXhq;pBiKRxtK zcocg3ABl=}VLT3s!WWSO`Qp*~L5?Na4w9S>733C8dk*yMku&M=$;3 z5KQS-It=OumF;FrLSw-gMLND)^s?dVVxIU59=_ZnTPE(b=(%8(T}*ZP(Q#vh8bw=1 z^!31U`vd(-W;;TRgxoy8iI@?WTzs!q95WFu8ra$rXoloe7dPJ&TjCGzML`D_M@fS! zfl5w_86yD`{eZnbwIQ*nh_e=5+1o8=RBTJaxG&62V`omSN|BpOv{d4E0wIWn=c6gj z@2N^^G;f_~YA=diF$#PwW8;KLpj?t55+O^dBUPPNt{SD}gXVjER^074kf05OYC_*K zt7QC~T4bf4)mqXum#2s+exE*0h;EpU#6(f>cra~ify z{{L2AJr>5^qCzI#7By)?=Fkr!TMtlS;^(J-_$q{sx}Z0NV%re}uLj6so}wAJ&2gTz zloZ{K1pv5!y123PDDTR`xnnftTp?v zr*6SBv&aa4^yc0N7dpCQ!ZmFx_1UG4^LWZV)*C-9zkxo_|G}+%AWYrSYusqkxOQOq)HhuJ3)v#a9JI%eoI`v47D?iCznoQg%kx?`7_) zm(}3Y_mG}gwZ@-mF}b-bDmpE5wB=m-W5(J(m33V*Cx_Rj8RrE(_ZklbTs?vjk;w(04ALb22DKY)#0!1`|gn=LKE z_iHr6PIlE}8n?7b3n3U+Ogd62G}B_$lU8L91_tq2mz& z_-#=PHK}<&532GFx~Q3np4Fz;#kb$}+}=>}RqPPoLUtitcaxvN`dqnRJl<+MYxxXT zAL6%!GHqXW;iPWP5iYyUjMeCF97~kQ4DpQ}r2BPPS8y<+;Z@&J|*I zVJPD!jouZ{Yqq>Zu`@FB0lRqf1D%16Z&DzcT0TV8^_&y6&M{bw{5`&SM-^tFv3yGyrtpUqMWcRz=X zNc@kBsyv4->f`Ozn_D++yKS@W$^t0jf^>*y2m6d+!>0A_!PxBg_fj9pXSa`C) ziw_)Y*r2!G^~zP}ts^^97#ZiDAJ1xh)q5+bJAIA$4HUa{jp8Yvt_>&On`QVib_GA$ zsCoFI(zU4bprzdTK|yq@!eN=HFJ;57IIP60Qkhp|({#4%;N9F} zVZV~fS;}1)1+{ng`BrV^uc13ibo$gJuTb3QQ~TCAD@@U5om#ku{(u3uTwXl4^p8dT z=2B-WFDDt#n)Mi%wTka}i#Hp8L4yJyH?wS{Q66%6WPW!iXpBh^>PFsO+kG_!c{tbMP-72Ul^vA94xl9 zS#hIn`N9SBZV>~w_rYW0wchUg)Oo8}mW#Z|C2(=D^`#s`WqxvpX_(C-&H90OVO?L_ zck7z^VUqLL>vnTZ+U2{$eQ$NReJ>lA$Vyj%n}ju$Y2z-it70li3B|Lk`igt&WPckhP4v;6NUcBXEXp~uSqcv=-d$)dKyX8R-R zf(M>vy;DP2{zwL;ZCMtp15Rna9jI8gE7w0!!CgYit3Yaoxhz4aERuAgS2krHC*ot$ ze7g9uqLnJaSD_>7cDY8CL0z3qzQItVisk9QW=O7BuU z7o`n!K5Fz((pdMtK-$>!z2k&sO_)LH>Eyjh725qPu^umyc_FOz)ChWAjpW8KlhfBy zUcG71saA`?9>|@cj^<&DwCoRbDz@qO4;>?S~7BUE!N|Fn_Yz`To)(;w}2ef+}am295F`jfN6lPQ){sHDwV2 zvZZfE8jMZ2vX|2qLd`r`0~xDG zPxK1HYU`8pCl?voF5DJp-rcKcrL3~4`vQt# zI_4$iL7?v2b=vW0MT~c3L}0sD6#sqbZlto-!Ux5!c+Kq{;o^Anp_+(RW_M}ym$EkO zVHVyDn=u+-6U@yYy-0f@ZmzZ3 zWmkQZ!$8#%K8ju0=<=qDH5JX}x4j5iX-$C(?xa>MUt#383znxEeF_eWx5d;4R_8_UTR3+ zL;67dH4wa%aNV!P{;=D7`*}%i(1ujwnfTs}^o@?% z2UN310bhDwGJbI@iR>Q?FN>Zx7Qr_U7%{0{k78F*2}gsGtb~sy+*^S;#{G2kqJ37m zzx;Ci_O@3mvr;_uyVLKUj_cIOo*&&4Ijy{T)45}8BJNQ!Z7@J7=k3mQF#(g-PQ9e7 zEe|a<7_DM8o#f#H)7<4UelFj`ZS*&7Qq|vBhSVKfc)G_Yr1i?sm*-xKcb5CBqi#dO zW+rci-ddho?;YgRc{DmGGQUAA@JSV8Re#>etJov#znkCB{O>k;QmgAw>~O=IPPHoBh{@O1TKvSFZAkUvz2Z)n&EY z8g=a>8-`hOau;3W=r_Z;|q8p$h0@#IUf8|BZcPKh#T`vpkREI#34+m7TRsXySgYXaV5B#xi99`>aQ@`*#B5t zf=@CwW23dUD*L4PSn~)aHFtU!Qz3R%XA=CP22^u@W$V#g@}Ar*Ke}>~h8VT6Zgg|V zrC=S6k9TwJg&uo{Was-0MTn*-$2Jk-GDR;3JPNSVEECq@q}cmmLcLk&4k}vq2@8Ly zf#u6pyjZ_UHB=ZWHeh)iFHL(9cUSa+h7XZ&wJZs$+24F`r;k7j-o)rf|p(ZO*8g{=p+SzO4lW6deqENGq z{cDU^0=ZnHx|P_V44$@>-86Iry}dy$>leY2Kp@c>+(%?d!rNAEa)3~YdL)Nt3|6wO z=dc^&e`*{w#l*3N=~z5Hc0FmsxYH^q$$4!s`x(;I)tW41X=3E2Po5$$;0lh0J&Q|8 zigWnhn3S{yu?|^R^$NU!2Jj#TkA9WQ=K#Be7QRFIbGUGcz6Fif6?w9Z)+n>U|F2TBZ0G`IYNykVJwW2KpBnF?b-+mA~Ij1+G72?!Ne~d+(&&8JI{~yc&9+&#yK@v`MbH=A)W;|S^aduPaHyJFbr1uHUJ)#$(_~Hi1 zoH}fkgUj?1mVMkQ4I9r~o0R)9QUZm=KJ5?)5=}|Y#|Ri@m@IM^&seiqD~KnpAKmXK z@$JpJk1jF=aGQkIg!su?avurkyOcreE9UyEWNf*Zyrxozl{CJ~yRD{F0!cT@5E-po)g8b+it~G`7}< z$QTiCy^4o*^>+hHbVT^&#s+0pyM3zWh{=aGh-9X*O~q@aa;4@0M7&g7b}`zOMD1NJ z1(qoeRKi}7Z%Zf=4R|DBOV!jDCswUdH~oDSqm0@%u>G8l+7se&Cr)7<9mZLkiCBw??ru)6flQN3RzWN!2_d4=`~o83CD%_8YM(JERD%8XsRY z_SfOXG_`8`!mh?K+{N23SRP=cSH$r{768S&6a#iA_uAqk#d%ZV~uD&ojSw<^FN8 zm&4))Q<rD;!p#p#ey~(d0h0J}x~?BBT4RhzOwXunnb6u{@s;LQ9}~ zyN<4s9AQF}cqg>i?gDPi#n+J+R0nWTzDrjs(ED={{1liDnU?2EYvfrMrl6LOW2Z?$ zF!r;At9d-;`E~}HL$Y8Ar&A>c&~dH3s-3hTa-+?vM$3|rdSzK-Nt#espDZZyf^Int z>C)tXP)$FMV~50C1s-Ti(-W%QGA2u$*XQHDaGjiRiILZy9(a5cTFsxKi&b86BOERj^22rqCIwjXFhR` z5|P?m{Y+WRq+%xFWLTeCHnhBLeOAj;S202+F+RlaLYt99UOp z=bZ|PpmZpv2l*?3Gs-+1Z__NnsmiU2{8Q!BzsEU((vlw@3-^>7Ms1kj*|lXn6BL|4M4- zn`GYO3GUi3&J{++OT2pTd&hOJ|K*})jZp=tWA)rg@utesB+~aEWoLBCduh&x9_Vl` zo4$LVWBUGL-g-C8j~2to)`wYl-XsS;8Rd{i$dBw*W41VIo^UM-A6&al;vLp>Q~Xm` z2_=u@ldJviUDWyDEVlE5c@&DB5iQC57{7Vjk+%V2_?EtA;og!3S!MTEZna&$;Htgg z`KQFI{8Yb?!}8&0#IYxmv!OT5rSX@X@#VqhQ4)^_@#k(B$}UvO^K2Qi?|jRbaJMl( zw$(P(@~YoU$KMzAhgOGn0N)tTZh2;d_Lb`I2GsYzj&j^~#!%sG$8An~@j-`$ZRFNv zQN4TNUDnFM{dERqdoo(ZBl^C3+x(3BA9VQf1F0GAw z8xgH^z-5Q2&Cph#C)g!^+j;id(4Lye*GDNMV(W5t)^`q4!U<1;T}Jz!xOD{YaN)Ps z%7mBrv|Qwq;)TZC}aK&aYxs<;x`OzpEJ{6InW1b%&pFPalYL zrKHE5{MRQnQ+pTf0B*c^ceP4uXPtJYXW4W3gUkm zHj*~!Lb1y@>f+n#%_rxRgAXb86>oHD(0!RyN*P?ejq-t<$n_sAkMV_1m--G}U9@Oo zAY$i}_5xc~r8}h`yqT_!V&L-gM&aI?`eTRhuhMV{SkNxK8~C=RFfC9jc>C*c4=-#1 z5mg*5>egRAsd2`_+`!f&0AD?jb$FF$c5BPXOJSHm!mYbgYwO({hRHaY2ywN-x{t5Ce`ArJdd3OkHW%yxdl}O2R@e zXwj9-A=fM15iDdQe6%WAu6~QRX0Kynn%I=RSNLjuRMgL>EzM*{d`UBP<*o>p(K8HP zuoGJ}s2@Js@6Pwgi9;^7N_)LN1D|v)+hcl>gJJh3=Wj?H)sy>vS&<#5ZqQWMI35b-q+`TjfK4^6=#s!8GgU zb4SlOMW|E#la|J9Xdc59*2_zsxYHQ!_R)jO@P#7Wa~EG;{<891yTN+*A#4`&V(OF3 z6XDtYrVq8M)BEdOHUz7n*qN@$(#v?=|6JH_{o0bsb7jUeKc4A-MK$ne9-r=dypLTh zvwNtbaPZ-)M~%kO_{mmz_6LQHT!qN15Tn&D*Ptx2cG=FUfezThVg9bRtFKxvufkpn z@23wB%dwNsx6M1zFm=Djo@>iN`~Nj|+T!WU~Mziw8D!jc9agKIl#NyjS=ncO?!J$3GU zVitsASD5HAbEs_&DW9>hl%Nja&jTNYF3G&ID+F_7Bg%QEoqGD_yRdc6IBZ_aLLax+ zRasfDtc`sa{odC6V+P{fTnBJArpG3xRealBn{ACxSYA2@%c4h}kLoya<}cbgtnfm! z;sIUW>2sfOT>NEDXhq6Bed?FmHt^h0bxl|=Cc-z@>i)OBWtnMK1u89ZS*ti7R+%?i z+VX?O{5xOPcO>|pJdZc?vG;^u=*y#dB%~N(=Tn9w^^M^y)K<<{is2i$b$nM^QtI&~ z?Ra%mt&@yiq+bCAb8^s#*6iRK=Teo-AN@pM8>Xkr;JuSH$-WlHr-U|^@jW~>8mPCm z^OQ5o{}zDfnq@y{Aogv$YjEK*>w(VysqRX{n!3{Pxw$tX3E4_O0t84{0*FYEU2M64 zA_io0DJqtT)Ut^|6v0ZkQC78rfWQc9SX96b6&G*|OI5T*q%JUuLoIHI8XYMhROW=X zGj{sSpP6TV_~b`^DKBB{{P4t5Q~vAC z>bK>~%C(vGw`WS>>GYRFca|*Lxb&cQp>DVzw~ zuHvlI+JG?fG0W-N%R&n3vUMAOv{kbFx1L=RzRBM;mkTWNFlTA(kB7IrKPlso`TtHp zADybXQ)jNEt-TvAz8C)Hu1DF!?T_887Z^6o+@beH+AQ5M%9nr$@nyrf0YfebXrlsh?R{eBE&Saqa9QE}d_-OE2;g0~bb~ zx-70gxhQcSHjJ)riAuUG-(RRad=W(amA+bN6MuP^wH?+yqT$0DvT5Px>ugd;9lVue z8X0jE=YF`%>JR}=Gudr)d4|=+Tfno>#K7G)(5%oTFVyPsJ3<1@6pl53QoJ5oj-nh_m;(F z@v^CfhqCSpG^f*okhH_2@&3j9`GxC{=iY6rj<;nVieG+QW^|uiuY6HFKfJgqtN$5@ zQGRp%@Y$sJs8sG|(yi)jU%iCKJpqlv$NoD53;z91!4il8&7YcotThnE?g^o>AEpg; zJWgpfU`^GZ9eKm$0bOyts$KssIk?Go){t}Kf(Y^67<;eW+Jvt+94XwqyDzdoc&Ph( z|19=})#nxz##3VV*A1?;O}Y7SrIPn4uq)xmq2`rU>urt5Udra_e)qr2_1CIiv20`E zbX-4q^OC5UiyD5BT<&+r3v9a`xR#-u+=qe476Xl_f}!T_ayh<{+KYB`610=~oKlaK7S)QRBxs_F0O*KbAm{X8+{ z=@eghwX(N`c#)+TBeZTFo2Dr&zG+V|2`x1g1)BH@!fwEktT)hab4jHapX{u>yfQLD zWN9^HOU_O|=C|b|fss_o5LZIv7z6lrFDGh{H*?5}Yl(tq5WOc6i!6)8k$K1=XcDm4 zaMqq2&KjXr((73YiqdgEnj~KMD>^`2vW-YUpC?-VMN@VkXAz5U!odKnzHjeVz}Pvu zVoJS+G1&lK46HJow{=fi92i&#)kpsCU8QHNq{#Tv(23gcIROkPs0$Wlt@y!aB@qD; z=}J{fkYmoi9sJp~1QXZlL}l`ck=>JIM?Gj^g2PTJ7!~mFU`C>sjNwk3b$^Q1w5eYv z-*s8(U3=DMUG~X1jNM_;|BMs-j(>gl&7PP*LUS8gW5X-jco{r{&R6Or#WjuW z=cX*kBoHdI`x)U)=`~Q-0Ei{tk`>@F>9^P^}jL zPJvIMVlN==JM`V6NyRJIDteMvlEzGoS{$|Biz85vXKSsc^wbZTvBfip_L7eSlW*JZ zdk48n&sOU<@rl_(`qQE!e2q2e*svE2Rm{{Dm23-aXYn>q>TnX!{(g>GH1CNuEXEfR zJ?3xj!~l>Ei<32ou=VBkO=btcUKqPBG7D0sg$+AvO?4eLfR!kXY?>xpqH5eu1$&FaE#e`$PakXktw1a+GC) zjdxu0%#m)2C8u+H+}160+X_b9EmgP6($$k{VSauc&wimwOQGae2N6bI<)xp$ypAkQ zoFan8Sy=8kSP2Fh6!2Qh|PBEg)L12pmPzp4@{{p$S8A!26SBZe#3!Dl+ zN@(g9FWybkiG@~rIe|xZAdSu?TUw2?7QF^gRJ$It* zbocwc$vx?Rt0HvZTGc1?q2(I;HpW&V-Kcz!ZK_{0M1s|m@7gp5%h-9?n zKVve3u6jF;;V6g1VG!C|eMR3u7us8L9vS`2=;z*T%yE*mqg))9e#e2fCLR$}G6YOI zFTlrx8MJi4&z^5YtM{G0aF&XDlW_-9h0+7*j-6NC73*8PG-%v?S$wtm>YbNT3bLwWjMD8j`^bJ4-d%W;JLid9=2GyzF?)Tv$U6c&?$I!DEjRbf6jNT*JU} zAO_Pw3aZrVb08@lKqyEs1bZyGKEWhtrK=M6iToIJ25GCx1<6Ko8H}~+u6Ofd67Ge= z$wj)&(sL{TE=uo(9@$ioOx^d81~{~R!)ZArwGVj1vj9BHiHG);81Tb9Z-aF-(?Ob) zM_%(Mr|b{MdUDMB!?9AI-v?h%(_!oe{XQ6hV)fXZ(J%oC7b?O~rQd|qe*(9ShJ_{b z5k=IA6`QKhbR0eURqrs{MMjdrdFK*lhG=~UkRu1}@{I^9Q9`R&ugB=C@l|$+>EKT@ zllS+bK6Fu0zUr3X#J0KKlE8NF;PPHtZ^Wi!O&4GzgAwRw%L8^^)P^iwSa8zPN$@Io zj1v?FzUDfU(fjGn4!Ogz{TO=>#%=&|DyRqLAP4CttHcPQMk2_Tr$>YZOurYe-bnN-Fpw zj8r*7#k~r`D0B9w#`rOG_Ev6tqP}q}Bu$!f>dgFCiVv^3h!A9z2atg_r7_fAhkU7{ zT3AleX`k!N@0fEj8u1gzVKq8Fo{z)E0)!n1L-}x`z$;LiEK+zUAXHsT;;Gh(ypa_e zoxG9b&l0!ls<&ysyrYMf6v{I-YmoHol7sUzgfskbjcv4;Qhnc_hb*Nt+P!t=GlN7? zx;yfzT|J(91;sDYlq5Cds+5ff`XFw4FAaU{i+3Fyd?G|Q%B>GywN39airY*zl~7G{ zsGbL?rn|u)qGD~6F#tUag!2wq5|Kq&1g%{ePUq5Uc!s=SS}t5=m8Z>wK(!Eyjv z{CEh77ec~7wwXga*)RQU35=be5MjTq$K1u?lFNt~C#!K(ss$K8Ns(etai+?Iu<}!N z#U{zDNm4EY=>h?~OIa39nW~!DW$WHn7|)3-x$#N~8wlis^Pd2Lj;ERnSYu-RZ7`%! z{oYFh5TlZP81TS?USk9myD@fw{G=JMnz`d%0uti*{*??#5fZ5k9l+3;c&-0TO3*;| zJ0(kD{)+-OC^#}Y5L(hG|2(}eMiS9F+!gV!v0&`bhkOjSIq>>$yidO-w42%EInsO} zEe1K4Y&A^Ffig+;4bq!;3QKC{nDsBPy{?6j)W@VehqXbwA=*6fRGZf3+D4YpCYk9pY5e9 z%7<607%-2aJq@gw2vtfP;R3p=;+NsN3V9QIsU{sVr201^lLW0dKM=ID% zJ^DlnFrbUYRZ2g|6=;@X&|SVdr+j*Prk3t2ji0+s_m}^r#|*QkKF|A-H`Y@Tm{lzHp$S89Os{0HU|^;SiCb zgu@r9LzOzHlO+q0eM|tVD2`noC3EzE6{wMBbh{viDr6TKgLUBHP;B3gBk8}5!i@e- zlwL{r1)0-dP60 zk02l;Aj&wwzoY6|dgy~)pb=9LrWIIPiePh98bOTR(PJ?LD3?YMSvyNblK?jGHEjL* zZ4Aa7BE%>0si-unf(G(HKQtB?{=nIBVkQFaJwfg$%CzGm~OWm0PB=T4W5j!{uY&Vk26 zS*cXM^{nv@(C0C8PXBN3KUWduZW9@rL@kU6oPY&U>&^7%tcOH~GH}5Eu?Q;EG@yn` zA+gj|<5WHkqo9fe{YhKZC7+dzCf8rc;B1Yf0g)mR_yJG050?ddL1-jg7O7xdFU%Ka zwn0NQYLMljTNi0KcEwqFQl+Iw3}QX$CbB)BU#X1`akh%MDV31B4k*-}awji)j z5otPzbQp7V7*h-g(-ttK1haks%|}O0*QHR7qC7%5kBkrzT7T|p0?G!fp+d4##SWp` zcqD0LMhLAix6bImRcFA{lRWBul#oAsF>>!W>_6t@i_^~kky4UgKLjhOJD<1^QJ5$u z&ifj`As}2q04#E!;(JXYICdqY_^&@d%Dnj8gkOm=q_n^XiQ9A`57OT0%;_^tU&vxVAfAi+xy~k|E#0GNub0T zICf=#=+-@$C~3d=<%pS&%JfyO=}Zlli_d!V+TM5XzZs zhPr7kRHq67!U9XdP~jpf_))Yg?h=FE`#^+Q^m{D`pWBckHv#O4tH!|~bEl+8>f-k} z0ia>dFoT7%(WNB!a5Geq26HP4%&^!uHlH^4b)2q&s06H^!Pv_kkMih_h#nX=LvA^D!%6J6=`|QT~JC$$oX-lw#8DPhlz3tMBWlGlZdj2 z7hi&_KRvbq#vB@yp}+-9zyaa;tVBgH2vtUsLw3JG)k66sOpnfG`L5D^@;=I)tkb7j zw`yC;qvtoaPrkXZ7LPbG9&s-P@TdULWr%VeL|_d<0Y-7!bqTY8sgN!FBp`&>AfY4P z#XbhS!sRr}LSF@AvcY9ERp4u$rQ)kNtD=AO^F1Vis>P&g#Vyytv8zIADF_ucC-%$M zfl-E|Kg01y6B|^tb6`x35J`Sy&ZdHDP+XWuLo_L5VY?;t1~Z4u_{w6H(LUlM`Lh?3 z{}iU9XgK&ngiwDn$DT&mbxEGjrH>56aN?}>tSlN=!8erWEZ=nh z1sU++^3tyeqa1{7Md><{rc1cVe#+vzn5;C{4@`ZB=MvG*Mr<0042iPs7J@75NT&( zKksX5r#LD=R3Hf|itG?jq1~(8saon(P=sILh%t~v1`)nBp|}Bdt?xXvqYOLm8LXjbfodh!^?z<>97;Q@Ms#9}q51 z)q}8`Zkk{L=AoP8dH0*%!FG&K1$|V2V^`nCR{e%3&L!@A58_?HodyaYr6DL~w=G7> zlF}LhZ&*9;Om4>nFZn1XY9RtpP&=j#+{xhZq|F_$8sNI7YBA%WL z-xE|JQ`5Jv1_8C z;4#!j?BuCIIjM*^1m6*gu5RJ_*_Sz711ORvqX&8C>(phDS48M<%6h4>y^sS1!0`+M zAWAr(jchh6G3MeUYgXMAz{3#yL3+B@%3UUi*Z{LE5;6dX zCXtnIjT}z0~0?9slc^;)zFO4nn58_MX`rd^vJr_?c{Qc)|U41zLCmd@I#%TRnOT`Oo>5;9*^xx-_c^tpLbDB0E2MUPXT>qAErJ**&sfwLD(Tiohw^Z8}RIvl%v z$GS|BJ!BgaMWeqosL$)q&0Xs3o45wSW&*Lfs^h zDie!_LJSxeM~I^#qKF=%!#f;1(?Sb>w>addFHMGKoSq6cTbQRQy#5Y6O+O)Q0;M?x z-xg(btush~0S&P7U7#UVmE1@Lh)l!yJn?TfBG|utWOK~j?Lsn|{DDtLCwlbo!3NAA zpthi!ud}n8mZYBkrV}FpITY_QM0n_3TCaWZ_Y?{yB4-AC1!!HEzdl?F=Og$hQAEP?l270`O^OX6R1q*x4Kx5m zvt5k&P(Nvx*c^_RmntSS&{V93&jptr2%@OZdkv`V@VK7}36TTJZjdO4@9zNEfI6YxV1$g>fCjwMDnThhd=qdi(;8T`7_v!QYUA>V+=n5r2^8 zk>yI)w|RvMK7;gcBH4BhL0}Aniccuj3D8gx!4d3mHUT_H z)+PRm9kMJO3TG|f`fixDdEsTFQscBtYMKgMpBb`S)FAiZnf=Bs%O55`St-Ih49r;U zHictXE>8)74s6Hc)R(}aNdrtoe#YgZ_TV&G=fEc#9kz=ypuE46ybjMpP>vJ$mJrIz+g2fw*zTPPL z^UyN-XX#~!5VSEchcqcbmfHDcFF=T>%THelk~4`FKnShNNAtR7ka-0$DKTw*QgZXyEaGR0)*>>SlaXRLSIfl{>=fO43}Vj&2$f!e0x zttW8qaPixRU7cTGk#9A)SX<-lobN#rdm=CfzSF&!odcNq|Nr5PkKRHm!bZbY@IeIP z%kF_P;7;<#@esrw>~NW-o736^fbFPF1l)f`a7%ru8+^70+DeZv>54q{FZ#v}gv9IErwBNzqJF0-26PQh5 z0LU(R$8B9uT6aSN-BGuf4ojmABiE;%f7q-% z0oC<&87%6%ket5M7EEH4@eow*4cO^xK;g&PzHRxATmsz0yMgsNk`Ta{T;tb9?lx9G z^KJi@g^BQLpr*f8Foy5D8uZbXKyIGc@b>IhpoM6iY#sXyA8GoIXsKfx#&1~NUV!5z zFq4UR<30O~YJgl~iX`&0EN^Qh*x{tyr(6QwcIBl%k14R`6xnVl+rR8b-|+QpFjoxy zDCEQ6f!1|GDix%En0-JJ29y@537+l|l!L@iGY}X%L!7za3q#O(n`*bdQ2B^&9RxEw z;T=lIDxMycXpTzhFIAad+J~@TOVj(~8`n7Khb$k*u1+W&v0e|566Sxd=uf9s@kGq0{Q(fx7axF&9kYk+K z@-^5=3LV5>fyC^j%kw!pwkZ#9pLIF$FX13IiySWRzhWu7bF#EwW@90Rzk2G!^4+Qd zP$m9qaMHm^_g`qpOaZo5v&`)HbY0b0g;vc3b7vs-x*W>LLYKH^Y1BX9hZ4mQkBl+d zo_P@0mQHzNZ?@M}c*kjIzhdt&d!9Py{RS(Y&SP^Uq%=c!i&r?Vsw~o+`Q?i=Sc+=y zox#@g*S z`AL-xk_``IE2YtZ0$$17cwS}baG1XXrSEmv!dI0l@EYlWm0c%HK{KKM=99i&Xy=>< zAE$7Atj03Qo-remo;wFm?Q42puhK`i z@h4x0mvNO;C!t6+rPa&s6Dou_cCG&6c0F09dgAX#zBm^hC(EVkLj9IXnx}oz5jOD8 z`Fgi7pb^}8BxAXeHwiz#F~o`4Lkb%!q{P7z4N#mG01GMn*(Rpmgk2n&TyGZz`R+tA zCt@r8DRN25^x83gC(B+LE7;+XC**wGpW(_I3#AEfzs@JelW`9P004tf$YyO;r& z2Ql&(9FPV@MzZuni7ueQQYtS8Ld-5BdZ14GJ-frz(N&NI*30Z!r;ceLICh2~A%i&g zt6n}u=uOBR+F~r$7^w{oFPF2M-+ztBzj*>}Hh%0&MirH|il`ai?QIu5T*5oAyxus7 zz>w~@&qE!Am&W3|pi4<_(+07dH+jU7Ed5^Slal^s&Biy6{5Bisw!u$qkKF~k2=&>q z3*;=_bEZsni3$3}z`koDjC|oX4=M-UvhAqSbX0X>(y0{IS28q2@3-S(O;f;n(jlMPX? z4x@YmbMt6mCm;TXgF7mYmskUP%j8={;A8)@qTYAC8-(C>`tHYbR^r#<41K~4!JBg+ zrUR%Q;7c0=hIPdHe-F=p+0|~oICg?q`4{%bUgE(sPnkOX=<$H@cjEa@E9Aj5R8IQx-I?_m^r#93V&KGC&-(6o#NDq%P-S4++SEXo5L~BUhr~?6pNV29%k!0&L3^hp_;b??zZ~-O!{ePrVPS9Jx(szU zp{2hIsNy*24|-F=%OOp2J|#-0(2Uz6^F5vSti7!;Uu%rEe{E)diyQ+rGT5RY0A7_v z2+@`>W&-#@DC0Br%YEhenA+n9)Y0;Td)_{l^bA2>3o z37x=aa0Iwu^#_fy$xQf55fq9fE}Y&ZoS*ks*O(p$B~IaobQ%OSL}Ic}3HnH4st(kC zC^@TdNGL6>TyVXBgF-*O4QUm!WB{o<`qKFfprzxxY9NS&b^>emnzAP}Tv$LD0>4O@ zN8L*`EEei~Oi1s+` zfmCAWh#24vk&DjeqI_pMKDn+W#AArF>KTruh=w{tJ9Xc;2q8Y1I7nn-i?8~2eLD`U zP_* zWmn1JPS1f(Lh|-TQ%fzK)8((__3kNTzTK%XD(`U9^st8CNfXb&K9{_5!3a_;_F7aM z)3et86nS6z5@?;Ec6IFHQQ^1dSj3AD<2?Z+Uo)=gl`&&srzFx$U%c~4LdDcvi=e##<(qEh}xW)YVmN9!xJZI%1+? z@zweM)f2iszU;IX#qf-tyX=o@(g$=LCw_argqSKjk}e)K4r=(Wev0Si#8Ww=bOz)v zx^*o$wqdghwrqhNP0+R6ZxBfrHqLO;bYDC#J?NwLrL=b*k?s!uY-t-h_DiUvsm1+n?c_dWYFx{NfhjlAXGD^xLkS=KpY*kr3kwW&+E6uRl^ z{Hue}InvpcEi(Jc|GKDG)%c>`56@}ppf}x^lhBG@5a)N8jq?pt>aJNMG(r*~w*QQM zhRNgB(ude)<%f`~g=zxfjGo5O%aGB#OATMkbQXU5?WC-#TKW|SexA2s>(kH>tsr{5 zoWZ{GMfr#4A$yLs#N*gm;ZhEdXK@P$*Hz3n+|Rr+A{arF%U=Z4?CV+E+qZ5N#Tr`I z@+@>wM}*MWl@~8P0_Q$w%0w}k?O!)pVNcuXw-2=%k60$u!1(kE+|?k-n8hhtYzYL?#+>raz69a5!%#lAL- zXLr304bIz>l%_T(OeVV6sO7xpu$|h%^6pPEo%Mc9r(9zyLPLWbDM zvTZZFx4lH(?f6% zxw_&wt!!h%&!t^s>fU0dj%-)1Fn92P!n1IpHMTI$Iw9n&m#=?g{_OxG!n^*(i>0L0 zxBDIMa|!dFaXw5TBiOZ>?%Ca-F`)C})7~KJ5_jOW!jzOb&9RyXEYwqmE6 zYb@p8_g|eI5wy=ygJ8#rPOw-mSNrw=zhS&t?{-#ae@OWJvnOTz?{eQSww+b$v7`Tc ziTV~&hfTN9OG}OU^v;TQW7}9qEuPm2>)Vv!MdygEw_V5V!3pCtgR2_m2Ub5tpN`xH zf&FcgGfCX=d&=tc>Mvj<DlebAXTyvCnpdZ$L-1!y*PKlq^i4NR5}GG{n|j&1>uux~ z8@HQoBP4aU_2`}Ok~<%Bj`$38^l)FZs~;Y7oJeh0%zs0UuBeuTv1%tg?M-C;bQF8D z?K5%)+zv1j=gn3Sy5-u$hPrt${A{e83-(w}beS8OI=P1z$hAf`MPCZMbnR98ZXCO61mI}ItCrMhhdUl?y5Lp~ zM6+s4SNA&9miOJWj@bXVyi(xGGyh(z%FHIsZv~4xVm93f`qcM24ee}rsLV1oA6QnV-W#oEa{(uZnur?35_f<*Cw*L3*(W!(z_NPxSH$E#1CGMlvNnqz(8XTz|=r(yej&+^vk78N`yN$!eFH@M>1 zcI(iCJI@{r-(99+$Kkz;+)AC%cuueI`D}XkW>`7&AU`x$wh5XZKJ!p-S#YXp^x0S6e&y%Dj%xpoNcI=aF%nk!e8kw(=OtY zisP{~C34VT@6^fZS~a<`BYI-s`=`4OpYHr1EQdplUBq$)6DxbpAkn6+6jJDFhWpu@ z*L@zOmoEu;^HHy^>al}Ryvg6u?y1|#m*y+{4hT}xkB5wB+>bxaTwErp3B$3g2VRHd zf)qv;7w!wz2|Zc#xV<0eJh4V+^jlw(B?sz3>R!+|vDwTTI`mwv*+y1+6wcddE_J0( z&a4q@_U%fMZdnr7$97C=V$fjS3kH3*UVApI$h&^lIg}VRPMTP9%|H0joW<+hj=B8x zPE}X$!53wIBRH;O<)*t19|xX%-PM|vQsJ*^MT+0+!E(L!7ogETIvY)FAmm6GbNC&1xZ#LsYy|BA(B>quMU-R8Kjx-Xq-UZsZJ zHh2iEg!3;3bUdt(a)zn`@1*Jvg?%Rp`cG0r^M#fE)|NQsc@}kcbQ(Y3+8m2$XR-n=JNb5VBCScAE*It0q%Ev{+bhU>`Ogga2d(4%U+}~& zla1#%+3uGdm~5n1pgV6_@+qmaPK(4icFnVA{c7I|5?-oobM2$~`@Ii$Ft$l8eA+%v z`?lb%Uv1WyctLO2eE6DLRJr5)`diN`?OXP(*2HFZ+bHh2Cs$B@jcJaF-hAzcA}BwW{hD`_*?$vmcHfw{#P3IDBX; ziJE0=SKH)&dWEr*k0%swuvK2#$Vc;h_&I=#wHH;bJ9bF=Epj<^7etLB)j!P^&WLUP zxwLvrb747iWM7q2OjGy)!}Z|@kR1C#=D_7Hn;;LVy*Z=()YJTq*7SuDa(>x;E7s8L z&X@PZ2kGUo0}nUp7T1NnD>AS5{p6o(AKPG;NJ-z8#;Ck?kLmsDOvuoe)wFf;#oBV= ziIvQA0V8PhOtIlD*s7%{>m(g^(ZOeJo9S75>6Jgv+JeXUtnKE3H}(MB{pVR58wE*B zL=7dA5$F(xUqxWXb>Uy|LE3TW4;2aikjRX?4vX=FH`4=iTpv6NrmrJ2GtoEsg3d&I zAePk^43)MLM3l?>JI6_`-BI(0o3w?YNA&I3G!HA4zR`k3%Y5-L1f+aj3+Kj z`bzE<2kh>==Hu*Vw!Tl@t@2Pzy8M%2#2EE z9+T0zP=!O(ZqH3ldej{ak9|NQz-WD_pRKWg(cu!Goc3IEXzwdNf7Cop9(ikxBG96+ z{)L=?ILoP@X@y7h9$_@x{-?_Jh-jE1idkA=BK*7tLb*&cVxmS+ii?Z0ub_&(GH%v@ z*&*rr%uG*sha?SDFbAHZ-63W^>1m_*Sr*6+;o)<#H@zoL26GOO$_S5o%`=7 zY6YLCN#<6eY^dS4^edwmEPN{vBm~xwEO+SmP0cRzSs1=xt zFx{7O240118_5`hz2f*lr4ZutnQ^#ky&#vZFua=|HTA`^aHT=yO*hK^DwNg+uId4Q zZvuW8E0qO|^?oTOve^1`9k?0CZeQ}BHEaK$h*|&sHAIQhcS@Xe!bt-6gf?g*4~&cPZ#sF>do@w9-6HeeVGB(qH(R(xmIh#! z9i962vor>)I-EYY!;RzbBlo0ui#~)wKb-Mw5!X0rV^dQTEO42K_2KcZvG1Ew$5#rX zjN&=rx*?9Q>h$(53&i5S5%hDYP-={7Rz9G`ghRO&t~`yj(2(T@KOfxLP15OFuN(TNuQ2mBxQxj6c)1i*bR9wcyiBrI%GISE4I8unnyEui& zhGPw8b>b$ELCLi5p*g*OI5QS$Lf4y*EQZ}zI~|-%GPs2*F%y zx<}HSdn=DCl0C8Cr^a`)Bn;fr8I^%i_`6z-<)tTYu_*lfQ9qI6Uj(i}0W_RR{*V#J z$%@l5Vn!2<6xi>5>Q>yO!?Eqh%e(02?y{*BF?x)S+v}MVc;p z<93^m4hS-T+OB%fD{Ez4)O&)wU%3Y5@JSC;uxC|3#Ya z{BxQL{QM{&2J+>h+M|)Ujp?8B<>(^wz>601j5Vhwxp-@`R(;bM**CMxB8(~hD$BQS zEMt%=2_E`vP34^|ff%gCy={^Jb$=hl!G2kv7;4Z=x-+ z8`Gsf@o;r+06R!~Rno62cIPs3SFL_`DFdB$XrlstUaBKs5~lCaMt2%7s4P_I@v6z% zR*<^E%;CA*4?)*uPFvm)w`R4shN*Ch)NU0-zQJ3Wc2U@VOv9BY z!zA`EBp;o^lN86!!V{Hlt=|J}ARmWv7s`|8pDKcExcVRDH0v!_y?*>6b=A}$)=j{q z#)^C;7rQNtT13yL#;Cy1?=@g6zQxjW05N~$s=Xd+0hSU%JLU@z2B4yH zLS79S7(oMGdb>(hXR?bW!87>|c(%^=_4)pIS7+$JRCE}9NxzG<8uEWZH#j= zG7FlGOw0ZJK@bjAlBUuwK2m`BwA4ju)82ssTEX43W>h?iJF`$TcWNl1LG+}_23B}NjpQlcV;NGKdt7|7#^4S_h9+;U6_AatjmSR z*XBv!FNKSODdH_v0&#RA<6u|n=_x!_Yf5{h;RZ3(=nVlWY~wD3uA@_~d?{{VgR&E< z!~!{UQ5s)EP;7hzPx)io+V7&!&BuW?s==WcKr1u3@?-#IRNYmxA`xi5?znha`daU-L^pEp5MG<7H!+`KCWgN z{a*TY=F7d75uf}?Ri<{lWKCD8UUE*+u&+}2Dc8{eD_^Cy*|uv3<2+<_yPtM80XwD0 z$I%U6^;9;{cjvC4KD5xrrL7Z(%^HL>n*_l!eCCr+&Nf;hR}$UAm?ymbG;96h$`uh# zQmG9~z|*<;K9!8p;H;aKYIRGB|E8S%Lysx@CEhwbRsVR4W_8!0>FUjiE5#G@54y@~ zHVkuaeoWu!GRU`*>WZ9byslj$#_fS~ynTJ<1tvw2q0i7$#Ty=8PT9vCtZ8E@cMy}b z7jDxv=Iy9h7w>Sv{qSp5M_yGB|F&VU?;>sKqx|z@ie7fbRV%jbzFnB|-NX4oQQ{mN zJF`=GFkp-2ahamX{?P$f%JUTK&NI&t_A{L|p_*(tOS1KXAVXaJ7jznrt-IY`cP8j- z0CAjb_;}4;Ig_{R4jp^^&(g*5l3&UhuOrs#hDl@ZiciW93HHwWWm;C;f0;X~c=J@v zBF)^-5rp5=o9*IHtGrK1$x%8@=^S7E^o-=IYm8oGmAP_qV4hF-^7K>E`C?-->F z7Kzn~4nn!5#8B&mVx`7j96Jjgfv%&0A7kW?WGOVp6thI6c&NuSiTL8?8lmAcS6gk| z9>6>mY0VCq2le!?JYwiusou0+q&j$~-OSzfE8$zp^8;6Wu9aCEJCz>3rgt~Zvi7ER zM9EE=2Y=N#@f(F}oMX5i(-mxaBREEjA9d-}+9Nm5-I>&voyVJ9rrA?j8?U{P741|T zkF9(&`^^(4H+aya^-jYF$OjvCNmn+O^Q(X7R-PsE-3iC8;FIDXRCmPl?pSP)&bo=L zFoA#5Ak#23+?tlBJ!Hz0a_&v-q3>PvaKLY2l#z?B=UX`)^LK7xA(1WJ5zIlI#%pE0 zO?oAUoW3fVbG|v)EqmhtWyhkF`%l*PH@z&WCT5C)U)4*6XI^dSIpTiyq>^AG_kZg# zf73WmweG=X_FKW$9kN!p96sOL9xFMLKwB@Py=2C3h4%s-wB#43bZ0fi$RBu{npY~o z%>jQK9J``)O`}1{m1b)dIpdh)##ZcmOeN77fkNr)OT}6Hr5dy4*WPJPcGS2=sSzGf zi;~O5($E)N+#g^>sT4OB=?vS>c3?+p=ZUhVuKrdM)__P{(p?qfB1ey0gPtFrccRET z@tEx<&zZfsC2OxKid=C4rgn8yac%qaVJ8EnOStl%tygv3<$TOimxKSpJD9#x=2O8M?0?^>e>kT&_G#~+{)SIn*R<0!OLBOr zaSpugz$MYaPqGWsFMCVQ=t?(*?8QRua+2jZr_0km?&%@zh0;Z%U$(WI9?pAWq`u8C z&C6ZfJ_3rxdjo1}d)6ku3lC)ZL+dOmQtDksb3+<O^u4K`Bj z`k1RWv=y@~53q&HOl=F&Y&=r^Jz574bKDTUubO{<#A(@rH{j_T)^->BnxN>MBcApQ zS+qR!lgt@`@25eg(fV+PXTI8f?}V9)hnd%Dr-eh`3LRgT&xS%`JS^&Uu~!`23=O@q z&noL!EC&(v&d zgPI+FyHhPiJ*^(LT;8?uXHKOOc1Q&`JcoUx^=&0S7J-~wIr~prj)uhzBzR8D?AIOO z{bf;QxXWWUx%q|q#I3Dk!_gmZ*{8OYzN)G`q9jUg4@&h=p#|?<;Cg5IwOdMs?RM_G z#lE(B(!9P%L+$%l#hqyy!ipqQ?hV`x{Yv^RD|#ZUnj4pmu00@*G=8t^CSDnr(p^&I z|1#U{c&5kpa&hOr**JE!SLSX(Yt?kT=3TcW>qNczU|}D4+v-UuJw9*m895s1(7i46 z6SsX=1g$DiN*Ws%Ra%)?tK;IoeuDPd$4kdrB|R0m_z&k*XsT?hq~BSRhW@3g_~l$p zp8e%qCEPHvJcpYDIo_V~gX^}22Ad{pslP9mey?;dS6b95na|%=*Hoy`xuP~vANSNo zxH#G6Ld&BYOV`eR7tz7m2)zcKYnvK&SH{}n*wtfb4+2Q1uHHQCBd1xhlY!hb63Kc$ z>DIS$_nP?;hF$j)72{Oyo%g=^+$+aRNXOlQfVSO%9WM?iwx61B`IYc}pN!%;x~lgH z(AeGkocU&^q5p{GM2WUjKCH}%8aTBsxY6iNZzoOeUzV!L4b$dn%*Go*Z^@MvrwxsQ zZyVjQ=aGwf${n&^&B7)(IfXie7l(VHHg~uK*L?@8@G1H!VX_~`uCcf%-Sw@FUtVLn zO%MBLn39NYvPox;y*w$Wyz7%pRFP#}m${R1q*DB4+#IN`99*7zT`Du@@}(e+agE_K z@_1V9RXiju5n5C`zv|uPZZ^V*$ zmzwn+${g-|tmc0W z8@5aKGZ+cI78>tQ6u-A+D^2^M<>OMXu4`^S^?1m3)y>_(=UO9TKvH%a^4taYL$hDK zENSR@?%3_`vk!Nsn_DUt+sKtAcbX=fCzZF2J#Xw)JCEduBJD}I*Lj9|TZcBWZ_>H{ z`oCP%q}km$xF2GHW1mX^Q@DjsURzPCU>vlMOthzKTtk(5vzH!9@LN1#7>)m-P`_8M zwQ3+((>iqBilnGzi4WVvlm;)g?U+2zdgVPg=RG#eS@h72-%fQ8l!2F z<5-rq+4%f*<{~SbI1REYmmb&cpOB-z_nETfd2p3vz@B{ zbG6jJpBICwf6aXIa*zK~bMW&z_<1qB>0bY9k~q`%yCN&)+F13eOPA58o;dMW-R(0- zRZA{7G?CwuZd0x^-yhjGufQl}h+SV6U7W$nzx~?$s9Hg3Z{gFC>qy`yId1-pZS^|R zCnzK_)E&@82u|-RG=J{;B?9kpyjr72OB{v|>T@&R3H2V|n`bZI^E|qN;ck`qlpSGM z;Tpv$v&|GUwR-jL;<+lnaZ%L`>xudoGJdq}Nhd7Kv)9#qGFcwQHV%2vKk?3c?Vc-h zHl4d2Cix@jXDBfKcCY}KNI=)EHEC7XUpA&3DeZ`gx#XR6*uyswccfOgC zjW1WtJ{b>|TF>lPNNL^BVG_`ZvW#!fleBBP`?F6T;p7*rvrHtcsR^%I!pf2E3O{6? zd!p~%@V47^wd(@!Tv#jfVoS`0wL^CrYuBx^N_pLSR?F%<{)5KThjHxYoBm650>bQ+ z=Pu4&SD~4=v-fQL&9ZZ8)oHGxA$s-w`QNu}9)8|@eO+L`Rk!BnbLV0{EWLt55Bz$1 R>%)glUr&bN&}+B#|1bQU^M(Kb literal 0 HcmV?d00001 diff --git a/internal/protocol/hls/testdata/olemovienews.m3u8 b/internal/protocol/hls/testdata/olemovienews.m3u8 new file mode 100644 index 00000000..6c908400 --- /dev/null +++ b/internal/protocol/hls/testdata/olemovienews.m3u8 @@ -0,0 +1,283 @@ +#EXTM3U +#EXT-X-TARGETDURATION:20 +#EXT-X-ALLOW-CACHE:YES +#EXT-X-PLAYLIST-TYPE:VOD +#EXT-X-VERSION:3 +#EXT-X-MEDIA-SEQUENCE:1 +#EXTINF:20.000, +seg-1-v1-a1.ts +#EXTINF:20.000, +seg-2-v1-a1.ts +#EXTINF:20.000, +seg-3-v1-a1.ts +#EXTINF:20.000, +seg-4-v1-a1.ts +#EXTINF:20.000, +seg-5-v1-a1.ts +#EXTINF:20.000, +seg-6-v1-a1.ts +#EXTINF:20.000, +seg-7-v1-a1.ts +#EXTINF:20.000, +seg-8-v1-a1.ts +#EXTINF:20.000, +seg-9-v1-a1.ts +#EXTINF:20.000, +seg-10-v1-a1.ts +#EXTINF:20.000, +seg-11-v1-a1.ts +#EXTINF:20.000, +seg-12-v1-a1.ts +#EXTINF:20.000, +seg-13-v1-a1.ts +#EXTINF:20.000, +seg-14-v1-a1.ts +#EXTINF:20.000, +seg-15-v1-a1.ts +#EXTINF:20.000, +seg-16-v1-a1.ts +#EXTINF:20.000, +seg-17-v1-a1.ts +#EXTINF:20.000, +seg-18-v1-a1.ts +#EXTINF:20.000, +seg-19-v1-a1.ts +#EXTINF:20.000, +seg-20-v1-a1.ts +#EXTINF:20.000, +seg-21-v1-a1.ts +#EXTINF:20.000, +seg-22-v1-a1.ts +#EXTINF:20.000, +seg-23-v1-a1.ts +#EXTINF:20.000, +seg-24-v1-a1.ts +#EXTINF:20.000, +seg-25-v1-a1.ts +#EXTINF:20.000, +seg-26-v1-a1.ts +#EXTINF:20.000, +seg-27-v1-a1.ts +#EXTINF:20.000, +seg-28-v1-a1.ts +#EXTINF:20.000, +seg-29-v1-a1.ts +#EXTINF:20.000, +seg-30-v1-a1.ts +#EXTINF:20.000, +seg-31-v1-a1.ts +#EXTINF:20.000, +seg-32-v1-a1.ts +#EXTINF:20.000, +seg-33-v1-a1.ts +#EXTINF:20.000, +seg-34-v1-a1.ts +#EXTINF:20.000, +seg-35-v1-a1.ts +#EXTINF:20.000, +seg-36-v1-a1.ts +#EXTINF:20.000, +seg-37-v1-a1.ts +#EXTINF:20.000, +seg-38-v1-a1.ts +#EXTINF:20.000, +seg-39-v1-a1.ts +#EXTINF:20.000, +seg-40-v1-a1.ts +#EXTINF:20.000, +seg-41-v1-a1.ts +#EXTINF:20.000, +seg-42-v1-a1.ts +#EXTINF:20.000, +seg-43-v1-a1.ts +#EXTINF:20.000, +seg-44-v1-a1.ts +#EXTINF:20.000, +seg-45-v1-a1.ts +#EXTINF:20.000, +seg-46-v1-a1.ts +#EXTINF:20.000, +seg-47-v1-a1.ts +#EXTINF:20.000, +seg-48-v1-a1.ts +#EXTINF:20.000, +seg-49-v1-a1.ts +#EXTINF:20.000, +seg-50-v1-a1.ts +#EXTINF:20.000, +seg-51-v1-a1.ts +#EXTINF:20.000, +seg-52-v1-a1.ts +#EXTINF:20.000, +seg-53-v1-a1.ts +#EXTINF:20.000, +seg-54-v1-a1.ts +#EXTINF:20.000, +seg-55-v1-a1.ts +#EXTINF:20.000, +seg-56-v1-a1.ts +#EXTINF:20.000, +seg-57-v1-a1.ts +#EXTINF:20.000, +seg-58-v1-a1.ts +#EXTINF:20.000, +seg-59-v1-a1.ts +#EXTINF:20.000, +seg-60-v1-a1.ts +#EXTINF:20.000, +seg-61-v1-a1.ts +#EXTINF:20.000, +seg-62-v1-a1.ts +#EXTINF:20.000, +seg-63-v1-a1.ts +#EXTINF:20.000, +seg-64-v1-a1.ts +#EXTINF:20.000, +seg-65-v1-a1.ts +#EXTINF:20.000, +seg-66-v1-a1.ts +#EXTINF:20.000, +seg-67-v1-a1.ts +#EXTINF:20.000, +seg-68-v1-a1.ts +#EXTINF:20.000, +seg-69-v1-a1.ts +#EXTINF:20.000, +seg-70-v1-a1.ts +#EXTINF:20.000, +seg-71-v1-a1.ts +#EXTINF:20.000, +seg-72-v1-a1.ts +#EXTINF:20.000, +seg-73-v1-a1.ts +#EXTINF:20.000, +seg-74-v1-a1.ts +#EXTINF:20.000, +seg-75-v1-a1.ts +#EXTINF:20.000, +seg-76-v1-a1.ts +#EXTINF:20.000, +seg-77-v1-a1.ts +#EXTINF:20.000, +seg-78-v1-a1.ts +#EXTINF:20.000, +seg-79-v1-a1.ts +#EXTINF:20.000, +seg-80-v1-a1.ts +#EXTINF:20.000, +seg-81-v1-a1.ts +#EXTINF:20.000, +seg-82-v1-a1.ts +#EXTINF:20.000, +seg-83-v1-a1.ts +#EXTINF:20.000, +seg-84-v1-a1.ts +#EXTINF:20.000, +seg-85-v1-a1.ts +#EXTINF:20.000, +seg-86-v1-a1.ts +#EXTINF:20.000, +seg-87-v1-a1.ts +#EXTINF:20.000, +seg-88-v1-a1.ts +#EXTINF:20.000, +seg-89-v1-a1.ts +#EXTINF:20.000, +seg-90-v1-a1.ts +#EXTINF:20.000, +seg-91-v1-a1.ts +#EXTINF:20.000, +seg-92-v1-a1.ts +#EXTINF:20.000, +seg-93-v1-a1.ts +#EXTINF:20.000, +seg-94-v1-a1.ts +#EXTINF:20.000, +seg-95-v1-a1.ts +#EXTINF:20.000, +seg-96-v1-a1.ts +#EXTINF:20.000, +seg-97-v1-a1.ts +#EXTINF:20.000, +seg-98-v1-a1.ts +#EXTINF:20.000, +seg-99-v1-a1.ts +#EXTINF:20.000, +seg-100-v1-a1.ts +#EXTINF:20.000, +seg-101-v1-a1.ts +#EXTINF:20.000, +seg-102-v1-a1.ts +#EXTINF:20.000, +seg-103-v1-a1.ts +#EXTINF:20.000, +seg-104-v1-a1.ts +#EXTINF:20.000, +seg-105-v1-a1.ts +#EXTINF:20.000, +seg-106-v1-a1.ts +#EXTINF:20.000, +seg-107-v1-a1.ts +#EXTINF:20.000, +seg-108-v1-a1.ts +#EXTINF:20.000, +seg-109-v1-a1.ts +#EXTINF:20.000, +seg-110-v1-a1.ts +#EXTINF:20.000, +seg-111-v1-a1.ts +#EXTINF:20.000, +seg-112-v1-a1.ts +#EXTINF:20.000, +seg-113-v1-a1.ts +#EXTINF:20.000, +seg-114-v1-a1.ts +#EXTINF:20.000, +seg-115-v1-a1.ts +#EXTINF:20.000, +seg-116-v1-a1.ts +#EXTINF:20.000, +seg-117-v1-a1.ts +#EXTINF:20.000, +seg-118-v1-a1.ts +#EXTINF:20.000, +seg-119-v1-a1.ts +#EXTINF:20.000, +seg-120-v1-a1.ts +#EXTINF:20.000, +seg-121-v1-a1.ts +#EXTINF:20.000, +seg-122-v1-a1.ts +#EXTINF:20.000, +seg-123-v1-a1.ts +#EXTINF:20.000, +seg-124-v1-a1.ts +#EXTINF:20.000, +seg-125-v1-a1.ts +#EXTINF:20.000, +seg-126-v1-a1.ts +#EXTINF:20.000, +seg-127-v1-a1.ts +#EXTINF:20.000, +seg-128-v1-a1.ts +#EXTINF:20.000, +seg-129-v1-a1.ts +#EXTINF:20.000, +seg-130-v1-a1.ts +#EXTINF:20.000, +seg-131-v1-a1.ts +#EXTINF:20.000, +seg-132-v1-a1.ts +#EXTINF:20.000, +seg-133-v1-a1.ts +#EXTINF:20.000, +seg-134-v1-a1.ts +#EXTINF:20.000, +seg-135-v1-a1.ts +#EXTINF:20.000, +seg-136-v1-a1.ts +#EXTINF:20.000, +seg-137-v1-a1.ts +#EXTINF:4.111, +seg-138-v1-a1.ts +#EXT-X-ENDLIST diff --git a/pkg/download/hls_live_test.go b/pkg/download/hls_live_test.go new file mode 100644 index 00000000..7fb41054 --- /dev/null +++ b/pkg/download/hls_live_test.go @@ -0,0 +1,97 @@ +package download + +import ( + "fmt" + "os" + "path/filepath" + "testing" + "time" + + "github.com/GopeedLab/gopeed/pkg/base" +) + +// TestDownloader_HlsLiveSite is a manual acceptance gate against a real site. +// It is skipped unless HLS_LIVE_URL is set, e.g.: +// +// HLS_LIVE_URL="https://host/path/index.m3u8" go test ./pkg/download/ -run TestDownloader_HlsLiveSite -v +func TestDownloader_HlsLiveSite(t *testing.T) { + playlistURL := os.Getenv("HLS_LIVE_URL") + if playlistURL == "" { + t.Skip("set HLS_LIVE_URL to run the live-site acceptance test") + } + referer := os.Getenv("HLS_LIVE_REFERER") + + downloader := NewDownloader(&DownloaderConfig{Storage: NewMemStorage(), StorageDir: t.TempDir()}) + if err := downloader.Setup(); err != nil { + t.Fatal(err) + } + defer downloader.Clear() + downloadDir := t.TempDir() + cfg, err := downloader.GetConfig() + if err != nil { + t.Fatal(err) + } + cfg.DownloadDir = downloadDir + if err := downloader.PutConfig(cfg); err != nil { + t.Fatal(err) + } + + req := &base.Request{URL: playlistURL} + if referer != "" { + req.Extra = map[string]any{"header": map[string]string{"Referer": referer}} + } + rr, err := downloader.Resolve(req, nil) + if err != nil { + t.Fatalf("resolve: %v", err) + } + name := rr.Res.Files[0].Name + size := rr.Res.Size + t.Logf("resolved: name=%s size=%d", name, size) + + taskID, err := downloader.Create(rr.ID) + if err != nil { + t.Fatal(err) + } + deadline := time.Now().Add(30 * time.Minute) + var lastLogged int64 + for { + task := downloader.GetTask(taskID) + if task == nil { + t.Fatal("task not found") + } + if task.Status == base.DownloadStatusDone { + break + } + if task.Status == base.DownloadStatusError { + t.Fatalf("task entered error state after %d bytes", task.Progress.Downloaded) + } + if time.Now().After(deadline) { + t.Fatalf("timeout, downloaded %d bytes", task.Progress.Downloaded) + } + if tick := task.Progress.Downloaded; tick-lastLogged > 10<<20 { + lastLogged = tick + t.Logf("progress: %d bytes", tick) + } + time.Sleep(500 * time.Millisecond) + } + + data, err := os.ReadFile(filepath.Join(downloadDir, name)) + if err != nil { + t.Fatal(err) + } + if int64(len(data)) == 0 { + t.Fatal("merged file is empty") + } + t.Logf("merged: %s (%d bytes)", name, len(data)) + if filepath.Ext(name) == ".ts" { + // MPEG-TS packets are 188 bytes and start with the 0x47 sync byte. + for i := 0; i < 188*4 && i < len(data); i += 188 { + if data[i] != 0x47 { + t.Fatalf("invalid TS sync byte at offset %d: %#x", i, data[i]) + } + } + } else if string(data[4:8]) != "ftyp" { + t.Fatalf("invalid fMP4 header: %q", data[:8]) + } + fmt.Printf("LIVE ACCEPTANCE OK: %s (%d bytes)\n", filepath.Join(downloadDir, name), len(data)) +} diff --git a/pkg/download/hls_protocol_test.go b/pkg/download/hls_protocol_test.go new file mode 100644 index 00000000..8ef48df8 --- /dev/null +++ b/pkg/download/hls_protocol_test.go @@ -0,0 +1,172 @@ +package download + +import ( + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/GopeedLab/gopeed/pkg/base" +) + +// hlsTestCdn spins up a master-playlist HLS site whose segments require a +// Referer header (anti-hotlink) and returns the server plus expected bytes. +func hlsTestCdn(t *testing.T, segmentDelay time.Duration) (*httptest.Server, []byte) { + t.Helper() + const total = 6 + segments := make(map[int][]byte) + for i := 0; i < total; i++ { + segments[i] = []byte(fmt.Sprintf("ENGINE-SEGMENT-%02d", i)) + } + var want []byte + for i := 0; i < total; i++ { + want = append(want, segments[i]...) + } + + mux := http.NewServeMux() + mux.HandleFunc("/master.m3u8", func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte("#EXTM3U\n" + + "#EXT-X-STREAM-INF:BANDWIDTH=500000\nlow/index.m3u8\n" + + "#EXT-X-STREAM-INF:BANDWIDTH=2000000\nhigh/index.m3u8\n")) + }) + mux.HandleFunc("/low/index.m3u8", func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte("#EXTM3U\n#EXTINF:4,\nseg-0.ts\n#EXT-X-ENDLIST\n")) + }) + mux.HandleFunc("/high/index.m3u8", func(w http.ResponseWriter, r *http.Request) { + var b strings.Builder + b.WriteString("#EXTM3U\n") + for i := 0; i < total; i++ { + b.WriteString(fmt.Sprintf("#EXTINF:4,\nseg-%d.ts\n", i)) + } + b.WriteString("#EXT-X-ENDLIST\n") + w.Write([]byte(b.String())) + }) + for i := 0; i < total; i++ { + idx := i + mux.HandleFunc(fmt.Sprintf("/high/seg-%d.ts", i), func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Referer") == "" { + w.WriteHeader(http.StatusForbidden) + return + } + if segmentDelay > 0 { + time.Sleep(segmentDelay) + } + w.Write(segments[idx]) + }) + } + server := httptest.NewServer(mux) + t.Cleanup(server.Close) + return server, want +} + +func hlsResolveReferer() *base.Request { + return &base.Request{ + URL: "", + Extra: map[string]any{"header": map[string]string{"Referer": "https://player.example.com/page"}}, + } +} + +func TestDownloader_HlsProtocolEndToEnd(t *testing.T) { + server, want := hlsTestCdn(t, 0) + downloader := NewDownloader(&DownloaderConfig{Storage: NewMemStorage(), StorageDir: t.TempDir()}) + if err := downloader.Setup(); err != nil { + t.Fatal(err) + } + defer downloader.Clear() + downloadDir := t.TempDir() + cfg, err := downloader.GetConfig() + if err != nil { + t.Fatal(err) + } + cfg.DownloadDir = downloadDir + if err := downloader.PutConfig(cfg); err != nil { + t.Fatal(err) + } + + req := hlsResolveReferer() + req.URL = server.URL + "/master.m3u8" + rr, err := downloader.Resolve(req, nil) + if err != nil { + t.Fatal(err) + } + if got := rr.Res.Files[0].Name; got != "index.ts" { + t.Errorf("resolved name = %s, want index.ts", got) + } + taskID, err := downloader.Create(rr.ID) + if err != nil { + t.Fatal(err) + } + waitForTaskStatus(t, downloader, taskID, base.DownloadStatusDone, 10*time.Second) + + task := downloader.GetTask(taskID) + if task.Protocol != "hls" { + t.Errorf("task protocol = %s, want hls", task.Protocol) + } + data, err := os.ReadFile(filepath.Join(downloadDir, "index.ts")) + if err != nil { + t.Fatal(err) + } + if string(data) != string(want) { + t.Errorf("merged file mismatch: %q", data) + } +} + +func TestDownloader_HlsProtocolPauseContinue(t *testing.T) { + server, want := hlsTestCdn(t, 120*time.Millisecond) + downloader := NewDownloader(&DownloaderConfig{Storage: NewMemStorage(), StorageDir: t.TempDir()}) + if err := downloader.Setup(); err != nil { + t.Fatal(err) + } + defer downloader.Clear() + downloadDir := t.TempDir() + cfg, err := downloader.GetConfig() + if err != nil { + t.Fatal(err) + } + cfg.DownloadDir = downloadDir + // Throttle the protocol so the pause lands mid-download. + cfg.ProtocolConfig["hls"] = map[string]any{ + "segmentConnections": 2, + "maxRetries": 3, + "timeoutSeconds": 5, + "prefetchContentLength": false, + } + if err := downloader.PutConfig(cfg); err != nil { + t.Fatal(err) + } + + req := hlsResolveReferer() + req.URL = server.URL + "/master.m3u8" + req.Extra = map[string]any{"header": map[string]string{"Referer": "https://player.example.com/page"}} + // CreateDirect: engine resolves lazily on start, like a direct task. + taskID, err := downloader.CreateDirect(req, &base.Options{Name: "movie.ts"}) + if err != nil { + t.Fatal(err) + } + waitForTaskStatus(t, downloader, taskID, base.DownloadStatusRunning, 10*time.Second) + if err := downloader.Pause(&TaskFilter{IDs: []string{taskID}}); err != nil { + t.Fatal(err) + } + waitForTaskStatus(t, downloader, taskID, base.DownloadStatusPause, 5*time.Second) + + if err := downloader.Continue(&TaskFilter{IDs: []string{taskID}}); err != nil { + t.Fatal(err) + } + waitForTaskStatus(t, downloader, taskID, base.DownloadStatusDone, 30*time.Second) + + task := downloader.GetTask(taskID) + if task.Meta.Res.Size != int64(len(want)) { + t.Errorf("final size = %d, want %d", task.Meta.Res.Size, len(want)) + } + data, err := os.ReadFile(filepath.Join(downloadDir, "movie.ts")) + if err != nil { + t.Fatal(err) + } + if string(data) != string(want) { + t.Errorf("merged file mismatch after pause/continue: %q", data) + } +} diff --git a/pkg/download/model.go b/pkg/download/model.go index 0705827b..1a3b7ae8 100644 --- a/pkg/download/model.go +++ b/pkg/download/model.go @@ -9,6 +9,7 @@ import ( "github.com/GopeedLab/gopeed/internal/fetcher" "github.com/GopeedLab/gopeed/internal/protocol/bt" "github.com/GopeedLab/gopeed/internal/protocol/ed2k" + "github.com/GopeedLab/gopeed/internal/protocol/hls" "github.com/GopeedLab/gopeed/internal/protocol/http" "github.com/GopeedLab/gopeed/pkg/base" enginewebview "github.com/GopeedLab/gopeed/pkg/download/engine/webview" @@ -197,6 +198,7 @@ func (cfg *DownloaderConfig) Init() *DownloaderConfig { } if len(cfg.FetchManagers) == 0 { cfg.FetchManagers = []fetcher.FetcherManager{ + new(hls.FetcherManager), new(http.FetcherManager), new(bt.FetcherManager), new(ed2k.FetcherManager), diff --git a/ui/flutter/lib/api/model/downloader_config.dart b/ui/flutter/lib/api/model/downloader_config.dart index 6a0fc47f..dae6a103 100644 --- a/ui/flutter/lib/api/model/downloader_config.dart +++ b/ui/flutter/lib/api/model/downloader_config.dart @@ -56,6 +56,7 @@ class ProtocolConfig { HttpConfig http = HttpConfig(); BtConfig bt = BtConfig(); Ed2kConfig ed2k = Ed2kConfig(); + HlsConfig hls = HlsConfig(); ProtocolConfig(); @@ -65,6 +66,25 @@ class ProtocolConfig { Map toJson() => _$ProtocolConfigToJson(this); } +@JsonSerializable() +class HlsConfig { + int segmentConnections; + int maxRetries; + int timeoutSeconds; + bool prefetchContentLength; + + HlsConfig({ + this.segmentConnections = 0, + this.maxRetries = 0, + this.timeoutSeconds = 0, + this.prefetchContentLength = false, + }); + + factory HlsConfig.fromJson(Map json) => _$HlsConfigFromJson(json); + + Map toJson() => _$HlsConfigToJson(this); +} + @JsonSerializable() class HttpConfig { String userAgent; diff --git a/ui/flutter/lib/api/model/downloader_config.g.dart b/ui/flutter/lib/api/model/downloader_config.g.dart index 33d6e05a..2ee3c2d8 100644 --- a/ui/flutter/lib/api/model/downloader_config.g.dart +++ b/ui/flutter/lib/api/model/downloader_config.g.dart @@ -56,12 +56,28 @@ Map _$ApiServerConfigToJson(ApiServerConfig instance) => json) => ProtocolConfig() ..http = HttpConfig.fromJson(json['http'] as Map) ..bt = BtConfig.fromJson(json['bt'] as Map) - ..ed2k = Ed2kConfig.fromJson(json['ed2k'] as Map); + ..ed2k = Ed2kConfig.fromJson(json['ed2k'] as Map) + ..hls = HlsConfig.fromJson(json['hls'] as Map); Map _$ProtocolConfigToJson(ProtocolConfig instance) => { 'http': instance.http.toJson(), 'bt': instance.bt.toJson(), 'ed2k': instance.ed2k.toJson(), + 'hls': instance.hls.toJson(), +}; + +HlsConfig _$HlsConfigFromJson(Map json) => HlsConfig( + segmentConnections: (json['segmentConnections'] as num?)?.toInt() ?? 0, + maxRetries: (json['maxRetries'] as num?)?.toInt() ?? 0, + timeoutSeconds: (json['timeoutSeconds'] as num?)?.toInt() ?? 0, + prefetchContentLength: json['prefetchContentLength'] as bool? ?? false, +); + +Map _$HlsConfigToJson(HlsConfig instance) => { + 'segmentConnections': instance.segmentConnections, + 'maxRetries': instance.maxRetries, + 'timeoutSeconds': instance.timeoutSeconds, + 'prefetchContentLength': instance.prefetchContentLength, }; HttpConfig _$HttpConfigFromJson(Map json) => HttpConfig( diff --git a/ui/flutter/lib/api/model/task.dart b/ui/flutter/lib/api/model/task.dart index cca65ed9..fd08e781 100644 --- a/ui/flutter/lib/api/model/task.dart +++ b/ui/flutter/lib/api/model/task.dart @@ -6,7 +6,7 @@ part 'task.g.dart'; enum Status { ready, running, pause, wait, error, done } -enum Protocol { http, bt, ed2k } +enum Protocol { http, bt, ed2k, hls } // ExtractStatus enum matching Go backend enum ExtractStatus { diff --git a/ui/flutter/lib/api/model/task.g.dart b/ui/flutter/lib/api/model/task.g.dart index 15b3b277..f127beee 100644 --- a/ui/flutter/lib/api/model/task.g.dart +++ b/ui/flutter/lib/api/model/task.g.dart @@ -38,7 +38,7 @@ const _$StatusEnumMap = { Status.done: 'done', }; -const _$ProtocolEnumMap = {Protocol.http: 'http', Protocol.bt: 'bt', Protocol.ed2k: 'ed2k'}; +const _$ProtocolEnumMap = {Protocol.http: 'http', Protocol.bt: 'bt', Protocol.ed2k: 'ed2k', Protocol.hls: 'hls'}; Progress _$ProgressFromJson(Map json) => Progress( used: (json['used'] as num).toInt(), diff --git a/ui/flutter/lib/api/model/task_stats.dart b/ui/flutter/lib/api/model/task_stats.dart index 4dafd0b0..dfe6c8c7 100644 --- a/ui/flutter/lib/api/model/task_stats.dart +++ b/ui/flutter/lib/api/model/task_stats.dart @@ -55,7 +55,7 @@ sealed class TaskStats { Protocol.http => HttpTaskStats.fromJson(snapshot), Protocol.bt => BtTaskStats.fromJson(snapshot: snapshot, runtime: runtime), Protocol.ed2k => Ed2kTaskStats.fromJson(snapshot: snapshot, runtime: runtime), - null => null, + Protocol.hls || null => null, }; } } diff --git a/ui/flutter/lib/features/settings/presentation/pages/settings_page.dart b/ui/flutter/lib/features/settings/presentation/pages/settings_page.dart index 291b504c..c78a18a1 100644 --- a/ui/flutter/lib/features/settings/presentation/pages/settings_page.dart +++ b/ui/flutter/lib/features/settings/presentation/pages/settings_page.dart @@ -65,6 +65,8 @@ class _SettingsPageState extends ConsumerState { final _maxRunningController = TextEditingController(); final _httpUserAgentController = TextEditingController(); final _httpConnectionsController = TextEditingController(); + final _hlsConnectionsController = TextEditingController(); + final _hlsMaxRetriesController = TextEditingController(); final _btListenPortController = TextEditingController(); final _btSeedRatioController = TextEditingController(); final _btSeedTimeController = TextEditingController(); @@ -96,6 +98,8 @@ class _SettingsPageState extends ConsumerState { _maxRunningController, _httpUserAgentController, _httpConnectionsController, + _hlsConnectionsController, + _hlsMaxRetriesController, _btListenPortController, _btSeedRatioController, _btSeedTimeController, @@ -572,6 +576,32 @@ class _SettingsPageState extends ConsumerState { ], ), ), + _SettingsBlock( + key: const ValueKey('settings-hls-block'), + title: 'HLS', + child: _SettingsGroup( + children: [ + SettingsItem( + title: context.l10n.connections, + child: _NumberSettingControl( + fieldKey: const ValueKey('hls-connections-input'), + controller: _hlsConnectionsController, + min: 1, + max: 256, + ), + ), + SettingsItem( + title: context.l10n.maxRetries, + child: _NumberSettingControl( + fieldKey: const ValueKey('hls-max-retries-input'), + controller: _hlsMaxRetriesController, + min: 0, + max: 30, + ), + ), + ], + ), + ), _SettingsBlock( title: 'BitTorrent', child: _SettingsGroup( @@ -953,6 +983,8 @@ class _SettingsPageState extends ConsumerState { _maxRunningController.text = _config!.maxRunning.clamp(1, 256).toString(); _httpUserAgentController.text = _config!.protocolConfig.http.userAgent; _httpConnectionsController.text = _config!.protocolConfig.http.connections.clamp(1, 256).toString(); + _hlsConnectionsController.text = _config!.protocolConfig.hls.segmentConnections.clamp(1, 256).toString(); + _hlsMaxRetriesController.text = _config!.protocolConfig.hls.maxRetries.clamp(0, 30).toString(); _btListenPortController.text = _config!.protocolConfig.bt.listenPort.clamp(0, 65535).toString(); _btSeedRatioController.text = _config!.protocolConfig.bt.seedRatio.toString(); _btSeedTimeController.text = (_config!.protocolConfig.bt.seedTime ~/ 60).clamp(0, 100000000).toString(); @@ -1048,6 +1080,18 @@ class _SettingsPageState extends ConsumerState { min: 1, max: 256, ); + config.protocolConfig.hls.segmentConnections = _boundedInt( + _hlsConnectionsController, + fallback: config.protocolConfig.hls.segmentConnections, + min: 1, + max: 256, + ); + config.protocolConfig.hls.maxRetries = _boundedInt( + _hlsMaxRetriesController, + fallback: config.protocolConfig.hls.maxRetries, + min: 0, + max: 30, + ); config.protocolConfig.bt.listenPort = _boundedInt( _btListenPortController, fallback: config.protocolConfig.bt.listenPort, diff --git a/ui/flutter/lib/features/tasks/domain/task_record.dart b/ui/flutter/lib/features/tasks/domain/task_record.dart index 47aa0cd4..94a94818 100644 --- a/ui/flutter/lib/features/tasks/domain/task_record.dart +++ b/ui/flutter/lib/features/tasks/domain/task_record.dart @@ -294,7 +294,7 @@ TaskAssetType _assetType(String name, {required bool isFolder, required api_task return switch (protocol) { api_task.Protocol.bt => TaskAssetType.torrent, api_task.Protocol.ed2k => TaskAssetType.ed2k, - api_task.Protocol.http || null => TaskAssetType.file, + api_task.Protocol.hls || api_task.Protocol.http || null => TaskAssetType.file, }; } diff --git a/ui/flutter/lib/l10n/app_ar.arb b/ui/flutter/lib/l10n/app_ar.arb index 372a5777..947af41b 100644 --- a/ui/flutter/lib/l10n/app_ar.arb +++ b/ui/flutter/lib/l10n/app_ar.arb @@ -38,6 +38,7 @@ "rememberLastDownloadDir": "حفظ كمجلد التنزيل الافتراضي", "downloadDirValid": "الرجاء تحديد مجلد التنزيل", "connections": "الاتصالات", + "maxRetries": "الحد الأقصى لإعادة المحاولات", "useServerCtime": "استخدام وقت الخادم لإنشاء الملف", "maxRunning": "الحد الأقصى للمهام المتزامنة", "defaultDirectDownload": "تفعيل التنزيل المباشر افتراضياً", diff --git a/ui/flutter/lib/l10n/app_az.arb b/ui/flutter/lib/l10n/app_az.arb index 564f9999..39c351b9 100644 --- a/ui/flutter/lib/l10n/app_az.arb +++ b/ui/flutter/lib/l10n/app_az.arb @@ -38,6 +38,7 @@ "rememberLastDownloadDir": "Standart endirmə qovluğu kimi yadda saxla", "downloadDirValid": "Zəhmət olmasa endirmə qovluğunu seçin", "connections": "Əlaqələr", + "maxRetries": "Maksimum cəhd sayı", "useServerCtime": "Fayl yaratma tarixi üçün server vaxtından istifadə et", "maxRunning": "Paralel Endirmələr", "defaultDirectDownload": "Standart olaraq Birbaşa Endirmə", diff --git a/ui/flutter/lib/l10n/app_ca.arb b/ui/flutter/lib/l10n/app_ca.arb index bee2f371..fe733610 100644 --- a/ui/flutter/lib/l10n/app_ca.arb +++ b/ui/flutter/lib/l10n/app_ca.arb @@ -38,6 +38,7 @@ "rememberLastDownloadDir": "Desa com a carpeta de baixades predeterminada", "downloadDirValid": "Seleccioneu la carpeta de baixades", "connections": "Connexions", + "maxRetries": "Reintents màxims", "useServerCtime": "Utilitza l'hora del servidor per a la creació de fitxers", "maxRunning": "Màxim de tasques en execució", "defaultDirectDownload": "Activa la baixada directa per defecte", diff --git a/ui/flutter/lib/l10n/app_de.arb b/ui/flutter/lib/l10n/app_de.arb index 5c95653f..058a4407 100644 --- a/ui/flutter/lib/l10n/app_de.arb +++ b/ui/flutter/lib/l10n/app_de.arb @@ -38,6 +38,7 @@ "rememberLastDownloadDir": "Als Standard-Downloadordner speichern", "downloadDirValid": "Bitte wählen Sie das Download-Verzeichnis", "connections": "Anschlüsse", + "maxRetries": "Maximale Wiederholungsversuche", "useServerCtime": "Serverzeit für die Dateierstellung verwenden", "maxRunning": "Maximale Anzahl ausgeführter Aufgaben", "defaultDirectDownload": "Direkten Download standardmäßig aktivieren", diff --git a/ui/flutter/lib/l10n/app_en.arb b/ui/flutter/lib/l10n/app_en.arb index dcf3d820..9af2806f 100644 --- a/ui/flutter/lib/l10n/app_en.arb +++ b/ui/flutter/lib/l10n/app_en.arb @@ -38,6 +38,7 @@ "rememberLastDownloadDir": "Save as default download directory", "downloadDirValid": "Please select the download directory", "connections": "Connections", + "maxRetries": "Max retries", "useServerCtime": "Use server time for file creation", "maxRunning": "Concurrent Downloads", "defaultDirectDownload": "Direct Download by Default", diff --git a/ui/flutter/lib/l10n/app_es.arb b/ui/flutter/lib/l10n/app_es.arb index a9588f9a..96a65113 100644 --- a/ui/flutter/lib/l10n/app_es.arb +++ b/ui/flutter/lib/l10n/app_es.arb @@ -38,6 +38,7 @@ "rememberLastDownloadDir": "Guardar como carpeta de descargas predeterminada", "downloadDirValid": "Por favor, selecciona el directorio de descarga", "connections": "Conexiones", + "maxRetries": "Reintentos máximos", "useServerCtime": "Usar tiempo del servidor para la creación de archivos", "maxRunning": "Máximo de Tareas en Ejecución", "defaultDirectDownload": "Ver descarga directa por defecto", diff --git a/ui/flutter/lib/l10n/app_fa.arb b/ui/flutter/lib/l10n/app_fa.arb index 66235220..a7cd2144 100644 --- a/ui/flutter/lib/l10n/app_fa.arb +++ b/ui/flutter/lib/l10n/app_fa.arb @@ -38,6 +38,7 @@ "rememberLastDownloadDir": "ذخیره به‌عنوان پوشه پیش‌فرض دانلود", "downloadDirValid": "لطفا دایرکتوری دانلود را انتخاب کنید", "connections": "اتصالات", + "maxRetries": "حداکثر تلاش مجدد", "useServerCtime": "استفاده از زمان سرور برای ایجاد فایل", "maxRunning": "حداکثر وظایف در حال اجرا", "defaultDirectDownload": "دانلود مستقیم را به صورت پیش فرض بررسی کنید", diff --git a/ui/flutter/lib/l10n/app_fr.arb b/ui/flutter/lib/l10n/app_fr.arb index de714190..9f269b01 100644 --- a/ui/flutter/lib/l10n/app_fr.arb +++ b/ui/flutter/lib/l10n/app_fr.arb @@ -38,6 +38,7 @@ "rememberLastDownloadDir": "Enregistrer comme dossier de téléchargement par défaut", "downloadDirValid": "Veuillez sélectionner le répertoire de téléchargement", "connections": "Connexions", + "maxRetries": "Nombre maximal de tentatives", "useServerCtime": "Utiliser l'heure du serveur pour la création de fichier", "maxRunning": "Tâches en cours maximum", "defaultDirectDownload": "Vérifiez le téléchargement direct par défaut", diff --git a/ui/flutter/lib/l10n/app_hu.arb b/ui/flutter/lib/l10n/app_hu.arb index 7b24e524..db44d5a7 100644 --- a/ui/flutter/lib/l10n/app_hu.arb +++ b/ui/flutter/lib/l10n/app_hu.arb @@ -38,6 +38,7 @@ "rememberLastDownloadDir": "Mentés alapértelmezett letöltési mappaként", "downloadDirValid": "Kérjük, válassza ki a letöltési könyvtárat", "connections": "Kapcsolatok", + "maxRetries": "Maximális újrapróbálkozások", "useServerCtime": "Használja a szerveridőt a fájl létrehozásához", "maxRunning": "Maximális futó feladatok", "defaultDirectDownload": "Alapértelmezettként ellenőrizze a közvetlen letöltést", diff --git a/ui/flutter/lib/l10n/app_id.arb b/ui/flutter/lib/l10n/app_id.arb index 4f883708..87a36a65 100644 --- a/ui/flutter/lib/l10n/app_id.arb +++ b/ui/flutter/lib/l10n/app_id.arb @@ -38,6 +38,7 @@ "rememberLastDownloadDir": "Simpan sebagai folder unduhan default", "downloadDirValid": "Silahkan pilih direktori unduhan", "connections": "Koneksi", + "maxRetries": "Percobaan ulang maks.", "useServerCtime": "Gunakan waktu server untuk pembuatan file", "maxRunning": "Tugas Maksimum yang Berjalan", "defaultDirectDownload": "Pilih unduhan langsung sebagai opsi standar", diff --git a/ui/flutter/lib/l10n/app_it.arb b/ui/flutter/lib/l10n/app_it.arb index 168e2fd6..b04fd493 100644 --- a/ui/flutter/lib/l10n/app_it.arb +++ b/ui/flutter/lib/l10n/app_it.arb @@ -38,6 +38,7 @@ "rememberLastDownloadDir": "Salva come cartella di download predefinita", "downloadDirValid": "Seleziona la directory di scaricamento", "connections": "Connessioni", + "maxRetries": "Tentativi massimi", "useServerCtime": "Utilizza l'ora del server per la creazione dei file", "maxRunning": "Numero massimo di attività in esecuzione", "defaultDirectDownload": "Controlla il download diretto per impostazione predefinita", diff --git a/ui/flutter/lib/l10n/app_ja.arb b/ui/flutter/lib/l10n/app_ja.arb index 98ea7b15..009a7bea 100644 --- a/ui/flutter/lib/l10n/app_ja.arb +++ b/ui/flutter/lib/l10n/app_ja.arb @@ -38,6 +38,7 @@ "rememberLastDownloadDir": "デフォルトのダウンロード先として保存", "downloadDirValid": "ダウンロードディレクトリを選択してください", "connections": "接続", + "maxRetries": "最大リトライ回数", "useServerCtime": "ファイル作成時にサーバー時間を使用する", "maxRunning": "最大実行タスク", "defaultDirectDownload": "デフォルトで直接ダウンロードにチェックを入れる", diff --git a/ui/flutter/lib/l10n/app_ko.arb b/ui/flutter/lib/l10n/app_ko.arb index 28d64a32..8f89366e 100644 --- a/ui/flutter/lib/l10n/app_ko.arb +++ b/ui/flutter/lib/l10n/app_ko.arb @@ -38,6 +38,7 @@ "rememberLastDownloadDir": "기본 다운로드 폴더로 저장", "downloadDirValid": "다운로드 폴더를 선택해 주세요", "connections": "연결 수", + "maxRetries": "최대 재시도 횟수", "useServerCtime": "파일 생성 시간에 서버 시간 사용", "maxRunning": "최대 동시 작업 수", "defaultDirectDownload": "기본으로 바로 다운로드 사용", diff --git a/ui/flutter/lib/l10n/app_pl.arb b/ui/flutter/lib/l10n/app_pl.arb index 6256937f..a6b9b031 100644 --- a/ui/flutter/lib/l10n/app_pl.arb +++ b/ui/flutter/lib/l10n/app_pl.arb @@ -38,6 +38,7 @@ "rememberLastDownloadDir": "Zapisz jako domyślny folder pobierania", "downloadDirValid": "Wskaż folder pobierania", "connections": "Połączenia", + "maxRetries": "Maks. liczba ponownych prób", "useServerCtime": "Użyj czasu serwera do utworzenia pliku", "maxRunning": "Maksymalna liczba zadań", "defaultDirectDownload": "Domyślnie sprawdź bezpośredni pobór", diff --git a/ui/flutter/lib/l10n/app_pt.arb b/ui/flutter/lib/l10n/app_pt.arb index eca4a034..0fb06371 100644 --- a/ui/flutter/lib/l10n/app_pt.arb +++ b/ui/flutter/lib/l10n/app_pt.arb @@ -38,6 +38,7 @@ "rememberLastDownloadDir": "Salvar como pasta de download padrão", "downloadDirValid": "Por favor, selecione a pasta", "connections": "Conexões", + "maxRetries": "Tentativas máximas", "useServerCtime": "Usar o tempo de servidor para a criação de ficheiros", "maxRunning": "Máximo de Tarefas Simultâneas", "defaultDirectDownload": "Marcar download direto por padrão", diff --git a/ui/flutter/lib/l10n/app_ru.arb b/ui/flutter/lib/l10n/app_ru.arb index 08811c7d..b8ebe8e9 100644 --- a/ui/flutter/lib/l10n/app_ru.arb +++ b/ui/flutter/lib/l10n/app_ru.arb @@ -38,6 +38,7 @@ "rememberLastDownloadDir": "Сохранить как папку загрузок по умолчанию", "downloadDirValid": "Пожалуйста, выберите папку для загрузки", "connections": "Соединений", + "maxRetries": "Макс. число повторов", "useServerCtime": "Использовать время сервера для создания файла", "maxRunning": "Максимальное количество активных задач", "defaultDirectDownload": "Прямая загрузка по умолчанию", diff --git a/ui/flutter/lib/l10n/app_ta.arb b/ui/flutter/lib/l10n/app_ta.arb index a4544b27..97e6b3c9 100644 --- a/ui/flutter/lib/l10n/app_ta.arb +++ b/ui/flutter/lib/l10n/app_ta.arb @@ -38,6 +38,7 @@ "rememberLastDownloadDir": "இயல்புநிலை பதிவிறக்க கோப்புறையாக சேமி", "downloadDirValid": "பதிவிரக்குமிடத்தை தேர்வு செயுங்கள்", "connections": "தொடர்புகள்", + "maxRetries": "அதிகபட்ச மறுமுயற்சிகள்", "useServerCtime": "கோப்பு உருவாக்க சேவையக நேரத்தைப் பயன்படுத்தவும்", "maxRunning": "அதிகபட்ச இயங்கும் பணிகள்", "defaultDirectDownload": "இயல்புநிலையாக நேரடிப் பதிவிறக்கத்தைச் சரிபார்க்கவும்", diff --git a/ui/flutter/lib/l10n/app_tr.arb b/ui/flutter/lib/l10n/app_tr.arb index e96f138a..127d7a0c 100644 --- a/ui/flutter/lib/l10n/app_tr.arb +++ b/ui/flutter/lib/l10n/app_tr.arb @@ -38,6 +38,7 @@ "rememberLastDownloadDir": "Varsayılan indirme klasörü olarak kaydet", "downloadDirValid": "Lütfen indirme dizinini seçin", "connections": "Bağlantılar", + "maxRetries": "Maksimum yeniden deneme", "useServerCtime": "Dosya oluşturma tarihi için sunucu zamanını kullan", "maxRunning": "Maksimum Çalışan İşlem", "defaultDirectDownload": "Varsayılan olarak direkt indirmeyi işaretle", diff --git a/ui/flutter/lib/l10n/app_uk.arb b/ui/flutter/lib/l10n/app_uk.arb index 2c659f04..24e20393 100644 --- a/ui/flutter/lib/l10n/app_uk.arb +++ b/ui/flutter/lib/l10n/app_uk.arb @@ -38,6 +38,7 @@ "rememberLastDownloadDir": "Зберегти як типову папку завантажень", "downloadDirValid": "Будь ласка, виберіть каталог завантажень", "connections": "З’єднання", + "maxRetries": "Макс. кількість повторів", "useServerCtime": "Використовувати час сервера для створення файлу", "maxRunning": "Макс. завдань у черзі", "defaultDirectDownload": "Пряме завантаження за замовчуванням", diff --git a/ui/flutter/lib/l10n/app_vi.arb b/ui/flutter/lib/l10n/app_vi.arb index 8e3e36f5..aa3e7a9c 100644 --- a/ui/flutter/lib/l10n/app_vi.arb +++ b/ui/flutter/lib/l10n/app_vi.arb @@ -38,6 +38,7 @@ "rememberLastDownloadDir": "Lưu làm thư mục tải xuống mặc định", "downloadDirValid": "Vui lòng chọn thư mục tải về", "connections": "Kết nối", + "maxRetries": "Số lần thử lại tối đa", "useServerCtime": "Sử dụng thời gian máy chủ cho việc tạo tệp", "maxRunning": "Số nhiệm vụ tối đa", "defaultDirectDownload": "Kiểm tra trực tiếp tải về theo mặc định", diff --git a/ui/flutter/lib/l10n/app_zh.arb b/ui/flutter/lib/l10n/app_zh.arb index 2395f69b..e199a7f7 100644 --- a/ui/flutter/lib/l10n/app_zh.arb +++ b/ui/flutter/lib/l10n/app_zh.arb @@ -38,6 +38,7 @@ "rememberLastDownloadDir": "保存为默认下载目录", "downloadDirValid": "请选择下载目录", "connections": "连接数", + "maxRetries": "最大重试次数", "useServerCtime": "创建文件使用服务器时间", "maxRunning": "最大下载数", "defaultDirectDownload": "默认启用直接下载", diff --git a/ui/flutter/lib/l10n/app_zh_TW.arb b/ui/flutter/lib/l10n/app_zh_TW.arb index 2692840b..1925dd0f 100644 --- a/ui/flutter/lib/l10n/app_zh_TW.arb +++ b/ui/flutter/lib/l10n/app_zh_TW.arb @@ -38,6 +38,7 @@ "rememberLastDownloadDir": "儲存為預設下載目錄", "downloadDirValid": "請選擇下載目錄", "connections": "連接數", + "maxRetries": "最大重試次數", "useServerCtime": "使用伺服器時間作為檔案建立時間", "maxRunning": "最大執行任務數", "defaultDirectDownload": "預設勾選直接下載", From c667f097dc5309b035e069769504aae33db3baaf Mon Sep 17 00:00:00 2001 From: Levi Date: Wed, 16 Sep 2026 00:38:22 +0800 Subject: [PATCH 2/5] feat: add FFmpeg WASM runtime (#1512) --- go.mod | 4 +- go.sum | 6 + internal/protocol/http/config.go | 3 + internal/protocol/http/fetcher_manager.go | 2 +- pkg/base/model.go | 3 - pkg/download/engine/engine.go | 19 +- pkg/download/engine/ffmpeg/README.md | 128 +++++++ pkg/download/engine/ffmpeg/input.go | 227 ++++++++++++ pkg/download/engine/ffmpeg/input_test.go | 222 ++++++++++++ pkg/download/engine/ffmpeg/runtime.go | 254 ++++++++++++++ pkg/download/engine/ffmpeg/runtime_test.go | 215 ++++++++++++ pkg/download/engine/ffmpeg/stream.go | 163 +++++++++ .../engine/ffmpeg_integration_test.go | 321 +++++++++++++++++ pkg/download/engine/inject/fetch/module.go | 8 +- pkg/download/engine/inject/ffmpeg/ffmpeg.js | 139 ++++++++ pkg/download/engine/inject/ffmpeg/module.go | 324 ++++++++++++++++++ pkg/download/engine/inject/stream/module.go | 13 +- .../inject/stream/module_integration_test.go | 54 +++ pkg/download/engine/inject/stream/stream.js | 17 +- pkg/download/engine/testdata/ffmpeg/README.md | 18 + pkg/download/engine/testdata/ffmpeg/audio.mp4 | Bin 0 -> 1125 bytes .../engine/testdata/ffmpeg/audio.webm | Bin 0 -> 681 bytes pkg/download/engine/testdata/ffmpeg/video.mp4 | Bin 0 -> 1597 bytes .../engine/testdata/ffmpeg/video.webm | Bin 0 -> 579 bytes pkg/download/extension_engine_session.go | 12 +- pkg/download/extension_runtime_webview.go | 3 + pkg/download/extension_test.go | 4 + .../extension_unnamed_resource_test.go | 37 ++ pkg/download/extension_user_agent_test.go | 108 ++++++ .../features/tasks/domain/task_record.dart | 5 +- .../tasks/domain/task_record_test.dart | 20 ++ 31 files changed, 2305 insertions(+), 24 deletions(-) create mode 100644 pkg/download/engine/ffmpeg/README.md create mode 100644 pkg/download/engine/ffmpeg/input.go create mode 100644 pkg/download/engine/ffmpeg/input_test.go create mode 100644 pkg/download/engine/ffmpeg/runtime.go create mode 100644 pkg/download/engine/ffmpeg/runtime_test.go create mode 100644 pkg/download/engine/ffmpeg/stream.go create mode 100644 pkg/download/engine/ffmpeg_integration_test.go create mode 100644 pkg/download/engine/inject/ffmpeg/ffmpeg.js create mode 100644 pkg/download/engine/inject/ffmpeg/module.go create mode 100644 pkg/download/engine/testdata/ffmpeg/README.md create mode 100644 pkg/download/engine/testdata/ffmpeg/audio.mp4 create mode 100644 pkg/download/engine/testdata/ffmpeg/audio.webm create mode 100644 pkg/download/engine/testdata/ffmpeg/video.mp4 create mode 100644 pkg/download/engine/testdata/ffmpeg/video.webm create mode 100644 pkg/download/extension_unnamed_resource_test.go create mode 100644 pkg/download/extension_user_agent_test.go create mode 100644 ui/flutter/test/features/tasks/domain/task_record_test.dart diff --git a/go.mod b/go.mod index 13db5774..f3575db8 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/GopeedLab/gopeed go 1.25.4 require ( + codeberg.org/gruf/go-ffmpreg v0.6.23-0.20260827095416-df1836cb5be0 github.com/GopeedLab/webview_go v0.0.0-20260423085439-7a2f88b6e9b5 github.com/anacrolix/torrent v1.60.1-0.20251217073903-486bcbe758e0 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 @@ -21,6 +22,7 @@ require ( github.com/pkg/errors v0.9.1 github.com/rs/dnscache v0.0.0-20230804202142-fc85eb664529 github.com/rs/zerolog v1.31.0 + github.com/tetratelabs/wazero v1.12.0 github.com/xiaoqidun/setft v0.0.0-20220310121541-be86327699ad go.etcd.io/bbolt v1.4.3 golang.org/x/exp v0.0.0-20251219203646-944ab1f22d93 @@ -156,7 +158,7 @@ require ( golang.org/x/crypto v0.48.0 // indirect golang.org/x/mod v0.33.0 // indirect golang.org/x/net v0.50.0 - golang.org/x/sys v0.41.0 // indirect + golang.org/x/sys v0.44.0 // indirect golang.org/x/text v0.34.0 golang.org/x/time v0.15.0 // indirect golang.org/x/tools v0.42.0 // indirect diff --git a/go.sum b/go.sum index 9c4680f4..b1f117c1 100644 --- a/go.sum +++ b/go.sum @@ -14,6 +14,8 @@ cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2k cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= +codeberg.org/gruf/go-ffmpreg v0.6.23-0.20260827095416-df1836cb5be0 h1:AqNFl60o4xhTTOxmzAvSf7pazXHKBGuV5ATw7sETIRA= +codeberg.org/gruf/go-ffmpreg v0.6.23-0.20260827095416-df1836cb5be0/go.mod h1:XM6eW8CSgAuZRKRynm5nDiSNedm0rav30IychqygOHE= crawshaw.io/iox v0.0.0-20181124134642-c51c3df30797/go.mod h1:sXBiorCo8c46JlQV3oXPKINnZ8mcqnye1EkVkqsectk= crawshaw.io/sqlite v0.3.2/go.mod h1:igAO5JulrQ1DbdZdtVq48mnZUBAPOeFzer7VhDWNtW4= dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= @@ -552,6 +554,8 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/tetratelabs/wazero v1.12.0 h1:DuWcpNu/FzgEXgGBDp8J1Spc+CWOvvtvVyjKlaZopYU= +github.com/tetratelabs/wazero v1.12.0/go.mod h1:LvKtzl2RqO4gyF27BiXU+nKAjcV8f38U+kP/q2vgxh0= github.com/tidwall/btree v1.8.1 h1:27ehoXvm5AG/g+1VxLS1SD3vRhp/H7LuEfwNvddEdmA= github.com/tidwall/btree v1.8.1/go.mod h1:jBbTdUWhSZClZWoDg54VnvV7/54modSOzDN7VXftj1A= github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= @@ -732,6 +736,8 @@ golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k= golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/sys v0.44.0 h1:ildZl3J4uzeKP07r2F++Op7E9B29JRUy+a27EibtBTQ= +golang.org/x/sys v0.44.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= diff --git a/internal/protocol/http/config.go b/internal/protocol/http/config.go index fdabb292..7a09e59a 100644 --- a/internal/protocol/http/config.go +++ b/internal/protocol/http/config.go @@ -1,5 +1,8 @@ package http +// DefaultUserAgent is shared by HTTP downloads and extension HTTP requests. +const DefaultUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36" + type config struct { UserAgent string `json:"userAgent"` Connections int `json:"connections"` diff --git a/internal/protocol/http/fetcher_manager.go b/internal/protocol/http/fetcher_manager.go index 99d9b571..833a4b08 100644 --- a/internal/protocol/http/fetcher_manager.go +++ b/internal/protocol/http/fetcher_manager.go @@ -72,7 +72,7 @@ func (fm *FetcherManager) AutoRename() bool { func (fm *FetcherManager) DefaultConfig() any { return &config{ - UserAgent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36", + UserAgent: DefaultUserAgent, Connections: 16, } } diff --git a/pkg/base/model.go b/pkg/base/model.go index 46bcace2..7b49b297 100644 --- a/pkg/base/model.go +++ b/pkg/base/model.go @@ -94,9 +94,6 @@ type Resource struct { } func (r *Resource) Validate() error { - if r.Name == "" { - return fmt.Errorf("invalid resource name") - } if len(r.Files) == 0 { return fmt.Errorf("invalid resource files") } diff --git a/pkg/download/engine/engine.go b/pkg/download/engine/engine.go index fb93662a..147f1f98 100644 --- a/pkg/download/engine/engine.go +++ b/pkg/download/engine/engine.go @@ -9,6 +9,7 @@ import ( "github.com/GopeedLab/gopeed/pkg/base" gojaerror "github.com/GopeedLab/gopeed/pkg/download/engine/inject/error" fetchapi "github.com/GopeedLab/gopeed/pkg/download/engine/inject/fetch" + ffmpegapi "github.com/GopeedLab/gopeed/pkg/download/engine/inject/ffmpeg" "github.com/GopeedLab/gopeed/pkg/download/engine/inject/file" "github.com/GopeedLab/gopeed/pkg/download/engine/inject/stream" "github.com/GopeedLab/gopeed/pkg/download/engine/inject/vm" @@ -159,8 +160,10 @@ func (e *Engine) addCleanup(cleanup func()) { } type Config struct { - ProxyConfig *base.DownloaderProxyConfig - StreamConfig *stream.Config + // HTTPUserAgent is a snapshot of the downloader HTTP default. Nil leaves standalone engines unchanged. + HTTPUserAgent *string + ProxyConfig *base.DownloaderProxyConfig + StreamConfig *stream.Config } func NewEngine(cfg *Config) *Engine { @@ -203,9 +206,17 @@ func NewEngine(cfg *Config) *Engine { if err := stream.Enable(runtime, loop, cfg.StreamConfig); err != nil { return } + if err := ffmpegapi.Enable(runtime, loop, &ffmpegapi.Config{ + ProxyHandler: cfg.ProxyConfig.ToHandler(), + DefaultUserAgent: cfg.HTTPUserAgent, + RegisterCleanup: engine.addCleanup, + }); err != nil { + return + } if err := fetchapi.Enable(runtime, loop, &fetchapi.Config{ - ProxyHandler: cfg.ProxyConfig.ToHandler(), - RegisterCleanup: engine.addCleanup, + ProxyHandler: cfg.ProxyConfig.ToHandler(), + DefaultUserAgent: cfg.HTTPUserAgent, + RegisterCleanup: engine.addCleanup, }); err != nil { return } diff --git a/pkg/download/engine/ffmpeg/README.md b/pkg/download/engine/ffmpeg/README.md new file mode 100644 index 00000000..c7ef41bd --- /dev/null +++ b/pkg/download/engine/ffmpeg/README.md @@ -0,0 +1,128 @@ +# Extension FFmpeg runtime + +`gopeed.runtime.ffmpeg.merge(options)` returns a `ReadableStream`. +It merges one video track and one audio track without re-encoding, using +[go-ffmpreg](https://codeberg.org/gruf/go-ffmpreg)'s embedded FFmpeg 5.1.10 and +wazero. No system FFmpeg installation is required. go-ffmpreg and its embedded +GPL-enabled FFmpeg are distributed under GPLv3; see the pinned dependency's +LICENSE, README and build scripts for its source and included components. + +## HTTP inputs and one downloadable output + +```js +gopeed.events.onResolve(async (ctx) => { + // Resolve these URLs and headers using the site's API in your extension. + const video = { url: videoURL, headers: { Referer: pageURL } }; + const audio = { url: audioURL, headers: { Referer: pageURL } }; + const url = await gopeed.runtime.blob.createObjectURL( + () => gopeed.runtime.ffmpeg.merge({ video, audio }), + { contentType: "video/mp4" } + ); + ctx.res = { + name: "video", + files: [{ name: "video.mp4", req: { url } }] + }; +}); +``` + +The opener creates a new merge on each open. Output size is unknown and range +support must remain disabled. Do not declare the sum of the input sizes as the +output size. The downloader reads the merged output as one file; FFmpeg errors +propagate through the stream instead of completing a partial download. + +## JS inputs, including SABR + +Stream inputs **must** be created by an `inputs({ signal })` callback. It returns +`{ video, audio }`, each an HTTP descriptor or `ReadableStream`, +including mixed inputs. Do not start producers before this callback. SABR/UMP decoding belongs +to the extension: pass the resulting media streams, with initialization data, +not the SABR protocol bytes. Keep each selected track's codec/configuration +stable for the duration of a merge. + +```js +const output = gopeed.runtime.ffmpeg.merge({ + inputs: async ({ signal }) => { + const session = await prepareSession({ signal }); + const { videoStream, audioStream, abort } = await session.openStreams(); + if (signal.aborted) { abort(); throw new Error("Cancelled"); } + signal.addEventListener("abort", abort, { once: true }); + return { video: videoStream, audio: audioStream }; + }, + signal: abortController.signal, + args: ["-shortest", "-metadata", "title=Example"] +}); +``` + +Creating the output does not start requests or acquire input readers until the +output is pulled and a global execution slot is acquired. Queued jobs do not +open HTTP inputs, invoke input factories, or allocate media ring buffers. +The callback runs once per output; its signal aborts on cancellation, engine +shutdown, failure, and completion. A callback must cooperate with cancellation; +returned streams are cancelled even if the callback finishes after cancellation. +A stream input is single-use. To retry a Blob-backed task, +its opener must create a new SABR session and fresh input streams. Arbitrary +output byte ranges / resumable merged output are not supported. + +`gopeed.runtime.ffmpeg.supportsInputFactory === true` identifies this API. +Older builds accept eager streams and must be upgraded before using callbacks. + +## Options + +- `inputs`: callback returning `{ video, audio }`, synchronously or asynchronously. + Required for stream inputs. Cannot be combined with top-level `video`/`audio`. +- `video`, `audio`: HTTP descriptors only when supplied at the top level; both + are required unless `inputs` is supplied. HTTP descriptors have `url` and optional + `headers` (standard `HeadersInit`). HTTP requests use Gopeed's configured + proxy. Supply required cookies/authorization explicitly; browser or `fetch` + cookie jars are not implicitly shared. When `User-Agent` is absent, requests + use the downloader HTTP protocol's configured UA, captured when the extension + engine is created, matching fetch and XHR. Explicit UA values (including an + empty string) take precedence. +- `format`: defaults to `"mp4"`. MP4 output always enables fragmentation + (`frag_keyframe+empty_moov+default_base_moof`) for non-seekable output. The + other accepted muxers are `"mkv"`/`"matroska"`, `"webm"`, and `"mpegts"`. + Codec/container incompatibility is an error, not an automatic conversion. + Choose the matching filename and Blob content type in the extension. +- `args`: optional array of **output options**, not a shell command or a full + FFmpeg invocation. Defaults select `0:v:0`, `1:a:0`, and `-c copy`. Additional + `-map` options are additive. Allowed options are listed in `outputOptions` in + `runtime.go`: metadata, dispositions, bitstream filters, duration/shortest, + fragmentation, timestamp/interleaving options and `-strict`. Codec options + must remain `copy`. Extra inputs, output paths, reports and arbitrary global + options are rejected. Required MP4 fragmentation is enforced by the host. +- `signal`: optional `AbortSignal`. Aborting or cancelling the output stops + network requests and native execution, cancels input readers, and removes + temporary buffers. Await/consume the output inside a live engine or retain + it through a Blob URL; unowned work ends with the extension engine. + +The backend defaults to strict input errors (`-xerror`). Native errors include +the FFmpeg exit code and a bounded stderr tail. No host directories are exposed +to the WASM module; it can access only its two input files and standard output. + +## Input behavior and resource limits + +HTTP inputs start with a 1 MiB Range GET, reusing the bytes for media reads. +A validated 206 response with a known total size enables seeking; a 200 +response is consumed as-is, sequentially. Subsequent ranges verify positions, +lengths, total size and available ETag/Last-Modified validators. Without a +server validator, same-size remote changes cannot be reliably detected. +Content encoding is forced to identity. Authentication/status failures and +malformed range responses are errors. There is no full-file seek fallback. + +Sequential JS inputs are drained independently into memory ring buffers that +grow on demand up to **32 MiB per input**, independent of total media size. +No temporary files are used. This does not enable seeking. Full buffers apply +async backpressure until space is available or the operation is cancelled. +Extensions must propagate backpressure through their upstream queues and +network reads. Independent upstream track requests avoid blocking one track +behind another in a shared response. Buffers are released on close. +Blob output writes also wait asynchronously for bounded downstream capacity, +so a slow consumer does not block the extension event loop. + +At most two WASM merges execute concurrently per process, with a 512 MiB +linear-memory limit per instance. Compilation is shared and lazy; the upstream +embedded package still decompresses its WASM at process initialization. Native +compilation memory, host buffers and JS allocations are additional to the +linear-memory limit. Playback support depends on the selected codecs, not just +the `.mp4` suffix. Platform execution and performance must be tested on target +devices, especially mobile/32-bit devices. diff --git a/pkg/download/engine/ffmpeg/input.go b/pkg/download/engine/ffmpeg/input.go new file mode 100644 index 00000000..74e9c453 --- /dev/null +++ b/pkg/download/engine/ffmpeg/input.go @@ -0,0 +1,227 @@ +// Package ffmpeg runs an embedded FFmpeg with host-controlled media inputs. +package ffmpeg + +import ( + "context" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "strconv" + "strings" +) + +const httpBlockSize = 1024 * 1024 + +// Input is a file visible to FFmpeg. Only verified HTTP Range sources seek. +type Input interface { + io.ReadCloser + Seek(int64, int) (int64, error) + Size() int64 + Seekable() bool +} + +var ErrSeek = errors.New("ffmpeg: input requires seeking but source is sequential") + +type HTTPSource struct { + URL string `json:"url"` + Headers map[string]string `json:"headers"` +} + +type httpInput struct { + ctx context.Context + client *http.Client + source HTTPSource + body io.ReadCloser + size, pos, blockStart int64 + block []byte + ranged bool + etag, modified string +} + +// OpenHTTP combines capability detection with the first media read. A 200 +// response remains open and is consumed sequentially without another request. +func OpenHTTP(ctx context.Context, client *http.Client, source HTTPSource) (Input, error) { + u, err := url.Parse(source.URL) + if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" { + return nil, errors.New("ffmpeg: expected an HTTP(S) input URL") + } + h := &httpInput{ctx: ctx, client: client, source: source, size: -1} + resp, err := h.request(0, httpBlockSize-1) + if err != nil { + return nil, err + } + if resp.StatusCode == http.StatusOK { + if err = identityEncoding(resp); err != nil { + resp.Body.Close() + return nil, err + } + h.body, h.size = resp.Body, resp.ContentLength + return h, nil + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusPartialContent { + return nil, fmt.Errorf("ffmpeg: HTTP input returned %d", resp.StatusCode) + } + start, end, size, err := contentRange(resp.Header.Get("Content-Range")) + if err != nil || start != 0 || end >= httpBlockSize { + return nil, errors.New("ffmpeg: invalid initial Content-Range") + } + h.size, h.ranged = size, true + h.etag = resp.Header.Get("ETag") + if strings.HasPrefix(h.etag, "W/") { + h.etag = "" + } + h.modified = resp.Header.Get("Last-Modified") + h.block, err = readBlock(resp, end-start+1) + return h, err +} + +func (h *httpInput) request(start, end int64) (*http.Response, error) { + req, err := http.NewRequestWithContext(h.ctx, http.MethodGet, h.source.URL, nil) + if err != nil { + return nil, err + } + for k, v := range h.source.Headers { + req.Header.Set(k, v) + } + // Offsets must refer to the actual media bytes, not a decompressed response. + req.Header.Set("Accept-Encoding", "identity") + req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", start, end)) + if h.etag != "" { + req.Header.Set("If-Range", h.etag) + } else if h.modified != "" { + req.Header.Set("If-Range", h.modified) + } + return h.client.Do(req) +} + +func identityEncoding(resp *http.Response) error { + if enc := resp.Header.Get("Content-Encoding"); enc != "" && !strings.EqualFold(enc, "identity") { + return errors.New("ffmpeg: encoded HTTP responses cannot represent media byte offsets") + } + return nil +} + +func contentRange(value string) (start, end, size int64, err error) { + err = errors.New("invalid Content-Range") + if !strings.HasPrefix(value, "bytes ") { + return + } + parts := strings.Split(strings.TrimPrefix(value, "bytes "), "/") + if len(parts) != 2 { + return + } + bounds := strings.Split(parts[0], "-") + if len(bounds) != 2 { + return + } + var e error + if start, e = strconv.ParseInt(bounds[0], 10, 64); e != nil { + return + } + if end, e = strconv.ParseInt(bounds[1], 10, 64); e != nil { + return + } + if size, e = strconv.ParseInt(parts[1], 10, 64); e != nil { + return + } + if start < 0 || end < start || size <= end { + return + } + err = nil + return +} + +func readBlock(resp *http.Response, length int64) ([]byte, error) { + if err := identityEncoding(resp); err != nil { + return nil, err + } + if resp.ContentLength >= 0 && resp.ContentLength != length { + return nil, errors.New("ffmpeg: Range response length mismatch") + } + b, err := io.ReadAll(io.LimitReader(resp.Body, length+1)) + if err != nil { + return nil, err + } + if int64(len(b)) != length { + return nil, errors.New("ffmpeg: truncated or oversized Range response") + } + return b, nil +} + +func (h *httpInput) Read(p []byte) (int, error) { + if len(p) == 0 { + return 0, nil + } + if err := h.ctx.Err(); err != nil { + return 0, err + } + if !h.ranged { + return h.body.Read(p) + } + if h.pos >= h.size { + return 0, io.EOF + } + if h.pos < h.blockStart || h.pos >= h.blockStart+int64(len(h.block)) { + end := min(h.pos+httpBlockSize-1, h.size-1) + resp, err := h.request(h.pos, end) + if err != nil { + return 0, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusPartialContent { + return 0, fmt.Errorf("ffmpeg: Range request returned %d; source changed or stopped supporting Range", resp.StatusCode) + } + start, actualEnd, size, err := contentRange(resp.Header.Get("Content-Range")) + if err != nil || start != h.pos || actualEnd > end || size != h.size { + return 0, errors.New("ffmpeg: inconsistent Content-Range") + } + if h.etag != "" && resp.Header.Get("ETag") != h.etag { + return 0, errors.New("ffmpeg: HTTP input ETag changed") + } + if h.etag == "" && h.modified != "" && resp.Header.Get("Last-Modified") != h.modified { + return 0, errors.New("ffmpeg: HTTP input Last-Modified changed") + } + h.block, err = readBlock(resp, actualEnd-start+1) + if err != nil { + return 0, err + } + h.blockStart = start + } + n := copy(p, h.block[h.pos-h.blockStart:]) + h.pos += int64(n) + return n, nil +} + +func (h *httpInput) Seek(offset int64, whence int) (int64, error) { + if !h.ranged { + return 0, ErrSeek + } + base := int64(0) + switch whence { + case io.SeekStart: + case io.SeekCurrent: + base = h.pos + case io.SeekEnd: + base = h.size + default: + return 0, errors.New("invalid seek origin") + } + next := base + offset + if next < 0 || (offset > 0 && next < base) { + return 0, errors.New("invalid seek offset") + } + h.pos = next + return next, nil +} + +func (h *httpInput) Size() int64 { return h.size } +func (h *httpInput) Seekable() bool { return h.ranged } +func (h *httpInput) Close() error { + if h.body != nil { + return h.body.Close() + } + return nil +} diff --git a/pkg/download/engine/ffmpeg/input_test.go b/pkg/download/engine/ffmpeg/input_test.go new file mode 100644 index 00000000..63e86c21 --- /dev/null +++ b/pkg/download/engine/ffmpeg/input_test.go @@ -0,0 +1,222 @@ +package ffmpeg + +import ( + "bytes" + "context" + "fmt" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" +) + +func TestHTTPRange(t *testing.T) { + data := bytes.Repeat([]byte("0123456789"), 300000) + var requests atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + count := requests.Add(1) + if r.Header.Get("X-Test") != "secret" { + t.Error("missing headers") + } + if r.Header.Get("Accept-Encoding") != "identity" { + t.Error("missing identity encoding") + } + if count > 1 && r.Header.Get("If-Range") != `"v1"` { + t.Error("missing If-Range") + } + w.Header().Set("ETag", `"v1"`) + http.ServeContent(w, r, "media", time.Time{}, bytes.NewReader(data)) + })) + defer srv.Close() + in, err := OpenHTTP(context.Background(), srv.Client(), HTTPSource{URL: srv.URL, Headers: map[string]string{"X-Test": "secret"}}) + if err != nil { + t.Fatal(err) + } + defer in.Close() + if !in.Seekable() || in.Size() != int64(len(data)) { + t.Fatal("range not detected") + } + for _, off := range []int64{0, int64(len(data) - 16), 12, int64(httpBlockSize + 35)} { + if _, err = in.Seek(off, io.SeekStart); err != nil { + t.Fatal(err) + } + b := make([]byte, 16) + if _, err = io.ReadFull(in, b); err != nil { + t.Fatal(err) + } + if !bytes.Equal(b, data[off:off+16]) { + t.Fatalf("wrong bytes at %d", off) + } + } +} + +func TestHTTPSequentialAndValidation(t *testing.T) { + for _, kind := range []string{"sequential", "bad-range", "changed", "truncated", "unauthorized"} { + t.Run(kind, func(t *testing.T) { + var requests atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + count := requests.Add(1) + switch kind { + case "sequential": + w.Write([]byte("media")) + case "unauthorized": + w.WriteHeader(403) + case "bad-range": + w.Header().Set("Content-Range", "bytes 2-6/10") + w.WriteHeader(206) + w.Write([]byte("media")) + case "truncated": + w.Header().Set("Content-Range", "bytes 0-9/10") + w.WriteHeader(206) + w.Write([]byte("media")) + case "changed": + if count == 1 { + w.Header().Set("Content-Range", "bytes 0-4/10") + w.Header().Set("ETag", `"v1"`) + w.WriteHeader(206) + } + w.Write([]byte("media")) + } + })) + defer srv.Close() + in, err := OpenHTTP(context.Background(), srv.Client(), HTTPSource{URL: srv.URL}) + if kind == "sequential" { + if err != nil { + t.Fatal(err) + } + defer in.Close() + if in.Seekable() { + t.Fatal("unexpected seeking") + } + if _, err = in.Seek(0, 0); err != ErrSeek { + t.Fatal(err) + } + b, err := io.ReadAll(in) + if err != nil || string(b) != "media" || requests.Load() != 1 { + t.Fatalf("%q %v count %d", b, err, requests.Load()) + } + } else if kind == "changed" { + if err != nil { + t.Fatal(err) + } + defer in.Close() + in.Seek(5, 0) + if _, err = in.Read(make([]byte, 2)); err == nil { + t.Fatal("accepted 200 after range") + } + } else if err == nil { + in.Close() + t.Fatal("accepted invalid response") + } + }) + } +} + +func TestStreamRing(t *testing.T) { + s := NewStreamInput(context.Background()) + defer s.Close() + // Start near the boundary to exercise wrap-around without a huge fixture. + s.buffer = make([]byte, 256*1024) + s.read = int64(len(s.buffer)) - 3 + s.written = s.read + if _, err := s.Write([]byte("abcdefgh")); err != nil { + t.Fatal(err) + } + s.End(nil) + b, err := io.ReadAll(s) + if err != nil || string(b) != "abcdefgh" { + t.Fatalf("%q %v", b, err) + } + if err = s.Close(); err != nil { + t.Fatal(err) + } + if s.buffer != nil { + t.Fatal("buffer retained after close") + } +} + +func TestOutputArguments(t *testing.T) { + a, err := Arguments("", []string{"-shortest", "-metadata", "title=test"}) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(strings.Join(a, " "), "-f mp4 pipe:1") { + t.Fatal(a) + } + for _, args := range [][]string{{"-i", "/etc/passwd"}, {"-report"}, {"-c:v", "libx264"}, {"-metadata"}, {"other.mp4"}} { + if _, err := Arguments("mp4", args); err == nil { + t.Fatal(fmt.Sprint(args)) + } + } +} + +func TestStreamBackpressureAndCancel(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + s := NewStreamInput(ctx) + defer s.Close() + s.buffer = make([]byte, StreamBufferLimit) + s.written = StreamBufferLimit + finished := make(chan error, 1) + go func() { _, err := s.Write([]byte{42}); finished <- err }() + select { + case err := <-finished: + t.Fatalf("did not apply backpressure: %v", err) + case <-time.After(10 * time.Millisecond): + } + if _, err := s.Read(make([]byte, 1)); err != nil { + t.Fatal(err) + } + select { + case err := <-finished: + if err != nil { + t.Fatal(err) + } + case <-time.After(time.Second): + t.Fatal("writer did not resume") + } + go func() { _, err := s.Write([]byte{43}); finished <- err }() + cancel() + select { + case err := <-finished: + if err == nil { + t.Fatal("cancelled write succeeded") + } + case <-time.After(time.Second): + t.Fatal("writer did not cancel") + } +} + +func TestStreamMemoryGrowthAndWrap(t *testing.T) { + s := NewStreamInput(context.Background()) + defer s.Close() + if len(s.buffer) != 0 { + t.Fatal("buffer allocated before first write") + } + a := bytes.Repeat([]byte{1}, 200*1024) + b := bytes.Repeat([]byte{2}, 180*1024) + s.Write(a) + consumed := make([]byte, 180*1024) + if _, err := io.ReadFull(s, consumed); err != nil { + t.Fatal(err) + } + s.Write(b) // Wrap in the initial 256 KiB allocation. + c := bytes.Repeat([]byte{3}, 300*1024) + s.Write(c) // Grow while unread data wraps around the old ring. + s.End(nil) + got, err := io.ReadAll(s) + want := append(append(append([]byte{}, a[len(consumed):]...), b...), c...) + if err != nil || !bytes.Equal(got, want) { + t.Fatalf("growth corrupted data: %v", err) + } + if len(s.buffer) > StreamBufferLimit { + t.Fatal("buffer exceeded limit") + } + s.Close() + if s.buffer != nil { + t.Fatal("buffer retained on close") + } +} diff --git a/pkg/download/engine/ffmpeg/runtime.go b/pkg/download/engine/ffmpeg/runtime.go new file mode 100644 index 00000000..2d6b192f --- /dev/null +++ b/pkg/download/engine/ffmpeg/runtime.go @@ -0,0 +1,254 @@ +package ffmpeg + +import ( + "context" + "errors" + "fmt" + "io" + "io/fs" + "strings" + "sync" + + ffembed "codeberg.org/gruf/go-ffmpreg/embed" + "codeberg.org/gruf/go-ffmpreg/wasm" + "github.com/tetratelabs/wazero" + expsys "github.com/tetratelabs/wazero/experimental/sys" + "github.com/tetratelabs/wazero/experimental/sysfs" + "github.com/tetratelabs/wazero/sys" +) + +var shared struct { + once sync.Once + runtime wazero.Runtime + module wazero.CompiledModule + err error +} + +// Limit active instances independently of the number of extension engines. +var slots = make(chan struct{}, 2) + +func initialize() { + shared.once.Do(func() { + defer func() { + if r := recover(); r != nil { + shared.err = fmt.Errorf("ffmpeg initialization: %v", r) + } + }() + cfg := wazero.NewRuntimeConfig().WithCloseOnContextDone(true).WithMemoryLimitPages(8192) + shared.runtime, shared.err = wasm.NewRuntime(context.Background(), cfg) + if shared.err != nil { + return + } + shared.module, shared.err = shared.runtime.CompileModule(context.Background(), ffembed.B()) + if shared.err != nil { + shared.runtime.Close(context.Background()) + return + } + ffembed.Free() + }) +} + +// Output arguments deliberately do not accept extra inputs, output paths or +// global filesystem/logging options. The host owns all I/O and cancellation. +var outputOptions = map[string]int{ + "-c": 1, "-codec": 1, "-c:v": 1, "-c:a": 1, "-codec:v": 1, "-codec:a": 1, + "-map": 1, "-map_metadata": 1, "-map_chapters": 1, "-metadata": 1, "-metadata:s:a:0": 1, "-metadata:s:v:0": 1, + "-movflags": 1, "-frag_duration": 1, "-frag_size": 1, "-min_frag_duration": 1, + "-bsf:v": 1, "-bsf:a": 1, "-strict": 1, "-shortest": 0, "-t": 1, "-to": 1, + "-avoid_negative_ts": 1, "-max_interleave_delta": 1, "-flush_packets": 1, + "-disposition:a:0": 1, "-disposition:v:0": 1, +} + +func Arguments(format string, extra []string) ([]string, error) { + if format == "" { + format = "mp4" + } + if format == "mkv" { + format = "matroska" + } + switch format { + case "mp4", "matroska", "webm", "mpegts": + default: + return nil, fmt.Errorf("ffmpeg: unsupported output format %q", format) + } + for i := 0; i < len(extra); { + count, ok := outputOptions[extra[i]] + if !ok { + return nil, fmt.Errorf("ffmpeg: unsupported output option %q", extra[i]) + } + if i+count >= len(extra) { + return nil, fmt.Errorf("ffmpeg: missing value for %s", extra[i]) + } + if strings.HasPrefix(extra[i], "-c") && extra[i+1] != "copy" { + return nil, errors.New("ffmpeg: merge only supports stream copy") + } + i += count + 1 + } + args := []string{"-hide_banner", "-nostdin", "-xerror", "-loglevel", "error", "-protocol_whitelist", "file,pipe", "-i", "/inputs/video", "-protocol_whitelist", "file,pipe", "-i", "/inputs/audio", "-map", "0:v:0", "-map", "1:a:0", "-c", "copy"} + fragmentFlags := "" + for i := 0; i < len(extra); { + count := outputOptions[extra[i]] + if format == "mp4" && extra[i] == "-movflags" { + if strings.Contains(extra[i+1], "faststart") || strings.Contains(extra[i+1], "global_sidx") { + return nil, errors.New("ffmpeg: these movflags require seekable output") + } + fragmentFlags = extra[i+1] + } else { + args = append(args, extra[i:i+count+1]...) + } + i += count + 1 + } + if format == "mp4" { + args = append(args, "-movflags", fragmentFlags+"+frag_keyframe+empty_moov+default_base_moof") + } + args = append(args, "-f", format, "pipe:1") + return args, nil +} + +// Run shares compilation, but gives every merge its own memory and filesystem. +// A clean stdout EOF is only delivered by the caller after this succeeds. +func Run(ctx context.Context, video, audio Input, out io.Writer, format string, extra []string) error { + _, err := Arguments(format, extra) + if err != nil { + return err + } + release, err := Acquire(ctx) + if err != nil { + return err + } + defer release() + return RunAcquired(ctx, video, audio, out, format, extra) +} + +// Acquire reserves execution capacity before input factories or HTTP requests start. +func Acquire(ctx context.Context) (func(), error) { + select { + case slots <- struct{}{}: + case <-ctx.Done(): + return nil, ctx.Err() + } + var once sync.Once + return func() { once.Do(func() { <-slots }) }, nil +} + +// RunAcquired requires a reservation held until this call returns. +func RunAcquired(ctx context.Context, video, audio Input, out io.Writer, format string, extra []string) error { + args, err := Arguments(format, extra) + if err != nil { + return err + } + ready := make(chan struct{}) + go func() { initialize(); close(ready) }() + select { + case <-ready: + case <-ctx.Done(): + return ctx.Err() + } + if shared.err != nil { + return shared.err + } + files := &inputFS{files: map[string]Input{"video": video, "audio": audio}} + fsconfig := wazero.NewFSConfig().(sysfs.FSConfig).WithSysFSMount(files, "/inputs") + logs := &tailLog{} + rc, err := wasm.Run(ctx, shared.runtime, shared.module, wasm.Args{ + Name: "ffmpeg", Args: args, Stdout: out, Stderr: logs, + Config: func(c wazero.ModuleConfig) wazero.ModuleConfig { return c.WithFSConfig(fsconfig) }, + }) + if ctx.Err() != nil { + return ctx.Err() + } + if files.err != nil { + return fmt.Errorf("ffmpeg input: %w (%s)", files.err, logs.String()) + } + if err != nil { + return fmt.Errorf("ffmpeg runtime: %w (%s)", err, logs.String()) + } + if rc != 0 { + return fmt.Errorf("ffmpeg exited with code %d: %s", rc, logs.String()) + } + return nil +} + +type tailLog struct{ data []byte } + +func (l *tailLog) Write(p []byte) (int, error) { + n := len(p) + if len(p) > 8192 { + p = p[len(p)-8192:] + } + l.data = append(l.data, p...) + if len(l.data) > 8192 { + l.data = l.data[len(l.data)-8192:] + } + return n, nil +} +func (l *tailLog) String() string { return strings.TrimSpace(string(l.data)) } + +type inputFS struct { + expsys.UnimplementedFS + files map[string]Input + err error +} + +func (f *inputFS) OpenFile(path string, flag expsys.Oflag, perm fs.FileMode) (expsys.File, expsys.Errno) { + if flag&(expsys.O_WRONLY|expsys.O_RDWR|expsys.O_CREAT|expsys.O_TRUNC) != 0 { + return nil, expsys.EROFS + } + path = strings.TrimPrefix(path, "/") + if path == "." || path == "" { + return &inputFile{owner: f}, 0 + } + in := f.files[path] + if in == nil { + return nil, expsys.ENOENT + } + return &inputFile{owner: f, in: in}, 0 +} +func (f *inputFS) Stat(path string) (sys.Stat_t, expsys.Errno) { + file, err := f.OpenFile(path, 0, 0) + if err != 0 { + return sys.Stat_t{}, err + } + return file.Stat() +} + +type inputFile struct { + expsys.UnimplementedFile + owner *inputFS + in Input +} + +func (f *inputFile) IsDir() (bool, expsys.Errno) { return f.in == nil, 0 } +func (f *inputFile) Stat() (sys.Stat_t, expsys.Errno) { + if f.in == nil { + return sys.Stat_t{Mode: fs.ModeDir | 0555}, 0 + } + mode := fs.FileMode(0444) + if !f.in.Seekable() { + mode |= fs.ModeNamedPipe + } + return sys.Stat_t{Mode: mode, Size: max(0, f.in.Size()), Nlink: 1}, 0 +} +func (f *inputFile) Read(p []byte) (int, expsys.Errno) { + if f.in == nil { + return 0, expsys.EISDIR + } + n, err := f.in.Read(p) + if err != nil && err != io.EOF { + f.owner.err = err + return n, expsys.EIO + } + return n, 0 +} +func (f *inputFile) Seek(off int64, whence int) (int64, expsys.Errno) { + if f.in == nil { + return 0, expsys.EISDIR + } + n, err := f.in.Seek(off, whence) + // Capability probes may legitimately fail; a failed seek alone is not a + // media error. FFmpeg decides whether it can continue sequentially. + if err != nil { + return 0, expsys.ENOSYS + } + return n, 0 +} diff --git a/pkg/download/engine/ffmpeg/runtime_test.go b/pkg/download/engine/ffmpeg/runtime_test.go new file mode 100644 index 00000000..ee57c180 --- /dev/null +++ b/pkg/download/engine/ffmpeg/runtime_test.go @@ -0,0 +1,215 @@ +package ffmpeg + +import ( + "bytes" + "context" + "encoding/binary" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + "time" + + "codeberg.org/gruf/go-ffmpreg/wasm" + "github.com/tetratelabs/wazero" +) + +// Generate tiny deterministic media with the exact embedded build under test; +// no system FFmpeg, network media or third-party fixture licensing is needed. +func fixture(t *testing.T, audio bool) []byte { + t.Helper() + initialize() + if shared.err != nil { + t.Fatal(shared.err) + } + args := []string{"-hide_banner", "-loglevel", "error", "-f", "lavfi", "-i", "color=c=black:s=32x32:r=10", "-t", "0.3", "-c:v", "libx264", "-preset", "ultrafast", "-movflags", "frag_keyframe+empty_moov", "-f", "mp4", "pipe:1"} + if audio { + args = []string{"-hide_banner", "-loglevel", "error", "-f", "lavfi", "-i", "anullsrc=r=48000:cl=mono", "-t", "0.3", "-c:a", "aac", "-movflags", "frag_keyframe+empty_moov", "-f", "mp4", "pipe:1"} + } + var out, logs bytes.Buffer + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + rc, err := wasm.Run(ctx, shared.runtime, shared.module, wasm.Args{Name: "ffmpeg", Args: args, Stdout: &out, Stderr: &logs}) + if err != nil || rc != 0 { + t.Fatalf("fixture: %d %v %s", rc, err, logs.String()) + } + return out.Bytes() +} + +func TestTailMoovNeedsRange(t *testing.T) { + initialize() + if shared.err != nil { + t.Fatal(shared.err) + } + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + dir := t.TempDir() + var logs bytes.Buffer + rc, err := wasm.Run(ctx, shared.runtime, shared.module, wasm.Args{Name: "ffmpeg", Args: []string{ + "-v", "error", "-f", "lavfi", "-i", "color=c=black:s=32x32:r=10", "-t", "0.3", "-c:v", "libx264", "-preset", "ultrafast", "/out/video.mp4", + }, Stderr: &logs, Config: func(c wazero.ModuleConfig) wazero.ModuleConfig { + return c.WithFSConfig(wazero.NewFSConfig().WithDirMount(dir, "/out")) + }}) + if rc != 0 || err != nil { + t.Fatalf("fixture %d %v %s", rc, err, logs.String()) + } + raw, err := os.ReadFile(filepath.Join(dir, "video.mp4")) + if err != nil { + t.Fatal(err) + } + var video []byte + for offset := 0; offset < len(raw); { + length := int(binary.BigEndian.Uint32(raw[offset : offset+4])) + if string(raw[offset+4:offset+8]) == "moov" { + free := make([]byte, 2*httpBlockSize) + binary.BigEndian.PutUint32(free, uint32(len(free))) + copy(free[4:], "free") + video = append(video, free...) + } + video = append(video, raw[offset:offset+length]...) + offset += length + } + audio := fixture(t, true) + for _, ranged := range []bool{true, false} { + t.Run(fmtBool(ranged), func(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + data := video + if r.URL.Path == "/audio" { + data = audio + } + if ranged { + http.ServeContent(w, r, "media", time.Time{}, bytes.NewReader(data)) + } else { + w.Write(data) + } + })) + defer srv.Close() + v, err := OpenHTTP(ctx, srv.Client(), HTTPSource{URL: srv.URL}) + if err != nil { + t.Fatal(err) + } + defer v.Close() + a, err := OpenHTTP(ctx, srv.Client(), HTTPSource{URL: srv.URL + "/audio"}) + if err != nil { + t.Fatal(err) + } + defer a.Close() + err = Run(ctx, v, a, io.Discard, "mp4", nil) + if ranged && err != nil { + t.Fatal(err) + } + if !ranged && err == nil { + t.Fatal("non-seekable tail-moov input unexpectedly succeeded") + } + }) + } +} + +func fmtBool(b bool) string { + if b { + return "range" + } + return "sequential" +} + +func TestWebMToMP4(t *testing.T) { + initialize() + if shared.err != nil { + t.Fatal(shared.err) + } + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + inputs := make([]*StreamInput, 2) + for i, name := range []string{"video.webm", "audio.webm"} { + data, err := os.ReadFile(filepath.Join("..", "testdata", "ffmpeg", name)) + if err != nil { + t.Fatal(err) + } + inputs[i] = NewStreamInput(ctx) + defer inputs[i].Close() + if _, err = inputs[i].Write(data); err != nil { + t.Fatal(err) + } + inputs[i].End(nil) + } + + var output, probe, logs bytes.Buffer + if err := Run(ctx, inputs[0], inputs[1], &output, "mp4", nil); err != nil { + t.Fatal(err) + } + rc, err := wasm.Run(ctx, shared.runtime, shared.module, wasm.Args{Name: "ffprobe", Args: []string{"-v", "error", "-show_entries", "stream=codec_name", "-of", "json", "pipe:0"}, Stdin: bytes.NewReader(output.Bytes()), Stdout: &probe, Stderr: &logs}) + if rc != 0 || err != nil { + t.Fatalf("probe %d %v %s", rc, err, logs.String()) + } + if !bytes.Contains(probe.Bytes(), []byte(`"vp9"`)) || !bytes.Contains(probe.Bytes(), []byte(`"opus"`)) { + t.Fatalf("codecs changed: %s", probe.String()) + } +} + +func TestWASMMerge(t *testing.T) { + video, audio := fixture(t, false), fixture(t, true) + for _, ranged := range []bool{false, true} { + t.Run(map[bool]string{false: "sequential", true: "range"}[ranged], func(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + data := video + if r.URL.Path == "/audio" { + data = audio + } + if ranged { + http.ServeContent(w, r, "media", time.Time{}, bytes.NewReader(data)) + } else { + w.Write(data) + } + })) + defer srv.Close() + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + v, err := OpenHTTP(ctx, srv.Client(), HTTPSource{URL: srv.URL + "/video"}) + if err != nil { + t.Fatal(err) + } + defer v.Close() + a, err := OpenHTTP(ctx, srv.Client(), HTTPSource{URL: srv.URL + "/audio"}) + if err != nil { + t.Fatal(err) + } + defer a.Close() + var output bytes.Buffer + if err = Run(ctx, v, a, &output, "", nil); err != nil { + t.Fatal(err) + } + if !bytes.Contains(output.Bytes(), []byte("moof")) { + t.Fatal("output is not fragmented MP4") + } + // Probe the actual result, rather than just checking its filename/header. + var probe, logs bytes.Buffer + rc, err := wasm.Run(ctx, shared.runtime, shared.module, wasm.Args{Name: "ffprobe", Args: []string{"-v", "error", "-show_streams", "-of", "json", "pipe:0"}, Stdin: bytes.NewReader(output.Bytes()), Stdout: &probe, Stderr: &logs}) + if rc != 0 || err != nil { + t.Fatalf("probe: %d %v %s", rc, err, logs.String()) + } + var result struct { + Streams []struct { + CodecType string `json:"codec_type"` + CodecName string `json:"codec_name"` + } + } + if err = json.Unmarshal(probe.Bytes(), &result); err != nil { + t.Fatal(err) + } + if len(result.Streams) != 2 || result.Streams[0].CodecName != "h264" || result.Streams[1].CodecName != "aac" { + t.Fatalf("unexpected streams %s", probe.String()) + } + }) + } + // A malformed input must produce an error, not successful empty output. + s := NewStreamInput(context.Background()) + defer s.Close() + s.Write([]byte("invalid")) + s.End(nil) + if err := Run(context.Background(), s, s, io.Discard, "", nil); err == nil { + t.Fatal("accepted invalid media") + } +} diff --git a/pkg/download/engine/ffmpeg/stream.go b/pkg/download/engine/ffmpeg/stream.go new file mode 100644 index 00000000..7db44f4f --- /dev/null +++ b/pkg/download/engine/ffmpeg/stream.go @@ -0,0 +1,163 @@ +package ffmpeg + +import ( + "context" + "errors" + "io" + "sync" +) + +// A bounded memory ring lets both JS inputs drain independently, including when +// they share a SABR producer. It does not make a sequential input seekable. +// A full ring waits for consumption or cancellation. Producers must propagate +// backpressure and allow each track to make progress independently. +const StreamBufferLimit = 32 * 1024 * 1024 + +type StreamInput struct { + ctx context.Context + mu sync.Mutex + buffer []byte + read, written int64 + ended, closed bool + err error + wake chan struct{} + space chan struct{} +} + +func NewStreamInput(ctx context.Context) *StreamInput { + return &StreamInput{ctx: ctx, wake: make(chan struct{}, 1), space: make(chan struct{}, 1)} +} + +func (s *StreamInput) Write(p []byte) (int, error) { + s.mu.Lock() + defer s.mu.Unlock() + if len(p) > StreamBufferLimit { + return 0, errors.New("ffmpeg: input chunk exceeds buffer capacity") + } + for int64(len(p)) > StreamBufferLimit-(s.written-s.read) && !s.closed && !s.ended { + s.mu.Unlock() + var err error + select { + case <-s.space: + case <-s.ctx.Done(): + err = s.ctx.Err() + } + s.mu.Lock() + if err != nil { + return 0, err + } + } + if s.closed || s.ended { + return 0, io.ErrClosedPipe + } + if err := s.ctx.Err(); err != nil { + return 0, err + } + if len(p) == 0 { + return 0, nil + } + // Grow only as needed. Rebase unread bytes when the ring grows. + need := int(s.written-s.read) + len(p) + if need > len(s.buffer) { + capacity := min(StreamBufferLimit, max(256*1024, max(need, len(s.buffer)*2))) + buffer := make([]byte, capacity) + unread := int(s.written - s.read) + if unread > 0 { + off := int(s.read % int64(len(s.buffer))) + first := min(unread, len(s.buffer)-off) + copy(buffer, s.buffer[off:off+first]) + copy(buffer[first:], s.buffer[:unread-first]) + } + s.buffer = buffer + s.read, s.written = 0, int64(unread) + } + off := int(s.written % int64(len(s.buffer))) + first := min(len(p), len(s.buffer)-off) + copy(s.buffer[off:], p[:first]) + copy(s.buffer, p[first:]) + s.written += int64(len(p)) + select { + case s.wake <- struct{}{}: + default: + } + return len(p), nil +} + +func (s *StreamInput) End(err error) { + s.mu.Lock() + s.ended = true + if s.err == nil { + s.err = err + } + s.mu.Unlock() + select { + case s.space <- struct{}{}: + default: + } + select { + case s.wake <- struct{}{}: + default: + } +} + +func (s *StreamInput) Read(p []byte) (int, error) { + if len(p) == 0 { + return 0, nil + } + for { + if err := s.ctx.Err(); err != nil { + return 0, err + } + s.mu.Lock() + if s.closed { + s.mu.Unlock() + return 0, io.ErrClosedPipe + } + if s.err != nil { + err := s.err + s.mu.Unlock() + return 0, err + } + if s.read < s.written { + off := s.read % int64(len(s.buffer)) + n := min(int64(len(p)), s.written-s.read, int64(len(s.buffer))-off) + count := copy(p, s.buffer[off:off+n]) + s.read += int64(count) + s.mu.Unlock() + select { + case s.space <- struct{}{}: + default: + } + return count, nil + } + ended := s.ended + s.mu.Unlock() + if ended { + return 0, io.EOF + } + select { + case <-s.wake: + case <-s.ctx.Done(): + return 0, s.ctx.Err() + } + } +} + +func (s *StreamInput) Close() error { + s.mu.Lock() + defer s.mu.Unlock() + s.closed = true + select { + case s.space <- struct{}{}: + default: + } + select { + case s.wake <- struct{}{}: + default: + } + s.buffer = nil + return nil +} +func (*StreamInput) Seek(int64, int) (int64, error) { return 0, ErrSeek } +func (*StreamInput) Size() int64 { return -1 } +func (*StreamInput) Seekable() bool { return false } diff --git a/pkg/download/engine/ffmpeg_integration_test.go b/pkg/download/engine/ffmpeg_integration_test.go new file mode 100644 index 00000000..29c988ce --- /dev/null +++ b/pkg/download/engine/ffmpeg_integration_test.go @@ -0,0 +1,321 @@ +package engine + +import ( + "bytes" + "context" + "fmt" + media "github.com/GopeedLab/gopeed/pkg/download/engine/ffmpeg" + "github.com/GopeedLab/gopeed/pkg/download/engine/inject/stream" + "io" + "net/http" + "net/http/httptest" + "os" + "strings" + "sync/atomic" + "testing" + "time" +) + +func TestFFmpegJSStreams(t *testing.T) { + video, err := os.ReadFile("testdata/ffmpeg/video.mp4") + if err != nil { + t.Fatal(err) + } + audio, err := os.ReadFile("testdata/ffmpeg/audio.mp4") + if err != nil { + t.Fatal(err) + } + for _, mode := range []string{"streams", "http", "mixed"} { + t.Run(mode, func(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + b := video + if r.URL.Path == "/audio" { + b = audio + } + http.ServeContent(w, r, "media", time.Time{}, bytes.NewReader(b)) + })) + defer srv.Close() + e := NewEngine(nil) + defer e.Close() + // Input data is deliberately interleaved through one JS producer. + // Native push/read operations must never block the event loop. + e.Runtime.Set("videoBytes", e.Runtime.NewArrayBuffer(video)) + e.Runtime.Set("audioBytes", e.Runtime.NewArrayBuffer(audio)) + e.Runtime.Set("sourceURL", srv.URL) + e.Runtime.Set("inputMode", mode) + result, err := e.RunString(`(async()=>{ + let vc, ac, offset=0; + const v=new Uint8Array(videoBytes), a=new Uint8Array(audioBytes); + const video=new ReadableStream({start(c){vc=c;}}); + const audio=new ReadableStream({start(c){ac=c;}}); + const timer=setInterval(()=>{ + if(offset=Math.max(v.length,a.length)){clearInterval(timer);vc.close();ac.close();} + },1); + const stream=__gopeed_ffmpeg.merge({ + inputs: () => ({ video:inputMode==='http'?{url:sourceURL+'/video'}:video, + audio:inputMode==='streams'?audio:{url:sourceURL+'/audio'} }), + args:['-metadata','title=gopeed-test'] + }); + const bytes=await new Response(stream).arrayBuffer(); + return new Uint8Array(bytes); + })()`) + if err != nil { + t.Fatal(err) + } + out, ok := result.([]byte) + if !ok { + t.Fatalf("output %T", result) + } + if !bytes.Contains(out, []byte("moof")) || !bytes.Contains(out, []byte("gopeed-test")) { + t.Fatalf("invalid output (%d bytes)", len(out)) + } + }) + } +} + +func TestFFmpegJSErrorAndCancel(t *testing.T) { + for _, scenario := range []string{"input-error", "abort", "output-cancel", "invalid-args"} { + t.Run(scenario, func(t *testing.T) { + e := NewEngine(nil) + defer e.Close() + e.Runtime.Set("scenario", scenario) + result, err := e.RunString(`(async()=>{ + let cancelled=0; + const source=()=>new ReadableStream({ + pull(c){if(scenario==='input-error')c.error(new Error('upstream failed'));else return new Promise(()=>{});}, + cancel(){cancelled++;} + }); + const controller=new AbortController(); + const output=__gopeed_ffmpeg.merge({inputs:()=>({video:source(),audio:source()}),signal:controller.signal, + args:scenario==='invalid-args'?['-i','/outside']:[]}); + const reader=output.getReader(); + const pending=reader.read().then(()=>'',e=>e.message); + if(scenario==='abort')setTimeout(()=>controller.abort(),20); + if(scenario==='output-cancel')setTimeout(()=>reader.cancel(),20); + const message=await pending; + await new Promise(r=>setTimeout(r,10)); + return {message,cancelled}; + })()`) + if err != nil { + t.Fatal(err) + } + m := result.(map[string]any) + if scenario != "output-cancel" && m["message"] == "" { + t.Fatalf("missing error: %v", m) + } + if scenario == "abort" || scenario == "output-cancel" { + if fmt.Sprint(m["cancelled"]) != "2" { + t.Fatalf("inputs not cancelled: %v", m) + } + } + }) + } +} + +func TestFFmpegMalformedMediaFails(t *testing.T) { + e := NewEngine(nil) + defer e.Close() + _, err := e.RunString(`(async()=>{ + const source=()=>new ReadableStream({start(c){c.enqueue(new Uint8Array([1,2,3]));c.close();}}); + return await new Response(__gopeed_ffmpeg.merge({inputs:()=>({video:source(),audio:source()})})).arrayBuffer(); + })()`) + if err == nil || !strings.Contains(err.Error(), "ffmpeg") { + t.Fatalf("expected FFmpeg error: %v", err) + } +} + +func TestFFmpegBlobOpener(t *testing.T) { + var open stream.ObjectURLOpener + e := NewEngine(&Config{StreamConfig: &stream.Config{CreateObjectURL: func(opts *stream.ObjectURLOptions, opener stream.ObjectURLOpener) (string, error) { + if opts.Range || opts.Size != 0 { + t.Fatal("merged output must not advertise range or a guessed size") + } + open = opener + return "blob:ffmpeg-test", nil + }}}) + defer e.Close() + video, _ := os.ReadFile("testdata/ffmpeg/video.mp4") + audio, _ := os.ReadFile("testdata/ffmpeg/audio.mp4") + e.Runtime.Set("videoBytes", e.Runtime.NewArrayBuffer(video)) + e.Runtime.Set("audioBytes", e.Runtime.NewArrayBuffer(audio)) + _, err := e.RunString(`__gopeed_blob_create_object_url(()=>{ + const source=b=>new ReadableStream({start(c){c.enqueue(new Uint8Array(b));c.close();}}); + return __gopeed_ffmpeg.merge({inputs:()=>({video:source(videoBytes),audio:source(audioBytes)})}); + },{contentType:'video/mp4'})`) + if err != nil { + t.Fatal(err) + } + for i := 0; i < 2; i++ { + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + r, err := open(ctx, stream.ObjectURLOpenRequest{End: -1}) + if err != nil { + cancel() + t.Fatal(err) + } + b, err := io.ReadAll(r) + r.Close() + cancel() + if err != nil || !bytes.Contains(b, []byte("moof")) { + t.Fatalf("Blob read: %v (%d bytes)", err, len(b)) + } + } +} + +func TestFFmpegEngineCloseCancelsRequest(t *testing.T) { + started, closed := make(chan struct{}), make(chan struct{}) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + close(started) + <-r.Context().Done() + close(closed) + })) + defer srv.Close() + e := NewEngine(nil) + defer e.Close() + _, err := e.RunString(fmt.Sprintf(` + globalThis.cancelCount=0; + const audio=new ReadableStream({cancel(){cancelCount++;}}); + __gopeed_ffmpeg.merge({inputs:()=>({video:{url:%q},audio})}).getReader().read().catch(()=>{}); + "started";`, srv.URL)) + if err != nil { + t.Fatal(err) + } + select { + case <-started: + case <-time.After(5 * time.Second): + t.Fatal("HTTP did not start") + } + e.Close() + select { + case <-closed: + case <-time.After(5 * time.Second): + t.Fatal("HTTP leaked after engine shutdown") + } + if e.Runtime.Get("cancelCount").ToInteger() != 1 { + t.Fatal("JS input not cancelled on shutdown") + } +} + +func TestFFmpegInputFactoriesWaitForCapacity(t *testing.T) { + for _, mode := range []string{"factory", "http", "cancel"} { + t.Run(mode, func(t *testing.T) { + release1, _ := media.Acquire(context.Background()) + defer release1() + release2, _ := media.Acquire(context.Background()) + defer release2() + var requests atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { requests.Add(1); http.Error(w, "test", 500) })) + defer srv.Close() + e := NewEngine(nil) + defer e.Close() + e.Runtime.Set("mode", mode) + e.Runtime.Set("sourceURL", srv.URL) + result, err := e.RunString(`(async()=>{ + globalThis.factoryCalls=0; + globalThis.abortQueue=new AbortController(); + const options=mode==='http'?{video:{url:sourceURL},audio:{url:sourceURL}}: + {inputs:()=>{factoryCalls++;throw new Error('factory failed');}}; + options.signal=abortQueue.signal; + globalThis.work=__gopeed_ffmpeg.merge(options).getReader().read().then(()=>'',e=>e.message); + await new Promise(r=>setTimeout(r,50)); + if(mode==='cancel')abortQueue.abort(); + return factoryCalls; + })()`) + if err != nil { + t.Fatal(err) + } + if fmt.Sprint(result) != "0" || requests.Load() != 0 { + t.Fatalf("queued inputs started: %v, requests=%d", result, requests.Load()) + } + if mode == "cancel" { + if _, err = e.RunString(`work`); err != nil { + t.Fatal(err) + } + } + release1() + if _, err = e.RunString(`work`); err != nil { + t.Fatal(err) + } + calls, err := e.RunString(`factoryCalls`) + if err != nil { + t.Fatal(err) + } + if mode == "factory" && fmt.Sprint(calls) != "1" { + t.Fatal(calls) + } + if mode == "cancel" && fmt.Sprint(calls) != "0" { + t.Fatal("cancelled factory ran") + } + if mode == "http" && requests.Load() != 1 { + t.Fatal("HTTP did not start after admission") + } + // Failure/cancellation must return the acquired slot even with the other held. + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + release, err := media.Acquire(ctx) + if err != nil { + t.Fatalf("slot leaked: %v", err) + } + release() + }) + } +} + +func TestFFmpegRequiresFactoryForStreams(t *testing.T) { + e := NewEngine(nil) + defer e.Close() + result, err := e.RunString(`(()=>{ + const source=()=>new ReadableStream({}); + let rejected=0; + for(const opts of [ + {video:source(),audio:source()}, + {video:{url:'https://example.test'},audio:source()}, + {inputs:()=>({}),video:{url:'https://example.test'}}, + {inputs:{video:source(),audio:source()}} + ]) {try{__gopeed_ffmpeg.merge(opts);}catch(e){if(e instanceof TypeError)rejected++;}} + return rejected; + })()`) + if err != nil || fmt.Sprint(result) != "4" { + t.Fatalf("validation: %v %v", result, err) + } +} + +func TestFFmpegCancelPendingFactoryCleansLateInputs(t *testing.T) { + e := NewEngine(nil) + defer e.Close() + result, err := e.RunString(`(async()=>{ + let ready,finish,cancelled=0,signal; + const entered=new Promise(r=>ready=r); + const abort=new AbortController(); + const output=__gopeed_ffmpeg.merge({signal:abort.signal,inputs:async opts=>{ + signal=opts.signal;ready();await new Promise(r=>finish=r); + const source=()=>new ReadableStream({cancel(){cancelled++;}}); + return {video:source(),audio:source()}; + }}); + const work=output.getReader().read().catch(e=>e.name); + await entered;abort.abort();await work;finish(); + await new Promise(r=>setTimeout(r,30)); + return {cancelled,aborted:signal.aborted}; + })()`) + if err != nil { + t.Fatal(err) + } + state := result.(map[string]any) + if fmt.Sprint(state["cancelled"]) != "2" || state["aborted"] != true { + t.Fatal(state) + } + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + first, err := media.Acquire(ctx) + if err != nil { + t.Fatal(err) + } + defer first() + second, err := media.Acquire(ctx) + if err != nil { + t.Fatal("factory cancellation leaked capacity") + } + second() +} diff --git a/pkg/download/engine/inject/fetch/module.go b/pkg/download/engine/inject/fetch/module.go index f882fd7c..49ccf8cb 100644 --- a/pkg/download/engine/inject/fetch/module.go +++ b/pkg/download/engine/inject/fetch/module.go @@ -29,8 +29,9 @@ const FingerprintMagicKey = "__gopeed_fetch_fingerprint" var script string type Config struct { - ProxyHandler func(*http.Request) (*url.URL, error) - RegisterCleanup func(func()) + DefaultUserAgent *string + ProxyHandler func(*http.Request) (*url.URL, error) + RegisterCleanup func(func()) } func Enable(runtime *goja.Runtime, loop *eventloop.EventLoop, cfg *Config) error { @@ -51,6 +52,9 @@ func Enable(runtime *goja.Runtime, loop *eventloop.EventLoop, cfg *Config) error if err != nil { panic(runtime.NewGoError(err)) } + if cfg.DefaultUserAgent != nil && !hasHeader(request.Headers, "User-Agent") { + request.Headers = append(request.Headers, [2]string{"User-Agent", *cfg.DefaultUserAgent}) + } fingerprint := util.SafeGet[string](runtime, FingerprintMagicKey) promise, resolve, reject := runtime.NewPromise() go func() { diff --git a/pkg/download/engine/inject/ffmpeg/ffmpeg.js b/pkg/download/engine/inject/ffmpeg/ffmpeg.js new file mode 100644 index 00000000..49b42f25 --- /dev/null +++ b/pkg/download/engine/inject/ffmpeg/ffmpeg.js @@ -0,0 +1,139 @@ +(function () { + const create = __gopeed_ffmpeg_create; + const read = __gopeed_ffmpeg_read; + const acquire = __gopeed_ffmpeg_acquire; + const start = __gopeed_ffmpeg_start; + const push = __gopeed_ffmpeg_push; + const end = __gopeed_ffmpeg_end; + const cancel = __gopeed_ffmpeg_cancel; + const active = new Set(); + globalThis.__gopeed_ffmpeg_close_all = () => { + for (const stop of Array.from(active)) stop(new Error("extension engine closed")); + }; + + function input(value, name, allowStream) { + if (value && typeof value.getReader === "function") { + if (!allowStream) throw new TypeError("stream inputs must be created by the inputs callback"); + if (value.locked) throw new TypeError(name + " stream is locked"); + return { stream: value, request: null }; + } + if (!value || typeof value.url !== "string" || !/^https?:\/\//i.test(value.url)) { + throw new TypeError(name + " must be an HTTP request or ReadableStream"); + } + const headers = {}; + new Headers(value.headers || {}).forEach((v, k) => { headers[k] = v; }); + return { request: { url: value.url, headers } }; + } + + function merge(options) { + if (!options || typeof options !== "object") throw new TypeError("merge options are required"); + const factory = options.inputs; + if (factory !== undefined && typeof factory !== "function") throw new TypeError("inputs must be a callback"); + if (factory && (options.video !== undefined || options.audio !== undefined)) throw new TypeError("inputs cannot be combined with video/audio"); + let video = factory ? null : input(options.video, "video", false); + let audio = factory ? null : input(options.audio, "audio", false); + const args = options.args === undefined ? [] : options.args; + if (!Array.isArray(args) || args.some(v => typeof v !== "string")) throw new TypeError("args must be a string array"); + const format = options.format === undefined ? "mp4" : options.format; + if (typeof format !== "string") throw new TypeError("format must be a string"); + const signal = options.signal; + let id, controller, stopped = false; + const readers = []; + const lifetime = new AbortController(); + let ownedInputs; + function discardInputs(inputs, reason) { + for (const source of new Set([inputs && inputs.video, inputs && inputs.audio])) { + if (source && typeof source.cancel === "function" && !source.locked) Promise.resolve(source.cancel(reason)).catch(() => {}); + } + } + + function stop(reason) { + if (stopped) return; + stopped = true; + active.delete(shutdown); + lifetime.abort(); + discardInputs(ownedInputs, reason); + if (id) cancel(id); + if (signal) signal.removeEventListener("abort", abort); + for (const reader of readers) { + Promise.resolve(reader.cancel(reason)).catch(() => {}).then(() => { + try { reader.releaseLock(); } catch (_) {} + }); + } + } + function shutdown(reason) { + controller.error(reason); + stop(reason); + } + function abort() { + const error = new Error("FFmpeg operation aborted"); + error.name = "AbortError"; + stop(error); + controller.error(error); + } + async function pump(reader, index) { + try { + while (!stopped) { + const result = await reader.read(); + if (stopped) return; + if (result.done) { end(id, index, ""); return; } + if (!(result.value instanceof Uint8Array)) throw new TypeError("media streams must contain Uint8Array chunks"); + // Keep individual crossings bounded, even if upstream emits a large chunk. + for (let offset = 0; offset < result.value.byteLength && !stopped; offset += 256 * 1024) { + if (!await push(id, index, result.value.subarray(offset, offset + 256 * 1024))) return; + } + } + } catch (error) { + if (!stopped) { + end(id, index, String(error && error.message || error)); + controller.error(error); + stop(error); + } + } + } + return new ReadableStream({ + start(c) { + controller = c; + active.add(shutdown); + if (signal) { + if (signal.aborted) abort(); + else signal.addEventListener("abort", abort); + } + }, + async pull(c) { + if (stopped) return; + try { + if (!id) { + id = create({ format, args }); + await acquire(id); + if (stopped) return; + if (factory) { + const inputs = await factory({ signal: lifetime.signal }); + if (stopped) { discardInputs(inputs, new Error("FFmpeg operation cancelled")); return; } + ownedInputs = inputs; + video = input(inputs && inputs.video, "video", true); + audio = input(inputs && inputs.audio, "audio", true); + } + if (video.stream && video.stream === audio.stream) throw new TypeError("video and audio must be distinct streams"); + for (const source of [video, audio]) { + if (source.stream) readers.push(source.stream.getReader()); + } + start(id, { video: video.request, audio: audio.request, format, args }); + let readerIndex = 0; + for (const [index, source] of [video, audio].entries()) { + if (source.stream) void pump(readers[readerIndex++], index); + } + } + const chunk = await read(id); + if (stopped) return; + if (chunk === null) { stop(); c.close(); } + else c.enqueue(new Uint8Array(chunk)); + } catch (error) { + if (!stopped) { c.error(error); stop(error); } + } + }, + cancel(reason) { stop(reason); } + }); + } + globalThis.__gopeed_ffmpeg = Object.freeze({ merge, supportsInputFactory: true }); +})(); diff --git a/pkg/download/engine/inject/ffmpeg/module.go b/pkg/download/engine/inject/ffmpeg/module.go new file mode 100644 index 00000000..cc73e990 --- /dev/null +++ b/pkg/download/engine/inject/ffmpeg/module.go @@ -0,0 +1,324 @@ +package ffmpeg + +import ( + "context" + _ "embed" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "sync" + "sync/atomic" + + "github.com/GopeedLab/gopeed/internal/httpclient" + media "github.com/GopeedLab/gopeed/pkg/download/engine/ffmpeg" + "github.com/dop251/goja" + "github.com/dop251/goja_nodejs/eventloop" +) + +//go:embed ffmpeg.js +var script string + +type Config struct { + DefaultUserAgent *string + ProxyHandler func(*http.Request) (*url.URL, error) + RegisterCleanup func(func()) +} +type options struct { + Video *media.HTTPSource `json:"video"` + Audio *media.HTTPSource `json:"audio"` + Format string `json:"format"` + Args []string `json:"args"` +} +type job struct { + ctx context.Context + cancel context.CancelFunc + output *io.PipeReader + streams [2]*media.StreamInput + mu sync.Mutex + ready chan struct{} + start chan options + started bool +} + +func (j *job) stop() { + j.cancel() + j.output.CloseWithError(context.Canceled) + j.mu.Lock() + defer j.mu.Unlock() + for _, s := range j.streams { + if s != nil { + s.Close() + } + } +} + +func Enable(vm *goja.Runtime, loop *eventloop.EventLoop, cfg *Config) error { + if cfg == nil { + cfg = &Config{} + } + client, err := httpclient.NewClient(httpclient.Options{Transport: httpclient.TransportOptions{Proxy: cfg.ProxyHandler}}) + if err != nil { + return err + } + var mu sync.Mutex + jobs := map[string]*job{} + var next atomic.Uint64 + get := func(id string) *job { mu.Lock(); defer mu.Unlock(); return jobs[id] } + remove := func(id string) { + mu.Lock() + j := jobs[id] + delete(jobs, id) + mu.Unlock() + if j != nil { + j.stop() + } + } + if cfg.RegisterCleanup != nil { + cfg.RegisterCleanup(func() { + done := make(chan struct{}) + if loop.RunOnLoop(func(vm *goja.Runtime) { + defer close(done) + if closeAll, ok := goja.AssertFunction(vm.Get("__gopeed_ffmpeg_close_all")); ok { + _, _ = closeAll(goja.Undefined()) + } + }) { + <-done + } + mu.Lock() + old := jobs + jobs = map[string]*job{} + mu.Unlock() + for _, j := range old { + j.stop() + } + client.CloseIdleConnections() + }) + } + if err := vm.Set("__gopeed_ffmpeg_create", func(call goja.FunctionCall) goja.Value { + var opts options + if err := vm.ExportTo(call.Argument(0), &opts); err != nil { + panic(vm.NewGoError(err)) + } + if _, err := media.Arguments(opts.Format, opts.Args); err != nil { + panic(vm.NewGoError(err)) + } + ctx, cancel := context.WithCancel(context.Background()) + r, w := io.Pipe() + j := &job{ctx: ctx, cancel: cancel, output: r, ready: make(chan struct{}), start: make(chan options, 1)} + id := fmt.Sprintf("ffmpeg-%d", next.Add(1)) + mu.Lock() + jobs[id] = j + mu.Unlock() + go func() { + var inputs [2]media.Input + var runErr error + defer func() { + if v := recover(); v != nil { + runErr = fmt.Errorf("ffmpeg panic: %v", v) + } + cancel() + for _, in := range inputs { + if in != nil { + in.Close() + } + } + w.CloseWithError(runErr) + }() + release, err := media.Acquire(ctx) + if err != nil { + runErr = err + return + } + defer release() + close(j.ready) + var opts options + select { + case opts = <-j.start: + case <-ctx.Done(): + runErr = ctx.Err() + return + } + for i, source := range []*media.HTTPSource{opts.Video, opts.Audio} { + if source == nil { + inputs[i] = j.streams[i] + } else { + inputs[i], runErr = media.OpenHTTP(ctx, client, *source) + if runErr != nil { + return + } + } + } + runErr = media.RunAcquired(ctx, inputs[0], inputs[1], w, opts.Format, opts.Args) + }() + return vm.ToValue(id) + }); err != nil { + return err + } + if err := vm.Set("__gopeed_ffmpeg_acquire", func(id string) goja.Value { + promise, resolve, reject := vm.NewPromise() + j := get(id) + go func() { + var err error + if j == nil { + err = io.ErrClosedPipe + } else { + select { + case <-j.ready: + case <-j.ctx.Done(): + err = j.ctx.Err() + } + if j.ctx.Err() != nil { + err = j.ctx.Err() + } + } + loop.RunOnLoop(func(vm *goja.Runtime) { + if err != nil { + reject(vm.NewGoError(err)) + } else { + resolve(true) + } + }) + }() + return vm.ToValue(promise) + }); err != nil { + return err + } + if err := vm.Set("__gopeed_ffmpeg_start", func(call goja.FunctionCall) goja.Value { + id := call.Argument(0).String() + j := get(id) + if j == nil { + panic(vm.NewGoError(io.ErrClosedPipe)) + } + var opts options + if err := vm.ExportTo(call.Argument(1), &opts); err != nil { + panic(vm.NewGoError(err)) + } + if _, err := media.Arguments(opts.Format, opts.Args); err != nil { + panic(vm.NewGoError(err)) + } + for _, source := range []*media.HTTPSource{opts.Video, opts.Audio} { + if source == nil || cfg.DefaultUserAgent == nil { + continue + } + specified := false + for key := range source.Headers { + if strings.EqualFold(key, "User-Agent") { + specified = true + break + } + } + if !specified { + if source.Headers == nil { + source.Headers = make(map[string]string) + } + source.Headers["User-Agent"] = *cfg.DefaultUserAgent + } + } + + j.mu.Lock() + defer j.mu.Unlock() + if j.started || j.ctx.Err() != nil { + panic(vm.NewGoError(io.ErrClosedPipe)) + } + j.started = true + for i, source := range []*media.HTTPSource{opts.Video, opts.Audio} { + if source == nil { + j.streams[i] = media.NewStreamInput(j.ctx) + } + } + j.start <- opts + return goja.Undefined() + }); err != nil { + return err + } + if err := vm.Set("__gopeed_ffmpeg_read", func(id string) goja.Value { + promise, resolve, reject := vm.NewPromise() + go func() { + var readErr error + buf := make([]byte, 64*1024) + n := 0 + j := get(id) + if j == nil { + readErr = io.ErrClosedPipe + } else { + n, readErr = j.output.Read(buf) + } + if !loop.RunOnLoop(func(vm *goja.Runtime) { + if readErr != nil { + remove(id) + if readErr == io.EOF { + resolve(goja.Null()) + } else { + reject(vm.NewGoError(readErr)) + } + return + } + resolve(vm.ToValue(vm.NewArrayBuffer(buf[:n]))) + }) { + remove(id) + } + }() + return vm.ToValue(promise) + }); err != nil { + return err + } + if err := vm.Set("__gopeed_ffmpeg_push", func(call goja.FunctionCall) goja.Value { + id := call.Argument(0).String() + index := int(call.Argument(1).ToInteger()) + var chunk []byte + switch b := call.Argument(2).Export().(type) { + case goja.ArrayBuffer: + chunk = append([]byte(nil), b.Bytes()...) + case []byte: + chunk = append([]byte(nil), b...) + default: + panic(vm.NewTypeError("ffmpeg input requires bytes")) + } + promise, resolve, reject := vm.NewPromise() + go func() { + var err error + j := get(id) + if j == nil || index < 0 || index > 1 || j.streams[index] == nil { + err = io.ErrClosedPipe + } else { + _, err = j.streams[index].Write(chunk) + } + if err != nil && j != nil && index >= 0 && index < 2 && j.streams[index] != nil { + j.streams[index].End(err) + } + loop.RunOnLoop(func(vm *goja.Runtime) { + if j != nil && j.ctx.Err() != nil { + resolve(false) + return + } + if err != nil { + reject(vm.NewGoError(err)) + } else { + resolve(true) + } + }) + }() + return vm.ToValue(promise) + }); err != nil { + return err + } + if err := vm.Set("__gopeed_ffmpeg_end", func(id string, index int, message string) { + if j := get(id); j != nil && index >= 0 && index < 2 && j.streams[index] != nil { + var err error + if message != "" { + err = errors.New(message) + } + j.streams[index].End(err) + } + }); err != nil { + return err + } + if err := vm.Set("__gopeed_ffmpeg_cancel", remove); err != nil { + return err + } + _, err = vm.RunString(script) + return err +} diff --git a/pkg/download/engine/inject/stream/module.go b/pkg/download/engine/inject/stream/module.go index 5aaa214f..89db11fc 100644 --- a/pkg/download/engine/inject/stream/module.go +++ b/pkg/download/engine/inject/stream/module.go @@ -86,18 +86,23 @@ func Enable(runtime *goja.Runtime, loop *eventloop.EventLoop, cfg *Config) error } if err := runtime.Set("__gopeed_blob_pipe_chunk", func(call goja.FunctionCall) goja.Value { id := call.Argument(0).String() - ok := blobPipes.Push(id, exportBytes(call.Argument(1))) - return runtime.ToValue(ok) + chunk := append([]byte(nil), exportBytes(call.Argument(1))...) + promise, resolve, _ := runtime.NewPromise() + go func() { + ok := blobPipes.Push(id, chunk) + loop.RunOnLoop(func(vm *goja.Runtime) { resolve(ok) }) + }() + return runtime.ToValue(promise) }); err != nil { return err } if err := runtime.Set("__gopeed_blob_pipe_close", func(id string) { - blobPipes.Close(id, nil) + go blobPipes.Close(id, nil) }); err != nil { return err } if err := runtime.Set("__gopeed_blob_pipe_error", func(id string, message string) { - blobPipes.Close(id, errors.New(message)) + go blobPipes.Close(id, errors.New(message)) }); err != nil { return err } diff --git a/pkg/download/engine/inject/stream/module_integration_test.go b/pkg/download/engine/inject/stream/module_integration_test.go index b2c11cec..e9e07411 100644 --- a/pkg/download/engine/inject/stream/module_integration_test.go +++ b/pkg/download/engine/inject/stream/module_integration_test.go @@ -269,3 +269,57 @@ func newStreamTestEngine(t *testing.T, created chan<- createdObjectURL) (*engine }) return e, e.Close } + +// A stalled disk/network consumer must pause the producer without blocking +// the JS event loop, so cancellation and other tracks can still run. +func TestBlobBackpressureLeavesEventLoopResponsive(t *testing.T) { + created := make(chan createdObjectURL, 1) + e, cleanup := newStreamTestEngine(t, created) + defer cleanup() + _, err := e.RunString(` + globalThis.produced = 0; + globalThis.sourceCancelled = false; + __gopeed_blob_create_object_url(() => new ReadableStream({ + pull(c) { produced++; c.enqueue(new Uint8Array(64*1024)); }, + cancel() { sourceCancelled = true; } + })); + `) + if err != nil { + t.Fatal(err) + } + object := <-created + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + reader, err := object.open(ctx, stream.ObjectURLOpenRequest{End: -1}) + if err != nil { + t.Fatal(err) + } + defer reader.Close() + time.Sleep(100 * time.Millisecond) + value, err := e.RunString("produced") + if err != nil { + t.Fatal(err) + } + before := value.(int64) + if before > 10 { + t.Fatalf("unbounded production: %d", before) + } + time.Sleep(50 * time.Millisecond) + after, err := e.RunString("produced") + if err != nil || after != value { + t.Fatalf("producer did not pause: %v -> %v (%v)", value, after, err) + } + if ctx.Err() != nil { + t.Fatal("event loop only resumed after cancellation") + } + if _, err = reader.Read(make([]byte, 64*1024)); err != nil { + t.Fatal(err) + } + time.Sleep(30 * time.Millisecond) + after, err = e.RunString("produced") + if err != nil || after.(int64) <= before { + t.Fatalf("producer did not resume: %v (%v)", after, err) + } + reader.Close() + waitForJSValue(t, e, "String(sourceCancelled)", "true") +} diff --git a/pkg/download/engine/inject/stream/stream.js b/pkg/download/engine/inject/stream/stream.js index ea3bd85b..829a9344 100644 --- a/pkg/download/engine/inject/stream/stream.js +++ b/pkg/download/engine/inject/stream/stream.js @@ -105,6 +105,10 @@ return Promise.resolve({ done: true, value: undefined }); } this._maybePull(); + // pull() may synchronously error before a waiter has been registered. + if (this._errored) { + return Promise.reject(this._errored); + } if (this._queue.length > 0) { return Promise.resolve({ done: false, value: this._queue.shift() }); } @@ -760,11 +764,14 @@ return; } const chunk = value instanceof Uint8Array ? value : new Uint8Array(value); - const buffer = chunk.buffer.slice(chunk.byteOffset, chunk.byteOffset + chunk.byteLength); - if (!globalThis.__gopeed_blob_pipe_chunk(pipeId, buffer)) { - state.cancelRequested = true; - state.cancelReason = "blob pipe closed"; - return; + for (let offset = 0; offset < chunk.byteLength; offset += 256 * 1024) { + const part = chunk.subarray(offset, offset + 256 * 1024); + const buffer = part.buffer.slice(part.byteOffset, part.byteOffset + part.byteLength); + if (!await globalThis.__gopeed_blob_pipe_chunk(pipeId, buffer)) { + state.cancelRequested = true; + state.cancelReason = "blob pipe closed"; + return; + } } await yieldBlobPipeTask(); } diff --git a/pkg/download/engine/testdata/ffmpeg/README.md b/pkg/download/engine/testdata/ffmpeg/README.md new file mode 100644 index 00000000..ba3e5992 --- /dev/null +++ b/pkg/download/engine/testdata/ffmpeg/README.md @@ -0,0 +1,18 @@ +# Synthetic media fixtures + +These files contain 0.3 seconds of a generated 32x32 black image (H.264) and +generated mono silence (AAC). They contain no third-party media. Generated +with the pinned go-ffmpreg FFmpeg binary using the equivalent commands: + +``` +ffmpeg -f lavfi -i color=c=black:s=32x32:r=10 -t 0.3 -c:v libx264 -preset ultrafast -movflags frag_keyframe+empty_moov -f mp4 video.mp4 +ffmpeg -f lavfi -i anullsrc=r=48000:cl=mono -t 0.3 -c:a aac -movflags frag_keyframe+empty_moov -f mp4 audio.mp4 +``` + +The backend tests also generate and probe media using the embedded FFmpeg and +FFprobe at test time, without relying on a system installation. + +`video.webm` and `audio.webm` use the same black/silent sources, encoded once +with native FFmpeg 7.1 using `-c:v libvpx-vp9` and `-c:a libopus`. The committed +fixtures let the tests verify WASM stream-copy remuxing without exercising its +VP9 encoder or requiring native FFmpeg on developer/CI machines. diff --git a/pkg/download/engine/testdata/ffmpeg/audio.mp4 b/pkg/download/engine/testdata/ffmpeg/audio.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1a16c3c499092f37e57bfd45338419dbb79fe23b GIT binary patch literal 1125 zcmcgrJ#W-N5FPuH&;UB5i=Y6>NQs1m5?ulWrL&ZVB3%?DJ{sSxt(@XLpLX|7K!L8z z58zMm8xW$QqNYu{NGVe02k_q7&N&tp1*5ayn>Vww@*Tv)qEuI zVms3E<3tT~nq0P??l*8r!=)HF;F=U)BwB@XyxSdeF=n;~>~5jecpNbVl2q$PhbUbWcTOFjbCU_Ex7JsXcCv8%Jc4$%Iq6wvAD4+4TBXS8&ru zD)Ebb@%?0$z1ZE{e!02T?<07L%j?fWXV>S!#NW7J)Yn|8?}As|g{LA*aAlPvg7sXc zToo)8kuT2hB6KM89Iom$6{3Kz@gGoP?9a{KXiVZP6wF|V89Xaz5ccD{!Gq}5zdk6u er!$iPpE_VQhO)*!0&1qy%+B&DXG%uy2k)f^4G2TBQG&pQkbF$uou-z>zN479DH!?k( zIKPo`Ylp-0032Fri~=1+0){Nw`-=_h8oE^c@A zbM|*}4RUEmaa(cm_2dKH$p}}ZpIC&fcq-6e$v}S@T7o>Lkdv8|540=s7_9WJ3k zjv=1@elG2k4GgTnNYB{7!T<~yg)t;Sj0F=Gwl^~BG60#Si&_}Y{!0KQmWjLj UH#ahD?*b;uk4=mRKQuA}0Cf=pS^xk5 literal 0 HcmV?d00001 diff --git a/pkg/download/engine/testdata/ffmpeg/video.mp4 b/pkg/download/engine/testdata/ffmpeg/video.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d5d3d385a25b6de9ff30292a620533189184c243 GIT binary patch literal 1597 zcmaJ>F>f426rQ__kzz%PK?YF}X%J3iiQL`p`h1Bkt>oksZ~+nxDJ;ZkZfDlJx}Dwi z%-s1dqM&erM2SQ}g+xV1M}9Nj zRokQ=0vCrOmzg3?qpQ!T!cNTNj%+ zUr8y}8;K^wNm3q|!Z?cA8lSbn{zR(4{w| zs1DNw3~@BHyw|2(74 z@ivbNqqyJSg?ZS4tNZ>358iF}_`Qb%#H5N47!>6^Q?cWL=k2ruFCcc}T)E%vo}Qj| zXnKoG>Qm>lEVW?Vo!ym@ zgAtu*>{1E&sK`|q__uwZ$HKZ{X~$`aL+1%j%3%@5R?#7s7#y$&W<@a;3Bd5eXR|cI zQ)-9Db5mUrrCK<8Mw&ZiG653&qikZtd>9qF6xD-J8_YqPBL#>?5GOW5D|+{-N{90GlX`Ow9ENg`iRbguzb7<0o`|=ym!q zOUhGeF$?>9a9F3xu*cI9RYmVm^d35N5_;@YOw~2@+1dWBZFaT!%Woe8uig8K2cIvE igFYDpjrT*p?V!(RwjSMDBTUj-Z&U-0Ps7w7@8DmJ^hj<1 literal 0 HcmV?d00001 diff --git a/pkg/download/engine/testdata/ffmpeg/video.webm b/pkg/download/engine/testdata/ffmpeg/video.webm new file mode 100644 index 0000000000000000000000000000000000000000..f88aa145d40f4ecc0a071bf83eba72ede9baa089 GIT binary patch literal 579 zcmb1gy}x+AQ(GgW({~{L)X3uWxsk)EsiizMDc7mJk;$pGkx3%BA)S!{1ek;ceP@^K z^4;AXyt+lyb7flan#P3?o><7bY#{HN@9rL;oKVQ&^x!p3jLHzMOz>`?WE11xHxP3e zBEiPdf&jT{gVyzp&HPRdz70J-iDhYKhI;0Dh6V=VjwoE0&JKs717J6R9DiaR)NS_H z8ySm_c8GF+zdmuhsl}W|2BocYnoILiIvN?;TEpVQ0xX*w8I>P5x3GSF$napohDL>5 zjS90GnL;ezPNaP@_~l*6Ej^Gx4Zf|`@6UXxwNC$UtD}E`9OCv!lv{S zOOX}N0J=RH=ypR(kmD6{GLy;*Ds;;VEYnXcbY0xx5*p+f;_2__(mvV1zzU4?j18cG zLEd>=>4oQw42u^wGB7kWc{V#SFn(bC#$cf$BjLrsV35GTAaMWx-$>C0hQ*T`8B!S9 w7#t?eVF+MgU{+vYaCQI*o`4GaJ%9>=qH*Hx{>_aH+q;0t ({ hasRuntime: !!gopeed.runtime, hasBlob: !!(gopeed.runtime && gopeed.runtime.blob), + hasFFmpegMerge: typeof gopeed.runtime.ffmpeg.merge, hasCreateObjectURL: typeof gopeed.runtime.blob.createObjectURL, hasRevokeObjectURL: typeof gopeed.runtime.blob.revokeObjectURL, hasWebView: !!(gopeed.runtime && gopeed.runtime.webview), @@ -1502,6 +1503,9 @@ func TestDownloader_ExtensionRuntimeWebViewInjected(t *testing.T) { if result["hasRuntime"] != true { t.Fatalf("expected runtime injection, got %#v", result) } + if result["hasFFmpegMerge"] != "function" { + t.Fatalf("expected ffmpeg injection, got %#v", result) + } if result["hasBlob"] != true { t.Fatalf("expected blob injection, got %#v", result) } diff --git a/pkg/download/extension_unnamed_resource_test.go b/pkg/download/extension_unnamed_resource_test.go new file mode 100644 index 00000000..7ef765eb --- /dev/null +++ b/pkg/download/extension_unnamed_resource_test.go @@ -0,0 +1,37 @@ +package download + +import ( + "github.com/GopeedLab/gopeed/pkg/base" + "os" + "path/filepath" + "testing" +) + +func TestExtensionUnnamedResource(t *testing.T) { + d, cleanup, err := newTestExtensionEngineDownloader() + if err != nil { + t.Fatal(err) + } + defer cleanup() + dir := t.TempDir() + manifest := `{"name":"unnamed","title":"Unnamed test","author":"test","version":"1.0.0","scripts":[{"event":"onResolve","match":{"urls":["https://example.com/unnamed"]},"entry":"index.js"}]}` + script := `gopeed.events.onResolve(async(ctx)=>{ctx.res={range:false,files:[{name:'video.mp4',req:{url:await gopeed.runtime.blob.createObjectURL(()=>new ReadableStream({start(c){c.enqueue(new Uint8Array([1,2,3]));c.close();}}))}}]};});` + for name, data := range map[string]string{"manifest.json": manifest, "index.js": script} { + if err = os.WriteFile(filepath.Join(dir, name), []byte(data), 0600); err != nil { + t.Fatal(err) + } + } + if _, err = d.InstallExtensionByFolder(dir, true); err != nil { + t.Fatal(err) + } + res, err := d.Resolve(&base.Request{URL: "https://example.com/unnamed"}, nil) + if err != nil { + t.Fatal(err) + } + if res.Res.Name != "" || len(res.Res.Files) != 1 || res.Res.Files[0].Name != "video.mp4" { + t.Fatalf("unexpected resource: %+v", res.Res) + } + if !d.blob.IsURL(res.Res.Files[0].Req.URL) { + t.Fatal("extension result was discarded") + } +} diff --git a/pkg/download/extension_user_agent_test.go b/pkg/download/extension_user_agent_test.go new file mode 100644 index 00000000..627a6f73 --- /dev/null +++ b/pkg/download/extension_user_agent_test.go @@ -0,0 +1,108 @@ +package download + +import ( + "bytes" + "fmt" + httpProtocol "github.com/GopeedLab/gopeed/internal/protocol/http" + "net/http" + "net/http/httptest" + "os" + "strings" + "testing" + "time" +) + +func TestExtensionHTTPUserAgent(t *testing.T) { + video, err := os.ReadFile("engine/testdata/ffmpeg/video.mp4") + if err != nil { + t.Fatal(err) + } + audio, err := os.ReadFile("engine/testdata/ffmpeg/audio.mp4") + if err != nil { + t.Fatal(err) + } + for _, configured := range []string{" Configured-UA/1.0 ", "", ""} { + for _, header := range []string{"default", "custom", "empty"} { + t.Run(fmt.Sprintf("configured=%q/header=%s", configured, header), func(t *testing.T) { + expected := strings.TrimSpace(configured) + if configured == "" { + expected = httpProtocol.DefaultUserAgent + } + headers := `{}` + if header == "custom" { + expected = "Explicit-UA/2.0" + headers = `{"uSeR-aGeNt":"Explicit-UA/2.0"}` + } + if header == "empty" { + expected = "" + headers = `{"user-agent":""}` + } + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.UserAgent() != expected { + t.Errorf("%s UA=%q, want %q", r.URL.Path, r.UserAgent(), expected) + } + if strings.HasPrefix(r.URL.Path, "/redirect/") { + http.Redirect(w, r, strings.TrimPrefix(r.URL.Path, "/redirect"), http.StatusFound) + return + } + data := video + if strings.HasSuffix(r.URL.Path, "audio") { + data = audio + } + http.ServeContent(w, r, "media", time.Time{}, bytes.NewReader(data)) + })) + defer server.Close() + d, cleanup, err := newTestExtensionEngineDownloader() + if err != nil { + t.Fatal(err) + } + defer cleanup() + cfg, err := d.GetConfig() + if err != nil { + t.Fatal(err) + } + cfg.ProtocolConfig["http"] = map[string]any{"userAgent": configured} + if configured == "" { + cfg.ProtocolConfig["http"] = map[string]any{"connections": 4} + } + if err = d.PutConfig(cfg); err != nil { + t.Fatal(err) + } + runtime, err := newTestExtensionEngine(t, d) + if err != nil { + t.Fatal(err) + } + defer runtime.Close() + // The existing engine keeps its UA snapshot when the downloader config changes. + cfg, err = d.GetConfig() + if err != nil { + t.Fatal(err) + } + cfg.ProtocolConfig["http"] = map[string]any{"userAgent": "Next-engine-UA"} + if err = d.PutConfig(cfg); err != nil { + t.Fatal(err) + } + result, err := runtime.Eval(fmt.Sprintf(`(async()=>{ + const headers=%s, url=%q; + await (await fetch(url+'/redirect/fetch',{headers})).arrayBuffer(); + await new Promise((resolve,reject)=>{ + const xhr=new XMLHttpRequest();xhr.open('GET',url+'/redirect/xhr'); + for(const key of Object.keys(headers))xhr.setRequestHeader(key,headers[key]); + xhr.onload=resolve;xhr.onerror=()=>reject(new Error('XHR failed'));xhr.send(); + }); + // Profile defaults must not replace the configured or explicitly empty UA. + __gopeed_setFingerprint('chrome'); + await (await fetch(url+'/fingerprint',{headers})).arrayBuffer(); + const output=gopeed.runtime.ffmpeg.merge({video:{url:url+'/redirect/video',headers},audio:{url:url+'/redirect/audio',headers}}); + return (await new Response(output).arrayBuffer()).byteLength > 0; + })()`, headers, server.URL)) + if err != nil { + t.Fatal(err) + } + if result != true { + t.Fatalf("merge result=%v", result) + } + }) + } + } +} diff --git a/ui/flutter/lib/features/tasks/domain/task_record.dart b/ui/flutter/lib/features/tasks/domain/task_record.dart index 94a94818..ca85a26b 100644 --- a/ui/flutter/lib/features/tasks/domain/task_record.dart +++ b/ui/flutter/lib/features/tasks/domain/task_record.dart @@ -299,8 +299,9 @@ TaskAssetType _assetType(String name, {required bool isFolder, required api_task } String _fileExtension(String name) { - final pathWithoutQuery = name.split(RegExp(r'[?#]')).first; - final extension = path.extension(pathWithoutQuery).replaceFirst('.', '').toLowerCase(); + final uri = Uri.tryParse(name); + final filePath = uri != null && uri.hasScheme && uri.hasAuthority ? uri.path : name; + final extension = path.extension(filePath).replaceFirst('.', '').toLowerCase(); return extension.length <= 10 ? extension : ''; } diff --git a/ui/flutter/test/features/tasks/domain/task_record_test.dart b/ui/flutter/test/features/tasks/domain/task_record_test.dart new file mode 100644 index 00000000..8b5c72ca --- /dev/null +++ b/ui/flutter/test/features/tasks/domain/task_record_test.dart @@ -0,0 +1,20 @@ +import 'package:flutter_test/flutter_test.dart'; +import 'package:gopeed/core/icons/gopeed_icons.dart'; +import 'package:gopeed/features/tasks/domain/task_record.dart'; + +void main() { + test('file names preserve hash and question mark characters for type detection', () { + expect( + taskFileTypeIcon('Slow English Podcast #2 _ Eating, Talking, and Learning Natural English.mp4'), + GopeedIcons.fileVideo, + ); + expect(taskFileTypeIcon('What is this? #2.MP4'), GopeedIcons.fileVideo); + expect(taskFileTypeIcon('Episode #2.mp3'), GopeedIcons.fileAudio); + }); + + test('URL fallback ignores query and fragment when detecting file type', () { + expect(taskFileTypeIcon('https://example.com/video.mp4?token=abc#part'), GopeedIcons.fileVideo); + expect(taskFileTypeIcon('https://example.com/download?name=video.mp4'), GopeedIcons.file); + expect(taskFileTypeIcon('Playlist #2.mp4', isFolder: true), GopeedIcons.folder); + }); +} From e99ef6c7f46d40eabab9652371131d073437ad8b Mon Sep 17 00:00:00 2001 From: Levi Date: Wed, 16 Sep 2026 00:39:18 +0800 Subject: [PATCH 3/5] fix(flutter): keep waiting task progress static (#1516) --- ui/flutter/lib/features/tasks/domain/task_record.dart | 2 +- .../tasks/presentation/widgets/task_card/task_card_desktop.dart | 2 +- .../tasks/presentation/widgets/task_card/task_card_mobile.dart | 2 +- .../lib/features/tasks/presentation/widgets/task_file_tree.dart | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ui/flutter/lib/features/tasks/domain/task_record.dart b/ui/flutter/lib/features/tasks/domain/task_record.dart index ca85a26b..9a5de0e5 100644 --- a/ui/flutter/lib/features/tasks/domain/task_record.dart +++ b/ui/flutter/lib/features/tasks/domain/task_record.dart @@ -110,7 +110,7 @@ class TaskRecord { bool get canUpdateUrl => protocol == api_task.Protocol.http && (status == TaskStatus.paused || status == TaskStatus.failed); - bool get isIndeterminate => status == TaskStatus.downloading && total == null; + bool get isIndeterminate => status == TaskStatus.downloading && !waiting && total == null; TaskAssetType get assetType => _assetType(name, isFolder: isFolder, protocol: protocol); diff --git a/ui/flutter/lib/features/tasks/presentation/widgets/task_card/task_card_desktop.dart b/ui/flutter/lib/features/tasks/presentation/widgets/task_card/task_card_desktop.dart index 480531ee..004092bd 100644 --- a/ui/flutter/lib/features/tasks/presentation/widgets/task_card/task_card_desktop.dart +++ b/ui/flutter/lib/features/tasks/presentation/widgets/task_card/task_card_desktop.dart @@ -225,7 +225,7 @@ class _ProgressContent extends StatelessWidget { fillColor: fillColor, highlightStartColor: fillColor, highlightEndColor: fillColor, - shimmer: task.status == TaskStatus.downloading, + shimmer: task.status == TaskStatus.downloading && !task.waiting, ), const SizedBox(height: 6), TaskCardFooter(task: task, isError: isError), diff --git a/ui/flutter/lib/features/tasks/presentation/widgets/task_card/task_card_mobile.dart b/ui/flutter/lib/features/tasks/presentation/widgets/task_card/task_card_mobile.dart index 59af346f..8724f0d0 100644 --- a/ui/flutter/lib/features/tasks/presentation/widgets/task_card/task_card_mobile.dart +++ b/ui/flutter/lib/features/tasks/presentation/widgets/task_card/task_card_mobile.dart @@ -149,7 +149,7 @@ class _TaskCardMobileState extends State { fillColor: fillColor, highlightStartColor: fillColor, highlightEndColor: fillColor, - shimmer: task.status == TaskStatus.downloading, + shimmer: task.status == TaskStatus.downloading && !task.waiting, ), const SizedBox(height: 6), TaskCardFooter(task: task, isError: task.status == TaskStatus.failed), diff --git a/ui/flutter/lib/features/tasks/presentation/widgets/task_file_tree.dart b/ui/flutter/lib/features/tasks/presentation/widgets/task_file_tree.dart index 88745fc6..9953f394 100644 --- a/ui/flutter/lib/features/tasks/presentation/widgets/task_file_tree.dart +++ b/ui/flutter/lib/features/tasks/presentation/widgets/task_file_tree.dart @@ -145,7 +145,7 @@ class _TaskDownloadProgress extends StatelessWidget { key: const ValueKey('task-files-total-progress'), value: progress, indeterminate: runtime == null && task.isIndeterminate, - shimmer: task.status == TaskStatus.downloading, + shimmer: task.status == TaskStatus.downloading && !task.waiting, height: 4, trackColor: palette.progressTrack, fillColor: palette.brandProgress, From 21702d5d462b8aa688dbba7d17e6759972421f3e Mon Sep 17 00:00:00 2001 From: Levi Date: Wed, 16 Sep 2026 23:17:43 +0800 Subject: [PATCH 4/5] feat(webview): add proxy support and per-extension profile lifecycle (#1517) --- .gitattributes | 2 + .github/workflows/test.yml | 3 +- go.mod | 4 +- go.sum | 4 +- internal/webview/README.md | 62 ++++ .../webview/goprovider/mainthread_linux.go | 47 +++ .../webview/goprovider/mainthread_other.go | 2 +- internal/webview/goprovider/provider.go | 32 +- .../goprovider/provider_integration_test.go | 2 + internal/webview/integrationtest/contract.go | 39 +- internal/webview/integrationtest/profiles.go | 247 +++++++++++++ internal/webview/proxy/server.go | 341 ++++++++++++++++++ internal/webview/proxy/server_test.go | 315 ++++++++++++++++ internal/webview/rpcprovider/README.md | 6 + internal/webview/rpcprovider/provider.go | 12 +- .../rpcprovider/provider_integration_test.go | 5 + internal/webview/rpcprovider/provider_test.go | 26 ++ pkg/download/downloader.go | 14 +- pkg/download/engine/webview/rpc.go | 12 + pkg/download/engine/webview/runtime.go | 9 + pkg/download/engine/webview/runtime_test.go | 12 + pkg/download/extension.go | 11 +- pkg/download/extension_engine.go | 2 +- pkg/download/extension_engine_session.go | 4 +- pkg/download/extension_webview_config.go | 183 ++++++++++ pkg/download/extension_webview_config_test.go | 51 +++ pkg/download/extension_webview_remove_test.go | 189 ++++++++++ ui/flutter/analysis_options.yaml | 4 + .../lib/app/rpc/webview_rpc_service.dart | 29 +- ui/flutter/macos/Runner/Info.plist | 5 + ui/flutter/pubspec.lock | 53 +-- ui/flutter/pubspec.yaml | 26 ++ 32 files changed, 1708 insertions(+), 45 deletions(-) create mode 100644 .gitattributes create mode 100644 internal/webview/README.md create mode 100644 internal/webview/goprovider/mainthread_linux.go create mode 100644 internal/webview/integrationtest/profiles.go create mode 100644 internal/webview/proxy/server.go create mode 100644 internal/webview/proxy/server_test.go create mode 100644 pkg/download/extension_webview_config.go create mode 100644 pkg/download/extension_webview_config_test.go create mode 100644 pkg/download/extension_webview_remove_test.go diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..437f6d74 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +# Keep upstream plugin sources recognizable and collapsed in GitHub reviews. +ui/flutter/third_party/** linguist-vendored whitespace=-blank-at-eol,-blank-at-eof diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a7138158..745fe371 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -91,7 +91,8 @@ jobs: flutter-version: ${{ env.FLUTTER_VERSION }} - name: Test webview if: runner.os == 'Windows' - run: go test -tags webview -v -count=1 -timeout 300s ./internal/webview/goprovider/ + # Repeat close/reopen coverage to catch browser shutdown races. + run: go test -tags webview -v -count=3 -timeout 300s ./internal/webview/goprovider/ env: CGO_ENABLED: 1 - name: Test webview diff --git a/go.mod b/go.mod index f3575db8..b01d1f1e 100644 --- a/go.mod +++ b/go.mod @@ -4,13 +4,14 @@ go 1.25.4 require ( codeberg.org/gruf/go-ffmpreg v0.6.23-0.20260827095416-df1836cb5be0 - github.com/GopeedLab/webview_go v0.0.0-20260423085439-7a2f88b6e9b5 + github.com/GopeedLab/webview_go v0.0.0-20260915010614-2567e492369a github.com/anacrolix/torrent v1.60.1-0.20251217073903-486bcbe758e0 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 github.com/bodgit/sevenzip v1.6.1 github.com/dop251/goja v0.0.0-20260106131823-651366fbe6e3 github.com/dop251/goja_nodejs v0.0.0-20240728170619-29b559befffc github.com/go-git/go-git/v5 v5.8.1 + github.com/google/jsonschema-go v0.4.3 github.com/gorilla/handlers v1.5.1 github.com/gorilla/mux v1.8.0 github.com/imroc/req/v3 v3.52.2 @@ -40,7 +41,6 @@ require ( github.com/felixge/fgprof v0.9.5 // indirect github.com/go-task/slim-sprig/v3 v3.0.0 // indirect github.com/google/go-cmp v0.7.0 // indirect - github.com/google/jsonschema-go v0.4.3 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect diff --git a/go.sum b/go.sum index b1f117c1..98c4c2eb 100644 --- a/go.sum +++ b/go.sum @@ -25,8 +25,8 @@ filippo.io/edwards25519 v1.0.0-rc.1 h1:m0VOOB23frXZvAOK44usCgLWvtsxIoMCTBGJZlpmG filippo.io/edwards25519 v1.0.0-rc.1/go.mod h1:N1IkdkCkiLB6tki+MYJoSx2JTY9NUlxZE7eHn5EwJns= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/GopeedLab/webview_go v0.0.0-20260423085439-7a2f88b6e9b5 h1:86LVTaNDM/j/uTAD2HedkXNO9LIQlt642VAdbU/0D/M= -github.com/GopeedLab/webview_go v0.0.0-20260423085439-7a2f88b6e9b5/go.mod h1:FvV19lu9rQ9NcbczN5y8pbWecURRRbWa8tbfFARBDa0= +github.com/GopeedLab/webview_go v0.0.0-20260915010614-2567e492369a h1:Vzz+8aMVX9ZPPpkxgxa7mCorrAgdXCZL5EunOpOBvao= +github.com/GopeedLab/webview_go v0.0.0-20260915010614-2567e492369a/go.mod h1:FvV19lu9rQ9NcbczN5y8pbWecURRRbWa8tbfFARBDa0= github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0rYXWg0= github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= diff --git a/internal/webview/README.md b/internal/webview/README.md new file mode 100644 index 00000000..bd46c554 --- /dev/null +++ b/internal/webview/README.md @@ -0,0 +1,62 @@ +# Extension browser host configuration + +The downloader wraps each provider with the extension's stable identity before +constructing the runtime. `OpenOptions.ProfileID`, `DataPath`, and `ProxyURL` are +host-only fields. `parseOpenOptions` never reads them from extension JavaScript. +The extension API is unchanged. The Flutter host uses the fork's public +`WebViewProfile`, `InAppWebViewSettings.profileId`, and profile-scoped +`CookieManager` APIs; the library has no Gopeed-specific method-channel bridge. +Only the Gopeed extension JavaScript boundary keeps these options host-only. + +Profiles are persistent and shared across invocations/pages of the same extension. +Different identities use different native stores. Go providers use a SHA-256-based +path under `/webview`; Apple RPC hosts use a stable UUID data store; +Android uses a named WebView profile. Cookie operations select that same store, +including `clearCookies`. Closing an execution closes its pages, not its profile. +Uninstall closes the extension's pages, prevents its old runtimes from opening +new pages, and removes its native profile before deleting the extension record. +Cleanup failures abort uninstall and can be retried. Reinstallation starts with +empty browser data; an ordinary extension update keeps the existing profile. +Windows/Linux delete the profile directory; Apple removes the named WebKit store. +Android cannot delete a profile already loaded by the current process: when +DELETE_BROWSING_DATA is supported it clears all site data immediately, and records +an empty-profile deletion in SharedPreferences for the next process start. A +reinstallation may reuse that cleared profile and cancels the pending deletion. +If the full-data clearing API is unavailable, uninstall fails with a restart +instruction; the pending profile is deleted at startup before it can be loaded. +No removal operation is exposed through extension JavaScript. + +Existing shared browser data is not copied into every extension's new profile. + +The downloader lazily owns one loopback forward proxy with HTTP and SOCKS5 +listeners. Windows/Linux/Android use HTTP; Apple uses SOCKS5 because its HTTP +CONNECT configuration may bypass plain HTTP. The forwarder reads the current +Gopeed proxy config for each new request/tunnel and supports direct, system, +HTTP(S) upstream proxies with Basic authentication, and authenticated SOCKS5. +Credentials remain in Go and are never sent over the WebView RPC. + +Changes affect new requests/connections. Existing CONNECT/SOCKS tunnels keep their +route until closed; opening another page in the same profile does not reset it. +SOCKS carries host/port rather than a full URL: system proxy/PAC selection for +Apple is host-based, with port 80 treated as HTTP and other ports as HTTPS. +Path-dependent PAC rules cannot be reproduced from a SOCKS connection. + +Apple requires macOS 14 / iOS 17. Android checks MULTI_PROFILE and PROXY_OVERRIDE. +Unsupported hosts fail explicitly instead of silently sharing browser storage or +ignoring the proxy. + +## Validation + +- `go test -race ./internal/webview/proxy ./pkg/download/engine/webview` +- `go test ./pkg/download -run 'TestWebViewProfileBound|TestDownloader_ExtensionRuntimeWebView'` +- Windows/Linux: `go test -tags webview -v ./internal/webview/goprovider` (Linux: run under `xvfb-run -a`). +- macOS: start the normal Flutter app, then run `go test -tags webview -v ./internal/webview/rpcprovider` with its socket address. See `rpcprovider/README.md`. +- In webview_go: `go test -tags webview_integration -v -count=1 .` (Linux: run under `xvfb-run -a`). + +The shared provider contract covers HTTP/HTTPS proxy routing, concurrent pages, +HttpOnly cookies, LocalStorage, IndexedDB, profile-scoped clearing, reopening, +and removal without affecting another profile. HTTPS routing is checked with a +rejecting upstream tunnel; no test-only certificate bypass is added to the app. +Go proxy unit tests separately cover successful TLS forwarding and authentication. +These cases run in the existing `test.yml` WebView jobs; existing mobile build jobs +compile the Android/iOS bridge. No separate Flutter test application is required. diff --git a/internal/webview/goprovider/mainthread_linux.go b/internal/webview/goprovider/mainthread_linux.go new file mode 100644 index 00000000..3e0a09da --- /dev/null +++ b/internal/webview/goprovider/mainthread_linux.go @@ -0,0 +1,47 @@ +//go:build linux && cgo + +package goprovider + +/* +#cgo pkg-config: gtk+-3.0 +#include +static void pumpGTK(void) { + // Bound each batch so Go creation/removal requests cannot be starved. + for (int i = 0; i < 64 && g_main_context_pending(NULL); ++i) + g_main_context_iteration(NULL, FALSE); +} +*/ +import "C" + +import ( + "runtime" + "sync" + "time" +) + +var gtkThreadOnce sync.Once +var gtkThreadTasks = make(chan func()) + +func RunMainThreadLoop(run func() int) int { return run() } + +func postMainThreadTask(task func()) bool { + gtkThreadOnce.Do(func() { + go func() { + runtime.LockOSThread() + // WebKit must always use its original UI thread, including after + // the last page closes. A page cannot stop this shared event loop. + ticker := time.NewTicker(10 * time.Millisecond) + defer ticker.Stop() + for { + select { + case task := <-gtkThreadTasks: + task() + case <-ticker.C: + } + C.pumpGTK() + } + }() + }) + gtkThreadTasks <- task + return true +} diff --git a/internal/webview/goprovider/mainthread_other.go b/internal/webview/goprovider/mainthread_other.go index aa3ee4a1..45ad0255 100644 --- a/internal/webview/goprovider/mainthread_other.go +++ b/internal/webview/goprovider/mainthread_other.go @@ -1,4 +1,4 @@ -//go:build !darwin +//go:build !darwin && (!linux || !cgo) package goprovider diff --git a/internal/webview/goprovider/provider.go b/internal/webview/goprovider/provider.go index a81c20df..c9d06a5e 100644 --- a/internal/webview/goprovider/provider.go +++ b/internal/webview/goprovider/provider.go @@ -105,7 +105,15 @@ func (p *pageWrapper) start() error { defer runtimepkg.UnlockOSThread() var w webview.WebView - if p.opts.Headless { + if p.opts.DataPath != "" { + var err error + w, err = webview.NewWithOptions(webview.Options{Debug: p.opts.Debug, Headless: p.opts.Headless, DataPath: p.opts.DataPath, ProxyURL: p.opts.ProxyURL}) + if err != nil { + p.ready <- err + close(p.done) + return + } + } else if p.opts.Headless { w = webview.NewHeadless(p.opts.Debug) } else { w = webview.New(p.opts.Debug) @@ -127,12 +135,18 @@ func (p *pageWrapper) start() error { if err := w.Bind(p.callbackName, func(payload string) error { return p.handleCallback(payload) }); err != nil { + w.Destroy() p.ready <- err + close(p.done) return } applyWindowOptions(w, p.opts) w.SetHtml(buildBootstrapHTML(p.callbackName, p.readyID)) + if runtimepkg.GOOS == "linux" { + // All GTK pages share the process-wide event loop below. + return + } w.Run() w.Destroy() close(p.done) @@ -318,7 +332,12 @@ func (p *pageWrapper) Close() error { // the quit message lands on the correct queue. terminateDone := make(chan struct{}) w.Dispatch(func() { - w.Terminate() + if runtimepkg.GOOS == "linux" { + w.Destroy() + close(p.done) + } else { + w.Terminate() + } close(terminateDone) }) <-terminateDone @@ -610,3 +629,12 @@ func firstNonEmpty(values ...string) string { } return "" } + +func (p *Provider) RemoveProfile(profileID, dataPath string) error { + done := make(chan error, 1) + run := func() { done <- webview.RemoveProfile(dataPath) } + if !postMainThreadTask(run) { + run() + } + return <-done +} diff --git a/internal/webview/goprovider/provider_integration_test.go b/internal/webview/goprovider/provider_integration_test.go index 92f226e7..e6d9e63e 100644 --- a/internal/webview/goprovider/provider_integration_test.go +++ b/internal/webview/goprovider/provider_integration_test.go @@ -13,3 +13,5 @@ func TestProviderContract(t *testing.T) { CookieDomainMode: integrationtest.CookieDomainModeRequired, }) } + +func TestProviderProfiles(t *testing.T) { integrationtest.RunProfileContract(t, New()) } diff --git a/internal/webview/integrationtest/contract.go b/internal/webview/integrationtest/contract.go index 1a508e35..364fe5c2 100644 --- a/internal/webview/integrationtest/contract.go +++ b/internal/webview/integrationtest/contract.go @@ -1,10 +1,14 @@ package integrationtest import ( + "crypto/sha256" "fmt" + webviewproxy "github.com/GopeedLab/gopeed/internal/webview/proxy" "net/http" "net/http/httptest" "net/url" + "path/filepath" + "runtime" "strings" "testing" "time" @@ -219,7 +223,19 @@ func openTestPage(t *testing.T, provider enginewebview.Provider, opts enginewebv })) t.Cleanup(server.Close) - runtime := enginewebview.NewRuntime(provider, true) + proxy, err := webviewproxy.New(nil) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { proxy.Close() }) + path := filepath.Join(t.TempDir(), "profile") + sum := sha256.Sum256([]byte(path)) + profileID := fmt.Sprintf("%x-%x-%x-%x-%x", sum[:4], sum[4:6], sum[6:8], sum[8:10], sum[10:16]) + proxyURL := proxy.URL() + if runtime.GOOS == "darwin" { + proxyURL = proxy.SOCKSURL() + } + runtime := enginewebview.NewRuntime(&configuredOpener{provider: provider, profileID: profileID, dataPath: path, proxyURL: proxyURL}, true) page, err := runtime.Open(map[string]any{ "headless": opts.Headless, "debug": opts.Debug, @@ -232,7 +248,14 @@ func openTestPage(t *testing.T, provider enginewebview.Provider, opts enginewebv t.Fatalf("failed to open webview: %v", err) } t.Cleanup(func() { - _ = runtime.Close() + if err := runtime.Close(); err != nil { + t.Errorf("close contract page: %v", err) + } + if remover, ok := provider.(enginewebview.ProfileRemover); ok { + if err := remover.RemoveProfile(profileID, path); err != nil { + t.Errorf("remove contract profile: %v", err) + } + } }) return page, server, server.URL @@ -380,3 +403,15 @@ func hasCookie(cookies []enginewebview.Cookie, name string, value string) bool { } return false } + +type configuredOpener struct { + provider enginewebview.Provider + profileID, dataPath, proxyURL string +} + +func (o *configuredOpener) Open(opts enginewebview.OpenOptions) (enginewebview.Page, error) { + opts.ProfileID = o.profileID + opts.DataPath = o.dataPath + opts.ProxyURL = o.proxyURL + return o.provider.Open(opts) +} diff --git a/internal/webview/integrationtest/profiles.go b/internal/webview/integrationtest/profiles.go new file mode 100644 index 00000000..72d297ba --- /dev/null +++ b/internal/webview/integrationtest/profiles.go @@ -0,0 +1,247 @@ +package integrationtest + +import ( + "crypto/sha256" + "fmt" + "io" + "net" + "net/http" + "net/http/httptest" + "net/url" + "path/filepath" + "reflect" + "runtime" + "sync/atomic" + "testing" + "time" + + webviewproxy "github.com/GopeedLab/gopeed/internal/webview/proxy" + webview "github.com/GopeedLab/gopeed/pkg/download/engine/webview" +) + +// RunProfileContract exercises the same host API through both native and RPC +// providers. It never bypasses the host bridge with direct Flutter plugin calls. +func RunProfileContract(t *testing.T, provider webview.Provider) { + t.Helper() + waitForProviderAvailable(t, provider, defaultAvailabilityTimeout) + remover, ok := provider.(webview.ProfileRemover) + if !ok { + t.Fatal("provider does not implement host profile removal") + } + proxyURL, secureHits := profileProxy(t) + a := newContractProfile(t, provider, remover, proxyURL) + b := newContractProfile(t, provider, remover, proxyURL) + first := a.open(t) + assertProfileValue(t, first, `async () => { localStorage.setItem('owner','a'); return 'a'; }`, "a") + assertProfileValue(t, first, profileDatabaseScript(true), "saved") + if err := first.SetCookie(webview.Cookie{Name: "session", Value: "a", Domain: "profiles.invalid", Path: "/", HTTPOnly: true, Expires: time.Now().Add(time.Hour)}); err != nil { + t.Fatal(err) + } + second := b.open(t) + assertProfileValue(t, second, `() => localStorage.getItem('owner')`, nil) + assertProfileValue(t, second, profileDatabaseScript(false), nil) + assertProfileCookies(t, second, "", "new profile B") + assertProfileValue(t, second, `() => { localStorage.setItem('owner','b'); return 'b'; }`, "b") + if err := second.SetCookie(webview.Cookie{Name: "session", Value: "b", Domain: "profiles.invalid", Path: "/", HTTPOnly: true}); err != nil { + t.Fatal(err) + } + shared := a.open(t) + assertProfileValue(t, shared, `() => localStorage.getItem('owner')`, "a") + assertProfileValue(t, shared, profileDatabaseScript(false), "saved") + assertProfileCookies(t, shared, "a", "shared profile A") + storedCookies, err := shared.GetCookies() + if err != nil || len(storedCookies) != 1 || storedCookies[0].Expires.Before(time.Now()) { + t.Fatalf("persistent cookie lost its expiration: %#v, error %v", storedCookies, err) + } + assertProfileValue(t, shared, `() => document.cookie`, "") // HttpOnly stays hidden. + if err := second.ClearCookies(); err != nil { + t.Fatal(err) + } + assertProfileCookies(t, first, "a", "clearing B leaves A intact") + if err := second.SetCookie(webview.Cookie{Name: "session", Value: "b", Domain: "profiles.invalid", Path: "/", HTTPOnly: true}); err != nil { + t.Fatal(err) + } + a.close(t) + reopened := a.open(t) + assertProfileValue(t, reopened, `() => localStorage.getItem('owner')`, "a") + assertProfileValue(t, reopened, profileDatabaseScript(false), "saved") + assertProfileCookies(t, reopened, "a", "reopened profile A") + // Include a session cookie immediately before deletion to catch stale + // in-process CookieStore caches when the same profile is recreated. + if err := reopened.SetCookie(webview.Cookie{Name: "session", Value: "remove-me", Domain: "profiles.invalid", Path: "/", HTTPOnly: true}); err != nil { + t.Fatal(err) + } + a.close(t) + a.remove(t) + a.remove(t) // Retrying a successful uninstall is harmless. + reinstalled := a.open(t) + assertProfileValue(t, reinstalled, `() => localStorage.getItem('owner')`, nil) + assertProfileValue(t, reinstalled, profileDatabaseScript(false), nil) + assertProfileCookies(t, reinstalled, "", "reinstalled profile A") + assertProfileValue(t, second, `() => localStorage.getItem('owner')`, "b") + assertProfileCookies(t, second, "b", "B after removing A") + // Refuse the HTTPS tunnel deliberately. Seeing it at our proxy proves HTTPS + // routing without adding a certificate-validation bypass to the test host. + // WebView2 may commit its built-in error page and report navigation as + // complete. The invariant here is that the HTTPS tunnel reaches the proxy. + _ = reinstalled.Goto("https://profiles.invalid/", webview.GotoOptions{TimeoutMS: 5000}) + deadline := time.Now().Add(5 * time.Second) + for secureHits.Load() == 0 && time.Now().Before(deadline) { + time.Sleep(10 * time.Millisecond) + } + if secureHits.Load() == 0 { + t.Fatal("HTTPS bypassed the host proxy") + } +} + +type contractProfile struct { + provider webview.Provider + remover webview.ProfileRemover + options webview.OpenOptions + pages []webview.Page +} + +func newContractProfile(t *testing.T, provider webview.Provider, remover webview.ProfileRemover, proxyURL string) *contractProfile { + t.Helper() + path := filepath.Join(t.TempDir(), "profile") + hash := sha256.Sum256([]byte(path)) + id := fmt.Sprintf("%x-%x-%x-%x-%x", hash[:4], hash[4:6], hash[6:8], hash[8:10], hash[10:16]) + p := &contractProfile{provider: provider, remover: remover, options: webview.OpenOptions{ProfileID: id, DataPath: path, ProxyURL: proxyURL, Headless: true, Title: "Gopeed profile contract"}} + t.Cleanup(func() { p.close(t); p.remove(t) }) + return p +} +func (p *contractProfile) open(t *testing.T) webview.Page { + t.Helper() + page, err := p.provider.Open(p.options) + if err != nil { + t.Fatal(err) + } + p.pages = append(p.pages, page) + if err := page.Goto("http://profiles.invalid/", webview.GotoOptions{TimeoutMS: 10000}); err != nil { + t.Fatal(err) + } + return page +} +func (p *contractProfile) close(t *testing.T) { + t.Helper() + for _, page := range p.pages { + if err := page.Close(); err != nil { + t.Errorf("close profile page: %v", err) + } + } + p.pages = nil +} +func (p *contractProfile) remove(t *testing.T) { + t.Helper() + if err := p.remover.RemoveProfile(p.options.ProfileID, p.options.DataPath); err != nil { + t.Errorf("remove profile: %v", err) + } +} +func assertProfileValue(t *testing.T, page webview.Page, script string, want any) { + t.Helper() + got, err := page.Execute(script) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("script %s: got %#v, want %#v", script, got, want) + } +} +func assertProfileCookies(t *testing.T, page webview.Page, want, stage string) { + t.Helper() + // Native cookie managers propagate updates across browser processes + // asynchronously. Wait for the expected state rather than assuming the + // first IPC read already includes the preceding write. + deadline := time.Now().Add(5 * time.Second) + for { + cookies, err := page.GetCookies() + if err != nil { + t.Fatalf("%s: %v", stage, err) + } + matches := len(cookies) == 0 && want == "" || len(cookies) == 1 && want != "" && cookies[0].Name == "session" && cookies[0].Value == want && cookies[0].HTTPOnly + if matches { + return + } + if time.Now().After(deadline) { + t.Fatalf("%s: cookies %#v, want %q", stage, cookies, want) + } + time.Sleep(20 * time.Millisecond) + } +} +func profileDatabaseScript(write bool) string { + operation := "store.get('owner')" + if write { + operation = "store.put('saved','owner')" + } + return fmt.Sprintf(`async () => await new Promise((resolve,reject)=>{ + const request=indexedDB.open('profile-contract',1); + request.onupgradeneeded=()=>request.result.createObjectStore('state'); + request.onerror=()=>reject(String(request.error)); + request.onsuccess=()=>{ + const db=request.result, tx=db.transaction('state','readwrite'), store=tx.objectStore('state'); + const op=%s; let value; + op.onsuccess=()=>value=%s; + tx.oncomplete=()=>{db.close();resolve(value===undefined?null:value)}; + tx.onerror=()=>{db.close();reject(String(tx.error))}; + }; +})`, operation, map[bool]string{true: "'saved'", false: "op.result"}[write]) +} + +func profileProxy(t *testing.T) (string, *atomic.Int32) { + t.Helper() + origin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/html") + w.Header().Set("Cache-Control", "no-store") + fmt.Fprint(w, "Profile contract") + })) + t.Cleanup(origin.Close) + secureHits := &atomic.Int32{} + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodConnect { + origin.Config.Handler.ServeHTTP(w, r) + return + } + if r.Host == "profiles.invalid:443" { + secureHits.Add(1) + http.Error(w, "HTTPS routing fixture rejects tunnels", http.StatusBadGateway) + return + } + if r.Host != "profiles.invalid:80" { + http.Error(w, "unexpected target", http.StatusBadGateway) + return + } + remote, err := net.DialTimeout("tcp", origin.Listener.Addr().String(), 5*time.Second) + if err != nil { + http.Error(w, "origin unavailable", http.StatusBadGateway) + return + } + defer remote.Close() + client, buffer, err := w.(http.Hijacker).Hijack() + if err != nil { + return + } + defer client.Close() + if _, err := buffer.WriteString("HTTP/1.1 200 Connection Established\r\n\r\n"); err != nil { + return + } + if err := buffer.Flush(); err != nil { + return + } + go func() { io.Copy(remote, buffer); remote.Close() }() + io.Copy(client, remote) + })) + t.Cleanup(upstream.Close) + upstreamURL, err := url.Parse(upstream.URL) + if err != nil { + t.Fatal(err) + } + proxy, err := webviewproxy.New(func(*http.Request) (*url.URL, error) { return upstreamURL, nil }) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { proxy.Close() }) + if runtime.GOOS == "darwin" { + return proxy.SOCKSURL(), secureHits + } + return proxy.URL(), secureHits +} diff --git a/internal/webview/proxy/server.go b/internal/webview/proxy/server.go new file mode 100644 index 00000000..17697d40 --- /dev/null +++ b/internal/webview/proxy/server.go @@ -0,0 +1,341 @@ +// Package proxy provides the host-owned WebView forward proxy. Browser engines +// always use this endpoint; Gopeed decides the upstream route for each request. +package proxy + +import ( + "bufio" + "context" + "crypto/tls" + "encoding/base64" + "errors" + "fmt" + "io" + "log" + "net" + "net/http" + "net/url" + "strings" + "sync" + "time" + + "github.com/armon/go-socks5" + xproxy "golang.org/x/net/proxy" +) + +type Selector func(*http.Request) (*url.URL, error) + +type Server struct { + listener net.Listener + socksListener net.Listener + server *http.Server + transport *http.Transport + selectProxy Selector + mu sync.Mutex + connections map[io.Closer]struct{} + closed bool +} + +func New(selectProxy Selector) (*Server, error) { + if selectProxy == nil { + selectProxy = func(*http.Request) (*url.URL, error) { return nil, nil } + } + listener, err := net.Listen("tcp4", "127.0.0.1:0") + if err != nil { + return nil, err + } + s := &Server{listener: listener, selectProxy: selectProxy, connections: make(map[io.Closer]struct{})} + s.transport = &http.Transport{Proxy: selectProxy, DisableKeepAlives: true, TLSHandshakeTimeout: 15 * time.Second, ResponseHeaderTimeout: 30 * time.Second} + s.server = &http.Server{Handler: s, ReadHeaderTimeout: 15 * time.Second} + s.socksListener, err = net.Listen("tcp4", "127.0.0.1:0") + if err != nil { + listener.Close() + return nil, err + } + socks, err := socks5.New(&socks5.Config{Logger: log.New(io.Discard, "", 0), Resolver: unresolvedDNS{}, Dial: func(ctx context.Context, network, address string) (net.Conn, error) { + _, port, _ := net.SplitHostPort(address) + scheme := "https" + if port == "80" { + scheme = "http" + } + request := &http.Request{URL: &url.URL{Scheme: scheme, Host: address}, Header: make(http.Header)} + upstream, err := selectProxy(request) + if err != nil { + return nil, err + } + timed, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + conn, err := dialTunnel(timed, address, upstream) + if err != nil { + return nil, err + } + return s.track(conn) + }}) + if err != nil { + listener.Close() + s.socksListener.Close() + return nil, err + } + go s.server.Serve(listener) + go socks.Serve(&trackedListener{Listener: s.socksListener, server: s}) + return s, nil +} + +func (s *Server) URL() string { return "http://" + s.listener.Addr().String() } + +func (s *Server) SOCKSURL() string { return "socks5://" + s.socksListener.Addr().String() } + +func (s *Server) Close() error { + s.mu.Lock() + s.closed = true + connections := make([]io.Closer, 0, len(s.connections)) + for c := range s.connections { + connections = append(connections, c) + } + s.mu.Unlock() + for _, c := range connections { + c.Close() + } + s.socksListener.Close() + s.transport.CloseIdleConnections() + return s.server.Close() +} + +func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodConnect { + s.connect(w, r) + return + } + if !r.URL.IsAbs() || (r.URL.Scheme != "http" && r.URL.Scheme != "https") { + http.Error(w, "absolute HTTP URL required", http.StatusBadRequest) + return + } + out := r.Clone(r.Context()) + out.RequestURI = "" + upgrade := out.Header.Get("Upgrade") + stripHopHeaders(out.Header) + if upgrade != "" { + out.Header.Set("Connection", "Upgrade") + out.Header.Set("Upgrade", upgrade) + } + resp, err := s.transport.RoundTrip(out) + if err != nil { + http.Error(w, "upstream request failed", http.StatusBadGateway) + return + } + defer resp.Body.Close() + if resp.StatusCode == http.StatusSwitchingProtocols { + remote, ok := resp.Body.(io.ReadWriteCloser) + if !ok { + http.Error(w, "invalid upstream upgrade", 502) + return + } + client, buffer, err := w.(http.Hijacker).Hijack() + if err != nil { + return + } + defer client.Close() + resp.Body = nil + if err := resp.Write(buffer); err != nil { + return + } + if err := buffer.Flush(); err != nil { + return + } + s.bridge(client, buffer, remote) + return + } + stripHopHeaders(resp.Header) + for k, v := range resp.Header { + w.Header()[k] = v + } + w.WriteHeader(resp.StatusCode) + io.Copy(w, resp.Body) +} + +func stripHopHeaders(h http.Header) { + for _, key := range strings.Split(h.Get("Connection"), ",") { + h.Del(strings.TrimSpace(key)) + } + for _, key := range []string{"Connection", "Proxy-Connection", "Proxy-Authorization", "Proxy-Authenticate", "Keep-Alive", "TE", "Trailer", "Transfer-Encoding", "Upgrade"} { + h.Del(key) + } +} + +func (s *Server) connect(w http.ResponseWriter, r *http.Request) { + if _, _, err := net.SplitHostPort(r.Host); err != nil { + http.Error(w, "invalid tunnel destination", 400) + return + } + target := r.Clone(r.Context()) + target.URL = &url.URL{Scheme: "https", Host: r.Host} + upstream, err := s.selectProxy(target) + if err != nil { + http.Error(w, "proxy selection failed", 502) + return + } + ctx, cancel := context.WithTimeout(r.Context(), 30*time.Second) + defer cancel() + remote, err := dialTunnel(ctx, r.Host, upstream) + if err != nil { + http.Error(w, "upstream connection failed", 502) + return + } + defer remote.Close() + client, buffer, err := w.(http.Hijacker).Hijack() + if err != nil { + return + } + defer client.Close() + if _, err := buffer.WriteString("HTTP/1.1 200 Connection Established\r\n\r\n"); err != nil { + return + } + if err := buffer.Flush(); err != nil { + return + } + s.bridge(client, buffer, remote) +} + +func (s *Server) bridge(client net.Conn, buffer *bufio.ReadWriter, remote io.ReadWriteCloser) { + s.mu.Lock() + if s.closed { + s.mu.Unlock() + return + } + s.connections[client] = struct{}{} + s.connections[remote] = struct{}{} + s.mu.Unlock() + defer func() { s.mu.Lock(); delete(s.connections, client); delete(s.connections, remote); s.mu.Unlock() }() + + done := make(chan struct{}) + go func() { io.Copy(remote, buffer); remote.Close(); close(done) }() + io.Copy(client, remote) + client.Close() + <-done +} + +type bufferedConn struct { + net.Conn + reader *bufio.Reader +} + +func (c *bufferedConn) Read(p []byte) (int, error) { return c.reader.Read(p) } + +func dialTunnel(ctx context.Context, target string, upstream *url.URL) (net.Conn, error) { + dialer := &net.Dialer{} + if upstream == nil { + return dialer.DialContext(ctx, "tcp", target) + } + if upstream.Scheme == "socks5" || upstream.Scheme == "socks5h" { + u := *upstream + u.Scheme = "socks5" + d, err := xproxy.FromURL(&u, dialer) + if err != nil { + return nil, err + } + return d.(xproxy.ContextDialer).DialContext(ctx, "tcp", target) + } + if upstream.Scheme != "http" && upstream.Scheme != "https" { + return nil, errors.New("unsupported proxy scheme") + } + host := upstream.Host + if upstream.Port() == "" { + port := "80" + if upstream.Scheme == "https" { + port = "443" + } + host = net.JoinHostPort(upstream.Hostname(), port) + } + conn, err := dialer.DialContext(ctx, "tcp", host) + if err != nil { + return nil, err + } + ok := false + defer func() { + if !ok { + conn.Close() + } + }() + if deadline, exists := ctx.Deadline(); exists { + conn.SetDeadline(deadline) + } + if upstream.Scheme == "https" { + secure := tls.Client(conn, &tls.Config{ServerName: upstream.Hostname(), MinVersion: tls.VersionTLS12}) + if err := secure.HandshakeContext(ctx); err != nil { + return nil, err + } + conn = secure + } + request := &http.Request{Method: http.MethodConnect, URL: &url.URL{Opaque: target}, Host: target, Header: make(http.Header)} + if upstream.User != nil { + password, _ := upstream.User.Password() + request.Header.Set("Proxy-Authorization", "Basic "+base64.StdEncoding.EncodeToString([]byte(upstream.User.Username()+":"+password))) + } + if err := request.Write(conn); err != nil { + return nil, err + } + reader := bufio.NewReader(conn) + response, err := http.ReadResponse(reader, request) + if err != nil { + return nil, err + } + if response.StatusCode != 200 { + response.Body.Close() + return nil, fmt.Errorf("upstream CONNECT status %d", response.StatusCode) + } + conn.SetDeadline(time.Time{}) + ok = true + return &bufferedConn{Conn: conn, reader: reader}, nil +} + +// Leave hostname resolution to the selected upstream (SOCKS remote DNS / CONNECT). +type unresolvedDNS struct{} + +func (unresolvedDNS) Resolve(ctx context.Context, name string) (context.Context, net.IP, error) { + return ctx, nil, nil +} + +type trackedListener struct { + net.Listener + server *Server +} + +func (l *trackedListener) Accept() (net.Conn, error) { + c, err := l.Listener.Accept() + if err != nil { + return nil, err + } + return l.server.track(c) +} + +type trackedConn struct { + net.Conn + server *Server +} + +func (c *trackedConn) Close() error { + err := c.Conn.Close() + c.server.mu.Lock() + delete(c.server.connections, c) + c.server.mu.Unlock() + return err +} +func (s *Server) track(c net.Conn) (net.Conn, error) { + s.mu.Lock() + defer s.mu.Unlock() + if s.closed { + c.Close() + return nil, net.ErrClosed + } + tracked := &trackedConn{Conn: c, server: s} + s.connections[tracked] = struct{}{} + return tracked, nil +} + +func (c *trackedConn) CloseWrite() error { return closeWrite(c.Conn) } +func (c *bufferedConn) CloseWrite() error { return closeWrite(c.Conn) } +func closeWrite(c net.Conn) error { + if writer, ok := c.(interface{ CloseWrite() error }); ok { + return writer.CloseWrite() + } + return c.Close() +} diff --git a/internal/webview/proxy/server_test.go b/internal/webview/proxy/server_test.go new file mode 100644 index 00000000..217de9e8 --- /dev/null +++ b/internal/webview/proxy/server_test.go @@ -0,0 +1,315 @@ +package proxy + +import ( + "bufio" + "context" + "crypto/tls" + "errors" + "io" + "net" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/armon/go-socks5" +) + +func TestRejectedProxyRequests(t *testing.T) { + secret := "upstream-password" + for _, tc := range []struct { + name, method, target string + status int + }{ + {"relative URL", "GET", "/relative", 400}, + {"unsupported URL", "GET", "ftp://example.com/file", 400}, + {"invalid CONNECT", "CONNECT", "example.com", 400}, + {"HTTP selection failure", "GET", "http://example.com/", 502}, + {"CONNECT selection failure", "CONNECT", "example.com:443", 502}, + } { + t.Run(tc.name, func(t *testing.T) { + s := &Server{selectProxy: func(*http.Request) (*url.URL, error) { return nil, errors.New(secret) }} + s.transport = &http.Transport{Proxy: s.selectProxy} + defer s.transport.CloseIdleConnections() + response := httptest.NewRecorder() + s.ServeHTTP(response, httptest.NewRequest(tc.method, tc.target, nil)) + if response.Code != tc.status || strings.Contains(response.Body.String(), secret) { + t.Fatalf("unexpected proxy rejection: %d %q", response.Code, response.Body.String()) + } + }) + } +} + +func TestTunnelRejectsUnsafeUpstreams(t *testing.T) { + rejected := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Error(w, "authentication required", http.StatusProxyAuthRequired) + })) + defer rejected.Close() + secure := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + t.Error("untrusted TLS proxy received a request") + })) + defer secure.Close() + for _, upstream := range []string{"ftp://127.0.0.1", rejected.URL, secure.URL} { + t.Run(upstream, func(t *testing.T) { + selected, err := url.Parse(upstream) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + conn, err := dialTunnel(ctx, "example.com:443", selected) + if conn != nil { + conn.Close() + } + if err == nil { + t.Fatal("unsafe upstream unexpectedly accepted") + } + }) + } +} + +func TestRoutesAndAuthentication(t *testing.T) { + origin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Proxy-Authorization") != "" { + t.Error("proxy credentials leaked to origin") + } + w.Write([]byte("origin")) + })) + defer origin.Close() + secure := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Write([]byte("secure")) })) + defer secure.Close() + upstream, err := New(func(*http.Request) (*url.URL, error) { return nil, nil }) + if err != nil { + t.Fatal(err) + } + defer upstream.Close() + var authenticated atomic.Int32 + authProxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Proxy-Authorization") != "Basic dXNlcjpwYXNz" { + http.Error(w, "authentication required", 407) + return + } + authenticated.Add(1) + upstream.ServeHTTP(w, r) + })) + defer authProxy.Close() + socks, err := socks5.New(&socks5.Config{Credentials: socks5.StaticCredentials{"user": "pass"}}) + if err != nil { + t.Fatal(err) + } + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer listener.Close() + go socks.Serve(listener) + for _, route := range []string{"", authProxy.URL, "socks5://" + listener.Addr().String()} { + t.Run(route, func(t *testing.T) { + var selected *url.URL + if route != "" { + selected, _ = url.Parse(route) + selected.User = url.UserPassword("user", "pass") + } + local, err := New(func(r *http.Request) (*url.URL, error) { return selected, nil }) + if err != nil { + t.Fatal(err) + } + defer local.Close() + proxyURL, _ := url.Parse(local.URL()) + transport := &http.Transport{Proxy: http.ProxyURL(proxyURL), TLSClientConfig: &tls.Config{InsecureSkipVerify: true}} + defer transport.CloseIdleConnections() + client := &http.Client{Transport: transport, Timeout: 5 * time.Second} + for _, test := range []struct{ url, want string }{{origin.URL, "origin"}, {secure.URL, "secure"}} { + response, err := client.Get(test.url) + if err != nil { + t.Fatal(err) + } + body, _ := io.ReadAll(response.Body) + response.Body.Close() + if string(body) != test.want { + t.Fatalf("got status %d, body %q", response.StatusCode, body) + } + } + }) + } + if authenticated.Load() != 2 { + t.Fatalf("expected authenticated HTTP and CONNECT, got %d", authenticated.Load()) + } +} + +func TestRoutingChangesAndFailureDoesNotBypassProxy(t *testing.T) { + var hits atomic.Int32 + origin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { hits.Add(1); w.Write([]byte("direct")) })) + defer origin.Close() + var useProxy atomic.Bool + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Write([]byte("proxied")) })) + upstreamURL, _ := url.Parse(upstream.URL) + local, err := New(func(*http.Request) (*url.URL, error) { + if useProxy.Load() { + return upstreamURL, nil + } + return nil, nil + }) + if err != nil { + t.Fatal(err) + } + defer local.Close() + endpoint, _ := url.Parse(local.URL()) + transport := &http.Transport{Proxy: http.ProxyURL(endpoint)} + defer transport.CloseIdleConnections() + client := &http.Client{Transport: transport, Timeout: 3 * time.Second} + for _, want := range []string{"direct", "proxied", "upstream request failed\n"} { + resp, err := client.Get(origin.URL) + if err != nil { + t.Fatal(err) + } + body, _ := io.ReadAll(resp.Body) + resp.Body.Close() + if string(body) != want { + t.Fatalf("got %q want %q", body, want) + } + if want == "direct" { + useProxy.Store(true) + } else if want == "proxied" { + upstream.Close() + } + } + if hits.Load() != 1 { + t.Fatal("failed proxy must not fall back to direct") + } +} + +func TestCloseTerminatesTunnels(t *testing.T) { + echo, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer echo.Close() + go func() { + c, e := echo.Accept() + if e == nil { + defer c.Close() + io.Copy(c, c) + } + }() + s, err := New(func(*http.Request) (*url.URL, error) { return nil, nil }) + if err != nil { + t.Fatal(err) + } + endpoint, _ := url.Parse(s.URL()) + c, err := dialTunnel(context.Background(), echo.Addr().String(), endpoint) + if err != nil { + t.Fatal(err) + } + defer c.Close() + c.SetDeadline(time.Now().Add(3 * time.Second)) + c.Write([]byte("hello\n")) + line, err := bufio.NewReader(c).ReadString('\n') + if err != nil || line != "hello\n" { + t.Fatalf("echo %q %v", line, err) + } + s.Close() + if _, err := c.Read(make([]byte, 1)); err == nil { + t.Fatal("tunnel stayed open") + } +} + +func TestSOCKSIngressPreservesRemoteDNS(t *testing.T) { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != "CONNECT" || r.Host != "unresolved.invalid:80" { + t.Errorf("lost destination: %s %s", r.Method, r.Host) + } + c, b, err := w.(http.Hijacker).Hijack() + if err != nil { + return + } + defer c.Close() + b.WriteString("HTTP/1.1 200 Connection Established\r\n\r\n") + b.Flush() + request, err := http.ReadRequest(b.Reader) + if err != nil { + t.Error(err) + return + } + request.Body.Close() + b.WriteString("HTTP/1.1 200 OK\r\nContent-Length: 7\r\nConnection: close\r\n\r\nproxied") + b.Flush() + })) + defer upstream.Close() + target, _ := url.Parse(upstream.URL) + local, err := New(func(r *http.Request) (*url.URL, error) { + if r.URL.Host != "unresolved.invalid:80" { + t.Errorf("selector lost hostname: %s", r.URL) + } + return target, nil + }) + if err != nil { + t.Fatal(err) + } + defer local.Close() + endpoint, _ := url.Parse(local.SOCKSURL()) + transport := &http.Transport{Proxy: http.ProxyURL(endpoint)} + defer transport.CloseIdleConnections() + client := &http.Client{Transport: transport, Timeout: 5 * time.Second} + response, err := client.Get("http://unresolved.invalid/") + if err != nil { + t.Fatal(err) + } + defer response.Body.Close() + body, _ := io.ReadAll(response.Body) + if string(body) != "proxied" { + t.Fatalf("unexpected body %q", body) + } +} + +func TestHTTPUpgradeIsForwarded(t *testing.T) { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Upgrade") != "fixture" { + http.Error(w, "upgrade lost", 400) + return + } + c, b, err := w.(http.Hijacker).Hijack() + if err != nil { + return + } + defer c.Close() + b.WriteString("HTTP/1.1 101 Switching Protocols\r\nConnection: Upgrade\r\nUpgrade: fixture\r\n\r\n") + b.Flush() + line, err := b.ReadString('\n') + if err != nil { + return + } + b.WriteString(line) + b.Flush() + })) + defer upstream.Close() + local, err := New(nil) + if err != nil { + t.Fatal(err) + } + defer local.Close() + endpoint, _ := url.Parse(local.URL()) + c, err := net.Dial("tcp", endpoint.Host) + if err != nil { + t.Fatal(err) + } + defer c.Close() + c.SetDeadline(time.Now().Add(3 * time.Second)) + io.WriteString(c, "GET "+upstream.URL+"/ HTTP/1.1\r\nHost: fixture\r\nConnection: Upgrade\r\nUpgrade: fixture\r\n\r\n") + reader := bufio.NewReader(c) + response, err := http.ReadResponse(reader, nil) + if err != nil { + t.Fatal(err) + } + if response.StatusCode != 101 { + t.Fatalf("upgrade failed: %s", response.Status) + } + io.WriteString(c, "hello\n") + line, err := reader.ReadString('\n') + if err != nil || line != "hello\n" { + t.Fatalf("upgraded echo %q %v", line, err) + } +} diff --git a/internal/webview/rpcprovider/README.md b/internal/webview/rpcprovider/README.md index 3590f557..5036ddfc 100644 --- a/internal/webview/rpcprovider/README.md +++ b/internal/webview/rpcprovider/README.md @@ -1,5 +1,11 @@ # rpcprovider +The private Flutter native profile bridge is maintained in the +[GopeedLab flutter_inappwebview fork](https://github.com/GopeedLab/flutter_inappwebview/tree/codex/gopeed-profiles). +All three platform overrides pin one commit in `ui/flutter/pubspec.yaml`; +see the fork’s [GOPEED.md](https://github.com/GopeedLab/flutter_inappwebview/blob/codex/gopeed-profiles/GOPEED.md) +for patch scope and upstream synchronization. + `rpcprovider` is the RPC-backed implementation of Gopeed's `webview.Provider` interface. It is intended for environments where WebView capability is owned by another process, most notably mobile hosts. This package does not implement a WebView itself. Instead, it translates the `pkg/download/engine/webview` page API into single-request `POST + JSON` calls sent to a local host-side RPC service. diff --git a/internal/webview/rpcprovider/provider.go b/internal/webview/rpcprovider/provider.go index 9a22bea2..607bb5a6 100644 --- a/internal/webview/rpcprovider/provider.go +++ b/internal/webview/rpcprovider/provider.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "encoding/json" + "errors" "fmt" "net" "net/http" @@ -109,9 +110,14 @@ func (p *page) ClearCookies() error { } func (p *page) Close() error { - return p.client.Call(enginewebview.MethodPageClose, enginewebview.PageCloseParams{ + err := p.client.Call(enginewebview.MethodPageClose, enginewebview.PageCloseParams{ PageID: p.id, }, nil) + var rpcErr *enginewebview.RPCError + if errors.As(err, &rpcErr) && rpcErr.Code == enginewebview.ErrorCodePageNotFound { + return nil + } + return err } type Client struct { @@ -193,3 +199,7 @@ func (c *Client) Call(method string, params any, result any) error { } return json.Unmarshal(body.Result, result) } + +func (p *Provider) RemoveProfile(profileID, dataPath string) error { + return p.client.Call(enginewebview.MethodProfileRemove, enginewebview.ProfileRemoveParams{ProfileID: profileID}, nil) +} diff --git a/internal/webview/rpcprovider/provider_integration_test.go b/internal/webview/rpcprovider/provider_integration_test.go index 6ade9458..5f26ca56 100644 --- a/internal/webview/rpcprovider/provider_integration_test.go +++ b/internal/webview/rpcprovider/provider_integration_test.go @@ -37,3 +37,8 @@ func defaultWebViewRPCAddress() string { } return filepath.Join(homeDir, "Library", "Application Support", "com.gopeed.gopeed", "gopeed_webview.sock") } + +func TestProviderProfiles(t *testing.T) { + provider := New(enginewebview.RPCConfig{Network: *webViewRPCNetwork, Address: *webViewRPCAddress, Token: *webViewRPCToken}) + integrationtest.RunProfileContract(t, provider) +} diff --git a/internal/webview/rpcprovider/provider_test.go b/internal/webview/rpcprovider/provider_test.go index 9ec03ac1..0a28cd9f 100644 --- a/internal/webview/rpcprovider/provider_test.go +++ b/internal/webview/rpcprovider/provider_test.go @@ -184,3 +184,29 @@ func TestPageSetCookieParamsOmitsZeroExpires(t *testing.T) { t.Fatalf("expected expires in payload: %s", got) } } + +func TestRemoveProfileUsesPrivateHostRPC(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var request struct { + Method string `json:"method"` + Params map[string]string `json:"params"` + } + if err := json.NewDecoder(r.Body).Decode(&request); err != nil { + t.Error(err) + } + if request.Method != enginewebview.MethodProfileRemove || request.Params["profileId"] != "extension-profile" || len(request.Params) != 1 { + t.Errorf("unexpected profile removal request: %+v", request) + } + if r.Header.Get("Authorization") != "Bearer secret" { + t.Error("missing host authentication") + } + w.Header().Set("Content-Type", "application/json") + w.Write([]byte(`{"error":{"code":"PROFILE_REMOVE_FAILED","message":"store busy"}}`)) + })) + defer server.Close() + provider := New(enginewebview.RPCConfig{Network: "tcp", Address: strings.TrimPrefix(server.URL, "http://"), Token: "secret"}) + err := provider.(enginewebview.ProfileRemover).RemoveProfile("extension-profile", "/host-only/path") + if err == nil || err.Error() != "store busy" { + t.Fatalf("cleanup failure was lost: %v", err) + } +} diff --git a/pkg/download/downloader.go b/pkg/download/downloader.go index 72953a8e..d3a4ff6a 100644 --- a/pkg/download/downloader.go +++ b/pkg/download/downloader.go @@ -21,6 +21,7 @@ import ( "github.com/GopeedLab/gopeed/internal/controller" "github.com/GopeedLab/gopeed/internal/fetcher" "github.com/GopeedLab/gopeed/internal/logger" + webviewproxy "github.com/GopeedLab/gopeed/internal/webview/proxy" "github.com/GopeedLab/gopeed/pkg/base" "github.com/GopeedLab/gopeed/pkg/protocol/http" "github.com/GopeedLab/gopeed/pkg/util" @@ -97,8 +98,12 @@ type Progress struct { } type Downloader struct { - Logger *logger.Logger - ExtensionLogger *logger.Logger + webviewProfilesLock sync.Mutex + webviewProfiles map[string]*extensionWebViewProfile + webviewProxyLock sync.Mutex + webviewProxy *webviewproxy.Server + Logger *logger.Logger + ExtensionLogger *logger.Logger cfg *DownloaderConfig fetcherCache map[string]fetcher.Fetcher @@ -1114,6 +1119,11 @@ func (d *Downloader) doDelete(task *Task, force bool) (err error) { func (d *Downloader) Close() error { d.closed.Store(true) + d.webviewProxyLock.Lock() + if d.webviewProxy != nil { + _ = d.webviewProxy.Close() + } + d.webviewProxyLock.Unlock() closeArr := []func() error{ d.pauseAll, diff --git a/pkg/download/engine/webview/rpc.go b/pkg/download/engine/webview/rpc.go index 5dab17ef..77c91976 100644 --- a/pkg/download/engine/webview/rpc.go +++ b/pkg/download/engine/webview/rpc.go @@ -5,6 +5,7 @@ import "encoding/json" const RPCEndpointPath = "/webview" const ( + MethodProfileRemove = "profile.remove" MethodIsAvailable = "webview.isAvailable" MethodPageOpen = "page.open" MethodPageAddInitScript = "page.addInitScript" @@ -68,6 +69,9 @@ type IsAvailableResult struct { } type PageOpenParams struct { + ProfileID string `json:"profileId,omitempty"` + DataPath string `json:"dataPath,omitempty"` + ProxyURL string `json:"proxyUrl,omitempty"` Headless bool `json:"headless,omitempty"` Debug bool `json:"debug,omitempty"` Title string `json:"title,omitempty"` @@ -78,6 +82,9 @@ type PageOpenParams struct { func NewPageOpenParams(opts OpenOptions) PageOpenParams { return PageOpenParams{ + ProfileID: opts.ProfileID, + DataPath: opts.DataPath, + ProxyURL: opts.ProxyURL, Headless: opts.Headless, Debug: opts.Debug, Title: opts.Title, @@ -141,3 +148,8 @@ type PageCloseParams struct { } type EmptyResult struct{} + +// ProfileRemoveParams belongs to the private host RPC. +type ProfileRemoveParams struct { + ProfileID string `json:"profileId"` +} diff --git a/pkg/download/engine/webview/runtime.go b/pkg/download/engine/webview/runtime.go index 64a236e4..cc115826 100644 --- a/pkg/download/engine/webview/runtime.go +++ b/pkg/download/engine/webview/runtime.go @@ -17,6 +17,11 @@ type Opener interface { Open(opts OpenOptions) (Page, error) } +// ProfileRemover is a host-only lifecycle operation, never exposed to extension JS. +type ProfileRemover interface { + RemoveProfile(profileID, dataPath string) error +} + type Page interface { AddInitScript(script string) error Goto(url string, opts GotoOptions) error @@ -29,6 +34,10 @@ type Page interface { } type OpenOptions struct { + // Host-owned fields are never parsed from extension JavaScript options. + ProfileID string + DataPath string + ProxyURL string Headless bool Debug bool Title string diff --git a/pkg/download/engine/webview/runtime_test.go b/pkg/download/engine/webview/runtime_test.go index 066dea2e..6f963055 100644 --- a/pkg/download/engine/webview/runtime_test.go +++ b/pkg/download/engine/webview/runtime_test.go @@ -336,3 +336,15 @@ func (p *fakePage) Close() error { p.closed = true return nil } + +func TestExtensionCannotSupplyHostProfileOrProxy(t *testing.T) { + opener := &fakeOpener{page: &fakePage{}} + runtime := NewRuntime(opener, true) + _, err := runtime.Open(map[string]any{"profileId": "other-extension", "dataPath": "/tmp/other-profile", "proxyUrl": "http://example.com:1234", "ProfileID": "other"}) + if err != nil { + t.Fatal(err) + } + if opener.opts.ProfileID != "" || opener.opts.DataPath != "" || opener.opts.ProxyURL != "" { + t.Fatalf("extension controlled host settings: %+v", opener.opts) + } +} diff --git a/pkg/download/extension.go b/pkg/download/extension.go index c9bb7832..53f9e989 100644 --- a/pkg/download/extension.go +++ b/pkg/download/extension.go @@ -65,6 +65,7 @@ func (d *Downloader) InstallExtensionByFolder(path string, devMode bool) (*Exten // if extension is not installed, add it to the list, otherwise update it installedExt := d.getExtension(ext.Identity) + newInstallation := installedExt == nil if installedExt == nil { ext.CreatedAt = time.Now() ext.UpdatedAt = ext.CreatedAt @@ -76,6 +77,11 @@ func (d *Downloader) InstallExtensionByFolder(path string, devMode bool) (*Exten if err = d.storage.Put(bucketExtension, installedExt.Identity, installedExt); err != nil { return nil, err } + if newInstallation { + d.webviewProfilesLock.Lock() + delete(d.webviewProfiles, ext.Identity) + d.webviewProfilesLock.Unlock() + } return installedExt, nil } @@ -144,6 +150,9 @@ func (d *Downloader) DeleteExtension(identity string) error { if err != nil { return err } + if err := d.removeExtensionWebViewProfile(identity); err != nil { + return fmt.Errorf("remove extension WebView data: %w", err) + } // remove from disk if !ext.DevMode { if err := os.RemoveAll(d.ExtensionPath(ext)); err != nil { @@ -365,7 +374,7 @@ func doTrigger[T any](d *Downloader, event ActivationEvent, req *base.Request, c engine, session := d.newExtensionEngine() defer session.CloseIfIdle() gopeed.Runtime = &InstanceRuntime{ - WebView: d.newExtensionWebViewRuntime(session), + WebView: d.newExtensionWebViewRuntime(session, ext.buildIdentity()), } err = injectGopeed(engine.Runtime, gopeed) if err != nil { diff --git a/pkg/download/extension_engine.go b/pkg/download/extension_engine.go index 39b832f1..4ef05d23 100644 --- a/pkg/download/extension_engine.go +++ b/pkg/download/extension_engine.go @@ -30,7 +30,7 @@ func (d *Downloader) NewExtensionEngine(ext *Extension, settings map[string]any) identity: ext.buildIdentity(), }, Runtime: &InstanceRuntime{ - WebView: d.newExtensionWebViewRuntime(session), + WebView: d.newExtensionWebViewRuntime(session, ext.buildIdentity()), }, } if err := injectGopeed(engine.Runtime, gopeed); err != nil { diff --git a/pkg/download/extension_engine_session.go b/pkg/download/extension_engine_session.go index 4f248c88..45b89dcb 100644 --- a/pkg/download/extension_engine_session.go +++ b/pkg/download/extension_engine_session.go @@ -135,13 +135,13 @@ func (d *Downloader) newExtensionEngine() (*engine.Engine, *engineSession) { return e, session } -func (d *Downloader) newExtensionWebViewRuntime(session *engineSession) *enginewebview.Runtime { +func (d *Downloader) newExtensionWebViewRuntime(session *engineSession, identity string) *enginewebview.Runtime { var ( opener enginewebview.Opener available bool ) if provider := d.cfg.WebViewProvider; provider != nil && provider.IsAvailable() { - opener = provider + opener = &extensionWebViewOpener{downloader: d, provider: provider, identity: identity, profile: d.extensionWebViewProfile(identity)} available = true } runtime := enginewebview.NewRuntime(opener, available) diff --git a/pkg/download/extension_webview_config.go b/pkg/download/extension_webview_config.go new file mode 100644 index 00000000..164df1e4 --- /dev/null +++ b/pkg/download/extension_webview_config.go @@ -0,0 +1,183 @@ +package download + +import ( + "crypto/sha256" + "fmt" + "net/http" + "net/url" + "os" + "sync" + + "github.com/GopeedLab/gopeed/pkg/base" + "path/filepath" + "runtime" + + webviewproxy "github.com/GopeedLab/gopeed/internal/webview/proxy" + enginewebview "github.com/GopeedLab/gopeed/pkg/download/engine/webview" +) + +// extensionWebViewOpener binds browser state to the owning extension. Neither +// identity, filesystem paths nor proxy settings are accepted from JavaScript. +type extensionWebViewOpener struct { + downloader *Downloader + provider enginewebview.Provider + identity string + profile *extensionWebViewProfile +} + +func (o *extensionWebViewOpener) Open(opts enginewebview.OpenOptions) (enginewebview.Page, error) { + if o.identity == "" { + return nil, fmt.Errorf("webview extension identity is required") + } + profile := o.profile + if profile == nil { + profile = o.downloader.extensionWebViewProfile(o.identity) + } + profile.mu.Lock() + defer profile.mu.Unlock() + if profile.revoked { + return nil, fmt.Errorf("extension WebView profile has been removed") + } + id, path, err := o.downloader.extensionWebViewLocation(o.identity) + if err != nil { + return nil, err + } + opts.ProfileID, opts.DataPath = id, path + opts.ProxyURL, err = o.downloader.webviewProxyURL() + if err != nil { + return nil, err + } + page, err := o.provider.Open(opts) + if err != nil { + return nil, err + } + tracked := &extensionWebViewPage{Page: page, profile: profile} + profile.pages[tracked] = struct{}{} + return tracked, nil +} + +func (d *Downloader) webviewProxyURL() (string, error) { + d.webviewProxyLock.Lock() + defer d.webviewProxyLock.Unlock() + if d.closed.Load() { + return "", fmt.Errorf("downloader is closed") + } + if d.webviewProxy == nil { + server, err := webviewproxy.New(func(r *http.Request) (*url.URL, error) { + if d.blob != nil && d.blob.IsURL(r.URL.String()) { + return nil, nil + } + d.configLock.RLock() + var cfg base.DownloaderProxyConfig + if d.cfg.Proxy != nil { + cfg = *d.cfg.Proxy + } + d.configLock.RUnlock() + handler := cfg.ToHandler() + if handler == nil { + return nil, nil + } + return handler(r) + }) + if err != nil { + return "", err + } + d.webviewProxy = server + } + if runtime.GOOS == "darwin" || runtime.GOOS == "ios" { + return d.webviewProxy.SOCKSURL(), nil + } + return d.webviewProxy.URL(), nil +} + +// A revoked profile is retained until a fresh installation. Existing runtimes +// retain this object, so reinstalling cannot let an old runtime recreate data. +type extensionWebViewProfile struct { + mu sync.Mutex + deleteMu sync.Mutex + revoked bool + pages map[*extensionWebViewPage]struct{} +} + +type extensionWebViewPage struct { + enginewebview.Page + profile *extensionWebViewProfile + mu sync.Mutex + closed bool +} + +func (p *extensionWebViewPage) Close() error { + p.mu.Lock() + defer p.mu.Unlock() + if p.closed { + return nil + } + if err := p.Page.Close(); err != nil { + return err + } + p.closed = true + p.profile.mu.Lock() + delete(p.profile.pages, p) + p.profile.mu.Unlock() + return nil +} + +func (d *Downloader) extensionWebViewProfile(identity string) *extensionWebViewProfile { + d.webviewProfilesLock.Lock() + defer d.webviewProfilesLock.Unlock() + if d.webviewProfiles == nil { + d.webviewProfiles = make(map[string]*extensionWebViewProfile) + } + profile := d.webviewProfiles[identity] + if profile == nil { + profile = &extensionWebViewProfile{pages: make(map[*extensionWebViewPage]struct{})} + d.webviewProfiles[identity] = profile + } + return profile +} + +func (d *Downloader) extensionWebViewLocation(identity string) (string, string, error) { + if identity == "" { + return "", "", fmt.Errorf("webview extension identity is required") + } + sum := sha256.Sum256([]byte(identity)) + id := fmt.Sprintf("%x-%x-%x-%x-%x", sum[:4], sum[4:6], sum[6:8], sum[8:10], sum[10:16]) + path, err := filepath.Abs(filepath.Join(d.cfg.StorageDir, "webview", fmt.Sprintf("%x", sum[:]))) + return id, path, err +} + +func (d *Downloader) removeExtensionWebViewProfile(identity string) error { + profile := d.extensionWebViewProfile(identity) + profile.deleteMu.Lock() + defer profile.deleteMu.Unlock() + profile.mu.Lock() + profile.revoked = true + pages := make([]*extensionWebViewPage, 0, len(profile.pages)) + for page := range profile.pages { + pages = append(pages, page) + } + profile.mu.Unlock() + for _, page := range pages { + if err := page.Close(); err != nil { + return err + } + } + id, path, err := d.extensionWebViewLocation(identity) + if err != nil { + return err + } + if provider := d.cfg.WebViewProvider; provider != nil { + remover, ok := provider.(enginewebview.ProfileRemover) + if !ok { + // An unavailable backend cannot have created a profile this run. + // Still fail if a persisted native profile directory exists. + _, statErr := os.Stat(path) + if provider.IsAvailable() || len(pages) != 0 || !os.IsNotExist(statErr) { + return fmt.Errorf("WebView provider does not support profile removal") + } + } else if err := remover.RemoveProfile(id, path); err != nil { + return err + } + } + return os.RemoveAll(path) +} diff --git a/pkg/download/extension_webview_config_test.go b/pkg/download/extension_webview_config_test.go new file mode 100644 index 00000000..2c972970 --- /dev/null +++ b/pkg/download/extension_webview_config_test.go @@ -0,0 +1,51 @@ +package download + +import ( + "path/filepath" + "sync" + "testing" + + "github.com/GopeedLab/gopeed/pkg/base" + enginewebview "github.com/GopeedLab/gopeed/pkg/download/engine/webview" +) + +type profileCaptureProvider struct{ options []enginewebview.OpenOptions } + +func (*profileCaptureProvider) IsAvailable() bool { return true } +func (p *profileCaptureProvider) Open(opts enginewebview.OpenOptions) (enginewebview.Page, error) { + p.options = append(p.options, opts) + return fakeRuntimeWebViewPage{}, nil +} + +func TestWebViewProfileBoundToExtension(t *testing.T) { + root := t.TempDir() + d := &Downloader{cfg: &DownloaderConfig{StorageDir: root, DownloaderStoreConfig: &base.DownloaderStoreConfig{}}, configLock: &sync.RWMutex{}} + t.Cleanup(func() { + if d.webviewProxy != nil { + d.webviewProxy.Close() + } + }) + provider := &profileCaptureProvider{} + for _, id := range []string{"extension-a", "extension-b", "extension-a", "../../escape"} { + opener := &extensionWebViewOpener{downloader: d, provider: provider, identity: id} + _, err := opener.Open(enginewebview.OpenOptions{ProfileID: "forged", DataPath: "/tmp/forged", ProxyURL: "http://wrong:80"}) + if err != nil { + t.Fatal(err) + } + } + a, b, reopen := provider.options[0], provider.options[1], provider.options[2] + if a.ProfileID == b.ProfileID || a.DataPath == b.DataPath { + t.Fatal("different extensions shared browser storage") + } + if a.ProfileID != reopen.ProfileID || a.DataPath != reopen.DataPath { + t.Fatal("same extension lost profile identity") + } + for _, opts := range provider.options { + if filepath.Dir(opts.DataPath) != filepath.Join(root, "webview") { + t.Fatalf("profile escaped storage root: %s", opts.DataPath) + } + if opts.ProxyURL != d.webviewProxy.URL() && opts.ProxyURL != d.webviewProxy.SOCKSURL() { + t.Fatal("host proxy was overridden") + } + } +} diff --git a/pkg/download/extension_webview_remove_test.go b/pkg/download/extension_webview_remove_test.go new file mode 100644 index 00000000..50bc3191 --- /dev/null +++ b/pkg/download/extension_webview_remove_test.go @@ -0,0 +1,189 @@ +package download + +import ( + "errors" + "os" + "path/filepath" + "sync" + "testing" + + "github.com/GopeedLab/gopeed/pkg/base" + webview "github.com/GopeedLab/gopeed/pkg/download/engine/webview" +) + +type removalProvider struct { + pages map[string][]*removalPage + removed []string + failure error + entered chan struct{} + resume chan struct{} +} +type removalPage struct { + fakeRuntimeWebViewPage + closed bool +} + +func (p *removalPage) Close() error { p.closed = true; return nil } +func (p *removalProvider) IsAvailable() bool { return true } +func (p *removalProvider) Open(opts webview.OpenOptions) (webview.Page, error) { + if p.entered != nil { + close(p.entered) + <-p.resume + } + if p.pages == nil { + p.pages = make(map[string][]*removalPage) + } + page := &removalPage{} + p.pages[opts.ProfileID] = append(p.pages[opts.ProfileID], page) + return page, nil +} +func (p *removalProvider) RemoveProfile(id, path string) error { + for _, page := range p.pages[id] { + if !page.closed { + return errors.New("profile removed before closing pages") + } + } + if p.failure != nil { + return p.failure + } + p.removed = append(p.removed, id) + return nil +} +func removalDownloader(t *testing.T, provider *removalProvider) *Downloader { + t.Helper() + storage := NewMemStorage() + if err := storage.Setup([]string{bucketExtension, bucketExtensionStorage}); err != nil { + t.Fatal(err) + } + d := &Downloader{cfg: &DownloaderConfig{StorageDir: t.TempDir(), DownloaderStoreConfig: &base.DownloaderStoreConfig{}, WebViewProvider: provider}, storage: storage, configLock: &sync.RWMutex{}} + t.Cleanup(func() { + if d.webviewProxy != nil { + d.webviewProxy.Close() + } + }) + return d +} +func profileSentinel(t *testing.T, d *Downloader, id string) string { + t.Helper() + _, path, err := d.extensionWebViewLocation(id) + if err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(path, 0700); err != nil { + t.Fatal(err) + } + sentinel := filepath.Join(path, "cookies.sqlite") + if err := os.WriteFile(sentinel, []byte("private browser state"), 0600); err != nil { + t.Fatal(err) + } + return sentinel +} +func TestDeleteExtensionRemovesWebViewData(t *testing.T) { + provider := &removalProvider{} + d := removalDownloader(t, provider) + ext, err := d.InstallExtensionByFolder("testdata/extensions/basic", true) + if err != nil { + t.Fatal(err) + } + identity := ext.Identity + opener := &extensionWebViewOpener{downloader: d, provider: provider, identity: identity, profile: d.extensionWebViewProfile(identity)} + for i := 0; i < 2; i++ { + if _, err := opener.Open(webview.OpenOptions{}); err != nil { + t.Fatal(err) + } + } + sentinel := profileSentinel(t, d, identity) + other := profileSentinel(t, d, "other-extension") + // Updating the installation keeps both its profile object and stored data. + if _, err := d.InstallExtensionByFolder("testdata/extensions/basic", true); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(sentinel); err != nil { + t.Fatal("upgrade removed browser data", err) + } + if err := d.DeleteExtension(identity); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(sentinel); !os.IsNotExist(err) { + t.Fatalf("browser data retained: %v", err) + } + if _, err := os.Stat(other); err != nil { + t.Fatal("other extension data was removed", err) + } + if _, err := os.Stat(ext.DevPath); err != nil { + t.Fatal("dev source was removed", err) + } + if len(provider.removed) != 1 { + t.Fatal("native profile was not removed") + } + if _, err := opener.Open(webview.OpenOptions{}); err == nil { + t.Fatal("old runtime recreated removed profile") + } + if _, err := d.InstallExtensionByFolder("testdata/extensions/basic", true); err != nil { + t.Fatal(err) + } + fresh := &extensionWebViewOpener{downloader: d, provider: provider, identity: identity, profile: d.extensionWebViewProfile(identity)} + if _, err := fresh.Open(webview.OpenOptions{}); err != nil { + t.Fatal("reinstall could not open profile", err) + } + if _, err := opener.Open(webview.OpenOptions{}); err == nil { + t.Fatal("reinstall reactivated old runtime") + } +} +func TestDeleteExtensionProfileFailureIsRetryable(t *testing.T) { + provider := &removalProvider{failure: errors.New("native store busy")} + d := removalDownloader(t, provider) + ext, err := d.InstallExtensionByFolder("testdata/extensions/basic", true) + if err != nil { + t.Fatal(err) + } + sentinel := profileSentinel(t, d, ext.Identity) + if err := d.DeleteExtension(ext.Identity); err == nil { + t.Fatal("cleanup failure was swallowed") + } + if _, err := d.GetExtension(ext.Identity); err != nil { + t.Fatal("extension removed before cleanup completed") + } + if _, err := os.Stat(sentinel); err != nil { + t.Fatal("profile path removed before native cleanup completed") + } + provider.failure = nil + if err := d.DeleteExtension(ext.Identity); err != nil { + t.Fatal(err) + } + if _, err := d.GetExtension(ext.Identity); !errors.Is(err, ErrExtensionNotFound) { + t.Fatal("extension was not uninstalled") + } +} +func TestRemoveProfileWaitsForOpeningPage(t *testing.T) { + provider := &removalProvider{entered: make(chan struct{}), resume: make(chan struct{})} + d := removalDownloader(t, provider) + opener := &extensionWebViewOpener{downloader: d, provider: provider, identity: "race", profile: d.extensionWebViewProfile("race")} + opened := make(chan error, 1) + go func() { _, err := opener.Open(webview.OpenOptions{}); opened <- err }() + <-provider.entered + removed := make(chan error, 1) + go func() { removed <- d.removeExtensionWebViewProfile("race") }() + close(provider.resume) + if err := <-opened; err != nil { + t.Fatal(err) + } + if err := <-removed; err != nil { + t.Fatal(err) + } + if len(provider.removed) != 1 { + t.Fatal("profile not removed") + } +} + +func TestUninstallWithoutAvailableWebView(t *testing.T) { + d := removalDownloader(t, &removalProvider{}) + d.cfg.WebViewProvider = webview.NewUnavailableProvider() + ext, err := d.InstallExtensionByFolder("testdata/extensions/basic", true) + if err != nil { + t.Fatal(err) + } + if err := d.DeleteExtension(ext.Identity); err != nil { + t.Fatal(err) + } +} diff --git a/ui/flutter/analysis_options.yaml b/ui/flutter/analysis_options.yaml index 89f7f921..a1a8bbfc 100644 --- a/ui/flutter/analysis_options.yaml +++ b/ui/flutter/analysis_options.yaml @@ -29,3 +29,7 @@ linter: formatter: page_width: 120 + +analyzer: + exclude: + - third_party/** diff --git a/ui/flutter/lib/app/rpc/webview_rpc_service.dart b/ui/flutter/lib/app/rpc/webview_rpc_service.dart index 653e78e6..b138abd0 100644 --- a/ui/flutter/lib/app/rpc/webview_rpc_service.dart +++ b/ui/flutter/lib/app/rpc/webview_rpc_service.dart @@ -3,6 +3,7 @@ import 'dart:collection'; import 'dart:convert'; import 'package:flutter/material.dart'; +import 'package:flutter/services.dart'; import 'package:flutter_inappwebview/flutter_inappwebview.dart'; import '../../core/common/start_config.dart'; @@ -73,7 +74,6 @@ class WebViewRpcService { final Map _pagesById = {}; final Completer _overlayReady = Completer(); - late final CookieManager _cookieManager = CookieManager.instance(); RpcServerHandle? _server; int _pageSeq = 0; @@ -129,6 +129,9 @@ class WebViewRpcService { switch (method) { case 'webview.isAvailable': return {'available': true}; + case 'profile.remove': + await _removeProfile(_string(params, 'profileId')); + return {}; case 'page.open': return _openPage(params); case 'page.addInitScript': @@ -162,10 +165,12 @@ class WebViewRpcService { } Future> _openPage(Map params) async { + final profile = WebViewProfile(_string(params, 'profileId')); + await profile.prepare(proxyUrl: Uri.parse(_string(params, 'proxyUrl'))); final pageId = 'page-${++_pageSeq}'; final session = WebViewRpcPageSession( pageId: pageId, - cookieManager: _cookieManager, + profile: profile, headless: params['headless'] as bool? ?? false, debug: params['debug'] as bool? ?? false, title: params['title'] as String? ?? '', @@ -185,6 +190,15 @@ class WebViewRpcService { return {'pageId': pageId}; } + Future _removeProfile(String id) async { + for (final page in _pagesById.values.where((page) => page.profile.id == id).toList()) { + await _closePage(page.pageId); + } + // Let visible platform views unmount before releasing their native store. + await WidgetsBinding.instance.endOfFrame; + await WebViewProfile(id).remove(); + } + WebViewRpcPageSession _page(Map params) { final page = _pagesById[_pageId(params)]; if (page == null) { @@ -271,6 +285,9 @@ class WebViewRpcService { if (error is WebViewRpcException) { return {'code': error.code, 'message': error.message}; } + if (error is PlatformException && error.code == 'UNAVAILABLE') { + return {'code': error.code, 'message': error.message ?? 'WebView is unavailable'}; + } final code = switch (method) { 'page.goto' => 'NAVIGATION_FAILED', 'page.execute' => 'EVALUATION_FAILED', @@ -283,7 +300,7 @@ class WebViewRpcService { class WebViewRpcPageSession { WebViewRpcPageSession({ required this.pageId, - required this.cookieManager, + required this.profile, required this.headless, required this.debug, required this.title, @@ -293,7 +310,8 @@ class WebViewRpcPageSession { }); final String pageId; - final CookieManager cookieManager; + final WebViewProfile profile; + late final CookieManager cookieManager = profile.cookieManager; final bool headless; final bool debug; final String title; @@ -449,6 +467,8 @@ class WebViewRpcPageSession { 'path': cookie.path ?? '/', 'secure': cookie.isSecure, 'httpOnly': cookie.isHttpOnly, + if (cookie.expiresDate != null) + 'expires': DateTime.fromMillisecondsSinceEpoch(cookie.expiresDate!, isUtc: true).toIso8601String(), }; }) .toList(growable: false); @@ -524,6 +544,7 @@ class WebViewRpcPageSession { } InAppWebViewSettings get _settings => InAppWebViewSettings( + profileId: profile.id, javaScriptEnabled: true, transparentBackground: true, isInspectable: debug, diff --git a/ui/flutter/macos/Runner/Info.plist b/ui/flutter/macos/Runner/Info.plist index cee06fe7..48add6d5 100644 --- a/ui/flutter/macos/Runner/Info.plist +++ b/ui/flutter/macos/Runner/Info.plist @@ -2,6 +2,11 @@ + NSAppTransportSecurity + + NSAllowsArbitraryLoadsInWebContent + + CFBundleDevelopmentRegion $(DEVELOPMENT_LANGUAGE) CFBundleExecutable diff --git a/ui/flutter/pubspec.lock b/ui/flutter/pubspec.lock index e3384198..06828e78 100644 --- a/ui/flutter/pubspec.lock +++ b/ui/flutter/pubspec.lock @@ -457,18 +457,20 @@ packages: flutter_inappwebview: dependency: "direct main" description: - name: flutter_inappwebview - sha256: "80092d13d3e29b6227e25b67973c67c7210bd5e35c4b747ca908e31eb71a46d5" - url: "https://pub.dev" - source: hosted + path: flutter_inappwebview + ref: d579215760e1c1e58a4ff7601a6055b5e6adf8e8 + resolved-ref: d579215760e1c1e58a4ff7601a6055b5e6adf8e8 + url: "https://github.com/GopeedLab/flutter_inappwebview.git" + source: git version: "6.1.5" flutter_inappwebview_android: - dependency: transitive + dependency: "direct overridden" description: - name: flutter_inappwebview_android - sha256: "62557c15a5c2db5d195cb3892aab74fcaec266d7b86d59a6f0027abd672cddba" - url: "https://pub.dev" - source: hosted + path: flutter_inappwebview_android + ref: d579215760e1c1e58a4ff7601a6055b5e6adf8e8 + resolved-ref: d579215760e1c1e58a4ff7601a6055b5e6adf8e8 + url: "https://github.com/GopeedLab/flutter_inappwebview.git" + source: git version: "1.1.3" flutter_inappwebview_internal_annotations: dependency: transitive @@ -479,28 +481,31 @@ packages: source: hosted version: "1.3.0" flutter_inappwebview_ios: - dependency: transitive + dependency: "direct overridden" description: - name: flutter_inappwebview_ios - sha256: "5818cf9b26cf0cbb0f62ff50772217d41ea8d3d9cc00279c45f8aabaa1b4025d" - url: "https://pub.dev" - source: hosted + path: flutter_inappwebview_ios + ref: d579215760e1c1e58a4ff7601a6055b5e6adf8e8 + resolved-ref: d579215760e1c1e58a4ff7601a6055b5e6adf8e8 + url: "https://github.com/GopeedLab/flutter_inappwebview.git" + source: git version: "1.1.2" flutter_inappwebview_macos: - dependency: transitive + dependency: "direct overridden" description: - name: flutter_inappwebview_macos - sha256: c1fbb86af1a3738e3541364d7d1866315ffb0468a1a77e34198c9be571287da1 - url: "https://pub.dev" - source: hosted + path: flutter_inappwebview_macos + ref: d579215760e1c1e58a4ff7601a6055b5e6adf8e8 + resolved-ref: d579215760e1c1e58a4ff7601a6055b5e6adf8e8 + url: "https://github.com/GopeedLab/flutter_inappwebview.git" + source: git version: "1.1.2" flutter_inappwebview_platform_interface: - dependency: transitive + dependency: "direct overridden" description: - name: flutter_inappwebview_platform_interface - sha256: cf5323e194096b6ede7a1ca808c3e0a078e4b33cc3f6338977d75b4024ba2500 - url: "https://pub.dev" - source: hosted + path: flutter_inappwebview_platform_interface + ref: d579215760e1c1e58a4ff7601a6055b5e6adf8e8 + resolved-ref: d579215760e1c1e58a4ff7601a6055b5e6adf8e8 + url: "https://github.com/GopeedLab/flutter_inappwebview.git" + source: git version: "1.3.0+1" flutter_inappwebview_web: dependency: transitive diff --git a/ui/flutter/pubspec.yaml b/ui/flutter/pubspec.yaml index cb27bf83..405a645f 100644 --- a/ui/flutter/pubspec.yaml +++ b/ui/flutter/pubspec.yaml @@ -86,6 +86,32 @@ dependencies: go_router: ^16.2.1 dependency_overrides: + # Public persistent-profile and proxy APIs; keep all packages on one commit. + flutter_inappwebview: + git: + url: https://github.com/GopeedLab/flutter_inappwebview.git + ref: d579215760e1c1e58a4ff7601a6055b5e6adf8e8 + path: flutter_inappwebview + flutter_inappwebview_platform_interface: + git: + url: https://github.com/GopeedLab/flutter_inappwebview.git + ref: d579215760e1c1e58a4ff7601a6055b5e6adf8e8 + path: flutter_inappwebview_platform_interface + flutter_inappwebview_macos: + git: + url: https://github.com/GopeedLab/flutter_inappwebview.git + ref: d579215760e1c1e58a4ff7601a6055b5e6adf8e8 + path: flutter_inappwebview_macos + flutter_inappwebview_ios: + git: + url: https://github.com/GopeedLab/flutter_inappwebview.git + ref: d579215760e1c1e58a4ff7601a6055b5e6adf8e8 + path: flutter_inappwebview_ios + flutter_inappwebview_android: + git: + url: https://github.com/GopeedLab/flutter_inappwebview.git + ref: d579215760e1c1e58a4ff7601a6055b5e6adf8e8 + path: flutter_inappwebview_android # Prevent a native Windows crash when the push notification service is unavailable. # Keep this fork until https://github.com/MaikuB/flutter_local_notifications/pull/2813 is merged and released. flutter_local_notifications: From 7c1447f6590517e07d3283ed03826fb827fbad5a Mon Sep 17 00:00:00 2001 From: UltraGeek0102 Date: Thu, 17 Sep 2026 06:02:21 +0530 Subject: [PATCH 5/5] fix(ios): recover continued processing after toggle (#1519) --- .../GopeedContinuedProcessingManager.swift | 81 +++++++++++++++++++ .../Runner/GopeedLiveActivityManager.swift | 35 +++++++- 2 files changed, 115 insertions(+), 1 deletion(-) diff --git a/ui/flutter/ios/Runner/GopeedContinuedProcessingManager.swift b/ui/flutter/ios/Runner/GopeedContinuedProcessingManager.swift index 7ef2b852..bc8f65a9 100644 --- a/ui/flutter/ios/Runner/GopeedContinuedProcessingManager.swift +++ b/ui/flutter/ios/Runner/GopeedContinuedProcessingManager.swift @@ -32,6 +32,7 @@ final class GopeedContinuedProcessingManager: NSObject { private var expiringTasks: [String: ExpiringTaskState] = [:] private var taskGenerations: [String: UUID] = [:] + private var progressRecoveryTokens: [String: UUID] = [:] private var activeTasks: [String: BGContinuedProcessingTask] = [:] @@ -224,6 +225,10 @@ final class GopeedContinuedProcessingManager: NSObject { switch type { case "task.start": + progressRecoveryTokens.removeValue( + forKey: taskID + ) + beginTask( taskID: taskID, name: name @@ -237,6 +242,11 @@ final class GopeedContinuedProcessingManager: NSObject { generation: generation, force: false ) + } else { + recoverTaskFromProgress( + taskID: taskID, + name: name + ) } case "task.pause": @@ -285,6 +295,72 @@ final class GopeedContinuedProcessingManager: NSObject { } + // MARK: - Recover active task after setting changes + + private func recoverTaskFromProgress( + taskID: String, + name: String + ) { + guard + currentEnabledSnapshot(), + taskGenerations[taskID] == nil, + !pendingTaskIDs.contains(taskID), + activeTasks[taskID] == nil, + expiringTasks[taskID] == nil, + progressRecoveryTokens[taskID] == nil + else { + return + } + + let recoveryToken = UUID() + + progressRecoveryTokens[taskID] = + recoveryToken + + getRuntimeStatus( + taskID: taskID + ) { [weak self] runtime in + guard + let self, + self.progressRecoveryTokens[taskID] + == recoveryToken + else { + return + } + + // The status request has completed. Remove the token so + // another progress event may retry if recovery is still + // necessary. + self.progressRecoveryTokens.removeValue( + forKey: taskID + ) + + guard + self.currentEnabledSnapshot(), + let runtime, + runtime.status == "running", + self.taskGenerations[taskID] == nil, + !self.pendingTaskIDs.contains(taskID), + self.activeTasks[taskID] == nil, + self.expiringTasks[taskID] == nil + else { + return + } + + print( + "ContinuedProcessing:", + "recovering active task:", + taskID + ) + + self.beginTask( + taskID: taskID, + name: name + ) + } + } + + // MARK: - Used by Objective-C forwarder @objc(isHandlingTaskId:) @@ -825,6 +901,10 @@ final class GopeedContinuedProcessingManager: NSObject { success: Bool, finalSubtitle: String ) { + progressRecoveryTokens.removeValue( + forKey: taskID + ) + taskGenerations.removeValue( forKey: taskID ) @@ -1063,6 +1143,7 @@ final class GopeedContinuedProcessingManager: NSObject { pendingTaskIDs.removeAll() expiringTasks.removeAll() taskGenerations.removeAll() + progressRecoveryTokens.removeAll() taskIdentifiers.removeAll() taskNames.removeAll() lastProgressUpdate.removeAll() diff --git a/ui/flutter/ios/Runner/GopeedLiveActivityManager.swift b/ui/flutter/ios/Runner/GopeedLiveActivityManager.swift index 84f965ba..6889dec9 100644 --- a/ui/flutter/ios/Runner/GopeedLiveActivityManager.swift +++ b/ui/flutter/ios/Runner/GopeedLiveActivityManager.swift @@ -108,6 +108,39 @@ final class GopeedLiveActivityManager: NSObject { return generation } + private func generationForProgress( + taskID: String + ) -> UUID? { + stateLock.lock() + defer { stateLock.unlock() } + + guard + !suppressedTaskIDs.contains(taskID) + else { + return nil + } + + if let generation = + taskGenerations[taskID] { + return generation + } + + // Progress may be the first event after Continued Processing + // releases ownership. Establish a fresh lifecycle generation. + // refreshActivity still verifies that Gopeed reports the task + // as running before it can create a Live Activity. + let generation = UUID() + + taskGenerations[taskID] = + generation + + lastUpdateTime.removeValue( + forKey: taskID + ) + + return generation + } + @discardableResult private func invalidateGeneration( taskID: String @@ -252,7 +285,7 @@ final class GopeedLiveActivityManager: NSObject { case "task.progress": guard let generation = - currentGeneration( + generationForProgress( taskID: taskID ) else {