diff --git a/internal/fetcher/fetcher.go b/internal/fetcher/fetcher.go index 4a1374c3..8d3a7821 100644 --- a/internal/fetcher/fetcher.go +++ b/internal/fetcher/fetcher.go @@ -106,7 +106,13 @@ func (s *SchemeFilter) Match(uri string) bool { case FilterTypeUrl: return strings.HasPrefix(uriUpper, patternUpper+":") case FilterTypeFile: - return strings.HasSuffix(uriUpper, "."+patternUpper) + // Strip query and fragment before matching so URLs like + // https://example.com/video.m3u8?sign=abc still route by extension. + clean := uri + if idx := strings.IndexAny(clean, "?#"); idx >= 0 { + clean = clean[:idx] + } + return strings.HasSuffix(strings.ToUpper(clean), "."+patternUpper) case FilterTypeBase64: return strings.HasPrefix(uriUpper, "DATA:"+patternUpper+";BASE64,") } diff --git a/internal/protocol/hls/config.go b/internal/protocol/hls/config.go new file mode 100644 index 00000000..2c06d5f0 --- /dev/null +++ b/internal/protocol/hls/config.go @@ -0,0 +1,30 @@ +package hls + +const ( + defaultSegmentConnections = 8 + defaultMaxRetries = 5 + defaultTimeoutSeconds = 30 + + // maxPrefetchSizeSegments caps the HEAD probing done at resolve time. + // Playlists larger than this skip size prefetch entirely. + maxPrefetchSizeSegments = 500 +) + +type config struct { + SegmentConnections int `json:"segmentConnections"` + MaxRetries int `json:"maxRetries"` + TimeoutSeconds int `json:"timeoutSeconds"` + PrefetchContentLength bool `json:"prefetchContentLength"` +} + +func (c *config) normalize() { + if c.SegmentConnections <= 0 { + c.SegmentConnections = defaultSegmentConnections + } + if c.MaxRetries < 0 { + c.MaxRetries = defaultMaxRetries + } + if c.TimeoutSeconds <= 0 { + c.TimeoutSeconds = defaultTimeoutSeconds + } +} diff --git a/internal/protocol/hls/crypto.go b/internal/protocol/hls/crypto.go new file mode 100644 index 00000000..58379a4f --- /dev/null +++ b/internal/protocol/hls/crypto.go @@ -0,0 +1,59 @@ +package hls + +import ( + "crypto/aes" + "crypto/cipher" + "fmt" +) + +// decryptAES128 decrypts an AES-128-CBC encrypted segment and strips PKCS7 +// padding. data length must be a multiple of the AES block size. +func decryptAES128(data, key, iv []byte) ([]byte, error) { + if len(key) != 16 { + return nil, fmt.Errorf("invalid AES-128 key size %d", len(key)) + } + if len(iv) != 16 { + return nil, fmt.Errorf("invalid AES-128 IV size %d", len(iv)) + } + if len(data) == 0 || len(data)%aes.BlockSize != 0 { + return nil, fmt.Errorf("invalid AES-128 data size %d", len(data)) + } + block, err := aes.NewCipher(key) + if err != nil { + return nil, err + } + plain := make([]byte, len(data)) + cipher.NewCBCDecrypter(block, iv).CryptBlocks(plain, data) + return pkcs7Unpad(plain, aes.BlockSize) +} + +func pkcs7Unpad(data []byte, blockSize int) ([]byte, error) { + if len(data) == 0 { + return nil, fmt.Errorf("empty plaintext") + } + pad := int(data[len(data)-1]) + if pad == 0 || pad > blockSize || pad > len(data) { + return nil, fmt.Errorf("invalid PKCS7 padding") + } + for _, b := range data[len(data)-pad:] { + if int(b) != pad { + return nil, fmt.Errorf("invalid PKCS7 padding") + } + } + return data[:len(data)-pad], nil +} + +// sequenceIV derives the default per-segment IV mandated by RFC 8216: the +// segment media sequence number as a 16-byte big-endian integer. +func sequenceIV(sequence int64) []byte { + iv := make([]byte, 16) + iv[8] = byte(sequence >> 56) + iv[9] = byte(sequence >> 48) + iv[10] = byte(sequence >> 40) + iv[11] = byte(sequence >> 32) + iv[12] = byte(sequence >> 24) + iv[13] = byte(sequence >> 16) + iv[14] = byte(sequence >> 8) + iv[15] = byte(sequence) + return iv +} diff --git a/internal/protocol/hls/crypto_test.go b/internal/protocol/hls/crypto_test.go new file mode 100644 index 00000000..05f18423 --- /dev/null +++ b/internal/protocol/hls/crypto_test.go @@ -0,0 +1,66 @@ +package hls + +import ( + "bytes" + "crypto/aes" + "crypto/cipher" + "crypto/rand" + "testing" +) + +func pkcs7Pad(data []byte, blockSize int) []byte { + pad := blockSize - len(data)%blockSize + return append(data, bytes.Repeat([]byte{byte(pad)}, pad)...) +} + +func TestDecryptAES128Roundtrip(t *testing.T) { + key := make([]byte, 16) + iv := make([]byte, 16) + if _, err := rand.Read(key); err != nil { + t.Fatal(err) + } + if _, err := rand.Read(iv); err != nil { + t.Fatal(err) + } + plain := []byte("hello hls segment payload, padded to block size!") + padded := pkcs7Pad(plain, aes.BlockSize) + block, err := aes.NewCipher(key) + if err != nil { + t.Fatal(err) + } + encrypted := make([]byte, len(padded)) + cipher.NewCBCEncrypter(block, iv).CryptBlocks(encrypted, padded) + + got, err := decryptAES128(encrypted, key, iv) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(got, plain) { + t.Errorf("roundtrip mismatch: %q", got) + } +} + +func TestDecryptAES128InvalidInput(t *testing.T) { + if _, err := decryptAES128([]byte{1, 2, 3}, make([]byte, 16), make([]byte, 16)); err == nil { + t.Error("non-block-aligned data should fail") + } + if _, err := decryptAES128(make([]byte, 16), make([]byte, 15), make([]byte, 16)); err == nil { + t.Error("short key should fail") + } + if _, err := decryptAES128(make([]byte, 16), make([]byte, 16), make([]byte, 16)); err == nil { + t.Error("all-zero block has invalid padding and should fail") + } +} + +func TestSequenceIV(t *testing.T) { + iv := sequenceIV(1) + if len(iv) != 16 || iv[15] != 1 { + t.Errorf("sequence 1 IV wrong: %v", iv) + } + iv = sequenceIV(0x0102030405060708) + for i, want := range []byte{1, 2, 3, 4, 5, 6, 7, 8} { + if iv[8+i] != want { + t.Errorf("sequence IV byte %d: want %d got %d", i, want, iv[8+i]) + } + } +} diff --git a/internal/protocol/hls/doc.go b/internal/protocol/hls/doc.go new file mode 100644 index 00000000..78699556 --- /dev/null +++ b/internal/protocol/hls/doc.go @@ -0,0 +1,24 @@ +// Package hls implements native HTTP Live Streaming downloads for m3u8 +// playlists. A task produces exactly one merged, playable output file: +// MPEG-2 TS segments are concatenated as-is, fMP4 streams are merged as +// init section followed by the media fragments, and AES-128 encrypted +// segments are decrypted before they are stored. +// +// Supported subset: +// - VOD media playlists (EXT-X-ENDLIST present) and master playlists whose +// selected variant carries its own audio +// - AES-128 with the identity KEYFORMAT, explicit IV or media-sequence +// derived IV, including key rotation mid-playlist +// - EXT-X-BYTERANGE (including offset inheritance) and fMP4 init sections +// via a single, constant EXT-X-MAP +// +// Everything the merge pipeline cannot reproduce faithfully is rejected with +// an explicit error instead of silently producing a broken output file: +// - live streams (no EXT-X-ENDLIST) and SAMPLE-AES / DRM encryption +// - EXT-X-GAP (would leave a hole) and EXT-X-DISCONTINUITY (broken timeline) +// - changing init sections (a repeated EXT-X-MAP is compared by all its +// attributes, not just the URI) and encrypted EXT-X-MAP without an +// explicit IV +// - separate audio renditions referenced by the selected variant (the +// merged output would have no audio) and non-identity KEYFORMATs +package hls diff --git a/internal/protocol/hls/fetcher.go b/internal/protocol/hls/fetcher.go new file mode 100644 index 00000000..7259a9b0 --- /dev/null +++ b/internal/protocol/hls/fetcher.go @@ -0,0 +1,1001 @@ +package hls + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "os" + "path/filepath" + "strconv" + "sync" + "sync/atomic" + "time" + + "github.com/GopeedLab/gopeed/internal/controller" + "github.com/GopeedLab/gopeed/internal/fetcher" + "github.com/GopeedLab/gopeed/internal/httpclient" + "github.com/GopeedLab/gopeed/pkg/base" +) + +// ReqExtra mirrors the HTTP protocol extra: the create dialog attaches HTTP +// style options to m3u8 URLs (same JSON keys), so the same shape is decoded. +type ReqExtra struct { + Method string `json:"method,omitempty"` + Header map[string]string `json:"header,omitempty"` + Body string `json:"body,omitempty"` +} + +func copyReqExtra(extra *ReqExtra) *ReqExtra { + copied := &ReqExtra{} + if extra == nil { + return copied + } + copied.Method = extra.Method + copied.Body = extra.Body + if extra.Header != nil { + copied.Header = make(map[string]string, len(extra.Header)) + for key, value := range extra.Header { + copied.Header[key] = value + } + } + return copied +} + +// fetcherState is the resolved download plan persisted for resume. StagingID +// gives the task its own staging folder (two tasks downloading the same URL +// into the same directory never share segment files) and survives restarts +// through the persisted state. StagingDir pins the absolute folder so user +// renames or moves of the task output do not orphan the staged segments. +type fetcherState struct { + MediaURL string `json:"mediaURL"` + OutputName string `json:"outputName"` + Segments []*Segment `json:"segments"` + StagingID string `json:"stagingID,omitempty"` + StagingDir string `json:"stagingDir,omitempty"` +} + +// Stats is the protocol task statistics snapshot. +type Stats struct { + SegmentsTotal int `json:"segmentsTotal"` + SegmentsDone int `json:"segmentsDone"` + SegmentsFailed int `json:"segmentsFailed"` +} + +// tempDirPrefix separates HLS staging folders from regular task files. +const tempDirPrefix = ".gopeed-hls-" + +// hlsRun holds every piece of state that belongs to one Start..Pause/complete +// cycle. Runs never share mutable state: a Pause that is immediately followed +// by a Start replaces the whole run, so workers of the previous run cannot +// race with the next one. The run also snapshots the plan, request headers, +// client and merge target so concurrent Patch calls cannot change them under +// a running download. +type hlsRun struct { + ctx context.Context + cancel context.CancelFunc + // wg covers the queue producer AND the segment workers: Pause must not + // return while the producer can still hand out segment indexes. + wg sync.WaitGroup + done chan struct{} // closed when the supervisor has fully exited + + segs []*Segment // plan snapshot, read-only for the duration of the run + extra *ReqExtra // header snapshot (deep copy) + client *http.Client + outputPath string // merge target snapshot + tempDir string + + keyCache map[string][]byte + keyCacheMu sync.Mutex + + stagingID string // identity of the staging folder, bound into the journal + planHash string // fingerprint of the download plan, validated on resume + journal map[int64]int64 + journalPath string + journalMu sync.Mutex + + // downloaded counts bytes of the final output that are safely on disk + // (completed segment files), not raw network traffic: discarded retry + // bytes must never show up as progress. + downloaded atomic.Int64 + doneCount atomic.Int64 + failCount atomic.Int64 + baseBytes int64 // stored bytes restored from the journal at startup + baseDone int // completed segments restored from the journal at startup + pending []int64 // segment indexes to download, set by prepareRun +} + +func (r *hlsRun) statsSnapshot() Stats { + return Stats{ + SegmentsTotal: len(r.segs), + SegmentsDone: r.baseDone + int(r.doneCount.Load()), + SegmentsFailed: int(r.failCount.Load()), + } +} + +type Fetcher struct { + fetcher.DefaultFetcher + meta *fetcher.FetcherMeta + config *config + + // mu guards the fields below and every lifecycle transition. Progress and + // Stats are read from the engine's checkpoint goroutine while a run is + // active; they only take mu to snapshot the active run pointer, then read + // run-owned atomics lock-free. + mu sync.Mutex + extra *ReqExtra + client *http.Client + state *fetcherState + activeRun *hlsRun + closed bool + lastStats Stats + lastDownloaded int64 + + impSession *httpclient.ImpersonationSession +} + +func (f *Fetcher) Setup(ctl *controller.Controller) { + // Initializes Ctl and DoneCh, which Wait() depends on. + f.DefaultFetcher.Setup(ctl) + if f.meta == nil { + f.meta = &fetcher.FetcherMeta{} + } + f.impSession = httpclient.NewImpersonationSession() + f.config = &config{} + f.Ctl.GetConfig(f.config) + f.config.normalize() + // Restored fetchers receive their request through Setup (the engine calls + // it after the Restore builder): parse the extra headers so resumed + // segment/key requests keep Referer/Cookie and other anti-hotlink headers. + f.mu.Lock() + f.reloadExtraLocked() + f.mu.Unlock() +} + +// reloadExtraLocked refreshes f.extra from the typed extra of meta.Req. +func (f *Fetcher) reloadExtraLocked() { + if f.meta == nil || f.meta.Req == nil { + return + } + if err := base.ParseReqExtra[ReqExtra](f.meta.Req); err != nil { + return + } + if e, ok := f.meta.Req.Extra.(*ReqExtra); ok { + f.extra = e + } +} + +func (f *Fetcher) Meta() *fetcher.FetcherMeta { + return f.meta +} + +func (f *Fetcher) Resolve(req *base.Request, opts *base.Options) (err error) { + f.mu.Lock() + defer f.mu.Unlock() + f.meta.Req = req + f.meta.Opts = opts + f.extra = &ReqExtra{} + if err = base.ParseReqExtra[ReqExtra](req); err != nil { + return + } + if e, ok := req.Extra.(*ReqExtra); ok { + f.extra = e + } + if f.client == nil { + f.client = f.buildClient() + } + + state, name, err := f.resolvePlanLocked(context.Background(), "") + if err != nil { + return err + } + state.StagingID = randomStagingID() + f.state = state + // Pin the staging folder now so the very first persisted state already + // carries it: a crash right after create still resumes into the same + // folder. + f.resolveStagingDirLocked() + size := planTotalSize(state.Segments) + f.meta.Res = &base.Resource{ + Range: false, + Files: []*base.FileInfo{ + { + Name: name, + Size: size, + }, + }, + Size: size, + } + return nil +} + +// resolvePlanLocked fetches and parses the playlist into a fresh state. It +// must be called with f.mu held while the fetcher is quiescent. When keepName +// is non-empty the output keeps that file name (used when re-resolving an +// existing task so a refreshed URL does not rename the file). +func (f *Fetcher) resolvePlanLocked(ctx context.Context, keepName string) (*fetcherState, string, error) { + content, finalURL, err := f.fetchText(ctx, f.playlistMethod(), f.meta.Req.URL) + if err != nil { + return nil, "", fmt.Errorf("fetch playlist failed: %w", err) + } + baseURL, err := url.Parse(finalURL) + if err != nil { + return nil, "", err + } + + mediaURL := finalURL + if IsMasterPlaylist(content) { + master, err := ParseMasterPlaylist(content, baseURL) + if err != nil { + return nil, "", err + } + best := PickBestVariant(master.Variants) + if err := checkAudioRendition(master, best); err != nil { + return nil, "", err + } + variantContent, variantURL, err := f.fetchText(ctx, http.MethodGet, best.URI) + if err != nil { + return nil, "", fmt.Errorf("fetch variant playlist failed: %w", err) + } + variantBase, err := url.Parse(variantURL) + if err != nil { + return nil, "", err + } + content = variantContent + baseURL = variantBase + mediaURL = variantURL + } + media, err := ParseMedia(content, baseURL) + if err != nil { + return nil, "", err + } + if err := validateSupported(media); err != nil { + return nil, "", err + } + + outputName := keepName + if outputName == "" { + outputName = DeriveOutputName(mediaURL, media) + } + state := &fetcherState{ + MediaURL: mediaURL, + OutputName: outputName, + Segments: media.Segments, + } + if f.config.PrefetchContentLength { + f.prefetchSize(media.Segments) + } + return state, outputName, nil +} + +func (f *Fetcher) Patch(req *base.Request, opts *base.Options) error { + f.mu.Lock() + defer f.mu.Unlock() + if f.activeRun != nil { + return errors.New("task is running; pause it before changing its settings") + } + if req != nil { + cur := f.meta.Req + if cur == nil { + f.meta.Req = req + cur = req + } + if req.URL != "" && req.URL != cur.URL { + cur.URL = req.URL + // The playlist URL changed (e.g. refreshed signed link): drop the + // plan so the engine re-resolves on next start (it does when Res + // is nil). Only explicitly supplied fields are applied, so a + // header-only or rename-only Patch keeps the resolved plan and + // the output path. + f.dropPlanLocked() + } + if err := mergeReqExtra(cur, req); err != nil { + return err + } + if req.Proxy != nil && cur.Proxy != req.Proxy { + cur.Proxy = req.Proxy + // Transport settings changed: rebuild the client before the + // next run. + f.client = nil + } + if len(req.Labels) > 0 { + if cur.Labels == nil { + cur.Labels = make(map[string]string, len(req.Labels)) + } + for key, value := range req.Labels { + cur.Labels[key] = value + } + } + // SkipVerifyCert is a plain bool, so a partial update cannot tell an + // absent field from an explicit false: it is intentionally not merged + // (the HTTP protocol Patch ignores it as well). + } + if opts != nil { + if f.meta.Opts == nil { + f.meta.Opts = opts + } else { + if opts.Name != "" { + f.meta.Opts.Name = opts.Name + } + if opts.Path != "" { + f.meta.Opts.Path = opts.Path + } + } + } + f.reloadExtraLocked() + return nil +} + +// mergeReqExtra merges the typed extra of incoming into cur following the +// HTTP protocol semantics: Method/Body are overridden when non-empty and +// headers are merged per key. +func mergeReqExtra(cur *base.Request, incoming *base.Request) error { + if incoming.Extra == nil { + return nil + } + if err := base.ParseReqExtra[ReqExtra](incoming); err != nil { + return err + } + inc, ok := incoming.Extra.(*ReqExtra) + if !ok { + return nil + } + if err := base.ParseReqExtra[ReqExtra](cur); err != nil { + return err + } + target, ok := cur.Extra.(*ReqExtra) + if !ok { + target = &ReqExtra{} + cur.Extra = target + } + if inc.Method != "" { + target.Method = inc.Method + } + if inc.Body != "" { + target.Body = inc.Body + } + if len(inc.Header) > 0 { + if target.Header == nil { + target.Header = make(map[string]string, len(inc.Header)) + } + for key, value := range inc.Header { + target.Header[key] = value + } + } + return nil +} + +// dropPlanLocked discards the resolved plan after a URL change. The staging +// folder of the dropped plan is removed too: this only runs while the +// fetcher is quiescent, so no run can still be writing into it. +func (f *Fetcher) dropPlanLocked() { + if f.state != nil && f.state.StagingDir != "" { + os.RemoveAll(f.state.StagingDir) + } + f.state = nil + f.meta.Res = nil +} + +// existingOutputNameLocked returns the output file name the task already +// uses, so a re-resolve does not rename an in-progress download. +func (f *Fetcher) existingOutputNameLocked() string { + if f.meta.Opts != nil && f.meta.Opts.Name != "" { + return f.meta.Opts.Name + } + if res := f.meta.Res; res != nil && len(res.Files) > 0 && res.Files[0] != nil { + return res.Files[0].Name + } + return "" +} + +func (f *Fetcher) Start() error { + f.mu.Lock() + if f.closed { + f.mu.Unlock() + return errors.New("hls fetcher is closed") + } + if f.activeRun != nil { + f.mu.Unlock() + return errors.New("hls fetcher already running") + } + if f.state == nil { + if f.meta.Req == nil || f.meta.Res == nil { + f.mu.Unlock() + return errors.New("hls fetcher is not resolved") + } + // Crash window: the engine persists the task (with its resource) + // before the first HLS state checkpoint lands, so a restored task + // can have a resource but no plan. Rebuild the plan instead of + // failing, keeping the existing output name. + if f.client == nil { + f.client = f.buildClient() + } + state, _, err := f.resolvePlanLocked(context.Background(), f.existingOutputNameLocked()) + if err != nil { + f.mu.Unlock() + return fmt.Errorf("re-resolve failed: %w", err) + } + f.state = state + size := planTotalSize(state.Segments) + if res := f.meta.Res; res != nil { + res.Size = size + if len(res.Files) > 0 && res.Files[0] != nil { + res.Files[0].Size = size + } + } + } + // The client is built here, before any worker exists, so segment + // goroutines never race on lazy initialization. + if f.client == nil { + f.client = f.buildClient() + } + if f.meta.Res == nil { + // A restored plan without a resource cannot determine its output + // path; the engine re-resolves when Res is nil. + f.mu.Unlock() + return errors.New("hls fetcher is not resolved") + } + + ctx, cancel := context.WithCancel(context.Background()) + run := &hlsRun{ + ctx: ctx, + cancel: cancel, + done: make(chan struct{}), + segs: f.state.Segments, + extra: copyReqExtra(f.extra), + client: f.client, + outputPath: f.meta.SingleFilepath(), + stagingID: f.state.StagingID, + planHash: planFingerprint(f.state), + tempDir: f.resolveStagingDirLocked(), + keyCache: make(map[string][]byte), + journal: make(map[int64]int64), + } + // Prepare staging and the completed-segment counters BEFORE publishing + // the run: Stats/Progress may observe it immediately afterwards and must + // never read half-initialized plain fields. + if err := f.prepareRun(run); err != nil { + f.mu.Unlock() + run.cancel() + close(run.done) + return err + } + f.activeRun = run + f.mu.Unlock() + + f.launchWorkers(run) + go f.supervise(run) + return nil +} + +// Pause stops all segment downloads synchronously. It waits for the queue +// producer, every worker AND the supervisor (including a merge in flight) +// before returning, so a Store right after Pause sees a consistent state. +func (f *Fetcher) Pause() error { + f.mu.Lock() + run := f.activeRun + f.mu.Unlock() + if run == nil { + return nil + } + run.cancel() + <-run.done + f.clearActiveRun(run) + return nil +} + +// Close pauses any active run and only removes the staging folder once every +// run goroutine has stopped writing to it. +func (f *Fetcher) Close() error { + f.mu.Lock() + f.closed = true + run := f.activeRun + tempDir := "" + if f.state != nil { + tempDir = f.resolveStagingDirLocked() + } + f.mu.Unlock() + + if run != nil { + run.cancel() + <-run.done + f.clearActiveRun(run) + } + if f.impSession != nil { + f.impSession.Clear() + } + if tempDir != "" { + os.RemoveAll(tempDir) + } + return nil +} + +// clearActiveRun detaches a finished run and keeps its counters as the last +// known values for idle Stats/Progress reporting. +func (f *Fetcher) clearActiveRun(run *hlsRun) { + f.mu.Lock() + if f.activeRun == run { + f.activeRun = nil + f.lastStats = run.statsSnapshot() + f.lastDownloaded = run.downloaded.Load() + } + f.mu.Unlock() +} + +// Progress reports final-output bytes stored on disk for the single file. +func (f *Fetcher) Progress() fetcher.Progress { + f.mu.Lock() + run := f.activeRun + if run != nil { + f.mu.Unlock() + return fetcher.Progress{run.downloaded.Load()} + } + last := f.lastDownloaded + f.mu.Unlock() + return fetcher.Progress{last} +} + +func (f *Fetcher) Stats() *fetcher.Stats { + f.mu.Lock() + run := f.activeRun + state := f.state + if run != nil { + f.mu.Unlock() + snap := run.statsSnapshot() + return &fetcher.Stats{Snapshot: &snap} + } + snap := f.lastStats + f.mu.Unlock() + if snap.SegmentsTotal == 0 && state != nil { + snap.SegmentsTotal = len(state.Segments) + } + return &fetcher.Stats{Snapshot: &snap} +} + +// supervise waits for the segment workers, then merges. A canceled context +// exits silently (the engine is pausing), everything else is reported through +// DoneCh exactly once per run. +func (f *Fetcher) supervise(run *hlsRun) { + run.wg.Wait() + defer close(run.done) + + if run.ctx.Err() != nil { + f.clearActiveRun(run) + return + } + var err error + if failed := run.failCount.Load(); failed > 0 { + err = fmt.Errorf("%d segment(s) failed to download", failed) + } else { + err = f.merge(run) + if err == nil { + if run.ctx.Err() != nil { + // Paused while the merge was in flight: keep the staged + // segments and exit silently; the next run re-merges. + f.clearActiveRun(run) + return + } + os.RemoveAll(run.tempDir) + } + } + f.clearActiveRun(run) + select { + case f.DoneCh <- err: + default: + } +} + +// prepareRun prepares the staging folder and loads the completed-segment +// journal into run.baseBytes/baseDone/pending. It performs no goroutine +// launches and no context-bound work, so it runs safely under f.mu before +// the run is published. +func (f *Fetcher) prepareRun(run *hlsRun) error { + if err := os.MkdirAll(run.tempDir, 0777); err != nil { + return err + } + run.journalPath = filepath.Join(run.tempDir, "journal.json") + if err := loadJournal(run); err != nil { + return err + } + for i := range run.segs { + if size, ok := run.journal[int64(i)]; ok { + if info, err := os.Stat(segmentPath(run.tempDir, int64(i))); err == nil && info.Size() == size { + run.baseBytes += size + run.baseDone++ + continue + } + } + run.pending = append(run.pending, int64(i)) + } + run.downloaded.Store(run.baseBytes) + return nil +} + +// launchWorkers starts the queue producer and the segment download workers +// for a published run. Every goroutine registers itself in run.wg so Pause +// waits for all of them. +func (f *Fetcher) launchWorkers(run *hlsRun) { + if len(run.pending) == 0 { + return + } + queue := make(chan int64) + run.wg.Add(1) + go func() { + defer run.wg.Done() + defer close(queue) + for _, idx := range run.pending { + select { + case <-run.ctx.Done(): + return + case queue <- idx: + } + } + }() + + workers := min(f.config.SegmentConnections, len(run.pending)) + for i := 0; i < workers; i++ { + run.wg.Add(1) + go func() { + defer run.wg.Done() + for { + select { + case <-run.ctx.Done(): + return + case idx, ok := <-queue: + if !ok { + return + } + if err := f.downloadSegment(run, idx); err != nil { + if errors.Is(err, context.Canceled) { + return + } + run.failCount.Add(1) + } + } + } + }() + } +} + +func (f *Fetcher) downloadSegment(run *hlsRun, idx int64) error { + seg := run.segs[idx] + data, err := f.fetchSegmentData(run, seg) + if err != nil { + return err + } + if seg.Key != nil { + key, err := f.fetchKey(run, seg.Key) + if err != nil { + return err + } + iv := seg.Key.IV + if iv == nil { + iv = sequenceIV(seg.Sequence) + } + data, err = decryptAES128(data, key, iv) + if err != nil { + return err + } + } + if err := storeSegment(run, idx, data); err != nil { + return err + } + run.doneCount.Add(1) + run.downloaded.Add(int64(len(data))) + return nil +} + +// fetchSegmentData downloads one segment with retries, returning its (still +// encrypted) content. +func (f *Fetcher) fetchSegmentData(run *hlsRun, seg *Segment) ([]byte, error) { + var lastErr error + for attempt := 0; attempt <= f.config.MaxRetries; attempt++ { + if err := run.ctx.Err(); err != nil { + return nil, err + } + if attempt > 0 { + select { + case <-run.ctx.Done(): + return nil, run.ctx.Err() + case <-time.After(time.Duration(attempt) * time.Second): + } + } + data, err := f.tryGetSegment(run, seg) + if err == nil { + return data, nil + } + if errors.Is(err, context.Canceled) { + return nil, err + } + lastErr = err + } + return nil, lastErr +} + +func (f *Fetcher) tryGetSegment(run *hlsRun, seg *Segment) ([]byte, error) { + resp, err := f.do(run.ctx, run.client, run.extra, http.MethodGet, seg.URI, seg.Byterange) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return nil, fmt.Errorf("segment %s: unexpected status %d", seg.URI, resp.StatusCode) + } + reader := &stallReader{rc: resp.Body, timeout: f.timeout()} + var buf bytes.Buffer + chunk := make([]byte, 64<<10) + for { + n, err := reader.Read(chunk) + if n > 0 { + buf.Write(chunk[:n]) + } + if err != nil { + if errors.Is(err, io.EOF) { + break + } + return nil, err + } + } + if seg.Byterange != nil && int64(buf.Len()) != seg.Byterange.Length { + return nil, fmt.Errorf("segment %s: size mismatch, got %d want %d", seg.URI, buf.Len(), seg.Byterange.Length) + } + return buf.Bytes(), nil +} + +func (f *Fetcher) fetchKey(run *hlsRun, key *Key) ([]byte, error) { + run.keyCacheMu.Lock() + if cached, ok := run.keyCache[key.URI]; ok { + run.keyCacheMu.Unlock() + return cached, nil + } + run.keyCacheMu.Unlock() + + var data []byte + var err error + for attempt := 0; attempt <= f.config.MaxRetries; attempt++ { + if err = run.ctx.Err(); err != nil { + return nil, err + } + if attempt > 0 { + select { + case <-run.ctx.Done(): + return nil, run.ctx.Err() + case <-time.After(time.Duration(attempt) * time.Second): + } + } + resp, reqErr := f.do(run.ctx, run.client, run.extra, http.MethodGet, key.URI, nil) + if reqErr != nil { + err = reqErr + continue + } + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + resp.Body.Close() + err = fmt.Errorf("key %s: unexpected status %d", key.URI, resp.StatusCode) + continue + } + data, err = io.ReadAll(io.LimitReader(resp.Body, 1<<20)) + resp.Body.Close() + if err == nil { + break + } + } + if err != nil { + return nil, err + } + if len(data) != 16 { + return nil, fmt.Errorf("key %s: invalid key size %d", key.URI, len(data)) + } + run.keyCacheMu.Lock() + run.keyCache[key.URI] = data + run.keyCacheMu.Unlock() + return data, nil +} + +// merge concatenates the downloaded segments in plan order into the final +// output file. Segments are stored decrypted, so this is pure I/O. The merge +// writes to a unique temporary file next to the target, checks write and +// close errors, and only then replaces the target: a failed merge leaves the +// staged segments intact so it can be retried, and the unique part name keeps +// concurrent tasks that share an output path from clobbering each other. The +// final size is published through Progress: the engine backfills Res.Size +// from Progress().TotalDownloaded() on completion. +func (f *Fetcher) merge(run *hlsRun) error { + target := run.outputPath + if err := os.MkdirAll(filepath.Dir(target), 0777); err != nil { + return err + } + partPath := target + tempDirPrefix + run.stagingID + ".part" + out, err := os.Create(partPath) + if err != nil { + return err + } + var total int64 + for i := range run.segs { + data, err := os.ReadFile(segmentPath(run.tempDir, int64(i))) + if err != nil { + out.Close() + os.Remove(partPath) + return fmt.Errorf("read segment %d failed: %w", i, err) + } + if _, err := out.Write(data); err != nil { + out.Close() + os.Remove(partPath) + return err + } + total += int64(len(data)) + } + if err := out.Close(); err != nil { + os.Remove(partPath) + return err + } + // Remove-then-rename keeps this idempotent: if a previous merge already + // renamed its output before a crash lost the bookkeeping, re-merging + // must still succeed. + if err := os.Remove(target); err != nil && !os.IsNotExist(err) { + os.Remove(partPath) + return err + } + if err := os.Rename(partPath, target); err != nil { + os.Remove(partPath) + return err + } + run.downloaded.Store(total) + return nil +} + +func storeSegment(run *hlsRun, idx int64, data []byte) error { + partPath := segmentPath(run.tempDir, idx) + ".part" + if err := os.WriteFile(partPath, data, 0666); err != nil { + return err + } + if err := os.Rename(partPath, segmentPath(run.tempDir, idx)); err != nil { + return err + } + run.journalMu.Lock() + defer run.journalMu.Unlock() + run.journal[idx] = int64(len(data)) + return saveJournal(run) +} + +func segmentPath(tempDir string, idx int64) string { + return filepath.Join(tempDir, "seg-"+strconv.FormatInt(idx, 10)) +} + +// resolveStagingDirLocked returns the staging folder of the current plan, +// deriving and pinning it on first use so later output renames or moves do +// not orphan the staged segments. It lives next to the task output, which is +// exactly opts.Path for a single-file task. Must be called with f.mu held. +func (f *Fetcher) resolveStagingDirLocked() string { + if f.state.StagingDir == "" { + if f.state.StagingID == "" { + // Legacy state (or a hand-crafted plan): migrate to a fresh task + // owned identity. The old URL-hash folder is deliberately not + // reused: its contents cannot be proven to belong to this plan, + // and sharing it with another same-URL task is exactly the bug + // being fixed. It is left behind as orphaned temp data. + f.state.StagingID = randomStagingID() + } + dir := "" + if f.meta.Opts != nil { + dir = f.meta.Opts.Path + } + f.state.StagingDir = filepath.Join(dir, tempDirPrefix+f.state.StagingID) + } + return f.state.StagingDir +} + +// prefetchSize probes segment Content-Length via HEAD. Any failure resets all +// sizes to zero: a partial size estimate would corrupt progress reporting. +func (f *Fetcher) prefetchSize(segments []*Segment) { + var need []*Segment + for _, seg := range segments { + if seg.Size <= 0 { + need = append(need, seg) + } + } + if len(need) == 0 || len(need) > maxPrefetchSizeSegments { + return + } + + workerCount := min(f.config.SegmentConnections, 8) + var failed atomic.Bool + queue := make(chan *Segment) + var wg sync.WaitGroup + for i := 0; i < workerCount; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for seg := range queue { + ctx, cancel := context.WithTimeout(context.Background(), f.timeout()) + seg.Size = f.headContentLength(ctx, seg.URI) + cancel() + if seg.Size <= 0 { + failed.Store(true) + } + } + }() + } + for _, seg := range need { + queue <- seg + } + close(queue) + wg.Wait() + if failed.Load() { + for _, seg := range segments { + seg.Size = 0 + } + } +} + +func (f *Fetcher) headContentLength(ctx context.Context, u string) int64 { + req, err := buildRequest(ctx, f.extra, http.MethodHead, u, nil, nil) + if err != nil { + return 0 + } + resp, err := f.client.Do(req) + if err != nil { + return 0 + } + defer resp.Body.Close() + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return 0 + } + return resp.ContentLength +} + +func (f *Fetcher) fetchText(ctx context.Context, method, u string) (string, string, error) { + resp, err := f.do(ctx, f.client, f.extra, method, u, nil) + if err != nil { + return "", "", err + } + defer resp.Body.Close() + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return "", "", fmt.Errorf("unexpected status %d for %s", resp.StatusCode, u) + } + data, err := io.ReadAll(io.LimitReader(&stallReader{rc: resp.Body, timeout: f.timeout()}, maxPlaylistSize)) + if err != nil { + return "", "", err + } + return string(data), resp.Request.URL.String(), nil +} + +func (f *Fetcher) do(ctx context.Context, client *http.Client, extra *ReqExtra, method, u string, byterange *ByteRange) (*http.Response, error) { + if method == "" { + method = http.MethodGet + } + req, err := buildRequest(ctx, extra, method, u, nil, byterange) + if err != nil { + return nil, err + } + return client.Do(req) +} + +func (f *Fetcher) playlistMethod() string { + if f.extra != nil && f.extra.Method != "" { + return f.extra.Method + } + return http.MethodGet +} + +// planTotalSize sums the known segment sizes. It returns 0 unless every +// segment is unencrypted with a known size: ciphertext lengths overstate the +// final output (CBC padding is removed on decrypt), and partial estimates +// would make progress stall below 100%. For indeterminate plans the engine +// backfills Res.Size from Progress().TotalDownloaded() when the task +// completes (downloader.go watch). +func planTotalSize(segments []*Segment) int64 { + var total int64 + for _, seg := range segments { + if seg.Key != nil { + return 0 + } + if seg.Size <= 0 { + return 0 + } + total += seg.Size + } + return total +} diff --git a/internal/protocol/hls/fetcher_manager.go b/internal/protocol/hls/fetcher_manager.go new file mode 100644 index 00000000..63941ec2 --- /dev/null +++ b/internal/protocol/hls/fetcher_manager.go @@ -0,0 +1,100 @@ +package hls + +import ( + "net/url" + "path" + "strings" + + "github.com/GopeedLab/gopeed/internal/fetcher" +) + +type FetcherManager struct { +} + +func (fm *FetcherManager) Name() string { + return "hls" +} + +// Filters registers the .m3u8 extension. The manager must be registered BEFORE +// the http manager in the downloader config: protocol routing picks the first +// manager whose filter matches, and the http filter claims every http(s) URL. +func (fm *FetcherManager) Filters() []*fetcher.SchemeFilter { + return []*fetcher.SchemeFilter{ + { + Type: fetcher.FilterTypeFile, + Pattern: "M3U8", + }, + } +} + +func (fm *FetcherManager) Build() fetcher.Fetcher { + return &Fetcher{} +} + +func (fm *FetcherManager) ParseName(u string) string { + parsed, err := url.Parse(u) + if err != nil { + return "" + } + name := path.Base(parsed.Path) + name = strings.TrimSuffix(name, path.Ext(name)) + if name == "" || name == "/" || name == "." { + name = parsed.Hostname() + } + return name +} + +func (fm *FetcherManager) AutoRename() bool { + return true +} + +func (fm *FetcherManager) DefaultConfig() any { + return &config{ + SegmentConnections: defaultSegmentConnections, + MaxRetries: defaultMaxRetries, + TimeoutSeconds: defaultTimeoutSeconds, + PrefetchContentLength: true, + } +} + +func (fm *FetcherManager) Store(f fetcher.Fetcher) (data any, err error) { + _f := f.(*Fetcher) + _f.mu.Lock() + defer _f.mu.Unlock() + if _f.state == nil { + return nil, nil + } + // The engine's periodic checkpoint calls Store without holding the task + // lock and serializes the returned value afterwards, while Resolve/Patch/ + // Start mutate the live state under the fetcher mutex: return a snapshot + // copy taken under that mutex, mirroring the HTTP protocol's Store. + return cloneState(_f.state), nil +} + +func cloneState(state *fetcherState) *fetcherState { + cloned := *state + cloned.Segments = make([]*Segment, len(state.Segments)) + for i, seg := range state.Segments { + s := *seg + cloned.Segments[i] = &s + } + return &cloned +} + +func (fm *FetcherManager) Restore() (v any, f func(meta *fetcher.FetcherMeta, v any) fetcher.Fetcher) { + return &fetcherState{}, func(meta *fetcher.FetcherMeta, v any) fetcher.Fetcher { + fb := &FetcherManager{} + restored := fb.Build().(*Fetcher) + restored.meta = meta + if state, ok := v.(*fetcherState); ok && state.MediaURL != "" { + restored.state = state + } + // Setup is invoked by the engine (restoreFetcher) after this builder + // returns; ReqExtra parsing happens there as well. + return restored + } +} + +func (fm *FetcherManager) Close() error { + return nil +} diff --git a/internal/protocol/hls/fetcher_test.go b/internal/protocol/hls/fetcher_test.go new file mode 100644 index 00000000..3dd785f0 --- /dev/null +++ b/internal/protocol/hls/fetcher_test.go @@ -0,0 +1,857 @@ +package hls + +import ( + "bytes" + "crypto/aes" + "crypto/cipher" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strconv" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/GopeedLab/gopeed/internal/controller" + "github.com/GopeedLab/gopeed/internal/fetcher" + "github.com/GopeedLab/gopeed/pkg/base" +) + +func pkcs7PadForTest(data []byte, blockSize int) []byte { + pad := blockSize - len(data)%blockSize + return append(data, bytes.Repeat([]byte{byte(pad)}, pad)...) +} + +func encryptSegment(plain, key, iv []byte) []byte { + padded := pkcs7PadForTest(plain, aes.BlockSize) + block, _ := aes.NewCipher(key) + encrypted := make([]byte, len(padded)) + cipher.NewCBCEncrypter(block, iv).CryptBlocks(encrypted, padded) + return encrypted +} + +// serveContentWithReferer is a helper handler that enforces an optional +// Referer requirement and reports request counts. +type cdnHandler struct { + mux *http.ServeMux + requireReferer string + requests atomic.Int64 +} + +func (h *cdnHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + h.requests.Add(1) + if h.requireReferer != "" && r.Header.Get("Referer") != h.requireReferer { + w.WriteHeader(http.StatusForbidden) + return + } + h.mux.ServeHTTP(w, r) +} + +func newTestFetcher(t *testing.T, cfg config) *Fetcher { + t.Helper() + fm := &FetcherManager{} + f := fm.Build().(*Fetcher) + ctl := controller.NewController() + ctl.GetConfig = func(v any) { + if c, ok := v.(*config); ok { + *c = cfg + } + } + f.Setup(ctl) + return f +} + +func testConfig() config { + return config{ + SegmentConnections: 4, + MaxRetries: 2, + TimeoutSeconds: 5, + PrefetchContentLength: true, + } +} + +func mustResolve(t *testing.T, f *Fetcher, playlistURL, referer string) { + t.Helper() + req := &base.Request{URL: playlistURL} + if referer != "" { + req.Extra = &ReqExtra{Header: map[string]string{"Referer": referer}} + } + if err := f.Resolve(req, &base.Options{Path: t.TempDir()}); err != nil { + t.Fatalf("resolve: %v", err) + } +} + +func mustRun(t *testing.T, f *Fetcher) { + t.Helper() + if err := f.Start(); err != nil { + t.Fatalf("start: %v", err) + } + if err := f.Wait(); err != nil { + t.Fatalf("wait: %v", err) + } +} + +func statsOf(t *testing.T, f *Fetcher) Stats { + t.Helper() + s, ok := f.Stats().Snapshot.(*Stats) + if !ok { + t.Fatalf("unexpected stats snapshot type %T", f.Stats().Snapshot) + } + return *s +} + +func readOutput(t *testing.T, f *Fetcher) []byte { + t.Helper() + data, err := os.ReadFile(f.meta.SingleFilepath()) + if err != nil { + t.Fatal(err) + } + return data +} + +func TestFetcherE2E_MasterTS(t *testing.T) { + seg := func(i int) []byte { + return []byte(fmt.Sprintf("SEGMENT-%02d-PAYLOAD", i)) + } + mux := http.NewServeMux() + mux.HandleFunc("/master.m3u8", func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte("#EXTM3U\n" + + "#EXT-X-STREAM-INF:BANDWIDTH=800000\nv1/index.m3u8\n" + + "#EXT-X-STREAM-INF:BANDWIDTH=2000000\nv2/index.m3u8\n")) + }) + mux.HandleFunc("/v1/index.m3u8", func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte("#EXTM3U\n#EXTINF:4,\nlow-0.ts\n#EXT-X-ENDLIST\n")) + }) + mux.HandleFunc("/v2/index.m3u8", func(w http.ResponseWriter, r *http.Request) { + var b strings.Builder + b.WriteString("#EXTM3U\n") + for i := 0; i < 5; i++ { + b.WriteString(fmt.Sprintf("#EXTINF:4,\nseg-%d.ts\n", i)) + } + b.WriteString("#EXT-X-ENDLIST\n") + w.Write([]byte(b.String())) + }) + for i := 0; i < 5; i++ { + content := seg(i) + path := "/v2/seg-" + strconv.Itoa(i) + ".ts" + mux.HandleFunc(path, func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Length", strconv.Itoa(len(content))) + w.Write(content) + }) + } + server := httptest.NewServer(&cdnHandler{mux: mux}) + defer server.Close() + + f := newTestFetcher(t, testConfig()) + mustResolve(t, f, server.URL+"/master.m3u8", "") + // Master resolution picked the 2000000 variant. + if f.state.MediaURL != server.URL+"/v2/index.m3u8" { + t.Fatalf("wrong variant picked: %s", f.state.MediaURL) + } + // HEAD prefetch should have computed the exact total size. + if f.meta.Res.Size != int64(5*len(seg(0))) { + t.Fatalf("prefetched size wrong: %d", f.meta.Res.Size) + } + mustRun(t, f) + + var want []byte + for i := 0; i < 5; i++ { + want = append(want, seg(i)...) + } + got := readOutput(t, f) + if !bytes.Equal(got, want) { + t.Errorf("merged output mismatch: %q", got) + } + // The staging folder must be cleaned up after a successful merge; only + // the output file remains in the task directory. + if entries, _ := os.ReadDir(filepath.Dir(f.meta.SingleFilepath())); len(entries) != 1 { + for _, e := range entries { + t.Logf("leftover entry: %s", e.Name()) + } + t.Errorf("staging dir should be cleaned up after merge") + } + if f.meta.Res.Size != int64(len(want)) { + t.Errorf("final size wrong: %d", f.meta.Res.Size) + } +} + +func TestFetcherE2E_RefererEnforced(t *testing.T) { + page := "https://player.example.com/watch/123" + var playlist strings.Builder + playlist.WriteString("#EXTM3U\n") + segments := make(map[string][]byte) + for i := 0; i < 3; i++ { + name := fmt.Sprintf("seg-%d.ts", i) + segments["/"+name] = []byte(fmt.Sprintf("DATA-%d", i)) + playlist.WriteString(fmt.Sprintf("#EXTINF:4,\n%s\n", name)) + } + playlist.WriteString("#EXT-X-ENDLIST\n") + + mux := http.NewServeMux() + mux.HandleFunc("/video.m3u8", func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte(playlist.String())) + }) + for path, content := range segments { + c := content + mux.HandleFunc(path, func(w http.ResponseWriter, r *http.Request) { + w.Write(c) + }) + } + server := httptest.NewServer(&cdnHandler{mux: mux, requireReferer: page}) + defer server.Close() + + // Without Referer the resolve itself must fail (403 on the playlist). + blocked := newTestFetcher(t, testConfig()) + req := &base.Request{URL: server.URL + "/video.m3u8"} + if err := blocked.Resolve(req, &base.Options{Path: t.TempDir()}); err == nil { + t.Fatal("resolve without Referer should fail") + } + + // With the Referer set through the task extra headers it must succeed. + allowed := newTestFetcher(t, testConfig()) + allowedReq := &base.Request{ + URL: server.URL + "/video.m3u8", + Extra: &ReqExtra{Header: map[string]string{"Referer": page}}, + } + if err := allowed.Resolve(allowedReq, &base.Options{Path: t.TempDir()}); err != nil { + t.Fatal(err) + } + mustRun(t, allowed) + want := []byte("DATA-0DATA-1DATA-2") + if !bytes.Equal(readOutput(t, allowed), want) { + t.Error("output mismatch with Referer download") + } +} + +func TestFetcherE2E_AES128KeyRotation(t *testing.T) { + key1 := []byte("0123456789abcdef") + key2 := []byte("fedcba9876543210") + iv1 := []byte("aaaaaaaaaaaaaaaa") + plain1 := []byte("ENCRYPTED-SEGMENT-ONE!!") // padded on encrypt + plain2 := []byte("ENCRYPTED-SEGMENT-TWO!!") + // seg-2 has no explicit IV: derived from media sequence 1. + enc1 := encryptSegment(plain1, key1, iv1) + enc2 := encryptSegment(plain2, key2, sequenceIV(1)) + + mux := http.NewServeMux() + mux.HandleFunc("/video.m3u8", func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte("#EXTM3U\n" + + "#EXT-X-MEDIA-SEQUENCE:0\n" + + "#EXT-X-KEY:METHOD=AES-128,URI=\"key-1.bin\",IV=0x" + hexEncode(iv1) + "\n" + + "#EXTINF:5,\nseg-1.ts\n" + + "#EXT-X-KEY:METHOD=AES-128,URI=\"key-2.bin\"\n" + + "#EXTINF:5,\nseg-2.ts\n" + + "#EXT-X-ENDLIST\n")) + }) + mux.HandleFunc("/key-1.bin", func(w http.ResponseWriter, r *http.Request) { + w.Write(key1) + }) + mux.HandleFunc("/key-2.bin", func(w http.ResponseWriter, r *http.Request) { + w.Write(key2) + }) + mux.HandleFunc("/seg-1.ts", func(w http.ResponseWriter, r *http.Request) { + w.Write(enc1) + }) + mux.HandleFunc("/seg-2.ts", func(w http.ResponseWriter, r *http.Request) { + w.Write(enc2) + }) + server := httptest.NewServer(mux) + defer server.Close() + + f := newTestFetcher(t, testConfig()) + mustResolve(t, f, server.URL+"/video.m3u8", "") + // Ciphertext sizes overstate the final output after decryption, so an + // encrypted plan must report an indeterminate size. + if f.meta.Res.Size != 0 { + t.Errorf("encrypted plan should report size 0, got %d", f.meta.Res.Size) + } + mustRun(t, f) + + want := append(append([]byte{}, plain1...), plain2...) + if !bytes.Equal(readOutput(t, f), want) { + t.Errorf("decrypted output mismatch: %q", readOutput(t, f)) + } +} + +func TestFetcherE2E_ByteRange(t *testing.T) { + data := []byte("0123456789ABCDEFGHIJ") // 20 bytes + mux := http.NewServeMux() + mux.HandleFunc("/video.m3u8", func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte("#EXTM3U\n" + + "#EXT-X-BYTERANGE:10@0\nblob.bin\n" + + "#EXT-X-BYTERANGE:10\nblob.bin\n" + + "#EXT-X-ENDLIST\n")) + }) + mux.HandleFunc("/blob.bin", func(w http.ResponseWriter, r *http.Request) { + rng := r.Header.Get("Range") + if rng == "" { + w.Write(data) + return + } + // bytes=start-end + var start, end int + fmt.Sscanf(rng, "bytes=%d-%d", &start, &end) + w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, len(data))) + w.WriteHeader(http.StatusPartialContent) + w.Write(data[start : end+1]) + }) + server := httptest.NewServer(mux) + defer server.Close() + + f := newTestFetcher(t, testConfig()) + mustResolve(t, f, server.URL+"/video.m3u8", "") + if f.meta.Res.Size != 20 { + t.Fatalf("byterange sizes should be known from the playlist, got %d", f.meta.Res.Size) + } + mustRun(t, f) + if !bytes.Equal(readOutput(t, f), data) { + t.Errorf("byterange merge mismatch: %q", readOutput(t, f)) + } +} + +func TestFetcherE2E_PauseResume(t *testing.T) { + const total = 8 + var served atomic.Int64 + mux := http.NewServeMux() + mux.HandleFunc("/video.m3u8", func(w http.ResponseWriter, r *http.Request) { + var b strings.Builder + b.WriteString("#EXTM3U\n") + for i := 0; i < total; i++ { + b.WriteString(fmt.Sprintf("#EXTINF:4,\nseg-%d.ts\n", i)) + } + b.WriteString("#EXT-X-ENDLIST\n") + w.Write([]byte(b.String())) + }) + segments := make(map[int][]byte) + for i := 0; i < total; i++ { + segments[i] = []byte(fmt.Sprintf("SLOW-SEGMENT-%02d", i)) + } + for i := 0; i < total; i++ { + idx := i + mux.HandleFunc(fmt.Sprintf("/seg-%d.ts", i), func(w http.ResponseWriter, r *http.Request) { + time.Sleep(150 * time.Millisecond) + served.Add(1) + w.Write(segments[idx]) + }) + } + server := httptest.NewServer(mux) + defer server.Close() + + cfg := testConfig() + cfg.SegmentConnections = 2 + cfg.PrefetchContentLength = false + f := newTestFetcher(t, cfg) + mustResolve(t, f, server.URL+"/video.m3u8", "") + + if err := f.Start(); err != nil { + t.Fatal(err) + } + // Let a couple of segments finish, then pause. + time.Sleep(400 * time.Millisecond) + if err := f.Pause(); err != nil { + t.Fatal(err) + } + doneBefore := statsOf(t, f).SegmentsDone + if doneBefore == 0 { + t.Fatal("expected some segments to complete before pause") + } + if doneBefore == total { + t.Fatal("all segments finished before pause, test is not exercising resume") + } + + // Resume must finish the rest and produce the full output. + if err := f.Start(); err != nil { + t.Fatal(err) + } + if err := f.Wait(); err != nil { + t.Fatal(err) + } + var want []byte + for i := 0; i < total; i++ { + want = append(want, segments[i]...) + } + if !bytes.Equal(readOutput(t, f), want) { + t.Errorf("resumed output mismatch: %q", readOutput(t, f)) + } +} + +func TestFetcherE2E_StoreRestoreResume(t *testing.T) { + const total = 6 + var served atomic.Int64 + mux := http.NewServeMux() + mux.HandleFunc("/video.m3u8", func(w http.ResponseWriter, r *http.Request) { + var b strings.Builder + b.WriteString("#EXTM3U\n") + for i := 0; i < total; i++ { + b.WriteString(fmt.Sprintf("#EXTINF:4,\nseg-%d.ts\n", i)) + } + b.WriteString("#EXT-X-ENDLIST\n") + w.Write([]byte(b.String())) + }) + segments := make(map[int][]byte) + for i := 0; i < total; i++ { + segments[i] = []byte(fmt.Sprintf("RESTORE-SEGMENT-%02d", i)) + } + for i := 0; i < total; i++ { + idx := i + mux.HandleFunc(fmt.Sprintf("/seg-%d.ts", i), func(w http.ResponseWriter, r *http.Request) { + time.Sleep(120 * time.Millisecond) + served.Add(1) + w.Write(segments[idx]) + }) + } + server := httptest.NewServer(mux) + defer server.Close() + + cfg := testConfig() + cfg.SegmentConnections = 2 + cfg.PrefetchContentLength = false + + f := newTestFetcher(t, cfg) + mustResolve(t, f, server.URL+"/video.m3u8", "") + if err := f.Start(); err != nil { + t.Fatal(err) + } + time.Sleep(350 * time.Millisecond) + if err := f.Pause(); err != nil { + t.Fatal(err) + } + doneBefore := statsOf(t, f).SegmentsDone + if doneBefore == 0 { + t.Fatal("expected some segments to finish before simulated restart") + } + + // Simulate an app restart: persist, then restore into a fresh fetcher. + fm := &FetcherManager{} + state, err := fm.Store(f) + if err != nil { + t.Fatal(err) + } + if state == nil { + t.Fatal("state should be persisted") + } + ctl := controller.NewController() + ctl.GetConfig = func(v any) { + if c, ok := v.(*config); ok { + *c = cfg + } + } + holder := fm.Build().(*Fetcher) + holder.Setup(ctl) + holder.meta = &fetcher.FetcherMeta{ + Req: f.meta.Req, + Opts: f.meta.Opts, + Res: f.meta.Res, + } + _, builder := fm.Restore() + rf := builder(holder.meta, state).(*Fetcher) + // The engine calls Setup on the builder result (restoreFetcher). + rf.Setup(ctl) + if rf.state == nil || len(rf.state.Segments) != total { + t.Fatal("restored state missing segment plan") + } + + if err := rf.Start(); err != nil { + t.Fatal(err) + } + if err := rf.Wait(); err != nil { + t.Fatal(err) + } + if done := statsOf(t, rf).SegmentsDone; done != total { + t.Errorf("want %d done, got %d", total, done) + } + var want []byte + for i := 0; i < total; i++ { + want = append(want, segments[i]...) + } + if !bytes.Equal(readOutput(t, rf), want) { + t.Errorf("restored output mismatch: %q", readOutput(t, rf)) + } +} + +func hexEncode(b []byte) string { + const digits = "0123456789abcdef" + out := make([]byte, 0, len(b)*2) + for _, v := range b { + out = append(out, digits[v>>4], digits[v&0x0f]) + } + return string(out) +} + +// restoreLikeEngine round-trips the persisted state and the task meta through +// JSON the way the engine storage does, then builds a fresh fetcher. This is +// the path that historically lost the request extra headers. +func restoreLikeEngine(t *testing.T, f *Fetcher, cfg config) *Fetcher { + t.Helper() + fm := &FetcherManager{} + stateAny, err := fm.Store(f) + if err != nil { + t.Fatal(err) + } + rawState, err := json.Marshal(stateAny) + if err != nil { + t.Fatal(err) + } + storedState := &fetcherState{} + if err := json.Unmarshal(rawState, storedState); err != nil { + t.Fatal(err) + } + rawMeta, err := json.Marshal(f.meta) + if err != nil { + t.Fatal(err) + } + restoredMeta := &fetcher.FetcherMeta{} + if err := json.Unmarshal(rawMeta, restoredMeta); err != nil { + t.Fatal(err) + } + if _, ok := restoredMeta.Req.Extra.(*ReqExtra); ok { + t.Fatal("storage round trip should yield a map-shaped extra, like the engine") + } + ctl := controller.NewController() + ctl.GetConfig = func(v any) { + if c, ok := v.(*config); ok { + *c = cfg + } + } + _, builder := fm.Restore() + rf := builder(restoredMeta, storedState).(*Fetcher) + // The engine calls Setup on the builder result (restoreFetcher). + rf.Setup(ctl) + return rf +} + +func TestFetcherE2E_StoreRestoreAuthenticated(t *testing.T) { + const total = 4 + page := "https://player.example.com/watch/42" + var served atomic.Int64 + mux := http.NewServeMux() + var playlist strings.Builder + playlist.WriteString("#EXTM3U\n") + for i := 0; i < total; i++ { + playlist.WriteString(fmt.Sprintf("#EXTINF:4,\nseg-%d.ts\n", i)) + } + playlist.WriteString("#EXT-X-ENDLIST\n") + mux.HandleFunc("/video.m3u8", func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte(playlist.String())) + }) + segments := make(map[int][]byte) + for i := 0; i < total; i++ { + segments[i] = []byte(fmt.Sprintf("AUTH-SEGMENT-%02d", i)) + idx := i + mux.HandleFunc(fmt.Sprintf("/seg-%d.ts", idx), func(w http.ResponseWriter, r *http.Request) { + time.Sleep(80 * time.Millisecond) + served.Add(1) + w.Write(segments[idx]) + }) + } + server := httptest.NewServer(&cdnHandler{mux: mux, requireReferer: page}) + defer server.Close() + + cfg := testConfig() + cfg.SegmentConnections = 2 + cfg.PrefetchContentLength = false + + f := newTestFetcher(t, cfg) + req := &base.Request{ + URL: server.URL + "/video.m3u8", + Extra: &ReqExtra{Header: map[string]string{"Referer": page}}, + } + if err := f.Resolve(req, &base.Options{Path: t.TempDir()}); err != nil { + t.Fatal(err) + } + if err := f.Start(); err != nil { + t.Fatal(err) + } + time.Sleep(150 * time.Millisecond) + if err := f.Pause(); err != nil { + t.Fatal(err) + } + + rf := restoreLikeEngine(t, f, cfg) + if rf.extra == nil || rf.extra.Header["Referer"] != page { + t.Fatalf("restored fetcher lost the request extra headers: %+v", rf.extra) + } + if err := rf.Start(); err != nil { + t.Fatal(err) + } + if err := rf.Wait(); err != nil { + t.Fatalf("resumed download failed (request headers lost?): %v", err) + } + var want []byte + for i := 0; i < total; i++ { + want = append(want, segments[i]...) + } + if !bytes.Equal(readOutput(t, rf), want) { + t.Errorf("restored output mismatch: %q", readOutput(t, rf)) + } +} + +func TestFetcherE2E_PatchPartialUpdate(t *testing.T) { + page := "https://player.example.com/watch/7" + var requiredReferer atomic.Value // string + requiredReferer.Store(page) + var playlist strings.Builder + playlist.WriteString("#EXTM3U\n") + segments := make(map[string][]byte) + for i := 0; i < 3; i++ { + segments[fmt.Sprintf("/s%d.ts", i)] = []byte(fmt.Sprintf("PATCH-%d", i)) + playlist.WriteString(fmt.Sprintf("#EXTINF:4,\ns%d.ts\n", i)) + } + playlist.WriteString("#EXT-X-ENDLIST\n") + mux := http.NewServeMux() + mux.HandleFunc("/video.m3u8", func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte(playlist.String())) + }) + for path, content := range segments { + c := content + mux.HandleFunc(path, func(w http.ResponseWriter, r *http.Request) { + w.Write(c) + }) + } + // The Referer requirement can be flipped mid-test to prove that a patched + // header is actually sent by the next run. + dyn := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if want, _ := requiredReferer.Load().(string); want != "" && r.Header.Get("Referer") != want { + w.WriteHeader(http.StatusForbidden) + return + } + mux.ServeHTTP(w, r) + }) + server := httptest.NewServer(dyn) + defer server.Close() + + f := newTestFetcher(t, testConfig()) + req := &base.Request{ + URL: server.URL + "/video.m3u8", + Extra: &ReqExtra{Header: map[string]string{"Referer": page}}, + } + opts := &base.Options{Path: t.TempDir()} + if err := f.Resolve(req, opts); err != nil { + t.Fatal(err) + } + + // A nil/nil patch must be a no-op, not a panic. + if err := f.Patch(nil, nil); err != nil { + t.Fatalf("nil patch should be accepted: %v", err) + } + + // Header-only patch: the resolved plan and output options are retained + // while the new header takes effect on the next run. + newReferer := "https://player.example.com/watch/9" + if err := f.Patch(&base.Request{ + URL: server.URL + "/video.m3u8", + Extra: &ReqExtra{Header: map[string]string{"Referer": newReferer}}, + }, nil); err != nil { + t.Fatal(err) + } + if f.state == nil || f.meta.Res == nil { + t.Fatal("header-only patch must keep the resolved plan") + } + if f.meta.Opts.Path != opts.Path { + t.Fatalf("header-only patch must keep the output path: %s", f.meta.Opts.Path) + } + if f.extra.Header["Referer"] != newReferer { + t.Fatalf("patched header not applied: %+v", f.extra.Header) + } + requiredReferer.Store(newReferer) + if err := f.Start(); err != nil { + t.Fatal(err) + } + if err := f.Wait(); err != nil { + t.Fatalf("run with patched header failed: %v", err) + } + if got := readOutput(t, f); string(got) != "PATCH-0PATCH-1PATCH-2" { + t.Errorf("output mismatch after header patch: %q", got) + } + + // Options-only patch: renaming the task keeps the resolved plan. + if err := f.Patch(nil, &base.Options{Name: "renamed.ts"}); err != nil { + t.Fatal(err) + } + if f.state == nil { + t.Fatal("options-only patch must keep the resolved plan") + } + if f.meta.Opts.Name != "renamed.ts" || f.meta.Opts.Path != opts.Path { + t.Fatalf("options patch misapplied: %+v", f.meta.Opts) + } + + // URL-only patch: the plan is dropped for a re-resolve but the output + // path and renamed file survive. + if err := f.Patch(&base.Request{URL: server.URL + "/refreshed.m3u8"}, nil); err != nil { + t.Fatal(err) + } + if f.state != nil || f.meta.Res != nil { + t.Fatal("URL patch must drop the resolved plan") + } + if f.meta.Opts.Path != opts.Path || f.meta.Opts.Name != "renamed.ts" { + t.Fatal("URL patch must keep the output options") + } +} + +func TestFetcherE2E_PatchWhileRunningRejected(t *testing.T) { + mux := http.NewServeMux() + var playlist strings.Builder + playlist.WriteString("#EXTM3U\n") + for i := 0; i < 6; i++ { + playlist.WriteString(fmt.Sprintf("#EXTINF:4,\nseg-%d.ts\n", i)) + } + playlist.WriteString("#EXT-X-ENDLIST\n") + mux.HandleFunc("/video.m3u8", func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte(playlist.String())) + }) + for i := 0; i < 6; i++ { + mux.HandleFunc(fmt.Sprintf("/seg-%d.ts", i), func(w http.ResponseWriter, r *http.Request) { + time.Sleep(150 * time.Millisecond) + w.Write([]byte("X")) + }) + } + server := httptest.NewServer(mux) + defer server.Close() + + f := newTestFetcher(t, testConfig()) + if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: t.TempDir()}); err != nil { + t.Fatal(err) + } + if err := f.Start(); err != nil { + t.Fatal(err) + } + if err := f.Patch(&base.Request{URL: server.URL + "/other.m3u8"}, nil); err == nil { + t.Fatal("patching a running task must be rejected") + } + if err := f.Pause(); err != nil { + t.Fatal(err) + } + // After the pause the patch goes through and only then drops the plan. + if err := f.Patch(&base.Request{URL: server.URL + "/other.m3u8"}, nil); err != nil { + t.Fatal(err) + } + if f.state != nil { + t.Fatal("URL patch after pause must drop the plan") + } +} + +func TestFetcherE2E_RetryAfterFailure(t *testing.T) { + const total = 4 + var healthy atomic.Bool + mux := http.NewServeMux() + var playlist strings.Builder + playlist.WriteString("#EXTM3U\n") + for i := 0; i < total; i++ { + playlist.WriteString(fmt.Sprintf("#EXTINF:4,\nseg-%d.ts\n", i)) + } + playlist.WriteString("#EXT-X-ENDLIST\n") + mux.HandleFunc("/video.m3u8", func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte(playlist.String())) + }) + segments := make(map[int][]byte) + for i := 0; i < total; i++ { + segments[i] = []byte(fmt.Sprintf("RETRY-SEGMENT-%02d", i)) + idx := i + mux.HandleFunc(fmt.Sprintf("/seg-%d.ts", idx), func(w http.ResponseWriter, r *http.Request) { + if idx == 1 && !healthy.Load() { + w.WriteHeader(http.StatusInternalServerError) + return + } + time.Sleep(60 * time.Millisecond) + w.Write(segments[idx]) + }) + } + server := httptest.NewServer(mux) + defer server.Close() + + cfg := testConfig() + cfg.MaxRetries = 0 // fail the run on the first bad segment + cfg.PrefetchContentLength = false + f := newTestFetcher(t, cfg) + if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: t.TempDir()}); err != nil { + t.Fatal(err) + } + if err := f.Start(); err != nil { + t.Fatal(err) + } + if err := f.Wait(); err == nil { + t.Fatal("first run should fail while the segment endpoint is down") + } + if got := statsOf(t, f).SegmentsFailed; got == 0 { + t.Fatal("failed segments should be reported in stats") + } + + // Server recovered: a fresh run must succeed and report zero failures. + healthy.Store(true) + if err := f.Start(); err != nil { + t.Fatal(err) + } + if err := f.Wait(); err != nil { + t.Fatalf("retry run failed: %v", err) + } + stats := statsOf(t, f) + if stats.SegmentsFailed != 0 { + t.Errorf("failures must reset per run, got %d", stats.SegmentsFailed) + } + if stats.SegmentsDone != total { + t.Errorf("want %d done, got %d", total, stats.SegmentsDone) + } + var want []byte + for i := 0; i < total; i++ { + want = append(want, segments[i]...) + } + if !bytes.Equal(readOutput(t, f), want) { + t.Errorf("retry output mismatch: %q", readOutput(t, f)) + } +} + +func TestFetcherE2E_StateMissingSelfHeal(t *testing.T) { + const total = 3 + mux := http.NewServeMux() + var playlist strings.Builder + playlist.WriteString("#EXTM3U\n") + for i := 0; i < total; i++ { + playlist.WriteString(fmt.Sprintf("#EXTINF:4,\nseg-%d.ts\n", i)) + } + playlist.WriteString("#EXT-X-ENDLIST\n") + mux.HandleFunc("/video.m3u8", func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte(playlist.String())) + }) + segments := make(map[int][]byte) + for i := 0; i < total; i++ { + segments[i] = []byte(fmt.Sprintf("SELFHEAL-%02d", i)) + idx := i + mux.HandleFunc(fmt.Sprintf("/seg-%d.ts", idx), func(w http.ResponseWriter, r *http.Request) { + w.Write(segments[idx]) + }) + } + server := httptest.NewServer(mux) + defer server.Close() + + f := newTestFetcher(t, testConfig()) + if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: t.TempDir()}); err != nil { + t.Fatal(err) + } + // Simulate the crash window where the engine persisted the task with its + // resource but the HLS plan was never checkpointed. + f.state = nil + outputName := f.meta.Res.Files[0].Name + if err := f.Start(); err != nil { + t.Fatalf("start should self-heal a missing plan: %v", err) + } + if err := f.Wait(); err != nil { + t.Fatalf("self-healed run failed: %v", err) + } + if f.state == nil || len(f.state.Segments) != total { + t.Fatal("self-heal should rebuild the segment plan") + } + if f.meta.Res.Files[0].Name != outputName { + t.Fatalf("self-heal renamed the output: %s", f.meta.Res.Files[0].Name) + } + var want []byte + for i := 0; i < total; i++ { + want = append(want, segments[i]...) + } + if !bytes.Equal(readOutput(t, f), want) { + t.Errorf("self-healed output mismatch: %q", readOutput(t, f)) + } +} diff --git a/internal/protocol/hls/helper.go b/internal/protocol/hls/helper.go new file mode 100644 index 00000000..ba4a4f32 --- /dev/null +++ b/internal/protocol/hls/helper.go @@ -0,0 +1,100 @@ +package hls + +import ( + "context" + "crypto/tls" + "fmt" + "io" + "net" + "net/http" + "net/http/cookiejar" + "time" + + "github.com/GopeedLab/gopeed/internal/httpclient" + "github.com/GopeedLab/gopeed/pkg/base" +) + +const defaultUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36" + +const maxPlaylistSize = 10 << 20 // 10MB, playlists are small text files + +func (f *Fetcher) timeout() time.Duration { + return time.Duration(f.config.TimeoutSeconds) * time.Second +} + +// buildClient builds an HTTP client honoring the task proxy and TLS settings. +// The client has no total timeout on purpose: large segments over slow links +// must not be aborted; body stalls are handled by stallReader instead. It is +// only called while the fetcher is quiescent (Resolve/Start), never from +// inside running worker goroutines. +func (f *Fetcher) buildClient() *http.Client { + var proxy *base.RequestProxy + skipVerify := false + if req := f.meta.Req; req != nil { + proxy = req.Proxy + skipVerify = req.SkipVerifyCert + } + jar, _ := cookiejar.New(nil) + client, err := httpclient.NewClient(httpclient.Options{ + Client: httpclient.ClientOptions{ + Jar: jar, + }, + Transport: httpclient.TransportOptions{ + DialContext: (&net.Dialer{ + Timeout: f.timeout(), + }).DialContext, + Proxy: f.Ctl.GetProxy(proxy), + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: skipVerify, + }, + TLSHandshakeTimeout: f.timeout(), + }, + Impersonation: httpclient.ImpersonationOptions{ + Mode: httpclient.ImpersonationAuto, + Session: f.impSession, + }, + }) + if err != nil { + panic(fmt.Sprintf("build HTTP client: %v", err)) + } + return client +} + +// buildRequest builds a request with task extra headers, the default user +// agent and an optional byte range. extra may be nil. +func buildRequest(ctx context.Context, extra *ReqExtra, method, u string, body io.Reader, byterange *ByteRange) (*http.Request, error) { + req, err := http.NewRequestWithContext(ctx, method, u, body) + if err != nil { + return nil, err + } + if extra != nil { + for key, value := range extra.Header { + req.Header.Set(key, value) + } + } + if req.Header.Get("User-Agent") == "" { + req.Header.Set("User-Agent", defaultUserAgent) + } + if byterange != nil { + req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", byterange.Offset, byterange.Offset+byterange.Length-1)) + } + return req, nil +} + +// stallReader aborts reads that see no data for the configured timeout. +type stallReader struct { + rc io.ReadCloser + timeout time.Duration +} + +func (r *stallReader) Read(p []byte) (int, error) { + timer := time.AfterFunc(r.timeout, func() { + r.rc.Close() + }) + defer timer.Stop() + return r.rc.Read(p) +} + +func (r *stallReader) Close() error { + return r.rc.Close() +} diff --git a/internal/protocol/hls/journal.go b/internal/protocol/hls/journal.go new file mode 100644 index 00000000..9d06a80b --- /dev/null +++ b/internal/protocol/hls/journal.go @@ -0,0 +1,112 @@ +package hls + +import ( + "crypto/rand" + "crypto/sha1" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "os" + "strconv" +) + +// journalFile is the persisted completed-segment ledger. It is bound to the +// exact download plan: a journal whose fingerprint does not match the current +// plan is discarded instead of being applied to the wrong segments. +type journalFile struct { + StagingID string `json:"stagingID"` + PlanHash string `json:"planHash"` + SegmentCount int `json:"segmentCount"` + Sizes map[string]int64 `json:"sizes"` +} + +// randomStagingID returns a random identifier used to give every task its own +// staging folder, stable across app restarts through the persisted state. +func randomStagingID() string { + buf := make([]byte, 6) + if _, err := rand.Read(buf); err != nil { + // crypto/rand failing is catastrophic anyway; fall back to address + // entropy rather than crashing the fetch. + return fmt.Sprintf("%x", sha1.Sum([]byte(fmt.Sprintf("%p", buf))))[:12] + } + return hex.EncodeToString(buf) +} + +// planFingerprint hashes everything that defines the download plan: the media +// URL, the ordered segments with their sequence numbers, byte ranges, keys +// and init markers. Two tasks resolve to different fingerprints even when +// they point at the same playlist URL. +func planFingerprint(state *fetcherState) string { + h := sha1.New() + io.WriteString(h, state.MediaURL+"\n") + io.WriteString(h, strconv.Itoa(len(state.Segments))+"\n") + for _, seg := range state.Segments { + fmt.Fprintf(h, "%d|%s|", seg.Sequence, seg.URI) + if seg.Byterange != nil { + fmt.Fprintf(h, "%d@%d|", seg.Byterange.Offset, seg.Byterange.Length) + } + if seg.Key != nil { + // The IV is part of the plan: same URI with a rotated IV would + // silently decrypt to garbage if the journal were reused. + fmt.Fprintf(h, "key=%s iv=%x|", seg.Key.URI, seg.Key.IV) + } + if seg.IsInit { + io.WriteString(h, "init|") + } + io.WriteString(h, "\n") + } + return hex.EncodeToString(h.Sum(nil))[:16] +} + +// loadJournal reads the journal of a run, discarding anything that cannot be +// proven to belong to the current plan: foreign or legacy journals, corrupted +// files, and entries whose recorded size no longer matches the plan. +func loadJournal(run *hlsRun) error { + data, err := os.ReadFile(run.journalPath) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return err + } + var file journalFile + if err := json.Unmarshal(data, &file); err != nil { + // A broken or legacy journal must not kill a restartable download. + return nil + } + if file.StagingID != run.stagingID || file.PlanHash != run.planHash || file.SegmentCount != len(run.segs) { + return nil + } + for key, size := range file.Sizes { + idx, err := strconv.ParseInt(key, 10, 64) + if err != nil || idx < 0 || idx >= int64(len(run.segs)) || size <= 0 { + continue + } + run.journal[idx] = size + } + return nil +} + +// saveJournal atomically rewrites the journal through a temp file + rename so +// a crash never leaves a half-written ledger behind. +func saveJournal(run *hlsRun) error { + file := journalFile{ + StagingID: run.stagingID, + PlanHash: run.planHash, + SegmentCount: len(run.segs), + Sizes: make(map[string]int64, len(run.journal)), + } + for idx, size := range run.journal { + file.Sizes[strconv.FormatInt(idx, 10)] = size + } + data, err := json.Marshal(file) + if err != nil { + return err + } + tmp := run.journalPath + ".tmp" + if err := os.WriteFile(tmp, data, 0666); err != nil { + return err + } + return os.Rename(tmp, run.journalPath) +} diff --git a/internal/protocol/hls/media_validate_test.go b/internal/protocol/hls/media_validate_test.go new file mode 100644 index 00000000..276d2fd0 --- /dev/null +++ b/internal/protocol/hls/media_validate_test.go @@ -0,0 +1,297 @@ +package hls + +import ( + "bytes" + "fmt" + "net/http" + "net/http/httptest" + "os" + "testing" + + "github.com/GopeedLab/gopeed/pkg/base" +) + +// The validators below check real media structure, not just byte +// concatenation: TS packets are verified on the 188 byte grid with +// continuity-counter rules, PAT/PMT presence and PES start codes; fMP4 +// output is verified by walking the ISO-BMFF box hierarchy. + +func be32(b []byte) int { + return int(b[0])<<24 | int(b[1])<<16 | int(b[2])<<8 | int(b[3]) +} + +// validateTS asserts structural validity of a merged MPEG-2 TS stream. +func validateTS(t *testing.T, data []byte) { + t.Helper() + if len(data) == 0 || len(data)%188 != 0 { + t.Fatalf("TS size %d is not a positive multiple of 188", len(data)) + } + type pidState struct{ cc int } + ccByPID := map[int]int{} + patSeen, pmtSeen, pesSeen := false, false, false + for off := 0; off < len(data); off += 188 { + pkt := data[off : off+188] + if pkt[0] != 0x47 { + t.Fatalf("bad TS sync byte at offset %d", off) + } + pid := (int(pkt[1]&0x1F) << 8) | int(pkt[2]) + pusi := pkt[1]&0x40 != 0 + afc := (pkt[3] >> 4) & 0x3 + if afc == 0 { + continue // reserved + } + idx := 4 + if afc&0x2 != 0 { // adaptation field present + afLen := int(pkt[4]) + if afLen > 0 && pkt[5]&0x80 != 0 { + // discontinuity_indicator: restart the CC expectation + delete(ccByPID, pid) + } + idx += 1 + afLen + if afc == 2 { + continue // adaptation only, no payload: no CC rule + } + } + if idx > 188 { + t.Fatalf("adaptation field overruns packet at offset %d", off) + } + if pid == 0x1FFF { + continue // null packet + } + payload := pkt[idx:] + if pusi && len(payload) >= 3 && pid != 0 && payload[0] == 0 && payload[1] == 0 && payload[2] == 1 { + pesSeen = true + } + if pid == 0 { + if ptr := int(payload[0]); 1+ptr < len(payload) && payload[1+ptr] == 0x00 { + patSeen = true // program_association_section table_id + } + } else if pusi && len(payload) >= 2 && payload[0] == 0 && payload[1] == 0x02 { + pmtSeen = true // program_map_section table_id + } + cc := int(pkt[3] & 0x0F) + prev, ok := ccByPID[pid] + if !ok { + ccByPID[pid] = cc + continue + } + // CC increments by one per payload-bearing packet; a repeated value + // models the single legal duplicate packet. + if cc != (prev+1)%16 && cc != prev { + t.Fatalf("continuity error on PID %d: %d after %d at offset %d", pid, cc, prev, off) + } + ccByPID[pid] = cc + } + if !patSeen { + t.Error("no PAT found in merged TS") + } + if !pmtSeen { + t.Error("no PMT found in merged TS") + } + if !pesSeen { + t.Error("no PES start code found in merged TS") + } +} + +// walkBoxes iterates the top-level ISO-BMFF boxes of data[off:end]. +func walkBoxes(t *testing.T, data []byte, off, end int) []string { + t.Helper() + var types []string + for off+8 <= end { + size := be32(data[off:]) + typ := string(data[off+4 : off+8]) + if size < 8 || off+size > end { + t.Fatalf("invalid box %q at offset %d (size %d)", typ, off, size) + } + types = append(types, typ) + off += size + } + return types +} + +// validateFMP4 asserts the merged fMP4 output is init section followed by +// well-formed moof+mdat fragments. +func validateFMP4(t *testing.T, merged []byte, wantFragments int) { + t.Helper() + types := walkBoxes(t, merged, 0, len(merged)) + hasFtyp, hasMoov := false, false + fragments := 0 + expectFragment := false + for i, typ := range types { + switch typ { + case "ftyp": + if i != 0 { + t.Errorf("ftyp not at stream start") + } + hasFtyp = true + case "moov": + if i != 1 { + t.Errorf("moov must follow ftyp, found at index %d", i) + } + hasMoov = true + case "styp": + expectFragment = true + case "moof": + if !hasMoov { + t.Errorf("fragment before moov") + } + expectFragment = true + case "mdat": + if !expectFragment { + t.Errorf("mdat outside of a fragment at index %d", i) + } + expectFragment = false + fragments++ + case "mfra": + // optional trailer + default: + t.Errorf("unexpected top-level box %q", typ) + } + } + if !hasFtyp || !hasMoov { + t.Error("merged fMP4 missing ftyp/moov init section") + } + if fragments != wantFragments { + t.Errorf("want %d fragments in merged fMP4, got %d", wantFragments, fragments) + } +} + +func mediaFixture(t *testing.T, name string) []byte { + t.Helper() + data, err := os.ReadFile("testdata/media/" + name) + if err != nil { + t.Fatal(err) + } + return data +} + +func serveFixtureServer(t *testing.T) *httptest.Server { + t.Helper() + mux := http.NewServeMux() + fs := http.FileServer(http.Dir("testdata/media")) + mux.Handle("/", fs) + return httptest.NewServer(mux) +} + +func TestHLSRealMedia_TS(t *testing.T) { + seg0 := mediaFixture(t, "sample-00.ts") + seg1 := mediaFixture(t, "sample-01.ts") + playlist := "#EXTM3U\n" + + "#EXTINF:0.5,\nsample-00.ts\n" + + "#EXTINF:0.5,\nsample-01.ts\n" + + "#EXT-X-ENDLIST\n" + mux := http.NewServeMux() + mux.HandleFunc("/video.m3u8", func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte(playlist)) + }) + mux.Handle("/", http.FileServer(http.Dir("testdata/media"))) + server := httptest.NewServer(mux) + defer server.Close() + + f := newTestFetcher(t, testConfig()) + if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: t.TempDir()}); err != nil { + t.Fatal(err) + } + if err := f.Start(); err != nil { + t.Fatal(err) + } + if err := f.Wait(); err != nil { + t.Fatal(err) + } + got := readOutput(t, f) + if !bytes.Equal(got, append(append([]byte{}, seg0...), seg1...)) { + t.Fatal("merged TS is not the exact concatenation of the segments") + } + validateTS(t, got) +} + +func TestHLSRealMedia_FMP4(t *testing.T) { + init := mediaFixture(t, "fmp4-init.mp4") + frag1 := mediaFixture(t, "fmp4-seg1.m4s") + frag2 := mediaFixture(t, "fmp4-seg2.m4s") + frag3 := mediaFixture(t, "fmp4-seg3.m4s") + playlist := "#EXTM3U\n" + + "#EXT-X-MAP:URI=\"fmp4-init.mp4\"\n" + + "#EXTINF:0.5,\nfmp4-seg1.m4s\n" + + "#EXTINF:0.5,\nfmp4-seg2.m4s\n" + + "#EXTINF:0.5,\nfmp4-seg3.m4s\n" + + "#EXT-X-ENDLIST\n" + mux := http.NewServeMux() + mux.HandleFunc("/video.m3u8", func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte(playlist)) + }) + mux.Handle("/", http.FileServer(http.Dir("testdata/media"))) + server := httptest.NewServer(mux) + defer server.Close() + + f := newTestFetcher(t, testConfig()) + if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: t.TempDir()}); err != nil { + t.Fatal(err) + } + if err := f.Start(); err != nil { + t.Fatal(err) + } + if err := f.Wait(); err != nil { + t.Fatal(err) + } + got := readOutput(t, f) + want := append(append(append(append([]byte{}, init...), frag1...), frag2...), frag3...) + if !bytes.Equal(got, want) { + t.Fatal("merged fMP4 is not the exact concatenation of init + fragments") + } + validateFMP4(t, got, 3) +} + +func TestHLSRealMedia_AES128(t *testing.T) { + keys := [][]byte{[]byte("key-A-0123456789"), []byte("key-B-9876543210")} + iv := []byte("iv-1337-01234567") + plain := [][]byte{mediaFixture(t, "sample-00.ts"), mediaFixture(t, "sample-01.ts")} + + playlist := "#EXTM3U\n" + + "#EXT-X-MEDIA-SEQUENCE:0\n" + + "#EXT-X-KEY:METHOD=AES-128,URI=\"key-0.bin\",IV=0x" + hexEncode(iv) + "\n" + + "#EXTINF:0.5,\nenc-0.ts\n" + + "#EXT-X-KEY:METHOD=AES-128,URI=\"key-1.bin\"\n" + + "#EXTINF:0.5,\nenc-1.ts\n" + + "#EXT-X-ENDLIST\n" + mux := http.NewServeMux() + mux.HandleFunc("/video.m3u8", func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte(playlist)) + }) + for i, key := range keys { + k := key + mux.HandleFunc(fmt.Sprintf("/key-%d.bin", i), func(w http.ResponseWriter, r *http.Request) { + w.Write(k) + }) + // Segment 0 has an explicit IV, segment 1 derives it from its media + // sequence number (1). + enc := encryptSegment(plain[i], k, map[bool][]byte{true: iv, false: sequenceIV(int64(i))}[i == 0]) + mux.HandleFunc(fmt.Sprintf("/enc-%d.ts", i), func(w http.ResponseWriter, r *http.Request) { + w.Write(enc) + }) + } + server := httptest.NewServer(mux) + defer server.Close() + + f := newTestFetcher(t, testConfig()) + if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: t.TempDir()}); err != nil { + t.Fatal(err) + } + if f.meta.Res.Size != 0 { + t.Errorf("encrypted plan must report indeterminate size, got %d", f.meta.Res.Size) + } + if err := f.Start(); err != nil { + t.Fatal(err) + } + if err := f.Wait(); err != nil { + t.Fatal(err) + } + got := readOutput(t, f) + if !bytes.Equal(got, append(append([]byte{}, plain[0]...), plain[1]...)) { + t.Fatal("decrypted output mismatch") + } + validateTS(t, got) + if p := f.Progress(); p.TotalDownloaded() != int64(len(got)) { + t.Errorf("final progress %d != merged size %d", p.TotalDownloaded(), len(got)) + } +} diff --git a/internal/protocol/hls/playlist.go b/internal/protocol/hls/playlist.go new file mode 100644 index 00000000..6dd1eeb2 --- /dev/null +++ b/internal/protocol/hls/playlist.go @@ -0,0 +1,479 @@ +package hls + +import ( + "encoding/hex" + "errors" + "fmt" + "net/url" + "path" + "strconv" + "strings" +) + +// Variant is one rendition entry of a master playlist. +type Variant struct { + URI string + Bandwidth int64 + AvgBandwidth int64 + Resolution string + Codecs string + Name string + AudioGroup string // AUDIO attribute: group id of the audio renditions +} + +// MediaRendition is one EXT-X-MEDIA entry of a master playlist. Only the +// fields needed to detect unsupported track layouts are kept. +type MediaRendition struct { + Type string + GroupID string + URI string +} + +// MasterPlaylist is a parsed master playlist with its rendition groups. +type MasterPlaylist struct { + Variants []*Variant + Rendition map[string][]*MediaRendition // group id -> all EXT-X-MEDIA members +} + +// Key is the decryption key applied to the segments that follow its tag, until +// the next EXT-X-KEY tag or the end of the playlist. +type Key struct { + Method string + URI string // absolute URI + IV []byte // explicit IV; nil means derive from the media sequence number +} + +// ByteRange is an EXT-X-BYTERANGE sub range of the segment resource. +type ByteRange struct { + Length int64 + Offset int64 +} + +// Segment is one downloadable piece of a media playlist. The init section of +// an fMP4 stream is represented as a segment with IsInit set. +type Segment struct { + URI string + Sequence int64 // media sequence number, used to derive the default AES IV + Byterange *ByteRange + Key *Key // nil means unencrypted + IsInit bool + Gap bool + Duration float64 + Size int64 // exact size in bytes when known (byterange / HEAD probe), 0 when unknown +} + +// Media is a parsed media playlist. +type Media struct { + Segments []*Segment + Version int + Live bool // no EXT-X-ENDLIST tag + IsFMP4 bool // playlist declares an init section via EXT-X-MAP + Discontinuity bool // playlist contains EXT-X-DISCONTINUITY +} + +const ( + keyMethodNone = "NONE" + keyMethodAES128 = "AES-128" + keyMethodSampleAES = "SAMPLE-AES" +) + +// IsMasterPlaylist reports whether the content looks like a master playlist. +func IsMasterPlaylist(content string) bool { + return strings.Contains(content, "#EXT-X-STREAM-INF") +} + +func validateM3U8Header(content string) error { + for _, line := range strings.Split(content, "\n") { + line = strings.TrimSpace(line) + if line == "" { + continue + } + if line != "#EXTM3U" { + return errors.New("content is not a valid m3u8 playlist") + } + return nil + } + return errors.New("empty m3u8 playlist") +} + +// ParseMaster parses a master playlist and returns all variants. +func ParseMaster(content string, baseURL *url.URL) ([]*Variant, error) { + master, err := ParseMasterPlaylist(content, baseURL) + if err != nil { + return nil, err + } + return master.Variants, nil +} + +// ParseMasterPlaylist parses a master playlist including its EXT-X-MEDIA +// rendition groups. +func ParseMasterPlaylist(content string, baseURL *url.URL) (*MasterPlaylist, error) { + if err := validateM3U8Header(content); err != nil { + return nil, err + } + master := &MasterPlaylist{Rendition: make(map[string][]*MediaRendition)} + var pending *Variant + for _, line := range strings.Split(content, "\n") { + line = strings.TrimSpace(line) + if line == "" { + continue + } + if strings.HasPrefix(line, "#EXT-X-STREAM-INF:") { + attrs := parseAttributes(line[len("#EXT-X-STREAM-INF:"):]) + pending = &Variant{ + Bandwidth: parseInt64(attrs["BANDWIDTH"]), + AvgBandwidth: parseInt64(attrs["AVERAGE-BANDWIDTH"]), + Resolution: attrs["RESOLUTION"], + Codecs: unquote(attrs["CODECS"]), + Name: unquote(attrs["NAME"]), + AudioGroup: unquote(attrs["AUDIO"]), + } + continue + } + if strings.HasPrefix(line, "#EXT-X-MEDIA:") { + attrs := parseAttributes(line[len("#EXT-X-MEDIA:"):]) + rendition := &MediaRendition{ + Type: strings.ToUpper(attrs["TYPE"]), + GroupID: unquote(attrs["GROUP-ID"]), + } + // Only an explicit URI means a separate track; resolving an empty + // reference would fabricate one out of the playlist URL. + if uri := unquote(attrs["URI"]); uri != "" { + rendition.URI = resolveURL(baseURL, uri) + } + if rendition.GroupID != "" { + // Keep every member: a group is unsupported as a whole when + // ANY of its members carries its own URI. + master.Rendition[rendition.GroupID] = append(master.Rendition[rendition.GroupID], rendition) + } + continue + } + if pending != nil { + if line[0] != '#' { + pending.URI = resolveURL(baseURL, line) + master.Variants = append(master.Variants, pending) + } + pending = nil + } + } + if len(master.Variants) == 0 { + return nil, errors.New("master playlist has no variants") + } + return master, nil +} + +// checkAudioRendition rejects the selected variant when its AUDIO group +// carries a separate audio track: downloading only the video variant would +// silently produce a silent file. Groups the variant does not reference are +// ignored (variants without an AUDIO attribute are assumed self-contained). +func checkAudioRendition(master *MasterPlaylist, best *Variant) error { + if best == nil || best.AudioGroup == "" { + return nil + } + for _, rendition := range master.Rendition[best.AudioGroup] { + if rendition == nil || rendition.Type != "AUDIO" || rendition.URI == "" { + continue + } + return fmt.Errorf( + "selected variant uses a separate audio rendition (group %q), which is not supported: the merged output would have no audio", + best.AudioGroup) + } + return nil +} + +// PickBestVariant selects the highest-bandwidth variant. +func PickBestVariant(variants []*Variant) *Variant { + var best *Variant + for _, v := range variants { + if best == nil || v.Bandwidth > best.Bandwidth { + best = v + } + } + return best +} + +// ParseMedia parses a media playlist into an ordered segment plan. +func ParseMedia(content string, baseURL *url.URL) (*Media, error) { + if err := validateM3U8Header(content); err != nil { + return nil, err + } + + media := &Media{Live: true} + var currentKey *Key + var pendingByterange *ByteRange + var pendingGap bool + var pendingDuration float64 + var lastRangeEnd int64 + var sequence int64 + var mapIdentity string // normalized identity of the current EXT-X-MAP + + for _, line := range strings.Split(content, "\n") { + line = strings.TrimSpace(line) + if line == "" { + continue + } + if line[0] != '#' { + // Every non-comment line in a media playlist is a segment URI. + seg := &Segment{ + URI: resolveURL(baseURL, line), + Sequence: sequence, + Byterange: pendingByterange, + Key: currentKey, + Gap: pendingGap, + Duration: pendingDuration, + } + if pendingByterange != nil { + // Byterange segments have exact sizes straight from the playlist. + seg.Size = pendingByterange.Length + } + sequence++ + pendingByterange = nil + pendingGap = false + pendingDuration = 0 + media.Segments = append(media.Segments, seg) + continue + } + + switch { + case strings.HasPrefix(line, "#EXTINF:"): + value := line[len("#EXTINF:"):] + if idx := strings.Index(value, ","); idx >= 0 { + value = value[:idx] + } + pendingDuration = parseFloat(value) + case strings.HasPrefix(line, "#EXT-X-KEY:"): + key, err := parseKey(line[len("#EXT-X-KEY:"):], baseURL) + if err != nil { + return nil, err + } + currentKey = key + case strings.HasPrefix(line, "#EXT-X-MAP:"): + attrs := parseAttributes(line[len("#EXT-X-MAP:"):]) + uriAttr := unquote(attrs["URI"]) + if uriAttr == "" { + return nil, errors.New("EXT-X-MAP without URI is not supported") + } + resolved := resolveURL(baseURL, uriAttr) + // The init section identity covers every attribute, not just the + // URI: a repeated tag with a different BYTERANGE addresses + // different bytes and changes the init section. NUL keeps the + // two parts unambiguously separated. + identity := resolved + "\x00" + attrs["BYTERANGE"] + if mapIdentity != "" { + if mapIdentity != identity { + return nil, errors.New("changing init sections are not supported") + } + media.IsFMP4 = true + continue + } + mapIdentity = identity + init := &Segment{ + URI: resolved, + Sequence: sequence, + IsInit: true, + Key: currentKey, + } + if attrs["BYTERANGE"] != "" { + init.Byterange = parseByteRange(attrs["BYTERANGE"], 0) + if init.Byterange == nil { + return nil, fmt.Errorf("invalid EXT-X-MAP BYTERANGE: %s", attrs["BYTERANGE"]) + } + } + if init.Key != nil && init.Key.IV == nil { + // RFC 8216 only defines sequence-derived IVs for media + // segments; guessing one for the init section produces + // corrupt output, so require the playlist to be explicit. + return nil, errors.New("encrypted EXT-X-MAP without an explicit IV is not supported") + } + // The init section is prepended to the segment plan, it takes + // the media sequence number of the playlist for IV derivation. + media.Segments = append(media.Segments, init) + media.IsFMP4 = true + case strings.HasPrefix(line, "#EXT-X-BYTERANGE:"): + pendingByterange = parseByteRange(line[len("#EXT-X-BYTERANGE:"):], lastRangeEnd) + if pendingByterange == nil { + return nil, fmt.Errorf("invalid EXT-X-BYTERANGE: %s", line) + } + case line == "#EXT-X-GAP": + pendingGap = true + case line == "#EXT-X-DISCONTINUITY": + media.Discontinuity = true + case line == "#EXT-X-ENDLIST": + media.Live = false + case strings.HasPrefix(line, "#EXT-X-MEDIA-SEQUENCE:"): + sequence = parseInt64(line[len("#EXT-X-MEDIA-SEQUENCE:"):]) + case strings.HasPrefix(line, "#EXT-X-VERSION:"): + media.Version = int(parseInt64(line[len("#EXT-X-VERSION:"):])) + } + + if pendingByterange != nil { + // Non-segment tags between BYTERANGE and its URI must not reset the + // running offset, so it is tracked here after each parse. + lastRangeEnd = pendingByterange.Offset + pendingByterange.Length + } + } + + if len(media.Segments) == 0 { + return nil, errors.New("media playlist has no segments") + } + return media, nil +} + +// validateSupported rejects playlist features the byte-concatenation merge +// cannot reproduce faithfully. Skipping such features silently would report +// success while producing a broken file. +func validateSupported(media *Media) error { + if media.Live { + return errors.New("live streams are not supported yet") + } + if media.Discontinuity { + return errors.New("EXT-X-DISCONTINUITY is not supported: the merged output would have a broken timeline") + } + for _, seg := range media.Segments { + if seg.Gap { + return errors.New("EXT-X-GAP is not supported: the missing segment would leave a hole in the merged output") + } + } + return nil +} + +func parseKey(attrs string, baseURL *url.URL) (*Key, error) { + parsed := parseAttributes(attrs) + switch method := parsed["METHOD"]; method { + case keyMethodNone: + return nil, nil + case keyMethodAES128: + uri := unquote(parsed["URI"]) + if uri == "" { + return nil, errors.New("AES-128 EXT-X-KEY without URI") + } + if format := unquote(parsed["KEYFORMAT"]); format != "" && format != "identity" { + return nil, fmt.Errorf("unsupported EXT-X-KEY KEYFORMAT: %s", format) + } + key := &Key{Method: keyMethodAES128, URI: resolveURL(baseURL, uri)} + if iv, ok := parsed["IV"]; ok { + value := strings.TrimPrefix(strings.ToLower(iv), "0x") + decoded, err := hex.DecodeString(value) + if err != nil || len(decoded) != 16 { + return nil, fmt.Errorf("invalid EXT-X-KEY IV: %s", iv) + } + key.IV = decoded + } + return key, nil + case keyMethodSampleAES: + return nil, errors.New("SAMPLE-AES encryption is not supported") + default: + return nil, fmt.Errorf("unsupported encryption method: %s", method) + } +} + +func parseByteRange(value string, defaultOffset int64) *ByteRange { + lengthPart := value + offsetPart := "" + if idx := strings.Index(value, "@"); idx >= 0 { + lengthPart = value[:idx] + offsetPart = value[idx+1:] + } + length := parseInt64(lengthPart) + if length <= 0 { + return nil + } + offset := defaultOffset + if offsetPart != "" { + offset = parseInt64(offsetPart) + if offset < 0 { + return nil + } + } + return &ByteRange{Length: length, Offset: offset} +} + +// resolveURL resolves a playlist URI reference against the playlist URL +// following RFC 3986. +func resolveURL(baseURL *url.URL, ref string) string { + refURL, err := url.Parse(ref) + if err != nil { + return ref + } + return baseURL.ResolveReference(refURL).String() +} + +// parseAttributes parses an HLS attribute list like FOO="bar",NUM=42. +func parseAttributes(s string) map[string]string { + attrs := make(map[string]string) + var key, val strings.Builder + inKey := true + inQuote := false + flush := func() { + name := strings.ToUpper(strings.TrimSpace(key.String())) + if name != "" { + attrs[name] = strings.TrimSpace(val.String()) + } + key.Reset() + val.Reset() + inKey = true + } + for i := 0; i < len(s); i++ { + c := s[i] + if inKey { + if c == '=' { + inKey = false + } else { + key.WriteByte(c) + } + continue + } + if c == '"' { + inQuote = !inQuote + continue + } + if c == ',' && !inQuote { + flush() + continue + } + val.WriteByte(c) + } + flush() + return attrs +} + +func unquote(s string) string { + s = strings.TrimSpace(s) + if len(s) >= 2 && strings.HasPrefix(s, "\"") && strings.HasSuffix(s, "\"") { + return s[1 : len(s)-1] + } + return s +} + +func parseInt64(s string) int64 { + value, _ := strconv.ParseInt(strings.TrimSpace(s), 10, 64) + return value +} + +func parseFloat(s string) float64 { + value, _ := strconv.ParseFloat(strings.TrimSpace(s), 64) + return value +} + +// OutputExtension returns the container file extension for the playlist. +func (m *Media) OutputExtension() string { + if m.IsFMP4 { + return ".mp4" + } + return ".ts" +} + +// DeriveOutputName builds the merged output file name from a playlist URL. +func DeriveOutputName(playlistURL string, media *Media) string { + name := "" + if u, err := url.Parse(playlistURL); err == nil { + name = path.Base(u.Path) + name = strings.TrimSuffix(name, path.Ext(name)) + } + if name == "" || name == "/" || name == "." { + if u, err := url.Parse(playlistURL); err == nil { + name = u.Hostname() + } + } + return name + media.OutputExtension() +} diff --git a/internal/protocol/hls/playlist_realworld_test.go b/internal/protocol/hls/playlist_realworld_test.go new file mode 100644 index 00000000..6a9de3c8 --- /dev/null +++ b/internal/protocol/hls/playlist_realworld_test.go @@ -0,0 +1,39 @@ +package hls + +import ( + _ "embed" + "net/url" + "testing" +) + +//go:embed testdata/olemovienews.m3u8 +var realWorldPlaylist string + +// TestParseMediaRealWorld guards parsing against a real site playlist +// (olemovienews.com VOD, 138 TS segments). +func TestParseMediaRealWorld(t *testing.T) { + base, err := url.Parse("https://europe.olemovienews.com/ts2/20230125/afAxfjit/mp4/afAxfjit.mp4/index-v1-a1.m3u8") + if err != nil { + t.Fatal(err) + } + media, err := ParseMedia(realWorldPlaylist, base) + if err != nil { + t.Fatal(err) + } + if media.Live { + t.Error("real VOD playlist should not be live") + } + if media.IsFMP4 { + t.Error("playlist should be detected as TS") + } + if len(media.Segments) != 138 { + t.Errorf("want 138 segments, got %d", len(media.Segments)) + } + first := media.Segments[0] + if first.URI != "https://europe.olemovienews.com/ts2/20230125/afAxfjit/mp4/afAxfjit.mp4/seg-1-v1-a1.ts" { + t.Errorf("segment URI resolution wrong: %s", first.URI) + } + if media.Segments[0].Sequence != 1 { + t.Errorf("first sequence = %d, want 1", media.Segments[0].Sequence) + } +} diff --git a/internal/protocol/hls/playlist_test.go b/internal/protocol/hls/playlist_test.go new file mode 100644 index 00000000..fb219aa4 --- /dev/null +++ b/internal/protocol/hls/playlist_test.go @@ -0,0 +1,252 @@ +package hls + +import ( + "net/url" + "testing" +) + +func mustURL(t *testing.T, raw string) *url.URL { + t.Helper() + u, err := url.Parse(raw) + if err != nil { + t.Fatal(err) + } + return u +} + +func TestParseMediaTS(t *testing.T) { + content := "#EXTM3U\n" + + "#EXT-X-TARGETDURATION:20\n" + + "#EXT-X-VERSION:3\n" + + "#EXT-X-MEDIA-SEQUENCE:1\n" + + "#EXTINF:20.000,\n" + + "seg-1.ts\n" + + "#EXTINF:20.000,\n" + + "seg-2.ts\n" + + "#EXT-X-ENDLIST\n" + media, err := ParseMedia(content, mustURL(t, "https://cdn.example.com/vod/index.m3u8")) + if err != nil { + t.Fatal(err) + } + if media.Live { + t.Error("VOD playlist should not be live") + } + if media.IsFMP4 { + t.Error("TS playlist should not be fMP4") + } + if len(media.Segments) != 2 { + t.Fatalf("want 2 segments, got %d", len(media.Segments)) + } + want := "https://cdn.example.com/vod/seg-1.ts" + if media.Segments[0].URI != want { + t.Errorf("want uri %s, got %s", want, media.Segments[0].URI) + } + if media.Segments[0].Sequence != 1 { + t.Errorf("want sequence 1, got %d", media.Segments[0].Sequence) + } + if media.Segments[0].Key != nil { + t.Error("segment should be unencrypted") + } +} + +func TestParseMediaWithoutExtinf(t *testing.T) { + // Some playlists list segment URIs without EXTINF lines. + content := "#EXTM3U\nseg-1.ts\nseg-2.ts\n#EXT-X-ENDLIST\n" + media, err := ParseMedia(content, mustURL(t, "https://cdn.example.com/a/index.m3u8")) + if err != nil { + t.Fatal(err) + } + if len(media.Segments) != 2 { + t.Fatalf("want 2 segments, got %d", len(media.Segments)) + } +} + +func TestParseMediaLive(t *testing.T) { + content := "#EXTM3U\n#EXT-X-TARGETDURATION:6\n#EXTINF:6.0,\nseg-1.ts\n" + media, err := ParseMedia(content, mustURL(t, "https://cdn.example.com/live/index.m3u8")) + if err != nil { + t.Fatal(err) + } + if !media.Live { + t.Error("playlist without ENDLIST should be live") + } +} + +func TestParseMediaFMP4(t *testing.T) { + content := "#EXTM3U\n" + + "#EXT-X-VERSION:6\n" + + "#EXT-X-MAP:URI=\"init.mp4\"\n" + + "#EXTINF:5.0,\n" + + "seg-1.m4s\n" + + "#EXT-X-ENDLIST\n" + media, err := ParseMedia(content, mustURL(t, "https://cdn.example.com/vod/index.m3u8")) + if err != nil { + t.Fatal(err) + } + if !media.IsFMP4 { + t.Error("playlist with MAP should be fMP4") + } + if len(media.Segments) != 2 { + t.Fatalf("want 2 segments (init + 1), got %d", len(media.Segments)) + } + if !media.Segments[0].IsInit { + t.Error("first segment should be the init section") + } + if media.Segments[0].URI != "https://cdn.example.com/vod/init.mp4" { + t.Errorf("init uri wrong: %s", media.Segments[0].URI) + } + if media.OutputExtension() != ".mp4" { + t.Errorf("want .mp4, got %s", media.OutputExtension()) + } +} + +func TestParseMediaMultiMapError(t *testing.T) { + content := "#EXTM3U\n" + + "#EXT-X-MAP:URI=\"init1.mp4\"\n" + + "#EXTINF:5.0,\nseg-1.m4s\n" + + "#EXT-X-DISCONTINUITY\n" + + "#EXT-X-MAP:URI=\"init2.mp4\"\n" + + "#EXTINF:5.0,\nseg-2.m4s\n" + + "#EXT-X-ENDLIST\n" + if _, err := ParseMedia(content, mustURL(t, "https://cdn.example.com/vod/index.m3u8")); err == nil { + t.Error("multi-period fMP4 should be rejected") + } +} + +func TestParseMediaKeyRotation(t *testing.T) { + content := "#EXTM3U\n" + + "#EXT-X-KEY:METHOD=AES-128,URI=\"key1.bin\",IV=0x00000000000000000000000000000001\n" + + "#EXTINF:5.0,\nseg-1.ts\n" + + "#EXT-X-KEY:METHOD=AES-128,URI=\"key2.bin\"\n" + + "#EXTINF:5.0,\nseg-2.ts\n" + + "#EXT-X-KEY:METHOD=NONE\n" + + "#EXTINF:5.0,\nseg-3.ts\n" + + "#EXT-X-ENDLIST\n" + media, err := ParseMedia(content, mustURL(t, "https://cdn.example.com/vod/index.m3u8")) + if err != nil { + t.Fatal(err) + } + if media.Segments[0].Key == nil || media.Segments[0].Key.URI != "https://cdn.example.com/vod/key1.bin" { + t.Fatalf("segment 1 key wrong: %+v", media.Segments[0].Key) + } + if len(media.Segments[0].Key.IV) != 16 || media.Segments[0].Key.IV[15] != 1 { + t.Error("explicit IV not parsed") + } + if media.Segments[1].Key == nil || media.Segments[1].Key.URI != "https://cdn.example.com/vod/key2.bin" { + t.Fatalf("segment 2 key wrong: %+v", media.Segments[1].Key) + } + if media.Segments[1].Key.IV != nil { + t.Error("key without IV should leave IV nil (sequence-derived)") + } + if media.Segments[2].Key != nil { + t.Error("segment 3 should be unencrypted after METHOD=NONE") + } +} + +func TestParseMediaSampleAESError(t *testing.T) { + content := "#EXTM3U\n#EXT-X-KEY:METHOD=SAMPLE-AES,URI=\"skd://key\"\n#EXTINF:5.0,\nseg-1.ts\n" + if _, err := ParseMedia(content, mustURL(t, "https://cdn.example.com/vod/index.m3u8")); err == nil { + t.Error("SAMPLE-AES should be rejected") + } +} + +func TestParseMediaByterange(t *testing.T) { + content := "#EXTM3U\n" + + "#EXT-X-BYTERANGE:1000@0\nseg.ts\n" + + "#EXT-X-BYTERANGE:2000\nseg.ts\n" + + "#EXT-X-BYTERANGE:3000\nseg.ts\n" + + "#EXT-X-ENDLIST\n" + media, err := ParseMedia(content, mustURL(t, "https://cdn.example.com/vod/index.m3u8")) + if err != nil { + t.Fatal(err) + } + ranges := [][2]int64{{0, 1000}, {1000, 2000}, {3000, 3000}} + for i, want := range ranges { + br := media.Segments[i].Byterange + if br == nil { + t.Fatalf("segment %d byterange missing", i) + } + if br.Offset != want[0] || br.Length != want[1] { + t.Errorf("segment %d: want offset %d length %d, got %d/%d", i, want[0], want[1], br.Offset, br.Length) + } + } +} + +func TestParseMediaGap(t *testing.T) { + content := "#EXTM3U\n#EXTINF:5.0,\nseg-1.ts\n#EXT-X-GAP\n#EXTINF:5.0,\nseg-2.ts\n#EXT-X-ENDLIST\n" + media, err := ParseMedia(content, mustURL(t, "https://cdn.example.com/vod/index.m3u8")) + if err != nil { + t.Fatal(err) + } + if media.Segments[0].Gap || !media.Segments[1].Gap { + t.Error("gap flags wrong") + } +} + +func TestParseMediaInvalidHeader(t *testing.T) { + if _, err := ParseMedia("not a playlist", mustURL(t, "https://cdn.example.com/a.m3u8")); err == nil { + t.Error("invalid content should be rejected") + } +} + +func TestParseMasterPickBest(t *testing.T) { + content := "#EXTM3U\n" + + "#EXT-X-MEDIA:TYPE=AUDIO,GROUP-ID=\"aud\",NAME=\"en\",DEFAULT=YES,URI=\"audio/index.m3u8\"\n" + + "#EXT-X-STREAM-INF:BANDWIDTH=800000,RESOLUTION=640x360\n" + + "360p/index.m3u8\n" + + "#EXT-X-STREAM-INF:BANDWIDTH=2500000,RESOLUTION=1920x1080,CODECS=\"avc1.640028,mp4a.40.2\"\n" + + "1080p/index.m3u8?sign=abc\n" + + "#EXT-X-ENDLIST\n" + variants, err := ParseMaster(content, mustURL(t, "https://cdn.example.com/vod/master.m3u8")) + if err != nil { + t.Fatal(err) + } + if len(variants) != 2 { + t.Fatalf("want 2 variants, got %d", len(variants)) + } + best := PickBestVariant(variants) + if best.Bandwidth != 2500000 { + t.Errorf("want highest bandwidth variant, got %d", best.Bandwidth) + } + if best.URI != "https://cdn.example.com/vod/1080p/index.m3u8?sign=abc" { + t.Errorf("variant uri wrong: %s", best.URI) + } +} + +func TestResolveURLQueryNotInherited(t *testing.T) { + base := mustURL(t, "https://cdn.example.com/vod/index.m3u8?sign=abc") + got := resolveURL(base, "seg-1.ts") + if got != "https://cdn.example.com/vod/seg-1.ts" { + t.Errorf("relative resolution should follow RFC3986 (no query inheritance), got %s", got) + } +} + +func TestDeriveOutputName(t *testing.T) { + media := &Media{} + if got := DeriveOutputName("https://cdn.example.com/ts2/a/index-v1-a1.m3u8", media); got != "index-v1-a1.ts" { + t.Errorf("want index-v1-a1.ts, got %s", got) + } + fmp4 := &Media{IsFMP4: true} + if got := DeriveOutputName("https://cdn.example.com/ts2/a/index.m3u8", fmp4); got != "index.mp4" { + t.Errorf("want index.mp4, got %s", got) + } + if got := DeriveOutputName("https://cdn.example.com/play?token=x", media); got != "play.ts" { + t.Errorf("want play.ts, got %s", got) + } + if got := DeriveOutputName("https://cdn.example.com/", media); got != "cdn.example.com.ts" { + t.Errorf("want host fallback, got %s", got) + } +} + +func TestParseAttributesQuotedComma(t *testing.T) { + attrs := parseAttributes(`URI="a,b.ts",IV=0xABCDEF00000000000000000000000001,NAME="x"`) + if attrs["URI"] != "a,b.ts" { + t.Errorf("quoted comma value wrong: %q", attrs["URI"]) + } + if attrs["IV"] != "0xABCDEF00000000000000000000000001" { + t.Errorf("hex value wrong: %q", attrs["IV"]) + } + if attrs["NAME"] != "x" { + t.Errorf("quoted value wrong: %q", attrs["NAME"]) + } +} diff --git a/internal/protocol/hls/staging_test.go b/internal/protocol/hls/staging_test.go new file mode 100644 index 00000000..ec25acbf --- /dev/null +++ b/internal/protocol/hls/staging_test.go @@ -0,0 +1,296 @@ +package hls + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/GopeedLab/gopeed/pkg/base" +) + +func TestFetcherE2E_SameURLTwoTasks(t *testing.T) { + const total = 6 + mux := http.NewServeMux() + var playlist strings.Builder + playlist.WriteString("#EXTM3U\n") + for i := 0; i < total; i++ { + playlist.WriteString(fmt.Sprintf("#EXTINF:4,\nseg-%d.ts\n", i)) + } + playlist.WriteString("#EXT-X-ENDLIST\n") + mux.HandleFunc("/video.m3u8", func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte(playlist.String())) + }) + segments := make(map[int][]byte) + for i := 0; i < total; i++ { + segments[i] = []byte(fmt.Sprintf("SHARED-SEGMENT-%02d", i)) + idx := i + mux.HandleFunc(fmt.Sprintf("/seg-%d.ts", idx), func(w http.ResponseWriter, r *http.Request) { + time.Sleep(120 * time.Millisecond) + w.Write(segments[idx]) + }) + } + server := httptest.NewServer(mux) + defer server.Close() + + cfg := testConfig() + cfg.SegmentConnections = 2 + cfg.PrefetchContentLength = false + dir := t.TempDir() + + // Two tasks for the same playlist URL into the same directory, like a + // user duplicating a task. The engine would rename the second output + // (AutoRename); mirror that here. + f1 := newTestFetcher(t, cfg) + if err := f1.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: dir}); err != nil { + t.Fatal(err) + } + f2 := newTestFetcher(t, cfg) + if err := f2.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: dir, Name: "video-2.ts"}); err != nil { + t.Fatal(err) + } + if f1.state.StagingDir == "" || f1.state.StagingDir == f2.state.StagingDir { + t.Fatalf("staging dirs must be distinct per task: %q vs %q", f1.state.StagingDir, f2.state.StagingDir) + } + + // Start task 1, let it stage some segments, then pause it. + if err := f1.Start(); err != nil { + t.Fatal(err) + } + time.Sleep(250 * time.Millisecond) + if err := f1.Pause(); err != nil { + t.Fatal(err) + } + staged1, err := os.ReadDir(f1.state.StagingDir) + if err != nil || len(staged1) == 0 { + t.Fatalf("task 1 should have staged segments, dir: %s", f1.state.StagingDir) + } + + // Task 2 must complete without touching task 1's staging data... + if err := f2.Start(); err != nil { + t.Fatal(err) + } + if err := f2.Wait(); err != nil { + t.Fatal(err) + } + stillThere, err := os.ReadDir(f1.state.StagingDir) + if err != nil || len(stillThere) != len(staged1) { + t.Fatalf("task 2 disturbed task 1 staging: had %d entries, now %d", len(staged1), len(stillThere)) + } + + // ...and task 1 resumes to a complete, correct output of its own. + if err := f1.Start(); err != nil { + t.Fatal(err) + } + if err := f1.Wait(); err != nil { + t.Fatal(err) + } + var want []byte + for i := 0; i < total; i++ { + want = append(want, segments[i]...) + } + if !bytes.Equal(readOutput(t, f1), want) || !bytes.Equal(readOutput(t, f2), want) { + t.Error("same-URL task outputs mismatch") + } + entries, _ := os.ReadDir(dir) + if len(entries) != 2 { + names := make([]string, 0, len(entries)) + for _, e := range entries { + names = append(names, e.Name()) + } + t.Errorf("expected only the two outputs to remain, got %v", names) + } +} + +func TestFetcherE2E_ForeignJournalDiscarded(t *testing.T) { + const total = 3 + var served atomic.Int64 + mux := http.NewServeMux() + var playlist strings.Builder + playlist.WriteString("#EXTM3U\n") + for i := 0; i < total; i++ { + playlist.WriteString(fmt.Sprintf("#EXTINF:4,\nseg-%d.ts\n", i)) + } + playlist.WriteString("#EXT-X-ENDLIST\n") + mux.HandleFunc("/video.m3u8", func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte(playlist.String())) + }) + for i := 0; i < total; i++ { + mux.HandleFunc(fmt.Sprintf("/seg-%d.ts", i), func(w http.ResponseWriter, r *http.Request) { + served.Add(1) + w.Write([]byte("DATA")) + }) + } + server := httptest.NewServer(mux) + defer server.Close() + + cfg := testConfig() + cfg.PrefetchContentLength = false + f := newTestFetcher(t, cfg) + if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: t.TempDir()}); err != nil { + t.Fatal(err) + } + // Plant a journal that does not belong to this plan (wrong fingerprint). + tempDir := f.resolveStagingDirLocked() + if err := os.MkdirAll(tempDir, 0777); err != nil { + t.Fatal(err) + } + foreign := journalFile{ + StagingID: "some-other-task", + PlanHash: "deadbeef", + SegmentCount: total, + Sizes: map[string]int64{"0": 4, "1": 4, "2": 4}, + } + data, _ := json.Marshal(foreign) + if err := os.WriteFile(filepath.Join(tempDir, "journal.json"), data, 0666); err != nil { + t.Fatal(err) + } + + if err := f.Start(); err != nil { + t.Fatal(err) + } + if err := f.Wait(); err != nil { + t.Fatal(err) + } + if got := served.Load(); got != total { + t.Errorf("foreign journal must be discarded, want %d downloads, got %d", total, got) + } +} + +func TestFetcherE2E_MergeFailurePreservesStaging(t *testing.T) { + mux := http.NewServeMux() + mux.HandleFunc("/video.m3u8", func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte("#EXTM3U\n#EXTINF:4,\na.ts\n#EXTINF:4,\nb.ts\n#EXT-X-ENDLIST\n")) + }) + mux.HandleFunc("/a.ts", func(w http.ResponseWriter, r *http.Request) { w.Write([]byte("AAAA")) }) + mux.HandleFunc("/b.ts", func(w http.ResponseWriter, r *http.Request) { w.Write([]byte("BBBB")) }) + server := httptest.NewServer(mux) + defer server.Close() + + f := newTestFetcher(t, testConfig()) + if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: t.TempDir()}); err != nil { + t.Fatal(err) + } + run := &hlsRun{ + ctx: context.Background(), + cancel: func() {}, + done: make(chan struct{}), + segs: f.state.Segments, + outputPath: f.meta.SingleFilepath(), + stagingID: f.state.StagingID, + planHash: planFingerprint(f.state), + tempDir: f.resolveStagingDirLocked(), + keyCache: make(map[string][]byte), + journal: make(map[int64]int64), + } + run.journalPath = filepath.Join(run.tempDir, "journal.json") + if err := os.MkdirAll(run.tempDir, 0777); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(segmentPath(run.tempDir, 0), []byte("AAAA"), 0666); err != nil { + t.Fatal(err) + } + // Segment 1 is missing on disk: the merge must fail without deleting the + // staged data or leaving a partial output behind. + if err := f.merge(run); err == nil { + t.Fatal("merge with a missing segment should fail") + } + if _, err := os.Stat(segmentPath(run.tempDir, 0)); err != nil { + t.Fatal("merge failure must preserve staged segments") + } + entries, _ := os.ReadDir(filepath.Dir(run.outputPath)) + for _, e := range entries { + if strings.HasSuffix(e.Name(), ".part") { + t.Errorf("failed merge left a partial output behind: %s", e.Name()) + } + } + + // Recover the segment: the retried merge succeeds. + if err := os.WriteFile(segmentPath(run.tempDir, 1), []byte("BBBB"), 0666); err != nil { + t.Fatal(err) + } + if err := f.merge(run); err != nil { + t.Fatalf("retried merge failed: %v", err) + } + got, err := os.ReadFile(run.outputPath) + if err != nil { + t.Fatal(err) + } + if string(got) != "AAAABBBB" { + t.Errorf("retried merge output mismatch: %q", got) + } +} + +func TestFetcherE2E_ReMergeOverExistingOutput(t *testing.T) { + const total = 2 + mux := http.NewServeMux() + var playlist strings.Builder + playlist.WriteString("#EXTM3U\n") + for i := 0; i < total; i++ { + playlist.WriteString(fmt.Sprintf("#EXTINF:4,\nseg-%d.ts\n", i)) + } + playlist.WriteString("#EXT-X-ENDLIST\n") + mux.HandleFunc("/video.m3u8", func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte(playlist.String())) + }) + for i := 0; i < total; i++ { + mux.HandleFunc(fmt.Sprintf("/seg-%d.ts", i), func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte("DATA")) + }) + } + server := httptest.NewServer(mux) + defer server.Close() + + f := newTestFetcher(t, testConfig()) + if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: t.TempDir()}); err != nil { + t.Fatal(err) + } + tempDir := f.resolveStagingDirLocked() + if err := os.MkdirAll(tempDir, 0777); err != nil { + t.Fatal(err) + } + // Simulate the crash window after a completed rename but before the task + // was marked done: a stale output file, a complete journal and all + // segments staged. + stale := filepath.Join(filepath.Dir(f.meta.SingleFilepath()), f.state.OutputName) + if err := os.WriteFile(stale, []byte("STALE-OUTPUT"), 0666); err != nil { + t.Fatal(err) + } + jr := &hlsRun{ + segs: f.state.Segments, + stagingID: f.state.StagingID, + planHash: planFingerprint(f.state), + journalPath: filepath.Join(tempDir, "journal.json"), + journal: map[int64]int64{0: 4, 1: 4}, + } + if err := saveJournal(jr); err != nil { + t.Fatal(err) + } + for i := 0; i < total; i++ { + if err := os.WriteFile(segmentPath(tempDir, int64(i)), []byte("DATA"), 0666); err != nil { + t.Fatal(err) + } + } + + if err := f.Start(); err != nil { + t.Fatal(err) + } + if err := f.Wait(); err != nil { + t.Fatalf("re-merge over an existing output failed: %v", err) + } + got, err := os.ReadFile(stale) + if err != nil { + t.Fatal(err) + } + if string(got) != "DATADATA" { + t.Errorf("stale output was not replaced: %q", got) + } +} diff --git a/internal/protocol/hls/subset_test.go b/internal/protocol/hls/subset_test.go new file mode 100644 index 00000000..18ef4328 --- /dev/null +++ b/internal/protocol/hls/subset_test.go @@ -0,0 +1,146 @@ +package hls + +import ( + "net/url" + "strings" + "testing" +) + +func mustParseURL(t *testing.T, raw string) *url.URL { + t.Helper() + u, err := url.Parse(raw) + if err != nil { + t.Fatal(err) + } + return u +} + +// subsetCase is a media playlist plus the error substring the pipeline must +// reject it with; wantOK playlists must parse AND validate cleanly. +type subsetCase struct { + name string + playlist string + wantErr string +} + +func TestSubsetRejections(t *testing.T) { + cases := []subsetCase{ + { + name: "gap", + playlist: "#EXTM3U\n#EXTINF:4,\na.ts\n#EXT-X-GAP\n#EXTINF:4,\nb.ts\n#EXT-X-ENDLIST\n", + wantErr: "EXT-X-GAP", + }, + { + name: "gap-before-first-segment", + playlist: "#EXTM3U\n#EXT-X-GAP\n#EXTINF:4,\na.ts\n#EXT-X-ENDLIST\n", + wantErr: "EXT-X-GAP", + }, + { + name: "discontinuity", + playlist: "#EXTM3U\n#EXTINF:4,\na.ts\n#EXT-X-DISCONTINUITY\n#EXTINF:4,\nb.ts\n#EXT-X-ENDLIST\n", + wantErr: "EXT-X-DISCONTINUITY", + }, + { + name: "live", + playlist: "#EXTM3U\n#EXTINF:4,\na.ts\n", + wantErr: "live", + }, + { + name: "keyformat", + playlist: "#EXTM3U\n#EXT-X-KEY:METHOD=AES-128,URI=\"k\",KEYFORMAT=\"com.apple.streamingkeydelivery\"\n#EXTINF:4,\na.ts\n#EXT-X-ENDLIST\n", + wantErr: "KEYFORMAT", + }, + { + name: "encrypted-map-without-iv", + playlist: "#EXTM3U\n#EXT-X-KEY:METHOD=AES-128,URI=\"k\"\n#EXT-X-MAP:URI=\"init.mp4\"\n#EXTINF:4,\na.m4s\n#EXT-X-ENDLIST\n", + wantErr: "explicit IV", + }, + { + name: "map-change-byterange", + playlist: "#EXTM3U\n#EXT-X-MAP:URI=\"init.mp4\",BYTERANGE=\"100@0\"\n#EXTINF:4,\na.m4s\n#EXT-X-MAP:URI=\"init.mp4\",BYTERANGE=\"100@100\"\n#EXTINF:4,\nb.m4s\n#EXT-X-ENDLIST\n", + wantErr: "changing init sections", + }, + { + name: "map-change-uri", + playlist: "#EXTM3U\n#EXT-X-MAP:URI=\"init1.mp4\"\n#EXTINF:4,\na.m4s\n#EXT-X-MAP:URI=\"init2.mp4\"\n#EXTINF:4,\nb.m4s\n#EXT-X-ENDLIST\n", + wantErr: "changing init sections", + }, + } + baseURL := mustParseURL(t, "https://cdn.example.com/vod/index.m3u8") + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + media, parseErr := ParseMedia(tc.playlist, baseURL) + if parseErr != nil { + if !strings.Contains(parseErr.Error(), tc.wantErr) { + t.Fatalf("parse error %q does not mention %q", parseErr, tc.wantErr) + } + return + } + err := validateSupported(media) + if err == nil { + t.Fatalf("playlist should be rejected with %q", tc.wantErr) + } + if !strings.Contains(err.Error(), tc.wantErr) { + t.Fatalf("error %q does not mention %q", err, tc.wantErr) + } + }) + } +} + +func TestSubsetRepeatedSameMapAllowed(t *testing.T) { + playlist := "#EXTM3U\n" + + "#EXT-X-MAP:URI=\"init.mp4\",BYTERANGE=\"100@0\"\n" + + "#EXTINF:4,\na.m4s\n" + + "#EXT-X-MAP:URI=\"init.mp4\",BYTERANGE=\"100@0\"\n" + + "#EXTINF:4,\nb.m4s\n" + + "#EXT-X-ENDLIST\n" + media, err := ParseMedia(playlist, mustParseURL(t, "https://cdn.example.com/vod/index.m3u8")) + if err != nil { + t.Fatal(err) + } + if err := validateSupported(media); err != nil { + t.Fatalf("identical repeated MAP must be allowed: %v", err) + } + if len(media.Segments) != 3 { // init + 2 segments, no duplicate init + t.Fatalf("want 3 segments (init + 2), got %d", len(media.Segments)) + } +} + +func TestMasterAudioRenditionDetection(t *testing.T) { + masterURL := mustParseURL(t, "https://cdn.example.com/master.m3u8") + + separateAudio := "#EXTM3U\n" + + "#EXT-X-MEDIA:TYPE=AUDIO,GROUP-ID=\"aud\",NAME=\"en\",DEFAULT=YES,URI=\"audio/index.m3u8\"\n" + + "#EXT-X-STREAM-INF:BANDWIDTH=1000000,AUDIO=\"aud\"\nvideo/index.m3u8\n" + master, err := ParseMasterPlaylist(separateAudio, masterURL) + if err != nil { + t.Fatal(err) + } + if err := checkAudioRendition(master, PickBestVariant(master.Variants)); err == nil { + t.Fatal("selected variant with a separate audio rendition must be rejected") + } + + // A group the variant does not reference must not cause a false positive. + unrelatedGroup := "#EXTM3U\n" + + "#EXT-X-MEDIA:TYPE=AUDIO,GROUP-ID=\"aud\",NAME=\"en\",DEFAULT=YES,URI=\"audio/index.m3u8\"\n" + + "#EXT-X-STREAM-INF:BANDWIDTH=1000000\nvideo/index.m3u8\n" + master, err = ParseMasterPlaylist(unrelatedGroup, masterURL) + if err != nil { + t.Fatal(err) + } + if err := checkAudioRendition(master, PickBestVariant(master.Variants)); err != nil { + t.Fatalf("unreferenced audio group must not be rejected: %v", err) + } + + // A rendition group without URIs (muxed tracks) is fine. + groupWithoutURI := "#EXTM3U\n" + + "#EXT-X-MEDIA:TYPE=AUDIO,GROUP-ID=\"aud\",NAME=\"en\",DEFAULT=YES\n" + + "#EXT-X-STREAM-INF:BANDWIDTH=1000000,AUDIO=\"aud\"\nvideo/index.m3u8\n" + master, err = ParseMasterPlaylist(groupWithoutURI, masterURL) + if err != nil { + t.Fatal(err) + } + if err := checkAudioRendition(master, PickBestVariant(master.Variants)); err != nil { + t.Fatalf("audio group without URI must not be rejected: %v", err) + } +} diff --git a/internal/protocol/hls/testdata/media/README.md b/internal/protocol/hls/testdata/media/README.md new file mode 100644 index 00000000..10afcfb9 --- /dev/null +++ b/internal/protocol/hls/testdata/media/README.md @@ -0,0 +1,10 @@ +Media fixtures generated locally with FFmpeg 7.0.2 (testsrc + sine sources, +no third-party content): + +- sample-00.ts / sample-01.ts: one continuous MPEG-2 TS encode split at a + packet boundary, so continuity counters and PCR are consistent across the + segment join exactly like segments cut from a single real-time stream. +- fmp4-init.mp4: ftyp+moov init section of a fragmented MP4. +- fmp4-seg1.m4s .. fmp4-seg3.m4s: moof+mdat media fragments (styp omitted by + the muxer, which is legal) split at moof boundaries from one fragmented MP4 + (-movflags +frag_keyframe+empty_moov+default_base_moof -frag_duration 500000). diff --git a/internal/protocol/hls/testdata/media/fmp4-init.mp4 b/internal/protocol/hls/testdata/media/fmp4-init.mp4 new file mode 100644 index 00000000..0c758c0b Binary files /dev/null and b/internal/protocol/hls/testdata/media/fmp4-init.mp4 differ diff --git a/internal/protocol/hls/testdata/media/fmp4-seg1.m4s b/internal/protocol/hls/testdata/media/fmp4-seg1.m4s new file mode 100644 index 00000000..e2735d3e Binary files /dev/null and b/internal/protocol/hls/testdata/media/fmp4-seg1.m4s differ diff --git a/internal/protocol/hls/testdata/media/fmp4-seg2.m4s b/internal/protocol/hls/testdata/media/fmp4-seg2.m4s new file mode 100644 index 00000000..f6fb5cad Binary files /dev/null and b/internal/protocol/hls/testdata/media/fmp4-seg2.m4s differ diff --git a/internal/protocol/hls/testdata/media/fmp4-seg3.m4s b/internal/protocol/hls/testdata/media/fmp4-seg3.m4s new file mode 100644 index 00000000..8d1b41aa Binary files /dev/null and b/internal/protocol/hls/testdata/media/fmp4-seg3.m4s differ diff --git a/internal/protocol/hls/testdata/media/sample-00.ts b/internal/protocol/hls/testdata/media/sample-00.ts new file mode 100644 index 00000000..4e301f77 Binary files /dev/null and b/internal/protocol/hls/testdata/media/sample-00.ts differ diff --git a/internal/protocol/hls/testdata/media/sample-01.ts b/internal/protocol/hls/testdata/media/sample-01.ts new file mode 100644 index 00000000..7b230d5b Binary files /dev/null and b/internal/protocol/hls/testdata/media/sample-01.ts differ diff --git a/internal/protocol/hls/testdata/olemovienews.m3u8 b/internal/protocol/hls/testdata/olemovienews.m3u8 new file mode 100644 index 00000000..6c908400 --- /dev/null +++ b/internal/protocol/hls/testdata/olemovienews.m3u8 @@ -0,0 +1,283 @@ +#EXTM3U +#EXT-X-TARGETDURATION:20 +#EXT-X-ALLOW-CACHE:YES +#EXT-X-PLAYLIST-TYPE:VOD +#EXT-X-VERSION:3 +#EXT-X-MEDIA-SEQUENCE:1 +#EXTINF:20.000, +seg-1-v1-a1.ts +#EXTINF:20.000, +seg-2-v1-a1.ts +#EXTINF:20.000, +seg-3-v1-a1.ts +#EXTINF:20.000, +seg-4-v1-a1.ts +#EXTINF:20.000, +seg-5-v1-a1.ts +#EXTINF:20.000, +seg-6-v1-a1.ts +#EXTINF:20.000, +seg-7-v1-a1.ts +#EXTINF:20.000, +seg-8-v1-a1.ts +#EXTINF:20.000, +seg-9-v1-a1.ts +#EXTINF:20.000, +seg-10-v1-a1.ts +#EXTINF:20.000, +seg-11-v1-a1.ts +#EXTINF:20.000, +seg-12-v1-a1.ts +#EXTINF:20.000, +seg-13-v1-a1.ts +#EXTINF:20.000, +seg-14-v1-a1.ts +#EXTINF:20.000, +seg-15-v1-a1.ts +#EXTINF:20.000, +seg-16-v1-a1.ts +#EXTINF:20.000, +seg-17-v1-a1.ts +#EXTINF:20.000, +seg-18-v1-a1.ts +#EXTINF:20.000, +seg-19-v1-a1.ts +#EXTINF:20.000, +seg-20-v1-a1.ts +#EXTINF:20.000, +seg-21-v1-a1.ts +#EXTINF:20.000, +seg-22-v1-a1.ts +#EXTINF:20.000, +seg-23-v1-a1.ts +#EXTINF:20.000, +seg-24-v1-a1.ts +#EXTINF:20.000, +seg-25-v1-a1.ts +#EXTINF:20.000, +seg-26-v1-a1.ts +#EXTINF:20.000, +seg-27-v1-a1.ts +#EXTINF:20.000, +seg-28-v1-a1.ts +#EXTINF:20.000, +seg-29-v1-a1.ts +#EXTINF:20.000, +seg-30-v1-a1.ts +#EXTINF:20.000, +seg-31-v1-a1.ts +#EXTINF:20.000, +seg-32-v1-a1.ts +#EXTINF:20.000, +seg-33-v1-a1.ts +#EXTINF:20.000, +seg-34-v1-a1.ts +#EXTINF:20.000, +seg-35-v1-a1.ts +#EXTINF:20.000, +seg-36-v1-a1.ts +#EXTINF:20.000, +seg-37-v1-a1.ts +#EXTINF:20.000, +seg-38-v1-a1.ts +#EXTINF:20.000, +seg-39-v1-a1.ts +#EXTINF:20.000, +seg-40-v1-a1.ts +#EXTINF:20.000, +seg-41-v1-a1.ts +#EXTINF:20.000, +seg-42-v1-a1.ts +#EXTINF:20.000, +seg-43-v1-a1.ts +#EXTINF:20.000, +seg-44-v1-a1.ts +#EXTINF:20.000, +seg-45-v1-a1.ts +#EXTINF:20.000, +seg-46-v1-a1.ts +#EXTINF:20.000, +seg-47-v1-a1.ts +#EXTINF:20.000, +seg-48-v1-a1.ts +#EXTINF:20.000, +seg-49-v1-a1.ts +#EXTINF:20.000, +seg-50-v1-a1.ts +#EXTINF:20.000, +seg-51-v1-a1.ts +#EXTINF:20.000, +seg-52-v1-a1.ts +#EXTINF:20.000, +seg-53-v1-a1.ts +#EXTINF:20.000, +seg-54-v1-a1.ts +#EXTINF:20.000, +seg-55-v1-a1.ts +#EXTINF:20.000, +seg-56-v1-a1.ts +#EXTINF:20.000, +seg-57-v1-a1.ts +#EXTINF:20.000, +seg-58-v1-a1.ts +#EXTINF:20.000, +seg-59-v1-a1.ts +#EXTINF:20.000, +seg-60-v1-a1.ts +#EXTINF:20.000, +seg-61-v1-a1.ts +#EXTINF:20.000, +seg-62-v1-a1.ts +#EXTINF:20.000, +seg-63-v1-a1.ts +#EXTINF:20.000, +seg-64-v1-a1.ts +#EXTINF:20.000, +seg-65-v1-a1.ts +#EXTINF:20.000, +seg-66-v1-a1.ts +#EXTINF:20.000, +seg-67-v1-a1.ts +#EXTINF:20.000, +seg-68-v1-a1.ts +#EXTINF:20.000, +seg-69-v1-a1.ts +#EXTINF:20.000, +seg-70-v1-a1.ts +#EXTINF:20.000, +seg-71-v1-a1.ts +#EXTINF:20.000, +seg-72-v1-a1.ts +#EXTINF:20.000, +seg-73-v1-a1.ts +#EXTINF:20.000, +seg-74-v1-a1.ts +#EXTINF:20.000, +seg-75-v1-a1.ts +#EXTINF:20.000, +seg-76-v1-a1.ts +#EXTINF:20.000, +seg-77-v1-a1.ts +#EXTINF:20.000, +seg-78-v1-a1.ts +#EXTINF:20.000, +seg-79-v1-a1.ts +#EXTINF:20.000, +seg-80-v1-a1.ts +#EXTINF:20.000, +seg-81-v1-a1.ts +#EXTINF:20.000, +seg-82-v1-a1.ts +#EXTINF:20.000, +seg-83-v1-a1.ts +#EXTINF:20.000, +seg-84-v1-a1.ts +#EXTINF:20.000, +seg-85-v1-a1.ts +#EXTINF:20.000, +seg-86-v1-a1.ts +#EXTINF:20.000, +seg-87-v1-a1.ts +#EXTINF:20.000, +seg-88-v1-a1.ts +#EXTINF:20.000, +seg-89-v1-a1.ts +#EXTINF:20.000, +seg-90-v1-a1.ts +#EXTINF:20.000, +seg-91-v1-a1.ts +#EXTINF:20.000, +seg-92-v1-a1.ts +#EXTINF:20.000, +seg-93-v1-a1.ts +#EXTINF:20.000, +seg-94-v1-a1.ts +#EXTINF:20.000, +seg-95-v1-a1.ts +#EXTINF:20.000, +seg-96-v1-a1.ts +#EXTINF:20.000, +seg-97-v1-a1.ts +#EXTINF:20.000, +seg-98-v1-a1.ts +#EXTINF:20.000, +seg-99-v1-a1.ts +#EXTINF:20.000, +seg-100-v1-a1.ts +#EXTINF:20.000, +seg-101-v1-a1.ts +#EXTINF:20.000, +seg-102-v1-a1.ts +#EXTINF:20.000, +seg-103-v1-a1.ts +#EXTINF:20.000, +seg-104-v1-a1.ts +#EXTINF:20.000, +seg-105-v1-a1.ts +#EXTINF:20.000, +seg-106-v1-a1.ts +#EXTINF:20.000, +seg-107-v1-a1.ts +#EXTINF:20.000, +seg-108-v1-a1.ts +#EXTINF:20.000, +seg-109-v1-a1.ts +#EXTINF:20.000, +seg-110-v1-a1.ts +#EXTINF:20.000, +seg-111-v1-a1.ts +#EXTINF:20.000, +seg-112-v1-a1.ts +#EXTINF:20.000, +seg-113-v1-a1.ts +#EXTINF:20.000, +seg-114-v1-a1.ts +#EXTINF:20.000, +seg-115-v1-a1.ts +#EXTINF:20.000, +seg-116-v1-a1.ts +#EXTINF:20.000, +seg-117-v1-a1.ts +#EXTINF:20.000, +seg-118-v1-a1.ts +#EXTINF:20.000, +seg-119-v1-a1.ts +#EXTINF:20.000, +seg-120-v1-a1.ts +#EXTINF:20.000, +seg-121-v1-a1.ts +#EXTINF:20.000, +seg-122-v1-a1.ts +#EXTINF:20.000, +seg-123-v1-a1.ts +#EXTINF:20.000, +seg-124-v1-a1.ts +#EXTINF:20.000, +seg-125-v1-a1.ts +#EXTINF:20.000, +seg-126-v1-a1.ts +#EXTINF:20.000, +seg-127-v1-a1.ts +#EXTINF:20.000, +seg-128-v1-a1.ts +#EXTINF:20.000, +seg-129-v1-a1.ts +#EXTINF:20.000, +seg-130-v1-a1.ts +#EXTINF:20.000, +seg-131-v1-a1.ts +#EXTINF:20.000, +seg-132-v1-a1.ts +#EXTINF:20.000, +seg-133-v1-a1.ts +#EXTINF:20.000, +seg-134-v1-a1.ts +#EXTINF:20.000, +seg-135-v1-a1.ts +#EXTINF:20.000, +seg-136-v1-a1.ts +#EXTINF:20.000, +seg-137-v1-a1.ts +#EXTINF:4.111, +seg-138-v1-a1.ts +#EXT-X-ENDLIST diff --git a/pkg/download/hls_live_test.go b/pkg/download/hls_live_test.go new file mode 100644 index 00000000..7fb41054 --- /dev/null +++ b/pkg/download/hls_live_test.go @@ -0,0 +1,97 @@ +package download + +import ( + "fmt" + "os" + "path/filepath" + "testing" + "time" + + "github.com/GopeedLab/gopeed/pkg/base" +) + +// TestDownloader_HlsLiveSite is a manual acceptance gate against a real site. +// It is skipped unless HLS_LIVE_URL is set, e.g.: +// +// HLS_LIVE_URL="https://host/path/index.m3u8" go test ./pkg/download/ -run TestDownloader_HlsLiveSite -v +func TestDownloader_HlsLiveSite(t *testing.T) { + playlistURL := os.Getenv("HLS_LIVE_URL") + if playlistURL == "" { + t.Skip("set HLS_LIVE_URL to run the live-site acceptance test") + } + referer := os.Getenv("HLS_LIVE_REFERER") + + downloader := NewDownloader(&DownloaderConfig{Storage: NewMemStorage(), StorageDir: t.TempDir()}) + if err := downloader.Setup(); err != nil { + t.Fatal(err) + } + defer downloader.Clear() + downloadDir := t.TempDir() + cfg, err := downloader.GetConfig() + if err != nil { + t.Fatal(err) + } + cfg.DownloadDir = downloadDir + if err := downloader.PutConfig(cfg); err != nil { + t.Fatal(err) + } + + req := &base.Request{URL: playlistURL} + if referer != "" { + req.Extra = map[string]any{"header": map[string]string{"Referer": referer}} + } + rr, err := downloader.Resolve(req, nil) + if err != nil { + t.Fatalf("resolve: %v", err) + } + name := rr.Res.Files[0].Name + size := rr.Res.Size + t.Logf("resolved: name=%s size=%d", name, size) + + taskID, err := downloader.Create(rr.ID) + if err != nil { + t.Fatal(err) + } + deadline := time.Now().Add(30 * time.Minute) + var lastLogged int64 + for { + task := downloader.GetTask(taskID) + if task == nil { + t.Fatal("task not found") + } + if task.Status == base.DownloadStatusDone { + break + } + if task.Status == base.DownloadStatusError { + t.Fatalf("task entered error state after %d bytes", task.Progress.Downloaded) + } + if time.Now().After(deadline) { + t.Fatalf("timeout, downloaded %d bytes", task.Progress.Downloaded) + } + if tick := task.Progress.Downloaded; tick-lastLogged > 10<<20 { + lastLogged = tick + t.Logf("progress: %d bytes", tick) + } + time.Sleep(500 * time.Millisecond) + } + + data, err := os.ReadFile(filepath.Join(downloadDir, name)) + if err != nil { + t.Fatal(err) + } + if int64(len(data)) == 0 { + t.Fatal("merged file is empty") + } + t.Logf("merged: %s (%d bytes)", name, len(data)) + if filepath.Ext(name) == ".ts" { + // MPEG-TS packets are 188 bytes and start with the 0x47 sync byte. + for i := 0; i < 188*4 && i < len(data); i += 188 { + if data[i] != 0x47 { + t.Fatalf("invalid TS sync byte at offset %d: %#x", i, data[i]) + } + } + } else if string(data[4:8]) != "ftyp" { + t.Fatalf("invalid fMP4 header: %q", data[:8]) + } + fmt.Printf("LIVE ACCEPTANCE OK: %s (%d bytes)\n", filepath.Join(downloadDir, name), len(data)) +} diff --git a/pkg/download/hls_protocol_test.go b/pkg/download/hls_protocol_test.go new file mode 100644 index 00000000..8ef48df8 --- /dev/null +++ b/pkg/download/hls_protocol_test.go @@ -0,0 +1,172 @@ +package download + +import ( + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/GopeedLab/gopeed/pkg/base" +) + +// hlsTestCdn spins up a master-playlist HLS site whose segments require a +// Referer header (anti-hotlink) and returns the server plus expected bytes. +func hlsTestCdn(t *testing.T, segmentDelay time.Duration) (*httptest.Server, []byte) { + t.Helper() + const total = 6 + segments := make(map[int][]byte) + for i := 0; i < total; i++ { + segments[i] = []byte(fmt.Sprintf("ENGINE-SEGMENT-%02d", i)) + } + var want []byte + for i := 0; i < total; i++ { + want = append(want, segments[i]...) + } + + mux := http.NewServeMux() + mux.HandleFunc("/master.m3u8", func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte("#EXTM3U\n" + + "#EXT-X-STREAM-INF:BANDWIDTH=500000\nlow/index.m3u8\n" + + "#EXT-X-STREAM-INF:BANDWIDTH=2000000\nhigh/index.m3u8\n")) + }) + mux.HandleFunc("/low/index.m3u8", func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte("#EXTM3U\n#EXTINF:4,\nseg-0.ts\n#EXT-X-ENDLIST\n")) + }) + mux.HandleFunc("/high/index.m3u8", func(w http.ResponseWriter, r *http.Request) { + var b strings.Builder + b.WriteString("#EXTM3U\n") + for i := 0; i < total; i++ { + b.WriteString(fmt.Sprintf("#EXTINF:4,\nseg-%d.ts\n", i)) + } + b.WriteString("#EXT-X-ENDLIST\n") + w.Write([]byte(b.String())) + }) + for i := 0; i < total; i++ { + idx := i + mux.HandleFunc(fmt.Sprintf("/high/seg-%d.ts", i), func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Referer") == "" { + w.WriteHeader(http.StatusForbidden) + return + } + if segmentDelay > 0 { + time.Sleep(segmentDelay) + } + w.Write(segments[idx]) + }) + } + server := httptest.NewServer(mux) + t.Cleanup(server.Close) + return server, want +} + +func hlsResolveReferer() *base.Request { + return &base.Request{ + URL: "", + Extra: map[string]any{"header": map[string]string{"Referer": "https://player.example.com/page"}}, + } +} + +func TestDownloader_HlsProtocolEndToEnd(t *testing.T) { + server, want := hlsTestCdn(t, 0) + downloader := NewDownloader(&DownloaderConfig{Storage: NewMemStorage(), StorageDir: t.TempDir()}) + if err := downloader.Setup(); err != nil { + t.Fatal(err) + } + defer downloader.Clear() + downloadDir := t.TempDir() + cfg, err := downloader.GetConfig() + if err != nil { + t.Fatal(err) + } + cfg.DownloadDir = downloadDir + if err := downloader.PutConfig(cfg); err != nil { + t.Fatal(err) + } + + req := hlsResolveReferer() + req.URL = server.URL + "/master.m3u8" + rr, err := downloader.Resolve(req, nil) + if err != nil { + t.Fatal(err) + } + if got := rr.Res.Files[0].Name; got != "index.ts" { + t.Errorf("resolved name = %s, want index.ts", got) + } + taskID, err := downloader.Create(rr.ID) + if err != nil { + t.Fatal(err) + } + waitForTaskStatus(t, downloader, taskID, base.DownloadStatusDone, 10*time.Second) + + task := downloader.GetTask(taskID) + if task.Protocol != "hls" { + t.Errorf("task protocol = %s, want hls", task.Protocol) + } + data, err := os.ReadFile(filepath.Join(downloadDir, "index.ts")) + if err != nil { + t.Fatal(err) + } + if string(data) != string(want) { + t.Errorf("merged file mismatch: %q", data) + } +} + +func TestDownloader_HlsProtocolPauseContinue(t *testing.T) { + server, want := hlsTestCdn(t, 120*time.Millisecond) + downloader := NewDownloader(&DownloaderConfig{Storage: NewMemStorage(), StorageDir: t.TempDir()}) + if err := downloader.Setup(); err != nil { + t.Fatal(err) + } + defer downloader.Clear() + downloadDir := t.TempDir() + cfg, err := downloader.GetConfig() + if err != nil { + t.Fatal(err) + } + cfg.DownloadDir = downloadDir + // Throttle the protocol so the pause lands mid-download. + cfg.ProtocolConfig["hls"] = map[string]any{ + "segmentConnections": 2, + "maxRetries": 3, + "timeoutSeconds": 5, + "prefetchContentLength": false, + } + if err := downloader.PutConfig(cfg); err != nil { + t.Fatal(err) + } + + req := hlsResolveReferer() + req.URL = server.URL + "/master.m3u8" + req.Extra = map[string]any{"header": map[string]string{"Referer": "https://player.example.com/page"}} + // CreateDirect: engine resolves lazily on start, like a direct task. + taskID, err := downloader.CreateDirect(req, &base.Options{Name: "movie.ts"}) + if err != nil { + t.Fatal(err) + } + waitForTaskStatus(t, downloader, taskID, base.DownloadStatusRunning, 10*time.Second) + if err := downloader.Pause(&TaskFilter{IDs: []string{taskID}}); err != nil { + t.Fatal(err) + } + waitForTaskStatus(t, downloader, taskID, base.DownloadStatusPause, 5*time.Second) + + if err := downloader.Continue(&TaskFilter{IDs: []string{taskID}}); err != nil { + t.Fatal(err) + } + waitForTaskStatus(t, downloader, taskID, base.DownloadStatusDone, 30*time.Second) + + task := downloader.GetTask(taskID) + if task.Meta.Res.Size != int64(len(want)) { + t.Errorf("final size = %d, want %d", task.Meta.Res.Size, len(want)) + } + data, err := os.ReadFile(filepath.Join(downloadDir, "movie.ts")) + if err != nil { + t.Fatal(err) + } + if string(data) != string(want) { + t.Errorf("merged file mismatch after pause/continue: %q", data) + } +} diff --git a/pkg/download/model.go b/pkg/download/model.go index 0705827b..1a3b7ae8 100644 --- a/pkg/download/model.go +++ b/pkg/download/model.go @@ -9,6 +9,7 @@ import ( "github.com/GopeedLab/gopeed/internal/fetcher" "github.com/GopeedLab/gopeed/internal/protocol/bt" "github.com/GopeedLab/gopeed/internal/protocol/ed2k" + "github.com/GopeedLab/gopeed/internal/protocol/hls" "github.com/GopeedLab/gopeed/internal/protocol/http" "github.com/GopeedLab/gopeed/pkg/base" enginewebview "github.com/GopeedLab/gopeed/pkg/download/engine/webview" @@ -197,6 +198,7 @@ func (cfg *DownloaderConfig) Init() *DownloaderConfig { } if len(cfg.FetchManagers) == 0 { cfg.FetchManagers = []fetcher.FetcherManager{ + new(hls.FetcherManager), new(http.FetcherManager), new(bt.FetcherManager), new(ed2k.FetcherManager), diff --git a/ui/flutter/lib/api/model/downloader_config.dart b/ui/flutter/lib/api/model/downloader_config.dart index 6a0fc47f..dae6a103 100644 --- a/ui/flutter/lib/api/model/downloader_config.dart +++ b/ui/flutter/lib/api/model/downloader_config.dart @@ -56,6 +56,7 @@ class ProtocolConfig { HttpConfig http = HttpConfig(); BtConfig bt = BtConfig(); Ed2kConfig ed2k = Ed2kConfig(); + HlsConfig hls = HlsConfig(); ProtocolConfig(); @@ -65,6 +66,25 @@ class ProtocolConfig { Map toJson() => _$ProtocolConfigToJson(this); } +@JsonSerializable() +class HlsConfig { + int segmentConnections; + int maxRetries; + int timeoutSeconds; + bool prefetchContentLength; + + HlsConfig({ + this.segmentConnections = 0, + this.maxRetries = 0, + this.timeoutSeconds = 0, + this.prefetchContentLength = false, + }); + + factory HlsConfig.fromJson(Map json) => _$HlsConfigFromJson(json); + + Map toJson() => _$HlsConfigToJson(this); +} + @JsonSerializable() class HttpConfig { String userAgent; diff --git a/ui/flutter/lib/api/model/downloader_config.g.dart b/ui/flutter/lib/api/model/downloader_config.g.dart index 33d6e05a..2ee3c2d8 100644 --- a/ui/flutter/lib/api/model/downloader_config.g.dart +++ b/ui/flutter/lib/api/model/downloader_config.g.dart @@ -56,12 +56,28 @@ Map _$ApiServerConfigToJson(ApiServerConfig instance) => json) => ProtocolConfig() ..http = HttpConfig.fromJson(json['http'] as Map) ..bt = BtConfig.fromJson(json['bt'] as Map) - ..ed2k = Ed2kConfig.fromJson(json['ed2k'] as Map); + ..ed2k = Ed2kConfig.fromJson(json['ed2k'] as Map) + ..hls = HlsConfig.fromJson(json['hls'] as Map); Map _$ProtocolConfigToJson(ProtocolConfig instance) => { 'http': instance.http.toJson(), 'bt': instance.bt.toJson(), 'ed2k': instance.ed2k.toJson(), + 'hls': instance.hls.toJson(), +}; + +HlsConfig _$HlsConfigFromJson(Map json) => HlsConfig( + segmentConnections: (json['segmentConnections'] as num?)?.toInt() ?? 0, + maxRetries: (json['maxRetries'] as num?)?.toInt() ?? 0, + timeoutSeconds: (json['timeoutSeconds'] as num?)?.toInt() ?? 0, + prefetchContentLength: json['prefetchContentLength'] as bool? ?? false, +); + +Map _$HlsConfigToJson(HlsConfig instance) => { + 'segmentConnections': instance.segmentConnections, + 'maxRetries': instance.maxRetries, + 'timeoutSeconds': instance.timeoutSeconds, + 'prefetchContentLength': instance.prefetchContentLength, }; HttpConfig _$HttpConfigFromJson(Map json) => HttpConfig( diff --git a/ui/flutter/lib/api/model/task.dart b/ui/flutter/lib/api/model/task.dart index cca65ed9..fd08e781 100644 --- a/ui/flutter/lib/api/model/task.dart +++ b/ui/flutter/lib/api/model/task.dart @@ -6,7 +6,7 @@ part 'task.g.dart'; enum Status { ready, running, pause, wait, error, done } -enum Protocol { http, bt, ed2k } +enum Protocol { http, bt, ed2k, hls } // ExtractStatus enum matching Go backend enum ExtractStatus { diff --git a/ui/flutter/lib/api/model/task.g.dart b/ui/flutter/lib/api/model/task.g.dart index 15b3b277..f127beee 100644 --- a/ui/flutter/lib/api/model/task.g.dart +++ b/ui/flutter/lib/api/model/task.g.dart @@ -38,7 +38,7 @@ const _$StatusEnumMap = { Status.done: 'done', }; -const _$ProtocolEnumMap = {Protocol.http: 'http', Protocol.bt: 'bt', Protocol.ed2k: 'ed2k'}; +const _$ProtocolEnumMap = {Protocol.http: 'http', Protocol.bt: 'bt', Protocol.ed2k: 'ed2k', Protocol.hls: 'hls'}; Progress _$ProgressFromJson(Map json) => Progress( used: (json['used'] as num).toInt(), diff --git a/ui/flutter/lib/api/model/task_stats.dart b/ui/flutter/lib/api/model/task_stats.dart index 4dafd0b0..dfe6c8c7 100644 --- a/ui/flutter/lib/api/model/task_stats.dart +++ b/ui/flutter/lib/api/model/task_stats.dart @@ -55,7 +55,7 @@ sealed class TaskStats { Protocol.http => HttpTaskStats.fromJson(snapshot), Protocol.bt => BtTaskStats.fromJson(snapshot: snapshot, runtime: runtime), Protocol.ed2k => Ed2kTaskStats.fromJson(snapshot: snapshot, runtime: runtime), - null => null, + Protocol.hls || null => null, }; } } diff --git a/ui/flutter/lib/features/settings/presentation/pages/settings_page.dart b/ui/flutter/lib/features/settings/presentation/pages/settings_page.dart index 291b504c..c78a18a1 100644 --- a/ui/flutter/lib/features/settings/presentation/pages/settings_page.dart +++ b/ui/flutter/lib/features/settings/presentation/pages/settings_page.dart @@ -65,6 +65,8 @@ class _SettingsPageState extends ConsumerState { final _maxRunningController = TextEditingController(); final _httpUserAgentController = TextEditingController(); final _httpConnectionsController = TextEditingController(); + final _hlsConnectionsController = TextEditingController(); + final _hlsMaxRetriesController = TextEditingController(); final _btListenPortController = TextEditingController(); final _btSeedRatioController = TextEditingController(); final _btSeedTimeController = TextEditingController(); @@ -96,6 +98,8 @@ class _SettingsPageState extends ConsumerState { _maxRunningController, _httpUserAgentController, _httpConnectionsController, + _hlsConnectionsController, + _hlsMaxRetriesController, _btListenPortController, _btSeedRatioController, _btSeedTimeController, @@ -572,6 +576,32 @@ class _SettingsPageState extends ConsumerState { ], ), ), + _SettingsBlock( + key: const ValueKey('settings-hls-block'), + title: 'HLS', + child: _SettingsGroup( + children: [ + SettingsItem( + title: context.l10n.connections, + child: _NumberSettingControl( + fieldKey: const ValueKey('hls-connections-input'), + controller: _hlsConnectionsController, + min: 1, + max: 256, + ), + ), + SettingsItem( + title: context.l10n.maxRetries, + child: _NumberSettingControl( + fieldKey: const ValueKey('hls-max-retries-input'), + controller: _hlsMaxRetriesController, + min: 0, + max: 30, + ), + ), + ], + ), + ), _SettingsBlock( title: 'BitTorrent', child: _SettingsGroup( @@ -953,6 +983,8 @@ class _SettingsPageState extends ConsumerState { _maxRunningController.text = _config!.maxRunning.clamp(1, 256).toString(); _httpUserAgentController.text = _config!.protocolConfig.http.userAgent; _httpConnectionsController.text = _config!.protocolConfig.http.connections.clamp(1, 256).toString(); + _hlsConnectionsController.text = _config!.protocolConfig.hls.segmentConnections.clamp(1, 256).toString(); + _hlsMaxRetriesController.text = _config!.protocolConfig.hls.maxRetries.clamp(0, 30).toString(); _btListenPortController.text = _config!.protocolConfig.bt.listenPort.clamp(0, 65535).toString(); _btSeedRatioController.text = _config!.protocolConfig.bt.seedRatio.toString(); _btSeedTimeController.text = (_config!.protocolConfig.bt.seedTime ~/ 60).clamp(0, 100000000).toString(); @@ -1048,6 +1080,18 @@ class _SettingsPageState extends ConsumerState { min: 1, max: 256, ); + config.protocolConfig.hls.segmentConnections = _boundedInt( + _hlsConnectionsController, + fallback: config.protocolConfig.hls.segmentConnections, + min: 1, + max: 256, + ); + config.protocolConfig.hls.maxRetries = _boundedInt( + _hlsMaxRetriesController, + fallback: config.protocolConfig.hls.maxRetries, + min: 0, + max: 30, + ); config.protocolConfig.bt.listenPort = _boundedInt( _btListenPortController, fallback: config.protocolConfig.bt.listenPort, diff --git a/ui/flutter/lib/features/tasks/domain/task_record.dart b/ui/flutter/lib/features/tasks/domain/task_record.dart index 47aa0cd4..94a94818 100644 --- a/ui/flutter/lib/features/tasks/domain/task_record.dart +++ b/ui/flutter/lib/features/tasks/domain/task_record.dart @@ -294,7 +294,7 @@ TaskAssetType _assetType(String name, {required bool isFolder, required api_task return switch (protocol) { api_task.Protocol.bt => TaskAssetType.torrent, api_task.Protocol.ed2k => TaskAssetType.ed2k, - api_task.Protocol.http || null => TaskAssetType.file, + api_task.Protocol.hls || api_task.Protocol.http || null => TaskAssetType.file, }; } diff --git a/ui/flutter/lib/l10n/app_ar.arb b/ui/flutter/lib/l10n/app_ar.arb index 372a5777..947af41b 100644 --- a/ui/flutter/lib/l10n/app_ar.arb +++ b/ui/flutter/lib/l10n/app_ar.arb @@ -38,6 +38,7 @@ "rememberLastDownloadDir": "حفظ كمجلد التنزيل الافتراضي", "downloadDirValid": "الرجاء تحديد مجلد التنزيل", "connections": "الاتصالات", + "maxRetries": "الحد الأقصى لإعادة المحاولات", "useServerCtime": "استخدام وقت الخادم لإنشاء الملف", "maxRunning": "الحد الأقصى للمهام المتزامنة", "defaultDirectDownload": "تفعيل التنزيل المباشر افتراضياً", diff --git a/ui/flutter/lib/l10n/app_az.arb b/ui/flutter/lib/l10n/app_az.arb index 564f9999..39c351b9 100644 --- a/ui/flutter/lib/l10n/app_az.arb +++ b/ui/flutter/lib/l10n/app_az.arb @@ -38,6 +38,7 @@ "rememberLastDownloadDir": "Standart endirmə qovluğu kimi yadda saxla", "downloadDirValid": "Zəhmət olmasa endirmə qovluğunu seçin", "connections": "Əlaqələr", + "maxRetries": "Maksimum cəhd sayı", "useServerCtime": "Fayl yaratma tarixi üçün server vaxtından istifadə et", "maxRunning": "Paralel Endirmələr", "defaultDirectDownload": "Standart olaraq Birbaşa Endirmə", diff --git a/ui/flutter/lib/l10n/app_ca.arb b/ui/flutter/lib/l10n/app_ca.arb index bee2f371..fe733610 100644 --- a/ui/flutter/lib/l10n/app_ca.arb +++ b/ui/flutter/lib/l10n/app_ca.arb @@ -38,6 +38,7 @@ "rememberLastDownloadDir": "Desa com a carpeta de baixades predeterminada", "downloadDirValid": "Seleccioneu la carpeta de baixades", "connections": "Connexions", + "maxRetries": "Reintents màxims", "useServerCtime": "Utilitza l'hora del servidor per a la creació de fitxers", "maxRunning": "Màxim de tasques en execució", "defaultDirectDownload": "Activa la baixada directa per defecte", diff --git a/ui/flutter/lib/l10n/app_de.arb b/ui/flutter/lib/l10n/app_de.arb index 5c95653f..058a4407 100644 --- a/ui/flutter/lib/l10n/app_de.arb +++ b/ui/flutter/lib/l10n/app_de.arb @@ -38,6 +38,7 @@ "rememberLastDownloadDir": "Als Standard-Downloadordner speichern", "downloadDirValid": "Bitte wählen Sie das Download-Verzeichnis", "connections": "Anschlüsse", + "maxRetries": "Maximale Wiederholungsversuche", "useServerCtime": "Serverzeit für die Dateierstellung verwenden", "maxRunning": "Maximale Anzahl ausgeführter Aufgaben", "defaultDirectDownload": "Direkten Download standardmäßig aktivieren", diff --git a/ui/flutter/lib/l10n/app_en.arb b/ui/flutter/lib/l10n/app_en.arb index dcf3d820..9af2806f 100644 --- a/ui/flutter/lib/l10n/app_en.arb +++ b/ui/flutter/lib/l10n/app_en.arb @@ -38,6 +38,7 @@ "rememberLastDownloadDir": "Save as default download directory", "downloadDirValid": "Please select the download directory", "connections": "Connections", + "maxRetries": "Max retries", "useServerCtime": "Use server time for file creation", "maxRunning": "Concurrent Downloads", "defaultDirectDownload": "Direct Download by Default", diff --git a/ui/flutter/lib/l10n/app_es.arb b/ui/flutter/lib/l10n/app_es.arb index a9588f9a..96a65113 100644 --- a/ui/flutter/lib/l10n/app_es.arb +++ b/ui/flutter/lib/l10n/app_es.arb @@ -38,6 +38,7 @@ "rememberLastDownloadDir": "Guardar como carpeta de descargas predeterminada", "downloadDirValid": "Por favor, selecciona el directorio de descarga", "connections": "Conexiones", + "maxRetries": "Reintentos máximos", "useServerCtime": "Usar tiempo del servidor para la creación de archivos", "maxRunning": "Máximo de Tareas en Ejecución", "defaultDirectDownload": "Ver descarga directa por defecto", diff --git a/ui/flutter/lib/l10n/app_fa.arb b/ui/flutter/lib/l10n/app_fa.arb index 66235220..a7cd2144 100644 --- a/ui/flutter/lib/l10n/app_fa.arb +++ b/ui/flutter/lib/l10n/app_fa.arb @@ -38,6 +38,7 @@ "rememberLastDownloadDir": "ذخیره به‌عنوان پوشه پیش‌فرض دانلود", "downloadDirValid": "لطفا دایرکتوری دانلود را انتخاب کنید", "connections": "اتصالات", + "maxRetries": "حداکثر تلاش مجدد", "useServerCtime": "استفاده از زمان سرور برای ایجاد فایل", "maxRunning": "حداکثر وظایف در حال اجرا", "defaultDirectDownload": "دانلود مستقیم را به صورت پیش فرض بررسی کنید", diff --git a/ui/flutter/lib/l10n/app_fr.arb b/ui/flutter/lib/l10n/app_fr.arb index de714190..9f269b01 100644 --- a/ui/flutter/lib/l10n/app_fr.arb +++ b/ui/flutter/lib/l10n/app_fr.arb @@ -38,6 +38,7 @@ "rememberLastDownloadDir": "Enregistrer comme dossier de téléchargement par défaut", "downloadDirValid": "Veuillez sélectionner le répertoire de téléchargement", "connections": "Connexions", + "maxRetries": "Nombre maximal de tentatives", "useServerCtime": "Utiliser l'heure du serveur pour la création de fichier", "maxRunning": "Tâches en cours maximum", "defaultDirectDownload": "Vérifiez le téléchargement direct par défaut", diff --git a/ui/flutter/lib/l10n/app_hu.arb b/ui/flutter/lib/l10n/app_hu.arb index 7b24e524..db44d5a7 100644 --- a/ui/flutter/lib/l10n/app_hu.arb +++ b/ui/flutter/lib/l10n/app_hu.arb @@ -38,6 +38,7 @@ "rememberLastDownloadDir": "Mentés alapértelmezett letöltési mappaként", "downloadDirValid": "Kérjük, válassza ki a letöltési könyvtárat", "connections": "Kapcsolatok", + "maxRetries": "Maximális újrapróbálkozások", "useServerCtime": "Használja a szerveridőt a fájl létrehozásához", "maxRunning": "Maximális futó feladatok", "defaultDirectDownload": "Alapértelmezettként ellenőrizze a közvetlen letöltést", diff --git a/ui/flutter/lib/l10n/app_id.arb b/ui/flutter/lib/l10n/app_id.arb index 4f883708..87a36a65 100644 --- a/ui/flutter/lib/l10n/app_id.arb +++ b/ui/flutter/lib/l10n/app_id.arb @@ -38,6 +38,7 @@ "rememberLastDownloadDir": "Simpan sebagai folder unduhan default", "downloadDirValid": "Silahkan pilih direktori unduhan", "connections": "Koneksi", + "maxRetries": "Percobaan ulang maks.", "useServerCtime": "Gunakan waktu server untuk pembuatan file", "maxRunning": "Tugas Maksimum yang Berjalan", "defaultDirectDownload": "Pilih unduhan langsung sebagai opsi standar", diff --git a/ui/flutter/lib/l10n/app_it.arb b/ui/flutter/lib/l10n/app_it.arb index 168e2fd6..b04fd493 100644 --- a/ui/flutter/lib/l10n/app_it.arb +++ b/ui/flutter/lib/l10n/app_it.arb @@ -38,6 +38,7 @@ "rememberLastDownloadDir": "Salva come cartella di download predefinita", "downloadDirValid": "Seleziona la directory di scaricamento", "connections": "Connessioni", + "maxRetries": "Tentativi massimi", "useServerCtime": "Utilizza l'ora del server per la creazione dei file", "maxRunning": "Numero massimo di attività in esecuzione", "defaultDirectDownload": "Controlla il download diretto per impostazione predefinita", diff --git a/ui/flutter/lib/l10n/app_ja.arb b/ui/flutter/lib/l10n/app_ja.arb index 98ea7b15..009a7bea 100644 --- a/ui/flutter/lib/l10n/app_ja.arb +++ b/ui/flutter/lib/l10n/app_ja.arb @@ -38,6 +38,7 @@ "rememberLastDownloadDir": "デフォルトのダウンロード先として保存", "downloadDirValid": "ダウンロードディレクトリを選択してください", "connections": "接続", + "maxRetries": "最大リトライ回数", "useServerCtime": "ファイル作成時にサーバー時間を使用する", "maxRunning": "最大実行タスク", "defaultDirectDownload": "デフォルトで直接ダウンロードにチェックを入れる", diff --git a/ui/flutter/lib/l10n/app_ko.arb b/ui/flutter/lib/l10n/app_ko.arb index 28d64a32..8f89366e 100644 --- a/ui/flutter/lib/l10n/app_ko.arb +++ b/ui/flutter/lib/l10n/app_ko.arb @@ -38,6 +38,7 @@ "rememberLastDownloadDir": "기본 다운로드 폴더로 저장", "downloadDirValid": "다운로드 폴더를 선택해 주세요", "connections": "연결 수", + "maxRetries": "최대 재시도 횟수", "useServerCtime": "파일 생성 시간에 서버 시간 사용", "maxRunning": "최대 동시 작업 수", "defaultDirectDownload": "기본으로 바로 다운로드 사용", diff --git a/ui/flutter/lib/l10n/app_pl.arb b/ui/flutter/lib/l10n/app_pl.arb index 6256937f..a6b9b031 100644 --- a/ui/flutter/lib/l10n/app_pl.arb +++ b/ui/flutter/lib/l10n/app_pl.arb @@ -38,6 +38,7 @@ "rememberLastDownloadDir": "Zapisz jako domyślny folder pobierania", "downloadDirValid": "Wskaż folder pobierania", "connections": "Połączenia", + "maxRetries": "Maks. liczba ponownych prób", "useServerCtime": "Użyj czasu serwera do utworzenia pliku", "maxRunning": "Maksymalna liczba zadań", "defaultDirectDownload": "Domyślnie sprawdź bezpośredni pobór", diff --git a/ui/flutter/lib/l10n/app_pt.arb b/ui/flutter/lib/l10n/app_pt.arb index eca4a034..0fb06371 100644 --- a/ui/flutter/lib/l10n/app_pt.arb +++ b/ui/flutter/lib/l10n/app_pt.arb @@ -38,6 +38,7 @@ "rememberLastDownloadDir": "Salvar como pasta de download padrão", "downloadDirValid": "Por favor, selecione a pasta", "connections": "Conexões", + "maxRetries": "Tentativas máximas", "useServerCtime": "Usar o tempo de servidor para a criação de ficheiros", "maxRunning": "Máximo de Tarefas Simultâneas", "defaultDirectDownload": "Marcar download direto por padrão", diff --git a/ui/flutter/lib/l10n/app_ru.arb b/ui/flutter/lib/l10n/app_ru.arb index 08811c7d..b8ebe8e9 100644 --- a/ui/flutter/lib/l10n/app_ru.arb +++ b/ui/flutter/lib/l10n/app_ru.arb @@ -38,6 +38,7 @@ "rememberLastDownloadDir": "Сохранить как папку загрузок по умолчанию", "downloadDirValid": "Пожалуйста, выберите папку для загрузки", "connections": "Соединений", + "maxRetries": "Макс. число повторов", "useServerCtime": "Использовать время сервера для создания файла", "maxRunning": "Максимальное количество активных задач", "defaultDirectDownload": "Прямая загрузка по умолчанию", diff --git a/ui/flutter/lib/l10n/app_ta.arb b/ui/flutter/lib/l10n/app_ta.arb index a4544b27..97e6b3c9 100644 --- a/ui/flutter/lib/l10n/app_ta.arb +++ b/ui/flutter/lib/l10n/app_ta.arb @@ -38,6 +38,7 @@ "rememberLastDownloadDir": "இயல்புநிலை பதிவிறக்க கோப்புறையாக சேமி", "downloadDirValid": "பதிவிரக்குமிடத்தை தேர்வு செயுங்கள்", "connections": "தொடர்புகள்", + "maxRetries": "அதிகபட்ச மறுமுயற்சிகள்", "useServerCtime": "கோப்பு உருவாக்க சேவையக நேரத்தைப் பயன்படுத்தவும்", "maxRunning": "அதிகபட்ச இயங்கும் பணிகள்", "defaultDirectDownload": "இயல்புநிலையாக நேரடிப் பதிவிறக்கத்தைச் சரிபார்க்கவும்", diff --git a/ui/flutter/lib/l10n/app_tr.arb b/ui/flutter/lib/l10n/app_tr.arb index e96f138a..127d7a0c 100644 --- a/ui/flutter/lib/l10n/app_tr.arb +++ b/ui/flutter/lib/l10n/app_tr.arb @@ -38,6 +38,7 @@ "rememberLastDownloadDir": "Varsayılan indirme klasörü olarak kaydet", "downloadDirValid": "Lütfen indirme dizinini seçin", "connections": "Bağlantılar", + "maxRetries": "Maksimum yeniden deneme", "useServerCtime": "Dosya oluşturma tarihi için sunucu zamanını kullan", "maxRunning": "Maksimum Çalışan İşlem", "defaultDirectDownload": "Varsayılan olarak direkt indirmeyi işaretle", diff --git a/ui/flutter/lib/l10n/app_uk.arb b/ui/flutter/lib/l10n/app_uk.arb index 2c659f04..24e20393 100644 --- a/ui/flutter/lib/l10n/app_uk.arb +++ b/ui/flutter/lib/l10n/app_uk.arb @@ -38,6 +38,7 @@ "rememberLastDownloadDir": "Зберегти як типову папку завантажень", "downloadDirValid": "Будь ласка, виберіть каталог завантажень", "connections": "З’єднання", + "maxRetries": "Макс. кількість повторів", "useServerCtime": "Використовувати час сервера для створення файлу", "maxRunning": "Макс. завдань у черзі", "defaultDirectDownload": "Пряме завантаження за замовчуванням", diff --git a/ui/flutter/lib/l10n/app_vi.arb b/ui/flutter/lib/l10n/app_vi.arb index 8e3e36f5..aa3e7a9c 100644 --- a/ui/flutter/lib/l10n/app_vi.arb +++ b/ui/flutter/lib/l10n/app_vi.arb @@ -38,6 +38,7 @@ "rememberLastDownloadDir": "Lưu làm thư mục tải xuống mặc định", "downloadDirValid": "Vui lòng chọn thư mục tải về", "connections": "Kết nối", + "maxRetries": "Số lần thử lại tối đa", "useServerCtime": "Sử dụng thời gian máy chủ cho việc tạo tệp", "maxRunning": "Số nhiệm vụ tối đa", "defaultDirectDownload": "Kiểm tra trực tiếp tải về theo mặc định", diff --git a/ui/flutter/lib/l10n/app_zh.arb b/ui/flutter/lib/l10n/app_zh.arb index 2395f69b..e199a7f7 100644 --- a/ui/flutter/lib/l10n/app_zh.arb +++ b/ui/flutter/lib/l10n/app_zh.arb @@ -38,6 +38,7 @@ "rememberLastDownloadDir": "保存为默认下载目录", "downloadDirValid": "请选择下载目录", "connections": "连接数", + "maxRetries": "最大重试次数", "useServerCtime": "创建文件使用服务器时间", "maxRunning": "最大下载数", "defaultDirectDownload": "默认启用直接下载", diff --git a/ui/flutter/lib/l10n/app_zh_TW.arb b/ui/flutter/lib/l10n/app_zh_TW.arb index 2692840b..1925dd0f 100644 --- a/ui/flutter/lib/l10n/app_zh_TW.arb +++ b/ui/flutter/lib/l10n/app_zh_TW.arb @@ -38,6 +38,7 @@ "rememberLastDownloadDir": "儲存為預設下載目錄", "downloadDirValid": "請選擇下載目錄", "connections": "連接數", + "maxRetries": "最大重試次數", "useServerCtime": "使用伺服器時間作為檔案建立時間", "maxRunning": "最大執行任務數", "defaultDirectDownload": "預設勾選直接下載",