diff --git a/internal/fetcher/fetcher.go b/internal/fetcher/fetcher.go index 4a1374c3..8d3a7821 100644 --- a/internal/fetcher/fetcher.go +++ b/internal/fetcher/fetcher.go @@ -106,7 +106,13 @@ func (s *SchemeFilter) Match(uri string) bool { case FilterTypeUrl: return strings.HasPrefix(uriUpper, patternUpper+":") case FilterTypeFile: - return strings.HasSuffix(uriUpper, "."+patternUpper) + // Strip query and fragment before matching so URLs like + // https://example.com/video.m3u8?sign=abc still route by extension. + clean := uri + if idx := strings.IndexAny(clean, "?#"); idx >= 0 { + clean = clean[:idx] + } + return strings.HasSuffix(strings.ToUpper(clean), "."+patternUpper) case FilterTypeBase64: return strings.HasPrefix(uriUpper, "DATA:"+patternUpper+";BASE64,") } diff --git a/internal/protocol/hls/config.go b/internal/protocol/hls/config.go new file mode 100644 index 00000000..2c06d5f0 --- /dev/null +++ b/internal/protocol/hls/config.go @@ -0,0 +1,30 @@ +package hls + +const ( + defaultSegmentConnections = 8 + defaultMaxRetries = 5 + defaultTimeoutSeconds = 30 + + // maxPrefetchSizeSegments caps the HEAD probing done at resolve time. + // Playlists larger than this skip size prefetch entirely. + maxPrefetchSizeSegments = 500 +) + +type config struct { + SegmentConnections int `json:"segmentConnections"` + MaxRetries int `json:"maxRetries"` + TimeoutSeconds int `json:"timeoutSeconds"` + PrefetchContentLength bool `json:"prefetchContentLength"` +} + +func (c *config) normalize() { + if c.SegmentConnections <= 0 { + c.SegmentConnections = defaultSegmentConnections + } + if c.MaxRetries < 0 { + c.MaxRetries = defaultMaxRetries + } + if c.TimeoutSeconds <= 0 { + c.TimeoutSeconds = defaultTimeoutSeconds + } +} diff --git a/internal/protocol/hls/crypto.go b/internal/protocol/hls/crypto.go new file mode 100644 index 00000000..58379a4f --- /dev/null +++ b/internal/protocol/hls/crypto.go @@ -0,0 +1,59 @@ +package hls + +import ( + "crypto/aes" + "crypto/cipher" + "fmt" +) + +// decryptAES128 decrypts an AES-128-CBC encrypted segment and strips PKCS7 +// padding. data length must be a multiple of the AES block size. +func decryptAES128(data, key, iv []byte) ([]byte, error) { + if len(key) != 16 { + return nil, fmt.Errorf("invalid AES-128 key size %d", len(key)) + } + if len(iv) != 16 { + return nil, fmt.Errorf("invalid AES-128 IV size %d", len(iv)) + } + if len(data) == 0 || len(data)%aes.BlockSize != 0 { + return nil, fmt.Errorf("invalid AES-128 data size %d", len(data)) + } + block, err := aes.NewCipher(key) + if err != nil { + return nil, err + } + plain := make([]byte, len(data)) + cipher.NewCBCDecrypter(block, iv).CryptBlocks(plain, data) + return pkcs7Unpad(plain, aes.BlockSize) +} + +func pkcs7Unpad(data []byte, blockSize int) ([]byte, error) { + if len(data) == 0 { + return nil, fmt.Errorf("empty plaintext") + } + pad := int(data[len(data)-1]) + if pad == 0 || pad > blockSize || pad > len(data) { + return nil, fmt.Errorf("invalid PKCS7 padding") + } + for _, b := range data[len(data)-pad:] { + if int(b) != pad { + return nil, fmt.Errorf("invalid PKCS7 padding") + } + } + return data[:len(data)-pad], nil +} + +// sequenceIV derives the default per-segment IV mandated by RFC 8216: the +// segment media sequence number as a 16-byte big-endian integer. +func sequenceIV(sequence int64) []byte { + iv := make([]byte, 16) + iv[8] = byte(sequence >> 56) + iv[9] = byte(sequence >> 48) + iv[10] = byte(sequence >> 40) + iv[11] = byte(sequence >> 32) + iv[12] = byte(sequence >> 24) + iv[13] = byte(sequence >> 16) + iv[14] = byte(sequence >> 8) + iv[15] = byte(sequence) + return iv +} diff --git a/internal/protocol/hls/crypto_test.go b/internal/protocol/hls/crypto_test.go new file mode 100644 index 00000000..05f18423 --- /dev/null +++ b/internal/protocol/hls/crypto_test.go @@ -0,0 +1,66 @@ +package hls + +import ( + "bytes" + "crypto/aes" + "crypto/cipher" + "crypto/rand" + "testing" +) + +func pkcs7Pad(data []byte, blockSize int) []byte { + pad := blockSize - len(data)%blockSize + return append(data, bytes.Repeat([]byte{byte(pad)}, pad)...) +} + +func TestDecryptAES128Roundtrip(t *testing.T) { + key := make([]byte, 16) + iv := make([]byte, 16) + if _, err := rand.Read(key); err != nil { + t.Fatal(err) + } + if _, err := rand.Read(iv); err != nil { + t.Fatal(err) + } + plain := []byte("hello hls segment payload, padded to block size!") + padded := pkcs7Pad(plain, aes.BlockSize) + block, err := aes.NewCipher(key) + if err != nil { + t.Fatal(err) + } + encrypted := make([]byte, len(padded)) + cipher.NewCBCEncrypter(block, iv).CryptBlocks(encrypted, padded) + + got, err := decryptAES128(encrypted, key, iv) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(got, plain) { + t.Errorf("roundtrip mismatch: %q", got) + } +} + +func TestDecryptAES128InvalidInput(t *testing.T) { + if _, err := decryptAES128([]byte{1, 2, 3}, make([]byte, 16), make([]byte, 16)); err == nil { + t.Error("non-block-aligned data should fail") + } + if _, err := decryptAES128(make([]byte, 16), make([]byte, 15), make([]byte, 16)); err == nil { + t.Error("short key should fail") + } + if _, err := decryptAES128(make([]byte, 16), make([]byte, 16), make([]byte, 16)); err == nil { + t.Error("all-zero block has invalid padding and should fail") + } +} + +func TestSequenceIV(t *testing.T) { + iv := sequenceIV(1) + if len(iv) != 16 || iv[15] != 1 { + t.Errorf("sequence 1 IV wrong: %v", iv) + } + iv = sequenceIV(0x0102030405060708) + for i, want := range []byte{1, 2, 3, 4, 5, 6, 7, 8} { + if iv[8+i] != want { + t.Errorf("sequence IV byte %d: want %d got %d", i, want, iv[8+i]) + } + } +} diff --git a/internal/protocol/hls/fetcher.go b/internal/protocol/hls/fetcher.go new file mode 100644 index 00000000..67ef9b5c --- /dev/null +++ b/internal/protocol/hls/fetcher.go @@ -0,0 +1,674 @@ +package hls + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "os" + "path/filepath" + "strconv" + "sync" + "sync/atomic" + "time" + + "github.com/GopeedLab/gopeed/internal/controller" + "github.com/GopeedLab/gopeed/internal/fetcher" + "github.com/GopeedLab/gopeed/internal/httpclient" + "github.com/GopeedLab/gopeed/pkg/base" +) + +// ReqExtra mirrors the HTTP protocol extra: the create dialog attaches HTTP +// style options to m3u8 URLs (same JSON keys), so the same shape is decoded. +type ReqExtra struct { + Method string `json:"method,omitempty"` + Header map[string]string `json:"header,omitempty"` + Body string `json:"body,omitempty"` +} + +// fetcherState is the resolved download plan persisted for resume. +type fetcherState struct { + MediaURL string `json:"mediaURL"` + OutputName string `json:"outputName"` + Segments []*Segment `json:"segments"` +} + +// Stats is the protocol task statistics snapshot. +type Stats struct { + SegmentsTotal int `json:"segmentsTotal"` + SegmentsDone int `json:"segmentsDone"` + SegmentsFailed int `json:"segmentsFailed"` +} + +// tempDirPrefix separates HLS staging folders from regular task files. +const tempDirPrefix = ".gopeed-hls-" + +type Fetcher struct { + fetcher.DefaultFetcher + meta *fetcher.FetcherMeta + config *config + + extra *ReqExtra + client *http.Client + impSession *httpclient.ImpersonationSession + + state *fetcherState + + ctx context.Context + cancel context.CancelFunc + wg sync.WaitGroup + mergeMu sync.Mutex + + downloaded atomic.Int64 // network bytes received, seeded from the journal + doneCount atomic.Int64 + failCount atomic.Int64 + + keyCache map[string][]byte + keyCacheMu sync.Mutex + + journal map[int64]int64 // plan index -> stored byte size + journalPath string + journalMu sync.Mutex +} + +func (f *Fetcher) Setup(ctl *controller.Controller) { + // Initializes Ctl and DoneCh, which Wait() depends on. + f.DefaultFetcher.Setup(ctl) + if f.meta == nil { + f.meta = &fetcher.FetcherMeta{} + } + f.impSession = httpclient.NewImpersonationSession() + f.keyCache = make(map[string][]byte) + f.config = &config{} + f.Ctl.GetConfig(f.config) + f.config.normalize() +} + +func (f *Fetcher) Meta() *fetcher.FetcherMeta { + return f.meta +} + +func (f *Fetcher) Resolve(req *base.Request, opts *base.Options) (err error) { + f.meta.Req = req + f.meta.Opts = opts + f.extra = &ReqExtra{} + if err = base.ParseReqExtra[ReqExtra](req); err != nil { + return + } + if e, ok := req.Extra.(*ReqExtra); ok { + f.extra = e + } + if f.client == nil { + f.client = f.buildClient() + } + + ctx := context.Background() + content, finalURL, err := f.fetchText(ctx, f.playlistMethod(), req.URL) + if err != nil { + return fmt.Errorf("fetch playlist failed: %w", err) + } + baseURL, err := url.Parse(finalURL) + if err != nil { + return err + } + + var media *Media + mediaURL := finalURL + if IsMasterPlaylist(content) { + variants, err := ParseMaster(content, baseURL) + if err != nil { + return err + } + best := PickBestVariant(variants) + variantContent, variantURL, err := f.fetchText(ctx, http.MethodGet, best.URI) + if err != nil { + return fmt.Errorf("fetch variant playlist failed: %w", err) + } + variantBase, err := url.Parse(variantURL) + if err != nil { + return err + } + content = variantContent + baseURL = variantBase + mediaURL = variantURL + } + media, err = ParseMedia(content, baseURL) + if err != nil { + return err + } + if media.Live { + return errors.New("live streams are not supported yet") + } + + outputName := DeriveOutputName(mediaURL, media) + f.state = &fetcherState{ + MediaURL: mediaURL, + OutputName: outputName, + Segments: media.Segments, + } + if f.config.PrefetchContentLength { + f.prefetchSize(media.Segments) + } + size := totalPlanSize(media.Segments) + + f.meta.Res = &base.Resource{ + Range: false, + Files: []*base.FileInfo{ + { + Name: outputName, + Size: size, + }, + }, + } + f.meta.Res.Size = size + return nil +} + +func (f *Fetcher) Patch(req *base.Request, opts *base.Options) (err error) { + urlChanged := f.meta.Req == nil || f.meta.Req.URL != req.URL + f.meta.Req = req + f.meta.Opts = opts + f.extra = &ReqExtra{} + if err = base.ParseReqExtra[ReqExtra](req); err != nil { + return + } + if e, ok := req.Extra.(*ReqExtra); ok { + f.extra = e + } + if urlChanged { + // The playlist URL changed (e.g. refreshed signed link): drop the plan + // so the engine re-resolves on next start (it does when Res is nil). + f.state = nil + f.meta.Res = nil + } + return +} + +func (f *Fetcher) Start() error { + if f.state == nil { + return errors.New("hls fetcher is not resolved") + } + ctx, cancel := context.WithCancel(context.Background()) + f.ctx, f.cancel = ctx, cancel + // Workers must be registered synchronously so a Pause() right after + // Start() is guaranteed to wait on all of them. + if err := f.startWorkers(); err != nil { + return err + } + // The supervisor owns this run's context: a Pause/Start cycle creates a + // fresh context and the previous supervisor must never act on it. + go f.supervise(ctx) + return nil +} + +// Pause stops all segment downloads synchronously. A merge already in flight +// is local I/O and is awaited rather than aborted. +func (f *Fetcher) Pause() error { + if f.cancel != nil { + f.cancel() + } + f.wg.Wait() + f.mergeMu.Lock() + f.mergeMu.Unlock() + return nil +} + +func (f *Fetcher) Close() error { + if f.cancel != nil { + f.cancel() + } + if f.impSession != nil { + f.impSession.Clear() + } + if f.state != nil && f.meta != nil { + os.RemoveAll(f.tempDir()) + } + return nil +} + +// Progress reports network bytes received for the single output file. +func (f *Fetcher) Progress() fetcher.Progress { + return fetcher.Progress{f.downloaded.Load()} +} + +func (f *Fetcher) Stats() *fetcher.Stats { + stats := &Stats{} + if f.state != nil { + stats.SegmentsTotal = len(f.state.Segments) + } + stats.SegmentsDone = int(f.doneCount.Load()) + stats.SegmentsFailed = int(f.failCount.Load()) + return &fetcher.Stats{Snapshot: stats} +} + +// supervise waits for the segment workers, then merges. A canceled context +// exits silently (the engine is pausing), everything else is reported through +// DoneCh. +func (f *Fetcher) supervise(ctx context.Context) { + f.wg.Wait() + if ctx.Err() != nil { + return + } + if failed := f.failCount.Load(); failed > 0 { + f.DoneCh <- fmt.Errorf("%d segment(s) failed to download", failed) + return + } + f.mergeMu.Lock() + err := f.merge() + f.mergeMu.Unlock() + if err == nil { + os.RemoveAll(f.tempDir()) + } + if errors.Is(err, context.Canceled) { + return + } + f.DoneCh <- err +} + +// startWorkers prepares the staging folder and launches the segment download +// workers. It returns immediately when every segment is already complete. +func (f *Fetcher) startWorkers() error { + tempDir := f.tempDir() + if err := os.MkdirAll(tempDir, 0777); err != nil { + return err + } + f.journalPath = filepath.Join(tempDir, "journal.json") + if err := f.loadJournal(); err != nil { + return err + } + + var pending []int64 + var baseBytes int64 + for i := range f.state.Segments { + if size, ok := f.journal[int64(i)]; ok { + if info, err := os.Stat(f.segmentPath(tempDir, int64(i))); err == nil && info.Size() == size { + baseBytes += size + f.doneCount.Add(1) + continue + } + } + pending = append(pending, int64(i)) + } + f.downloaded.Store(baseBytes) + if len(pending) == 0 { + return nil + } + + queue := make(chan int64) + go func() { + defer close(queue) + for _, idx := range pending { + select { + case <-f.ctx.Done(): + return + case queue <- idx: + } + } + }() + + workers := f.config.SegmentConnections + if workers > len(pending) { + workers = len(pending) + } + for i := 0; i < workers; i++ { + f.wg.Add(1) + go func() { + defer f.wg.Done() + for { + select { + case <-f.ctx.Done(): + return + case idx, ok := <-queue: + if !ok { + return + } + if err := f.downloadSegment(idx); err != nil { + if errors.Is(err, context.Canceled) { + return + } + f.failCount.Add(1) + } + } + } + }() + } + return nil +} + +func (f *Fetcher) downloadSegment(idx int64) error { + seg := f.state.Segments[idx] + data, err := f.fetchSegmentData(seg) + if err != nil { + return err + } + if seg.Key != nil { + key, err := f.fetchKey(seg.Key) + if err != nil { + return err + } + iv := seg.Key.IV + if iv == nil { + iv = sequenceIV(seg.Sequence) + } + data, err = decryptAES128(data, key, iv) + if err != nil { + return err + } + } + if err := f.storeSegment(idx, data); err != nil { + return err + } + f.doneCount.Add(1) + return nil +} + +// fetchSegmentData downloads one segment with retries, returning its (still +// encrypted) content. Network bytes are counted for progress as they arrive. +func (f *Fetcher) fetchSegmentData(seg *Segment) ([]byte, error) { + var lastErr error + for attempt := 0; attempt <= f.config.MaxRetries; attempt++ { + if err := f.ctx.Err(); err != nil { + return nil, err + } + if attempt > 0 { + select { + case <-f.ctx.Done(): + return nil, f.ctx.Err() + case <-time.After(time.Duration(attempt) * time.Second): + } + } + data, err := f.tryGetSegment(seg) + if err == nil { + return data, nil + } + if errors.Is(err, context.Canceled) { + return nil, err + } + lastErr = err + } + return nil, lastErr +} + +func (f *Fetcher) tryGetSegment(seg *Segment) ([]byte, error) { + resp, err := f.do(f.ctx, http.MethodGet, seg.URI, seg.Byterange) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return nil, fmt.Errorf("segment %s: unexpected status %d", seg.URI, resp.StatusCode) + } + reader := &stallReader{rc: resp.Body, timeout: f.timeout()} + var buf bytes.Buffer + chunk := make([]byte, 64<<10) + for { + n, err := reader.Read(chunk) + if n > 0 { + buf.Write(chunk[:n]) + f.downloaded.Add(int64(n)) + } + if err != nil { + if errors.Is(err, io.EOF) { + break + } + return nil, err + } + } + if seg.Byterange != nil && int64(buf.Len()) != seg.Byterange.Length { + return nil, fmt.Errorf("segment %s: size mismatch, got %d want %d", seg.URI, buf.Len(), seg.Byterange.Length) + } + return buf.Bytes(), nil +} + +func (f *Fetcher) fetchKey(key *Key) ([]byte, error) { + f.keyCacheMu.Lock() + if cached, ok := f.keyCache[key.URI]; ok { + f.keyCacheMu.Unlock() + return cached, nil + } + f.keyCacheMu.Unlock() + + var data []byte + var err error + for attempt := 0; attempt <= f.config.MaxRetries; attempt++ { + if err = f.ctx.Err(); err != nil { + return nil, err + } + if attempt > 0 { + select { + case <-f.ctx.Done(): + return nil, f.ctx.Err() + case <-time.After(time.Duration(attempt) * time.Second): + } + } + resp, reqErr := f.do(f.ctx, http.MethodGet, key.URI, nil) + if reqErr != nil { + err = reqErr + continue + } + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + resp.Body.Close() + err = fmt.Errorf("key %s: unexpected status %d", key.URI, resp.StatusCode) + continue + } + data, err = io.ReadAll(io.LimitReader(resp.Body, 1<<20)) + resp.Body.Close() + if err == nil { + break + } + } + if err != nil { + return nil, err + } + if len(data) != 16 { + return nil, fmt.Errorf("key %s: invalid key size %d", key.URI, len(data)) + } + f.keyCacheMu.Lock() + f.keyCache[key.URI] = data + f.keyCacheMu.Unlock() + return data, nil +} + +// merge concatenates the downloaded segments in plan order into the final +// output file. Segments are stored decrypted, so this is pure I/O. +func (f *Fetcher) merge() error { + target := f.meta.SingleFilepath() + if err := os.MkdirAll(filepath.Dir(target), 0777); err != nil { + return err + } + out, err := os.Create(target) + if err != nil { + return err + } + defer out.Close() + var total int64 + for i := range f.state.Segments { + data, err := os.ReadFile(f.segmentPath(f.tempDir(), int64(i))) + if err != nil { + return fmt.Errorf("read segment %d failed: %w", i, err) + } + if _, err := out.Write(data); err != nil { + return err + } + total += int64(len(data)) + } + f.meta.Res.Size = total + if len(f.meta.Res.Files) > 0 && f.meta.Res.Files[0] != nil { + f.meta.Res.Files[0].Size = total + } + return nil +} + +func (f *Fetcher) storeSegment(idx int64, data []byte) error { + tempDir := f.tempDir() + partPath := f.segmentPath(tempDir, idx) + ".part" + if err := os.WriteFile(partPath, data, 0666); err != nil { + return err + } + if err := os.Rename(partPath, f.segmentPath(tempDir, idx)); err != nil { + return err + } + f.journalMu.Lock() + defer f.journalMu.Unlock() + f.journal[idx] = int64(len(data)) + return f.saveJournal() +} + +func (f *Fetcher) segmentPath(tempDir string, idx int64) string { + return filepath.Join(tempDir, "seg-"+strconv.FormatInt(idx, 10)) +} + +// tempDir derives a stable staging folder from the playlist URL so resumes +// find their segments across app restarts. +func (f *Fetcher) tempDir() string { + return filepath.Join(filepath.Dir(f.meta.SingleFilepath()), tempDirPrefix+sha1Hex(f.state.MediaURL)[:12]) +} + +func (f *Fetcher) loadJournal() error { + f.journal = make(map[int64]int64) + data, err := os.ReadFile(f.journalPath) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return err + } + stored := make(map[string]int64) + if err := json.Unmarshal(data, &stored); err != nil { + // A broken journal must not kill a restartable download. + return nil + } + for key, size := range stored { + idx, err := strconv.ParseInt(key, 10, 64) + if err != nil { + continue + } + f.journal[idx] = size + } + return nil +} + +func (f *Fetcher) saveJournal() error { + stored := make(map[string]int64, len(f.journal)) + for idx, size := range f.journal { + stored[strconv.FormatInt(idx, 10)] = size + } + data, err := json.Marshal(stored) + if err != nil { + return err + } + return os.WriteFile(f.journalPath, data, 0666) +} + +// prefetchSize probes segment Content-Length via HEAD so the task can show a +// percentage. Any failure resets all sizes to zero (indeterminate progress). +func (f *Fetcher) prefetchSize(segments []*Segment) { + var need []*Segment + for _, seg := range segments { + if seg.Byterange == nil { + need = append(need, seg) + } + } + if len(need) == 0 || len(need) > maxPrefetchSizeSegments { + return + } + + workerCount := f.config.SegmentConnections + if workerCount > 8 { + workerCount = 8 + } + var failed atomic.Bool + queue := make(chan *Segment) + var wg sync.WaitGroup + for i := 0; i < workerCount; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for seg := range queue { + ctx, cancel := context.WithTimeout(context.Background(), f.timeout()) + seg.Size = f.headContentLength(ctx, seg.URI) + cancel() + if seg.Size <= 0 { + failed.Store(true) + } + } + }() + } + for _, seg := range need { + queue <- seg + } + close(queue) + wg.Wait() + if failed.Load() { + for _, seg := range segments { + seg.Size = 0 + } + } +} + +func (f *Fetcher) headContentLength(ctx context.Context, u string) int64 { + req, err := f.buildRequest(ctx, http.MethodHead, u, nil, nil) + if err != nil { + return 0 + } + resp, err := f.client.Do(req) + if err != nil { + return 0 + } + defer resp.Body.Close() + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return 0 + } + return resp.ContentLength +} + +func (f *Fetcher) fetchText(ctx context.Context, method, u string) (string, string, error) { + resp, err := f.do(ctx, method, u, nil) + if err != nil { + return "", "", err + } + defer resp.Body.Close() + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return "", "", fmt.Errorf("unexpected status %d for %s", resp.StatusCode, u) + } + data, err := io.ReadAll(io.LimitReader(&stallReader{rc: resp.Body, timeout: f.timeout()}, maxPlaylistSize)) + if err != nil { + return "", "", err + } + return string(data), resp.Request.URL.String(), nil +} + +func (f *Fetcher) do(ctx context.Context, method, u string, byterange *ByteRange) (*http.Response, error) { + if method == "" { + method = http.MethodGet + } + if f.client == nil { + f.client = f.buildClient() + } + req, err := f.buildRequest(ctx, method, u, nil, byterange) + if err != nil { + return nil, err + } + return f.client.Do(req) +} + +func (f *Fetcher) playlistMethod() string { + if f.extra != nil && f.extra.Method != "" { + return f.extra.Method + } + return http.MethodGet +} + +func totalPlanSize(segments []*Segment) int64 { + var total int64 + for _, seg := range segments { + if seg.Size > 0 { + total += seg.Size + } + } + return total +} diff --git a/internal/protocol/hls/fetcher_manager.go b/internal/protocol/hls/fetcher_manager.go new file mode 100644 index 00000000..5de89225 --- /dev/null +++ b/internal/protocol/hls/fetcher_manager.go @@ -0,0 +1,86 @@ +package hls + +import ( + "net/url" + "path" + "strings" + + "github.com/GopeedLab/gopeed/internal/fetcher" +) + +type FetcherManager struct { +} + +func (fm *FetcherManager) Name() string { + return "hls" +} + +// Filters registers the .m3u8 extension. The manager must be registered BEFORE +// the http manager in the downloader config: protocol routing picks the first +// manager whose filter matches, and the http filter claims every http(s) URL. +func (fm *FetcherManager) Filters() []*fetcher.SchemeFilter { + return []*fetcher.SchemeFilter{ + { + Type: fetcher.FilterTypeFile, + Pattern: "M3U8", + }, + } +} + +func (fm *FetcherManager) Build() fetcher.Fetcher { + return &Fetcher{} +} + +func (fm *FetcherManager) ParseName(u string) string { + parsed, err := url.Parse(u) + if err != nil { + return "" + } + name := path.Base(parsed.Path) + name = strings.TrimSuffix(name, path.Ext(name)) + if name == "" || name == "/" || name == "." { + name = parsed.Hostname() + } + return name +} + +func (fm *FetcherManager) AutoRename() bool { + return true +} + +func (fm *FetcherManager) DefaultConfig() any { + return &config{ + SegmentConnections: defaultSegmentConnections, + MaxRetries: defaultMaxRetries, + TimeoutSeconds: defaultTimeoutSeconds, + PrefetchContentLength: true, + } +} + +func (fm *FetcherManager) Store(f fetcher.Fetcher) (data any, err error) { + _f := f.(*Fetcher) + if _f.state == nil { + return nil, nil + } + // The segment plan is immutable after resolve; the completed-segment + // journal lives on disk next to the segments. + return _f.state, nil +} + +func (fm *FetcherManager) Restore() (v any, f func(meta *fetcher.FetcherMeta, v any) fetcher.Fetcher) { + return &fetcherState{}, func(meta *fetcher.FetcherMeta, v any) fetcher.Fetcher { + fb := &FetcherManager{} + restored := fb.Build().(*Fetcher) + restored.meta = meta + if state, ok := v.(*fetcherState); ok && state.MediaURL != "" { + restored.state = state + } + // Setup is invoked by the engine (restoreFetcher) after this builder + // returns; ReqExtra parsing happens there as well. + return restored + } +} + +func (fm *FetcherManager) Close() error { + return nil +} diff --git a/internal/protocol/hls/fetcher_test.go b/internal/protocol/hls/fetcher_test.go new file mode 100644 index 00000000..867d0b93 --- /dev/null +++ b/internal/protocol/hls/fetcher_test.go @@ -0,0 +1,472 @@ +package hls + +import ( + "bytes" + "crypto/aes" + "crypto/cipher" + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strconv" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/GopeedLab/gopeed/internal/controller" + "github.com/GopeedLab/gopeed/internal/fetcher" + "github.com/GopeedLab/gopeed/pkg/base" +) + +func pkcs7PadForTest(data []byte, blockSize int) []byte { + pad := blockSize - len(data)%blockSize + return append(data, bytes.Repeat([]byte{byte(pad)}, pad)...) +} + +func encryptSegment(plain, key, iv []byte) []byte { + padded := pkcs7PadForTest(plain, aes.BlockSize) + block, _ := aes.NewCipher(key) + encrypted := make([]byte, len(padded)) + cipher.NewCBCEncrypter(block, iv).CryptBlocks(encrypted, padded) + return encrypted +} + +// serveContentWithReferer is a helper handler that enforces an optional +// Referer requirement and reports request counts. +type cdnHandler struct { + mux *http.ServeMux + requireReferer string + requests atomic.Int64 +} + +func (h *cdnHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + h.requests.Add(1) + if h.requireReferer != "" && r.Header.Get("Referer") != h.requireReferer { + w.WriteHeader(http.StatusForbidden) + return + } + h.mux.ServeHTTP(w, r) +} + +func newTestFetcher(t *testing.T, cfg config) *Fetcher { + t.Helper() + fm := &FetcherManager{} + f := fm.Build().(*Fetcher) + ctl := controller.NewController() + ctl.GetConfig = func(v any) { + if c, ok := v.(*config); ok { + *c = cfg + } + } + f.Setup(ctl) + return f +} + +func testConfig() config { + return config{ + SegmentConnections: 4, + MaxRetries: 2, + TimeoutSeconds: 5, + PrefetchContentLength: true, + } +} + +func mustResolve(t *testing.T, f *Fetcher, playlistURL, referer string) { + t.Helper() + req := &base.Request{URL: playlistURL} + if referer != "" { + req.Extra = &ReqExtra{Header: map[string]string{"Referer": referer}} + } + if err := f.Resolve(req, &base.Options{Path: t.TempDir()}); err != nil { + t.Fatalf("resolve: %v", err) + } +} + +func mustRun(t *testing.T, f *Fetcher) { + t.Helper() + if err := f.Start(); err != nil { + t.Fatalf("start: %v", err) + } + if err := f.Wait(); err != nil { + t.Fatalf("wait: %v", err) + } +} + +func readOutput(t *testing.T, f *Fetcher) []byte { + t.Helper() + data, err := os.ReadFile(f.meta.SingleFilepath()) + if err != nil { + t.Fatal(err) + } + return data +} + +func TestFetcherE2E_MasterTS(t *testing.T) { + seg := func(i int) []byte { + return []byte(fmt.Sprintf("SEGMENT-%02d-PAYLOAD", i)) + } + mux := http.NewServeMux() + mux.HandleFunc("/master.m3u8", func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte("#EXTM3U\n" + + "#EXT-X-STREAM-INF:BANDWIDTH=800000\nv1/index.m3u8\n" + + "#EXT-X-STREAM-INF:BANDWIDTH=2000000\nv2/index.m3u8\n")) + }) + mux.HandleFunc("/v1/index.m3u8", func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte("#EXTM3U\n#EXTINF:4,\nlow-0.ts\n#EXT-X-ENDLIST\n")) + }) + mux.HandleFunc("/v2/index.m3u8", func(w http.ResponseWriter, r *http.Request) { + var b strings.Builder + b.WriteString("#EXTM3U\n") + for i := 0; i < 5; i++ { + b.WriteString(fmt.Sprintf("#EXTINF:4,\nseg-%d.ts\n", i)) + } + b.WriteString("#EXT-X-ENDLIST\n") + w.Write([]byte(b.String())) + }) + for i := 0; i < 5; i++ { + content := seg(i) + path := "/v2/seg-" + strconv.Itoa(i) + ".ts" + mux.HandleFunc(path, func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Length", strconv.Itoa(len(content))) + w.Write(content) + }) + } + server := httptest.NewServer(&cdnHandler{mux: mux}) + defer server.Close() + + f := newTestFetcher(t, testConfig()) + mustResolve(t, f, server.URL+"/master.m3u8", "") + // Master resolution picked the 2000000 variant. + if f.state.MediaURL != server.URL+"/v2/index.m3u8" { + t.Fatalf("wrong variant picked: %s", f.state.MediaURL) + } + // HEAD prefetch should have computed the exact total size. + if f.meta.Res.Size != int64(5*len(seg(0))) { + t.Fatalf("prefetched size wrong: %d", f.meta.Res.Size) + } + mustRun(t, f) + + var want []byte + for i := 0; i < 5; i++ { + want = append(want, seg(i)...) + } + got := readOutput(t, f) + if !bytes.Equal(got, want) { + t.Errorf("merged output mismatch: %q", got) + } + // The staging folder must be cleaned up after a successful merge; only + // the output file remains in the task directory. + if entries, _ := os.ReadDir(filepath.Dir(f.meta.SingleFilepath())); len(entries) != 1 { + for _, e := range entries { + t.Logf("leftover entry: %s", e.Name()) + } + t.Errorf("staging dir should be cleaned up after merge") + } + if f.meta.Res.Size != int64(len(want)) { + t.Errorf("final size wrong: %d", f.meta.Res.Size) + } +} + +func TestFetcherE2E_RefererEnforced(t *testing.T) { + page := "https://player.example.com/watch/123" + var playlist strings.Builder + playlist.WriteString("#EXTM3U\n") + segments := make(map[string][]byte) + for i := 0; i < 3; i++ { + name := fmt.Sprintf("seg-%d.ts", i) + segments["/"+name] = []byte(fmt.Sprintf("DATA-%d", i)) + playlist.WriteString(fmt.Sprintf("#EXTINF:4,\n%s\n", name)) + } + playlist.WriteString("#EXT-X-ENDLIST\n") + + mux := http.NewServeMux() + mux.HandleFunc("/video.m3u8", func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte(playlist.String())) + }) + for path, content := range segments { + c := content + mux.HandleFunc(path, func(w http.ResponseWriter, r *http.Request) { + w.Write(c) + }) + } + server := httptest.NewServer(&cdnHandler{mux: mux, requireReferer: page}) + defer server.Close() + + // Without Referer the resolve itself must fail (403 on the playlist). + blocked := newTestFetcher(t, testConfig()) + req := &base.Request{URL: server.URL + "/video.m3u8"} + if err := blocked.Resolve(req, &base.Options{Path: t.TempDir()}); err == nil { + t.Fatal("resolve without Referer should fail") + } + + // With the Referer set through the task extra headers it must succeed. + allowed := newTestFetcher(t, testConfig()) + allowedReq := &base.Request{ + URL: server.URL + "/video.m3u8", + Extra: &ReqExtra{Header: map[string]string{"Referer": page}}, + } + if err := allowed.Resolve(allowedReq, &base.Options{Path: t.TempDir()}); err != nil { + t.Fatal(err) + } + mustRun(t, allowed) + want := []byte("DATA-0DATA-1DATA-2") + if !bytes.Equal(readOutput(t, allowed), want) { + t.Error("output mismatch with Referer download") + } +} + +func TestFetcherE2E_AES128KeyRotation(t *testing.T) { + key1 := []byte("0123456789abcdef") + key2 := []byte("fedcba9876543210") + iv1 := []byte("aaaaaaaaaaaaaaaa") + plain1 := []byte("ENCRYPTED-SEGMENT-ONE!!") // padded on encrypt + plain2 := []byte("ENCRYPTED-SEGMENT-TWO!!") + // seg-2 has no explicit IV: derived from media sequence 1. + enc1 := encryptSegment(plain1, key1, iv1) + enc2 := encryptSegment(plain2, key2, sequenceIV(1)) + + mux := http.NewServeMux() + mux.HandleFunc("/video.m3u8", func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte("#EXTM3U\n" + + "#EXT-X-MEDIA-SEQUENCE:0\n" + + "#EXT-X-KEY:METHOD=AES-128,URI=\"key-1.bin\",IV=0x" + hexEncode(iv1) + "\n" + + "#EXTINF:5,\nseg-1.ts\n" + + "#EXT-X-KEY:METHOD=AES-128,URI=\"key-2.bin\"\n" + + "#EXTINF:5,\nseg-2.ts\n" + + "#EXT-X-ENDLIST\n")) + }) + mux.HandleFunc("/key-1.bin", func(w http.ResponseWriter, r *http.Request) { + w.Write(key1) + }) + mux.HandleFunc("/key-2.bin", func(w http.ResponseWriter, r *http.Request) { + w.Write(key2) + }) + mux.HandleFunc("/seg-1.ts", func(w http.ResponseWriter, r *http.Request) { + w.Write(enc1) + }) + mux.HandleFunc("/seg-2.ts", func(w http.ResponseWriter, r *http.Request) { + w.Write(enc2) + }) + server := httptest.NewServer(mux) + defer server.Close() + + f := newTestFetcher(t, testConfig()) + mustResolve(t, f, server.URL+"/video.m3u8", "") + if f.meta.Res.Size != 0 { + // Encrypted segment sizes are probed as ciphertext length, decryption + // shrinks them, so an all-or-nothing probe may still report a size. + t.Logf("prefetched size: %d", f.meta.Res.Size) + } + mustRun(t, f) + + want := append(append([]byte{}, plain1...), plain2...) + if !bytes.Equal(readOutput(t, f), want) { + t.Errorf("decrypted output mismatch: %q", readOutput(t, f)) + } +} + +func TestFetcherE2E_ByteRange(t *testing.T) { + data := []byte("0123456789ABCDEFGHIJ") // 20 bytes + mux := http.NewServeMux() + mux.HandleFunc("/video.m3u8", func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte("#EXTM3U\n" + + "#EXT-X-BYTERANGE:10@0\nblob.bin\n" + + "#EXT-X-BYTERANGE:10\nblob.bin\n" + + "#EXT-X-ENDLIST\n")) + }) + mux.HandleFunc("/blob.bin", func(w http.ResponseWriter, r *http.Request) { + rng := r.Header.Get("Range") + if rng == "" { + w.Write(data) + return + } + // bytes=start-end + var start, end int + fmt.Sscanf(rng, "bytes=%d-%d", &start, &end) + w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, len(data))) + w.WriteHeader(http.StatusPartialContent) + w.Write(data[start : end+1]) + }) + server := httptest.NewServer(mux) + defer server.Close() + + f := newTestFetcher(t, testConfig()) + mustResolve(t, f, server.URL+"/video.m3u8", "") + if f.meta.Res.Size != 20 { + t.Fatalf("byterange sizes should be known from the playlist, got %d", f.meta.Res.Size) + } + mustRun(t, f) + if !bytes.Equal(readOutput(t, f), data) { + t.Errorf("byterange merge mismatch: %q", readOutput(t, f)) + } +} + +func TestFetcherE2E_PauseResume(t *testing.T) { + const total = 8 + var served atomic.Int64 + mux := http.NewServeMux() + mux.HandleFunc("/video.m3u8", func(w http.ResponseWriter, r *http.Request) { + var b strings.Builder + b.WriteString("#EXTM3U\n") + for i := 0; i < total; i++ { + b.WriteString(fmt.Sprintf("#EXTINF:4,\nseg-%d.ts\n", i)) + } + b.WriteString("#EXT-X-ENDLIST\n") + w.Write([]byte(b.String())) + }) + segments := make(map[int][]byte) + for i := 0; i < total; i++ { + segments[i] = []byte(fmt.Sprintf("SLOW-SEGMENT-%02d", i)) + } + for i := 0; i < total; i++ { + idx := i + mux.HandleFunc(fmt.Sprintf("/seg-%d.ts", i), func(w http.ResponseWriter, r *http.Request) { + time.Sleep(150 * time.Millisecond) + served.Add(1) + w.Write(segments[idx]) + }) + } + server := httptest.NewServer(mux) + defer server.Close() + + cfg := testConfig() + cfg.SegmentConnections = 2 + cfg.PrefetchContentLength = false + f := newTestFetcher(t, cfg) + mustResolve(t, f, server.URL+"/video.m3u8", "") + + if err := f.Start(); err != nil { + t.Fatal(err) + } + // Let a couple of segments finish, then pause. + time.Sleep(400 * time.Millisecond) + if err := f.Pause(); err != nil { + t.Fatal(err) + } + if f.doneCount.Load() == 0 { + t.Fatal("expected some segments to complete before pause") + } + if f.doneCount.Load() == total { + t.Fatal("all segments finished before pause, test is not exercising resume") + } + + // Resume must finish the rest and produce the full output. + if err := f.Start(); err != nil { + t.Fatal(err) + } + if err := f.Wait(); err != nil { + t.Fatal(err) + } + var want []byte + for i := 0; i < total; i++ { + want = append(want, segments[i]...) + } + if !bytes.Equal(readOutput(t, f), want) { + t.Errorf("resumed output mismatch: %q", readOutput(t, f)) + } +} + +func TestFetcherE2E_StoreRestoreResume(t *testing.T) { + const total = 6 + var served atomic.Int64 + mux := http.NewServeMux() + mux.HandleFunc("/video.m3u8", func(w http.ResponseWriter, r *http.Request) { + var b strings.Builder + b.WriteString("#EXTM3U\n") + for i := 0; i < total; i++ { + b.WriteString(fmt.Sprintf("#EXTINF:4,\nseg-%d.ts\n", i)) + } + b.WriteString("#EXT-X-ENDLIST\n") + w.Write([]byte(b.String())) + }) + segments := make(map[int][]byte) + for i := 0; i < total; i++ { + segments[i] = []byte(fmt.Sprintf("RESTORE-SEGMENT-%02d", i)) + } + for i := 0; i < total; i++ { + idx := i + mux.HandleFunc(fmt.Sprintf("/seg-%d.ts", i), func(w http.ResponseWriter, r *http.Request) { + time.Sleep(120 * time.Millisecond) + served.Add(1) + w.Write(segments[idx]) + }) + } + server := httptest.NewServer(mux) + defer server.Close() + + cfg := testConfig() + cfg.SegmentConnections = 2 + cfg.PrefetchContentLength = false + + f := newTestFetcher(t, cfg) + mustResolve(t, f, server.URL+"/video.m3u8", "") + if err := f.Start(); err != nil { + t.Fatal(err) + } + time.Sleep(350 * time.Millisecond) + if err := f.Pause(); err != nil { + t.Fatal(err) + } + doneBefore := f.doneCount.Load() + if doneBefore == 0 { + t.Fatal("expected some segments to finish before simulated restart") + } + + // Simulate an app restart: persist, then restore into a fresh fetcher. + fm := &FetcherManager{} + state, err := fm.Store(f) + if err != nil { + t.Fatal(err) + } + if state == nil { + t.Fatal("state should be persisted") + } + ctl := controller.NewController() + ctl.GetConfig = func(v any) { + if c, ok := v.(*config); ok { + *c = cfg + } + } + holder := fm.Build().(*Fetcher) + holder.Setup(ctl) + holder.meta = &fetcher.FetcherMeta{ + Req: f.meta.Req, + Opts: f.meta.Opts, + Res: f.meta.Res, + } + _, builder := fm.Restore() + rf := builder(holder.meta, state).(*Fetcher) + // The engine calls Setup on the builder result (restoreFetcher). + rf.Setup(ctl) + if rf.state == nil || len(rf.state.Segments) != total { + t.Fatal("restored state missing segment plan") + } + + if err := rf.Start(); err != nil { + t.Fatal(err) + } + if err := rf.Wait(); err != nil { + t.Fatal(err) + } + if rf.doneCount.Load() != total { + t.Errorf("want %d done, got %d", total, rf.doneCount.Load()) + } + var want []byte + for i := 0; i < total; i++ { + want = append(want, segments[i]...) + } + if !bytes.Equal(readOutput(t, rf), want) { + t.Errorf("restored output mismatch: %q", readOutput(t, rf)) + } +} + +func hexEncode(b []byte) string { + const digits = "0123456789abcdef" + out := make([]byte, 0, len(b)*2) + for _, v := range b { + out = append(out, digits[v>>4], digits[v&0x0f]) + } + return string(out) +} diff --git a/internal/protocol/hls/helper.go b/internal/protocol/hls/helper.go new file mode 100644 index 00000000..4c75220b --- /dev/null +++ b/internal/protocol/hls/helper.go @@ -0,0 +1,99 @@ +package hls + +import ( + "context" + "crypto/sha1" + "crypto/tls" + "encoding/hex" + "fmt" + "io" + "net" + "net/http" + "net/http/cookiejar" + "time" + + "github.com/GopeedLab/gopeed/internal/httpclient" +) + +const defaultUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36" + +const maxPlaylistSize = 10 << 20 // 10MB, playlists are small text files + +func (f *Fetcher) timeout() time.Duration { + return time.Duration(f.config.TimeoutSeconds) * time.Second +} + +// buildClient builds an HTTP client honoring the task proxy and TLS settings. +// The client has no total timeout on purpose: large segments over slow links +// must not be aborted; body stalls are handled by stallReader instead. +func (f *Fetcher) buildClient() *http.Client { + jar, _ := cookiejar.New(nil) + client, err := httpclient.NewClient(httpclient.Options{ + Client: httpclient.ClientOptions{ + Jar: jar, + }, + Transport: httpclient.TransportOptions{ + DialContext: (&net.Dialer{ + Timeout: f.timeout(), + }).DialContext, + Proxy: f.Ctl.GetProxy(f.meta.Req.Proxy), + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: f.meta.Req.SkipVerifyCert, + }, + TLSHandshakeTimeout: f.timeout(), + }, + Impersonation: httpclient.ImpersonationOptions{ + Mode: httpclient.ImpersonationAuto, + Session: f.impSession, + }, + }) + if err != nil { + panic(fmt.Sprintf("build HTTP client: %v", err)) + } + return client +} + +// buildRequest builds a request with task extra headers, the default user +// agent and an optional byte range. +func (f *Fetcher) buildRequest(ctx context.Context, method, u string, body io.Reader, byterange *ByteRange) (*http.Request, error) { + req, err := http.NewRequestWithContext(ctx, method, u, body) + if err != nil { + return nil, err + } + if f.extra != nil { + for key, value := range f.extra.Header { + req.Header.Set(key, value) + } + } + if req.Header.Get("User-Agent") == "" { + req.Header.Set("User-Agent", defaultUserAgent) + } + if byterange != nil { + req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", byterange.Offset, byterange.Offset+byterange.Length-1)) + } + return req, nil +} + +// stallReader aborts reads that see no data for the configured timeout. +type stallReader struct { + rc io.ReadCloser + timeout time.Duration +} + +func (r *stallReader) Read(p []byte) (int, error) { + timer := time.AfterFunc(r.timeout, func() { + r.rc.Close() + }) + defer timer.Stop() + return r.rc.Read(p) +} + +func (r *stallReader) Close() error { + return r.rc.Close() +} + +// sha1Hex returns the hex SHA-1 digest of s. +func sha1Hex(s string) string { + sum := sha1.Sum([]byte(s)) + return hex.EncodeToString(sum[:]) +} diff --git a/internal/protocol/hls/playlist.go b/internal/protocol/hls/playlist.go new file mode 100644 index 00000000..6c6b8d68 --- /dev/null +++ b/internal/protocol/hls/playlist.go @@ -0,0 +1,379 @@ +package hls + +import ( + "encoding/hex" + "errors" + "fmt" + "net/url" + "path" + "strconv" + "strings" +) + +// Variant is one rendition entry of a master playlist. +type Variant struct { + URI string + Bandwidth int64 + AvgBandwidth int64 + Resolution string + Codecs string + Name string +} + +// Key is the decryption key applied to the segments that follow its tag, until +// the next EXT-X-KEY tag or the end of the playlist. +type Key struct { + Method string + URI string // absolute URI + IV []byte // explicit IV; nil means derive from the media sequence number +} + +// ByteRange is an EXT-X-BYTERANGE sub range of the segment resource. +type ByteRange struct { + Length int64 + Offset int64 +} + +// Segment is one downloadable piece of a media playlist. The init section of +// an fMP4 stream is represented as a segment with IsInit set. +type Segment struct { + URI string + Sequence int64 // media sequence number, used to derive the default AES IV + Byterange *ByteRange + Key *Key // nil means unencrypted + IsInit bool + Gap bool + Duration float64 + Size int64 // exact size in bytes when known (byterange / HEAD probe), 0 when unknown +} + +// Media is a parsed media playlist. +type Media struct { + Segments []*Segment + Version int + Live bool // no EXT-X-ENDLIST tag + IsFMP4 bool // playlist declares an init section via EXT-X-MAP +} + +const ( + keyMethodNone = "NONE" + keyMethodAES128 = "AES-128" + keyMethodSampleAES = "SAMPLE-AES" +) + +// IsMasterPlaylist reports whether the content looks like a master playlist. +func IsMasterPlaylist(content string) bool { + return strings.Contains(content, "#EXT-X-STREAM-INF") +} + +func validateM3U8Header(content string) error { + for _, line := range strings.Split(content, "\n") { + line = strings.TrimSpace(line) + if line == "" { + continue + } + if line != "#EXTM3U" { + return errors.New("content is not a valid m3u8 playlist") + } + return nil + } + return errors.New("empty m3u8 playlist") +} + +// ParseMaster parses a master playlist and returns all variants. +func ParseMaster(content string, baseURL *url.URL) ([]*Variant, error) { + if err := validateM3U8Header(content); err != nil { + return nil, err + } + var variants []*Variant + var pending *Variant + for _, line := range strings.Split(content, "\n") { + line = strings.TrimSpace(line) + if line == "" { + continue + } + if strings.HasPrefix(line, "#EXT-X-STREAM-INF:") { + attrs := parseAttributes(line[len("#EXT-X-STREAM-INF:"):]) + pending = &Variant{ + Bandwidth: parseInt64(attrs["BANDWIDTH"]), + AvgBandwidth: parseInt64(attrs["AVERAGE-BANDWIDTH"]), + Resolution: attrs["RESOLUTION"], + Codecs: unquote(attrs["CODECS"]), + Name: unquote(attrs["NAME"]), + } + continue + } + if pending != nil { + if line[0] != '#' { + pending.URI = resolveURL(baseURL, line) + variants = append(variants, pending) + } + pending = nil + } + } + if len(variants) == 0 { + return nil, errors.New("master playlist has no variants") + } + return variants, nil +} + +// PickBestVariant selects the highest-bandwidth variant. +func PickBestVariant(variants []*Variant) *Variant { + var best *Variant + for _, v := range variants { + if best == nil || v.Bandwidth > best.Bandwidth { + best = v + } + } + return best +} + +// ParseMedia parses a media playlist into an ordered segment plan. +func ParseMedia(content string, baseURL *url.URL) (*Media, error) { + if err := validateM3U8Header(content); err != nil { + return nil, err + } + + media := &Media{Live: true} + var currentKey *Key + var pendingByterange *ByteRange + var pendingGap bool + var pendingDuration float64 + var lastRangeEnd int64 + var sequence int64 + var initURI string + + for _, line := range strings.Split(content, "\n") { + line = strings.TrimSpace(line) + if line == "" { + continue + } + if line[0] != '#' { + // Every non-comment line in a media playlist is a segment URI. + seg := &Segment{ + URI: resolveURL(baseURL, line), + Sequence: sequence, + Byterange: pendingByterange, + Key: currentKey, + Gap: pendingGap, + Duration: pendingDuration, + } + if pendingByterange != nil { + // Byterange segments have exact sizes straight from the playlist. + seg.Size = pendingByterange.Length + } + sequence++ + pendingByterange = nil + pendingGap = false + pendingDuration = 0 + media.Segments = append(media.Segments, seg) + continue + } + + switch { + case strings.HasPrefix(line, "#EXTINF:"): + value := line[len("#EXTINF:"):] + if idx := strings.Index(value, ","); idx >= 0 { + value = value[:idx] + } + pendingDuration = parseFloat(value) + case strings.HasPrefix(line, "#EXT-X-KEY:"): + key, err := parseKey(line[len("#EXT-X-KEY:"):], baseURL) + if err != nil { + return nil, err + } + currentKey = key + case strings.HasPrefix(line, "#EXT-X-MAP:"): + attrs := parseAttributes(line[len("#EXT-X-MAP:"):]) + uriAttr := unquote(attrs["URI"]) + if uriAttr == "" { + return nil, errors.New("EXT-X-MAP without URI is not supported") + } + resolved := resolveURL(baseURL, uriAttr) + if initURI != "" && initURI != resolved { + return nil, errors.New("multi-period fMP4 streams are not supported") + } + initURI = resolved + if !media.IsFMP4 { + // The init section is prepended to the segment plan, it takes + // the media sequence number of the playlist for IV derivation. + init := &Segment{ + URI: resolved, + Sequence: sequence, + IsInit: true, + Key: currentKey, + } + if attrs["BYTERANGE"] != "" { + init.Byterange = parseByteRange(attrs["BYTERANGE"], 0) + if init.Byterange == nil { + return nil, fmt.Errorf("invalid EXT-X-MAP BYTERANGE: %s", attrs["BYTERANGE"]) + } + } + media.Segments = append(media.Segments, init) + } + media.IsFMP4 = true + case strings.HasPrefix(line, "#EXT-X-BYTERANGE:"): + pendingByterange = parseByteRange(line[len("#EXT-X-BYTERANGE:"):], lastRangeEnd) + if pendingByterange == nil { + return nil, fmt.Errorf("invalid EXT-X-BYTERANGE: %s", line) + } + case line == "#EXT-X-GAP": + pendingGap = true + case line == "#EXT-X-ENDLIST": + media.Live = false + case strings.HasPrefix(line, "#EXT-X-MEDIA-SEQUENCE:"): + sequence = parseInt64(line[len("#EXT-X-MEDIA-SEQUENCE:"):]) + case strings.HasPrefix(line, "#EXT-X-VERSION:"): + media.Version = int(parseInt64(line[len("#EXT-X-VERSION:"):])) + } + + if pendingByterange != nil { + // Non-segment tags between BYTERANGE and its URI must not reset the + // running offset, so it is tracked here after each parse. + lastRangeEnd = pendingByterange.Offset + pendingByterange.Length + } + } + + if len(media.Segments) == 0 { + return nil, errors.New("media playlist has no segments") + } + return media, nil +} + +func parseKey(attrs string, baseURL *url.URL) (*Key, error) { + parsed := parseAttributes(attrs) + switch method := parsed["METHOD"]; method { + case keyMethodNone: + return nil, nil + case keyMethodAES128: + uri := unquote(parsed["URI"]) + if uri == "" { + return nil, errors.New("AES-128 EXT-X-KEY without URI") + } + key := &Key{Method: keyMethodAES128, URI: resolveURL(baseURL, uri)} + if iv, ok := parsed["IV"]; ok { + value := strings.TrimPrefix(strings.ToLower(iv), "0x") + decoded, err := hex.DecodeString(value) + if err != nil || len(decoded) != 16 { + return nil, fmt.Errorf("invalid EXT-X-KEY IV: %s", iv) + } + key.IV = decoded + } + return key, nil + case keyMethodSampleAES: + return nil, errors.New("SAMPLE-AES encryption is not supported") + default: + return nil, fmt.Errorf("unsupported encryption method: %s", method) + } +} + +func parseByteRange(value string, defaultOffset int64) *ByteRange { + lengthPart := value + offsetPart := "" + if idx := strings.Index(value, "@"); idx >= 0 { + lengthPart = value[:idx] + offsetPart = value[idx+1:] + } + length := parseInt64(lengthPart) + if length <= 0 { + return nil + } + offset := defaultOffset + if offsetPart != "" { + offset = parseInt64(offsetPart) + if offset < 0 { + return nil + } + } + return &ByteRange{Length: length, Offset: offset} +} + +// resolveURL resolves a playlist URI reference against the playlist URL +// following RFC 3986. +func resolveURL(baseURL *url.URL, ref string) string { + refURL, err := url.Parse(ref) + if err != nil { + return ref + } + return baseURL.ResolveReference(refURL).String() +} + +// parseAttributes parses an HLS attribute list like FOO="bar",NUM=42. +func parseAttributes(s string) map[string]string { + attrs := make(map[string]string) + var key, val strings.Builder + inKey := true + inQuote := false + flush := func() { + name := strings.ToUpper(strings.TrimSpace(key.String())) + if name != "" { + attrs[name] = strings.TrimSpace(val.String()) + } + key.Reset() + val.Reset() + inKey = true + } + for i := 0; i < len(s); i++ { + c := s[i] + if inKey { + if c == '=' { + inKey = false + } else { + key.WriteByte(c) + } + continue + } + if c == '"' { + inQuote = !inQuote + continue + } + if c == ',' && !inQuote { + flush() + continue + } + val.WriteByte(c) + } + flush() + return attrs +} + +func unquote(s string) string { + s = strings.TrimSpace(s) + if len(s) >= 2 && strings.HasPrefix(s, "\"") && strings.HasSuffix(s, "\"") { + return s[1 : len(s)-1] + } + return s +} + +func parseInt64(s string) int64 { + value, _ := strconv.ParseInt(strings.TrimSpace(s), 10, 64) + return value +} + +func parseFloat(s string) float64 { + value, _ := strconv.ParseFloat(strings.TrimSpace(s), 64) + return value +} + +// OutputExtension returns the container file extension for the playlist. +func (m *Media) OutputExtension() string { + if m.IsFMP4 { + return ".mp4" + } + return ".ts" +} + +// DeriveOutputName builds the merged output file name from a playlist URL. +func DeriveOutputName(playlistURL string, media *Media) string { + name := "" + if u, err := url.Parse(playlistURL); err == nil { + name = path.Base(u.Path) + name = strings.TrimSuffix(name, path.Ext(name)) + } + if name == "" || name == "/" || name == "." { + if u, err := url.Parse(playlistURL); err == nil { + name = u.Hostname() + } + } + return name + media.OutputExtension() +} diff --git a/internal/protocol/hls/playlist_realworld_test.go b/internal/protocol/hls/playlist_realworld_test.go new file mode 100644 index 00000000..6a9de3c8 --- /dev/null +++ b/internal/protocol/hls/playlist_realworld_test.go @@ -0,0 +1,39 @@ +package hls + +import ( + _ "embed" + "net/url" + "testing" +) + +//go:embed testdata/olemovienews.m3u8 +var realWorldPlaylist string + +// TestParseMediaRealWorld guards parsing against a real site playlist +// (olemovienews.com VOD, 138 TS segments). +func TestParseMediaRealWorld(t *testing.T) { + base, err := url.Parse("https://europe.olemovienews.com/ts2/20230125/afAxfjit/mp4/afAxfjit.mp4/index-v1-a1.m3u8") + if err != nil { + t.Fatal(err) + } + media, err := ParseMedia(realWorldPlaylist, base) + if err != nil { + t.Fatal(err) + } + if media.Live { + t.Error("real VOD playlist should not be live") + } + if media.IsFMP4 { + t.Error("playlist should be detected as TS") + } + if len(media.Segments) != 138 { + t.Errorf("want 138 segments, got %d", len(media.Segments)) + } + first := media.Segments[0] + if first.URI != "https://europe.olemovienews.com/ts2/20230125/afAxfjit/mp4/afAxfjit.mp4/seg-1-v1-a1.ts" { + t.Errorf("segment URI resolution wrong: %s", first.URI) + } + if media.Segments[0].Sequence != 1 { + t.Errorf("first sequence = %d, want 1", media.Segments[0].Sequence) + } +} diff --git a/internal/protocol/hls/playlist_test.go b/internal/protocol/hls/playlist_test.go new file mode 100644 index 00000000..fb219aa4 --- /dev/null +++ b/internal/protocol/hls/playlist_test.go @@ -0,0 +1,252 @@ +package hls + +import ( + "net/url" + "testing" +) + +func mustURL(t *testing.T, raw string) *url.URL { + t.Helper() + u, err := url.Parse(raw) + if err != nil { + t.Fatal(err) + } + return u +} + +func TestParseMediaTS(t *testing.T) { + content := "#EXTM3U\n" + + "#EXT-X-TARGETDURATION:20\n" + + "#EXT-X-VERSION:3\n" + + "#EXT-X-MEDIA-SEQUENCE:1\n" + + "#EXTINF:20.000,\n" + + "seg-1.ts\n" + + "#EXTINF:20.000,\n" + + "seg-2.ts\n" + + "#EXT-X-ENDLIST\n" + media, err := ParseMedia(content, mustURL(t, "https://cdn.example.com/vod/index.m3u8")) + if err != nil { + t.Fatal(err) + } + if media.Live { + t.Error("VOD playlist should not be live") + } + if media.IsFMP4 { + t.Error("TS playlist should not be fMP4") + } + if len(media.Segments) != 2 { + t.Fatalf("want 2 segments, got %d", len(media.Segments)) + } + want := "https://cdn.example.com/vod/seg-1.ts" + if media.Segments[0].URI != want { + t.Errorf("want uri %s, got %s", want, media.Segments[0].URI) + } + if media.Segments[0].Sequence != 1 { + t.Errorf("want sequence 1, got %d", media.Segments[0].Sequence) + } + if media.Segments[0].Key != nil { + t.Error("segment should be unencrypted") + } +} + +func TestParseMediaWithoutExtinf(t *testing.T) { + // Some playlists list segment URIs without EXTINF lines. + content := "#EXTM3U\nseg-1.ts\nseg-2.ts\n#EXT-X-ENDLIST\n" + media, err := ParseMedia(content, mustURL(t, "https://cdn.example.com/a/index.m3u8")) + if err != nil { + t.Fatal(err) + } + if len(media.Segments) != 2 { + t.Fatalf("want 2 segments, got %d", len(media.Segments)) + } +} + +func TestParseMediaLive(t *testing.T) { + content := "#EXTM3U\n#EXT-X-TARGETDURATION:6\n#EXTINF:6.0,\nseg-1.ts\n" + media, err := ParseMedia(content, mustURL(t, "https://cdn.example.com/live/index.m3u8")) + if err != nil { + t.Fatal(err) + } + if !media.Live { + t.Error("playlist without ENDLIST should be live") + } +} + +func TestParseMediaFMP4(t *testing.T) { + content := "#EXTM3U\n" + + "#EXT-X-VERSION:6\n" + + "#EXT-X-MAP:URI=\"init.mp4\"\n" + + "#EXTINF:5.0,\n" + + "seg-1.m4s\n" + + "#EXT-X-ENDLIST\n" + media, err := ParseMedia(content, mustURL(t, "https://cdn.example.com/vod/index.m3u8")) + if err != nil { + t.Fatal(err) + } + if !media.IsFMP4 { + t.Error("playlist with MAP should be fMP4") + } + if len(media.Segments) != 2 { + t.Fatalf("want 2 segments (init + 1), got %d", len(media.Segments)) + } + if !media.Segments[0].IsInit { + t.Error("first segment should be the init section") + } + if media.Segments[0].URI != "https://cdn.example.com/vod/init.mp4" { + t.Errorf("init uri wrong: %s", media.Segments[0].URI) + } + if media.OutputExtension() != ".mp4" { + t.Errorf("want .mp4, got %s", media.OutputExtension()) + } +} + +func TestParseMediaMultiMapError(t *testing.T) { + content := "#EXTM3U\n" + + "#EXT-X-MAP:URI=\"init1.mp4\"\n" + + "#EXTINF:5.0,\nseg-1.m4s\n" + + "#EXT-X-DISCONTINUITY\n" + + "#EXT-X-MAP:URI=\"init2.mp4\"\n" + + "#EXTINF:5.0,\nseg-2.m4s\n" + + "#EXT-X-ENDLIST\n" + if _, err := ParseMedia(content, mustURL(t, "https://cdn.example.com/vod/index.m3u8")); err == nil { + t.Error("multi-period fMP4 should be rejected") + } +} + +func TestParseMediaKeyRotation(t *testing.T) { + content := "#EXTM3U\n" + + "#EXT-X-KEY:METHOD=AES-128,URI=\"key1.bin\",IV=0x00000000000000000000000000000001\n" + + "#EXTINF:5.0,\nseg-1.ts\n" + + "#EXT-X-KEY:METHOD=AES-128,URI=\"key2.bin\"\n" + + "#EXTINF:5.0,\nseg-2.ts\n" + + "#EXT-X-KEY:METHOD=NONE\n" + + "#EXTINF:5.0,\nseg-3.ts\n" + + "#EXT-X-ENDLIST\n" + media, err := ParseMedia(content, mustURL(t, "https://cdn.example.com/vod/index.m3u8")) + if err != nil { + t.Fatal(err) + } + if media.Segments[0].Key == nil || media.Segments[0].Key.URI != "https://cdn.example.com/vod/key1.bin" { + t.Fatalf("segment 1 key wrong: %+v", media.Segments[0].Key) + } + if len(media.Segments[0].Key.IV) != 16 || media.Segments[0].Key.IV[15] != 1 { + t.Error("explicit IV not parsed") + } + if media.Segments[1].Key == nil || media.Segments[1].Key.URI != "https://cdn.example.com/vod/key2.bin" { + t.Fatalf("segment 2 key wrong: %+v", media.Segments[1].Key) + } + if media.Segments[1].Key.IV != nil { + t.Error("key without IV should leave IV nil (sequence-derived)") + } + if media.Segments[2].Key != nil { + t.Error("segment 3 should be unencrypted after METHOD=NONE") + } +} + +func TestParseMediaSampleAESError(t *testing.T) { + content := "#EXTM3U\n#EXT-X-KEY:METHOD=SAMPLE-AES,URI=\"skd://key\"\n#EXTINF:5.0,\nseg-1.ts\n" + if _, err := ParseMedia(content, mustURL(t, "https://cdn.example.com/vod/index.m3u8")); err == nil { + t.Error("SAMPLE-AES should be rejected") + } +} + +func TestParseMediaByterange(t *testing.T) { + content := "#EXTM3U\n" + + "#EXT-X-BYTERANGE:1000@0\nseg.ts\n" + + "#EXT-X-BYTERANGE:2000\nseg.ts\n" + + "#EXT-X-BYTERANGE:3000\nseg.ts\n" + + "#EXT-X-ENDLIST\n" + media, err := ParseMedia(content, mustURL(t, "https://cdn.example.com/vod/index.m3u8")) + if err != nil { + t.Fatal(err) + } + ranges := [][2]int64{{0, 1000}, {1000, 2000}, {3000, 3000}} + for i, want := range ranges { + br := media.Segments[i].Byterange + if br == nil { + t.Fatalf("segment %d byterange missing", i) + } + if br.Offset != want[0] || br.Length != want[1] { + t.Errorf("segment %d: want offset %d length %d, got %d/%d", i, want[0], want[1], br.Offset, br.Length) + } + } +} + +func TestParseMediaGap(t *testing.T) { + content := "#EXTM3U\n#EXTINF:5.0,\nseg-1.ts\n#EXT-X-GAP\n#EXTINF:5.0,\nseg-2.ts\n#EXT-X-ENDLIST\n" + media, err := ParseMedia(content, mustURL(t, "https://cdn.example.com/vod/index.m3u8")) + if err != nil { + t.Fatal(err) + } + if media.Segments[0].Gap || !media.Segments[1].Gap { + t.Error("gap flags wrong") + } +} + +func TestParseMediaInvalidHeader(t *testing.T) { + if _, err := ParseMedia("not a playlist", mustURL(t, "https://cdn.example.com/a.m3u8")); err == nil { + t.Error("invalid content should be rejected") + } +} + +func TestParseMasterPickBest(t *testing.T) { + content := "#EXTM3U\n" + + "#EXT-X-MEDIA:TYPE=AUDIO,GROUP-ID=\"aud\",NAME=\"en\",DEFAULT=YES,URI=\"audio/index.m3u8\"\n" + + "#EXT-X-STREAM-INF:BANDWIDTH=800000,RESOLUTION=640x360\n" + + "360p/index.m3u8\n" + + "#EXT-X-STREAM-INF:BANDWIDTH=2500000,RESOLUTION=1920x1080,CODECS=\"avc1.640028,mp4a.40.2\"\n" + + "1080p/index.m3u8?sign=abc\n" + + "#EXT-X-ENDLIST\n" + variants, err := ParseMaster(content, mustURL(t, "https://cdn.example.com/vod/master.m3u8")) + if err != nil { + t.Fatal(err) + } + if len(variants) != 2 { + t.Fatalf("want 2 variants, got %d", len(variants)) + } + best := PickBestVariant(variants) + if best.Bandwidth != 2500000 { + t.Errorf("want highest bandwidth variant, got %d", best.Bandwidth) + } + if best.URI != "https://cdn.example.com/vod/1080p/index.m3u8?sign=abc" { + t.Errorf("variant uri wrong: %s", best.URI) + } +} + +func TestResolveURLQueryNotInherited(t *testing.T) { + base := mustURL(t, "https://cdn.example.com/vod/index.m3u8?sign=abc") + got := resolveURL(base, "seg-1.ts") + if got != "https://cdn.example.com/vod/seg-1.ts" { + t.Errorf("relative resolution should follow RFC3986 (no query inheritance), got %s", got) + } +} + +func TestDeriveOutputName(t *testing.T) { + media := &Media{} + if got := DeriveOutputName("https://cdn.example.com/ts2/a/index-v1-a1.m3u8", media); got != "index-v1-a1.ts" { + t.Errorf("want index-v1-a1.ts, got %s", got) + } + fmp4 := &Media{IsFMP4: true} + if got := DeriveOutputName("https://cdn.example.com/ts2/a/index.m3u8", fmp4); got != "index.mp4" { + t.Errorf("want index.mp4, got %s", got) + } + if got := DeriveOutputName("https://cdn.example.com/play?token=x", media); got != "play.ts" { + t.Errorf("want play.ts, got %s", got) + } + if got := DeriveOutputName("https://cdn.example.com/", media); got != "cdn.example.com.ts" { + t.Errorf("want host fallback, got %s", got) + } +} + +func TestParseAttributesQuotedComma(t *testing.T) { + attrs := parseAttributes(`URI="a,b.ts",IV=0xABCDEF00000000000000000000000001,NAME="x"`) + if attrs["URI"] != "a,b.ts" { + t.Errorf("quoted comma value wrong: %q", attrs["URI"]) + } + if attrs["IV"] != "0xABCDEF00000000000000000000000001" { + t.Errorf("hex value wrong: %q", attrs["IV"]) + } + if attrs["NAME"] != "x" { + t.Errorf("quoted value wrong: %q", attrs["NAME"]) + } +} diff --git a/internal/protocol/hls/testdata/olemovienews.m3u8 b/internal/protocol/hls/testdata/olemovienews.m3u8 new file mode 100644 index 00000000..6c908400 --- /dev/null +++ b/internal/protocol/hls/testdata/olemovienews.m3u8 @@ -0,0 +1,283 @@ +#EXTM3U +#EXT-X-TARGETDURATION:20 +#EXT-X-ALLOW-CACHE:YES +#EXT-X-PLAYLIST-TYPE:VOD +#EXT-X-VERSION:3 +#EXT-X-MEDIA-SEQUENCE:1 +#EXTINF:20.000, +seg-1-v1-a1.ts +#EXTINF:20.000, +seg-2-v1-a1.ts +#EXTINF:20.000, +seg-3-v1-a1.ts +#EXTINF:20.000, +seg-4-v1-a1.ts +#EXTINF:20.000, +seg-5-v1-a1.ts +#EXTINF:20.000, +seg-6-v1-a1.ts +#EXTINF:20.000, +seg-7-v1-a1.ts +#EXTINF:20.000, +seg-8-v1-a1.ts +#EXTINF:20.000, +seg-9-v1-a1.ts +#EXTINF:20.000, +seg-10-v1-a1.ts +#EXTINF:20.000, +seg-11-v1-a1.ts +#EXTINF:20.000, +seg-12-v1-a1.ts +#EXTINF:20.000, +seg-13-v1-a1.ts +#EXTINF:20.000, +seg-14-v1-a1.ts +#EXTINF:20.000, +seg-15-v1-a1.ts +#EXTINF:20.000, +seg-16-v1-a1.ts +#EXTINF:20.000, +seg-17-v1-a1.ts +#EXTINF:20.000, +seg-18-v1-a1.ts +#EXTINF:20.000, +seg-19-v1-a1.ts +#EXTINF:20.000, +seg-20-v1-a1.ts +#EXTINF:20.000, +seg-21-v1-a1.ts +#EXTINF:20.000, +seg-22-v1-a1.ts +#EXTINF:20.000, +seg-23-v1-a1.ts +#EXTINF:20.000, +seg-24-v1-a1.ts +#EXTINF:20.000, +seg-25-v1-a1.ts +#EXTINF:20.000, +seg-26-v1-a1.ts +#EXTINF:20.000, +seg-27-v1-a1.ts +#EXTINF:20.000, +seg-28-v1-a1.ts +#EXTINF:20.000, +seg-29-v1-a1.ts +#EXTINF:20.000, +seg-30-v1-a1.ts +#EXTINF:20.000, +seg-31-v1-a1.ts +#EXTINF:20.000, +seg-32-v1-a1.ts +#EXTINF:20.000, +seg-33-v1-a1.ts +#EXTINF:20.000, +seg-34-v1-a1.ts +#EXTINF:20.000, +seg-35-v1-a1.ts +#EXTINF:20.000, +seg-36-v1-a1.ts +#EXTINF:20.000, +seg-37-v1-a1.ts +#EXTINF:20.000, +seg-38-v1-a1.ts +#EXTINF:20.000, +seg-39-v1-a1.ts +#EXTINF:20.000, +seg-40-v1-a1.ts +#EXTINF:20.000, +seg-41-v1-a1.ts +#EXTINF:20.000, +seg-42-v1-a1.ts +#EXTINF:20.000, +seg-43-v1-a1.ts +#EXTINF:20.000, +seg-44-v1-a1.ts +#EXTINF:20.000, +seg-45-v1-a1.ts +#EXTINF:20.000, +seg-46-v1-a1.ts +#EXTINF:20.000, +seg-47-v1-a1.ts +#EXTINF:20.000, +seg-48-v1-a1.ts +#EXTINF:20.000, +seg-49-v1-a1.ts +#EXTINF:20.000, +seg-50-v1-a1.ts +#EXTINF:20.000, +seg-51-v1-a1.ts +#EXTINF:20.000, +seg-52-v1-a1.ts +#EXTINF:20.000, +seg-53-v1-a1.ts +#EXTINF:20.000, +seg-54-v1-a1.ts +#EXTINF:20.000, +seg-55-v1-a1.ts +#EXTINF:20.000, +seg-56-v1-a1.ts +#EXTINF:20.000, +seg-57-v1-a1.ts +#EXTINF:20.000, +seg-58-v1-a1.ts +#EXTINF:20.000, +seg-59-v1-a1.ts +#EXTINF:20.000, +seg-60-v1-a1.ts +#EXTINF:20.000, +seg-61-v1-a1.ts +#EXTINF:20.000, +seg-62-v1-a1.ts +#EXTINF:20.000, +seg-63-v1-a1.ts +#EXTINF:20.000, +seg-64-v1-a1.ts +#EXTINF:20.000, +seg-65-v1-a1.ts +#EXTINF:20.000, +seg-66-v1-a1.ts +#EXTINF:20.000, +seg-67-v1-a1.ts +#EXTINF:20.000, +seg-68-v1-a1.ts +#EXTINF:20.000, +seg-69-v1-a1.ts +#EXTINF:20.000, +seg-70-v1-a1.ts +#EXTINF:20.000, +seg-71-v1-a1.ts +#EXTINF:20.000, +seg-72-v1-a1.ts +#EXTINF:20.000, +seg-73-v1-a1.ts +#EXTINF:20.000, +seg-74-v1-a1.ts +#EXTINF:20.000, +seg-75-v1-a1.ts +#EXTINF:20.000, +seg-76-v1-a1.ts +#EXTINF:20.000, +seg-77-v1-a1.ts +#EXTINF:20.000, +seg-78-v1-a1.ts +#EXTINF:20.000, +seg-79-v1-a1.ts +#EXTINF:20.000, +seg-80-v1-a1.ts +#EXTINF:20.000, +seg-81-v1-a1.ts +#EXTINF:20.000, +seg-82-v1-a1.ts +#EXTINF:20.000, +seg-83-v1-a1.ts +#EXTINF:20.000, +seg-84-v1-a1.ts +#EXTINF:20.000, +seg-85-v1-a1.ts +#EXTINF:20.000, +seg-86-v1-a1.ts +#EXTINF:20.000, +seg-87-v1-a1.ts +#EXTINF:20.000, +seg-88-v1-a1.ts +#EXTINF:20.000, +seg-89-v1-a1.ts +#EXTINF:20.000, +seg-90-v1-a1.ts +#EXTINF:20.000, +seg-91-v1-a1.ts +#EXTINF:20.000, +seg-92-v1-a1.ts +#EXTINF:20.000, +seg-93-v1-a1.ts +#EXTINF:20.000, +seg-94-v1-a1.ts +#EXTINF:20.000, +seg-95-v1-a1.ts +#EXTINF:20.000, +seg-96-v1-a1.ts +#EXTINF:20.000, +seg-97-v1-a1.ts +#EXTINF:20.000, +seg-98-v1-a1.ts +#EXTINF:20.000, +seg-99-v1-a1.ts +#EXTINF:20.000, +seg-100-v1-a1.ts +#EXTINF:20.000, +seg-101-v1-a1.ts +#EXTINF:20.000, +seg-102-v1-a1.ts +#EXTINF:20.000, +seg-103-v1-a1.ts +#EXTINF:20.000, +seg-104-v1-a1.ts +#EXTINF:20.000, +seg-105-v1-a1.ts +#EXTINF:20.000, +seg-106-v1-a1.ts +#EXTINF:20.000, +seg-107-v1-a1.ts +#EXTINF:20.000, +seg-108-v1-a1.ts +#EXTINF:20.000, +seg-109-v1-a1.ts +#EXTINF:20.000, +seg-110-v1-a1.ts +#EXTINF:20.000, +seg-111-v1-a1.ts +#EXTINF:20.000, +seg-112-v1-a1.ts +#EXTINF:20.000, +seg-113-v1-a1.ts +#EXTINF:20.000, +seg-114-v1-a1.ts +#EXTINF:20.000, +seg-115-v1-a1.ts +#EXTINF:20.000, +seg-116-v1-a1.ts +#EXTINF:20.000, +seg-117-v1-a1.ts +#EXTINF:20.000, +seg-118-v1-a1.ts +#EXTINF:20.000, +seg-119-v1-a1.ts +#EXTINF:20.000, +seg-120-v1-a1.ts +#EXTINF:20.000, +seg-121-v1-a1.ts +#EXTINF:20.000, +seg-122-v1-a1.ts +#EXTINF:20.000, +seg-123-v1-a1.ts +#EXTINF:20.000, +seg-124-v1-a1.ts +#EXTINF:20.000, +seg-125-v1-a1.ts +#EXTINF:20.000, +seg-126-v1-a1.ts +#EXTINF:20.000, +seg-127-v1-a1.ts +#EXTINF:20.000, +seg-128-v1-a1.ts +#EXTINF:20.000, +seg-129-v1-a1.ts +#EXTINF:20.000, +seg-130-v1-a1.ts +#EXTINF:20.000, +seg-131-v1-a1.ts +#EXTINF:20.000, +seg-132-v1-a1.ts +#EXTINF:20.000, +seg-133-v1-a1.ts +#EXTINF:20.000, +seg-134-v1-a1.ts +#EXTINF:20.000, +seg-135-v1-a1.ts +#EXTINF:20.000, +seg-136-v1-a1.ts +#EXTINF:20.000, +seg-137-v1-a1.ts +#EXTINF:4.111, +seg-138-v1-a1.ts +#EXT-X-ENDLIST diff --git a/pkg/download/hls_live_test.go b/pkg/download/hls_live_test.go new file mode 100644 index 00000000..7fb41054 --- /dev/null +++ b/pkg/download/hls_live_test.go @@ -0,0 +1,97 @@ +package download + +import ( + "fmt" + "os" + "path/filepath" + "testing" + "time" + + "github.com/GopeedLab/gopeed/pkg/base" +) + +// TestDownloader_HlsLiveSite is a manual acceptance gate against a real site. +// It is skipped unless HLS_LIVE_URL is set, e.g.: +// +// HLS_LIVE_URL="https://host/path/index.m3u8" go test ./pkg/download/ -run TestDownloader_HlsLiveSite -v +func TestDownloader_HlsLiveSite(t *testing.T) { + playlistURL := os.Getenv("HLS_LIVE_URL") + if playlistURL == "" { + t.Skip("set HLS_LIVE_URL to run the live-site acceptance test") + } + referer := os.Getenv("HLS_LIVE_REFERER") + + downloader := NewDownloader(&DownloaderConfig{Storage: NewMemStorage(), StorageDir: t.TempDir()}) + if err := downloader.Setup(); err != nil { + t.Fatal(err) + } + defer downloader.Clear() + downloadDir := t.TempDir() + cfg, err := downloader.GetConfig() + if err != nil { + t.Fatal(err) + } + cfg.DownloadDir = downloadDir + if err := downloader.PutConfig(cfg); err != nil { + t.Fatal(err) + } + + req := &base.Request{URL: playlistURL} + if referer != "" { + req.Extra = map[string]any{"header": map[string]string{"Referer": referer}} + } + rr, err := downloader.Resolve(req, nil) + if err != nil { + t.Fatalf("resolve: %v", err) + } + name := rr.Res.Files[0].Name + size := rr.Res.Size + t.Logf("resolved: name=%s size=%d", name, size) + + taskID, err := downloader.Create(rr.ID) + if err != nil { + t.Fatal(err) + } + deadline := time.Now().Add(30 * time.Minute) + var lastLogged int64 + for { + task := downloader.GetTask(taskID) + if task == nil { + t.Fatal("task not found") + } + if task.Status == base.DownloadStatusDone { + break + } + if task.Status == base.DownloadStatusError { + t.Fatalf("task entered error state after %d bytes", task.Progress.Downloaded) + } + if time.Now().After(deadline) { + t.Fatalf("timeout, downloaded %d bytes", task.Progress.Downloaded) + } + if tick := task.Progress.Downloaded; tick-lastLogged > 10<<20 { + lastLogged = tick + t.Logf("progress: %d bytes", tick) + } + time.Sleep(500 * time.Millisecond) + } + + data, err := os.ReadFile(filepath.Join(downloadDir, name)) + if err != nil { + t.Fatal(err) + } + if int64(len(data)) == 0 { + t.Fatal("merged file is empty") + } + t.Logf("merged: %s (%d bytes)", name, len(data)) + if filepath.Ext(name) == ".ts" { + // MPEG-TS packets are 188 bytes and start with the 0x47 sync byte. + for i := 0; i < 188*4 && i < len(data); i += 188 { + if data[i] != 0x47 { + t.Fatalf("invalid TS sync byte at offset %d: %#x", i, data[i]) + } + } + } else if string(data[4:8]) != "ftyp" { + t.Fatalf("invalid fMP4 header: %q", data[:8]) + } + fmt.Printf("LIVE ACCEPTANCE OK: %s (%d bytes)\n", filepath.Join(downloadDir, name), len(data)) +} diff --git a/pkg/download/hls_protocol_test.go b/pkg/download/hls_protocol_test.go new file mode 100644 index 00000000..8ef48df8 --- /dev/null +++ b/pkg/download/hls_protocol_test.go @@ -0,0 +1,172 @@ +package download + +import ( + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/GopeedLab/gopeed/pkg/base" +) + +// hlsTestCdn spins up a master-playlist HLS site whose segments require a +// Referer header (anti-hotlink) and returns the server plus expected bytes. +func hlsTestCdn(t *testing.T, segmentDelay time.Duration) (*httptest.Server, []byte) { + t.Helper() + const total = 6 + segments := make(map[int][]byte) + for i := 0; i < total; i++ { + segments[i] = []byte(fmt.Sprintf("ENGINE-SEGMENT-%02d", i)) + } + var want []byte + for i := 0; i < total; i++ { + want = append(want, segments[i]...) + } + + mux := http.NewServeMux() + mux.HandleFunc("/master.m3u8", func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte("#EXTM3U\n" + + "#EXT-X-STREAM-INF:BANDWIDTH=500000\nlow/index.m3u8\n" + + "#EXT-X-STREAM-INF:BANDWIDTH=2000000\nhigh/index.m3u8\n")) + }) + mux.HandleFunc("/low/index.m3u8", func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte("#EXTM3U\n#EXTINF:4,\nseg-0.ts\n#EXT-X-ENDLIST\n")) + }) + mux.HandleFunc("/high/index.m3u8", func(w http.ResponseWriter, r *http.Request) { + var b strings.Builder + b.WriteString("#EXTM3U\n") + for i := 0; i < total; i++ { + b.WriteString(fmt.Sprintf("#EXTINF:4,\nseg-%d.ts\n", i)) + } + b.WriteString("#EXT-X-ENDLIST\n") + w.Write([]byte(b.String())) + }) + for i := 0; i < total; i++ { + idx := i + mux.HandleFunc(fmt.Sprintf("/high/seg-%d.ts", i), func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Referer") == "" { + w.WriteHeader(http.StatusForbidden) + return + } + if segmentDelay > 0 { + time.Sleep(segmentDelay) + } + w.Write(segments[idx]) + }) + } + server := httptest.NewServer(mux) + t.Cleanup(server.Close) + return server, want +} + +func hlsResolveReferer() *base.Request { + return &base.Request{ + URL: "", + Extra: map[string]any{"header": map[string]string{"Referer": "https://player.example.com/page"}}, + } +} + +func TestDownloader_HlsProtocolEndToEnd(t *testing.T) { + server, want := hlsTestCdn(t, 0) + downloader := NewDownloader(&DownloaderConfig{Storage: NewMemStorage(), StorageDir: t.TempDir()}) + if err := downloader.Setup(); err != nil { + t.Fatal(err) + } + defer downloader.Clear() + downloadDir := t.TempDir() + cfg, err := downloader.GetConfig() + if err != nil { + t.Fatal(err) + } + cfg.DownloadDir = downloadDir + if err := downloader.PutConfig(cfg); err != nil { + t.Fatal(err) + } + + req := hlsResolveReferer() + req.URL = server.URL + "/master.m3u8" + rr, err := downloader.Resolve(req, nil) + if err != nil { + t.Fatal(err) + } + if got := rr.Res.Files[0].Name; got != "index.ts" { + t.Errorf("resolved name = %s, want index.ts", got) + } + taskID, err := downloader.Create(rr.ID) + if err != nil { + t.Fatal(err) + } + waitForTaskStatus(t, downloader, taskID, base.DownloadStatusDone, 10*time.Second) + + task := downloader.GetTask(taskID) + if task.Protocol != "hls" { + t.Errorf("task protocol = %s, want hls", task.Protocol) + } + data, err := os.ReadFile(filepath.Join(downloadDir, "index.ts")) + if err != nil { + t.Fatal(err) + } + if string(data) != string(want) { + t.Errorf("merged file mismatch: %q", data) + } +} + +func TestDownloader_HlsProtocolPauseContinue(t *testing.T) { + server, want := hlsTestCdn(t, 120*time.Millisecond) + downloader := NewDownloader(&DownloaderConfig{Storage: NewMemStorage(), StorageDir: t.TempDir()}) + if err := downloader.Setup(); err != nil { + t.Fatal(err) + } + defer downloader.Clear() + downloadDir := t.TempDir() + cfg, err := downloader.GetConfig() + if err != nil { + t.Fatal(err) + } + cfg.DownloadDir = downloadDir + // Throttle the protocol so the pause lands mid-download. + cfg.ProtocolConfig["hls"] = map[string]any{ + "segmentConnections": 2, + "maxRetries": 3, + "timeoutSeconds": 5, + "prefetchContentLength": false, + } + if err := downloader.PutConfig(cfg); err != nil { + t.Fatal(err) + } + + req := hlsResolveReferer() + req.URL = server.URL + "/master.m3u8" + req.Extra = map[string]any{"header": map[string]string{"Referer": "https://player.example.com/page"}} + // CreateDirect: engine resolves lazily on start, like a direct task. + taskID, err := downloader.CreateDirect(req, &base.Options{Name: "movie.ts"}) + if err != nil { + t.Fatal(err) + } + waitForTaskStatus(t, downloader, taskID, base.DownloadStatusRunning, 10*time.Second) + if err := downloader.Pause(&TaskFilter{IDs: []string{taskID}}); err != nil { + t.Fatal(err) + } + waitForTaskStatus(t, downloader, taskID, base.DownloadStatusPause, 5*time.Second) + + if err := downloader.Continue(&TaskFilter{IDs: []string{taskID}}); err != nil { + t.Fatal(err) + } + waitForTaskStatus(t, downloader, taskID, base.DownloadStatusDone, 30*time.Second) + + task := downloader.GetTask(taskID) + if task.Meta.Res.Size != int64(len(want)) { + t.Errorf("final size = %d, want %d", task.Meta.Res.Size, len(want)) + } + data, err := os.ReadFile(filepath.Join(downloadDir, "movie.ts")) + if err != nil { + t.Fatal(err) + } + if string(data) != string(want) { + t.Errorf("merged file mismatch after pause/continue: %q", data) + } +} diff --git a/pkg/download/model.go b/pkg/download/model.go index 0705827b..1a3b7ae8 100644 --- a/pkg/download/model.go +++ b/pkg/download/model.go @@ -9,6 +9,7 @@ import ( "github.com/GopeedLab/gopeed/internal/fetcher" "github.com/GopeedLab/gopeed/internal/protocol/bt" "github.com/GopeedLab/gopeed/internal/protocol/ed2k" + "github.com/GopeedLab/gopeed/internal/protocol/hls" "github.com/GopeedLab/gopeed/internal/protocol/http" "github.com/GopeedLab/gopeed/pkg/base" enginewebview "github.com/GopeedLab/gopeed/pkg/download/engine/webview" @@ -197,6 +198,7 @@ func (cfg *DownloaderConfig) Init() *DownloaderConfig { } if len(cfg.FetchManagers) == 0 { cfg.FetchManagers = []fetcher.FetcherManager{ + new(hls.FetcherManager), new(http.FetcherManager), new(bt.FetcherManager), new(ed2k.FetcherManager), diff --git a/ui/flutter/lib/api/model/downloader_config.dart b/ui/flutter/lib/api/model/downloader_config.dart index 39b4bbee..d824f6d8 100644 --- a/ui/flutter/lib/api/model/downloader_config.dart +++ b/ui/flutter/lib/api/model/downloader_config.dart @@ -56,6 +56,7 @@ class ProtocolConfig { HttpConfig http = HttpConfig(); BtConfig bt = BtConfig(); Ed2kConfig ed2k = Ed2kConfig(); + HlsConfig hls = HlsConfig(); ProtocolConfig(); @@ -65,6 +66,25 @@ class ProtocolConfig { Map toJson() => _$ProtocolConfigToJson(this); } +@JsonSerializable() +class HlsConfig { + int segmentConnections; + int maxRetries; + int timeoutSeconds; + bool prefetchContentLength; + + HlsConfig({ + this.segmentConnections = 0, + this.maxRetries = 0, + this.timeoutSeconds = 0, + this.prefetchContentLength = false, + }); + + factory HlsConfig.fromJson(Map json) => _$HlsConfigFromJson(json); + + Map toJson() => _$HlsConfigToJson(this); +} + @JsonSerializable() class HttpConfig { String userAgent; diff --git a/ui/flutter/lib/api/model/downloader_config.g.dart b/ui/flutter/lib/api/model/downloader_config.g.dart index 9c82c3ce..2457ce2b 100644 --- a/ui/flutter/lib/api/model/downloader_config.g.dart +++ b/ui/flutter/lib/api/model/downloader_config.g.dart @@ -56,12 +56,28 @@ Map _$ApiServerConfigToJson(ApiServerConfig instance) => json) => ProtocolConfig() ..http = HttpConfig.fromJson(json['http'] as Map) ..bt = BtConfig.fromJson(json['bt'] as Map) - ..ed2k = Ed2kConfig.fromJson(json['ed2k'] as Map); + ..ed2k = Ed2kConfig.fromJson(json['ed2k'] as Map) + ..hls = HlsConfig.fromJson(json['hls'] as Map); Map _$ProtocolConfigToJson(ProtocolConfig instance) => { 'http': instance.http.toJson(), 'bt': instance.bt.toJson(), 'ed2k': instance.ed2k.toJson(), + 'hls': instance.hls.toJson(), +}; + +HlsConfig _$HlsConfigFromJson(Map json) => HlsConfig( + segmentConnections: (json['segmentConnections'] as num?)?.toInt() ?? 0, + maxRetries: (json['maxRetries'] as num?)?.toInt() ?? 0, + timeoutSeconds: (json['timeoutSeconds'] as num?)?.toInt() ?? 0, + prefetchContentLength: json['prefetchContentLength'] as bool? ?? false, +); + +Map _$HlsConfigToJson(HlsConfig instance) => { + 'segmentConnections': instance.segmentConnections, + 'maxRetries': instance.maxRetries, + 'timeoutSeconds': instance.timeoutSeconds, + 'prefetchContentLength': instance.prefetchContentLength, }; HttpConfig _$HttpConfigFromJson(Map json) => HttpConfig( diff --git a/ui/flutter/lib/api/model/task.dart b/ui/flutter/lib/api/model/task.dart index cca65ed9..fd08e781 100644 --- a/ui/flutter/lib/api/model/task.dart +++ b/ui/flutter/lib/api/model/task.dart @@ -6,7 +6,7 @@ part 'task.g.dart'; enum Status { ready, running, pause, wait, error, done } -enum Protocol { http, bt, ed2k } +enum Protocol { http, bt, ed2k, hls } // ExtractStatus enum matching Go backend enum ExtractStatus { diff --git a/ui/flutter/lib/api/model/task.g.dart b/ui/flutter/lib/api/model/task.g.dart index 15b3b277..f127beee 100644 --- a/ui/flutter/lib/api/model/task.g.dart +++ b/ui/flutter/lib/api/model/task.g.dart @@ -38,7 +38,7 @@ const _$StatusEnumMap = { Status.done: 'done', }; -const _$ProtocolEnumMap = {Protocol.http: 'http', Protocol.bt: 'bt', Protocol.ed2k: 'ed2k'}; +const _$ProtocolEnumMap = {Protocol.http: 'http', Protocol.bt: 'bt', Protocol.ed2k: 'ed2k', Protocol.hls: 'hls'}; Progress _$ProgressFromJson(Map json) => Progress( used: (json['used'] as num).toInt(), diff --git a/ui/flutter/lib/api/model/task_stats.dart b/ui/flutter/lib/api/model/task_stats.dart index 4dafd0b0..dfe6c8c7 100644 --- a/ui/flutter/lib/api/model/task_stats.dart +++ b/ui/flutter/lib/api/model/task_stats.dart @@ -55,7 +55,7 @@ sealed class TaskStats { Protocol.http => HttpTaskStats.fromJson(snapshot), Protocol.bt => BtTaskStats.fromJson(snapshot: snapshot, runtime: runtime), Protocol.ed2k => Ed2kTaskStats.fromJson(snapshot: snapshot, runtime: runtime), - null => null, + Protocol.hls || null => null, }; } } diff --git a/ui/flutter/lib/features/settings/presentation/pages/settings_page.dart b/ui/flutter/lib/features/settings/presentation/pages/settings_page.dart index 67d41e74..f772b883 100644 --- a/ui/flutter/lib/features/settings/presentation/pages/settings_page.dart +++ b/ui/flutter/lib/features/settings/presentation/pages/settings_page.dart @@ -64,6 +64,8 @@ class _SettingsPageState extends ConsumerState { final _maxRunningController = TextEditingController(); final _httpUserAgentController = TextEditingController(); final _httpConnectionsController = TextEditingController(); + final _hlsConnectionsController = TextEditingController(); + final _hlsMaxRetriesController = TextEditingController(); final _btListenPortController = TextEditingController(); final _btSeedRatioController = TextEditingController(); final _btSeedTimeController = TextEditingController(); @@ -95,6 +97,8 @@ class _SettingsPageState extends ConsumerState { _maxRunningController, _httpUserAgentController, _httpConnectionsController, + _hlsConnectionsController, + _hlsMaxRetriesController, _btListenPortController, _btSeedRatioController, _btSeedTimeController, @@ -558,6 +562,32 @@ class _SettingsPageState extends ConsumerState { ], ), ), + _SettingsBlock( + key: const ValueKey('settings-hls-block'), + title: 'HLS', + child: _SettingsGroup( + children: [ + SettingsItem( + title: context.l10n.connections, + child: _NumberSettingControl( + fieldKey: const ValueKey('hls-connections-input'), + controller: _hlsConnectionsController, + min: 1, + max: 256, + ), + ), + SettingsItem( + title: context.l10n.maxRetries, + child: _NumberSettingControl( + fieldKey: const ValueKey('hls-max-retries-input'), + controller: _hlsMaxRetriesController, + min: 0, + max: 30, + ), + ), + ], + ), + ), _SettingsBlock( title: 'BitTorrent', child: _SettingsGroup( @@ -939,6 +969,8 @@ class _SettingsPageState extends ConsumerState { _maxRunningController.text = _config!.maxRunning.clamp(1, 256).toString(); _httpUserAgentController.text = _config!.protocolConfig.http.userAgent; _httpConnectionsController.text = _config!.protocolConfig.http.connections.clamp(1, 256).toString(); + _hlsConnectionsController.text = _config!.protocolConfig.hls.segmentConnections.clamp(1, 256).toString(); + _hlsMaxRetriesController.text = _config!.protocolConfig.hls.maxRetries.clamp(0, 30).toString(); _btListenPortController.text = _config!.protocolConfig.bt.listenPort.clamp(0, 65535).toString(); _btSeedRatioController.text = _config!.protocolConfig.bt.seedRatio.toString(); _btSeedTimeController.text = (_config!.protocolConfig.bt.seedTime ~/ 60).clamp(0, 100000000).toString(); @@ -1034,6 +1066,18 @@ class _SettingsPageState extends ConsumerState { min: 1, max: 256, ); + config.protocolConfig.hls.segmentConnections = _boundedInt( + _hlsConnectionsController, + fallback: config.protocolConfig.hls.segmentConnections, + min: 1, + max: 256, + ); + config.protocolConfig.hls.maxRetries = _boundedInt( + _hlsMaxRetriesController, + fallback: config.protocolConfig.hls.maxRetries, + min: 0, + max: 30, + ); config.protocolConfig.bt.listenPort = _boundedInt( _btListenPortController, fallback: config.protocolConfig.bt.listenPort, diff --git a/ui/flutter/lib/features/tasks/domain/task_record.dart b/ui/flutter/lib/features/tasks/domain/task_record.dart index 44a5a426..dfaeb67e 100644 --- a/ui/flutter/lib/features/tasks/domain/task_record.dart +++ b/ui/flutter/lib/features/tasks/domain/task_record.dart @@ -294,7 +294,7 @@ TaskAssetType _assetType(String name, {required bool isFolder, required api_task return switch (protocol) { api_task.Protocol.bt => TaskAssetType.torrent, api_task.Protocol.ed2k => TaskAssetType.ed2k, - api_task.Protocol.http || null => TaskAssetType.file, + api_task.Protocol.hls || api_task.Protocol.http || null => TaskAssetType.file, }; } diff --git a/ui/flutter/lib/l10n/app_ar.arb b/ui/flutter/lib/l10n/app_ar.arb index 3b7c407d..0168ed5e 100644 --- a/ui/flutter/lib/l10n/app_ar.arb +++ b/ui/flutter/lib/l10n/app_ar.arb @@ -38,6 +38,7 @@ "rememberLastDownloadDir": "حفظ كمجلد التنزيل الافتراضي", "downloadDirValid": "الرجاء تحديد مجلد التنزيل", "connections": "الاتصالات", + "maxRetries": "الحد الأقصى لإعادة المحاولات", "useServerCtime": "استخدام وقت الخادم لإنشاء الملف", "maxRunning": "الحد الأقصى للمهام المتزامنة", "defaultDirectDownload": "تفعيل التنزيل المباشر افتراضياً", diff --git a/ui/flutter/lib/l10n/app_az.arb b/ui/flutter/lib/l10n/app_az.arb index 4fccb3de..d0b85272 100644 --- a/ui/flutter/lib/l10n/app_az.arb +++ b/ui/flutter/lib/l10n/app_az.arb @@ -38,6 +38,7 @@ "rememberLastDownloadDir": "Standart endirmə qovluğu kimi yadda saxla", "downloadDirValid": "Zəhmət olmasa endirmə qovluğunu seçin", "connections": "Əlaqələr", + "maxRetries": "Maksimum cəhd sayı", "useServerCtime": "Fayl yaratma tarixi üçün server vaxtından istifadə et", "maxRunning": "Paralel Endirmələr", "defaultDirectDownload": "Standart olaraq Birbaşa Endirmə", diff --git a/ui/flutter/lib/l10n/app_ca.arb b/ui/flutter/lib/l10n/app_ca.arb index bb030cf5..4f11f102 100644 --- a/ui/flutter/lib/l10n/app_ca.arb +++ b/ui/flutter/lib/l10n/app_ca.arb @@ -38,6 +38,7 @@ "rememberLastDownloadDir": "Desa com a carpeta de baixades predeterminada", "downloadDirValid": "Seleccioneu la carpeta de baixades", "connections": "Connexions", + "maxRetries": "Reintents màxims", "useServerCtime": "Utilitza l'hora del servidor per a la creació de fitxers", "maxRunning": "Màxim de tasques en execució", "defaultDirectDownload": "Activa la baixada directa per defecte", diff --git a/ui/flutter/lib/l10n/app_de.arb b/ui/flutter/lib/l10n/app_de.arb index 64d7683c..53bf4e5d 100644 --- a/ui/flutter/lib/l10n/app_de.arb +++ b/ui/flutter/lib/l10n/app_de.arb @@ -38,6 +38,7 @@ "rememberLastDownloadDir": "Als Standard-Downloadordner speichern", "downloadDirValid": "Bitte wählen Sie das Download-Verzeichnis", "connections": "Anschlüsse", + "maxRetries": "Maximale Wiederholungsversuche", "useServerCtime": "Serverzeit für die Dateierstellung verwenden", "maxRunning": "Maximale Anzahl ausgeführter Aufgaben", "defaultDirectDownload": "Direkten Download standardmäßig aktivieren", diff --git a/ui/flutter/lib/l10n/app_en.arb b/ui/flutter/lib/l10n/app_en.arb index 5aab7de5..3ad1437a 100644 --- a/ui/flutter/lib/l10n/app_en.arb +++ b/ui/flutter/lib/l10n/app_en.arb @@ -38,6 +38,7 @@ "rememberLastDownloadDir": "Save as default download directory", "downloadDirValid": "Please select the download directory", "connections": "Connections", + "maxRetries": "Max retries", "useServerCtime": "Use server time for file creation", "maxRunning": "Concurrent Downloads", "defaultDirectDownload": "Direct Download by Default", diff --git a/ui/flutter/lib/l10n/app_es.arb b/ui/flutter/lib/l10n/app_es.arb index 89ed2ad0..2337c1fc 100644 --- a/ui/flutter/lib/l10n/app_es.arb +++ b/ui/flutter/lib/l10n/app_es.arb @@ -38,6 +38,7 @@ "rememberLastDownloadDir": "Guardar como carpeta de descargas predeterminada", "downloadDirValid": "Por favor, selecciona el directorio de descarga", "connections": "Conexiones", + "maxRetries": "Reintentos máximos", "useServerCtime": "Usar tiempo del servidor para la creación de archivos", "maxRunning": "Máximo de Tareas en Ejecución", "defaultDirectDownload": "Ver descarga directa por defecto", diff --git a/ui/flutter/lib/l10n/app_fa.arb b/ui/flutter/lib/l10n/app_fa.arb index 6486d00d..4317ff26 100644 --- a/ui/flutter/lib/l10n/app_fa.arb +++ b/ui/flutter/lib/l10n/app_fa.arb @@ -38,6 +38,7 @@ "rememberLastDownloadDir": "ذخیره به‌عنوان پوشه پیش‌فرض دانلود", "downloadDirValid": "لطفا دایرکتوری دانلود را انتخاب کنید", "connections": "اتصالات", + "maxRetries": "حداکثر تلاش مجدد", "useServerCtime": "استفاده از زمان سرور برای ایجاد فایل", "maxRunning": "حداکثر وظایف در حال اجرا", "defaultDirectDownload": "دانلود مستقیم را به صورت پیش فرض بررسی کنید", diff --git a/ui/flutter/lib/l10n/app_fr.arb b/ui/flutter/lib/l10n/app_fr.arb index cd764419..a81b941a 100644 --- a/ui/flutter/lib/l10n/app_fr.arb +++ b/ui/flutter/lib/l10n/app_fr.arb @@ -38,6 +38,7 @@ "rememberLastDownloadDir": "Enregistrer comme dossier de téléchargement par défaut", "downloadDirValid": "Veuillez sélectionner le répertoire de téléchargement", "connections": "Connexions", + "maxRetries": "Nombre maximal de tentatives", "useServerCtime": "Utiliser l'heure du serveur pour la création de fichier", "maxRunning": "Tâches en cours maximum", "defaultDirectDownload": "Vérifiez le téléchargement direct par défaut", diff --git a/ui/flutter/lib/l10n/app_hu.arb b/ui/flutter/lib/l10n/app_hu.arb index 034eff81..0ca0fb3f 100644 --- a/ui/flutter/lib/l10n/app_hu.arb +++ b/ui/flutter/lib/l10n/app_hu.arb @@ -38,6 +38,7 @@ "rememberLastDownloadDir": "Mentés alapértelmezett letöltési mappaként", "downloadDirValid": "Kérjük, válassza ki a letöltési könyvtárat", "connections": "Kapcsolatok", + "maxRetries": "Maximális újrapróbálkozások", "useServerCtime": "Használja a szerveridőt a fájl létrehozásához", "maxRunning": "Maximális futó feladatok", "defaultDirectDownload": "Alapértelmezettként ellenőrizze a közvetlen letöltést", diff --git a/ui/flutter/lib/l10n/app_id.arb b/ui/flutter/lib/l10n/app_id.arb index 40acc714..2b114f43 100644 --- a/ui/flutter/lib/l10n/app_id.arb +++ b/ui/flutter/lib/l10n/app_id.arb @@ -38,6 +38,7 @@ "rememberLastDownloadDir": "Simpan sebagai folder unduhan default", "downloadDirValid": "Silahkan pilih direktori unduhan", "connections": "Koneksi", + "maxRetries": "Percobaan ulang maks.", "useServerCtime": "Gunakan waktu server untuk pembuatan file", "maxRunning": "Tugas Maksimum yang Berjalan", "defaultDirectDownload": "Pilih unduhan langsung sebagai opsi standar", diff --git a/ui/flutter/lib/l10n/app_it.arb b/ui/flutter/lib/l10n/app_it.arb index aba9620f..2a38ac83 100644 --- a/ui/flutter/lib/l10n/app_it.arb +++ b/ui/flutter/lib/l10n/app_it.arb @@ -38,6 +38,7 @@ "rememberLastDownloadDir": "Salva come cartella di download predefinita", "downloadDirValid": "Seleziona la directory di scaricamento", "connections": "Connessioni", + "maxRetries": "Tentativi massimi", "useServerCtime": "Utilizza l'ora del server per la creazione dei file", "maxRunning": "Numero massimo di attività in esecuzione", "defaultDirectDownload": "Controlla il download diretto per impostazione predefinita", diff --git a/ui/flutter/lib/l10n/app_ja.arb b/ui/flutter/lib/l10n/app_ja.arb index 8f15b2d1..92b085da 100644 --- a/ui/flutter/lib/l10n/app_ja.arb +++ b/ui/flutter/lib/l10n/app_ja.arb @@ -38,6 +38,7 @@ "rememberLastDownloadDir": "デフォルトのダウンロード先として保存", "downloadDirValid": "ダウンロードディレクトリを選択してください", "connections": "接続", + "maxRetries": "最大リトライ回数", "useServerCtime": "ファイル作成時にサーバー時間を使用する", "maxRunning": "最大実行タスク", "defaultDirectDownload": "デフォルトで直接ダウンロードにチェックを入れる", diff --git a/ui/flutter/lib/l10n/app_ko.arb b/ui/flutter/lib/l10n/app_ko.arb index 23c42a68..def6088a 100644 --- a/ui/flutter/lib/l10n/app_ko.arb +++ b/ui/flutter/lib/l10n/app_ko.arb @@ -38,6 +38,7 @@ "rememberLastDownloadDir": "기본 다운로드 폴더로 저장", "downloadDirValid": "다운로드 폴더를 선택해 주세요", "connections": "연결 수", + "maxRetries": "최대 재시도 횟수", "useServerCtime": "파일 생성 시간에 서버 시간 사용", "maxRunning": "최대 동시 작업 수", "defaultDirectDownload": "기본으로 바로 다운로드 사용", diff --git a/ui/flutter/lib/l10n/app_pl.arb b/ui/flutter/lib/l10n/app_pl.arb index 85b61144..122955df 100644 --- a/ui/flutter/lib/l10n/app_pl.arb +++ b/ui/flutter/lib/l10n/app_pl.arb @@ -38,6 +38,7 @@ "rememberLastDownloadDir": "Zapisz jako domyślny folder pobierania", "downloadDirValid": "Wskaż folder pobierania", "connections": "Połączenia", + "maxRetries": "Maks. liczba ponownych prób", "useServerCtime": "Użyj czasu serwera do utworzenia pliku", "maxRunning": "Maksymalna liczba zadań", "defaultDirectDownload": "Domyślnie sprawdź bezpośredni pobór", diff --git a/ui/flutter/lib/l10n/app_pt.arb b/ui/flutter/lib/l10n/app_pt.arb index 46674d2c..64401539 100644 --- a/ui/flutter/lib/l10n/app_pt.arb +++ b/ui/flutter/lib/l10n/app_pt.arb @@ -38,6 +38,7 @@ "rememberLastDownloadDir": "Salvar como pasta de download padrão", "downloadDirValid": "Por favor, selecione a pasta", "connections": "Conexões", + "maxRetries": "Tentativas máximas", "useServerCtime": "Usar o tempo de servidor para a criação de ficheiros", "maxRunning": "Máximo de Tarefas Simultâneas", "defaultDirectDownload": "Marcar download direto por padrão", diff --git a/ui/flutter/lib/l10n/app_ru.arb b/ui/flutter/lib/l10n/app_ru.arb index 53f5f2f6..c70e1fe4 100644 --- a/ui/flutter/lib/l10n/app_ru.arb +++ b/ui/flutter/lib/l10n/app_ru.arb @@ -38,6 +38,7 @@ "rememberLastDownloadDir": "Сохранить как папку загрузок по умолчанию", "downloadDirValid": "Пожалуйста, выберите папку для загрузки", "connections": "Соединений", + "maxRetries": "Макс. число повторов", "useServerCtime": "Использовать время сервера для создания файла", "maxRunning": "Максимальное количество активных задач", "defaultDirectDownload": "Прямая загрузка по умолчанию", diff --git a/ui/flutter/lib/l10n/app_ta.arb b/ui/flutter/lib/l10n/app_ta.arb index bccefc16..d7adf3ad 100644 --- a/ui/flutter/lib/l10n/app_ta.arb +++ b/ui/flutter/lib/l10n/app_ta.arb @@ -38,6 +38,7 @@ "rememberLastDownloadDir": "இயல்புநிலை பதிவிறக்க கோப்புறையாக சேமி", "downloadDirValid": "பதிவிரக்குமிடத்தை தேர்வு செயுங்கள்", "connections": "தொடர்புகள்", + "maxRetries": "அதிகபட்ச மறுமுயற்சிகள்", "useServerCtime": "கோப்பு உருவாக்க சேவையக நேரத்தைப் பயன்படுத்தவும்", "maxRunning": "அதிகபட்ச இயங்கும் பணிகள்", "defaultDirectDownload": "இயல்புநிலையாக நேரடிப் பதிவிறக்கத்தைச் சரிபார்க்கவும்", diff --git a/ui/flutter/lib/l10n/app_tr.arb b/ui/flutter/lib/l10n/app_tr.arb index 9b3b4954..ad850e4b 100644 --- a/ui/flutter/lib/l10n/app_tr.arb +++ b/ui/flutter/lib/l10n/app_tr.arb @@ -38,6 +38,7 @@ "rememberLastDownloadDir": "Varsayılan indirme klasörü olarak kaydet", "downloadDirValid": "Lütfen indirme dizinini seçin", "connections": "Bağlantılar", + "maxRetries": "Maksimum yeniden deneme", "useServerCtime": "Dosya oluşturma tarihi için sunucu zamanını kullan", "maxRunning": "Maksimum Çalışan İşlem", "defaultDirectDownload": "Varsayılan olarak direkt indirmeyi işaretle", diff --git a/ui/flutter/lib/l10n/app_uk.arb b/ui/flutter/lib/l10n/app_uk.arb index e4b3e923..85bef551 100644 --- a/ui/flutter/lib/l10n/app_uk.arb +++ b/ui/flutter/lib/l10n/app_uk.arb @@ -38,6 +38,7 @@ "rememberLastDownloadDir": "Зберегти як типову папку завантажень", "downloadDirValid": "Будь ласка, виберіть каталог завантажень", "connections": "З’єднання", + "maxRetries": "Макс. кількість повторів", "useServerCtime": "Використовувати час сервера для створення файлу", "maxRunning": "Макс. завдань у черзі", "defaultDirectDownload": "Пряме завантаження за замовчуванням", diff --git a/ui/flutter/lib/l10n/app_vi.arb b/ui/flutter/lib/l10n/app_vi.arb index f7a6db52..f84ac9ba 100644 --- a/ui/flutter/lib/l10n/app_vi.arb +++ b/ui/flutter/lib/l10n/app_vi.arb @@ -38,6 +38,7 @@ "rememberLastDownloadDir": "Lưu làm thư mục tải xuống mặc định", "downloadDirValid": "Vui lòng chọn thư mục tải về", "connections": "Kết nối", + "maxRetries": "Số lần thử lại tối đa", "useServerCtime": "Sử dụng thời gian máy chủ cho việc tạo tệp", "maxRunning": "Số nhiệm vụ tối đa", "defaultDirectDownload": "Kiểm tra trực tiếp tải về theo mặc định", diff --git a/ui/flutter/lib/l10n/app_zh.arb b/ui/flutter/lib/l10n/app_zh.arb index 0bd33ecd..20a0fda3 100644 --- a/ui/flutter/lib/l10n/app_zh.arb +++ b/ui/flutter/lib/l10n/app_zh.arb @@ -38,6 +38,7 @@ "rememberLastDownloadDir": "保存为默认下载目录", "downloadDirValid": "请选择下载目录", "connections": "连接数", + "maxRetries": "最大重试次数", "useServerCtime": "创建文件使用服务器时间", "maxRunning": "最大下载数", "defaultDirectDownload": "默认启用直接下载", diff --git a/ui/flutter/lib/l10n/app_zh_TW.arb b/ui/flutter/lib/l10n/app_zh_TW.arb index 8220df52..94419c20 100644 --- a/ui/flutter/lib/l10n/app_zh_TW.arb +++ b/ui/flutter/lib/l10n/app_zh_TW.arb @@ -38,6 +38,7 @@ "rememberLastDownloadDir": "儲存為預設下載目錄", "downloadDirValid": "請選擇下載目錄", "connections": "連接數", + "maxRetries": "最大重試次數", "useServerCtime": "使用伺服器時間作為檔案建立時間", "maxRunning": "最大執行任務數", "defaultDirectDownload": "預設勾選直接下載",