feat(hls): add native HLS (m3u8) download protocol (#1515)

This commit is contained in:
sun authored and GitHub committed 2026-09-15 23:40:31 +08:00
1 parent e47a20a1bd
commit 46c34f2e92
56 files changed
+4536 -6

No files matched your search

+7 -1
View File
@@ -106,7 +106,13 @@ func (s *SchemeFilter) Match(uri string) bool {
case FilterTypeUrl:
return strings.HasPrefix(uriUpper, patternUpper+":")
case FilterTypeFile:
return strings.HasSuffix(uriUpper, "."+patternUpper)
// Strip query and fragment before matching so URLs like
// https://example.com/video.m3u8?sign=abc still route by extension.
clean := uri
if idx := strings.IndexAny(clean, "?#"); idx >= 0 {
clean = clean[:idx]
}
return strings.HasSuffix(strings.ToUpper(clean), "."+patternUpper)
case FilterTypeBase64:
return strings.HasPrefix(uriUpper, "DATA:"+patternUpper+";BASE64,")
}
+30
View File
@@ -0,0 +1,30 @@
package hls
const (
defaultSegmentConnections = 8
defaultMaxRetries = 5
defaultTimeoutSeconds = 30
// maxPrefetchSizeSegments caps the HEAD probing done at resolve time.
// Playlists larger than this skip size prefetch entirely.
maxPrefetchSizeSegments = 500
)
type config struct {
SegmentConnections int `json:"segmentConnections"`
MaxRetries int `json:"maxRetries"`
TimeoutSeconds int `json:"timeoutSeconds"`
PrefetchContentLength bool `json:"prefetchContentLength"`
}
func (c *config) normalize() {
if c.SegmentConnections <= 0 {
c.SegmentConnections = defaultSegmentConnections
}
if c.MaxRetries < 0 {
c.MaxRetries = defaultMaxRetries
}
if c.TimeoutSeconds <= 0 {
c.TimeoutSeconds = defaultTimeoutSeconds
}
}
+59
View File
@@ -0,0 +1,59 @@
package hls
import (
"crypto/aes"
"crypto/cipher"
"fmt"
)
// decryptAES128 decrypts an AES-128-CBC encrypted segment and strips PKCS7
// padding. data length must be a multiple of the AES block size.
func decryptAES128(data, key, iv []byte) ([]byte, error) {
if len(key) != 16 {
return nil, fmt.Errorf("invalid AES-128 key size %d", len(key))
}
if len(iv) != 16 {
return nil, fmt.Errorf("invalid AES-128 IV size %d", len(iv))
}
if len(data) == 0 || len(data)%aes.BlockSize != 0 {
return nil, fmt.Errorf("invalid AES-128 data size %d", len(data))
}
block, err := aes.NewCipher(key)
if err != nil {
return nil, err
}
plain := make([]byte, len(data))
cipher.NewCBCDecrypter(block, iv).CryptBlocks(plain, data)
return pkcs7Unpad(plain, aes.BlockSize)
}
func pkcs7Unpad(data []byte, blockSize int) ([]byte, error) {
if len(data) == 0 {
return nil, fmt.Errorf("empty plaintext")
}
pad := int(data[len(data)-1])
if pad == 0 || pad > blockSize || pad > len(data) {
return nil, fmt.Errorf("invalid PKCS7 padding")
}
for _, b := range data[len(data)-pad:] {
if int(b) != pad {
return nil, fmt.Errorf("invalid PKCS7 padding")
}
}
return data[:len(data)-pad], nil
}
// sequenceIV derives the default per-segment IV mandated by RFC 8216: the
// segment media sequence number as a 16-byte big-endian integer.
func sequenceIV(sequence int64) []byte {
iv := make([]byte, 16)
iv[8] = byte(sequence >> 56)
iv[9] = byte(sequence >> 48)
iv[10] = byte(sequence >> 40)
iv[11] = byte(sequence >> 32)
iv[12] = byte(sequence >> 24)
iv[13] = byte(sequence >> 16)
iv[14] = byte(sequence >> 8)
iv[15] = byte(sequence)
return iv
}
+66
View File
@@ -0,0 +1,66 @@
package hls
import (
"bytes"
"crypto/aes"
"crypto/cipher"
"crypto/rand"
"testing"
)
func pkcs7Pad(data []byte, blockSize int) []byte {
pad := blockSize - len(data)%blockSize
return append(data, bytes.Repeat([]byte{byte(pad)}, pad)...)
}
func TestDecryptAES128Roundtrip(t *testing.T) {
key := make([]byte, 16)
iv := make([]byte, 16)
if _, err := rand.Read(key); err != nil {
t.Fatal(err)
}
if _, err := rand.Read(iv); err != nil {
t.Fatal(err)
}
plain := []byte("hello hls segment payload, padded to block size!")
padded := pkcs7Pad(plain, aes.BlockSize)
block, err := aes.NewCipher(key)
if err != nil {
t.Fatal(err)
}
encrypted := make([]byte, len(padded))
cipher.NewCBCEncrypter(block, iv).CryptBlocks(encrypted, padded)
got, err := decryptAES128(encrypted, key, iv)
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(got, plain) {
t.Errorf("roundtrip mismatch: %q", got)
}
}
func TestDecryptAES128InvalidInput(t *testing.T) {
if _, err := decryptAES128([]byte{1, 2, 3}, make([]byte, 16), make([]byte, 16)); err == nil {
t.Error("non-block-aligned data should fail")
}
if _, err := decryptAES128(make([]byte, 16), make([]byte, 15), make([]byte, 16)); err == nil {
t.Error("short key should fail")
}
if _, err := decryptAES128(make([]byte, 16), make([]byte, 16), make([]byte, 16)); err == nil {
t.Error("all-zero block has invalid padding and should fail")
}
}
func TestSequenceIV(t *testing.T) {
iv := sequenceIV(1)
if len(iv) != 16 || iv[15] != 1 {
t.Errorf("sequence 1 IV wrong: %v", iv)
}
iv = sequenceIV(0x0102030405060708)
for i, want := range []byte{1, 2, 3, 4, 5, 6, 7, 8} {
if iv[8+i] != want {
t.Errorf("sequence IV byte %d: want %d got %d", i, want, iv[8+i])
}
}
}
+24
View File
@@ -0,0 +1,24 @@
// Package hls implements native HTTP Live Streaming downloads for m3u8
// playlists. A task produces exactly one merged, playable output file:
// MPEG-2 TS segments are concatenated as-is, fMP4 streams are merged as
// init section followed by the media fragments, and AES-128 encrypted
// segments are decrypted before they are stored.
//
// Supported subset:
// - VOD media playlists (EXT-X-ENDLIST present) and master playlists whose
// selected variant carries its own audio
// - AES-128 with the identity KEYFORMAT, explicit IV or media-sequence
// derived IV, including key rotation mid-playlist
// - EXT-X-BYTERANGE (including offset inheritance) and fMP4 init sections
// via a single, constant EXT-X-MAP
//
// Everything the merge pipeline cannot reproduce faithfully is rejected with
// an explicit error instead of silently producing a broken output file:
// - live streams (no EXT-X-ENDLIST) and SAMPLE-AES / DRM encryption
// - EXT-X-GAP (would leave a hole) and EXT-X-DISCONTINUITY (broken timeline)
// - changing init sections (a repeated EXT-X-MAP is compared by all its
// attributes, not just the URI) and encrypted EXT-X-MAP without an
// explicit IV
// - separate audio renditions referenced by the selected variant (the
// merged output would have no audio) and non-identity KEYFORMATs
package hls
File diff suppressed because it is too large. Load diff
+100
View File
@@ -0,0 +1,100 @@
package hls
import (
"net/url"
"path"
"strings"
"github.com/GopeedLab/gopeed/internal/fetcher"
)
type FetcherManager struct {
}
func (fm *FetcherManager) Name() string {
return "hls"
}
// Filters registers the .m3u8 extension. The manager must be registered BEFORE
// the http manager in the downloader config: protocol routing picks the first
// manager whose filter matches, and the http filter claims every http(s) URL.
func (fm *FetcherManager) Filters() []*fetcher.SchemeFilter {
return []*fetcher.SchemeFilter{
{
Type: fetcher.FilterTypeFile,
Pattern: "M3U8",
},
}
}
func (fm *FetcherManager) Build() fetcher.Fetcher {
return &Fetcher{}
}
func (fm *FetcherManager) ParseName(u string) string {
parsed, err := url.Parse(u)
if err != nil {
return ""
}
name := path.Base(parsed.Path)
name = strings.TrimSuffix(name, path.Ext(name))
if name == "" || name == "/" || name == "." {
name = parsed.Hostname()
}
return name
}
func (fm *FetcherManager) AutoRename() bool {
return true
}
func (fm *FetcherManager) DefaultConfig() any {
return &config{
SegmentConnections: defaultSegmentConnections,
MaxRetries: defaultMaxRetries,
TimeoutSeconds: defaultTimeoutSeconds,
PrefetchContentLength: true,
}
}
func (fm *FetcherManager) Store(f fetcher.Fetcher) (data any, err error) {
_f := f.(*Fetcher)
_f.mu.Lock()
defer _f.mu.Unlock()
if _f.state == nil {
return nil, nil
}
// The engine's periodic checkpoint calls Store without holding the task
// lock and serializes the returned value afterwards, while Resolve/Patch/
// Start mutate the live state under the fetcher mutex: return a snapshot
// copy taken under that mutex, mirroring the HTTP protocol's Store.
return cloneState(_f.state), nil
}
func cloneState(state *fetcherState) *fetcherState {
cloned := *state
cloned.Segments = make([]*Segment, len(state.Segments))
for i, seg := range state.Segments {
s := *seg
cloned.Segments[i] = &s
}
return &cloned
}
func (fm *FetcherManager) Restore() (v any, f func(meta *fetcher.FetcherMeta, v any) fetcher.Fetcher) {
return &fetcherState{}, func(meta *fetcher.FetcherMeta, v any) fetcher.Fetcher {
fb := &FetcherManager{}
restored := fb.Build().(*Fetcher)
restored.meta = meta
if state, ok := v.(*fetcherState); ok && state.MediaURL != "" {
restored.state = state
}
// Setup is invoked by the engine (restoreFetcher) after this builder
// returns; ReqExtra parsing happens there as well.
return restored
}
}
func (fm *FetcherManager) Close() error {
return nil
}
+857
View File
@@ -0,0 +1,857 @@
package hls
import (
"bytes"
"crypto/aes"
"crypto/cipher"
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strconv"
"strings"
"sync/atomic"
"testing"
"time"
"github.com/GopeedLab/gopeed/internal/controller"
"github.com/GopeedLab/gopeed/internal/fetcher"
"github.com/GopeedLab/gopeed/pkg/base"
)
func pkcs7PadForTest(data []byte, blockSize int) []byte {
pad := blockSize - len(data)%blockSize
return append(data, bytes.Repeat([]byte{byte(pad)}, pad)...)
}
func encryptSegment(plain, key, iv []byte) []byte {
padded := pkcs7PadForTest(plain, aes.BlockSize)
block, _ := aes.NewCipher(key)
encrypted := make([]byte, len(padded))
cipher.NewCBCEncrypter(block, iv).CryptBlocks(encrypted, padded)
return encrypted
}
// serveContentWithReferer is a helper handler that enforces an optional
// Referer requirement and reports request counts.
type cdnHandler struct {
mux *http.ServeMux
requireReferer string
requests atomic.Int64
}
func (h *cdnHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
h.requests.Add(1)
if h.requireReferer != "" && r.Header.Get("Referer") != h.requireReferer {
w.WriteHeader(http.StatusForbidden)
return
}
h.mux.ServeHTTP(w, r)
}
func newTestFetcher(t *testing.T, cfg config) *Fetcher {
t.Helper()
fm := &FetcherManager{}
f := fm.Build().(*Fetcher)
ctl := controller.NewController()
ctl.GetConfig = func(v any) {
if c, ok := v.(*config); ok {
*c = cfg
}
}
f.Setup(ctl)
return f
}
func testConfig() config {
return config{
SegmentConnections: 4,
MaxRetries: 2,
TimeoutSeconds: 5,
PrefetchContentLength: true,
}
}
func mustResolve(t *testing.T, f *Fetcher, playlistURL, referer string) {
t.Helper()
req := &base.Request{URL: playlistURL}
if referer != "" {
req.Extra = &ReqExtra{Header: map[string]string{"Referer": referer}}
}
if err := f.Resolve(req, &base.Options{Path: t.TempDir()}); err != nil {
t.Fatalf("resolve: %v", err)
}
}
func mustRun(t *testing.T, f *Fetcher) {
t.Helper()
if err := f.Start(); err != nil {
t.Fatalf("start: %v", err)
}
if err := f.Wait(); err != nil {
t.Fatalf("wait: %v", err)
}
}
func statsOf(t *testing.T, f *Fetcher) Stats {
t.Helper()
s, ok := f.Stats().Snapshot.(*Stats)
if !ok {
t.Fatalf("unexpected stats snapshot type %T", f.Stats().Snapshot)
}
return *s
}
func readOutput(t *testing.T, f *Fetcher) []byte {
t.Helper()
data, err := os.ReadFile(f.meta.SingleFilepath())
if err != nil {
t.Fatal(err)
}
return data
}
func TestFetcherE2E_MasterTS(t *testing.T) {
seg := func(i int) []byte {
return []byte(fmt.Sprintf("SEGMENT-%02d-PAYLOAD", i))
}
mux := http.NewServeMux()
mux.HandleFunc("/master.m3u8", func(w http.ResponseWriter, r *http.Request) {
w.Write([]byte("#EXTM3U\n" +
"#EXT-X-STREAM-INF:BANDWIDTH=800000\nv1/index.m3u8\n" +
"#EXT-X-STREAM-INF:BANDWIDTH=2000000\nv2/index.m3u8\n"))
})
mux.HandleFunc("/v1/index.m3u8", func(w http.ResponseWriter, r *http.Request) {
w.Write([]byte("#EXTM3U\n#EXTINF:4,\nlow-0.ts\n#EXT-X-ENDLIST\n"))
})
mux.HandleFunc("/v2/index.m3u8", func(w http.ResponseWriter, r *http.Request) {
var b strings.Builder
b.WriteString("#EXTM3U\n")
for i := 0; i < 5; i++ {
b.WriteString(fmt.Sprintf("#EXTINF:4,\nseg-%d.ts\n", i))
}
b.WriteString("#EXT-X-ENDLIST\n")
w.Write([]byte(b.String()))
})
for i := 0; i < 5; i++ {
content := seg(i)
path := "/v2/seg-" + strconv.Itoa(i) + ".ts"
mux.HandleFunc(path, func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Length", strconv.Itoa(len(content)))
w.Write(content)
})
}
server := httptest.NewServer(&cdnHandler{mux: mux})
defer server.Close()
f := newTestFetcher(t, testConfig())
mustResolve(t, f, server.URL+"/master.m3u8", "")
// Master resolution picked the 2000000 variant.
if f.state.MediaURL != server.URL+"/v2/index.m3u8" {
t.Fatalf("wrong variant picked: %s", f.state.MediaURL)
}
// HEAD prefetch should have computed the exact total size.
if f.meta.Res.Size != int64(5*len(seg(0))) {
t.Fatalf("prefetched size wrong: %d", f.meta.Res.Size)
}
mustRun(t, f)
var want []byte
for i := 0; i < 5; i++ {
want = append(want, seg(i)...)
}
got := readOutput(t, f)
if !bytes.Equal(got, want) {
t.Errorf("merged output mismatch: %q", got)
}
// The staging folder must be cleaned up after a successful merge; only
// the output file remains in the task directory.
if entries, _ := os.ReadDir(filepath.Dir(f.meta.SingleFilepath())); len(entries) != 1 {
for _, e := range entries {
t.Logf("leftover entry: %s", e.Name())
}
t.Errorf("staging dir should be cleaned up after merge")
}
if f.meta.Res.Size != int64(len(want)) {
t.Errorf("final size wrong: %d", f.meta.Res.Size)
}
}
func TestFetcherE2E_RefererEnforced(t *testing.T) {
page := "https://player.example.com/watch/123"
var playlist strings.Builder
playlist.WriteString("#EXTM3U\n")
segments := make(map[string][]byte)
for i := 0; i < 3; i++ {
name := fmt.Sprintf("seg-%d.ts", i)
segments["/"+name] = []byte(fmt.Sprintf("DATA-%d", i))
playlist.WriteString(fmt.Sprintf("#EXTINF:4,\n%s\n", name))
}
playlist.WriteString("#EXT-X-ENDLIST\n")
mux := http.NewServeMux()
mux.HandleFunc("/video.m3u8", func(w http.ResponseWriter, r *http.Request) {
w.Write([]byte(playlist.String()))
})
for path, content := range segments {
c := content
mux.HandleFunc(path, func(w http.ResponseWriter, r *http.Request) {
w.Write(c)
})
}
server := httptest.NewServer(&cdnHandler{mux: mux, requireReferer: page})
defer server.Close()
// Without Referer the resolve itself must fail (403 on the playlist).
blocked := newTestFetcher(t, testConfig())
req := &base.Request{URL: server.URL + "/video.m3u8"}
if err := blocked.Resolve(req, &base.Options{Path: t.TempDir()}); err == nil {
t.Fatal("resolve without Referer should fail")
}
// With the Referer set through the task extra headers it must succeed.
allowed := newTestFetcher(t, testConfig())
allowedReq := &base.Request{
URL: server.URL + "/video.m3u8",
Extra: &ReqExtra{Header: map[string]string{"Referer": page}},
}
if err := allowed.Resolve(allowedReq, &base.Options{Path: t.TempDir()}); err != nil {
t.Fatal(err)
}
mustRun(t, allowed)
want := []byte("DATA-0DATA-1DATA-2")
if !bytes.Equal(readOutput(t, allowed), want) {
t.Error("output mismatch with Referer download")
}
}
func TestFetcherE2E_AES128KeyRotation(t *testing.T) {
key1 := []byte("0123456789abcdef")
key2 := []byte("fedcba9876543210")
iv1 := []byte("aaaaaaaaaaaaaaaa")
plain1 := []byte("ENCRYPTED-SEGMENT-ONE!!") // padded on encrypt
plain2 := []byte("ENCRYPTED-SEGMENT-TWO!!")
// seg-2 has no explicit IV: derived from media sequence 1.
enc1 := encryptSegment(plain1, key1, iv1)
enc2 := encryptSegment(plain2, key2, sequenceIV(1))
mux := http.NewServeMux()
mux.HandleFunc("/video.m3u8", func(w http.ResponseWriter, r *http.Request) {
w.Write([]byte("#EXTM3U\n" +
"#EXT-X-MEDIA-SEQUENCE:0\n" +
"#EXT-X-KEY:METHOD=AES-128,URI=\"key-1.bin\",IV=0x" + hexEncode(iv1) + "\n" +
"#EXTINF:5,\nseg-1.ts\n" +
"#EXT-X-KEY:METHOD=AES-128,URI=\"key-2.bin\"\n" +
"#EXTINF:5,\nseg-2.ts\n" +
"#EXT-X-ENDLIST\n"))
})
mux.HandleFunc("/key-1.bin", func(w http.ResponseWriter, r *http.Request) {
w.Write(key1)
})
mux.HandleFunc("/key-2.bin", func(w http.ResponseWriter, r *http.Request) {
w.Write(key2)
})
mux.HandleFunc("/seg-1.ts", func(w http.ResponseWriter, r *http.Request) {
w.Write(enc1)
})
mux.HandleFunc("/seg-2.ts", func(w http.ResponseWriter, r *http.Request) {
w.Write(enc2)
})
server := httptest.NewServer(mux)
defer server.Close()
f := newTestFetcher(t, testConfig())
mustResolve(t, f, server.URL+"/video.m3u8", "")
// Ciphertext sizes overstate the final output after decryption, so an
// encrypted plan must report an indeterminate size.
if f.meta.Res.Size != 0 {
t.Errorf("encrypted plan should report size 0, got %d", f.meta.Res.Size)
}
mustRun(t, f)
want := append(append([]byte{}, plain1...), plain2...)
if !bytes.Equal(readOutput(t, f), want) {
t.Errorf("decrypted output mismatch: %q", readOutput(t, f))
}
}
func TestFetcherE2E_ByteRange(t *testing.T) {
data := []byte("0123456789ABCDEFGHIJ") // 20 bytes
mux := http.NewServeMux()
mux.HandleFunc("/video.m3u8", func(w http.ResponseWriter, r *http.Request) {
w.Write([]byte("#EXTM3U\n" +
"#EXT-X-BYTERANGE:10@0\nblob.bin\n" +
"#EXT-X-BYTERANGE:10\nblob.bin\n" +
"#EXT-X-ENDLIST\n"))
})
mux.HandleFunc("/blob.bin", func(w http.ResponseWriter, r *http.Request) {
rng := r.Header.Get("Range")
if rng == "" {
w.Write(data)
return
}
// bytes=start-end
var start, end int
fmt.Sscanf(rng, "bytes=%d-%d", &start, &end)
w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, len(data)))
w.WriteHeader(http.StatusPartialContent)
w.Write(data[start : end+1])
})
server := httptest.NewServer(mux)
defer server.Close()
f := newTestFetcher(t, testConfig())
mustResolve(t, f, server.URL+"/video.m3u8", "")
if f.meta.Res.Size != 20 {
t.Fatalf("byterange sizes should be known from the playlist, got %d", f.meta.Res.Size)
}
mustRun(t, f)
if !bytes.Equal(readOutput(t, f), data) {
t.Errorf("byterange merge mismatch: %q", readOutput(t, f))
}
}
func TestFetcherE2E_PauseResume(t *testing.T) {
const total = 8
var served atomic.Int64
mux := http.NewServeMux()
mux.HandleFunc("/video.m3u8", func(w http.ResponseWriter, r *http.Request) {
var b strings.Builder
b.WriteString("#EXTM3U\n")
for i := 0; i < total; i++ {
b.WriteString(fmt.Sprintf("#EXTINF:4,\nseg-%d.ts\n", i))
}
b.WriteString("#EXT-X-ENDLIST\n")
w.Write([]byte(b.String()))
})
segments := make(map[int][]byte)
for i := 0; i < total; i++ {
segments[i] = []byte(fmt.Sprintf("SLOW-SEGMENT-%02d", i))
}
for i := 0; i < total; i++ {
idx := i
mux.HandleFunc(fmt.Sprintf("/seg-%d.ts", i), func(w http.ResponseWriter, r *http.Request) {
time.Sleep(150 * time.Millisecond)
served.Add(1)
w.Write(segments[idx])
})
}
server := httptest.NewServer(mux)
defer server.Close()
cfg := testConfig()
cfg.SegmentConnections = 2
cfg.PrefetchContentLength = false
f := newTestFetcher(t, cfg)
mustResolve(t, f, server.URL+"/video.m3u8", "")
if err := f.Start(); err != nil {
t.Fatal(err)
}
// Let a couple of segments finish, then pause.
time.Sleep(400 * time.Millisecond)
if err := f.Pause(); err != nil {
t.Fatal(err)
}
doneBefore := statsOf(t, f).SegmentsDone
if doneBefore == 0 {
t.Fatal("expected some segments to complete before pause")
}
if doneBefore == total {
t.Fatal("all segments finished before pause, test is not exercising resume")
}
// Resume must finish the rest and produce the full output.
if err := f.Start(); err != nil {
t.Fatal(err)
}
if err := f.Wait(); err != nil {
t.Fatal(err)
}
var want []byte
for i := 0; i < total; i++ {
want = append(want, segments[i]...)
}
if !bytes.Equal(readOutput(t, f), want) {
t.Errorf("resumed output mismatch: %q", readOutput(t, f))
}
}
func TestFetcherE2E_StoreRestoreResume(t *testing.T) {
const total = 6
var served atomic.Int64
mux := http.NewServeMux()
mux.HandleFunc("/video.m3u8", func(w http.ResponseWriter, r *http.Request) {
var b strings.Builder
b.WriteString("#EXTM3U\n")
for i := 0; i < total; i++ {
b.WriteString(fmt.Sprintf("#EXTINF:4,\nseg-%d.ts\n", i))
}
b.WriteString("#EXT-X-ENDLIST\n")
w.Write([]byte(b.String()))
})
segments := make(map[int][]byte)
for i := 0; i < total; i++ {
segments[i] = []byte(fmt.Sprintf("RESTORE-SEGMENT-%02d", i))
}
for i := 0; i < total; i++ {
idx := i
mux.HandleFunc(fmt.Sprintf("/seg-%d.ts", i), func(w http.ResponseWriter, r *http.Request) {
time.Sleep(120 * time.Millisecond)
served.Add(1)
w.Write(segments[idx])
})
}
server := httptest.NewServer(mux)
defer server.Close()
cfg := testConfig()
cfg.SegmentConnections = 2
cfg.PrefetchContentLength = false
f := newTestFetcher(t, cfg)
mustResolve(t, f, server.URL+"/video.m3u8", "")
if err := f.Start(); err != nil {
t.Fatal(err)
}
time.Sleep(350 * time.Millisecond)
if err := f.Pause(); err != nil {
t.Fatal(err)
}
doneBefore := statsOf(t, f).SegmentsDone
if doneBefore == 0 {
t.Fatal("expected some segments to finish before simulated restart")
}
// Simulate an app restart: persist, then restore into a fresh fetcher.
fm := &FetcherManager{}
state, err := fm.Store(f)
if err != nil {
t.Fatal(err)
}
if state == nil {
t.Fatal("state should be persisted")
}
ctl := controller.NewController()
ctl.GetConfig = func(v any) {
if c, ok := v.(*config); ok {
*c = cfg
}
}
holder := fm.Build().(*Fetcher)
holder.Setup(ctl)
holder.meta = &fetcher.FetcherMeta{
Req: f.meta.Req,
Opts: f.meta.Opts,
Res: f.meta.Res,
}
_, builder := fm.Restore()
rf := builder(holder.meta, state).(*Fetcher)
// The engine calls Setup on the builder result (restoreFetcher).
rf.Setup(ctl)
if rf.state == nil || len(rf.state.Segments) != total {
t.Fatal("restored state missing segment plan")
}
if err := rf.Start(); err != nil {
t.Fatal(err)
}
if err := rf.Wait(); err != nil {
t.Fatal(err)
}
if done := statsOf(t, rf).SegmentsDone; done != total {
t.Errorf("want %d done, got %d", total, done)
}
var want []byte
for i := 0; i < total; i++ {
want = append(want, segments[i]...)
}
if !bytes.Equal(readOutput(t, rf), want) {
t.Errorf("restored output mismatch: %q", readOutput(t, rf))
}
}
func hexEncode(b []byte) string {
const digits = "0123456789abcdef"
out := make([]byte, 0, len(b)*2)
for _, v := range b {
out = append(out, digits[v>>4], digits[v&0x0f])
}
return string(out)
}
// restoreLikeEngine round-trips the persisted state and the task meta through
// JSON the way the engine storage does, then builds a fresh fetcher. This is
// the path that historically lost the request extra headers.
func restoreLikeEngine(t *testing.T, f *Fetcher, cfg config) *Fetcher {
t.Helper()
fm := &FetcherManager{}
stateAny, err := fm.Store(f)
if err != nil {
t.Fatal(err)
}
rawState, err := json.Marshal(stateAny)
if err != nil {
t.Fatal(err)
}
storedState := &fetcherState{}
if err := json.Unmarshal(rawState, storedState); err != nil {
t.Fatal(err)
}
rawMeta, err := json.Marshal(f.meta)
if err != nil {
t.Fatal(err)
}
restoredMeta := &fetcher.FetcherMeta{}
if err := json.Unmarshal(rawMeta, restoredMeta); err != nil {
t.Fatal(err)
}
if _, ok := restoredMeta.Req.Extra.(*ReqExtra); ok {
t.Fatal("storage round trip should yield a map-shaped extra, like the engine")
}
ctl := controller.NewController()
ctl.GetConfig = func(v any) {
if c, ok := v.(*config); ok {
*c = cfg
}
}
_, builder := fm.Restore()
rf := builder(restoredMeta, storedState).(*Fetcher)
// The engine calls Setup on the builder result (restoreFetcher).
rf.Setup(ctl)
return rf
}
func TestFetcherE2E_StoreRestoreAuthenticated(t *testing.T) {
const total = 4
page := "https://player.example.com/watch/42"
var served atomic.Int64
mux := http.NewServeMux()
var playlist strings.Builder
playlist.WriteString("#EXTM3U\n")
for i := 0; i < total; i++ {
playlist.WriteString(fmt.Sprintf("#EXTINF:4,\nseg-%d.ts\n", i))
}
playlist.WriteString("#EXT-X-ENDLIST\n")
mux.HandleFunc("/video.m3u8", func(w http.ResponseWriter, r *http.Request) {
w.Write([]byte(playlist.String()))
})
segments := make(map[int][]byte)
for i := 0; i < total; i++ {
segments[i] = []byte(fmt.Sprintf("AUTH-SEGMENT-%02d", i))
idx := i
mux.HandleFunc(fmt.Sprintf("/seg-%d.ts", idx), func(w http.ResponseWriter, r *http.Request) {
time.Sleep(80 * time.Millisecond)
served.Add(1)
w.Write(segments[idx])
})
}
server := httptest.NewServer(&cdnHandler{mux: mux, requireReferer: page})
defer server.Close()
cfg := testConfig()
cfg.SegmentConnections = 2
cfg.PrefetchContentLength = false
f := newTestFetcher(t, cfg)
req := &base.Request{
URL: server.URL + "/video.m3u8",
Extra: &ReqExtra{Header: map[string]string{"Referer": page}},
}
if err := f.Resolve(req, &base.Options{Path: t.TempDir()}); err != nil {
t.Fatal(err)
}
if err := f.Start(); err != nil {
t.Fatal(err)
}
time.Sleep(150 * time.Millisecond)
if err := f.Pause(); err != nil {
t.Fatal(err)
}
rf := restoreLikeEngine(t, f, cfg)
if rf.extra == nil || rf.extra.Header["Referer"] != page {
t.Fatalf("restored fetcher lost the request extra headers: %+v", rf.extra)
}
if err := rf.Start(); err != nil {
t.Fatal(err)
}
if err := rf.Wait(); err != nil {
t.Fatalf("resumed download failed (request headers lost?): %v", err)
}
var want []byte
for i := 0; i < total; i++ {
want = append(want, segments[i]...)
}
if !bytes.Equal(readOutput(t, rf), want) {
t.Errorf("restored output mismatch: %q", readOutput(t, rf))
}
}
func TestFetcherE2E_PatchPartialUpdate(t *testing.T) {
page := "https://player.example.com/watch/7"
var requiredReferer atomic.Value // string
requiredReferer.Store(page)
var playlist strings.Builder
playlist.WriteString("#EXTM3U\n")
segments := make(map[string][]byte)
for i := 0; i < 3; i++ {
segments[fmt.Sprintf("/s%d.ts", i)] = []byte(fmt.Sprintf("PATCH-%d", i))
playlist.WriteString(fmt.Sprintf("#EXTINF:4,\ns%d.ts\n", i))
}
playlist.WriteString("#EXT-X-ENDLIST\n")
mux := http.NewServeMux()
mux.HandleFunc("/video.m3u8", func(w http.ResponseWriter, r *http.Request) {
w.Write([]byte(playlist.String()))
})
for path, content := range segments {
c := content
mux.HandleFunc(path, func(w http.ResponseWriter, r *http.Request) {
w.Write(c)
})
}
// The Referer requirement can be flipped mid-test to prove that a patched
// header is actually sent by the next run.
dyn := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if want, _ := requiredReferer.Load().(string); want != "" && r.Header.Get("Referer") != want {
w.WriteHeader(http.StatusForbidden)
return
}
mux.ServeHTTP(w, r)
})
server := httptest.NewServer(dyn)
defer server.Close()
f := newTestFetcher(t, testConfig())
req := &base.Request{
URL: server.URL + "/video.m3u8",
Extra: &ReqExtra{Header: map[string]string{"Referer": page}},
}
opts := &base.Options{Path: t.TempDir()}
if err := f.Resolve(req, opts); err != nil {
t.Fatal(err)
}
// A nil/nil patch must be a no-op, not a panic.
if err := f.Patch(nil, nil); err != nil {
t.Fatalf("nil patch should be accepted: %v", err)
}
// Header-only patch: the resolved plan and output options are retained
// while the new header takes effect on the next run.
newReferer := "https://player.example.com/watch/9"
if err := f.Patch(&base.Request{
URL: server.URL + "/video.m3u8",
Extra: &ReqExtra{Header: map[string]string{"Referer": newReferer}},
}, nil); err != nil {
t.Fatal(err)
}
if f.state == nil || f.meta.Res == nil {
t.Fatal("header-only patch must keep the resolved plan")
}
if f.meta.Opts.Path != opts.Path {
t.Fatalf("header-only patch must keep the output path: %s", f.meta.Opts.Path)
}
if f.extra.Header["Referer"] != newReferer {
t.Fatalf("patched header not applied: %+v", f.extra.Header)
}
requiredReferer.Store(newReferer)
if err := f.Start(); err != nil {
t.Fatal(err)
}
if err := f.Wait(); err != nil {
t.Fatalf("run with patched header failed: %v", err)
}
if got := readOutput(t, f); string(got) != "PATCH-0PATCH-1PATCH-2" {
t.Errorf("output mismatch after header patch: %q", got)
}
// Options-only patch: renaming the task keeps the resolved plan.
if err := f.Patch(nil, &base.Options{Name: "renamed.ts"}); err != nil {
t.Fatal(err)
}
if f.state == nil {
t.Fatal("options-only patch must keep the resolved plan")
}
if f.meta.Opts.Name != "renamed.ts" || f.meta.Opts.Path != opts.Path {
t.Fatalf("options patch misapplied: %+v", f.meta.Opts)
}
// URL-only patch: the plan is dropped for a re-resolve but the output
// path and renamed file survive.
if err := f.Patch(&base.Request{URL: server.URL + "/refreshed.m3u8"}, nil); err != nil {
t.Fatal(err)
}
if f.state != nil || f.meta.Res != nil {
t.Fatal("URL patch must drop the resolved plan")
}
if f.meta.Opts.Path != opts.Path || f.meta.Opts.Name != "renamed.ts" {
t.Fatal("URL patch must keep the output options")
}
}
func TestFetcherE2E_PatchWhileRunningRejected(t *testing.T) {
mux := http.NewServeMux()
var playlist strings.Builder
playlist.WriteString("#EXTM3U\n")
for i := 0; i < 6; i++ {
playlist.WriteString(fmt.Sprintf("#EXTINF:4,\nseg-%d.ts\n", i))
}
playlist.WriteString("#EXT-X-ENDLIST\n")
mux.HandleFunc("/video.m3u8", func(w http.ResponseWriter, r *http.Request) {
w.Write([]byte(playlist.String()))
})
for i := 0; i < 6; i++ {
mux.HandleFunc(fmt.Sprintf("/seg-%d.ts", i), func(w http.ResponseWriter, r *http.Request) {
time.Sleep(150 * time.Millisecond)
w.Write([]byte("X"))
})
}
server := httptest.NewServer(mux)
defer server.Close()
f := newTestFetcher(t, testConfig())
if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: t.TempDir()}); err != nil {
t.Fatal(err)
}
if err := f.Start(); err != nil {
t.Fatal(err)
}
if err := f.Patch(&base.Request{URL: server.URL + "/other.m3u8"}, nil); err == nil {
t.Fatal("patching a running task must be rejected")
}
if err := f.Pause(); err != nil {
t.Fatal(err)
}
// After the pause the patch goes through and only then drops the plan.
if err := f.Patch(&base.Request{URL: server.URL + "/other.m3u8"}, nil); err != nil {
t.Fatal(err)
}
if f.state != nil {
t.Fatal("URL patch after pause must drop the plan")
}
}
func TestFetcherE2E_RetryAfterFailure(t *testing.T) {
const total = 4
var healthy atomic.Bool
mux := http.NewServeMux()
var playlist strings.Builder
playlist.WriteString("#EXTM3U\n")
for i := 0; i < total; i++ {
playlist.WriteString(fmt.Sprintf("#EXTINF:4,\nseg-%d.ts\n", i))
}
playlist.WriteString("#EXT-X-ENDLIST\n")
mux.HandleFunc("/video.m3u8", func(w http.ResponseWriter, r *http.Request) {
w.Write([]byte(playlist.String()))
})
segments := make(map[int][]byte)
for i := 0; i < total; i++ {
segments[i] = []byte(fmt.Sprintf("RETRY-SEGMENT-%02d", i))
idx := i
mux.HandleFunc(fmt.Sprintf("/seg-%d.ts", idx), func(w http.ResponseWriter, r *http.Request) {
if idx == 1 && !healthy.Load() {
w.WriteHeader(http.StatusInternalServerError)
return
}
time.Sleep(60 * time.Millisecond)
w.Write(segments[idx])
})
}
server := httptest.NewServer(mux)
defer server.Close()
cfg := testConfig()
cfg.MaxRetries = 0 // fail the run on the first bad segment
cfg.PrefetchContentLength = false
f := newTestFetcher(t, cfg)
if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: t.TempDir()}); err != nil {
t.Fatal(err)
}
if err := f.Start(); err != nil {
t.Fatal(err)
}
if err := f.Wait(); err == nil {
t.Fatal("first run should fail while the segment endpoint is down")
}
if got := statsOf(t, f).SegmentsFailed; got == 0 {
t.Fatal("failed segments should be reported in stats")
}
// Server recovered: a fresh run must succeed and report zero failures.
healthy.Store(true)
if err := f.Start(); err != nil {
t.Fatal(err)
}
if err := f.Wait(); err != nil {
t.Fatalf("retry run failed: %v", err)
}
stats := statsOf(t, f)
if stats.SegmentsFailed != 0 {
t.Errorf("failures must reset per run, got %d", stats.SegmentsFailed)
}
if stats.SegmentsDone != total {
t.Errorf("want %d done, got %d", total, stats.SegmentsDone)
}
var want []byte
for i := 0; i < total; i++ {
want = append(want, segments[i]...)
}
if !bytes.Equal(readOutput(t, f), want) {
t.Errorf("retry output mismatch: %q", readOutput(t, f))
}
}
func TestFetcherE2E_StateMissingSelfHeal(t *testing.T) {
const total = 3
mux := http.NewServeMux()
var playlist strings.Builder
playlist.WriteString("#EXTM3U\n")
for i := 0; i < total; i++ {
playlist.WriteString(fmt.Sprintf("#EXTINF:4,\nseg-%d.ts\n", i))
}
playlist.WriteString("#EXT-X-ENDLIST\n")
mux.HandleFunc("/video.m3u8", func(w http.ResponseWriter, r *http.Request) {
w.Write([]byte(playlist.String()))
})
segments := make(map[int][]byte)
for i := 0; i < total; i++ {
segments[i] = []byte(fmt.Sprintf("SELFHEAL-%02d", i))
idx := i
mux.HandleFunc(fmt.Sprintf("/seg-%d.ts", idx), func(w http.ResponseWriter, r *http.Request) {
w.Write(segments[idx])
})
}
server := httptest.NewServer(mux)
defer server.Close()
f := newTestFetcher(t, testConfig())
if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: t.TempDir()}); err != nil {
t.Fatal(err)
}
// Simulate the crash window where the engine persisted the task with its
// resource but the HLS plan was never checkpointed.
f.state = nil
outputName := f.meta.Res.Files[0].Name
if err := f.Start(); err != nil {
t.Fatalf("start should self-heal a missing plan: %v", err)
}
if err := f.Wait(); err != nil {
t.Fatalf("self-healed run failed: %v", err)
}
if f.state == nil || len(f.state.Segments) != total {
t.Fatal("self-heal should rebuild the segment plan")
}
if f.meta.Res.Files[0].Name != outputName {
t.Fatalf("self-heal renamed the output: %s", f.meta.Res.Files[0].Name)
}
var want []byte
for i := 0; i < total; i++ {
want = append(want, segments[i]...)
}
if !bytes.Equal(readOutput(t, f), want) {
t.Errorf("self-healed output mismatch: %q", readOutput(t, f))
}
}
+100
View File
@@ -0,0 +1,100 @@
package hls
import (
"context"
"crypto/tls"
"fmt"
"io"
"net"
"net/http"
"net/http/cookiejar"
"time"
"github.com/GopeedLab/gopeed/internal/httpclient"
"github.com/GopeedLab/gopeed/pkg/base"
)
const defaultUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36"
const maxPlaylistSize = 10 << 20 // 10MB, playlists are small text files
func (f *Fetcher) timeout() time.Duration {
return time.Duration(f.config.TimeoutSeconds) * time.Second
}
// buildClient builds an HTTP client honoring the task proxy and TLS settings.
// The client has no total timeout on purpose: large segments over slow links
// must not be aborted; body stalls are handled by stallReader instead. It is
// only called while the fetcher is quiescent (Resolve/Start), never from
// inside running worker goroutines.
func (f *Fetcher) buildClient() *http.Client {
var proxy *base.RequestProxy
skipVerify := false
if req := f.meta.Req; req != nil {
proxy = req.Proxy
skipVerify = req.SkipVerifyCert
}
jar, _ := cookiejar.New(nil)
client, err := httpclient.NewClient(httpclient.Options{
Client: httpclient.ClientOptions{
Jar: jar,
},
Transport: httpclient.TransportOptions{
DialContext: (&net.Dialer{
Timeout: f.timeout(),
}).DialContext,
Proxy: f.Ctl.GetProxy(proxy),
TLSClientConfig: &tls.Config{
InsecureSkipVerify: skipVerify,
},
TLSHandshakeTimeout: f.timeout(),
},
Impersonation: httpclient.ImpersonationOptions{
Mode: httpclient.ImpersonationAuto,
Session: f.impSession,
},
})
if err != nil {
panic(fmt.Sprintf("build HTTP client: %v", err))
}
return client
}
// buildRequest builds a request with task extra headers, the default user
// agent and an optional byte range. extra may be nil.
func buildRequest(ctx context.Context, extra *ReqExtra, method, u string, body io.Reader, byterange *ByteRange) (*http.Request, error) {
req, err := http.NewRequestWithContext(ctx, method, u, body)
if err != nil {
return nil, err
}
if extra != nil {
for key, value := range extra.Header {
req.Header.Set(key, value)
}
}
if req.Header.Get("User-Agent") == "" {
req.Header.Set("User-Agent", defaultUserAgent)
}
if byterange != nil {
req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", byterange.Offset, byterange.Offset+byterange.Length-1))
}
return req, nil
}
// stallReader aborts reads that see no data for the configured timeout.
type stallReader struct {
rc io.ReadCloser
timeout time.Duration
}
func (r *stallReader) Read(p []byte) (int, error) {
timer := time.AfterFunc(r.timeout, func() {
r.rc.Close()
})
defer timer.Stop()
return r.rc.Read(p)
}
func (r *stallReader) Close() error {
return r.rc.Close()
}
+112
View File
@@ -0,0 +1,112 @@
package hls
import (
"crypto/rand"
"crypto/sha1"
"encoding/hex"
"encoding/json"
"fmt"
"io"
"os"
"strconv"
)
// journalFile is the persisted completed-segment ledger. It is bound to the
// exact download plan: a journal whose fingerprint does not match the current
// plan is discarded instead of being applied to the wrong segments.
type journalFile struct {
StagingID string `json:"stagingID"`
PlanHash string `json:"planHash"`
SegmentCount int `json:"segmentCount"`
Sizes map[string]int64 `json:"sizes"`
}
// randomStagingID returns a random identifier used to give every task its own
// staging folder, stable across app restarts through the persisted state.
func randomStagingID() string {
buf := make([]byte, 6)
if _, err := rand.Read(buf); err != nil {
// crypto/rand failing is catastrophic anyway; fall back to address
// entropy rather than crashing the fetch.
return fmt.Sprintf("%x", sha1.Sum([]byte(fmt.Sprintf("%p", buf))))[:12]
}
return hex.EncodeToString(buf)
}
// planFingerprint hashes everything that defines the download plan: the media
// URL, the ordered segments with their sequence numbers, byte ranges, keys
// and init markers. Two tasks resolve to different fingerprints even when
// they point at the same playlist URL.
func planFingerprint(state *fetcherState) string {
h := sha1.New()
io.WriteString(h, state.MediaURL+"\n")
io.WriteString(h, strconv.Itoa(len(state.Segments))+"\n")
for _, seg := range state.Segments {
fmt.Fprintf(h, "%d|%s|", seg.Sequence, seg.URI)
if seg.Byterange != nil {
fmt.Fprintf(h, "%d@%d|", seg.Byterange.Offset, seg.Byterange.Length)
}
if seg.Key != nil {
// The IV is part of the plan: same URI with a rotated IV would
// silently decrypt to garbage if the journal were reused.
fmt.Fprintf(h, "key=%s iv=%x|", seg.Key.URI, seg.Key.IV)
}
if seg.IsInit {
io.WriteString(h, "init|")
}
io.WriteString(h, "\n")
}
return hex.EncodeToString(h.Sum(nil))[:16]
}
// loadJournal reads the journal of a run, discarding anything that cannot be
// proven to belong to the current plan: foreign or legacy journals, corrupted
// files, and entries whose recorded size no longer matches the plan.
func loadJournal(run *hlsRun) error {
data, err := os.ReadFile(run.journalPath)
if err != nil {
if os.IsNotExist(err) {
return nil
}
return err
}
var file journalFile
if err := json.Unmarshal(data, &file); err != nil {
// A broken or legacy journal must not kill a restartable download.
return nil
}
if file.StagingID != run.stagingID || file.PlanHash != run.planHash || file.SegmentCount != len(run.segs) {
return nil
}
for key, size := range file.Sizes {
idx, err := strconv.ParseInt(key, 10, 64)
if err != nil || idx < 0 || idx >= int64(len(run.segs)) || size <= 0 {
continue
}
run.journal[idx] = size
}
return nil
}
// saveJournal atomically rewrites the journal through a temp file + rename so
// a crash never leaves a half-written ledger behind.
func saveJournal(run *hlsRun) error {
file := journalFile{
StagingID: run.stagingID,
PlanHash: run.planHash,
SegmentCount: len(run.segs),
Sizes: make(map[string]int64, len(run.journal)),
}
for idx, size := range run.journal {
file.Sizes[strconv.FormatInt(idx, 10)] = size
}
data, err := json.Marshal(file)
if err != nil {
return err
}
tmp := run.journalPath + ".tmp"
if err := os.WriteFile(tmp, data, 0666); err != nil {
return err
}
return os.Rename(tmp, run.journalPath)
}
@@ -0,0 +1,297 @@
package hls
import (
"bytes"
"fmt"
"net/http"
"net/http/httptest"
"os"
"testing"
"github.com/GopeedLab/gopeed/pkg/base"
)
// The validators below check real media structure, not just byte
// concatenation: TS packets are verified on the 188 byte grid with
// continuity-counter rules, PAT/PMT presence and PES start codes; fMP4
// output is verified by walking the ISO-BMFF box hierarchy.
func be32(b []byte) int {
return int(b[0])<<24 | int(b[1])<<16 | int(b[2])<<8 | int(b[3])
}
// validateTS asserts structural validity of a merged MPEG-2 TS stream.
func validateTS(t *testing.T, data []byte) {
t.Helper()
if len(data) == 0 || len(data)%188 != 0 {
t.Fatalf("TS size %d is not a positive multiple of 188", len(data))
}
type pidState struct{ cc int }
ccByPID := map[int]int{}
patSeen, pmtSeen, pesSeen := false, false, false
for off := 0; off < len(data); off += 188 {
pkt := data[off : off+188]
if pkt[0] != 0x47 {
t.Fatalf("bad TS sync byte at offset %d", off)
}
pid := (int(pkt[1]&0x1F) << 8) | int(pkt[2])
pusi := pkt[1]&0x40 != 0
afc := (pkt[3] >> 4) & 0x3
if afc == 0 {
continue // reserved
}
idx := 4
if afc&0x2 != 0 { // adaptation field present
afLen := int(pkt[4])
if afLen > 0 && pkt[5]&0x80 != 0 {
// discontinuity_indicator: restart the CC expectation
delete(ccByPID, pid)
}
idx += 1 + afLen
if afc == 2 {
continue // adaptation only, no payload: no CC rule
}
}
if idx > 188 {
t.Fatalf("adaptation field overruns packet at offset %d", off)
}
if pid == 0x1FFF {
continue // null packet
}
payload := pkt[idx:]
if pusi && len(payload) >= 3 && pid != 0 && payload[0] == 0 && payload[1] == 0 && payload[2] == 1 {
pesSeen = true
}
if pid == 0 {
if ptr := int(payload[0]); 1+ptr < len(payload) && payload[1+ptr] == 0x00 {
patSeen = true // program_association_section table_id
}
} else if pusi && len(payload) >= 2 && payload[0] == 0 && payload[1] == 0x02 {
pmtSeen = true // program_map_section table_id
}
cc := int(pkt[3] & 0x0F)
prev, ok := ccByPID[pid]
if !ok {
ccByPID[pid] = cc
continue
}
// CC increments by one per payload-bearing packet; a repeated value
// models the single legal duplicate packet.
if cc != (prev+1)%16 && cc != prev {
t.Fatalf("continuity error on PID %d: %d after %d at offset %d", pid, cc, prev, off)
}
ccByPID[pid] = cc
}
if !patSeen {
t.Error("no PAT found in merged TS")
}
if !pmtSeen {
t.Error("no PMT found in merged TS")
}
if !pesSeen {
t.Error("no PES start code found in merged TS")
}
}
// walkBoxes iterates the top-level ISO-BMFF boxes of data[off:end].
func walkBoxes(t *testing.T, data []byte, off, end int) []string {
t.Helper()
var types []string
for off+8 <= end {
size := be32(data[off:])
typ := string(data[off+4 : off+8])
if size < 8 || off+size > end {
t.Fatalf("invalid box %q at offset %d (size %d)", typ, off, size)
}
types = append(types, typ)
off += size
}
return types
}
// validateFMP4 asserts the merged fMP4 output is init section followed by
// well-formed moof+mdat fragments.
func validateFMP4(t *testing.T, merged []byte, wantFragments int) {
t.Helper()
types := walkBoxes(t, merged, 0, len(merged))
hasFtyp, hasMoov := false, false
fragments := 0
expectFragment := false
for i, typ := range types {
switch typ {
case "ftyp":
if i != 0 {
t.Errorf("ftyp not at stream start")
}
hasFtyp = true
case "moov":
if i != 1 {
t.Errorf("moov must follow ftyp, found at index %d", i)
}
hasMoov = true
case "styp":
expectFragment = true
case "moof":
if !hasMoov {
t.Errorf("fragment before moov")
}
expectFragment = true
case "mdat":
if !expectFragment {
t.Errorf("mdat outside of a fragment at index %d", i)
}
expectFragment = false
fragments++
case "mfra":
// optional trailer
default:
t.Errorf("unexpected top-level box %q", typ)
}
}
if !hasFtyp || !hasMoov {
t.Error("merged fMP4 missing ftyp/moov init section")
}
if fragments != wantFragments {
t.Errorf("want %d fragments in merged fMP4, got %d", wantFragments, fragments)
}
}
func mediaFixture(t *testing.T, name string) []byte {
t.Helper()
data, err := os.ReadFile("testdata/media/" + name)
if err != nil {
t.Fatal(err)
}
return data
}
func serveFixtureServer(t *testing.T) *httptest.Server {
t.Helper()
mux := http.NewServeMux()
fs := http.FileServer(http.Dir("testdata/media"))
mux.Handle("/", fs)
return httptest.NewServer(mux)
}
func TestHLSRealMedia_TS(t *testing.T) {
seg0 := mediaFixture(t, "sample-00.ts")
seg1 := mediaFixture(t, "sample-01.ts")
playlist := "#EXTM3U\n" +
"#EXTINF:0.5,\nsample-00.ts\n" +
"#EXTINF:0.5,\nsample-01.ts\n" +
"#EXT-X-ENDLIST\n"
mux := http.NewServeMux()
mux.HandleFunc("/video.m3u8", func(w http.ResponseWriter, r *http.Request) {
w.Write([]byte(playlist))
})
mux.Handle("/", http.FileServer(http.Dir("testdata/media")))
server := httptest.NewServer(mux)
defer server.Close()
f := newTestFetcher(t, testConfig())
if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: t.TempDir()}); err != nil {
t.Fatal(err)
}
if err := f.Start(); err != nil {
t.Fatal(err)
}
if err := f.Wait(); err != nil {
t.Fatal(err)
}
got := readOutput(t, f)
if !bytes.Equal(got, append(append([]byte{}, seg0...), seg1...)) {
t.Fatal("merged TS is not the exact concatenation of the segments")
}
validateTS(t, got)
}
func TestHLSRealMedia_FMP4(t *testing.T) {
init := mediaFixture(t, "fmp4-init.mp4")
frag1 := mediaFixture(t, "fmp4-seg1.m4s")
frag2 := mediaFixture(t, "fmp4-seg2.m4s")
frag3 := mediaFixture(t, "fmp4-seg3.m4s")
playlist := "#EXTM3U\n" +
"#EXT-X-MAP:URI=\"fmp4-init.mp4\"\n" +
"#EXTINF:0.5,\nfmp4-seg1.m4s\n" +
"#EXTINF:0.5,\nfmp4-seg2.m4s\n" +
"#EXTINF:0.5,\nfmp4-seg3.m4s\n" +
"#EXT-X-ENDLIST\n"
mux := http.NewServeMux()
mux.HandleFunc("/video.m3u8", func(w http.ResponseWriter, r *http.Request) {
w.Write([]byte(playlist))
})
mux.Handle("/", http.FileServer(http.Dir("testdata/media")))
server := httptest.NewServer(mux)
defer server.Close()
f := newTestFetcher(t, testConfig())
if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: t.TempDir()}); err != nil {
t.Fatal(err)
}
if err := f.Start(); err != nil {
t.Fatal(err)
}
if err := f.Wait(); err != nil {
t.Fatal(err)
}
got := readOutput(t, f)
want := append(append(append(append([]byte{}, init...), frag1...), frag2...), frag3...)
if !bytes.Equal(got, want) {
t.Fatal("merged fMP4 is not the exact concatenation of init + fragments")
}
validateFMP4(t, got, 3)
}
func TestHLSRealMedia_AES128(t *testing.T) {
keys := [][]byte{[]byte("key-A-0123456789"), []byte("key-B-9876543210")}
iv := []byte("iv-1337-01234567")
plain := [][]byte{mediaFixture(t, "sample-00.ts"), mediaFixture(t, "sample-01.ts")}
playlist := "#EXTM3U\n" +
"#EXT-X-MEDIA-SEQUENCE:0\n" +
"#EXT-X-KEY:METHOD=AES-128,URI=\"key-0.bin\",IV=0x" + hexEncode(iv) + "\n" +
"#EXTINF:0.5,\nenc-0.ts\n" +
"#EXT-X-KEY:METHOD=AES-128,URI=\"key-1.bin\"\n" +
"#EXTINF:0.5,\nenc-1.ts\n" +
"#EXT-X-ENDLIST\n"
mux := http.NewServeMux()
mux.HandleFunc("/video.m3u8", func(w http.ResponseWriter, r *http.Request) {
w.Write([]byte(playlist))
})
for i, key := range keys {
k := key
mux.HandleFunc(fmt.Sprintf("/key-%d.bin", i), func(w http.ResponseWriter, r *http.Request) {
w.Write(k)
})
// Segment 0 has an explicit IV, segment 1 derives it from its media
// sequence number (1).
enc := encryptSegment(plain[i], k, map[bool][]byte{true: iv, false: sequenceIV(int64(i))}[i == 0])
mux.HandleFunc(fmt.Sprintf("/enc-%d.ts", i), func(w http.ResponseWriter, r *http.Request) {
w.Write(enc)
})
}
server := httptest.NewServer(mux)
defer server.Close()
f := newTestFetcher(t, testConfig())
if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: t.TempDir()}); err != nil {
t.Fatal(err)
}
if f.meta.Res.Size != 0 {
t.Errorf("encrypted plan must report indeterminate size, got %d", f.meta.Res.Size)
}
if err := f.Start(); err != nil {
t.Fatal(err)
}
if err := f.Wait(); err != nil {
t.Fatal(err)
}
got := readOutput(t, f)
if !bytes.Equal(got, append(append([]byte{}, plain[0]...), plain[1]...)) {
t.Fatal("decrypted output mismatch")
}
validateTS(t, got)
if p := f.Progress(); p.TotalDownloaded() != int64(len(got)) {
t.Errorf("final progress %d != merged size %d", p.TotalDownloaded(), len(got))
}
}
+479
View File
@@ -0,0 +1,479 @@
package hls
import (
"encoding/hex"
"errors"
"fmt"
"net/url"
"path"
"strconv"
"strings"
)
// Variant is one rendition entry of a master playlist.
type Variant struct {
URI string
Bandwidth int64
AvgBandwidth int64
Resolution string
Codecs string
Name string
AudioGroup string // AUDIO attribute: group id of the audio renditions
}
// MediaRendition is one EXT-X-MEDIA entry of a master playlist. Only the
// fields needed to detect unsupported track layouts are kept.
type MediaRendition struct {
Type string
GroupID string
URI string
}
// MasterPlaylist is a parsed master playlist with its rendition groups.
type MasterPlaylist struct {
Variants []*Variant
Rendition map[string][]*MediaRendition // group id -> all EXT-X-MEDIA members
}
// Key is the decryption key applied to the segments that follow its tag, until
// the next EXT-X-KEY tag or the end of the playlist.
type Key struct {
Method string
URI string // absolute URI
IV []byte // explicit IV; nil means derive from the media sequence number
}
// ByteRange is an EXT-X-BYTERANGE sub range of the segment resource.
type ByteRange struct {
Length int64
Offset int64
}
// Segment is one downloadable piece of a media playlist. The init section of
// an fMP4 stream is represented as a segment with IsInit set.
type Segment struct {
URI string
Sequence int64 // media sequence number, used to derive the default AES IV
Byterange *ByteRange
Key *Key // nil means unencrypted
IsInit bool
Gap bool
Duration float64
Size int64 // exact size in bytes when known (byterange / HEAD probe), 0 when unknown
}
// Media is a parsed media playlist.
type Media struct {
Segments []*Segment
Version int
Live bool // no EXT-X-ENDLIST tag
IsFMP4 bool // playlist declares an init section via EXT-X-MAP
Discontinuity bool // playlist contains EXT-X-DISCONTINUITY
}
const (
keyMethodNone = "NONE"
keyMethodAES128 = "AES-128"
keyMethodSampleAES = "SAMPLE-AES"
)
// IsMasterPlaylist reports whether the content looks like a master playlist.
func IsMasterPlaylist(content string) bool {
return strings.Contains(content, "#EXT-X-STREAM-INF")
}
func validateM3U8Header(content string) error {
for _, line := range strings.Split(content, "\n") {
line = strings.TrimSpace(line)
if line == "" {
continue
}
if line != "#EXTM3U" {
return errors.New("content is not a valid m3u8 playlist")
}
return nil
}
return errors.New("empty m3u8 playlist")
}
// ParseMaster parses a master playlist and returns all variants.
func ParseMaster(content string, baseURL *url.URL) ([]*Variant, error) {
master, err := ParseMasterPlaylist(content, baseURL)
if err != nil {
return nil, err
}
return master.Variants, nil
}
// ParseMasterPlaylist parses a master playlist including its EXT-X-MEDIA
// rendition groups.
func ParseMasterPlaylist(content string, baseURL *url.URL) (*MasterPlaylist, error) {
if err := validateM3U8Header(content); err != nil {
return nil, err
}
master := &MasterPlaylist{Rendition: make(map[string][]*MediaRendition)}
var pending *Variant
for _, line := range strings.Split(content, "\n") {
line = strings.TrimSpace(line)
if line == "" {
continue
}
if strings.HasPrefix(line, "#EXT-X-STREAM-INF:") {
attrs := parseAttributes(line[len("#EXT-X-STREAM-INF:"):])
pending = &Variant{
Bandwidth: parseInt64(attrs["BANDWIDTH"]),
AvgBandwidth: parseInt64(attrs["AVERAGE-BANDWIDTH"]),
Resolution: attrs["RESOLUTION"],
Codecs: unquote(attrs["CODECS"]),
Name: unquote(attrs["NAME"]),
AudioGroup: unquote(attrs["AUDIO"]),
}
continue
}
if strings.HasPrefix(line, "#EXT-X-MEDIA:") {
attrs := parseAttributes(line[len("#EXT-X-MEDIA:"):])
rendition := &MediaRendition{
Type: strings.ToUpper(attrs["TYPE"]),
GroupID: unquote(attrs["GROUP-ID"]),
}
// Only an explicit URI means a separate track; resolving an empty
// reference would fabricate one out of the playlist URL.
if uri := unquote(attrs["URI"]); uri != "" {
rendition.URI = resolveURL(baseURL, uri)
}
if rendition.GroupID != "" {
// Keep every member: a group is unsupported as a whole when
// ANY of its members carries its own URI.
master.Rendition[rendition.GroupID] = append(master.Rendition[rendition.GroupID], rendition)
}
continue
}
if pending != nil {
if line[0] != '#' {
pending.URI = resolveURL(baseURL, line)
master.Variants = append(master.Variants, pending)
}
pending = nil
}
}
if len(master.Variants) == 0 {
return nil, errors.New("master playlist has no variants")
}
return master, nil
}
// checkAudioRendition rejects the selected variant when its AUDIO group
// carries a separate audio track: downloading only the video variant would
// silently produce a silent file. Groups the variant does not reference are
// ignored (variants without an AUDIO attribute are assumed self-contained).
func checkAudioRendition(master *MasterPlaylist, best *Variant) error {
if best == nil || best.AudioGroup == "" {
return nil
}
for _, rendition := range master.Rendition[best.AudioGroup] {
if rendition == nil || rendition.Type != "AUDIO" || rendition.URI == "" {
continue
}
return fmt.Errorf(
"selected variant uses a separate audio rendition (group %q), which is not supported: the merged output would have no audio",
best.AudioGroup)
}
return nil
}
// PickBestVariant selects the highest-bandwidth variant.
func PickBestVariant(variants []*Variant) *Variant {
var best *Variant
for _, v := range variants {
if best == nil || v.Bandwidth > best.Bandwidth {
best = v
}
}
return best
}
// ParseMedia parses a media playlist into an ordered segment plan.
func ParseMedia(content string, baseURL *url.URL) (*Media, error) {
if err := validateM3U8Header(content); err != nil {
return nil, err
}
media := &Media{Live: true}
var currentKey *Key
var pendingByterange *ByteRange
var pendingGap bool
var pendingDuration float64
var lastRangeEnd int64
var sequence int64
var mapIdentity string // normalized identity of the current EXT-X-MAP
for _, line := range strings.Split(content, "\n") {
line = strings.TrimSpace(line)
if line == "" {
continue
}
if line[0] != '#' {
// Every non-comment line in a media playlist is a segment URI.
seg := &Segment{
URI: resolveURL(baseURL, line),
Sequence: sequence,
Byterange: pendingByterange,
Key: currentKey,
Gap: pendingGap,
Duration: pendingDuration,
}
if pendingByterange != nil {
// Byterange segments have exact sizes straight from the playlist.
seg.Size = pendingByterange.Length
}
sequence++
pendingByterange = nil
pendingGap = false
pendingDuration = 0
media.Segments = append(media.Segments, seg)
continue
}
switch {
case strings.HasPrefix(line, "#EXTINF:"):
value := line[len("#EXTINF:"):]
if idx := strings.Index(value, ","); idx >= 0 {
value = value[:idx]
}
pendingDuration = parseFloat(value)
case strings.HasPrefix(line, "#EXT-X-KEY:"):
key, err := parseKey(line[len("#EXT-X-KEY:"):], baseURL)
if err != nil {
return nil, err
}
currentKey = key
case strings.HasPrefix(line, "#EXT-X-MAP:"):
attrs := parseAttributes(line[len("#EXT-X-MAP:"):])
uriAttr := unquote(attrs["URI"])
if uriAttr == "" {
return nil, errors.New("EXT-X-MAP without URI is not supported")
}
resolved := resolveURL(baseURL, uriAttr)
// The init section identity covers every attribute, not just the
// URI: a repeated tag with a different BYTERANGE addresses
// different bytes and changes the init section. NUL keeps the
// two parts unambiguously separated.
identity := resolved + "\x00" + attrs["BYTERANGE"]
if mapIdentity != "" {
if mapIdentity != identity {
return nil, errors.New("changing init sections are not supported")
}
media.IsFMP4 = true
continue
}
mapIdentity = identity
init := &Segment{
URI: resolved,
Sequence: sequence,
IsInit: true,
Key: currentKey,
}
if attrs["BYTERANGE"] != "" {
init.Byterange = parseByteRange(attrs["BYTERANGE"], 0)
if init.Byterange == nil {
return nil, fmt.Errorf("invalid EXT-X-MAP BYTERANGE: %s", attrs["BYTERANGE"])
}
}
if init.Key != nil && init.Key.IV == nil {
// RFC 8216 only defines sequence-derived IVs for media
// segments; guessing one for the init section produces
// corrupt output, so require the playlist to be explicit.
return nil, errors.New("encrypted EXT-X-MAP without an explicit IV is not supported")
}
// The init section is prepended to the segment plan, it takes
// the media sequence number of the playlist for IV derivation.
media.Segments = append(media.Segments, init)
media.IsFMP4 = true
case strings.HasPrefix(line, "#EXT-X-BYTERANGE:"):
pendingByterange = parseByteRange(line[len("#EXT-X-BYTERANGE:"):], lastRangeEnd)
if pendingByterange == nil {
return nil, fmt.Errorf("invalid EXT-X-BYTERANGE: %s", line)
}
case line == "#EXT-X-GAP":
pendingGap = true
case line == "#EXT-X-DISCONTINUITY":
media.Discontinuity = true
case line == "#EXT-X-ENDLIST":
media.Live = false
case strings.HasPrefix(line, "#EXT-X-MEDIA-SEQUENCE:"):
sequence = parseInt64(line[len("#EXT-X-MEDIA-SEQUENCE:"):])
case strings.HasPrefix(line, "#EXT-X-VERSION:"):
media.Version = int(parseInt64(line[len("#EXT-X-VERSION:"):]))
}
if pendingByterange != nil {
// Non-segment tags between BYTERANGE and its URI must not reset the
// running offset, so it is tracked here after each parse.
lastRangeEnd = pendingByterange.Offset + pendingByterange.Length
}
}
if len(media.Segments) == 0 {
return nil, errors.New("media playlist has no segments")
}
return media, nil
}
// validateSupported rejects playlist features the byte-concatenation merge
// cannot reproduce faithfully. Skipping such features silently would report
// success while producing a broken file.
func validateSupported(media *Media) error {
if media.Live {
return errors.New("live streams are not supported yet")
}
if media.Discontinuity {
return errors.New("EXT-X-DISCONTINUITY is not supported: the merged output would have a broken timeline")
}
for _, seg := range media.Segments {
if seg.Gap {
return errors.New("EXT-X-GAP is not supported: the missing segment would leave a hole in the merged output")
}
}
return nil
}
func parseKey(attrs string, baseURL *url.URL) (*Key, error) {
parsed := parseAttributes(attrs)
switch method := parsed["METHOD"]; method {
case keyMethodNone:
return nil, nil
case keyMethodAES128:
uri := unquote(parsed["URI"])
if uri == "" {
return nil, errors.New("AES-128 EXT-X-KEY without URI")
}
if format := unquote(parsed["KEYFORMAT"]); format != "" && format != "identity" {
return nil, fmt.Errorf("unsupported EXT-X-KEY KEYFORMAT: %s", format)
}
key := &Key{Method: keyMethodAES128, URI: resolveURL(baseURL, uri)}
if iv, ok := parsed["IV"]; ok {
value := strings.TrimPrefix(strings.ToLower(iv), "0x")
decoded, err := hex.DecodeString(value)
if err != nil || len(decoded) != 16 {
return nil, fmt.Errorf("invalid EXT-X-KEY IV: %s", iv)
}
key.IV = decoded
}
return key, nil
case keyMethodSampleAES:
return nil, errors.New("SAMPLE-AES encryption is not supported")
default:
return nil, fmt.Errorf("unsupported encryption method: %s", method)
}
}
func parseByteRange(value string, defaultOffset int64) *ByteRange {
lengthPart := value
offsetPart := ""
if idx := strings.Index(value, "@"); idx >= 0 {
lengthPart = value[:idx]
offsetPart = value[idx+1:]
}
length := parseInt64(lengthPart)
if length <= 0 {
return nil
}
offset := defaultOffset
if offsetPart != "" {
offset = parseInt64(offsetPart)
if offset < 0 {
return nil
}
}
return &ByteRange{Length: length, Offset: offset}
}
// resolveURL resolves a playlist URI reference against the playlist URL
// following RFC 3986.
func resolveURL(baseURL *url.URL, ref string) string {
refURL, err := url.Parse(ref)
if err != nil {
return ref
}
return baseURL.ResolveReference(refURL).String()
}
// parseAttributes parses an HLS attribute list like FOO="bar",NUM=42.
func parseAttributes(s string) map[string]string {
attrs := make(map[string]string)
var key, val strings.Builder
inKey := true
inQuote := false
flush := func() {
name := strings.ToUpper(strings.TrimSpace(key.String()))
if name != "" {
attrs[name] = strings.TrimSpace(val.String())
}
key.Reset()
val.Reset()
inKey = true
}
for i := 0; i < len(s); i++ {
c := s[i]
if inKey {
if c == '=' {
inKey = false
} else {
key.WriteByte(c)
}
continue
}
if c == '"' {
inQuote = !inQuote
continue
}
if c == ',' && !inQuote {
flush()
continue
}
val.WriteByte(c)
}
flush()
return attrs
}
func unquote(s string) string {
s = strings.TrimSpace(s)
if len(s) >= 2 && strings.HasPrefix(s, "\"") && strings.HasSuffix(s, "\"") {
return s[1 : len(s)-1]
}
return s
}
func parseInt64(s string) int64 {
value, _ := strconv.ParseInt(strings.TrimSpace(s), 10, 64)
return value
}
func parseFloat(s string) float64 {
value, _ := strconv.ParseFloat(strings.TrimSpace(s), 64)
return value
}
// OutputExtension returns the container file extension for the playlist.
func (m *Media) OutputExtension() string {
if m.IsFMP4 {
return ".mp4"
}
return ".ts"
}
// DeriveOutputName builds the merged output file name from a playlist URL.
func DeriveOutputName(playlistURL string, media *Media) string {
name := ""
if u, err := url.Parse(playlistURL); err == nil {
name = path.Base(u.Path)
name = strings.TrimSuffix(name, path.Ext(name))
}
if name == "" || name == "/" || name == "." {
if u, err := url.Parse(playlistURL); err == nil {
name = u.Hostname()
}
}
return name + media.OutputExtension()
}
@@ -0,0 +1,39 @@
package hls
import (
_ "embed"
"net/url"
"testing"
)
//go:embed testdata/olemovienews.m3u8
var realWorldPlaylist string
// TestParseMediaRealWorld guards parsing against a real site playlist
// (olemovienews.com VOD, 138 TS segments).
func TestParseMediaRealWorld(t *testing.T) {
base, err := url.Parse("https://europe.olemovienews.com/ts2/20230125/afAxfjit/mp4/afAxfjit.mp4/index-v1-a1.m3u8")
if err != nil {
t.Fatal(err)
}
media, err := ParseMedia(realWorldPlaylist, base)
if err != nil {
t.Fatal(err)
}
if media.Live {
t.Error("real VOD playlist should not be live")
}
if media.IsFMP4 {
t.Error("playlist should be detected as TS")
}
if len(media.Segments) != 138 {
t.Errorf("want 138 segments, got %d", len(media.Segments))
}
first := media.Segments[0]
if first.URI != "https://europe.olemovienews.com/ts2/20230125/afAxfjit/mp4/afAxfjit.mp4/seg-1-v1-a1.ts" {
t.Errorf("segment URI resolution wrong: %s", first.URI)
}
if media.Segments[0].Sequence != 1 {
t.Errorf("first sequence = %d, want 1", media.Segments[0].Sequence)
}
}
+252
View File
@@ -0,0 +1,252 @@
package hls
import (
"net/url"
"testing"
)
func mustURL(t *testing.T, raw string) *url.URL {
t.Helper()
u, err := url.Parse(raw)
if err != nil {
t.Fatal(err)
}
return u
}
func TestParseMediaTS(t *testing.T) {
content := "#EXTM3U\n" +
"#EXT-X-TARGETDURATION:20\n" +
"#EXT-X-VERSION:3\n" +
"#EXT-X-MEDIA-SEQUENCE:1\n" +
"#EXTINF:20.000,\n" +
"seg-1.ts\n" +
"#EXTINF:20.000,\n" +
"seg-2.ts\n" +
"#EXT-X-ENDLIST\n"
media, err := ParseMedia(content, mustURL(t, "https://cdn.example.com/vod/index.m3u8"))
if err != nil {
t.Fatal(err)
}
if media.Live {
t.Error("VOD playlist should not be live")
}
if media.IsFMP4 {
t.Error("TS playlist should not be fMP4")
}
if len(media.Segments) != 2 {
t.Fatalf("want 2 segments, got %d", len(media.Segments))
}
want := "https://cdn.example.com/vod/seg-1.ts"
if media.Segments[0].URI != want {
t.Errorf("want uri %s, got %s", want, media.Segments[0].URI)
}
if media.Segments[0].Sequence != 1 {
t.Errorf("want sequence 1, got %d", media.Segments[0].Sequence)
}
if media.Segments[0].Key != nil {
t.Error("segment should be unencrypted")
}
}
func TestParseMediaWithoutExtinf(t *testing.T) {
// Some playlists list segment URIs without EXTINF lines.
content := "#EXTM3U\nseg-1.ts\nseg-2.ts\n#EXT-X-ENDLIST\n"
media, err := ParseMedia(content, mustURL(t, "https://cdn.example.com/a/index.m3u8"))
if err != nil {
t.Fatal(err)
}
if len(media.Segments) != 2 {
t.Fatalf("want 2 segments, got %d", len(media.Segments))
}
}
func TestParseMediaLive(t *testing.T) {
content := "#EXTM3U\n#EXT-X-TARGETDURATION:6\n#EXTINF:6.0,\nseg-1.ts\n"
media, err := ParseMedia(content, mustURL(t, "https://cdn.example.com/live/index.m3u8"))
if err != nil {
t.Fatal(err)
}
if !media.Live {
t.Error("playlist without ENDLIST should be live")
}
}
func TestParseMediaFMP4(t *testing.T) {
content := "#EXTM3U\n" +
"#EXT-X-VERSION:6\n" +
"#EXT-X-MAP:URI=\"init.mp4\"\n" +
"#EXTINF:5.0,\n" +
"seg-1.m4s\n" +
"#EXT-X-ENDLIST\n"
media, err := ParseMedia(content, mustURL(t, "https://cdn.example.com/vod/index.m3u8"))
if err != nil {
t.Fatal(err)
}
if !media.IsFMP4 {
t.Error("playlist with MAP should be fMP4")
}
if len(media.Segments) != 2 {
t.Fatalf("want 2 segments (init + 1), got %d", len(media.Segments))
}
if !media.Segments[0].IsInit {
t.Error("first segment should be the init section")
}
if media.Segments[0].URI != "https://cdn.example.com/vod/init.mp4" {
t.Errorf("init uri wrong: %s", media.Segments[0].URI)
}
if media.OutputExtension() != ".mp4" {
t.Errorf("want .mp4, got %s", media.OutputExtension())
}
}
func TestParseMediaMultiMapError(t *testing.T) {
content := "#EXTM3U\n" +
"#EXT-X-MAP:URI=\"init1.mp4\"\n" +
"#EXTINF:5.0,\nseg-1.m4s\n" +
"#EXT-X-DISCONTINUITY\n" +
"#EXT-X-MAP:URI=\"init2.mp4\"\n" +
"#EXTINF:5.0,\nseg-2.m4s\n" +
"#EXT-X-ENDLIST\n"
if _, err := ParseMedia(content, mustURL(t, "https://cdn.example.com/vod/index.m3u8")); err == nil {
t.Error("multi-period fMP4 should be rejected")
}
}
func TestParseMediaKeyRotation(t *testing.T) {
content := "#EXTM3U\n" +
"#EXT-X-KEY:METHOD=AES-128,URI=\"key1.bin\",IV=0x00000000000000000000000000000001\n" +
"#EXTINF:5.0,\nseg-1.ts\n" +
"#EXT-X-KEY:METHOD=AES-128,URI=\"key2.bin\"\n" +
"#EXTINF:5.0,\nseg-2.ts\n" +
"#EXT-X-KEY:METHOD=NONE\n" +
"#EXTINF:5.0,\nseg-3.ts\n" +
"#EXT-X-ENDLIST\n"
media, err := ParseMedia(content, mustURL(t, "https://cdn.example.com/vod/index.m3u8"))
if err != nil {
t.Fatal(err)
}
if media.Segments[0].Key == nil || media.Segments[0].Key.URI != "https://cdn.example.com/vod/key1.bin" {
t.Fatalf("segment 1 key wrong: %+v", media.Segments[0].Key)
}
if len(media.Segments[0].Key.IV) != 16 || media.Segments[0].Key.IV[15] != 1 {
t.Error("explicit IV not parsed")
}
if media.Segments[1].Key == nil || media.Segments[1].Key.URI != "https://cdn.example.com/vod/key2.bin" {
t.Fatalf("segment 2 key wrong: %+v", media.Segments[1].Key)
}
if media.Segments[1].Key.IV != nil {
t.Error("key without IV should leave IV nil (sequence-derived)")
}
if media.Segments[2].Key != nil {
t.Error("segment 3 should be unencrypted after METHOD=NONE")
}
}
func TestParseMediaSampleAESError(t *testing.T) {
content := "#EXTM3U\n#EXT-X-KEY:METHOD=SAMPLE-AES,URI=\"skd://key\"\n#EXTINF:5.0,\nseg-1.ts\n"
if _, err := ParseMedia(content, mustURL(t, "https://cdn.example.com/vod/index.m3u8")); err == nil {
t.Error("SAMPLE-AES should be rejected")
}
}
func TestParseMediaByterange(t *testing.T) {
content := "#EXTM3U\n" +
"#EXT-X-BYTERANGE:1000@0\nseg.ts\n" +
"#EXT-X-BYTERANGE:2000\nseg.ts\n" +
"#EXT-X-BYTERANGE:3000\nseg.ts\n" +
"#EXT-X-ENDLIST\n"
media, err := ParseMedia(content, mustURL(t, "https://cdn.example.com/vod/index.m3u8"))
if err != nil {
t.Fatal(err)
}
ranges := [][2]int64{{0, 1000}, {1000, 2000}, {3000, 3000}}
for i, want := range ranges {
br := media.Segments[i].Byterange
if br == nil {
t.Fatalf("segment %d byterange missing", i)
}
if br.Offset != want[0] || br.Length != want[1] {
t.Errorf("segment %d: want offset %d length %d, got %d/%d", i, want[0], want[1], br.Offset, br.Length)
}
}
}
func TestParseMediaGap(t *testing.T) {
content := "#EXTM3U\n#EXTINF:5.0,\nseg-1.ts\n#EXT-X-GAP\n#EXTINF:5.0,\nseg-2.ts\n#EXT-X-ENDLIST\n"
media, err := ParseMedia(content, mustURL(t, "https://cdn.example.com/vod/index.m3u8"))
if err != nil {
t.Fatal(err)
}
if media.Segments[0].Gap || !media.Segments[1].Gap {
t.Error("gap flags wrong")
}
}
func TestParseMediaInvalidHeader(t *testing.T) {
if _, err := ParseMedia("not a playlist", mustURL(t, "https://cdn.example.com/a.m3u8")); err == nil {
t.Error("invalid content should be rejected")
}
}
func TestParseMasterPickBest(t *testing.T) {
content := "#EXTM3U\n" +
"#EXT-X-MEDIA:TYPE=AUDIO,GROUP-ID=\"aud\",NAME=\"en\",DEFAULT=YES,URI=\"audio/index.m3u8\"\n" +
"#EXT-X-STREAM-INF:BANDWIDTH=800000,RESOLUTION=640x360\n" +
"360p/index.m3u8\n" +
"#EXT-X-STREAM-INF:BANDWIDTH=2500000,RESOLUTION=1920x1080,CODECS=\"avc1.640028,mp4a.40.2\"\n" +
"1080p/index.m3u8?sign=abc\n" +
"#EXT-X-ENDLIST\n"
variants, err := ParseMaster(content, mustURL(t, "https://cdn.example.com/vod/master.m3u8"))
if err != nil {
t.Fatal(err)
}
if len(variants) != 2 {
t.Fatalf("want 2 variants, got %d", len(variants))
}
best := PickBestVariant(variants)
if best.Bandwidth != 2500000 {
t.Errorf("want highest bandwidth variant, got %d", best.Bandwidth)
}
if best.URI != "https://cdn.example.com/vod/1080p/index.m3u8?sign=abc" {
t.Errorf("variant uri wrong: %s", best.URI)
}
}
func TestResolveURLQueryNotInherited(t *testing.T) {
base := mustURL(t, "https://cdn.example.com/vod/index.m3u8?sign=abc")
got := resolveURL(base, "seg-1.ts")
if got != "https://cdn.example.com/vod/seg-1.ts" {
t.Errorf("relative resolution should follow RFC3986 (no query inheritance), got %s", got)
}
}
func TestDeriveOutputName(t *testing.T) {
media := &Media{}
if got := DeriveOutputName("https://cdn.example.com/ts2/a/index-v1-a1.m3u8", media); got != "index-v1-a1.ts" {
t.Errorf("want index-v1-a1.ts, got %s", got)
}
fmp4 := &Media{IsFMP4: true}
if got := DeriveOutputName("https://cdn.example.com/ts2/a/index.m3u8", fmp4); got != "index.mp4" {
t.Errorf("want index.mp4, got %s", got)
}
if got := DeriveOutputName("https://cdn.example.com/play?token=x", media); got != "play.ts" {
t.Errorf("want play.ts, got %s", got)
}
if got := DeriveOutputName("https://cdn.example.com/", media); got != "cdn.example.com.ts" {
t.Errorf("want host fallback, got %s", got)
}
}
func TestParseAttributesQuotedComma(t *testing.T) {
attrs := parseAttributes(`URI="a,b.ts",IV=0xABCDEF00000000000000000000000001,NAME="x"`)
if attrs["URI"] != "a,b.ts" {
t.Errorf("quoted comma value wrong: %q", attrs["URI"])
}
if attrs["IV"] != "0xABCDEF00000000000000000000000001" {
t.Errorf("hex value wrong: %q", attrs["IV"])
}
if attrs["NAME"] != "x" {
t.Errorf("quoted value wrong: %q", attrs["NAME"])
}
}
+296
View File
@@ -0,0 +1,296 @@
package hls
import (
"bytes"
"context"
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"sync/atomic"
"testing"
"time"
"github.com/GopeedLab/gopeed/pkg/base"
)
func TestFetcherE2E_SameURLTwoTasks(t *testing.T) {
const total = 6
mux := http.NewServeMux()
var playlist strings.Builder
playlist.WriteString("#EXTM3U\n")
for i := 0; i < total; i++ {
playlist.WriteString(fmt.Sprintf("#EXTINF:4,\nseg-%d.ts\n", i))
}
playlist.WriteString("#EXT-X-ENDLIST\n")
mux.HandleFunc("/video.m3u8", func(w http.ResponseWriter, r *http.Request) {
w.Write([]byte(playlist.String()))
})
segments := make(map[int][]byte)
for i := 0; i < total; i++ {
segments[i] = []byte(fmt.Sprintf("SHARED-SEGMENT-%02d", i))
idx := i
mux.HandleFunc(fmt.Sprintf("/seg-%d.ts", idx), func(w http.ResponseWriter, r *http.Request) {
time.Sleep(120 * time.Millisecond)
w.Write(segments[idx])
})
}
server := httptest.NewServer(mux)
defer server.Close()
cfg := testConfig()
cfg.SegmentConnections = 2
cfg.PrefetchContentLength = false
dir := t.TempDir()
// Two tasks for the same playlist URL into the same directory, like a
// user duplicating a task. The engine would rename the second output
// (AutoRename); mirror that here.
f1 := newTestFetcher(t, cfg)
if err := f1.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: dir}); err != nil {
t.Fatal(err)
}
f2 := newTestFetcher(t, cfg)
if err := f2.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: dir, Name: "video-2.ts"}); err != nil {
t.Fatal(err)
}
if f1.state.StagingDir == "" || f1.state.StagingDir == f2.state.StagingDir {
t.Fatalf("staging dirs must be distinct per task: %q vs %q", f1.state.StagingDir, f2.state.StagingDir)
}
// Start task 1, let it stage some segments, then pause it.
if err := f1.Start(); err != nil {
t.Fatal(err)
}
time.Sleep(250 * time.Millisecond)
if err := f1.Pause(); err != nil {
t.Fatal(err)
}
staged1, err := os.ReadDir(f1.state.StagingDir)
if err != nil || len(staged1) == 0 {
t.Fatalf("task 1 should have staged segments, dir: %s", f1.state.StagingDir)
}
// Task 2 must complete without touching task 1's staging data...
if err := f2.Start(); err != nil {
t.Fatal(err)
}
if err := f2.Wait(); err != nil {
t.Fatal(err)
}
stillThere, err := os.ReadDir(f1.state.StagingDir)
if err != nil || len(stillThere) != len(staged1) {
t.Fatalf("task 2 disturbed task 1 staging: had %d entries, now %d", len(staged1), len(stillThere))
}
// ...and task 1 resumes to a complete, correct output of its own.
if err := f1.Start(); err != nil {
t.Fatal(err)
}
if err := f1.Wait(); err != nil {
t.Fatal(err)
}
var want []byte
for i := 0; i < total; i++ {
want = append(want, segments[i]...)
}
if !bytes.Equal(readOutput(t, f1), want) || !bytes.Equal(readOutput(t, f2), want) {
t.Error("same-URL task outputs mismatch")
}
entries, _ := os.ReadDir(dir)
if len(entries) != 2 {
names := make([]string, 0, len(entries))
for _, e := range entries {
names = append(names, e.Name())
}
t.Errorf("expected only the two outputs to remain, got %v", names)
}
}
func TestFetcherE2E_ForeignJournalDiscarded(t *testing.T) {
const total = 3
var served atomic.Int64
mux := http.NewServeMux()
var playlist strings.Builder
playlist.WriteString("#EXTM3U\n")
for i := 0; i < total; i++ {
playlist.WriteString(fmt.Sprintf("#EXTINF:4,\nseg-%d.ts\n", i))
}
playlist.WriteString("#EXT-X-ENDLIST\n")
mux.HandleFunc("/video.m3u8", func(w http.ResponseWriter, r *http.Request) {
w.Write([]byte(playlist.String()))
})
for i := 0; i < total; i++ {
mux.HandleFunc(fmt.Sprintf("/seg-%d.ts", i), func(w http.ResponseWriter, r *http.Request) {
served.Add(1)
w.Write([]byte("DATA"))
})
}
server := httptest.NewServer(mux)
defer server.Close()
cfg := testConfig()
cfg.PrefetchContentLength = false
f := newTestFetcher(t, cfg)
if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: t.TempDir()}); err != nil {
t.Fatal(err)
}
// Plant a journal that does not belong to this plan (wrong fingerprint).
tempDir := f.resolveStagingDirLocked()
if err := os.MkdirAll(tempDir, 0777); err != nil {
t.Fatal(err)
}
foreign := journalFile{
StagingID: "some-other-task",
PlanHash: "deadbeef",
SegmentCount: total,
Sizes: map[string]int64{"0": 4, "1": 4, "2": 4},
}
data, _ := json.Marshal(foreign)
if err := os.WriteFile(filepath.Join(tempDir, "journal.json"), data, 0666); err != nil {
t.Fatal(err)
}
if err := f.Start(); err != nil {
t.Fatal(err)
}
if err := f.Wait(); err != nil {
t.Fatal(err)
}
if got := served.Load(); got != total {
t.Errorf("foreign journal must be discarded, want %d downloads, got %d", total, got)
}
}
func TestFetcherE2E_MergeFailurePreservesStaging(t *testing.T) {
mux := http.NewServeMux()
mux.HandleFunc("/video.m3u8", func(w http.ResponseWriter, r *http.Request) {
w.Write([]byte("#EXTM3U\n#EXTINF:4,\na.ts\n#EXTINF:4,\nb.ts\n#EXT-X-ENDLIST\n"))
})
mux.HandleFunc("/a.ts", func(w http.ResponseWriter, r *http.Request) { w.Write([]byte("AAAA")) })
mux.HandleFunc("/b.ts", func(w http.ResponseWriter, r *http.Request) { w.Write([]byte("BBBB")) })
server := httptest.NewServer(mux)
defer server.Close()
f := newTestFetcher(t, testConfig())
if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: t.TempDir()}); err != nil {
t.Fatal(err)
}
run := &hlsRun{
ctx: context.Background(),
cancel: func() {},
done: make(chan struct{}),
segs: f.state.Segments,
outputPath: f.meta.SingleFilepath(),
stagingID: f.state.StagingID,
planHash: planFingerprint(f.state),
tempDir: f.resolveStagingDirLocked(),
keyCache: make(map[string][]byte),
journal: make(map[int64]int64),
}
run.journalPath = filepath.Join(run.tempDir, "journal.json")
if err := os.MkdirAll(run.tempDir, 0777); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(segmentPath(run.tempDir, 0), []byte("AAAA"), 0666); err != nil {
t.Fatal(err)
}
// Segment 1 is missing on disk: the merge must fail without deleting the
// staged data or leaving a partial output behind.
if err := f.merge(run); err == nil {
t.Fatal("merge with a missing segment should fail")
}
if _, err := os.Stat(segmentPath(run.tempDir, 0)); err != nil {
t.Fatal("merge failure must preserve staged segments")
}
entries, _ := os.ReadDir(filepath.Dir(run.outputPath))
for _, e := range entries {
if strings.HasSuffix(e.Name(), ".part") {
t.Errorf("failed merge left a partial output behind: %s", e.Name())
}
}
// Recover the segment: the retried merge succeeds.
if err := os.WriteFile(segmentPath(run.tempDir, 1), []byte("BBBB"), 0666); err != nil {
t.Fatal(err)
}
if err := f.merge(run); err != nil {
t.Fatalf("retried merge failed: %v", err)
}
got, err := os.ReadFile(run.outputPath)
if err != nil {
t.Fatal(err)
}
if string(got) != "AAAABBBB" {
t.Errorf("retried merge output mismatch: %q", got)
}
}
func TestFetcherE2E_ReMergeOverExistingOutput(t *testing.T) {
const total = 2
mux := http.NewServeMux()
var playlist strings.Builder
playlist.WriteString("#EXTM3U\n")
for i := 0; i < total; i++ {
playlist.WriteString(fmt.Sprintf("#EXTINF:4,\nseg-%d.ts\n", i))
}
playlist.WriteString("#EXT-X-ENDLIST\n")
mux.HandleFunc("/video.m3u8", func(w http.ResponseWriter, r *http.Request) {
w.Write([]byte(playlist.String()))
})
for i := 0; i < total; i++ {
mux.HandleFunc(fmt.Sprintf("/seg-%d.ts", i), func(w http.ResponseWriter, r *http.Request) {
w.Write([]byte("DATA"))
})
}
server := httptest.NewServer(mux)
defer server.Close()
f := newTestFetcher(t, testConfig())
if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: t.TempDir()}); err != nil {
t.Fatal(err)
}
tempDir := f.resolveStagingDirLocked()
if err := os.MkdirAll(tempDir, 0777); err != nil {
t.Fatal(err)
}
// Simulate the crash window after a completed rename but before the task
// was marked done: a stale output file, a complete journal and all
// segments staged.
stale := filepath.Join(filepath.Dir(f.meta.SingleFilepath()), f.state.OutputName)
if err := os.WriteFile(stale, []byte("STALE-OUTPUT"), 0666); err != nil {
t.Fatal(err)
}
jr := &hlsRun{
segs: f.state.Segments,
stagingID: f.state.StagingID,
planHash: planFingerprint(f.state),
journalPath: filepath.Join(tempDir, "journal.json"),
journal: map[int64]int64{0: 4, 1: 4},
}
if err := saveJournal(jr); err != nil {
t.Fatal(err)
}
for i := 0; i < total; i++ {
if err := os.WriteFile(segmentPath(tempDir, int64(i)), []byte("DATA"), 0666); err != nil {
t.Fatal(err)
}
}
if err := f.Start(); err != nil {
t.Fatal(err)
}
if err := f.Wait(); err != nil {
t.Fatalf("re-merge over an existing output failed: %v", err)
}
got, err := os.ReadFile(stale)
if err != nil {
t.Fatal(err)
}
if string(got) != "DATADATA" {
t.Errorf("stale output was not replaced: %q", got)
}
}
+146
View File
@@ -0,0 +1,146 @@
package hls
import (
"net/url"
"strings"
"testing"
)
func mustParseURL(t *testing.T, raw string) *url.URL {
t.Helper()
u, err := url.Parse(raw)
if err != nil {
t.Fatal(err)
}
return u
}
// subsetCase is a media playlist plus the error substring the pipeline must
// reject it with; wantOK playlists must parse AND validate cleanly.
type subsetCase struct {
name string
playlist string
wantErr string
}
func TestSubsetRejections(t *testing.T) {
cases := []subsetCase{
{
name: "gap",
playlist: "#EXTM3U\n#EXTINF:4,\na.ts\n#EXT-X-GAP\n#EXTINF:4,\nb.ts\n#EXT-X-ENDLIST\n",
wantErr: "EXT-X-GAP",
},
{
name: "gap-before-first-segment",
playlist: "#EXTM3U\n#EXT-X-GAP\n#EXTINF:4,\na.ts\n#EXT-X-ENDLIST\n",
wantErr: "EXT-X-GAP",
},
{
name: "discontinuity",
playlist: "#EXTM3U\n#EXTINF:4,\na.ts\n#EXT-X-DISCONTINUITY\n#EXTINF:4,\nb.ts\n#EXT-X-ENDLIST\n",
wantErr: "EXT-X-DISCONTINUITY",
},
{
name: "live",
playlist: "#EXTM3U\n#EXTINF:4,\na.ts\n",
wantErr: "live",
},
{
name: "keyformat",
playlist: "#EXTM3U\n#EXT-X-KEY:METHOD=AES-128,URI=\"k\",KEYFORMAT=\"com.apple.streamingkeydelivery\"\n#EXTINF:4,\na.ts\n#EXT-X-ENDLIST\n",
wantErr: "KEYFORMAT",
},
{
name: "encrypted-map-without-iv",
playlist: "#EXTM3U\n#EXT-X-KEY:METHOD=AES-128,URI=\"k\"\n#EXT-X-MAP:URI=\"init.mp4\"\n#EXTINF:4,\na.m4s\n#EXT-X-ENDLIST\n",
wantErr: "explicit IV",
},
{
name: "map-change-byterange",
playlist: "#EXTM3U\n#EXT-X-MAP:URI=\"init.mp4\",BYTERANGE=\"100@0\"\n#EXTINF:4,\na.m4s\n#EXT-X-MAP:URI=\"init.mp4\",BYTERANGE=\"100@100\"\n#EXTINF:4,\nb.m4s\n#EXT-X-ENDLIST\n",
wantErr: "changing init sections",
},
{
name: "map-change-uri",
playlist: "#EXTM3U\n#EXT-X-MAP:URI=\"init1.mp4\"\n#EXTINF:4,\na.m4s\n#EXT-X-MAP:URI=\"init2.mp4\"\n#EXTINF:4,\nb.m4s\n#EXT-X-ENDLIST\n",
wantErr: "changing init sections",
},
}
baseURL := mustParseURL(t, "https://cdn.example.com/vod/index.m3u8")
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
media, parseErr := ParseMedia(tc.playlist, baseURL)
if parseErr != nil {
if !strings.Contains(parseErr.Error(), tc.wantErr) {
t.Fatalf("parse error %q does not mention %q", parseErr, tc.wantErr)
}
return
}
err := validateSupported(media)
if err == nil {
t.Fatalf("playlist should be rejected with %q", tc.wantErr)
}
if !strings.Contains(err.Error(), tc.wantErr) {
t.Fatalf("error %q does not mention %q", err, tc.wantErr)
}
})
}
}
func TestSubsetRepeatedSameMapAllowed(t *testing.T) {
playlist := "#EXTM3U\n" +
"#EXT-X-MAP:URI=\"init.mp4\",BYTERANGE=\"100@0\"\n" +
"#EXTINF:4,\na.m4s\n" +
"#EXT-X-MAP:URI=\"init.mp4\",BYTERANGE=\"100@0\"\n" +
"#EXTINF:4,\nb.m4s\n" +
"#EXT-X-ENDLIST\n"
media, err := ParseMedia(playlist, mustParseURL(t, "https://cdn.example.com/vod/index.m3u8"))
if err != nil {
t.Fatal(err)
}
if err := validateSupported(media); err != nil {
t.Fatalf("identical repeated MAP must be allowed: %v", err)
}
if len(media.Segments) != 3 { // init + 2 segments, no duplicate init
t.Fatalf("want 3 segments (init + 2), got %d", len(media.Segments))
}
}
func TestMasterAudioRenditionDetection(t *testing.T) {
masterURL := mustParseURL(t, "https://cdn.example.com/master.m3u8")
separateAudio := "#EXTM3U\n" +
"#EXT-X-MEDIA:TYPE=AUDIO,GROUP-ID=\"aud\",NAME=\"en\",DEFAULT=YES,URI=\"audio/index.m3u8\"\n" +
"#EXT-X-STREAM-INF:BANDWIDTH=1000000,AUDIO=\"aud\"\nvideo/index.m3u8\n"
master, err := ParseMasterPlaylist(separateAudio, masterURL)
if err != nil {
t.Fatal(err)
}
if err := checkAudioRendition(master, PickBestVariant(master.Variants)); err == nil {
t.Fatal("selected variant with a separate audio rendition must be rejected")
}
// A group the variant does not reference must not cause a false positive.
unrelatedGroup := "#EXTM3U\n" +
"#EXT-X-MEDIA:TYPE=AUDIO,GROUP-ID=\"aud\",NAME=\"en\",DEFAULT=YES,URI=\"audio/index.m3u8\"\n" +
"#EXT-X-STREAM-INF:BANDWIDTH=1000000\nvideo/index.m3u8\n"
master, err = ParseMasterPlaylist(unrelatedGroup, masterURL)
if err != nil {
t.Fatal(err)
}
if err := checkAudioRendition(master, PickBestVariant(master.Variants)); err != nil {
t.Fatalf("unreferenced audio group must not be rejected: %v", err)
}
// A rendition group without URIs (muxed tracks) is fine.
groupWithoutURI := "#EXTM3U\n" +
"#EXT-X-MEDIA:TYPE=AUDIO,GROUP-ID=\"aud\",NAME=\"en\",DEFAULT=YES\n" +
"#EXT-X-STREAM-INF:BANDWIDTH=1000000,AUDIO=\"aud\"\nvideo/index.m3u8\n"
master, err = ParseMasterPlaylist(groupWithoutURI, masterURL)
if err != nil {
t.Fatal(err)
}
if err := checkAudioRendition(master, PickBestVariant(master.Variants)); err != nil {
t.Fatalf("audio group without URI must not be rejected: %v", err)
}
}
+10
View File
@@ -0,0 +1,10 @@
Media fixtures generated locally with FFmpeg 7.0.2 (testsrc + sine sources,
no third-party content):
- sample-00.ts / sample-01.ts: one continuous MPEG-2 TS encode split at a
packet boundary, so continuity counters and PCR are consistent across the
segment join exactly like segments cut from a single real-time stream.
- fmp4-init.mp4: ftyp+moov init section of a fragmented MP4.
- fmp4-seg1.m4s .. fmp4-seg3.m4s: moof+mdat media fragments (styp omitted by
the muxer, which is legal) split at moof boundaries from one fragmented MP4
(-movflags +frag_keyframe+empty_moov+default_base_moof -frag_duration 500000).
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
+283
View File
@@ -0,0 +1,283 @@
#EXTM3U
#EXT-X-TARGETDURATION:20
#EXT-X-ALLOW-CACHE:YES
#EXT-X-PLAYLIST-TYPE:VOD
#EXT-X-VERSION:3
#EXT-X-MEDIA-SEQUENCE:1
#EXTINF:20.000,
seg-1-v1-a1.ts
#EXTINF:20.000,
seg-2-v1-a1.ts
#EXTINF:20.000,
seg-3-v1-a1.ts
#EXTINF:20.000,
seg-4-v1-a1.ts
#EXTINF:20.000,
seg-5-v1-a1.ts
#EXTINF:20.000,
seg-6-v1-a1.ts
#EXTINF:20.000,
seg-7-v1-a1.ts
#EXTINF:20.000,
seg-8-v1-a1.ts
#EXTINF:20.000,
seg-9-v1-a1.ts
#EXTINF:20.000,
seg-10-v1-a1.ts
#EXTINF:20.000,
seg-11-v1-a1.ts
#EXTINF:20.000,
seg-12-v1-a1.ts
#EXTINF:20.000,
seg-13-v1-a1.ts
#EXTINF:20.000,
seg-14-v1-a1.ts
#EXTINF:20.000,
seg-15-v1-a1.ts
#EXTINF:20.000,
seg-16-v1-a1.ts
#EXTINF:20.000,
seg-17-v1-a1.ts
#EXTINF:20.000,
seg-18-v1-a1.ts
#EXTINF:20.000,
seg-19-v1-a1.ts
#EXTINF:20.000,
seg-20-v1-a1.ts
#EXTINF:20.000,
seg-21-v1-a1.ts
#EXTINF:20.000,
seg-22-v1-a1.ts
#EXTINF:20.000,
seg-23-v1-a1.ts
#EXTINF:20.000,
seg-24-v1-a1.ts
#EXTINF:20.000,
seg-25-v1-a1.ts
#EXTINF:20.000,
seg-26-v1-a1.ts
#EXTINF:20.000,
seg-27-v1-a1.ts
#EXTINF:20.000,
seg-28-v1-a1.ts
#EXTINF:20.000,
seg-29-v1-a1.ts
#EXTINF:20.000,
seg-30-v1-a1.ts
#EXTINF:20.000,
seg-31-v1-a1.ts
#EXTINF:20.000,
seg-32-v1-a1.ts
#EXTINF:20.000,
seg-33-v1-a1.ts
#EXTINF:20.000,
seg-34-v1-a1.ts
#EXTINF:20.000,
seg-35-v1-a1.ts
#EXTINF:20.000,
seg-36-v1-a1.ts
#EXTINF:20.000,
seg-37-v1-a1.ts
#EXTINF:20.000,
seg-38-v1-a1.ts
#EXTINF:20.000,
seg-39-v1-a1.ts
#EXTINF:20.000,
seg-40-v1-a1.ts
#EXTINF:20.000,
seg-41-v1-a1.ts
#EXTINF:20.000,
seg-42-v1-a1.ts
#EXTINF:20.000,
seg-43-v1-a1.ts
#EXTINF:20.000,
seg-44-v1-a1.ts
#EXTINF:20.000,
seg-45-v1-a1.ts
#EXTINF:20.000,
seg-46-v1-a1.ts
#EXTINF:20.000,
seg-47-v1-a1.ts
#EXTINF:20.000,
seg-48-v1-a1.ts
#EXTINF:20.000,
seg-49-v1-a1.ts
#EXTINF:20.000,
seg-50-v1-a1.ts
#EXTINF:20.000,
seg-51-v1-a1.ts
#EXTINF:20.000,
seg-52-v1-a1.ts
#EXTINF:20.000,
seg-53-v1-a1.ts
#EXTINF:20.000,
seg-54-v1-a1.ts
#EXTINF:20.000,
seg-55-v1-a1.ts
#EXTINF:20.000,
seg-56-v1-a1.ts
#EXTINF:20.000,
seg-57-v1-a1.ts
#EXTINF:20.000,
seg-58-v1-a1.ts
#EXTINF:20.000,
seg-59-v1-a1.ts
#EXTINF:20.000,
seg-60-v1-a1.ts
#EXTINF:20.000,
seg-61-v1-a1.ts
#EXTINF:20.000,
seg-62-v1-a1.ts
#EXTINF:20.000,
seg-63-v1-a1.ts
#EXTINF:20.000,
seg-64-v1-a1.ts
#EXTINF:20.000,
seg-65-v1-a1.ts
#EXTINF:20.000,
seg-66-v1-a1.ts
#EXTINF:20.000,
seg-67-v1-a1.ts
#EXTINF:20.000,
seg-68-v1-a1.ts
#EXTINF:20.000,
seg-69-v1-a1.ts
#EXTINF:20.000,
seg-70-v1-a1.ts
#EXTINF:20.000,
seg-71-v1-a1.ts
#EXTINF:20.000,
seg-72-v1-a1.ts
#EXTINF:20.000,
seg-73-v1-a1.ts
#EXTINF:20.000,
seg-74-v1-a1.ts
#EXTINF:20.000,
seg-75-v1-a1.ts
#EXTINF:20.000,
seg-76-v1-a1.ts
#EXTINF:20.000,
seg-77-v1-a1.ts
#EXTINF:20.000,
seg-78-v1-a1.ts
#EXTINF:20.000,
seg-79-v1-a1.ts
#EXTINF:20.000,
seg-80-v1-a1.ts
#EXTINF:20.000,
seg-81-v1-a1.ts
#EXTINF:20.000,
seg-82-v1-a1.ts
#EXTINF:20.000,
seg-83-v1-a1.ts
#EXTINF:20.000,
seg-84-v1-a1.ts
#EXTINF:20.000,
seg-85-v1-a1.ts
#EXTINF:20.000,
seg-86-v1-a1.ts
#EXTINF:20.000,
seg-87-v1-a1.ts
#EXTINF:20.000,
seg-88-v1-a1.ts
#EXTINF:20.000,
seg-89-v1-a1.ts
#EXTINF:20.000,
seg-90-v1-a1.ts
#EXTINF:20.000,
seg-91-v1-a1.ts
#EXTINF:20.000,
seg-92-v1-a1.ts
#EXTINF:20.000,
seg-93-v1-a1.ts
#EXTINF:20.000,
seg-94-v1-a1.ts
#EXTINF:20.000,
seg-95-v1-a1.ts
#EXTINF:20.000,
seg-96-v1-a1.ts
#EXTINF:20.000,
seg-97-v1-a1.ts
#EXTINF:20.000,
seg-98-v1-a1.ts
#EXTINF:20.000,
seg-99-v1-a1.ts
#EXTINF:20.000,
seg-100-v1-a1.ts
#EXTINF:20.000,
seg-101-v1-a1.ts
#EXTINF:20.000,
seg-102-v1-a1.ts
#EXTINF:20.000,
seg-103-v1-a1.ts
#EXTINF:20.000,
seg-104-v1-a1.ts
#EXTINF:20.000,
seg-105-v1-a1.ts
#EXTINF:20.000,
seg-106-v1-a1.ts
#EXTINF:20.000,
seg-107-v1-a1.ts
#EXTINF:20.000,
seg-108-v1-a1.ts
#EXTINF:20.000,
seg-109-v1-a1.ts
#EXTINF:20.000,
seg-110-v1-a1.ts
#EXTINF:20.000,
seg-111-v1-a1.ts
#EXTINF:20.000,
seg-112-v1-a1.ts
#EXTINF:20.000,
seg-113-v1-a1.ts
#EXTINF:20.000,
seg-114-v1-a1.ts
#EXTINF:20.000,
seg-115-v1-a1.ts
#EXTINF:20.000,
seg-116-v1-a1.ts
#EXTINF:20.000,
seg-117-v1-a1.ts
#EXTINF:20.000,
seg-118-v1-a1.ts
#EXTINF:20.000,
seg-119-v1-a1.ts
#EXTINF:20.000,
seg-120-v1-a1.ts
#EXTINF:20.000,
seg-121-v1-a1.ts
#EXTINF:20.000,
seg-122-v1-a1.ts
#EXTINF:20.000,
seg-123-v1-a1.ts
#EXTINF:20.000,
seg-124-v1-a1.ts
#EXTINF:20.000,
seg-125-v1-a1.ts
#EXTINF:20.000,
seg-126-v1-a1.ts
#EXTINF:20.000,
seg-127-v1-a1.ts
#EXTINF:20.000,
seg-128-v1-a1.ts
#EXTINF:20.000,
seg-129-v1-a1.ts
#EXTINF:20.000,
seg-130-v1-a1.ts
#EXTINF:20.000,
seg-131-v1-a1.ts
#EXTINF:20.000,
seg-132-v1-a1.ts
#EXTINF:20.000,
seg-133-v1-a1.ts
#EXTINF:20.000,
seg-134-v1-a1.ts
#EXTINF:20.000,
seg-135-v1-a1.ts
#EXTINF:20.000,
seg-136-v1-a1.ts
#EXTINF:20.000,
seg-137-v1-a1.ts
#EXTINF:4.111,
seg-138-v1-a1.ts
#EXT-X-ENDLIST
+97
View File
@@ -0,0 +1,97 @@
package download
import (
"fmt"
"os"
"path/filepath"
"testing"
"time"
"github.com/GopeedLab/gopeed/pkg/base"
)
// TestDownloader_HlsLiveSite is a manual acceptance gate against a real site.
// It is skipped unless HLS_LIVE_URL is set, e.g.:
//
// HLS_LIVE_URL="https://host/path/index.m3u8" go test ./pkg/download/ -run TestDownloader_HlsLiveSite -v
func TestDownloader_HlsLiveSite(t *testing.T) {
playlistURL := os.Getenv("HLS_LIVE_URL")
if playlistURL == "" {
t.Skip("set HLS_LIVE_URL to run the live-site acceptance test")
}
referer := os.Getenv("HLS_LIVE_REFERER")
downloader := NewDownloader(&DownloaderConfig{Storage: NewMemStorage(), StorageDir: t.TempDir()})
if err := downloader.Setup(); err != nil {
t.Fatal(err)
}
defer downloader.Clear()
downloadDir := t.TempDir()
cfg, err := downloader.GetConfig()
if err != nil {
t.Fatal(err)
}
cfg.DownloadDir = downloadDir
if err := downloader.PutConfig(cfg); err != nil {
t.Fatal(err)
}
req := &base.Request{URL: playlistURL}
if referer != "" {
req.Extra = map[string]any{"header": map[string]string{"Referer": referer}}
}
rr, err := downloader.Resolve(req, nil)
if err != nil {
t.Fatalf("resolve: %v", err)
}
name := rr.Res.Files[0].Name
size := rr.Res.Size
t.Logf("resolved: name=%s size=%d", name, size)
taskID, err := downloader.Create(rr.ID)
if err != nil {
t.Fatal(err)
}
deadline := time.Now().Add(30 * time.Minute)
var lastLogged int64
for {
task := downloader.GetTask(taskID)
if task == nil {
t.Fatal("task not found")
}
if task.Status == base.DownloadStatusDone {
break
}
if task.Status == base.DownloadStatusError {
t.Fatalf("task entered error state after %d bytes", task.Progress.Downloaded)
}
if time.Now().After(deadline) {
t.Fatalf("timeout, downloaded %d bytes", task.Progress.Downloaded)
}
if tick := task.Progress.Downloaded; tick-lastLogged > 10<<20 {
lastLogged = tick
t.Logf("progress: %d bytes", tick)
}
time.Sleep(500 * time.Millisecond)
}
data, err := os.ReadFile(filepath.Join(downloadDir, name))
if err != nil {
t.Fatal(err)
}
if int64(len(data)) == 0 {
t.Fatal("merged file is empty")
}
t.Logf("merged: %s (%d bytes)", name, len(data))
if filepath.Ext(name) == ".ts" {
// MPEG-TS packets are 188 bytes and start with the 0x47 sync byte.
for i := 0; i < 188*4 && i < len(data); i += 188 {
if data[i] != 0x47 {
t.Fatalf("invalid TS sync byte at offset %d: %#x", i, data[i])
}
}
} else if string(data[4:8]) != "ftyp" {
t.Fatalf("invalid fMP4 header: %q", data[:8])
}
fmt.Printf("LIVE ACCEPTANCE OK: %s (%d bytes)\n", filepath.Join(downloadDir, name), len(data))
}
+172
View File
@@ -0,0 +1,172 @@
package download
import (
"fmt"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"testing"
"time"
"github.com/GopeedLab/gopeed/pkg/base"
)
// hlsTestCdn spins up a master-playlist HLS site whose segments require a
// Referer header (anti-hotlink) and returns the server plus expected bytes.
func hlsTestCdn(t *testing.T, segmentDelay time.Duration) (*httptest.Server, []byte) {
t.Helper()
const total = 6
segments := make(map[int][]byte)
for i := 0; i < total; i++ {
segments[i] = []byte(fmt.Sprintf("ENGINE-SEGMENT-%02d", i))
}
var want []byte
for i := 0; i < total; i++ {
want = append(want, segments[i]...)
}
mux := http.NewServeMux()
mux.HandleFunc("/master.m3u8", func(w http.ResponseWriter, r *http.Request) {
w.Write([]byte("#EXTM3U\n" +
"#EXT-X-STREAM-INF:BANDWIDTH=500000\nlow/index.m3u8\n" +
"#EXT-X-STREAM-INF:BANDWIDTH=2000000\nhigh/index.m3u8\n"))
})
mux.HandleFunc("/low/index.m3u8", func(w http.ResponseWriter, r *http.Request) {
w.Write([]byte("#EXTM3U\n#EXTINF:4,\nseg-0.ts\n#EXT-X-ENDLIST\n"))
})
mux.HandleFunc("/high/index.m3u8", func(w http.ResponseWriter, r *http.Request) {
var b strings.Builder
b.WriteString("#EXTM3U\n")
for i := 0; i < total; i++ {
b.WriteString(fmt.Sprintf("#EXTINF:4,\nseg-%d.ts\n", i))
}
b.WriteString("#EXT-X-ENDLIST\n")
w.Write([]byte(b.String()))
})
for i := 0; i < total; i++ {
idx := i
mux.HandleFunc(fmt.Sprintf("/high/seg-%d.ts", i), func(w http.ResponseWriter, r *http.Request) {
if r.Header.Get("Referer") == "" {
w.WriteHeader(http.StatusForbidden)
return
}
if segmentDelay > 0 {
time.Sleep(segmentDelay)
}
w.Write(segments[idx])
})
}
server := httptest.NewServer(mux)
t.Cleanup(server.Close)
return server, want
}
func hlsResolveReferer() *base.Request {
return &base.Request{
URL: "",
Extra: map[string]any{"header": map[string]string{"Referer": "https://player.example.com/page"}},
}
}
func TestDownloader_HlsProtocolEndToEnd(t *testing.T) {
server, want := hlsTestCdn(t, 0)
downloader := NewDownloader(&DownloaderConfig{Storage: NewMemStorage(), StorageDir: t.TempDir()})
if err := downloader.Setup(); err != nil {
t.Fatal(err)
}
defer downloader.Clear()
downloadDir := t.TempDir()
cfg, err := downloader.GetConfig()
if err != nil {
t.Fatal(err)
}
cfg.DownloadDir = downloadDir
if err := downloader.PutConfig(cfg); err != nil {
t.Fatal(err)
}
req := hlsResolveReferer()
req.URL = server.URL + "/master.m3u8"
rr, err := downloader.Resolve(req, nil)
if err != nil {
t.Fatal(err)
}
if got := rr.Res.Files[0].Name; got != "index.ts" {
t.Errorf("resolved name = %s, want index.ts", got)
}
taskID, err := downloader.Create(rr.ID)
if err != nil {
t.Fatal(err)
}
waitForTaskStatus(t, downloader, taskID, base.DownloadStatusDone, 10*time.Second)
task := downloader.GetTask(taskID)
if task.Protocol != "hls" {
t.Errorf("task protocol = %s, want hls", task.Protocol)
}
data, err := os.ReadFile(filepath.Join(downloadDir, "index.ts"))
if err != nil {
t.Fatal(err)
}
if string(data) != string(want) {
t.Errorf("merged file mismatch: %q", data)
}
}
func TestDownloader_HlsProtocolPauseContinue(t *testing.T) {
server, want := hlsTestCdn(t, 120*time.Millisecond)
downloader := NewDownloader(&DownloaderConfig{Storage: NewMemStorage(), StorageDir: t.TempDir()})
if err := downloader.Setup(); err != nil {
t.Fatal(err)
}
defer downloader.Clear()
downloadDir := t.TempDir()
cfg, err := downloader.GetConfig()
if err != nil {
t.Fatal(err)
}
cfg.DownloadDir = downloadDir
// Throttle the protocol so the pause lands mid-download.
cfg.ProtocolConfig["hls"] = map[string]any{
"segmentConnections": 2,
"maxRetries": 3,
"timeoutSeconds": 5,
"prefetchContentLength": false,
}
if err := downloader.PutConfig(cfg); err != nil {
t.Fatal(err)
}
req := hlsResolveReferer()
req.URL = server.URL + "/master.m3u8"
req.Extra = map[string]any{"header": map[string]string{"Referer": "https://player.example.com/page"}}
// CreateDirect: engine resolves lazily on start, like a direct task.
taskID, err := downloader.CreateDirect(req, &base.Options{Name: "movie.ts"})
if err != nil {
t.Fatal(err)
}
waitForTaskStatus(t, downloader, taskID, base.DownloadStatusRunning, 10*time.Second)
if err := downloader.Pause(&TaskFilter{IDs: []string{taskID}}); err != nil {
t.Fatal(err)
}
waitForTaskStatus(t, downloader, taskID, base.DownloadStatusPause, 5*time.Second)
if err := downloader.Continue(&TaskFilter{IDs: []string{taskID}}); err != nil {
t.Fatal(err)
}
waitForTaskStatus(t, downloader, taskID, base.DownloadStatusDone, 30*time.Second)
task := downloader.GetTask(taskID)
if task.Meta.Res.Size != int64(len(want)) {
t.Errorf("final size = %d, want %d", task.Meta.Res.Size, len(want))
}
data, err := os.ReadFile(filepath.Join(downloadDir, "movie.ts"))
if err != nil {
t.Fatal(err)
}
if string(data) != string(want) {
t.Errorf("merged file mismatch after pause/continue: %q", data)
}
}
+2
View File
@@ -9,6 +9,7 @@ import (
"github.com/GopeedLab/gopeed/internal/fetcher"
"github.com/GopeedLab/gopeed/internal/protocol/bt"
"github.com/GopeedLab/gopeed/internal/protocol/ed2k"
"github.com/GopeedLab/gopeed/internal/protocol/hls"
"github.com/GopeedLab/gopeed/internal/protocol/http"
"github.com/GopeedLab/gopeed/pkg/base"
enginewebview "github.com/GopeedLab/gopeed/pkg/download/engine/webview"
@@ -197,6 +198,7 @@ func (cfg *DownloaderConfig) Init() *DownloaderConfig {
}
if len(cfg.FetchManagers) == 0 {
cfg.FetchManagers = []fetcher.FetcherManager{
new(hls.FetcherManager),
new(http.FetcherManager),
new(bt.FetcherManager),
new(ed2k.FetcherManager),
@@ -56,6 +56,7 @@ class ProtocolConfig {
HttpConfig http = HttpConfig();
BtConfig bt = BtConfig();
Ed2kConfig ed2k = Ed2kConfig();
HlsConfig hls = HlsConfig();
ProtocolConfig();
@@ -65,6 +66,25 @@ class ProtocolConfig {
Map<String, dynamic> toJson() => _$ProtocolConfigToJson(this);
}
@JsonSerializable()
class HlsConfig {
int segmentConnections;
int maxRetries;
int timeoutSeconds;
bool prefetchContentLength;
HlsConfig({
this.segmentConnections = 0,
this.maxRetries = 0,
this.timeoutSeconds = 0,
this.prefetchContentLength = false,
});
factory HlsConfig.fromJson(Map<String, dynamic> json) => _$HlsConfigFromJson(json);
Map<String, dynamic> toJson() => _$HlsConfigToJson(this);
}
@JsonSerializable()
class HttpConfig {
String userAgent;
@@ -56,12 +56,28 @@ Map<String, dynamic> _$ApiServerConfigToJson(ApiServerConfig instance) => <Strin
ProtocolConfig _$ProtocolConfigFromJson(Map<String, dynamic> json) => ProtocolConfig()
..http = HttpConfig.fromJson(json['http'] as Map<String, dynamic>)
..bt = BtConfig.fromJson(json['bt'] as Map<String, dynamic>)
..ed2k = Ed2kConfig.fromJson(json['ed2k'] as Map<String, dynamic>);
..ed2k = Ed2kConfig.fromJson(json['ed2k'] as Map<String, dynamic>)
..hls = HlsConfig.fromJson(json['hls'] as Map<String, dynamic>);
Map<String, dynamic> _$ProtocolConfigToJson(ProtocolConfig instance) => <String, dynamic>{
'http': instance.http.toJson(),
'bt': instance.bt.toJson(),
'ed2k': instance.ed2k.toJson(),
'hls': instance.hls.toJson(),
};
HlsConfig _$HlsConfigFromJson(Map<String, dynamic> json) => HlsConfig(
segmentConnections: (json['segmentConnections'] as num?)?.toInt() ?? 0,
maxRetries: (json['maxRetries'] as num?)?.toInt() ?? 0,
timeoutSeconds: (json['timeoutSeconds'] as num?)?.toInt() ?? 0,
prefetchContentLength: json['prefetchContentLength'] as bool? ?? false,
);
Map<String, dynamic> _$HlsConfigToJson(HlsConfig instance) => <String, dynamic>{
'segmentConnections': instance.segmentConnections,
'maxRetries': instance.maxRetries,
'timeoutSeconds': instance.timeoutSeconds,
'prefetchContentLength': instance.prefetchContentLength,
};
HttpConfig _$HttpConfigFromJson(Map<String, dynamic> json) => HttpConfig(
+1 -1
View File
@@ -6,7 +6,7 @@ part 'task.g.dart';
enum Status { ready, running, pause, wait, error, done }
enum Protocol { http, bt, ed2k }
enum Protocol { http, bt, ed2k, hls }
// ExtractStatus enum matching Go backend
enum ExtractStatus {
+1 -1
View File
@@ -38,7 +38,7 @@ const _$StatusEnumMap = {
Status.done: 'done',
};
const _$ProtocolEnumMap = {Protocol.http: 'http', Protocol.bt: 'bt', Protocol.ed2k: 'ed2k'};
const _$ProtocolEnumMap = {Protocol.http: 'http', Protocol.bt: 'bt', Protocol.ed2k: 'ed2k', Protocol.hls: 'hls'};
Progress _$ProgressFromJson(Map<String, dynamic> json) => Progress(
used: (json['used'] as num).toInt(),
+1 -1
View File
@@ -55,7 +55,7 @@ sealed class TaskStats {
Protocol.http => HttpTaskStats.fromJson(snapshot),
Protocol.bt => BtTaskStats.fromJson(snapshot: snapshot, runtime: runtime),
Protocol.ed2k => Ed2kTaskStats.fromJson(snapshot: snapshot, runtime: runtime),
null => null,
Protocol.hls || null => null,
};
}
}
@@ -65,6 +65,8 @@ class _SettingsPageState extends ConsumerState<SettingsPage> {
final _maxRunningController = TextEditingController();
final _httpUserAgentController = TextEditingController();
final _httpConnectionsController = TextEditingController();
final _hlsConnectionsController = TextEditingController();
final _hlsMaxRetriesController = TextEditingController();
final _btListenPortController = TextEditingController();
final _btSeedRatioController = TextEditingController();
final _btSeedTimeController = TextEditingController();
@@ -96,6 +98,8 @@ class _SettingsPageState extends ConsumerState<SettingsPage> {
_maxRunningController,
_httpUserAgentController,
_httpConnectionsController,
_hlsConnectionsController,
_hlsMaxRetriesController,
_btListenPortController,
_btSeedRatioController,
_btSeedTimeController,
@@ -572,6 +576,32 @@ class _SettingsPageState extends ConsumerState<SettingsPage> {
],
),
),
_SettingsBlock(
key: const ValueKey('settings-hls-block'),
title: 'HLS',
child: _SettingsGroup(
children: [
SettingsItem(
title: context.l10n.connections,
child: _NumberSettingControl(
fieldKey: const ValueKey('hls-connections-input'),
controller: _hlsConnectionsController,
min: 1,
max: 256,
),
),
SettingsItem(
title: context.l10n.maxRetries,
child: _NumberSettingControl(
fieldKey: const ValueKey('hls-max-retries-input'),
controller: _hlsMaxRetriesController,
min: 0,
max: 30,
),
),
],
),
),
_SettingsBlock(
title: 'BitTorrent',
child: _SettingsGroup(
@@ -953,6 +983,8 @@ class _SettingsPageState extends ConsumerState<SettingsPage> {
_maxRunningController.text = _config!.maxRunning.clamp(1, 256).toString();
_httpUserAgentController.text = _config!.protocolConfig.http.userAgent;
_httpConnectionsController.text = _config!.protocolConfig.http.connections.clamp(1, 256).toString();
_hlsConnectionsController.text = _config!.protocolConfig.hls.segmentConnections.clamp(1, 256).toString();
_hlsMaxRetriesController.text = _config!.protocolConfig.hls.maxRetries.clamp(0, 30).toString();
_btListenPortController.text = _config!.protocolConfig.bt.listenPort.clamp(0, 65535).toString();
_btSeedRatioController.text = _config!.protocolConfig.bt.seedRatio.toString();
_btSeedTimeController.text = (_config!.protocolConfig.bt.seedTime ~/ 60).clamp(0, 100000000).toString();
@@ -1048,6 +1080,18 @@ class _SettingsPageState extends ConsumerState<SettingsPage> {
min: 1,
max: 256,
);
config.protocolConfig.hls.segmentConnections = _boundedInt(
_hlsConnectionsController,
fallback: config.protocolConfig.hls.segmentConnections,
min: 1,
max: 256,
);
config.protocolConfig.hls.maxRetries = _boundedInt(
_hlsMaxRetriesController,
fallback: config.protocolConfig.hls.maxRetries,
min: 0,
max: 30,
);
config.protocolConfig.bt.listenPort = _boundedInt(
_btListenPortController,
fallback: config.protocolConfig.bt.listenPort,
@@ -294,7 +294,7 @@ TaskAssetType _assetType(String name, {required bool isFolder, required api_task
return switch (protocol) {
api_task.Protocol.bt => TaskAssetType.torrent,
api_task.Protocol.ed2k => TaskAssetType.ed2k,
api_task.Protocol.http || null => TaskAssetType.file,
api_task.Protocol.hls || api_task.Protocol.http || null => TaskAssetType.file,
};
}
+1
View File
@@ -38,6 +38,7 @@
"rememberLastDownloadDir": "حفظ كمجلد التنزيل الافتراضي",
"downloadDirValid": "الرجاء تحديد مجلد التنزيل",
"connections": "الاتصالات",
"maxRetries": "الحد الأقصى لإعادة المحاولات",
"useServerCtime": "استخدام وقت الخادم لإنشاء الملف",
"maxRunning": "الحد الأقصى للمهام المتزامنة",
"defaultDirectDownload": "تفعيل التنزيل المباشر افتراضياً",
+1
View File
@@ -38,6 +38,7 @@
"rememberLastDownloadDir": "Standart endirmə qovluğu kimi yadda saxla",
"downloadDirValid": "Zəhmət olmasa endirmə qovluğunu seçin",
"connections": "Əlaqələr",
"maxRetries": "Maksimum cəhd sayı",
"useServerCtime": "Fayl yaratma tarixi üçün server vaxtından istifadə et",
"maxRunning": "Paralel Endirmələr",
"defaultDirectDownload": "Standart olaraq Birbaşa Endirmə",
+1
View File
@@ -38,6 +38,7 @@
"rememberLastDownloadDir": "Desa com a carpeta de baixades predeterminada",
"downloadDirValid": "Seleccioneu la carpeta de baixades",
"connections": "Connexions",
"maxRetries": "Reintents màxims",
"useServerCtime": "Utilitza l'hora del servidor per a la creació de fitxers",
"maxRunning": "Màxim de tasques en execució",
"defaultDirectDownload": "Activa la baixada directa per defecte",
+1
View File
@@ -38,6 +38,7 @@
"rememberLastDownloadDir": "Als Standard-Downloadordner speichern",
"downloadDirValid": "Bitte wählen Sie das Download-Verzeichnis",
"connections": "Anschlüsse",
"maxRetries": "Maximale Wiederholungsversuche",
"useServerCtime": "Serverzeit für die Dateierstellung verwenden",
"maxRunning": "Maximale Anzahl ausgeführter Aufgaben",
"defaultDirectDownload": "Direkten Download standardmäßig aktivieren",
+1
View File
@@ -38,6 +38,7 @@
"rememberLastDownloadDir": "Save as default download directory",
"downloadDirValid": "Please select the download directory",
"connections": "Connections",
"maxRetries": "Max retries",
"useServerCtime": "Use server time for file creation",
"maxRunning": "Concurrent Downloads",
"defaultDirectDownload": "Direct Download by Default",
+1
View File
@@ -38,6 +38,7 @@
"rememberLastDownloadDir": "Guardar como carpeta de descargas predeterminada",
"downloadDirValid": "Por favor, selecciona el directorio de descarga",
"connections": "Conexiones",
"maxRetries": "Reintentos máximos",
"useServerCtime": "Usar tiempo del servidor para la creación de archivos",
"maxRunning": "Máximo de Tareas en Ejecución",
"defaultDirectDownload": "Ver descarga directa por defecto",
+1
View File
@@ -38,6 +38,7 @@
"rememberLastDownloadDir": "ذخیره به‌عنوان پوشه پیش‌فرض دانلود",
"downloadDirValid": "لطفا دایرکتوری دانلود را انتخاب کنید",
"connections": "اتصالات",
"maxRetries": "حداکثر تلاش مجدد",
"useServerCtime": "استفاده از زمان سرور برای ایجاد فایل",
"maxRunning": "حداکثر وظایف در حال اجرا",
"defaultDirectDownload": "دانلود مستقیم را به صورت پیش فرض بررسی کنید",
+1
View File
@@ -38,6 +38,7 @@
"rememberLastDownloadDir": "Enregistrer comme dossier de téléchargement par défaut",
"downloadDirValid": "Veuillez sélectionner le répertoire de téléchargement",
"connections": "Connexions",
"maxRetries": "Nombre maximal de tentatives",
"useServerCtime": "Utiliser l'heure du serveur pour la création de fichier",
"maxRunning": "Tâches en cours maximum",
"defaultDirectDownload": "Vérifiez le téléchargement direct par défaut",
+1
View File
@@ -38,6 +38,7 @@
"rememberLastDownloadDir": "Mentés alapértelmezett letöltési mappaként",
"downloadDirValid": "Kérjük, válassza ki a letöltési könyvtárat",
"connections": "Kapcsolatok",
"maxRetries": "Maximális újrapróbálkozások",
"useServerCtime": "Használja a szerveridőt a fájl létrehozásához",
"maxRunning": "Maximális futó feladatok",
"defaultDirectDownload": "Alapértelmezettként ellenőrizze a közvetlen letöltést",
+1
View File
@@ -38,6 +38,7 @@
"rememberLastDownloadDir": "Simpan sebagai folder unduhan default",
"downloadDirValid": "Silahkan pilih direktori unduhan",
"connections": "Koneksi",
"maxRetries": "Percobaan ulang maks.",
"useServerCtime": "Gunakan waktu server untuk pembuatan file",
"maxRunning": "Tugas Maksimum yang Berjalan",
"defaultDirectDownload": "Pilih unduhan langsung sebagai opsi standar",
+1
View File
@@ -38,6 +38,7 @@
"rememberLastDownloadDir": "Salva come cartella di download predefinita",
"downloadDirValid": "Seleziona la directory di scaricamento",
"connections": "Connessioni",
"maxRetries": "Tentativi massimi",
"useServerCtime": "Utilizza l'ora del server per la creazione dei file",
"maxRunning": "Numero massimo di attività in esecuzione",
"defaultDirectDownload": "Controlla il download diretto per impostazione predefinita",
+1
View File
@@ -38,6 +38,7 @@
"rememberLastDownloadDir": "デフォルトのダウンロード先として保存",
"downloadDirValid": "ダウンロードディレクトリを選択してください",
"connections": "接続",
"maxRetries": "最大リトライ回数",
"useServerCtime": "ファイル作成時にサーバー時間を使用する",
"maxRunning": "最大実行タスク",
"defaultDirectDownload": "デフォルトで直接ダウンロードにチェックを入れる",
+1
View File
@@ -38,6 +38,7 @@
"rememberLastDownloadDir": "기본 다운로드 폴더로 저장",
"downloadDirValid": "다운로드 폴더를 선택해 주세요",
"connections": "연결 수",
"maxRetries": "최대 재시도 횟수",
"useServerCtime": "파일 생성 시간에 서버 시간 사용",
"maxRunning": "최대 동시 작업 수",
"defaultDirectDownload": "기본으로 바로 다운로드 사용",
+1
View File
@@ -38,6 +38,7 @@
"rememberLastDownloadDir": "Zapisz jako domyślny folder pobierania",
"downloadDirValid": "Wskaż folder pobierania",
"connections": "Połączenia",
"maxRetries": "Maks. liczba ponownych prób",
"useServerCtime": "Użyj czasu serwera do utworzenia pliku",
"maxRunning": "Maksymalna liczba zadań",
"defaultDirectDownload": "Domyślnie sprawdź bezpośredni pobór",
+1
View File
@@ -38,6 +38,7 @@
"rememberLastDownloadDir": "Salvar como pasta de download padrão",
"downloadDirValid": "Por favor, selecione a pasta",
"connections": "Conexões",
"maxRetries": "Tentativas máximas",
"useServerCtime": "Usar o tempo de servidor para a criação de ficheiros",
"maxRunning": "Máximo de Tarefas Simultâneas",
"defaultDirectDownload": "Marcar download direto por padrão",
+1
View File
@@ -38,6 +38,7 @@
"rememberLastDownloadDir": "Сохранить как папку загрузок по умолчанию",
"downloadDirValid": "Пожалуйста, выберите папку для загрузки",
"connections": "Соединений",
"maxRetries": "Макс. число повторов",
"useServerCtime": "Использовать время сервера для создания файла",
"maxRunning": "Максимальное количество активных задач",
"defaultDirectDownload": "Прямая загрузка по умолчанию",
+1
View File
@@ -38,6 +38,7 @@
"rememberLastDownloadDir": "இயல்புநிலை பதிவிறக்க கோப்புறையாக சேமி",
"downloadDirValid": "பதிவிரக்குமிடத்தை தேர்வு செயுங்கள்",
"connections": "தொடர்புகள்",
"maxRetries": "அதிகபட்ச மறுமுயற்சிகள்",
"useServerCtime": "கோப்பு உருவாக்க சேவையக நேரத்தைப் பயன்படுத்தவும்",
"maxRunning": "அதிகபட்ச இயங்கும் பணிகள்",
"defaultDirectDownload": "இயல்புநிலையாக நேரடிப் பதிவிறக்கத்தைச் சரிபார்க்கவும்",
+1
View File
@@ -38,6 +38,7 @@
"rememberLastDownloadDir": "Varsayılan indirme klasörü olarak kaydet",
"downloadDirValid": "Lütfen indirme dizinini seçin",
"connections": "Bağlantılar",
"maxRetries": "Maksimum yeniden deneme",
"useServerCtime": "Dosya oluşturma tarihi için sunucu zamanını kullan",
"maxRunning": "Maksimum Çalışan İşlem",
"defaultDirectDownload": "Varsayılan olarak direkt indirmeyi işaretle",
+1
View File
@@ -38,6 +38,7 @@
"rememberLastDownloadDir": "Зберегти як типову папку завантажень",
"downloadDirValid": "Будь ласка, виберіть каталог завантажень",
"connections": "З’єднання",
"maxRetries": "Макс. кількість повторів",
"useServerCtime": "Використовувати час сервера для створення файлу",
"maxRunning": "Макс. завдань у черзі",
"defaultDirectDownload": "Пряме завантаження за замовчуванням",
+1
View File
@@ -38,6 +38,7 @@
"rememberLastDownloadDir": "Lưu làm thư mục tải xuống mặc định",
"downloadDirValid": "Vui lòng chọn thư mục tải về",
"connections": "Kết nối",
"maxRetries": "Số lần thử lại tối đa",
"useServerCtime": "Sử dụng thời gian máy chủ cho việc tạo tệp",
"maxRunning": "Số nhiệm vụ tối đa",
"defaultDirectDownload": "Kiểm tra trực tiếp tải về theo mặc định",
+1
View File
@@ -38,6 +38,7 @@
"rememberLastDownloadDir": "保存为默认下载目录",
"downloadDirValid": "请选择下载目录",
"connections": "连接数",
"maxRetries": "最大重试次数",
"useServerCtime": "创建文件使用服务器时间",
"maxRunning": "最大下载数",
"defaultDirectDownload": "默认启用直接下载",
+1
View File
@@ -38,6 +38,7 @@
"rememberLastDownloadDir": "儲存為預設下載目錄",
"downloadDirValid": "請選擇下載目錄",
"connections": "連接數",
"maxRetries": "最大重試次數",
"useServerCtime": "使用伺服器時間作為檔案建立時間",
"maxRunning": "最大執行任務數",
"defaultDirectDownload": "預設勾選直接下載",