feat(hls): add native HLS (m3u8) download protocol (#1515)

pkg/download/hls_live_test.go

@@ -0,0 +1,97 @@
+package download
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/GopeedLab/gopeed/pkg/base"
+)
+
+// TestDownloader_HlsLiveSite is a manual acceptance gate against a real site.
+// It is skipped unless HLS_LIVE_URL is set, e.g.:
+//
+//	HLS_LIVE_URL="https://host/path/index.m3u8" go test ./pkg/download/ -run TestDownloader_HlsLiveSite -v
+func TestDownloader_HlsLiveSite(t *testing.T) {
+	playlistURL := os.Getenv("HLS_LIVE_URL")
+	if playlistURL == "" {
+		t.Skip("set HLS_LIVE_URL to run the live-site acceptance test")
+	}
+	referer := os.Getenv("HLS_LIVE_REFERER")
+
+	downloader := NewDownloader(&DownloaderConfig{Storage: NewMemStorage(), StorageDir: t.TempDir()})
+	if err := downloader.Setup(); err != nil {
+		t.Fatal(err)
+	}
+	defer downloader.Clear()
+	downloadDir := t.TempDir()
+	cfg, err := downloader.GetConfig()
+	if err != nil {
+		t.Fatal(err)
+	}
+	cfg.DownloadDir = downloadDir
+	if err := downloader.PutConfig(cfg); err != nil {
+		t.Fatal(err)
+	}
+
+	req := &base.Request{URL: playlistURL}
+	if referer != "" {
+		req.Extra = map[string]any{"header": map[string]string{"Referer": referer}}
+	}
+	rr, err := downloader.Resolve(req, nil)
+	if err != nil {
+		t.Fatalf("resolve: %v", err)
+	}
+	name := rr.Res.Files[0].Name
+	size := rr.Res.Size
+	t.Logf("resolved: name=%s size=%d", name, size)
+
+	taskID, err := downloader.Create(rr.ID)
+	if err != nil {
+		t.Fatal(err)
+	}
+	deadline := time.Now().Add(30 * time.Minute)
+	var lastLogged int64
+	for {
+		task := downloader.GetTask(taskID)
+		if task == nil {
+			t.Fatal("task not found")
+		}
+		if task.Status == base.DownloadStatusDone {
+			break
+		}
+		if task.Status == base.DownloadStatusError {
+			t.Fatalf("task entered error state after %d bytes", task.Progress.Downloaded)
+		}
+		if time.Now().After(deadline) {
+			t.Fatalf("timeout, downloaded %d bytes", task.Progress.Downloaded)
+		}
+		if tick := task.Progress.Downloaded; tick-lastLogged > 10<<20 {
+			lastLogged = tick
+			t.Logf("progress: %d bytes", tick)
+		}
+		time.Sleep(500 * time.Millisecond)
+	}
+
+	data, err := os.ReadFile(filepath.Join(downloadDir, name))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if int64(len(data)) == 0 {
+		t.Fatal("merged file is empty")
+	}
+	t.Logf("merged: %s (%d bytes)", name, len(data))
+	if filepath.Ext(name) == ".ts" {
+		// MPEG-TS packets are 188 bytes and start with the 0x47 sync byte.
+		for i := 0; i < 188*4 && i < len(data); i += 188 {
+			if data[i] != 0x47 {
+				t.Fatalf("invalid TS sync byte at offset %d: %#x", i, data[i])
+			}
+		}
+	} else if string(data[4:8]) != "ftyp" {
+		t.Fatalf("invalid fMP4 header: %q", data[:8])
+	}
+	fmt.Printf("LIVE ACCEPTANCE OK: %s (%d bytes)\n", filepath.Join(downloadDir, name), len(data))
+}

pkg/download/hls_protocol_test.go

@@ -0,0 +1,172 @@
+package download
+
+import (
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/GopeedLab/gopeed/pkg/base"
+)
+
+// hlsTestCdn spins up a master-playlist HLS site whose segments require a
+// Referer header (anti-hotlink) and returns the server plus expected bytes.
+func hlsTestCdn(t *testing.T, segmentDelay time.Duration) (*httptest.Server, []byte) {
+	t.Helper()
+	const total = 6
+	segments := make(map[int][]byte)
+	for i := 0; i < total; i++ {
+		segments[i] = []byte(fmt.Sprintf("ENGINE-SEGMENT-%02d", i))
+	}
+	var want []byte
+	for i := 0; i < total; i++ {
+		want = append(want, segments[i]...)
+	}
+
+	mux := http.NewServeMux()
+	mux.HandleFunc("/master.m3u8", func(w http.ResponseWriter, r *http.Request) {
+		w.Write([]byte("#EXTM3U\n" +
+			"#EXT-X-STREAM-INF:BANDWIDTH=500000\nlow/index.m3u8\n" +
+			"#EXT-X-STREAM-INF:BANDWIDTH=2000000\nhigh/index.m3u8\n"))
+	})
+	mux.HandleFunc("/low/index.m3u8", func(w http.ResponseWriter, r *http.Request) {
+		w.Write([]byte("#EXTM3U\n#EXTINF:4,\nseg-0.ts\n#EXT-X-ENDLIST\n"))
+	})
+	mux.HandleFunc("/high/index.m3u8", func(w http.ResponseWriter, r *http.Request) {
+		var b strings.Builder
+		b.WriteString("#EXTM3U\n")
+		for i := 0; i < total; i++ {
+			b.WriteString(fmt.Sprintf("#EXTINF:4,\nseg-%d.ts\n", i))
+		}
+		b.WriteString("#EXT-X-ENDLIST\n")
+		w.Write([]byte(b.String()))
+	})
+	for i := 0; i < total; i++ {
+		idx := i
+		mux.HandleFunc(fmt.Sprintf("/high/seg-%d.ts", i), func(w http.ResponseWriter, r *http.Request) {
+			if r.Header.Get("Referer") == "" {
+				w.WriteHeader(http.StatusForbidden)
+				return
+			}
+			if segmentDelay > 0 {
+				time.Sleep(segmentDelay)
+			}
+			w.Write(segments[idx])
+		})
+	}
+	server := httptest.NewServer(mux)
+	t.Cleanup(server.Close)
+	return server, want
+}
+
+func hlsResolveReferer() *base.Request {
+	return &base.Request{
+		URL:   "",
+		Extra: map[string]any{"header": map[string]string{"Referer": "https://player.example.com/page"}},
+	}
+}
+
+func TestDownloader_HlsProtocolEndToEnd(t *testing.T) {
+	server, want := hlsTestCdn(t, 0)
+	downloader := NewDownloader(&DownloaderConfig{Storage: NewMemStorage(), StorageDir: t.TempDir()})
+	if err := downloader.Setup(); err != nil {
+		t.Fatal(err)
+	}
+	defer downloader.Clear()
+	downloadDir := t.TempDir()
+	cfg, err := downloader.GetConfig()
+	if err != nil {
+		t.Fatal(err)
+	}
+	cfg.DownloadDir = downloadDir
+	if err := downloader.PutConfig(cfg); err != nil {
+		t.Fatal(err)
+	}
+
+	req := hlsResolveReferer()
+	req.URL = server.URL + "/master.m3u8"
+	rr, err := downloader.Resolve(req, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got := rr.Res.Files[0].Name; got != "index.ts" {
+		t.Errorf("resolved name = %s, want index.ts", got)
+	}
+	taskID, err := downloader.Create(rr.ID)
+	if err != nil {
+		t.Fatal(err)
+	}
+	waitForTaskStatus(t, downloader, taskID, base.DownloadStatusDone, 10*time.Second)
+
+	task := downloader.GetTask(taskID)
+	if task.Protocol != "hls" {
+		t.Errorf("task protocol = %s, want hls", task.Protocol)
+	}
+	data, err := os.ReadFile(filepath.Join(downloadDir, "index.ts"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if string(data) != string(want) {
+		t.Errorf("merged file mismatch: %q", data)
+	}
+}
+
+func TestDownloader_HlsProtocolPauseContinue(t *testing.T) {
+	server, want := hlsTestCdn(t, 120*time.Millisecond)
+	downloader := NewDownloader(&DownloaderConfig{Storage: NewMemStorage(), StorageDir: t.TempDir()})
+	if err := downloader.Setup(); err != nil {
+		t.Fatal(err)
+	}
+	defer downloader.Clear()
+	downloadDir := t.TempDir()
+	cfg, err := downloader.GetConfig()
+	if err != nil {
+		t.Fatal(err)
+	}
+	cfg.DownloadDir = downloadDir
+	// Throttle the protocol so the pause lands mid-download.
+	cfg.ProtocolConfig["hls"] = map[string]any{
+		"segmentConnections":    2,
+		"maxRetries":            3,
+		"timeoutSeconds":        5,
+		"prefetchContentLength": false,
+	}
+	if err := downloader.PutConfig(cfg); err != nil {
+		t.Fatal(err)
+	}
+
+	req := hlsResolveReferer()
+	req.URL = server.URL + "/master.m3u8"
+	req.Extra = map[string]any{"header": map[string]string{"Referer": "https://player.example.com/page"}}
+	// CreateDirect: engine resolves lazily on start, like a direct task.
+	taskID, err := downloader.CreateDirect(req, &base.Options{Name: "movie.ts"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	waitForTaskStatus(t, downloader, taskID, base.DownloadStatusRunning, 10*time.Second)
+	if err := downloader.Pause(&TaskFilter{IDs: []string{taskID}}); err != nil {
+		t.Fatal(err)
+	}
+	waitForTaskStatus(t, downloader, taskID, base.DownloadStatusPause, 5*time.Second)
+
+	if err := downloader.Continue(&TaskFilter{IDs: []string{taskID}}); err != nil {
+		t.Fatal(err)
+	}
+	waitForTaskStatus(t, downloader, taskID, base.DownloadStatusDone, 30*time.Second)
+
+	task := downloader.GetTask(taskID)
+	if task.Meta.Res.Size != int64(len(want)) {
+		t.Errorf("final size = %d, want %d", task.Meta.Res.Size, len(want))
+	}
+	data, err := os.ReadFile(filepath.Join(downloadDir, "movie.ts"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if string(data) != string(want) {
+		t.Errorf("merged file mismatch after pause/continue: %q", data)
+	}
+}

pkg/download/model.go

@@ -9,6 +9,7 @@ import (
 	"github.com/GopeedLab/gopeed/internal/fetcher"
 	"github.com/GopeedLab/gopeed/internal/protocol/bt"
 	"github.com/GopeedLab/gopeed/internal/protocol/ed2k"
+	"github.com/GopeedLab/gopeed/internal/protocol/hls"
 	"github.com/GopeedLab/gopeed/internal/protocol/http"
 	"github.com/GopeedLab/gopeed/pkg/base"
 	enginewebview "github.com/GopeedLab/gopeed/pkg/download/engine/webview"
@@ -197,6 +198,7 @@ func (cfg *DownloaderConfig) Init() *DownloaderConfig {
 	}
 	if len(cfg.FetchManagers) == 0 {
 		cfg.FetchManagers = []fetcher.FetcherManager{
+			new(hls.FetcherManager),
 			new(http.FetcherManager),
 			new(bt.FetcherManager),
 			new(ed2k.FetcherManager),