Merge remote-tracking branch 'origin/main' into codex/fix-bt-file-selection

# Conflicts:
#	ui/flutter/test/features/tasks/domain/task_record_test.dart
This commit is contained in:
Levi committed 2026-09-17 00:03:36 +08:00
commit a83e3a2b84
272 files changed
+17212 -1065

No files matched your search

+7 -1
View File
@@ -106,7 +106,13 @@ func (s *SchemeFilter) Match(uri string) bool {
case FilterTypeUrl:
return strings.HasPrefix(uriUpper, patternUpper+":")
case FilterTypeFile:
return strings.HasSuffix(uriUpper, "."+patternUpper)
// Strip query and fragment before matching so URLs like
// https://example.com/video.m3u8?sign=abc still route by extension.
clean := uri
if idx := strings.IndexAny(clean, "?#"); idx >= 0 {
clean = clean[:idx]
}
return strings.HasSuffix(strings.ToUpper(clean), "."+patternUpper)
case FilterTypeBase64:
return strings.HasPrefix(uriUpper, "DATA:"+patternUpper+";BASE64,")
}
+30
View File
@@ -0,0 +1,30 @@
package hls
const (
defaultSegmentConnections = 8
defaultMaxRetries = 5
defaultTimeoutSeconds = 30
// maxPrefetchSizeSegments caps the HEAD probing done at resolve time.
// Playlists larger than this skip size prefetch entirely.
maxPrefetchSizeSegments = 500
)
type config struct {
SegmentConnections int `json:"segmentConnections"`
MaxRetries int `json:"maxRetries"`
TimeoutSeconds int `json:"timeoutSeconds"`
PrefetchContentLength bool `json:"prefetchContentLength"`
}
func (c *config) normalize() {
if c.SegmentConnections <= 0 {
c.SegmentConnections = defaultSegmentConnections
}
if c.MaxRetries < 0 {
c.MaxRetries = defaultMaxRetries
}
if c.TimeoutSeconds <= 0 {
c.TimeoutSeconds = defaultTimeoutSeconds
}
}
+59
View File
@@ -0,0 +1,59 @@
package hls
import (
"crypto/aes"
"crypto/cipher"
"fmt"
)
// decryptAES128 decrypts an AES-128-CBC encrypted segment and strips PKCS7
// padding. data length must be a multiple of the AES block size.
func decryptAES128(data, key, iv []byte) ([]byte, error) {
if len(key) != 16 {
return nil, fmt.Errorf("invalid AES-128 key size %d", len(key))
}
if len(iv) != 16 {
return nil, fmt.Errorf("invalid AES-128 IV size %d", len(iv))
}
if len(data) == 0 || len(data)%aes.BlockSize != 0 {
return nil, fmt.Errorf("invalid AES-128 data size %d", len(data))
}
block, err := aes.NewCipher(key)
if err != nil {
return nil, err
}
plain := make([]byte, len(data))
cipher.NewCBCDecrypter(block, iv).CryptBlocks(plain, data)
return pkcs7Unpad(plain, aes.BlockSize)
}
func pkcs7Unpad(data []byte, blockSize int) ([]byte, error) {
if len(data) == 0 {
return nil, fmt.Errorf("empty plaintext")
}
pad := int(data[len(data)-1])
if pad == 0 || pad > blockSize || pad > len(data) {
return nil, fmt.Errorf("invalid PKCS7 padding")
}
for _, b := range data[len(data)-pad:] {
if int(b) != pad {
return nil, fmt.Errorf("invalid PKCS7 padding")
}
}
return data[:len(data)-pad], nil
}
// sequenceIV derives the default per-segment IV mandated by RFC 8216: the
// segment media sequence number as a 16-byte big-endian integer.
func sequenceIV(sequence int64) []byte {
iv := make([]byte, 16)
iv[8] = byte(sequence >> 56)
iv[9] = byte(sequence >> 48)
iv[10] = byte(sequence >> 40)
iv[11] = byte(sequence >> 32)
iv[12] = byte(sequence >> 24)
iv[13] = byte(sequence >> 16)
iv[14] = byte(sequence >> 8)
iv[15] = byte(sequence)
return iv
}
+66
View File
@@ -0,0 +1,66 @@
package hls
import (
"bytes"
"crypto/aes"
"crypto/cipher"
"crypto/rand"
"testing"
)
func pkcs7Pad(data []byte, blockSize int) []byte {
pad := blockSize - len(data)%blockSize
return append(data, bytes.Repeat([]byte{byte(pad)}, pad)...)
}
func TestDecryptAES128Roundtrip(t *testing.T) {
key := make([]byte, 16)
iv := make([]byte, 16)
if _, err := rand.Read(key); err != nil {
t.Fatal(err)
}
if _, err := rand.Read(iv); err != nil {
t.Fatal(err)
}
plain := []byte("hello hls segment payload, padded to block size!")
padded := pkcs7Pad(plain, aes.BlockSize)
block, err := aes.NewCipher(key)
if err != nil {
t.Fatal(err)
}
encrypted := make([]byte, len(padded))
cipher.NewCBCEncrypter(block, iv).CryptBlocks(encrypted, padded)
got, err := decryptAES128(encrypted, key, iv)
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(got, plain) {
t.Errorf("roundtrip mismatch: %q", got)
}
}
func TestDecryptAES128InvalidInput(t *testing.T) {
if _, err := decryptAES128([]byte{1, 2, 3}, make([]byte, 16), make([]byte, 16)); err == nil {
t.Error("non-block-aligned data should fail")
}
if _, err := decryptAES128(make([]byte, 16), make([]byte, 15), make([]byte, 16)); err == nil {
t.Error("short key should fail")
}
if _, err := decryptAES128(make([]byte, 16), make([]byte, 16), make([]byte, 16)); err == nil {
t.Error("all-zero block has invalid padding and should fail")
}
}
func TestSequenceIV(t *testing.T) {
iv := sequenceIV(1)
if len(iv) != 16 || iv[15] != 1 {
t.Errorf("sequence 1 IV wrong: %v", iv)
}
iv = sequenceIV(0x0102030405060708)
for i, want := range []byte{1, 2, 3, 4, 5, 6, 7, 8} {
if iv[8+i] != want {
t.Errorf("sequence IV byte %d: want %d got %d", i, want, iv[8+i])
}
}
}
+24
View File
@@ -0,0 +1,24 @@
// Package hls implements native HTTP Live Streaming downloads for m3u8
// playlists. A task produces exactly one merged, playable output file:
// MPEG-2 TS segments are concatenated as-is, fMP4 streams are merged as
// init section followed by the media fragments, and AES-128 encrypted
// segments are decrypted before they are stored.
//
// Supported subset:
// - VOD media playlists (EXT-X-ENDLIST present) and master playlists whose
// selected variant carries its own audio
// - AES-128 with the identity KEYFORMAT, explicit IV or media-sequence
// derived IV, including key rotation mid-playlist
// - EXT-X-BYTERANGE (including offset inheritance) and fMP4 init sections
// via a single, constant EXT-X-MAP
//
// Everything the merge pipeline cannot reproduce faithfully is rejected with
// an explicit error instead of silently producing a broken output file:
// - live streams (no EXT-X-ENDLIST) and SAMPLE-AES / DRM encryption
// - EXT-X-GAP (would leave a hole) and EXT-X-DISCONTINUITY (broken timeline)
// - changing init sections (a repeated EXT-X-MAP is compared by all its
// attributes, not just the URI) and encrypted EXT-X-MAP without an
// explicit IV
// - separate audio renditions referenced by the selected variant (the
// merged output would have no audio) and non-identity KEYFORMATs
package hls
File diff suppressed because it is too large. Load diff
+100
View File
@@ -0,0 +1,100 @@
package hls
import (
"net/url"
"path"
"strings"
"github.com/GopeedLab/gopeed/internal/fetcher"
)
type FetcherManager struct {
}
func (fm *FetcherManager) Name() string {
return "hls"
}
// Filters registers the .m3u8 extension. The manager must be registered BEFORE
// the http manager in the downloader config: protocol routing picks the first
// manager whose filter matches, and the http filter claims every http(s) URL.
func (fm *FetcherManager) Filters() []*fetcher.SchemeFilter {
return []*fetcher.SchemeFilter{
{
Type: fetcher.FilterTypeFile,
Pattern: "M3U8",
},
}
}
func (fm *FetcherManager) Build() fetcher.Fetcher {
return &Fetcher{}
}
func (fm *FetcherManager) ParseName(u string) string {
parsed, err := url.Parse(u)
if err != nil {
return ""
}
name := path.Base(parsed.Path)
name = strings.TrimSuffix(name, path.Ext(name))
if name == "" || name == "/" || name == "." {
name = parsed.Hostname()
}
return name
}
func (fm *FetcherManager) AutoRename() bool {
return true
}
func (fm *FetcherManager) DefaultConfig() any {
return &config{
SegmentConnections: defaultSegmentConnections,
MaxRetries: defaultMaxRetries,
TimeoutSeconds: defaultTimeoutSeconds,
PrefetchContentLength: true,
}
}
func (fm *FetcherManager) Store(f fetcher.Fetcher) (data any, err error) {
_f := f.(*Fetcher)
_f.mu.Lock()
defer _f.mu.Unlock()
if _f.state == nil {
return nil, nil
}
// The engine's periodic checkpoint calls Store without holding the task
// lock and serializes the returned value afterwards, while Resolve/Patch/
// Start mutate the live state under the fetcher mutex: return a snapshot
// copy taken under that mutex, mirroring the HTTP protocol's Store.
return cloneState(_f.state), nil
}
func cloneState(state *fetcherState) *fetcherState {
cloned := *state
cloned.Segments = make([]*Segment, len(state.Segments))
for i, seg := range state.Segments {
s := *seg
cloned.Segments[i] = &s
}
return &cloned
}
func (fm *FetcherManager) Restore() (v any, f func(meta *fetcher.FetcherMeta, v any) fetcher.Fetcher) {
return &fetcherState{}, func(meta *fetcher.FetcherMeta, v any) fetcher.Fetcher {
fb := &FetcherManager{}
restored := fb.Build().(*Fetcher)
restored.meta = meta
if state, ok := v.(*fetcherState); ok && state.MediaURL != "" {
restored.state = state
}
// Setup is invoked by the engine (restoreFetcher) after this builder
// returns; ReqExtra parsing happens there as well.
return restored
}
}
func (fm *FetcherManager) Close() error {
return nil
}
File diff suppressed because it is too large. Load diff
+100
View File
@@ -0,0 +1,100 @@
package hls
import (
"context"
"crypto/tls"
"fmt"
"io"
"net"
"net/http"
"net/http/cookiejar"
"time"
"github.com/GopeedLab/gopeed/internal/httpclient"
"github.com/GopeedLab/gopeed/pkg/base"
)
const defaultUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36"
const maxPlaylistSize = 10 << 20 // 10MB, playlists are small text files
func (f *Fetcher) timeout() time.Duration {
return time.Duration(f.config.TimeoutSeconds) * time.Second
}
// buildClient builds an HTTP client honoring the task proxy and TLS settings.
// The client has no total timeout on purpose: large segments over slow links
// must not be aborted; body stalls are handled by stallReader instead. It is
// only called while the fetcher is quiescent (Resolve/Start), never from
// inside running worker goroutines.
func (f *Fetcher) buildClient() *http.Client {
var proxy *base.RequestProxy
skipVerify := false
if req := f.meta.Req; req != nil {
proxy = req.Proxy
skipVerify = req.SkipVerifyCert
}
jar, _ := cookiejar.New(nil)
client, err := httpclient.NewClient(httpclient.Options{
Client: httpclient.ClientOptions{
Jar: jar,
},
Transport: httpclient.TransportOptions{
DialContext: (&net.Dialer{
Timeout: f.timeout(),
}).DialContext,
Proxy: f.Ctl.GetProxy(proxy),
TLSClientConfig: &tls.Config{
InsecureSkipVerify: skipVerify,
},
TLSHandshakeTimeout: f.timeout(),
},
Impersonation: httpclient.ImpersonationOptions{
Mode: httpclient.ImpersonationAuto,
Session: f.impSession,
},
})
if err != nil {
panic(fmt.Sprintf("build HTTP client: %v", err))
}
return client
}
// buildRequest builds a request with task extra headers, the default user
// agent and an optional byte range. extra may be nil.
func buildRequest(ctx context.Context, extra *ReqExtra, method, u string, body io.Reader, byterange *ByteRange) (*http.Request, error) {
req, err := http.NewRequestWithContext(ctx, method, u, body)
if err != nil {
return nil, err
}
if extra != nil {
for key, value := range extra.Header {
req.Header.Set(key, value)
}
}
if req.Header.Get("User-Agent") == "" {
req.Header.Set("User-Agent", defaultUserAgent)
}
if byterange != nil {
req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", byterange.Offset, byterange.Offset+byterange.Length-1))
}
return req, nil
}
// stallReader aborts reads that see no data for the configured timeout.
type stallReader struct {
rc io.ReadCloser
timeout time.Duration
}
func (r *stallReader) Read(p []byte) (int, error) {
timer := time.AfterFunc(r.timeout, func() {
r.rc.Close()
})
defer timer.Stop()
return r.rc.Read(p)
}
func (r *stallReader) Close() error {
return r.rc.Close()
}
+112
View File
@@ -0,0 +1,112 @@
package hls
import (
"crypto/rand"
"crypto/sha1"
"encoding/hex"
"encoding/json"
"fmt"
"io"
"os"
"strconv"
)
// journalFile is the persisted completed-segment ledger. It is bound to the
// exact download plan: a journal whose fingerprint does not match the current
// plan is discarded instead of being applied to the wrong segments.
type journalFile struct {
StagingID string `json:"stagingID"`
PlanHash string `json:"planHash"`
SegmentCount int `json:"segmentCount"`
Sizes map[string]int64 `json:"sizes"`
}
// randomStagingID returns a random identifier used to give every task its own
// staging folder, stable across app restarts through the persisted state.
func randomStagingID() string {
buf := make([]byte, 6)
if _, err := rand.Read(buf); err != nil {
// crypto/rand failing is catastrophic anyway; fall back to address
// entropy rather than crashing the fetch.
return fmt.Sprintf("%x", sha1.Sum([]byte(fmt.Sprintf("%p", buf))))[:12]
}
return hex.EncodeToString(buf)
}
// planFingerprint hashes everything that defines the download plan: the media
// URL, the ordered segments with their sequence numbers, byte ranges, keys
// and init markers. Two tasks resolve to different fingerprints even when
// they point at the same playlist URL.
func planFingerprint(state *fetcherState) string {
h := sha1.New()
io.WriteString(h, state.MediaURL+"\n")
io.WriteString(h, strconv.Itoa(len(state.Segments))+"\n")
for _, seg := range state.Segments {
fmt.Fprintf(h, "%d|%s|", seg.Sequence, seg.URI)
if seg.Byterange != nil {
fmt.Fprintf(h, "%d@%d|", seg.Byterange.Offset, seg.Byterange.Length)
}
if seg.Key != nil {
// The IV is part of the plan: same URI with a rotated IV would
// silently decrypt to garbage if the journal were reused.
fmt.Fprintf(h, "key=%s iv=%x|", seg.Key.URI, seg.Key.IV)
}
if seg.IsInit {
io.WriteString(h, "init|")
}
io.WriteString(h, "\n")
}
return hex.EncodeToString(h.Sum(nil))[:16]
}
// loadJournal reads the journal of a run, discarding anything that cannot be
// proven to belong to the current plan: foreign or legacy journals, corrupted
// files, and entries whose recorded size no longer matches the plan.
func loadJournal(run *hlsRun) error {
data, err := os.ReadFile(run.journalPath)
if err != nil {
if os.IsNotExist(err) {
return nil
}
return err
}
var file journalFile
if err := json.Unmarshal(data, &file); err != nil {
// A broken or legacy journal must not kill a restartable download.
return nil
}
if file.StagingID != run.stagingID || file.PlanHash != run.planHash || file.SegmentCount != len(run.segs) {
return nil
}
for key, size := range file.Sizes {
idx, err := strconv.ParseInt(key, 10, 64)
if err != nil || idx < 0 || idx >= int64(len(run.segs)) || size <= 0 {
continue
}
run.journal[idx] = size
}
return nil
}
// saveJournal atomically rewrites the journal through a temp file + rename so
// a crash never leaves a half-written ledger behind.
func saveJournal(run *hlsRun) error {
file := journalFile{
StagingID: run.stagingID,
PlanHash: run.planHash,
SegmentCount: len(run.segs),
Sizes: make(map[string]int64, len(run.journal)),
}
for idx, size := range run.journal {
file.Sizes[strconv.FormatInt(idx, 10)] = size
}
data, err := json.Marshal(file)
if err != nil {
return err
}
tmp := run.journalPath + ".tmp"
if err := os.WriteFile(tmp, data, 0666); err != nil {
return err
}
return os.Rename(tmp, run.journalPath)
}
@@ -0,0 +1,297 @@
package hls
import (
"bytes"
"fmt"
"net/http"
"net/http/httptest"
"os"
"testing"
"github.com/GopeedLab/gopeed/pkg/base"
)
// The validators below check real media structure, not just byte
// concatenation: TS packets are verified on the 188 byte grid with
// continuity-counter rules, PAT/PMT presence and PES start codes; fMP4
// output is verified by walking the ISO-BMFF box hierarchy.
func be32(b []byte) int {
return int(b[0])<<24 | int(b[1])<<16 | int(b[2])<<8 | int(b[3])
}
// validateTS asserts structural validity of a merged MPEG-2 TS stream.
func validateTS(t *testing.T, data []byte) {
t.Helper()
if len(data) == 0 || len(data)%188 != 0 {
t.Fatalf("TS size %d is not a positive multiple of 188", len(data))
}
type pidState struct{ cc int }
ccByPID := map[int]int{}
patSeen, pmtSeen, pesSeen := false, false, false
for off := 0; off < len(data); off += 188 {
pkt := data[off : off+188]
if pkt[0] != 0x47 {
t.Fatalf("bad TS sync byte at offset %d", off)
}
pid := (int(pkt[1]&0x1F) << 8) | int(pkt[2])
pusi := pkt[1]&0x40 != 0
afc := (pkt[3] >> 4) & 0x3
if afc == 0 {
continue // reserved
}
idx := 4
if afc&0x2 != 0 { // adaptation field present
afLen := int(pkt[4])
if afLen > 0 && pkt[5]&0x80 != 0 {
// discontinuity_indicator: restart the CC expectation
delete(ccByPID, pid)
}
idx += 1 + afLen
if afc == 2 {
continue // adaptation only, no payload: no CC rule
}
}
if idx > 188 {
t.Fatalf("adaptation field overruns packet at offset %d", off)
}
if pid == 0x1FFF {
continue // null packet
}
payload := pkt[idx:]
if pusi && len(payload) >= 3 && pid != 0 && payload[0] == 0 && payload[1] == 0 && payload[2] == 1 {
pesSeen = true
}
if pid == 0 {
if ptr := int(payload[0]); 1+ptr < len(payload) && payload[1+ptr] == 0x00 {
patSeen = true // program_association_section table_id
}
} else if pusi && len(payload) >= 2 && payload[0] == 0 && payload[1] == 0x02 {
pmtSeen = true // program_map_section table_id
}
cc := int(pkt[3] & 0x0F)
prev, ok := ccByPID[pid]
if !ok {
ccByPID[pid] = cc
continue
}
// CC increments by one per payload-bearing packet; a repeated value
// models the single legal duplicate packet.
if cc != (prev+1)%16 && cc != prev {
t.Fatalf("continuity error on PID %d: %d after %d at offset %d", pid, cc, prev, off)
}
ccByPID[pid] = cc
}
if !patSeen {
t.Error("no PAT found in merged TS")
}
if !pmtSeen {
t.Error("no PMT found in merged TS")
}
if !pesSeen {
t.Error("no PES start code found in merged TS")
}
}
// walkBoxes iterates the top-level ISO-BMFF boxes of data[off:end].
func walkBoxes(t *testing.T, data []byte, off, end int) []string {
t.Helper()
var types []string
for off+8 <= end {
size := be32(data[off:])
typ := string(data[off+4 : off+8])
if size < 8 || off+size > end {
t.Fatalf("invalid box %q at offset %d (size %d)", typ, off, size)
}
types = append(types, typ)
off += size
}
return types
}
// validateFMP4 asserts the merged fMP4 output is init section followed by
// well-formed moof+mdat fragments.
func validateFMP4(t *testing.T, merged []byte, wantFragments int) {
t.Helper()
types := walkBoxes(t, merged, 0, len(merged))
hasFtyp, hasMoov := false, false
fragments := 0
expectFragment := false
for i, typ := range types {
switch typ {
case "ftyp":
if i != 0 {
t.Errorf("ftyp not at stream start")
}
hasFtyp = true
case "moov":
if i != 1 {
t.Errorf("moov must follow ftyp, found at index %d", i)
}
hasMoov = true
case "styp":
expectFragment = true
case "moof":
if !hasMoov {
t.Errorf("fragment before moov")
}
expectFragment = true
case "mdat":
if !expectFragment {
t.Errorf("mdat outside of a fragment at index %d", i)
}
expectFragment = false
fragments++
case "mfra":
// optional trailer
default:
t.Errorf("unexpected top-level box %q", typ)
}
}
if !hasFtyp || !hasMoov {
t.Error("merged fMP4 missing ftyp/moov init section")
}
if fragments != wantFragments {
t.Errorf("want %d fragments in merged fMP4, got %d", wantFragments, fragments)
}
}
func mediaFixture(t *testing.T, name string) []byte {
t.Helper()
data, err := os.ReadFile("testdata/media/" + name)
if err != nil {
t.Fatal(err)
}
return data
}
func serveFixtureServer(t *testing.T) *httptest.Server {
t.Helper()
mux := http.NewServeMux()
fs := http.FileServer(http.Dir("testdata/media"))
mux.Handle("/", fs)
return httptest.NewServer(mux)
}
func TestHLSRealMedia_TS(t *testing.T) {
seg0 := mediaFixture(t, "sample-00.ts")
seg1 := mediaFixture(t, "sample-01.ts")
playlist := "#EXTM3U\n" +
"#EXTINF:0.5,\nsample-00.ts\n" +
"#EXTINF:0.5,\nsample-01.ts\n" +
"#EXT-X-ENDLIST\n"
mux := http.NewServeMux()
mux.HandleFunc("/video.m3u8", func(w http.ResponseWriter, r *http.Request) {
w.Write([]byte(playlist))
})
mux.Handle("/", http.FileServer(http.Dir("testdata/media")))
server := httptest.NewServer(mux)
defer server.Close()
f := newTestFetcher(t, testConfig())
if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: t.TempDir()}); err != nil {
t.Fatal(err)
}
if err := f.Start(); err != nil {
t.Fatal(err)
}
if err := f.Wait(); err != nil {
t.Fatal(err)
}
got := readOutput(t, f)
if !bytes.Equal(got, append(append([]byte{}, seg0...), seg1...)) {
t.Fatal("merged TS is not the exact concatenation of the segments")
}
validateTS(t, got)
}
func TestHLSRealMedia_FMP4(t *testing.T) {
init := mediaFixture(t, "fmp4-init.mp4")
frag1 := mediaFixture(t, "fmp4-seg1.m4s")
frag2 := mediaFixture(t, "fmp4-seg2.m4s")
frag3 := mediaFixture(t, "fmp4-seg3.m4s")
playlist := "#EXTM3U\n" +
"#EXT-X-MAP:URI=\"fmp4-init.mp4\"\n" +
"#EXTINF:0.5,\nfmp4-seg1.m4s\n" +
"#EXTINF:0.5,\nfmp4-seg2.m4s\n" +
"#EXTINF:0.5,\nfmp4-seg3.m4s\n" +
"#EXT-X-ENDLIST\n"
mux := http.NewServeMux()
mux.HandleFunc("/video.m3u8", func(w http.ResponseWriter, r *http.Request) {
w.Write([]byte(playlist))
})
mux.Handle("/", http.FileServer(http.Dir("testdata/media")))
server := httptest.NewServer(mux)
defer server.Close()
f := newTestFetcher(t, testConfig())
if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: t.TempDir()}); err != nil {
t.Fatal(err)
}
if err := f.Start(); err != nil {
t.Fatal(err)
}
if err := f.Wait(); err != nil {
t.Fatal(err)
}
got := readOutput(t, f)
want := append(append(append(append([]byte{}, init...), frag1...), frag2...), frag3...)
if !bytes.Equal(got, want) {
t.Fatal("merged fMP4 is not the exact concatenation of init + fragments")
}
validateFMP4(t, got, 3)
}
func TestHLSRealMedia_AES128(t *testing.T) {
keys := [][]byte{[]byte("key-A-0123456789"), []byte("key-B-9876543210")}
iv := []byte("iv-1337-01234567")
plain := [][]byte{mediaFixture(t, "sample-00.ts"), mediaFixture(t, "sample-01.ts")}
playlist := "#EXTM3U\n" +
"#EXT-X-MEDIA-SEQUENCE:0\n" +
"#EXT-X-KEY:METHOD=AES-128,URI=\"key-0.bin\",IV=0x" + hexEncode(iv) + "\n" +
"#EXTINF:0.5,\nenc-0.ts\n" +
"#EXT-X-KEY:METHOD=AES-128,URI=\"key-1.bin\"\n" +
"#EXTINF:0.5,\nenc-1.ts\n" +
"#EXT-X-ENDLIST\n"
mux := http.NewServeMux()
mux.HandleFunc("/video.m3u8", func(w http.ResponseWriter, r *http.Request) {
w.Write([]byte(playlist))
})
for i, key := range keys {
k := key
mux.HandleFunc(fmt.Sprintf("/key-%d.bin", i), func(w http.ResponseWriter, r *http.Request) {
w.Write(k)
})
// Segment 0 has an explicit IV, segment 1 derives it from its media
// sequence number (1).
enc := encryptSegment(plain[i], k, map[bool][]byte{true: iv, false: sequenceIV(int64(i))}[i == 0])
mux.HandleFunc(fmt.Sprintf("/enc-%d.ts", i), func(w http.ResponseWriter, r *http.Request) {
w.Write(enc)
})
}
server := httptest.NewServer(mux)
defer server.Close()
f := newTestFetcher(t, testConfig())
if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: t.TempDir()}); err != nil {
t.Fatal(err)
}
if f.meta.Res.Size != 0 {
t.Errorf("encrypted plan must report indeterminate size, got %d", f.meta.Res.Size)
}
if err := f.Start(); err != nil {
t.Fatal(err)
}
if err := f.Wait(); err != nil {
t.Fatal(err)
}
got := readOutput(t, f)
if !bytes.Equal(got, append(append([]byte{}, plain[0]...), plain[1]...)) {
t.Fatal("decrypted output mismatch")
}
validateTS(t, got)
if p := f.Progress(); p.TotalDownloaded() != int64(len(got)) {
t.Errorf("final progress %d != merged size %d", p.TotalDownloaded(), len(got))
}
}
+479
View File
@@ -0,0 +1,479 @@
package hls
import (
"encoding/hex"
"errors"
"fmt"
"net/url"
"path"
"strconv"
"strings"
)
// Variant is one rendition entry of a master playlist.
type Variant struct {
URI string
Bandwidth int64
AvgBandwidth int64
Resolution string
Codecs string
Name string
AudioGroup string // AUDIO attribute: group id of the audio renditions
}
// MediaRendition is one EXT-X-MEDIA entry of a master playlist. Only the
// fields needed to detect unsupported track layouts are kept.
type MediaRendition struct {
Type string
GroupID string
URI string
}
// MasterPlaylist is a parsed master playlist with its rendition groups.
type MasterPlaylist struct {
Variants []*Variant
Rendition map[string][]*MediaRendition // group id -> all EXT-X-MEDIA members
}
// Key is the decryption key applied to the segments that follow its tag, until
// the next EXT-X-KEY tag or the end of the playlist.
type Key struct {
Method string
URI string // absolute URI
IV []byte // explicit IV; nil means derive from the media sequence number
}
// ByteRange is an EXT-X-BYTERANGE sub range of the segment resource.
type ByteRange struct {
Length int64
Offset int64
}
// Segment is one downloadable piece of a media playlist. The init section of
// an fMP4 stream is represented as a segment with IsInit set.
type Segment struct {
URI string
Sequence int64 // media sequence number, used to derive the default AES IV
Byterange *ByteRange
Key *Key // nil means unencrypted
IsInit bool
Gap bool
Duration float64
Size int64 // exact size in bytes when known (byterange / HEAD probe), 0 when unknown
}
// Media is a parsed media playlist.
type Media struct {
Segments []*Segment
Version int
Live bool // no EXT-X-ENDLIST tag
IsFMP4 bool // playlist declares an init section via EXT-X-MAP
Discontinuity bool // playlist contains EXT-X-DISCONTINUITY
}
const (
keyMethodNone = "NONE"
keyMethodAES128 = "AES-128"
keyMethodSampleAES = "SAMPLE-AES"
)
// IsMasterPlaylist reports whether the content looks like a master playlist.
func IsMasterPlaylist(content string) bool {
return strings.Contains(content, "#EXT-X-STREAM-INF")
}
func validateM3U8Header(content string) error {
for _, line := range strings.Split(content, "\n") {
line = strings.TrimSpace(line)
if line == "" {
continue
}
if line != "#EXTM3U" {
return errors.New("content is not a valid m3u8 playlist")
}
return nil
}
return errors.New("empty m3u8 playlist")
}
// ParseMaster parses a master playlist and returns all variants.
func ParseMaster(content string, baseURL *url.URL) ([]*Variant, error) {
master, err := ParseMasterPlaylist(content, baseURL)
if err != nil {
return nil, err
}
return master.Variants, nil
}
// ParseMasterPlaylist parses a master playlist including its EXT-X-MEDIA
// rendition groups.
func ParseMasterPlaylist(content string, baseURL *url.URL) (*MasterPlaylist, error) {
if err := validateM3U8Header(content); err != nil {
return nil, err
}
master := &MasterPlaylist{Rendition: make(map[string][]*MediaRendition)}
var pending *Variant
for _, line := range strings.Split(content, "\n") {
line = strings.TrimSpace(line)
if line == "" {
continue
}
if strings.HasPrefix(line, "#EXT-X-STREAM-INF:") {
attrs := parseAttributes(line[len("#EXT-X-STREAM-INF:"):])
pending = &Variant{
Bandwidth: parseInt64(attrs["BANDWIDTH"]),
AvgBandwidth: parseInt64(attrs["AVERAGE-BANDWIDTH"]),
Resolution: attrs["RESOLUTION"],
Codecs: unquote(attrs["CODECS"]),
Name: unquote(attrs["NAME"]),
AudioGroup: unquote(attrs["AUDIO"]),
}
continue
}
if strings.HasPrefix(line, "#EXT-X-MEDIA:") {
attrs := parseAttributes(line[len("#EXT-X-MEDIA:"):])
rendition := &MediaRendition{
Type: strings.ToUpper(attrs["TYPE"]),
GroupID: unquote(attrs["GROUP-ID"]),
}
// Only an explicit URI means a separate track; resolving an empty
// reference would fabricate one out of the playlist URL.
if uri := unquote(attrs["URI"]); uri != "" {
rendition.URI = resolveURL(baseURL, uri)
}
if rendition.GroupID != "" {
// Keep every member: a group is unsupported as a whole when
// ANY of its members carries its own URI.
master.Rendition[rendition.GroupID] = append(master.Rendition[rendition.GroupID], rendition)
}
continue
}
if pending != nil {
if line[0] != '#' {
pending.URI = resolveURL(baseURL, line)
master.Variants = append(master.Variants, pending)
}
pending = nil
}
}
if len(master.Variants) == 0 {
return nil, errors.New("master playlist has no variants")
}
return master, nil
}
// checkAudioRendition rejects the selected variant when its AUDIO group
// carries a separate audio track: downloading only the video variant would
// silently produce a silent file. Groups the variant does not reference are
// ignored (variants without an AUDIO attribute are assumed self-contained).
func checkAudioRendition(master *MasterPlaylist, best *Variant) error {
if best == nil || best.AudioGroup == "" {
return nil
}
for _, rendition := range master.Rendition[best.AudioGroup] {
if rendition == nil || rendition.Type != "AUDIO" || rendition.URI == "" {
continue
}
return fmt.Errorf(
"selected variant uses a separate audio rendition (group %q), which is not supported: the merged output would have no audio",
best.AudioGroup)
}
return nil
}
// PickBestVariant selects the highest-bandwidth variant.
func PickBestVariant(variants []*Variant) *Variant {
var best *Variant
for _, v := range variants {
if best == nil || v.Bandwidth > best.Bandwidth {
best = v
}
}
return best
}
// ParseMedia parses a media playlist into an ordered segment plan.
func ParseMedia(content string, baseURL *url.URL) (*Media, error) {
if err := validateM3U8Header(content); err != nil {
return nil, err
}
media := &Media{Live: true}
var currentKey *Key
var pendingByterange *ByteRange
var pendingGap bool
var pendingDuration float64
var lastRangeEnd int64
var sequence int64
var mapIdentity string // normalized identity of the current EXT-X-MAP
for _, line := range strings.Split(content, "\n") {
line = strings.TrimSpace(line)
if line == "" {
continue
}
if line[0] != '#' {
// Every non-comment line in a media playlist is a segment URI.
seg := &Segment{
URI: resolveURL(baseURL, line),
Sequence: sequence,
Byterange: pendingByterange,
Key: currentKey,
Gap: pendingGap,
Duration: pendingDuration,
}
if pendingByterange != nil {
// Byterange segments have exact sizes straight from the playlist.
seg.Size = pendingByterange.Length
}
sequence++
pendingByterange = nil
pendingGap = false
pendingDuration = 0
media.Segments = append(media.Segments, seg)
continue
}
switch {
case strings.HasPrefix(line, "#EXTINF:"):
value := line[len("#EXTINF:"):]
if idx := strings.Index(value, ","); idx >= 0 {
value = value[:idx]
}
pendingDuration = parseFloat(value)
case strings.HasPrefix(line, "#EXT-X-KEY:"):
key, err := parseKey(line[len("#EXT-X-KEY:"):], baseURL)
if err != nil {
return nil, err
}
currentKey = key
case strings.HasPrefix(line, "#EXT-X-MAP:"):
attrs := parseAttributes(line[len("#EXT-X-MAP:"):])
uriAttr := unquote(attrs["URI"])
if uriAttr == "" {
return nil, errors.New("EXT-X-MAP without URI is not supported")
}
resolved := resolveURL(baseURL, uriAttr)
// The init section identity covers every attribute, not just the
// URI: a repeated tag with a different BYTERANGE addresses
// different bytes and changes the init section. NUL keeps the
// two parts unambiguously separated.
identity := resolved + "\x00" + attrs["BYTERANGE"]
if mapIdentity != "" {
if mapIdentity != identity {
return nil, errors.New("changing init sections are not supported")
}
media.IsFMP4 = true
continue
}
mapIdentity = identity
init := &Segment{
URI: resolved,
Sequence: sequence,
IsInit: true,
Key: currentKey,
}
if attrs["BYTERANGE"] != "" {
init.Byterange = parseByteRange(attrs["BYTERANGE"], 0)
if init.Byterange == nil {
return nil, fmt.Errorf("invalid EXT-X-MAP BYTERANGE: %s", attrs["BYTERANGE"])
}
}
if init.Key != nil && init.Key.IV == nil {
// RFC 8216 only defines sequence-derived IVs for media
// segments; guessing one for the init section produces
// corrupt output, so require the playlist to be explicit.
return nil, errors.New("encrypted EXT-X-MAP without an explicit IV is not supported")
}
// The init section is prepended to the segment plan, it takes
// the media sequence number of the playlist for IV derivation.
media.Segments = append(media.Segments, init)
media.IsFMP4 = true
case strings.HasPrefix(line, "#EXT-X-BYTERANGE:"):
pendingByterange = parseByteRange(line[len("#EXT-X-BYTERANGE:"):], lastRangeEnd)
if pendingByterange == nil {
return nil, fmt.Errorf("invalid EXT-X-BYTERANGE: %s", line)
}
case line == "#EXT-X-GAP":
pendingGap = true
case line == "#EXT-X-DISCONTINUITY":
media.Discontinuity = true
case line == "#EXT-X-ENDLIST":
media.Live = false
case strings.HasPrefix(line, "#EXT-X-MEDIA-SEQUENCE:"):
sequence = parseInt64(line[len("#EXT-X-MEDIA-SEQUENCE:"):])
case strings.HasPrefix(line, "#EXT-X-VERSION:"):
media.Version = int(parseInt64(line[len("#EXT-X-VERSION:"):]))
}
if pendingByterange != nil {
// Non-segment tags between BYTERANGE and its URI must not reset the
// running offset, so it is tracked here after each parse.
lastRangeEnd = pendingByterange.Offset + pendingByterange.Length
}
}
if len(media.Segments) == 0 {
return nil, errors.New("media playlist has no segments")
}
return media, nil
}
// validateSupported rejects playlist features the byte-concatenation merge
// cannot reproduce faithfully. Skipping such features silently would report
// success while producing a broken file.
func validateSupported(media *Media) error {
if media.Live {
return errors.New("live streams are not supported yet")
}
if media.Discontinuity {
return errors.New("EXT-X-DISCONTINUITY is not supported: the merged output would have a broken timeline")
}
for _, seg := range media.Segments {
if seg.Gap {
return errors.New("EXT-X-GAP is not supported: the missing segment would leave a hole in the merged output")
}
}
return nil
}
func parseKey(attrs string, baseURL *url.URL) (*Key, error) {
parsed := parseAttributes(attrs)
switch method := parsed["METHOD"]; method {
case keyMethodNone:
return nil, nil
case keyMethodAES128:
uri := unquote(parsed["URI"])
if uri == "" {
return nil, errors.New("AES-128 EXT-X-KEY without URI")
}
if format := unquote(parsed["KEYFORMAT"]); format != "" && format != "identity" {
return nil, fmt.Errorf("unsupported EXT-X-KEY KEYFORMAT: %s", format)
}
key := &Key{Method: keyMethodAES128, URI: resolveURL(baseURL, uri)}
if iv, ok := parsed["IV"]; ok {
value := strings.TrimPrefix(strings.ToLower(iv), "0x")
decoded, err := hex.DecodeString(value)
if err != nil || len(decoded) != 16 {
return nil, fmt.Errorf("invalid EXT-X-KEY IV: %s", iv)
}
key.IV = decoded
}
return key, nil
case keyMethodSampleAES:
return nil, errors.New("SAMPLE-AES encryption is not supported")
default:
return nil, fmt.Errorf("unsupported encryption method: %s", method)
}
}
func parseByteRange(value string, defaultOffset int64) *ByteRange {
lengthPart := value
offsetPart := ""
if idx := strings.Index(value, "@"); idx >= 0 {
lengthPart = value[:idx]
offsetPart = value[idx+1:]
}
length := parseInt64(lengthPart)
if length <= 0 {
return nil
}
offset := defaultOffset
if offsetPart != "" {
offset = parseInt64(offsetPart)
if offset < 0 {
return nil
}
}
return &ByteRange{Length: length, Offset: offset}
}
// resolveURL resolves a playlist URI reference against the playlist URL
// following RFC 3986.
func resolveURL(baseURL *url.URL, ref string) string {
refURL, err := url.Parse(ref)
if err != nil {
return ref
}
return baseURL.ResolveReference(refURL).String()
}
// parseAttributes parses an HLS attribute list like FOO="bar",NUM=42.
func parseAttributes(s string) map[string]string {
attrs := make(map[string]string)
var key, val strings.Builder
inKey := true
inQuote := false
flush := func() {
name := strings.ToUpper(strings.TrimSpace(key.String()))
if name != "" {
attrs[name] = strings.TrimSpace(val.String())
}
key.Reset()
val.Reset()
inKey = true
}
for i := 0; i < len(s); i++ {
c := s[i]
if inKey {
if c == '=' {
inKey = false
} else {
key.WriteByte(c)
}
continue
}
if c == '"' {
inQuote = !inQuote
continue
}
if c == ',' && !inQuote {
flush()
continue
}
val.WriteByte(c)
}
flush()
return attrs
}
func unquote(s string) string {
s = strings.TrimSpace(s)
if len(s) >= 2 && strings.HasPrefix(s, "\"") && strings.HasSuffix(s, "\"") {
return s[1 : len(s)-1]
}
return s
}
func parseInt64(s string) int64 {
value, _ := strconv.ParseInt(strings.TrimSpace(s), 10, 64)
return value
}
func parseFloat(s string) float64 {
value, _ := strconv.ParseFloat(strings.TrimSpace(s), 64)
return value
}
// OutputExtension returns the container file extension for the playlist.
func (m *Media) OutputExtension() string {
if m.IsFMP4 {
return ".mp4"
}
return ".ts"
}
// DeriveOutputName builds the merged output file name from a playlist URL.
func DeriveOutputName(playlistURL string, media *Media) string {
name := ""
if u, err := url.Parse(playlistURL); err == nil {
name = path.Base(u.Path)
name = strings.TrimSuffix(name, path.Ext(name))
}
if name == "" || name == "/" || name == "." {
if u, err := url.Parse(playlistURL); err == nil {
name = u.Hostname()
}
}
return name + media.OutputExtension()
}
@@ -0,0 +1,39 @@
package hls
import (
_ "embed"
"net/url"
"testing"
)
//go:embed testdata/olemovienews.m3u8
var realWorldPlaylist string
// TestParseMediaRealWorld guards parsing against a real site playlist
// (olemovienews.com VOD, 138 TS segments).
func TestParseMediaRealWorld(t *testing.T) {
base, err := url.Parse("https://europe.olemovienews.com/ts2/20230125/afAxfjit/mp4/afAxfjit.mp4/index-v1-a1.m3u8")
if err != nil {
t.Fatal(err)
}
media, err := ParseMedia(realWorldPlaylist, base)
if err != nil {
t.Fatal(err)
}
if media.Live {
t.Error("real VOD playlist should not be live")
}
if media.IsFMP4 {
t.Error("playlist should be detected as TS")
}
if len(media.Segments) != 138 {
t.Errorf("want 138 segments, got %d", len(media.Segments))
}
first := media.Segments[0]
if first.URI != "https://europe.olemovienews.com/ts2/20230125/afAxfjit/mp4/afAxfjit.mp4/seg-1-v1-a1.ts" {
t.Errorf("segment URI resolution wrong: %s", first.URI)
}
if media.Segments[0].Sequence != 1 {
t.Errorf("first sequence = %d, want 1", media.Segments[0].Sequence)
}
}
+252
View File
@@ -0,0 +1,252 @@
package hls
import (
"net/url"
"testing"
)
func mustURL(t *testing.T, raw string) *url.URL {
t.Helper()
u, err := url.Parse(raw)
if err != nil {
t.Fatal(err)
}
return u
}
func TestParseMediaTS(t *testing.T) {
content := "#EXTM3U\n" +
"#EXT-X-TARGETDURATION:20\n" +
"#EXT-X-VERSION:3\n" +
"#EXT-X-MEDIA-SEQUENCE:1\n" +
"#EXTINF:20.000,\n" +
"seg-1.ts\n" +
"#EXTINF:20.000,\n" +
"seg-2.ts\n" +
"#EXT-X-ENDLIST\n"
media, err := ParseMedia(content, mustURL(t, "https://cdn.example.com/vod/index.m3u8"))
if err != nil {
t.Fatal(err)
}
if media.Live {
t.Error("VOD playlist should not be live")
}
if media.IsFMP4 {
t.Error("TS playlist should not be fMP4")
}
if len(media.Segments) != 2 {
t.Fatalf("want 2 segments, got %d", len(media.Segments))
}
want := "https://cdn.example.com/vod/seg-1.ts"
if media.Segments[0].URI != want {
t.Errorf("want uri %s, got %s", want, media.Segments[0].URI)
}
if media.Segments[0].Sequence != 1 {
t.Errorf("want sequence 1, got %d", media.Segments[0].Sequence)
}
if media.Segments[0].Key != nil {
t.Error("segment should be unencrypted")
}
}
func TestParseMediaWithoutExtinf(t *testing.T) {
// Some playlists list segment URIs without EXTINF lines.
content := "#EXTM3U\nseg-1.ts\nseg-2.ts\n#EXT-X-ENDLIST\n"
media, err := ParseMedia(content, mustURL(t, "https://cdn.example.com/a/index.m3u8"))
if err != nil {
t.Fatal(err)
}
if len(media.Segments) != 2 {
t.Fatalf("want 2 segments, got %d", len(media.Segments))
}
}
func TestParseMediaLive(t *testing.T) {
content := "#EXTM3U\n#EXT-X-TARGETDURATION:6\n#EXTINF:6.0,\nseg-1.ts\n"
media, err := ParseMedia(content, mustURL(t, "https://cdn.example.com/live/index.m3u8"))
if err != nil {
t.Fatal(err)
}
if !media.Live {
t.Error("playlist without ENDLIST should be live")
}
}
func TestParseMediaFMP4(t *testing.T) {
content := "#EXTM3U\n" +
"#EXT-X-VERSION:6\n" +
"#EXT-X-MAP:URI=\"init.mp4\"\n" +
"#EXTINF:5.0,\n" +
"seg-1.m4s\n" +
"#EXT-X-ENDLIST\n"
media, err := ParseMedia(content, mustURL(t, "https://cdn.example.com/vod/index.m3u8"))
if err != nil {
t.Fatal(err)
}
if !media.IsFMP4 {
t.Error("playlist with MAP should be fMP4")
}
if len(media.Segments) != 2 {
t.Fatalf("want 2 segments (init + 1), got %d", len(media.Segments))
}
if !media.Segments[0].IsInit {
t.Error("first segment should be the init section")
}
if media.Segments[0].URI != "https://cdn.example.com/vod/init.mp4" {
t.Errorf("init uri wrong: %s", media.Segments[0].URI)
}
if media.OutputExtension() != ".mp4" {
t.Errorf("want .mp4, got %s", media.OutputExtension())
}
}
func TestParseMediaMultiMapError(t *testing.T) {
content := "#EXTM3U\n" +
"#EXT-X-MAP:URI=\"init1.mp4\"\n" +
"#EXTINF:5.0,\nseg-1.m4s\n" +
"#EXT-X-DISCONTINUITY\n" +
"#EXT-X-MAP:URI=\"init2.mp4\"\n" +
"#EXTINF:5.0,\nseg-2.m4s\n" +
"#EXT-X-ENDLIST\n"
if _, err := ParseMedia(content, mustURL(t, "https://cdn.example.com/vod/index.m3u8")); err == nil {
t.Error("multi-period fMP4 should be rejected")
}
}
func TestParseMediaKeyRotation(t *testing.T) {
content := "#EXTM3U\n" +
"#EXT-X-KEY:METHOD=AES-128,URI=\"key1.bin\",IV=0x00000000000000000000000000000001\n" +
"#EXTINF:5.0,\nseg-1.ts\n" +
"#EXT-X-KEY:METHOD=AES-128,URI=\"key2.bin\"\n" +
"#EXTINF:5.0,\nseg-2.ts\n" +
"#EXT-X-KEY:METHOD=NONE\n" +
"#EXTINF:5.0,\nseg-3.ts\n" +
"#EXT-X-ENDLIST\n"
media, err := ParseMedia(content, mustURL(t, "https://cdn.example.com/vod/index.m3u8"))
if err != nil {
t.Fatal(err)
}
if media.Segments[0].Key == nil || media.Segments[0].Key.URI != "https://cdn.example.com/vod/key1.bin" {
t.Fatalf("segment 1 key wrong: %+v", media.Segments[0].Key)
}
if len(media.Segments[0].Key.IV) != 16 || media.Segments[0].Key.IV[15] != 1 {
t.Error("explicit IV not parsed")
}
if media.Segments[1].Key == nil || media.Segments[1].Key.URI != "https://cdn.example.com/vod/key2.bin" {
t.Fatalf("segment 2 key wrong: %+v", media.Segments[1].Key)
}
if media.Segments[1].Key.IV != nil {
t.Error("key without IV should leave IV nil (sequence-derived)")
}
if media.Segments[2].Key != nil {
t.Error("segment 3 should be unencrypted after METHOD=NONE")
}
}
func TestParseMediaSampleAESError(t *testing.T) {
content := "#EXTM3U\n#EXT-X-KEY:METHOD=SAMPLE-AES,URI=\"skd://key\"\n#EXTINF:5.0,\nseg-1.ts\n"
if _, err := ParseMedia(content, mustURL(t, "https://cdn.example.com/vod/index.m3u8")); err == nil {
t.Error("SAMPLE-AES should be rejected")
}
}
func TestParseMediaByterange(t *testing.T) {
content := "#EXTM3U\n" +
"#EXT-X-BYTERANGE:1000@0\nseg.ts\n" +
"#EXT-X-BYTERANGE:2000\nseg.ts\n" +
"#EXT-X-BYTERANGE:3000\nseg.ts\n" +
"#EXT-X-ENDLIST\n"
media, err := ParseMedia(content, mustURL(t, "https://cdn.example.com/vod/index.m3u8"))
if err != nil {
t.Fatal(err)
}
ranges := [][2]int64{{0, 1000}, {1000, 2000}, {3000, 3000}}
for i, want := range ranges {
br := media.Segments[i].Byterange
if br == nil {
t.Fatalf("segment %d byterange missing", i)
}
if br.Offset != want[0] || br.Length != want[1] {
t.Errorf("segment %d: want offset %d length %d, got %d/%d", i, want[0], want[1], br.Offset, br.Length)
}
}
}
func TestParseMediaGap(t *testing.T) {
content := "#EXTM3U\n#EXTINF:5.0,\nseg-1.ts\n#EXT-X-GAP\n#EXTINF:5.0,\nseg-2.ts\n#EXT-X-ENDLIST\n"
media, err := ParseMedia(content, mustURL(t, "https://cdn.example.com/vod/index.m3u8"))
if err != nil {
t.Fatal(err)
}
if media.Segments[0].Gap || !media.Segments[1].Gap {
t.Error("gap flags wrong")
}
}
func TestParseMediaInvalidHeader(t *testing.T) {
if _, err := ParseMedia("not a playlist", mustURL(t, "https://cdn.example.com/a.m3u8")); err == nil {
t.Error("invalid content should be rejected")
}
}
func TestParseMasterPickBest(t *testing.T) {
content := "#EXTM3U\n" +
"#EXT-X-MEDIA:TYPE=AUDIO,GROUP-ID=\"aud\",NAME=\"en\",DEFAULT=YES,URI=\"audio/index.m3u8\"\n" +
"#EXT-X-STREAM-INF:BANDWIDTH=800000,RESOLUTION=640x360\n" +
"360p/index.m3u8\n" +
"#EXT-X-STREAM-INF:BANDWIDTH=2500000,RESOLUTION=1920x1080,CODECS=\"avc1.640028,mp4a.40.2\"\n" +
"1080p/index.m3u8?sign=abc\n" +
"#EXT-X-ENDLIST\n"
variants, err := ParseMaster(content, mustURL(t, "https://cdn.example.com/vod/master.m3u8"))
if err != nil {
t.Fatal(err)
}
if len(variants) != 2 {
t.Fatalf("want 2 variants, got %d", len(variants))
}
best := PickBestVariant(variants)
if best.Bandwidth != 2500000 {
t.Errorf("want highest bandwidth variant, got %d", best.Bandwidth)
}
if best.URI != "https://cdn.example.com/vod/1080p/index.m3u8?sign=abc" {
t.Errorf("variant uri wrong: %s", best.URI)
}
}
func TestResolveURLQueryNotInherited(t *testing.T) {
base := mustURL(t, "https://cdn.example.com/vod/index.m3u8?sign=abc")
got := resolveURL(base, "seg-1.ts")
if got != "https://cdn.example.com/vod/seg-1.ts" {
t.Errorf("relative resolution should follow RFC3986 (no query inheritance), got %s", got)
}
}
func TestDeriveOutputName(t *testing.T) {
media := &Media{}
if got := DeriveOutputName("https://cdn.example.com/ts2/a/index-v1-a1.m3u8", media); got != "index-v1-a1.ts" {
t.Errorf("want index-v1-a1.ts, got %s", got)
}
fmp4 := &Media{IsFMP4: true}
if got := DeriveOutputName("https://cdn.example.com/ts2/a/index.m3u8", fmp4); got != "index.mp4" {
t.Errorf("want index.mp4, got %s", got)
}
if got := DeriveOutputName("https://cdn.example.com/play?token=x", media); got != "play.ts" {
t.Errorf("want play.ts, got %s", got)
}
if got := DeriveOutputName("https://cdn.example.com/", media); got != "cdn.example.com.ts" {
t.Errorf("want host fallback, got %s", got)
}
}
func TestParseAttributesQuotedComma(t *testing.T) {
attrs := parseAttributes(`URI="a,b.ts",IV=0xABCDEF00000000000000000000000001,NAME="x"`)
if attrs["URI"] != "a,b.ts" {
t.Errorf("quoted comma value wrong: %q", attrs["URI"])
}
if attrs["IV"] != "0xABCDEF00000000000000000000000001" {
t.Errorf("hex value wrong: %q", attrs["IV"])
}
if attrs["NAME"] != "x" {
t.Errorf("quoted value wrong: %q", attrs["NAME"])
}
}
+296
View File
@@ -0,0 +1,296 @@
package hls
import (
"bytes"
"context"
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"sync/atomic"
"testing"
"time"
"github.com/GopeedLab/gopeed/pkg/base"
)
func TestFetcherE2E_SameURLTwoTasks(t *testing.T) {
const total = 6
mux := http.NewServeMux()
var playlist strings.Builder
playlist.WriteString("#EXTM3U\n")
for i := 0; i < total; i++ {
playlist.WriteString(fmt.Sprintf("#EXTINF:4,\nseg-%d.ts\n", i))
}
playlist.WriteString("#EXT-X-ENDLIST\n")
mux.HandleFunc("/video.m3u8", func(w http.ResponseWriter, r *http.Request) {
w.Write([]byte(playlist.String()))
})
segments := make(map[int][]byte)
for i := 0; i < total; i++ {
segments[i] = []byte(fmt.Sprintf("SHARED-SEGMENT-%02d", i))
idx := i
mux.HandleFunc(fmt.Sprintf("/seg-%d.ts", idx), func(w http.ResponseWriter, r *http.Request) {
time.Sleep(120 * time.Millisecond)
w.Write(segments[idx])
})
}
server := httptest.NewServer(mux)
defer server.Close()
cfg := testConfig()
cfg.SegmentConnections = 2
cfg.PrefetchContentLength = false
dir := t.TempDir()
// Two tasks for the same playlist URL into the same directory, like a
// user duplicating a task. The engine would rename the second output
// (AutoRename); mirror that here.
f1 := newTestFetcher(t, cfg)
if err := f1.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: dir}); err != nil {
t.Fatal(err)
}
f2 := newTestFetcher(t, cfg)
if err := f2.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: dir, Name: "video-2.ts"}); err != nil {
t.Fatal(err)
}
if f1.state.StagingDir == "" || f1.state.StagingDir == f2.state.StagingDir {
t.Fatalf("staging dirs must be distinct per task: %q vs %q", f1.state.StagingDir, f2.state.StagingDir)
}
// Start task 1, let it stage some segments, then pause it.
if err := f1.Start(); err != nil {
t.Fatal(err)
}
time.Sleep(250 * time.Millisecond)
if err := f1.Pause(); err != nil {
t.Fatal(err)
}
staged1, err := os.ReadDir(f1.state.StagingDir)
if err != nil || len(staged1) == 0 {
t.Fatalf("task 1 should have staged segments, dir: %s", f1.state.StagingDir)
}
// Task 2 must complete without touching task 1's staging data...
if err := f2.Start(); err != nil {
t.Fatal(err)
}
if err := f2.Wait(); err != nil {
t.Fatal(err)
}
stillThere, err := os.ReadDir(f1.state.StagingDir)
if err != nil || len(stillThere) != len(staged1) {
t.Fatalf("task 2 disturbed task 1 staging: had %d entries, now %d", len(staged1), len(stillThere))
}
// ...and task 1 resumes to a complete, correct output of its own.
if err := f1.Start(); err != nil {
t.Fatal(err)
}
if err := f1.Wait(); err != nil {
t.Fatal(err)
}
var want []byte
for i := 0; i < total; i++ {
want = append(want, segments[i]...)
}
if !bytes.Equal(readOutput(t, f1), want) || !bytes.Equal(readOutput(t, f2), want) {
t.Error("same-URL task outputs mismatch")
}
entries, _ := os.ReadDir(dir)
if len(entries) != 2 {
names := make([]string, 0, len(entries))
for _, e := range entries {
names = append(names, e.Name())
}
t.Errorf("expected only the two outputs to remain, got %v", names)
}
}
func TestFetcherE2E_ForeignJournalDiscarded(t *testing.T) {
const total = 3
var served atomic.Int64
mux := http.NewServeMux()
var playlist strings.Builder
playlist.WriteString("#EXTM3U\n")
for i := 0; i < total; i++ {
playlist.WriteString(fmt.Sprintf("#EXTINF:4,\nseg-%d.ts\n", i))
}
playlist.WriteString("#EXT-X-ENDLIST\n")
mux.HandleFunc("/video.m3u8", func(w http.ResponseWriter, r *http.Request) {
w.Write([]byte(playlist.String()))
})
for i := 0; i < total; i++ {
mux.HandleFunc(fmt.Sprintf("/seg-%d.ts", i), func(w http.ResponseWriter, r *http.Request) {
served.Add(1)
w.Write([]byte("DATA"))
})
}
server := httptest.NewServer(mux)
defer server.Close()
cfg := testConfig()
cfg.PrefetchContentLength = false
f := newTestFetcher(t, cfg)
if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: t.TempDir()}); err != nil {
t.Fatal(err)
}
// Plant a journal that does not belong to this plan (wrong fingerprint).
tempDir := f.resolveStagingDirLocked()
if err := os.MkdirAll(tempDir, 0777); err != nil {
t.Fatal(err)
}
foreign := journalFile{
StagingID: "some-other-task",
PlanHash: "deadbeef",
SegmentCount: total,
Sizes: map[string]int64{"0": 4, "1": 4, "2": 4},
}
data, _ := json.Marshal(foreign)
if err := os.WriteFile(filepath.Join(tempDir, "journal.json"), data, 0666); err != nil {
t.Fatal(err)
}
if err := f.Start(); err != nil {
t.Fatal(err)
}
if err := f.Wait(); err != nil {
t.Fatal(err)
}
if got := served.Load(); got != total {
t.Errorf("foreign journal must be discarded, want %d downloads, got %d", total, got)
}
}
func TestFetcherE2E_MergeFailurePreservesStaging(t *testing.T) {
mux := http.NewServeMux()
mux.HandleFunc("/video.m3u8", func(w http.ResponseWriter, r *http.Request) {
w.Write([]byte("#EXTM3U\n#EXTINF:4,\na.ts\n#EXTINF:4,\nb.ts\n#EXT-X-ENDLIST\n"))
})
mux.HandleFunc("/a.ts", func(w http.ResponseWriter, r *http.Request) { w.Write([]byte("AAAA")) })
mux.HandleFunc("/b.ts", func(w http.ResponseWriter, r *http.Request) { w.Write([]byte("BBBB")) })
server := httptest.NewServer(mux)
defer server.Close()
f := newTestFetcher(t, testConfig())
if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: t.TempDir()}); err != nil {
t.Fatal(err)
}
run := &hlsRun{
ctx: context.Background(),
cancel: func() {},
done: make(chan struct{}),
segs: f.state.Segments,
outputPath: f.meta.SingleFilepath(),
stagingID: f.state.StagingID,
planHash: planFingerprint(f.state),
tempDir: f.resolveStagingDirLocked(),
keyCache: make(map[string][]byte),
journal: make(map[int64]int64),
}
run.journalPath = filepath.Join(run.tempDir, "journal.json")
if err := os.MkdirAll(run.tempDir, 0777); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(segmentPath(run.tempDir, 0), []byte("AAAA"), 0666); err != nil {
t.Fatal(err)
}
// Segment 1 is missing on disk: the merge must fail without deleting the
// staged data or leaving a partial output behind.
if err := f.merge(run); err == nil {
t.Fatal("merge with a missing segment should fail")
}
if _, err := os.Stat(segmentPath(run.tempDir, 0)); err != nil {
t.Fatal("merge failure must preserve staged segments")
}
entries, _ := os.ReadDir(filepath.Dir(run.outputPath))
for _, e := range entries {
if strings.HasSuffix(e.Name(), ".part") {
t.Errorf("failed merge left a partial output behind: %s", e.Name())
}
}
// Recover the segment: the retried merge succeeds.
if err := os.WriteFile(segmentPath(run.tempDir, 1), []byte("BBBB"), 0666); err != nil {
t.Fatal(err)
}
if err := f.merge(run); err != nil {
t.Fatalf("retried merge failed: %v", err)
}
got, err := os.ReadFile(run.outputPath)
if err != nil {
t.Fatal(err)
}
if string(got) != "AAAABBBB" {
t.Errorf("retried merge output mismatch: %q", got)
}
}
func TestFetcherE2E_ReMergeOverExistingOutput(t *testing.T) {
const total = 2
mux := http.NewServeMux()
var playlist strings.Builder
playlist.WriteString("#EXTM3U\n")
for i := 0; i < total; i++ {
playlist.WriteString(fmt.Sprintf("#EXTINF:4,\nseg-%d.ts\n", i))
}
playlist.WriteString("#EXT-X-ENDLIST\n")
mux.HandleFunc("/video.m3u8", func(w http.ResponseWriter, r *http.Request) {
w.Write([]byte(playlist.String()))
})
for i := 0; i < total; i++ {
mux.HandleFunc(fmt.Sprintf("/seg-%d.ts", i), func(w http.ResponseWriter, r *http.Request) {
w.Write([]byte("DATA"))
})
}
server := httptest.NewServer(mux)
defer server.Close()
f := newTestFetcher(t, testConfig())
if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: t.TempDir()}); err != nil {
t.Fatal(err)
}
tempDir := f.resolveStagingDirLocked()
if err := os.MkdirAll(tempDir, 0777); err != nil {
t.Fatal(err)
}
// Simulate the crash window after a completed rename but before the task
// was marked done: a stale output file, a complete journal and all
// segments staged.
stale := filepath.Join(filepath.Dir(f.meta.SingleFilepath()), f.state.OutputName)
if err := os.WriteFile(stale, []byte("STALE-OUTPUT"), 0666); err != nil {
t.Fatal(err)
}
jr := &hlsRun{
segs: f.state.Segments,
stagingID: f.state.StagingID,
planHash: planFingerprint(f.state),
journalPath: filepath.Join(tempDir, "journal.json"),
journal: map[int64]int64{0: 4, 1: 4},
}
if err := saveJournal(jr); err != nil {
t.Fatal(err)
}
for i := 0; i < total; i++ {
if err := os.WriteFile(segmentPath(tempDir, int64(i)), []byte("DATA"), 0666); err != nil {
t.Fatal(err)
}
}
if err := f.Start(); err != nil {
t.Fatal(err)
}
if err := f.Wait(); err != nil {
t.Fatalf("re-merge over an existing output failed: %v", err)
}
got, err := os.ReadFile(stale)
if err != nil {
t.Fatal(err)
}
if string(got) != "DATADATA" {
t.Errorf("stale output was not replaced: %q", got)
}
}
+146
View File
@@ -0,0 +1,146 @@
package hls
import (
"net/url"
"strings"
"testing"
)
func mustParseURL(t *testing.T, raw string) *url.URL {
t.Helper()
u, err := url.Parse(raw)
if err != nil {
t.Fatal(err)
}
return u
}
// subsetCase is a media playlist plus the error substring the pipeline must
// reject it with; wantOK playlists must parse AND validate cleanly.
type subsetCase struct {
name string
playlist string
wantErr string
}
func TestSubsetRejections(t *testing.T) {
cases := []subsetCase{
{
name: "gap",
playlist: "#EXTM3U\n#EXTINF:4,\na.ts\n#EXT-X-GAP\n#EXTINF:4,\nb.ts\n#EXT-X-ENDLIST\n",
wantErr: "EXT-X-GAP",
},
{
name: "gap-before-first-segment",
playlist: "#EXTM3U\n#EXT-X-GAP\n#EXTINF:4,\na.ts\n#EXT-X-ENDLIST\n",
wantErr: "EXT-X-GAP",
},
{
name: "discontinuity",
playlist: "#EXTM3U\n#EXTINF:4,\na.ts\n#EXT-X-DISCONTINUITY\n#EXTINF:4,\nb.ts\n#EXT-X-ENDLIST\n",
wantErr: "EXT-X-DISCONTINUITY",
},
{
name: "live",
playlist: "#EXTM3U\n#EXTINF:4,\na.ts\n",
wantErr: "live",
},
{
name: "keyformat",
playlist: "#EXTM3U\n#EXT-X-KEY:METHOD=AES-128,URI=\"k\",KEYFORMAT=\"com.apple.streamingkeydelivery\"\n#EXTINF:4,\na.ts\n#EXT-X-ENDLIST\n",
wantErr: "KEYFORMAT",
},
{
name: "encrypted-map-without-iv",
playlist: "#EXTM3U\n#EXT-X-KEY:METHOD=AES-128,URI=\"k\"\n#EXT-X-MAP:URI=\"init.mp4\"\n#EXTINF:4,\na.m4s\n#EXT-X-ENDLIST\n",
wantErr: "explicit IV",
},
{
name: "map-change-byterange",
playlist: "#EXTM3U\n#EXT-X-MAP:URI=\"init.mp4\",BYTERANGE=\"100@0\"\n#EXTINF:4,\na.m4s\n#EXT-X-MAP:URI=\"init.mp4\",BYTERANGE=\"100@100\"\n#EXTINF:4,\nb.m4s\n#EXT-X-ENDLIST\n",
wantErr: "changing init sections",
},
{
name: "map-change-uri",
playlist: "#EXTM3U\n#EXT-X-MAP:URI=\"init1.mp4\"\n#EXTINF:4,\na.m4s\n#EXT-X-MAP:URI=\"init2.mp4\"\n#EXTINF:4,\nb.m4s\n#EXT-X-ENDLIST\n",
wantErr: "changing init sections",
},
}
baseURL := mustParseURL(t, "https://cdn.example.com/vod/index.m3u8")
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
media, parseErr := ParseMedia(tc.playlist, baseURL)
if parseErr != nil {
if !strings.Contains(parseErr.Error(), tc.wantErr) {
t.Fatalf("parse error %q does not mention %q", parseErr, tc.wantErr)
}
return
}
err := validateSupported(media)
if err == nil {
t.Fatalf("playlist should be rejected with %q", tc.wantErr)
}
if !strings.Contains(err.Error(), tc.wantErr) {
t.Fatalf("error %q does not mention %q", err, tc.wantErr)
}
})
}
}
func TestSubsetRepeatedSameMapAllowed(t *testing.T) {
playlist := "#EXTM3U\n" +
"#EXT-X-MAP:URI=\"init.mp4\",BYTERANGE=\"100@0\"\n" +
"#EXTINF:4,\na.m4s\n" +
"#EXT-X-MAP:URI=\"init.mp4\",BYTERANGE=\"100@0\"\n" +
"#EXTINF:4,\nb.m4s\n" +
"#EXT-X-ENDLIST\n"
media, err := ParseMedia(playlist, mustParseURL(t, "https://cdn.example.com/vod/index.m3u8"))
if err != nil {
t.Fatal(err)
}
if err := validateSupported(media); err != nil {
t.Fatalf("identical repeated MAP must be allowed: %v", err)
}
if len(media.Segments) != 3 { // init + 2 segments, no duplicate init
t.Fatalf("want 3 segments (init + 2), got %d", len(media.Segments))
}
}
func TestMasterAudioRenditionDetection(t *testing.T) {
masterURL := mustParseURL(t, "https://cdn.example.com/master.m3u8")
separateAudio := "#EXTM3U\n" +
"#EXT-X-MEDIA:TYPE=AUDIO,GROUP-ID=\"aud\",NAME=\"en\",DEFAULT=YES,URI=\"audio/index.m3u8\"\n" +
"#EXT-X-STREAM-INF:BANDWIDTH=1000000,AUDIO=\"aud\"\nvideo/index.m3u8\n"
master, err := ParseMasterPlaylist(separateAudio, masterURL)
if err != nil {
t.Fatal(err)
}
if err := checkAudioRendition(master, PickBestVariant(master.Variants)); err == nil {
t.Fatal("selected variant with a separate audio rendition must be rejected")
}
// A group the variant does not reference must not cause a false positive.
unrelatedGroup := "#EXTM3U\n" +
"#EXT-X-MEDIA:TYPE=AUDIO,GROUP-ID=\"aud\",NAME=\"en\",DEFAULT=YES,URI=\"audio/index.m3u8\"\n" +
"#EXT-X-STREAM-INF:BANDWIDTH=1000000\nvideo/index.m3u8\n"
master, err = ParseMasterPlaylist(unrelatedGroup, masterURL)
if err != nil {
t.Fatal(err)
}
if err := checkAudioRendition(master, PickBestVariant(master.Variants)); err != nil {
t.Fatalf("unreferenced audio group must not be rejected: %v", err)
}
// A rendition group without URIs (muxed tracks) is fine.
groupWithoutURI := "#EXTM3U\n" +
"#EXT-X-MEDIA:TYPE=AUDIO,GROUP-ID=\"aud\",NAME=\"en\",DEFAULT=YES\n" +
"#EXT-X-STREAM-INF:BANDWIDTH=1000000,AUDIO=\"aud\"\nvideo/index.m3u8\n"
master, err = ParseMasterPlaylist(groupWithoutURI, masterURL)
if err != nil {
t.Fatal(err)
}
if err := checkAudioRendition(master, PickBestVariant(master.Variants)); err != nil {
t.Fatalf("audio group without URI must not be rejected: %v", err)
}
}
+10
View File
@@ -0,0 +1,10 @@
Media fixtures generated locally with FFmpeg 7.0.2 (testsrc + sine sources,
no third-party content):
- sample-00.ts / sample-01.ts: one continuous MPEG-2 TS encode split at a
packet boundary, so continuity counters and PCR are consistent across the
segment join exactly like segments cut from a single real-time stream.
- fmp4-init.mp4: ftyp+moov init section of a fragmented MP4.
- fmp4-seg1.m4s .. fmp4-seg3.m4s: moof+mdat media fragments (styp omitted by
the muxer, which is legal) split at moof boundaries from one fragmented MP4
(-movflags +frag_keyframe+empty_moov+default_base_moof -frag_duration 500000).
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
+283
View File
@@ -0,0 +1,283 @@
#EXTM3U
#EXT-X-TARGETDURATION:20
#EXT-X-ALLOW-CACHE:YES
#EXT-X-PLAYLIST-TYPE:VOD
#EXT-X-VERSION:3
#EXT-X-MEDIA-SEQUENCE:1
#EXTINF:20.000,
seg-1-v1-a1.ts
#EXTINF:20.000,
seg-2-v1-a1.ts
#EXTINF:20.000,
seg-3-v1-a1.ts
#EXTINF:20.000,
seg-4-v1-a1.ts
#EXTINF:20.000,
seg-5-v1-a1.ts
#EXTINF:20.000,
seg-6-v1-a1.ts
#EXTINF:20.000,
seg-7-v1-a1.ts
#EXTINF:20.000,
seg-8-v1-a1.ts
#EXTINF:20.000,
seg-9-v1-a1.ts
#EXTINF:20.000,
seg-10-v1-a1.ts
#EXTINF:20.000,
seg-11-v1-a1.ts
#EXTINF:20.000,
seg-12-v1-a1.ts
#EXTINF:20.000,
seg-13-v1-a1.ts
#EXTINF:20.000,
seg-14-v1-a1.ts
#EXTINF:20.000,
seg-15-v1-a1.ts
#EXTINF:20.000,
seg-16-v1-a1.ts
#EXTINF:20.000,
seg-17-v1-a1.ts
#EXTINF:20.000,
seg-18-v1-a1.ts
#EXTINF:20.000,
seg-19-v1-a1.ts
#EXTINF:20.000,
seg-20-v1-a1.ts
#EXTINF:20.000,
seg-21-v1-a1.ts
#EXTINF:20.000,
seg-22-v1-a1.ts
#EXTINF:20.000,
seg-23-v1-a1.ts
#EXTINF:20.000,
seg-24-v1-a1.ts
#EXTINF:20.000,
seg-25-v1-a1.ts
#EXTINF:20.000,
seg-26-v1-a1.ts
#EXTINF:20.000,
seg-27-v1-a1.ts
#EXTINF:20.000,
seg-28-v1-a1.ts
#EXTINF:20.000,
seg-29-v1-a1.ts
#EXTINF:20.000,
seg-30-v1-a1.ts
#EXTINF:20.000,
seg-31-v1-a1.ts
#EXTINF:20.000,
seg-32-v1-a1.ts
#EXTINF:20.000,
seg-33-v1-a1.ts
#EXTINF:20.000,
seg-34-v1-a1.ts
#EXTINF:20.000,
seg-35-v1-a1.ts
#EXTINF:20.000,
seg-36-v1-a1.ts
#EXTINF:20.000,
seg-37-v1-a1.ts
#EXTINF:20.000,
seg-38-v1-a1.ts
#EXTINF:20.000,
seg-39-v1-a1.ts
#EXTINF:20.000,
seg-40-v1-a1.ts
#EXTINF:20.000,
seg-41-v1-a1.ts
#EXTINF:20.000,
seg-42-v1-a1.ts
#EXTINF:20.000,
seg-43-v1-a1.ts
#EXTINF:20.000,
seg-44-v1-a1.ts
#EXTINF:20.000,
seg-45-v1-a1.ts
#EXTINF:20.000,
seg-46-v1-a1.ts
#EXTINF:20.000,
seg-47-v1-a1.ts
#EXTINF:20.000,
seg-48-v1-a1.ts
#EXTINF:20.000,
seg-49-v1-a1.ts
#EXTINF:20.000,
seg-50-v1-a1.ts
#EXTINF:20.000,
seg-51-v1-a1.ts
#EXTINF:20.000,
seg-52-v1-a1.ts
#EXTINF:20.000,
seg-53-v1-a1.ts
#EXTINF:20.000,
seg-54-v1-a1.ts
#EXTINF:20.000,
seg-55-v1-a1.ts
#EXTINF:20.000,
seg-56-v1-a1.ts
#EXTINF:20.000,
seg-57-v1-a1.ts
#EXTINF:20.000,
seg-58-v1-a1.ts
#EXTINF:20.000,
seg-59-v1-a1.ts
#EXTINF:20.000,
seg-60-v1-a1.ts
#EXTINF:20.000,
seg-61-v1-a1.ts
#EXTINF:20.000,
seg-62-v1-a1.ts
#EXTINF:20.000,
seg-63-v1-a1.ts
#EXTINF:20.000,
seg-64-v1-a1.ts
#EXTINF:20.000,
seg-65-v1-a1.ts
#EXTINF:20.000,
seg-66-v1-a1.ts
#EXTINF:20.000,
seg-67-v1-a1.ts
#EXTINF:20.000,
seg-68-v1-a1.ts
#EXTINF:20.000,
seg-69-v1-a1.ts
#EXTINF:20.000,
seg-70-v1-a1.ts
#EXTINF:20.000,
seg-71-v1-a1.ts
#EXTINF:20.000,
seg-72-v1-a1.ts
#EXTINF:20.000,
seg-73-v1-a1.ts
#EXTINF:20.000,
seg-74-v1-a1.ts
#EXTINF:20.000,
seg-75-v1-a1.ts
#EXTINF:20.000,
seg-76-v1-a1.ts
#EXTINF:20.000,
seg-77-v1-a1.ts
#EXTINF:20.000,
seg-78-v1-a1.ts
#EXTINF:20.000,
seg-79-v1-a1.ts
#EXTINF:20.000,
seg-80-v1-a1.ts
#EXTINF:20.000,
seg-81-v1-a1.ts
#EXTINF:20.000,
seg-82-v1-a1.ts
#EXTINF:20.000,
seg-83-v1-a1.ts
#EXTINF:20.000,
seg-84-v1-a1.ts
#EXTINF:20.000,
seg-85-v1-a1.ts
#EXTINF:20.000,
seg-86-v1-a1.ts
#EXTINF:20.000,
seg-87-v1-a1.ts
#EXTINF:20.000,
seg-88-v1-a1.ts
#EXTINF:20.000,
seg-89-v1-a1.ts
#EXTINF:20.000,
seg-90-v1-a1.ts
#EXTINF:20.000,
seg-91-v1-a1.ts
#EXTINF:20.000,
seg-92-v1-a1.ts
#EXTINF:20.000,
seg-93-v1-a1.ts
#EXTINF:20.000,
seg-94-v1-a1.ts
#EXTINF:20.000,
seg-95-v1-a1.ts
#EXTINF:20.000,
seg-96-v1-a1.ts
#EXTINF:20.000,
seg-97-v1-a1.ts
#EXTINF:20.000,
seg-98-v1-a1.ts
#EXTINF:20.000,
seg-99-v1-a1.ts
#EXTINF:20.000,
seg-100-v1-a1.ts
#EXTINF:20.000,
seg-101-v1-a1.ts
#EXTINF:20.000,
seg-102-v1-a1.ts
#EXTINF:20.000,
seg-103-v1-a1.ts
#EXTINF:20.000,
seg-104-v1-a1.ts
#EXTINF:20.000,
seg-105-v1-a1.ts
#EXTINF:20.000,
seg-106-v1-a1.ts
#EXTINF:20.000,
seg-107-v1-a1.ts
#EXTINF:20.000,
seg-108-v1-a1.ts
#EXTINF:20.000,
seg-109-v1-a1.ts
#EXTINF:20.000,
seg-110-v1-a1.ts
#EXTINF:20.000,
seg-111-v1-a1.ts
#EXTINF:20.000,
seg-112-v1-a1.ts
#EXTINF:20.000,
seg-113-v1-a1.ts
#EXTINF:20.000,
seg-114-v1-a1.ts
#EXTINF:20.000,
seg-115-v1-a1.ts
#EXTINF:20.000,
seg-116-v1-a1.ts
#EXTINF:20.000,
seg-117-v1-a1.ts
#EXTINF:20.000,
seg-118-v1-a1.ts
#EXTINF:20.000,
seg-119-v1-a1.ts
#EXTINF:20.000,
seg-120-v1-a1.ts
#EXTINF:20.000,
seg-121-v1-a1.ts
#EXTINF:20.000,
seg-122-v1-a1.ts
#EXTINF:20.000,
seg-123-v1-a1.ts
#EXTINF:20.000,
seg-124-v1-a1.ts
#EXTINF:20.000,
seg-125-v1-a1.ts
#EXTINF:20.000,
seg-126-v1-a1.ts
#EXTINF:20.000,
seg-127-v1-a1.ts
#EXTINF:20.000,
seg-128-v1-a1.ts
#EXTINF:20.000,
seg-129-v1-a1.ts
#EXTINF:20.000,
seg-130-v1-a1.ts
#EXTINF:20.000,
seg-131-v1-a1.ts
#EXTINF:20.000,
seg-132-v1-a1.ts
#EXTINF:20.000,
seg-133-v1-a1.ts
#EXTINF:20.000,
seg-134-v1-a1.ts
#EXTINF:20.000,
seg-135-v1-a1.ts
#EXTINF:20.000,
seg-136-v1-a1.ts
#EXTINF:20.000,
seg-137-v1-a1.ts
#EXTINF:4.111,
seg-138-v1-a1.ts
#EXT-X-ENDLIST
+3
View File
@@ -1,5 +1,8 @@
package http
// DefaultUserAgent is shared by HTTP downloads and extension HTTP requests.
const DefaultUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36"
type config struct {
UserAgent string `json:"userAgent"`
Connections int `json:"connections"`
+1
View File
@@ -448,6 +448,7 @@ func (f *Fetcher) Resolve(req *base.Request, opts *base.Options) error {
if file.Name == "" || file.Name == "/" || file.Name == "." {
file.Name = httpReq.URL.Hostname()
}
file.Name = appendFilenameExtension(file.Name, resp.Header.Get("Content-Type"))
res.Files = append(res.Files, file)
f.meta.Res = res
+1 -1
View File
@@ -72,7 +72,7 @@ func (fm *FetcherManager) AutoRename() bool {
func (fm *FetcherManager) DefaultConfig() any {
return &config{
UserAgent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36",
UserAgent: DefaultUserAgent,
Connections: 16,
}
}
+24
View File
@@ -221,6 +221,30 @@ func TestFetcher_Resolve(t *testing.T) {
})
}
func TestFetcher_ResolveFilenameFromContentType(t *testing.T) {
server := httptest.NewServer(gohttp.HandlerFunc(func(w gohttp.ResponseWriter, r *gohttp.Request) {
w.Header().Set("Content-Type", "image/webp")
w.WriteHeader(gohttp.StatusOK)
}))
defer server.Close()
fetcher := buildFetcher()
defer fetcher.Pause()
const opaqueName = "OIP-C.u5nEXTXND3PJF0gNTgw5YgHaFP"
err := fetcher.Resolve(&base.Request{URL: server.URL + "/" + opaqueName}, &base.Options{
Name: test.DownloadName,
Path: test.Dir,
})
if err != nil {
t.Fatal(err)
}
if got := fetcher.meta.Res.Files[0].Name; got != opaqueName+".webp" {
t.Fatalf("resolved filename = %q, want %q", got, opaqueName+".webp")
}
}
func TestFetcher_ResolveWithHostHeader(t *testing.T) {
listener := test.StartTestHostHeaderServer()
defer listener.Close()
@@ -136,3 +136,52 @@ func TestUnescapeHTMLEntities(t *testing.T) {
})
}
}
func TestAppendFilenameExtension(t *testing.T) {
tests := []struct {
name string
filename string
contentType string
want string
}{
{
name: "adds extension from content type",
filename: "download",
contentType: "image/png",
want: "download.png",
},
{
name: "parses MIME parameters",
filename: "download",
contentType: "image/png; charset=binary",
want: "download.png",
},
{
name: "preserves existing extension",
filename: "download.zip",
contentType: "application/zip",
want: "download.zip",
},
{
name: "ignores invalid content type",
filename: "download",
contentType: "not a content type",
want: "download",
},
{
name: "ignores unknown content type",
filename: "download",
contentType: "application/x-gopeed-unknown",
want: "download",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := appendFilenameExtension(tt.filename, tt.contentType)
if got != tt.want {
t.Errorf("appendFilenameExtension() = %q, want %q", got, tt.want)
}
})
}
}
+23
View File
@@ -12,6 +12,7 @@ import (
"net/http"
"net/http/cookiejar"
"net/url"
"path"
"strings"
"time"
"unicode/utf8"
@@ -314,6 +315,28 @@ func parseFilename(contentDisposition string) string {
return parseFilenameFallback(contentDisposition)
}
// appendFilenameExtension adds a MIME-derived extension when the resolved
// filename has no usable extension. Some image hosts use opaque path segments
// that look like extensions, even though the response Content-Type is the
// only reliable source for the file type.
func appendFilenameExtension(filename, contentType string) string {
mediaType, _, err := mime.ParseMediaType(contentType)
if err != nil {
return filename
}
extensions, err := mime.ExtensionsByType(mediaType)
if err != nil || len(extensions) == 0 {
return filename
}
ext := path.Ext(filename)
if len(ext) > 1 && len(ext) <= 20 {
return filename
}
return filename + extensions[0]
}
// parseFilenameExtended handles RFC 5987 extended notation (filename*=)
// Format: filename*=charset'language'value (e.g., UTF-8”%E6%B5%8B%E8%AF%95.zip)
func parseFilenameExtended(cd string) string {
+62
View File
@@ -0,0 +1,62 @@
# Extension browser host configuration
The downloader wraps each provider with the extension's stable identity before
constructing the runtime. `OpenOptions.ProfileID`, `DataPath`, and `ProxyURL` are
host-only fields. `parseOpenOptions` never reads them from extension JavaScript.
The extension API is unchanged. The Flutter host uses the fork's public
`WebViewProfile`, `InAppWebViewSettings.profileId`, and profile-scoped
`CookieManager` APIs; the library has no Gopeed-specific method-channel bridge.
Only the Gopeed extension JavaScript boundary keeps these options host-only.
Profiles are persistent and shared across invocations/pages of the same extension.
Different identities use different native stores. Go providers use a SHA-256-based
path under `<StorageDir>/webview`; Apple RPC hosts use a stable UUID data store;
Android uses a named WebView profile. Cookie operations select that same store,
including `clearCookies`. Closing an execution closes its pages, not its profile.
Uninstall closes the extension's pages, prevents its old runtimes from opening
new pages, and removes its native profile before deleting the extension record.
Cleanup failures abort uninstall and can be retried. Reinstallation starts with
empty browser data; an ordinary extension update keeps the existing profile.
Windows/Linux delete the profile directory; Apple removes the named WebKit store.
Android cannot delete a profile already loaded by the current process: when
DELETE_BROWSING_DATA is supported it clears all site data immediately, and records
an empty-profile deletion in SharedPreferences for the next process start. A
reinstallation may reuse that cleared profile and cancels the pending deletion.
If the full-data clearing API is unavailable, uninstall fails with a restart
instruction; the pending profile is deleted at startup before it can be loaded.
No removal operation is exposed through extension JavaScript.
Existing shared browser data is not copied into every extension's new profile.
The downloader lazily owns one loopback forward proxy with HTTP and SOCKS5
listeners. Windows/Linux/Android use HTTP; Apple uses SOCKS5 because its HTTP
CONNECT configuration may bypass plain HTTP. The forwarder reads the current
Gopeed proxy config for each new request/tunnel and supports direct, system,
HTTP(S) upstream proxies with Basic authentication, and authenticated SOCKS5.
Credentials remain in Go and are never sent over the WebView RPC.
Changes affect new requests/connections. Existing CONNECT/SOCKS tunnels keep their
route until closed; opening another page in the same profile does not reset it.
SOCKS carries host/port rather than a full URL: system proxy/PAC selection for
Apple is host-based, with port 80 treated as HTTP and other ports as HTTPS.
Path-dependent PAC rules cannot be reproduced from a SOCKS connection.
Apple requires macOS 14 / iOS 17. Android checks MULTI_PROFILE and PROXY_OVERRIDE.
Unsupported hosts fail explicitly instead of silently sharing browser storage or
ignoring the proxy.
## Validation
- `go test -race ./internal/webview/proxy ./pkg/download/engine/webview`
- `go test ./pkg/download -run 'TestWebViewProfileBound|TestDownloader_ExtensionRuntimeWebView'`
- Windows/Linux: `go test -tags webview -v ./internal/webview/goprovider` (Linux: run under `xvfb-run -a`).
- macOS: start the normal Flutter app, then run `go test -tags webview -v ./internal/webview/rpcprovider` with its socket address. See `rpcprovider/README.md`.
- In webview_go: `go test -tags webview_integration -v -count=1 .` (Linux: run under `xvfb-run -a`).
The shared provider contract covers HTTP/HTTPS proxy routing, concurrent pages,
HttpOnly cookies, LocalStorage, IndexedDB, profile-scoped clearing, reopening,
and removal without affecting another profile. HTTPS routing is checked with a
rejecting upstream tunnel; no test-only certificate bypass is added to the app.
Go proxy unit tests separately cover successful TLS forwarding and authentication.
These cases run in the existing `test.yml` WebView jobs; existing mobile build jobs
compile the Android/iOS bridge. No separate Flutter test application is required.
@@ -0,0 +1,47 @@
//go:build linux && cgo
package goprovider
/*
#cgo pkg-config: gtk+-3.0
#include <gtk/gtk.h>
static void pumpGTK(void) {
// Bound each batch so Go creation/removal requests cannot be starved.
for (int i = 0; i < 64 && g_main_context_pending(NULL); ++i)
g_main_context_iteration(NULL, FALSE);
}
*/
import "C"
import (
"runtime"
"sync"
"time"
)
var gtkThreadOnce sync.Once
var gtkThreadTasks = make(chan func())
func RunMainThreadLoop(run func() int) int { return run() }
func postMainThreadTask(task func()) bool {
gtkThreadOnce.Do(func() {
go func() {
runtime.LockOSThread()
// WebKit must always use its original UI thread, including after
// the last page closes. A page cannot stop this shared event loop.
ticker := time.NewTicker(10 * time.Millisecond)
defer ticker.Stop()
for {
select {
case task := <-gtkThreadTasks:
task()
case <-ticker.C:
}
C.pumpGTK()
}
}()
})
gtkThreadTasks <- task
return true
}
@@ -1,4 +1,4 @@
//go:build !darwin
//go:build !darwin && (!linux || !cgo)
package goprovider
+30 -2
View File
@@ -105,7 +105,15 @@ func (p *pageWrapper) start() error {
defer runtimepkg.UnlockOSThread()
var w webview.WebView
if p.opts.Headless {
if p.opts.DataPath != "" {
var err error
w, err = webview.NewWithOptions(webview.Options{Debug: p.opts.Debug, Headless: p.opts.Headless, DataPath: p.opts.DataPath, ProxyURL: p.opts.ProxyURL})
if err != nil {
p.ready <- err
close(p.done)
return
}
} else if p.opts.Headless {
w = webview.NewHeadless(p.opts.Debug)
} else {
w = webview.New(p.opts.Debug)
@@ -127,12 +135,18 @@ func (p *pageWrapper) start() error {
if err := w.Bind(p.callbackName, func(payload string) error {
return p.handleCallback(payload)
}); err != nil {
w.Destroy()
p.ready <- err
close(p.done)
return
}
applyWindowOptions(w, p.opts)
w.SetHtml(buildBootstrapHTML(p.callbackName, p.readyID))
if runtimepkg.GOOS == "linux" {
// All GTK pages share the process-wide event loop below.
return
}
w.Run()
w.Destroy()
close(p.done)
@@ -318,7 +332,12 @@ func (p *pageWrapper) Close() error {
// the quit message lands on the correct queue.
terminateDone := make(chan struct{})
w.Dispatch(func() {
w.Terminate()
if runtimepkg.GOOS == "linux" {
w.Destroy()
close(p.done)
} else {
w.Terminate()
}
close(terminateDone)
})
<-terminateDone
@@ -610,3 +629,12 @@ func firstNonEmpty(values ...string) string {
}
return ""
}
func (p *Provider) RemoveProfile(profileID, dataPath string) error {
done := make(chan error, 1)
run := func() { done <- webview.RemoveProfile(dataPath) }
if !postMainThreadTask(run) {
run()
}
return <-done
}
@@ -13,3 +13,5 @@ func TestProviderContract(t *testing.T) {
CookieDomainMode: integrationtest.CookieDomainModeRequired,
})
}
func TestProviderProfiles(t *testing.T) { integrationtest.RunProfileContract(t, New()) }
+37 -2
View File
@@ -1,10 +1,14 @@
package integrationtest
import (
"crypto/sha256"
"fmt"
webviewproxy "github.com/GopeedLab/gopeed/internal/webview/proxy"
"net/http"
"net/http/httptest"
"net/url"
"path/filepath"
"runtime"
"strings"
"testing"
"time"
@@ -219,7 +223,19 @@ func openTestPage(t *testing.T, provider enginewebview.Provider, opts enginewebv
}))
t.Cleanup(server.Close)
runtime := enginewebview.NewRuntime(provider, true)
proxy, err := webviewproxy.New(nil)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { proxy.Close() })
path := filepath.Join(t.TempDir(), "profile")
sum := sha256.Sum256([]byte(path))
profileID := fmt.Sprintf("%x-%x-%x-%x-%x", sum[:4], sum[4:6], sum[6:8], sum[8:10], sum[10:16])
proxyURL := proxy.URL()
if runtime.GOOS == "darwin" {
proxyURL = proxy.SOCKSURL()
}
runtime := enginewebview.NewRuntime(&configuredOpener{provider: provider, profileID: profileID, dataPath: path, proxyURL: proxyURL}, true)
page, err := runtime.Open(map[string]any{
"headless": opts.Headless,
"debug": opts.Debug,
@@ -232,7 +248,14 @@ func openTestPage(t *testing.T, provider enginewebview.Provider, opts enginewebv
t.Fatalf("failed to open webview: %v", err)
}
t.Cleanup(func() {
_ = runtime.Close()
if err := runtime.Close(); err != nil {
t.Errorf("close contract page: %v", err)
}
if remover, ok := provider.(enginewebview.ProfileRemover); ok {
if err := remover.RemoveProfile(profileID, path); err != nil {
t.Errorf("remove contract profile: %v", err)
}
}
})
return page, server, server.URL
@@ -380,3 +403,15 @@ func hasCookie(cookies []enginewebview.Cookie, name string, value string) bool {
}
return false
}
type configuredOpener struct {
provider enginewebview.Provider
profileID, dataPath, proxyURL string
}
func (o *configuredOpener) Open(opts enginewebview.OpenOptions) (enginewebview.Page, error) {
opts.ProfileID = o.profileID
opts.DataPath = o.dataPath
opts.ProxyURL = o.proxyURL
return o.provider.Open(opts)
}
@@ -0,0 +1,247 @@
package integrationtest
import (
"crypto/sha256"
"fmt"
"io"
"net"
"net/http"
"net/http/httptest"
"net/url"
"path/filepath"
"reflect"
"runtime"
"sync/atomic"
"testing"
"time"
webviewproxy "github.com/GopeedLab/gopeed/internal/webview/proxy"
webview "github.com/GopeedLab/gopeed/pkg/download/engine/webview"
)
// RunProfileContract exercises the same host API through both native and RPC
// providers. It never bypasses the host bridge with direct Flutter plugin calls.
func RunProfileContract(t *testing.T, provider webview.Provider) {
t.Helper()
waitForProviderAvailable(t, provider, defaultAvailabilityTimeout)
remover, ok := provider.(webview.ProfileRemover)
if !ok {
t.Fatal("provider does not implement host profile removal")
}
proxyURL, secureHits := profileProxy(t)
a := newContractProfile(t, provider, remover, proxyURL)
b := newContractProfile(t, provider, remover, proxyURL)
first := a.open(t)
assertProfileValue(t, first, `async () => { localStorage.setItem('owner','a'); return 'a'; }`, "a")
assertProfileValue(t, first, profileDatabaseScript(true), "saved")
if err := first.SetCookie(webview.Cookie{Name: "session", Value: "a", Domain: "profiles.invalid", Path: "/", HTTPOnly: true, Expires: time.Now().Add(time.Hour)}); err != nil {
t.Fatal(err)
}
second := b.open(t)
assertProfileValue(t, second, `() => localStorage.getItem('owner')`, nil)
assertProfileValue(t, second, profileDatabaseScript(false), nil)
assertProfileCookies(t, second, "", "new profile B")
assertProfileValue(t, second, `() => { localStorage.setItem('owner','b'); return 'b'; }`, "b")
if err := second.SetCookie(webview.Cookie{Name: "session", Value: "b", Domain: "profiles.invalid", Path: "/", HTTPOnly: true}); err != nil {
t.Fatal(err)
}
shared := a.open(t)
assertProfileValue(t, shared, `() => localStorage.getItem('owner')`, "a")
assertProfileValue(t, shared, profileDatabaseScript(false), "saved")
assertProfileCookies(t, shared, "a", "shared profile A")
storedCookies, err := shared.GetCookies()
if err != nil || len(storedCookies) != 1 || storedCookies[0].Expires.Before(time.Now()) {
t.Fatalf("persistent cookie lost its expiration: %#v, error %v", storedCookies, err)
}
assertProfileValue(t, shared, `() => document.cookie`, "") // HttpOnly stays hidden.
if err := second.ClearCookies(); err != nil {
t.Fatal(err)
}
assertProfileCookies(t, first, "a", "clearing B leaves A intact")
if err := second.SetCookie(webview.Cookie{Name: "session", Value: "b", Domain: "profiles.invalid", Path: "/", HTTPOnly: true}); err != nil {
t.Fatal(err)
}
a.close(t)
reopened := a.open(t)
assertProfileValue(t, reopened, `() => localStorage.getItem('owner')`, "a")
assertProfileValue(t, reopened, profileDatabaseScript(false), "saved")
assertProfileCookies(t, reopened, "a", "reopened profile A")
// Include a session cookie immediately before deletion to catch stale
// in-process CookieStore caches when the same profile is recreated.
if err := reopened.SetCookie(webview.Cookie{Name: "session", Value: "remove-me", Domain: "profiles.invalid", Path: "/", HTTPOnly: true}); err != nil {
t.Fatal(err)
}
a.close(t)
a.remove(t)
a.remove(t) // Retrying a successful uninstall is harmless.
reinstalled := a.open(t)
assertProfileValue(t, reinstalled, `() => localStorage.getItem('owner')`, nil)
assertProfileValue(t, reinstalled, profileDatabaseScript(false), nil)
assertProfileCookies(t, reinstalled, "", "reinstalled profile A")
assertProfileValue(t, second, `() => localStorage.getItem('owner')`, "b")
assertProfileCookies(t, second, "b", "B after removing A")
// Refuse the HTTPS tunnel deliberately. Seeing it at our proxy proves HTTPS
// routing without adding a certificate-validation bypass to the test host.
// WebView2 may commit its built-in error page and report navigation as
// complete. The invariant here is that the HTTPS tunnel reaches the proxy.
_ = reinstalled.Goto("https://profiles.invalid/", webview.GotoOptions{TimeoutMS: 5000})
deadline := time.Now().Add(5 * time.Second)
for secureHits.Load() == 0 && time.Now().Before(deadline) {
time.Sleep(10 * time.Millisecond)
}
if secureHits.Load() == 0 {
t.Fatal("HTTPS bypassed the host proxy")
}
}
type contractProfile struct {
provider webview.Provider
remover webview.ProfileRemover
options webview.OpenOptions
pages []webview.Page
}
func newContractProfile(t *testing.T, provider webview.Provider, remover webview.ProfileRemover, proxyURL string) *contractProfile {
t.Helper()
path := filepath.Join(t.TempDir(), "profile")
hash := sha256.Sum256([]byte(path))
id := fmt.Sprintf("%x-%x-%x-%x-%x", hash[:4], hash[4:6], hash[6:8], hash[8:10], hash[10:16])
p := &contractProfile{provider: provider, remover: remover, options: webview.OpenOptions{ProfileID: id, DataPath: path, ProxyURL: proxyURL, Headless: true, Title: "Gopeed profile contract"}}
t.Cleanup(func() { p.close(t); p.remove(t) })
return p
}
func (p *contractProfile) open(t *testing.T) webview.Page {
t.Helper()
page, err := p.provider.Open(p.options)
if err != nil {
t.Fatal(err)
}
p.pages = append(p.pages, page)
if err := page.Goto("http://profiles.invalid/", webview.GotoOptions{TimeoutMS: 10000}); err != nil {
t.Fatal(err)
}
return page
}
func (p *contractProfile) close(t *testing.T) {
t.Helper()
for _, page := range p.pages {
if err := page.Close(); err != nil {
t.Errorf("close profile page: %v", err)
}
}
p.pages = nil
}
func (p *contractProfile) remove(t *testing.T) {
t.Helper()
if err := p.remover.RemoveProfile(p.options.ProfileID, p.options.DataPath); err != nil {
t.Errorf("remove profile: %v", err)
}
}
func assertProfileValue(t *testing.T, page webview.Page, script string, want any) {
t.Helper()
got, err := page.Execute(script)
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(got, want) {
t.Fatalf("script %s: got %#v, want %#v", script, got, want)
}
}
func assertProfileCookies(t *testing.T, page webview.Page, want, stage string) {
t.Helper()
// Native cookie managers propagate updates across browser processes
// asynchronously. Wait for the expected state rather than assuming the
// first IPC read already includes the preceding write.
deadline := time.Now().Add(5 * time.Second)
for {
cookies, err := page.GetCookies()
if err != nil {
t.Fatalf("%s: %v", stage, err)
}
matches := len(cookies) == 0 && want == "" || len(cookies) == 1 && want != "" && cookies[0].Name == "session" && cookies[0].Value == want && cookies[0].HTTPOnly
if matches {
return
}
if time.Now().After(deadline) {
t.Fatalf("%s: cookies %#v, want %q", stage, cookies, want)
}
time.Sleep(20 * time.Millisecond)
}
}
func profileDatabaseScript(write bool) string {
operation := "store.get('owner')"
if write {
operation = "store.put('saved','owner')"
}
return fmt.Sprintf(`async () => await new Promise((resolve,reject)=>{
const request=indexedDB.open('profile-contract',1);
request.onupgradeneeded=()=>request.result.createObjectStore('state');
request.onerror=()=>reject(String(request.error));
request.onsuccess=()=>{
const db=request.result, tx=db.transaction('state','readwrite'), store=tx.objectStore('state');
const op=%s; let value;
op.onsuccess=()=>value=%s;
tx.oncomplete=()=>{db.close();resolve(value===undefined?null:value)};
tx.onerror=()=>{db.close();reject(String(tx.error))};
};
})`, operation, map[bool]string{true: "'saved'", false: "op.result"}[write])
}
func profileProxy(t *testing.T) (string, *atomic.Int32) {
t.Helper()
origin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/html")
w.Header().Set("Cache-Control", "no-store")
fmt.Fprint(w, "<!doctype html><title>Profile contract</title>")
}))
t.Cleanup(origin.Close)
secureHits := &atomic.Int32{}
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodConnect {
origin.Config.Handler.ServeHTTP(w, r)
return
}
if r.Host == "profiles.invalid:443" {
secureHits.Add(1)
http.Error(w, "HTTPS routing fixture rejects tunnels", http.StatusBadGateway)
return
}
if r.Host != "profiles.invalid:80" {
http.Error(w, "unexpected target", http.StatusBadGateway)
return
}
remote, err := net.DialTimeout("tcp", origin.Listener.Addr().String(), 5*time.Second)
if err != nil {
http.Error(w, "origin unavailable", http.StatusBadGateway)
return
}
defer remote.Close()
client, buffer, err := w.(http.Hijacker).Hijack()
if err != nil {
return
}
defer client.Close()
if _, err := buffer.WriteString("HTTP/1.1 200 Connection Established\r\n\r\n"); err != nil {
return
}
if err := buffer.Flush(); err != nil {
return
}
go func() { io.Copy(remote, buffer); remote.Close() }()
io.Copy(client, remote)
}))
t.Cleanup(upstream.Close)
upstreamURL, err := url.Parse(upstream.URL)
if err != nil {
t.Fatal(err)
}
proxy, err := webviewproxy.New(func(*http.Request) (*url.URL, error) { return upstreamURL, nil })
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { proxy.Close() })
if runtime.GOOS == "darwin" {
return proxy.SOCKSURL(), secureHits
}
return proxy.URL(), secureHits
}
+341
View File
@@ -0,0 +1,341 @@
// Package proxy provides the host-owned WebView forward proxy. Browser engines
// always use this endpoint; Gopeed decides the upstream route for each request.
package proxy
import (
"bufio"
"context"
"crypto/tls"
"encoding/base64"
"errors"
"fmt"
"io"
"log"
"net"
"net/http"
"net/url"
"strings"
"sync"
"time"
"github.com/armon/go-socks5"
xproxy "golang.org/x/net/proxy"
)
type Selector func(*http.Request) (*url.URL, error)
type Server struct {
listener net.Listener
socksListener net.Listener
server *http.Server
transport *http.Transport
selectProxy Selector
mu sync.Mutex
connections map[io.Closer]struct{}
closed bool
}
func New(selectProxy Selector) (*Server, error) {
if selectProxy == nil {
selectProxy = func(*http.Request) (*url.URL, error) { return nil, nil }
}
listener, err := net.Listen("tcp4", "127.0.0.1:0")
if err != nil {
return nil, err
}
s := &Server{listener: listener, selectProxy: selectProxy, connections: make(map[io.Closer]struct{})}
s.transport = &http.Transport{Proxy: selectProxy, DisableKeepAlives: true, TLSHandshakeTimeout: 15 * time.Second, ResponseHeaderTimeout: 30 * time.Second}
s.server = &http.Server{Handler: s, ReadHeaderTimeout: 15 * time.Second}
s.socksListener, err = net.Listen("tcp4", "127.0.0.1:0")
if err != nil {
listener.Close()
return nil, err
}
socks, err := socks5.New(&socks5.Config{Logger: log.New(io.Discard, "", 0), Resolver: unresolvedDNS{}, Dial: func(ctx context.Context, network, address string) (net.Conn, error) {
_, port, _ := net.SplitHostPort(address)
scheme := "https"
if port == "80" {
scheme = "http"
}
request := &http.Request{URL: &url.URL{Scheme: scheme, Host: address}, Header: make(http.Header)}
upstream, err := selectProxy(request)
if err != nil {
return nil, err
}
timed, cancel := context.WithTimeout(ctx, 30*time.Second)
defer cancel()
conn, err := dialTunnel(timed, address, upstream)
if err != nil {
return nil, err
}
return s.track(conn)
}})
if err != nil {
listener.Close()
s.socksListener.Close()
return nil, err
}
go s.server.Serve(listener)
go socks.Serve(&trackedListener{Listener: s.socksListener, server: s})
return s, nil
}
func (s *Server) URL() string { return "http://" + s.listener.Addr().String() }
func (s *Server) SOCKSURL() string { return "socks5://" + s.socksListener.Addr().String() }
func (s *Server) Close() error {
s.mu.Lock()
s.closed = true
connections := make([]io.Closer, 0, len(s.connections))
for c := range s.connections {
connections = append(connections, c)
}
s.mu.Unlock()
for _, c := range connections {
c.Close()
}
s.socksListener.Close()
s.transport.CloseIdleConnections()
return s.server.Close()
}
func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
if r.Method == http.MethodConnect {
s.connect(w, r)
return
}
if !r.URL.IsAbs() || (r.URL.Scheme != "http" && r.URL.Scheme != "https") {
http.Error(w, "absolute HTTP URL required", http.StatusBadRequest)
return
}
out := r.Clone(r.Context())
out.RequestURI = ""
upgrade := out.Header.Get("Upgrade")
stripHopHeaders(out.Header)
if upgrade != "" {
out.Header.Set("Connection", "Upgrade")
out.Header.Set("Upgrade", upgrade)
}
resp, err := s.transport.RoundTrip(out)
if err != nil {
http.Error(w, "upstream request failed", http.StatusBadGateway)
return
}
defer resp.Body.Close()
if resp.StatusCode == http.StatusSwitchingProtocols {
remote, ok := resp.Body.(io.ReadWriteCloser)
if !ok {
http.Error(w, "invalid upstream upgrade", 502)
return
}
client, buffer, err := w.(http.Hijacker).Hijack()
if err != nil {
return
}
defer client.Close()
resp.Body = nil
if err := resp.Write(buffer); err != nil {
return
}
if err := buffer.Flush(); err != nil {
return
}
s.bridge(client, buffer, remote)
return
}
stripHopHeaders(resp.Header)
for k, v := range resp.Header {
w.Header()[k] = v
}
w.WriteHeader(resp.StatusCode)
io.Copy(w, resp.Body)
}
func stripHopHeaders(h http.Header) {
for _, key := range strings.Split(h.Get("Connection"), ",") {
h.Del(strings.TrimSpace(key))
}
for _, key := range []string{"Connection", "Proxy-Connection", "Proxy-Authorization", "Proxy-Authenticate", "Keep-Alive", "TE", "Trailer", "Transfer-Encoding", "Upgrade"} {
h.Del(key)
}
}
func (s *Server) connect(w http.ResponseWriter, r *http.Request) {
if _, _, err := net.SplitHostPort(r.Host); err != nil {
http.Error(w, "invalid tunnel destination", 400)
return
}
target := r.Clone(r.Context())
target.URL = &url.URL{Scheme: "https", Host: r.Host}
upstream, err := s.selectProxy(target)
if err != nil {
http.Error(w, "proxy selection failed", 502)
return
}
ctx, cancel := context.WithTimeout(r.Context(), 30*time.Second)
defer cancel()
remote, err := dialTunnel(ctx, r.Host, upstream)
if err != nil {
http.Error(w, "upstream connection failed", 502)
return
}
defer remote.Close()
client, buffer, err := w.(http.Hijacker).Hijack()
if err != nil {
return
}
defer client.Close()
if _, err := buffer.WriteString("HTTP/1.1 200 Connection Established\r\n\r\n"); err != nil {
return
}
if err := buffer.Flush(); err != nil {
return
}
s.bridge(client, buffer, remote)
}
func (s *Server) bridge(client net.Conn, buffer *bufio.ReadWriter, remote io.ReadWriteCloser) {
s.mu.Lock()
if s.closed {
s.mu.Unlock()
return
}
s.connections[client] = struct{}{}
s.connections[remote] = struct{}{}
s.mu.Unlock()
defer func() { s.mu.Lock(); delete(s.connections, client); delete(s.connections, remote); s.mu.Unlock() }()
done := make(chan struct{})
go func() { io.Copy(remote, buffer); remote.Close(); close(done) }()
io.Copy(client, remote)
client.Close()
<-done
}
type bufferedConn struct {
net.Conn
reader *bufio.Reader
}
func (c *bufferedConn) Read(p []byte) (int, error) { return c.reader.Read(p) }
func dialTunnel(ctx context.Context, target string, upstream *url.URL) (net.Conn, error) {
dialer := &net.Dialer{}
if upstream == nil {
return dialer.DialContext(ctx, "tcp", target)
}
if upstream.Scheme == "socks5" || upstream.Scheme == "socks5h" {
u := *upstream
u.Scheme = "socks5"
d, err := xproxy.FromURL(&u, dialer)
if err != nil {
return nil, err
}
return d.(xproxy.ContextDialer).DialContext(ctx, "tcp", target)
}
if upstream.Scheme != "http" && upstream.Scheme != "https" {
return nil, errors.New("unsupported proxy scheme")
}
host := upstream.Host
if upstream.Port() == "" {
port := "80"
if upstream.Scheme == "https" {
port = "443"
}
host = net.JoinHostPort(upstream.Hostname(), port)
}
conn, err := dialer.DialContext(ctx, "tcp", host)
if err != nil {
return nil, err
}
ok := false
defer func() {
if !ok {
conn.Close()
}
}()
if deadline, exists := ctx.Deadline(); exists {
conn.SetDeadline(deadline)
}
if upstream.Scheme == "https" {
secure := tls.Client(conn, &tls.Config{ServerName: upstream.Hostname(), MinVersion: tls.VersionTLS12})
if err := secure.HandshakeContext(ctx); err != nil {
return nil, err
}
conn = secure
}
request := &http.Request{Method: http.MethodConnect, URL: &url.URL{Opaque: target}, Host: target, Header: make(http.Header)}
if upstream.User != nil {
password, _ := upstream.User.Password()
request.Header.Set("Proxy-Authorization", "Basic "+base64.StdEncoding.EncodeToString([]byte(upstream.User.Username()+":"+password)))
}
if err := request.Write(conn); err != nil {
return nil, err
}
reader := bufio.NewReader(conn)
response, err := http.ReadResponse(reader, request)
if err != nil {
return nil, err
}
if response.StatusCode != 200 {
response.Body.Close()
return nil, fmt.Errorf("upstream CONNECT status %d", response.StatusCode)
}
conn.SetDeadline(time.Time{})
ok = true
return &bufferedConn{Conn: conn, reader: reader}, nil
}
// Leave hostname resolution to the selected upstream (SOCKS remote DNS / CONNECT).
type unresolvedDNS struct{}
func (unresolvedDNS) Resolve(ctx context.Context, name string) (context.Context, net.IP, error) {
return ctx, nil, nil
}
type trackedListener struct {
net.Listener
server *Server
}
func (l *trackedListener) Accept() (net.Conn, error) {
c, err := l.Listener.Accept()
if err != nil {
return nil, err
}
return l.server.track(c)
}
type trackedConn struct {
net.Conn
server *Server
}
func (c *trackedConn) Close() error {
err := c.Conn.Close()
c.server.mu.Lock()
delete(c.server.connections, c)
c.server.mu.Unlock()
return err
}
func (s *Server) track(c net.Conn) (net.Conn, error) {
s.mu.Lock()
defer s.mu.Unlock()
if s.closed {
c.Close()
return nil, net.ErrClosed
}
tracked := &trackedConn{Conn: c, server: s}
s.connections[tracked] = struct{}{}
return tracked, nil
}
func (c *trackedConn) CloseWrite() error { return closeWrite(c.Conn) }
func (c *bufferedConn) CloseWrite() error { return closeWrite(c.Conn) }
func closeWrite(c net.Conn) error {
if writer, ok := c.(interface{ CloseWrite() error }); ok {
return writer.CloseWrite()
}
return c.Close()
}
+315
View File
@@ -0,0 +1,315 @@
package proxy
import (
"bufio"
"context"
"crypto/tls"
"errors"
"io"
"net"
"net/http"
"net/http/httptest"
"net/url"
"strings"
"sync/atomic"
"testing"
"time"
"github.com/armon/go-socks5"
)
func TestRejectedProxyRequests(t *testing.T) {
secret := "upstream-password"
for _, tc := range []struct {
name, method, target string
status int
}{
{"relative URL", "GET", "/relative", 400},
{"unsupported URL", "GET", "ftp://example.com/file", 400},
{"invalid CONNECT", "CONNECT", "example.com", 400},
{"HTTP selection failure", "GET", "http://example.com/", 502},
{"CONNECT selection failure", "CONNECT", "example.com:443", 502},
} {
t.Run(tc.name, func(t *testing.T) {
s := &Server{selectProxy: func(*http.Request) (*url.URL, error) { return nil, errors.New(secret) }}
s.transport = &http.Transport{Proxy: s.selectProxy}
defer s.transport.CloseIdleConnections()
response := httptest.NewRecorder()
s.ServeHTTP(response, httptest.NewRequest(tc.method, tc.target, nil))
if response.Code != tc.status || strings.Contains(response.Body.String(), secret) {
t.Fatalf("unexpected proxy rejection: %d %q", response.Code, response.Body.String())
}
})
}
}
func TestTunnelRejectsUnsafeUpstreams(t *testing.T) {
rejected := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
http.Error(w, "authentication required", http.StatusProxyAuthRequired)
}))
defer rejected.Close()
secure := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
t.Error("untrusted TLS proxy received a request")
}))
defer secure.Close()
for _, upstream := range []string{"ftp://127.0.0.1", rejected.URL, secure.URL} {
t.Run(upstream, func(t *testing.T) {
selected, err := url.Parse(upstream)
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
defer cancel()
conn, err := dialTunnel(ctx, "example.com:443", selected)
if conn != nil {
conn.Close()
}
if err == nil {
t.Fatal("unsafe upstream unexpectedly accepted")
}
})
}
}
func TestRoutesAndAuthentication(t *testing.T) {
origin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Header.Get("Proxy-Authorization") != "" {
t.Error("proxy credentials leaked to origin")
}
w.Write([]byte("origin"))
}))
defer origin.Close()
secure := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Write([]byte("secure")) }))
defer secure.Close()
upstream, err := New(func(*http.Request) (*url.URL, error) { return nil, nil })
if err != nil {
t.Fatal(err)
}
defer upstream.Close()
var authenticated atomic.Int32
authProxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Header.Get("Proxy-Authorization") != "Basic dXNlcjpwYXNz" {
http.Error(w, "authentication required", 407)
return
}
authenticated.Add(1)
upstream.ServeHTTP(w, r)
}))
defer authProxy.Close()
socks, err := socks5.New(&socks5.Config{Credentials: socks5.StaticCredentials{"user": "pass"}})
if err != nil {
t.Fatal(err)
}
listener, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
defer listener.Close()
go socks.Serve(listener)
for _, route := range []string{"", authProxy.URL, "socks5://" + listener.Addr().String()} {
t.Run(route, func(t *testing.T) {
var selected *url.URL
if route != "" {
selected, _ = url.Parse(route)
selected.User = url.UserPassword("user", "pass")
}
local, err := New(func(r *http.Request) (*url.URL, error) { return selected, nil })
if err != nil {
t.Fatal(err)
}
defer local.Close()
proxyURL, _ := url.Parse(local.URL())
transport := &http.Transport{Proxy: http.ProxyURL(proxyURL), TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}
defer transport.CloseIdleConnections()
client := &http.Client{Transport: transport, Timeout: 5 * time.Second}
for _, test := range []struct{ url, want string }{{origin.URL, "origin"}, {secure.URL, "secure"}} {
response, err := client.Get(test.url)
if err != nil {
t.Fatal(err)
}
body, _ := io.ReadAll(response.Body)
response.Body.Close()
if string(body) != test.want {
t.Fatalf("got status %d, body %q", response.StatusCode, body)
}
}
})
}
if authenticated.Load() != 2 {
t.Fatalf("expected authenticated HTTP and CONNECT, got %d", authenticated.Load())
}
}
func TestRoutingChangesAndFailureDoesNotBypassProxy(t *testing.T) {
var hits atomic.Int32
origin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { hits.Add(1); w.Write([]byte("direct")) }))
defer origin.Close()
var useProxy atomic.Bool
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Write([]byte("proxied")) }))
upstreamURL, _ := url.Parse(upstream.URL)
local, err := New(func(*http.Request) (*url.URL, error) {
if useProxy.Load() {
return upstreamURL, nil
}
return nil, nil
})
if err != nil {
t.Fatal(err)
}
defer local.Close()
endpoint, _ := url.Parse(local.URL())
transport := &http.Transport{Proxy: http.ProxyURL(endpoint)}
defer transport.CloseIdleConnections()
client := &http.Client{Transport: transport, Timeout: 3 * time.Second}
for _, want := range []string{"direct", "proxied", "upstream request failed\n"} {
resp, err := client.Get(origin.URL)
if err != nil {
t.Fatal(err)
}
body, _ := io.ReadAll(resp.Body)
resp.Body.Close()
if string(body) != want {
t.Fatalf("got %q want %q", body, want)
}
if want == "direct" {
useProxy.Store(true)
} else if want == "proxied" {
upstream.Close()
}
}
if hits.Load() != 1 {
t.Fatal("failed proxy must not fall back to direct")
}
}
func TestCloseTerminatesTunnels(t *testing.T) {
echo, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
defer echo.Close()
go func() {
c, e := echo.Accept()
if e == nil {
defer c.Close()
io.Copy(c, c)
}
}()
s, err := New(func(*http.Request) (*url.URL, error) { return nil, nil })
if err != nil {
t.Fatal(err)
}
endpoint, _ := url.Parse(s.URL())
c, err := dialTunnel(context.Background(), echo.Addr().String(), endpoint)
if err != nil {
t.Fatal(err)
}
defer c.Close()
c.SetDeadline(time.Now().Add(3 * time.Second))
c.Write([]byte("hello\n"))
line, err := bufio.NewReader(c).ReadString('\n')
if err != nil || line != "hello\n" {
t.Fatalf("echo %q %v", line, err)
}
s.Close()
if _, err := c.Read(make([]byte, 1)); err == nil {
t.Fatal("tunnel stayed open")
}
}
func TestSOCKSIngressPreservesRemoteDNS(t *testing.T) {
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method != "CONNECT" || r.Host != "unresolved.invalid:80" {
t.Errorf("lost destination: %s %s", r.Method, r.Host)
}
c, b, err := w.(http.Hijacker).Hijack()
if err != nil {
return
}
defer c.Close()
b.WriteString("HTTP/1.1 200 Connection Established\r\n\r\n")
b.Flush()
request, err := http.ReadRequest(b.Reader)
if err != nil {
t.Error(err)
return
}
request.Body.Close()
b.WriteString("HTTP/1.1 200 OK\r\nContent-Length: 7\r\nConnection: close\r\n\r\nproxied")
b.Flush()
}))
defer upstream.Close()
target, _ := url.Parse(upstream.URL)
local, err := New(func(r *http.Request) (*url.URL, error) {
if r.URL.Host != "unresolved.invalid:80" {
t.Errorf("selector lost hostname: %s", r.URL)
}
return target, nil
})
if err != nil {
t.Fatal(err)
}
defer local.Close()
endpoint, _ := url.Parse(local.SOCKSURL())
transport := &http.Transport{Proxy: http.ProxyURL(endpoint)}
defer transport.CloseIdleConnections()
client := &http.Client{Transport: transport, Timeout: 5 * time.Second}
response, err := client.Get("http://unresolved.invalid/")
if err != nil {
t.Fatal(err)
}
defer response.Body.Close()
body, _ := io.ReadAll(response.Body)
if string(body) != "proxied" {
t.Fatalf("unexpected body %q", body)
}
}
func TestHTTPUpgradeIsForwarded(t *testing.T) {
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Header.Get("Upgrade") != "fixture" {
http.Error(w, "upgrade lost", 400)
return
}
c, b, err := w.(http.Hijacker).Hijack()
if err != nil {
return
}
defer c.Close()
b.WriteString("HTTP/1.1 101 Switching Protocols\r\nConnection: Upgrade\r\nUpgrade: fixture\r\n\r\n")
b.Flush()
line, err := b.ReadString('\n')
if err != nil {
return
}
b.WriteString(line)
b.Flush()
}))
defer upstream.Close()
local, err := New(nil)
if err != nil {
t.Fatal(err)
}
defer local.Close()
endpoint, _ := url.Parse(local.URL())
c, err := net.Dial("tcp", endpoint.Host)
if err != nil {
t.Fatal(err)
}
defer c.Close()
c.SetDeadline(time.Now().Add(3 * time.Second))
io.WriteString(c, "GET "+upstream.URL+"/ HTTP/1.1\r\nHost: fixture\r\nConnection: Upgrade\r\nUpgrade: fixture\r\n\r\n")
reader := bufio.NewReader(c)
response, err := http.ReadResponse(reader, nil)
if err != nil {
t.Fatal(err)
}
if response.StatusCode != 101 {
t.Fatalf("upgrade failed: %s", response.Status)
}
io.WriteString(c, "hello\n")
line, err := reader.ReadString('\n')
if err != nil || line != "hello\n" {
t.Fatalf("upgraded echo %q %v", line, err)
}
}
+6
View File
@@ -1,5 +1,11 @@
# rpcprovider
The private Flutter native profile bridge is maintained in the
[GopeedLab flutter_inappwebview fork](https://github.com/GopeedLab/flutter_inappwebview/tree/codex/gopeed-profiles).
All three platform overrides pin one commit in `ui/flutter/pubspec.yaml`;
see the fork’s [GOPEED.md](https://github.com/GopeedLab/flutter_inappwebview/blob/codex/gopeed-profiles/GOPEED.md)
for patch scope and upstream synchronization.
`rpcprovider` is the RPC-backed implementation of Gopeed's `webview.Provider` interface.
It is intended for environments where WebView capability is owned by another process, most notably mobile hosts. This package does not implement a WebView itself. Instead, it translates the `pkg/download/engine/webview` page API into single-request `POST + JSON` calls sent to a local host-side RPC service.
+11 -1
View File
@@ -4,6 +4,7 @@ import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"net"
"net/http"
@@ -109,9 +110,14 @@ func (p *page) ClearCookies() error {
}
func (p *page) Close() error {
return p.client.Call(enginewebview.MethodPageClose, enginewebview.PageCloseParams{
err := p.client.Call(enginewebview.MethodPageClose, enginewebview.PageCloseParams{
PageID: p.id,
}, nil)
var rpcErr *enginewebview.RPCError
if errors.As(err, &rpcErr) && rpcErr.Code == enginewebview.ErrorCodePageNotFound {
return nil
}
return err
}
type Client struct {
@@ -193,3 +199,7 @@ func (c *Client) Call(method string, params any, result any) error {
}
return json.Unmarshal(body.Result, result)
}
func (p *Provider) RemoveProfile(profileID, dataPath string) error {
return p.client.Call(enginewebview.MethodProfileRemove, enginewebview.ProfileRemoveParams{ProfileID: profileID}, nil)
}
@@ -37,3 +37,8 @@ func defaultWebViewRPCAddress() string {
}
return filepath.Join(homeDir, "Library", "Application Support", "com.gopeed.gopeed", "gopeed_webview.sock")
}
func TestProviderProfiles(t *testing.T) {
provider := New(enginewebview.RPCConfig{Network: *webViewRPCNetwork, Address: *webViewRPCAddress, Token: *webViewRPCToken})
integrationtest.RunProfileContract(t, provider)
}
@@ -184,3 +184,29 @@ func TestPageSetCookieParamsOmitsZeroExpires(t *testing.T) {
t.Fatalf("expected expires in payload: %s", got)
}
}
func TestRemoveProfileUsesPrivateHostRPC(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
var request struct {
Method string `json:"method"`
Params map[string]string `json:"params"`
}
if err := json.NewDecoder(r.Body).Decode(&request); err != nil {
t.Error(err)
}
if request.Method != enginewebview.MethodProfileRemove || request.Params["profileId"] != "extension-profile" || len(request.Params) != 1 {
t.Errorf("unexpected profile removal request: %+v", request)
}
if r.Header.Get("Authorization") != "Bearer secret" {
t.Error("missing host authentication")
}
w.Header().Set("Content-Type", "application/json")
w.Write([]byte(`{"error":{"code":"PROFILE_REMOVE_FAILED","message":"store busy"}}`))
}))
defer server.Close()
provider := New(enginewebview.RPCConfig{Network: "tcp", Address: strings.TrimPrefix(server.URL, "http://"), Token: "secret"})
err := provider.(enginewebview.ProfileRemover).RemoveProfile("extension-profile", "/host-only/path")
if err == nil || err.Error() != "store busy" {
t.Fatalf("cleanup failure was lost: %v", err)
}
}