Compare commits

..
Author SHA1 Message Date
Levi cc96cff2ea Merge PR #1583: optimize extension API loading
# Conflicts:
#	ui/flutter/lib/features/extensions/presentation/widgets/extension_detail_view.dart
2026-10-10 17:40:56 +08:00
Levi 15738644b0 perf(extensions): load store summaries without waiting for update checks 2026-10-10 17:32:43 +08:00
Levi 44a5f9eefc Merge PR #1582: widen macOS navigation rail 2026-10-10 14:43:06 +08:00
Levi c90c690836 Merge PR #1581: synchronize extension cookie range coverage
# Conflicts:
#	ui/flutter/test/widget_test.dart
2026-10-10 14:43:04 +08:00
Levi dee1c2515f Merge PR #1535: keep extension update chip within card header 2026-10-10 14:42:53 +08:00
Levi bbf1a9fd7d fix(ui): widen navigation rail for macOS traffic lights 2026-10-10 14:34:11 +08:00
Levi b3f462bb7d test(http): synchronize ignored Range response coverage 2026-10-09 21:15:51 +08:00
Levi 0d08004c49 revert: remove macOS traffic light adjustments 2026-10-09 21:13:55 +08:00
Levi 1589d4cbe7 fix: update macOS traffic lights through window events 2026-10-09 20:58:24 +08:00
Levi b6ce7756fb fix: keep macOS traffic lights within the navigation rail 2026-10-09 20:49:14 +08:00
Levi 342c55fbd6 fix: use server-side fetch header permissions for extensions 2026-10-09 19:59:06 +08:00
Levi fe2f45e40d fix: preserve explicit Cookie headers in extension requests 2026-10-09 19:39:05 +08:00
WuFeng ba84a04a43 fix(settings): use Save as the primary label on settings form dialogs (#1571) 2026-10-08 16:31:16 +08:00
Levi 2d4a8cddc1 fix(ui): keep extension update chip within card header 2026-09-26 11:05:43 +08:00
Levi 224b488087 feat(task): record and display task completion time (#1536) 2026-09-25 19:15:31 +08:00
Levi 7ce4fb7f0e fix(ui): smooth task progress bar updates (#1532) 2026-09-25 19:08:20 +08:00
Levi 76c22c50d8 test: align beta2 integration checks 2026-09-25 16:36:27 +08:00
Levi f400137ead Merge PR #1542 into 2.0.0-beta.2 2026-09-25 16:23:55 +08:00
Levi a5ebbca318 Merge PR #1536 into 2.0.0-beta.2
# Conflicts:
#	ui/flutter/test/widget_test.dart
2026-09-25 16:23:55 +08:00
Levi 8f2af5660e Merge PR #1535 into 2.0.0-beta.2 2026-09-25 16:23:39 +08:00
Levi 932a49c1d1 Merge PR #1532 into 2.0.0-beta.2 2026-09-25 16:23:39 +08:00
Levi 212c735b66 Merge latest main into 2.0.0-beta.2 2026-09-25 16:23:39 +08:00
UltraGeek0102 8d1ae93b03 refactor(ios): clean up continued processing task handling (#1547) 2026-09-25 16:21:09 +08:00
ashish6298 b5e8d61f01 fix: move checksum to HTTP opts, fix mismatch retry, scope get_download_speed to aggregate-only 2026-09-25 12:15:50 +05:30
JiangJiang 363153fa33 fix(http): use Content-Range total size for 206 responses (#1545) 2026-09-23 09:52:59 +08:00
Levi f0dab90e09 feat(extension): expose request mutation methods on task.meta.req (#1538) 2026-09-23 09:33:28 +08:00
doctor-martin-sowa 6e60889ce8 feat(lang): Add Slovak localization (app_sk.arb) (#1541) 2026-09-23 09:32:07 +08:00
ashish6298 929e2d2d7c test(base): add unit tests for ChecksumOption serialization 2026-09-22 16:36:41 +05:30
ashish6298 e93437c95f feat: implement Model Context Protocol (MCP) server and download manager capabilities 2026-09-22 16:16:01 +05:30
Levi 88d150b933 test(bt): isolate cleanup fixtures from part-file promotion (#1531) 2026-09-21 22:30:54 +08:00
Levi 7bc7efed5b feat(task): show a dash for completed tasks without a done time
Tasks completed before this change have no doneAt value; the drawer now
always renders the Completed at row for completed tasks and falls back
to a dash instead of hiding the row.
2026-09-20 09:28:46 +08:00
Levi 1d8a502ac4 fix(task): keep DoneAt once written instead of clearing it
Done tasks never leave the done state (doStart, pause and markTaskError
all guard against Done), so the else branch clearing DoneAt was
unreachable dead code. DoneAt is now written once on completion and
kept for the task lifetime.
2026-09-20 09:21:36 +08:00
Levi 4b231f52c5 feat(task): record and display task completion time
Add DoneAt to the download task model: set when a task finishes and
cleared when it is downloaded again. The value flows through the API
model and task record, and is shown as "Completed at" in the task
drawer info tab next to the creation time.
2026-09-20 08:56:54 +08:00
Levi e956bd1ce5 style(ui): tighten card update chip spacing to 4px 2026-09-20 08:05:00 +08:00
Levi 68402c85d3 revert(ui): keep extension update check silent on failure
A failed check simply means no update chip; the failure toast and its
state field are removed.
2026-09-20 08:02:22 +08:00
Levi a480c5bd14 fix(ui): align extension update dialog title with app upgrade dialog 2026-09-19 22:16:37 +08:00
Levi e779de285d fix(ui): card chip spacing, instant flag clearing, locale-wide label
- Space the update chip off the card title to match the detail header
- Clear the extension's update flag right after a successful upgrade so
  the chip disappears without waiting for the background re-check
- Use the concise Update label across all locales
2026-09-19 21:56:46 +08:00
Levi fac17b0ccc style(ui): shorten update button label to Update
The shared newVersionUpdate label now reads Update on the app upgrade
dialog, the tracker update button and the extension update dialog, so
the extension flow matches the host dialog wording.
2026-09-19 21:47:44 +08:00
Levi 8c728ffda7 style(ui): render the update status as a bordered chip
Pill shape with a brand-green tint, outline and dot so the status
reads as a badge next to the title instead of plain text.
2026-09-19 21:41:56 +08:00
Levi 1b4956b0ec fix(ui): reserve the pointer cursor for the update chip
Extension cards no longer show a pointer cursor on hover; the tappable
update status chip shows it instead, on cards and detail headers alike.
2026-09-19 21:18:57 +08:00
Levi 2e5f8948be fix(ui): restore homepage and GitHub buttons on extension cards 2026-09-19 20:55:22 +08:00
Levi 04ebca60cf refactor(ui): make the status chip the only update entry
Remove the update button from the detail actions; the green chip next
to the extension title opens the update dialog on cards, the desktop
drawer and the mobile details page alike.
2026-09-19 20:49:20 +08:00
Levi 948f6d5487 fix(ui): let the update status chip shrink instead of overflow
Long translations (e.g. Italian, Tamil) could exceed the card title row
on minimum-width cards. The chip is now flex-sized and its label
ellipsizes, so the title and chip degrade gracefully at any width and
locale.
2026-09-19 20:02:28 +08:00
Levi 51b7abed48 refactor(ui): replace update dot with tappable status chip
Card and detail headers now show a green update chip next to the
extension title; on cards it opens the update dialog directly. The red
dot and the red card button are gone, and the status chip no longer
duplicates inside the detail content. Shorten the English label so the
chip fits narrow cards.
2026-09-19 19:47:02 +08:00
Levi d8758267c4 refactor(ui): polish extension update affordance per review
- Move the update dot from the icon corner to beside the card title
- Restyle the card update button to match the row's ghost buttons
- Drop the detail-view dot; the existing update status chip covers it
- Rename the detail update button label to just Update
2026-09-19 19:04:58 +08:00
Levi d94b59b165 refactor(ui): open extension details from every card
Installed extensions missing from the store index previously had no tap
handler on their card, so they kept on-card homepage/GitHub buttons as
the only metadata entry. Make all cards open the detail view and drop
the homepage/GitHub buttons entirely; details remain the single place
for links.
2026-09-19 18:43:57 +08:00
Levi f34cb9c971 refactor(ui): declutter extension card actions
Drop homepage/GitHub buttons from store extension cards (they remain on
the detail page); keep them only for unlisted extensions whose cards do
not open a detail view. With the labeled update button this caps card
actions at update/settings/delete.
2026-09-19 18:07:28 +08:00
Levi 172f4a3650 fix(ui): unify extension update check and improve update affordance
- Update availability is now decided solely by the backend live GitHub
  check (updateFlags); the store index may lag and previously masked
  new versions in the market view.
- Replace the ambiguous refresh icon on extension cards with a labeled
  update button, and add a red dot badge on extension icons (card and
  detail view) when an update is available.
- Surface a toast when the update check fails instead of failing silently.
2026-09-19 17:32:44 +08:00
Levi 694b43e3f9 docs: update beta download link to v2.0.0-beta.3 2026-09-19 08:16:57 +08:00
Levi 085ebf357a fix(ui): smooth task progress bar updates 2026-09-19 00:14:15 +08:00
Levi fe25ccab1d release: v2.0.0-beta.3 2026-09-18 23:34:09 +08:00
Levi 8301b1f9ce feat(http): accelerate connection expansion with squared batches (#1530) 2026-09-18 23:22:57 +08:00
Levi 49aab4ea7c feat(extension): add host environment and WebView page events (#1527) 2026-09-18 23:21:53 +08:00
Levi 5d8267fa56 fix(ui): fix various UI issues (#1518) 2026-09-18 22:40:36 +08:00
Levi 1ab452ced8 fix(bt): honor selected files when creating resolved tasks (#1523) 2026-09-18 22:32:09 +08:00
Levi 743f356567 feat(flutter): add action buttons to desktop download notifications (#1524) 2026-09-18 22:18:47 +08:00
Levi b1dd46cc7c feat: spool media inputs to disk and report producer progress (#1520) 2026-09-18 22:17:22 +08:00
sysy 38750d8505 fix: sanitize archive entry paths before extraction to prevent path traversal (#1526) 2026-09-18 22:12:43 +08:00
Levi dae13dec01 Merge latest PR #1527 into 2.0.0-beta.2 2026-09-18 09:43:51 +08:00
Levi a8f682559f Merge latest PR #1523 into 2.0.0-beta.2 2026-09-18 09:43:51 +08:00
Levi 3c0c019bc9 fix(bt): derive cleanup boundaries from torrent storage layout 2026-09-18 09:33:42 +08:00
Levi 69b3e223ad fix(bt): always preserve the torrent root directory 2026-09-18 09:23:46 +08:00
Levi 183616fcc0 fix(webview): suppress GTK fallback error document events 2026-09-18 09:13:40 +08:00
Levi 3415364ab7 test(webview): allow for throttled timers and diagnose selector timeouts 2026-09-18 09:04:39 +08:00
Levi 1b00a16277 fix(webview): update native callback lifetime and test coverage instrumentation 2026-09-18 09:00:50 +08:00
Levi b33c00d90d ci: resolve Linux native crash address in covered test binary 2026-09-18 00:36:41 +08:00
Levi 7e07ea31cc test(webview): cover event stream failures and cleanup 2026-09-18 00:31:06 +08:00
Levi c2da22f371 fix(webview): update GTK lifecycle cleanup and repeat Linux integration tests 2026-09-18 00:21:13 +08:00
Levi 2fd8a96c5d fix(webview): ignore load notifications from browser error documents 2026-09-18 00:07:55 +08:00
Levi 1776b05782 test(webview): trace navigation completion in event contract 2026-09-18 00:03:03 +08:00
Levi ffbae58e6b test(webview): trigger navigation failure consistently through proxies 2026-09-17 23:58:30 +08:00
Levi 2c7a86b21c build(deps): use webview_go with Go 1.17 minimum 2026-09-17 23:52:58 +08:00
Levi 4157e0a5bc fix(deps): update webview_go with legacy Go compatibility fix 2026-09-17 23:51:26 +08:00
Levi dfe8e9a3fd feat(extension): expose host version and WebView page events 2026-09-17 23:43:39 +08:00
Levi 7c33d978c4 fix(bt): clean partial files and empty parent directories 2026-09-17 23:35:40 +08:00
Levi 534fbbb3a2 Merge latest PR #1524 into 2.0.0-beta.2 2026-09-17 23:03:45 +08:00
Levi fdb2768008 Merge latest PR #1523 into 2.0.0-beta.2 2026-09-17 23:03:45 +08:00
Levi 27202cf99e Merge latest PR #1518 into 2.0.0-beta.2 2026-09-17 23:03:45 +08:00
Levi a05eea3268 fix(ui): restore tray menu dismissal and use upstream plugin 2026-09-17 23:02:20 +08:00
Levi 7496331ffd fix(flutter): align notification file actions with task cards 2026-09-17 23:02:14 +08:00
Levi 967ef25408 fix(bt): remove unselected files from the correct download path 2026-09-17 22:47:07 +08:00
Levi d1b2ae42af fix: finish migration to main WebView profile library 2026-09-17 21:48:41 +08:00
Levi adeb7a2895 Merge PR #1526 into 2.0.0-beta.2 2026-09-17 21:47:44 +08:00
Levi edfecefc16 Merge PR #1524 into 2.0.0-beta.2 2026-09-17 21:47:44 +08:00
Levi f7dfc4226b Merge PR #1523 into 2.0.0-beta.2 2026-09-17 21:47:44 +08:00
Levi edb8d3cbdc Merge PR #1520 into 2.0.0-beta.2 2026-09-17 21:47:43 +08:00
Levi c6dd58848d Merge latest PR #1518 into 2.0.0-beta.2 2026-09-17 21:47:43 +08:00
Levi 88dd35cc46 Merge latest main into 2.0.0-beta.2
# Conflicts:
#	internal/webview/README.md
#	ui/flutter/lib/app/rpc/webview_rpc_service.dart
#	ui/flutter/pubspec.lock
#	ui/flutter/pubspec.yaml
2026-09-17 21:47:43 +08:00
Levi 051d4267a5 fix(ui): restore path placeholders and window shortcut 2026-09-17 21:08:30 +08:00
sysy 022fbf633b fix: sanitize archive entry paths before extraction to prevent path traversal
Move path sanitization before directory handling in extractFile so
directory entries with "../" prefixes are rejected the same way file
entries already are. Add the same sanitization to extractSevenZipMultiPart
and both zip extraction loops in the Windows updater, which had no path
validation at all.
2026-09-16 23:31:35 -04:00
Levi 014221a1a7 fix(flutter): drop action buttons from error notifications
A failed download has no usable output file, so error notifications now
carry no buttons at all; only the done event offers open file / open
folder actions.
2026-09-17 09:55:07 +08:00
Levi 6f4635e2eb feat(flutter): add action buttons to desktop download notifications
Terminal task notifications now carry actionable buttons and respond to
clicks instead of being inert:

- Done notifications offer Open File and Open Folder for single-file
  tasks, and Open Folder only for folder tasks (e.g. multi-file
  torrents), following the same rule as the task details page. Error
  notifications offer Open Folder only, since the output is rarely
  usable.
- Clicking the notification body brings the app window back to the
  front (show + focus), covering the close-to-tray case.
- The click target is snapshotted into the notification (Windows: action
  arguments; macOS/Linux: notification payload) so buttons keep working
  after the task record is deleted, and a missing file is logged instead
  of surfacing an OS error dialog.
- macOS resolves buttons through three pre-registered notification
  categories (done single-file / done folder / error); Windows uses the
  actions API of the pinned notification plugin fork; Linux uses
  freedesktop notification actions.
- New openFolder label localized in all 22 ARB catalogs.

Cold-start activation is intentionally out of scope: when the process is
not running, OS activation opens the main window without dispatching the
button action.
2026-09-17 09:49:20 +08:00
UltraGeek0102 7c1447f659 fix(ios): recover continued processing after toggle (#1519) 2026-09-17 08:32:21 +08:00
Levi a83e3a2b84 Merge remote-tracking branch 'origin/main' into codex/fix-bt-file-selection
# Conflicts:
#	ui/flutter/test/features/tasks/domain/task_record_test.dart
2026-09-17 00:03:36 +08:00
Levi 5bb040eaaf fix(bt): honor selected files when creating resolved tasks 2026-09-17 00:03:03 +08:00
Levi e27c7dddb4 fix(http): close prefetch response when temporary-file setup fails 2026-09-16 23:43:43 +08:00
Levi 5342be16d9 Merge main into codex/tempdir-media-spooling 2026-09-16 23:20:29 +08:00
Levi 21702d5d46 feat(webview): add proxy support and per-extension profile lifecycle (#1517) 2026-09-16 23:17:43 +08:00
Levi e0ced260e2 feat: spool media inputs to disk and report producer progress 2026-09-16 23:16:19 +08:00
Levi e99ef6c7f4 fix(flutter): keep waiting task progress static (#1516) 2026-09-16 00:39:18 +08:00
Levi c667f097dc feat: add FFmpeg WASM runtime (#1512) 2026-09-16 00:38:22 +08:00
sun 46c34f2e92 feat(hls): add native HLS (m3u8) download protocol (#1515) 2026-09-15 23:40:31 +08:00
Levi d41e27234d Merge PR #1518 into 2.0.0-beta.2 2026-09-15 23:06:45 +08:00
Levi c703c52cbc fix(ui): refine narrow layouts and enable tab swipes 2026-09-15 23:01:48 +08:00
Levi b88144c864 Merge PR #1517 into 2.0.0-beta.2
# Conflicts:
#	go.mod
2026-09-15 22:51:56 +08:00
Levi 273850ae3f Merge PR #1516 into 2.0.0-beta.2 2026-09-15 22:51:41 +08:00
Levi 07356033f9 Merge PR #1515 into 2.0.0-beta.2 2026-09-15 22:51:41 +08:00
Levi 166aee81be Merge latest PR #1512 into 2.0.0-beta.2 2026-09-15 22:51:40 +08:00
Levi 45d7b6f5bc Merge latest main into 2.0.0-beta.2 2026-09-15 22:51:31 +08:00
Levi 5845e9ea5f Merge branch 'main' into main 2026-09-15 22:49:37 +08:00
UltraGeek0102 e47a20a1bd feat(ios): add iOS 26+ continued processing and Live Activities (#1514) 2026-09-15 22:49:11 +08:00
sun f9410bbcc7 fix(hls): snapshot state in Store and publish fully initialized runs
Address the completion review findings on top of the upstream review
fixes:

- Store runs on the engine's checkpoint goroutine without the task
  lock and serializes the returned value afterwards: take the fetcher
  mutex and return a deep copy instead of the live state pointer,
  matching the HTTP protocol's Store pattern
- Load the journal and per-run base counters before publishing the run
  as active, so Stats/Progress never observe half-initialized fields
- Bind the journal fingerprint to the key IV as well: a rotated IV on
  an otherwise identical plan would silently decrypt to garbage
- Keep every EXT-X-MEDIA member per group instead of only the first
  one, so a group with a URI on a later member is still detected
- After a merge that raced with a pause, keep the staged segments and
  exit silently instead of cleaning them up; drop a dead context
  check and an unused helper, and make the MAP identity and merge
  part-file names unambiguous
2026-09-15 16:31:35 +08:00
sun f1f17948c6 fix(hls): enforce supported HLS subset with explicit errors
The merge pipeline concatenates stored bytes, so features it cannot
reproduce faithfully are now detected and rejected with a clear error
instead of silently succeeding with a broken output:

- EXT-X-GAP (a hole in the merged file) and EXT-X-DISCONTINUITY
  (a broken timeline)
- separate audio renditions referenced by the selected variant: the
  master EXT-X-MEDIA groups are parsed and an AUDIO group with its own
  URI fails the resolve; unreferenced groups are ignored
- changing init sections: a repeated EXT-X-MAP is compared by all of
  its attributes (URI and BYTERANGE), not just the URI
- encrypted EXT-X-MAP without an explicit IV: the spec does not define
  a sequence-derived IV for init sections
- non-identity AES-128 KEYFORMAT values

The supported matrix is documented in the package doc.

Add regression tests for every rejection plus real media validation:
ffmpeg-generated TS segments and fMP4 init/fragments are committed as
fixtures, downloaded through the fetcher (including an AES-128 key
rotation variant), and the merged outputs are validated structurally -
TS on the 188 byte grid with continuity counter rules, PAT/PMT and PES
presence, fMP4 by walking the ISO-BMFF box hierarchy.
2026-09-15 11:42:06 +08:00
sun 675028c64a fix(hls): isolate staging per task and finalize output atomically
- Give every task a random staging identity persisted in the saved
  state: two tasks downloading the same playlist into the same
  directory get separate segment folders and journals instead of
  deleting each other's progress
- Bind the journal to a fingerprint of the full download plan (media
  URL, ordered segments, ranges, keys); foreign or unreadable journals
  are discarded rather than applied to the wrong segments, and the
  ledger is rewritten through a temp file + rename
- Legacy states without a staging identity migrate to a fresh folder
  instead of reusing the shared URL-hash folder
- Merge into a unique temporary file next to the target, check write
  and close errors, then remove-then-rename over the final name: a
  failed merge preserves the staged segments for a retry and re-merging
  after a crash stays idempotent
- Drop the staging folder when a URL patch invalidates the plan
2026-09-15 11:27:26 +08:00
sun b6d18a267b fix(hls): rebuild request extras on restore and adopt merge semantics in Patch
- Parse the request extra in Setup so restored fetchers resend Referer,
  Cookie and other configured headers on resumed segment/key requests;
  the store/restore regression test round-trips the state and the task
  meta through JSON exactly like the engine storage does
- Rebuild the resolved plan on Start when the resource survived but the
  plan was lost to a crash between the task save and the first
  checkpoint, keeping the existing output name
- Patch now merges partial updates instead of replacing the request and
  options wholesale: URL-only patches keep the output path and renamed
  file, header-only patches keep the resolved plan and take effect on
  the next run, Labels merge per key, Proxy swaps rebuild the client,
  and nil arguments are no longer a panic. Editing a running task is
  rejected, consistent with the ed2k protocol
- Cover retry-after-failure counter resets, running-task patch
  rejection and the missing-state self-heal with regression tests
2026-09-15 11:18:40 +08:00
sun 40cfb167f6 fix(hls): isolate per-run lifecycle and use final-output byte accounting
- Give every Start its own run context, WaitGroup (including the queue
  producer), supervisor done channel and snapshots of the plan, request
  headers, HTTP client and merge target; Pause/Close wait for all run
  goroutines, including a merge in flight, before returning
- Build the HTTP client in Resolve/Start before workers launch instead of
  lazily inside do(), which raced between concurrent segment downloads
- Reject task edits while a run is active instead of mutating shared state
- Reset counters per run and seed them from the completed-segment journal;
  count only stored output bytes as progress, never discarded retry bytes
- Report Res.Size only when every segment is unencrypted with a known
  length; ciphertext sums overstate the final output, indeterminate plans
  rely on the engine's TotalDownloaded backfill at completion
2026-09-15 11:13:38 +08:00
Levi 4fd1939732 fix(ffmpeg): reserve capacity before creating media inputs 2026-09-15 10:08:29 +08:00
Levi 3ed54ba93a fix(macos): resign WebView input focus before disposal 2026-09-15 09:15:27 +08:00
Levi fe64a328ac fix(webview): synchronize Windows browser shutdown before reopening profiles 2026-09-15 09:06:56 +08:00
Levi 7e49b678f2 fix(webview): release retained macOS views before profile removal 2026-09-15 09:01:58 +08:00
Levi 311e002e55 Return persistent cookie expiration through the existing RPC contract 2026-09-15 00:56:50 +08:00
Levi 119fad9a94 Use explicit WebView2 shutdown and verify persistent cookie expiration 2026-09-15 00:54:12 +08:00
Levi e2601ea9fa Keep Linux WebViews on one persistent GTK UI thread 2026-09-15 00:52:05 +08:00
Levi a5eaa6b528 Wait for cookie propagation and identify profile assertion stages 2026-09-15 00:51:07 +08:00
Levi bed624bfce Assert HTTPS proxy routing independently of native error-page behavior 2026-09-15 00:44:20 +08:00
Levi eec53c15b2 Extend existing provider contracts and remove duplicate WebView CI 2026-09-15 00:41:47 +08:00
Levi d58135db9f Wait for Android native views to close before clearing profiles 2026-09-14 10:13:08 +08:00
Levi d642ae5018 Remove extension WebView data during uninstall 2026-09-14 10:11:53 +08:00
Levi e358123b22 fix(flutter): keep waiting task progress static 2026-09-14 10:02:19 +08:00
sun ee9b16794e Merge branch 'GopeedLab:main' into main 2026-09-14 09:55:31 +08:00
sun 4f74101b3a feat(hls): add native HLS (m3u8) download protocol
New hls protocol in internal/protocol/hls (registered ahead of http):
- master playlist variant picking (highest bandwidth), RFC3986 segment
  URL resolution, fMP4 init segments, EXT-X-BYTERANGE, EXT-X-GAP skip
- AES-128 decryption with mid-playlist key rotation and sequence-derived
  IVs; explicit errors for live streams, SAMPLE-AES and multi-period fMP4
- concurrent segment downloads with retry, journal-based resume across
  pause/app restarts, automatic in-order merge into a single .ts/.mp4
- best-effort Content-Length probing for determinate progress
- anti-hotlink Referer/Cookie/UA via task extra headers (Go transport
  does not filter them)

Core: fix SchemeFilter extension match for URLs with query strings
(also benefits .torrent links); Flutter: Protocol.hls enum, hls protocol
config model (prevents config round-trip wipe) and HLS settings block
with localized labels in all 22 locales.

Tests: parser/crypto unit tests, httptest e2e (Referer-enforced CDN,
encrypted streams, byterange, pause/resume, store/restore) and an
opt-in live-site acceptance gate (HLS_LIVE_URL).
2026-09-14 09:55:31 +08:00
Levi c47b735fc4 Preserve existing profile connections when opening more pages 2026-09-14 00:43:50 +08:00
Levi ca98806dcc Pin verified native backend and complete proxy regression checks 2026-09-14 00:34:27 +08:00
Levi a0e84a75b1 Isolate extension WebView profiles and follow host proxy settings 2026-09-14 00:30:52 +08:00
Levi cee5f3336e fix: extension install by deeplink (#1510) 2026-09-13 23:40:59 +08:00
Levi a760290213 fix(ui): enable task detail text selection and compact extension stats (#1511) 2026-09-13 23:30:45 +08:00
Levi 14b22cf485 test: align failed task speed assertion with PR #1510 behavior 2026-09-13 22:27:32 +08:00
Levi a3f6ec5885 Merge latest PR #1512 into 2.0.0-beta.2 2026-09-13 22:25:20 +08:00
Levi f839c3ac93 Merge latest PR #1511 into 2.0.0-beta.2 2026-09-13 22:25:20 +08:00
Levi 67d4aa9554 Merge latest PR #1510 into 2.0.0-beta.2
# Conflicts:
#	ui/flutter/lib/features/extensions/presentation/pages/extensions_page.dart
#	ui/flutter/test/widget_test.dart
2026-09-13 22:25:20 +08:00
Levi c355f9ff5c fix: retain zero download speed for known-size tasks 2026-09-13 18:56:37 +08:00
Levi 59a7d40bfe fix: select downloading tab after mobile task creation 2026-09-13 18:46:19 +08:00
Levi 5d83e91ce5 fix(stream): bound FFmpeg memory buffers and propagate Blob backpressure 2026-09-13 18:24:32 +08:00
Levi 637004c8a8 fix(ffmpeg): use app-writable temporary directory for stream buffers 2026-09-13 18:00:10 +08:00
Levi e79b37d08c fix: remove mobile reveal-to-open-file fallback 2026-09-13 17:39:54 +08:00
Levi d393d27ec7 fix: hide completed task folder action on mobile and web 2026-09-13 17:38:34 +08:00
Levi 2e8672273c feat: confirm extension updates with progress and retry 2026-09-13 17:25:31 +08:00
Levi 4802802828 fix: limit URL install loading to its submit button 2026-09-13 17:20:08 +08:00
Levi 3d372f98de fix: preserve install URL paste focus and limit local installs to desktop 2026-09-13 17:18:13 +08:00
Levi d578fe96bc test(ui): cover minimum extension card width 2026-09-13 15:39:04 +08:00
Levi 1262844363 fix(ui): keep three extension columns at default width 2026-09-13 15:36:36 +08:00
Levi 91bea83692 Merge PR #1512 into 2.0.0-beta.2 2026-09-13 15:30:10 +08:00
Levi e21bc5eeae Merge PR #1511 into 2.0.0-beta.2 2026-09-13 15:30:10 +08:00
Levi 6331060171 Merge PR #1510 into 2.0.0-beta.2 2026-09-13 15:30:10 +08:00
Levi f3ec6b8ee9 Merge latest main into 2.0.0-beta.2 2026-09-13 15:30:00 +08:00
Levi 9124976b5c feat(extension): add streaming FFmpeg WASM merge runtime 2026-09-13 15:23:29 +08:00
Levi 9820e58de9 Merge branch 'main' of github.com:GopeedLab/gopeed into codex/fix-task-detail-text-selection 2026-09-13 15:07:02 +08:00
Levi e890e593d9 feat(ui): compact number display in extension stats 2026-09-13 15:06:42 +08:00
Levi 6572e9e899 fix: automatically submit scheme extension installs 2026-09-13 15:01:50 +08:00
Levi 1c57b07575 fix: prefill scheme extension installs and forward iOS deep links 2026-09-13 14:31:36 +08:00
Levi a29f4bb42a fix(ui): use typed controls for extension settings (#1509) 2026-09-13 14:29:46 +08:00
Levi 62e8ab3cef fix(ui): show downloaded bytes for tasks with unknown size (#1508) 2026-09-13 11:57:26 +08:00
Levi 8fa9a1e902 fix(ui): preserve mobile home route and confirm Android exit (#1507) 2026-09-12 21:44:27 +08:00
Levi 6f12eb1f07 Merge Android cold-start back fix and bottom exit prompt into beta 2026-09-12 20:40:25 +08:00
Levi bd03d3b31c fix(android): intercept back during cold startup and move exit toast down 2026-09-12 20:40:07 +08:00
Levi 46675f26af fix(ui): align history search placeholder and prevent clipping (#1506) 2026-09-12 20:04:28 +08:00
Levi d330d8bea8 Merge mobile back navigation and exit confirmation into beta 2026-09-12 19:03:24 +08:00
Levi ee6b073bc1 fix(ui): preserve mobile home route and confirm Android exit 2026-09-12 18:56:52 +08:00
Levi 0d52b840a2 feat(ui): emphasize task creation and add empty-list actions (#1505) 2026-09-12 18:09:36 +08:00
Levi a41c6c145c Merge latest PR #1505: compact creation button and inline empty-list guidance 2026-09-12 17:49:31 +08:00
Levi 86c8383584 feat(ui): simplify empty-list guidance with inline create action 2026-09-12 17:48:14 +08:00
Levi 75462fe2be fix(ui): make mobile task creation button compact 2026-09-12 17:28:42 +08:00
Levi f35111e072 fix(ui): make mobile task creation button compact 2026-09-12 17:27:44 +08:00
Levi 6aabe69f15 test(ui): align window title expectations with main 2026-09-12 16:56:06 +08:00
Levi 527539f3a1 Merge PR #1506: fix history search text alignment and clipping 2026-09-12 16:55:20 +08:00
Levi 2d239495a0 Merge PR #1505: emphasize task creation and empty-list actions 2026-09-12 16:55:20 +08:00
Levi ddce649c8b Merge latest main into 2.0.0-beta.2 2026-09-12 16:55:20 +08:00
Levi 29675858f2 fix(ui): align history search placeholder and prevent clipping 2026-09-12 10:03:08 +08:00
Levi 1fc0ff6738 feat: emphasize task creation and add empty-list actions 2026-09-12 09:57:02 +08:00
Levi 091e7b8936 fix(ui): keep mobile create task actions in safe area (#1504) 2026-09-12 09:28:04 +08:00
Levi ff8f61423e fix(ui): refine app update dialog layout and scrolling (#1503) 2026-09-12 07:25:37 +08:00
Levi e5ac13cfdc docs: update beta download links 2026-09-11 08:36:01 +08:00
Levi 073e1413ab release: 2.0.0-beta.2 2026-09-11 08:20:45 +08:00
Wu Shuwen 04b9aa18ec fix(http): infer extension for opaque URLs (#1495) 2026-09-11 08:01:32 +08:00
Levi 08fe4eef4e fix(ui): refine create task actions and relocate icon sources (#1500) 2026-09-11 08:00:03 +08:00
Levi 8f9bac9e6b fix(ui): reuse shared file path picker for script settings (#1496) 2026-09-10 23:51:16 +08:00
Levi 0851e27c6d Merge PR #1496: fix script settings path picker 2026-09-10 23:10:58 +08:00
Levi 3abc8d42fa Merge PR #1498: fix macOS download notifications 2026-09-10 23:10:53 +08:00
Levi 71d7d96cc0 Merge PR #1499: fix Windows notification icon
# Conflicts:
#	ui/flutter/test/features/tasks/presentation/create_task_window_page_test.dart
2026-09-10 23:10:49 +08:00
Levi f254a82d9c refactor(windows): use bundled notification icon path directly 2026-09-10 23:07:14 +08:00
Levi c4ae3330b8 fix(windows): persist PNG icon for download notifications 2026-09-10 23:00:24 +08:00
Levi 979988a7f4 fix(macos): request permission for download notifications 2026-09-10 13:03:51 +08:00
Levi 9efe7ff4c9 Merge PR #1488 latest head: create task panel mobile UI 2026-09-10 00:26:07 +08:00
Levi 77734c69bc Merge PR #1494 latest head: unify platform window titles 2026-09-10 00:26:03 +08:00
Levi 22db46c68a Merge PR #1495 latest head: infer extension for opaque URLs 2026-09-10 00:26:00 +08:00
Levi 99a62357ab Merge PR #1496 latest head: reuse shared file path picker 2026-09-10 00:25:56 +08:00
Levi 9558042513 Merge PR #1497 latest head: fix(android) restore notification permission prompt 2026-09-10 00:25:50 +08:00
Levi 39912718c4 fix(android): request notification permission on startup 2026-09-10 00:19:53 +08:00
Levi 3b6b134f24 ci: warn on incomplete localization catalogs 2026-09-10 00:02:31 +08:00
Levi f506b8f92f Merge branch 'main' into fix/create-task-panel-mobile-ui 2026-09-09 23:30:41 +08:00
Levi 9b9fd12884 Merge branch 'main' into fix/create-task-panel-mobile-ui 2026-09-09 23:27:11 +08:00
dajiaohuang cea912b414 fix(http): preserve MIME extension for opaque URLs 2026-09-09 03:45:15 +08:00
Levi 7c3d7269fe fix: unify platform window titles as Gopeed 2026-09-08 23:15:55 +08:00
Levi 1554791e17 merge: PR #1493 refresh repeated Android download intents
# Conflicts:
#	ui/flutter/test/features/tasks/presentation/create_task_window_page_test.dart
2026-09-08 22:14:48 +08:00
Levi b0aecdcfc6 merge: latest PR #1488 mobile resolve action layout 2026-09-08 22:14:14 +08:00
Levi d2eb7b2a67 fix(android): refresh download URL when create page is already open 2026-09-08 22:05:45 +08:00
Levi 7112845c82 fix: keep resolve dialog actions equal in width 2026-09-08 21:53:25 +08:00
Levi 39820707be merge: PR #1488 extension toolbar alignment fix 2026-09-08 21:28:33 +08:00
Levi 47db539d91 fix: align extension search with compact toolbar controls 2026-09-08 21:27:52 +08:00
Levi f32f7583e4 fix: align extension search with compact toolbar controls 2026-09-08 21:25:05 +08:00
Levi e3ae2865d6 fix: reconcile Azerbaijani locale with merged create action 2026-09-08 20:26:38 +08:00
Levi 1a8d73fbe6 merge: PR #1489 Android browser takeover handling 2026-09-08 20:24:00 +08:00
Levi 5728caf7db merge: PR #1488 create task mobile UI 2026-09-08 20:23:52 +08:00
Levi 77d95971ca merge: PR #1476 async native API callbacks 2026-09-08 20:23:52 +08:00
Levi 2c81520029 merge: minimal macOS window reopen fix 2026-09-08 20:23:52 +08:00
Levi 8d45eb4fef test: remove native window regression tests 2026-09-08 20:03:57 +08:00
Levi 31fa0e5bc9 fix: restore only main macOS window and select active downloads 2026-09-08 18:14:16 +08:00
Levi 0f60f40e3e Merge branch 'main' into codex/async-ffi-worker-callbacks 2026-09-07 09:23:59 +08:00
Levi b17c0fd7c8 fix(android): preserve browser takeover handling 2026-09-07 09:11:17 +08:00
Levi a834ca0d74 fix(ui): unify text field appearance 2026-09-06 22:26:35 +08:00
Levi ecfd07ccb0 fix(ui): refine create task panel behavior 2026-09-06 22:17:26 +08:00
Levi d51eeb3f2c fix: prevent native requests from blocking shutdown 2026-09-06 21:49:11 +08:00
Levi 1bb3f74c9f Merge branch 'main' into codex/async-ffi-worker-callbacks 2026-09-06 20:37:14 +08:00
Levi f7be97667f Merge remote-tracking branch 'origin/main' into codex/async-ffi-worker-callbacks 2026-09-06 13:00:59 +08:00
Levi 607d1729e6 refactor: share async invoke bridge across native platforms 2026-09-06 11:59:11 +08:00
Levi d12122ea86 feat: make desktop FFI invocation asynchronous 2026-09-06 11:23:27 +08:00
324 changed files with 22683 additions and 1630 deletions

No files matched your search

+2
View File
@@ -0,0 +1,2 @@
# Keep upstream plugin sources recognizable and collapsed in GitHub reviews.
ui/flutter/third_party/** linguist-vendored whitespace=-blank-at-eol,-blank-at-eof
+19 -1
View File
@@ -77,6 +77,8 @@ jobs:
include:
- name: Windows
os: windows-latest
- name: Linux
os: ubuntu-latest
- name: macOS
os: macos-latest
runs-on: ${{ matrix.os }}
@@ -89,9 +91,25 @@ jobs:
if: runner.os == 'macOS'
with:
flutter-version: ${{ env.FLUTTER_VERSION }}
- name: Install Linux WebView dependencies
if: runner.os == 'Linux'
run: |
sudo apt-get update -y
sudo apt-get install -y libgtk-3-dev libwebkit2gtk-4.1-dev xvfb
- name: Test webview
if: runner.os == 'Linux'
# Match the full suite's coverage instrumentation and repeat on the same
# GTK event loop to catch queued callbacks after engine destruction.
run: |
PACKAGES=$(go list ./... | awk '!/\/bind\// && (!/\/cmd\// || /\/cmd\/web$/)')
go test -c -tags webview -coverpkg=$(echo "$PACKAGES" | paste -sd "," -) -covermode=count -o "$RUNNER_TEMP/goprovider.test" ./internal/webview/goprovider
xvfb-run -a "$RUNNER_TEMP/goprovider.test" -test.v -test.count=5 -test.timeout=300s -test.coverprofile="$RUNNER_TEMP/webview-coverage.out"
env:
CGO_ENABLED: 1
- name: Test webview
if: runner.os == 'Windows'
run: go test -tags webview -v -count=1 -timeout 300s ./internal/webview/goprovider/
# Repeat close/reopen coverage to catch browser shutdown races.
run: go test -tags webview -v -count=3 -timeout 300s ./internal/webview/goprovider/
env:
CGO_ENABLED: 1
- name: Test webview
+1 -1
View File
@@ -63,7 +63,7 @@ Connect Gopeed to an AI agent and manage downloads with natural language. For ex
Gopeed 2.0.0 is currently in public beta, introducing a redesigned interface, a native communication architecture that connects desktop and mobile clients directly to the Go core through FFI, a more consistent cross-platform experience, improved task management, more flexible API support, and MCP-based AI agent integration. Some features may still be incomplete or unstable, so please try it and report any issues you encounter.
- [Download Gopeed 2.0.0 Beta 1](https://github.com/GopeedLab/gopeed/releases/tag/v2.0.0-beta.1)
- [Download Gopeed 2.0.0 Beta 3](https://github.com/GopeedLab/gopeed/releases/tag/v2.0.0-beta.3)
Once the features and stability meet our release standards, we will publish the official Gopeed 2.0.0 release. Beta users will be able to upgrade directly to the final release, while existing stable users will not be automatically moved onto the beta channel.
+1 -1
View File
@@ -63,7 +63,7 @@ Gopeed を AI Agent と連携すると、自然言語でダウンロードを管
Gopeed 2.0.0 は現在パブリックベータ版です。刷新されたインターフェース、デスクトップおよびモバイルから FFI で Go コアへ直接接続するネイティブ通信アーキテクチャ、より統一されたクロスプラットフォーム体験、強化されたタスク管理、より柔軟な API、MCP による AI Agent 連携機能を導入しています。一部の機能はまだ不完全または不安定な場合がありますので、ぜひお試しいただき、問題があればフィードバックをお寄せください。
- [Gopeed 2.0.0 Beta 1 をダウンロード](https://github.com/GopeedLab/gopeed/releases/tag/v2.0.0-beta.1)
- [Gopeed 2.0.0 Beta 3 をダウンロード](https://github.com/GopeedLab/gopeed/releases/tag/v2.0.0-beta.3)
機能と安定性が正式リリースの基準に達した時点で、Gopeed 2.0.0 正式版を公開します。ベータ版のユーザーは正式版へ直接アップデートできます。既存の安定版ユーザーが自動的にベータチャンネルへ移行することはありません。
+1 -1
View File
@@ -63,7 +63,7 @@ Sau khi kết nối Gopeed với AI Agent, bạn có thể quản lý tải xu
Gopeed 2.0.0 hiện đang trong giai đoạn beta công khai, với giao diện được thiết kế lại, kiến trúc giao tiếp native kết nối trực tiếp ứng dụng desktop và di động với lõi Go qua FFI, trải nghiệm đa nền tảng nhất quán hơn, khả năng quản lý tác vụ được cải thiện, API linh hoạt hơn và khả năng tích hợp AI Agent qua MCP. Một số tính năng có thể vẫn chưa hoàn thiện hoặc chưa ổn định, vì vậy hãy dùng thử và gửi phản hồi nếu bạn gặp vấn đề.
- [Tải Gopeed 2.0.0 Beta 1](https://github.com/GopeedLab/gopeed/releases/tag/v2.0.0-beta.1)
- [Tải Gopeed 2.0.0 Beta 3](https://github.com/GopeedLab/gopeed/releases/tag/v2.0.0-beta.3)
Khi các tính năng và độ ổn định đáp ứng tiêu chuẩn phát hành, chúng tôi sẽ phát hành Gopeed 2.0.0 chính thức. Người dùng bản beta có thể nâng cấp trực tiếp lên bản chính thức, trong khi người dùng bản ổn định hiện tại sẽ không tự động được chuyển sang kênh beta.
+1 -1
View File
@@ -63,7 +63,7 @@ Gopeed(全称 **Go Speed**,也叫“够快下载器”)是一款使用 Go
Gopeed 2.0.0 目前处于公开 Beta 测试阶段,引入了全新界面、通过 FFI 直连 Go 核心的原生通信架构、更一致的跨平台体验、更完善的任务管理、更灵活的 API,以及用于 AI Agent 接入的 MCP 能力。部分功能仍可能不够完善或稳定,欢迎提前体验并向我们反馈问题。
- [下载 Gopeed 2.0.0 Beta 1](https://github.com/GopeedLab/gopeed/releases/tag/v2.0.0-beta.1)
- [下载 Gopeed 2.0.0 Beta 3](https://github.com/GopeedLab/gopeed/releases/tag/v2.0.0-beta.3)
当功能完整度和稳定性达到正式发布标准后,我们会发布 Gopeed 2.0.0 正式版。已安装 Beta 版本的用户可以直接升级到最终正式版,现有稳定版用户则不会被自动切换到 Beta 渠道。
+1 -1
View File
@@ -63,7 +63,7 @@ Gopeed(**Go Speed** 的縮寫)是一款以 Go 與 Flutter 開發的高速、
Gopeed 2.0.0 目前處於公開 Beta 測試階段,引入了全新介面、透過 FFI 直連 Go 核心的原生通訊架構、更一致的跨平台體驗、更完善的任務管理、更靈活的 API,以及透過 MCP 串接 AI Agent 的能力。部分功能仍可能不夠完善或穩定,歡迎搶先體驗並向我們回報問題。
- [下載 Gopeed 2.0.0 Beta 1](https://github.com/GopeedLab/gopeed/releases/tag/v2.0.0-beta.1)
- [下載 Gopeed 2.0.0 Beta 3](https://github.com/GopeedLab/gopeed/releases/tag/v2.0.0-beta.3)
當功能完整度與穩定性達到正式發布標準後,我們會發布 Gopeed 2.0.0 正式版。已安裝 Beta 版本的使用者可以直接升級至最終正式版,現有穩定版使用者則不會被自動切換至 Beta 頻道。
Binary file not shown.

Before

Width:  |  Height:  |  Size: 391 KiB

After

Width:  |  Height:  |  Size: 1.3 MiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 357 KiB

After

Width:  |  Height:  |  Size: 1.3 MiB

+12
View File
@@ -7,6 +7,7 @@ import (
nativebridge "github.com/GopeedLab/gopeed/bind/native"
"github.com/GopeedLab/gopeed/internal/webview/rpcprovider"
goapi "github.com/GopeedLab/gopeed/pkg/api"
"github.com/GopeedLab/gopeed/pkg/rest"
"github.com/GopeedLab/gopeed/pkg/rest/model"
)
@@ -90,3 +91,14 @@ func applyWebViewProvider(config *model.StartConfig) {
}
config.WebViewProvider = rpcprovider.New(*config.WebViewRPCConfig)
}
func LiveActivityTaskEventMask() int64 {
return int64(
goapi.TaskEventDone |
goapi.TaskEventError |
goapi.TaskEventStart |
goapi.TaskEventProgress |
goapi.TaskEventPause |
goapi.TaskEventDelete,
)
}
+7 -3
View File
@@ -7,9 +7,11 @@ import (
)
type args struct {
url string
connections *int
dir *string
url string
connections *int
dir *string
checksum *string
checksumAlgo *string
}
func parse() *args {
@@ -20,6 +22,8 @@ func parse() *args {
var args args
args.connections = flag.Int("C", 16, "Concurrent connections.")
args.dir = flag.String("D", dir, "Store directory.")
args.checksum = flag.String("checksum", "", "Expected checksum hash (hex encoded).")
args.checksumAlgo = flag.String("checksum-algo", "md5", "Checksum algorithm: md5, sha1, or sha256.")
flag.Parse()
t := flag.Args()
if len(t) > 0 {
+17 -2
View File
@@ -19,6 +19,18 @@ func main() {
var wg sync.WaitGroup
wg.Add(1)
var checksumOpt *http.ChecksumOption
if args.checksum != nil && *args.checksum != "" {
algo := "md5"
if args.checksumAlgo != nil && *args.checksumAlgo != "" {
algo = *args.checksumAlgo
}
checksumOpt = &http.ChecksumOption{
Algorithm: algo,
Expected: *args.checksum,
}
}
_, err := download.Boot().
URL(args.url).
Listener(func(event *download.Event) {
@@ -43,8 +55,11 @@ func main() {
}
}).
Create(&base.Options{
Path: *args.dir,
Extra: http.OptsExtra{Connections: *args.connections},
Path: *args.dir,
Extra: http.OptsExtra{
Connections: *args.connections,
Checksum: checksumOpt,
},
})
if err != nil {
panic(err)
+10 -4
View File
@@ -37,15 +37,17 @@ func installByInstaller(killSignalChan chan<- any, packagePath, destDir string)
}
defer reader.Close()
// Find the installer file
var installerPath string
for _, file := range reader.File {
cleanName := filepath.Clean(file.Name)
if strings.HasPrefix(cleanName, "..") || filepath.IsAbs(cleanName) {
continue
}
if file.FileInfo().IsDir() {
continue
}
// Extract file
path := filepath.Join(tempDir, file.Name)
path := filepath.Join(tempDir, cleanName)
if err := os.MkdirAll(filepath.Dir(path), os.ModePerm); err != nil {
return err
@@ -101,7 +103,11 @@ func installByPortable(killSignalChan chan<- any, packagePath, destDir string) e
defer reader.Close()
for _, file := range reader.File {
path := filepath.Join(destDir, file.Name)
cleanName := filepath.Clean(file.Name)
if strings.HasPrefix(cleanName, "..") || filepath.IsAbs(cleanName) {
continue
}
path := filepath.Join(destDir, cleanName)
if file.FileInfo().IsDir() {
os.MkdirAll(path, file.Mode())
+7
View File
@@ -13,6 +13,7 @@ import (
)
type args struct {
TempDir *string `json:"tempDir"`
Address *string `json:"address"`
Port *int `json:"port"`
Username *string `json:"username"`
@@ -54,6 +55,7 @@ func loadCliArgs() *args {
cfg.ApiToken = flag.String("T", "", "API token, it must be configured when using HTTP API in the case of enabling web authentication")
flag.StringVar(cfg.ApiToken, "api-token", "", "API token, it must be configured when using HTTP API in the case of enabling web authentication")
flag.BoolVar(&cfg.MCPEnable, "mcp-enable", false, "Enable MCP endpoint")
cfg.TempDir = flag.String("temp-dir", "", "Temporary directory (defaults to the OS temporary directory)")
cfg.StorageDir = flag.String("d", "", "Storage directory")
flag.StringVar(cfg.StorageDir, "storage-dir", "", "Storage directory")
whiteDownloadDirs := flag.String("white-download-dirs", "", "White download directories, comma-separated")
@@ -89,6 +91,8 @@ func overrideWithCliArgs(cfg *args, cliConfig *args) {
cfg.ApiToken = cliConfig.ApiToken
case "mcp-enable":
cfg.MCPEnable = cliConfig.MCPEnable
case "temp-dir":
cfg.TempDir = cliConfig.TempDir
case "d", "storage-dir":
cfg.StorageDir = cliConfig.StorageDir
case "white-download-dirs":
@@ -101,6 +105,9 @@ func overrideWithCliArgs(cfg *args, cliConfig *args) {
// setDefaults sets default values for any unset configuration fields
func setDefaults(cfg *args, cliConfig *args) {
if cfg.TempDir == nil {
cfg.TempDir = cliConfig.TempDir
}
if cfg.Address == nil {
cfg.Address = cliConfig.Address
}
+5
View File
@@ -43,11 +43,16 @@ func main() {
storageDir = filepath.Join(filepath.Dir(exe), "storage")
}
tempDir := ""
if args.TempDir != nil {
tempDir = *args.TempDir
}
cfg := &model.StartConfig{
Network: "tcp",
Address: fmt.Sprintf("%s:%d", *args.Address, *args.Port),
Storage: model.StorageBolt,
StorageDir: storageDir,
TempDir: tempDir,
WhiteDownloadDirs: args.WhiteDownloadDirs,
ApiToken: *args.ApiToken,
MCPEnable: args.MCPEnable,
+5 -3
View File
@@ -3,13 +3,15 @@ module github.com/GopeedLab/gopeed
go 1.25.4
require (
github.com/GopeedLab/webview_go v0.0.0-20260423085439-7a2f88b6e9b5
codeberg.org/gruf/go-ffmpreg v0.6.23-0.20260827095416-df1836cb5be0
github.com/GopeedLab/webview_go v0.0.0-20260918010912-0d4c27bb7914
github.com/anacrolix/torrent v1.60.1-0.20251217073903-486bcbe758e0
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5
github.com/bodgit/sevenzip v1.6.1
github.com/dop251/goja v0.0.0-20260106131823-651366fbe6e3
github.com/dop251/goja_nodejs v0.0.0-20240728170619-29b559befffc
github.com/go-git/go-git/v5 v5.8.1
github.com/google/jsonschema-go v0.4.3
github.com/gorilla/handlers v1.5.1
github.com/gorilla/mux v1.8.0
github.com/imroc/req/v3 v3.52.2
@@ -21,6 +23,7 @@ require (
github.com/pkg/errors v0.9.1
github.com/rs/dnscache v0.0.0-20230804202142-fc85eb664529
github.com/rs/zerolog v1.31.0
github.com/tetratelabs/wazero v1.12.0
github.com/xiaoqidun/setft v0.0.0-20220310121541-be86327699ad
go.etcd.io/bbolt v1.4.3
golang.org/x/exp v0.0.0-20251219203646-944ab1f22d93
@@ -38,7 +41,6 @@ require (
github.com/felixge/fgprof v0.9.5 // indirect
github.com/go-task/slim-sprig/v3 v3.0.0 // indirect
github.com/google/go-cmp v0.7.0 // indirect
github.com/google/jsonschema-go v0.4.3 // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/hashicorp/go-multierror v1.1.1 // indirect
github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
@@ -156,7 +158,7 @@ require (
golang.org/x/crypto v0.48.0 // indirect
golang.org/x/mod v0.33.0 // indirect
golang.org/x/net v0.50.0
golang.org/x/sys v0.41.0 // indirect
golang.org/x/sys v0.44.0 // indirect
golang.org/x/text v0.34.0
golang.org/x/time v0.15.0 // indirect
golang.org/x/tools v0.42.0 // indirect
+20 -2
View File
@@ -14,6 +14,8 @@ cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2k
cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw=
cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=
codeberg.org/gruf/go-ffmpreg v0.6.23-0.20260827095416-df1836cb5be0 h1:AqNFl60o4xhTTOxmzAvSf7pazXHKBGuV5ATw7sETIRA=
codeberg.org/gruf/go-ffmpreg v0.6.23-0.20260827095416-df1836cb5be0/go.mod h1:XM6eW8CSgAuZRKRynm5nDiSNedm0rav30IychqygOHE=
crawshaw.io/iox v0.0.0-20181124134642-c51c3df30797/go.mod h1:sXBiorCo8c46JlQV3oXPKINnZ8mcqnye1EkVkqsectk=
crawshaw.io/sqlite v0.3.2/go.mod h1:igAO5JulrQ1DbdZdtVq48mnZUBAPOeFzer7VhDWNtW4=
dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk=
@@ -23,8 +25,20 @@ filippo.io/edwards25519 v1.0.0-rc.1 h1:m0VOOB23frXZvAOK44usCgLWvtsxIoMCTBGJZlpmG
filippo.io/edwards25519 v1.0.0-rc.1/go.mod h1:N1IkdkCkiLB6tki+MYJoSx2JTY9NUlxZE7eHn5EwJns=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
github.com/GopeedLab/webview_go v0.0.0-20260423085439-7a2f88b6e9b5 h1:86LVTaNDM/j/uTAD2HedkXNO9LIQlt642VAdbU/0D/M=
github.com/GopeedLab/webview_go v0.0.0-20260423085439-7a2f88b6e9b5/go.mod h1:FvV19lu9rQ9NcbczN5y8pbWecURRRbWa8tbfFARBDa0=
github.com/GopeedLab/webview_go v0.0.0-20260915010614-2567e492369a h1:Vzz+8aMVX9ZPPpkxgxa7mCorrAgdXCZL5EunOpOBvao=
github.com/GopeedLab/webview_go v0.0.0-20260915010614-2567e492369a/go.mod h1:FvV19lu9rQ9NcbczN5y8pbWecURRRbWa8tbfFARBDa0=
github.com/GopeedLab/webview_go v0.0.0-20260917153919-dcf00737f32a h1:nz1bC6TGwQ1kv/OVLahNZ0EEtb7v5LHfceAkW4MvS6I=
github.com/GopeedLab/webview_go v0.0.0-20260917153919-dcf00737f32a/go.mod h1:FvV19lu9rQ9NcbczN5y8pbWecURRRbWa8tbfFARBDa0=
github.com/GopeedLab/webview_go v0.0.0-20260917155039-1ee7ffb4169a h1:yudcfXpipKLnMaaw1losn6YeOBjvj2rAFicfjqYJexY=
github.com/GopeedLab/webview_go v0.0.0-20260917155039-1ee7ffb4169a/go.mod h1:FvV19lu9rQ9NcbczN5y8pbWecURRRbWa8tbfFARBDa0=
github.com/GopeedLab/webview_go v0.0.0-20260917155218-bd16da0d6f53 h1:nMSHKhbrEaWPE/x3mYl/270GPjswbUmS+E7yYjly4xk=
github.com/GopeedLab/webview_go v0.0.0-20260917155218-bd16da0d6f53/go.mod h1:oNO07SCX8xgoHLds2OBwML8TU4xils88vMQ4gadKUHQ=
github.com/GopeedLab/webview_go v0.0.0-20260917161756-2a4f989dcb1d h1:4fjlhJpBWa6IcdKVPkEMMIDd6Xo9JM09dJHdKU+1V7k=
github.com/GopeedLab/webview_go v0.0.0-20260917161756-2a4f989dcb1d/go.mod h1:oNO07SCX8xgoHLds2OBwML8TU4xils88vMQ4gadKUHQ=
github.com/GopeedLab/webview_go v0.0.0-20260917163929-ca26c3e6bcfa h1:VkdBMcNHYTqCMwOOCimA7b3xZKV/qFIGrhh3+vm9n3s=
github.com/GopeedLab/webview_go v0.0.0-20260917163929-ca26c3e6bcfa/go.mod h1:oNO07SCX8xgoHLds2OBwML8TU4xils88vMQ4gadKUHQ=
github.com/GopeedLab/webview_go v0.0.0-20260918010912-0d4c27bb7914 h1:9VJQZIZ1R6T4m+4vlNQYNXSPNUv2gf2PiTfIXiQ3MTw=
github.com/GopeedLab/webview_go v0.0.0-20260918010912-0d4c27bb7914/go.mod h1:oNO07SCX8xgoHLds2OBwML8TU4xils88vMQ4gadKUHQ=
github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0rYXWg0=
github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ=
github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY=
@@ -552,6 +566,8 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
github.com/tetratelabs/wazero v1.12.0 h1:DuWcpNu/FzgEXgGBDp8J1Spc+CWOvvtvVyjKlaZopYU=
github.com/tetratelabs/wazero v1.12.0/go.mod h1:LvKtzl2RqO4gyF27BiXU+nKAjcV8f38U+kP/q2vgxh0=
github.com/tidwall/btree v1.8.1 h1:27ehoXvm5AG/g+1VxLS1SD3vRhp/H7LuEfwNvddEdmA=
github.com/tidwall/btree v1.8.1/go.mod h1:jBbTdUWhSZClZWoDg54VnvV7/54modSOzDN7VXftj1A=
github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE=
@@ -732,6 +748,8 @@ golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k=
golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
golang.org/x/sys v0.44.0 h1:ildZl3J4uzeKP07r2F++Op7E9B29JRUy+a27EibtBTQ=
golang.org/x/sys v0.44.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc=
+44
View File
@@ -7,6 +7,7 @@ import (
"encoding/hex"
"errors"
"fmt"
"github.com/GopeedLab/gopeed/internal/production"
"io"
"net"
"net/http"
@@ -61,6 +62,7 @@ type Metadata struct {
}
type Source struct {
producer production.Reader
ID string
URL string
ContentType string
@@ -359,11 +361,35 @@ func (r *Registry) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return
}
defer reader.Close()
if producer, ok := reader.(production.Reader); ok {
src.mu.Lock()
src.producer = producer
src.mu.Unlock()
if err := producer.WaitProduction(req.Context()); err != nil {
return
}
if producer.Production() != nil {
meta.Size = 0
meta.Range = false
start = 0
end = -1
ranged = false
src.mu.Lock()
src.size = 0
src.rangeEnabled = false
src.mu.Unlock()
}
}
writeHeaders(w, meta, start, end, ranged)
if ranged {
w.WriteHeader(http.StatusPartialContent)
}
if producer, ok := reader.(production.Reader); ok && producer.Production() != nil {
if flusher, ok := w.(http.Flusher); ok {
flusher.Flush()
}
}
limit := int64(-1)
if end >= start {
limit = end - start + 1
@@ -675,3 +701,21 @@ func randomID(size int) (string, error) {
}
return hex.EncodeToString(buf), nil
}
// Production returns the active opener's input counter, if it generates output.
func (r *Registry) Production(raw string) production.Source {
if r == nil {
return nil
}
src, err := r.get(raw)
if err != nil {
return nil
}
src.mu.Lock()
reader := src.producer
src.mu.Unlock()
if reader == nil {
return nil
}
return reader.Production()
}
+6 -2
View File
@@ -1,6 +1,7 @@
package controller
import (
"github.com/GopeedLab/gopeed/internal/tempfiles"
"github.com/GopeedLab/gopeed/pkg/base"
"net/http"
"net/url"
@@ -9,8 +10,11 @@ import (
)
type Controller struct {
GetConfig func(v any)
GetProxy func(requestProxy *base.RequestProxy) func(*http.Request) (*url.URL, error)
TempFiles *tempfiles.Scope
ManagedProduction func(string) bool
TempDir string
GetConfig func(v any)
GetProxy func(requestProxy *base.RequestProxy) func(*http.Request) (*url.URL, error)
FileController
//ContextDialer() (proxy.Dialer, error)
}
+7 -1
View File
@@ -106,7 +106,13 @@ func (s *SchemeFilter) Match(uri string) bool {
case FilterTypeUrl:
return strings.HasPrefix(uriUpper, patternUpper+":")
case FilterTypeFile:
return strings.HasSuffix(uriUpper, "."+patternUpper)
// Strip query and fragment before matching so URLs like
// https://example.com/video.m3u8?sign=abc still route by extension.
clean := uri
if idx := strings.IndexAny(clean, "?#"); idx >= 0 {
clean = clean[:idx]
}
return strings.HasSuffix(strings.ToUpper(clean), "."+patternUpper)
case FilterTypeBase64:
return strings.HasPrefix(uriUpper, "DATA:"+patternUpper+";BASE64,")
}
+35
View File
@@ -0,0 +1,35 @@
// Package production associates generated resources with their input progress.
package production
import (
"context"
"sync"
)
// Source counts unique input bytes and total received bytes (including refetches).
// Implementations must support concurrent snapshots.
type Source interface {
Progress() (downloaded, received int64)
}
// Reader resolves its optional producer before response headers are sent.
// A nil producer preserves ordinary Blob behavior.
type Reader interface {
Production() Source
WaitProduction(context.Context) error
}
type Registry struct {
mu sync.Mutex
sources map[string]Source
}
func (r *Registry) Set(id string, source Source) {
r.mu.Lock()
defer r.mu.Unlock()
if r.sources == nil {
r.sources = make(map[string]Source)
}
r.sources[id] = source
}
func (r *Registry) Get(id string) Source { r.mu.Lock(); defer r.mu.Unlock(); return r.sources[id] }
func (r *Registry) Remove(id string) { r.mu.Lock(); defer r.mu.Unlock(); delete(r.sources, id) }
+64 -1
View File
@@ -399,7 +399,10 @@ func (f *Fetcher) Wait() (err error) {
}
}
if !selected {
util.SafeRemove(filepath.Join(f.meta.Opts.Path, f.meta.Res.Name, file.Path()))
root, name := torrentFileLayout(f.torrent.Info(), file.FileInfo())
if err := removeUnselectedFile(f.meta.Opts.Path, root, name); err != nil {
return err
}
}
}
return
@@ -409,6 +412,62 @@ func (f *Fetcher) Wait() (err error) {
}
}
// Match the file storage layout, including its NoName convention for rootless
// torrents. File.Path() always prepends BestName, even when storage omits it.
func torrentFileLayout(info *metainfo.Info, file metainfo.FileInfo) (root, name string) {
parts := file.BestPath()
if info.BestName() != metainfo.NoName {
if len(parts) > 0 {
root = info.BestName()
}
parts = append([]string{info.BestName()}, parts...)
}
// A v2 file tree can itself have a named root without an extra name wrapper.
if root == "" && info.HasV2() && len(info.FileTree.Dir) == 1 {
for dir, tree := range info.FileTree.Dir {
if tree.IsDir() {
root = dir
}
}
}
return root, filepath.Join(parts...)
}
// Preserve the actual torrent root, or the download directory for rootless
// torrents. Only prune empty directories strictly below that boundary.
func removeUnselectedFile(downloadDir, torrentRoot, torrentPath string) error {
if !filepath.IsLocal(torrentPath) || (torrentRoot != "" && !filepath.IsLocal(torrentRoot)) {
return fmt.Errorf("invalid torrent file path %q", torrentPath)
}
root := filepath.Join(downloadDir, torrentRoot)
name := filepath.Join(downloadDir, torrentPath)
rel, err := filepath.Rel(root, name)
if err != nil || rel == "." || !filepath.IsLocal(rel) {
return fmt.Errorf("torrent file %q is outside cleanup root %q", name, root)
}
for _, candidate := range []string{name, name + ".part"} {
if err := util.SafeRemove(candidate); err != nil {
return fmt.Errorf("remove unselected torrent file %q: %w", candidate, err)
}
}
for dir := filepath.Dir(name); dir != root && dir != filepath.Dir(dir); dir = filepath.Dir(dir) {
entries, err := os.ReadDir(dir)
if os.IsNotExist(err) {
continue
}
if err != nil {
return fmt.Errorf("read torrent directory %q: %w", dir, err)
}
if len(entries) != 0 {
break
}
if err := util.SafeRemove(dir); err != nil {
return fmt.Errorf("remove empty torrent directory %q: %w", dir, err)
}
}
return nil
}
func (f *Fetcher) isDone() bool {
if f.meta.Opts == nil {
return false
@@ -638,6 +697,10 @@ func (f *Fetcher) addTorrent(req *base.Request, fromUpload bool) (err error) {
}
spec.Storage = storage.NewFileOpts(storage.NewFileClientOpts{
ClientBaseDir: cfg.DataDir,
FilePathMaker: func(opts storage.FilePathMakerOpts) string {
_, name := torrentFileLayout(opts.Info, *opts.File)
return name
},
TorrentDirMaker: func(baseDir string, info *metainfo.Info, infoHash metainfo.Hash) string {
return f.meta.Opts.Path
},
+307
View File
@@ -0,0 +1,307 @@
package bt
import (
"context"
"crypto/sha1"
"fmt"
"os"
"path/filepath"
"testing"
"time"
"github.com/GopeedLab/gopeed/internal/fetcher"
"github.com/GopeedLab/gopeed/pkg/base"
"github.com/anacrolix/torrent"
"github.com/anacrolix/torrent/bencode"
"github.com/anacrolix/torrent/metainfo"
"github.com/anacrolix/torrent/storage"
)
func TestStartUsesSelectionSubmittedAfterResolve(t *testing.T) {
config := torrent.TestingConfig(t)
config.DataDir = t.TempDir()
config.DisableTCP = true
config.DisableUTP = true
c, err := torrent.NewClient(config)
if err != nil {
t.Fatal(err)
}
defer c.Close()
tor, err := c.AddTorrentFromFile("testdata/test.torrent")
if err != nil {
t.Fatal(err)
}
f := &Fetcher{torrent: tor, meta: &fetcher.FetcherMeta{Opts: &base.Options{}}, data: &fetcherData{}}
f.torrentReady.Store(true)
f.updateRes()
// Resolve initializes all files; creation replaces the selection before Start.
f.meta.Opts = &base.Options{SelectFiles: []int{2}}
if err := f.Start(); err != nil {
t.Fatal(err)
}
for i, file := range tor.Files() {
want := torrent.PiecePriorityNone
if i == 2 {
want = torrent.PiecePriorityNormal
}
if file.Priority() != want {
t.Errorf("file %d priority = %v, want %v", i, file.Priority(), want)
}
}
if len(f.data.Progress) != 1 {
t.Fatalf("progress tracks %d files, want 1", len(f.data.Progress))
}
if err := f.Pause(); err != nil {
t.Fatal(err)
}
if err := f.Start(); err != nil {
t.Fatal(err)
}
if tor.Files()[0].Priority() != torrent.PiecePriorityNone {
t.Fatal("resume selected an unselected file")
}
}
func TestWaitCleansUnselectedFilesFromSharedPiece(t *testing.T) {
for _, mode := range []string{"partial", "part only", "both paths", "all", "already removed", "remove error"} {
t.Run(mode, func(t *testing.T) {
dir := t.TempDir()
keepPath := filepath.Join(dir, "bundle", "keep.txt")
skipPath := filepath.Join(dir, "bundle", "nested", "skip.txt")
if err := os.MkdirAll(filepath.Dir(skipPath), 0755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(keepPath, []byte("keep"), 0644); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(skipPath, []byte("skip"), 0644); err != nil {
t.Fatal(err)
}
// Both files occupy the same piece, so downloading keep.txt also writes skip.txt.
hash := sha1.Sum([]byte("keepskip"))
info := metainfo.Info{Name: "bundle", PieceLength: 16, Pieces: hash[:], Files: []metainfo.FileInfo{
{Length: 4, Path: []string{"keep.txt"}},
{Length: 4, Path: []string{"nested", "skip.txt"}},
}}
storageOpts := storage.NewFileClientOpts{
ClientBaseDir: dir,
TorrentDirMaker: func(baseDir string, _ *metainfo.Info, _ metainfo.Hash) string { return baseDir },
}
// VerifyDataContext can return before storage finishes marking the piece
// complete. Disable automatic part-file promotion so it cannot rename
// the .part fixtures that this test creates to exercise cleanup.
storageOpts.UsePartFiles.Set(false)
store := storage.NewFileOpts(storageOpts)
defer store.Close()
config := torrent.TestingConfig(t)
config.DisableTCP = true
config.DisableUTP = true
c, err := torrent.NewClient(config)
if err != nil {
t.Fatal(err)
}
defer c.Close()
spec, err := torrent.TorrentSpecFromMetaInfoErr(&metainfo.MetaInfo{InfoBytes: bencode.MustMarshal(&info)})
if err != nil {
t.Fatal(err)
}
spec.Storage = store
tor, _, err := c.AddTorrentSpec(spec)
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
if err := tor.VerifyDataContext(ctx); err != nil {
t.Fatal(err)
}
f := &Fetcher{torrent: tor, meta: &fetcher.FetcherMeta{Opts: &base.Options{Path: dir, SelectFiles: []int{0}}}, data: &fetcherData{}, torrentDropCtx: ctx}
f.torrentReady.Store(true)
f.updateRes()
if mode == "all" {
f.meta.Opts.SelectFiles = []int{0, 1}
}
if !f.isDone() {
t.Fatal("selected file is not complete")
}
if mode == "part only" || mode == "both paths" || mode == "all" {
if err := os.WriteFile(skipPath+".part", []byte("skip"), 0644); err != nil {
t.Fatal(err)
}
}
if mode == "part only" || mode == "already removed" || mode == "remove error" {
if err := os.Remove(skipPath); err != nil {
t.Fatal(err)
}
}
if mode == "remove error" {
if err := os.Mkdir(skipPath, 0755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(skipPath, "child"), []byte("preserve"), 0644); err != nil {
t.Fatal(err)
}
}
err = f.Wait()
if ctx.Err() != nil {
t.Fatal(ctx.Err())
}
if mode == "remove error" {
if err == nil {
t.Fatal("cleanup failure was silently ignored")
}
} else if err != nil {
t.Fatal(err)
}
data, err := os.ReadFile(keepPath)
if err != nil || string(data) != "keep" {
t.Fatalf("selected file changed: %q, %v", data, err)
}
if mode == "all" {
if _, err := os.Stat(skipPath + ".part"); err != nil {
t.Fatalf("selected part file was removed: %v", err)
}
data, err := os.ReadFile(skipPath)
if err != nil || string(data) != "skip" {
t.Fatalf("selected file changed: %q, %v", data, err)
}
} else if mode != "remove error" {
for _, removed := range []string{skipPath, skipPath + ".part", filepath.Dir(skipPath)} {
if _, err := os.Stat(removed); !os.IsNotExist(err) {
t.Fatalf("unselected file or empty directory remains: %s: %v", removed, err)
}
}
}
})
}
}
func TestRemoveUnselectedFilePrunesOnlyEmptyParents(t *testing.T) {
for _, preserveSibling := range []bool{false, true} {
t.Run(fmt.Sprint(preserveSibling), func(t *testing.T) {
root := t.TempDir()
relative := filepath.Join("bundle", "nested", "deep", "skip.txt")
name := filepath.Join(root, relative)
if err := os.MkdirAll(filepath.Dir(name), 0755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(name+".part", []byte("partial"), 0644); err != nil {
t.Fatal(err)
}
sibling := filepath.Join(root, "bundle", "keep.txt")
if preserveSibling {
if err := os.WriteFile(sibling, []byte("keep"), 0644); err != nil {
t.Fatal(err)
}
}
if err := removeUnselectedFile(root, "bundle", relative); err != nil {
t.Fatal(err)
}
if _, err := os.Stat(filepath.Join(root, "bundle", "nested")); !os.IsNotExist(err) {
t.Fatalf("empty ancestors remain: %v", err)
}
if preserveSibling {
data, err := os.ReadFile(sibling)
if err != nil || string(data) != "keep" {
t.Fatalf("sibling changed: %q, %v", data, err)
}
}
if info, err := os.Stat(filepath.Join(root, "bundle")); err != nil || !info.IsDir() {
t.Fatalf("torrent root directory was removed: %v", err)
}
if _, err := os.Stat(root); err != nil {
t.Fatalf("download directory was removed: %v", err)
}
// A file directly inside the torrent root must not cause that root to be pruned.
rootFile := filepath.Join("bundle", "root.txt")
if err := os.WriteFile(filepath.Join(root, rootFile)+".part", []byte("partial"), 0644); err != nil {
t.Fatal(err)
}
if err := removeUnselectedFile(root, "bundle", rootFile); err != nil {
t.Fatal(err)
}
if _, err := os.Stat(filepath.Join(root, "bundle")); err != nil {
t.Fatalf("torrent root directory was removed after root-level cleanup: %v", err)
}
// Repeating cleanup after the file and its subdirectories are gone is harmless.
if err := removeUnselectedFile(root, "bundle", relative); err != nil {
t.Fatal(err)
}
})
}
}
func TestCleanupMatchesNamedAndRootlessStorage(t *testing.T) {
for _, version := range []int64{1, 2} {
for _, layout := range []string{"rootless", "wrapped", "file-tree-root"} {
if version == 1 && layout == "file-tree-root" {
continue
}
t.Run(fmt.Sprintf("v%d/%s", version, layout), func(t *testing.T) {
dir := t.TempDir()
info := metainfo.Info{Name: metainfo.NoName, PieceLength: 16384}
root := dir
if layout == "wrapped" {
info.Name = "bundle"
root = filepath.Join(dir, "bundle")
}
if version == 1 {
info.Files = []metainfo.FileInfo{{Path: []string{"keep.txt"}}, {Path: []string{"nested", "deep", "skip.txt"}}}
} else {
info.MetaVersion = 2
info.FileTree = metainfo.FileTree{Dir: map[string]metainfo.FileTree{
"keep.txt": {},
"nested": {Dir: map[string]metainfo.FileTree{"deep": {Dir: map[string]metainfo.FileTree{"skip.txt": {}}}}},
}}
}
if layout == "file-tree-root" {
info.FileTree = metainfo.FileTree{Dir: map[string]metainfo.FileTree{"tree-root": info.FileTree}}
root = filepath.Join(dir, "tree-root")
}
// Keep the library's default FilePathMaker here to independently verify
// that cleanup agrees with its real disk layout, including NoName.
store := storage.NewFileOpts(storage.NewFileClientOpts{
ClientBaseDir: dir,
TorrentDirMaker: func(baseDir string, _ *metainfo.Info, _ metainfo.Hash) string { return baseDir },
})
defer store.Close()
torrentStorage, err := store.OpenTorrent(context.Background(), &info, metainfo.Hash{1})
if err != nil {
t.Fatal(err)
}
defer torrentStorage.Close()
skip := filepath.Join(root, "nested", "deep", "skip.txt")
if _, err := os.Stat(skip); err != nil {
t.Fatalf("unexpected storage layout: %v", err)
}
if err := os.WriteFile(skip+".part", []byte("partial"), 0644); err != nil {
t.Fatal(err)
}
files := info.UpvertedFiles()
boundary, name := torrentFileLayout(&info, files[1])
if err := removeUnselectedFile(dir, boundary, name); err != nil {
t.Fatal(err)
}
if _, err := os.Stat(filepath.Join(root, "nested")); !os.IsNotExist(err) {
t.Fatalf("empty subtree remains: %v", err)
}
if _, err := os.Stat(filepath.Join(root, "keep.txt")); err != nil {
t.Fatalf("selected file removed: %v", err)
}
// Also verify the boundary when the protected directory becomes empty.
if err := os.Remove(filepath.Join(root, "keep.txt")); err != nil {
t.Fatal(err)
}
for _, file := range files {
boundary, name := torrentFileLayout(&info, file)
if err := removeUnselectedFile(dir, boundary, name); err != nil {
t.Fatal(err)
}
}
if stat, err := os.Stat(root); err != nil || !stat.IsDir() {
t.Fatalf("cleanup boundary was removed: %v", err)
}
})
}
}
}
+30
View File
@@ -0,0 +1,30 @@
package hls
const (
defaultSegmentConnections = 8
defaultMaxRetries = 5
defaultTimeoutSeconds = 30
// maxPrefetchSizeSegments caps the HEAD probing done at resolve time.
// Playlists larger than this skip size prefetch entirely.
maxPrefetchSizeSegments = 500
)
type config struct {
SegmentConnections int `json:"segmentConnections"`
MaxRetries int `json:"maxRetries"`
TimeoutSeconds int `json:"timeoutSeconds"`
PrefetchContentLength bool `json:"prefetchContentLength"`
}
func (c *config) normalize() {
if c.SegmentConnections <= 0 {
c.SegmentConnections = defaultSegmentConnections
}
if c.MaxRetries < 0 {
c.MaxRetries = defaultMaxRetries
}
if c.TimeoutSeconds <= 0 {
c.TimeoutSeconds = defaultTimeoutSeconds
}
}
+59
View File
@@ -0,0 +1,59 @@
package hls
import (
"crypto/aes"
"crypto/cipher"
"fmt"
)
// decryptAES128 decrypts an AES-128-CBC encrypted segment and strips PKCS7
// padding. data length must be a multiple of the AES block size.
func decryptAES128(data, key, iv []byte) ([]byte, error) {
if len(key) != 16 {
return nil, fmt.Errorf("invalid AES-128 key size %d", len(key))
}
if len(iv) != 16 {
return nil, fmt.Errorf("invalid AES-128 IV size %d", len(iv))
}
if len(data) == 0 || len(data)%aes.BlockSize != 0 {
return nil, fmt.Errorf("invalid AES-128 data size %d", len(data))
}
block, err := aes.NewCipher(key)
if err != nil {
return nil, err
}
plain := make([]byte, len(data))
cipher.NewCBCDecrypter(block, iv).CryptBlocks(plain, data)
return pkcs7Unpad(plain, aes.BlockSize)
}
func pkcs7Unpad(data []byte, blockSize int) ([]byte, error) {
if len(data) == 0 {
return nil, fmt.Errorf("empty plaintext")
}
pad := int(data[len(data)-1])
if pad == 0 || pad > blockSize || pad > len(data) {
return nil, fmt.Errorf("invalid PKCS7 padding")
}
for _, b := range data[len(data)-pad:] {
if int(b) != pad {
return nil, fmt.Errorf("invalid PKCS7 padding")
}
}
return data[:len(data)-pad], nil
}
// sequenceIV derives the default per-segment IV mandated by RFC 8216: the
// segment media sequence number as a 16-byte big-endian integer.
func sequenceIV(sequence int64) []byte {
iv := make([]byte, 16)
iv[8] = byte(sequence >> 56)
iv[9] = byte(sequence >> 48)
iv[10] = byte(sequence >> 40)
iv[11] = byte(sequence >> 32)
iv[12] = byte(sequence >> 24)
iv[13] = byte(sequence >> 16)
iv[14] = byte(sequence >> 8)
iv[15] = byte(sequence)
return iv
}
+66
View File
@@ -0,0 +1,66 @@
package hls
import (
"bytes"
"crypto/aes"
"crypto/cipher"
"crypto/rand"
"testing"
)
func pkcs7Pad(data []byte, blockSize int) []byte {
pad := blockSize - len(data)%blockSize
return append(data, bytes.Repeat([]byte{byte(pad)}, pad)...)
}
func TestDecryptAES128Roundtrip(t *testing.T) {
key := make([]byte, 16)
iv := make([]byte, 16)
if _, err := rand.Read(key); err != nil {
t.Fatal(err)
}
if _, err := rand.Read(iv); err != nil {
t.Fatal(err)
}
plain := []byte("hello hls segment payload, padded to block size!")
padded := pkcs7Pad(plain, aes.BlockSize)
block, err := aes.NewCipher(key)
if err != nil {
t.Fatal(err)
}
encrypted := make([]byte, len(padded))
cipher.NewCBCEncrypter(block, iv).CryptBlocks(encrypted, padded)
got, err := decryptAES128(encrypted, key, iv)
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(got, plain) {
t.Errorf("roundtrip mismatch: %q", got)
}
}
func TestDecryptAES128InvalidInput(t *testing.T) {
if _, err := decryptAES128([]byte{1, 2, 3}, make([]byte, 16), make([]byte, 16)); err == nil {
t.Error("non-block-aligned data should fail")
}
if _, err := decryptAES128(make([]byte, 16), make([]byte, 15), make([]byte, 16)); err == nil {
t.Error("short key should fail")
}
if _, err := decryptAES128(make([]byte, 16), make([]byte, 16), make([]byte, 16)); err == nil {
t.Error("all-zero block has invalid padding and should fail")
}
}
func TestSequenceIV(t *testing.T) {
iv := sequenceIV(1)
if len(iv) != 16 || iv[15] != 1 {
t.Errorf("sequence 1 IV wrong: %v", iv)
}
iv = sequenceIV(0x0102030405060708)
for i, want := range []byte{1, 2, 3, 4, 5, 6, 7, 8} {
if iv[8+i] != want {
t.Errorf("sequence IV byte %d: want %d got %d", i, want, iv[8+i])
}
}
}
+24
View File
@@ -0,0 +1,24 @@
// Package hls implements native HTTP Live Streaming downloads for m3u8
// playlists. A task produces exactly one merged, playable output file:
// MPEG-2 TS segments are concatenated as-is, fMP4 streams are merged as
// init section followed by the media fragments, and AES-128 encrypted
// segments are decrypted before they are stored.
//
// Supported subset:
// - VOD media playlists (EXT-X-ENDLIST present) and master playlists whose
// selected variant carries its own audio
// - AES-128 with the identity KEYFORMAT, explicit IV or media-sequence
// derived IV, including key rotation mid-playlist
// - EXT-X-BYTERANGE (including offset inheritance) and fMP4 init sections
// via a single, constant EXT-X-MAP
//
// Everything the merge pipeline cannot reproduce faithfully is rejected with
// an explicit error instead of silently producing a broken output file:
// - live streams (no EXT-X-ENDLIST) and SAMPLE-AES / DRM encryption
// - EXT-X-GAP (would leave a hole) and EXT-X-DISCONTINUITY (broken timeline)
// - changing init sections (a repeated EXT-X-MAP is compared by all its
// attributes, not just the URI) and encrypted EXT-X-MAP without an
// explicit IV
// - separate audio renditions referenced by the selected variant (the
// merged output would have no audio) and non-identity KEYFORMATs
package hls
File diff suppressed because it is too large. Load diff
+100
View File
@@ -0,0 +1,100 @@
package hls
import (
"net/url"
"path"
"strings"
"github.com/GopeedLab/gopeed/internal/fetcher"
)
type FetcherManager struct {
}
func (fm *FetcherManager) Name() string {
return "hls"
}
// Filters registers the .m3u8 extension. The manager must be registered BEFORE
// the http manager in the downloader config: protocol routing picks the first
// manager whose filter matches, and the http filter claims every http(s) URL.
func (fm *FetcherManager) Filters() []*fetcher.SchemeFilter {
return []*fetcher.SchemeFilter{
{
Type: fetcher.FilterTypeFile,
Pattern: "M3U8",
},
}
}
func (fm *FetcherManager) Build() fetcher.Fetcher {
return &Fetcher{}
}
func (fm *FetcherManager) ParseName(u string) string {
parsed, err := url.Parse(u)
if err != nil {
return ""
}
name := path.Base(parsed.Path)
name = strings.TrimSuffix(name, path.Ext(name))
if name == "" || name == "/" || name == "." {
name = parsed.Hostname()
}
return name
}
func (fm *FetcherManager) AutoRename() bool {
return true
}
func (fm *FetcherManager) DefaultConfig() any {
return &config{
SegmentConnections: defaultSegmentConnections,
MaxRetries: defaultMaxRetries,
TimeoutSeconds: defaultTimeoutSeconds,
PrefetchContentLength: true,
}
}
func (fm *FetcherManager) Store(f fetcher.Fetcher) (data any, err error) {
_f := f.(*Fetcher)
_f.mu.Lock()
defer _f.mu.Unlock()
if _f.state == nil {
return nil, nil
}
// The engine's periodic checkpoint calls Store without holding the task
// lock and serializes the returned value afterwards, while Resolve/Patch/
// Start mutate the live state under the fetcher mutex: return a snapshot
// copy taken under that mutex, mirroring the HTTP protocol's Store.
return cloneState(_f.state), nil
}
func cloneState(state *fetcherState) *fetcherState {
cloned := *state
cloned.Segments = make([]*Segment, len(state.Segments))
for i, seg := range state.Segments {
s := *seg
cloned.Segments[i] = &s
}
return &cloned
}
func (fm *FetcherManager) Restore() (v any, f func(meta *fetcher.FetcherMeta, v any) fetcher.Fetcher) {
return &fetcherState{}, func(meta *fetcher.FetcherMeta, v any) fetcher.Fetcher {
fb := &FetcherManager{}
restored := fb.Build().(*Fetcher)
restored.meta = meta
if state, ok := v.(*fetcherState); ok && state.MediaURL != "" {
restored.state = state
}
// Setup is invoked by the engine (restoreFetcher) after this builder
// returns; ReqExtra parsing happens there as well.
return restored
}
}
func (fm *FetcherManager) Close() error {
return nil
}
+857
View File
@@ -0,0 +1,857 @@
package hls
import (
"bytes"
"crypto/aes"
"crypto/cipher"
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strconv"
"strings"
"sync/atomic"
"testing"
"time"
"github.com/GopeedLab/gopeed/internal/controller"
"github.com/GopeedLab/gopeed/internal/fetcher"
"github.com/GopeedLab/gopeed/pkg/base"
)
func pkcs7PadForTest(data []byte, blockSize int) []byte {
pad := blockSize - len(data)%blockSize
return append(data, bytes.Repeat([]byte{byte(pad)}, pad)...)
}
func encryptSegment(plain, key, iv []byte) []byte {
padded := pkcs7PadForTest(plain, aes.BlockSize)
block, _ := aes.NewCipher(key)
encrypted := make([]byte, len(padded))
cipher.NewCBCEncrypter(block, iv).CryptBlocks(encrypted, padded)
return encrypted
}
// serveContentWithReferer is a helper handler that enforces an optional
// Referer requirement and reports request counts.
type cdnHandler struct {
mux *http.ServeMux
requireReferer string
requests atomic.Int64
}
func (h *cdnHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
h.requests.Add(1)
if h.requireReferer != "" && r.Header.Get("Referer") != h.requireReferer {
w.WriteHeader(http.StatusForbidden)
return
}
h.mux.ServeHTTP(w, r)
}
func newTestFetcher(t *testing.T, cfg config) *Fetcher {
t.Helper()
fm := &FetcherManager{}
f := fm.Build().(*Fetcher)
ctl := controller.NewController()
ctl.GetConfig = func(v any) {
if c, ok := v.(*config); ok {
*c = cfg
}
}
f.Setup(ctl)
return f
}
func testConfig() config {
return config{
SegmentConnections: 4,
MaxRetries: 2,
TimeoutSeconds: 5,
PrefetchContentLength: true,
}
}
func mustResolve(t *testing.T, f *Fetcher, playlistURL, referer string) {
t.Helper()
req := &base.Request{URL: playlistURL}
if referer != "" {
req.Extra = &ReqExtra{Header: map[string]string{"Referer": referer}}
}
if err := f.Resolve(req, &base.Options{Path: t.TempDir()}); err != nil {
t.Fatalf("resolve: %v", err)
}
}
func mustRun(t *testing.T, f *Fetcher) {
t.Helper()
if err := f.Start(); err != nil {
t.Fatalf("start: %v", err)
}
if err := f.Wait(); err != nil {
t.Fatalf("wait: %v", err)
}
}
func statsOf(t *testing.T, f *Fetcher) Stats {
t.Helper()
s, ok := f.Stats().Snapshot.(*Stats)
if !ok {
t.Fatalf("unexpected stats snapshot type %T", f.Stats().Snapshot)
}
return *s
}
func readOutput(t *testing.T, f *Fetcher) []byte {
t.Helper()
data, err := os.ReadFile(f.meta.SingleFilepath())
if err != nil {
t.Fatal(err)
}
return data
}
func TestFetcherE2E_MasterTS(t *testing.T) {
seg := func(i int) []byte {
return []byte(fmt.Sprintf("SEGMENT-%02d-PAYLOAD", i))
}
mux := http.NewServeMux()
mux.HandleFunc("/master.m3u8", func(w http.ResponseWriter, r *http.Request) {
w.Write([]byte("#EXTM3U\n" +
"#EXT-X-STREAM-INF:BANDWIDTH=800000\nv1/index.m3u8\n" +
"#EXT-X-STREAM-INF:BANDWIDTH=2000000\nv2/index.m3u8\n"))
})
mux.HandleFunc("/v1/index.m3u8", func(w http.ResponseWriter, r *http.Request) {
w.Write([]byte("#EXTM3U\n#EXTINF:4,\nlow-0.ts\n#EXT-X-ENDLIST\n"))
})
mux.HandleFunc("/v2/index.m3u8", func(w http.ResponseWriter, r *http.Request) {
var b strings.Builder
b.WriteString("#EXTM3U\n")
for i := 0; i < 5; i++ {
b.WriteString(fmt.Sprintf("#EXTINF:4,\nseg-%d.ts\n", i))
}
b.WriteString("#EXT-X-ENDLIST\n")
w.Write([]byte(b.String()))
})
for i := 0; i < 5; i++ {
content := seg(i)
path := "/v2/seg-" + strconv.Itoa(i) + ".ts"
mux.HandleFunc(path, func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Length", strconv.Itoa(len(content)))
w.Write(content)
})
}
server := httptest.NewServer(&cdnHandler{mux: mux})
defer server.Close()
f := newTestFetcher(t, testConfig())
mustResolve(t, f, server.URL+"/master.m3u8", "")
// Master resolution picked the 2000000 variant.
if f.state.MediaURL != server.URL+"/v2/index.m3u8" {
t.Fatalf("wrong variant picked: %s", f.state.MediaURL)
}
// HEAD prefetch should have computed the exact total size.
if f.meta.Res.Size != int64(5*len(seg(0))) {
t.Fatalf("prefetched size wrong: %d", f.meta.Res.Size)
}
mustRun(t, f)
var want []byte
for i := 0; i < 5; i++ {
want = append(want, seg(i)...)
}
got := readOutput(t, f)
if !bytes.Equal(got, want) {
t.Errorf("merged output mismatch: %q", got)
}
// The staging folder must be cleaned up after a successful merge; only
// the output file remains in the task directory.
if entries, _ := os.ReadDir(filepath.Dir(f.meta.SingleFilepath())); len(entries) != 1 {
for _, e := range entries {
t.Logf("leftover entry: %s", e.Name())
}
t.Errorf("staging dir should be cleaned up after merge")
}
if f.meta.Res.Size != int64(len(want)) {
t.Errorf("final size wrong: %d", f.meta.Res.Size)
}
}
func TestFetcherE2E_RefererEnforced(t *testing.T) {
page := "https://player.example.com/watch/123"
var playlist strings.Builder
playlist.WriteString("#EXTM3U\n")
segments := make(map[string][]byte)
for i := 0; i < 3; i++ {
name := fmt.Sprintf("seg-%d.ts", i)
segments["/"+name] = []byte(fmt.Sprintf("DATA-%d", i))
playlist.WriteString(fmt.Sprintf("#EXTINF:4,\n%s\n", name))
}
playlist.WriteString("#EXT-X-ENDLIST\n")
mux := http.NewServeMux()
mux.HandleFunc("/video.m3u8", func(w http.ResponseWriter, r *http.Request) {
w.Write([]byte(playlist.String()))
})
for path, content := range segments {
c := content
mux.HandleFunc(path, func(w http.ResponseWriter, r *http.Request) {
w.Write(c)
})
}
server := httptest.NewServer(&cdnHandler{mux: mux, requireReferer: page})
defer server.Close()
// Without Referer the resolve itself must fail (403 on the playlist).
blocked := newTestFetcher(t, testConfig())
req := &base.Request{URL: server.URL + "/video.m3u8"}
if err := blocked.Resolve(req, &base.Options{Path: t.TempDir()}); err == nil {
t.Fatal("resolve without Referer should fail")
}
// With the Referer set through the task extra headers it must succeed.
allowed := newTestFetcher(t, testConfig())
allowedReq := &base.Request{
URL: server.URL + "/video.m3u8",
Extra: &ReqExtra{Header: map[string]string{"Referer": page}},
}
if err := allowed.Resolve(allowedReq, &base.Options{Path: t.TempDir()}); err != nil {
t.Fatal(err)
}
mustRun(t, allowed)
want := []byte("DATA-0DATA-1DATA-2")
if !bytes.Equal(readOutput(t, allowed), want) {
t.Error("output mismatch with Referer download")
}
}
func TestFetcherE2E_AES128KeyRotation(t *testing.T) {
key1 := []byte("0123456789abcdef")
key2 := []byte("fedcba9876543210")
iv1 := []byte("aaaaaaaaaaaaaaaa")
plain1 := []byte("ENCRYPTED-SEGMENT-ONE!!") // padded on encrypt
plain2 := []byte("ENCRYPTED-SEGMENT-TWO!!")
// seg-2 has no explicit IV: derived from media sequence 1.
enc1 := encryptSegment(plain1, key1, iv1)
enc2 := encryptSegment(plain2, key2, sequenceIV(1))
mux := http.NewServeMux()
mux.HandleFunc("/video.m3u8", func(w http.ResponseWriter, r *http.Request) {
w.Write([]byte("#EXTM3U\n" +
"#EXT-X-MEDIA-SEQUENCE:0\n" +
"#EXT-X-KEY:METHOD=AES-128,URI=\"key-1.bin\",IV=0x" + hexEncode(iv1) + "\n" +
"#EXTINF:5,\nseg-1.ts\n" +
"#EXT-X-KEY:METHOD=AES-128,URI=\"key-2.bin\"\n" +
"#EXTINF:5,\nseg-2.ts\n" +
"#EXT-X-ENDLIST\n"))
})
mux.HandleFunc("/key-1.bin", func(w http.ResponseWriter, r *http.Request) {
w.Write(key1)
})
mux.HandleFunc("/key-2.bin", func(w http.ResponseWriter, r *http.Request) {
w.Write(key2)
})
mux.HandleFunc("/seg-1.ts", func(w http.ResponseWriter, r *http.Request) {
w.Write(enc1)
})
mux.HandleFunc("/seg-2.ts", func(w http.ResponseWriter, r *http.Request) {
w.Write(enc2)
})
server := httptest.NewServer(mux)
defer server.Close()
f := newTestFetcher(t, testConfig())
mustResolve(t, f, server.URL+"/video.m3u8", "")
// Ciphertext sizes overstate the final output after decryption, so an
// encrypted plan must report an indeterminate size.
if f.meta.Res.Size != 0 {
t.Errorf("encrypted plan should report size 0, got %d", f.meta.Res.Size)
}
mustRun(t, f)
want := append(append([]byte{}, plain1...), plain2...)
if !bytes.Equal(readOutput(t, f), want) {
t.Errorf("decrypted output mismatch: %q", readOutput(t, f))
}
}
func TestFetcherE2E_ByteRange(t *testing.T) {
data := []byte("0123456789ABCDEFGHIJ") // 20 bytes
mux := http.NewServeMux()
mux.HandleFunc("/video.m3u8", func(w http.ResponseWriter, r *http.Request) {
w.Write([]byte("#EXTM3U\n" +
"#EXT-X-BYTERANGE:10@0\nblob.bin\n" +
"#EXT-X-BYTERANGE:10\nblob.bin\n" +
"#EXT-X-ENDLIST\n"))
})
mux.HandleFunc("/blob.bin", func(w http.ResponseWriter, r *http.Request) {
rng := r.Header.Get("Range")
if rng == "" {
w.Write(data)
return
}
// bytes=start-end
var start, end int
fmt.Sscanf(rng, "bytes=%d-%d", &start, &end)
w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, len(data)))
w.WriteHeader(http.StatusPartialContent)
w.Write(data[start : end+1])
})
server := httptest.NewServer(mux)
defer server.Close()
f := newTestFetcher(t, testConfig())
mustResolve(t, f, server.URL+"/video.m3u8", "")
if f.meta.Res.Size != 20 {
t.Fatalf("byterange sizes should be known from the playlist, got %d", f.meta.Res.Size)
}
mustRun(t, f)
if !bytes.Equal(readOutput(t, f), data) {
t.Errorf("byterange merge mismatch: %q", readOutput(t, f))
}
}
func TestFetcherE2E_PauseResume(t *testing.T) {
const total = 8
var served atomic.Int64
mux := http.NewServeMux()
mux.HandleFunc("/video.m3u8", func(w http.ResponseWriter, r *http.Request) {
var b strings.Builder
b.WriteString("#EXTM3U\n")
for i := 0; i < total; i++ {
b.WriteString(fmt.Sprintf("#EXTINF:4,\nseg-%d.ts\n", i))
}
b.WriteString("#EXT-X-ENDLIST\n")
w.Write([]byte(b.String()))
})
segments := make(map[int][]byte)
for i := 0; i < total; i++ {
segments[i] = []byte(fmt.Sprintf("SLOW-SEGMENT-%02d", i))
}
for i := 0; i < total; i++ {
idx := i
mux.HandleFunc(fmt.Sprintf("/seg-%d.ts", i), func(w http.ResponseWriter, r *http.Request) {
time.Sleep(150 * time.Millisecond)
served.Add(1)
w.Write(segments[idx])
})
}
server := httptest.NewServer(mux)
defer server.Close()
cfg := testConfig()
cfg.SegmentConnections = 2
cfg.PrefetchContentLength = false
f := newTestFetcher(t, cfg)
mustResolve(t, f, server.URL+"/video.m3u8", "")
if err := f.Start(); err != nil {
t.Fatal(err)
}
// Let a couple of segments finish, then pause.
time.Sleep(400 * time.Millisecond)
if err := f.Pause(); err != nil {
t.Fatal(err)
}
doneBefore := statsOf(t, f).SegmentsDone
if doneBefore == 0 {
t.Fatal("expected some segments to complete before pause")
}
if doneBefore == total {
t.Fatal("all segments finished before pause, test is not exercising resume")
}
// Resume must finish the rest and produce the full output.
if err := f.Start(); err != nil {
t.Fatal(err)
}
if err := f.Wait(); err != nil {
t.Fatal(err)
}
var want []byte
for i := 0; i < total; i++ {
want = append(want, segments[i]...)
}
if !bytes.Equal(readOutput(t, f), want) {
t.Errorf("resumed output mismatch: %q", readOutput(t, f))
}
}
func TestFetcherE2E_StoreRestoreResume(t *testing.T) {
const total = 6
var served atomic.Int64
mux := http.NewServeMux()
mux.HandleFunc("/video.m3u8", func(w http.ResponseWriter, r *http.Request) {
var b strings.Builder
b.WriteString("#EXTM3U\n")
for i := 0; i < total; i++ {
b.WriteString(fmt.Sprintf("#EXTINF:4,\nseg-%d.ts\n", i))
}
b.WriteString("#EXT-X-ENDLIST\n")
w.Write([]byte(b.String()))
})
segments := make(map[int][]byte)
for i := 0; i < total; i++ {
segments[i] = []byte(fmt.Sprintf("RESTORE-SEGMENT-%02d", i))
}
for i := 0; i < total; i++ {
idx := i
mux.HandleFunc(fmt.Sprintf("/seg-%d.ts", i), func(w http.ResponseWriter, r *http.Request) {
time.Sleep(120 * time.Millisecond)
served.Add(1)
w.Write(segments[idx])
})
}
server := httptest.NewServer(mux)
defer server.Close()
cfg := testConfig()
cfg.SegmentConnections = 2
cfg.PrefetchContentLength = false
f := newTestFetcher(t, cfg)
mustResolve(t, f, server.URL+"/video.m3u8", "")
if err := f.Start(); err != nil {
t.Fatal(err)
}
time.Sleep(350 * time.Millisecond)
if err := f.Pause(); err != nil {
t.Fatal(err)
}
doneBefore := statsOf(t, f).SegmentsDone
if doneBefore == 0 {
t.Fatal("expected some segments to finish before simulated restart")
}
// Simulate an app restart: persist, then restore into a fresh fetcher.
fm := &FetcherManager{}
state, err := fm.Store(f)
if err != nil {
t.Fatal(err)
}
if state == nil {
t.Fatal("state should be persisted")
}
ctl := controller.NewController()
ctl.GetConfig = func(v any) {
if c, ok := v.(*config); ok {
*c = cfg
}
}
holder := fm.Build().(*Fetcher)
holder.Setup(ctl)
holder.meta = &fetcher.FetcherMeta{
Req: f.meta.Req,
Opts: f.meta.Opts,
Res: f.meta.Res,
}
_, builder := fm.Restore()
rf := builder(holder.meta, state).(*Fetcher)
// The engine calls Setup on the builder result (restoreFetcher).
rf.Setup(ctl)
if rf.state == nil || len(rf.state.Segments) != total {
t.Fatal("restored state missing segment plan")
}
if err := rf.Start(); err != nil {
t.Fatal(err)
}
if err := rf.Wait(); err != nil {
t.Fatal(err)
}
if done := statsOf(t, rf).SegmentsDone; done != total {
t.Errorf("want %d done, got %d", total, done)
}
var want []byte
for i := 0; i < total; i++ {
want = append(want, segments[i]...)
}
if !bytes.Equal(readOutput(t, rf), want) {
t.Errorf("restored output mismatch: %q", readOutput(t, rf))
}
}
func hexEncode(b []byte) string {
const digits = "0123456789abcdef"
out := make([]byte, 0, len(b)*2)
for _, v := range b {
out = append(out, digits[v>>4], digits[v&0x0f])
}
return string(out)
}
// restoreLikeEngine round-trips the persisted state and the task meta through
// JSON the way the engine storage does, then builds a fresh fetcher. This is
// the path that historically lost the request extra headers.
func restoreLikeEngine(t *testing.T, f *Fetcher, cfg config) *Fetcher {
t.Helper()
fm := &FetcherManager{}
stateAny, err := fm.Store(f)
if err != nil {
t.Fatal(err)
}
rawState, err := json.Marshal(stateAny)
if err != nil {
t.Fatal(err)
}
storedState := &fetcherState{}
if err := json.Unmarshal(rawState, storedState); err != nil {
t.Fatal(err)
}
rawMeta, err := json.Marshal(f.meta)
if err != nil {
t.Fatal(err)
}
restoredMeta := &fetcher.FetcherMeta{}
if err := json.Unmarshal(rawMeta, restoredMeta); err != nil {
t.Fatal(err)
}
if _, ok := restoredMeta.Req.Extra.(*ReqExtra); ok {
t.Fatal("storage round trip should yield a map-shaped extra, like the engine")
}
ctl := controller.NewController()
ctl.GetConfig = func(v any) {
if c, ok := v.(*config); ok {
*c = cfg
}
}
_, builder := fm.Restore()
rf := builder(restoredMeta, storedState).(*Fetcher)
// The engine calls Setup on the builder result (restoreFetcher).
rf.Setup(ctl)
return rf
}
func TestFetcherE2E_StoreRestoreAuthenticated(t *testing.T) {
const total = 4
page := "https://player.example.com/watch/42"
var served atomic.Int64
mux := http.NewServeMux()
var playlist strings.Builder
playlist.WriteString("#EXTM3U\n")
for i := 0; i < total; i++ {
playlist.WriteString(fmt.Sprintf("#EXTINF:4,\nseg-%d.ts\n", i))
}
playlist.WriteString("#EXT-X-ENDLIST\n")
mux.HandleFunc("/video.m3u8", func(w http.ResponseWriter, r *http.Request) {
w.Write([]byte(playlist.String()))
})
segments := make(map[int][]byte)
for i := 0; i < total; i++ {
segments[i] = []byte(fmt.Sprintf("AUTH-SEGMENT-%02d", i))
idx := i
mux.HandleFunc(fmt.Sprintf("/seg-%d.ts", idx), func(w http.ResponseWriter, r *http.Request) {
time.Sleep(80 * time.Millisecond)
served.Add(1)
w.Write(segments[idx])
})
}
server := httptest.NewServer(&cdnHandler{mux: mux, requireReferer: page})
defer server.Close()
cfg := testConfig()
cfg.SegmentConnections = 2
cfg.PrefetchContentLength = false
f := newTestFetcher(t, cfg)
req := &base.Request{
URL: server.URL + "/video.m3u8",
Extra: &ReqExtra{Header: map[string]string{"Referer": page}},
}
if err := f.Resolve(req, &base.Options{Path: t.TempDir()}); err != nil {
t.Fatal(err)
}
if err := f.Start(); err != nil {
t.Fatal(err)
}
time.Sleep(150 * time.Millisecond)
if err := f.Pause(); err != nil {
t.Fatal(err)
}
rf := restoreLikeEngine(t, f, cfg)
if rf.extra == nil || rf.extra.Header["Referer"] != page {
t.Fatalf("restored fetcher lost the request extra headers: %+v", rf.extra)
}
if err := rf.Start(); err != nil {
t.Fatal(err)
}
if err := rf.Wait(); err != nil {
t.Fatalf("resumed download failed (request headers lost?): %v", err)
}
var want []byte
for i := 0; i < total; i++ {
want = append(want, segments[i]...)
}
if !bytes.Equal(readOutput(t, rf), want) {
t.Errorf("restored output mismatch: %q", readOutput(t, rf))
}
}
func TestFetcherE2E_PatchPartialUpdate(t *testing.T) {
page := "https://player.example.com/watch/7"
var requiredReferer atomic.Value // string
requiredReferer.Store(page)
var playlist strings.Builder
playlist.WriteString("#EXTM3U\n")
segments := make(map[string][]byte)
for i := 0; i < 3; i++ {
segments[fmt.Sprintf("/s%d.ts", i)] = []byte(fmt.Sprintf("PATCH-%d", i))
playlist.WriteString(fmt.Sprintf("#EXTINF:4,\ns%d.ts\n", i))
}
playlist.WriteString("#EXT-X-ENDLIST\n")
mux := http.NewServeMux()
mux.HandleFunc("/video.m3u8", func(w http.ResponseWriter, r *http.Request) {
w.Write([]byte(playlist.String()))
})
for path, content := range segments {
c := content
mux.HandleFunc(path, func(w http.ResponseWriter, r *http.Request) {
w.Write(c)
})
}
// The Referer requirement can be flipped mid-test to prove that a patched
// header is actually sent by the next run.
dyn := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if want, _ := requiredReferer.Load().(string); want != "" && r.Header.Get("Referer") != want {
w.WriteHeader(http.StatusForbidden)
return
}
mux.ServeHTTP(w, r)
})
server := httptest.NewServer(dyn)
defer server.Close()
f := newTestFetcher(t, testConfig())
req := &base.Request{
URL: server.URL + "/video.m3u8",
Extra: &ReqExtra{Header: map[string]string{"Referer": page}},
}
opts := &base.Options{Path: t.TempDir()}
if err := f.Resolve(req, opts); err != nil {
t.Fatal(err)
}
// A nil/nil patch must be a no-op, not a panic.
if err := f.Patch(nil, nil); err != nil {
t.Fatalf("nil patch should be accepted: %v", err)
}
// Header-only patch: the resolved plan and output options are retained
// while the new header takes effect on the next run.
newReferer := "https://player.example.com/watch/9"
if err := f.Patch(&base.Request{
URL: server.URL + "/video.m3u8",
Extra: &ReqExtra{Header: map[string]string{"Referer": newReferer}},
}, nil); err != nil {
t.Fatal(err)
}
if f.state == nil || f.meta.Res == nil {
t.Fatal("header-only patch must keep the resolved plan")
}
if f.meta.Opts.Path != opts.Path {
t.Fatalf("header-only patch must keep the output path: %s", f.meta.Opts.Path)
}
if f.extra.Header["Referer"] != newReferer {
t.Fatalf("patched header not applied: %+v", f.extra.Header)
}
requiredReferer.Store(newReferer)
if err := f.Start(); err != nil {
t.Fatal(err)
}
if err := f.Wait(); err != nil {
t.Fatalf("run with patched header failed: %v", err)
}
if got := readOutput(t, f); string(got) != "PATCH-0PATCH-1PATCH-2" {
t.Errorf("output mismatch after header patch: %q", got)
}
// Options-only patch: renaming the task keeps the resolved plan.
if err := f.Patch(nil, &base.Options{Name: "renamed.ts"}); err != nil {
t.Fatal(err)
}
if f.state == nil {
t.Fatal("options-only patch must keep the resolved plan")
}
if f.meta.Opts.Name != "renamed.ts" || f.meta.Opts.Path != opts.Path {
t.Fatalf("options patch misapplied: %+v", f.meta.Opts)
}
// URL-only patch: the plan is dropped for a re-resolve but the output
// path and renamed file survive.
if err := f.Patch(&base.Request{URL: server.URL + "/refreshed.m3u8"}, nil); err != nil {
t.Fatal(err)
}
if f.state != nil || f.meta.Res != nil {
t.Fatal("URL patch must drop the resolved plan")
}
if f.meta.Opts.Path != opts.Path || f.meta.Opts.Name != "renamed.ts" {
t.Fatal("URL patch must keep the output options")
}
}
func TestFetcherE2E_PatchWhileRunningRejected(t *testing.T) {
mux := http.NewServeMux()
var playlist strings.Builder
playlist.WriteString("#EXTM3U\n")
for i := 0; i < 6; i++ {
playlist.WriteString(fmt.Sprintf("#EXTINF:4,\nseg-%d.ts\n", i))
}
playlist.WriteString("#EXT-X-ENDLIST\n")
mux.HandleFunc("/video.m3u8", func(w http.ResponseWriter, r *http.Request) {
w.Write([]byte(playlist.String()))
})
for i := 0; i < 6; i++ {
mux.HandleFunc(fmt.Sprintf("/seg-%d.ts", i), func(w http.ResponseWriter, r *http.Request) {
time.Sleep(150 * time.Millisecond)
w.Write([]byte("X"))
})
}
server := httptest.NewServer(mux)
defer server.Close()
f := newTestFetcher(t, testConfig())
if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: t.TempDir()}); err != nil {
t.Fatal(err)
}
if err := f.Start(); err != nil {
t.Fatal(err)
}
if err := f.Patch(&base.Request{URL: server.URL + "/other.m3u8"}, nil); err == nil {
t.Fatal("patching a running task must be rejected")
}
if err := f.Pause(); err != nil {
t.Fatal(err)
}
// After the pause the patch goes through and only then drops the plan.
if err := f.Patch(&base.Request{URL: server.URL + "/other.m3u8"}, nil); err != nil {
t.Fatal(err)
}
if f.state != nil {
t.Fatal("URL patch after pause must drop the plan")
}
}
func TestFetcherE2E_RetryAfterFailure(t *testing.T) {
const total = 4
var healthy atomic.Bool
mux := http.NewServeMux()
var playlist strings.Builder
playlist.WriteString("#EXTM3U\n")
for i := 0; i < total; i++ {
playlist.WriteString(fmt.Sprintf("#EXTINF:4,\nseg-%d.ts\n", i))
}
playlist.WriteString("#EXT-X-ENDLIST\n")
mux.HandleFunc("/video.m3u8", func(w http.ResponseWriter, r *http.Request) {
w.Write([]byte(playlist.String()))
})
segments := make(map[int][]byte)
for i := 0; i < total; i++ {
segments[i] = []byte(fmt.Sprintf("RETRY-SEGMENT-%02d", i))
idx := i
mux.HandleFunc(fmt.Sprintf("/seg-%d.ts", idx), func(w http.ResponseWriter, r *http.Request) {
if idx == 1 && !healthy.Load() {
w.WriteHeader(http.StatusInternalServerError)
return
}
time.Sleep(60 * time.Millisecond)
w.Write(segments[idx])
})
}
server := httptest.NewServer(mux)
defer server.Close()
cfg := testConfig()
cfg.MaxRetries = 0 // fail the run on the first bad segment
cfg.PrefetchContentLength = false
f := newTestFetcher(t, cfg)
if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: t.TempDir()}); err != nil {
t.Fatal(err)
}
if err := f.Start(); err != nil {
t.Fatal(err)
}
if err := f.Wait(); err == nil {
t.Fatal("first run should fail while the segment endpoint is down")
}
if got := statsOf(t, f).SegmentsFailed; got == 0 {
t.Fatal("failed segments should be reported in stats")
}
// Server recovered: a fresh run must succeed and report zero failures.
healthy.Store(true)
if err := f.Start(); err != nil {
t.Fatal(err)
}
if err := f.Wait(); err != nil {
t.Fatalf("retry run failed: %v", err)
}
stats := statsOf(t, f)
if stats.SegmentsFailed != 0 {
t.Errorf("failures must reset per run, got %d", stats.SegmentsFailed)
}
if stats.SegmentsDone != total {
t.Errorf("want %d done, got %d", total, stats.SegmentsDone)
}
var want []byte
for i := 0; i < total; i++ {
want = append(want, segments[i]...)
}
if !bytes.Equal(readOutput(t, f), want) {
t.Errorf("retry output mismatch: %q", readOutput(t, f))
}
}
func TestFetcherE2E_StateMissingSelfHeal(t *testing.T) {
const total = 3
mux := http.NewServeMux()
var playlist strings.Builder
playlist.WriteString("#EXTM3U\n")
for i := 0; i < total; i++ {
playlist.WriteString(fmt.Sprintf("#EXTINF:4,\nseg-%d.ts\n", i))
}
playlist.WriteString("#EXT-X-ENDLIST\n")
mux.HandleFunc("/video.m3u8", func(w http.ResponseWriter, r *http.Request) {
w.Write([]byte(playlist.String()))
})
segments := make(map[int][]byte)
for i := 0; i < total; i++ {
segments[i] = []byte(fmt.Sprintf("SELFHEAL-%02d", i))
idx := i
mux.HandleFunc(fmt.Sprintf("/seg-%d.ts", idx), func(w http.ResponseWriter, r *http.Request) {
w.Write(segments[idx])
})
}
server := httptest.NewServer(mux)
defer server.Close()
f := newTestFetcher(t, testConfig())
if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: t.TempDir()}); err != nil {
t.Fatal(err)
}
// Simulate the crash window where the engine persisted the task with its
// resource but the HLS plan was never checkpointed.
f.state = nil
outputName := f.meta.Res.Files[0].Name
if err := f.Start(); err != nil {
t.Fatalf("start should self-heal a missing plan: %v", err)
}
if err := f.Wait(); err != nil {
t.Fatalf("self-healed run failed: %v", err)
}
if f.state == nil || len(f.state.Segments) != total {
t.Fatal("self-heal should rebuild the segment plan")
}
if f.meta.Res.Files[0].Name != outputName {
t.Fatalf("self-heal renamed the output: %s", f.meta.Res.Files[0].Name)
}
var want []byte
for i := 0; i < total; i++ {
want = append(want, segments[i]...)
}
if !bytes.Equal(readOutput(t, f), want) {
t.Errorf("self-healed output mismatch: %q", readOutput(t, f))
}
}
+100
View File
@@ -0,0 +1,100 @@
package hls
import (
"context"
"crypto/tls"
"fmt"
"io"
"net"
"net/http"
"net/http/cookiejar"
"time"
"github.com/GopeedLab/gopeed/internal/httpclient"
"github.com/GopeedLab/gopeed/pkg/base"
)
const defaultUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36"
const maxPlaylistSize = 10 << 20 // 10MB, playlists are small text files
func (f *Fetcher) timeout() time.Duration {
return time.Duration(f.config.TimeoutSeconds) * time.Second
}
// buildClient builds an HTTP client honoring the task proxy and TLS settings.
// The client has no total timeout on purpose: large segments over slow links
// must not be aborted; body stalls are handled by stallReader instead. It is
// only called while the fetcher is quiescent (Resolve/Start), never from
// inside running worker goroutines.
func (f *Fetcher) buildClient() *http.Client {
var proxy *base.RequestProxy
skipVerify := false
if req := f.meta.Req; req != nil {
proxy = req.Proxy
skipVerify = req.SkipVerifyCert
}
jar, _ := cookiejar.New(nil)
client, err := httpclient.NewClient(httpclient.Options{
Client: httpclient.ClientOptions{
Jar: jar,
},
Transport: httpclient.TransportOptions{
DialContext: (&net.Dialer{
Timeout: f.timeout(),
}).DialContext,
Proxy: f.Ctl.GetProxy(proxy),
TLSClientConfig: &tls.Config{
InsecureSkipVerify: skipVerify,
},
TLSHandshakeTimeout: f.timeout(),
},
Impersonation: httpclient.ImpersonationOptions{
Mode: httpclient.ImpersonationAuto,
Session: f.impSession,
},
})
if err != nil {
panic(fmt.Sprintf("build HTTP client: %v", err))
}
return client
}
// buildRequest builds a request with task extra headers, the default user
// agent and an optional byte range. extra may be nil.
func buildRequest(ctx context.Context, extra *ReqExtra, method, u string, body io.Reader, byterange *ByteRange) (*http.Request, error) {
req, err := http.NewRequestWithContext(ctx, method, u, body)
if err != nil {
return nil, err
}
if extra != nil {
for key, value := range extra.Header {
req.Header.Set(key, value)
}
}
if req.Header.Get("User-Agent") == "" {
req.Header.Set("User-Agent", defaultUserAgent)
}
if byterange != nil {
req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", byterange.Offset, byterange.Offset+byterange.Length-1))
}
return req, nil
}
// stallReader aborts reads that see no data for the configured timeout.
type stallReader struct {
rc io.ReadCloser
timeout time.Duration
}
func (r *stallReader) Read(p []byte) (int, error) {
timer := time.AfterFunc(r.timeout, func() {
r.rc.Close()
})
defer timer.Stop()
return r.rc.Read(p)
}
func (r *stallReader) Close() error {
return r.rc.Close()
}
+112
View File
@@ -0,0 +1,112 @@
package hls
import (
"crypto/rand"
"crypto/sha1"
"encoding/hex"
"encoding/json"
"fmt"
"io"
"os"
"strconv"
)
// journalFile is the persisted completed-segment ledger. It is bound to the
// exact download plan: a journal whose fingerprint does not match the current
// plan is discarded instead of being applied to the wrong segments.
type journalFile struct {
StagingID string `json:"stagingID"`
PlanHash string `json:"planHash"`
SegmentCount int `json:"segmentCount"`
Sizes map[string]int64 `json:"sizes"`
}
// randomStagingID returns a random identifier used to give every task its own
// staging folder, stable across app restarts through the persisted state.
func randomStagingID() string {
buf := make([]byte, 6)
if _, err := rand.Read(buf); err != nil {
// crypto/rand failing is catastrophic anyway; fall back to address
// entropy rather than crashing the fetch.
return fmt.Sprintf("%x", sha1.Sum([]byte(fmt.Sprintf("%p", buf))))[:12]
}
return hex.EncodeToString(buf)
}
// planFingerprint hashes everything that defines the download plan: the media
// URL, the ordered segments with their sequence numbers, byte ranges, keys
// and init markers. Two tasks resolve to different fingerprints even when
// they point at the same playlist URL.
func planFingerprint(state *fetcherState) string {
h := sha1.New()
io.WriteString(h, state.MediaURL+"\n")
io.WriteString(h, strconv.Itoa(len(state.Segments))+"\n")
for _, seg := range state.Segments {
fmt.Fprintf(h, "%d|%s|", seg.Sequence, seg.URI)
if seg.Byterange != nil {
fmt.Fprintf(h, "%d@%d|", seg.Byterange.Offset, seg.Byterange.Length)
}
if seg.Key != nil {
// The IV is part of the plan: same URI with a rotated IV would
// silently decrypt to garbage if the journal were reused.
fmt.Fprintf(h, "key=%s iv=%x|", seg.Key.URI, seg.Key.IV)
}
if seg.IsInit {
io.WriteString(h, "init|")
}
io.WriteString(h, "\n")
}
return hex.EncodeToString(h.Sum(nil))[:16]
}
// loadJournal reads the journal of a run, discarding anything that cannot be
// proven to belong to the current plan: foreign or legacy journals, corrupted
// files, and entries whose recorded size no longer matches the plan.
func loadJournal(run *hlsRun) error {
data, err := os.ReadFile(run.journalPath)
if err != nil {
if os.IsNotExist(err) {
return nil
}
return err
}
var file journalFile
if err := json.Unmarshal(data, &file); err != nil {
// A broken or legacy journal must not kill a restartable download.
return nil
}
if file.StagingID != run.stagingID || file.PlanHash != run.planHash || file.SegmentCount != len(run.segs) {
return nil
}
for key, size := range file.Sizes {
idx, err := strconv.ParseInt(key, 10, 64)
if err != nil || idx < 0 || idx >= int64(len(run.segs)) || size <= 0 {
continue
}
run.journal[idx] = size
}
return nil
}
// saveJournal atomically rewrites the journal through a temp file + rename so
// a crash never leaves a half-written ledger behind.
func saveJournal(run *hlsRun) error {
file := journalFile{
StagingID: run.stagingID,
PlanHash: run.planHash,
SegmentCount: len(run.segs),
Sizes: make(map[string]int64, len(run.journal)),
}
for idx, size := range run.journal {
file.Sizes[strconv.FormatInt(idx, 10)] = size
}
data, err := json.Marshal(file)
if err != nil {
return err
}
tmp := run.journalPath + ".tmp"
if err := os.WriteFile(tmp, data, 0666); err != nil {
return err
}
return os.Rename(tmp, run.journalPath)
}
@@ -0,0 +1,297 @@
package hls
import (
"bytes"
"fmt"
"net/http"
"net/http/httptest"
"os"
"testing"
"github.com/GopeedLab/gopeed/pkg/base"
)
// The validators below check real media structure, not just byte
// concatenation: TS packets are verified on the 188 byte grid with
// continuity-counter rules, PAT/PMT presence and PES start codes; fMP4
// output is verified by walking the ISO-BMFF box hierarchy.
func be32(b []byte) int {
return int(b[0])<<24 | int(b[1])<<16 | int(b[2])<<8 | int(b[3])
}
// validateTS asserts structural validity of a merged MPEG-2 TS stream.
func validateTS(t *testing.T, data []byte) {
t.Helper()
if len(data) == 0 || len(data)%188 != 0 {
t.Fatalf("TS size %d is not a positive multiple of 188", len(data))
}
type pidState struct{ cc int }
ccByPID := map[int]int{}
patSeen, pmtSeen, pesSeen := false, false, false
for off := 0; off < len(data); off += 188 {
pkt := data[off : off+188]
if pkt[0] != 0x47 {
t.Fatalf("bad TS sync byte at offset %d", off)
}
pid := (int(pkt[1]&0x1F) << 8) | int(pkt[2])
pusi := pkt[1]&0x40 != 0
afc := (pkt[3] >> 4) & 0x3
if afc == 0 {
continue // reserved
}
idx := 4
if afc&0x2 != 0 { // adaptation field present
afLen := int(pkt[4])
if afLen > 0 && pkt[5]&0x80 != 0 {
// discontinuity_indicator: restart the CC expectation
delete(ccByPID, pid)
}
idx += 1 + afLen
if afc == 2 {
continue // adaptation only, no payload: no CC rule
}
}
if idx > 188 {
t.Fatalf("adaptation field overruns packet at offset %d", off)
}
if pid == 0x1FFF {
continue // null packet
}
payload := pkt[idx:]
if pusi && len(payload) >= 3 && pid != 0 && payload[0] == 0 && payload[1] == 0 && payload[2] == 1 {
pesSeen = true
}
if pid == 0 {
if ptr := int(payload[0]); 1+ptr < len(payload) && payload[1+ptr] == 0x00 {
patSeen = true // program_association_section table_id
}
} else if pusi && len(payload) >= 2 && payload[0] == 0 && payload[1] == 0x02 {
pmtSeen = true // program_map_section table_id
}
cc := int(pkt[3] & 0x0F)
prev, ok := ccByPID[pid]
if !ok {
ccByPID[pid] = cc
continue
}
// CC increments by one per payload-bearing packet; a repeated value
// models the single legal duplicate packet.
if cc != (prev+1)%16 && cc != prev {
t.Fatalf("continuity error on PID %d: %d after %d at offset %d", pid, cc, prev, off)
}
ccByPID[pid] = cc
}
if !patSeen {
t.Error("no PAT found in merged TS")
}
if !pmtSeen {
t.Error("no PMT found in merged TS")
}
if !pesSeen {
t.Error("no PES start code found in merged TS")
}
}
// walkBoxes iterates the top-level ISO-BMFF boxes of data[off:end].
func walkBoxes(t *testing.T, data []byte, off, end int) []string {
t.Helper()
var types []string
for off+8 <= end {
size := be32(data[off:])
typ := string(data[off+4 : off+8])
if size < 8 || off+size > end {
t.Fatalf("invalid box %q at offset %d (size %d)", typ, off, size)
}
types = append(types, typ)
off += size
}
return types
}
// validateFMP4 asserts the merged fMP4 output is init section followed by
// well-formed moof+mdat fragments.
func validateFMP4(t *testing.T, merged []byte, wantFragments int) {
t.Helper()
types := walkBoxes(t, merged, 0, len(merged))
hasFtyp, hasMoov := false, false
fragments := 0
expectFragment := false
for i, typ := range types {
switch typ {
case "ftyp":
if i != 0 {
t.Errorf("ftyp not at stream start")
}
hasFtyp = true
case "moov":
if i != 1 {
t.Errorf("moov must follow ftyp, found at index %d", i)
}
hasMoov = true
case "styp":
expectFragment = true
case "moof":
if !hasMoov {
t.Errorf("fragment before moov")
}
expectFragment = true
case "mdat":
if !expectFragment {
t.Errorf("mdat outside of a fragment at index %d", i)
}
expectFragment = false
fragments++
case "mfra":
// optional trailer
default:
t.Errorf("unexpected top-level box %q", typ)
}
}
if !hasFtyp || !hasMoov {
t.Error("merged fMP4 missing ftyp/moov init section")
}
if fragments != wantFragments {
t.Errorf("want %d fragments in merged fMP4, got %d", wantFragments, fragments)
}
}
func mediaFixture(t *testing.T, name string) []byte {
t.Helper()
data, err := os.ReadFile("testdata/media/" + name)
if err != nil {
t.Fatal(err)
}
return data
}
func serveFixtureServer(t *testing.T) *httptest.Server {
t.Helper()
mux := http.NewServeMux()
fs := http.FileServer(http.Dir("testdata/media"))
mux.Handle("/", fs)
return httptest.NewServer(mux)
}
func TestHLSRealMedia_TS(t *testing.T) {
seg0 := mediaFixture(t, "sample-00.ts")
seg1 := mediaFixture(t, "sample-01.ts")
playlist := "#EXTM3U\n" +
"#EXTINF:0.5,\nsample-00.ts\n" +
"#EXTINF:0.5,\nsample-01.ts\n" +
"#EXT-X-ENDLIST\n"
mux := http.NewServeMux()
mux.HandleFunc("/video.m3u8", func(w http.ResponseWriter, r *http.Request) {
w.Write([]byte(playlist))
})
mux.Handle("/", http.FileServer(http.Dir("testdata/media")))
server := httptest.NewServer(mux)
defer server.Close()
f := newTestFetcher(t, testConfig())
if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: t.TempDir()}); err != nil {
t.Fatal(err)
}
if err := f.Start(); err != nil {
t.Fatal(err)
}
if err := f.Wait(); err != nil {
t.Fatal(err)
}
got := readOutput(t, f)
if !bytes.Equal(got, append(append([]byte{}, seg0...), seg1...)) {
t.Fatal("merged TS is not the exact concatenation of the segments")
}
validateTS(t, got)
}
func TestHLSRealMedia_FMP4(t *testing.T) {
init := mediaFixture(t, "fmp4-init.mp4")
frag1 := mediaFixture(t, "fmp4-seg1.m4s")
frag2 := mediaFixture(t, "fmp4-seg2.m4s")
frag3 := mediaFixture(t, "fmp4-seg3.m4s")
playlist := "#EXTM3U\n" +
"#EXT-X-MAP:URI=\"fmp4-init.mp4\"\n" +
"#EXTINF:0.5,\nfmp4-seg1.m4s\n" +
"#EXTINF:0.5,\nfmp4-seg2.m4s\n" +
"#EXTINF:0.5,\nfmp4-seg3.m4s\n" +
"#EXT-X-ENDLIST\n"
mux := http.NewServeMux()
mux.HandleFunc("/video.m3u8", func(w http.ResponseWriter, r *http.Request) {
w.Write([]byte(playlist))
})
mux.Handle("/", http.FileServer(http.Dir("testdata/media")))
server := httptest.NewServer(mux)
defer server.Close()
f := newTestFetcher(t, testConfig())
if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: t.TempDir()}); err != nil {
t.Fatal(err)
}
if err := f.Start(); err != nil {
t.Fatal(err)
}
if err := f.Wait(); err != nil {
t.Fatal(err)
}
got := readOutput(t, f)
want := append(append(append(append([]byte{}, init...), frag1...), frag2...), frag3...)
if !bytes.Equal(got, want) {
t.Fatal("merged fMP4 is not the exact concatenation of init + fragments")
}
validateFMP4(t, got, 3)
}
func TestHLSRealMedia_AES128(t *testing.T) {
keys := [][]byte{[]byte("key-A-0123456789"), []byte("key-B-9876543210")}
iv := []byte("iv-1337-01234567")
plain := [][]byte{mediaFixture(t, "sample-00.ts"), mediaFixture(t, "sample-01.ts")}
playlist := "#EXTM3U\n" +
"#EXT-X-MEDIA-SEQUENCE:0\n" +
"#EXT-X-KEY:METHOD=AES-128,URI=\"key-0.bin\",IV=0x" + hexEncode(iv) + "\n" +
"#EXTINF:0.5,\nenc-0.ts\n" +
"#EXT-X-KEY:METHOD=AES-128,URI=\"key-1.bin\"\n" +
"#EXTINF:0.5,\nenc-1.ts\n" +
"#EXT-X-ENDLIST\n"
mux := http.NewServeMux()
mux.HandleFunc("/video.m3u8", func(w http.ResponseWriter, r *http.Request) {
w.Write([]byte(playlist))
})
for i, key := range keys {
k := key
mux.HandleFunc(fmt.Sprintf("/key-%d.bin", i), func(w http.ResponseWriter, r *http.Request) {
w.Write(k)
})
// Segment 0 has an explicit IV, segment 1 derives it from its media
// sequence number (1).
enc := encryptSegment(plain[i], k, map[bool][]byte{true: iv, false: sequenceIV(int64(i))}[i == 0])
mux.HandleFunc(fmt.Sprintf("/enc-%d.ts", i), func(w http.ResponseWriter, r *http.Request) {
w.Write(enc)
})
}
server := httptest.NewServer(mux)
defer server.Close()
f := newTestFetcher(t, testConfig())
if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: t.TempDir()}); err != nil {
t.Fatal(err)
}
if f.meta.Res.Size != 0 {
t.Errorf("encrypted plan must report indeterminate size, got %d", f.meta.Res.Size)
}
if err := f.Start(); err != nil {
t.Fatal(err)
}
if err := f.Wait(); err != nil {
t.Fatal(err)
}
got := readOutput(t, f)
if !bytes.Equal(got, append(append([]byte{}, plain[0]...), plain[1]...)) {
t.Fatal("decrypted output mismatch")
}
validateTS(t, got)
if p := f.Progress(); p.TotalDownloaded() != int64(len(got)) {
t.Errorf("final progress %d != merged size %d", p.TotalDownloaded(), len(got))
}
}
+479
View File
@@ -0,0 +1,479 @@
package hls
import (
"encoding/hex"
"errors"
"fmt"
"net/url"
"path"
"strconv"
"strings"
)
// Variant is one rendition entry of a master playlist.
type Variant struct {
URI string
Bandwidth int64
AvgBandwidth int64
Resolution string
Codecs string
Name string
AudioGroup string // AUDIO attribute: group id of the audio renditions
}
// MediaRendition is one EXT-X-MEDIA entry of a master playlist. Only the
// fields needed to detect unsupported track layouts are kept.
type MediaRendition struct {
Type string
GroupID string
URI string
}
// MasterPlaylist is a parsed master playlist with its rendition groups.
type MasterPlaylist struct {
Variants []*Variant
Rendition map[string][]*MediaRendition // group id -> all EXT-X-MEDIA members
}
// Key is the decryption key applied to the segments that follow its tag, until
// the next EXT-X-KEY tag or the end of the playlist.
type Key struct {
Method string
URI string // absolute URI
IV []byte // explicit IV; nil means derive from the media sequence number
}
// ByteRange is an EXT-X-BYTERANGE sub range of the segment resource.
type ByteRange struct {
Length int64
Offset int64
}
// Segment is one downloadable piece of a media playlist. The init section of
// an fMP4 stream is represented as a segment with IsInit set.
type Segment struct {
URI string
Sequence int64 // media sequence number, used to derive the default AES IV
Byterange *ByteRange
Key *Key // nil means unencrypted
IsInit bool
Gap bool
Duration float64
Size int64 // exact size in bytes when known (byterange / HEAD probe), 0 when unknown
}
// Media is a parsed media playlist.
type Media struct {
Segments []*Segment
Version int
Live bool // no EXT-X-ENDLIST tag
IsFMP4 bool // playlist declares an init section via EXT-X-MAP
Discontinuity bool // playlist contains EXT-X-DISCONTINUITY
}
const (
keyMethodNone = "NONE"
keyMethodAES128 = "AES-128"
keyMethodSampleAES = "SAMPLE-AES"
)
// IsMasterPlaylist reports whether the content looks like a master playlist.
func IsMasterPlaylist(content string) bool {
return strings.Contains(content, "#EXT-X-STREAM-INF")
}
func validateM3U8Header(content string) error {
for _, line := range strings.Split(content, "\n") {
line = strings.TrimSpace(line)
if line == "" {
continue
}
if line != "#EXTM3U" {
return errors.New("content is not a valid m3u8 playlist")
}
return nil
}
return errors.New("empty m3u8 playlist")
}
// ParseMaster parses a master playlist and returns all variants.
func ParseMaster(content string, baseURL *url.URL) ([]*Variant, error) {
master, err := ParseMasterPlaylist(content, baseURL)
if err != nil {
return nil, err
}
return master.Variants, nil
}
// ParseMasterPlaylist parses a master playlist including its EXT-X-MEDIA
// rendition groups.
func ParseMasterPlaylist(content string, baseURL *url.URL) (*MasterPlaylist, error) {
if err := validateM3U8Header(content); err != nil {
return nil, err
}
master := &MasterPlaylist{Rendition: make(map[string][]*MediaRendition)}
var pending *Variant
for _, line := range strings.Split(content, "\n") {
line = strings.TrimSpace(line)
if line == "" {
continue
}
if strings.HasPrefix(line, "#EXT-X-STREAM-INF:") {
attrs := parseAttributes(line[len("#EXT-X-STREAM-INF:"):])
pending = &Variant{
Bandwidth: parseInt64(attrs["BANDWIDTH"]),
AvgBandwidth: parseInt64(attrs["AVERAGE-BANDWIDTH"]),
Resolution: attrs["RESOLUTION"],
Codecs: unquote(attrs["CODECS"]),
Name: unquote(attrs["NAME"]),
AudioGroup: unquote(attrs["AUDIO"]),
}
continue
}
if strings.HasPrefix(line, "#EXT-X-MEDIA:") {
attrs := parseAttributes(line[len("#EXT-X-MEDIA:"):])
rendition := &MediaRendition{
Type: strings.ToUpper(attrs["TYPE"]),
GroupID: unquote(attrs["GROUP-ID"]),
}
// Only an explicit URI means a separate track; resolving an empty
// reference would fabricate one out of the playlist URL.
if uri := unquote(attrs["URI"]); uri != "" {
rendition.URI = resolveURL(baseURL, uri)
}
if rendition.GroupID != "" {
// Keep every member: a group is unsupported as a whole when
// ANY of its members carries its own URI.
master.Rendition[rendition.GroupID] = append(master.Rendition[rendition.GroupID], rendition)
}
continue
}
if pending != nil {
if line[0] != '#' {
pending.URI = resolveURL(baseURL, line)
master.Variants = append(master.Variants, pending)
}
pending = nil
}
}
if len(master.Variants) == 0 {
return nil, errors.New("master playlist has no variants")
}
return master, nil
}
// checkAudioRendition rejects the selected variant when its AUDIO group
// carries a separate audio track: downloading only the video variant would
// silently produce a silent file. Groups the variant does not reference are
// ignored (variants without an AUDIO attribute are assumed self-contained).
func checkAudioRendition(master *MasterPlaylist, best *Variant) error {
if best == nil || best.AudioGroup == "" {
return nil
}
for _, rendition := range master.Rendition[best.AudioGroup] {
if rendition == nil || rendition.Type != "AUDIO" || rendition.URI == "" {
continue
}
return fmt.Errorf(
"selected variant uses a separate audio rendition (group %q), which is not supported: the merged output would have no audio",
best.AudioGroup)
}
return nil
}
// PickBestVariant selects the highest-bandwidth variant.
func PickBestVariant(variants []*Variant) *Variant {
var best *Variant
for _, v := range variants {
if best == nil || v.Bandwidth > best.Bandwidth {
best = v
}
}
return best
}
// ParseMedia parses a media playlist into an ordered segment plan.
func ParseMedia(content string, baseURL *url.URL) (*Media, error) {
if err := validateM3U8Header(content); err != nil {
return nil, err
}
media := &Media{Live: true}
var currentKey *Key
var pendingByterange *ByteRange
var pendingGap bool
var pendingDuration float64
var lastRangeEnd int64
var sequence int64
var mapIdentity string // normalized identity of the current EXT-X-MAP
for _, line := range strings.Split(content, "\n") {
line = strings.TrimSpace(line)
if line == "" {
continue
}
if line[0] != '#' {
// Every non-comment line in a media playlist is a segment URI.
seg := &Segment{
URI: resolveURL(baseURL, line),
Sequence: sequence,
Byterange: pendingByterange,
Key: currentKey,
Gap: pendingGap,
Duration: pendingDuration,
}
if pendingByterange != nil {
// Byterange segments have exact sizes straight from the playlist.
seg.Size = pendingByterange.Length
}
sequence++
pendingByterange = nil
pendingGap = false
pendingDuration = 0
media.Segments = append(media.Segments, seg)
continue
}
switch {
case strings.HasPrefix(line, "#EXTINF:"):
value := line[len("#EXTINF:"):]
if idx := strings.Index(value, ","); idx >= 0 {
value = value[:idx]
}
pendingDuration = parseFloat(value)
case strings.HasPrefix(line, "#EXT-X-KEY:"):
key, err := parseKey(line[len("#EXT-X-KEY:"):], baseURL)
if err != nil {
return nil, err
}
currentKey = key
case strings.HasPrefix(line, "#EXT-X-MAP:"):
attrs := parseAttributes(line[len("#EXT-X-MAP:"):])
uriAttr := unquote(attrs["URI"])
if uriAttr == "" {
return nil, errors.New("EXT-X-MAP without URI is not supported")
}
resolved := resolveURL(baseURL, uriAttr)
// The init section identity covers every attribute, not just the
// URI: a repeated tag with a different BYTERANGE addresses
// different bytes and changes the init section. NUL keeps the
// two parts unambiguously separated.
identity := resolved + "\x00" + attrs["BYTERANGE"]
if mapIdentity != "" {
if mapIdentity != identity {
return nil, errors.New("changing init sections are not supported")
}
media.IsFMP4 = true
continue
}
mapIdentity = identity
init := &Segment{
URI: resolved,
Sequence: sequence,
IsInit: true,
Key: currentKey,
}
if attrs["BYTERANGE"] != "" {
init.Byterange = parseByteRange(attrs["BYTERANGE"], 0)
if init.Byterange == nil {
return nil, fmt.Errorf("invalid EXT-X-MAP BYTERANGE: %s", attrs["BYTERANGE"])
}
}
if init.Key != nil && init.Key.IV == nil {
// RFC 8216 only defines sequence-derived IVs for media
// segments; guessing one for the init section produces
// corrupt output, so require the playlist to be explicit.
return nil, errors.New("encrypted EXT-X-MAP without an explicit IV is not supported")
}
// The init section is prepended to the segment plan, it takes
// the media sequence number of the playlist for IV derivation.
media.Segments = append(media.Segments, init)
media.IsFMP4 = true
case strings.HasPrefix(line, "#EXT-X-BYTERANGE:"):
pendingByterange = parseByteRange(line[len("#EXT-X-BYTERANGE:"):], lastRangeEnd)
if pendingByterange == nil {
return nil, fmt.Errorf("invalid EXT-X-BYTERANGE: %s", line)
}
case line == "#EXT-X-GAP":
pendingGap = true
case line == "#EXT-X-DISCONTINUITY":
media.Discontinuity = true
case line == "#EXT-X-ENDLIST":
media.Live = false
case strings.HasPrefix(line, "#EXT-X-MEDIA-SEQUENCE:"):
sequence = parseInt64(line[len("#EXT-X-MEDIA-SEQUENCE:"):])
case strings.HasPrefix(line, "#EXT-X-VERSION:"):
media.Version = int(parseInt64(line[len("#EXT-X-VERSION:"):]))
}
if pendingByterange != nil {
// Non-segment tags between BYTERANGE and its URI must not reset the
// running offset, so it is tracked here after each parse.
lastRangeEnd = pendingByterange.Offset + pendingByterange.Length
}
}
if len(media.Segments) == 0 {
return nil, errors.New("media playlist has no segments")
}
return media, nil
}
// validateSupported rejects playlist features the byte-concatenation merge
// cannot reproduce faithfully. Skipping such features silently would report
// success while producing a broken file.
func validateSupported(media *Media) error {
if media.Live {
return errors.New("live streams are not supported yet")
}
if media.Discontinuity {
return errors.New("EXT-X-DISCONTINUITY is not supported: the merged output would have a broken timeline")
}
for _, seg := range media.Segments {
if seg.Gap {
return errors.New("EXT-X-GAP is not supported: the missing segment would leave a hole in the merged output")
}
}
return nil
}
func parseKey(attrs string, baseURL *url.URL) (*Key, error) {
parsed := parseAttributes(attrs)
switch method := parsed["METHOD"]; method {
case keyMethodNone:
return nil, nil
case keyMethodAES128:
uri := unquote(parsed["URI"])
if uri == "" {
return nil, errors.New("AES-128 EXT-X-KEY without URI")
}
if format := unquote(parsed["KEYFORMAT"]); format != "" && format != "identity" {
return nil, fmt.Errorf("unsupported EXT-X-KEY KEYFORMAT: %s", format)
}
key := &Key{Method: keyMethodAES128, URI: resolveURL(baseURL, uri)}
if iv, ok := parsed["IV"]; ok {
value := strings.TrimPrefix(strings.ToLower(iv), "0x")
decoded, err := hex.DecodeString(value)
if err != nil || len(decoded) != 16 {
return nil, fmt.Errorf("invalid EXT-X-KEY IV: %s", iv)
}
key.IV = decoded
}
return key, nil
case keyMethodSampleAES:
return nil, errors.New("SAMPLE-AES encryption is not supported")
default:
return nil, fmt.Errorf("unsupported encryption method: %s", method)
}
}
func parseByteRange(value string, defaultOffset int64) *ByteRange {
lengthPart := value
offsetPart := ""
if idx := strings.Index(value, "@"); idx >= 0 {
lengthPart = value[:idx]
offsetPart = value[idx+1:]
}
length := parseInt64(lengthPart)
if length <= 0 {
return nil
}
offset := defaultOffset
if offsetPart != "" {
offset = parseInt64(offsetPart)
if offset < 0 {
return nil
}
}
return &ByteRange{Length: length, Offset: offset}
}
// resolveURL resolves a playlist URI reference against the playlist URL
// following RFC 3986.
func resolveURL(baseURL *url.URL, ref string) string {
refURL, err := url.Parse(ref)
if err != nil {
return ref
}
return baseURL.ResolveReference(refURL).String()
}
// parseAttributes parses an HLS attribute list like FOO="bar",NUM=42.
func parseAttributes(s string) map[string]string {
attrs := make(map[string]string)
var key, val strings.Builder
inKey := true
inQuote := false
flush := func() {
name := strings.ToUpper(strings.TrimSpace(key.String()))
if name != "" {
attrs[name] = strings.TrimSpace(val.String())
}
key.Reset()
val.Reset()
inKey = true
}
for i := 0; i < len(s); i++ {
c := s[i]
if inKey {
if c == '=' {
inKey = false
} else {
key.WriteByte(c)
}
continue
}
if c == '"' {
inQuote = !inQuote
continue
}
if c == ',' && !inQuote {
flush()
continue
}
val.WriteByte(c)
}
flush()
return attrs
}
func unquote(s string) string {
s = strings.TrimSpace(s)
if len(s) >= 2 && strings.HasPrefix(s, "\"") && strings.HasSuffix(s, "\"") {
return s[1 : len(s)-1]
}
return s
}
func parseInt64(s string) int64 {
value, _ := strconv.ParseInt(strings.TrimSpace(s), 10, 64)
return value
}
func parseFloat(s string) float64 {
value, _ := strconv.ParseFloat(strings.TrimSpace(s), 64)
return value
}
// OutputExtension returns the container file extension for the playlist.
func (m *Media) OutputExtension() string {
if m.IsFMP4 {
return ".mp4"
}
return ".ts"
}
// DeriveOutputName builds the merged output file name from a playlist URL.
func DeriveOutputName(playlistURL string, media *Media) string {
name := ""
if u, err := url.Parse(playlistURL); err == nil {
name = path.Base(u.Path)
name = strings.TrimSuffix(name, path.Ext(name))
}
if name == "" || name == "/" || name == "." {
if u, err := url.Parse(playlistURL); err == nil {
name = u.Hostname()
}
}
return name + media.OutputExtension()
}
@@ -0,0 +1,39 @@
package hls
import (
_ "embed"
"net/url"
"testing"
)
//go:embed testdata/olemovienews.m3u8
var realWorldPlaylist string
// TestParseMediaRealWorld guards parsing against a real site playlist
// (olemovienews.com VOD, 138 TS segments).
func TestParseMediaRealWorld(t *testing.T) {
base, err := url.Parse("https://europe.olemovienews.com/ts2/20230125/afAxfjit/mp4/afAxfjit.mp4/index-v1-a1.m3u8")
if err != nil {
t.Fatal(err)
}
media, err := ParseMedia(realWorldPlaylist, base)
if err != nil {
t.Fatal(err)
}
if media.Live {
t.Error("real VOD playlist should not be live")
}
if media.IsFMP4 {
t.Error("playlist should be detected as TS")
}
if len(media.Segments) != 138 {
t.Errorf("want 138 segments, got %d", len(media.Segments))
}
first := media.Segments[0]
if first.URI != "https://europe.olemovienews.com/ts2/20230125/afAxfjit/mp4/afAxfjit.mp4/seg-1-v1-a1.ts" {
t.Errorf("segment URI resolution wrong: %s", first.URI)
}
if media.Segments[0].Sequence != 1 {
t.Errorf("first sequence = %d, want 1", media.Segments[0].Sequence)
}
}
+252
View File
@@ -0,0 +1,252 @@
package hls
import (
"net/url"
"testing"
)
func mustURL(t *testing.T, raw string) *url.URL {
t.Helper()
u, err := url.Parse(raw)
if err != nil {
t.Fatal(err)
}
return u
}
func TestParseMediaTS(t *testing.T) {
content := "#EXTM3U\n" +
"#EXT-X-TARGETDURATION:20\n" +
"#EXT-X-VERSION:3\n" +
"#EXT-X-MEDIA-SEQUENCE:1\n" +
"#EXTINF:20.000,\n" +
"seg-1.ts\n" +
"#EXTINF:20.000,\n" +
"seg-2.ts\n" +
"#EXT-X-ENDLIST\n"
media, err := ParseMedia(content, mustURL(t, "https://cdn.example.com/vod/index.m3u8"))
if err != nil {
t.Fatal(err)
}
if media.Live {
t.Error("VOD playlist should not be live")
}
if media.IsFMP4 {
t.Error("TS playlist should not be fMP4")
}
if len(media.Segments) != 2 {
t.Fatalf("want 2 segments, got %d", len(media.Segments))
}
want := "https://cdn.example.com/vod/seg-1.ts"
if media.Segments[0].URI != want {
t.Errorf("want uri %s, got %s", want, media.Segments[0].URI)
}
if media.Segments[0].Sequence != 1 {
t.Errorf("want sequence 1, got %d", media.Segments[0].Sequence)
}
if media.Segments[0].Key != nil {
t.Error("segment should be unencrypted")
}
}
func TestParseMediaWithoutExtinf(t *testing.T) {
// Some playlists list segment URIs without EXTINF lines.
content := "#EXTM3U\nseg-1.ts\nseg-2.ts\n#EXT-X-ENDLIST\n"
media, err := ParseMedia(content, mustURL(t, "https://cdn.example.com/a/index.m3u8"))
if err != nil {
t.Fatal(err)
}
if len(media.Segments) != 2 {
t.Fatalf("want 2 segments, got %d", len(media.Segments))
}
}
func TestParseMediaLive(t *testing.T) {
content := "#EXTM3U\n#EXT-X-TARGETDURATION:6\n#EXTINF:6.0,\nseg-1.ts\n"
media, err := ParseMedia(content, mustURL(t, "https://cdn.example.com/live/index.m3u8"))
if err != nil {
t.Fatal(err)
}
if !media.Live {
t.Error("playlist without ENDLIST should be live")
}
}
func TestParseMediaFMP4(t *testing.T) {
content := "#EXTM3U\n" +
"#EXT-X-VERSION:6\n" +
"#EXT-X-MAP:URI=\"init.mp4\"\n" +
"#EXTINF:5.0,\n" +
"seg-1.m4s\n" +
"#EXT-X-ENDLIST\n"
media, err := ParseMedia(content, mustURL(t, "https://cdn.example.com/vod/index.m3u8"))
if err != nil {
t.Fatal(err)
}
if !media.IsFMP4 {
t.Error("playlist with MAP should be fMP4")
}
if len(media.Segments) != 2 {
t.Fatalf("want 2 segments (init + 1), got %d", len(media.Segments))
}
if !media.Segments[0].IsInit {
t.Error("first segment should be the init section")
}
if media.Segments[0].URI != "https://cdn.example.com/vod/init.mp4" {
t.Errorf("init uri wrong: %s", media.Segments[0].URI)
}
if media.OutputExtension() != ".mp4" {
t.Errorf("want .mp4, got %s", media.OutputExtension())
}
}
func TestParseMediaMultiMapError(t *testing.T) {
content := "#EXTM3U\n" +
"#EXT-X-MAP:URI=\"init1.mp4\"\n" +
"#EXTINF:5.0,\nseg-1.m4s\n" +
"#EXT-X-DISCONTINUITY\n" +
"#EXT-X-MAP:URI=\"init2.mp4\"\n" +
"#EXTINF:5.0,\nseg-2.m4s\n" +
"#EXT-X-ENDLIST\n"
if _, err := ParseMedia(content, mustURL(t, "https://cdn.example.com/vod/index.m3u8")); err == nil {
t.Error("multi-period fMP4 should be rejected")
}
}
func TestParseMediaKeyRotation(t *testing.T) {
content := "#EXTM3U\n" +
"#EXT-X-KEY:METHOD=AES-128,URI=\"key1.bin\",IV=0x00000000000000000000000000000001\n" +
"#EXTINF:5.0,\nseg-1.ts\n" +
"#EXT-X-KEY:METHOD=AES-128,URI=\"key2.bin\"\n" +
"#EXTINF:5.0,\nseg-2.ts\n" +
"#EXT-X-KEY:METHOD=NONE\n" +
"#EXTINF:5.0,\nseg-3.ts\n" +
"#EXT-X-ENDLIST\n"
media, err := ParseMedia(content, mustURL(t, "https://cdn.example.com/vod/index.m3u8"))
if err != nil {
t.Fatal(err)
}
if media.Segments[0].Key == nil || media.Segments[0].Key.URI != "https://cdn.example.com/vod/key1.bin" {
t.Fatalf("segment 1 key wrong: %+v", media.Segments[0].Key)
}
if len(media.Segments[0].Key.IV) != 16 || media.Segments[0].Key.IV[15] != 1 {
t.Error("explicit IV not parsed")
}
if media.Segments[1].Key == nil || media.Segments[1].Key.URI != "https://cdn.example.com/vod/key2.bin" {
t.Fatalf("segment 2 key wrong: %+v", media.Segments[1].Key)
}
if media.Segments[1].Key.IV != nil {
t.Error("key without IV should leave IV nil (sequence-derived)")
}
if media.Segments[2].Key != nil {
t.Error("segment 3 should be unencrypted after METHOD=NONE")
}
}
func TestParseMediaSampleAESError(t *testing.T) {
content := "#EXTM3U\n#EXT-X-KEY:METHOD=SAMPLE-AES,URI=\"skd://key\"\n#EXTINF:5.0,\nseg-1.ts\n"
if _, err := ParseMedia(content, mustURL(t, "https://cdn.example.com/vod/index.m3u8")); err == nil {
t.Error("SAMPLE-AES should be rejected")
}
}
func TestParseMediaByterange(t *testing.T) {
content := "#EXTM3U\n" +
"#EXT-X-BYTERANGE:1000@0\nseg.ts\n" +
"#EXT-X-BYTERANGE:2000\nseg.ts\n" +
"#EXT-X-BYTERANGE:3000\nseg.ts\n" +
"#EXT-X-ENDLIST\n"
media, err := ParseMedia(content, mustURL(t, "https://cdn.example.com/vod/index.m3u8"))
if err != nil {
t.Fatal(err)
}
ranges := [][2]int64{{0, 1000}, {1000, 2000}, {3000, 3000}}
for i, want := range ranges {
br := media.Segments[i].Byterange
if br == nil {
t.Fatalf("segment %d byterange missing", i)
}
if br.Offset != want[0] || br.Length != want[1] {
t.Errorf("segment %d: want offset %d length %d, got %d/%d", i, want[0], want[1], br.Offset, br.Length)
}
}
}
func TestParseMediaGap(t *testing.T) {
content := "#EXTM3U\n#EXTINF:5.0,\nseg-1.ts\n#EXT-X-GAP\n#EXTINF:5.0,\nseg-2.ts\n#EXT-X-ENDLIST\n"
media, err := ParseMedia(content, mustURL(t, "https://cdn.example.com/vod/index.m3u8"))
if err != nil {
t.Fatal(err)
}
if media.Segments[0].Gap || !media.Segments[1].Gap {
t.Error("gap flags wrong")
}
}
func TestParseMediaInvalidHeader(t *testing.T) {
if _, err := ParseMedia("not a playlist", mustURL(t, "https://cdn.example.com/a.m3u8")); err == nil {
t.Error("invalid content should be rejected")
}
}
func TestParseMasterPickBest(t *testing.T) {
content := "#EXTM3U\n" +
"#EXT-X-MEDIA:TYPE=AUDIO,GROUP-ID=\"aud\",NAME=\"en\",DEFAULT=YES,URI=\"audio/index.m3u8\"\n" +
"#EXT-X-STREAM-INF:BANDWIDTH=800000,RESOLUTION=640x360\n" +
"360p/index.m3u8\n" +
"#EXT-X-STREAM-INF:BANDWIDTH=2500000,RESOLUTION=1920x1080,CODECS=\"avc1.640028,mp4a.40.2\"\n" +
"1080p/index.m3u8?sign=abc\n" +
"#EXT-X-ENDLIST\n"
variants, err := ParseMaster(content, mustURL(t, "https://cdn.example.com/vod/master.m3u8"))
if err != nil {
t.Fatal(err)
}
if len(variants) != 2 {
t.Fatalf("want 2 variants, got %d", len(variants))
}
best := PickBestVariant(variants)
if best.Bandwidth != 2500000 {
t.Errorf("want highest bandwidth variant, got %d", best.Bandwidth)
}
if best.URI != "https://cdn.example.com/vod/1080p/index.m3u8?sign=abc" {
t.Errorf("variant uri wrong: %s", best.URI)
}
}
func TestResolveURLQueryNotInherited(t *testing.T) {
base := mustURL(t, "https://cdn.example.com/vod/index.m3u8?sign=abc")
got := resolveURL(base, "seg-1.ts")
if got != "https://cdn.example.com/vod/seg-1.ts" {
t.Errorf("relative resolution should follow RFC3986 (no query inheritance), got %s", got)
}
}
func TestDeriveOutputName(t *testing.T) {
media := &Media{}
if got := DeriveOutputName("https://cdn.example.com/ts2/a/index-v1-a1.m3u8", media); got != "index-v1-a1.ts" {
t.Errorf("want index-v1-a1.ts, got %s", got)
}
fmp4 := &Media{IsFMP4: true}
if got := DeriveOutputName("https://cdn.example.com/ts2/a/index.m3u8", fmp4); got != "index.mp4" {
t.Errorf("want index.mp4, got %s", got)
}
if got := DeriveOutputName("https://cdn.example.com/play?token=x", media); got != "play.ts" {
t.Errorf("want play.ts, got %s", got)
}
if got := DeriveOutputName("https://cdn.example.com/", media); got != "cdn.example.com.ts" {
t.Errorf("want host fallback, got %s", got)
}
}
func TestParseAttributesQuotedComma(t *testing.T) {
attrs := parseAttributes(`URI="a,b.ts",IV=0xABCDEF00000000000000000000000001,NAME="x"`)
if attrs["URI"] != "a,b.ts" {
t.Errorf("quoted comma value wrong: %q", attrs["URI"])
}
if attrs["IV"] != "0xABCDEF00000000000000000000000001" {
t.Errorf("hex value wrong: %q", attrs["IV"])
}
if attrs["NAME"] != "x" {
t.Errorf("quoted value wrong: %q", attrs["NAME"])
}
}
+296
View File
@@ -0,0 +1,296 @@
package hls
import (
"bytes"
"context"
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"sync/atomic"
"testing"
"time"
"github.com/GopeedLab/gopeed/pkg/base"
)
func TestFetcherE2E_SameURLTwoTasks(t *testing.T) {
const total = 6
mux := http.NewServeMux()
var playlist strings.Builder
playlist.WriteString("#EXTM3U\n")
for i := 0; i < total; i++ {
playlist.WriteString(fmt.Sprintf("#EXTINF:4,\nseg-%d.ts\n", i))
}
playlist.WriteString("#EXT-X-ENDLIST\n")
mux.HandleFunc("/video.m3u8", func(w http.ResponseWriter, r *http.Request) {
w.Write([]byte(playlist.String()))
})
segments := make(map[int][]byte)
for i := 0; i < total; i++ {
segments[i] = []byte(fmt.Sprintf("SHARED-SEGMENT-%02d", i))
idx := i
mux.HandleFunc(fmt.Sprintf("/seg-%d.ts", idx), func(w http.ResponseWriter, r *http.Request) {
time.Sleep(120 * time.Millisecond)
w.Write(segments[idx])
})
}
server := httptest.NewServer(mux)
defer server.Close()
cfg := testConfig()
cfg.SegmentConnections = 2
cfg.PrefetchContentLength = false
dir := t.TempDir()
// Two tasks for the same playlist URL into the same directory, like a
// user duplicating a task. The engine would rename the second output
// (AutoRename); mirror that here.
f1 := newTestFetcher(t, cfg)
if err := f1.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: dir}); err != nil {
t.Fatal(err)
}
f2 := newTestFetcher(t, cfg)
if err := f2.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: dir, Name: "video-2.ts"}); err != nil {
t.Fatal(err)
}
if f1.state.StagingDir == "" || f1.state.StagingDir == f2.state.StagingDir {
t.Fatalf("staging dirs must be distinct per task: %q vs %q", f1.state.StagingDir, f2.state.StagingDir)
}
// Start task 1, let it stage some segments, then pause it.
if err := f1.Start(); err != nil {
t.Fatal(err)
}
time.Sleep(250 * time.Millisecond)
if err := f1.Pause(); err != nil {
t.Fatal(err)
}
staged1, err := os.ReadDir(f1.state.StagingDir)
if err != nil || len(staged1) == 0 {
t.Fatalf("task 1 should have staged segments, dir: %s", f1.state.StagingDir)
}
// Task 2 must complete without touching task 1's staging data...
if err := f2.Start(); err != nil {
t.Fatal(err)
}
if err := f2.Wait(); err != nil {
t.Fatal(err)
}
stillThere, err := os.ReadDir(f1.state.StagingDir)
if err != nil || len(stillThere) != len(staged1) {
t.Fatalf("task 2 disturbed task 1 staging: had %d entries, now %d", len(staged1), len(stillThere))
}
// ...and task 1 resumes to a complete, correct output of its own.
if err := f1.Start(); err != nil {
t.Fatal(err)
}
if err := f1.Wait(); err != nil {
t.Fatal(err)
}
var want []byte
for i := 0; i < total; i++ {
want = append(want, segments[i]...)
}
if !bytes.Equal(readOutput(t, f1), want) || !bytes.Equal(readOutput(t, f2), want) {
t.Error("same-URL task outputs mismatch")
}
entries, _ := os.ReadDir(dir)
if len(entries) != 2 {
names := make([]string, 0, len(entries))
for _, e := range entries {
names = append(names, e.Name())
}
t.Errorf("expected only the two outputs to remain, got %v", names)
}
}
func TestFetcherE2E_ForeignJournalDiscarded(t *testing.T) {
const total = 3
var served atomic.Int64
mux := http.NewServeMux()
var playlist strings.Builder
playlist.WriteString("#EXTM3U\n")
for i := 0; i < total; i++ {
playlist.WriteString(fmt.Sprintf("#EXTINF:4,\nseg-%d.ts\n", i))
}
playlist.WriteString("#EXT-X-ENDLIST\n")
mux.HandleFunc("/video.m3u8", func(w http.ResponseWriter, r *http.Request) {
w.Write([]byte(playlist.String()))
})
for i := 0; i < total; i++ {
mux.HandleFunc(fmt.Sprintf("/seg-%d.ts", i), func(w http.ResponseWriter, r *http.Request) {
served.Add(1)
w.Write([]byte("DATA"))
})
}
server := httptest.NewServer(mux)
defer server.Close()
cfg := testConfig()
cfg.PrefetchContentLength = false
f := newTestFetcher(t, cfg)
if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: t.TempDir()}); err != nil {
t.Fatal(err)
}
// Plant a journal that does not belong to this plan (wrong fingerprint).
tempDir := f.resolveStagingDirLocked()
if err := os.MkdirAll(tempDir, 0777); err != nil {
t.Fatal(err)
}
foreign := journalFile{
StagingID: "some-other-task",
PlanHash: "deadbeef",
SegmentCount: total,
Sizes: map[string]int64{"0": 4, "1": 4, "2": 4},
}
data, _ := json.Marshal(foreign)
if err := os.WriteFile(filepath.Join(tempDir, "journal.json"), data, 0666); err != nil {
t.Fatal(err)
}
if err := f.Start(); err != nil {
t.Fatal(err)
}
if err := f.Wait(); err != nil {
t.Fatal(err)
}
if got := served.Load(); got != total {
t.Errorf("foreign journal must be discarded, want %d downloads, got %d", total, got)
}
}
func TestFetcherE2E_MergeFailurePreservesStaging(t *testing.T) {
mux := http.NewServeMux()
mux.HandleFunc("/video.m3u8", func(w http.ResponseWriter, r *http.Request) {
w.Write([]byte("#EXTM3U\n#EXTINF:4,\na.ts\n#EXTINF:4,\nb.ts\n#EXT-X-ENDLIST\n"))
})
mux.HandleFunc("/a.ts", func(w http.ResponseWriter, r *http.Request) { w.Write([]byte("AAAA")) })
mux.HandleFunc("/b.ts", func(w http.ResponseWriter, r *http.Request) { w.Write([]byte("BBBB")) })
server := httptest.NewServer(mux)
defer server.Close()
f := newTestFetcher(t, testConfig())
if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: t.TempDir()}); err != nil {
t.Fatal(err)
}
run := &hlsRun{
ctx: context.Background(),
cancel: func() {},
done: make(chan struct{}),
segs: f.state.Segments,
outputPath: f.meta.SingleFilepath(),
stagingID: f.state.StagingID,
planHash: planFingerprint(f.state),
tempDir: f.resolveStagingDirLocked(),
keyCache: make(map[string][]byte),
journal: make(map[int64]int64),
}
run.journalPath = filepath.Join(run.tempDir, "journal.json")
if err := os.MkdirAll(run.tempDir, 0777); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(segmentPath(run.tempDir, 0), []byte("AAAA"), 0666); err != nil {
t.Fatal(err)
}
// Segment 1 is missing on disk: the merge must fail without deleting the
// staged data or leaving a partial output behind.
if err := f.merge(run); err == nil {
t.Fatal("merge with a missing segment should fail")
}
if _, err := os.Stat(segmentPath(run.tempDir, 0)); err != nil {
t.Fatal("merge failure must preserve staged segments")
}
entries, _ := os.ReadDir(filepath.Dir(run.outputPath))
for _, e := range entries {
if strings.HasSuffix(e.Name(), ".part") {
t.Errorf("failed merge left a partial output behind: %s", e.Name())
}
}
// Recover the segment: the retried merge succeeds.
if err := os.WriteFile(segmentPath(run.tempDir, 1), []byte("BBBB"), 0666); err != nil {
t.Fatal(err)
}
if err := f.merge(run); err != nil {
t.Fatalf("retried merge failed: %v", err)
}
got, err := os.ReadFile(run.outputPath)
if err != nil {
t.Fatal(err)
}
if string(got) != "AAAABBBB" {
t.Errorf("retried merge output mismatch: %q", got)
}
}
func TestFetcherE2E_ReMergeOverExistingOutput(t *testing.T) {
const total = 2
mux := http.NewServeMux()
var playlist strings.Builder
playlist.WriteString("#EXTM3U\n")
for i := 0; i < total; i++ {
playlist.WriteString(fmt.Sprintf("#EXTINF:4,\nseg-%d.ts\n", i))
}
playlist.WriteString("#EXT-X-ENDLIST\n")
mux.HandleFunc("/video.m3u8", func(w http.ResponseWriter, r *http.Request) {
w.Write([]byte(playlist.String()))
})
for i := 0; i < total; i++ {
mux.HandleFunc(fmt.Sprintf("/seg-%d.ts", i), func(w http.ResponseWriter, r *http.Request) {
w.Write([]byte("DATA"))
})
}
server := httptest.NewServer(mux)
defer server.Close()
f := newTestFetcher(t, testConfig())
if err := f.Resolve(&base.Request{URL: server.URL + "/video.m3u8"}, &base.Options{Path: t.TempDir()}); err != nil {
t.Fatal(err)
}
tempDir := f.resolveStagingDirLocked()
if err := os.MkdirAll(tempDir, 0777); err != nil {
t.Fatal(err)
}
// Simulate the crash window after a completed rename but before the task
// was marked done: a stale output file, a complete journal and all
// segments staged.
stale := filepath.Join(filepath.Dir(f.meta.SingleFilepath()), f.state.OutputName)
if err := os.WriteFile(stale, []byte("STALE-OUTPUT"), 0666); err != nil {
t.Fatal(err)
}
jr := &hlsRun{
segs: f.state.Segments,
stagingID: f.state.StagingID,
planHash: planFingerprint(f.state),
journalPath: filepath.Join(tempDir, "journal.json"),
journal: map[int64]int64{0: 4, 1: 4},
}
if err := saveJournal(jr); err != nil {
t.Fatal(err)
}
for i := 0; i < total; i++ {
if err := os.WriteFile(segmentPath(tempDir, int64(i)), []byte("DATA"), 0666); err != nil {
t.Fatal(err)
}
}
if err := f.Start(); err != nil {
t.Fatal(err)
}
if err := f.Wait(); err != nil {
t.Fatalf("re-merge over an existing output failed: %v", err)
}
got, err := os.ReadFile(stale)
if err != nil {
t.Fatal(err)
}
if string(got) != "DATADATA" {
t.Errorf("stale output was not replaced: %q", got)
}
}
+146
View File
@@ -0,0 +1,146 @@
package hls
import (
"net/url"
"strings"
"testing"
)
func mustParseURL(t *testing.T, raw string) *url.URL {
t.Helper()
u, err := url.Parse(raw)
if err != nil {
t.Fatal(err)
}
return u
}
// subsetCase is a media playlist plus the error substring the pipeline must
// reject it with; wantOK playlists must parse AND validate cleanly.
type subsetCase struct {
name string
playlist string
wantErr string
}
func TestSubsetRejections(t *testing.T) {
cases := []subsetCase{
{
name: "gap",
playlist: "#EXTM3U\n#EXTINF:4,\na.ts\n#EXT-X-GAP\n#EXTINF:4,\nb.ts\n#EXT-X-ENDLIST\n",
wantErr: "EXT-X-GAP",
},
{
name: "gap-before-first-segment",
playlist: "#EXTM3U\n#EXT-X-GAP\n#EXTINF:4,\na.ts\n#EXT-X-ENDLIST\n",
wantErr: "EXT-X-GAP",
},
{
name: "discontinuity",
playlist: "#EXTM3U\n#EXTINF:4,\na.ts\n#EXT-X-DISCONTINUITY\n#EXTINF:4,\nb.ts\n#EXT-X-ENDLIST\n",
wantErr: "EXT-X-DISCONTINUITY",
},
{
name: "live",
playlist: "#EXTM3U\n#EXTINF:4,\na.ts\n",
wantErr: "live",
},
{
name: "keyformat",
playlist: "#EXTM3U\n#EXT-X-KEY:METHOD=AES-128,URI=\"k\",KEYFORMAT=\"com.apple.streamingkeydelivery\"\n#EXTINF:4,\na.ts\n#EXT-X-ENDLIST\n",
wantErr: "KEYFORMAT",
},
{
name: "encrypted-map-without-iv",
playlist: "#EXTM3U\n#EXT-X-KEY:METHOD=AES-128,URI=\"k\"\n#EXT-X-MAP:URI=\"init.mp4\"\n#EXTINF:4,\na.m4s\n#EXT-X-ENDLIST\n",
wantErr: "explicit IV",
},
{
name: "map-change-byterange",
playlist: "#EXTM3U\n#EXT-X-MAP:URI=\"init.mp4\",BYTERANGE=\"100@0\"\n#EXTINF:4,\na.m4s\n#EXT-X-MAP:URI=\"init.mp4\",BYTERANGE=\"100@100\"\n#EXTINF:4,\nb.m4s\n#EXT-X-ENDLIST\n",
wantErr: "changing init sections",
},
{
name: "map-change-uri",
playlist: "#EXTM3U\n#EXT-X-MAP:URI=\"init1.mp4\"\n#EXTINF:4,\na.m4s\n#EXT-X-MAP:URI=\"init2.mp4\"\n#EXTINF:4,\nb.m4s\n#EXT-X-ENDLIST\n",
wantErr: "changing init sections",
},
}
baseURL := mustParseURL(t, "https://cdn.example.com/vod/index.m3u8")
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
media, parseErr := ParseMedia(tc.playlist, baseURL)
if parseErr != nil {
if !strings.Contains(parseErr.Error(), tc.wantErr) {
t.Fatalf("parse error %q does not mention %q", parseErr, tc.wantErr)
}
return
}
err := validateSupported(media)
if err == nil {
t.Fatalf("playlist should be rejected with %q", tc.wantErr)
}
if !strings.Contains(err.Error(), tc.wantErr) {
t.Fatalf("error %q does not mention %q", err, tc.wantErr)
}
})
}
}
func TestSubsetRepeatedSameMapAllowed(t *testing.T) {
playlist := "#EXTM3U\n" +
"#EXT-X-MAP:URI=\"init.mp4\",BYTERANGE=\"100@0\"\n" +
"#EXTINF:4,\na.m4s\n" +
"#EXT-X-MAP:URI=\"init.mp4\",BYTERANGE=\"100@0\"\n" +
"#EXTINF:4,\nb.m4s\n" +
"#EXT-X-ENDLIST\n"
media, err := ParseMedia(playlist, mustParseURL(t, "https://cdn.example.com/vod/index.m3u8"))
if err != nil {
t.Fatal(err)
}
if err := validateSupported(media); err != nil {
t.Fatalf("identical repeated MAP must be allowed: %v", err)
}
if len(media.Segments) != 3 { // init + 2 segments, no duplicate init
t.Fatalf("want 3 segments (init + 2), got %d", len(media.Segments))
}
}
func TestMasterAudioRenditionDetection(t *testing.T) {
masterURL := mustParseURL(t, "https://cdn.example.com/master.m3u8")
separateAudio := "#EXTM3U\n" +
"#EXT-X-MEDIA:TYPE=AUDIO,GROUP-ID=\"aud\",NAME=\"en\",DEFAULT=YES,URI=\"audio/index.m3u8\"\n" +
"#EXT-X-STREAM-INF:BANDWIDTH=1000000,AUDIO=\"aud\"\nvideo/index.m3u8\n"
master, err := ParseMasterPlaylist(separateAudio, masterURL)
if err != nil {
t.Fatal(err)
}
if err := checkAudioRendition(master, PickBestVariant(master.Variants)); err == nil {
t.Fatal("selected variant with a separate audio rendition must be rejected")
}
// A group the variant does not reference must not cause a false positive.
unrelatedGroup := "#EXTM3U\n" +
"#EXT-X-MEDIA:TYPE=AUDIO,GROUP-ID=\"aud\",NAME=\"en\",DEFAULT=YES,URI=\"audio/index.m3u8\"\n" +
"#EXT-X-STREAM-INF:BANDWIDTH=1000000\nvideo/index.m3u8\n"
master, err = ParseMasterPlaylist(unrelatedGroup, masterURL)
if err != nil {
t.Fatal(err)
}
if err := checkAudioRendition(master, PickBestVariant(master.Variants)); err != nil {
t.Fatalf("unreferenced audio group must not be rejected: %v", err)
}
// A rendition group without URIs (muxed tracks) is fine.
groupWithoutURI := "#EXTM3U\n" +
"#EXT-X-MEDIA:TYPE=AUDIO,GROUP-ID=\"aud\",NAME=\"en\",DEFAULT=YES\n" +
"#EXT-X-STREAM-INF:BANDWIDTH=1000000,AUDIO=\"aud\"\nvideo/index.m3u8\n"
master, err = ParseMasterPlaylist(groupWithoutURI, masterURL)
if err != nil {
t.Fatal(err)
}
if err := checkAudioRendition(master, PickBestVariant(master.Variants)); err != nil {
t.Fatalf("audio group without URI must not be rejected: %v", err)
}
}
+10
View File
@@ -0,0 +1,10 @@
Media fixtures generated locally with FFmpeg 7.0.2 (testsrc + sine sources,
no third-party content):
- sample-00.ts / sample-01.ts: one continuous MPEG-2 TS encode split at a
packet boundary, so continuity counters and PCR are consistent across the
segment join exactly like segments cut from a single real-time stream.
- fmp4-init.mp4: ftyp+moov init section of a fragmented MP4.
- fmp4-seg1.m4s .. fmp4-seg3.m4s: moof+mdat media fragments (styp omitted by
the muxer, which is legal) split at moof boundaries from one fragmented MP4
(-movflags +frag_keyframe+empty_moov+default_base_moof -frag_duration 500000).
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
+283
View File
@@ -0,0 +1,283 @@
#EXTM3U
#EXT-X-TARGETDURATION:20
#EXT-X-ALLOW-CACHE:YES
#EXT-X-PLAYLIST-TYPE:VOD
#EXT-X-VERSION:3
#EXT-X-MEDIA-SEQUENCE:1
#EXTINF:20.000,
seg-1-v1-a1.ts
#EXTINF:20.000,
seg-2-v1-a1.ts
#EXTINF:20.000,
seg-3-v1-a1.ts
#EXTINF:20.000,
seg-4-v1-a1.ts
#EXTINF:20.000,
seg-5-v1-a1.ts
#EXTINF:20.000,
seg-6-v1-a1.ts
#EXTINF:20.000,
seg-7-v1-a1.ts
#EXTINF:20.000,
seg-8-v1-a1.ts
#EXTINF:20.000,
seg-9-v1-a1.ts
#EXTINF:20.000,
seg-10-v1-a1.ts
#EXTINF:20.000,
seg-11-v1-a1.ts
#EXTINF:20.000,
seg-12-v1-a1.ts
#EXTINF:20.000,
seg-13-v1-a1.ts
#EXTINF:20.000,
seg-14-v1-a1.ts
#EXTINF:20.000,
seg-15-v1-a1.ts
#EXTINF:20.000,
seg-16-v1-a1.ts
#EXTINF:20.000,
seg-17-v1-a1.ts
#EXTINF:20.000,
seg-18-v1-a1.ts
#EXTINF:20.000,
seg-19-v1-a1.ts
#EXTINF:20.000,
seg-20-v1-a1.ts
#EXTINF:20.000,
seg-21-v1-a1.ts
#EXTINF:20.000,
seg-22-v1-a1.ts
#EXTINF:20.000,
seg-23-v1-a1.ts
#EXTINF:20.000,
seg-24-v1-a1.ts
#EXTINF:20.000,
seg-25-v1-a1.ts
#EXTINF:20.000,
seg-26-v1-a1.ts
#EXTINF:20.000,
seg-27-v1-a1.ts
#EXTINF:20.000,
seg-28-v1-a1.ts
#EXTINF:20.000,
seg-29-v1-a1.ts
#EXTINF:20.000,
seg-30-v1-a1.ts
#EXTINF:20.000,
seg-31-v1-a1.ts
#EXTINF:20.000,
seg-32-v1-a1.ts
#EXTINF:20.000,
seg-33-v1-a1.ts
#EXTINF:20.000,
seg-34-v1-a1.ts
#EXTINF:20.000,
seg-35-v1-a1.ts
#EXTINF:20.000,
seg-36-v1-a1.ts
#EXTINF:20.000,
seg-37-v1-a1.ts
#EXTINF:20.000,
seg-38-v1-a1.ts
#EXTINF:20.000,
seg-39-v1-a1.ts
#EXTINF:20.000,
seg-40-v1-a1.ts
#EXTINF:20.000,
seg-41-v1-a1.ts
#EXTINF:20.000,
seg-42-v1-a1.ts
#EXTINF:20.000,
seg-43-v1-a1.ts
#EXTINF:20.000,
seg-44-v1-a1.ts
#EXTINF:20.000,
seg-45-v1-a1.ts
#EXTINF:20.000,
seg-46-v1-a1.ts
#EXTINF:20.000,
seg-47-v1-a1.ts
#EXTINF:20.000,
seg-48-v1-a1.ts
#EXTINF:20.000,
seg-49-v1-a1.ts
#EXTINF:20.000,
seg-50-v1-a1.ts
#EXTINF:20.000,
seg-51-v1-a1.ts
#EXTINF:20.000,
seg-52-v1-a1.ts
#EXTINF:20.000,
seg-53-v1-a1.ts
#EXTINF:20.000,
seg-54-v1-a1.ts
#EXTINF:20.000,
seg-55-v1-a1.ts
#EXTINF:20.000,
seg-56-v1-a1.ts
#EXTINF:20.000,
seg-57-v1-a1.ts
#EXTINF:20.000,
seg-58-v1-a1.ts
#EXTINF:20.000,
seg-59-v1-a1.ts
#EXTINF:20.000,
seg-60-v1-a1.ts
#EXTINF:20.000,
seg-61-v1-a1.ts
#EXTINF:20.000,
seg-62-v1-a1.ts
#EXTINF:20.000,
seg-63-v1-a1.ts
#EXTINF:20.000,
seg-64-v1-a1.ts
#EXTINF:20.000,
seg-65-v1-a1.ts
#EXTINF:20.000,
seg-66-v1-a1.ts
#EXTINF:20.000,
seg-67-v1-a1.ts
#EXTINF:20.000,
seg-68-v1-a1.ts
#EXTINF:20.000,
seg-69-v1-a1.ts
#EXTINF:20.000,
seg-70-v1-a1.ts
#EXTINF:20.000,
seg-71-v1-a1.ts
#EXTINF:20.000,
seg-72-v1-a1.ts
#EXTINF:20.000,
seg-73-v1-a1.ts
#EXTINF:20.000,
seg-74-v1-a1.ts
#EXTINF:20.000,
seg-75-v1-a1.ts
#EXTINF:20.000,
seg-76-v1-a1.ts
#EXTINF:20.000,
seg-77-v1-a1.ts
#EXTINF:20.000,
seg-78-v1-a1.ts
#EXTINF:20.000,
seg-79-v1-a1.ts
#EXTINF:20.000,
seg-80-v1-a1.ts
#EXTINF:20.000,
seg-81-v1-a1.ts
#EXTINF:20.000,
seg-82-v1-a1.ts
#EXTINF:20.000,
seg-83-v1-a1.ts
#EXTINF:20.000,
seg-84-v1-a1.ts
#EXTINF:20.000,
seg-85-v1-a1.ts
#EXTINF:20.000,
seg-86-v1-a1.ts
#EXTINF:20.000,
seg-87-v1-a1.ts
#EXTINF:20.000,
seg-88-v1-a1.ts
#EXTINF:20.000,
seg-89-v1-a1.ts
#EXTINF:20.000,
seg-90-v1-a1.ts
#EXTINF:20.000,
seg-91-v1-a1.ts
#EXTINF:20.000,
seg-92-v1-a1.ts
#EXTINF:20.000,
seg-93-v1-a1.ts
#EXTINF:20.000,
seg-94-v1-a1.ts
#EXTINF:20.000,
seg-95-v1-a1.ts
#EXTINF:20.000,
seg-96-v1-a1.ts
#EXTINF:20.000,
seg-97-v1-a1.ts
#EXTINF:20.000,
seg-98-v1-a1.ts
#EXTINF:20.000,
seg-99-v1-a1.ts
#EXTINF:20.000,
seg-100-v1-a1.ts
#EXTINF:20.000,
seg-101-v1-a1.ts
#EXTINF:20.000,
seg-102-v1-a1.ts
#EXTINF:20.000,
seg-103-v1-a1.ts
#EXTINF:20.000,
seg-104-v1-a1.ts
#EXTINF:20.000,
seg-105-v1-a1.ts
#EXTINF:20.000,
seg-106-v1-a1.ts
#EXTINF:20.000,
seg-107-v1-a1.ts
#EXTINF:20.000,
seg-108-v1-a1.ts
#EXTINF:20.000,
seg-109-v1-a1.ts
#EXTINF:20.000,
seg-110-v1-a1.ts
#EXTINF:20.000,
seg-111-v1-a1.ts
#EXTINF:20.000,
seg-112-v1-a1.ts
#EXTINF:20.000,
seg-113-v1-a1.ts
#EXTINF:20.000,
seg-114-v1-a1.ts
#EXTINF:20.000,
seg-115-v1-a1.ts
#EXTINF:20.000,
seg-116-v1-a1.ts
#EXTINF:20.000,
seg-117-v1-a1.ts
#EXTINF:20.000,
seg-118-v1-a1.ts
#EXTINF:20.000,
seg-119-v1-a1.ts
#EXTINF:20.000,
seg-120-v1-a1.ts
#EXTINF:20.000,
seg-121-v1-a1.ts
#EXTINF:20.000,
seg-122-v1-a1.ts
#EXTINF:20.000,
seg-123-v1-a1.ts
#EXTINF:20.000,
seg-124-v1-a1.ts
#EXTINF:20.000,
seg-125-v1-a1.ts
#EXTINF:20.000,
seg-126-v1-a1.ts
#EXTINF:20.000,
seg-127-v1-a1.ts
#EXTINF:20.000,
seg-128-v1-a1.ts
#EXTINF:20.000,
seg-129-v1-a1.ts
#EXTINF:20.000,
seg-130-v1-a1.ts
#EXTINF:20.000,
seg-131-v1-a1.ts
#EXTINF:20.000,
seg-132-v1-a1.ts
#EXTINF:20.000,
seg-133-v1-a1.ts
#EXTINF:20.000,
seg-134-v1-a1.ts
#EXTINF:20.000,
seg-135-v1-a1.ts
#EXTINF:20.000,
seg-136-v1-a1.ts
#EXTINF:20.000,
seg-137-v1-a1.ts
#EXTINF:4.111,
seg-138-v1-a1.ts
#EXT-X-ENDLIST
+3
View File
@@ -1,5 +1,8 @@
package http
// DefaultUserAgent is shared by HTTP downloads and extension HTTP requests.
const DefaultUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36"
type config struct {
UserAgent string `json:"userAgent"`
Connections int `json:"connections"`
+234 -63
View File
@@ -2,8 +2,13 @@ package http
import (
"context"
"crypto/md5"
"crypto/sha1"
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"hash"
"io"
"net/http"
"net/url"
@@ -150,7 +155,7 @@ type slowStartController struct {
totalLaunched int
batchPending int // Connections in current batch waiting for HTTP response
batchReady int // Connections in current batch that succeeded
nextBatchSize int // Next batch size: 1, 2, 4, 8...
nextBatchSize int // Next batch size: 1, 2, 4, 16, 256... (capped)
expansionCh chan struct{} // Signal to trigger next expansion
paused bool // Pause expansion (e.g., on 429)
}
@@ -229,7 +234,15 @@ func (s *slowStartController) commitBatch(count int) {
defer s.mu.Unlock()
s.totalLaunched += count
s.nextBatchSize = s.nextBatchSize * 2 // Exponential growth: 1, 2, 4, 8...
if s.nextBatchSize == 1 {
// Bootstrap growth: squaring one would never increase the batch size.
s.nextBatchSize = min(2, s.maxConnections)
} else if s.nextBatchSize > s.maxConnections/s.nextBatchSize {
// Cap before multiplying to avoid integer overflow.
s.nextBatchSize = s.maxConnections
} else {
s.nextBatchSize *= s.nextBatchSize
}
s.batchPending = count
s.batchReady = 0
}
@@ -272,6 +285,12 @@ type Fetcher struct {
primaryReadyOnce sync.Once
primaryReadyCh chan struct{}
// checksumFailed marks that the last completed download failed
// checksum verification, so the next retry must force a full
// re-download instead of skipping connections that already
// reported Completed.
checksumFailed atomic.Bool
// Start pending mechanism
startPending atomic.Bool
resolvedCh chan struct{} // Signal when resolve completes
@@ -363,20 +382,9 @@ func (f *Fetcher) Resolve(req *base.Request, opts *base.Options) error {
f.meta.Opts = &base.Options{}
}
// Parse options
if err := base.ParseOptExtra[fhttp.OptsExtra](opts); err != nil {
if err := f.initOptions(); err != nil {
return err
}
if opts.Extra == nil {
opts.Extra = &fhttp.OptsExtra{}
}
extra := opts.Extra.(*fhttp.OptsExtra)
if extra.Connections <= 0 {
extra.Connections = f.config.Connections
if extra.Connections <= 0 {
extra.Connections = 1
}
}
f.setState(stateResolving)
@@ -420,12 +428,26 @@ func (f *Fetcher) Resolve(req *base.Request, opts *base.Options) error {
res.Range = true
}
// Get content length from Content-Length header
contentLength := resp.Header.Get(base.HttpHeaderContentLength)
if contentLength != "" {
parse, err := strconv.ParseInt(contentLength, 10, 64)
if err == nil {
res.Size = parse
// A partial response describes one segment, not the whole resource.
reuseResolveBody := true
if resp.StatusCode == base.HttpCodePartialContent {
start, _, total, err := parseRangeResponse(resp)
if err != nil {
resp.Body.Close()
f.setState(stateError)
return err
}
res.Size = total
res.Range = true
// Prefetch is a contiguous prefix copied at offset zero. A segment
// starting elsewhere must be fetched again through the Range path.
reuseResolveBody = start == 0
} else {
contentLength := resp.Header.Get(base.HttpHeaderContentLength)
if contentLength != "" {
if size, err := strconv.ParseInt(contentLength, 10, 64); err == nil {
res.Size = size
}
}
}
@@ -459,6 +481,7 @@ func (f *Fetcher) Resolve(req *base.Request, opts *base.Options) error {
if file.Name == "" || file.Name == "/" || file.Name == "." {
file.Name = httpReq.URL.Hostname()
}
file.Name = appendFilenameExtension(file.Name, resp.Header.Get("Content-Type"))
res.Files = append(res.Files, file)
f.meta.Res = res
@@ -469,7 +492,12 @@ func (f *Fetcher) Resolve(req *base.Request, opts *base.Options) error {
// IMPORTANT: Keep the response body open for downloading in Start phase
// This is crucial for one-time URLs that can only be accessed once
f.resolveRespLock.Lock()
f.resolveResp = resp
if reuseResolveBody {
f.resolveResp = resp
} else {
resp.Body.Close()
f.prefetchDone.Store(true)
}
f.resolveRespLock.Unlock()
f.setState(stateResolved)
@@ -481,7 +509,7 @@ func (f *Fetcher) Resolve(req *base.Request, opts *base.Options) error {
// Start async prefetch in background (only for range-supported resources)
// For non-range resources, the response will be used directly in Start
if res.Range {
if res.Range && reuseResolveBody {
f.prefetchStopCh = make(chan struct{})
f.resolveFallback.Store(true)
go f.asyncPrefetch()
@@ -511,17 +539,9 @@ func (f *Fetcher) asyncPrefetch() {
return
}
// Create temporary file for prefetch data
tmpFile, err := os.CreateTemp("", "gopeed-prefetch-*")
if err != nil {
f.prefetchErr = err
return
}
f.prefetchFile = tmpFile
f.prefetchFilePath = tmpFile.Name()
defer func() {
// Close response body when prefetch stops
// Close the response even if temporary-file setup fails or shutdown
// rejects a late prefetch. Otherwise the server can remain blocked writing.
f.resolveRespLock.Lock()
if f.resolveResp != nil {
f.resolveResp.Body.Close()
@@ -530,8 +550,33 @@ func (f *Fetcher) asyncPrefetch() {
f.resolveRespLock.Unlock()
}()
// Create temporary file for prefetch data
tempDir := ""
if f.ctl != nil && f.ctl.TempDir != "" {
tempDir = f.ctl.TempDir
if err := os.MkdirAll(tempDir, 0700); err != nil {
f.prefetchErr = err
return
}
}
tmpFile, err := os.CreateTemp(tempDir, "gopeed-prefetch-*")
if err != nil {
f.prefetchErr = err
return
}
if f.ctl != nil {
if err := f.ctl.TempFiles.Track(tmpFile.Name()); err != nil {
tmpFile.Close()
os.Remove(tmpFile.Name())
f.prefetchErr = err
return
}
}
f.prefetchFile = tmpFile
f.prefetchFilePath = tmpFile.Name()
buf := make([]byte, 32*1024) // 32KB buffer
reader := NewTimeoutReader(resp.Body, readTimeout)
reader := f.responseReader(resp)
for {
select {
@@ -616,7 +661,11 @@ func (f *Fetcher) cleanupPrefetchFile() {
f.prefetchFile = nil
}
if f.prefetchFilePath != "" {
os.Remove(f.prefetchFilePath)
if f.ctl != nil {
f.ctl.TempFiles.Remove(f.prefetchFilePath)
} else {
os.Remove(f.prefetchFilePath)
}
f.prefetchFilePath = ""
}
// The byte count is only reusable while the backing prefetch file exists.
@@ -626,7 +675,31 @@ func (f *Fetcher) cleanupPrefetchFile() {
f.resolveFallback.Store(false)
}
// Options may be replaced between Resolve and Start when creating a resolved task.
func (f *Fetcher) initOptions() error {
opts := f.meta.Opts
// Parse options
if err := base.ParseOptExtra[fhttp.OptsExtra](opts); err != nil {
return err
}
if opts.Extra == nil {
opts.Extra = &fhttp.OptsExtra{}
}
extra := opts.Extra.(*fhttp.OptsExtra)
if extra.Connections <= 0 {
extra.Connections = f.config.Connections
if extra.Connections <= 0 {
extra.Connections = 1
}
}
return nil
}
func (f *Fetcher) Start() error {
if err := f.initOptions(); err != nil {
return err
}
state := f.getState()
switch state {
@@ -673,12 +746,26 @@ func (f *Fetcher) doStart() error {
// If retrying after error, reset connection states for retry
if state == stateError {
forceFullReset := f.checksumFailed.Swap(false)
if forceFullReset {
// Checksum mismatch: the file content itself is wrong even
// though every connection reported Completed. Truncate the
// corrupt file so stale bytes cannot linger if the retry is
// interrupted before every connection re-writes its chunk.
if err := os.Truncate(f.meta.SingleFilepath(), 0); err != nil && !os.IsNotExist(err) {
return fmt.Errorf("failed to reset file after checksum mismatch: %w", err)
}
}
f.connMu.Lock()
for _, conn := range f.connections {
// Reset connections that can be retried
if !conn.Completed && conn.State != connCompleted {
// On a normal error, only reset connections that didn't
// finish. On a checksum mismatch, reset EVERY connection —
// "Completed" only means the download loop finished, not
// that the bytes were verified correct.
shouldReset := forceFullReset || (!conn.Completed && conn.State != connCompleted)
if shouldReset {
if !f.hasSequentialPrefixLocked(conn) {
f.resetConnectionForRestart(conn)
f.resetConnectionForRestart(conn, forceFullReset)
}
conn.State = connNotStarted
conn.failed = false
@@ -908,8 +995,14 @@ func (f *Fetcher) expandConnections() {
}
f.fileMu.Unlock()
f.setState(stateDone)
f.doneCh <- nil
var err error
if chkErr := f.verifyChecksum(); chkErr != nil {
err = chkErr
f.setState(stateError)
} else {
f.setState(stateDone)
}
f.doneCh <- err
return
}
@@ -1045,7 +1138,7 @@ func (f *Fetcher) runConnection(conn *connection) {
// it only when no safe validator exists or If-Range returns a full 200.
f.connMu.Lock()
if !f.hasSequentialPrefixLocked(conn) {
f.resetConnectionForRestart(conn)
f.resetConnectionForRestart(conn, false)
}
f.connMu.Unlock()
}
@@ -1152,7 +1245,7 @@ func (f *Fetcher) downloadChunkOnce(conn *connection, client *http.Client, buf [
resumeProbe := f.canProbeSequentialResumeLocked(conn)
intentionalRestart := sequentialSizeUnknown
if sequentialSizeUnknown || (hasSequentialPrefix && !resumeProbe) {
f.resetConnectionForRestart(conn)
f.resetConnectionForRestart(conn, false)
f.resolveDataPos.Store(0)
f.rangeValidatorPinned = false
intentionalRestart = true
@@ -1328,7 +1421,7 @@ func (f *Fetcher) downloadChunkOnce(conn *connection, client *http.Client, buf [
f.slowStart.onConnectSuccess()
}
reader := NewTimeoutReader(resp.Body, readTimeout)
reader := f.responseReader(resp)
var responseBytesRead int64
for {
if conn.ctx.Err() != nil {
@@ -1501,27 +1594,41 @@ func (f *Fetcher) completeFromResolveFallback(conn *connection) bool {
return true
}
func validateRangeResponse(resp *http.Response, requestedStart int64, requestedEnd *int64, totalSize int64) (int64, int64, error) {
// parseRangeResponse validates a single byte range with a known complete length.
// Unknown totals are rejected: a segment length cannot stand in for resource size.
func parseRangeResponse(resp *http.Response) (int64, int64, int64, error) {
contentRange := resp.Header.Get(base.HttpHeaderContentRange)
fields := strings.Fields(contentRange)
if len(fields) != 2 || !strings.EqualFold(fields[0], base.HttpHeaderBytes) {
return 0, 0, fmt.Errorf("%w: malformed Content-Range %q", errInvalidRangeResponse, contentRange)
return 0, 0, 0, fmt.Errorf("%w: malformed Content-Range %q", errInvalidRangeResponse, contentRange)
}
rangeAndTotal := strings.Split(fields[1], "/")
if len(rangeAndTotal) != 2 {
return 0, 0, fmt.Errorf("%w: malformed Content-Range %q", errInvalidRangeResponse, contentRange)
return 0, 0, 0, fmt.Errorf("%w: malformed Content-Range %q", errInvalidRangeResponse, contentRange)
}
bounds := strings.Split(rangeAndTotal[0], "-")
if len(bounds) != 2 {
return 0, 0, fmt.Errorf("%w: malformed Content-Range %q", errInvalidRangeResponse, contentRange)
return 0, 0, 0, fmt.Errorf("%w: malformed Content-Range %q", errInvalidRangeResponse, contentRange)
}
start, startErr := strconv.ParseInt(bounds[0], 10, 64)
end, endErr := strconv.ParseInt(bounds[1], 10, 64)
total, totalErr := strconv.ParseInt(rangeAndTotal[1], 10, 64)
if startErr != nil || endErr != nil || totalErr != nil || start < 0 || end < start || total <= end {
return 0, 0, fmt.Errorf("%w: malformed Content-Range %q", errInvalidRangeResponse, contentRange)
return 0, 0, 0, fmt.Errorf("%w: malformed Content-Range %q", errInvalidRangeResponse, contentRange)
}
expectedLength := end - start + 1
if resp.ContentLength >= 0 && resp.ContentLength != expectedLength {
return 0, 0, 0, fmt.Errorf("%w: Content-Length %d does not match range length %d", errInvalidRangeResponse, resp.ContentLength, expectedLength)
}
return start, end, total, nil
}
func validateRangeResponse(resp *http.Response, requestedStart int64, requestedEnd *int64, totalSize int64) (int64, int64, error) {
start, end, total, err := parseRangeResponse(resp)
if err != nil {
return 0, 0, err
}
if start != requestedStart || (requestedEnd != nil && end != *requestedEnd) || (requestedEnd == nil && end != total-1) {
return 0, 0, fmt.Errorf("%w: Content-Range %d-%d does not match requested range", errInvalidRangeResponse, start, end)
@@ -1530,11 +1637,7 @@ func validateRangeResponse(resp *http.Response, requestedStart int64, requestedE
return 0, 0, fmt.Errorf("%w: Content-Range total %d does not match resource size %d", errInvalidRangeResponse, total, totalSize)
}
expectedLength := end - start + 1
if resp.ContentLength >= 0 && resp.ContentLength != expectedLength {
return 0, 0, fmt.Errorf("%w: Content-Length %d does not match range length %d", errInvalidRangeResponse, resp.ContentLength, expectedLength)
}
return expectedLength, total, nil
return end - start + 1, total, nil
}
func (f *Fetcher) resolveOpenEndedRange(conn *connection, totalSize int64) error {
@@ -1587,7 +1690,7 @@ func (f *Fetcher) fallbackToSequentialDownload(conn *connection, ifRange string)
f.rangeReprobeEligible = true
f.rangeValidatorPinned = false
f.ifRange = ifRange
f.resetConnectionForRestart(conn)
f.resetConnectionForRestart(conn, false)
f.resolveDataPos.Store(0)
return nil
}
@@ -1603,7 +1706,7 @@ func (f *Fetcher) restartSequentialDownload(conn *connection, ifRange string) {
f.rangeReprobeEligible = true
f.rangeValidatorPinned = false
f.ifRange = ifRange
f.resetConnectionForRestart(conn)
f.resetConnectionForRestart(conn, false)
f.resolveDataPos.Store(0)
}
@@ -1710,7 +1813,7 @@ func (f *Fetcher) runConnectionWithResolveResp(conn *connection) {
}
// Download data from resolve response
reader := NewTimeoutReader(resp.Body, readTimeout)
reader := f.responseReader(resp)
for {
if conn.ctx.Err() != nil {
return
@@ -1827,7 +1930,7 @@ func (f *Fetcher) runConnectionFallback(conn *connection) {
f.slowStart.onConnectSuccess()
}
reader := NewTimeoutReader(resp.Body, readTimeout)
reader := f.responseReader(resp)
for {
if conn.ctx.Err() != nil {
return conn.ctx.Err()
@@ -1986,16 +2089,25 @@ func (f *Fetcher) helpOtherConnection(helper *connection) bool {
return true
}
func (f *Fetcher) resetConnectionForRestart(conn *connection) {
if f.meta.Res.Range {
func (f *Fetcher) resetConnectionForRestart(conn *connection, force bool) {
if f.meta.Res.Range && !force {
return
}
if f.meta.Res.Range && force {
// Checksum mismatch: the on-disk content for this chunk is
// confirmed wrong, so its bytes must be re-fetched from the
// origin server even though range support would normally let
// us just resume from the existing Downloaded position.
if conn.Chunk != nil {
conn.Chunk.Downloaded = 0
}
}
// Without range support a new request always starts from byte 0,
// so pause/retry must restart instead of continuing from the old offset.
if conn.Chunk == nil {
conn.Chunk = newChunk(0, 0)
} else {
} else if !f.meta.Res.Range {
conn.Chunk.Begin = 0
conn.Chunk.setEnd(0)
conn.Chunk.Downloaded = 0
@@ -2059,7 +2171,7 @@ func (f *Fetcher) resumeConnections() {
}
}
if !f.hasSequentialPrefixLocked(conn) {
f.resetConnectionForRestart(conn)
f.resetConnectionForRestart(conn, false)
}
// Reset the connection state for resume
conn.ctx, conn.cancel = context.WithCancel(f.ctx)
@@ -2166,10 +2278,15 @@ func (f *Fetcher) onDownloadComplete() {
}
f.fileMu.Unlock()
if finalErr != nil {
f.setState(stateError)
if finalErr == nil {
if chkErr := f.verifyChecksum(); chkErr != nil {
finalErr = chkErr
f.setState(stateError)
} else {
f.setState(stateDone)
}
} else {
f.setState(stateDone)
f.setState(stateError)
}
select {
@@ -2178,6 +2295,51 @@ func (f *Fetcher) onDownloadComplete() {
}
}
func (f *Fetcher) verifyChecksum() error {
if f.meta == nil || f.meta.Opts == nil || f.meta.Opts.Extra == nil {
return nil
}
extra, ok := f.meta.Opts.Extra.(*fhttp.OptsExtra)
if !ok || extra.Checksum == nil {
return nil
}
chk := extra.Checksum
if chk.Algorithm == "" && chk.Expected == "" {
return nil
}
var h hash.Hash
switch strings.ToLower(strings.TrimSpace(chk.Algorithm)) {
case "md5":
h = md5.New()
case "sha1", "sha-1":
h = sha1.New()
case "sha256", "sha-256":
h = sha256.New()
default:
return fmt.Errorf("unsupported checksum algorithm: %s", chk.Algorithm)
}
filePath := f.meta.SingleFilepath()
file, err := os.Open(filePath)
if err != nil {
return fmt.Errorf("open file for checksum verification failed: %w", err)
}
defer file.Close()
buf := make([]byte, 64*1024)
if _, err := io.CopyBuffer(h, file, buf); err != nil {
return fmt.Errorf("calculate checksum failed: %w", err)
}
actual := hex.EncodeToString(h.Sum(nil))
expected := strings.ToLower(strings.TrimSpace(chk.Expected))
if !strings.EqualFold(actual, expected) {
f.checksumFailed.Store(true)
return fmt.Errorf("checksum mismatch: expected %s, got %s", expected, actual)
}
return nil
}
func (f *Fetcher) checkCompletion() bool {
// Check if all data has been downloaded
f.connMu.Lock()
@@ -2450,3 +2612,12 @@ func (f *Fetcher) Progress() fetcher.Progress {
func (f *Fetcher) Wait() error {
return <-f.doneCh
}
// Generated output can pause while its producer waits for processing capacity.
// Its producer owns upstream deadlines; cancellation still closes the response.
func (f *Fetcher) responseReader(resp *http.Response) io.Reader {
if f.ctl != nil && f.ctl.ManagedProduction != nil && resp.Request != nil && f.ctl.ManagedProduction(resp.Request.URL.String()) {
return resp.Body
}
return NewTimeoutReader(resp.Body, readTimeout)
}
@@ -0,0 +1,122 @@
package http
import (
"crypto/sha256"
"encoding/hex"
"fmt"
gohttp "net/http"
"net/http/httptest"
"os"
"strings"
"sync/atomic"
"testing"
"time"
"github.com/GopeedLab/gopeed/pkg/base"
"github.com/GopeedLab/gopeed/pkg/protocol/http"
)
func TestFetcher_ChecksumMismatch_Retry(t *testing.T) {
corruptContent := []byte("Corrupt Initial Download Data!")
correctContent := []byte("Correct Retried Download Data!")
correctSha256Hasher := sha256.New()
correctSha256Hasher.Write(correctContent)
correctSha256 := hex.EncodeToString(correctSha256Hasher.Sum(nil))
var reqCount atomic.Int32
server := httptest.NewServer(gohttp.HandlerFunc(func(w gohttp.ResponseWriter, r *gohttp.Request) {
count := reqCount.Add(1)
var served []byte
if count == 1 {
// First request: serve corrupted content
served = corruptContent
} else {
// Second / subsequent request: serve correct content
served = correctContent
}
w.Header().Set(base.HttpHeaderContentLength, fmt.Sprintf("%d", len(served)))
w.Header().Set(base.HttpHeaderAcceptRanges, base.HttpHeaderBytes)
w.WriteHeader(gohttp.StatusOK)
_, _ = w.Write(served)
}))
defer server.Close()
downloadDir := t.TempDir()
fileName := "test_checksum_retry.txt"
f := buildFetcher()
defer f.Close()
opts := &base.Options{
Path: downloadDir,
Name: fileName,
Extra: &http.OptsExtra{
Connections: 1,
Checksum: &http.ChecksumOption{
Algorithm: "sha256",
Expected: correctSha256,
},
},
}
// 1. Initial attempt -> will download corruptContent, failing checksum verification
if err := f.Resolve(&base.Request{URL: server.URL + "/retry_test.txt"}, opts); err != nil {
t.Fatalf("Resolve error: %v", err)
}
// Ensure the reusable resolve response has been fully prefetched so the
// first attempt deterministically verifies the corrupt initial payload.
deadline := time.NewTimer(5 * time.Second)
defer deadline.Stop()
ticker := time.NewTicker(10 * time.Millisecond)
defer ticker.Stop()
for !f.prefetchDone.Load() {
select {
case <-deadline.C:
t.Fatal("timed out waiting for checksum test prefetch")
case <-ticker.C:
}
}
if err := f.Start(); err != nil {
t.Fatalf("Start error: %v", err)
}
err := f.Wait()
if err == nil {
t.Fatal("expected checksum mismatch error on initial attempt, got nil")
}
if !strings.Contains(err.Error(), "checksum mismatch") {
t.Fatalf("expected 'checksum mismatch' in error, got %v", err)
}
if f.getState() != stateError {
t.Fatalf("expected stateError after checksum mismatch, got %v", f.getState())
}
// 2. Retry download -> should trigger forced full reset, truncate file, and re-download correctContent
if err := f.Start(); err != nil {
t.Fatalf("Start error on retry: %v", err)
}
if err := f.Wait(); err != nil {
t.Fatalf("Wait error on retry (expected success): %v", err)
}
if f.getState() != stateDone {
t.Fatalf("expected stateDone after successful retry, got %v", f.getState())
}
// 3. Verify on-disk file content matches the re-downloaded bytes
filePath := f.meta.SingleFilepath()
fileBytes, err := os.ReadFile(filePath)
if err != nil {
t.Fatalf("failed to read completed file: %v", err)
}
if string(fileBytes) != string(correctContent) {
t.Fatalf("on-disk file content mismatch:\ngot: %q\nwant: %q", string(fileBytes), string(correctContent))
}
// 4. Compute hash independently and verify
actualHasher := sha256.New()
actualHasher.Write(fileBytes)
actualHash := hex.EncodeToString(actualHasher.Sum(nil))
if actualHash != correctSha256 {
t.Fatalf("re-computed on-disk hash mismatch: got %s, want %s", actualHash, correctSha256)
}
}
@@ -0,0 +1,307 @@
package http
import (
"crypto/md5"
"crypto/sha256"
"encoding/hex"
"fmt"
gohttp "net/http"
"net/http/httptest"
"strings"
"testing"
"github.com/GopeedLab/gopeed/internal/fetcher"
"github.com/GopeedLab/gopeed/internal/test"
"github.com/GopeedLab/gopeed/pkg/base"
"github.com/GopeedLab/gopeed/pkg/protocol/http"
)
func TestFetcher_ChecksumVerification(t *testing.T) {
content := []byte("Hello, Gopeed Checksum Verification Test Data!")
md5Hasher := md5.New()
md5Hasher.Write(content)
correctMd5 := hex.EncodeToString(md5Hasher.Sum(nil))
sha256Hasher := sha256.New()
sha256Hasher.Write(content)
correctSha256 := hex.EncodeToString(sha256Hasher.Sum(nil))
wrongHash := "0123456789abcdef0123456789abcdef"
server := httptest.NewServer(gohttp.HandlerFunc(func(w gohttp.ResponseWriter, r *gohttp.Request) {
w.Header().Set(base.HttpHeaderContentLength, fmt.Sprintf("%d", len(content)))
w.Header().Set(base.HttpHeaderAcceptRanges, base.HttpHeaderBytes)
w.WriteHeader(gohttp.StatusOK)
_, _ = w.Write(content)
}))
defer server.Close()
t.Run("Valid MD5", func(t *testing.T) {
f := buildFetcher()
defer f.Close()
opts := &base.Options{
Path: t.TempDir(),
Name: "test_valid_md5.txt",
Extra: &http.OptsExtra{
Connections: 1,
Checksum: &http.ChecksumOption{
Algorithm: "md5",
Expected: correctMd5,
},
},
}
if err := f.Resolve(&base.Request{URL: server.URL + "/test.txt"}, opts); err != nil {
t.Fatalf("Resolve error: %v", err)
}
if err := f.Start(); err != nil {
t.Fatalf("Start error: %v", err)
}
if err := f.Wait(); err != nil {
t.Fatalf("Wait error (expected success): %v", err)
}
})
t.Run("Valid SHA256", func(t *testing.T) {
f := buildFetcher()
defer f.Close()
opts := &base.Options{
Path: t.TempDir(),
Name: "test_valid_sha256.txt",
Extra: &http.OptsExtra{
Connections: 1,
Checksum: &http.ChecksumOption{
Algorithm: "sha256",
Expected: correctSha256,
},
},
}
if err := f.Resolve(&base.Request{URL: server.URL + "/test.txt"}, opts); err != nil {
t.Fatalf("Resolve error: %v", err)
}
if err := f.Start(); err != nil {
t.Fatalf("Start error: %v", err)
}
if err := f.Wait(); err != nil {
t.Fatalf("Wait error (expected success): %v", err)
}
})
t.Run("Wrong MD5 mismatch", func(t *testing.T) {
f := buildFetcher()
defer f.Close()
opts := &base.Options{
Path: t.TempDir(),
Name: "test_wrong_md5.txt",
Extra: &http.OptsExtra{
Connections: 1,
Checksum: &http.ChecksumOption{
Algorithm: "md5",
Expected: wrongHash,
},
},
}
if err := f.Resolve(&base.Request{URL: server.URL + "/test.txt"}, opts); err != nil {
t.Fatalf("Resolve error: %v", err)
}
if err := f.Start(); err != nil {
t.Fatalf("Start error: %v", err)
}
err := f.Wait()
if err == nil {
t.Fatal("Wait expected checksum mismatch error, got nil")
}
if !strings.Contains(err.Error(), "checksum mismatch") {
t.Fatalf("expected 'checksum mismatch' in error, got %v", err)
}
})
t.Run("Unsupported Algorithm", func(t *testing.T) {
f := buildFetcher()
defer f.Close()
opts := &base.Options{
Path: t.TempDir(),
Name: "test_unsupported.txt",
Extra: &http.OptsExtra{
Connections: 1,
Checksum: &http.ChecksumOption{
Algorithm: "invalid-algo",
Expected: wrongHash,
},
},
}
if err := f.Resolve(&base.Request{URL: server.URL + "/test.txt"}, opts); err != nil {
t.Fatalf("Resolve error: %v", err)
}
if err := f.Start(); err != nil {
t.Fatalf("Start error: %v", err)
}
err := f.Wait()
if err == nil {
t.Fatal("Wait expected error for unsupported algorithm, got nil")
}
if !strings.Contains(err.Error(), "unsupported checksum algorithm") {
t.Fatalf("expected 'unsupported checksum algorithm' in error, got %v", err)
}
if f.checksumFailed.Load() {
t.Fatal("checksumFailed flag should NOT be set on unsupported algorithm error")
}
})
t.Run("Case-Insensitive Hash Comparison", func(t *testing.T) {
f := buildFetcher()
defer f.Close()
opts := &base.Options{
Path: t.TempDir(),
Name: "test_case_insensitive.txt",
Extra: &http.OptsExtra{
Connections: 1,
Checksum: &http.ChecksumOption{
Algorithm: "sha256",
Expected: strings.ToUpper(correctSha256),
},
},
}
if err := f.Resolve(&base.Request{URL: server.URL + "/test.txt"}, opts); err != nil {
t.Fatalf("Resolve error: %v", err)
}
if err := f.Start(); err != nil {
t.Fatalf("Start error: %v", err)
}
if err := f.Wait(); err != nil {
t.Fatalf("Wait error (expected success with upper case hash): %v", err)
}
})
t.Run("Empty Expected Checksum (Skipped)", func(t *testing.T) {
f := buildFetcher()
defer f.Close()
opts := &base.Options{
Path: t.TempDir(),
Name: "test_empty_expected.txt",
Extra: &http.OptsExtra{
Connections: 1,
Checksum: &http.ChecksumOption{
Algorithm: "",
Expected: "",
},
},
}
if err := f.Resolve(&base.Request{URL: server.URL + "/test.txt"}, opts); err != nil {
t.Fatalf("Resolve error: %v", err)
}
if err := f.Start(); err != nil {
t.Fatalf("Start error: %v", err)
}
if err := f.Wait(); err != nil {
t.Fatalf("Wait error (expected skip and success): %v", err)
}
})
t.Run("Large File Multi-Chunk Streaming (>64KB)", func(t *testing.T) {
largeData := make([]byte, 128*1024)
for i := range largeData {
largeData[i] = byte(i % 256)
}
largeSha256Hasher := sha256.New()
largeSha256Hasher.Write(largeData)
largeSha256 := hex.EncodeToString(largeSha256Hasher.Sum(nil))
largeServer := httptest.NewServer(gohttp.HandlerFunc(func(w gohttp.ResponseWriter, r *gohttp.Request) {
w.Header().Set(base.HttpHeaderContentLength, fmt.Sprintf("%d", len(largeData)))
w.Header().Set(base.HttpHeaderAcceptRanges, base.HttpHeaderBytes)
w.WriteHeader(gohttp.StatusOK)
_, _ = w.Write(largeData)
}))
defer largeServer.Close()
f := buildFetcher()
defer f.Close()
opts := &base.Options{
Path: t.TempDir(),
Name: "test_large_file.bin",
Extra: &http.OptsExtra{
Connections: 1,
Checksum: &http.ChecksumOption{
Algorithm: "sha256",
Expected: largeSha256,
},
},
}
if err := f.Resolve(&base.Request{URL: largeServer.URL + "/large.bin"}, opts); err != nil {
t.Fatalf("Resolve error: %v", err)
}
if err := f.Start(); err != nil {
t.Fatalf("Start error: %v", err)
}
if err := f.Wait(); err != nil {
t.Fatalf("Wait error (expected large file checksum success): %v", err)
}
})
t.Run("No Checksum (Opt-in backward compatibility)", func(t *testing.T) {
f := buildFetcher()
defer f.Close()
opts := &base.Options{
Path: t.TempDir(),
Name: test.DownloadName,
Extra: &http.OptsExtra{
Connections: 1,
},
}
if err := f.Resolve(&base.Request{URL: server.URL + "/test.txt"}, opts); err != nil {
t.Fatalf("Resolve error: %v", err)
}
if err := f.Start(); err != nil {
t.Fatalf("Start error: %v", err)
}
if err := f.Wait(); err != nil {
t.Fatalf("Wait error: %v", err)
}
})
}
func TestResetConnectionForRestart_ForceOnRangeDownload(t *testing.T) {
f := buildFetcher()
f.meta = &fetcher.FetcherMeta{
Res: &base.Resource{
Range: true,
},
}
conn := &connection{
Chunk: newChunk(0, 1024),
Downloaded: 512,
Completed: true,
}
conn.Chunk.Downloaded = 512
// Non-forced reset on range-supported download does not reset Chunk.Downloaded
f.resetConnectionForRestart(conn, false)
if conn.Chunk.Downloaded != 512 {
t.Fatalf("expected Chunk.Downloaded to remain 512 when force is false, got %d", conn.Chunk.Downloaded)
}
// Forced reset on range-supported download resets Chunk.Downloaded to 0
f.resetConnectionForRestart(conn, true)
if conn.Chunk.Downloaded != 0 {
t.Fatalf("expected Chunk.Downloaded to be reset to 0 when force is true, got %d", conn.Chunk.Downloaded)
}
if conn.Completed {
t.Fatal("expected Completed to be false after reset")
}
}
+1 -1
View File
@@ -72,7 +72,7 @@ func (fm *FetcherManager) AutoRename() bool {
func (fm *FetcherManager) DefaultConfig() any {
return &config{
UserAgent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36",
UserAgent: DefaultUserAgent,
Connections: 16,
}
}
+37
View File
@@ -13,6 +13,7 @@ import (
"net/url"
"os"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
@@ -221,6 +222,30 @@ func TestFetcher_Resolve(t *testing.T) {
})
}
func TestFetcher_ResolveFilenameFromContentType(t *testing.T) {
server := httptest.NewServer(gohttp.HandlerFunc(func(w gohttp.ResponseWriter, r *gohttp.Request) {
w.Header().Set("Content-Type", "image/webp")
w.WriteHeader(gohttp.StatusOK)
}))
defer server.Close()
fetcher := buildFetcher()
defer fetcher.Pause()
const opaqueName = "OIP-C.u5nEXTXND3PJF0gNTgw5YgHaFP"
err := fetcher.Resolve(&base.Request{URL: server.URL + "/" + opaqueName}, &base.Options{
Name: test.DownloadName,
Path: test.Dir,
})
if err != nil {
t.Fatal(err)
}
if got := fetcher.meta.Res.Files[0].Name; got != opaqueName+".webp" {
t.Fatalf("resolved filename = %q, want %q", got, opaqueName+".webp")
}
}
func TestFetcher_ResolveWithHostHeader(t *testing.T) {
listener := test.StartTestHostHeaderServer()
defer listener.Close()
@@ -829,22 +854,34 @@ func TestFetcher_InitialIgnoredRangeUsesResponseSize(t *testing.T) {
resolvePayload := bytes.Repeat([]byte("resolve"), 64*1024)
replacement := bytes.Repeat([]byte("replacement"), 24*1024)
var rangeRequests atomic.Int32
rangeStarted := make(chan struct{})
releaseResolve := sync.OnceFunc(func() { close(rangeStarted) })
server := httptest.NewServer(gohttp.HandlerFunc(func(w gohttp.ResponseWriter, r *gohttp.Request) {
w.Header().Set(base.HttpHeaderAcceptRanges, base.HttpHeaderBytes)
if r.Header.Get(base.HttpHeaderRange) == "" {
w.Header().Set(base.HttpHeaderContentLength, fmt.Sprintf("%d", len(resolvePayload)))
w.WriteHeader(gohttp.StatusOK)
w.(gohttp.Flusher).Flush()
// Keep Resolve prefetch incomplete until the Range request starts.
// Otherwise Start may reuse the completed body and skip this scenario.
select {
case <-rangeStarted:
case <-r.Context().Done():
return
}
_, _ = w.Write(resolvePayload)
return
}
rangeRequests.Add(1)
releaseResolve()
w.Header().Set(base.HttpHeaderContentLength, fmt.Sprintf("%d", len(replacement)))
w.WriteHeader(gohttp.StatusOK)
_, _ = w.Write(replacement)
}))
defer server.Close()
defer releaseResolve()
f := buildFetcher()
if err := f.Resolve(&base.Request{URL: server.URL + "/ignored-range.data"}, &base.Options{
@@ -136,3 +136,52 @@ func TestUnescapeHTMLEntities(t *testing.T) {
})
}
}
func TestAppendFilenameExtension(t *testing.T) {
tests := []struct {
name string
filename string
contentType string
want string
}{
{
name: "adds extension from content type",
filename: "download",
contentType: "image/png",
want: "download.png",
},
{
name: "parses MIME parameters",
filename: "download",
contentType: "image/png; charset=binary",
want: "download.png",
},
{
name: "preserves existing extension",
filename: "download.zip",
contentType: "application/zip",
want: "download.zip",
},
{
name: "ignores invalid content type",
filename: "download",
contentType: "not a content type",
want: "download",
},
{
name: "ignores unknown content type",
filename: "download",
contentType: "application/x-gopeed-unknown",
want: "download",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := appendFilenameExtension(tt.filename, tt.contentType)
if got != tt.want {
t.Errorf("appendFilenameExtension() = %q, want %q", got, tt.want)
}
})
}
}
+23
View File
@@ -12,6 +12,7 @@ import (
"net/http"
"net/http/cookiejar"
"net/url"
"path"
"strings"
"time"
"unicode/utf8"
@@ -314,6 +315,28 @@ func parseFilename(contentDisposition string) string {
return parseFilenameFallback(contentDisposition)
}
// appendFilenameExtension adds a MIME-derived extension when the resolved
// filename has no usable extension. Some image hosts use opaque path segments
// that look like extensions, even though the response Content-Type is the
// only reliable source for the file type.
func appendFilenameExtension(filename, contentType string) string {
mediaType, _, err := mime.ParseMediaType(contentType)
if err != nil {
return filename
}
extensions, err := mime.ExtensionsByType(mediaType)
if err != nil || len(extensions) == 0 {
return filename
}
ext := path.Ext(filename)
if len(ext) > 1 && len(ext) <= 20 {
return filename
}
return filename + extensions[0]
}
// parseFilenameExtended handles RFC 5987 extended notation (filename*=)
// Format: filename*=charset'language'value (e.g., UTF-8”%E6%B5%8B%E8%AF%95.zip)
func parseFilenameExtended(cd string) string {
@@ -0,0 +1,74 @@
package http
import (
"io"
"net/http"
"os"
"path/filepath"
"strings"
"testing"
"github.com/GopeedLab/gopeed/internal/controller"
"github.com/GopeedLab/gopeed/internal/tempfiles"
)
type prefetchSetupBody struct {
io.Reader
closed bool
}
func (b *prefetchSetupBody) Close() error {
b.closed = true
return nil
}
func TestPrefetchSetupFailureClosesResponse(t *testing.T) {
for _, closedScope := range []bool{false, true} {
name := "invalid_temp_directory"
if closedScope {
name = "downloader_already_closed"
}
t.Run(name, func(t *testing.T) {
root := t.TempDir()
scope := &tempfiles.Scope{}
dir := root
if closedScope {
if err := scope.Close(); err != nil {
t.Fatal(err)
}
} else {
dir = filepath.Join(root, "file")
if err := os.WriteFile(dir, nil, 0600); err != nil {
t.Fatal(err)
}
}
body := &prefetchSetupBody{Reader: strings.NewReader("unused response")}
f := &Fetcher{
ctl: &controller.Controller{TempDir: dir, TempFiles: scope},
resolveResp: &http.Response{Body: body},
}
f.asyncPrefetch()
if f.prefetchErr == nil {
t.Fatal("expected temporary-file initialization to fail")
}
if !body.closed {
t.Error("prefetch setup failure left the response body open")
}
if f.resolveResp != nil {
t.Error("prefetch retained its failed resolve response")
}
if !f.prefetchDone.Load() {
t.Error("prefetch did not report completion")
}
if closedScope {
entries, err := os.ReadDir(root)
if err != nil {
t.Fatal(err)
}
if len(entries) != 0 {
t.Errorf("late prefetch left temporary files: %v", entries)
}
}
})
}
}
@@ -0,0 +1,119 @@
package http
import (
"bytes"
"errors"
"fmt"
"net/http"
"net/http/httptest"
"os"
"testing"
"github.com/GopeedLab/gopeed/pkg/base"
fhttp "github.com/GopeedLab/gopeed/pkg/protocol/http"
)
func TestResolveResponseSize(t *testing.T) {
for _, tc := range []struct {
name string
status int
length, contentRange string
size int64
invalid bool
}{
{"full", 200, "801571015", "", 801571015, false},
{"partial_prefix", 206, "472321339", "bytes 0-472321338/801571015", 801571015, false},
{"partial_nonzero", 206, "100000", "bytes 100000-199999/801571015", 801571015, false},
{"second_report", 206, "335134409", "bytes 0-335134408/552140234", 552140234, false},
{"length_mismatch", 206, "10", "bytes 0-19/100", 0, true},
{"unknown_total", 206, "472321339", "bytes 0-472321338/*", 0, true},
{"missing_range", 206, "10", "", 0, true},
{"malformed_range", 206, "10", "bytes invalid", 0, true},
{"invalid_bounds", 206, "10", "bytes 20-10/100", 0, true},
{"invalid_total", 206, "10", "bytes 0-9/9", 0, true},
{"overflow", 206, "10", "bytes 0-9/9223372036854775808", 0, true},
{"chunked_partial", 206, "", "bytes 0-9/100", 100, false},
{"unknown_full", 200, "", "", 0, false},
} {
t.Run(tc.name, func(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if tc.length != "" {
w.Header().Set("Content-Length", tc.length)
}
if tc.contentRange != "" {
w.Header().Set("Content-Range", tc.contentRange)
}
w.WriteHeader(tc.status)
w.(http.Flusher).Flush()
}))
defer server.Close()
f := buildFetcher()
defer f.Close()
err := f.Resolve(&base.Request{URL: server.URL + "/file"}, &base.Options{Path: t.TempDir()})
if tc.invalid {
if !errors.Is(err, errInvalidRangeResponse) {
t.Fatalf("Resolve error = %v, want invalid range", err)
}
if f.meta.Res != nil || f.resolveResp != nil || f.getState() != stateError {
t.Fatal("invalid response was retained or published")
}
return
}
if err != nil {
t.Fatal(err)
}
if f.meta.Res.Size != tc.size || f.meta.Res.Files[0].Size != tc.size {
t.Fatalf("resource/file size = %d/%d, want %d", f.meta.Res.Size, f.meta.Res.Files[0].Size, tc.size)
}
if tc.status == 206 && !f.meta.Res.Range {
t.Fatal("partial response must enable Range downloads")
}
})
}
}
func TestDownloadPartialResolve(t *testing.T) {
payload := make([]byte, 256*1024)
for i := range payload {
payload[i] = byte(i*31 + i/251)
}
for _, start := range []int{0, 100000} {
for _, connections := range []int{1, 4} {
t.Run(fmt.Sprintf("start_%d_connections_%d", start, connections), func(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
begin, end := start, start+9999
if h := r.Header.Get("Range"); h != "" {
if _, err := fmt.Sscanf(h, "bytes=%d-%d", &begin, &end); err != nil {
t.Errorf("invalid request range %q: %v", h, err)
w.WriteHeader(400)
return
}
}
w.Header().Set("Content-Length", fmt.Sprint(end-begin+1))
w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", begin, end, len(payload)))
w.WriteHeader(206)
w.Write(payload[begin : end+1])
}))
defer server.Close()
f := buildFetcher()
defer f.Close()
if err := f.Resolve(&base.Request{URL: server.URL + "/file"}, &base.Options{Path: t.TempDir(), Name: "file", Extra: &fhttp.OptsExtra{Connections: connections}}); err != nil {
t.Fatal(err)
}
if err := f.Start(); err != nil {
t.Fatal(err)
}
if err := f.Wait(); err != nil {
t.Fatal(err)
}
got, err := os.ReadFile(f.meta.SingleFilepath())
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(got, payload) {
t.Fatalf("download content mismatch: got %d bytes, want %d", len(got), len(payload))
}
})
}
}
}
+63
View File
@@ -0,0 +1,63 @@
// Package tempfiles tracks temporary paths owned by one downloader, without
// requiring a private parent directory or scanning the shared system temp root.
package tempfiles
import (
"errors"
"io"
"os"
"sync"
)
type Scope struct {
mu sync.Mutex
paths map[string]struct{}
closed bool
}
// Track adopts a newly created unique path. Late creations are removed when
// shutdown has already begun. A nil scope leaves cleanup to the resource owner.
func (s *Scope) Track(path string) error {
if s == nil {
return nil
}
s.mu.Lock()
defer s.mu.Unlock()
if s.closed {
return errors.Join(io.ErrClosedPipe, os.RemoveAll(path))
}
if s.paths == nil {
s.paths = make(map[string]struct{})
}
s.paths[path] = struct{}{}
return nil
}
func (s *Scope) Remove(path string) error {
if s == nil {
return os.RemoveAll(path)
}
s.mu.Lock()
defer s.mu.Unlock()
if err := os.RemoveAll(path); err != nil {
return err
}
delete(s.paths, path)
return nil
}
func (s *Scope) Close() error {
if s == nil {
return nil
}
s.mu.Lock()
defer s.mu.Unlock()
s.closed = true
var result error
for path := range s.paths {
if err := os.RemoveAll(path); err != nil {
result = errors.Join(result, err)
} else {
delete(s.paths, path)
}
}
return result
}
+76
View File
@@ -0,0 +1,76 @@
package tempfiles
import (
"errors"
"io"
"os"
"sync"
"testing"
)
func TestSharedDirectoryOwnersAndLateCreation(t *testing.T) {
root := t.TempDir()
a, b := &Scope{}, &Scope{}
own := func(scope *Scope) string {
t.Helper()
dir, err := os.MkdirTemp(root, "gopeed-media-")
if err != nil {
t.Fatal(err)
}
if err := scope.Track(dir); err != nil {
t.Fatal(err)
}
return dir
}
first, second := own(a), own(b)
if err := a.Close(); err != nil {
t.Fatal(err)
}
if _, err := os.Stat(first); !os.IsNotExist(err) {
t.Fatal(err)
}
if _, err := os.Stat(second); err != nil {
t.Fatal("removed other owner's path", err)
}
late, err := os.MkdirTemp(root, "gopeed-media-")
if err != nil {
t.Fatal(err)
}
if err := a.Track(late); !errors.Is(err, io.ErrClosedPipe) {
t.Fatal(err)
}
if _, err := os.Stat(late); !os.IsNotExist(err) {
t.Fatal("late creation leaked", err)
}
if err := b.Close(); err != nil {
t.Fatal(err)
}
}
func TestConcurrentCreationAndShutdown(t *testing.T) {
root := t.TempDir()
scope := &Scope{}
var wg sync.WaitGroup
for i := 0; i < 20; i++ {
wg.Add(1)
go func() {
defer wg.Done()
dir, err := os.MkdirTemp(root, "gopeed-media-")
if err != nil {
t.Error(err)
return
}
if err := scope.Track(dir); err != nil && !errors.Is(err, io.ErrClosedPipe) {
t.Error(err)
}
}()
}
if err := scope.Close(); err != nil {
t.Fatal(err)
}
wg.Wait()
entries, err := os.ReadDir(root)
if err != nil || len(entries) != 0 {
t.Fatal("shutdown leaked paths", entries, err)
}
}
+109
View File
@@ -0,0 +1,109 @@
# Extension browser host configuration
The downloader wraps each provider with the extension's stable identity before
constructing the runtime. `OpenOptions.ProfileID`, `DataPath`, and `ProxyURL` are
host-only fields. `parseOpenOptions` never reads them from extension JavaScript.
The extension API is unchanged. The Flutter host uses the fork's public
`WebViewProfile`, `InAppWebViewSettings.profileId`, and profile-scoped
`CookieManager` APIs; the library has no Gopeed-specific method-channel bridge.
Only the Gopeed extension JavaScript boundary keeps these options host-only.
Profiles are persistent and shared across invocations/pages of the same extension.
Different identities use different native stores. Go providers use a SHA-256-based
path under `<StorageDir>/webview`; Apple RPC hosts use a stable UUID data store;
Android uses a named WebView profile. Cookie operations select that same store,
including `clearCookies`. Closing an execution closes its pages, not its profile.
Uninstall closes the extension's pages, prevents its old runtimes from opening
new pages, and removes its native profile before deleting the extension record.
Cleanup failures abort uninstall and can be retried. Reinstallation starts with
empty browser data; an ordinary extension update keeps the existing profile.
Windows/Linux delete the profile directory; Apple removes the named WebKit store.
Android cannot delete a profile already loaded by the current process: when
DELETE_BROWSING_DATA is supported it clears all site data immediately, and records
an empty-profile deletion in SharedPreferences for the next process start. A
reinstallation may reuse that cleared profile and cancels the pending deletion.
If the full-data clearing API is unavailable, uninstall fails with a restart
instruction; the pending profile is deleted at startup before it can be loaded.
No removal operation is exposed through extension JavaScript.
Existing shared browser data is not copied into every extension's new profile.
The downloader lazily owns one loopback forward proxy with HTTP and SOCKS5
listeners. Windows/Linux/Android use HTTP; Apple uses SOCKS5 because its HTTP
CONNECT configuration may bypass plain HTTP. The forwarder reads the current
Gopeed proxy config for each new request/tunnel and supports direct, system,
HTTP(S) upstream proxies with Basic authentication, and authenticated SOCKS5.
Credentials remain in Go and are never sent over the WebView RPC.
Changes affect new requests/connections. Existing CONNECT/SOCKS tunnels keep their
route until closed; opening another page in the same profile does not reset it.
SOCKS carries host/port rather than a full URL: system proxy/PAC selection for
Apple is host-based, with port 80 treated as HTTP and other ports as HTTPS.
Path-dependent PAC rules cannot be reproduced from a SOCKS connection.
Apple requires macOS 14 / iOS 17. Android checks MULTI_PROFILE and PROXY_OVERRIDE.
Unsupported hosts fail explicitly instead of silently sharing browser storage or
ignoring the proxy.
## Validation
- `go test -race ./internal/webview/proxy ./pkg/download/engine/webview`
- `go test ./pkg/download -run 'TestWebViewProfileBound|TestDownloader_ExtensionRuntimeWebView'`
- Windows/Linux: `go test -tags webview -v ./internal/webview/goprovider` (Linux: run under `xvfb-run -a`).
- macOS: start the normal Flutter app, then run `go test -tags webview -v ./internal/webview/rpcprovider` with its socket address. See `rpcprovider/README.md`.
- In webview_go: `go test -tags webview_integration -v -count=1 .` (Linux: run under `xvfb-run -a`).
The shared provider contract covers HTTP/HTTPS proxy routing, concurrent pages,
HttpOnly cookies, LocalStorage, IndexedDB, profile-scoped clearing, reopening,
and removal without affecting another profile. HTTPS routing is checked with a
rejecting upstream tunnel; no test-only certificate bypass is added to the app.
Go proxy unit tests separately cover successful TLS forwarding and authentication.
These cases run in the existing `test.yml` WebView jobs; existing mobile build jobs
compile the Android/iOS bridge. No separate Flutter test application is required.
## Extension events and host environment
`gopeed.host.env.version` is the host's build version (`dev` for development
builds). `gopeed.info.version` continues to identify the extension's own version.
`gopeed.host.env.os` and `gopeed.host.env.arch` match the REST info endpoint's `os` and
`arch`: the host's Go `runtime.GOOS` and `runtime.GOARCH` values (for example,
`windows` / `amd64` or `darwin` / `arm64`).
See [extension-api.d.ts](extension-api.d.ts) for the event payloads.
```js
const page = await gopeed.runtime.webview.open();
const unsubscribe = await page.on("url-changed", ({ url, sameDocument }) => {
gopeed.logger.info(url, sameDocument);
});
await page.on("closed", ({ reason }) => gopeed.logger.info(reason));
await page.addInitScript("globalThis.extensionReady = true");
await page.goto("https://example.com");
unsubscribe();
await page.close();
```
Registration resolves only after the listener is installed; it does not replay
current state. `url-changed` reports actual address changes, including hash,
pushState, replaceState and history navigation. `load` follows the main document's
`window.load`, including reloads, and does not wait for business-level async work.
Iframe loads and subresource failures do not become main-page notifications.
`closed` distinguishes the user's close action from API closure and is terminal.
Unsubscribe is idempotent. Notifications cannot block navigation; the engine does
not await promises returned by handlers. Catch async errors inside the handler.
`addInitScript` applies to future document contexts only. Use `execute` to run
code on the current page. Executable functions are serialized and cannot capture
extension-side lexical variables; pass values explicitly as arguments.
The RPC host uses an NDJSON `page.events` response whose first record is
`{"ready":true}`. Subsequent records are `{event,data}`. Closing a page sends the
terminal record before ending the stream. The native backend uses document-start
notifications for main-document URL/load events and native callbacks for
navigation failures and user closure.
Additional validation:
- `go test -race ./pkg/download/engine/webview ./internal/webview/rpcprovider`
- `go test ./pkg/download -run 'TestExtension(WebViewEvents|InfoHostVersion)'`
- macOS native: `go test -tags webview_native ./internal/webview/goprovider -run TestProviderEvents`
- Running Flutter host: `go test -tags webview ./internal/webview/rpcprovider -run TestProviderEvents`
+60
View File
@@ -0,0 +1,60 @@
/** Public extension API additions. info.version remains the extension version. */
export interface ExtensionInfo {
identity: string;
name: string;
author: string;
title: string;
version: string;
}
/** Available as gopeed.host. */
export interface Host {
env: HostEnv;
}
export interface HostEnv {
/** Host build version; "dev" for development builds. */
version: string;
/** Host OS and architecture, matching the REST info endpoint (Go GOOS/GOARCH). */
os: string;
arch: string;
}
/** Functions are serialized; they cannot capture extension-side variables. */
export type WebviewExecutable<T = unknown> =
| string
| ((...args: unknown[]) => T | Promise<T>);
export type WebviewUnsubscribe = () => void;
/** Navigation and load notifications are for the main document only. */
export interface WebviewEventMap {
"url-changed": { url: string; sameDocument: boolean };
/** window.load, including reloads, excluding hash/History API navigation. */
"load": { url: string };
"load-error": { url: string; message: string };
"closed": { reason: "user" | "api" };
}
export interface WebviewGotoOptions {
timeoutMs?: number;
waitUntil?: "load" | "domcontentloaded";
}
/** These members extend the existing WebviewPage API. */
export interface WebviewPage {
/**
* Resolves when listening is ready. Does not replay current state.
* Register before goto() to observe navigation. Unsubscribe is idempotent.
* Closed is delivered once, then listeners are released automatically.
* Callbacks cannot cancel navigation; returned promises are not awaited.
* Callers handle errors in asynchronous callbacks.
*/
on<K extends keyof WebviewEventMap>(
event: K,
handler: (data: WebviewEventMap[K]) => void,
): Promise<WebviewUnsubscribe>;
/** Runs at document creation on future navigations, never immediately. */
addInitScript(script: string): Promise<void>;
goto(url: string, opts?: WebviewGotoOptions): Promise<void>;
}
@@ -0,0 +1,12 @@
//go:build cgo && darwin && webview_native
package goprovider
import (
"github.com/GopeedLab/gopeed/internal/webview/integrationtest"
"os"
"testing"
)
func TestMain(m *testing.M) { os.Exit(RunMainThreadLoop(m.Run)) }
func TestProviderEvents(t *testing.T) { integrationtest.RunEventContract(t, New()) }
@@ -0,0 +1,49 @@
//go:build cgo
package goprovider
import (
enginewebview "github.com/GopeedLab/gopeed/pkg/download/engine/webview"
webview "github.com/GopeedLab/webview_go"
"github.com/dop251/goja"
"testing"
"time"
)
func TestInternalErrorDocumentDoesNotReportPageEvents(t *testing.T) {
vm := goja.New()
if _, err := vm.RunString(`
var window = globalThis;
window.top = window;
var location = {protocol: "chrome-error:", href: "chrome-error://chromewebdata/"};
var notifications = [];
var listeners = [];
function notify(payload) { notifications.push(payload); }
window.addEventListener = (name) => listeners.push(name);
`); err != nil {
t.Fatal(err)
}
if _, err := vm.RunString(buildLoadNotifyScript("notify")); err != nil {
t.Fatal(err)
}
for _, name := range []string{"notifications", "listeners"} {
value, err := vm.RunString(name + ".length")
if err != nil || value.ToInteger() != 0 {
t.Fatalf("internal error page registered %s: %v (%v)", name, value, err)
}
}
// The provider must wait for the native failure, preserving the failed URL.
page := newPageWrapper(enginewebview.OpenOptions{})
got := make(chan string, 1)
_, err := page.On("load-error", func(e enginewebview.Event) { got <- e.Data["url"].(string) })
if err != nil {
t.Fatal(err)
}
page.handleNativeEvent(webview.Event{Name: "load-error", URL: "https://example.test/failure", Message: "TLS handshake failed"})
if err := page.waitForNavigation(time.Second, "load"); err == nil {
t.Fatal("native failure completed navigation successfully")
}
if url := <-got; url != "https://example.test/failure" {
t.Fatal(url)
}
}
@@ -0,0 +1,47 @@
//go:build linux && cgo
package goprovider
/*
#cgo pkg-config: gtk+-3.0
#include <gtk/gtk.h>
static void pumpGTK(void) {
// Bound each batch so Go creation/removal requests cannot be starved.
for (int i = 0; i < 64 && g_main_context_pending(NULL); ++i)
g_main_context_iteration(NULL, FALSE);
}
*/
import "C"
import (
"runtime"
"sync"
"time"
)
var gtkThreadOnce sync.Once
var gtkThreadTasks = make(chan func())
func RunMainThreadLoop(run func() int) int { return run() }
func postMainThreadTask(task func()) bool {
gtkThreadOnce.Do(func() {
go func() {
runtime.LockOSThread()
// WebKit must always use its original UI thread, including after
// the last page closes. A page cannot stop this shared event loop.
ticker := time.NewTicker(10 * time.Millisecond)
defer ticker.Stop()
for {
select {
case task := <-gtkThreadTasks:
task()
case <-ticker.C:
}
C.pumpGTK()
}
}()
})
gtkThreadTasks <- task
return true
}
@@ -1,4 +1,4 @@
//go:build !darwin
//go:build !darwin && (!linux || !cgo)
package goprovider
+207 -119
View File
@@ -6,7 +6,6 @@ import (
"encoding/json"
"errors"
"fmt"
"net/url"
"strings"
"sync"
"sync/atomic"
@@ -37,7 +36,9 @@ func (p *Provider) Open(opts enginewebview.OpenOptions) (enginewebview.Page, err
}
type pageWrapper struct {
opts enginewebview.OpenOptions
opts enginewebview.OpenOptions
events enginewebview.EventHub
navigationErrors chan error
callbackName string
readyID string
@@ -49,19 +50,23 @@ type pageWrapper struct {
url string
closed bool
ready chan error
done chan struct{}
loads chan string
ready chan error
done chan struct{}
loads chan string
domReady chan struct{}
sameDocument chan struct{}
}
type evalMessage struct {
ID string `json:"id"`
Ready bool `json:"ready,omitempty"`
Load bool `json:"load,omitempty"`
URL string `json:"url,omitempty"`
OK bool `json:"ok,omitempty"`
Value any `json:"value,omitempty"`
Error string `json:"error,omitempty"`
Event string `json:"event,omitempty"`
SameDocument bool `json:"sameDocument,omitempty"`
ID string `json:"id"`
Ready bool `json:"ready,omitempty"`
Load bool `json:"load,omitempty"`
URL string `json:"url,omitempty"`
OK bool `json:"ok,omitempty"`
Value any `json:"value,omitempty"`
Error string `json:"error,omitempty"`
}
type evalResult struct {
@@ -88,13 +93,16 @@ const startupTimeout = time.Minute
func newPageWrapper(opts enginewebview.OpenOptions) *pageWrapper {
seq := providerSeq.Add(1)
return &pageWrapper{
opts: opts,
callbackName: fmt.Sprintf("__gopeedWebViewCallback_%d", seq),
readyID: fmt.Sprintf("__ready__%d", seq),
pending: make(map[string]chan evalResult),
ready: make(chan error, 1),
done: make(chan struct{}),
loads: make(chan string, 8),
opts: opts,
navigationErrors: make(chan error, 8),
callbackName: fmt.Sprintf("__gopeedWebViewCallback_%d", seq),
readyID: fmt.Sprintf("__ready__%d", seq),
pending: make(map[string]chan evalResult),
ready: make(chan error, 1),
done: make(chan struct{}),
loads: make(chan string, 8),
domReady: make(chan struct{}, 8),
sameDocument: make(chan struct{}, 8),
}
}
@@ -105,12 +113,21 @@ func (p *pageWrapper) start() error {
defer runtimepkg.UnlockOSThread()
var w webview.WebView
if p.opts.Headless {
if p.opts.DataPath != "" {
var err error
w, err = webview.NewWithOptions(webview.Options{Debug: p.opts.Debug, Headless: p.opts.Headless, DataPath: p.opts.DataPath, ProxyURL: p.opts.ProxyURL})
if err != nil {
p.ready <- err
close(p.done)
return
}
} else if p.opts.Headless {
w = webview.NewHeadless(p.opts.Debug)
} else {
w = webview.New(p.opts.Debug)
}
p.view = w
w.SetEventHandler(p.handleNativeEvent)
w.SetTitle(firstNonEmpty(p.opts.Title, "Gopeed WebView"))
w.SetSize(defaultWindowDimension(p.opts.Width, 1280), defaultWindowDimension(p.opts.Height, 800), webview.HintNone)
@@ -122,18 +139,25 @@ func (p *pageWrapper) start() error {
setter.SetUserAgent(p.opts.UserAgent)
}
}
w.Init(buildLoadNotifyScript(p.callbackName))
if err := w.Bind(p.callbackName, func(payload string) error {
return p.handleCallback(payload)
}); err != nil {
w.Destroy()
p.ready <- err
close(p.done)
return
}
w.Init(buildLoadNotifyScript(p.callbackName))
applyWindowOptions(w, p.opts)
w.SetHtml(buildBootstrapHTML(p.callbackName, p.readyID))
if runtimepkg.GOOS == "linux" {
// All GTK pages share the process-wide event loop below.
return
}
w.Run()
p.handleNativeEvent(webview.Event{Name: "closed"})
w.Destroy()
close(p.done)
}
@@ -162,10 +186,15 @@ func (p *pageWrapper) AddInitScript(script string) error {
func (p *pageWrapper) Goto(url string, opts enginewebview.GotoOptions) error {
startedAt := time.Now()
p.drainLoads()
for {
select {
case <-p.navigationErrors:
continue
default:
}
break
}
if err := p.dispatch(func(w webview.WebView) error {
p.mu.Lock()
p.url = url
p.mu.Unlock()
w.Navigate(url)
return nil
}); err != nil {
@@ -179,7 +208,7 @@ func (p *pageWrapper) Goto(url string, opts enginewebview.GotoOptions) error {
if waitUntil == "" {
waitUntil = "load"
}
err := p.waitForNavigation(url, timeout, waitUntil)
err := p.waitForNavigation(timeout, waitUntil)
if err != nil {
p.tracef("goto failed %dms url=%s err=%v", time.Since(startedAt).Milliseconds(), url, err)
return err
@@ -219,7 +248,12 @@ func (p *pageWrapper) Execute(expression string, args ...any) (any, error) {
return nil, err
}
result := <-ch
var result evalResult
select {
case result = <-ch:
case <-p.done:
return nil, fmt.Errorf("webview page is closed")
}
if result.err != nil {
p.tracef("execute failed %dms err=%v", time.Since(startedAt).Milliseconds(), result.err)
return result.value, result.err
@@ -307,6 +341,7 @@ func (p *pageWrapper) Close() error {
return nil
}
p.closed = true
p.failPendingLocked()
w := p.view
wait := p.done
p.mu.Unlock()
@@ -318,12 +353,18 @@ func (p *pageWrapper) Close() error {
// the quit message lands on the correct queue.
terminateDone := make(chan struct{})
w.Dispatch(func() {
w.Terminate()
if runtimepkg.GOOS == "linux" {
w.Destroy()
close(p.done)
} else {
w.Terminate()
}
close(terminateDone)
})
<-terminateDone
}
<-wait
p.events.Emit(enginewebview.Event{Name: "closed", Data: map[string]any{"reason": "api"}})
return nil
}
@@ -334,94 +375,29 @@ func (p *pageWrapper) tracef(format string, args ...any) {
fmt.Printf("goprovider "+format+"\n", args...)
}
func (p *pageWrapper) waitForNavigation(targetURL string, timeout time.Duration, waitUntil string) error {
deadline := time.Now().Add(timeout)
ticker := time.NewTicker(100 * time.Millisecond)
defer ticker.Stop()
func (p *pageWrapper) waitForNavigation(timeout time.Duration, waitUntil string) error {
timer := time.NewTimer(timeout)
defer timer.Stop()
for {
select {
case <-p.done:
return fmt.Errorf("webview page is closed")
case err := <-p.navigationErrors:
return err
case <-timer.C:
return fmt.Errorf("webview navigation timeout")
case <-p.sameDocument:
return nil
case <-p.domReady:
if waitUntil == "domcontentloaded" {
return nil
}
case <-p.loads:
if waitUntil == "load" {
return nil
}
case <-ticker.C:
state, err := p.navigationState()
if err == nil && state.ready(targetURL, waitUntil) {
return nil
}
if time.Now().After(deadline) {
return fmt.Errorf("webview navigation timeout")
}
return nil
}
}
}
type navigationState struct {
URL string `json:"url"`
ReadyState string `json:"readyState"`
}
func (s navigationState) ready(targetURL string, waitUntil string) bool {
if s.URL == "" || s.URL == "about:blank" {
return false
}
switch waitUntil {
case "domcontentloaded":
if s.ReadyState == "loading" || s.ReadyState == "" {
return false
}
default:
if s.ReadyState != "complete" {
return false
}
}
if targetURL == "" {
return true
}
if s.URL == targetURL {
return true
}
target, err := parseURL(targetURL)
if err != nil {
return true
}
current, err := parseURL(s.URL)
if err != nil {
return true
}
if current.Scheme == target.Scheme && current.Host == target.Host && current.Path == target.Path {
return true
}
return false
}
func parseURL(raw string) (*url.URL, error) {
return url.Parse(raw)
}
func (p *pageWrapper) navigationState() (navigationState, error) {
value, err := p.Execute(`() => ({
url: String(location.href || ""),
readyState: document.readyState || "",
})`)
if err != nil {
return navigationState{}, err
}
stateMap, ok := value.(map[string]any)
if !ok {
return navigationState{}, fmt.Errorf("invalid navigation state")
}
state := navigationState{}
if urlValue, ok := stateMap["url"].(string); ok {
state.URL = urlValue
}
if readyValue, ok := stateMap["readyState"].(string); ok {
state.ReadyState = readyValue
}
return state, nil
}
func (p *pageWrapper) dispatch(fn func(w webview.WebView) error) error {
p.mu.Lock()
if p.closed {
@@ -438,7 +414,12 @@ func (p *pageWrapper) dispatch(fn func(w webview.WebView) error) error {
w.Dispatch(func() {
done <- fn(w)
})
return <-done
select {
case err := <-done:
return err
case <-p.done:
return fmt.Errorf("webview page is closed")
}
}
func (p *pageWrapper) nativeCookies() ([]nativeCookieJSON, error) {
@@ -496,7 +477,31 @@ func (p *pageWrapper) handleCallback(payload string) error {
}
return nil
}
if msg.Event == "domcontentloaded" {
select {
case p.domReady <- struct{}{}:
default:
}
return nil
}
if msg.Event == "url-changed" {
if msg.SameDocument {
select {
case p.sameDocument <- struct{}{}:
default:
}
}
p.mu.Lock()
changed := p.url != msg.URL
p.url = msg.URL
p.mu.Unlock()
if changed {
p.events.Emit(enginewebview.Event{Name: "url-changed", Data: map[string]any{"url": msg.URL, "sameDocument": msg.SameDocument}})
}
return nil
}
if msg.Load {
p.events.Emit(enginewebview.Event{Name: "load", Data: map[string]any{"url": msg.URL}})
select {
case p.loads <- msg.URL:
default:
@@ -521,6 +526,22 @@ func (p *pageWrapper) handleCallback(payload string) error {
}
func (p *pageWrapper) drainLoads() {
for {
select {
case <-p.domReady:
continue
default:
}
break
}
for {
select {
case <-p.sameDocument:
continue
default:
}
break
}
for {
select {
case <-p.loads:
@@ -582,19 +603,77 @@ func buildExecuteScript(callbackName string, reqID string, expression string, ar
func buildLoadNotifyScript(callbackName string) string {
callbackJSON, _ := json.Marshal(callbackName)
return fmt.Sprintf(`(() => {
window.addEventListener("load", () => {
const __cb = globalThis[%s];
if (typeof __cb !== "function") {
return;
}
__cb(JSON.stringify({
load: true,
url: String(location.href || ""),
}));
}, { once: true });
if (window !== window.top) return;
// WebView2 runs init scripts in its internal error document too. Its load
// must not complete goto() before the native NavigationCompleted failure.
if (location.protocol === "chrome-error:") return;
const send = (message) => {
const cb = globalThis[%s];
if (typeof cb === "function") cb(JSON.stringify(message));
};
let lastURL = String(location.href);
const changed = () => {
const url = String(location.href);
if (url === lastURL) return;
lastURL = url;
send({event: "url-changed", url, sameDocument: true});
};
send({event: "url-changed", url: lastURL, sameDocument: false});
for (const name of ["pushState", "replaceState"]) {
const original = history[name];
history[name] = function(...args) {
const result = Reflect.apply(original, this, args);
changed();
return result;
};
}
window.addEventListener("hashchange", changed);
window.addEventListener("popstate", changed);
window.addEventListener("pageshow", (event) => {
if (event.persisted) send({event: "url-changed", url: String(location.href), sameDocument: false});
else changed();
});
window.addEventListener("DOMContentLoaded", () => send({event: "domcontentloaded"}), {once:true});
window.addEventListener("load", () => send({load: true, url: String(location.href)}), {once: true});
})();`, string(callbackJSON))
}
func (p *pageWrapper) On(name string, handler func(enginewebview.Event)) (func(), error) {
return p.events.On(name, handler)
}
func (p *pageWrapper) failPendingLocked() {
for id, ch := range p.pending {
ch <- evalResult{err: fmt.Errorf("webview page is closed")}
delete(p.pending, id)
}
}
func (p *pageWrapper) handleNativeEvent(event webview.Event) {
switch event.Name {
case "load-error":
p.events.Emit(enginewebview.Event{Name: "load-error", Data: map[string]any{"url": event.URL, "message": event.Message}})
select {
case p.navigationErrors <- errors.New(event.Message):
default:
}
case "closed":
p.mu.Lock()
api := p.closed
p.closed = true
p.failPendingLocked()
p.mu.Unlock()
reason := "user"
if api {
reason = "api"
}
p.events.Emit(enginewebview.Event{Name: "closed", Data: map[string]any{"reason": reason}})
if runtimepkg.GOOS == "linux" && !api {
p.view.Dispatch(func() { p.view.Destroy(); close(p.done) })
}
}
}
func defaultWindowDimension(value int, fallback int) int {
if value > 0 {
return value
@@ -610,3 +689,12 @@ func firstNonEmpty(values ...string) string {
}
return ""
}
func (p *Provider) RemoveProfile(profileID, dataPath string) error {
done := make(chan error, 1)
run := func() { done <- webview.RemoveProfile(dataPath) }
if !postMainThreadTask(run) {
run()
}
return <-done
}
@@ -13,3 +13,7 @@ func TestProviderContract(t *testing.T) {
CookieDomainMode: integrationtest.CookieDomainModeRequired,
})
}
func TestProviderProfiles(t *testing.T) { integrationtest.RunProfileContract(t, New()) }
func TestProviderEvents(t *testing.T) { integrationtest.RunEventContract(t, New()) }
+49 -5
View File
@@ -1,10 +1,14 @@
package integrationtest
import (
"crypto/sha256"
"fmt"
webviewproxy "github.com/GopeedLab/gopeed/internal/webview/proxy"
"net/http"
"net/http/httptest"
"net/url"
"path/filepath"
"runtime"
"strings"
"testing"
"time"
@@ -104,8 +108,10 @@ func RunProviderContract(t *testing.T, provider enginewebview.Provider, opts Con
t.Fatalf("expected click count to be 1, got %v", clickCount)
}
// Hidden WebViews can throttle timers on shared CI runners. Keep the
// positive asynchronous waits within the same budget as navigation.
found, err := page.WaitForSelector("#delayed", map[string]any{
"timeoutMs": 5000,
"timeoutMs": 10000,
"pollIntervalMs": 25,
"visible": true,
})
@@ -113,11 +119,18 @@ func RunProviderContract(t *testing.T, provider enginewebview.Provider, opts Con
t.Fatalf("waitForSelector failed: %v", err)
}
if !found {
t.Fatal("expected delayed selector to appear")
diagnostic, diagnosticErr := page.Execute(`() => {
const element = document.querySelector('#delayed');
const rect = element && element.getBoundingClientRect();
return {url: location.href, readyState: document.readyState,
visibility: document.visibilityState, exists: !!element,
width: rect && rect.width, height: rect && rect.height};
}`)
t.Fatalf("expected delayed selector to appear: state=%v, diagnostic error=%v", diagnostic, diagnosticErr)
}
value, err := page.WaitForFunction(`() => window.__readyValue`, map[string]any{
"timeoutMs": 5000,
"timeoutMs": 10000,
"pollIntervalMs": 25,
})
if err != nil {
@@ -219,7 +232,19 @@ func openTestPage(t *testing.T, provider enginewebview.Provider, opts enginewebv
}))
t.Cleanup(server.Close)
runtime := enginewebview.NewRuntime(provider, true)
proxy, err := webviewproxy.New(nil)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { proxy.Close() })
path := filepath.Join(t.TempDir(), "profile")
sum := sha256.Sum256([]byte(path))
profileID := fmt.Sprintf("%x-%x-%x-%x-%x", sum[:4], sum[4:6], sum[6:8], sum[8:10], sum[10:16])
proxyURL := proxy.URL()
if runtime.GOOS == "darwin" {
proxyURL = proxy.SOCKSURL()
}
runtime := enginewebview.NewRuntime(&configuredOpener{provider: provider, profileID: profileID, dataPath: path, proxyURL: proxyURL}, true)
page, err := runtime.Open(map[string]any{
"headless": opts.Headless,
"debug": opts.Debug,
@@ -232,7 +257,14 @@ func openTestPage(t *testing.T, provider enginewebview.Provider, opts enginewebv
t.Fatalf("failed to open webview: %v", err)
}
t.Cleanup(func() {
_ = runtime.Close()
if err := runtime.Close(); err != nil {
t.Errorf("close contract page: %v", err)
}
if remover, ok := provider.(enginewebview.ProfileRemover); ok {
if err := remover.RemoveProfile(profileID, path); err != nil {
t.Errorf("remove contract profile: %v", err)
}
}
})
return page, server, server.URL
@@ -380,3 +412,15 @@ func hasCookie(cookies []enginewebview.Cookie, name string, value string) bool {
}
return false
}
type configuredOpener struct {
provider enginewebview.Provider
profileID, dataPath, proxyURL string
}
func (o *configuredOpener) Open(opts enginewebview.OpenOptions) (enginewebview.Page, error) {
opts.ProfileID = o.profileID
opts.DataPath = o.dataPath
opts.ProxyURL = o.proxyURL
return o.provider.Open(opts)
}
+110
View File
@@ -0,0 +1,110 @@
package integrationtest
import (
"fmt"
"strings"
"testing"
"time"
wv "github.com/GopeedLab/gopeed/pkg/download/engine/webview"
)
// RunEventContract exercises the same public semantics on native and RPC hosts.
func RunEventContract(t *testing.T, provider wv.Provider) {
t.Helper()
page, _, url := openTestPage(t, provider, wv.OpenOptions{Headless: true, Title: "WebView event contract"})
events := make(chan wv.Event, 100)
for _, name := range []string{"url-changed", "load", "load-error", "closed"} {
if _, err := page.On(name, func(e wv.Event) { events <- e }); err != nil {
t.Fatal(err)
}
}
next := func(name string) wv.Event {
t.Helper()
select {
case event := <-events:
if event.Name != name {
t.Fatalf("got event %s (%v), want %s", event.Name, event.Data, name)
}
return event
case <-time.After(10 * time.Second):
t.Fatalf("timed out waiting for %s", name)
}
return wv.Event{}
}
quiet := func() {
t.Helper()
select {
case e := <-events:
t.Fatalf("unexpected event: %+v", e)
case <-time.After(150 * time.Millisecond):
}
}
quiet()
if err := page.AddInitScript(`globalThis.__initBeforePage = document.readyState`); err != nil {
t.Fatal(err)
}
if err := page.Goto(url, map[string]any{"timeoutMs": 10000}); err != nil {
t.Fatal(err)
}
e := next("url-changed")
if strings.TrimRight(fmt.Sprint(e.Data["url"]), "/") != strings.TrimRight(url, "/") || e.Data["sameDocument"] != false {
t.Fatal(e)
}
next("load")
if result, err := page.Execute(`() => globalThis.__initBeforePage`); err != nil || result != "loading" {
t.Fatalf("init script: %v %v", result, err)
}
if err := page.AddInitScript(`globalThis.__onlyNext = true`); err != nil {
t.Fatal(err)
}
if result, err := page.Execute(`() => typeof globalThis.__onlyNext`); err != nil || result != "undefined" {
t.Fatalf("init ran in current page: %v %v", result, err)
}
for _, expression := range []string{
`() => { history.pushState({}, '', '#push'); }`,
`() => { history.replaceState({}, '', '#replace'); }`,
`() => { location.hash = 'hash'; }`,
} {
if _, err := page.Execute(expression); err != nil {
t.Fatal(err)
}
e := next("url-changed")
if e.Data["sameDocument"] != true {
t.Fatal(e)
}
quiet()
}
if _, err := page.Execute(`() => { location.reload(); }`); err != nil {
t.Fatal(err)
}
next("load") // Same address on refresh is not a URL change.
if result, err := page.Execute(`() => globalThis.__onlyNext`); err != nil || result != true {
t.Fatalf("next init: %v %v", result, err)
}
if _, err := page.Execute(`() => { const f=document.createElement('iframe'); f.src='/next'; document.body.append(f); }`); err != nil {
t.Fatal(err)
}
quiet()
// The fixture serves plain HTTP. Attempting TLS against it produces a
// real navigation failure through both HTTP CONNECT and SOCKS proxies.
// An unreachable HTTP origin would instead become a valid 502 document
// when the host's HTTP proxy handles the connection error.
failedURL := strings.Replace(url, "http://", "https://", 1) + "/failure"
if err := page.Goto(failedURL, map[string]any{"timeoutMs": 3000}); err == nil {
t.Fatal("failed navigation succeeded")
}
e = next("load-error")
if e.Data["message"] == "" || !strings.Contains(fmt.Sprint(e.Data["url"]), "/failure") {
t.Fatal(e)
}
quiet() // A handled failure must not load a browser-generated fallback page.
if err := page.Close(); err != nil {
t.Fatal(err)
}
e = next("closed")
if e.Data["reason"] != "api" {
t.Fatal(e)
}
quiet()
}
@@ -0,0 +1,247 @@
package integrationtest
import (
"crypto/sha256"
"fmt"
"io"
"net"
"net/http"
"net/http/httptest"
"net/url"
"path/filepath"
"reflect"
"runtime"
"sync/atomic"
"testing"
"time"
webviewproxy "github.com/GopeedLab/gopeed/internal/webview/proxy"
webview "github.com/GopeedLab/gopeed/pkg/download/engine/webview"
)
// RunProfileContract exercises the same host API through both native and RPC
// providers. It never bypasses the host bridge with direct Flutter plugin calls.
func RunProfileContract(t *testing.T, provider webview.Provider) {
t.Helper()
waitForProviderAvailable(t, provider, defaultAvailabilityTimeout)
remover, ok := provider.(webview.ProfileRemover)
if !ok {
t.Fatal("provider does not implement host profile removal")
}
proxyURL, secureHits := profileProxy(t)
a := newContractProfile(t, provider, remover, proxyURL)
b := newContractProfile(t, provider, remover, proxyURL)
first := a.open(t)
assertProfileValue(t, first, `async () => { localStorage.setItem('owner','a'); return 'a'; }`, "a")
assertProfileValue(t, first, profileDatabaseScript(true), "saved")
if err := first.SetCookie(webview.Cookie{Name: "session", Value: "a", Domain: "profiles.invalid", Path: "/", HTTPOnly: true, Expires: time.Now().Add(time.Hour)}); err != nil {
t.Fatal(err)
}
second := b.open(t)
assertProfileValue(t, second, `() => localStorage.getItem('owner')`, nil)
assertProfileValue(t, second, profileDatabaseScript(false), nil)
assertProfileCookies(t, second, "", "new profile B")
assertProfileValue(t, second, `() => { localStorage.setItem('owner','b'); return 'b'; }`, "b")
if err := second.SetCookie(webview.Cookie{Name: "session", Value: "b", Domain: "profiles.invalid", Path: "/", HTTPOnly: true}); err != nil {
t.Fatal(err)
}
shared := a.open(t)
assertProfileValue(t, shared, `() => localStorage.getItem('owner')`, "a")
assertProfileValue(t, shared, profileDatabaseScript(false), "saved")
assertProfileCookies(t, shared, "a", "shared profile A")
storedCookies, err := shared.GetCookies()
if err != nil || len(storedCookies) != 1 || storedCookies[0].Expires.Before(time.Now()) {
t.Fatalf("persistent cookie lost its expiration: %#v, error %v", storedCookies, err)
}
assertProfileValue(t, shared, `() => document.cookie`, "") // HttpOnly stays hidden.
if err := second.ClearCookies(); err != nil {
t.Fatal(err)
}
assertProfileCookies(t, first, "a", "clearing B leaves A intact")
if err := second.SetCookie(webview.Cookie{Name: "session", Value: "b", Domain: "profiles.invalid", Path: "/", HTTPOnly: true}); err != nil {
t.Fatal(err)
}
a.close(t)
reopened := a.open(t)
assertProfileValue(t, reopened, `() => localStorage.getItem('owner')`, "a")
assertProfileValue(t, reopened, profileDatabaseScript(false), "saved")
assertProfileCookies(t, reopened, "a", "reopened profile A")
// Include a session cookie immediately before deletion to catch stale
// in-process CookieStore caches when the same profile is recreated.
if err := reopened.SetCookie(webview.Cookie{Name: "session", Value: "remove-me", Domain: "profiles.invalid", Path: "/", HTTPOnly: true}); err != nil {
t.Fatal(err)
}
a.close(t)
a.remove(t)
a.remove(t) // Retrying a successful uninstall is harmless.
reinstalled := a.open(t)
assertProfileValue(t, reinstalled, `() => localStorage.getItem('owner')`, nil)
assertProfileValue(t, reinstalled, profileDatabaseScript(false), nil)
assertProfileCookies(t, reinstalled, "", "reinstalled profile A")
assertProfileValue(t, second, `() => localStorage.getItem('owner')`, "b")
assertProfileCookies(t, second, "b", "B after removing A")
// Refuse the HTTPS tunnel deliberately. Seeing it at our proxy proves HTTPS
// routing without adding a certificate-validation bypass to the test host.
// WebView2 may commit its built-in error page and report navigation as
// complete. The invariant here is that the HTTPS tunnel reaches the proxy.
_ = reinstalled.Goto("https://profiles.invalid/", webview.GotoOptions{TimeoutMS: 5000})
deadline := time.Now().Add(5 * time.Second)
for secureHits.Load() == 0 && time.Now().Before(deadline) {
time.Sleep(10 * time.Millisecond)
}
if secureHits.Load() == 0 {
t.Fatal("HTTPS bypassed the host proxy")
}
}
type contractProfile struct {
provider webview.Provider
remover webview.ProfileRemover
options webview.OpenOptions
pages []webview.Page
}
func newContractProfile(t *testing.T, provider webview.Provider, remover webview.ProfileRemover, proxyURL string) *contractProfile {
t.Helper()
path := filepath.Join(t.TempDir(), "profile")
hash := sha256.Sum256([]byte(path))
id := fmt.Sprintf("%x-%x-%x-%x-%x", hash[:4], hash[4:6], hash[6:8], hash[8:10], hash[10:16])
p := &contractProfile{provider: provider, remover: remover, options: webview.OpenOptions{ProfileID: id, DataPath: path, ProxyURL: proxyURL, Headless: true, Title: "Gopeed profile contract"}}
t.Cleanup(func() { p.close(t); p.remove(t) })
return p
}
func (p *contractProfile) open(t *testing.T) webview.Page {
t.Helper()
page, err := p.provider.Open(p.options)
if err != nil {
t.Fatal(err)
}
p.pages = append(p.pages, page)
if err := page.Goto("http://profiles.invalid/", webview.GotoOptions{TimeoutMS: 10000}); err != nil {
t.Fatal(err)
}
return page
}
func (p *contractProfile) close(t *testing.T) {
t.Helper()
for _, page := range p.pages {
if err := page.Close(); err != nil {
t.Errorf("close profile page: %v", err)
}
}
p.pages = nil
}
func (p *contractProfile) remove(t *testing.T) {
t.Helper()
if err := p.remover.RemoveProfile(p.options.ProfileID, p.options.DataPath); err != nil {
t.Errorf("remove profile: %v", err)
}
}
func assertProfileValue(t *testing.T, page webview.Page, script string, want any) {
t.Helper()
got, err := page.Execute(script)
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(got, want) {
t.Fatalf("script %s: got %#v, want %#v", script, got, want)
}
}
func assertProfileCookies(t *testing.T, page webview.Page, want, stage string) {
t.Helper()
// Native cookie managers propagate updates across browser processes
// asynchronously. Wait for the expected state rather than assuming the
// first IPC read already includes the preceding write.
deadline := time.Now().Add(5 * time.Second)
for {
cookies, err := page.GetCookies()
if err != nil {
t.Fatalf("%s: %v", stage, err)
}
matches := len(cookies) == 0 && want == "" || len(cookies) == 1 && want != "" && cookies[0].Name == "session" && cookies[0].Value == want && cookies[0].HTTPOnly
if matches {
return
}
if time.Now().After(deadline) {
t.Fatalf("%s: cookies %#v, want %q", stage, cookies, want)
}
time.Sleep(20 * time.Millisecond)
}
}
func profileDatabaseScript(write bool) string {
operation := "store.get('owner')"
if write {
operation = "store.put('saved','owner')"
}
return fmt.Sprintf(`async () => await new Promise((resolve,reject)=>{
const request=indexedDB.open('profile-contract',1);
request.onupgradeneeded=()=>request.result.createObjectStore('state');
request.onerror=()=>reject(String(request.error));
request.onsuccess=()=>{
const db=request.result, tx=db.transaction('state','readwrite'), store=tx.objectStore('state');
const op=%s; let value;
op.onsuccess=()=>value=%s;
tx.oncomplete=()=>{db.close();resolve(value===undefined?null:value)};
tx.onerror=()=>{db.close();reject(String(tx.error))};
};
})`, operation, map[bool]string{true: "'saved'", false: "op.result"}[write])
}
func profileProxy(t *testing.T) (string, *atomic.Int32) {
t.Helper()
origin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/html")
w.Header().Set("Cache-Control", "no-store")
fmt.Fprint(w, "<!doctype html><title>Profile contract</title>")
}))
t.Cleanup(origin.Close)
secureHits := &atomic.Int32{}
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodConnect {
origin.Config.Handler.ServeHTTP(w, r)
return
}
if r.Host == "profiles.invalid:443" {
secureHits.Add(1)
http.Error(w, "HTTPS routing fixture rejects tunnels", http.StatusBadGateway)
return
}
if r.Host != "profiles.invalid:80" {
http.Error(w, "unexpected target", http.StatusBadGateway)
return
}
remote, err := net.DialTimeout("tcp", origin.Listener.Addr().String(), 5*time.Second)
if err != nil {
http.Error(w, "origin unavailable", http.StatusBadGateway)
return
}
defer remote.Close()
client, buffer, err := w.(http.Hijacker).Hijack()
if err != nil {
return
}
defer client.Close()
if _, err := buffer.WriteString("HTTP/1.1 200 Connection Established\r\n\r\n"); err != nil {
return
}
if err := buffer.Flush(); err != nil {
return
}
go func() { io.Copy(remote, buffer); remote.Close() }()
io.Copy(client, remote)
}))
t.Cleanup(upstream.Close)
upstreamURL, err := url.Parse(upstream.URL)
if err != nil {
t.Fatal(err)
}
proxy, err := webviewproxy.New(func(*http.Request) (*url.URL, error) { return upstreamURL, nil })
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { proxy.Close() })
if runtime.GOOS == "darwin" {
return proxy.SOCKSURL(), secureHits
}
return proxy.URL(), secureHits
}
+341
View File
@@ -0,0 +1,341 @@
// Package proxy provides the host-owned WebView forward proxy. Browser engines
// always use this endpoint; Gopeed decides the upstream route for each request.
package proxy
import (
"bufio"
"context"
"crypto/tls"
"encoding/base64"
"errors"
"fmt"
"io"
"log"
"net"
"net/http"
"net/url"
"strings"
"sync"
"time"
"github.com/armon/go-socks5"
xproxy "golang.org/x/net/proxy"
)
type Selector func(*http.Request) (*url.URL, error)
type Server struct {
listener net.Listener
socksListener net.Listener
server *http.Server
transport *http.Transport
selectProxy Selector
mu sync.Mutex
connections map[io.Closer]struct{}
closed bool
}
func New(selectProxy Selector) (*Server, error) {
if selectProxy == nil {
selectProxy = func(*http.Request) (*url.URL, error) { return nil, nil }
}
listener, err := net.Listen("tcp4", "127.0.0.1:0")
if err != nil {
return nil, err
}
s := &Server{listener: listener, selectProxy: selectProxy, connections: make(map[io.Closer]struct{})}
s.transport = &http.Transport{Proxy: selectProxy, DisableKeepAlives: true, TLSHandshakeTimeout: 15 * time.Second, ResponseHeaderTimeout: 30 * time.Second}
s.server = &http.Server{Handler: s, ReadHeaderTimeout: 15 * time.Second}
s.socksListener, err = net.Listen("tcp4", "127.0.0.1:0")
if err != nil {
listener.Close()
return nil, err
}
socks, err := socks5.New(&socks5.Config{Logger: log.New(io.Discard, "", 0), Resolver: unresolvedDNS{}, Dial: func(ctx context.Context, network, address string) (net.Conn, error) {
_, port, _ := net.SplitHostPort(address)
scheme := "https"
if port == "80" {
scheme = "http"
}
request := &http.Request{URL: &url.URL{Scheme: scheme, Host: address}, Header: make(http.Header)}
upstream, err := selectProxy(request)
if err != nil {
return nil, err
}
timed, cancel := context.WithTimeout(ctx, 30*time.Second)
defer cancel()
conn, err := dialTunnel(timed, address, upstream)
if err != nil {
return nil, err
}
return s.track(conn)
}})
if err != nil {
listener.Close()
s.socksListener.Close()
return nil, err
}
go s.server.Serve(listener)
go socks.Serve(&trackedListener{Listener: s.socksListener, server: s})
return s, nil
}
func (s *Server) URL() string { return "http://" + s.listener.Addr().String() }
func (s *Server) SOCKSURL() string { return "socks5://" + s.socksListener.Addr().String() }
func (s *Server) Close() error {
s.mu.Lock()
s.closed = true
connections := make([]io.Closer, 0, len(s.connections))
for c := range s.connections {
connections = append(connections, c)
}
s.mu.Unlock()
for _, c := range connections {
c.Close()
}
s.socksListener.Close()
s.transport.CloseIdleConnections()
return s.server.Close()
}
func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
if r.Method == http.MethodConnect {
s.connect(w, r)
return
}
if !r.URL.IsAbs() || (r.URL.Scheme != "http" && r.URL.Scheme != "https") {
http.Error(w, "absolute HTTP URL required", http.StatusBadRequest)
return
}
out := r.Clone(r.Context())
out.RequestURI = ""
upgrade := out.Header.Get("Upgrade")
stripHopHeaders(out.Header)
if upgrade != "" {
out.Header.Set("Connection", "Upgrade")
out.Header.Set("Upgrade", upgrade)
}
resp, err := s.transport.RoundTrip(out)
if err != nil {
http.Error(w, "upstream request failed", http.StatusBadGateway)
return
}
defer resp.Body.Close()
if resp.StatusCode == http.StatusSwitchingProtocols {
remote, ok := resp.Body.(io.ReadWriteCloser)
if !ok {
http.Error(w, "invalid upstream upgrade", 502)
return
}
client, buffer, err := w.(http.Hijacker).Hijack()
if err != nil {
return
}
defer client.Close()
resp.Body = nil
if err := resp.Write(buffer); err != nil {
return
}
if err := buffer.Flush(); err != nil {
return
}
s.bridge(client, buffer, remote)
return
}
stripHopHeaders(resp.Header)
for k, v := range resp.Header {
w.Header()[k] = v
}
w.WriteHeader(resp.StatusCode)
io.Copy(w, resp.Body)
}
func stripHopHeaders(h http.Header) {
for _, key := range strings.Split(h.Get("Connection"), ",") {
h.Del(strings.TrimSpace(key))
}
for _, key := range []string{"Connection", "Proxy-Connection", "Proxy-Authorization", "Proxy-Authenticate", "Keep-Alive", "TE", "Trailer", "Transfer-Encoding", "Upgrade"} {
h.Del(key)
}
}
func (s *Server) connect(w http.ResponseWriter, r *http.Request) {
if _, _, err := net.SplitHostPort(r.Host); err != nil {
http.Error(w, "invalid tunnel destination", 400)
return
}
target := r.Clone(r.Context())
target.URL = &url.URL{Scheme: "https", Host: r.Host}
upstream, err := s.selectProxy(target)
if err != nil {
http.Error(w, "proxy selection failed", 502)
return
}
ctx, cancel := context.WithTimeout(r.Context(), 30*time.Second)
defer cancel()
remote, err := dialTunnel(ctx, r.Host, upstream)
if err != nil {
http.Error(w, "upstream connection failed", 502)
return
}
defer remote.Close()
client, buffer, err := w.(http.Hijacker).Hijack()
if err != nil {
return
}
defer client.Close()
if _, err := buffer.WriteString("HTTP/1.1 200 Connection Established\r\n\r\n"); err != nil {
return
}
if err := buffer.Flush(); err != nil {
return
}
s.bridge(client, buffer, remote)
}
func (s *Server) bridge(client net.Conn, buffer *bufio.ReadWriter, remote io.ReadWriteCloser) {
s.mu.Lock()
if s.closed {
s.mu.Unlock()
return
}
s.connections[client] = struct{}{}
s.connections[remote] = struct{}{}
s.mu.Unlock()
defer func() { s.mu.Lock(); delete(s.connections, client); delete(s.connections, remote); s.mu.Unlock() }()
done := make(chan struct{})
go func() { io.Copy(remote, buffer); remote.Close(); close(done) }()
io.Copy(client, remote)
client.Close()
<-done
}
type bufferedConn struct {
net.Conn
reader *bufio.Reader
}
func (c *bufferedConn) Read(p []byte) (int, error) { return c.reader.Read(p) }
func dialTunnel(ctx context.Context, target string, upstream *url.URL) (net.Conn, error) {
dialer := &net.Dialer{}
if upstream == nil {
return dialer.DialContext(ctx, "tcp", target)
}
if upstream.Scheme == "socks5" || upstream.Scheme == "socks5h" {
u := *upstream
u.Scheme = "socks5"
d, err := xproxy.FromURL(&u, dialer)
if err != nil {
return nil, err
}
return d.(xproxy.ContextDialer).DialContext(ctx, "tcp", target)
}
if upstream.Scheme != "http" && upstream.Scheme != "https" {
return nil, errors.New("unsupported proxy scheme")
}
host := upstream.Host
if upstream.Port() == "" {
port := "80"
if upstream.Scheme == "https" {
port = "443"
}
host = net.JoinHostPort(upstream.Hostname(), port)
}
conn, err := dialer.DialContext(ctx, "tcp", host)
if err != nil {
return nil, err
}
ok := false
defer func() {
if !ok {
conn.Close()
}
}()
if deadline, exists := ctx.Deadline(); exists {
conn.SetDeadline(deadline)
}
if upstream.Scheme == "https" {
secure := tls.Client(conn, &tls.Config{ServerName: upstream.Hostname(), MinVersion: tls.VersionTLS12})
if err := secure.HandshakeContext(ctx); err != nil {
return nil, err
}
conn = secure
}
request := &http.Request{Method: http.MethodConnect, URL: &url.URL{Opaque: target}, Host: target, Header: make(http.Header)}
if upstream.User != nil {
password, _ := upstream.User.Password()
request.Header.Set("Proxy-Authorization", "Basic "+base64.StdEncoding.EncodeToString([]byte(upstream.User.Username()+":"+password)))
}
if err := request.Write(conn); err != nil {
return nil, err
}
reader := bufio.NewReader(conn)
response, err := http.ReadResponse(reader, request)
if err != nil {
return nil, err
}
if response.StatusCode != 200 {
response.Body.Close()
return nil, fmt.Errorf("upstream CONNECT status %d", response.StatusCode)
}
conn.SetDeadline(time.Time{})
ok = true
return &bufferedConn{Conn: conn, reader: reader}, nil
}
// Leave hostname resolution to the selected upstream (SOCKS remote DNS / CONNECT).
type unresolvedDNS struct{}
func (unresolvedDNS) Resolve(ctx context.Context, name string) (context.Context, net.IP, error) {
return ctx, nil, nil
}
type trackedListener struct {
net.Listener
server *Server
}
func (l *trackedListener) Accept() (net.Conn, error) {
c, err := l.Listener.Accept()
if err != nil {
return nil, err
}
return l.server.track(c)
}
type trackedConn struct {
net.Conn
server *Server
}
func (c *trackedConn) Close() error {
err := c.Conn.Close()
c.server.mu.Lock()
delete(c.server.connections, c)
c.server.mu.Unlock()
return err
}
func (s *Server) track(c net.Conn) (net.Conn, error) {
s.mu.Lock()
defer s.mu.Unlock()
if s.closed {
c.Close()
return nil, net.ErrClosed
}
tracked := &trackedConn{Conn: c, server: s}
s.connections[tracked] = struct{}{}
return tracked, nil
}
func (c *trackedConn) CloseWrite() error { return closeWrite(c.Conn) }
func (c *bufferedConn) CloseWrite() error { return closeWrite(c.Conn) }
func closeWrite(c net.Conn) error {
if writer, ok := c.(interface{ CloseWrite() error }); ok {
return writer.CloseWrite()
}
return c.Close()
}
+315
View File
@@ -0,0 +1,315 @@
package proxy
import (
"bufio"
"context"
"crypto/tls"
"errors"
"io"
"net"
"net/http"
"net/http/httptest"
"net/url"
"strings"
"sync/atomic"
"testing"
"time"
"github.com/armon/go-socks5"
)
func TestRejectedProxyRequests(t *testing.T) {
secret := "upstream-password"
for _, tc := range []struct {
name, method, target string
status int
}{
{"relative URL", "GET", "/relative", 400},
{"unsupported URL", "GET", "ftp://example.com/file", 400},
{"invalid CONNECT", "CONNECT", "example.com", 400},
{"HTTP selection failure", "GET", "http://example.com/", 502},
{"CONNECT selection failure", "CONNECT", "example.com:443", 502},
} {
t.Run(tc.name, func(t *testing.T) {
s := &Server{selectProxy: func(*http.Request) (*url.URL, error) { return nil, errors.New(secret) }}
s.transport = &http.Transport{Proxy: s.selectProxy}
defer s.transport.CloseIdleConnections()
response := httptest.NewRecorder()
s.ServeHTTP(response, httptest.NewRequest(tc.method, tc.target, nil))
if response.Code != tc.status || strings.Contains(response.Body.String(), secret) {
t.Fatalf("unexpected proxy rejection: %d %q", response.Code, response.Body.String())
}
})
}
}
func TestTunnelRejectsUnsafeUpstreams(t *testing.T) {
rejected := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
http.Error(w, "authentication required", http.StatusProxyAuthRequired)
}))
defer rejected.Close()
secure := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
t.Error("untrusted TLS proxy received a request")
}))
defer secure.Close()
for _, upstream := range []string{"ftp://127.0.0.1", rejected.URL, secure.URL} {
t.Run(upstream, func(t *testing.T) {
selected, err := url.Parse(upstream)
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
defer cancel()
conn, err := dialTunnel(ctx, "example.com:443", selected)
if conn != nil {
conn.Close()
}
if err == nil {
t.Fatal("unsafe upstream unexpectedly accepted")
}
})
}
}
func TestRoutesAndAuthentication(t *testing.T) {
origin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Header.Get("Proxy-Authorization") != "" {
t.Error("proxy credentials leaked to origin")
}
w.Write([]byte("origin"))
}))
defer origin.Close()
secure := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Write([]byte("secure")) }))
defer secure.Close()
upstream, err := New(func(*http.Request) (*url.URL, error) { return nil, nil })
if err != nil {
t.Fatal(err)
}
defer upstream.Close()
var authenticated atomic.Int32
authProxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Header.Get("Proxy-Authorization") != "Basic dXNlcjpwYXNz" {
http.Error(w, "authentication required", 407)
return
}
authenticated.Add(1)
upstream.ServeHTTP(w, r)
}))
defer authProxy.Close()
socks, err := socks5.New(&socks5.Config{Credentials: socks5.StaticCredentials{"user": "pass"}})
if err != nil {
t.Fatal(err)
}
listener, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
defer listener.Close()
go socks.Serve(listener)
for _, route := range []string{"", authProxy.URL, "socks5://" + listener.Addr().String()} {
t.Run(route, func(t *testing.T) {
var selected *url.URL
if route != "" {
selected, _ = url.Parse(route)
selected.User = url.UserPassword("user", "pass")
}
local, err := New(func(r *http.Request) (*url.URL, error) { return selected, nil })
if err != nil {
t.Fatal(err)
}
defer local.Close()
proxyURL, _ := url.Parse(local.URL())
transport := &http.Transport{Proxy: http.ProxyURL(proxyURL), TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}
defer transport.CloseIdleConnections()
client := &http.Client{Transport: transport, Timeout: 5 * time.Second}
for _, test := range []struct{ url, want string }{{origin.URL, "origin"}, {secure.URL, "secure"}} {
response, err := client.Get(test.url)
if err != nil {
t.Fatal(err)
}
body, _ := io.ReadAll(response.Body)
response.Body.Close()
if string(body) != test.want {
t.Fatalf("got status %d, body %q", response.StatusCode, body)
}
}
})
}
if authenticated.Load() != 2 {
t.Fatalf("expected authenticated HTTP and CONNECT, got %d", authenticated.Load())
}
}
func TestRoutingChangesAndFailureDoesNotBypassProxy(t *testing.T) {
var hits atomic.Int32
origin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { hits.Add(1); w.Write([]byte("direct")) }))
defer origin.Close()
var useProxy atomic.Bool
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Write([]byte("proxied")) }))
upstreamURL, _ := url.Parse(upstream.URL)
local, err := New(func(*http.Request) (*url.URL, error) {
if useProxy.Load() {
return upstreamURL, nil
}
return nil, nil
})
if err != nil {
t.Fatal(err)
}
defer local.Close()
endpoint, _ := url.Parse(local.URL())
transport := &http.Transport{Proxy: http.ProxyURL(endpoint)}
defer transport.CloseIdleConnections()
client := &http.Client{Transport: transport, Timeout: 3 * time.Second}
for _, want := range []string{"direct", "proxied", "upstream request failed\n"} {
resp, err := client.Get(origin.URL)
if err != nil {
t.Fatal(err)
}
body, _ := io.ReadAll(resp.Body)
resp.Body.Close()
if string(body) != want {
t.Fatalf("got %q want %q", body, want)
}
if want == "direct" {
useProxy.Store(true)
} else if want == "proxied" {
upstream.Close()
}
}
if hits.Load() != 1 {
t.Fatal("failed proxy must not fall back to direct")
}
}
func TestCloseTerminatesTunnels(t *testing.T) {
echo, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
defer echo.Close()
go func() {
c, e := echo.Accept()
if e == nil {
defer c.Close()
io.Copy(c, c)
}
}()
s, err := New(func(*http.Request) (*url.URL, error) { return nil, nil })
if err != nil {
t.Fatal(err)
}
endpoint, _ := url.Parse(s.URL())
c, err := dialTunnel(context.Background(), echo.Addr().String(), endpoint)
if err != nil {
t.Fatal(err)
}
defer c.Close()
c.SetDeadline(time.Now().Add(3 * time.Second))
c.Write([]byte("hello\n"))
line, err := bufio.NewReader(c).ReadString('\n')
if err != nil || line != "hello\n" {
t.Fatalf("echo %q %v", line, err)
}
s.Close()
if _, err := c.Read(make([]byte, 1)); err == nil {
t.Fatal("tunnel stayed open")
}
}
func TestSOCKSIngressPreservesRemoteDNS(t *testing.T) {
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method != "CONNECT" || r.Host != "unresolved.invalid:80" {
t.Errorf("lost destination: %s %s", r.Method, r.Host)
}
c, b, err := w.(http.Hijacker).Hijack()
if err != nil {
return
}
defer c.Close()
b.WriteString("HTTP/1.1 200 Connection Established\r\n\r\n")
b.Flush()
request, err := http.ReadRequest(b.Reader)
if err != nil {
t.Error(err)
return
}
request.Body.Close()
b.WriteString("HTTP/1.1 200 OK\r\nContent-Length: 7\r\nConnection: close\r\n\r\nproxied")
b.Flush()
}))
defer upstream.Close()
target, _ := url.Parse(upstream.URL)
local, err := New(func(r *http.Request) (*url.URL, error) {
if r.URL.Host != "unresolved.invalid:80" {
t.Errorf("selector lost hostname: %s", r.URL)
}
return target, nil
})
if err != nil {
t.Fatal(err)
}
defer local.Close()
endpoint, _ := url.Parse(local.SOCKSURL())
transport := &http.Transport{Proxy: http.ProxyURL(endpoint)}
defer transport.CloseIdleConnections()
client := &http.Client{Transport: transport, Timeout: 5 * time.Second}
response, err := client.Get("http://unresolved.invalid/")
if err != nil {
t.Fatal(err)
}
defer response.Body.Close()
body, _ := io.ReadAll(response.Body)
if string(body) != "proxied" {
t.Fatalf("unexpected body %q", body)
}
}
func TestHTTPUpgradeIsForwarded(t *testing.T) {
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Header.Get("Upgrade") != "fixture" {
http.Error(w, "upgrade lost", 400)
return
}
c, b, err := w.(http.Hijacker).Hijack()
if err != nil {
return
}
defer c.Close()
b.WriteString("HTTP/1.1 101 Switching Protocols\r\nConnection: Upgrade\r\nUpgrade: fixture\r\n\r\n")
b.Flush()
line, err := b.ReadString('\n')
if err != nil {
return
}
b.WriteString(line)
b.Flush()
}))
defer upstream.Close()
local, err := New(nil)
if err != nil {
t.Fatal(err)
}
defer local.Close()
endpoint, _ := url.Parse(local.URL())
c, err := net.Dial("tcp", endpoint.Host)
if err != nil {
t.Fatal(err)
}
defer c.Close()
c.SetDeadline(time.Now().Add(3 * time.Second))
io.WriteString(c, "GET "+upstream.URL+"/ HTTP/1.1\r\nHost: fixture\r\nConnection: Upgrade\r\nUpgrade: fixture\r\n\r\n")
reader := bufio.NewReader(c)
response, err := http.ReadResponse(reader, nil)
if err != nil {
t.Fatal(err)
}
if response.StatusCode != 101 {
t.Fatalf("upgrade failed: %s", response.Status)
}
io.WriteString(c, "hello\n")
line, err := reader.ReadString('\n')
if err != nil || line != "hello\n" {
t.Fatalf("upgraded echo %q %v", line, err)
}
}
+6
View File
@@ -1,5 +1,11 @@
# rpcprovider
The private Flutter native profile bridge is maintained in the
[GopeedLab flutter_inappwebview fork](https://github.com/GopeedLab/flutter_inappwebview/tree/codex/gopeed-profiles).
All three platform overrides pin one commit in `ui/flutter/pubspec.yaml`;
see the fork’s [GOPEED.md](https://github.com/GopeedLab/flutter_inappwebview/blob/codex/gopeed-profiles/GOPEED.md)
for patch scope and upstream synchronization.
`rpcprovider` is the RPC-backed implementation of Gopeed's `webview.Provider` interface.
It is intended for environments where WebView capability is owned by another process, most notably mobile hosts. This package does not implement a WebView itself. Instead, it translates the `pkg/download/engine/webview` page API into single-request `POST + JSON` calls sent to a local host-side RPC service.
+207
View File
@@ -0,0 +1,207 @@
package rpcprovider
import (
"encoding/json"
"io"
"net/http"
"net/http/httptest"
"strings"
"sync/atomic"
"testing"
"time"
wv "github.com/GopeedLab/gopeed/pkg/download/engine/webview"
)
func TestPageEventStream(t *testing.T) {
events := make(chan wv.Event, 8)
disconnected := make(chan struct{})
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
var request wv.RPCRequest
if err := json.NewDecoder(r.Body).Decode(&request); err != nil {
t.Error(err)
return
}
if request.Method != "page.events" {
t.Errorf("unexpected method %s", request.Method)
return
}
if r.Header.Get("Authorization") != "Bearer test" {
t.Error("missing authorization")
}
defer close(disconnected)
enc := json.NewEncoder(w)
_ = enc.Encode(map[string]any{"ready": true})
w.(http.Flusher).Flush()
for {
select {
case e := <-events:
_ = enc.Encode(e)
w.(http.Flusher).Flush()
if e.Name == "closed" {
return
}
case <-r.Context().Done():
return
}
}
}))
defer server.Close()
p := &page{client: NewClient(wv.RPCConfig{Network: "tcp", Address: strings.TrimPrefix(server.URL, "http://"), Token: "test"}), id: "page-1"}
got := make(chan wv.Event, 8)
off, err := p.On("load", func(e wv.Event) { got <- e })
if err != nil {
t.Fatal(err)
}
_, err = p.On("closed", func(e wv.Event) { got <- e })
if err != nil {
t.Fatal(err)
}
events <- wv.Event{Name: "load", Data: map[string]any{"url": "https://example.test"}}
select {
case e := <-got:
if e.Name != "load" {
t.Fatal(e)
}
case <-time.After(time.Second):
t.Fatal("no load event")
}
off()
off()
events <- wv.Event{Name: "load"}
events <- wv.Event{Name: "closed", Data: map[string]any{"reason": "user"}}
select {
case e := <-got:
if e.Name != "closed" || e.Data["reason"] != "user" {
t.Fatal(e)
}
case <-time.After(time.Second):
t.Fatal("no closed event")
}
select {
case <-disconnected:
case <-time.After(time.Second):
t.Fatal("stream retained after close")
}
if _, err = p.On("load", func(wv.Event) {}); err == nil {
t.Fatal("registration after close")
}
}
func TestPageEventStreamHandshakeFailureCanRetry(t *testing.T) {
for _, tc := range []struct {
name string
status int
body string
}{
{"http_error", http.StatusUnauthorized, `{"ready":true}`},
{"invalid_json", http.StatusOK, `not json`},
{"not_ready", http.StatusOK, `{"ready":false}`},
} {
t.Run(tc.name, func(t *testing.T) {
var requests atomic.Int32
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if requests.Add(1) == 1 {
w.WriteHeader(tc.status)
_, _ = io.WriteString(w, tc.body)
return
}
_, _ = io.WriteString(w, "{\"ready\":true}\n{\"event\":\"load\",\"data\":{\"url\":\"https://example.test\"}}\n")
w.(http.Flusher).Flush()
<-r.Context().Done()
}))
defer server.Close()
p := &page{client: NewClient(wv.RPCConfig{Network: "tcp", Address: strings.TrimPrefix(server.URL, "http://")}), id: "retry-page"}
if off, err := p.On("load", func(wv.Event) { t.Error("failed registration retained its listener") }); err == nil || off != nil {
t.Fatal("failed handshake accepted")
}
got := make(chan wv.Event, 1)
off, err := p.On("load", func(e wv.Event) { got <- e })
if err != nil {
t.Fatalf("retry: %v", err)
}
defer func() {
p.eventCancel()
<-p.eventDone
}()
defer off()
select {
case e := <-got:
if e.Data["url"] != "https://example.test" {
t.Fatal(e)
}
case <-time.After(time.Second):
t.Fatal("retry did not receive load notification")
}
if requests.Load() != 2 {
t.Fatalf("unexpected request count: %d", requests.Load())
}
})
}
}
func TestPageEventStreamDisconnectAndClose(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
var request wv.RPCRequest
if err := json.NewDecoder(r.Body).Decode(&request); err != nil {
t.Error(err)
return
}
switch request.Method {
case wv.MethodPageEvents:
// End the stream without a terminal event, as if the host dropped it.
_, _ = io.WriteString(w, "{\"ready\":true}\n")
case wv.MethodPageClose:
_ = json.NewEncoder(w).Encode(wv.RPCResponse{})
default:
t.Errorf("unexpected method: %s", request.Method)
}
}))
defer server.Close()
p := &page{client: NewClient(wv.RPCConfig{Network: "tcp", Address: strings.TrimPrefix(server.URL, "http://")}), id: "disconnected-page"}
closed := make(chan wv.Event, 1)
if _, err := p.On("closed", func(e wv.Event) { closed <- e }); err != nil {
t.Fatal(err)
}
select {
case <-p.eventDone:
case <-time.After(time.Second):
t.Fatal("disconnected stream did not stop")
}
if _, err := p.On("load", func(wv.Event) {}); err == nil || !strings.Contains(err.Error(), "disconnected") {
t.Fatalf("registration hid stream failure: %v", err)
}
if err := p.Close(); err != nil {
t.Fatal(err)
}
select {
case event := <-closed:
if event.Data["reason"] != "api" {
t.Fatal(event)
}
default:
t.Fatal("API close did not release listeners after disconnect")
}
if err := p.Close(); err != nil {
t.Fatal(err)
}
select {
case event := <-closed:
t.Fatalf("duplicate close notification: %+v", event)
default:
}
}
func TestPageEventStreamConnectionFailureCanRetry(t *testing.T) {
server := httptest.NewServer(http.NotFoundHandler())
server.Close()
p := &page{client: NewClient(wv.RPCConfig{Network: "tcp", Address: strings.TrimPrefix(server.URL, "http://")}), id: "offline-page"}
for i := 0; i < 2; i++ {
if off, err := p.On("load", func(wv.Event) {}); err == nil || off != nil {
t.Fatal("registration to unavailable host succeeded")
}
}
if p.eventCancel != nil {
t.Fatal("failed connection retained an active stream")
}
}
+112 -5
View File
@@ -4,10 +4,13 @@ import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"net"
"net/http"
"strings"
"sync"
"time"
enginewebview "github.com/GopeedLab/gopeed/pkg/download/engine/webview"
)
@@ -48,8 +51,13 @@ func (p *Provider) Open(opts enginewebview.OpenOptions) (enginewebview.Page, err
}
type page struct {
client *Client
id string
eventMu sync.Mutex
events enginewebview.EventHub
eventCancel context.CancelFunc
eventDone chan struct{}
eventErr error
client *Client
id string
}
func (p *page) AddInitScript(script string) error {
@@ -109,9 +117,28 @@ func (p *page) ClearCookies() error {
}
func (p *page) Close() error {
return p.client.Call(enginewebview.MethodPageClose, enginewebview.PageCloseParams{
PageID: p.id,
}, nil)
err := p.client.Call(enginewebview.MethodPageClose, enginewebview.PageCloseParams{PageID: p.id}, nil)
var rpcErr *enginewebview.RPCError
if errors.As(err, &rpcErr) && rpcErr.Code == enginewebview.ErrorCodePageNotFound {
err = nil
}
if err != nil {
return err
}
p.eventMu.Lock()
cancel, done := p.eventCancel, p.eventDone
p.eventMu.Unlock()
if cancel != nil {
// Drain the terminal notification before cancelling the transport, so
// already-sent navigation events keep their order before closed.
select {
case <-done:
case <-time.After(5 * time.Second):
}
cancel()
}
p.events.Emit(enginewebview.Event{Name: "closed", Data: map[string]any{"reason": "api"}})
return nil
}
type Client struct {
@@ -193,3 +220,83 @@ func (c *Client) Call(method string, params any, result any) error {
}
return json.Unmarshal(body.Result, result)
}
func (p *Provider) RemoveProfile(profileID, dataPath string) error {
return p.client.Call(enginewebview.MethodProfileRemove, enginewebview.ProfileRemoveParams{ProfileID: profileID}, nil)
}
// On starts one ordered notification stream per page. The ready record is sent
// only after the host has installed its subscription.
func (p *page) On(name string, handler func(enginewebview.Event)) (func(), error) {
p.eventMu.Lock()
defer p.eventMu.Unlock()
if p.eventErr != nil {
return nil, p.eventErr
}
unsubscribe, err := p.events.On(name, handler)
if err != nil {
return nil, err
}
if p.eventCancel != nil {
return unsubscribe, nil
}
ctx, cancel := context.WithCancel(context.Background())
payload, err := json.Marshal(enginewebview.RPCRequest{Method: enginewebview.MethodPageEvents, Params: enginewebview.PageCloseParams{PageID: p.id}})
if err != nil {
cancel()
unsubscribe()
return nil, err
}
req, err := http.NewRequestWithContext(ctx, http.MethodPost, p.client.endpoint, bytes.NewReader(payload))
if err != nil {
cancel()
unsubscribe()
return nil, err
}
req.Header.Set("Content-Type", "application/json")
if p.client.token != "" {
req.Header.Set("Authorization", "Bearer "+p.client.token)
}
// A broken host must not leave registration pending indefinitely.
timer := time.AfterFunc(30*time.Second, cancel)
resp, err := p.client.http.Do(req)
if err != nil {
timer.Stop()
cancel()
unsubscribe()
return nil, err
}
decoder := json.NewDecoder(resp.Body)
var ready struct {
Ready bool `json:"ready"`
}
err = decoder.Decode(&ready)
timer.Stop()
if err != nil || !ready.Ready || resp.StatusCode != http.StatusOK {
resp.Body.Close()
cancel()
unsubscribe()
return nil, fmt.Errorf("webview event stream did not become ready: %v", err)
}
p.eventCancel = cancel
p.eventDone = make(chan struct{})
go func() {
defer resp.Body.Close()
defer cancel()
defer close(p.eventDone)
for {
var event enginewebview.Event
if err := decoder.Decode(&event); err != nil {
p.eventMu.Lock()
p.eventErr = fmt.Errorf("webview event stream disconnected: %w", err)
p.eventMu.Unlock()
return
}
p.events.Emit(event)
if event.Name == "closed" {
return
}
}
}()
return unsubscribe, nil
}
@@ -37,3 +37,12 @@ func defaultWebViewRPCAddress() string {
}
return filepath.Join(homeDir, "Library", "Application Support", "com.gopeed.gopeed", "gopeed_webview.sock")
}
func TestProviderProfiles(t *testing.T) {
provider := New(enginewebview.RPCConfig{Network: *webViewRPCNetwork, Address: *webViewRPCAddress, Token: *webViewRPCToken})
integrationtest.RunProfileContract(t, provider)
}
func TestProviderEvents(t *testing.T) {
integrationtest.RunEventContract(t, New(enginewebview.RPCConfig{Network: *webViewRPCNetwork, Address: *webViewRPCAddress, Token: *webViewRPCToken}))
}
@@ -184,3 +184,29 @@ func TestPageSetCookieParamsOmitsZeroExpires(t *testing.T) {
t.Fatalf("expected expires in payload: %s", got)
}
}
func TestRemoveProfileUsesPrivateHostRPC(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
var request struct {
Method string `json:"method"`
Params map[string]string `json:"params"`
}
if err := json.NewDecoder(r.Body).Decode(&request); err != nil {
t.Error(err)
}
if request.Method != enginewebview.MethodProfileRemove || request.Params["profileId"] != "extension-profile" || len(request.Params) != 1 {
t.Errorf("unexpected profile removal request: %+v", request)
}
if r.Header.Get("Authorization") != "Bearer secret" {
t.Error("missing host authentication")
}
w.Header().Set("Content-Type", "application/json")
w.Write([]byte(`{"error":{"code":"PROFILE_REMOVE_FAILED","message":"store busy"}}`))
}))
defer server.Close()
provider := New(enginewebview.RPCConfig{Network: "tcp", Address: strings.TrimPrefix(server.URL, "http://"), Token: "secret"})
err := provider.(enginewebview.ProfileRemover).RemoveProfile("extension-profile", "/host-only/path")
if err == nil || err.Error() != "store busy" {
t.Fatalf("cleanup failure was lost: %v", err)
}
}
+22
View File
@@ -10,8 +10,15 @@ import (
type TaskEventMask uint64
const (
// Keep these first two unchanged because Flutter already uses
// bit 0 for done and bit 1 for error.
TaskEventDone TaskEventMask = 1 << iota
TaskEventError
TaskEventStart
TaskEventProgress
TaskEventPause
TaskEventDelete
)
type TaskEvent struct {
@@ -42,11 +49,26 @@ func (s *Service) emitTaskEvent(event *download.Event) {
}
var mask TaskEventMask
switch event.Key {
case download.EventKeyDone:
mask = TaskEventDone
case download.EventKeyError:
mask = TaskEventError
case download.EventKeyStart:
mask = TaskEventStart
case download.EventKeyProgress:
mask = TaskEventProgress
case download.EventKeyPause:
mask = TaskEventPause
case download.EventKeyDelete:
mask = TaskEventDelete
default:
return
}
+1 -1
View File
@@ -156,7 +156,7 @@ func (s *Service) createTask(ctx *Context) *Response {
err error
)
if request.Rid != "" {
taskID, err = s.Downloader.Create(request.Rid)
taskID, err = s.Downloader.CreateWithOptions(request.Rid, request.Opts)
} else if request.Req != nil {
taskID, err = s.Downloader.CreateDirect(request.Req, request.Opts)
} else {
-21
View File
@@ -25,24 +25,6 @@ type Request struct {
SkipVerifyCert bool `json:"skipVerifyCert"`
}
// SetLabels replaces all labels on the request.
func (r *Request) SetLabels(labels map[string]string) {
r.Labels = labels
}
// PutLabel sets a label on the request.
func (r *Request) PutLabel(key, value string) {
if r.Labels == nil {
r.Labels = make(map[string]string)
}
r.Labels[key] = value
}
// DelLabel deletes a label from the request.
func (r *Request) DelLabel(key string) {
delete(r.Labels, key)
}
func (r *Request) Validate() error {
if r.URL == "" {
return fmt.Errorf("invalid request url")
@@ -94,9 +76,6 @@ type Resource struct {
}
func (r *Resource) Validate() error {
if r.Name == "" {
return fmt.Errorf("invalid resource name")
}
if len(r.Files) == 0 {
return fmt.Errorf("invalid resource files")
}
+87 -15
View File
@@ -21,6 +21,8 @@ import (
"github.com/GopeedLab/gopeed/internal/controller"
"github.com/GopeedLab/gopeed/internal/fetcher"
"github.com/GopeedLab/gopeed/internal/logger"
"github.com/GopeedLab/gopeed/internal/tempfiles"
webviewproxy "github.com/GopeedLab/gopeed/internal/webview/proxy"
"github.com/GopeedLab/gopeed/pkg/base"
"github.com/GopeedLab/gopeed/pkg/protocol/http"
"github.com/GopeedLab/gopeed/pkg/util"
@@ -97,8 +99,13 @@ type Progress struct {
}
type Downloader struct {
Logger *logger.Logger
ExtensionLogger *logger.Logger
tempFiles *tempfiles.Scope
webviewProfilesLock sync.Mutex
webviewProfiles map[string]*extensionWebViewProfile
webviewProxyLock sync.Mutex
webviewProxy *webviewproxy.Server
Logger *logger.Logger
ExtensionLogger *logger.Logger
cfg *DownloaderConfig
fetcherCache map[string]fetcher.Fetcher
@@ -152,6 +159,9 @@ func NewDownloader(cfg *DownloaderConfig) *Downloader {
}
func (d *Downloader) Setup() error {
if err := d.setupTempDir(); err != nil {
return err
}
d.blob = internalblob.NewRegistry("")
// setup storage
@@ -263,24 +273,45 @@ func (d *Downloader) Setup() error {
if d.GetTask(task.ID) == nil {
return
}
// Startup can replace resource metadata while resolving a Blob.
// Skip that short transition instead of reading half-initialized state.
if !task.lock.TryLock() {
return
}
task.statusLock.Lock()
if task.Status != base.DownloadStatusRunning && !task.Uploading {
task.statusLock.Unlock()
task.lock.Unlock()
return
}
if task.fetcher == nil {
task.statusLock.Unlock()
task.lock.Unlock()
return
}
current := task.fetcher.Progress().TotalDownloaded()
delta := current - task.Progress.Downloaded
if task.Meta != nil && task.Meta.Req != nil {
if producer := d.blob.Production(task.Meta.Req.URL); producer != nil {
downloaded, received := producer.Progress()
if task.producer != producer {
task.producer = producer
task.producerReceived = 0
task.speedArr = nil
}
current = downloaded
delta = max(0, received-task.producerReceived)
task.producerReceived = received
}
}
tick := float64(d.cfg.RefreshInterval) / 1000
downloadDataChanged := false
uploading := task.Uploading
if task.Status == base.DownloadStatusRunning {
downloadDataChanged = current != task.Progress.Downloaded
task.Progress.Used = task.timer.Used()
task.Progress.Speed = task.updateSpeed(current-task.Progress.Downloaded, tick)
task.Progress.Speed = task.updateSpeed(max(0, delta), tick)
task.Progress.Downloaded = current
}
@@ -299,6 +330,7 @@ func (d *Downloader) Setup() error {
if err := d.captureTaskStats(task, true, false); err != nil {
d.Logger.Warn().Err(err).Msgf("persist task stats failed, task id: %s", task.ID)
}
task.lock.Unlock()
// Listener callbacks may Pause/Continue and acquire statusLock.
d.emit(EventKeyProgress, task)
@@ -389,6 +421,9 @@ func (d *Downloader) parseFm(url string) (fetcher.FetcherManager, error) {
func (d *Downloader) setupFetcher(fm fetcher.FetcherManager, fetcher fetcher.Fetcher) {
ctl := controller.NewController()
ctl.TempDir = d.cfg.TempDir
ctl.TempFiles = d.tempFiles
ctl.ManagedProduction = func(raw string) bool { return d.blob.Production(raw) != nil }
ctl.GetConfig = func(v any) {
d.getProtocolConfig(fm.Name(), v)
}
@@ -582,12 +617,36 @@ func (d *Downloader) CreateDirectBatch(req *base.CreateTaskBatch) (taskId []stri
}
func (d *Downloader) Create(rrId string) (taskId string, err error) {
return d.CreateWithOptions(rrId, nil)
}
// CreateWithOptions creates a resolved task using the submitted options, or the
// resolve-time options when opts is nil.
func (d *Downloader) CreateWithOptions(rrId string, opts *base.Options) (taskId string, err error) {
d.fetcherMapLock.RLock()
fetcher, ok := d.fetcherCache[rrId]
d.fetcherMapLock.RUnlock()
if !ok {
return "", errors.New("invalid resource id")
}
if opts != nil {
opts, err = d.initOptions(opts.Clone())
if err != nil {
return "", err
}
if res := fetcher.Meta().Res; res != nil {
for _, index := range opts.SelectFiles {
if index < 0 || index >= len(res.Files) {
return "", fmt.Errorf("invalid selected file index: %d", index)
}
}
opts.InitSelectFiles(len(res.Files))
}
fetcher.Meta().Opts = opts
}
if res := fetcher.Meta().Res; res != nil {
res.CalcSize(fetcher.Meta().Opts.SelectFiles)
}
defer func() {
d.fetcherMapLock.Lock()
delete(d.fetcherCache, rrId)
@@ -1114,6 +1173,11 @@ func (d *Downloader) doDelete(task *Task, force bool) (err error) {
func (d *Downloader) Close() error {
d.closed.Store(true)
d.webviewProxyLock.Lock()
if d.webviewProxy != nil {
_ = d.webviewProxy.Close()
}
d.webviewProxyLock.Unlock()
closeArr := []func() error{
d.pauseAll,
@@ -1124,7 +1188,7 @@ func (d *Downloader) Close() error {
if d.blob != nil {
closeArr = append(closeArr, d.blob.Close)
}
closeArr = append(closeArr, d.storage.Close)
closeArr = append(closeArr, d.storage.Close, d.tempFiles.Close)
// Make sure all resources are released, if had error, return the last error
var lastErr error
for i, close := range closeArr {
@@ -1399,17 +1463,6 @@ func (d *Downloader) watch(task *Task) {
}
}
task.Progress.Used = task.timer.Used()
if task.Meta.Res.Size == 0 {
task.Meta.Res.Size = task.fetcher.Progress().TotalDownloaded()
}
used := task.Progress.Used / int64(time.Second)
if used == 0 {
used = 1
}
totalSize := task.Meta.Res.Size
task.Progress.Speed = totalSize / used
task.Progress.Downloaded = totalSize
if !d.markTaskDone(task) {
return
}
@@ -1808,7 +1861,9 @@ func (d *Downloader) doStart(task *Task) (err error) {
task.Meta.Res.CalcSize(task.Meta.Opts.SelectFiles)
}
task.statusLock.Lock()
task.Progress.Speed = 0
task.statusLock.Unlock()
if !d.taskIsRunningGeneration(task, generation) {
return nil
}
@@ -1857,6 +1912,7 @@ func (d *Downloader) doPause(task *Task) (err error) {
}
if handled {
d.emit(EventKeyPause, task)
d.triggerWebhooks(WebhookEventDownloadPause, task, nil)
}
return nil
}
@@ -1962,11 +2018,27 @@ func (d *Downloader) markTaskDone(task *Task) bool {
if task == nil || task.statusLock == nil {
return false
}
task.lock.Lock()
defer task.lock.Unlock()
task.statusLock.Lock()
defer task.statusLock.Unlock()
if task.Status != base.DownloadStatusRunning {
return false
}
task.Progress.Used = task.timer.Used()
if task.Meta.Res.Size == 0 {
task.Meta.Res.Size = task.fetcher.Progress().TotalDownloaded()
}
used := task.Progress.Used / int64(time.Second)
if used == 0 {
used = 1
}
totalSize := task.Meta.Res.Size
task.Progress.Speed = totalSize / used
task.Progress.Downloaded = totalSize
task.producer = nil
task.producerReceived = 0
task.updateStatus(base.DownloadStatusDone)
task.runGeneration++
return true
+26 -6
View File
@@ -45,6 +45,32 @@ type generationTestManager struct {
func TestExtensionTaskMethodsMatchEventCapabilities(t *testing.T) {
taskType := reflect.TypeOf((*Task)(nil))
for _, typ := range []reflect.Type{taskType, reflect.TypeOf((*ExtensionTask)(nil)), reflect.TypeOf((*OnErrorExtensionTask)(nil))} {
for _, name := range []string{"SetUrl", "SetExtra", "PatchExtra", "PatchHeaders", "SetMethod", "SetBody", "SetHeaders", "PutHeader", "DelHeader", "SetTrackers"} {
if _, ok := typ.MethodByName(name); ok {
t.Fatalf("%v unexpectedly exposes %s", typ, name)
}
}
}
requestType := reflect.TypeOf((*ExtensionTaskRequest)(nil))
for _, name := range []string{"SetUrl", "SetLabels", "PutLabel", "DelLabel", "SetMethod", "SetBody", "SetHeaders", "PutHeader", "DelHeader", "SetTrackers"} {
if _, ok := requestType.MethodByName(name); !ok {
t.Fatalf("request wrapper does not expose %s", name)
}
}
for _, typ := range []reflect.Type{reflect.TypeOf(ExtensionTask{}), reflect.TypeOf(OnErrorExtensionTask{})} {
if _, ok := typ.FieldByName("Req"); ok {
t.Fatalf("%v exposes the removed task.req entry", typ)
}
field, ok := typ.FieldByName("Meta")
if !ok || field.Type != reflect.TypeOf((*ExtensionTaskMeta)(nil)) {
t.Fatalf("%v has no metadata wrapper", typ)
}
}
field, ok := reflect.TypeOf(ExtensionTaskMeta{}).FieldByName("Req")
if !ok || field.Type != requestType {
t.Fatal("meta.req has no request wrapper")
}
if _, ok := taskType.MethodByName("Continue"); ok {
t.Fatal("regular task unexpectedly exposes Continue")
}
@@ -53,17 +79,11 @@ func TestExtensionTaskMethodsMatchEventCapabilities(t *testing.T) {
}
extensionTaskType := reflect.TypeOf((*ExtensionTask)(nil))
if _, ok := extensionTaskType.MethodByName("SetUrl"); !ok {
t.Fatal("extension task does not expose SetUrl")
}
if _, ok := extensionTaskType.MethodByName("Continue"); ok {
t.Fatal("non-error extension task unexpectedly exposes Continue")
}
errorTaskType := reflect.TypeOf((*OnErrorExtensionTask)(nil))
if _, ok := errorTaskType.MethodByName("SetUrl"); !ok {
t.Fatal("onError task does not expose SetUrl")
}
if _, ok := errorTaskType.MethodByName("Continue"); !ok {
t.Fatal("onError task does not expose Continue")
}
+34 -5
View File
@@ -6,9 +6,12 @@ import (
"fmt"
"sync"
"github.com/GopeedLab/gopeed/internal/production"
"github.com/GopeedLab/gopeed/internal/tempfiles"
"github.com/GopeedLab/gopeed/pkg/base"
gojaerror "github.com/GopeedLab/gopeed/pkg/download/engine/inject/error"
fetchapi "github.com/GopeedLab/gopeed/pkg/download/engine/inject/fetch"
ffmpegapi "github.com/GopeedLab/gopeed/pkg/download/engine/inject/ffmpeg"
"github.com/GopeedLab/gopeed/pkg/download/engine/inject/file"
"github.com/GopeedLab/gopeed/pkg/download/engine/inject/stream"
"github.com/GopeedLab/gopeed/pkg/download/engine/inject/vm"
@@ -159,8 +162,12 @@ func (e *Engine) addCleanup(cleanup func()) {
}
type Config struct {
ProxyConfig *base.DownloaderProxyConfig
StreamConfig *stream.Config
TempFiles *tempfiles.Scope
// HTTPUserAgent is a snapshot of the downloader HTTP default. Nil leaves standalone engines unchanged.
TempDir string
HTTPUserAgent *string
ProxyConfig *base.DownloaderProxyConfig
StreamConfig *stream.Config
}
func NewEngine(cfg *Config) *Engine {
@@ -200,12 +207,29 @@ func NewEngine(cfg *Config) *Engine {
if _, err := runtime.RunString("global.location = new URL('http://localhost');"); err != nil {
return
}
if err := stream.Enable(runtime, loop, cfg.StreamConfig); err != nil {
producers := &production.Registry{}
streamCfg := stream.Config{}
if cfg.StreamConfig != nil {
streamCfg = *cfg.StreamConfig
}
streamCfg.Producers = producers
if err := stream.Enable(runtime, loop, &streamCfg); err != nil {
return
}
if err := ffmpegapi.Enable(runtime, loop, &ffmpegapi.Config{
Producers: producers,
TempDir: cfg.TempDir,
TempFiles: cfg.TempFiles,
ProxyHandler: cfg.ProxyConfig.ToHandler(),
DefaultUserAgent: cfg.HTTPUserAgent,
RegisterCleanup: engine.addCleanup,
}); err != nil {
return
}
if err := fetchapi.Enable(runtime, loop, &fetchapi.Config{
ProxyHandler: cfg.ProxyConfig.ToHandler(),
RegisterCleanup: engine.addCleanup,
ProxyHandler: cfg.ProxyConfig.ToHandler(),
DefaultUserAgent: cfg.HTTPUserAgent,
RegisterCleanup: engine.addCleanup,
}); err != nil {
return
}
@@ -265,3 +289,8 @@ func callExportedFunction(fn func(goja.FunctionCall) goja.Value, args ...goja.Va
Arguments: args,
}), nil
}
// Post queues a notification without waiting for its callback or returned promise.
func (e *Engine) Post(fn func(*goja.Runtime)) bool {
return e.loop.RunOnLoop(fn)
}
@@ -0,0 +1,222 @@
package engine_test
import (
"encoding/json"
"fmt"
"io"
"net/http"
"net/http/httptest"
"strings"
"testing"
"github.com/GopeedLab/gopeed/pkg/download/engine"
)
// Exercise server-side header permissions, including formerly forbidden names,
// download authentication, metadata and non-safelisted no-cors values.
var extensionRequestHeaders = [][2]string{
{"Accept-Charset", "utf-8"}, {"Accept-Encoding", "identity"},
{"Access-Control-Request-Headers", "authorization, x-download"},
{"Access-Control-Request-Method", "POST"}, {"Connection", "close"},
{"Connection", "keep-alive"},
{"Content-Length", "4"}, {"Cookie", "sid=manual"}, {"Cookie2", "legacy=1"},
{"Date", "Fri, 09 Oct 2026 00:00:00 GMT"}, {"DNT", "1"},
{"Host", "download.example:1234"}, {"Origin", "https://download.example"},
{"Permissions-Policy", "camera=()"}, {"Referer", "https://download.example/file"},
{"TE", "trailers"}, {"Trailer", "X-Checksum"}, {"Set-Cookie", "sid=request"},
{"Set-Cookie2", "legacy=request"}, {"Via", "1.1 proxy"},
{"Proxy-Authorization", "Basic dGVzdA=="}, {"Proxy-", "custom"},
{"Sec-Fetch-Site", "same-origin"}, {"Sec-Fetch-Mode", "custom"},
{"Sec-CH-UA", `"Download client"`}, {"Sec-", "custom"},
{"X-HTTP-Method", "TRACE"}, {"X-HTTP-Method-Override", "CONNECT"},
{"X-Method-Override", "TRACK"}, {"Authorization", "Bearer token"},
{"User-Agent", "Gopeed extension"}, {"Content-Type", "application/json"},
{"Range", "bytes=0-1,3-4"}, {"X-Download-Token", "custom"},
{"Accept-Language", strings.Repeat("x", 129)},
}
func TestExtensionRequestsSendNodeHeaders(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) {
body, err := io.ReadAll(request.Body)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
name := request.URL.Query().Get("header")
value := request.Header.Get(name)
if name == "Host" {
value = request.Host
}
_ = json.NewEncoder(w).Encode([]string{value, string(body)})
}))
t.Cleanup(server.Close)
clients := []struct{ name, script string }{
{"fetch record", `const response = await fetch(url, {method: "POST", body: "test", headers: {[name]: value}}); return (await response.text()).trim();`},
{"fetch no-cors Headers", `const response = await fetch(url, {method: "POST", body: "test", mode: "no-cors", headers: new Headers([[name.toLowerCase(), value]])}); return (await response.text()).trim();`},
{"fetch Request clone", `const request = new Request(url, {method: "POST", body: "test", mode: "same-origin", headers: [[name.toUpperCase(), value]]}); const response = await fetch(request.clone()); return (await response.text()).trim();`},
}
for _, credentials := range []bool{false, true} {
clients = append(clients, struct{ name, script string }{
fmt.Sprintf("XHR credentials=%t", credentials),
fmt.Sprintf(`return new Promise((resolve, reject) => {
const xhr = new XMLHttpRequest();
xhr.open("POST", url);
xhr.withCredentials = %t;
xhr.setRequestHeader(name.toLowerCase(), value);
xhr.onload = () => resolve(xhr.responseText.trim());
xhr.onerror = () => reject(new Error("XHR failed"));
xhr.send("test");
});`, credentials),
})
}
for _, header := range extensionRequestHeaders {
for _, client := range clients {
t.Run(header[0]+"/"+client.name, func(t *testing.T) {
runtime := engine.NewEngine(nil)
t.Cleanup(runtime.Close)
value, err := runtime.RunString(fmt.Sprintf(`(async () => {
const name = %q, value = %q;
const url = %q + "?header=" + encodeURIComponent(name);
%s
})()`, header[0], header[1], server.URL, client.script))
if err != nil {
t.Fatal(err)
}
want, _ := json.Marshal([]string{header[1], "test"})
if value != string(want) {
t.Fatalf("server received %#v, want %s", value, want)
}
})
}
}
}
func TestExtensionHeaderMutationsMatchNode(t *testing.T) {
headers := append([][2]string(nil), extensionRequestHeaders...)
// Node stores these in Headers/Request, but rejects them during fetch.
headers = append(headers, [2]string{"Expect", "100-continue"}, [2]string{"Keep-Alive", "timeout=5"}, [2]string{"Transfer-Encoding", "chunked"}, [2]string{"Upgrade", "websocket"})
data, _ := json.Marshal(headers)
runtime := engine.NewEngine(nil)
t.Cleanup(runtime.Close)
_, err := runtime.RunString(fmt.Sprintf(`(() => {
for (const mode of ["cors", "same-origin", "no-cors"]) {
for (const [name, value] of %s) {
const request = new Request("http://example.com", {mode, headers: {[name]: value}});
const headers = request.headers;
if (headers.get(name) !== value) throw new Error("Filtered init " + name + " in " + mode);
headers.delete(name.toUpperCase());
if (headers.has(name)) throw new Error("Cannot delete " + name);
headers.append(name, value);
if (headers.get(name) !== value) throw new Error("Cannot append " + name);
headers.set(name, value);
if (request.clone().headers.get(name) !== value) throw new Error("Cannot clone " + name);
}
}
return true;
})()`, data))
if err != nil {
t.Fatal(err)
}
}
func TestExtensionRequestsIgnoreUnsupportedTransportHeaders(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) {
body, err := io.ReadAll(request.Body)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
value := request.Header.Get(request.URL.Query().Get("header"))
_ = json.NewEncoder(w).Encode([]string{value, string(body)})
}))
t.Cleanup(server.Close)
for _, header := range [][2]string{
{"Expect", "100-continue"}, {"Keep-Alive", "timeout=5"},
{"Transfer-Encoding", "chunked"}, {"Upgrade", "websocket"},
{"Connection", "upgrade"}, {"Content-Length", "3"},
{"Content-Length", "5"}, {"Content-Length", "invalid"},
{"Content-Length", "-1"}, {"Content-Length", ""},
} {
for _, xhr := range []bool{false, true} {
t.Run(fmt.Sprintf("%s=%s/XHR=%t", header[0], header[1], xhr), func(t *testing.T) {
runtime := engine.NewEngine(nil)
t.Cleanup(runtime.Close)
value, err := runtime.RunString(fmt.Sprintf(`(async () => {
const name = %q, value = %q;
const url = %q + "?header=" + encodeURIComponent(name);
if (%t) return new Promise((resolve, reject) => {
const xhr = new XMLHttpRequest();
xhr.open("POST", url);
xhr.setRequestHeader(name, value);
xhr.onload = () => resolve(xhr.responseText.trim());
xhr.onerror = () => reject(new Error("XHR failed"));
xhr.send("test");
});
const response = await fetch(url, {method: "POST", body: "test", headers: {[name]: value}});
return (await response.text()).trim();
})()`, header[0], header[1], server.URL, xhr))
if err != nil {
t.Fatal(err)
}
wireValue := ""
if header[0] == "Content-Length" {
wireValue = "4"
}
want, _ := json.Marshal([]string{wireValue, "test"})
if value != string(want) {
t.Fatalf("server received %#v, want %s", value, want)
}
})
}
}
}
func TestExtensionResponseExposesSetCookie(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.Header().Add("Set-Cookie", "sid=one; HttpOnly")
w.Header().Add("Set-Cookie", "token=two")
w.Header().Set("Set-Cookie2", "legacy=three")
_, _ = w.Write([]byte("ok"))
}))
t.Cleanup(server.Close)
runtime := engine.NewEngine(nil)
t.Cleanup(runtime.Close)
value, err := runtime.RunString(fmt.Sprintf(`(async () => {
const response = await fetch(%q);
const clone = response.clone();
await response.text(); await clone.text();
let immutable = false;
try { response.headers.set("Set-Cookie", "changed=1"); } catch (error) { immutable = error instanceof TypeError; }
const synthetic = new Response(null, {headers: {"Set-Cookie": "initial=1"}});
synthetic.headers.append("Set-Cookie", "appended=2");
return JSON.stringify({cookies: clone.headers.getSetCookie(), legacy: response.headers.get("set-cookie2"), immutable, synthetic: synthetic.headers.getSetCookie()});
})()`, server.URL))
if err != nil {
t.Fatal(err)
}
const want = `{"cookies":["sid=one; HttpOnly","token=two"],"legacy":"legacy=three","immutable":true,"synthetic":["initial=1","appended=2"]}`
if value != want {
t.Fatalf("unexpected response headers: got %#v want %s", value, want)
}
}
func TestExtensionHeadersRejectInvalidSyntax(t *testing.T) {
runtime := engine.NewEngine(nil)
t.Cleanup(runtime.Close)
_, err := runtime.RunString(`(() => {
for (const [name, value] of [["Bad Header", "ok"], ["", "ok"], ["X:Header", "ok"],
["X-Test", "a\nb"], ["X-Test", "a\rb"], ["X-Test", "\0"], ["X-Test", "\u0100"]]) {
for (const create of [() => new Headers([[name, value]]),
() => new Request("http://example.com", {mode: "no-cors", headers: {[name]: value}}),
() => { const xhr = new XMLHttpRequest(); xhr.open("GET", "http://example.com"); xhr.setRequestHeader(name, value); }]) {
let rejected = false;
try { create(); } catch (error) { rejected = error instanceof TypeError; }
if (!rejected) throw new Error("Accepted invalid HTTP header syntax");
}
}
return true;
})()`)
if err != nil {
t.Fatal(err)
}
}
@@ -105,6 +105,105 @@ func TestXMLHttpRequestUsesFetch(t *testing.T) {
}
}
func TestExtensionRequestsSendExplicitCookie(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) {
_, _ = w.Write([]byte(request.Header.Get("Cookie")))
}))
t.Cleanup(server.Close)
type requestTest struct {
name string
script string
}
tests := []requestTest{
{"fetch record", `const response = await fetch(url, {headers: {Cookie: cookie}}); return response.text();`},
{"fetch Headers", `const response = await fetch(url, {headers: new Headers({cookie})}); return response.text();`},
{"fetch pairs", `const response = await fetch(url, {headers: [["cOoKiE", cookie]]}); return response.text();`},
{"fetch repeated Cookie", `const headers = new Headers([["Cookie", "sid=manual"], ["cookie", "token=provided"]]); if (headers.get("cookie") !== cookie) throw new Error("Incorrect Cookie separator"); const response = await fetch(url, {headers}); return response.text();`},
{"fetch Request clone", `const request = new Request(url, {headers: {Cookie: cookie}}); const response = await fetch(request.clone()); return response.text();`},
{"fetch Request set", `const request = new Request(url); request.headers.set("Cookie", cookie); const response = await fetch(request); return response.text();`},
{"fetch Request append", `const request = new Request(url); request.headers.append("Cookie", cookie); const response = await fetch(request); return response.text();`},
{"fetch no-cors mutations", `const request = new Request(url, {mode: "no-cors"}); request.headers.set("Cookie", "removed=1"); request.headers.delete("COOKIE"); if (request.headers.has("cookie")) throw new Error("Cookie was not deleted"); request.headers.append("Cookie", cookie); const response = await fetch(request.clone()); return response.text();`},
}
for _, mode := range []string{"cors", "same-origin", "no-cors"} {
for _, credentials := range []string{"omit", "same-origin", "include"} {
tests = append(tests, requestTest{
name: "fetch " + mode + " " + credentials,
script: fmt.Sprintf(`const response = await fetch(url, {headers: {Cookie: cookie}, mode: %q, credentials: %q}); return response.text();`, mode, credentials),
})
}
}
for _, withCredentials := range []bool{false, true} {
tests = append(tests, requestTest{
name: fmt.Sprintf("XHR withCredentials=%t", withCredentials),
script: fmt.Sprintf(`return new Promise((resolve, reject) => {
const xhr = new XMLHttpRequest();
xhr.open("GET", url);
xhr.withCredentials = %t;
xhr.setRequestHeader("cOoKiE", cookie);
xhr.onload = () => resolve(xhr.responseText);
xhr.onerror = () => reject(new Error("XHR failed"));
xhr.send();
});`, withCredentials),
})
}
tests = append(tests, requestTest{"XHR repeated Cookie", `return new Promise((resolve, reject) => {
const xhr = new XMLHttpRequest();
xhr.open("GET", url);
xhr.setRequestHeader("Cookie", "sid=manual");
xhr.setRequestHeader("cookie", "token=provided");
xhr.onload = () => resolve(xhr.responseText);
xhr.onerror = () => reject(new Error("XHR failed"));
xhr.send();
});`})
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
runtime := engine.NewEngine(nil)
t.Cleanup(runtime.Close)
const cookie = "sid=manual; token=provided"
value, err := runtime.RunString(fmt.Sprintf(`(async () => {
const url = %q;
const cookie = %q;
%s
})()`, server.URL, cookie, test.script))
if err != nil {
t.Fatal(err)
}
if value != cookie {
t.Fatalf("server received Cookie %q, want %q", value, cookie)
}
})
}
}
func TestFetchExplicitCookieWithCookieJar(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) {
if request.URL.Path == "/seed" {
http.SetCookie(w, &http.Cookie{Name: "stored", Value: "jar", Path: "/"})
}
_, _ = w.Write([]byte(request.Header.Get("Cookie")))
}))
t.Cleanup(server.Close)
runtime := engine.NewEngine(nil)
t.Cleanup(runtime.Close)
value, err := runtime.RunString(fmt.Sprintf(`(async () => {
const url = %q;
await (await fetch(url + "/seed")).text();
const automatic = await (await fetch(url)).text();
const combined = await (await fetch(url, {headers: {Cookie: "manual=provided"}})).text();
const omitted = await (await fetch(url, {credentials: "omit", headers: {Cookie: "manual=provided"}})).text();
const empty = await (await fetch(url, {credentials: "omit"})).text();
return JSON.stringify({automatic, combined, omitted, empty});
})()`, server.URL))
if err != nil {
t.Fatal(err)
}
const want = `{"automatic":"stored=jar","combined":"manual=provided; stored=jar","omitted":"manual=provided","empty":""}`
if value != want {
t.Fatalf("unexpected CookieJar result: got %#v want %s", value, want)
}
}
func TestFetchResponseAndRequestCloneHaveIndependentBodies(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) {
if request.Method == http.MethodPost {
+160
View File
@@ -0,0 +1,160 @@
# Extension FFmpeg runtime
`gopeed.runtime.ffmpeg.merge(options)` returns a `ReadableStream<Uint8Array>`.
It merges one video track and one audio track without re-encoding, using
[go-ffmpreg](https://codeberg.org/gruf/go-ffmpreg)'s embedded FFmpeg 5.1.10 and
wazero. No system FFmpeg installation is required. go-ffmpreg and its embedded
GPL-enabled FFmpeg are distributed under GPLv3; see the pinned dependency's
LICENSE, README and build scripts for its source and included components.
## HTTP inputs and one downloadable output
```js
gopeed.events.onResolve(async (ctx) => {
// Resolve these URLs and headers using the site's API in your extension.
const video = { url: videoURL, headers: { Referer: pageURL } };
const audio = { url: audioURL, headers: { Referer: pageURL } };
const url = await gopeed.runtime.blob.createObjectURL(
() => gopeed.runtime.ffmpeg.merge({ video, audio }),
{ contentType: "video/mp4" }
);
ctx.res = {
name: "video",
files: [{ name: "video.mp4", req: { url } }]
};
});
```
The opener creates a new merge on each open. Output size is unknown and range
support must remain disabled. Do not declare the sum of the input sizes as the
output size. The downloader reads the merged output as one file; FFmpeg errors
propagate through the stream instead of completing a partial download.
## JS inputs, including SABR
Stream inputs **must** be created by an `inputs({ signal })` callback. It returns
`{ video, audio }`, each an HTTP descriptor or `ReadableStream<Uint8Array>`,
including mixed inputs. Do not start producers before this callback. SABR/UMP decoding belongs
to the extension: pass the resulting media streams, with initialization data,
not the SABR protocol bytes. Keep each selected track's codec/configuration
stable for the duration of a merge.
```js
const output = gopeed.runtime.ffmpeg.merge({
inputs: async ({ signal }) => {
const session = await prepareSession({ signal });
const { videoStream, audioStream, abort } = await session.openStreams();
if (signal.aborted) { abort(); throw new Error("Cancelled"); }
signal.addEventListener("abort", abort, { once: true });
return { video: videoStream, audio: audioStream };
},
signal: abortController.signal,
args: ["-shortest", "-metadata", "title=Example"]
});
```
Creating the output does not start requests or acquire input readers until the
output is pulled. Inputs then download to temporary disk chunks even while the
job waits for a global FFmpeg execution slot. Only merging occupies a slot.
The callback runs once per output; its signal aborts on cancellation, engine
shutdown, failure, and completion. A callback must cooperate with cancellation;
returned streams are cancelled even if the callback finishes after cancellation.
A stream input is single-use. To retry a Blob-backed task,
its opener must create a new SABR session and fresh input streams. Arbitrary
output byte ranges / resumable merged output are not supported.
`gopeed.runtime.ffmpeg.supportsInputFactory === true` identifies the callback API.
`gopeed.runtime.ffmpeg.supportsProducerProgress === true` identifies disk-backed
inputs and automatic Blob production progress. Require the latter before using
one shared SABR session for both tracks; older bounded buffers can deadlock.
## Options
- `inputs`: callback returning `{ video, audio }`, synchronously or asynchronously.
Required for stream inputs. Cannot be combined with top-level `video`/`audio`.
- `video`, `audio`: HTTP descriptors only when supplied at the top level; both
are required unless `inputs` is supplied. HTTP descriptors have `url` and optional
`headers` (standard `HeadersInit`). HTTP requests use Gopeed's configured
proxy. Supply required cookies/authorization explicitly; browser or `fetch`
cookie jars are not implicitly shared. When `User-Agent` is absent, requests
use the downloader HTTP protocol's configured UA, captured when the extension
engine is created, matching fetch and XHR. Explicit UA values (including an
empty string) take precedence.
- `format`: defaults to `"mp4"`. MP4 output always enables fragmentation
(`frag_keyframe+empty_moov+default_base_moof`) for non-seekable output. The
other accepted muxers are `"mkv"`/`"matroska"`, `"webm"`, and `"mpegts"`.
Codec/container incompatibility is an error, not an automatic conversion.
Choose the matching filename and Blob content type in the extension.
- `args`: optional array of **output options**, not a shell command or a full
FFmpeg invocation. Defaults select `0:v:0`, `1:a:0`, and `-c copy`. Additional
`-map` options are additive. Allowed options are listed in `outputOptions` in
`runtime.go`: metadata, dispositions, bitstream filters, duration/shortest,
fragmentation, timestamp/interleaving options and `-strict`. Codec options
must remain `copy`. Extra inputs, output paths, reports and arbitrary global
options are rejected. Required MP4 fragmentation is enforced by the host.
- `signal`: optional `AbortSignal`. Aborting or cancelling the output stops
network requests and native execution, cancels input readers, and removes
temporary buffers. Await/consume the output inside a live engine or retain
it through a Blob URL; unowned work ends with the extension engine.
The backend defaults to strict input errors (`-xerror`). Native errors include
the FFmpeg exit code and a bounded stderr tail. No host directories are exposed
to the WASM module; it can access only its two input files and standard output.
## Input behavior and resource limits
HTTP inputs start with a 1 MiB Range GET, reusing the bytes for media reads.
A validated 206 response with a known total size enables seeking; a 200
response is consumed as-is, sequentially. Subsequent ranges verify positions,
lengths, total size and available ETag/Last-Modified validators. Without a
server validator, same-size remote changes cannot be reliably detected.
Content encoding is forced to identity. Authentication/status failures and
malformed range responses are errors. There is no full-file seek fallback.
HTTP and sequential JS inputs are drained into **8 MiB disk chunks** beneath
`TempDir/gopeed-media-<random>`. Range downloads stream each response into as many chunks as
needed, tracking valid byte intervals independently of sparse file length.
Verified HTTP Range inputs can seek, skip missing blocks, and re-fetch deleted
blocks. Sequential inputs cannot seek even though their bytes are on disk.
Consumed chunks are deleted; cancellation, errors and completion remove the
remaining input directories. Disk space is not capped; write failures stop the
merge. Media size does not determine the size of in-memory input queues.
JS writes await disk writes, and Blob output writes await bounded downstream
capacity. Extensions must still propagate backpressure through upstream queues
and network reads. Both tracks can share a SABR response because disk spooling
continues independently of which track FFmpeg currently reads.
When a Blob opener returns the merge stream directly, the runtime automatically
associates its input progress with that Blob. No public progress callback or
extra `createObjectURL` argument is required. Return the merge stream itself;
wrapping it in another stream loses this association. Downloaded bytes count
unique input offsets; speed counts received input bytes, including re-fetches.
The task stays downloading with unknown total size while queued or merging;
once input traffic stops the displayed speed settles to zero. Completion uses
the actual merged output size. Generated output waits are governed by producer
cancellation/errors rather than the HTTP body idle timeout; HTTP media inputs
still enforce a 15-second network idle deadline.
`tempDir` defaults to Go's `os.TempDir()` when omitted. Flutter only supplies
an explicit directory on Android/iOS, using `getTemporaryDirectory()` directly;
no extra `gopeed` parent directory is added. Desktop/server callers can override
it via startup JSON or `--temp-dir`.
Initialization never clears this directory. Media inputs, HTTP prefetch and
extension installation staging create unique `gopeed-media-*`,
`gopeed-prefetch-*`, and `gopeed-extension-*` paths directly beneath it. The
downloader tracks only its own paths and removes remaining ones on normal
shutdown, without scanning or deleting other files in a shared temporary
directory. Resources still clean up on completion/cancellation, and consumed
media chunks are deleted promptly. Forced process termination can leave files
behind; the next startup does not delete them. Persistent HLS resume data and
WebView profiles are unchanged.
At most two WASM merges execute concurrently per process, with a 512 MiB
linear-memory limit per instance. Compilation is shared and lazy; the upstream
embedded package still decompresses its WASM at process initialization. Native
compilation memory, host buffers and JS allocations are additional to the
linear-memory limit. Playback support depends on the selected codecs, not just
the `.mp4` suffix. Platform execution and performance must be tested on target
devices, especially mobile/32-bit devices.
+248
View File
@@ -0,0 +1,248 @@
package ffmpeg
import (
"context"
"errors"
"fmt"
"io"
"net/http"
"sync"
"github.com/GopeedLab/gopeed/internal/tempfiles"
)
// HTTPSpool downloads ahead independently of the FFmpeg semaphore. Range
// sources can fetch a missing block on demand, including blocks already evicted.
type HTTPSpool struct {
demand chan int64
demandDone chan struct{}
ctx context.Context
cancel context.CancelFunc
source Input
disk *DiskInput
network sync.Mutex
readMu sync.Mutex
pos int64
done chan struct{}
}
func OpenSpoolingHTTP(ctx context.Context, client *http.Client, source HTTPSource, root string, owners ...*tempfiles.Scope) (Input, error) {
ctx, cancel := context.WithCancel(ctx)
in, err := OpenHTTP(ctx, client, source)
if err != nil {
cancel()
return nil, err
}
disk, err := NewDiskInput(ctx, root, owners...)
if err != nil {
cancel()
in.Close()
return nil, err
}
if ranged := in.(*httpInput); ranged.ranged {
if _, err = disk.spool.WriteAt(ranged.block, 0); err != nil {
cancel()
in.Close()
disk.Close()
return nil, err
}
ranged.block = nil
}
h := &HTTPSpool{ctx: ctx, cancel: cancel, source: in, disk: disk, done: make(chan struct{}), demand: make(chan int64, 1), demandDone: make(chan struct{})}
go func() {
defer close(h.demandDone)
for {
select {
case offset := <-h.demand:
// A prefetch may have filled this position while demand was queued.
var probe [1]byte
if n, err, _ := h.disk.spool.readAt(probe[:], offset); n > 0 || err != nil {
continue
}
if err := h.fetch(offset / SpoolBlockSize * SpoolBlockSize); err != nil {
h.disk.End(err)
return
}
case <-ctx.Done():
return
}
}
}()
go func() {
defer close(h.done)
var err error
if !in.Seekable() {
_, err = io.CopyBuffer(disk, in, make([]byte, 64*1024))
} else {
for offset := int64(0); offset < in.Size(); offset += 3 * SpoolBlockSize {
if err = h.fetch(offset); err != nil {
break
}
}
}
disk.End(err)
}()
return h, nil
}
func (h *HTTPSpool) fetch(offset int64) error {
h.network.Lock()
defer h.network.Unlock()
if err := h.ctx.Err(); err != nil {
return err
}
source := h.source.(*httpInput)
end := offset + min(3*SpoolBlockSize, h.source.Size()-offset)
buffer := make([]byte, 64*1024)
for offset < end {
n, err, _ := h.disk.spool.readAt(buffer[:min(int64(len(buffer)), end-offset)], offset)
if err != nil {
return err
}
if n > 0 {
offset += int64(n)
continue
}
resp, err := source.request(offset, end-1)
if err != nil {
return err
}
err = func() error {
defer resp.Body.Close()
if resp.StatusCode != http.StatusPartialContent {
return fmt.Errorf("ffmpeg: Range request returned %d; source changed or stopped supporting Range", resp.StatusCode)
}
start, last, size, err := contentRange(resp.Header.Get("Content-Range"))
if err != nil || start != offset || last >= end || size != source.size {
return errors.New("ffmpeg: inconsistent Content-Range")
}
if source.etag != "" && resp.Header.Get("ETag") != source.etag {
return errors.New("ffmpeg: HTTP input ETag changed")
}
if source.etag == "" && source.modified != "" && resp.Header.Get("Last-Modified") != source.modified {
return errors.New("ffmpeg: HTTP input Last-Modified changed")
}
if err := identityEncoding(resp); err != nil {
return err
}
remaining := last - start + 1
if resp.ContentLength >= 0 && resp.ContentLength != remaining {
return errors.New("ffmpeg: Range response length mismatch")
}
for remaining > 0 {
n, err := resp.Body.Read(buffer[:min(int64(len(buffer)), remaining)])
if n > 0 {
if _, e := h.disk.spool.WriteAt(buffer[:n], offset); e != nil {
return e
}
offset += int64(n)
remaining -= int64(n)
}
if err != nil {
if err == io.EOF && remaining == 0 {
return nil
}
if err == io.EOF {
return io.ErrUnexpectedEOF
}
return err
}
if n == 0 {
return io.ErrNoProgress
}
}
n, err := resp.Body.Read(buffer[:1])
if n != 0 || err != io.EOF {
if err != nil && err != io.EOF {
return err
}
return errors.New("ffmpeg: oversized Range response")
}
return nil
}()
if err != nil {
return err
}
}
return nil
}
func (h *HTTPSpool) Read(p []byte) (int, error) {
if !h.Seekable() {
return h.disk.Read(p)
}
h.readMu.Lock()
defer h.readMu.Unlock()
if len(p) == 0 {
return 0, nil
}
if h.pos >= h.Size() {
if err := h.disk.spool.discard((h.Size() - 1) / SpoolBlockSize); err != nil {
return 0, err
}
return 0, io.EOF
}
for {
if err := h.ctx.Err(); err != nil {
return 0, err
}
n, err, wake := h.disk.spool.readAt(p, h.pos)
if n > 0 {
block := h.pos / SpoolBlockSize
h.pos += int64(n)
if h.pos/SpoolBlockSize > block {
if e := h.disk.spool.discard(block); err == nil {
err = e
}
}
return n, err
}
if err != nil {
return 0, err
}
select {
case h.demand <- h.pos:
default:
}
// Read as soon as a partial response supplies these bytes, rather than
// waiting for its entire multi-block Range request to finish.
select {
case <-wake:
case <-h.ctx.Done():
return 0, h.ctx.Err()
}
}
}
func (h *HTTPSpool) Seek(offset int64, whence int) (int64, error) {
if !h.Seekable() {
return 0, ErrSeek
}
h.readMu.Lock()
defer h.readMu.Unlock()
base := int64(0)
switch whence {
case io.SeekStart:
case io.SeekCurrent:
base = h.pos
case io.SeekEnd:
base = h.Size()
default:
return 0, errors.New("invalid seek origin")
}
next := base + offset
if next < 0 || (offset > 0 && next < base) {
return 0, errors.New("invalid seek offset")
}
h.pos = next
return next, nil
}
func (h *HTTPSpool) Size() int64 { return h.source.Size() }
func (h *HTTPSpool) Seekable() bool { return h.source.Seekable() }
func (h *HTTPSpool) Progress() (int64, int64) { return h.disk.Progress() }
func (h *HTTPSpool) Close() error {
h.cancel()
h.source.Close()
<-h.done
<-h.demandDone
return h.disk.Close()
}
func (h *HTTPSpool) Failures() <-chan error { return h.disk.Failures() }
@@ -0,0 +1,198 @@
package ffmpeg
import (
"bytes"
"context"
"fmt"
"io"
"net/http"
"net/http/httptest"
"sync/atomic"
"testing"
"time"
)
func TestHTTPSpoolRangeCrossBlocksAndRefetch(t *testing.T) {
data := make([]byte, 4*SpoolBlockSize+57)
for i := range data {
data[i] = byte(i*31 + i/257)
}
var large atomic.Bool
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
var start, end int64
fmt.Sscanf(r.Header.Get("Range"), "bytes=%d-%d", &start, &end)
if end-start+1 > SpoolBlockSize {
large.Store(true)
}
w.Header().Set("ETag", `"stable"`)
http.ServeContent(w, r, "media", time.Time{}, bytes.NewReader(data))
}))
defer srv.Close()
input, err := OpenSpoolingHTTP(context.Background(), srv.Client(), HTTPSource{URL: srv.URL}, t.TempDir())
if err != nil {
t.Fatal(err)
}
defer input.Close()
h := input.(*HTTPSpool)
select {
case <-h.done:
case <-time.After(10 * time.Second):
t.Fatal("prefetch did not finish")
}
if !large.Load() {
t.Fatal("did not fetch multiple blocks in one response")
}
for _, offset := range []int64{3*SpoolBlockSize - 123, 0, 2*SpoolBlockSize + 41, SpoolBlockSize - 5} {
if _, err := h.Seek(offset, io.SeekStart); err != nil {
t.Fatal(err)
}
got := make([]byte, min(int64(2*SpoolBlockSize), int64(len(data))-offset))
if _, err := io.ReadFull(h, got); err != nil {
t.Fatal(err)
}
if !bytes.Equal(got, data[offset:offset+int64(len(got))]) {
t.Fatal("incorrect random/cross-block read", offset)
}
}
unique, received := h.Progress()
if unique != int64(len(data)) || received <= unique {
t.Fatal("refetch progress must not double-count stored bytes", unique, received)
}
}
func TestHTTPSpoolSequentialAndCancellation(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
entered := make(chan struct{})
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Write([]byte("hello"))
w.(http.Flusher).Flush()
close(entered)
<-r.Context().Done()
}))
defer srv.Close()
in, err := OpenSpoolingHTTP(ctx, srv.Client(), HTTPSource{URL: srv.URL}, t.TempDir())
if err != nil {
t.Fatal(err)
}
<-entered
if in.Seekable() {
t.Fatal("200 response advertised seek")
}
got := make([]byte, 5)
if _, err := io.ReadFull(in, got); err != nil || string(got) != "hello" {
t.Fatal(string(got), err)
}
cancel()
if err := in.Close(); err != nil {
t.Fatal(err)
}
}
func TestHTTPSpoolRejectsBadRanges(t *testing.T) {
for _, mode := range []string{"etag", "length", "offset", "status", "truncated", "encoding"} {
t.Run(mode, func(t *testing.T) {
var calls atomic.Int32
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
first := calls.Add(1) == 1
start, end := int64(0), int64(httpBlockSize-1)
if !first {
fmt.Sscanf(r.Header.Get("Range"), "bytes=%d-%d", &start, &end)
}
tag := `"stable"`
if !first && mode == "etag" {
tag = `"changed"`
}
w.Header().Set("ETag", tag)
if !first && mode == "status" {
w.WriteHeader(200)
return
}
if !first && mode == "offset" {
start++
}
if !first && mode == "encoding" {
w.Header().Set("Content-Encoding", "gzip")
}
w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, 2*SpoolBlockSize))
length := end - start + 1
if !first && mode == "length" {
length++
}
w.Header().Set("Content-Length", fmt.Sprint(length))
w.WriteHeader(206)
if !first && mode == "truncated" {
length /= 2
}
chunk := make([]byte, 64*1024)
for length > 0 {
n := min(int64(len(chunk)), length)
if _, err := w.Write(chunk[:n]); err != nil {
return
}
length -= n
}
}))
defer srv.Close()
in, err := OpenSpoolingHTTP(context.Background(), srv.Client(), HTTPSource{URL: srv.URL}, t.TempDir())
if err != nil {
t.Fatal(err)
}
defer in.Close()
h := in.(*HTTPSpool)
select {
case err := <-h.Failures():
if err == nil {
t.Fatal("nil failure")
}
case <-time.After(5 * time.Second):
t.Fatal("bad range did not fail")
}
})
}
}
func TestHTTPSpoolReadsPartialRangeBeforeResponseFinishes(t *testing.T) {
ready, finish := make(chan struct{}), make(chan struct{})
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
var start, end int64
fmt.Sscanf(r.Header.Get("Range"), "bytes=%d-%d", &start, &end)
end = min(end, 4*1024*1024-1)
w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, 4*1024*1024))
w.Header().Set("Content-Length", fmt.Sprint(end-start+1))
w.WriteHeader(206)
if start == 0 {
w.Write(make([]byte, end+1))
return
}
w.Write(bytes.Repeat([]byte{93}, 64*1024))
w.(http.Flusher).Flush()
close(ready)
select {
case <-finish:
w.Write(make([]byte, end-start+1-64*1024))
case <-r.Context().Done():
}
}))
defer srv.Close()
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
defer cancel()
input, err := OpenSpoolingHTTP(ctx, srv.Client(), HTTPSource{URL: srv.URL}, t.TempDir())
if err != nil {
t.Fatal(err)
}
defer input.Close()
select {
case <-ready:
case <-ctx.Done():
t.Fatal(ctx.Err())
}
if _, err := input.Seek(1024*1024, io.SeekStart); err != nil {
t.Fatal(err)
}
b := make([]byte, 1)
if _, err := io.ReadFull(input, b); err != nil || b[0] != 93 {
t.Fatal("waited for complete response", b, err)
}
close(finish)
}
+252
View File
@@ -0,0 +1,252 @@
// Package ffmpeg runs an embedded FFmpeg with host-controlled media inputs.
package ffmpeg
import (
"context"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"strconv"
"strings"
"time"
)
const httpBlockSize = 1024 * 1024
// Input is a file visible to FFmpeg. Only verified HTTP Range sources seek.
type Input interface {
io.ReadCloser
Seek(int64, int) (int64, error)
Size() int64
Seekable() bool
}
var ErrSeek = errors.New("ffmpeg: input requires seeking but source is sequential")
type HTTPSource struct {
URL string `json:"url"`
Headers map[string]string `json:"headers"`
}
type httpInput struct {
ctx context.Context
client *http.Client
source HTTPSource
body io.ReadCloser
size, pos, blockStart int64
block []byte
ranged bool
etag, modified string
}
// OpenHTTP combines capability detection with the first media read. A 200
// response remains open and is consumed sequentially without another request.
func OpenHTTP(ctx context.Context, client *http.Client, source HTTPSource) (Input, error) {
u, err := url.Parse(source.URL)
if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" {
return nil, errors.New("ffmpeg: expected an HTTP(S) input URL")
}
h := &httpInput{ctx: ctx, client: client, source: source, size: -1}
resp, err := h.request(0, httpBlockSize-1)
if err != nil {
return nil, err
}
if resp.StatusCode == http.StatusOK {
if err = identityEncoding(resp); err != nil {
resp.Body.Close()
return nil, err
}
h.body, h.size = resp.Body, resp.ContentLength
return h, nil
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusPartialContent {
return nil, fmt.Errorf("ffmpeg: HTTP input returned %d", resp.StatusCode)
}
start, end, size, err := contentRange(resp.Header.Get("Content-Range"))
if err != nil || start != 0 || end >= httpBlockSize {
return nil, errors.New("ffmpeg: invalid initial Content-Range")
}
h.size, h.ranged = size, true
h.etag = resp.Header.Get("ETag")
if strings.HasPrefix(h.etag, "W/") {
h.etag = ""
}
h.modified = resp.Header.Get("Last-Modified")
h.block, err = readBlock(resp, end-start+1)
return h, err
}
func (h *httpInput) request(start, end int64) (*http.Response, error) {
ctx, cancel := context.WithCancel(h.ctx)
req, err := http.NewRequestWithContext(ctx, http.MethodGet, h.source.URL, nil)
if err != nil {
cancel()
return nil, err
}
for k, v := range h.source.Headers {
req.Header.Set(k, v)
}
// Offsets must refer to the actual media bytes, not a decompressed response.
req.Header.Set("Accept-Encoding", "identity")
req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", start, end))
if h.etag != "" {
req.Header.Set("If-Range", h.etag)
} else if h.modified != "" {
req.Header.Set("If-Range", h.modified)
}
timer := time.AfterFunc(15*time.Second, cancel)
resp, err := h.client.Do(req)
timer.Stop()
if err != nil {
cancel()
return nil, err
}
resp.Body = &idleMediaBody{ReadCloser: resp.Body, cancel: cancel}
return resp, nil
}
func identityEncoding(resp *http.Response) error {
if enc := resp.Header.Get("Content-Encoding"); enc != "" && !strings.EqualFold(enc, "identity") {
return errors.New("ffmpeg: encoded HTTP responses cannot represent media byte offsets")
}
return nil
}
func contentRange(value string) (start, end, size int64, err error) {
err = errors.New("invalid Content-Range")
if !strings.HasPrefix(value, "bytes ") {
return
}
parts := strings.Split(strings.TrimPrefix(value, "bytes "), "/")
if len(parts) != 2 {
return
}
bounds := strings.Split(parts[0], "-")
if len(bounds) != 2 {
return
}
var e error
if start, e = strconv.ParseInt(bounds[0], 10, 64); e != nil {
return
}
if end, e = strconv.ParseInt(bounds[1], 10, 64); e != nil {
return
}
if size, e = strconv.ParseInt(parts[1], 10, 64); e != nil {
return
}
if start < 0 || end < start || size <= end {
return
}
err = nil
return
}
func readBlock(resp *http.Response, length int64) ([]byte, error) {
if err := identityEncoding(resp); err != nil {
return nil, err
}
if resp.ContentLength >= 0 && resp.ContentLength != length {
return nil, errors.New("ffmpeg: Range response length mismatch")
}
b, err := io.ReadAll(io.LimitReader(resp.Body, length+1))
if err != nil {
return nil, err
}
if int64(len(b)) != length {
return nil, errors.New("ffmpeg: truncated or oversized Range response")
}
return b, nil
}
func (h *httpInput) Read(p []byte) (int, error) {
if len(p) == 0 {
return 0, nil
}
if err := h.ctx.Err(); err != nil {
return 0, err
}
if !h.ranged {
return h.body.Read(p)
}
if h.pos >= h.size {
return 0, io.EOF
}
if h.pos < h.blockStart || h.pos >= h.blockStart+int64(len(h.block)) {
end := min(h.pos+httpBlockSize-1, h.size-1)
resp, err := h.request(h.pos, end)
if err != nil {
return 0, err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusPartialContent {
return 0, fmt.Errorf("ffmpeg: Range request returned %d; source changed or stopped supporting Range", resp.StatusCode)
}
start, actualEnd, size, err := contentRange(resp.Header.Get("Content-Range"))
if err != nil || start != h.pos || actualEnd > end || size != h.size {
return 0, errors.New("ffmpeg: inconsistent Content-Range")
}
if h.etag != "" && resp.Header.Get("ETag") != h.etag {
return 0, errors.New("ffmpeg: HTTP input ETag changed")
}
if h.etag == "" && h.modified != "" && resp.Header.Get("Last-Modified") != h.modified {
return 0, errors.New("ffmpeg: HTTP input Last-Modified changed")
}
h.block, err = readBlock(resp, actualEnd-start+1)
if err != nil {
return 0, err
}
h.blockStart = start
}
n := copy(p, h.block[h.pos-h.blockStart:])
h.pos += int64(n)
return n, nil
}
func (h *httpInput) Seek(offset int64, whence int) (int64, error) {
if !h.ranged {
return 0, ErrSeek
}
base := int64(0)
switch whence {
case io.SeekStart:
case io.SeekCurrent:
base = h.pos
case io.SeekEnd:
base = h.size
default:
return 0, errors.New("invalid seek origin")
}
next := base + offset
if next < 0 || (offset > 0 && next < base) {
return 0, errors.New("invalid seek offset")
}
h.pos = next
return next, nil
}
func (h *httpInput) Size() int64 { return h.size }
func (h *httpInput) Seekable() bool { return h.ranged }
func (h *httpInput) Close() error {
if h.body != nil {
return h.body.Close()
}
return nil
}
// Only time spent waiting on the network counts toward this idle deadline.
// Pauses between reads (disk writes or FFmpeg scheduling) do not count.
type idleMediaBody struct {
io.ReadCloser
cancel context.CancelFunc
}
func (b *idleMediaBody) Read(p []byte) (int, error) {
timer := time.AfterFunc(15*time.Second, b.cancel)
defer timer.Stop()
return b.ReadCloser.Read(p)
}
func (b *idleMediaBody) Close() error { b.cancel(); return b.ReadCloser.Close() }
+131
View File
@@ -0,0 +1,131 @@
package ffmpeg
import (
"bytes"
"context"
"fmt"
"io"
"net/http"
"net/http/httptest"
"strings"
"sync/atomic"
"testing"
"time"
)
func TestHTTPRange(t *testing.T) {
data := bytes.Repeat([]byte("0123456789"), 300000)
var requests atomic.Int32
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
count := requests.Add(1)
if r.Header.Get("X-Test") != "secret" {
t.Error("missing headers")
}
if r.Header.Get("Accept-Encoding") != "identity" {
t.Error("missing identity encoding")
}
if count > 1 && r.Header.Get("If-Range") != `"v1"` {
t.Error("missing If-Range")
}
w.Header().Set("ETag", `"v1"`)
http.ServeContent(w, r, "media", time.Time{}, bytes.NewReader(data))
}))
defer srv.Close()
in, err := OpenHTTP(context.Background(), srv.Client(), HTTPSource{URL: srv.URL, Headers: map[string]string{"X-Test": "secret"}})
if err != nil {
t.Fatal(err)
}
defer in.Close()
if !in.Seekable() || in.Size() != int64(len(data)) {
t.Fatal("range not detected")
}
for _, off := range []int64{0, int64(len(data) - 16), 12, int64(httpBlockSize + 35)} {
if _, err = in.Seek(off, io.SeekStart); err != nil {
t.Fatal(err)
}
b := make([]byte, 16)
if _, err = io.ReadFull(in, b); err != nil {
t.Fatal(err)
}
if !bytes.Equal(b, data[off:off+16]) {
t.Fatalf("wrong bytes at %d", off)
}
}
}
func TestHTTPSequentialAndValidation(t *testing.T) {
for _, kind := range []string{"sequential", "bad-range", "changed", "truncated", "unauthorized"} {
t.Run(kind, func(t *testing.T) {
var requests atomic.Int32
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
count := requests.Add(1)
switch kind {
case "sequential":
w.Write([]byte("media"))
case "unauthorized":
w.WriteHeader(403)
case "bad-range":
w.Header().Set("Content-Range", "bytes 2-6/10")
w.WriteHeader(206)
w.Write([]byte("media"))
case "truncated":
w.Header().Set("Content-Range", "bytes 0-9/10")
w.WriteHeader(206)
w.Write([]byte("media"))
case "changed":
if count == 1 {
w.Header().Set("Content-Range", "bytes 0-4/10")
w.Header().Set("ETag", `"v1"`)
w.WriteHeader(206)
}
w.Write([]byte("media"))
}
}))
defer srv.Close()
in, err := OpenHTTP(context.Background(), srv.Client(), HTTPSource{URL: srv.URL})
if kind == "sequential" {
if err != nil {
t.Fatal(err)
}
defer in.Close()
if in.Seekable() {
t.Fatal("unexpected seeking")
}
if _, err = in.Seek(0, 0); err != ErrSeek {
t.Fatal(err)
}
b, err := io.ReadAll(in)
if err != nil || string(b) != "media" || requests.Load() != 1 {
t.Fatalf("%q %v count %d", b, err, requests.Load())
}
} else if kind == "changed" {
if err != nil {
t.Fatal(err)
}
defer in.Close()
in.Seek(5, 0)
if _, err = in.Read(make([]byte, 2)); err == nil {
t.Fatal("accepted 200 after range")
}
} else if err == nil {
in.Close()
t.Fatal("accepted invalid response")
}
})
}
}
func TestOutputArguments(t *testing.T) {
a, err := Arguments("", []string{"-shortest", "-metadata", "title=test"})
if err != nil {
t.Fatal(err)
}
if !strings.Contains(strings.Join(a, " "), "-f mp4 pipe:1") {
t.Fatal(a)
}
for _, args := range [][]string{{"-i", "/etc/passwd"}, {"-report"}, {"-c:v", "libx264"}, {"-metadata"}, {"other.mp4"}} {
if _, err := Arguments("mp4", args); err == nil {
t.Fatal(fmt.Sprint(args))
}
}
}
+255
View File
@@ -0,0 +1,255 @@
package ffmpeg
import (
"context"
"errors"
"fmt"
"io"
"io/fs"
"strings"
"sync"
ffembed "codeberg.org/gruf/go-ffmpreg/embed"
"codeberg.org/gruf/go-ffmpreg/wasm"
"github.com/tetratelabs/wazero"
expsys "github.com/tetratelabs/wazero/experimental/sys"
"github.com/tetratelabs/wazero/experimental/sysfs"
"github.com/tetratelabs/wazero/sys"
)
var shared struct {
once sync.Once
runtime wazero.Runtime
module wazero.CompiledModule
err error
}
// Limit active instances independently of the number of extension engines.
var slots = make(chan struct{}, 2)
func initialize() {
shared.once.Do(func() {
defer func() {
if r := recover(); r != nil {
shared.err = fmt.Errorf("ffmpeg initialization: %v", r)
}
}()
cfg := wazero.NewRuntimeConfig().WithCloseOnContextDone(true).WithMemoryLimitPages(8192)
shared.runtime, shared.err = wasm.NewRuntime(context.Background(), cfg)
if shared.err != nil {
return
}
shared.module, shared.err = shared.runtime.CompileModule(context.Background(), ffembed.B())
if shared.err != nil {
shared.runtime.Close(context.Background())
return
}
ffembed.Free()
})
}
// Output arguments deliberately do not accept extra inputs, output paths or
// global filesystem/logging options. The host owns all I/O and cancellation.
var outputOptions = map[string]int{
"-c": 1, "-codec": 1, "-c:v": 1, "-c:a": 1, "-codec:v": 1, "-codec:a": 1,
"-map": 1, "-map_metadata": 1, "-map_chapters": 1, "-metadata": 1, "-metadata:s:a:0": 1, "-metadata:s:v:0": 1,
"-movflags": 1, "-frag_duration": 1, "-frag_size": 1, "-min_frag_duration": 1,
"-bsf:v": 1, "-bsf:a": 1, "-strict": 1, "-shortest": 0, "-t": 1, "-to": 1,
"-avoid_negative_ts": 1, "-max_interleave_delta": 1, "-flush_packets": 1,
"-disposition:a:0": 1, "-disposition:v:0": 1,
}
func Arguments(format string, extra []string) ([]string, error) {
if format == "" {
format = "mp4"
}
if format == "mkv" {
format = "matroska"
}
switch format {
case "mp4", "matroska", "webm", "mpegts":
default:
return nil, fmt.Errorf("ffmpeg: unsupported output format %q", format)
}
for i := 0; i < len(extra); {
count, ok := outputOptions[extra[i]]
if !ok {
return nil, fmt.Errorf("ffmpeg: unsupported output option %q", extra[i])
}
if i+count >= len(extra) {
return nil, fmt.Errorf("ffmpeg: missing value for %s", extra[i])
}
if strings.HasPrefix(extra[i], "-c") && extra[i+1] != "copy" {
return nil, errors.New("ffmpeg: merge only supports stream copy")
}
i += count + 1
}
args := []string{"-hide_banner", "-nostdin", "-xerror", "-loglevel", "error", "-protocol_whitelist", "file,pipe", "-i", "/inputs/video", "-protocol_whitelist", "file,pipe", "-i", "/inputs/audio", "-map", "0:v:0", "-map", "1:a:0", "-c", "copy"}
fragmentFlags := ""
for i := 0; i < len(extra); {
count := outputOptions[extra[i]]
if format == "mp4" && extra[i] == "-movflags" {
if strings.Contains(extra[i+1], "faststart") || strings.Contains(extra[i+1], "global_sidx") {
return nil, errors.New("ffmpeg: these movflags require seekable output")
}
fragmentFlags = extra[i+1]
} else {
args = append(args, extra[i:i+count+1]...)
}
i += count + 1
}
if format == "mp4" {
args = append(args, "-movflags", fragmentFlags+"+frag_keyframe+empty_moov+default_base_moof")
}
args = append(args, "-f", format, "pipe:1")
return args, nil
}
// Run shares compilation, but gives every merge its own memory and filesystem.
// A clean stdout EOF is only delivered by the caller after this succeeds.
func Run(ctx context.Context, video, audio Input, out io.Writer, format string, extra []string) error {
_, err := Arguments(format, extra)
if err != nil {
return err
}
release, err := Acquire(ctx)
if err != nil {
return err
}
defer release()
return RunAcquired(ctx, video, audio, out, format, extra)
}
// Acquire reserves WASM execution capacity. Disk-backed input downloads may
// already be running while this call waits.
func Acquire(ctx context.Context) (func(), error) {
select {
case slots <- struct{}{}:
case <-ctx.Done():
return nil, ctx.Err()
}
var once sync.Once
return func() { once.Do(func() { <-slots }) }, nil
}
// RunAcquired requires a reservation held until this call returns.
func RunAcquired(ctx context.Context, video, audio Input, out io.Writer, format string, extra []string) error {
args, err := Arguments(format, extra)
if err != nil {
return err
}
ready := make(chan struct{})
go func() { initialize(); close(ready) }()
select {
case <-ready:
case <-ctx.Done():
return ctx.Err()
}
if shared.err != nil {
return shared.err
}
files := &inputFS{files: map[string]Input{"video": video, "audio": audio}}
fsconfig := wazero.NewFSConfig().(sysfs.FSConfig).WithSysFSMount(files, "/inputs")
logs := &tailLog{}
rc, err := wasm.Run(ctx, shared.runtime, shared.module, wasm.Args{
Name: "ffmpeg", Args: args, Stdout: out, Stderr: logs,
Config: func(c wazero.ModuleConfig) wazero.ModuleConfig { return c.WithFSConfig(fsconfig) },
})
if ctx.Err() != nil {
return ctx.Err()
}
if files.err != nil {
return fmt.Errorf("ffmpeg input: %w (%s)", files.err, logs.String())
}
if err != nil {
return fmt.Errorf("ffmpeg runtime: %w (%s)", err, logs.String())
}
if rc != 0 {
return fmt.Errorf("ffmpeg exited with code %d: %s", rc, logs.String())
}
return nil
}
type tailLog struct{ data []byte }
func (l *tailLog) Write(p []byte) (int, error) {
n := len(p)
if len(p) > 8192 {
p = p[len(p)-8192:]
}
l.data = append(l.data, p...)
if len(l.data) > 8192 {
l.data = l.data[len(l.data)-8192:]
}
return n, nil
}
func (l *tailLog) String() string { return strings.TrimSpace(string(l.data)) }
type inputFS struct {
expsys.UnimplementedFS
files map[string]Input
err error
}
func (f *inputFS) OpenFile(path string, flag expsys.Oflag, perm fs.FileMode) (expsys.File, expsys.Errno) {
if flag&(expsys.O_WRONLY|expsys.O_RDWR|expsys.O_CREAT|expsys.O_TRUNC) != 0 {
return nil, expsys.EROFS
}
path = strings.TrimPrefix(path, "/")
if path == "." || path == "" {
return &inputFile{owner: f}, 0
}
in := f.files[path]
if in == nil {
return nil, expsys.ENOENT
}
return &inputFile{owner: f, in: in}, 0
}
func (f *inputFS) Stat(path string) (sys.Stat_t, expsys.Errno) {
file, err := f.OpenFile(path, 0, 0)
if err != 0 {
return sys.Stat_t{}, err
}
return file.Stat()
}
type inputFile struct {
expsys.UnimplementedFile
owner *inputFS
in Input
}
func (f *inputFile) IsDir() (bool, expsys.Errno) { return f.in == nil, 0 }
func (f *inputFile) Stat() (sys.Stat_t, expsys.Errno) {
if f.in == nil {
return sys.Stat_t{Mode: fs.ModeDir | 0555}, 0
}
mode := fs.FileMode(0444)
if !f.in.Seekable() {
mode |= fs.ModeNamedPipe
}
return sys.Stat_t{Mode: mode, Size: max(0, f.in.Size()), Nlink: 1}, 0
}
func (f *inputFile) Read(p []byte) (int, expsys.Errno) {
if f.in == nil {
return 0, expsys.EISDIR
}
n, err := f.in.Read(p)
if err != nil && err != io.EOF {
f.owner.err = err
return n, expsys.EIO
}
return n, 0
}
func (f *inputFile) Seek(off int64, whence int) (int64, expsys.Errno) {
if f.in == nil {
return 0, expsys.EISDIR
}
n, err := f.in.Seek(off, whence)
// Capability probes may legitimately fail; a failed seek alone is not a
// media error. FFmpeg decides whether it can continue sequentially.
if err != nil {
return 0, expsys.ENOSYS
}
return n, 0
}
+221
View File
@@ -0,0 +1,221 @@
package ffmpeg
import (
"bytes"
"context"
"encoding/binary"
"encoding/json"
"io"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"testing"
"time"
"codeberg.org/gruf/go-ffmpreg/wasm"
"github.com/tetratelabs/wazero"
)
// Generate tiny deterministic media with the exact embedded build under test;
// no system FFmpeg, network media or third-party fixture licensing is needed.
func fixture(t *testing.T, audio bool) []byte {
t.Helper()
initialize()
if shared.err != nil {
t.Fatal(shared.err)
}
args := []string{"-hide_banner", "-loglevel", "error", "-f", "lavfi", "-i", "color=c=black:s=32x32:r=10", "-t", "0.3", "-c:v", "libx264", "-preset", "ultrafast", "-movflags", "frag_keyframe+empty_moov", "-f", "mp4", "pipe:1"}
if audio {
args = []string{"-hide_banner", "-loglevel", "error", "-f", "lavfi", "-i", "anullsrc=r=48000:cl=mono", "-t", "0.3", "-c:a", "aac", "-movflags", "frag_keyframe+empty_moov", "-f", "mp4", "pipe:1"}
}
var out, logs bytes.Buffer
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
rc, err := wasm.Run(ctx, shared.runtime, shared.module, wasm.Args{Name: "ffmpeg", Args: args, Stdout: &out, Stderr: &logs})
if err != nil || rc != 0 {
t.Fatalf("fixture: %d %v %s", rc, err, logs.String())
}
return out.Bytes()
}
func TestTailMoovNeedsRange(t *testing.T) {
initialize()
if shared.err != nil {
t.Fatal(shared.err)
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
dir := t.TempDir()
var logs bytes.Buffer
rc, err := wasm.Run(ctx, shared.runtime, shared.module, wasm.Args{Name: "ffmpeg", Args: []string{
"-v", "error", "-f", "lavfi", "-i", "color=c=black:s=32x32:r=10", "-t", "0.3", "-c:v", "libx264", "-preset", "ultrafast", "/out/video.mp4",
}, Stderr: &logs, Config: func(c wazero.ModuleConfig) wazero.ModuleConfig {
return c.WithFSConfig(wazero.NewFSConfig().WithDirMount(dir, "/out"))
}})
if rc != 0 || err != nil {
t.Fatalf("fixture %d %v %s", rc, err, logs.String())
}
raw, err := os.ReadFile(filepath.Join(dir, "video.mp4"))
if err != nil {
t.Fatal(err)
}
var video []byte
for offset := 0; offset < len(raw); {
length := int(binary.BigEndian.Uint32(raw[offset : offset+4]))
if string(raw[offset+4:offset+8]) == "moov" {
free := make([]byte, 2*httpBlockSize)
binary.BigEndian.PutUint32(free, uint32(len(free)))
copy(free[4:], "free")
video = append(video, free...)
}
video = append(video, raw[offset:offset+length]...)
offset += length
}
audio := fixture(t, true)
for _, ranged := range []bool{true, false} {
t.Run(fmtBool(ranged), func(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
data := video
if r.URL.Path == "/audio" {
data = audio
}
if ranged {
http.ServeContent(w, r, "media", time.Time{}, bytes.NewReader(data))
} else {
w.Write(data)
}
}))
defer srv.Close()
v, err := OpenHTTP(ctx, srv.Client(), HTTPSource{URL: srv.URL})
if err != nil {
t.Fatal(err)
}
defer v.Close()
a, err := OpenHTTP(ctx, srv.Client(), HTTPSource{URL: srv.URL + "/audio"})
if err != nil {
t.Fatal(err)
}
defer a.Close()
err = Run(ctx, v, a, io.Discard, "mp4", nil)
if ranged && err != nil {
t.Fatal(err)
}
if !ranged && err == nil {
t.Fatal("non-seekable tail-moov input unexpectedly succeeded")
}
})
}
}
func fmtBool(b bool) string {
if b {
return "range"
}
return "sequential"
}
func TestWebMToMP4(t *testing.T) {
initialize()
if shared.err != nil {
t.Fatal(shared.err)
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
inputs := make([]*DiskInput, 2)
for i, name := range []string{"video.webm", "audio.webm"} {
data, err := os.ReadFile(filepath.Join("..", "testdata", "ffmpeg", name))
if err != nil {
t.Fatal(err)
}
inputs[i], err = NewDiskInput(ctx, t.TempDir())
if err != nil {
t.Fatal(err)
}
defer inputs[i].Close()
if _, err = inputs[i].Write(data); err != nil {
t.Fatal(err)
}
inputs[i].End(nil)
}
var output, probe, logs bytes.Buffer
if err := Run(ctx, inputs[0], inputs[1], &output, "mp4", nil); err != nil {
t.Fatal(err)
}
rc, err := wasm.Run(ctx, shared.runtime, shared.module, wasm.Args{Name: "ffprobe", Args: []string{"-v", "error", "-show_entries", "stream=codec_name", "-of", "json", "pipe:0"}, Stdin: bytes.NewReader(output.Bytes()), Stdout: &probe, Stderr: &logs})
if rc != 0 || err != nil {
t.Fatalf("probe %d %v %s", rc, err, logs.String())
}
if !bytes.Contains(probe.Bytes(), []byte(`"vp9"`)) || !bytes.Contains(probe.Bytes(), []byte(`"opus"`)) {
t.Fatalf("codecs changed: %s", probe.String())
}
}
func TestWASMMerge(t *testing.T) {
video, audio := fixture(t, false), fixture(t, true)
for _, ranged := range []bool{false, true} {
t.Run(map[bool]string{false: "sequential", true: "range"}[ranged], func(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
data := video
if r.URL.Path == "/audio" {
data = audio
}
if ranged {
http.ServeContent(w, r, "media", time.Time{}, bytes.NewReader(data))
} else {
w.Write(data)
}
}))
defer srv.Close()
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
v, err := OpenHTTP(ctx, srv.Client(), HTTPSource{URL: srv.URL + "/video"})
if err != nil {
t.Fatal(err)
}
defer v.Close()
a, err := OpenHTTP(ctx, srv.Client(), HTTPSource{URL: srv.URL + "/audio"})
if err != nil {
t.Fatal(err)
}
defer a.Close()
var output bytes.Buffer
if err = Run(ctx, v, a, &output, "", nil); err != nil {
t.Fatal(err)
}
if !bytes.Contains(output.Bytes(), []byte("moof")) {
t.Fatal("output is not fragmented MP4")
}
// Probe the actual result, rather than just checking its filename/header.
var probe, logs bytes.Buffer
rc, err := wasm.Run(ctx, shared.runtime, shared.module, wasm.Args{Name: "ffprobe", Args: []string{"-v", "error", "-show_streams", "-of", "json", "pipe:0"}, Stdin: bytes.NewReader(output.Bytes()), Stdout: &probe, Stderr: &logs})
if rc != 0 || err != nil {
t.Fatalf("probe: %d %v %s", rc, err, logs.String())
}
var result struct {
Streams []struct {
CodecType string `json:"codec_type"`
CodecName string `json:"codec_name"`
}
}
if err = json.Unmarshal(probe.Bytes(), &result); err != nil {
t.Fatal(err)
}
if len(result.Streams) != 2 || result.Streams[0].CodecName != "h264" || result.Streams[1].CodecName != "aac" {
t.Fatalf("unexpected streams %s", probe.String())
}
})
}
// A malformed input must produce an error, not successful empty output.
s, err := NewDiskInput(context.Background(), t.TempDir())
if err != nil {
t.Fatal(err)
}
defer s.Close()
s.Write([]byte("invalid"))
s.End(nil)
if err := Run(context.Background(), s, s, io.Discard, "", nil); err == nil {
t.Fatal("accepted invalid media")
}
}
+284
View File
@@ -0,0 +1,284 @@
package ffmpeg
import (
"context"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"sync"
"github.com/GopeedLab/gopeed/internal/tempfiles"
)
const SpoolBlockSize int64 = 8 * 1024 * 1024
type byteSpan struct{ start, end int64 }
// blockSpool tracks valid byte intervals separately from file length: a sparse
// file's unwritten holes are never valid media. No file handles survive a call.
type blockSpool struct {
owner *tempfiles.Scope
failures chan error
mu sync.Mutex
dir string
blocks map[int64][]byteSpan
seen []byteSpan
wake chan struct{}
ended, closed bool
err error
downloaded, received int64
}
func newBlockSpool(root string, owners ...*tempfiles.Scope) (*blockSpool, error) {
if root != "" {
if err := os.MkdirAll(root, 0700); err != nil {
return nil, err
}
}
dir, err := os.MkdirTemp(root, "gopeed-media-")
if err != nil {
return nil, err
}
var owner *tempfiles.Scope
if len(owners) > 0 {
owner = owners[0]
}
if err := owner.Track(dir); err != nil {
return nil, err
}
return &blockSpool{owner: owner, failures: make(chan error, 1), dir: dir, blocks: make(map[int64][]byteSpan), wake: make(chan struct{})}, nil
}
func (s *blockSpool) notify() { close(s.wake); s.wake = make(chan struct{}) }
func (s *blockSpool) path(index int64) string { return filepath.Join(s.dir, fmt.Sprintf("%d", index)) }
func addSpan(spans []byteSpan, next byteSpan) ([]byteSpan, int64) {
before := int64(0)
for _, span := range spans {
before += span.end - span.start
}
out := make([]byteSpan, 0, len(spans)+1)
for _, span := range spans {
if span.end < next.start {
out = append(out, span)
} else if next.end < span.start {
out = append(out, next)
next = span
} else {
next.start = min(next.start, span.start)
next.end = max(next.end, span.end)
}
}
out = append(out, next)
after := int64(0)
for _, span := range out {
after += span.end - span.start
}
return out, after - before
}
func (s *blockSpool) WriteAt(p []byte, offset int64) (int, error) {
s.mu.Lock()
defer s.mu.Unlock()
if offset < 0 {
return 0, errors.New("negative media offset")
}
if s.closed {
return 0, io.ErrClosedPipe
}
if s.err != nil {
return 0, s.err
}
written := 0
for len(p) > 0 {
index, local := offset/SpoolBlockSize, offset%SpoolBlockSize
count := int(min(int64(len(p)), SpoolBlockSize-local))
f, err := os.OpenFile(s.path(index), os.O_CREATE|os.O_WRONLY, 0600)
if err != nil {
s.fail(err)
s.notify()
return written, err
}
n, err := f.WriteAt(p[:count], local)
closeErr := f.Close()
if err == nil {
err = closeErr
}
if n > 0 {
s.blocks[index], _ = addSpan(s.blocks[index], byteSpan{local, local + int64(n)})
var added int64
s.seen, added = addSpan(s.seen, byteSpan{offset, offset + int64(n)})
s.downloaded += added
s.received += int64(n)
written += n
offset += int64(n)
p = p[n:]
}
if err == nil && n != count {
err = io.ErrShortWrite
}
if err != nil {
s.fail(err)
s.notify()
return written, err
}
}
s.notify()
return written, nil
}
func (s *blockSpool) readAt(p []byte, offset int64) (int, error, <-chan struct{}) {
s.mu.Lock()
defer s.mu.Unlock()
if s.closed {
return 0, io.ErrClosedPipe, nil
}
if s.err != nil {
return 0, s.err, nil
}
index, local := offset/SpoolBlockSize, offset%SpoolBlockSize
for _, span := range s.blocks[index] {
if span.start <= local && local < span.end {
f, err := os.Open(s.path(index))
if err != nil {
return 0, err, nil
}
n, err := f.ReadAt(p[:min(int64(len(p)), span.end-local)], local)
f.Close()
return n, err, nil
}
}
return 0, nil, s.wake
}
func (s *blockSpool) discard(index int64) error {
s.mu.Lock()
defer s.mu.Unlock()
if err := os.Remove(s.path(index)); err != nil && !os.IsNotExist(err) {
return err
}
delete(s.blocks, index)
return nil
}
func (s *blockSpool) end(err error) {
s.mu.Lock()
defer s.mu.Unlock()
s.ended = true
if s.err == nil {
s.fail(err)
}
s.notify()
}
func (s *blockSpool) Close() error {
s.mu.Lock()
defer s.mu.Unlock()
s.closed = true
clear(s.blocks)
s.seen = nil
s.notify()
return s.owner.Remove(s.dir)
}
func (s *blockSpool) progress() (int64, int64) {
s.mu.Lock()
defer s.mu.Unlock()
return s.downloaded, s.received
}
// DiskInput consumes a sequential producer without retaining its payload in RAM.
// Writes acknowledge durable-to-the-OS chunks, allowing both SABR tracks to drain
// even while FFmpeg is queued or currently reading only the other track.
type DiskInput struct {
ctx context.Context
spool *blockSpool
writeMu, readMu sync.Mutex
written, read int64
}
func NewDiskInput(ctx context.Context, root string, owners ...*tempfiles.Scope) (*DiskInput, error) {
spool, err := newBlockSpool(root, owners...)
if err != nil {
return nil, err
}
return &DiskInput{ctx: ctx, spool: spool}, nil
}
func (s *DiskInput) Write(p []byte) (int, error) {
s.writeMu.Lock()
defer s.writeMu.Unlock()
if err := s.ctx.Err(); err != nil {
return 0, err
}
s.spool.mu.Lock()
ended := s.spool.ended
s.spool.mu.Unlock()
if ended {
return 0, io.ErrClosedPipe
}
n, err := s.spool.WriteAt(p, s.written)
s.written += int64(n)
return n, err
}
func (s *DiskInput) End(err error) { s.writeMu.Lock(); defer s.writeMu.Unlock(); s.spool.end(err) }
func (s *DiskInput) Read(p []byte) (int, error) {
s.readMu.Lock()
defer s.readMu.Unlock()
if len(p) == 0 {
return 0, nil
}
for {
if err := s.ctx.Err(); err != nil {
return 0, err
}
n, err, wake := s.spool.readAt(p, s.read)
if n > 0 {
previous := s.read / SpoolBlockSize
s.read += int64(n)
if s.read/SpoolBlockSize > previous {
if e := s.spool.discard(previous); err == nil {
err = e
}
}
return n, err
}
if err != nil {
return 0, err
}
s.spool.mu.Lock()
ended := s.spool.ended
s.spool.mu.Unlock()
// End can race the missing-data check; recheck after observing it.
if ended {
n, err, _ = s.spool.readAt(p, s.read)
if n > 0 {
// Use the regular consumption path, including block eviction.
continue
}
if err != nil {
return 0, err
}
if err := s.spool.discard(s.read / SpoolBlockSize); err != nil {
return 0, err
}
return 0, io.EOF
}
select {
case <-wake:
case <-s.ctx.Done():
return 0, s.ctx.Err()
}
}
}
func (s *DiskInput) Close() error { return s.spool.Close() }
func (*DiskInput) Seek(int64, int) (int64, error) { return 0, ErrSeek }
func (*DiskInput) Size() int64 { return -1 }
func (*DiskInput) Seekable() bool { return false }
func (s *DiskInput) Progress() (int64, int64) { return s.spool.progress() }
// fail is called with mu held. The first error wakes a queued FFmpeg job.
func (s *blockSpool) fail(err error) {
if err == nil || s.err != nil {
return
}
s.err = err
select {
case s.failures <- err:
default:
}
}
func (s *DiskInput) Failures() <-chan error { return s.spool.failures }
+101
View File
@@ -0,0 +1,101 @@
package ffmpeg
import (
"bytes"
"context"
"io"
"os"
"testing"
)
func TestSpoolSparseCrossBlock(t *testing.T) {
s, err := newBlockSpool(t.TempDir())
if err != nil {
t.Fatal(err)
}
defer s.Close()
offset := 7*SpoolBlockSize + 123
data := bytes.Repeat([]byte{91}, int(3*SpoolBlockSize+37))
if n, err := s.WriteAt(data, offset); err != nil || n != len(data) {
t.Fatal(n, err)
}
if n, err, wake := s.readAt(make([]byte, 16), 7*SpoolBlockSize); n != 0 || err != nil || wake == nil {
t.Fatal("sparse hole was exposed", n, err)
}
got := make([]byte, len(data))
pos := 0
for pos < len(got) {
n, err, _ := s.readAt(got[pos:], offset+int64(pos))
if err != nil || n == 0 {
t.Fatal(n, err)
}
pos += n
}
if !bytes.Equal(data, got) {
t.Fatal("cross-block contents differ")
}
if err := s.discard(8); err != nil {
t.Fatal(err)
}
if n, _, wake := s.readAt(got[:1], 8*SpoolBlockSize); n != 0 || wake == nil {
t.Fatal("deleted block remained readable")
}
if _, err := s.WriteAt(data, offset); err != nil {
t.Fatal(err)
}
unique, received := s.progress()
if unique != int64(len(data)) || received != 2*unique {
t.Fatal(unique, received)
}
}
func TestDiskInputDrainAndCleanup(t *testing.T) {
s, err := NewDiskInput(context.Background(), t.TempDir())
if err != nil {
t.Fatal(err)
}
data := bytes.Repeat([]byte{33}, int(3*SpoolBlockSize+99))
if _, err := s.Write(data); err != nil {
t.Fatal(err)
}
s.End(nil)
got, err := io.ReadAll(s)
if err != nil || !bytes.Equal(data, got) {
t.Fatal("drain", err)
}
files, err := os.ReadDir(s.spool.dir)
if err != nil || len(files) != 0 {
t.Fatal("consumed blocks retained", len(files), err)
}
if err := s.Close(); err != nil {
t.Fatal(err)
}
if _, err := os.Stat(s.spool.dir); !os.IsNotExist(err) {
t.Fatal("spool not removed", err)
}
}
func TestDiskInputCancelAndWriteFailure(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
s, err := NewDiskInput(ctx, t.TempDir())
if err != nil {
t.Fatal(err)
}
defer s.Close()
cancel()
if _, err := s.Read(make([]byte, 1)); err != context.Canceled {
t.Fatal(err)
}
broken, err := NewDiskInput(context.Background(), t.TempDir())
if err != nil {
t.Fatal(err)
}
defer broken.Close()
if err := os.RemoveAll(broken.spool.dir); err != nil {
t.Fatal(err)
}
if _, err := broken.Write([]byte{1}); err == nil {
t.Fatal("missing disk error")
}
if _, err := broken.Read(make([]byte, 1)); err == nil {
t.Fatal("disk error not propagated")
}
}
@@ -0,0 +1,433 @@
package engine
import (
"bytes"
"context"
"fmt"
"github.com/GopeedLab/gopeed/internal/production"
media "github.com/GopeedLab/gopeed/pkg/download/engine/ffmpeg"
"github.com/GopeedLab/gopeed/pkg/download/engine/inject/stream"
"io"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"sync/atomic"
"testing"
"time"
)
func TestFFmpegJSStreams(t *testing.T) {
video, err := os.ReadFile("testdata/ffmpeg/video.mp4")
if err != nil {
t.Fatal(err)
}
audio, err := os.ReadFile("testdata/ffmpeg/audio.mp4")
if err != nil {
t.Fatal(err)
}
for _, mode := range []string{"streams", "http", "mixed"} {
t.Run(mode, func(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
b := video
if r.URL.Path == "/audio" {
b = audio
}
http.ServeContent(w, r, "media", time.Time{}, bytes.NewReader(b))
}))
defer srv.Close()
e := NewEngine(nil)
defer e.Close()
// Input data is deliberately interleaved through one JS producer.
// Native push/read operations must never block the event loop.
e.Runtime.Set("videoBytes", e.Runtime.NewArrayBuffer(video))
e.Runtime.Set("audioBytes", e.Runtime.NewArrayBuffer(audio))
e.Runtime.Set("sourceURL", srv.URL)
e.Runtime.Set("inputMode", mode)
result, err := e.RunString(`(async()=>{
let vc, ac, offset=0;
const v=new Uint8Array(videoBytes), a=new Uint8Array(audioBytes);
const video=new ReadableStream({start(c){vc=c;}});
const audio=new ReadableStream({start(c){ac=c;}});
const timer=setInterval(()=>{
if(offset<v.length)vc.enqueue(v.slice(offset,offset+97));
if(offset<a.length)ac.enqueue(a.slice(offset,offset+97));
offset+=97;
if(offset>=Math.max(v.length,a.length)){clearInterval(timer);vc.close();ac.close();}
},1);
const stream=__gopeed_ffmpeg.merge({
inputs: () => ({ video:inputMode==='http'?{url:sourceURL+'/video'}:video,
audio:inputMode==='streams'?audio:{url:sourceURL+'/audio'} }),
args:['-metadata','title=gopeed-test']
});
const bytes=await new Response(stream).arrayBuffer();
return new Uint8Array(bytes);
})()`)
if err != nil {
t.Fatal(err)
}
out, ok := result.([]byte)
if !ok {
t.Fatalf("output %T", result)
}
if !bytes.Contains(out, []byte("moof")) || !bytes.Contains(out, []byte("gopeed-test")) {
t.Fatalf("invalid output (%d bytes)", len(out))
}
})
}
}
func TestFFmpegJSErrorAndCancel(t *testing.T) {
for _, scenario := range []string{"input-error", "abort", "output-cancel", "invalid-args"} {
t.Run(scenario, func(t *testing.T) {
e := NewEngine(nil)
defer e.Close()
e.Runtime.Set("scenario", scenario)
result, err := e.RunString(`(async()=>{
let cancelled=0;
const source=()=>new ReadableStream({
pull(c){if(scenario==='input-error')c.error(new Error('upstream failed'));else return new Promise(()=>{});},
cancel(){cancelled++;}
});
const controller=new AbortController();
const output=__gopeed_ffmpeg.merge({inputs:()=>({video:source(),audio:source()}),signal:controller.signal,
args:scenario==='invalid-args'?['-i','/outside']:[]});
const reader=output.getReader();
const pending=reader.read().then(()=>'',e=>e.message);
if(scenario==='abort')setTimeout(()=>controller.abort(),20);
if(scenario==='output-cancel')setTimeout(()=>reader.cancel(),20);
const message=await pending;
await new Promise(r=>setTimeout(r,10));
return {message,cancelled};
})()`)
if err != nil {
t.Fatal(err)
}
m := result.(map[string]any)
if scenario != "output-cancel" && m["message"] == "" {
t.Fatalf("missing error: %v", m)
}
if scenario == "abort" || scenario == "output-cancel" {
if fmt.Sprint(m["cancelled"]) != "2" {
t.Fatalf("inputs not cancelled: %v", m)
}
}
})
}
}
func TestFFmpegMalformedMediaFails(t *testing.T) {
e := NewEngine(nil)
defer e.Close()
_, err := e.RunString(`(async()=>{
const source=()=>new ReadableStream({start(c){c.enqueue(new Uint8Array([1,2,3]));c.close();}});
return await new Response(__gopeed_ffmpeg.merge({inputs:()=>({video:source(),audio:source()})})).arrayBuffer();
})()`)
if err == nil || !strings.Contains(err.Error(), "ffmpeg") {
t.Fatalf("expected FFmpeg error: %v", err)
}
}
func TestFFmpegBlobOpener(t *testing.T) {
var open stream.ObjectURLOpener
e := NewEngine(&Config{StreamConfig: &stream.Config{CreateObjectURL: func(opts *stream.ObjectURLOptions, opener stream.ObjectURLOpener) (string, error) {
if opts.Range || opts.Size != 0 {
t.Fatal("merged output must not advertise range or a guessed size")
}
open = opener
return "blob:ffmpeg-test", nil
}}})
defer e.Close()
video, _ := os.ReadFile("testdata/ffmpeg/video.mp4")
audio, _ := os.ReadFile("testdata/ffmpeg/audio.mp4")
e.Runtime.Set("videoBytes", e.Runtime.NewArrayBuffer(video))
e.Runtime.Set("audioBytes", e.Runtime.NewArrayBuffer(audio))
_, err := e.RunString(`__gopeed_blob_create_object_url(()=>{
const source=b=>new ReadableStream({start(c){c.enqueue(new Uint8Array(b));c.close();}});
return __gopeed_ffmpeg.merge({inputs:()=>({video:source(videoBytes),audio:source(audioBytes)})});
},{contentType:'video/mp4'})`)
if err != nil {
t.Fatal(err)
}
for i := 0; i < 2; i++ {
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
r, err := open(ctx, stream.ObjectURLOpenRequest{End: -1})
if err != nil {
cancel()
t.Fatal(err)
}
b, err := io.ReadAll(r)
r.Close()
cancel()
if err != nil || !bytes.Contains(b, []byte("moof")) {
t.Fatalf("Blob read: %v (%d bytes)", err, len(b))
}
}
}
func TestFFmpegEngineCloseCancelsRequest(t *testing.T) {
started, closed := make(chan struct{}), make(chan struct{})
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
close(started)
<-r.Context().Done()
close(closed)
}))
defer srv.Close()
e := NewEngine(nil)
defer e.Close()
_, err := e.RunString(fmt.Sprintf(`
globalThis.cancelCount=0;
const audio=new ReadableStream({cancel(){cancelCount++;}});
__gopeed_ffmpeg.merge({inputs:()=>({video:{url:%q},audio})}).getReader().read().catch(()=>{});
"started";`, srv.URL))
if err != nil {
t.Fatal(err)
}
select {
case <-started:
case <-time.After(5 * time.Second):
t.Fatal("HTTP did not start")
}
e.Close()
select {
case <-closed:
case <-time.After(5 * time.Second):
t.Fatal("HTTP leaked after engine shutdown")
}
if e.Runtime.Get("cancelCount").ToInteger() != 1 {
t.Fatal("JS input not cancelled on shutdown")
}
}
func TestFFmpegInputsStartBeforeCapacity(t *testing.T) {
for _, mode := range []string{"factory", "http", "cancel"} {
t.Run(mode, func(t *testing.T) {
release1, _ := media.Acquire(context.Background())
defer release1()
release2, _ := media.Acquire(context.Background())
defer release2()
var requests atomic.Int32
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { requests.Add(1); http.Error(w, "test", 500) }))
defer srv.Close()
e := NewEngine(nil)
defer e.Close()
e.Runtime.Set("mode", mode)
e.Runtime.Set("sourceURL", srv.URL)
result, err := e.RunString(`(async()=>{
globalThis.factoryCalls=0;
globalThis.abortQueue=new AbortController();
const options=mode==='http'?{video:{url:sourceURL},audio:{url:sourceURL}}:
{inputs:()=>{factoryCalls++;throw new Error('factory failed');}};
options.signal=abortQueue.signal;
globalThis.work=__gopeed_ffmpeg.merge(options).getReader().read().then(()=>'',e=>e.message);
await new Promise(r=>setTimeout(r,50));
if(mode==='cancel')abortQueue.abort();
return factoryCalls;
})()`)
if err != nil {
t.Fatal(err)
}
if mode == "http" {
if requests.Load() != 1 {
t.Fatalf("queued HTTP did not start: %d", requests.Load())
}
} else if fmt.Sprint(result) != "1" {
t.Fatalf("queued factory did not start: %v", result)
}
if mode == "cancel" {
if _, err = e.RunString(`work`); err != nil {
t.Fatal(err)
}
}
release1()
if _, err = e.RunString(`work`); err != nil {
t.Fatal(err)
}
calls, err := e.RunString(`factoryCalls`)
if err != nil {
t.Fatal(err)
}
if mode == "factory" && fmt.Sprint(calls) != "1" {
t.Fatal(calls)
}
if mode == "cancel" && fmt.Sprint(calls) != "1" {
t.Fatal("cancelled factory ran")
}
if mode == "http" && requests.Load() != 1 {
t.Fatal("HTTP was restarted unnecessarily")
}
// Failure/cancellation must return the acquired slot even with the other held.
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
defer cancel()
release, err := media.Acquire(ctx)
if err != nil {
t.Fatalf("slot leaked: %v", err)
}
release()
})
}
}
func TestFFmpegRequiresFactoryForStreams(t *testing.T) {
e := NewEngine(nil)
defer e.Close()
result, err := e.RunString(`(()=>{
const source=()=>new ReadableStream({});
let rejected=0;
for(const opts of [
{video:source(),audio:source()},
{video:{url:'https://example.test'},audio:source()},
{inputs:()=>({}),video:{url:'https://example.test'}},
{inputs:{video:source(),audio:source()}}
]) {try{__gopeed_ffmpeg.merge(opts);}catch(e){if(e instanceof TypeError)rejected++;}}
return rejected;
})()`)
if err != nil || fmt.Sprint(result) != "4" {
t.Fatalf("validation: %v %v", result, err)
}
}
func TestFFmpegCancelPendingFactoryCleansLateInputs(t *testing.T) {
e := NewEngine(nil)
defer e.Close()
result, err := e.RunString(`(async()=>{
let ready,finish,cancelled=0,signal;
const entered=new Promise(r=>ready=r);
const abort=new AbortController();
const output=__gopeed_ffmpeg.merge({signal:abort.signal,inputs:async opts=>{
signal=opts.signal;ready();await new Promise(r=>finish=r);
const source=()=>new ReadableStream({cancel(){cancelled++;}});
return {video:source(),audio:source()};
}});
const work=output.getReader().read().catch(e=>e.name);
await entered;abort.abort();await work;finish();
await new Promise(r=>setTimeout(r,30));
return {cancelled,aborted:signal.aborted};
})()`)
if err != nil {
t.Fatal(err)
}
state := result.(map[string]any)
if fmt.Sprint(state["cancelled"]) != "2" || state["aborted"] != true {
t.Fatal(state)
}
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
defer cancel()
first, err := media.Acquire(ctx)
if err != nil {
t.Fatal(err)
}
defer first()
second, err := media.Acquire(ctx)
if err != nil {
t.Fatal("factory cancellation leaked capacity")
}
second()
}
func TestQueuedMergePrefetchFailureAndCancel(t *testing.T) {
for _, mode := range []string{"failure", "cancel"} {
t.Run(mode, func(t *testing.T) {
release1, _ := media.Acquire(context.Background())
defer release1()
release2, _ := media.Acquire(context.Background())
defer release2()
second := make(chan struct{})
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Header.Get("Range") == "bytes=0-1048575" {
w.Header().Set("Content-Range", "bytes 0-1048575/33554432")
w.Header().Set("Content-Length", "1048576")
w.WriteHeader(206)
w.Write(make([]byte, 1048576))
return
}
close(second)
if mode == "cancel" {
<-r.Context().Done()
return
}
http.Error(w, "upstream failed", 503)
}))
defer srv.Close()
root := t.TempDir()
e := NewEngine(&Config{TempDir: root})
defer e.Close()
e.Runtime.Set("sourceURL", srv.URL)
_, err := e.RunString(`
globalThis.abortJob=new AbortController();
globalThis.result=__gopeed_ffmpeg.merge({signal:abortJob.signal,inputs:()=>({video:{url:sourceURL},audio:new ReadableStream({start(c){c.close();}})})})
.getReader().read().then(()=>'',e=>String(e.message||e));
'started';`)
if err != nil {
t.Fatal(err)
}
select {
case <-second:
case <-time.After(3 * time.Second):
t.Fatal("queued input did not download")
}
if mode == "cancel" {
if _, err := e.RunString(`abortJob.abort()`); err != nil {
t.Fatal(err)
}
}
result, err := e.RunString(`Promise.race([result,new Promise(r=>setTimeout(()=>r('test timeout'),3000))])`)
if err != nil || fmt.Sprint(result) == "test timeout" || fmt.Sprint(result) == "" {
t.Fatal(result, err)
}
if mode == "failure" && !strings.Contains(fmt.Sprint(result), "503") {
t.Fatal("lost upstream failure", result)
}
deadline := time.Now().Add(3 * time.Second)
for {
entries, err := os.ReadDir(root)
if err == nil && len(entries) == 0 {
break
}
if time.Now().After(deadline) {
t.Fatal("queued input files leaked", entries, err)
}
time.Sleep(10 * time.Millisecond)
}
})
}
}
func TestFFmpegUnavailableTempDirReportsBlobError(t *testing.T) {
root := filepath.Join(t.TempDir(), "blocked")
if err := os.WriteFile(root, []byte("not a directory"), 0600); err != nil {
t.Fatal(err)
}
var open stream.ObjectURLOpener
e := NewEngine(&Config{TempDir: root, StreamConfig: &stream.Config{CreateObjectURL: func(_ *stream.ObjectURLOptions, opener stream.ObjectURLOpener) (string, error) {
open = opener
return "blob:test", nil
}}})
defer e.Close()
_, err := e.RunString(`__gopeed_blob_create_object_url(()=>__gopeed_ffmpeg.merge({inputs:()=>({video:new ReadableStream(),audio:new ReadableStream()})}),{})`)
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
defer cancel()
reader, err := open(ctx, stream.ObjectURLOpenRequest{End: -1})
if err != nil {
t.Fatal(err)
}
defer reader.Close()
produced := reader.(production.Reader)
if err := produced.WaitProduction(ctx); err != nil {
t.Fatal(err)
}
source := produced.Production()
if source == nil {
t.Fatal("missing producer")
}
if _, err := io.ReadAll(reader); err == nil || !strings.Contains(err.Error(), "blocked") {
t.Fatal("missing useful disk error", err)
}
if downloaded, received := source.Progress(); downloaded != 0 || received != 0 {
t.Fatal(downloaded, received)
}
}
+15 -60
View File
@@ -122,7 +122,7 @@
if (name === "set-cookie") {
for (const value of values) result.push([name, value]);
} else {
result.push([name, values.join(", ")]);
result.push([name, values.join(name === "cookie" ? "; " : ", ")]);
}
}
return result;
@@ -158,49 +158,10 @@
return iterator;
}
function forbiddenRequestHeader(name, value) {
if (name.startsWith("proxy-") || name.startsWith("sec-")) return true;
if ([
"accept-charset", "accept-encoding", "access-control-request-headers",
"access-control-request-method", "connection", "content-length", "cookie",
"cookie2", "date", "dnt", "expect", "host", "keep-alive", "origin",
"permissions-policy", "referer", "te", "trailer", "transfer-encoding",
"set-cookie", "upgrade", "via",
].includes(name)) return true;
if (["x-http-method", "x-http-method-override", "x-method-override"].includes(name)) {
return String(value).split(",").some((method) => {
method = method.trim().toUpperCase();
return method === "CONNECT" || method === "TRACE" || method === "TRACK";
});
}
return false;
}
function noCORSSafelistedRequestHeader(name, value) {
if (value.length > 128) return false;
const hasUnsafeByte = /[\x00-\x08\x0A-\x1F\x7F"():<>?@\[\\\]{}]/.test(value);
if (name === "accept") return !hasUnsafeByte;
if (name === "accept-language" || name === "content-language") {
return /^[0-9A-Za-z *,\-.;=]*$/.test(value);
}
if (name === "content-type") {
if (hasUnsafeByte) return false;
const essence = value.split(";", 1)[0].trim().toLowerCase();
return essence === "application/x-www-form-urlencoded" ||
essence === "multipart/form-data" || essence === "text/plain";
}
if (name === "range") return /^bytes=[0-9]+-[0-9]*$/.test(value);
return false;
}
function blockedByGuard(headers, name, value) {
function assertMutableHeaders(headers) {
// Match Node/Undici: HTTP syntax and immutability still apply, but browser
// forbidden-header and no-cors safelist rules do not apply to extensions.
if (headers._guard === "immutable") throw new TypeError("Headers are immutable");
if (headers._guard === "request") return forbiddenRequestHeader(name, value);
if (headers._guard === "request-no-cors") {
return forbiddenRequestHeader(name, value) || !noCORSSafelistedRequestHeader(name, value);
}
if (headers._guard === "response") return name === "set-cookie" || name === "set-cookie2";
return false;
}
class Headers {
@@ -235,27 +196,21 @@
name = normalizeHeaderName(name);
value = normalizeHeaderValue(value);
const values = this._values.get(name);
const combinedValue = values && name !== "set-cookie" ? values.join(", ") + ", " + value : value;
if (blockedByGuard(this, name, combinedValue)) return;
assertMutableHeaders(this);
if (values) values.push(value);
else this._values.set(name, [value]);
}
delete(name) {
name = normalizeHeaderName(name);
if (this._guard === "immutable") throw new TypeError("Headers are immutable");
if (this._guard === "request-no-cors") {
if (forbiddenRequestHeader(name, "") || !["accept", "accept-language", "content-language", "content-type", "range"].includes(name)) return;
this._values.delete(name);
return;
}
if (blockedByGuard(this, name, "")) return;
name = normalizeHeaderName(name);
assertMutableHeaders(this);
this._values.delete(name);
}
get(name) {
const values = this._values.get(normalizeHeaderName(name));
return values ? values.join(", ") : null;
name = normalizeHeaderName(name);
const values = this._values.get(name);
return values ? values.join(name === "cookie" ? "; " : ", ") : null;
}
getSetCookie() {
@@ -268,10 +223,10 @@
}
set(name, value) {
name = normalizeHeaderName(name);
value = normalizeHeaderValue(value);
if (blockedByGuard(this, name, value)) return;
this._values.set(name, [value]);
name = normalizeHeaderName(name);
value = normalizeHeaderValue(value);
assertMutableHeaders(this);
this._values.set(name, [value]);
}
keys() { return createHeadersIterator(this, "key"); }
@@ -484,7 +439,7 @@
}
this._duplex = "half";
const headerInit = init.headers !== undefined ? init.headers : (source ? source.headers : undefined);
this._headers = new Headers(headerInit, this._mode === "no-cors" ? "request-no-cors" : "request");
this._headers = new Headers(headerInit, "request");
let body = init.body !== undefined ? init.body : (source ? source._bodyInit : null);
if (source && !bodyWasOverridden && source.body) body = transferReadableStream(source.body);
const bodyIsStream = typeof ReadableStream !== "undefined" && body instanceof ReadableStream;
+23 -2
View File
@@ -29,8 +29,9 @@ const FingerprintMagicKey = "__gopeed_fetch_fingerprint"
var script string
type Config struct {
ProxyHandler func(*http.Request) (*url.URL, error)
RegisterCleanup func(func())
DefaultUserAgent *string
ProxyHandler func(*http.Request) (*url.URL, error)
RegisterCleanup func(func())
}
func Enable(runtime *goja.Runtime, loop *eventloop.EventLoop, cfg *Config) error {
@@ -51,6 +52,9 @@ func Enable(runtime *goja.Runtime, loop *eventloop.EventLoop, cfg *Config) error
if err != nil {
panic(runtime.NewGoError(err))
}
if cfg.DefaultUserAgent != nil && !hasHeader(request.Headers, "User-Agent") {
request.Headers = append(request.Headers, [2]string{"User-Agent", *cfg.DefaultUserAgent})
}
fingerprint := util.SafeGet[string](runtime, FingerprintMagicKey)
promise, resolve, reject := runtime.NewPromise()
go func() {
@@ -298,6 +302,23 @@ func (r *registry) Open(fingerprint string, proxyHandler func(*http.Request) (*u
return nil, err
}
for _, header := range metadata.Headers {
// Ignore transport controls unsupported by Node/Undici rather than
// applying a browser forbidden-header list or failing the whole request.
switch strings.ToLower(header[0]) {
case "expect", "keep-alive", "transfer-encoding", "upgrade", "content-length":
// Content-Length is generated from the buffered request body below.
continue
case "connection":
if value := strings.ToLower(header[1]); value != "close" && value != "keep-alive" {
continue
}
case "trailer":
// Fetch supplies a declaration, not a Go Request.Trailer payload.
// Keep its lower-case wire name so Go's writer does not omit it as
// the canonical Trailer field it normally generates itself.
httpRequest.Header["trailer"] = append(httpRequest.Header["trailer"], header[1])
continue
}
httpRequest.Header.Add(header[0], header[1])
}
if host := httpRequest.Header.Get("Host"); host != "" {
+144
View File
@@ -0,0 +1,144 @@
(function () {
const create = __gopeed_ffmpeg_create;
const read = __gopeed_ffmpeg_read;
const start = __gopeed_ffmpeg_start;
const push = __gopeed_ffmpeg_push;
const end = __gopeed_ffmpeg_end;
const cancel = __gopeed_ffmpeg_cancel;
const active = new Set();
globalThis.__gopeed_ffmpeg_close_all = () => {
for (const stop of Array.from(active)) stop(new Error("extension engine closed"));
};
function input(value, name, allowStream) {
if (value && typeof value.getReader === "function") {
if (!allowStream) throw new TypeError("stream inputs must be created by the inputs callback");
if (value.locked) throw new TypeError(name + " stream is locked");
return { stream: value, request: null };
}
if (!value || typeof value.url !== "string" || !/^https?:\/\//i.test(value.url)) {
throw new TypeError(name + " must be an HTTP request or ReadableStream");
}
const headers = {};
new Headers(value.headers || {}).forEach((v, k) => { headers[k] = v; });
return { request: { url: value.url, headers } };
}
function merge(options) {
if (!options || typeof options !== "object") throw new TypeError("merge options are required");
const factory = options.inputs;
if (factory !== undefined && typeof factory !== "function") throw new TypeError("inputs must be a callback");
if (factory && (options.video !== undefined || options.audio !== undefined)) throw new TypeError("inputs cannot be combined with video/audio");
let video = factory ? null : input(options.video, "video", false);
let audio = factory ? null : input(options.audio, "audio", false);
const args = options.args === undefined ? [] : options.args;
if (!Array.isArray(args) || args.some(v => typeof v !== "string")) throw new TypeError("args must be a string array");
const format = options.format === undefined ? "mp4" : options.format;
if (typeof format !== "string") throw new TypeError("format must be a string");
const signal = options.signal;
const id = create({ format, args });
let controller, initialized = false, stopped = false;
const readers = [];
const lifetime = new AbortController();
let ownedInputs;
function discardInputs(inputs, reason) {
for (const source of new Set([inputs && inputs.video, inputs && inputs.audio])) {
if (source && typeof source.cancel === "function" && !source.locked) Promise.resolve(source.cancel(reason)).catch(() => {});
}
}
function stop(reason) {
if (stopped) return;
stopped = true;
active.delete(shutdown);
lifetime.abort();
discardInputs(ownedInputs, reason);
if (id) cancel(id);
if (signal) signal.removeEventListener("abort", abort);
for (const reader of readers) {
Promise.resolve(reader.cancel(reason)).catch(() => {}).then(() => {
try { reader.releaseLock(); } catch (_) {}
});
}
}
function shutdown(reason) {
controller.error(reason);
stop(reason);
}
function abort() {
const error = new Error("FFmpeg operation aborted");
error.name = "AbortError";
stop(error);
controller.error(error);
}
async function pump(reader, index) {
try {
while (!stopped) {
const result = await reader.read();
if (stopped) return;
if (result.done) { end(id, index, ""); return; }
if (!(result.value instanceof Uint8Array)) throw new TypeError("media streams must contain Uint8Array chunks");
// Keep individual crossings bounded, even if upstream emits a large chunk.
for (let offset = 0; offset < result.value.byteLength && !stopped; offset += 256 * 1024) {
if (!await push(id, index, result.value.subarray(offset, offset + 256 * 1024))) return;
}
}
} catch (error) {
if (!stopped) {
end(id, index, String(error && error.message || error));
controller.error(error);
stop(error);
}
}
}
const output = new ReadableStream({
start(c) {
controller = c;
active.add(shutdown);
if (signal) {
if (signal.aborted) abort();
else signal.addEventListener("abort", abort);
}
},
async pull(c) {
if (stopped) return;
try {
if (!initialized) {
initialized = true;
if (stopped) return;
if (factory) {
const inputs = await factory({ signal: lifetime.signal });
if (stopped) { discardInputs(inputs, new Error("FFmpeg operation cancelled")); return; }
ownedInputs = inputs;
video = input(inputs && inputs.video, "video", true);
audio = input(inputs && inputs.audio, "audio", true);
}
if (video.stream && video.stream === audio.stream) throw new TypeError("video and audio must be distinct streams");
for (const source of [video, audio]) {
if (source.stream) readers.push(source.stream.getReader());
}
start(id, { video: video.request, audio: audio.request, format, args });
let readerIndex = 0;
for (const [index, source] of [video, audio].entries()) {
if (source.stream) void pump(readers[readerIndex++], index);
}
}
const chunk = await read(id);
if (stopped) return;
if (chunk === null) { stop(); c.close(); }
else c.enqueue(new Uint8Array(chunk));
} catch (error) {
if (!stopped) {
const failure = typeof MessageError === "function" && !(error instanceof MessageError)
? new MessageError(String(error && error.message || error)) : error;
c.error(failure); stop(failure);
}
}
},
cancel(reason) { stop(reason); }
}, { highWaterMark: 0 });
if (globalThis.__gopeed_bind_producer) globalThis.__gopeed_bind_producer(output, id);
return output;
}
globalThis.__gopeed_ffmpeg = Object.freeze({ merge, supportsInputFactory: true, supportsProducerProgress: true });
})();
+337
View File
@@ -0,0 +1,337 @@
package ffmpeg
import (
"context"
_ "embed"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"strings"
"sync"
"sync/atomic"
"github.com/GopeedLab/gopeed/internal/httpclient"
"github.com/GopeedLab/gopeed/internal/production"
"github.com/GopeedLab/gopeed/internal/tempfiles"
media "github.com/GopeedLab/gopeed/pkg/download/engine/ffmpeg"
"github.com/dop251/goja"
"github.com/dop251/goja_nodejs/eventloop"
)
//go:embed ffmpeg.js
var script string
type Config struct {
TempFiles *tempfiles.Scope
Producers *production.Registry
TempDir string
DefaultUserAgent *string
ProxyHandler func(*http.Request) (*url.URL, error)
RegisterCleanup func(func())
}
type options struct {
Video *media.HTTPSource `json:"video"`
Audio *media.HTTPSource `json:"audio"`
Format string `json:"format"`
Args []string `json:"args"`
}
type job struct {
inputs [2]media.Input
ctx context.Context
cancel context.CancelFunc
output *io.PipeReader
streams [2]*media.DiskInput
mu sync.Mutex
start chan options
started bool
}
func (j *job) Progress() (downloaded, received int64) {
j.mu.Lock()
defer j.mu.Unlock()
for _, input := range j.inputs {
if p, ok := input.(production.Source); ok {
d, r := p.Progress()
downloaded += d
received += r
}
}
return
}
func (j *job) stop() {
j.cancel()
j.output.CloseWithError(context.Canceled)
j.mu.Lock()
defer j.mu.Unlock()
for _, s := range j.streams {
if s != nil {
s.Close()
}
}
}
func Enable(vm *goja.Runtime, loop *eventloop.EventLoop, cfg *Config) error {
if cfg == nil {
cfg = &Config{}
}
spoolDir := cfg.TempDir
if cfg.Producers == nil {
cfg.Producers = &production.Registry{}
}
client, err := httpclient.NewClient(httpclient.Options{Transport: httpclient.TransportOptions{Proxy: cfg.ProxyHandler}})
if err != nil {
return err
}
var mu sync.Mutex
jobs := map[string]*job{}
var next atomic.Uint64
get := func(id string) *job { mu.Lock(); defer mu.Unlock(); return jobs[id] }
remove := func(id string) {
mu.Lock()
j := jobs[id]
delete(jobs, id)
cfg.Producers.Remove(id)
mu.Unlock()
if j != nil {
j.stop()
}
}
if cfg.RegisterCleanup != nil {
cfg.RegisterCleanup(func() {
done := make(chan struct{})
if loop.RunOnLoop(func(vm *goja.Runtime) {
defer close(done)
if closeAll, ok := goja.AssertFunction(vm.Get("__gopeed_ffmpeg_close_all")); ok {
_, _ = closeAll(goja.Undefined())
}
}) {
<-done
}
mu.Lock()
old := jobs
jobs = map[string]*job{}
mu.Unlock()
for _, j := range old {
j.stop()
}
client.CloseIdleConnections()
})
}
if err := vm.Set("__gopeed_ffmpeg_create", func(call goja.FunctionCall) goja.Value {
var opts options
if err := vm.ExportTo(call.Argument(0), &opts); err != nil {
panic(vm.NewGoError(err))
}
ctx, cancelCause := context.WithCancelCause(context.Background())
cancel := func() { cancelCause(context.Canceled) }
r, w := io.Pipe()
j := &job{ctx: ctx, cancel: cancel, output: r, start: make(chan options, 1)}
id := fmt.Sprintf("ffmpeg-%d", next.Add(1))
mu.Lock()
jobs[id] = j
cfg.Producers.Set(id, j)
mu.Unlock()
go func() {
var inputs [2]media.Input
var runErr error
defer func() {
if v := recover(); v != nil {
runErr = fmt.Errorf("ffmpeg panic: %v", v)
}
if errors.Is(runErr, context.Canceled) {
runErr = context.Cause(ctx)
}
cancel()
for _, in := range inputs {
if in != nil {
in.Close()
}
}
w.CloseWithError(runErr)
}()
var opts options
select {
case opts = <-j.start:
case <-ctx.Done():
runErr = ctx.Err()
return
}
for i, source := range []*media.HTTPSource{opts.Video, opts.Audio} {
if source == nil {
inputs[i] = j.streams[i]
} else {
inputs[i], runErr = media.OpenSpoolingHTTP(ctx, client, *source, spoolDir, cfg.TempFiles)
if runErr != nil {
return
}
}
j.mu.Lock()
j.inputs[i] = inputs[i]
j.mu.Unlock()
if input, ok := inputs[i].(interface{ Failures() <-chan error }); ok {
go func() {
select {
case err := <-input.Failures():
cancelCause(err)
case <-ctx.Done():
}
}()
}
}
release, err := media.Acquire(ctx)
if err != nil {
runErr = err
return
}
defer release()
runErr = media.RunAcquired(ctx, inputs[0], inputs[1], w, opts.Format, opts.Args)
}()
return vm.ToValue(id)
}); err != nil {
return err
}
if err := vm.Set("__gopeed_ffmpeg_start", func(call goja.FunctionCall) goja.Value {
id := call.Argument(0).String()
j := get(id)
if j == nil {
panic(vm.NewGoError(io.ErrClosedPipe))
}
var opts options
if err := vm.ExportTo(call.Argument(1), &opts); err != nil {
panic(vm.NewGoError(err))
}
if _, err := media.Arguments(opts.Format, opts.Args); err != nil {
panic(vm.NewGoError(err))
}
for _, source := range []*media.HTTPSource{opts.Video, opts.Audio} {
if source == nil || cfg.DefaultUserAgent == nil {
continue
}
specified := false
for key := range source.Headers {
if strings.EqualFold(key, "User-Agent") {
specified = true
break
}
}
if !specified {
if source.Headers == nil {
source.Headers = make(map[string]string)
}
source.Headers["User-Agent"] = *cfg.DefaultUserAgent
}
}
j.mu.Lock()
defer j.mu.Unlock()
if j.started || j.ctx.Err() != nil {
panic(vm.NewGoError(io.ErrClosedPipe))
}
j.started = true
for i, source := range []*media.HTTPSource{opts.Video, opts.Audio} {
if source == nil {
var err error
j.streams[i], err = media.NewDiskInput(j.ctx, spoolDir, cfg.TempFiles)
if err != nil {
j.cancel()
panic(vm.NewGoError(err))
}
j.inputs[i] = j.streams[i]
}
}
j.start <- opts
return goja.Undefined()
}); err != nil {
return err
}
if err := vm.Set("__gopeed_ffmpeg_read", func(id string) goja.Value {
promise, resolve, reject := vm.NewPromise()
go func() {
var readErr error
buf := make([]byte, 64*1024)
n := 0
j := get(id)
if j == nil {
readErr = io.ErrClosedPipe
} else {
n, readErr = j.output.Read(buf)
}
if !loop.RunOnLoop(func(vm *goja.Runtime) {
if readErr != nil {
remove(id)
if readErr == io.EOF {
resolve(goja.Null())
} else {
reject(vm.NewGoError(readErr))
}
return
}
resolve(vm.ToValue(vm.NewArrayBuffer(buf[:n])))
}) {
remove(id)
}
}()
return vm.ToValue(promise)
}); err != nil {
return err
}
if err := vm.Set("__gopeed_ffmpeg_push", func(call goja.FunctionCall) goja.Value {
id := call.Argument(0).String()
index := int(call.Argument(1).ToInteger())
var chunk []byte
switch b := call.Argument(2).Export().(type) {
case goja.ArrayBuffer:
chunk = append([]byte(nil), b.Bytes()...)
case []byte:
chunk = append([]byte(nil), b...)
default:
panic(vm.NewTypeError("ffmpeg input requires bytes"))
}
promise, resolve, reject := vm.NewPromise()
go func() {
var err error
j := get(id)
if j == nil || index < 0 || index > 1 || j.streams[index] == nil {
err = io.ErrClosedPipe
} else {
_, err = j.streams[index].Write(chunk)
}
if err != nil && j != nil && index >= 0 && index < 2 && j.streams[index] != nil {
j.streams[index].End(err)
}
loop.RunOnLoop(func(vm *goja.Runtime) {
if j != nil && j.ctx.Err() != nil {
resolve(false)
return
}
if err != nil {
reject(vm.NewGoError(err))
} else {
resolve(true)
}
})
}()
return vm.ToValue(promise)
}); err != nil {
return err
}
if err := vm.Set("__gopeed_ffmpeg_end", func(id string, index int, message string) {
if j := get(id); j != nil && index >= 0 && index < 2 && j.streams[index] != nil {
var err error
if message != "" {
err = errors.New(message)
}
j.streams[index].End(err)
}
}); err != nil {
return err
}
if err := vm.Set("__gopeed_ffmpeg_cancel", remove); err != nil {
return err
}
_, err = vm.RunString(script)
return err
}
+56 -10
View File
@@ -9,6 +9,7 @@ import (
"sync"
"sync/atomic"
"github.com/GopeedLab/gopeed/internal/production"
"github.com/dop251/goja"
"github.com/dop251/goja_nodejs/eventloop"
)
@@ -17,6 +18,7 @@ import (
var script string
type Config struct {
Producers *production.Registry
CreateObjectURL func(opts *ObjectURLOptions, open ObjectURLOpener) (string, error)
RevokeObjectURL func(url string) error
}
@@ -39,6 +41,23 @@ func Enable(runtime *goja.Runtime, loop *eventloop.EventLoop, cfg *Config) error
cfg = &Config{}
}
blobPipes := newBlobPipeRegistry()
if err := runtime.Set("__gopeed_blob_pipe_bind", func(pipeID, producerID string) {
var source production.Source
if cfg.Producers != nil {
source = cfg.Producers.Get(producerID)
}
blobPipes.mu.Lock()
pipe := blobPipes.pipes[pipeID]
blobPipes.mu.Unlock()
if pipe != nil {
pipe.mu.Lock()
pipe.producer = source
pipe.mu.Unlock()
pipe.readyOnce.Do(func() { close(pipe.ready) })
}
}); err != nil {
return err
}
if err := runtime.Set("__gopeed_create_blob_object_url", func(call goja.FunctionCall) goja.Value {
if cfg.CreateObjectURL == nil {
panic(runtime.NewGoError(fmt.Errorf("blob object url handler not configured")))
@@ -86,18 +105,23 @@ func Enable(runtime *goja.Runtime, loop *eventloop.EventLoop, cfg *Config) error
}
if err := runtime.Set("__gopeed_blob_pipe_chunk", func(call goja.FunctionCall) goja.Value {
id := call.Argument(0).String()
ok := blobPipes.Push(id, exportBytes(call.Argument(1)))
return runtime.ToValue(ok)
chunk := append([]byte(nil), exportBytes(call.Argument(1))...)
promise, resolve, _ := runtime.NewPromise()
go func() {
ok := blobPipes.Push(id, chunk)
loop.RunOnLoop(func(vm *goja.Runtime) { resolve(ok) })
}()
return runtime.ToValue(promise)
}); err != nil {
return err
}
if err := runtime.Set("__gopeed_blob_pipe_close", func(id string) {
blobPipes.Close(id, nil)
go blobPipes.Close(id, nil)
}); err != nil {
return err
}
if err := runtime.Set("__gopeed_blob_pipe_error", func(id string, message string) {
blobPipes.Close(id, errors.New(message))
go blobPipes.Close(id, errors.New(message))
}); err != nil {
return err
}
@@ -184,12 +208,15 @@ type blobPipeItem struct {
}
type blobPipe struct {
id string
loop *eventloop.EventLoop
registry *blobPipeRegistry
ctx context.Context
cancel context.CancelFunc
ch chan blobPipeItem
ready chan struct{}
readyOnce sync.Once
producer production.Source
id string
loop *eventloop.EventLoop
registry *blobPipeRegistry
ctx context.Context
cancel context.CancelFunc
ch chan blobPipeItem
mu sync.Mutex
buf []byte
@@ -200,6 +227,7 @@ type blobPipe struct {
func newBlobPipe(ctx context.Context, loop *eventloop.EventLoop, registry *blobPipeRegistry) *blobPipe {
pipeCtx, cancel := context.WithCancel(ctx)
return &blobPipe{
ready: make(chan struct{}),
id: registry.NewID(),
loop: loop,
registry: registry,
@@ -223,6 +251,7 @@ func (p *blobPipe) Push(chunk []byte) bool {
}
func (p *blobPipe) CloseWithError(err error) {
p.readyOnce.Do(func() { close(p.ready) })
p.once.Do(func() {
p.registry.Remove(p.id)
select {
@@ -475,3 +504,20 @@ func exportJSError(value goja.Value) error {
}
return errors.New(stack)
}
func (p *blobPipe) Production() production.Source {
p.mu.Lock()
defer p.mu.Unlock()
return p.producer
}
func (p *blobPipe) WaitProduction(ctx context.Context) error {
select {
case <-p.ready:
return nil
case <-p.ctx.Done():
return p.ctx.Err()
case <-ctx.Done():
return ctx.Err()
}
}
@@ -269,3 +269,57 @@ func newStreamTestEngine(t *testing.T, created chan<- createdObjectURL) (*engine
})
return e, e.Close
}
// A stalled disk/network consumer must pause the producer without blocking
// the JS event loop, so cancellation and other tracks can still run.
func TestBlobBackpressureLeavesEventLoopResponsive(t *testing.T) {
created := make(chan createdObjectURL, 1)
e, cleanup := newStreamTestEngine(t, created)
defer cleanup()
_, err := e.RunString(`
globalThis.produced = 0;
globalThis.sourceCancelled = false;
__gopeed_blob_create_object_url(() => new ReadableStream({
pull(c) { produced++; c.enqueue(new Uint8Array(64*1024)); },
cancel() { sourceCancelled = true; }
}));
`)
if err != nil {
t.Fatal(err)
}
object := <-created
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
defer cancel()
reader, err := object.open(ctx, stream.ObjectURLOpenRequest{End: -1})
if err != nil {
t.Fatal(err)
}
defer reader.Close()
time.Sleep(100 * time.Millisecond)
value, err := e.RunString("produced")
if err != nil {
t.Fatal(err)
}
before := value.(int64)
if before > 10 {
t.Fatalf("unbounded production: %d", before)
}
time.Sleep(50 * time.Millisecond)
after, err := e.RunString("produced")
if err != nil || after != value {
t.Fatalf("producer did not pause: %v -> %v (%v)", value, after, err)
}
if ctx.Err() != nil {
t.Fatal("event loop only resumed after cancellation")
}
if _, err = reader.Read(make([]byte, 64*1024)); err != nil {
t.Fatal(err)
}
time.Sleep(30 * time.Millisecond)
after, err = e.RunString("produced")
if err != nil || after.(int64) <= before {
t.Fatalf("producer did not resume: %v (%v)", after, err)
}
reader.Close()
waitForJSValue(t, e, "String(sourceCancelled)", "true")
}
Loaded 100 of 324 files, more files were not shown because too many files have changed in this diff. Show more