Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c7fefd2b9b | ||
|
|
b8e36b483e | ||
|
|
91ac841b9d | ||
|
|
5134446c3f | ||
|
|
82b6ebd6f1 | ||
|
|
f8ea009fe9 |
No files matched your search
@@ -1,16 +0,0 @@
|
|||||||
.git
|
|
||||||
|
|
||||||
*.data
|
|
||||||
*.log
|
|
||||||
|
|
||||||
node_modules
|
|
||||||
**/node_modules
|
|
||||||
|
|
||||||
Dockerfile
|
|
||||||
.dockerignore
|
|
||||||
|
|
||||||
.github
|
|
||||||
_docs
|
|
||||||
_examples
|
|
||||||
bin
|
|
||||||
ui
|
|
||||||
@@ -1,2 +0,0 @@
|
|||||||
# Keep upstream plugin sources recognizable and collapsed in GitHub reviews.
|
|
||||||
ui/flutter/third_party/** linguist-vendored whitespace=-blank-at-eol,-blank-at-eof
|
|
||||||
@@ -1 +0,0 @@
|
|||||||
* @monkeyWie
|
|
||||||
@@ -12,4 +12,4 @@
|
|||||||
#custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2']
|
#custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2']
|
||||||
# github: monkeyWie
|
# github: monkeyWie
|
||||||
ko_fi: gopeed
|
ko_fi: gopeed
|
||||||
custom: https://gopeed.com/docs/donate
|
custom: https://docs.gopeed.com/donate.html
|
||||||
@@ -1,189 +1,43 @@
|
|||||||
name-template: "v$NEXT_PATCH_VERSION 🌈"
|
name-template: 'v$NEXT_PATCH_VERSION 🌈'
|
||||||
tag-template: "v$NEXT_PATCH_VERSION"
|
tag-template: 'v$NEXT_PATCH_VERSION'
|
||||||
categories:
|
categories:
|
||||||
- title: "🆕 Features"
|
- title: '🆕 Features'
|
||||||
labels:
|
labels:
|
||||||
- "feat"
|
- 'feat'
|
||||||
- "feature"
|
- 'feature'
|
||||||
- "enhancement"
|
- 'enhancement'
|
||||||
- title: "🐛 Bug Fixes"
|
- title: '🐛 Bug Fixes'
|
||||||
labels:
|
labels:
|
||||||
- "fix"
|
- 'fix'
|
||||||
- "bugfix"
|
- 'bugfix'
|
||||||
- "bug"
|
- 'bug'
|
||||||
- title: "🔧 Performance Improvements"
|
- title: '🧪 Tests'
|
||||||
labels:
|
label: 'test'
|
||||||
- "perf"
|
- title: '🧰 Maintenance'
|
||||||
- title: "🧪 Tests"
|
label: 'chore'
|
||||||
label: "test"
|
- title: '📖 Document'
|
||||||
- title: "🧰 Maintenance"
|
label: 'docs'
|
||||||
label: "chore"
|
- title: '🚀 CI/CD'
|
||||||
- title: "📖 Document"
|
label: 'ci'
|
||||||
label: "docs"
|
- title: '🌎 Internationalization'
|
||||||
- title: "🚀 CI/CD"
|
label: 'translation'
|
||||||
label: "ci"
|
change-template: '- $TITLE @$AUTHOR (#$NUMBER)'
|
||||||
- title: "🌎 Internationalization"
|
|
||||||
label: "translation"
|
|
||||||
change-template: "- $TITLE @$AUTHOR (#$NUMBER)"
|
|
||||||
version-resolver:
|
version-resolver:
|
||||||
major:
|
major:
|
||||||
labels:
|
labels:
|
||||||
- "major"
|
- 'major'
|
||||||
minor:
|
minor:
|
||||||
labels:
|
labels:
|
||||||
- "minor"
|
- 'minor'
|
||||||
patch:
|
patch:
|
||||||
labels:
|
labels:
|
||||||
- "patch"
|
- 'patch'
|
||||||
default: patch
|
default: patch
|
||||||
template: |
|
template: |
|
||||||
<table>
|
|
||||||
<tbody>
|
|
||||||
<tr>
|
|
||||||
<td rowspan="4">🪟 Windows</td>
|
|
||||||
<td rowspan="2"><code>EXE</code></td>
|
|
||||||
<td>amd64</td>
|
|
||||||
<td><a href="https://gopeed.com/api/download?tpl=Gopeed-v$NEXT_PATCH_VERSION-windows-amd64.zip">📥</a></td>
|
|
||||||
</tr>
|
|
||||||
<tr>
|
|
||||||
<td>arm64</td>
|
|
||||||
<td><a href="https://gopeed.com/api/download?tpl=Gopeed-v$NEXT_PATCH_VERSION-windows-arm64.zip">📥</a></td>
|
|
||||||
</tr>
|
|
||||||
<tr>
|
|
||||||
<td rowspan="2"><code>Portable</code></td>
|
|
||||||
<td>amd64</td>
|
|
||||||
<td><a href="https://gopeed.com/api/download?tpl=Gopeed-v$NEXT_PATCH_VERSION-windows-amd64-portable.zip">📥</a></td>
|
|
||||||
</tr>
|
|
||||||
<tr>
|
|
||||||
<td>arm64</td>
|
|
||||||
<td><a href="https://gopeed.com/api/download?tpl=Gopeed-v$NEXT_PATCH_VERSION-windows-arm64-portable.zip">📥</a></td>
|
|
||||||
</tr>
|
|
||||||
<tr>
|
|
||||||
<td rowspan="3">🍎 MacOS</td>
|
|
||||||
<td rowspan="3"><code>DMG</code></td>
|
|
||||||
<td>universal</td>
|
|
||||||
<td><a href="https://gopeed.com/api/download?tpl=Gopeed-v$NEXT_PATCH_VERSION-macos.dmg">📥</a></td>
|
|
||||||
</tr>
|
|
||||||
<tr>
|
|
||||||
<td>amd64</td>
|
|
||||||
<td><a href="https://gopeed.com/api/download?tpl=Gopeed-v$NEXT_PATCH_VERSION-macos-amd64.dmg">📥</a></td>
|
|
||||||
</tr>
|
|
||||||
<tr>
|
|
||||||
<td>arm64</td>
|
|
||||||
<td><a href="https://gopeed.com/api/download?tpl=Gopeed-v$NEXT_PATCH_VERSION-macos-arm64.dmg">📥</a></td>
|
|
||||||
</tr>
|
|
||||||
<tr>
|
|
||||||
<td rowspan="6">🐧 Linux</td>
|
|
||||||
<td><code>Flathub</code></td>
|
|
||||||
<td>amd64</td>
|
|
||||||
<td><a href="https://flathub.org/apps/com.gopeed.Gopeed">📥</a></td>
|
|
||||||
</tr>
|
|
||||||
<tr>
|
|
||||||
<td><code>SNAP</code></td>
|
|
||||||
<td>amd64</td>
|
|
||||||
<td><a href="https://snapcraft.io/gopeed">📥</a></td>
|
|
||||||
</tr>
|
|
||||||
<tr>
|
|
||||||
<td rowspan="2"><code>DEB</code></td>
|
|
||||||
<td>amd64</td>
|
|
||||||
<td><a href="https://gopeed.com/api/download?tpl=Gopeed-v$NEXT_PATCH_VERSION-linux-amd64.deb">📥</a></td>
|
|
||||||
</tr>
|
|
||||||
<tr>
|
|
||||||
<td>arm64</td>
|
|
||||||
<td><a href="https://gopeed.com/api/download?tpl=Gopeed-v$NEXT_PATCH_VERSION-linux-arm64.deb">📥</a></td>
|
|
||||||
</tr>
|
|
||||||
<tr>
|
|
||||||
<td rowspan="2"><code>AppImage</code></td>
|
|
||||||
<td>amd64</td>
|
|
||||||
<td><a href="https://gopeed.com/api/download?tpl=Gopeed-v$NEXT_PATCH_VERSION-linux-amd64.AppImage">📥</a></td>
|
|
||||||
</tr>
|
|
||||||
<tr>
|
|
||||||
<td>arm64</td>
|
|
||||||
<td><a href="https://gopeed.com/api/download?tpl=Gopeed-v$NEXT_PATCH_VERSION-linux-arm64.AppImage">📥</a></td>
|
|
||||||
</tr>
|
|
||||||
<tr>
|
|
||||||
<td rowspan="4">🤖 Android</td>
|
|
||||||
<td rowspan="4"><code>APK</code></td>
|
|
||||||
<td>universal</td>
|
|
||||||
<td><a href="https://gopeed.com/api/download?tpl=Gopeed-v$NEXT_PATCH_VERSION-android.apk">📥</a></td>
|
|
||||||
</tr>
|
|
||||||
<tr>
|
|
||||||
<td>armeabi-v7a</td>
|
|
||||||
<td><a href="https://gopeed.com/api/download?tpl=Gopeed-v$NEXT_PATCH_VERSION-android-armeabi-v7a.apk">📥</a></td>
|
|
||||||
</tr>
|
|
||||||
<tr>
|
|
||||||
<td>arm64-v8a</td>
|
|
||||||
<td><a href="https://gopeed.com/api/download?tpl=Gopeed-v$NEXT_PATCH_VERSION-android-arm64-v8a.apk">📥</a></td>
|
|
||||||
</tr>
|
|
||||||
<tr>
|
|
||||||
<td>x86_64</td>
|
|
||||||
<td><a href="https://gopeed.com/api/download?tpl=Gopeed-v$NEXT_PATCH_VERSION-android-x86_64.apk">📥</a></td>
|
|
||||||
</tr>
|
|
||||||
<tr>
|
|
||||||
<td>📱 iOS</td>
|
|
||||||
<td><code>IPA</code></td>
|
|
||||||
<td>universal</td>
|
|
||||||
<td><a href="https://gopeed.com/api/download?tpl=Gopeed-v$NEXT_PATCH_VERSION-ios.ipa">📥</a></td>
|
|
||||||
</tr>
|
|
||||||
<tr>
|
|
||||||
<td>🐳 Docker</td>
|
|
||||||
<td>-</td>
|
|
||||||
<td>universal</td>
|
|
||||||
<td><a href="https://hub.docker.com/r/liwei2633/gopeed">📥</a></td>
|
|
||||||
</tr>
|
|
||||||
<tr>
|
|
||||||
<td rowspan="2">💾 Qnap</td>
|
|
||||||
<td rowspan="2"><code>QPKG</code></td>
|
|
||||||
<td>amd64</td>
|
|
||||||
<td><a href="https://gopeed.com/api/download?tpl=gopeed-v$NEXT_PATCH_VERSION-qnap-amd64.qpkg">📥</a></td>
|
|
||||||
</tr>
|
|
||||||
<tr>
|
|
||||||
<td>arm64</td>
|
|
||||||
<td><a href="https://gopeed.com/api/download?tpl=gopeed-v$NEXT_PATCH_VERSION-qnap-arm64.qpkg">📥</a></td>
|
|
||||||
</tr>
|
|
||||||
<tr>
|
|
||||||
<td rowspan="8">🌐 Web</td>
|
|
||||||
<td rowspan="3"><code>Windows</code></td>
|
|
||||||
<td>amd64</td>
|
|
||||||
<td><a href="https://gopeed.com/api/download?tpl=gopeed-web-v$NEXT_PATCH_VERSION-windows-amd64.zip">📥</a></td>
|
|
||||||
</tr>
|
|
||||||
<tr>
|
|
||||||
<td>arm64</td>
|
|
||||||
<td><a href="https://gopeed.com/api/download?tpl=gopeed-web-v$NEXT_PATCH_VERSION-windows-arm64.zip">📥</a></td>
|
|
||||||
</tr>
|
|
||||||
<tr>
|
|
||||||
<td>386</td>
|
|
||||||
<td><a href="https://gopeed.com/api/download?tpl=gopeed-web-v$NEXT_PATCH_VERSION-windows-386.zip">📥</a></td>
|
|
||||||
</tr>
|
|
||||||
<tr>
|
|
||||||
<td rowspan="2"><code>MacOS</code></td>
|
|
||||||
<td>amd64</td>
|
|
||||||
<td><a href="https://gopeed.com/api/download?tpl=gopeed-web-v$NEXT_PATCH_VERSION-macos-amd64.zip">📥</a></td>
|
|
||||||
</tr>
|
|
||||||
<tr>
|
|
||||||
<td>arm64</td>
|
|
||||||
<td><a href="https://gopeed.com/api/download?tpl=gopeed-web-v$NEXT_PATCH_VERSION-macos-arm64.zip">📥</a></td>
|
|
||||||
</tr>
|
|
||||||
<tr>
|
|
||||||
<td rowspan="3"><code>Linux</code></td>
|
|
||||||
<td>amd64</td>
|
|
||||||
<td><a href="https://gopeed.com/api/download?tpl=gopeed-web-v$NEXT_PATCH_VERSION-linux-amd64.zip">📥</a></td>
|
|
||||||
</tr>
|
|
||||||
<tr>
|
|
||||||
<td>arm64</td>
|
|
||||||
<td><a href="https://gopeed.com/api/download?tpl=gopeed-web-v$NEXT_PATCH_VERSION-linux-arm64.zip">📥</a></td>
|
|
||||||
</tr>
|
|
||||||
<tr>
|
|
||||||
<td>386</td>
|
|
||||||
<td><a href="https://gopeed.com/api/download?tpl=gopeed-web-v$NEXT_PATCH_VERSION-linux-386.zip">📥</a></td>
|
|
||||||
</tr>
|
|
||||||
</tbody>
|
|
||||||
</table>
|
|
||||||
|
|
||||||
# Release notes
|
# Release notes
|
||||||
|
|
||||||
$CHANGES
|
$CHANGES
|
||||||
|
|
||||||
# 更新日志
|
# 更新日志
|
||||||
|
|
||||||
$CHANGES
|
$CHANGES
|
||||||
@@ -4,19 +4,17 @@ on:
|
|||||||
workflow_dispatch:
|
workflow_dispatch:
|
||||||
inputs:
|
inputs:
|
||||||
platform:
|
platform:
|
||||||
description: "Build platform"
|
description: 'Build platform'
|
||||||
required: true
|
required: true
|
||||||
default: "all"
|
default: 'all'
|
||||||
test:
|
test:
|
||||||
description: "Test mode"
|
description: 'Test mode'
|
||||||
required: true
|
required: true
|
||||||
default: "false"
|
default: "false"
|
||||||
|
|
||||||
env:
|
env:
|
||||||
GO_VERSION: "1.25.4"
|
GO_VERSION: "1.21"
|
||||||
FLUTTER_VERSION: "3.41.6"
|
FLUTTER_VERSION: "3.16.8"
|
||||||
GA4_MEASUREMENT_ID: ${{ secrets.GA4_MEASUREMENT_ID }}
|
|
||||||
GA4_API_SECRET: ${{ secrets.GA4_API_SECRET }}
|
|
||||||
|
|
||||||
permissions:
|
permissions:
|
||||||
contents: write
|
contents: write
|
||||||
@@ -36,110 +34,40 @@ jobs:
|
|||||||
|
|
||||||
build-windows:
|
build-windows:
|
||||||
if: ${{ github.event.inputs.platform == 'all' || github.event.inputs.platform == 'windows' }}
|
if: ${{ github.event.inputs.platform == 'all' || github.event.inputs.platform == 'windows' }}
|
||||||
needs: [get-release]
|
runs-on: windows-latest
|
||||||
strategy:
|
needs: [ get-release ]
|
||||||
fail-fast: false
|
|
||||||
matrix:
|
|
||||||
include:
|
|
||||||
- arch: arm64
|
|
||||||
os: windows-11-arm
|
|
||||||
llvm_ver: "20251202" # Only ARM64
|
|
||||||
flutter_channel: "main"
|
|
||||||
flutter_version: "7e1c8868"
|
|
||||||
- arch: amd64
|
|
||||||
os: windows-2022
|
|
||||||
runs-on: ${{ matrix.os }}
|
|
||||||
|
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v3
|
- uses: actions/checkout@v3
|
||||||
- name: Enable long paths for flutter main branch checks
|
- uses: actions/setup-go@v4
|
||||||
run: |
|
|
||||||
git config --global core.longpaths true
|
|
||||||
|
|
||||||
# Install LLVM environment only on ARM64
|
|
||||||
- name: Install llvm-mingw-ucrt-aarch64 (ARM64 only)
|
|
||||||
if: matrix.arch == 'arm64'
|
|
||||||
run: |
|
|
||||||
$ver = "${{ matrix.llvm_ver }}"
|
|
||||||
$url = "https://github.com/mstorsjo/llvm-mingw/releases/download/$ver/llvm-mingw-$ver-ucrt-aarch64.zip"
|
|
||||||
$zip = "$env:RUNNER_TEMP\\llvm.zip"
|
|
||||||
$extract = "$env:RUNNER_TEMP\\extract"
|
|
||||||
$target = "C:\\clangarm64"
|
|
||||||
|
|
||||||
curl -L $url -o $zip
|
|
||||||
rm -r -fo $extract,$target -ea Ignore
|
|
||||||
mkdir $extract | Out-Null
|
|
||||||
|
|
||||||
tar -xf $zip -C $extract
|
|
||||||
mv (Get-ChildItem $extract)[0].FullName $target
|
|
||||||
|
|
||||||
$b = "$target\\bin"
|
|
||||||
"CC=$b\\clang.exe" >> $env:GITHUB_ENV
|
|
||||||
"CXX=$b\\clang++.exe" >> $env:GITHUB_ENV
|
|
||||||
"CLANGARM64_BIN=$b" >> $env:GITHUB_ENV
|
|
||||||
"CGO_ENABLED=1" >> $env:GITHUB_ENV
|
|
||||||
"CLANGARM64_ROOT=$target" >> $env:GITHUB_ENV
|
|
||||||
$b >> $env:GITHUB_PATH
|
|
||||||
|
|
||||||
- uses: actions/setup-go@v5
|
|
||||||
with:
|
with:
|
||||||
go-version: ${{ env.GO_VERSION }}
|
go-version: ${{ env.GO_VERSION }}
|
||||||
- uses: subosito/flutter-action@v2
|
- uses: subosito/flutter-action@v2
|
||||||
with:
|
with:
|
||||||
channel: ${{ matrix.flutter_channel || 'stable' }}
|
flutter-version: ${{ env.FLUTTER_VERSION }}
|
||||||
flutter-version: ${{ matrix.flutter_version || env.FLUTTER_VERSION }}
|
|
||||||
- name: Build
|
- name: Build
|
||||||
env:
|
env:
|
||||||
VERSION: ${{ needs.get-release.outputs.version }}
|
VERSION: ${{ needs.get-release.outputs.version }}
|
||||||
GOARCH: ${{ matrix.arch }}
|
|
||||||
run: |
|
run: |
|
||||||
Invoke-WebRequest -Uri "https://github.com/jrsoftware/issrc/raw/main/Files/Languages/ChineseSimplified.isl" -OutFile "ChineseSimplified.isl"
|
Invoke-WebRequest -Uri "https://github.com/jrsoftware/issrc/raw/main/Files/Languages/Unofficial/ChineseSimplified.isl" -OutFile "ChineseSimplified.isl"
|
||||||
mv ChineseSimplified.isl "C:\Program Files (x86)\Inno Setup 6\Languages\"
|
mv ChineseSimplified.isl "C:\Program Files (x86)\Inno Setup 6\Languages\"
|
||||||
|
|
||||||
go build -tags nosqlite -ldflags="-w -s -X github.com/GopeedLab/gopeed/pkg/base.Version=$env:VERSION" -buildmode=c-shared -o ui/flutter/windows/libgopeed.dll github.com/GopeedLab/gopeed/bind/desktop
|
go build -tags nosqlite -ldflags="-w -s -X github.com/GopeedLab/gopeed/pkg/base.Version=$VERSION" -buildmode=c-shared -o ui/flutter/windows/libgopeed.dll github.com/GopeedLab/gopeed/bind/desktop
|
||||||
go build -ldflags="-w -s" -o ui/flutter/assets/exec/host.exe github.com/GopeedLab/gopeed/cmd/host
|
|
||||||
go build -ldflags="-w -s" -o ui/flutter/assets/exec/updater.exe github.com/GopeedLab/gopeed/cmd/updater
|
|
||||||
cd ui/flutter
|
cd ui/flutter
|
||||||
$TAG = "v$env:VERSION"
|
$TAG = "v$env:VERSION"
|
||||||
flutter build windows --dart-define="UPDATE_CHANNEL=windowsPortable" --dart-define="GA4_MEASUREMENT_ID=${{ env.GA4_MEASUREMENT_ID }}" --dart-define="GA4_API_SECRET=${{ env.GA4_API_SECRET }}"
|
flutter build windows
|
||||||
|
$mingw = "C:\Program Files\Git\mingw64\bin"
|
||||||
$system = "C:\Windows\System32"
|
$system = "C:\Windows\System32"
|
||||||
if ("${{ matrix.arch }}" -eq "amd64") {
|
$release = "build\windows\x64\runner\Release\"
|
||||||
# for amd64
|
cp $mingw\libstdc++-6.dll $release
|
||||||
$release = "build\windows\x64\runner\Release\"
|
cp $mingw\libgcc_s_seh-1.dll $release
|
||||||
# Use the same toolchain as cgo instead of Git's bundled DLLs.
|
cp $mingw\libwinpthread-1.dll $release
|
||||||
$cc = (Get-Command -Name (go env CC) -CommandType Application -ErrorAction Stop | Select-Object -First 1).Source
|
|
||||||
foreach ($dll in @("libstdc++-6.dll", "libgcc_s_seh-1.dll", "libwinpthread-1.dll")) {
|
|
||||||
$runtimePath = [string](& $cc "-print-file-name=$dll")
|
|
||||||
if ($LASTEXITCODE -ne 0 -or [string]::IsNullOrWhiteSpace($runtimePath)) {
|
|
||||||
throw "Failed to locate $dll using $cc"
|
|
||||||
}
|
|
||||||
$runtimePath = $runtimePath.Trim()
|
|
||||||
# Some MinGW distributions keep runtime DLLs beside gcc.exe.
|
|
||||||
if ($runtimePath -eq $dll) {
|
|
||||||
$runtimePath = Join-Path (Split-Path -Parent $cc) $dll
|
|
||||||
}
|
|
||||||
if (!(Test-Path -LiteralPath $runtimePath -PathType Leaf)) {
|
|
||||||
throw "MinGW runtime $dll was not found by $cc (reported path: $runtimePath)"
|
|
||||||
}
|
|
||||||
Write-Host "Copying MinGW runtime: $runtimePath"
|
|
||||||
Copy-Item -LiteralPath $runtimePath -Destination $release
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
# for ARM64
|
|
||||||
$release = "build\windows\arm64\runner\Release\"
|
|
||||||
$mingw = "$env:CLANGARM64_ROOT\aarch64-w64-mingw32\bin"
|
|
||||||
cp $mingw\libc++.dll $release
|
|
||||||
cp $mingw\libunwind.dll $release
|
|
||||||
cp $mingw\libwinpthread-1.dll $release
|
|
||||||
}
|
|
||||||
cp $system\msvcp140.dll $release
|
cp $system\msvcp140.dll $release
|
||||||
cp $system\vcruntime140.dll $release
|
cp $system\vcruntime140.dll $release
|
||||||
cp $system\vcruntime140_1.dll $release
|
cp $system\vcruntime140_1.dll $release
|
||||||
|
|
||||||
New-Item -Path build\windows\Output -ItemType Directory
|
New-Item -Path build\windows\Output -ItemType Directory
|
||||||
Compress-Archive -Path "$release*" -DestinationPath "build\windows\Output\Gopeed-$TAG-windows-${{ matrix.arch }}-portable.zip"
|
Compress-Archive -Path "$release*" -DestinationPath "build\windows\Output\Gopeed-$TAG-windows-amd64-portable.zip"
|
||||||
|
|
||||||
flutter build windows --dart-define="UPDATE_CHANNEL=windowsInstaller" --dart-define="GA4_MEASUREMENT_ID=${{ env.GA4_MEASUREMENT_ID }}" --dart-define="GA4_API_SECRET=${{ env.GA4_API_SECRET }}"
|
|
||||||
cd build/windows
|
cd build/windows
|
||||||
echo @"
|
echo @"
|
||||||
; Script generated by the Inno Setup Script Wizard.
|
; Script generated by the Inno Setup Script Wizard.
|
||||||
@@ -169,13 +97,11 @@ jobs:
|
|||||||
PrivilegesRequired=lowest
|
PrivilegesRequired=lowest
|
||||||
OutputBaseFilename=gopeed
|
OutputBaseFilename=gopeed
|
||||||
SetupIconFile=..\..\assets\icon\icon.ico
|
SetupIconFile=..\..\assets\icon\icon.ico
|
||||||
UninstallDisplayIcon={app}\{#MyAppExeName}
|
|
||||||
Compression=lzma
|
Compression=lzma
|
||||||
SolidCompression=yes
|
SolidCompression=yes
|
||||||
WizardStyle=modern
|
WizardStyle=modern
|
||||||
LanguageDetectionMethod=uilanguage
|
LanguageDetectionMethod=uilanguage
|
||||||
ShowLanguageDialog=yes
|
ShowLanguageDialog=yes
|
||||||
CloseApplications=force
|
|
||||||
|
|
||||||
[Languages]
|
[Languages]
|
||||||
Name: "english"; MessagesFile: "compiler:Default.isl"
|
Name: "english"; MessagesFile: "compiler:Default.isl"
|
||||||
@@ -185,7 +111,7 @@ jobs:
|
|||||||
Name: "desktopicon"; Description: "{cm:CreateDesktopIcon}"; GroupDescription: "{cm:AdditionalIcons}"; Flags: unchecked
|
Name: "desktopicon"; Description: "{cm:CreateDesktopIcon}"; GroupDescription: "{cm:AdditionalIcons}"; Flags: unchecked
|
||||||
|
|
||||||
[Files]
|
[Files]
|
||||||
Source: ".\${{ matrix.arch == 'amd64' && 'x64' || 'arm64' }}\runner\Release\*"; DestDir: "{app}"; Flags: ignoreversion recursesubdirs;
|
Source: ".\x64\runner\Release\*"; DestDir: "{app}"; Flags: ignoreversion recursesubdirs
|
||||||
; NOTE: Don't use "Flags: ignoreversion" on any shared system files
|
; NOTE: Don't use "Flags: ignoreversion" on any shared system files
|
||||||
|
|
||||||
[UninstallDelete]
|
[UninstallDelete]
|
||||||
@@ -199,47 +125,42 @@ jobs:
|
|||||||
Filename: "{app}\{#MyAppExeName}"; Description: "{cm:LaunchProgram,{#StringChange(MyAppName, '&', '&&')}}"; Flags: nowait postinstall skipifsilent
|
Filename: "{app}\{#MyAppExeName}"; Description: "{cm:LaunchProgram,{#StringChange(MyAppName, '&', '&&')}}"; Flags: nowait postinstall skipifsilent
|
||||||
"@ > setup.iss
|
"@ > setup.iss
|
||||||
iscc.exe setup.iss
|
iscc.exe setup.iss
|
||||||
mv "Output\gopeed.exe" "Output\Gopeed-$TAG-windows-${{ matrix.arch }}.exe"
|
mv "Output\gopeed.exe" "Output\Gopeed-$TAG-windows-amd64.exe"
|
||||||
Compress-Archive -Path "Output\Gopeed-$TAG-windows-${{ matrix.arch }}.exe" -DestinationPath "Output\Gopeed-$TAG-windows-${{ matrix.arch }}.zip"
|
Compress-Archive -Path "Output\Gopeed-$TAG-windows-amd64.exe" -DestinationPath "Output\Gopeed-$TAG-windows-amd64.zip"
|
||||||
|
Remove-Item -Path "Output\Gopeed-$TAG-windows-amd64.exe"
|
||||||
- name: Upload
|
- name: Upload
|
||||||
uses: shogo82148/actions-upload-release-asset@v1.9.2
|
uses: shogo82148/actions-upload-release-asset@v1
|
||||||
with:
|
with:
|
||||||
upload_url: ${{ needs.get-release.outputs.upload_url }}
|
upload_url: ${{ needs.get-release.outputs.upload_url }}
|
||||||
asset_path: ui/flutter/build/windows/Output/*
|
asset_path: ui/flutter/build/windows/Output/*
|
||||||
overwrite: true
|
overwrite: true
|
||||||
build-macos-amd64-lib:
|
build-macos-arm64-lib:
|
||||||
if: ${{ github.event.inputs.platform == 'all' || github.event.inputs.platform == 'macos' }}
|
if: ${{ github.event.inputs.platform == 'all' || github.event.inputs.platform == 'macos' }}
|
||||||
runs-on: macos-15-intel
|
runs-on: ubuntu-latest
|
||||||
needs: [get-release]
|
needs: [ get-release ]
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v3
|
- uses: actions/checkout@v3
|
||||||
- uses: actions/setup-go@v5
|
- uses: actions/setup-go@v4
|
||||||
with:
|
with:
|
||||||
go-version: ${{ env.GO_VERSION }}
|
go-version: ${{ env.GO_VERSION }}
|
||||||
- name: Build
|
- name: Build
|
||||||
env:
|
env:
|
||||||
VERSION: ${{ needs.get-release.outputs.version }}
|
VERSION: ${{ needs.get-release.outputs.version }}
|
||||||
run: |
|
run: |
|
||||||
go build -tags nosqlite -ldflags="-w -s -X github.com/GopeedLab/gopeed/pkg/base.Version=$VERSION" -buildmode=c-shared -o libgopeed.dylib github.com/GopeedLab/gopeed/bind/desktop
|
go install src.techknowlogick.com/xgo@latest
|
||||||
go build -ldflags="-w -s" github.com/GopeedLab/gopeed/cmd/host
|
xgo -go go-$GO_VERSION.x --targets=darwin/arm64 -tags="nosqlite" -ldflags="-w -s -X github.com/GopeedLab/gopeed/pkg/base.Version=$VERSION" -buildmode=c-shared -pkg=bind/desktop -out=libgopeed .
|
||||||
go build -ldflags="-w -s" github.com/GopeedLab/gopeed/cmd/updater
|
mv libgopeed-*.dylib libgopeed.dylib
|
||||||
- uses: actions/upload-artifact@v4
|
- uses: actions/upload-artifact@v3
|
||||||
with:
|
with:
|
||||||
name: macos-amd64-lib
|
name: macos-arm64-lib
|
||||||
path: |
|
path: libgopeed.dylib
|
||||||
libgopeed.dylib
|
|
||||||
host
|
|
||||||
updater
|
|
||||||
build-macos:
|
build-macos:
|
||||||
if: ${{ github.event.inputs.platform == 'all' || github.event.inputs.platform == 'macos' }}
|
if: ${{ github.event.inputs.platform == 'all' || github.event.inputs.platform == 'macos' }}
|
||||||
runs-on: macos-latest
|
runs-on: macos-latest
|
||||||
strategy:
|
needs: [ get-release, build-macos-arm64-lib ]
|
||||||
matrix:
|
|
||||||
channel: [arm64, amd64, universal]
|
|
||||||
needs: [get-release, build-macos-amd64-lib]
|
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v3
|
- uses: actions/checkout@v3
|
||||||
- uses: actions/setup-go@v5
|
- uses: actions/setup-go@v4
|
||||||
with:
|
with:
|
||||||
go-version: ${{ env.GO_VERSION }}
|
go-version: ${{ env.GO_VERSION }}
|
||||||
- uses: actions/setup-node@v3
|
- uses: actions/setup-node@v3
|
||||||
@@ -250,51 +171,29 @@ jobs:
|
|||||||
flutter-version: ${{ env.FLUTTER_VERSION }}
|
flutter-version: ${{ env.FLUTTER_VERSION }}
|
||||||
- name: Install appdmg
|
- name: Install appdmg
|
||||||
run: |
|
run: |
|
||||||
python3 -m pip install setuptools --break-system-packages
|
python3 -m pip install setuptools
|
||||||
npm install -g appdmg
|
npm install -g appdmg
|
||||||
- uses: actions/download-artifact@v4
|
- uses: actions/download-artifact@v3
|
||||||
with:
|
with:
|
||||||
name: macos-amd64-lib
|
name: macos-arm64-lib
|
||||||
path: ui/flutter/lib-amd64
|
path: ui/flutter/lib-arm64
|
||||||
- if: ${{ matrix.channel == 'arm64' }}
|
|
||||||
name: Build Arm64 Libraries
|
|
||||||
run: |
|
|
||||||
go build -tags nosqlite -ldflags="-w -s -X github.com/GopeedLab/gopeed/pkg/base.Version=$VERSION" -buildmode=c-shared -o ui/flutter/macos/Frameworks/libgopeed.dylib github.com/GopeedLab/gopeed/bind/desktop
|
|
||||||
go build -ldflags="-w -s" -o ui/flutter/assets/exec/host github.com/GopeedLab/gopeed/cmd/host
|
|
||||||
go build -ldflags="-w -s" -o ui/flutter/assets/exec/updater github.com/GopeedLab/gopeed/cmd/updater
|
|
||||||
- if: ${{ matrix.channel == 'amd64' }}
|
|
||||||
name: Build Amd64 Libraries
|
|
||||||
run: |
|
|
||||||
mkdir -p ui/flutter/macos/Frameworks
|
|
||||||
|
|
||||||
cp ui/flutter/lib-amd64/libgopeed.dylib ui/flutter/macos/Frameworks/libgopeed.dylib
|
|
||||||
cp ui/flutter/lib-amd64/host ui/flutter/assets/exec/host
|
|
||||||
cp ui/flutter/lib-amd64/updater ui/flutter/assets/exec/updater
|
|
||||||
- if: ${{ matrix.channel == 'universal' }}
|
|
||||||
name: Build Universal Libraries
|
|
||||||
run: |
|
|
||||||
mkdir -p ui/flutter/macos/Frameworks
|
|
||||||
|
|
||||||
# arm64 lib
|
|
||||||
go build -tags nosqlite -ldflags="-w -s -X github.com/GopeedLab/gopeed/pkg/base.Version=$VERSION" -buildmode=c-shared -o ui/flutter/.temp/libgopeed.dylib.arm64 github.com/GopeedLab/gopeed/bind/desktop
|
|
||||||
go build -ldflags="-w -s" -o ui/flutter/.temp/host.arm64 github.com/GopeedLab/gopeed/cmd/host
|
|
||||||
go build -ldflags="-w -s" -o ui/flutter/.temp/updater.arm64 github.com/GopeedLab/gopeed/cmd/updater
|
|
||||||
|
|
||||||
# amd64 lib
|
|
||||||
cp ui/flutter/lib-amd64/libgopeed.dylib ui/flutter/.temp/libgopeed.dylib.amd64
|
|
||||||
cp ui/flutter/lib-amd64/host ui/flutter/.temp/host.amd64
|
|
||||||
cp ui/flutter/lib-amd64/updater ui/flutter/.temp/updater.amd64
|
|
||||||
|
|
||||||
# universal lib
|
|
||||||
lipo -create -output ui/flutter/macos/Frameworks/libgopeed.dylib ui/flutter/.temp/libgopeed.dylib.arm64 ui/flutter/.temp/libgopeed.dylib.amd64
|
|
||||||
lipo -create -output ui/flutter/assets/exec/host ui/flutter/.temp/host.arm64 ui/flutter/.temp/host.amd64
|
|
||||||
lipo -create -output ui/flutter/assets/exec/updater ui/flutter/.temp/updater.arm64 ui/flutter/.temp/updater.amd64
|
|
||||||
- name: Build
|
- name: Build
|
||||||
env:
|
env:
|
||||||
VERSION: ${{ needs.get-release.outputs.version }}
|
VERSION: ${{ needs.get-release.outputs.version }}
|
||||||
run: |
|
run: |
|
||||||
cd ui/flutter
|
PROJECT_DIR=$(pwd)
|
||||||
flutter build macos --dart-define="UPDATE_CHANNEL=macosDmg" --dart-define="GA4_MEASUREMENT_ID=${{ env.GA4_MEASUREMENT_ID }}" --dart-define="GA4_API_SECRET=${{ env.GA4_API_SECRET }}"
|
|
||||||
|
# amd64 lib
|
||||||
|
go build -tags nosqlite -ldflags="-w -s -X github.com/GopeedLab/gopeed/pkg/base.Version=$VERSION" -buildmode=c-shared -o ui/flutter/lib-amd64/libgopeed.dylib github.com/GopeedLab/gopeed/bind/desktop
|
||||||
|
# universal binary
|
||||||
|
mkdir -p ui/flutter/macos/Frameworks
|
||||||
|
cp ui/flutter/lib-amd64/libgopeed.dylib ui/flutter/macos/Frameworks/amd64-lib
|
||||||
|
cp ui/flutter/lib-arm64/libgopeed.dylib ui/flutter/macos/Frameworks/arm64-lib
|
||||||
|
cd ui/flutter/macos/Frameworks
|
||||||
|
lipo -create -output libgopeed.dylib amd64-lib arm64-lib
|
||||||
|
|
||||||
|
cd $PROJECT_DIR/ui/flutter
|
||||||
|
flutter build macos
|
||||||
cd build/macos/Build/Products/Release
|
cd build/macos/Build/Products/Release
|
||||||
cat>appdmg.json<<EOF
|
cat>appdmg.json<<EOF
|
||||||
{
|
{
|
||||||
@@ -308,13 +207,6 @@ jobs:
|
|||||||
EOF
|
EOF
|
||||||
mkdir dist
|
mkdir dist
|
||||||
appdmg appdmg.json dist/Gopeed-v$VERSION-macos.dmg
|
appdmg appdmg.json dist/Gopeed-v$VERSION-macos.dmg
|
||||||
|
|
||||||
if [[ "${{ matrix.channel }}" == "arm64" ]]; then
|
|
||||||
mv dist/Gopeed-v$VERSION-macos.dmg dist/Gopeed-v$VERSION-macos-arm64.dmg
|
|
||||||
fi
|
|
||||||
if [[ "${{ matrix.channel }}" == "amd64" ]]; then
|
|
||||||
mv dist/Gopeed-v$VERSION-macos.dmg dist/Gopeed-v$VERSION-macos-amd64.dmg
|
|
||||||
fi
|
|
||||||
- name: Upload
|
- name: Upload
|
||||||
uses: shogo82148/actions-upload-release-asset@v1
|
uses: shogo82148/actions-upload-release-asset@v1
|
||||||
with:
|
with:
|
||||||
@@ -323,25 +215,23 @@ jobs:
|
|||||||
overwrite: true
|
overwrite: true
|
||||||
build-linux:
|
build-linux:
|
||||||
if: ${{ github.event.inputs.platform == 'all' || github.event.inputs.platform == 'linux' }}
|
if: ${{ github.event.inputs.platform == 'all' || github.event.inputs.platform == 'linux' }}
|
||||||
runs-on: ${{ matrix.os }}
|
runs-on: ubuntu-latest
|
||||||
strategy:
|
needs: [ get-release ]
|
||||||
matrix:
|
|
||||||
os: [ubuntu-22.04, ubuntu-22.04-arm]
|
|
||||||
needs: [get-release]
|
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v3
|
- uses: actions/checkout@v3
|
||||||
- uses: actions/setup-go@v5
|
- uses: actions/setup-go@v4
|
||||||
with:
|
with:
|
||||||
go-version: ${{ env.GO_VERSION }}
|
go-version: ${{ env.GO_VERSION }}
|
||||||
- uses: subosito/flutter-action@v2
|
- uses: subosito/flutter-action@v2
|
||||||
with:
|
with:
|
||||||
flutter-version: ${{ env.FLUTTER_VERSION }}
|
flutter-version: ${{ env.FLUTTER_VERSION }}
|
||||||
channel: master
|
|
||||||
- run: |
|
- run: |
|
||||||
sudo apt update -y
|
sudo apt update -y
|
||||||
sudo apt install -y ninja-build libgtk-3-dev libwebkit2gtk-4.1-dev libayatana-appindicator3-1 libayatana-appindicator3-dev rpm patchelf libfuse2 locate libkeybinder-3.0-dev
|
# flutter build dependencies
|
||||||
arch=$(uname -m)
|
sudo apt install -y ninja-build libgtk-3-dev libayatana-appindicator3-1 libayatana-appindicator3-dev
|
||||||
wget -O appimagetool "https://github.com/AppImage/appimagetool/releases/download/continuous/appimagetool-${arch}.AppImage"
|
# appimage build dependencies
|
||||||
|
sudo apt install -y libfuse2 locate
|
||||||
|
wget -O appimagetool "https://github.com/AppImage/AppImageKit/releases/download/continuous/appimagetool-x86_64.AppImage"
|
||||||
chmod +x appimagetool
|
chmod +x appimagetool
|
||||||
sudo mv appimagetool /usr/local/bin/
|
sudo mv appimagetool /usr/local/bin/
|
||||||
- name: Build
|
- name: Build
|
||||||
@@ -349,71 +239,12 @@ jobs:
|
|||||||
VERSION: ${{ needs.get-release.outputs.version }}
|
VERSION: ${{ needs.get-release.outputs.version }}
|
||||||
run: |
|
run: |
|
||||||
go build -tags nosqlite -ldflags="-w -s -X github.com/GopeedLab/gopeed/pkg/base.Version=$VERSION" -buildmode=c-shared -o ui/flutter/linux/bundle/lib/libgopeed.so github.com/GopeedLab/gopeed/bind/desktop
|
go build -tags nosqlite -ldflags="-w -s -X github.com/GopeedLab/gopeed/pkg/base.Version=$VERSION" -buildmode=c-shared -o ui/flutter/linux/bundle/lib/libgopeed.so github.com/GopeedLab/gopeed/bind/desktop
|
||||||
go build -ldflags="-w -s" -o ui/flutter/assets/exec/host github.com/GopeedLab/gopeed/cmd/host
|
|
||||||
go build -ldflags="-w -s" -o ui/flutter/assets/exec/updater github.com/GopeedLab/gopeed/cmd/updater
|
|
||||||
|
|
||||||
cd ui/flutter
|
cd ui/flutter
|
||||||
|
|
||||||
RPM_ARCH=$(uname -m)
|
|
||||||
mkdir -p linux/packaging/rpm
|
|
||||||
cat << EOF > linux/packaging/rpm/make_config.yaml
|
|
||||||
display_name: Gopeed
|
|
||||||
icon: assets/icon/icon.svg
|
|
||||||
summary: High speed downloader.
|
|
||||||
description: A high speed downloader that supports all platforms.
|
|
||||||
group: Applications/Internet
|
|
||||||
license: GPL-3.0
|
|
||||||
url: https://github.com/GopeedLab/gopeed
|
|
||||||
vendor: GopeedLab
|
|
||||||
maintainer: GopeedLab <[email protected]>
|
|
||||||
build_arch: $RPM_ARCH
|
|
||||||
EOF
|
|
||||||
|
|
||||||
cat << EOF > distribute_options.yaml
|
|
||||||
output: dist/
|
|
||||||
releases:
|
|
||||||
- name: linux
|
|
||||||
jobs:
|
|
||||||
- name: appimage
|
|
||||||
package:
|
|
||||||
platform: linux
|
|
||||||
target: appimage
|
|
||||||
build_args:
|
|
||||||
dart-define:
|
|
||||||
UPDATE_CHANNEL: linuxAppImage
|
|
||||||
GA4_MEASUREMENT_ID: ${{ env.GA4_MEASUREMENT_ID }}
|
|
||||||
GA4_API_SECRET: ${{ env.GA4_API_SECRET }}
|
|
||||||
- name: deb
|
|
||||||
package:
|
|
||||||
platform: linux
|
|
||||||
target: deb
|
|
||||||
build_args:
|
|
||||||
dart-define:
|
|
||||||
UPDATE_CHANNEL: linuxDeb
|
|
||||||
GA4_MEASUREMENT_ID: ${{ env.GA4_MEASUREMENT_ID }}
|
|
||||||
GA4_API_SECRET: ${{ env.GA4_API_SECRET }}
|
|
||||||
- name: rpm
|
|
||||||
package:
|
|
||||||
platform: linux
|
|
||||||
target: rpm
|
|
||||||
build_args:
|
|
||||||
dart-define:
|
|
||||||
UPDATE_CHANNEL: linuxRpm
|
|
||||||
GA4_MEASUREMENT_ID: ${{ env.GA4_MEASUREMENT_ID }}
|
|
||||||
GA4_API_SECRET: ${{ env.GA4_API_SECRET }}
|
|
||||||
EOF
|
|
||||||
dart pub global activate -sgit https://github.com/GopeedLab/flutter_distributor.git --git-path packages/flutter_distributor
|
dart pub global activate -sgit https://github.com/GopeedLab/flutter_distributor.git --git-path packages/flutter_distributor
|
||||||
flutter_distributor release --name linux
|
flutter_distributor package --platform linux --targets appimage,deb
|
||||||
cd dist/*
|
cd dist/*
|
||||||
|
mv gopeed-*-linux.AppImage Gopeed-v${VERSION}-linux-x86_64.AppImage
|
||||||
ARCH="amd64"
|
mv gopeed-*-linux.deb Gopeed-v${VERSION}-linux-amd64.deb
|
||||||
if [[ "${{ matrix.os }}" == *-arm ]]; then
|
|
||||||
ARCH="arm64"
|
|
||||||
fi
|
|
||||||
|
|
||||||
mv gopeed-*-linux.AppImage Gopeed-v${VERSION}-linux-${ARCH}.AppImage
|
|
||||||
mv gopeed-*-linux.deb Gopeed-v${VERSION}-linux-${ARCH}.deb
|
|
||||||
mv gopeed-*-linux.rpm Gopeed-v${VERSION}-linux-${ARCH}.rpm
|
|
||||||
- name: Upload
|
- name: Upload
|
||||||
uses: shogo82148/actions-upload-release-asset@v1
|
uses: shogo82148/actions-upload-release-asset@v1
|
||||||
with:
|
with:
|
||||||
@@ -422,17 +253,16 @@ jobs:
|
|||||||
overwrite: true
|
overwrite: true
|
||||||
build-snap:
|
build-snap:
|
||||||
if: ${{ github.event.inputs.platform == 'all' || github.event.inputs.platform == 'snap' }}
|
if: ${{ github.event.inputs.platform == 'all' || github.event.inputs.platform == 'snap' }}
|
||||||
runs-on: ubuntu-24.04
|
runs-on: ubuntu-latest
|
||||||
needs: [get-release]
|
needs: [ get-release ]
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v3
|
- uses: actions/checkout@v3
|
||||||
- uses: actions/setup-go@v5
|
- uses: actions/setup-go@v4
|
||||||
with:
|
with:
|
||||||
go-version: ${{ env.GO_VERSION }}
|
go-version: ${{ env.GO_VERSION }}
|
||||||
- name: Install Snap host build dependencies
|
- uses: subosito/flutter-action@v2
|
||||||
run: |
|
with:
|
||||||
sudo apt-get update
|
flutter-version: ${{ env.FLUTTER_VERSION }}
|
||||||
sudo apt-get install -y --no-install-recommends pkg-config libgtk-3-dev libwebkit2gtk-4.1-dev
|
|
||||||
- name: Setup LXD
|
- name: Setup LXD
|
||||||
uses: canonical/[email protected]
|
uses: canonical/[email protected]
|
||||||
with:
|
with:
|
||||||
@@ -442,124 +272,53 @@ jobs:
|
|||||||
VERSION: ${{ needs.get-release.outputs.version }}
|
VERSION: ${{ needs.get-release.outputs.version }}
|
||||||
run: |
|
run: |
|
||||||
go build -tags nosqlite -ldflags="-w -s -X github.com/GopeedLab/gopeed/pkg/base.Version=$VERSION" -buildmode=c-shared -o ui/flutter/linux/bundle/lib/libgopeed.so github.com/GopeedLab/gopeed/bind/desktop
|
go build -tags nosqlite -ldflags="-w -s -X github.com/GopeedLab/gopeed/pkg/base.Version=$VERSION" -buildmode=c-shared -o ui/flutter/linux/bundle/lib/libgopeed.so github.com/GopeedLab/gopeed/bind/desktop
|
||||||
go build -ldflags="-w -s" -o ui/flutter/assets/exec/host github.com/GopeedLab/gopeed/cmd/host
|
|
||||||
go build -ldflags="-w -s" -o ui/flutter/assets/exec/updater github.com/GopeedLab/gopeed/cmd/updater
|
|
||||||
cd ui/flutter
|
cd ui/flutter
|
||||||
|
|
||||||
sudo snap install snapcraft --classic
|
sudo snap install snapcraft --classic
|
||||||
mkdir -p snap/gui
|
mkdir -p snap/gui
|
||||||
cp assets/icon/icon.svg snap/gui/gopeed.svg
|
cp assets/icon/icon.svg snap/gui/gopeed.svg
|
||||||
|
|
||||||
cat>snap/snapcraft.yaml<<EOF
|
cat>snap/snapcraft.yaml<<EOF
|
||||||
name: gopeed
|
name: gopeed
|
||||||
version: $VERSION
|
version: $VERSION
|
||||||
summary: A modern download manager
|
summary: A modern download manager
|
||||||
description: High speed downloader that supports all platforms.
|
description: High speed downloader that supports all platforms.
|
||||||
|
|
||||||
confinement: strict
|
confinement: strict
|
||||||
base: core24
|
base: core22
|
||||||
grade: stable
|
grade: stable
|
||||||
|
|
||||||
platforms:
|
architectures:
|
||||||
amd64:
|
- build-on: amd64
|
||||||
build-on: amd64
|
|
||||||
build-for: amd64
|
build-for: amd64
|
||||||
|
|
||||||
slots:
|
slots:
|
||||||
dbus-gopeed:
|
dbus-gopeed:
|
||||||
interface: dbus
|
interface: dbus
|
||||||
bus: session
|
bus: session
|
||||||
name: com.gopeed.gopeed
|
name: com.gopeed.gopeed
|
||||||
|
|
||||||
apps:
|
apps:
|
||||||
gopeed:
|
gopeed:
|
||||||
command-chain:
|
command: gopeed
|
||||||
- bin/gpu-2404-wrapper
|
|
||||||
command: bin/gopeed
|
|
||||||
extensions: [gnome]
|
extensions: [gnome]
|
||||||
common-id: com.gopeed.Gopeed
|
|
||||||
plugs:
|
plugs:
|
||||||
- network
|
- network
|
||||||
- home
|
slots:
|
||||||
|
- dbus-gopeed
|
||||||
parts:
|
parts:
|
||||||
flutter-git:
|
|
||||||
source: https://github.com/flutter/flutter.git
|
|
||||||
source-tag: $FLUTTER_VERSION
|
|
||||||
source-depth: 1
|
|
||||||
plugin: nil
|
|
||||||
override-build: |
|
|
||||||
mkdir -p \$CRAFT_PART_INSTALL/usr/bin
|
|
||||||
mkdir -p \$CRAFT_PART_INSTALL/usr/libexec
|
|
||||||
cp -r \$CRAFT_PART_SRC \$CRAFT_PART_INSTALL/usr/libexec/flutter
|
|
||||||
ln -s \$CRAFT_PART_INSTALL/usr/libexec/flutter/bin/flutter \$CRAFT_PART_INSTALL/usr/bin/flutter
|
|
||||||
ln -s \$SNAPCRAFT_PART_INSTALL/usr/libexec/flutter/bin/dart \$SNAPCRAFT_PART_INSTALL/usr/bin/dart
|
|
||||||
\$CRAFT_PART_INSTALL/usr/bin/flutter doctor
|
|
||||||
build-packages:
|
|
||||||
- clang
|
|
||||||
- llvm-18
|
|
||||||
- lld
|
|
||||||
- cmake
|
|
||||||
- curl
|
|
||||||
- libgtk-3-dev
|
|
||||||
- libwebkit2gtk-4.1-dev
|
|
||||||
- libkeybinder-3.0-dev
|
|
||||||
- ninja-build
|
|
||||||
- unzip
|
|
||||||
- xz-utils
|
|
||||||
- zip
|
|
||||||
override-prime: ''
|
|
||||||
zenity:
|
|
||||||
plugin: nil
|
|
||||||
stage-packages:
|
|
||||||
- zenity
|
|
||||||
gopeed:
|
gopeed:
|
||||||
after: [flutter-git,zenity]
|
|
||||||
source: .
|
source: .
|
||||||
plugin: nil
|
plugin: flutter
|
||||||
stage-packages:
|
stage-packages:
|
||||||
- libwebkit2gtk-4.1-0
|
|
||||||
- libgtk-3-dev
|
- libgtk-3-dev
|
||||||
- libkeybinder-3.0-dev
|
|
||||||
- libappindicator3-dev
|
- libappindicator3-dev
|
||||||
override-build: |
|
flutter-target: lib/main.dart
|
||||||
# work around pub get stack overflow # https://github.com/dart-lang/sdk/issues/51068#issuecomment-1396588253
|
|
||||||
set +e
|
|
||||||
dart pub get
|
|
||||||
set -eux
|
|
||||||
llvm_bin_dir="\$(dirname "\$(readlink -f "\$(command -v clang)")")"
|
|
||||||
test -x "\$llvm_bin_dir/llvm-ar"
|
|
||||||
test -x "\$llvm_bin_dir/ld.lld"
|
|
||||||
flutter build linux --dart-define="UPDATE_CHANNEL=linuxSnap" --dart-define="GA4_MEASUREMENT_ID=$GA4_MEASUREMENT_ID" --dart-define="GA4_API_SECRET=$GA4_API_SECRET"
|
|
||||||
mkdir -p \$CRAFT_PART_INSTALL/bin/
|
|
||||||
cp -r build/linux/*/release/bundle/* \$CRAFT_PART_INSTALL/bin/
|
|
||||||
gpu-2404:
|
|
||||||
after: [gopeed]
|
|
||||||
source: https://github.com/canonical/gpu-snap.git
|
|
||||||
plugin: dump
|
|
||||||
override-prime: |
|
|
||||||
craftctl default
|
|
||||||
\${CRAFT_PART_SRC}/bin/gpu-2404-cleanup mesa-2404
|
|
||||||
prime:
|
|
||||||
- bin/gpu-2404-wrapper
|
|
||||||
|
|
||||||
plugs:
|
|
||||||
gpu-2404:
|
|
||||||
interface: content
|
|
||||||
target: \$SNAP/gpu-2404
|
|
||||||
default-provider: mesa-2404
|
|
||||||
|
|
||||||
layout:
|
|
||||||
/usr/share/libdrm:
|
|
||||||
bind: \$SNAP/gpu-2404/libdrm
|
|
||||||
/usr/share/drirc.d:
|
|
||||||
symlink: \$SNAP/gpu-2404/drirc.d
|
|
||||||
/usr/share/X11/XErrorDB:
|
|
||||||
symlink: \$SNAP/gpu-2404/X11/XErrorDB
|
|
||||||
EOF
|
EOF
|
||||||
|
|
||||||
cp linux/assets/com.gopeed.Gopeed.desktop snap/gui/gopeed.desktop
|
cp linux/assets/com.gopeed.Gopeed.desktop snap/gui/gopeed.desktop
|
||||||
sed -i 's/Icon=com.gopeed.Gopeed/Icon=\${SNAP}\/meta\/gui\/gopeed.svg/g' snap/gui/gopeed.desktop
|
sed -i 's/Icon=com.gopeed.Gopeed/Icon=\${SNAP}\/meta\/gui\/gopeed.svg/g' snap/gui/gopeed.desktop
|
||||||
|
|
||||||
snapcraft --use-lxd
|
snapcraft --use-lxd
|
||||||
|
|
||||||
# Snapcraft login
|
# Snapcraft login
|
||||||
@@ -568,16 +327,16 @@ jobs:
|
|||||||
build-android:
|
build-android:
|
||||||
if: ${{ github.event.inputs.platform == 'all' || github.event.inputs.platform == 'android' }}
|
if: ${{ github.event.inputs.platform == 'all' || github.event.inputs.platform == 'android' }}
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
needs: [get-release]
|
needs: [ get-release ]
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v3
|
- uses: actions/checkout@v3
|
||||||
- uses: actions/setup-go@v5
|
- uses: actions/setup-go@v4
|
||||||
with:
|
with:
|
||||||
go-version: ${{ env.GO_VERSION }}
|
go-version: ${{ env.GO_VERSION }}
|
||||||
- uses: actions/setup-java@v3
|
- uses: actions/setup-java@v3
|
||||||
with:
|
with:
|
||||||
distribution: "zulu"
|
distribution: "zulu"
|
||||||
java-version: "17"
|
java-version: "11"
|
||||||
- uses: subosito/flutter-action@v2
|
- uses: subosito/flutter-action@v2
|
||||||
with:
|
with:
|
||||||
flutter-version: ${{ env.FLUTTER_VERSION }}
|
flutter-version: ${{ env.FLUTTER_VERSION }}
|
||||||
@@ -591,16 +350,12 @@ jobs:
|
|||||||
go install golang.org/x/mobile/cmd/gomobile@latest
|
go install golang.org/x/mobile/cmd/gomobile@latest
|
||||||
go get golang.org/x/mobile/bind
|
go get golang.org/x/mobile/bind
|
||||||
gomobile init
|
gomobile init
|
||||||
gomobile bind -tags nosqlite -ldflags="-w -s -checklinkname=0 -X github.com/GopeedLab/gopeed/pkg/base.Version=$VERSION" -o ui/flutter/android/app/libs/libgopeed.aar -target=android -androidapi 21 -javapkg=com.gopeed github.com/GopeedLab/gopeed/bind/mobile
|
gomobile bind -tags nosqlite -ldflags="-w -s -X github.com/GopeedLab/gopeed/pkg/base.Version=$VERSION" -o ui/flutter/android/app/libs/libgopeed.aar -target=android -androidapi 19 -javapkg=com.gopeed github.com/GopeedLab/gopeed/bind/mobile
|
||||||
cd ui/flutter
|
cd ui/flutter
|
||||||
echo $APK_KEYSTORE | base64 -di > android/app/upload-keystore.jks
|
echo $APK_KEYSTORE | base64 -di > android/app/upload-keystore.jks
|
||||||
flutter build apk --dart-define="UPDATE_CHANNEL=androidApk" --dart-define="GA4_MEASUREMENT_ID=${{ env.GA4_MEASUREMENT_ID }}" --dart-define="GA4_API_SECRET=${{ env.GA4_API_SECRET }}"
|
flutter build apk
|
||||||
flutter build apk --split-per-abi --dart-define="UPDATE_CHANNEL=androidApk" --dart-define="GA4_MEASUREMENT_ID=${{ env.GA4_MEASUREMENT_ID }}" --dart-define="GA4_API_SECRET=${{ env.GA4_API_SECRET }}"
|
|
||||||
mkdir dist
|
mkdir dist
|
||||||
cp build/app/outputs/flutter-apk/app-release.apk dist/Gopeed-v$VERSION-android.apk
|
cp build/app/outputs/flutter-apk/app-release.apk dist/Gopeed-v$VERSION-android.apk
|
||||||
cp build/app/outputs/flutter-apk/app-armeabi-v7a-release.apk dist/Gopeed-v$VERSION-android-armeabi-v7a.apk
|
|
||||||
cp build/app/outputs/flutter-apk/app-arm64-v8a-release.apk dist/Gopeed-v$VERSION-android-arm64-v8a.apk
|
|
||||||
cp build/app/outputs/flutter-apk/app-x86_64-release.apk dist/Gopeed-v$VERSION-android-x86_64.apk
|
|
||||||
- name: Upload
|
- name: Upload
|
||||||
uses: shogo82148/actions-upload-release-asset@v1
|
uses: shogo82148/actions-upload-release-asset@v1
|
||||||
with:
|
with:
|
||||||
@@ -609,11 +364,11 @@ jobs:
|
|||||||
overwrite: true
|
overwrite: true
|
||||||
build-ios:
|
build-ios:
|
||||||
if: ${{ github.event.inputs.platform == 'all' || github.event.inputs.platform == 'ios' }}
|
if: ${{ github.event.inputs.platform == 'all' || github.event.inputs.platform == 'ios' }}
|
||||||
runs-on: macos-26
|
runs-on: macos-latest
|
||||||
needs: [get-release]
|
needs: [ get-release ]
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v3
|
- uses: actions/checkout@v3
|
||||||
- uses: actions/setup-go@v5
|
- uses: actions/setup-go@v4
|
||||||
with:
|
with:
|
||||||
go-version: ${{ env.GO_VERSION }}
|
go-version: ${{ env.GO_VERSION }}
|
||||||
- uses: subosito/flutter-action@v2
|
- uses: subosito/flutter-action@v2
|
||||||
@@ -626,9 +381,9 @@ jobs:
|
|||||||
go install golang.org/x/mobile/cmd/gomobile@latest
|
go install golang.org/x/mobile/cmd/gomobile@latest
|
||||||
go get golang.org/x/mobile/bind
|
go get golang.org/x/mobile/bind
|
||||||
gomobile init
|
gomobile init
|
||||||
gomobile bind -tags nosqlite -ldflags="-w -s -X github.com/GopeedLab/gopeed/pkg/base.Version=$VERSION" -o ui/flutter/ios/Frameworks/Libgopeed.xcframework -target=ios github.com/GopeedLab/gopeed/bind/mobile
|
gomobile bind -tags nosqlite,netgo -ldflags="-w -s -X github.com/GopeedLab/gopeed/pkg/base.Version=$VERSION" -o ui/flutter/ios/Frameworks/Libgopeed.xcframework -target=ios github.com/GopeedLab/gopeed/bind/mobile
|
||||||
cd ui/flutter
|
cd ui/flutter
|
||||||
flutter build ios --no-codesign --dart-define="UPDATE_CHANNEL=iosIpa" --dart-define="GA4_MEASUREMENT_ID=${{ env.GA4_MEASUREMENT_ID }}" --dart-define="GA4_API_SECRET=${{ env.GA4_API_SECRET }}"
|
flutter build ios --no-codesign
|
||||||
mkdir Payload
|
mkdir Payload
|
||||||
cp -r build/ios/iphoneos/Runner.app Payload
|
cp -r build/ios/iphoneos/Runner.app Payload
|
||||||
zip -r -y Payload.zip Payload/Runner.app
|
zip -r -y Payload.zip Payload/Runner.app
|
||||||
@@ -641,12 +396,12 @@ jobs:
|
|||||||
asset_path: ui/flutter/dist/*
|
asset_path: ui/flutter/dist/*
|
||||||
overwrite: true
|
overwrite: true
|
||||||
build-web:
|
build-web:
|
||||||
if: ${{ github.event.inputs.platform == 'all' || github.event.inputs.platform == 'web' || github.event.inputs.platform == 'qnap' }}
|
if: ${{ github.event.inputs.platform == 'all' || github.event.inputs.platform == 'web' }}
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
needs: [get-release]
|
needs: [ get-release ]
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v3
|
- uses: actions/checkout@v3
|
||||||
- uses: actions/setup-go@v5
|
- uses: actions/setup-go@v4
|
||||||
with:
|
with:
|
||||||
go-version: ${{ env.GO_VERSION }}
|
go-version: ${{ env.GO_VERSION }}
|
||||||
- uses: subosito/flutter-action@v2
|
- uses: subosito/flutter-action@v2
|
||||||
@@ -657,12 +412,11 @@ jobs:
|
|||||||
VERSION: ${{ needs.get-release.outputs.version }}
|
VERSION: ${{ needs.get-release.outputs.version }}
|
||||||
run: |
|
run: |
|
||||||
cd ui/flutter
|
cd ui/flutter
|
||||||
flutter build web --no-web-resources-cdn --dart-define="GA4_MEASUREMENT_ID=${{ env.GA4_MEASUREMENT_ID }}" --dart-define="GA4_API_SECRET=${{ env.GA4_API_SECRET }}"
|
flutter build web --web-renderer html
|
||||||
dart ../../.github/workflows/scripts/flutter_local_font.dart
|
|
||||||
cd ../../
|
cd ../../
|
||||||
cp -r ui/flutter/build/web cmd/web/dist
|
cp -r ui/flutter/build/web cmd/web/dist
|
||||||
mkdir -p dist/zip
|
mkdir -p dist/zip
|
||||||
|
|
||||||
goos_arr=(windows darwin linux)
|
goos_arr=(windows darwin linux)
|
||||||
goarch_arr=(386 amd64 arm64)
|
goarch_arr=(386 amd64 arm64)
|
||||||
export CGO_ENABLED=0
|
export CGO_ENABLED=0
|
||||||
@@ -684,86 +438,21 @@ jobs:
|
|||||||
|| true
|
|| true
|
||||||
done
|
done
|
||||||
done
|
done
|
||||||
- uses: actions/upload-artifact@v4
|
|
||||||
with:
|
|
||||||
name: web-dist
|
|
||||||
path: dist/zip/
|
|
||||||
- name: Upload
|
- name: Upload
|
||||||
uses: shogo82148/actions-upload-release-asset@v1
|
uses: shogo82148/actions-upload-release-asset@v1
|
||||||
with:
|
with:
|
||||||
upload_url: ${{ needs.get-release.outputs.upload_url }}
|
upload_url: ${{ needs.get-release.outputs.upload_url }}
|
||||||
asset_path: dist/zip/*
|
asset_path: dist/zip/*
|
||||||
overwrite: true
|
overwrite: true
|
||||||
build-qnap:
|
|
||||||
if: ${{ github.event.inputs.platform == 'all' || github.event.inputs.platform == 'qnap' }}
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
needs: [get-release, build-web]
|
|
||||||
steps:
|
|
||||||
- uses: actions/setup-python@v5
|
|
||||||
with:
|
|
||||||
python-version: "3.8.18"
|
|
||||||
- uses: actions/download-artifact@v4
|
|
||||||
with:
|
|
||||||
name: web-dist
|
|
||||||
path: dist/zip/
|
|
||||||
- name: Build
|
|
||||||
env:
|
|
||||||
VERSION: ${{ needs.get-release.outputs.version }}
|
|
||||||
run: |
|
|
||||||
sudo apt update -y
|
|
||||||
qdk_version=2.5.3
|
|
||||||
qdk_package="qdk_${qdk_version}_amd64.deb"
|
|
||||||
curl --fail --location --retry 3 --retry-all-errors \
|
|
||||||
--output "${qdk_package}" \
|
|
||||||
"https://github.com/qnap-dev/QDK/releases/download/v${qdk_version}/${qdk_package}"
|
|
||||||
echo "17b3841b7d4590a4ee025844ba583304b5e3c497d9fa8934d5175131d3908022 ${qdk_package}" | sha256sum --check --strict
|
|
||||||
sudo apt-get install -y "./${qdk_package}"
|
|
||||||
command -v qbuild
|
|
||||||
|
|
||||||
wget -O Gopeed.template.tar.gz "https://github.com/GopeedLab/QpkgBuild/raw/refs/heads/master/template/Gopeed.template.tar.gz"
|
|
||||||
tar -zxf Gopeed.template.tar.gz
|
|
||||||
[[ -d Gopeed ]] || exit 1
|
|
||||||
|
|
||||||
goos=linux
|
|
||||||
goarch_arr=(amd64 arm64)
|
|
||||||
for goarch in "${goarch_arr[@]}"; do
|
|
||||||
qarch=x86_64
|
|
||||||
[[ "${goarch}" == "arm64" ]] && qarch=arm_64
|
|
||||||
name=gopeed-web-v${VERSION}-${goos}-${goarch}
|
|
||||||
unzip dist/zip/${name}.zip -d dist/${name}
|
|
||||||
cp dist/${name}/${name}/* Gopeed/${qarch}/
|
|
||||||
done
|
|
||||||
cd Gopeed
|
|
||||||
sed -i -e 's/__QPKG_VER__/${VERSION}/g' qpkg.cfg
|
|
||||||
qbuild || exit 1
|
|
||||||
|
|
||||||
mkdir -p ../dist/qnap
|
|
||||||
goos=qnap
|
|
||||||
for goarch in "${goarch_arr[@]}"; do
|
|
||||||
qarch=x86_64
|
|
||||||
[[ "${goarch}" == "arm64" ]] && qarch=arm_64
|
|
||||||
sname=Gopeed_${VERSION}_${qarch}.qpkg
|
|
||||||
dname=gopeed-v${VERSION}-${goos}-${goarch}.qpkg
|
|
||||||
[[ -f build/${sname} ]] && cp -ra build/${sname} ../dist/qnap/${dname}
|
|
||||||
done
|
|
||||||
- name: Upload
|
|
||||||
uses: shogo82148/actions-upload-release-asset@v1
|
|
||||||
with:
|
|
||||||
upload_url: ${{ needs.get-release.outputs.upload_url }}
|
|
||||||
asset_path: dist/qnap/*
|
|
||||||
overwrite: true
|
|
||||||
build-docker:
|
build-docker:
|
||||||
if: ${{ github.event.inputs.platform == 'all' || github.event.inputs.platform == 'docker' }}
|
if: ${{ github.event.inputs.platform == 'all' || github.event.inputs.platform == 'docker' }}
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
needs: [get-release]
|
needs: [ get-release ]
|
||||||
steps:
|
steps:
|
||||||
- name: Remove unnecessary files
|
- name: Remove unnecessary files
|
||||||
run: |
|
run: |
|
||||||
sudo rm -rf /usr/share/dotnet
|
sudo rm -rf /usr/share/dotnet
|
||||||
sudo rm -rf "$AGENT_TOOLSDIRECTORY"
|
sudo rm -rf "$AGENT_TOOLSDIRECTORY"
|
||||||
- uses: subosito/flutter-action@v2
|
|
||||||
with:
|
|
||||||
flutter-version: ${{ env.FLUTTER_VERSION }}
|
|
||||||
- uses: actions/checkout@v3
|
- uses: actions/checkout@v3
|
||||||
- name: Set up QEMU
|
- name: Set up QEMU
|
||||||
uses: docker/setup-qemu-action@v2
|
uses: docker/setup-qemu-action@v2
|
||||||
@@ -774,20 +463,11 @@ jobs:
|
|||||||
with:
|
with:
|
||||||
username: liwei2633
|
username: liwei2633
|
||||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||||
- name: Build flutter web
|
|
||||||
run: |
|
|
||||||
cd ui/flutter
|
|
||||||
flutter build web --no-web-resources-cdn --dart-define="UPDATE_CHANNEL=docker" --dart-define="GA4_MEASUREMENT_ID=${{ env.GA4_MEASUREMENT_ID }}" --dart-define="GA4_API_SECRET=${{ env.GA4_API_SECRET }}"
|
|
||||||
dart ../../.github/workflows/scripts/flutter_local_font.dart
|
|
||||||
cd ../../
|
|
||||||
rm -rf cmd/web/dist
|
|
||||||
cp -r ui/flutter/build/web cmd/web/dist
|
|
||||||
- name: Build and push
|
- name: Build and push
|
||||||
uses: docker/build-push-action@v2
|
uses: docker/build-push-action@v2
|
||||||
env:
|
env:
|
||||||
VERSION: ${{ needs.get-release.outputs.version }}
|
VERSION: ${{ needs.get-release.outputs.version }}
|
||||||
with:
|
with:
|
||||||
context: .
|
|
||||||
push: ${{ github.event.inputs.test == 'true' && 'false' || 'true' }}
|
push: ${{ github.event.inputs.test == 'true' && 'false' || 'true' }}
|
||||||
build-args: |
|
build-args: |
|
||||||
VERSION=${{ env.VERSION }}
|
VERSION=${{ env.VERSION }}
|
||||||
@@ -797,6 +477,7 @@ jobs:
|
|||||||
linux/amd64
|
linux/amd64
|
||||||
linux/arm64
|
linux/arm64
|
||||||
linux/arm/v7
|
linux/arm/v7
|
||||||
|
linux/arm64/v8
|
||||||
tags: |
|
tags: |
|
||||||
${{ !contains(env.VERSION, 'beta') && 'liwei2633/gopeed:latest' || '' }}
|
liwei2633/gopeed:latest
|
||||||
liwei2633/gopeed:v${{ env.VERSION }}
|
liwei2633/gopeed:v${{ env.VERSION }}
|
||||||
@@ -1,50 +0,0 @@
|
|||||||
name: l10n
|
|
||||||
|
|
||||||
on:
|
|
||||||
pull_request:
|
|
||||||
branches:
|
|
||||||
- main
|
|
||||||
paths:
|
|
||||||
- "ui/flutter/lib/l10n/**"
|
|
||||||
- "ui/flutter/l10n.yaml"
|
|
||||||
- ".github/workflows/scripts/check_l10n.dart"
|
|
||||||
- ".github/workflows/l10n.yml"
|
|
||||||
push:
|
|
||||||
branches:
|
|
||||||
- main
|
|
||||||
paths:
|
|
||||||
- "ui/flutter/lib/l10n/**"
|
|
||||||
- "ui/flutter/l10n.yaml"
|
|
||||||
- ".github/workflows/scripts/check_l10n.dart"
|
|
||||||
- ".github/workflows/l10n.yml"
|
|
||||||
workflow_dispatch:
|
|
||||||
|
|
||||||
permissions:
|
|
||||||
contents: read
|
|
||||||
|
|
||||||
concurrency:
|
|
||||||
group: l10n-${{ github.event.pull_request.number || github.ref }}
|
|
||||||
cancel-in-progress: true
|
|
||||||
|
|
||||||
env:
|
|
||||||
FLUTTER_VERSION: "3.41.6"
|
|
||||||
|
|
||||||
jobs:
|
|
||||||
validate:
|
|
||||||
name: Validate Flutter translations
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
defaults:
|
|
||||||
run:
|
|
||||||
working-directory: ui/flutter
|
|
||||||
steps:
|
|
||||||
- uses: actions/checkout@v4
|
|
||||||
- uses: subosito/flutter-action@v2
|
|
||||||
with:
|
|
||||||
flutter-version: ${{ env.FLUTTER_VERSION }}
|
|
||||||
- run: flutter pub get
|
|
||||||
- name: Validate ARB catalogs
|
|
||||||
run: dart run ../../.github/workflows/scripts/check_l10n.dart
|
|
||||||
- name: Generate localizations
|
|
||||||
run: flutter gen-l10n
|
|
||||||
- name: Analyze generated localization integration
|
|
||||||
run: flutter analyze
|
|
||||||
@@ -17,7 +17,7 @@ on:
|
|||||||
- "Dockerfile"
|
- "Dockerfile"
|
||||||
|
|
||||||
env:
|
env:
|
||||||
GO_VERSION: "1.25.4"
|
GO_VERSION: "1.19"
|
||||||
|
|
||||||
permissions:
|
permissions:
|
||||||
contents: write
|
contents: write
|
||||||
@@ -31,7 +31,7 @@ jobs:
|
|||||||
|
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v3
|
- uses: actions/checkout@v3
|
||||||
- uses: actions/setup-go@v5
|
- uses: actions/setup-go@v4
|
||||||
with:
|
with:
|
||||||
go-version: ${{ env.GO_VERSION }}
|
go-version: ${{ env.GO_VERSION }}
|
||||||
- uses: release-drafter/release-drafter@v5
|
- uses: release-drafter/release-drafter@v5
|
||||||
|
|||||||
@@ -1,107 +0,0 @@
|
|||||||
// Validate Flutter ARB translation catalogs before localization code generation.
|
|
||||||
//
|
|
||||||
// Usage (from ui/flutter):
|
|
||||||
// dart run ../../.github/workflows/scripts/check_l10n.dart
|
|
||||||
//
|
|
||||||
// What it checks:
|
|
||||||
// - Every ARB file uses the repository's canonical two-space JSON formatting.
|
|
||||||
// - Every app_<locale>.arb file declares the matching @@locale value.
|
|
||||||
// - Message values are not empty (warning only).
|
|
||||||
// - Every locale has the same message keys as app_en.arb (warning only).
|
|
||||||
// - Every translation preserves the placeholder variables used by app_en.arb
|
|
||||||
// (warning only).
|
|
||||||
//
|
|
||||||
// Notes:
|
|
||||||
// - This script does not judge translation quality.
|
|
||||||
// - It does not generate localization code or fully parse ICU MessageFormat.
|
|
||||||
// `flutter gen-l10n` runs immediately afterwards in CI to validate ICU syntax
|
|
||||||
// and generate app_localizations*.dart.
|
|
||||||
// - The script exits with a failure only for malformed JSON, invalid ARB
|
|
||||||
// formatting, or a mismatched @@locale value. Translation completeness is
|
|
||||||
// reported as warnings; generation and analysis remain the compile-time gate.
|
|
||||||
|
|
||||||
import 'dart:convert';
|
|
||||||
import 'dart:io';
|
|
||||||
|
|
||||||
/// Runs repository-specific checks that `flutter gen-l10n` does not enforce.
|
|
||||||
void main() {
|
|
||||||
final directory = Directory('lib/l10n');
|
|
||||||
final templateFile = File('${directory.path}/app_en.arb');
|
|
||||||
final template = _readArb(templateFile);
|
|
||||||
final templateKeys = _messageKeys(template);
|
|
||||||
var failed = false;
|
|
||||||
|
|
||||||
for (final file in directory.listSync().whereType<File>().where(
|
|
||||||
(file) => file.path.endsWith('.arb'),
|
|
||||||
)) {
|
|
||||||
final messages = _readArb(file);
|
|
||||||
if (!_hasCanonicalFormat(file, messages)) {
|
|
||||||
stderr.writeln(
|
|
||||||
'${file.path}: ARB formatting is not canonical (use two-space JSON indentation and a trailing newline)',
|
|
||||||
);
|
|
||||||
failed = true;
|
|
||||||
}
|
|
||||||
final expectedLocale = file.uri.pathSegments.last
|
|
||||||
.replaceFirst('app_', '')
|
|
||||||
.replaceFirst('.arb', '');
|
|
||||||
if (messages['@@locale'] != expectedLocale) {
|
|
||||||
stderr.writeln('${file.path}: @@locale must be $expectedLocale');
|
|
||||||
failed = true;
|
|
||||||
}
|
|
||||||
final keys = _messageKeys(messages);
|
|
||||||
final empty =
|
|
||||||
keys.where((key) => (messages[key] as String).trim().isEmpty).toList()
|
|
||||||
..sort();
|
|
||||||
if (empty.isNotEmpty) {
|
|
||||||
_warning('${file.path}: empty messages $empty');
|
|
||||||
}
|
|
||||||
if (file.path == templateFile.path) continue;
|
|
||||||
final missing = templateKeys.difference(keys);
|
|
||||||
final extra = keys.difference(templateKeys);
|
|
||||||
if (missing.isNotEmpty || extra.isNotEmpty) {
|
|
||||||
_warning(
|
|
||||||
'${file.path}: missing ${missing.toList()..sort()}, extra ${extra.toList()..sort()}',
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
for (final key in templateKeys.intersection(keys)) {
|
|
||||||
final expected = _placeholders(template[key] as String);
|
|
||||||
final actual = _placeholders(messages[key] as String);
|
|
||||||
if (!_sameSet(expected, actual)) {
|
|
||||||
_warning(
|
|
||||||
'${file.path}: $key placeholders $actual do not match $expected',
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (failed) exitCode = 1;
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Reads one ARB file as its top-level JSON object.
|
|
||||||
Map<String, Object?> _readArb(File file) =>
|
|
||||||
jsonDecode(file.readAsStringSync()) as Map<String, Object?>;
|
|
||||||
|
|
||||||
/// Checks the exact JSON layout so whitespace-only formatting drift fails CI.
|
|
||||||
bool _hasCanonicalFormat(File file, Map<String, Object?> arb) {
|
|
||||||
const encoder = JsonEncoder.withIndent(' ');
|
|
||||||
return file.readAsStringSync() == '${encoder.convert(arb)}\n';
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Returns user-facing message keys and excludes ARB metadata keys (`@`/`@@`).
|
|
||||||
Set<String> _messageKeys(Map<String, Object?> arb) =>
|
|
||||||
arb.keys.where((key) => !key.startsWith('@')).toSet();
|
|
||||||
|
|
||||||
/// Extracts both simple placeholders (`{name}`) and ICU selector variables.
|
|
||||||
///
|
|
||||||
/// For example, this returns `count` from
|
|
||||||
/// `{count, plural, =1{1 file} other{{count} files}}`.
|
|
||||||
Set<String> _placeholders(String message) => RegExp(
|
|
||||||
r'\{([A-Za-z_][A-Za-z0-9_]*)(?=\s*(?:\}|,\s*(?:plural|select|selectordinal)\s*,))',
|
|
||||||
).allMatches(message).map((match) => match.group(1)!).toSet();
|
|
||||||
|
|
||||||
/// Compares sets without depending on iteration order.
|
|
||||||
bool _sameSet(Set<String> left, Set<String> right) =>
|
|
||||||
left.length == right.length && left.containsAll(right);
|
|
||||||
|
|
||||||
void _warning(String message) => stderr.writeln('warning: $message');
|
|
||||||
@@ -1,413 +0,0 @@
|
|||||||
// Localize external Google Fonts (fonts.gstatic.com) references in Flutter Web build output.
|
|
||||||
//
|
|
||||||
// Usage (post-build):
|
|
||||||
// # Run AFTER: flutter build web --no-web-resources-cdn
|
|
||||||
// dart ../../.github/workflows/scripts/flutter_local_font.dart
|
|
||||||
//
|
|
||||||
// What it does:
|
|
||||||
// In main.dart.js:
|
|
||||||
// - Find any referenced resources under: https://fonts.gstatic.com/s/
|
|
||||||
// (including the common pattern where Flutter concatenates the base URL with a relative path)
|
|
||||||
// - Download them into: build/web/assets/gstatic/<path>
|
|
||||||
// - Rewrite "https://fonts.gstatic.com/s/" -> "assets/gstatic/" so runtime loads locally
|
|
||||||
// - Only download font subsets needed by supported locales (parsed from lib/l10n/*.arb)
|
|
||||||
//
|
|
||||||
// Notes:
|
|
||||||
// - This script DOES NOT call `flutter build web`. It only patches the output.
|
|
||||||
// - This is a best-effort post-process. Always verify in browser devtools.
|
|
||||||
|
|
||||||
import 'dart:async';
|
|
||||||
import 'dart:io';
|
|
||||||
|
|
||||||
/// Get font families required for a specific locale based on language/script.
|
|
||||||
/// Uses language code prefix to automatically detect the script system.
|
|
||||||
/// Returns empty set if base fonts (Latin/Cyrillic/Greek) are sufficient.
|
|
||||||
Set<String> _getFontFamiliesForLocale(String locale) {
|
|
||||||
final normalizedLocale = locale.toLowerCase().replaceAll('-', '_');
|
|
||||||
final lang = normalizedLocale.split('_').first;
|
|
||||||
|
|
||||||
// CJK languages - need specific large font files
|
|
||||||
if (lang == 'zh') {
|
|
||||||
// Simplified vs Traditional Chinese
|
|
||||||
if (normalizedLocale.contains('_tw') ||
|
|
||||||
normalizedLocale.contains('_hk') ||
|
|
||||||
normalizedLocale.contains('_mo')) {
|
|
||||||
return {'notosanstc', 'notosanshk'}; // Traditional Chinese (TW, HK, MO)
|
|
||||||
}
|
|
||||||
return {'notosanssc'}; // Simplified Chinese, including generic zh
|
|
||||||
}
|
|
||||||
if (lang == 'ja') return {'notosansjp'}; // Japanese
|
|
||||||
if (lang == 'ko') return {'notosanskr'}; // Korean
|
|
||||||
|
|
||||||
// Arabic script languages
|
|
||||||
if (lang == 'ar') return {'notosansarabic'}; // Arabic
|
|
||||||
if (lang == 'fa') return {'notosansarabic'}; // Persian/Farsi
|
|
||||||
if (lang == 'ur') return {'notosansarabic'}; // Urdu
|
|
||||||
if (lang == 'ps') return {'notosansarabic'}; // Pashto
|
|
||||||
if (lang == 'ku') return {'notosansarabic'}; // Kurdish (Arabic script)
|
|
||||||
|
|
||||||
// Hebrew script
|
|
||||||
if (lang == 'he' || lang == 'yi') return {'notosanshebrew'};
|
|
||||||
|
|
||||||
// South Asian scripts
|
|
||||||
if (lang == 'hi' || lang == 'mr' || lang == 'ne' || lang == 'sa') {
|
|
||||||
return {'notosansdevanagari'}; // Hindi, Marathi, Nepali, Sanskrit
|
|
||||||
}
|
|
||||||
if (lang == 'bn' || lang == 'as')
|
|
||||||
return {'notosansbengali'}; // Bengali, Assamese
|
|
||||||
if (lang == 'ta')
|
|
||||||
return {'notosanstamil', 'notosanstamilsupplement'}; // Tamil
|
|
||||||
if (lang == 'te') return {'notosanstelugu'}; // Telugu
|
|
||||||
if (lang == 'kn') return {'notosanskannada'}; // Kannada
|
|
||||||
if (lang == 'ml') return {'notosansmalayalam'}; // Malayalam
|
|
||||||
if (lang == 'gu') return {'notosansgujarati'}; // Gujarati
|
|
||||||
if (lang == 'pa') return {'notosansgurmukhi'}; // Punjabi (Gurmukhi)
|
|
||||||
if (lang == 'or') return {'notosansoriya'}; // Odia/Oriya
|
|
||||||
if (lang == 'si') return {'notosanssinhala'}; // Sinhala
|
|
||||||
|
|
||||||
// Southeast Asian scripts
|
|
||||||
if (lang == 'th') return {'notosansthai'}; // Thai
|
|
||||||
if (lang == 'lo') return {'notosanslao'}; // Lao
|
|
||||||
if (lang == 'my') return {'notosansmyanmar'}; // Myanmar/Burmese
|
|
||||||
if (lang == 'km') return {'notosanskhmer'}; // Khmer/Cambodian
|
|
||||||
if (lang == 'jv') return {'notosansjavanese'}; // Javanese
|
|
||||||
|
|
||||||
// Other scripts
|
|
||||||
if (lang == 'ka') return {'notosansgeorgian'}; // Georgian
|
|
||||||
if (lang == 'hy') return {'notosansarmenian'}; // Armenian
|
|
||||||
if (lang == 'am' || lang == 'ti')
|
|
||||||
return {'notosansethiopic'}; // Amharic, Tigrinya
|
|
||||||
if (lang == 'mn') return {'notosansmongolian'}; // Mongolian
|
|
||||||
|
|
||||||
// Latin/Cyrillic/Greek based languages - base notosans is sufficient
|
|
||||||
// Includes: English, German, French, Spanish, Italian, Portuguese,
|
|
||||||
// Russian, Ukrainian, Polish, Turkish, Vietnamese, Greek, etc.
|
|
||||||
return {};
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Base font families that are always included regardless of locale.
|
|
||||||
/// These provide core functionality and are relatively small.
|
|
||||||
const _baseFontFamilies = <String>{
|
|
||||||
'notosans', // Base Latin/Cyrillic/Greek/Vietnamese
|
|
||||||
'roboto', // Material Design default font
|
|
||||||
'notosanssymbols', // Common symbols
|
|
||||||
'notosanssymbols2', // Additional symbols
|
|
||||||
'notosansmath', // Math symbols
|
|
||||||
'notomusic', // Music notation symbols
|
|
||||||
// Note: notocoloremoji (~24MB) and notoemoji (~860KB) are excluded
|
|
||||||
// to reduce bundle size. Add them back if emoji support is needed.
|
|
||||||
};
|
|
||||||
|
|
||||||
/// Parse supported locales from Flutter gen-l10n ARB filenames.
|
|
||||||
/// Filenames follow app_<locale>.arb, for example app_zh_CN.arb or app_en.arb.
|
|
||||||
Set<String> _parseSupportedLocales(Directory l10nDirectory) {
|
|
||||||
final locales = <String>{};
|
|
||||||
|
|
||||||
if (!l10nDirectory.existsSync()) {
|
|
||||||
_fail('Localization directory not found: ${l10nDirectory.path}');
|
|
||||||
}
|
|
||||||
|
|
||||||
final arbName = RegExp(r'^app_(.+)\.arb$');
|
|
||||||
for (final entity in l10nDirectory.listSync()) {
|
|
||||||
if (entity is! File) continue;
|
|
||||||
final filename = entity.uri.pathSegments.last;
|
|
||||||
final locale = arbName.firstMatch(filename)?.group(1);
|
|
||||||
if (locale != null && locale.isNotEmpty) locales.add(locale);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (locales.isEmpty) {
|
|
||||||
_fail('No app_<locale>.arb files found in ${l10nDirectory.path}');
|
|
||||||
}
|
|
||||||
|
|
||||||
return locales;
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Get required font families for the given locales.
|
|
||||||
/// Returns a set of font family names that should be included.
|
|
||||||
/// Automatically detects required fonts based on language code prefix.
|
|
||||||
Set<String> _getRequiredFontFamilies(Set<String> locales) {
|
|
||||||
final families = <String>{..._baseFontFamilies};
|
|
||||||
for (final locale in locales) {
|
|
||||||
final localeFamilies = _getFontFamiliesForLocale(locale);
|
|
||||||
families.addAll(localeFamilies);
|
|
||||||
}
|
|
||||||
return families;
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Check if a font file path should be included based on required font families.
|
|
||||||
/// Font paths have the format: "fontfamily/version/filename.ext"
|
|
||||||
/// Example: "notosanssc/v36/xxx.ttf" -> font family is "notosanssc"
|
|
||||||
bool _fontMatchesRequiredFamilies(
|
|
||||||
String fontPath,
|
|
||||||
Set<String> requiredFamilies,
|
|
||||||
) {
|
|
||||||
// Extract font family from path (first segment before '/')
|
|
||||||
final slashIndex = fontPath.indexOf('/');
|
|
||||||
if (slashIndex == -1) {
|
|
||||||
// No slash found, include by default (unusual path format)
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
final fontFamily = fontPath.substring(0, slashIndex).toLowerCase();
|
|
||||||
return requiredFamilies.contains(fontFamily);
|
|
||||||
}
|
|
||||||
|
|
||||||
Future<void> main(List<String> args) async {
|
|
||||||
try {
|
|
||||||
// This script is intentionally argument-free for CI convenience.
|
|
||||||
// It assumes it is executed from the Flutter project directory (ui/flutter),
|
|
||||||
// but will also try to locate pubspec.yaml by walking up.
|
|
||||||
final flutterDir = _findFlutterProjectRoot(Directory.current);
|
|
||||||
|
|
||||||
// Parse supported locales from Flutter gen-l10n ARB files.
|
|
||||||
final l10nDirectory = Directory(_join(flutterDir.path, 'lib', 'l10n'));
|
|
||||||
final supportedLocales = _parseSupportedLocales(l10nDirectory);
|
|
||||||
stdout.writeln('Supported locales: ${supportedLocales.join(', ')}');
|
|
||||||
|
|
||||||
// Get required font families based on supported locales
|
|
||||||
final requiredFamilies = _getRequiredFontFamilies(supportedLocales);
|
|
||||||
stdout.writeln('Required font families: ${requiredFamilies.join(', ')}');
|
|
||||||
|
|
||||||
final webDir = Directory(_join(flutterDir.path, 'build', 'web'));
|
|
||||||
if (!webDir.existsSync()) {
|
|
||||||
_fail(
|
|
||||||
'Web build directory not found: ${webDir.path}\n'
|
|
||||||
'Did you run: flutter build web --no-web-resources-cdn ?',
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
final mainJs = File(_join(webDir.path, 'main.dart.js'));
|
|
||||||
if (!mainJs.existsSync()) {
|
|
||||||
_fail('main.dart.js not found at: ${mainJs.path}');
|
|
||||||
}
|
|
||||||
|
|
||||||
final gstaticRoot = Directory(_join(webDir.path, 'assets', 'gstatic'));
|
|
||||||
if (!gstaticRoot.existsSync()) gstaticRoot.createSync(recursive: true);
|
|
||||||
|
|
||||||
const gstaticSPrefix = 'https://fonts.gstatic.com/s/';
|
|
||||||
final original = mainJs.readAsStringSync();
|
|
||||||
|
|
||||||
// Relative font asset paths which Flutter's loader typically concatenates with:
|
|
||||||
// https://fonts.gstatic.com/s/
|
|
||||||
// Keep this as a best-effort heuristic to cover the common concatenation pattern.
|
|
||||||
// Some gstatic assets include extra dot segments like: notocoloremoji/v32/Yq6P-KqIXTD0t4D9z1ESnKM3-HpFabsE4tq3luCC7p-aXxcn.0.woff2
|
|
||||||
// Allow dots in the path, while still restricting to font-like extensions.
|
|
||||||
final relAssetRegex = RegExp(
|
|
||||||
r"""["']([a-zA-Z0-9/_\.-]+\.(?:woff2|woff|ttf|otf|eot|svg))["']""",
|
|
||||||
caseSensitive: false,
|
|
||||||
);
|
|
||||||
final relAssetsUnderS = <String>{};
|
|
||||||
for (final m in relAssetRegex.allMatches(original)) {
|
|
||||||
final p = m.group(1);
|
|
||||||
if (p == null || p.isEmpty) continue;
|
|
||||||
if (p.contains('://') || p.startsWith('data:')) continue;
|
|
||||||
if (p.startsWith('/') ||
|
|
||||||
p.startsWith('assets/') ||
|
|
||||||
p.startsWith('packages/'))
|
|
||||||
continue;
|
|
||||||
if (p.contains('..')) continue;
|
|
||||||
relAssetsUnderS.add(p);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (relAssetsUnderS.isEmpty) {
|
|
||||||
_fail('No fonts.gstatic.com assets found in main.dart.js.');
|
|
||||||
}
|
|
||||||
|
|
||||||
// Build a download plan: dest relative path (under assets/gstatic/) -> URL.
|
|
||||||
// Filter fonts based on required font families to reduce package size.
|
|
||||||
final downloads = <String, Uri>{};
|
|
||||||
final skippedFonts = <String, Set<String>>{};
|
|
||||||
|
|
||||||
for (final rel in relAssetsUnderS) {
|
|
||||||
if (_fontMatchesRequiredFamilies(rel, requiredFamilies)) {
|
|
||||||
downloads[rel] = Uri.parse('$gstaticSPrefix$rel');
|
|
||||||
} else {
|
|
||||||
// Track skipped fonts and their paths for fallback generation
|
|
||||||
final slashIndex = rel.indexOf('/');
|
|
||||||
final family = slashIndex > 0 ? rel.substring(0, slashIndex) : rel;
|
|
||||||
skippedFonts.putIfAbsent(family, () => <String>{}).add(rel);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (skippedFonts.isNotEmpty) {
|
|
||||||
stdout.writeln(
|
|
||||||
'Skipped ${skippedFonts.length} font families not required by supported locales:',
|
|
||||||
);
|
|
||||||
stdout.writeln(' ${skippedFonts.keys.join(', ')}');
|
|
||||||
}
|
|
||||||
|
|
||||||
if (downloads.isNotEmpty) {
|
|
||||||
stdout.writeln(
|
|
||||||
'Found ${downloads.length} fonts.gstatic.com assets to download...',
|
|
||||||
);
|
|
||||||
|
|
||||||
// Reuse a single HttpClient to avoid creating hundreds of short-lived
|
|
||||||
// connections (can be flaky on some environments).
|
|
||||||
final httpClient = HttpClient()
|
|
||||||
..connectionTimeout = const Duration(seconds: 30)
|
|
||||||
..idleTimeout = const Duration(seconds: 30)
|
|
||||||
..maxConnectionsPerHost = 6;
|
|
||||||
try {
|
|
||||||
for (final entry in downloads.entries) {
|
|
||||||
final relPath = entry.key;
|
|
||||||
final url = entry.value;
|
|
||||||
final destPath = relPath.replaceAll('/', Platform.pathSeparator);
|
|
||||||
final dest = File(_join(gstaticRoot.path, destPath));
|
|
||||||
await _downloadIfMissing(httpClient, url, dest);
|
|
||||||
}
|
|
||||||
} finally {
|
|
||||||
httpClient.close(force: true);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Generate empty fallback font files for skipped fonts to prevent infinite loading
|
|
||||||
// Empty files will cause browsers to fail fast instead of retrying indefinitely
|
|
||||||
if (skippedFonts.isNotEmpty) {
|
|
||||||
var fallbackCount = 0;
|
|
||||||
for (final entry in skippedFonts.entries) {
|
|
||||||
for (final relPath in entry.value) {
|
|
||||||
final destPath = relPath.replaceAll('/', Platform.pathSeparator);
|
|
||||||
final dest = File(_join(gstaticRoot.path, destPath));
|
|
||||||
if (!dest.existsSync()) {
|
|
||||||
dest.parent.createSync(recursive: true);
|
|
||||||
// Create an empty file - browsers will recognize it as invalid and skip
|
|
||||||
dest.writeAsBytesSync([]);
|
|
||||||
fallbackCount++;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (fallbackCount > 0) {
|
|
||||||
stdout.writeln(
|
|
||||||
'Generated $fallbackCount empty fallback font files to prevent infinite loading',
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Rewrite remote prefix so requests become:
|
|
||||||
// <origin>/assets/gstatic/<...>.woff2
|
|
||||||
final replacedGstatic = original.replaceAll(
|
|
||||||
gstaticSPrefix,
|
|
||||||
'assets/gstatic/',
|
|
||||||
);
|
|
||||||
|
|
||||||
if (replacedGstatic == original && downloads.isEmpty) {
|
|
||||||
stdout.writeln(
|
|
||||||
'No changes applied (no fonts.gstatic.com references found).',
|
|
||||||
);
|
|
||||||
exitCode = 0;
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
mainJs.writeAsStringSync(replacedGstatic);
|
|
||||||
|
|
||||||
stdout.writeln('Patched: ${mainJs.path}');
|
|
||||||
if (downloads.isNotEmpty) {
|
|
||||||
stdout.writeln(
|
|
||||||
' - downloaded ${downloads.length} files into: ${gstaticRoot.path}',
|
|
||||||
);
|
|
||||||
}
|
|
||||||
stdout.writeln(' - fonts.gstatic.com rewritten -> assets/gstatic/');
|
|
||||||
} catch (e, st) {
|
|
||||||
stderr.writeln('ERROR: $e');
|
|
||||||
stderr.writeln(st);
|
|
||||||
exitCode = 1;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
Future<void> _downloadIfMissing(HttpClient client, Uri url, File dest) async {
|
|
||||||
if (dest.existsSync() && dest.lengthSync() > 0) return;
|
|
||||||
|
|
||||||
dest.parent.createSync(recursive: true);
|
|
||||||
|
|
||||||
// Network can be flaky in CI. Retry a few times on transient errors.
|
|
||||||
const maxAttempts = 4;
|
|
||||||
|
|
||||||
for (var attempt = 1; attempt <= maxAttempts; attempt++) {
|
|
||||||
final tmp = File('${dest.path}.tmp');
|
|
||||||
try {
|
|
||||||
try {
|
|
||||||
final req = await client.getUrl(url);
|
|
||||||
|
|
||||||
final resp = await req.close().timeout(const Duration(seconds: 60));
|
|
||||||
if (resp.statusCode != 200) {
|
|
||||||
// 404 is not transient in practice - fail fast with a clear message.
|
|
||||||
if (resp.statusCode == 404) {
|
|
||||||
_fail('Missing gstatic asset (404): $url');
|
|
||||||
}
|
|
||||||
throw HttpException('HTTP ${resp.statusCode}', uri: url);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Stream to temp file first, then rename to avoid leaving partial files.
|
|
||||||
if (tmp.existsSync()) tmp.deleteSync();
|
|
||||||
final sink = tmp.openWrite();
|
|
||||||
await resp.pipe(sink).timeout(const Duration(seconds: 120));
|
|
||||||
|
|
||||||
if (!tmp.existsSync() || tmp.lengthSync() == 0) {
|
|
||||||
throw const FormatException('Downloaded asset is empty');
|
|
||||||
}
|
|
||||||
if (dest.existsSync()) dest.deleteSync();
|
|
||||||
tmp.renameSync(dest.path);
|
|
||||||
return;
|
|
||||||
} finally {
|
|
||||||
// no-op: HttpClient lifecycle managed by the caller
|
|
||||||
}
|
|
||||||
} on TimeoutException {
|
|
||||||
if (tmp.existsSync()) tmp.deleteSync();
|
|
||||||
if (attempt == maxAttempts) {
|
|
||||||
_fail('Timeout downloading asset: $url');
|
|
||||||
}
|
|
||||||
} on SocketException catch (e) {
|
|
||||||
if (tmp.existsSync()) tmp.deleteSync();
|
|
||||||
if (attempt == maxAttempts) {
|
|
||||||
_fail('Socket error downloading asset: $url ($e)');
|
|
||||||
}
|
|
||||||
} on HttpException catch (e) {
|
|
||||||
if (tmp.existsSync()) tmp.deleteSync();
|
|
||||||
if (attempt == maxAttempts) {
|
|
||||||
_fail('HTTP error downloading asset: $url ($e)');
|
|
||||||
}
|
|
||||||
} on FileSystemException catch (e) {
|
|
||||||
if (tmp.existsSync()) tmp.deleteSync();
|
|
||||||
_fail('File write error downloading asset: $url ($e)');
|
|
||||||
} catch (e) {
|
|
||||||
if (tmp.existsSync()) tmp.deleteSync();
|
|
||||||
if (attempt == maxAttempts) {
|
|
||||||
_fail('Unexpected error downloading asset: $url ($e)');
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Backoff before retrying.
|
|
||||||
final backoffSeconds = 1 << (attempt - 1);
|
|
||||||
stdout.writeln(
|
|
||||||
'Retrying (${attempt + 1}/$maxAttempts) for: $url (wait ${backoffSeconds}s)',
|
|
||||||
);
|
|
||||||
await Future.delayed(Duration(seconds: backoffSeconds));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
Directory _findFlutterProjectRoot(Directory start) {
|
|
||||||
var dir = start;
|
|
||||||
for (var i = 0; i < 10; i++) {
|
|
||||||
final pubspec = File(_join(dir.path, 'pubspec.yaml'));
|
|
||||||
if (pubspec.existsSync()) return dir;
|
|
||||||
final parent = dir.parent;
|
|
||||||
if (parent.path == dir.path) break;
|
|
||||||
dir = parent;
|
|
||||||
}
|
|
||||||
// Fallback: use current directory, error messages will explain what is missing.
|
|
||||||
return start;
|
|
||||||
}
|
|
||||||
|
|
||||||
Never _fail(String msg) {
|
|
||||||
throw FormatException(msg);
|
|
||||||
}
|
|
||||||
|
|
||||||
String _join(String a, [String? b, String? c, String? d]) {
|
|
||||||
final parts = <String>[
|
|
||||||
a,
|
|
||||||
if (b != null) b,
|
|
||||||
if (c != null) c,
|
|
||||||
if (d != null) d,
|
|
||||||
];
|
|
||||||
return parts.join(Platform.pathSeparator);
|
|
||||||
}
|
|
||||||
@@ -25,11 +25,10 @@ on:
|
|||||||
- ".github/workflows/test.yml"
|
- ".github/workflows/test.yml"
|
||||||
- "go.mod"
|
- "go.mod"
|
||||||
- "go.sum"
|
- "go.sum"
|
||||||
workflow_dispatch:
|
|
||||||
|
|
||||||
env:
|
env:
|
||||||
GO_VERSION: "1.25.4"
|
GO_VERSION: "1.21"
|
||||||
FLUTTER_VERSION: "3.41.6"
|
FLUTTER_VERSION: "3.16.8"
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
# lint:
|
# lint:
|
||||||
@@ -50,298 +49,32 @@ jobs:
|
|||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v3
|
- uses: actions/checkout@v3
|
||||||
- uses: actions/setup-go@v5
|
- uses: actions/setup-go@v4
|
||||||
with:
|
with:
|
||||||
go-version: ${{ env.GO_VERSION }}
|
go-version: ${{ env.GO_VERSION }}
|
||||||
- name: Install webview dependencies
|
|
||||||
run: |
|
|
||||||
sudo apt-get update -y
|
|
||||||
sudo apt-get install -y libgtk-3-dev libwebkit2gtk-4.1-dev xvfb
|
|
||||||
- name: Run Unit tests.
|
- name: Run Unit tests.
|
||||||
run: |
|
run: go test $(go list ./... | grep -v /cmd/ | grep -v /bind/ ) -v -covermode=count -coverprofile=coverage.txt
|
||||||
# Get packages for testing and coverage (including cmd/web for flags_test.go)
|
- uses: codecov/codecov-action@v3
|
||||||
PACKAGES=$(go list ./... | awk '!/\/bind\// && (!/\/cmd\// || /\/cmd\/web$/)')
|
|
||||||
env CGO_ENABLED=1 xvfb-run -a go test -tags webview -v -coverpkg=$(echo "$PACKAGES" | paste -sd "," -) -covermode=count -coverprofile=coverage.txt $PACKAGES
|
|
||||||
# Filter out main.go from coverage report to avoid lowering coverage percentage
|
|
||||||
grep -v "main.go" coverage.txt > coverage_filtered.txt || true
|
|
||||||
mv coverage_filtered.txt coverage.txt
|
|
||||||
- uses: codecov/codecov-action@v4
|
|
||||||
with:
|
with:
|
||||||
files: ./coverage.txt
|
files: ./coverage.txt
|
||||||
token: ${{ secrets.CODECOV_UPLOAD_TOKEN }}
|
build:
|
||||||
test-flutter-category:
|
name: Build check
|
||||||
# The settings editor and the backend have to read an extension list the
|
|
||||||
# same way, so both sides run the shared rule table. The Go side is covered
|
|
||||||
# by the test job above, this runs the ui side of it.
|
|
||||||
name: Test shared category rules
|
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
|
# needs: [lint, test]
|
||||||
|
needs: [test]
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v3
|
- uses: actions/checkout@v3
|
||||||
- uses: subosito/flutter-action@v2
|
- uses: actions/setup-go@v4
|
||||||
with:
|
|
||||||
flutter-version: ${{ env.FLUTTER_VERSION }}
|
|
||||||
- name: Run the shared category rule table on the ui side
|
|
||||||
working-directory: ui/flutter
|
|
||||||
run: |
|
|
||||||
flutter pub get
|
|
||||||
flutter gen-l10n
|
|
||||||
flutter test test/api/downloader_config_test.dart
|
|
||||||
test-flutter-task-context-menu:
|
|
||||||
name: Test task context menus and text inputs
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
timeout-minutes: 20
|
|
||||||
env:
|
|
||||||
CHROME_EXECUTABLE: /usr/bin/google-chrome
|
|
||||||
steps:
|
|
||||||
- uses: actions/checkout@v3
|
|
||||||
- uses: subosito/flutter-action@v2
|
|
||||||
with:
|
|
||||||
flutter-version: ${{ env.FLUTTER_VERSION }}
|
|
||||||
- name: Prepare Flutter tests
|
|
||||||
working-directory: ui/flutter
|
|
||||||
run: |
|
|
||||||
flutter pub get
|
|
||||||
flutter gen-l10n
|
|
||||||
- name: Test task menu actions and native input policy
|
|
||||||
working-directory: ui/flutter
|
|
||||||
run: |
|
|
||||||
flutter test --no-pub test/features/tasks/presentation/widgets/task_context_menu_test.dart test/shared/widgets/app_text_field_test.dart
|
|
||||||
flutter test --no-pub test/widget_test.dart --plain-name 'task context menu exposes'
|
|
||||||
- name: Test browser menu cancellation and input policy
|
|
||||||
working-directory: ui/flutter
|
|
||||||
run: |
|
|
||||||
"$CHROME_EXECUTABLE" --version
|
|
||||||
flutter test --no-pub --platform chrome test/features/tasks/presentation/widgets/task_context_menu_test.dart test/shared/widgets/app_text_field_test.dart
|
|
||||||
test-webview-integration:
|
|
||||||
name: Test webview integration (${{ matrix.name }})
|
|
||||||
strategy:
|
|
||||||
fail-fast: false
|
|
||||||
matrix:
|
|
||||||
include:
|
|
||||||
- name: Windows
|
|
||||||
os: windows-latest
|
|
||||||
- name: Linux
|
|
||||||
os: ubuntu-latest
|
|
||||||
- name: macOS
|
|
||||||
os: macos-latest
|
|
||||||
runs-on: ${{ matrix.os }}
|
|
||||||
steps:
|
|
||||||
- uses: actions/checkout@v3
|
|
||||||
- uses: actions/setup-go@v5
|
|
||||||
with:
|
|
||||||
go-version: ${{ env.GO_VERSION }}
|
|
||||||
- uses: subosito/flutter-action@v2
|
|
||||||
if: runner.os == 'macOS'
|
|
||||||
with:
|
|
||||||
flutter-version: ${{ env.FLUTTER_VERSION }}
|
|
||||||
- name: Install Linux WebView dependencies
|
|
||||||
if: runner.os == 'Linux'
|
|
||||||
run: |
|
|
||||||
sudo apt-get update -y
|
|
||||||
sudo apt-get install -y libgtk-3-dev libwebkit2gtk-4.1-dev xvfb
|
|
||||||
- name: Test webview
|
|
||||||
if: runner.os == 'Linux'
|
|
||||||
# Match the full suite's coverage instrumentation and repeat on the same
|
|
||||||
# GTK event loop to catch queued callbacks after engine destruction.
|
|
||||||
run: |
|
|
||||||
PACKAGES=$(go list ./... | awk '!/\/bind\// && (!/\/cmd\// || /\/cmd\/web$/)')
|
|
||||||
go test -c -tags webview -coverpkg=$(echo "$PACKAGES" | paste -sd "," -) -covermode=count -o "$RUNNER_TEMP/goprovider.test" ./internal/webview/goprovider
|
|
||||||
xvfb-run -a "$RUNNER_TEMP/goprovider.test" -test.v -test.count=5 -test.timeout=300s -test.coverprofile="$RUNNER_TEMP/webview-coverage.out"
|
|
||||||
env:
|
|
||||||
CGO_ENABLED: 1
|
|
||||||
- name: Test webview
|
|
||||||
if: runner.os == 'Windows'
|
|
||||||
# Repeat close/reopen coverage to catch browser shutdown races.
|
|
||||||
run: go test -tags webview -v -count=3 -timeout 300s ./internal/webview/goprovider/
|
|
||||||
env:
|
|
||||||
CGO_ENABLED: 1
|
|
||||||
- name: Test webview
|
|
||||||
if: runner.os == 'macOS'
|
|
||||||
shell: bash
|
|
||||||
run: |
|
|
||||||
set -euo pipefail
|
|
||||||
SOCKET_PATH="$HOME/Library/Application Support/com.gopeed.gopeed/gopeed_webview.sock"
|
|
||||||
LOG_PATH="$RUNNER_TEMP/gopeed-webview-rpc.log"
|
|
||||||
rm -f "$SOCKET_PATH"
|
|
||||||
|
|
||||||
go build -tags nosqlite -ldflags="-w -s" -buildmode=c-shared -o ui/flutter/macos/Frameworks/libgopeed.dylib github.com/GopeedLab/gopeed/bind/desktop
|
|
||||||
|
|
||||||
cd ui/flutter
|
|
||||||
flutter pub get
|
|
||||||
flutter run --machine -d macos >"$LOG_PATH" 2>&1 &
|
|
||||||
FLUTTER_PID=$!
|
|
||||||
cd ../..
|
|
||||||
|
|
||||||
cleanup() {
|
|
||||||
kill "$FLUTTER_PID" >/dev/null 2>&1 || true
|
|
||||||
}
|
|
||||||
trap cleanup EXIT
|
|
||||||
|
|
||||||
FLUTTER_STARTED=false
|
|
||||||
for _ in $(seq 1 600); do
|
|
||||||
if grep -Eq '"event"[[:space:]]*:[[:space:]]*"app\.started"' "$LOG_PATH"; then
|
|
||||||
FLUTTER_STARTED=true
|
|
||||||
break
|
|
||||||
fi
|
|
||||||
if ! kill -0 "$FLUTTER_PID" >/dev/null 2>&1; then
|
|
||||||
echo "Flutter exited before reporting app.started"
|
|
||||||
cat "$LOG_PATH"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
sleep 1
|
|
||||||
done
|
|
||||||
if [[ "$FLUTTER_STARTED" != true ]]; then
|
|
||||||
echo "Timed out waiting for Flutter to report app.started"
|
|
||||||
cat "$LOG_PATH"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
if ! go test -tags webview -v -count=1 -timeout 600s ./internal/webview/rpcprovider/ -args -webview-rpc-network unix -webview-rpc-address "$SOCKET_PATH"; then
|
|
||||||
cat "$LOG_PATH"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
build-desktop:
|
|
||||||
strategy:
|
|
||||||
matrix:
|
|
||||||
include:
|
|
||||||
- os: windows-2022
|
|
||||||
- os: windows-11-arm
|
|
||||||
llvm_ver: "20251202" # Only ARM64
|
|
||||||
flutter_channel: "main"
|
|
||||||
flutter_version: "7e1c8868"
|
|
||||||
- os: macos-latest
|
|
||||||
- os: ubuntu-22.04
|
|
||||||
- os: ubuntu-22.04-arm
|
|
||||||
flutter_channel: "main"
|
|
||||||
name: Build desktop check (${{ matrix.os }})
|
|
||||||
runs-on: ${{ matrix.os }}
|
|
||||||
needs: [ test ]
|
|
||||||
steps:
|
|
||||||
- uses: actions/checkout@v3
|
|
||||||
- name: Enable long paths for flutter main branch checks
|
|
||||||
run: |
|
|
||||||
git config --global core.longpaths true
|
|
||||||
- name: Install llvm-mingw-ucrt-aarch64 (ARM64 only)
|
|
||||||
if: matrix.os == 'windows-11-arm'
|
|
||||||
run: |
|
|
||||||
$ver = "${{ matrix.llvm_ver }}"
|
|
||||||
$url = "https://github.com/mstorsjo/llvm-mingw/releases/download/$ver/llvm-mingw-$ver-ucrt-aarch64.zip"
|
|
||||||
$zip = "$env:RUNNER_TEMP\\llvm.zip"
|
|
||||||
$extract = "$env:RUNNER_TEMP\\extract"
|
|
||||||
$target = "C:\\clangarm64"
|
|
||||||
|
|
||||||
curl -L $url -o $zip
|
|
||||||
rm -r -fo $extract,$target -ea Ignore
|
|
||||||
mkdir $extract | Out-Null
|
|
||||||
|
|
||||||
tar -xf $zip -C $extract
|
|
||||||
mv (Get-ChildItem $extract)[0].FullName $target
|
|
||||||
|
|
||||||
$b = "$target\\bin"
|
|
||||||
"CC=$b\\clang.exe" >> $env:GITHUB_ENV
|
|
||||||
"CXX=$b\\clang++.exe" >> $env:GITHUB_ENV
|
|
||||||
"CLANGARM64_BIN=$b" >> $env:GITHUB_ENV
|
|
||||||
"CGO_ENABLED=1" >> $env:GITHUB_ENV
|
|
||||||
"CLANGARM64_ROOT=$target" >> $env:GITHUB_ENV
|
|
||||||
$b >> $env:GITHUB_PATH
|
|
||||||
- uses: actions/setup-go@v5
|
|
||||||
with:
|
|
||||||
go-version: ${{ env.GO_VERSION }}
|
|
||||||
- uses: subosito/flutter-action@v2
|
|
||||||
with:
|
|
||||||
channel: ${{ matrix.flutter_channel || 'stable' }}
|
|
||||||
flutter-version: ${{ matrix.flutter_version || env.FLUTTER_VERSION }}
|
|
||||||
- if: runner.os == 'Windows'
|
|
||||||
run: |
|
|
||||||
go build -tags nosqlite -ldflags="-w -s" -buildmode=c-shared -o ui/flutter/windows/libgopeed.dll github.com/GopeedLab/gopeed/bind/desktop
|
|
||||||
cd ui/flutter
|
|
||||||
flutter build windows
|
|
||||||
- if: runner.os == 'macOS'
|
|
||||||
run: |
|
|
||||||
go build -tags nosqlite -ldflags="-w -s" -buildmode=c-shared -o ui/flutter/macos/Frameworks/libgopeed.dylib github.com/GopeedLab/gopeed/bind/desktop
|
|
||||||
cd ui/flutter
|
|
||||||
flutter build macos
|
|
||||||
- if: runner.os == 'Linux'
|
|
||||||
run: |
|
|
||||||
sudo apt-get update -y
|
|
||||||
sudo apt-get install -y ninja-build libgtk-3-dev libwebkit2gtk-4.1-dev libayatana-appindicator3-dev libkeybinder-3.0-dev
|
|
||||||
go build -tags nosqlite -ldflags="-w -s" -buildmode=c-shared -o ui/flutter/linux/bundle/lib/libgopeed.so github.com/GopeedLab/gopeed/bind/desktop
|
|
||||||
cd ui/flutter
|
|
||||||
flutter build linux
|
|
||||||
build-mobile:
|
|
||||||
name: Build mobile check
|
|
||||||
runs-on: ${{ matrix.os }}
|
|
||||||
strategy:
|
|
||||||
matrix:
|
|
||||||
os: [ macos-26, ubuntu-latest ]
|
|
||||||
needs: [ test ]
|
|
||||||
steps:
|
|
||||||
- uses: actions/checkout@v3
|
|
||||||
- uses: actions/setup-go@v5
|
|
||||||
with:
|
with:
|
||||||
go-version: ${{ env.GO_VERSION }}
|
go-version: ${{ env.GO_VERSION }}
|
||||||
- uses: subosito/flutter-action@v2
|
- uses: subosito/flutter-action@v2
|
||||||
with:
|
with:
|
||||||
flutter-version: ${{ env.FLUTTER_VERSION }}
|
flutter-version: ${{ env.FLUTTER_VERSION }}
|
||||||
- run: |
|
- run: |
|
||||||
go install golang.org/x/mobile/cmd/gomobile@latest
|
sudo apt-get update -y
|
||||||
go get golang.org/x/mobile/bind
|
sudo apt-get install -y ninja-build libgtk-3-dev libayatana-appindicator3-dev
|
||||||
gomobile init
|
|
||||||
- if: runner.os == 'macOS'
|
|
||||||
run: |
|
|
||||||
gomobile bind -tags nosqlite -ldflags="-w -s" -o ui/flutter/ios/Frameworks/Libgopeed.xcframework -target=ios github.com/GopeedLab/gopeed/bind/mobile
|
|
||||||
cd ui/flutter
|
|
||||||
flutter build ipa --no-codesign
|
|
||||||
- if: runner.os == 'Linux'
|
|
||||||
uses: actions/setup-java@v3
|
|
||||||
with:
|
|
||||||
distribution: "zulu"
|
|
||||||
java-version: "17"
|
|
||||||
- if: runner.os == 'Linux'
|
|
||||||
run: |
|
|
||||||
gomobile bind -tags nosqlite -ldflags="-w -s -checklinkname=0" -o ui/flutter/android/app/libs/libgopeed.aar -target=android -androidapi 21 -javapkg=com.gopeed github.com/GopeedLab/gopeed/bind/mobile
|
|
||||||
cd ui/flutter
|
|
||||||
flutter build apk
|
|
||||||
|
|
||||||
build-web:
|
|
||||||
name: Build web check
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
needs: [ test ]
|
|
||||||
steps:
|
|
||||||
- uses: actions/checkout@v3
|
|
||||||
- uses: actions/setup-go@v5
|
|
||||||
with:
|
|
||||||
go-version: ${{ env.GO_VERSION }}
|
|
||||||
- uses: subosito/flutter-action@v2
|
|
||||||
with:
|
|
||||||
flutter-version: ${{ env.FLUTTER_VERSION }}
|
|
||||||
- name: Build
|
- name: Build
|
||||||
run: |
|
run: |
|
||||||
|
go build -tags nosqlite -ldflags="-w -s" -buildmode=c-shared -o ui/flutter/linux/bundle/lib/libgopeed.so github.com/GopeedLab/gopeed/bind/desktop
|
||||||
cd ui/flutter
|
cd ui/flutter
|
||||||
flutter build web --no-web-resources-cdn
|
flutter build linux
|
||||||
dart ../../.github/workflows/scripts/flutter_local_font.dart
|
|
||||||
cd ../../
|
|
||||||
rm -rf cmd/web/dist
|
|
||||||
cp -r ui/flutter/build/web cmd/web/dist
|
|
||||||
go build -tags nosqlite,web -ldflags="-s -w" github.com/GopeedLab/gopeed/cmd/web
|
|
||||||
build-docker:
|
|
||||||
name: Build docker check
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
needs: [ test ]
|
|
||||||
steps:
|
|
||||||
- uses: actions/checkout@v3
|
|
||||||
- uses: actions/setup-go@v5
|
|
||||||
with:
|
|
||||||
go-version: ${{ env.GO_VERSION }}
|
|
||||||
- uses: subosito/flutter-action@v2
|
|
||||||
with:
|
|
||||||
flutter-version: ${{ env.FLUTTER_VERSION }}
|
|
||||||
- name: Build
|
|
||||||
run: |
|
|
||||||
cd ui/flutter
|
|
||||||
flutter build web --no-web-resources-cdn
|
|
||||||
cd ../../
|
|
||||||
rm -rf cmd/web/dist
|
|
||||||
cp -r ui/flutter/build/web cmd/web/dist
|
|
||||||
docker build -t gopeed .
|
|
||||||
@@ -17,60 +17,10 @@ start the front-end flutter project in `debug` mode to run.
|
|||||||
|
|
||||||
## Translation
|
## Translation
|
||||||
|
|
||||||
Flutter translations are ARB files in `ui/flutter/lib/l10n`. Use `app_en.arb` as the source template,
|
The internationalization files of Gopeed are located in the `ui/flutter/lib/i18n/langs` directory.
|
||||||
then edit or add `app_<locale>.arb` (for example, `app_de.arb` or `app_zh_TW.arb`). Every locale must
|
You only need to add the corresponding language file in this directory.
|
||||||
contain the same message keys as the English template.
|
|
||||||
|
|
||||||
Translate message values only, and do not rename variables inside braces such as `{count}` and `{name}`.
|
Please refer to `en_us.dart` for translation.
|
||||||
Do not translate metadata keys prefixed with `@`. Submit only the ARB source files you changed; do not generate or commit
|
|
||||||
`app_localizations*.dart`. The pull-request CI automatically validates the catalogs, generates the
|
|
||||||
localization code, and checks its integration.
|
|
||||||
|
|
||||||
The following examples show both the ARB definition and the text users see after the app supplies values.
|
|
||||||
|
|
||||||
1. Insert a variable into a message
|
|
||||||
|
|
||||||
ARB definition:
|
|
||||||
|
|
||||||
```json
|
|
||||||
"welcomeUser": "Hola, {name}"
|
|
||||||
```
|
|
||||||
|
|
||||||
Rendered result: when `name = Ana`, users see “Hola, Ana”. You may reorder the sentence, but keep `{name}`.
|
|
||||||
|
|
||||||
2. Display different text for different quantities
|
|
||||||
|
|
||||||
ARB definition:
|
|
||||||
|
|
||||||
```json
|
|
||||||
"fileCount": "{count, plural, =0{No hay archivos} =1{1 archivo} other{{count} archivos}}"
|
|
||||||
```
|
|
||||||
|
|
||||||
Rendered results:
|
|
||||||
|
|
||||||
- `count = 0` → “No hay archivos”
|
|
||||||
- `count = 1` → “1 archivo”
|
|
||||||
- `count = 3` → “3 archivos”
|
|
||||||
|
|
||||||
`=0`, `=1`, and `other` mean zero, one, and every other quantity. Translate only the user-visible text
|
|
||||||
inside each branch.
|
|
||||||
|
|
||||||
3. Display different text for different values
|
|
||||||
|
|
||||||
ARB definition:
|
|
||||||
|
|
||||||
```json
|
|
||||||
"taskState": "{state, select, running{En curso} paused{En pausa} other{Desconocido}}"
|
|
||||||
```
|
|
||||||
|
|
||||||
Rendered results:
|
|
||||||
|
|
||||||
- `state = running` → “En curso”
|
|
||||||
- `state = paused` → “En pausa”
|
|
||||||
- Any other value → “Desconocido”
|
|
||||||
|
|
||||||
`running`, `paused`, and `other` are values supplied by the app. Do not translate them; translate only
|
|
||||||
the display text inside the following braces.
|
|
||||||
|
|
||||||
## flutter development
|
## flutter development
|
||||||
|
|
||||||
@@ -80,4 +30,4 @@ Turn on build_runner watcher if you want to edit api/models:
|
|||||||
|
|
||||||
```
|
```
|
||||||
flutter pub run build_runner watch
|
flutter pub run build_runner watch
|
||||||
```
|
```
|
||||||
@@ -17,58 +17,10 @@
|
|||||||
|
|
||||||
## 翻訳
|
## 翻訳
|
||||||
|
|
||||||
Flutter の翻訳ファイルは `ui/flutter/lib/l10n` にあります。`app_en.arb` を原文テンプレートとして、
|
Gopeed の国際化ファイルは `ui/flutter/lib/i18n/langs` ディレクトリにあります。
|
||||||
`app_<locale>.arb`(例: `app_de.arb`、`app_zh_TW.arb`)を編集または追加してください。すべての言語で
|
このディレクトリに対応する言語ファイルを追加するだけでよいです。
|
||||||
英語テンプレートと同じメッセージ key が必要です。
|
|
||||||
|
|
||||||
翻訳するのはメッセージ値だけです。`{count}` や `{name}` など、中括弧内の変数名は変更しないでください。
|
翻訳については `en_us.dart` を参照してください。
|
||||||
`@` で始まるメタデータ key は翻訳しません。変更した ARB ソースファイルだけをコミットし、
|
|
||||||
`app_localizations*.dart` は生成もコミットも不要です。PR の CI が全言語を検証し、国際化コードを生成して
|
|
||||||
統合結果を確認します。
|
|
||||||
|
|
||||||
以下の例では、ARB に記述する定義と、アプリが値を渡した後にユーザーが実際に見る表示を示します。
|
|
||||||
|
|
||||||
1. メッセージに変数を挿入する
|
|
||||||
|
|
||||||
ARB 定義:
|
|
||||||
|
|
||||||
```json
|
|
||||||
"welcomeUser": "こんにちは、{name}"
|
|
||||||
```
|
|
||||||
|
|
||||||
最終表示:`name = 太郎` の場合は「こんにちは、太郎」と表示されます。語順は変更できますが、`{name}` は残します。
|
|
||||||
|
|
||||||
2. 数量に応じて表示を切り替える
|
|
||||||
|
|
||||||
ARB 定義:
|
|
||||||
|
|
||||||
```json
|
|
||||||
"fileCount": "{count, plural, =0{ファイルなし} =1{1 個のファイル} other{{count} 個のファイル}}"
|
|
||||||
```
|
|
||||||
|
|
||||||
最終表示:
|
|
||||||
|
|
||||||
- `count = 0` →「ファイルなし」
|
|
||||||
- `count = 1` →「1 個のファイル」
|
|
||||||
- `count = 3` →「3 個のファイル」
|
|
||||||
|
|
||||||
`=0`、`=1`、`other` はそれぞれ 0、1、その他の数量を表します。各分岐の中で画面に表示される文字だけを翻訳します。
|
|
||||||
|
|
||||||
3. 変数の値に応じて表示を切り替える
|
|
||||||
|
|
||||||
ARB 定義:
|
|
||||||
|
|
||||||
```json
|
|
||||||
"taskState": "{state, select, running{実行中} paused{一時停止} other{不明}}"
|
|
||||||
```
|
|
||||||
|
|
||||||
最終表示:
|
|
||||||
|
|
||||||
- `state = running` →「実行中」
|
|
||||||
- `state = paused` →「一時停止」
|
|
||||||
- その他の値 →「不明」
|
|
||||||
|
|
||||||
`running`、`paused`、`other` はアプリが渡す分岐値なので翻訳せず、その後の中括弧内にある表示文字だけを翻訳します。
|
|
||||||
|
|
||||||
## flutter での開発
|
## flutter での開発
|
||||||
|
|
||||||
|
|||||||
@@ -17,60 +17,10 @@ khởi động dự án flutter frontend trong chế độ `debug` để chạy.
|
|||||||
|
|
||||||
## Dịch thuật
|
## Dịch thuật
|
||||||
|
|
||||||
Các bản dịch Flutter là tệp ARB trong `ui/flutter/lib/l10n`. Hãy dùng `app_en.arb` làm mẫu nguồn,
|
Các tệp quốc tế hóa của Gopeed được đặt trong thư mục `ui/flutter/lib/i18n/langs`.
|
||||||
sau đó sửa hoặc thêm `app_<locale>.arb` (ví dụ `app_de.arb` hoặc `app_zh_TW.arb`). Mỗi ngôn ngữ phải
|
Bạn chỉ cần thêm tệp ngôn ngữ tương ứng trong thư mục này.
|
||||||
có cùng các khóa thông điệp như mẫu tiếng Anh.
|
|
||||||
|
|
||||||
Chỉ dịch giá trị thông điệp; không đổi tên biến trong dấu ngoặc nhọn như `{count}` và `{name}`. Không dịch khóa
|
Vui lòng tham khảo `en_us.dart` để biết cách dịch thuật.
|
||||||
siêu dữ liệu bắt đầu bằng `@`. Chỉ commit các tệp nguồn ARB đã thay đổi; không cần tạo hoặc commit
|
|
||||||
`app_localizations*.dart`. CI của pull request sẽ tự động
|
|
||||||
kiểm tra tất cả ngôn ngữ, tạo mã bản địa hóa và xác minh việc tích hợp.
|
|
||||||
|
|
||||||
Các ví dụ sau hiển thị cả định nghĩa trong ARB và nội dung người dùng thực sự thấy sau khi ứng dụng truyền giá trị.
|
|
||||||
|
|
||||||
1. Chèn biến vào thông điệp
|
|
||||||
|
|
||||||
Định nghĩa ARB:
|
|
||||||
|
|
||||||
```json
|
|
||||||
"welcomeUser": "Xin chào, {name}"
|
|
||||||
```
|
|
||||||
|
|
||||||
Kết quả hiển thị: khi `name = An`, người dùng thấy “Xin chào, An”. Có thể đổi trật tự câu nhưng phải giữ `{name}`.
|
|
||||||
|
|
||||||
2. Hiển thị nội dung khác nhau theo số lượng
|
|
||||||
|
|
||||||
Định nghĩa ARB:
|
|
||||||
|
|
||||||
```json
|
|
||||||
"fileCount": "{count, plural, =0{Không có tệp} =1{1 tệp} other{{count} tệp}}"
|
|
||||||
```
|
|
||||||
|
|
||||||
Kết quả hiển thị:
|
|
||||||
|
|
||||||
- `count = 0` → “Không có tệp”
|
|
||||||
- `count = 1` → “1 tệp”
|
|
||||||
- `count = 3` → “3 tệp”
|
|
||||||
|
|
||||||
`=0`, `=1` và `other` lần lượt biểu thị số lượng bằng 0, bằng 1 và các số lượng còn lại. Chỉ dịch phần chữ
|
|
||||||
hiển thị cho người dùng bên trong mỗi nhánh.
|
|
||||||
|
|
||||||
3. Hiển thị nội dung khác nhau theo giá trị biến
|
|
||||||
|
|
||||||
Định nghĩa ARB:
|
|
||||||
|
|
||||||
```json
|
|
||||||
"taskState": "{state, select, running{Đang chạy} paused{Đã tạm dừng} other{Không xác định}}"
|
|
||||||
```
|
|
||||||
|
|
||||||
Kết quả hiển thị:
|
|
||||||
|
|
||||||
- `state = running` → “Đang chạy”
|
|
||||||
- `state = paused` → “Đã tạm dừng”
|
|
||||||
- Giá trị khác → “Không xác định”
|
|
||||||
|
|
||||||
`running`, `paused` và `other` là các giá trị do ứng dụng truyền vào nên không được dịch; chỉ dịch phần chữ
|
|
||||||
hiển thị trong dấu ngoặc theo sau chúng.
|
|
||||||
|
|
||||||
## Phát triển flutter
|
## Phát triển flutter
|
||||||
|
|
||||||
|
|||||||
@@ -13,57 +13,10 @@ PR,合并到 `main` 分支。
|
|||||||
flutter 项目即可运行。
|
flutter 项目即可运行。
|
||||||
|
|
||||||
## 翻译
|
## 翻译
|
||||||
|
|
||||||
|
Gopeed 的国际化文件位于 `ui/flutter/lib/i18n/langs` 目录下,只需要在该目录下添加对应的语言文件即可。
|
||||||
|
|
||||||
Flutter 国际化文件位于 `ui/flutter/lib/l10n`,请以 `app_en.arb` 为源模板,修改或新增
|
请注意以 `en_us.dart` 为参照进行翻译。
|
||||||
`app_<locale>.arb`(例如 `app_de.arb` 或 `app_zh_TW.arb`)。每个语种必须与英文模板包含相同的文案 key。
|
|
||||||
|
|
||||||
只翻译文案值,不要修改 `{count}`、`{name}` 等花括号中的变量名。以 `@` 开头的元数据 key 不需要翻译。
|
|
||||||
提交时只包含修改过的 ARB 源文件,不需要生成或提交 `app_localizations*.dart`。PR 的 CI 会自动校验所有语种、
|
|
||||||
生成国际化代码并检查集成结果。
|
|
||||||
|
|
||||||
下面的例子同时给出 ARB 中填写的文案定义,以及程序传值后用户最终看到的内容。
|
|
||||||
|
|
||||||
1. 在文案中插入变量
|
|
||||||
|
|
||||||
ARB 定义:
|
|
||||||
|
|
||||||
```json
|
|
||||||
"welcomeUser": "你好,{name}"
|
|
||||||
```
|
|
||||||
|
|
||||||
最终显示:当 `name = 小明` 时,显示“你好,小明”。翻译时可以调整整句话,但必须保留 `{name}`。
|
|
||||||
|
|
||||||
2. 根据数量显示不同文案
|
|
||||||
|
|
||||||
ARB 定义:
|
|
||||||
|
|
||||||
```json
|
|
||||||
"fileCount": "{count, plural, =0{没有文件} =1{1 个文件} other{{count} 个文件}}"
|
|
||||||
```
|
|
||||||
|
|
||||||
最终显示:
|
|
||||||
|
|
||||||
- `count = 0` → “没有文件”
|
|
||||||
- `count = 1` → “1 个文件”
|
|
||||||
- `count = 3` → “3 个文件”
|
|
||||||
|
|
||||||
`=0`、`=1`、`other` 分别表示数量为 0、数量为 1 和其他数量。只翻译每个分支花括号内显示给用户的文字。
|
|
||||||
|
|
||||||
3. 根据变量值显示不同文案
|
|
||||||
|
|
||||||
ARB 定义:
|
|
||||||
|
|
||||||
```json
|
|
||||||
"taskState": "{state, select, running{运行中} paused{已暂停} other{未知状态}}"
|
|
||||||
```
|
|
||||||
|
|
||||||
最终显示:
|
|
||||||
|
|
||||||
- `state = running` → “运行中”
|
|
||||||
- `state = paused` → “已暂停”
|
|
||||||
- 其他值 → “未知状态”
|
|
||||||
|
|
||||||
`running`、`paused`、`other` 是程序传入的分支值,不能翻译;只翻译它们后面花括号内的显示文字。
|
|
||||||
|
|
||||||
## flutter开发
|
## flutter开发
|
||||||
|
|
||||||
@@ -74,3 +27,4 @@ Flutter 国际化文件位于 `ui/flutter/lib/l10n`,请以 `app_en.arb` 为源
|
|||||||
```
|
```
|
||||||
flutter pub run build_runner watch
|
flutter pub run build_runner watch
|
||||||
```
|
```
|
||||||
|
|
||||||
@@ -11,57 +11,11 @@
|
|||||||
建議使用 web 端進行開發與調試,首先啟動服務,使用指令 `go run cmd/api/main.go` 啟動 ,該服務默認連接埠為 `9999`,接著以 `debug` 模式啟動前端 flutter 項目即可。
|
建議使用 web 端進行開發與調試,首先啟動服務,使用指令 `go run cmd/api/main.go` 啟動 ,該服務默認連接埠為 `9999`,接著以 `debug` 模式啟動前端 flutter 項目即可。
|
||||||
|
|
||||||
## 翻譯
|
## 翻譯
|
||||||
|
|
||||||
|
Gopeed 的翻譯文件位於 `ui/flutter/lib/i18n/langs` 目錄中,只需要修改或新建翻譯文件即可。
|
||||||
|
|
||||||
Flutter 翻譯檔位於 `ui/flutter/lib/l10n`,請以 `app_en.arb` 為來源範本,修改或新增
|
|
||||||
`app_<locale>.arb`(例如 `app_de.arb` 或 `app_zh_TW.arb`)。每個語言都必須與英文範本包含相同的訊息 key。
|
|
||||||
|
|
||||||
只翻譯訊息值,不要修改 `{count}`、`{name}` 等大括號中的變數名稱。以 `@` 開頭的中繼資料 key 不需要翻譯。
|
請以 `en_us.dart` 作為參照。
|
||||||
提交時只需包含修改過的 ARB 來源檔,不需要產生或提交 `app_localizations*.dart`。PR 的 CI
|
|
||||||
會自動驗證所有語言、產生國際化程式碼並檢查整合結果。
|
|
||||||
|
|
||||||
以下範例同時列出 ARB 中填寫的訊息定義,以及程式傳值後使用者最終看到的內容。
|
|
||||||
|
|
||||||
1. 在訊息中插入變數
|
|
||||||
|
|
||||||
ARB 定義:
|
|
||||||
|
|
||||||
```json
|
|
||||||
"welcomeUser": "你好,{name}"
|
|
||||||
```
|
|
||||||
|
|
||||||
最終顯示:當 `name = 小明` 時,顯示「你好,小明」。翻譯時可以調整整句話,但必須保留 `{name}`。
|
|
||||||
|
|
||||||
2. 依數量顯示不同訊息
|
|
||||||
|
|
||||||
ARB 定義:
|
|
||||||
|
|
||||||
```json
|
|
||||||
"fileCount": "{count, plural, =0{沒有檔案} =1{1 個檔案} other{{count} 個檔案}}"
|
|
||||||
```
|
|
||||||
|
|
||||||
最終顯示:
|
|
||||||
|
|
||||||
- `count = 0` →「沒有檔案」
|
|
||||||
- `count = 1` →「1 個檔案」
|
|
||||||
- `count = 3` →「3 個檔案」
|
|
||||||
|
|
||||||
`=0`、`=1`、`other` 分別代表數量為 0、數量為 1 和其他數量。只翻譯各分支大括號內顯示給使用者的文字。
|
|
||||||
|
|
||||||
3. 依變數值顯示不同訊息
|
|
||||||
|
|
||||||
ARB 定義:
|
|
||||||
|
|
||||||
```json
|
|
||||||
"taskState": "{state, select, running{執行中} paused{已暫停} other{未知狀態}}"
|
|
||||||
```
|
|
||||||
|
|
||||||
最終顯示:
|
|
||||||
|
|
||||||
- `state = running` →「執行中」
|
|
||||||
- `state = paused` →「已暫停」
|
|
||||||
- 其他值 →「未知狀態」
|
|
||||||
|
|
||||||
`running`、`paused`、`other` 是程式傳入的分支值,不能翻譯;只翻譯其後大括號內的顯示文字。
|
|
||||||
|
|
||||||
## flutter開發
|
## flutter開發
|
||||||
|
|
||||||
|
|||||||
@@ -1,20 +1,29 @@
|
|||||||
FROM golang:1.25.4-alpine3.22 AS go
|
FROM instrumentisto/flutter:3.16.8 AS flutter
|
||||||
|
WORKDIR /app
|
||||||
|
COPY ./ui/flutter/pubspec.yaml ./ui/flutter/pubspec.lock ./
|
||||||
|
RUN flutter pub get
|
||||||
|
COPY ./ui/flutter ./
|
||||||
|
RUN flutter build web --web-renderer html
|
||||||
|
|
||||||
|
FROM golang:1.21.6 AS go
|
||||||
WORKDIR /app
|
WORKDIR /app
|
||||||
COPY ./go.mod ./go.sum ./
|
COPY ./go.mod ./go.sum ./
|
||||||
RUN go mod download
|
RUN go mod download
|
||||||
COPY . .
|
COPY . .
|
||||||
|
COPY --from=flutter /app/build/web ./cmd/web/dist
|
||||||
ARG VERSION=dev
|
ARG VERSION=dev
|
||||||
RUN CGO_ENABLED=0 go build -tags nosqlite,web \
|
RUN CGO_ENABLED=0 go build -tags nosqlite,web \
|
||||||
-ldflags="-s -w -X github.com/GopeedLab/gopeed/pkg/base.Version=$VERSION -X github.com/GopeedLab/gopeed/pkg/base.InDocker=true" \
|
-ldflags="-s -w -X github.com/GopeedLab/gopeed/pkg/base.Version=$VERSION -X github.com/GopeedLab/gopeed/pkg/base.InDocker=true" \
|
||||||
-o dist/gopeed github.com/GopeedLab/gopeed/cmd/web
|
-o dist/gopeed github.com/GopeedLab/gopeed/cmd/web
|
||||||
|
|
||||||
FROM alpine:3.23
|
FROM alpine:3.14.2
|
||||||
LABEL maintainer="monkeyWie"
|
LABEL maintainer="monkeyWie"
|
||||||
WORKDIR /app
|
WORKDIR /app
|
||||||
COPY --from=go /app/dist/gopeed ./
|
COPY --from=go /app/dist/gopeed ./
|
||||||
COPY entrypoint.sh ./entrypoint.sh
|
COPY entrypoint.sh ./entrypoint.sh
|
||||||
RUN apk update && \
|
RUN apk update && \
|
||||||
apk add --no-cache su-exec ; \
|
apk upgrade --no-cache && \
|
||||||
|
apk add --no-cache bash su-exec; \
|
||||||
chmod +x ./entrypoint.sh && \
|
chmod +x ./entrypoint.sh && \
|
||||||
rm -rf /var/cache/apk/*
|
rm -rf /var/cache/apk/*
|
||||||
VOLUME ["/app/storage"]
|
VOLUME ["/app/storage"]
|
||||||
|
|||||||
@@ -1,129 +1,139 @@
|
|||||||
# [](https://gopeed.com)
|
|
||||||
|
|
||||||
[](https://github.com/GopeedLab/gopeed/actions?query=workflow%3Atest)
|
[](https://github.com/GopeedLab/gopeed/actions?query=workflow%3Atest)
|
||||||
[](https://codecov.io/gh/GopeedLab/gopeed)
|
[](https://codecov.io/gh/GopeedLab/gopeed)
|
||||||
[](https://github.com/GopeedLab/gopeed/releases)
|
[](https://github.com/GopeedLab/gopeed/releases)
|
||||||
[](https://github.com/GopeedLab/gopeed/releases)
|
[](https://github.com/GopeedLab/gopeed/releases)
|
||||||
[](https://gopeed.com/docs/donate)
|
[](https://docs.gopeed.com/donate.html)
|
||||||
[](https://raw.githubusercontent.com/GopeedLab/gopeed/main/_docs/img/weixin.png)
|
|
||||||
[](https://discord.gg/ZUJqJrwCGB)
|
[](https://discord.gg/ZUJqJrwCGB)
|
||||||
|
|
||||||
<a href="https://trendshift.io/repositories/7953" target="_blank"><img src="https://trendshift.io/api/badge/repositories/7953" alt="GopeedLab%2Fgopeed | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
|

|
||||||
|
|
||||||
|
[](https://ko-fi.com/R6R6IJGN6)
|
||||||
|
|
||||||
[English](/README.md) | [中文](/README_zh-CN.md) | [日本語](/README_ja-JP.md) | [正體中文](/README_zh-TW.md) | [Tiếng Việt](/README_vi-VN.md)
|
[English](/README.md) | [中文](/README_zh-CN.md) | [日本語](/README_ja-JP.md) | [正體中文](/README_zh-TW.md) | [Tiếng Việt](/README_vi-VN.md)
|
||||||
|
|
||||||
## 🚀 Introduction
|
## Introduction
|
||||||
|
|
||||||
Gopeed (short for **Go Speed**) is a fast, modern, free, and open-source download manager built with Go and Flutter. It supports HTTP, HTTPS, BitTorrent, magnet links, and ed2k on desktop, mobile, and the web.
|
Gopeed (full name Go Speed), a high-speed downloader developed by `Golang` + `Flutter`, supports (HTTP, BitTorrent, Magnet) protocol, and supports all platforms. In addition to basic download functions, Gopeed is also a highly customizable downloader that supports implementing more features through integration with [APIs](https://docs.gopeed.com/dev-api.html) or installation and development of [extensions](https://docs.gopeed.com/dev-extension.html).
|
||||||
|
|
||||||
Beyond core download management, Gopeed offers browser integration, JavaScript extensions, a REST API, a CLI, and a self-hosted web UI for customization and automation.
|
Visit ✈ [Official Website](https://gopeed.com) | 📖 [Official Docs](https://docs.gopeed.com)
|
||||||
|
|
||||||
Visit ✈ [Official Website](https://gopeed.com)
|
## Download
|
||||||
|
|
||||||

|
<table>
|
||||||
|
<thead>
|
||||||
|
<tr>
|
||||||
|
<th>Platform</th>
|
||||||
|
<th>Package Type</th>
|
||||||
|
<th>Download Link</th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody>
|
||||||
|
<tr>
|
||||||
|
<td rowspan=2>Windows</td>
|
||||||
|
<td><code>EXE Installer</code></td>
|
||||||
|
<td><a href="https://gopeed.com/api/download?tpl=Gopeed-$version-windows-amd64.zip">Link</a></td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td><code>Portable ZIP</code></td>
|
||||||
|
<td><a href="https://gopeed.com/api/download?tpl=Gopeed-$version-windows-amd64-portable.zip">Link</a></td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td>MacOS</td>
|
||||||
|
<td><code>DMG Installer</code></td>
|
||||||
|
<td><a href="https://gopeed.com/api/download?tpl=Gopeed-$version-macos.dmg">Link</a></td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td rowspan=4>Linux</td>
|
||||||
|
<td><code>Flathub</code></td>
|
||||||
|
<td><a href="https://flathub.org/apps/com.gopeed.Gopeed">Link</a></td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td><code>SNAP</code></td>
|
||||||
|
<td><a href="https://snapcraft.io/gopeed">Link</a></td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td><code>DEB</code></td>
|
||||||
|
<td><a href="https://gopeed.com/api/download?tpl=Gopeed-$version-linux-amd64.deb">Link</a></td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td><code>AppImage</code></td>
|
||||||
|
<td><a href="https://gopeed.com/api/download?tpl=Gopeed-$version-linux-x86_64.AppImage">Link</a></td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td>Android</td>
|
||||||
|
<td><code>APK</code></td>
|
||||||
|
<td><a href="https://gopeed.com/api/download?tpl=Gopeed-$version-android.apk">Link</a></td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td>iOS</td>
|
||||||
|
<td><code>IPA</code></td>
|
||||||
|
<td><a href="https://gopeed.com/api/download?tpl=Gopeed-$version-ios.ipa">Link</a></td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td>Web</td>
|
||||||
|
<td></td>
|
||||||
|
<td><a href="https://github.com/GopeedLab/gopeed/releases/latest">Link</a></td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td>Docker</td>
|
||||||
|
<td></td>
|
||||||
|
<td><a href="https://hub.docker.com/r/liwei2633/gopeed">Link</a></td>
|
||||||
|
</tr>
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
|
||||||
## ✨ Features
|
More about installation, please refer to [Installation](https://docs.gopeed.com/install.html)
|
||||||
|
|
||||||
- ⚡ **High-speed downloads** — combine concurrent tasks, multi-connection HTTP transfers, and peer-to-peer BitTorrent downloads to make the most of your bandwidth.
|
### Command tool
|
||||||
- 🧲 **Multiple protocols** — download HTTP/HTTPS files, torrents, magnet links, and ed2k resources from a single app.
|
|
||||||
- 🌱 **Full-featured BitTorrent** — use DHT peer discovery, uTP transport, Web Seeds, selective file downloads, tracker management, peer and piece statistics, and ratio- or time-based seeding limits.
|
|
||||||
- 📋 **Flexible task management** — pause, resume, retry, run batch operations, search, filter by status, organize with categories, and recover tasks after a restart.
|
|
||||||
- 🪶 **Lightweight native experience** — the main interface is rendered natively with Flutter. No Electron. No WebView shell. Enjoy a smaller footprint, lower overhead, and responsive performance.
|
|
||||||
- 💻 **Cross-platform** — available for Windows, macOS, Linux, Android, iOS, and the web, with Docker and QNAP deployment options.
|
|
||||||
- 🎨 **Customizable appearance** — follow your system theme or choose light or dark mode, with eight accent colors.
|
|
||||||
- 📐 **Responsive interface** — task lists, navigation, settings, and detail views adapt to phones, tablets, and resizable desktop windows.
|
|
||||||
- 🗣️ **Available in 20+ languages** — including English, Simplified and Traditional Chinese, Japanese, Korean, and many more.
|
|
||||||
- 🌐 **Browser integration** — send downloads from Chrome, Edge, Firefox, and other compatible browsers directly to Gopeed.
|
|
||||||
- 🧩 **JavaScript extensions** — add support for video platforms, AI model hubs, cloud storage services, and other download sources.
|
|
||||||
- 🤖 **AI integration** — use Gopeed's MCP endpoint to connect compatible AI agents and create, inspect, or manage downloads with natural language.
|
|
||||||
- 🔌 **Automation-ready** — integrate with Gopeed through its REST API, CLI, authenticated web UI, webhooks, and post-download scripts.
|
|
||||||
- 🛠️ **Built-in essentials** — customize headers and the User-Agent, use proxies and GitHub mirrors, receive notifications, and extract archives automatically.
|
|
||||||
|
|
||||||
## 🤖 AI Integration
|
use `go install`:
|
||||||
|
|
||||||
Connect Gopeed to an AI agent and manage downloads with natural language. For example, you can say:
|
|
||||||
|
|
||||||
> Download the latest Gopeed client for Windows.
|
|
||||||
|
|
||||||
| Tool | Description |
|
|
||||||
| --- | --- |
|
|
||||||
| `resolve_task` | Resolve a download URL or URI and return its resource metadata and files before creating a task. |
|
|
||||||
| `create_task` | Create and start a task from a resolved resource ID or a direct download request. |
|
|
||||||
| `list_tasks` | List tasks, optionally filtering them by ID or status. |
|
|
||||||
| `get_task` | Get the request, resource, options, and current progress for one task. |
|
|
||||||
| `get_task_status` | Get lightweight runtime status and per-file progress for one task. |
|
|
||||||
| `get_task_stats` | Get protocol-specific statistics, including HTTP connections or BitTorrent peers and seeding data. |
|
|
||||||
| `pause_task` | Pause a task. |
|
|
||||||
| `continue_task` | Continue a paused or failed task. |
|
|
||||||
| `delete_task` | Delete a task, optionally deleting its downloaded files. |
|
|
||||||
|
|
||||||
## ⬇️ Download
|
|
||||||
|
|
||||||
### 🧪 Gopeed 2.0.0 Beta
|
|
||||||
|
|
||||||
Gopeed 2.0.0 is currently in public beta, introducing a redesigned interface, a native communication architecture that connects desktop and mobile clients directly to the Go core through FFI, a more consistent cross-platform experience, improved task management, more flexible API support, and MCP-based AI agent integration. Some features may still be incomplete or unstable, so please try it and report any issues you encounter.
|
|
||||||
|
|
||||||
- [Download Gopeed 2.0.0 Beta 3](https://github.com/GopeedLab/gopeed/releases/tag/v2.0.0-beta.3)
|
|
||||||
|
|
||||||
Once the features and stability meet our release standards, we will publish the official Gopeed 2.0.0 release. Beta users will be able to upgrade directly to the final release, while existing stable users will not be automatically moved onto the beta channel.
|
|
||||||
|
|
||||||
### Stable release
|
|
||||||
|
|
||||||
- [Official Download](https://gopeed.com)
|
|
||||||
- [GitHub Releases](https://github.com/GopeedLab/gopeed/releases/latest)
|
|
||||||
|
|
||||||
### 🛠️ Command-line tool
|
|
||||||
|
|
||||||
Install the CLI with `go install`:
|
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
go install github.com/GopeedLab/gopeed/cmd/gopeed@latest
|
go install github.com/GopeedLab/gopeed/cmd/gopeed@latest
|
||||||
```
|
```
|
||||||
|
|
||||||
## 🔌 Browser Extension
|
## Browser Extension
|
||||||
|
|
||||||
Use the Gopeed browser extension to send downloads from Chrome, Edge, Firefox, and other compatible browsers directly to Gopeed: [GopeedLab/browser-extension](https://github.com/GopeedLab/browser-extension)
|
Gopeed also provides a browser extension to take over browser downloads, supporting browsers such as Chrome, Edge, Firefox, etc., please refer to: [https://github.com/GopeedLab/browser-extension](https://github.com/GopeedLab/browser-extension)
|
||||||
|
|
||||||
## 📱 WeChat Official Account
|
## Donate
|
||||||
|
|
||||||
Follow Gopeed's official WeChat account for updates and news.
|
If you like this project, please consider [donating](https://docs.gopeed.com/donate.html) to support the development of this project, thank you!
|
||||||
|
|
||||||
<img src="_docs/img/weixin.png" width="200" />
|
## Showcase
|
||||||
|
|
||||||
## 💝 Donate
|

|
||||||
|
|
||||||
If Gopeed is useful to you, please consider [supporting its development](https://gopeed.com/docs/donate). Thank you!
|
## Development
|
||||||
|
|
||||||
## 👨💻 Development
|
This project is divided into two parts, the front end uses `flutter`, the back end uses `Golang`, and the two sides communicate through the `http` protocol. On the unix system, `unix socket` is used, and on the windows system, `tcp` protocol is used.
|
||||||
|
|
||||||
Gopeed consists of a Flutter front end and a Go back end. They communicate over HTTP, using Unix sockets on Unix-like systems and TCP on Windows.
|
> The front code is located in the `ui/flutter` directory.
|
||||||
|
|
||||||
> The front-end source is located in the `ui/flutter` directory.
|
### Environment
|
||||||
|
|
||||||
### 🌍 Environment
|
1. Golang 1.21+
|
||||||
|
2. Flutter 3.16+
|
||||||
|
|
||||||
1. Go 1.25+
|
### Clone
|
||||||
2. Flutter 3.41+
|
|
||||||
|
|
||||||
### 📋 Clone
|
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
git clone [email protected]:GopeedLab/gopeed.git
|
git clone [email protected]:GopeedLab/gopeed.git
|
||||||
```
|
```
|
||||||
|
|
||||||
### 🤝 Contributing
|
### Contributing
|
||||||
|
|
||||||
See [CONTRIBUTING.md](/CONTRIBUTING.md).
|
Please refer to [CONTRIBUTING.md](/CONTRIBUTING.md)
|
||||||
|
|
||||||
### 🏗️ Build
|
### Build
|
||||||
|
|
||||||
#### Desktop
|
#### Desktop
|
||||||
|
|
||||||
Set up Flutter desktop development using the official [Flutter desktop guide](https://docs.flutter.dev/development/platform-integration/desktop), and make sure a working C toolchain is available for cgo. Then run the commands for your platform.
|
First, you need to configure the environment according to the official [Flutter desktop website documention](https://docs.flutter.dev/development/platform-integration/desktop), then you will need to ensure the cgo environment is set up accordingly. For detailed instructions on setting up the cgo environment, please refer to relevant resources available online.
|
||||||
|
|
||||||
Commands:
|
command:
|
||||||
|
|
||||||
- Windows
|
- windows
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
go build -tags nosqlite -ldflags="-w -s" -buildmode=c-shared -o ui/flutter/windows/libgopeed.dll github.com/GopeedLab/gopeed/bind/desktop
|
go build -tags nosqlite -ldflags="-w -s" -buildmode=c-shared -o ui/flutter/windows/libgopeed.dll github.com/GopeedLab/gopeed/bind/desktop
|
||||||
@@ -131,7 +141,7 @@ cd ui/flutter
|
|||||||
flutter build windows
|
flutter build windows
|
||||||
```
|
```
|
||||||
|
|
||||||
- macOS
|
- macos
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
go build -tags nosqlite -ldflags="-w -s" -buildmode=c-shared -o ui/flutter/macos/Frameworks/libgopeed.dylib github.com/GopeedLab/gopeed/bind/desktop
|
go build -tags nosqlite -ldflags="-w -s" -buildmode=c-shared -o ui/flutter/macos/Frameworks/libgopeed.dylib github.com/GopeedLab/gopeed/bind/desktop
|
||||||
@@ -139,7 +149,7 @@ cd ui/flutter
|
|||||||
flutter build macos
|
flutter build macos
|
||||||
```
|
```
|
||||||
|
|
||||||
- Linux
|
- linux
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
go build -tags nosqlite -ldflags="-w -s" -buildmode=c-shared -o ui/flutter/linux/bundle/lib/libgopeed.so github.com/GopeedLab/gopeed/bind/desktop
|
go build -tags nosqlite -ldflags="-w -s" -buildmode=c-shared -o ui/flutter/linux/bundle/lib/libgopeed.so github.com/GopeedLab/gopeed/bind/desktop
|
||||||
@@ -149,7 +159,7 @@ flutter build linux
|
|||||||
|
|
||||||
#### Mobile
|
#### Mobile
|
||||||
|
|
||||||
Mobile builds also require a working cgo toolchain. Install and initialize `gomobile`:
|
Same as before, you also need to prepare the `cgo` environment, and then install `gomobile`:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
go install golang.org/x/mobile/cmd/gomobile@latest
|
go install golang.org/x/mobile/cmd/gomobile@latest
|
||||||
@@ -157,17 +167,17 @@ go get golang.org/x/mobile/bind
|
|||||||
gomobile init
|
gomobile init
|
||||||
```
|
```
|
||||||
|
|
||||||
Commands:
|
command:
|
||||||
|
|
||||||
- Android
|
- android
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
gomobile bind -tags nosqlite -ldflags="-w -s -checklinkname=0" -o ui/flutter/android/app/libs/libgopeed.aar -target=android -androidapi 21 -javapkg="com.gopeed" github.com/GopeedLab/gopeed/bind/mobile
|
gomobile bind -tags nosqlite -ldflags="-w -s" -o ui/flutter/android/app/libs/libgopeed.aar -target=android -androidapi 19 -javapkg="com.gopeed" github.com/GopeedLab/gopeed/bind/mobile
|
||||||
cd ui/flutter
|
cd ui/flutter
|
||||||
flutter build apk
|
flutter build apk
|
||||||
```
|
```
|
||||||
|
|
||||||
- iOS
|
- ios
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
gomobile bind -tags nosqlite -ldflags="-w -s" -o ui/flutter/ios/Frameworks/Libgopeed.xcframework -target=ios github.com/GopeedLab/gopeed/bind/mobile
|
gomobile bind -tags nosqlite -ldflags="-w -s" -o ui/flutter/ios/Frameworks/Libgopeed.xcframework -target=ios github.com/GopeedLab/gopeed/bind/mobile
|
||||||
@@ -177,7 +187,7 @@ flutter build ios --no-codesign
|
|||||||
|
|
||||||
#### Web
|
#### Web
|
||||||
|
|
||||||
Build the web app and server:
|
command:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
cd ui/flutter
|
cd ui/flutter
|
||||||
@@ -188,18 +198,18 @@ cp -r ui/flutter/build/web cmd/web/dist
|
|||||||
go build -tags nosqlite,web -ldflags="-s -w" -o bin/ github.com/GopeedLab/gopeed/cmd/web
|
go build -tags nosqlite,web -ldflags="-s -w" -o bin/ github.com/GopeedLab/gopeed/cmd/web
|
||||||
```
|
```
|
||||||
|
|
||||||
## ❤️ Credits
|
## Credits
|
||||||
|
|
||||||
### 👥 Contributors
|
### Contributors
|
||||||
|
|
||||||
<a href="https://github.com/GopeedLab/gopeed/graphs/contributors">
|
<a href="https://github.com/GopeedLab/gopeed/graphs/contributors">
|
||||||
<img src="https://contrib.rocks/image?repo=GopeedLab/gopeed" />
|
<img src="https://contrib.rocks/image?repo=GopeedLab/gopeed" />
|
||||||
</a>
|
</a>
|
||||||
|
|
||||||
### 🏢 JetBrains
|
### JetBrains
|
||||||
|
|
||||||
[](https://www.jetbrains.com/?from=gopeed)
|
[](https://www.jetbrains.com/?from=gopeed)
|
||||||
|
|
||||||
## 📄 License
|
## License
|
||||||
|
|
||||||
[GPLv3](LICENSE)
|
[GPLv3](LICENSE)
|
||||||
@@ -1,111 +1,119 @@
|
|||||||
# [](https://gopeed.com)
|
|
||||||
|
|
||||||
[](https://github.com/GopeedLab/gopeed/actions?query=workflow%3Atest)
|
[](https://github.com/GopeedLab/gopeed/actions?query=workflow%3Atest)
|
||||||
[](https://codecov.io/gh/GopeedLab/gopeed)
|
[](https://codecov.io/gh/GopeedLab/gopeed)
|
||||||
[](https://github.com/GopeedLab/gopeed/releases)
|
[](https://github.com/GopeedLab/gopeed/releases)
|
||||||
[](https://github.com/GopeedLab/gopeed/releases)
|
[](https://github.com/GopeedLab/gopeed/releases)
|
||||||
[](https://gopeed.com/docs/donate)
|
[](https://docs.gopeed.com/donate.html)
|
||||||
[](https://raw.githubusercontent.com/GopeedLab/gopeed/main/_docs/img/weixin.png)
|
|
||||||
[](https://discord.gg/ZUJqJrwCGB)
|
[](https://discord.gg/ZUJqJrwCGB)
|
||||||
|
|
||||||
<a href="https://trendshift.io/repositories/7953" target="_blank"><img src="https://trendshift.io/api/badge/repositories/7953" alt="GopeedLab%2Fgopeed | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
|

|
||||||
|
|
||||||
|
[](https://ko-fi.com/R6R6IJGN6)
|
||||||
|
|
||||||
[English](/README.md) | [中文](/README_zh-CN.md) | [日本語](/README_ja-JP.md) | [正體中文](/README_zh-TW.md) | [Tiếng Việt](/README_vi-VN.md)
|
[English](/README.md) | [中文](/README_zh-CN.md) | [日本語](/README_ja-JP.md) | [正體中文](/README_zh-TW.md) | [Tiếng Việt](/README_vi-VN.md)
|
||||||
|
|
||||||
## 🚀 はじめに
|
## はじめに
|
||||||
|
|
||||||
Gopeed(**Go Speed** の略)は、Go と Flutter で開発された高速でモダンな無料オープンソース・ダウンロードマネージャーです。HTTP、HTTPS、BitTorrent、Magnet、ed2k に対応し、デスクトップ、モバイル、Web で利用できます。
|
Gopeed (正式名 Go Speed) は `Golang` + `Flutter` によって開発された高速ダウンローダーで、(HTTP、BitTorrent、Magnet) プロトコルをサポートし、すべてのプラットフォームをサポートします。基本的なダウンロード機能に加え、[APIs](https://docs.gopeed.com/dev-api.html)との連動や[拡張機能](https://docs.gopeed.com/dev-extension.html)のインストール・開発による追加機能にも対応した、カスタマイズ性の高いダウンローダーです。
|
||||||
|
|
||||||
ブラウザー連携、JavaScript 拡張、REST API、CLI、セルフホスト可能な Web UI により、日常利用から高度な自動化まで幅広く対応します。
|
見て下さい ✈ [公式ウェブサイト](https://gopeed.com) | 📖 [開発ドキュメント](https://docs.gopeed.com)
|
||||||
|
|
||||||
✈ [公式サイト](https://gopeed.com)
|
## インストール
|
||||||
|
|
||||||

|
<table>
|
||||||
|
<thead>
|
||||||
|
<tr>
|
||||||
|
<th>Platform</th>
|
||||||
|
<th>Package Type</th>
|
||||||
|
<th>Download Link</th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody>
|
||||||
|
<tr>
|
||||||
|
<td rowspan=2>Windows</td>
|
||||||
|
<td><code>EXE Installer</code></td>
|
||||||
|
<td><a href="https://gopeed.com/api/download?tpl=Gopeed-$version-windows-amd64.zip">Link</a></td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td><code>Portable ZIP</code></td>
|
||||||
|
<td><a href="https://gopeed.com/api/download?tpl=Gopeed-$version-windows-amd64-portable.zip">Link</a></td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td>MacOS</td>
|
||||||
|
<td><code>DMG Installer</code></td>
|
||||||
|
<td><a href="https://gopeed.com/api/download?tpl=Gopeed-$version-macos.dmg">Link</a></td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td rowspan=4>Linux</td>
|
||||||
|
<td><code>Flathub</code></td>
|
||||||
|
<td><a href="https://flathub.org/apps/com.gopeed.Gopeed">Link</a></td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td><code>SNAP</code></td>
|
||||||
|
<td><a href="https://snapcraft.io/gopeed">Link</a></td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td><code>DEB</code></td>
|
||||||
|
<td><a href="https://gopeed.com/api/download?tpl=Gopeed-$version-linux-amd64.deb">Link</a></td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td><code>AppImage</code></td>
|
||||||
|
<td><a href="https://gopeed.com/api/download?tpl=Gopeed-$version-linux-x86_64.AppImage">Link</a></td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td>Android</td>
|
||||||
|
<td><code>APK</code></td>
|
||||||
|
<td><a href="https://gopeed.com/api/download?tpl=Gopeed-$version-android.apk">Link</a></td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td>iOS</td>
|
||||||
|
<td><code>IPA</code></td>
|
||||||
|
<td><a href="https://gopeed.com/api/download?tpl=Gopeed-$version-ios.ipa">Link</a></td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td>Web</td>
|
||||||
|
<td></td>
|
||||||
|
<td><a href="https://github.com/GopeedLab/gopeed/releases/latest">Link</a></td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td>Docker</td>
|
||||||
|
<td></td>
|
||||||
|
<td><a href="https://hub.docker.com/r/liwei2633/gopeed">Link</a></td>
|
||||||
|
</tr>
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
|
||||||
## ✨ 主な機能
|
インストールについての詳細は、[インストール](https://docs.gopeed.com/install.html)を参照してください。
|
||||||
|
|
||||||
- ⚡ **高速ダウンロード** — 複数タスク、HTTP のマルチ接続分割転送、BitTorrent の P2P ダウンロードで帯域を最大限に活用します。
|
## 寄付
|
||||||
- 🧲 **マルチプロトコル** — HTTP、HTTPS、BitTorrent、Magnet、ed2k を一つの画面で管理します。
|
|
||||||
- 🌱 **充実した BT 機能** — DHT によるピア検出、uTP 転送、Web Seed、ファイル選択、Tracker 管理、Peer/ピース統計、共有率または時間によるシード制御。
|
|
||||||
- 📋 **タスク管理** — 一時停止、再開、再試行、一括操作、検索、状態フィルター、カテゴリ、起動時復元。
|
|
||||||
- 🪶 **軽量なネイティブ体験** — メイン UI は Flutter でネイティブ描画。Electron 非採用。WebView ラッパーでもありません。パッケージサイズと実行時オーバーヘッドを抑え、軽快な応答性を実現します。
|
|
||||||
- 💻 **クロスプラットフォーム** — Windows、macOS、Linux、Android、iOS、Web、Docker、QNAP に対応。
|
|
||||||
- 🎨 **カスタマイズ可能なテーマ** — システム連動、ライト、ダークの各モードと 8 色のアクセントカラーに対応。
|
|
||||||
- 📐 **レスポンシブ UI** — タスクリスト、ナビゲーション、設定、詳細画面がスマートフォン、タブレット、サイズ変更可能なデスクトップウィンドウに適応。
|
|
||||||
- 🗣️ **20 以上の UI 言語** — 日本語、英語、簡体字・繁体字中国語、韓国語など多くの言語に対応。
|
|
||||||
- 🌐 **ブラウザー連携** — Chrome、Edge、Firefox などのダウンロードを Gopeed に送信できます。
|
|
||||||
- 🧩 **JavaScript 拡張** — 動画サイト、AI モデルハブ、クラウドストレージなどのダウンロード元を追加できます。
|
|
||||||
- 🤖 **AI 連携** — MCP インターフェースを通じて対応する AI Agent と連携し、自然言語でダウンロードタスクを作成・確認・管理できます。
|
|
||||||
- 🔌 **オープンなインターフェース** — REST API、CLI、認証付き Web UI、Webhook、完了後スクリプトに対応。
|
|
||||||
- 🛠️ **便利な内蔵機能** — カスタム Header/User-Agent、Proxy、GitHub ミラー、通知、自動展開。
|
|
||||||
|
|
||||||
## 🤖 AI 連携
|
もしこのプロジェクトがお気に召しましたら、このプロジェクトの発展を支援するために[寄付](https://docs.gopeed.com/donate.html)をご検討ください!
|
||||||
|
|
||||||
Gopeed を AI Agent と連携すると、自然言語でダウンロードを管理できます。たとえば、AI Agent に次のように依頼できます:
|
## ショーケース
|
||||||
|
|
||||||
> 最新の Gopeed Windows クライアントをダウンロードして
|

|
||||||
|
|
||||||
| Tool | 説明 |
|
## 開発
|
||||||
| --- | --- |
|
|
||||||
| `resolve_task` | ダウンロード URL または URI を解析し、タスク作成前にリソース情報とファイル一覧を返します。 |
|
|
||||||
| `create_task` | 解析済みのリソース ID または直接のダウンロードリクエストからタスクを作成して開始します。 |
|
|
||||||
| `list_tasks` | タスクを一覧表示し、必要に応じて ID または状態で絞り込みます。 |
|
|
||||||
| `get_task` | 一つのタスクについて、リクエスト、リソース、オプション、現在の進捗を取得します。 |
|
|
||||||
| `get_task_status` | 一つのタスクの簡易的な実行状態とファイル別の進捗を取得します。 |
|
|
||||||
| `get_task_stats` | HTTP 接続数や BitTorrent の Peer・シード情報など、プロトコル固有の統計を取得します。 |
|
|
||||||
| `pause_task` | タスクを一時停止します。 |
|
|
||||||
| `continue_task` | 一時停止または失敗したタスクを再開します。 |
|
|
||||||
| `delete_task` | タスクを削除し、必要に応じてダウンロード済みファイルも削除します。 |
|
|
||||||
|
|
||||||
## ⬇️ インストール
|
|
||||||
|
|
||||||
### 🧪 Gopeed 2.0.0 Beta
|
|
||||||
|
|
||||||
Gopeed 2.0.0 は現在パブリックベータ版です。刷新されたインターフェース、デスクトップおよびモバイルから FFI で Go コアへ直接接続するネイティブ通信アーキテクチャ、より統一されたクロスプラットフォーム体験、強化されたタスク管理、より柔軟な API、MCP による AI Agent 連携機能を導入しています。一部の機能はまだ不完全または不安定な場合がありますので、ぜひお試しいただき、問題があればフィードバックをお寄せください。
|
|
||||||
|
|
||||||
- [Gopeed 2.0.0 Beta 3 をダウンロード](https://github.com/GopeedLab/gopeed/releases/tag/v2.0.0-beta.3)
|
|
||||||
|
|
||||||
機能と安定性が正式リリースの基準に達した時点で、Gopeed 2.0.0 正式版を公開します。ベータ版のユーザーは正式版へ直接アップデートできます。既存の安定版ユーザーが自動的にベータチャンネルへ移行することはありません。
|
|
||||||
|
|
||||||
### 安定版
|
|
||||||
|
|
||||||
- [公式サイトからダウンロード](https://gopeed.com)
|
|
||||||
- [GitHub Releases](https://github.com/GopeedLab/gopeed/releases/latest)
|
|
||||||
|
|
||||||
### 🛠️ コマンドツール
|
|
||||||
|
|
||||||
## 📱 WeChat 公式アカウント
|
|
||||||
|
|
||||||
公式アカウントをフォローして、最新のアップデートやニュースを入手してください。
|
|
||||||
|
|
||||||
<img src="_docs/img/weixin.png" width="200" />
|
|
||||||
|
|
||||||
## 💝 寄付
|
|
||||||
|
|
||||||
もしこのプロジェクトがお気に召しましたら、このプロジェクトの発展を支援するために[寄付](https://gopeed.com/docs/donate)をご検討ください!
|
|
||||||
|
|
||||||
## 👨💻 開発
|
|
||||||
|
|
||||||
このプロジェクトは二つの部分に分かれており、フロントエンドでは `flutter` を、バックエンドでは `Golang` を使用し、両者は `http` プロトコルで通信する。ユニックスシステムでは `unix socket` を、ウィンドウズシステムでは `tcp` プロトコルを使用します。
|
このプロジェクトは二つの部分に分かれており、フロントエンドでは `flutter` を、バックエンドでは `Golang` を使用し、両者は `http` プロトコルで通信する。ユニックスシステムでは `unix socket` を、ウィンドウズシステムでは `tcp` プロトコルを使用します。
|
||||||
|
|
||||||
> フロントコードは `ui/flutter` ディレクトリにあります。
|
> フロントコードは `ui/flutter` ディレクトリにあります。
|
||||||
|
|
||||||
### 🌍 環境
|
### 環境
|
||||||
|
|
||||||
1. Go 言語 1.25+
|
1. Go 言語 1.21+
|
||||||
2. Flutter 3.41+
|
2. Flutter 3.16+
|
||||||
|
|
||||||
### 📋 クローン
|
### クローン
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
git clone [email protected]:GopeedLab/gopeed.git
|
git clone [email protected]:GopeedLab/gopeed.git
|
||||||
```
|
```
|
||||||
|
|
||||||
### 🤝 コントリビュート
|
### コントリビュート
|
||||||
|
|
||||||
[CONTRIBUTING.md](/CONTRIBUTING_ja-JP.md) をご参照ください
|
[CONTRIBUTING.md](/CONTRIBUTING_ja-JP.md) をご参照ください
|
||||||
|
|
||||||
### 🏗️ ビルド
|
### ビルド
|
||||||
|
|
||||||
#### デスクトップ
|
#### デスクトップ
|
||||||
|
|
||||||
@@ -152,7 +160,7 @@ gomobile init
|
|||||||
- android
|
- android
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
gomobile bind -tags nosqlite -ldflags="-w -s -checklinkname=0" -o ui/flutter/android/app/libs/libgopeed.aar -target=android -androidapi 21 -javapkg="com.gopeed" github.com/GopeedLab/gopeed/bind/mobile
|
gomobile bind -tags nosqlite -ldflags="-w -s" -o ui/flutter/android/app/libs/libgopeed.aar -target=android -androidapi 19 -javapkg="com.gopeed" github.com/GopeedLab/gopeed/bind/mobile
|
||||||
cd ui/flutter
|
cd ui/flutter
|
||||||
flutter build apk
|
flutter build apk
|
||||||
```
|
```
|
||||||
@@ -178,7 +186,7 @@ cp -r ui/flutter/build/web cmd/web/dist
|
|||||||
go build -tags nosqlite,web -ldflags="-s -w" -o bin/ github.com/GopeedLab/gopeed/cmd/web
|
go build -tags nosqlite,web -ldflags="-s -w" -o bin/ github.com/GopeedLab/gopeed/cmd/web
|
||||||
```
|
```
|
||||||
|
|
||||||
## ❤️ 感謝
|
## 感謝
|
||||||
|
|
||||||
### コントリビューター
|
### コントリビューター
|
||||||
|
|
||||||
|
|||||||
@@ -1,78 +1,90 @@
|
|||||||
# [](https://gopeed.com)
|
|
||||||
|
|
||||||
[](https://github.com/GopeedLab/gopeed/actions?query=workflow%3Atest)
|
[](https://github.com/GopeedLab/gopeed/actions?query=workflow%3Atest)
|
||||||
[](https://codecov.io/gh/GopeedLab/gopeed)
|
[](https://codecov.io/gh/GopeedLab/gopeed)
|
||||||
[](https://github.com/GopeedLab/gopeed/releases)
|
[](https://github.com/GopeedLab/gopeed/releases)
|
||||||
[](https://github.com/GopeedLab/gopeed/releases)
|
[](https://github.com/GopeedLab/gopeed/releases)
|
||||||
[](https://gopeed.com/docs/donate)
|
[](https://docs.gopeed.com/donate.html)
|
||||||
[](https://raw.githubusercontent.com/GopeedLab/gopeed/main/_docs/img/weixin.png)
|
|
||||||
[](https://discord.gg/ZUJqJrwCGB)
|
[](https://discord.gg/ZUJqJrwCGB)
|
||||||
|
|
||||||
<a href="https://trendshift.io/repositories/7953" target="_blank"><img src="https://trendshift.io/api/badge/repositories/7953" alt="GopeedLab%2Fgopeed | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
|

|
||||||
|
|
||||||
|
[](https://ko-fi.com/R6R6IJGN6)
|
||||||
|
|
||||||
[English](/README.md) | [中文](/README_zh-CN.md) | [日本語](/README_ja-JP.md) | [正體中文](/README_zh-TW.md) | [Tiếng Việt](/README_vi-VN.md)
|
[English](/README.md) | [中文](/README_zh-CN.md) | [日本語](/README_ja-JP.md) | [正體中文](/README_zh-TW.md) | [Tiếng Việt](/README_vi-VN.md)
|
||||||
|
|
||||||
## 🚀 Giới thiệu
|
## Giới thiệu
|
||||||
|
|
||||||
Gopeed (viết tắt của **Go Speed**) là trình quản lý tải xuống nhanh, hiện đại, miễn phí và mã nguồn mở, được xây dựng bằng Go và Flutter. Gopeed hỗ trợ HTTP, HTTPS, BitTorrent, Magnet và ed2k trên máy tính, thiết bị di động và Web.
|
Gopeed (tên đầy đủ Go Speed), một công cụ tải xuống tốc độ cao được phát triển bởi `Golang` + `Flutter`, hỗ trợ giao thức (HTTP, BitTorrent, Magnet) và hỗ trợ tất cả các nền tảng. Ngoài các chức năng tải xuống cơ bản, Gopeed còn là một công cụ tải xuống có thể tùy chỉnh cao cho phép triển khai thêm tính năng thông qua việc tích hợp với [APIs](https://docs.gopeed.com/dev-api.html) hoặc cài đặt và phát triển các [tiện ích mở rộng](https://docs.gopeed.com/dev-extension.html).
|
||||||
|
|
||||||
Ngoài việc quản lý tải xuống hằng ngày, Gopeed còn hỗ trợ tích hợp trình duyệt, tiện ích JavaScript, REST API, CLI và Web UI có thể tự lưu trữ để mở rộng và tự động hóa quy trình.
|
Truy cập ✈ [Trang web chính thức](https://gopeed.com) | 📖 [Tài liệu chính thức](https://docs.gopeed.com)
|
||||||
|
|
||||||
Truy cập ✈ [Trang web chính thức](https://gopeed.com)
|
## Tải về
|
||||||
|
|
||||||

|
<table>
|
||||||
|
<thead>
|
||||||
|
<tr>
|
||||||
|
<th>Nền tảng</th>
|
||||||
|
<th>Loại gói</th>
|
||||||
|
<th>Liên kết tải xuống</th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody>
|
||||||
|
<tr>
|
||||||
|
<td rowspan=2>Windows</td>
|
||||||
|
<td><code>Bộ cài đặt EXE</code></td>
|
||||||
|
<td><a href="https://gopeed.com/api/download?tpl=Gopeed-$version-windows-amd64.zip">Liên kết</a></td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td><code>ZIP Portable</code></td>
|
||||||
|
<td><a href="https://gopeed.com/api/download?tpl=Gopeed-$version-windows-amd64-portable.zip">Liên kết</a></td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td>MacOS</td>
|
||||||
|
<td><code>Bộ cài đặt DMG</code></td>
|
||||||
|
<td><a href="https://gopeed.com/api/download?tpl=Gopeed-$version-macos.dmg">Liên kết</a></td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td rowspan=4>Linux</td>
|
||||||
|
<td><code>Flathub</code></td>
|
||||||
|
<td><a href="https://flathub.org/apps/com.gopeed.Gopeed">Liên kết</a></td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td><code>SNAP</code></td>
|
||||||
|
<td><a href="https://snapcraft.io/gopeed">Liên kết</a></td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td><code>DEB</code></td>
|
||||||
|
<td><a href="https://gopeed.com/api/download?tpl=Gopeed-$version-linux-amd64.deb">Liên kết</a></td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td><code>AppImage</code></td>
|
||||||
|
<td><a href="https://gopeed.com/api/download?tpl=Gopeed-$version-linux-x86_64.AppImage">Liên kết</a></td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td>Android</td>
|
||||||
|
<td><code>APK</code></td>
|
||||||
|
<td><a href="https://gopeed.com/api/download?tpl=Gopeed-$version-android.apk">Liên kết</a></td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td>iOS</td>
|
||||||
|
<td><code>IPA</code></td>
|
||||||
|
<td><a href="https://gopeed.com/api/download?tpl=Gopeed-$version-ios.ipa">Liên kết</a></td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td>Web</td>
|
||||||
|
<td></td>
|
||||||
|
<td><a href="https://github.com/GopeedLab/gopeed/releases/latest">Liên kết</a></td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td>Docker</td>
|
||||||
|
<td></td>
|
||||||
|
<td><a href="https://hub.docker.com/r/liwei2633/gopeed">Liên kết</a></td>
|
||||||
|
</tr>
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
|
||||||
## ✨ Tính năng chính
|
Thêm thông tin về cài đặt, vui lòng tham khảo [Cài đặt](https://docs.gopeed.com/install.html)
|
||||||
|
|
||||||
- ⚡ **Tải xuống tốc độ cao** — kết hợp nhiều tác vụ, truyền HTTP phân đoạn qua nhiều kết nối và tải BitTorrent P2P để tận dụng tối đa băng thông.
|
### Command tool
|
||||||
- 🧲 **Đa giao thức** — quản lý HTTP, HTTPS, BitTorrent, Magnet và ed2k trong một giao diện.
|
|
||||||
- 🌱 **Bộ công cụ BT đầy đủ** — hỗ trợ khám phá Peer qua DHT, truyền tải uTP, Web Seed, chọn tệp, quản lý Tracker, xem Peer/mảnh và kiểm soát seed theo tỷ lệ hoặc thời gian.
|
|
||||||
- 📋 **Quản lý tác vụ thực tế** — tạm dừng, tiếp tục, thử lại, thao tác hàng loạt, tìm kiếm, bộ lọc, danh mục và khôi phục khi khởi động.
|
|
||||||
- 🪶 **Trải nghiệm native gọn nhẹ** — giao diện chính được Flutter kết xuất native. Không Electron. Không vỏ bọc WebView. Kích thước nhỏ hơn, ít tốn tài nguyên hơn và phản hồi nhanh hơn.
|
|
||||||
- 💻 **Đa nền tảng** — Windows, macOS, Linux, Android, iOS, Web, Docker và QNAP.
|
|
||||||
- 🎨 **Giao diện tùy biến** — hỗ trợ chế độ theo hệ thống, sáng, tối và 8 màu nhấn.
|
|
||||||
- 📐 **Bố cục thích ứng** — danh sách tác vụ, điều hướng, cài đặt và màn hình chi tiết tự điều chỉnh cho điện thoại, máy tính bảng và cửa sổ desktop có thể thay đổi kích thước.
|
|
||||||
- 🗣️ **Hơn 20 ngôn ngữ giao diện** — gồm tiếng Việt, tiếng Anh, tiếng Trung giản thể và phồn thể, tiếng Nhật, tiếng Hàn cùng nhiều ngôn ngữ khác.
|
|
||||||
- 🌐 **Tích hợp trình duyệt** — gửi tải xuống từ Chrome, Edge, Firefox và các trình duyệt tương thích sang Gopeed.
|
|
||||||
- 🧩 **Tiện ích JavaScript** — thêm nguồn tải từ nền tảng video, kho mô hình AI, lưu trữ đám mây và nhiều dịch vụ khác.
|
|
||||||
- 🤖 **Tích hợp AI** — cung cấp giao diện MCP để kết nối với các AI Agent tương thích và tạo, kiểm tra hoặc quản lý tác vụ tải xuống bằng ngôn ngữ tự nhiên.
|
|
||||||
- 🔌 **Giao diện mở** — REST API, CLI, Web UI có xác thực, webhook và script sau khi tải.
|
|
||||||
- 🛠️ **Công cụ tích hợp** — Header/User-Agent tùy chỉnh, proxy, mirror GitHub, thông báo và tự động giải nén.
|
|
||||||
|
|
||||||
## 🤖 Tích hợp AI
|
|
||||||
|
|
||||||
Sau khi kết nối Gopeed với AI Agent, bạn có thể quản lý tải xuống bằng ngôn ngữ tự nhiên. Ví dụ, hãy nói với AI Agent:
|
|
||||||
|
|
||||||
> Hãy tải xuống phiên bản Gopeed mới nhất dành cho Windows
|
|
||||||
|
|
||||||
| Tool | Mô tả |
|
|
||||||
| --- | --- |
|
|
||||||
| `resolve_task` | Phân tích URL hoặc URI tải xuống và trả về thông tin tài nguyên cùng danh sách tệp trước khi tạo tác vụ. |
|
|
||||||
| `create_task` | Tạo và bắt đầu tác vụ từ ID tài nguyên đã phân tích hoặc từ yêu cầu tải xuống trực tiếp. |
|
|
||||||
| `list_tasks` | Liệt kê tác vụ và có thể lọc theo ID hoặc trạng thái. |
|
|
||||||
| `get_task` | Lấy yêu cầu, tài nguyên, tùy chọn và tiến độ hiện tại của một tác vụ. |
|
|
||||||
| `get_task_status` | Lấy trạng thái chạy rút gọn và tiến độ của từng tệp trong một tác vụ. |
|
|
||||||
| `get_task_stats` | Lấy số liệu thống kê theo giao thức, gồm kết nối HTTP hoặc Peer và dữ liệu seed của BitTorrent. |
|
|
||||||
| `pause_task` | Tạm dừng tác vụ. |
|
|
||||||
| `continue_task` | Tiếp tục tác vụ đã tạm dừng hoặc bị lỗi. |
|
|
||||||
| `delete_task` | Xóa tác vụ và có thể xóa cả các tệp đã tải xuống. |
|
|
||||||
|
|
||||||
## ⬇️ Tải về
|
|
||||||
|
|
||||||
### 🧪 Gopeed 2.0.0 Beta
|
|
||||||
|
|
||||||
Gopeed 2.0.0 hiện đang trong giai đoạn beta công khai, với giao diện được thiết kế lại, kiến trúc giao tiếp native kết nối trực tiếp ứng dụng desktop và di động với lõi Go qua FFI, trải nghiệm đa nền tảng nhất quán hơn, khả năng quản lý tác vụ được cải thiện, API linh hoạt hơn và khả năng tích hợp AI Agent qua MCP. Một số tính năng có thể vẫn chưa hoàn thiện hoặc chưa ổn định, vì vậy hãy dùng thử và gửi phản hồi nếu bạn gặp vấn đề.
|
|
||||||
|
|
||||||
- [Tải Gopeed 2.0.0 Beta 3](https://github.com/GopeedLab/gopeed/releases/tag/v2.0.0-beta.3)
|
|
||||||
|
|
||||||
Khi các tính năng và độ ổn định đáp ứng tiêu chuẩn phát hành, chúng tôi sẽ phát hành Gopeed 2.0.0 chính thức. Người dùng bản beta có thể nâng cấp trực tiếp lên bản chính thức, trong khi người dùng bản ổn định hiện tại sẽ không tự động được chuyển sang kênh beta.
|
|
||||||
|
|
||||||
### Bản ổn định
|
|
||||||
|
|
||||||
- [Tải xuống từ trang web chính thức](https://gopeed.com)
|
|
||||||
- [GitHub Releases](https://github.com/GopeedLab/gopeed/releases/latest)
|
|
||||||
|
|
||||||
### 🛠️ Công cụ lệnh
|
|
||||||
|
|
||||||
Sử dụng `go install`:
|
Sử dụng `go install`:
|
||||||
|
|
||||||
@@ -80,38 +92,36 @@ Sử dụng `go install`:
|
|||||||
go install github.com/GopeedLab/gopeed/cmd/gopeed@latest
|
go install github.com/GopeedLab/gopeed/cmd/gopeed@latest
|
||||||
```
|
```
|
||||||
|
|
||||||
## 📱 WeChat Official Account
|
## Quyên góp
|
||||||
|
|
||||||
Theo dõi tài khoản chính thức để nhận các cập nhật và tin tức mới nhất.
|
Nếu bạn thích dự án này, xin vui lòng xem xét [quyên góp](https://docs.gopeed.com/donate.html) để hỗ trợ phát triển dự án này, cảm ơn bạn!
|
||||||
|
|
||||||
<img src="_docs/img/weixin.png" width="200" />
|
## Trưng bày
|
||||||
|
|
||||||
## 💝 Quyên góp
|

|
||||||
|
|
||||||
Nếu bạn thích dự án này, xin vui lòng xem xét [quyên góp](https://gopeed.com/docs/donate) để hỗ trợ phát triển dự án này, cảm ơn bạn!
|
## Development
|
||||||
|
|
||||||
## 👨💻 Development
|
|
||||||
|
|
||||||
Dự án này được chia thành hai phần, phần giao diện sử dụng `flutter`, phần backend sử dụng `Golang`, và hai phía giao tiếp thông qua giao thức `http`. Trên hệ thống unix, sử dụng `unix socket`, và trên hệ thống windows, sử dụng giao thức `tcp`.
|
Dự án này được chia thành hai phần, phần giao diện sử dụng `flutter`, phần backend sử dụng `Golang`, và hai phía giao tiếp thông qua giao thức `http`. Trên hệ thống unix, sử dụng `unix socket`, và trên hệ thống windows, sử dụng giao thức `tcp`.
|
||||||
|
|
||||||
> Mã giao diện nằm trong thư mục `ui/flutter`.
|
> Mã giao diện nằm trong thư mục `ui/flutter`.
|
||||||
|
|
||||||
### 🌍 Environment
|
### Environment
|
||||||
|
|
||||||
1. Golang 1.25+
|
1. Golang 1.21+
|
||||||
2. Flutter 3.41+
|
2. Flutter 3.16+
|
||||||
|
|
||||||
### 📋 Clone
|
### Clone
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
git clone [email protected]:GopeedLab/gopeed.git
|
git clone [email protected]:GopeedLab/gopeed.git
|
||||||
```
|
```
|
||||||
|
|
||||||
### 🤝 Đóng góp
|
### Đóng góp
|
||||||
|
|
||||||
Vui lòng tham khảo [CONTRIBUTING_vi-VN.md](/CONTRIBUTING_vi-VN.md)
|
Vui lòng tham khảo [CONTRIBUTING_vi-VN.md](/CONTRIBUTING_vi-VN.md)
|
||||||
|
|
||||||
### 🏗️ Xây dựng
|
### Xây dựng
|
||||||
|
|
||||||
#### Desktop
|
#### Desktop
|
||||||
|
|
||||||
@@ -158,7 +168,7 @@ command:
|
|||||||
- android
|
- android
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
gomobile bind -tags nosqlite -ldflags="-w -s -checklinkname=0" -o ui/flutter/android/app/libs/libgopeed.aar -target=android -androidapi 21 -javapkg="com.gopeed" github.com/GopeedLab/gopeed/bind/mobile
|
gomobile bind -tags nosqlite -ldflags="-w -s" -o ui/flutter/android/app/libs/libgopeed.aar -target=android -androidapi 19 -javapkg="com.gopeed" github.com/GopeedLab/gopeed/bind/mobile
|
||||||
cd ui/flutter
|
cd ui/flutter
|
||||||
flutter build apk
|
flutter build apk
|
||||||
```
|
```
|
||||||
@@ -184,7 +194,7 @@ cp -r ui/flutter/build/web cmd/web/dist
|
|||||||
go build -tags nosqlite,web -ldflags="-s -w" -o bin/ github.com/GopeedLab/gopeed/cmd/web
|
go build -tags nosqlite,web -ldflags="-s -w" -o bin/ github.com/GopeedLab/gopeed/cmd/web
|
||||||
```
|
```
|
||||||
|
|
||||||
## ❤️ Tín dụng
|
## Tín dụng
|
||||||
|
|
||||||
### Người đóng góp
|
### Người đóng góp
|
||||||
|
|
||||||
@@ -198,4 +208,4 @@ go build -tags nosqlite,web -ldflags="-s -w" -o bin/ github.com/GopeedLab/gopeed
|
|||||||
|
|
||||||
## Giấy phép
|
## Giấy phép
|
||||||
|
|
||||||
[GPLv3](LICENSE)
|
[GPLv3](LICENSE)
|
||||||
@@ -1,78 +1,90 @@
|
|||||||
# [](https://gopeed.com)
|
|
||||||
|
|
||||||
[](https://github.com/GopeedLab/gopeed/actions?query=workflow%3Atest)
|
[](https://github.com/GopeedLab/gopeed/actions?query=workflow%3Atest)
|
||||||
[](https://codecov.io/gh/GopeedLab/gopeed)
|
[](https://codecov.io/gh/GopeedLab/gopeed)
|
||||||
[](https://github.com/GopeedLab/gopeed/releases)
|
[](https://github.com/GopeedLab/gopeed/releases)
|
||||||
[](https://github.com/GopeedLab/gopeed/releases)
|
[](https://github.com/GopeedLab/gopeed/releases)
|
||||||
[](https://gopeed.com/docs/donate)
|
[](https://docs.gopeed.com/zh/donate.html)
|
||||||
[](https://raw.githubusercontent.com/GopeedLab/gopeed/main/_docs/img/weixin.png)
|
|
||||||
[](https://discord.gg/ZUJqJrwCGB)
|
[](https://discord.gg/ZUJqJrwCGB)
|
||||||
|
|
||||||
<a href="https://trendshift.io/repositories/7953" target="_blank"><img src="https://trendshift.io/api/badge/repositories/7953" alt="GopeedLab%2Fgopeed | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
|

|
||||||
|
|
||||||
|
[](https://ko-fi.com/R6R6IJGN6)
|
||||||
|
|
||||||
[English](/README.md) | [中文](/README_zh-CN.md) | [日本語](/README_ja-JP.md) | [正體中文](/README_zh-TW.md) | [Tiếng Việt](/README_vi-VN.md)
|
[English](/README.md) | [中文](/README_zh-CN.md) | [日本語](/README_ja-JP.md) | [正體中文](/README_zh-TW.md) | [Tiếng Việt](/README_vi-VN.md)
|
||||||
|
|
||||||
## 🚀 介绍
|
## 介绍
|
||||||
|
|
||||||
Gopeed(全称 **Go Speed**,也叫“够快下载器”)是一款使用 Go 和 Flutter 开发的高速、现代、免费开源下载器,支持 HTTP、HTTPS、BitTorrent、磁力链接和 ed2k,并覆盖桌面端、移动端与 Web。
|
Gopeed(全称 Go Speed),直译过来中文名叫做`够快下载器`(不是狗屁下载器!),是一款由`Golang`+`Flutter`开发的高速下载器,支持(HTTP、BitTorrent、Magnet)协议下载,并且支持全平台使用。除了基本的下载功能外,Gopeed 还是一款高度可定制化的下载器,支持通过对接[APIs](https://docs.gopeed.com/zh/dev-api.html)或者安装和开发[扩展](https://docs.gopeed.com/zh/dev-extension.html)来实现更多的功能。
|
||||||
|
|
||||||
除了日常下载任务管理,Gopeed 还提供浏览器接管、JavaScript 扩展、REST API、命令行工具和可自托管 Web UI,方便高级用户扩展和自动化自己的下载流程。
|
访问 ✈ [官方网站](https://gopeed.com/zh-CN) | 📖 [官方文档](https://docs.gopeed.com/zh/)
|
||||||
|
|
||||||
访问 ✈ [官方网站](https://gopeed.com)
|
## 下载
|
||||||
|
|
||||||

|
<table>
|
||||||
|
<thead>
|
||||||
|
<tr>
|
||||||
|
<th>运行平台</th>
|
||||||
|
<th>安装包类型</th>
|
||||||
|
<th>下载链接</th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody>
|
||||||
|
<tr>
|
||||||
|
<td rowspan=2>Windows</td>
|
||||||
|
<td><code>EXE Installer</code></td>
|
||||||
|
<td><a href="https://gopeed.com/api/download?tpl=Gopeed-$version-windows-amd64.zip">前往</a></td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td><code>Portable ZIP</code></td>
|
||||||
|
<td><a href="https://gopeed.com/api/download?tpl=Gopeed-$version-windows-amd64-portable.zip">前往</a></td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td>MacOS</td>
|
||||||
|
<td><code>DMG Installer</code></td>
|
||||||
|
<td><a href="https://gopeed.com/api/download?tpl=Gopeed-$version-macos.dmg">前往</a></td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td rowspan=4>Linux</td>
|
||||||
|
<td><code>Flathub</code></td>
|
||||||
|
<td><a href="https://flathub.org/apps/com.gopeed.Gopeed">前往</a></td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td><code>SNAP</code></td>
|
||||||
|
<td><a href="https://snapcraft.io/gopeed">前往</a></td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td><code>DEB</code></td>
|
||||||
|
<td><a href="https://gopeed.com/api/download?tpl=Gopeed-$version-linux-amd64.deb">前往</a></td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td><code>AppImage</code></td>
|
||||||
|
<td><a href="https://gopeed.com/api/download?tpl=Gopeed-$version-linux-x86_64.AppImage">前往</a></td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td>Android</td>
|
||||||
|
<td><code>APK</code></td>
|
||||||
|
<td><a href="https://gopeed.com/api/download?tpl=Gopeed-$version-android.apk">前往</a></td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td>iOS</td>
|
||||||
|
<td><code>IPA</code></td>
|
||||||
|
<td><a href="https://gopeed.com/api/download?tpl=Gopeed-$version-ios.ipa">前往</a></td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td>Web</td>
|
||||||
|
<td></td>
|
||||||
|
<td><a href="https://github.com/GopeedLab/gopeed/releases/latest">前往</a></td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td>Docker</td>
|
||||||
|
<td></td>
|
||||||
|
<td><a href="https://hub.docker.com/r/liwei2633/gopeed">前往</a></td>
|
||||||
|
</tr>
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
|
||||||
## ✨ 主要特性
|
更多关于安装的内容请参考[安装文档](https://docs.gopeed.com/zh/install.html)
|
||||||
|
|
||||||
- ⚡ **高速下载** — 多任务并发、HTTP 多连接分段传输和 BitTorrent P2P 下载,充分利用可用带宽。
|
### 命令行工具
|
||||||
- 🧲 **多协议支持** — 在同一个界面管理 HTTP、HTTPS、BitTorrent、磁力链接和 ed2k。
|
|
||||||
- 🌱 **完整 BT 能力** — 支持 DHT 节点发现、uTP 传输、Web Seed、按文件选择下载、Tracker 管理、Peer/分片统计,以及按分享率或时间控制做种。
|
|
||||||
- 📋 **实用任务管理** — 暂停、续传、重试、批量操作、搜索、状态筛选、下载分类和启动恢复。
|
|
||||||
- 🪶 **轻量原生体验** — 主界面使用 Flutter 原生渲染,非 Electron,非 WebView 的套壳,安装包更轻量、运行开销更低、响应更流畅。
|
|
||||||
- 💻 **全平台覆盖** — 支持 Windows、macOS、Linux、Android、iOS 和 Web,并提供 Docker、QNAP 部署方式。
|
|
||||||
- 🎨 **个性化主题** — 支持跟随系统、浅色、深色三种模式,以及 8 种主题强调色。
|
|
||||||
- 📐 **响应式界面** — 任务列表、导航、设置和详情视图会针对手机、平板及可调整大小的桌面窗口自动布局。
|
|
||||||
- 🗣️ **20+ 种界面语言** — 支持简体中文、繁体中文、英语、日语、韩语等多种语言。
|
|
||||||
- 🌐 **浏览器接管** — 支持 Chrome、Edge、Firefox 等兼容浏览器,将下载任务直接发送到 Gopeed。
|
|
||||||
- 🧩 **JavaScript 扩展** — 扩展视频网站、AI 模型站、网盘等更多下载源。
|
|
||||||
- 🤖 **AI 接入** — 开放 MCP 接口,可接入 AI Agent,通过自然语言创建、查询和管理下载任务。
|
|
||||||
- 🔌 **开放接口** — 通过 REST API、CLI、带身份认证的 Web UI、Webhook 和下载后脚本进行自动化。
|
|
||||||
- 🛠️ **常用内置能力** — 自定义 Header 与 User-Agent、代理、GitHub 镜像、通知和自动解压。
|
|
||||||
|
|
||||||
## 🤖 AI 接入
|
|
||||||
|
|
||||||
将 Gopeed 接入 AI Agent 后,即可通过自然语言管理下载。例如,直接对 AI Agent 说:
|
|
||||||
|
|
||||||
> 帮我下载最新的 Gopeed Windows 客户端
|
|
||||||
|
|
||||||
| Tool | 功能说明 |
|
|
||||||
| --- | --- |
|
|
||||||
| `resolve_task` | 解析下载 URL 或 URI,在创建任务前返回资源信息及文件列表。 |
|
|
||||||
| `create_task` | 使用已解析的资源 ID 或直接下载请求创建并启动任务。 |
|
|
||||||
| `list_tasks` | 查询任务列表,并可按任务 ID 或状态筛选。 |
|
|
||||||
| `get_task` | 获取单个任务的请求、资源、选项及当前进度等完整信息。 |
|
|
||||||
| `get_task_status` | 获取单个任务的轻量运行状态和各文件下载进度。 |
|
|
||||||
| `get_task_stats` | 获取协议相关统计信息,包括 HTTP 连接或 BitTorrent Peer 与做种数据。 |
|
|
||||||
| `pause_task` | 暂停任务。 |
|
|
||||||
| `continue_task` | 继续已暂停或失败的任务。 |
|
|
||||||
| `delete_task` | 删除任务,并可选择同时删除已下载文件。 |
|
|
||||||
|
|
||||||
## ⬇️ 下载
|
|
||||||
|
|
||||||
### 🧪 Gopeed 2.0.0 Beta
|
|
||||||
|
|
||||||
Gopeed 2.0.0 目前处于公开 Beta 测试阶段,引入了全新界面、通过 FFI 直连 Go 核心的原生通信架构、更一致的跨平台体验、更完善的任务管理、更灵活的 API,以及用于 AI Agent 接入的 MCP 能力。部分功能仍可能不够完善或稳定,欢迎提前体验并向我们反馈问题。
|
|
||||||
|
|
||||||
- [下载 Gopeed 2.0.0 Beta 3](https://github.com/GopeedLab/gopeed/releases/tag/v2.0.0-beta.3)
|
|
||||||
|
|
||||||
当功能完整度和稳定性达到正式发布标准后,我们会发布 Gopeed 2.0.0 正式版。已安装 Beta 版本的用户可以直接升级到最终正式版,现有稳定版用户则不会被自动切换到 Beta 渠道。
|
|
||||||
|
|
||||||
### 正式稳定版
|
|
||||||
|
|
||||||
- [官方网站下载](https://gopeed.com)
|
|
||||||
- [GitHub Releases](https://github.com/GopeedLab/gopeed/releases/latest)
|
|
||||||
|
|
||||||
### 🛠️ 命令行工具
|
|
||||||
|
|
||||||
使用`go install`安装:
|
使用`go install`安装:
|
||||||
|
|
||||||
@@ -80,42 +92,40 @@ Gopeed 2.0.0 目前处于公开 Beta 测试阶段,引入了全新界面、通
|
|||||||
go install github.com/GopeedLab/gopeed/cmd/gopeed@latest
|
go install github.com/GopeedLab/gopeed/cmd/gopeed@latest
|
||||||
```
|
```
|
||||||
|
|
||||||
## 🔌 浏览器扩展
|
## 浏览器扩展
|
||||||
|
|
||||||
Gopeed 还提供了浏览器扩展用于接管浏览器下载,支持 Chrome、Edge、Firefox 等浏览器,具体请参考:[https://github.com/GopeedLab/browser-extension](https://github.com/GopeedLab/browser-extension)
|
Gopeed 还提供了浏览器扩展用于接管浏览器下载,支持 Chrome、Edge、Firefox 等浏览器,具体请参考:[https://github.com/GopeedLab/browser-extension](https://github.com/GopeedLab/browser-extension)
|
||||||
|
|
||||||
## 📱 微信公众号
|
## 赞助
|
||||||
|
|
||||||
关注公众号获取项目最新动态和资讯。
|
如果觉得项目对你有帮助,请考虑[赞助](https://docs.gopeed.com/zh/donate)以支持这个项目的发展,非常感谢!
|
||||||
|
|
||||||
<img src="_docs/img/weixin.png" width="200" />
|
## 界面展示
|
||||||
|
|
||||||
## 💝 赞助
|

|
||||||
|
|
||||||
如果觉得项目对你有帮助,请考虑[赞助](https://gopeed.com/docs/donate)以支持这个项目的发展,非常感谢!
|
## 开发
|
||||||
|
|
||||||
## 👨💻 开发
|
|
||||||
|
|
||||||
本项目分为前端和后端两个部分,前端使用`flutter`,后端使用`Golang`,两边通过`http`协议进行通讯,在 unix 系统下,使用的是`unix socket`,在 windows 系统下,使用的是`tcp`协议。
|
本项目分为前端和后端两个部分,前端使用`flutter`,后端使用`Golang`,两边通过`http`协议进行通讯,在 unix 系统下,使用的是`unix socket`,在 windows 系统下,使用的是`tcp`协议。
|
||||||
|
|
||||||
> 前端代码位于`ui/flutter`目录下。
|
> 前端代码位于`ui/flutter`目录下。
|
||||||
|
|
||||||
### 🌍 环境要求
|
### 环境要求
|
||||||
|
|
||||||
1. Golang 1.25+
|
1. Golang 1.21+
|
||||||
2. Flutter 3.41+
|
2. Flutter 3.16+
|
||||||
|
|
||||||
### 📋 克隆项目
|
### 克隆项目
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
git clone [email protected]:GopeedLab/gopeed.git
|
git clone [email protected]:GopeedLab/gopeed.git
|
||||||
```
|
```
|
||||||
|
|
||||||
### 🤝 贡献代码
|
### 贡献代码
|
||||||
|
|
||||||
请参考[贡献指南](CONTRIBUTING_zh-CN.md)
|
请参考[贡献指南](CONTRIBUTING_zh-CN.md)
|
||||||
|
|
||||||
### 🏗️ 编译
|
### 编译
|
||||||
|
|
||||||
#### 桌面端
|
#### 桌面端
|
||||||
|
|
||||||
@@ -162,7 +172,7 @@ gomobile init
|
|||||||
- android
|
- android
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
gomobile bind -tags nosqlite -ldflags="-w -s -checklinkname=0" -o ui/flutter/android/app/libs/libgopeed.aar -target=android -androidapi 21 -javapkg="com.gopeed" github.com/GopeedLab/gopeed/bind/mobile
|
gomobile bind -tags nosqlite -ldflags="-w -s" -o ui/flutter/android/app/libs/libgopeed.aar -target=android -androidapi 19 -javapkg="com.gopeed" github.com/GopeedLab/gopeed/bind/mobile
|
||||||
cd ui/flutter
|
cd ui/flutter
|
||||||
flutter build apk
|
flutter build apk
|
||||||
```
|
```
|
||||||
@@ -188,7 +198,7 @@ cp -r ui/flutter/build/web cmd/web/dist
|
|||||||
go build -tags nosqlite,web -ldflags="-s -w" -o bin/ github.com/GopeedLab/gopeed/cmd/web
|
go build -tags nosqlite,web -ldflags="-s -w" -o bin/ github.com/GopeedLab/gopeed/cmd/web
|
||||||
```
|
```
|
||||||
|
|
||||||
## ❤️ 感谢
|
## 感谢
|
||||||
|
|
||||||
### 贡献者
|
### 贡献者
|
||||||
|
|
||||||
|
|||||||
@@ -1,78 +1,90 @@
|
|||||||
# [](https://gopeed.com)
|
|
||||||
|
|
||||||
[](https://github.com/GopeedLab/gopeed/actions?query=workflow%3Atest)
|
[](https://github.com/GopeedLab/gopeed/actions?query=workflow%3Atest)
|
||||||
[](https://codecov.io/gh/GopeedLab/gopeed)
|
[](https://codecov.io/gh/GopeedLab/gopeed)
|
||||||
[](https://github.com/GopeedLab/gopeed/releases)
|
[](https://github.com/GopeedLab/gopeed/releases)
|
||||||
[](https://github.com/GopeedLab/gopeed/releases)
|
[](https://github.com/GopeedLab/gopeed/releases)
|
||||||
[](https://gopeed.com/docs/donate)
|
[](https://docs.gopeed.com/donate.html)
|
||||||
[](https://raw.githubusercontent.com/GopeedLab/gopeed/main/_docs/img/weixin.png)
|
|
||||||
[](https://discord.gg/ZUJqJrwCGB)
|
[](https://discord.gg/ZUJqJrwCGB)
|
||||||
|
|
||||||
<a href="https://trendshift.io/repositories/7953" target="_blank"><img src="https://trendshift.io/api/badge/repositories/7953" alt="GopeedLab%2Fgopeed | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
|

|
||||||
|
|
||||||
|
[](https://ko-fi.com/R6R6IJGN6)
|
||||||
|
|
||||||
[English](/README.md) | [中文](/README_zh-CN.md) | [日本語](/README_ja-JP.md) | [正體中文](/README_zh-TW.md) | [Tiếng Việt](/README_vi-VN.md)
|
[English](/README.md) | [中文](/README_zh-CN.md) | [日本語](/README_ja-JP.md) | [正體中文](/README_zh-TW.md) | [Tiếng Việt](/README_vi-VN.md)
|
||||||
|
|
||||||
## 🚀 簡介
|
## 簡介
|
||||||
|
|
||||||
Gopeed(**Go Speed** 的縮寫)是一款以 Go 與 Flutter 開發的高速、現代化、免費開源下載器,支援 HTTP、HTTPS、BitTorrent、磁力連結與 ed2k,並涵蓋桌面、行動裝置及 Web。
|
Gopeed(全稱 Go Speed),是一款使用`Golang`+`Flutter`編寫的高速下載軟體,支援(HTTP、BitTorrent、Magnet)協定,同時支援所有的平台。
|
||||||
|
|
||||||
除了日常任務管理,Gopeed 也提供瀏覽器整合、JavaScript 擴充、REST API、命令列工具及可自行託管的 Web UI,方便進階使用者擴充與自動化下載流程。
|
前往 ✈ [主頁](https://gopeed.com/zh-CN) | 📖 [文檔](https://docs.gopeed.com/zh/)
|
||||||
|
|
||||||
瀏覽 ✈ [官方網站](https://gopeed.com)
|
## 下載
|
||||||
|
|
||||||

|
<table>
|
||||||
|
<thead>
|
||||||
|
<tr>
|
||||||
|
<th>運行平台</th>
|
||||||
|
<th>安裝包類型</th>
|
||||||
|
<th>下載鏈接</th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody>
|
||||||
|
<tr>
|
||||||
|
<td rowspan=2>Windows</td>
|
||||||
|
<td><code>EXE Installer</code></td>
|
||||||
|
<td><a href="https://gopeed.com/api/download?tpl=Gopeed-$version-windows-amd64.zip">前往</a></td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td><code>Portable ZIP</code></td>
|
||||||
|
<td><a href="https://gopeed.com/api/download?tpl=Gopeed-$version-windows-amd64-portable.zip">前往</a></td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td>MacOS</td>
|
||||||
|
<td><code>DMG Installer</code></td>
|
||||||
|
<td><a href="https://gopeed.com/api/download?tpl=Gopeed-$version-macos.dmg">前往</a></td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td rowspan=4>Linux</td>
|
||||||
|
<td><code>Flathub</code></td>
|
||||||
|
<td><a href="https://flathub.org/apps/com.gopeed.Gopeed">前往</a></td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td><code>SNAP</code></td>
|
||||||
|
<td><a href="https://snapcraft.io/gopeed">前往</a></td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td><code>DEB</code></td>
|
||||||
|
<td><a href="https://gopeed.com/api/download?tpl=Gopeed-$version-linux-amd64.deb">前往</a></td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td><code>AppImage</code></td>
|
||||||
|
<td><a href="https://gopeed.com/api/download?tpl=Gopeed-$version-linux-x86_64.AppImage">前往</a></td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td>Android</td>
|
||||||
|
<td><code>APK</code></td>
|
||||||
|
<td><a href="https://gopeed.com/api/download?tpl=Gopeed-$version-android.apk">前往</a></td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td>iOS</td>
|
||||||
|
<td><code>IPA</code></td>
|
||||||
|
<td><a href="https://gopeed.com/api/download?tpl=Gopeed-$version-ios.ipa">前往</a></td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td>Web</td>
|
||||||
|
<td></td>
|
||||||
|
<td><a href="https://github.com/GopeedLab/gopeed/releases/latest">前往</a></td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td>Docker</td>
|
||||||
|
<td></td>
|
||||||
|
<td><a href="https://hub.docker.com/r/liwei2633/gopeed">前往</a></td>
|
||||||
|
</tr>
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
|
||||||
## ✨ 主要功能
|
更多關於安裝的內容請參考[安裝文檔](https://docs.gopeed.com/zh/install.html)
|
||||||
|
|
||||||
- ⚡ **高速下載** — 多任務並行、HTTP 多連線分段傳輸與 BitTorrent P2P 下載,充分利用可用頻寬。
|
### 使用CLI安裝
|
||||||
- 🧲 **多協定支援** — 在同一介面管理 HTTP、HTTPS、BitTorrent、磁力連結與 ed2k。
|
|
||||||
- 🌱 **完整 BT 功能** — 支援 DHT 節點探索、uTP 傳輸、Web Seed、逐檔選擇、Tracker 管理、Peer/分片統計,以及依分享率或時間控制做種。
|
|
||||||
- 📋 **實用任務管理** — 暫停、續傳、重試、批次操作、搜尋、狀態篩選、分類及啟動恢復。
|
|
||||||
- 🪶 **輕量原生體驗** — 主介面使用 Flutter 原生渲染,非 Electron,非 WebView 套殼,安裝包更輕量、執行開銷更低、操作回應更流暢。
|
|
||||||
- 💻 **跨平台** — 支援 Windows、macOS、Linux、Android、iOS、Web、Docker 與 QNAP。
|
|
||||||
- 🎨 **個人化主題** — 支援跟隨系統、淺色、深色模式及 8 種主題強調色。
|
|
||||||
- 📐 **響應式介面** — 任務列表、導覽、設定與詳細資訊會配合手機、平板及可調整大小的桌面視窗自動配置。
|
|
||||||
- 🗣️ **20+ 種介面語言** — 支援繁體中文、簡體中文、英文、日文、韓文等多種語言。
|
|
||||||
- 🌐 **瀏覽器整合** — 可接管 Chrome、Edge、Firefox 等相容瀏覽器的下載。
|
|
||||||
- 🧩 **JavaScript 擴充** — 新增影音網站、AI 模型平台、雲端儲存等下載來源。
|
|
||||||
- 🤖 **AI 整合** — 提供 MCP 介面,可串接相容的 AI Agent,以自然語言建立、查詢及管理下載任務。
|
|
||||||
- 🔌 **開放介面** — 支援 REST API、CLI、認證 Web UI、Webhook 及下載後腳本。
|
|
||||||
- 🛠️ **實用內建能力** — 自訂 Header/User-Agent、Proxy、GitHub 鏡像、通知及自動解壓縮。
|
|
||||||
|
|
||||||
## 🤖 AI 整合
|
|
||||||
|
|
||||||
將 Gopeed 與 AI Agent 串接後,即可透過自然語言管理下載。例如,直接對 AI Agent 說:
|
|
||||||
|
|
||||||
> 幫我下載最新的 Gopeed Windows 用戶端
|
|
||||||
|
|
||||||
| Tool | 功能說明 |
|
|
||||||
| --- | --- |
|
|
||||||
| `resolve_task` | 解析下載 URL 或 URI,在建立任務前回傳資源資訊及檔案清單。 |
|
|
||||||
| `create_task` | 使用已解析的資源 ID 或直接下載請求建立並啟動任務。 |
|
|
||||||
| `list_tasks` | 查詢任務清單,並可依任務 ID 或狀態篩選。 |
|
|
||||||
| `get_task` | 取得單一任務的請求、資源、選項及目前進度等完整資訊。 |
|
|
||||||
| `get_task_status` | 取得單一任務的簡要執行狀態及各檔案下載進度。 |
|
|
||||||
| `get_task_stats` | 取得協定相關統計資訊,包括 HTTP 連線或 BitTorrent Peer 與做種資料。 |
|
|
||||||
| `pause_task` | 暫停任務。 |
|
|
||||||
| `continue_task` | 繼續已暫停或失敗的任務。 |
|
|
||||||
| `delete_task` | 刪除任務,並可選擇同時刪除已下載檔案。 |
|
|
||||||
|
|
||||||
## ⬇️ 下載
|
|
||||||
|
|
||||||
### 🧪 Gopeed 2.0.0 Beta
|
|
||||||
|
|
||||||
Gopeed 2.0.0 目前處於公開 Beta 測試階段,引入了全新介面、透過 FFI 直連 Go 核心的原生通訊架構、更一致的跨平台體驗、更完善的任務管理、更靈活的 API,以及透過 MCP 串接 AI Agent 的能力。部分功能仍可能不夠完善或穩定,歡迎搶先體驗並向我們回報問題。
|
|
||||||
|
|
||||||
- [下載 Gopeed 2.0.0 Beta 3](https://github.com/GopeedLab/gopeed/releases/tag/v2.0.0-beta.3)
|
|
||||||
|
|
||||||
當功能完整度與穩定性達到正式發布標準後,我們會發布 Gopeed 2.0.0 正式版。已安裝 Beta 版本的使用者可以直接升級至最終正式版,現有穩定版使用者則不會被自動切換至 Beta 頻道。
|
|
||||||
|
|
||||||
### 正式穩定版
|
|
||||||
|
|
||||||
- [官方網站下載](https://gopeed.com)
|
|
||||||
- [GitHub Releases](https://github.com/GopeedLab/gopeed/releases/latest)
|
|
||||||
|
|
||||||
### 🛠️ 使用 CLI 安裝
|
|
||||||
|
|
||||||
使用`go install`安裝:
|
使用`go install`安裝:
|
||||||
|
|
||||||
@@ -80,38 +92,36 @@ Gopeed 2.0.0 目前處於公開 Beta 測試階段,引入了全新介面、透
|
|||||||
go install github.com/GopeedLab/gopeed/cmd/gopeed@latest
|
go install github.com/GopeedLab/gopeed/cmd/gopeed@latest
|
||||||
```
|
```
|
||||||
|
|
||||||
## 📱 微信公眾號
|
## 贊助
|
||||||
|
|
||||||
關注公眾號獲取項目最新動態和資訊。
|
如果你認為該項目對你有所幫助,請考慮[贊助](https://docs.gopeed.com/zh/donate)以支持該項目的持續發展,謝謝!
|
||||||
|
|
||||||
<img src="_docs/img/weixin.png" width="200" />
|
## 軟體介面
|
||||||
|
|
||||||
## 💝 贊助
|

|
||||||
|
|
||||||
如果你認為該項目對你有所幫助,請考慮[贊助](https://gopeed.com/docs/donate)以支持該項目的持續發展,謝謝!
|
## 開發
|
||||||
|
|
||||||
## 👨💻 開發
|
|
||||||
|
|
||||||
該項目分為前端與後端,前端使用`flutter`編寫,後端使用`Golang`編寫,兩邊通過`http`協定進行通訊,在 unix 系統下,則使用`unix socket`,在 windows 系統下,則使用`tcp`協定。
|
該項目分為前端與後端,前端使用`flutter`編寫,後端使用`Golang`編寫,兩邊通過`http`協定進行通訊,在 unix 系統下,則使用`unix socket`,在 windows 系統下,則使用`tcp`協定。
|
||||||
|
|
||||||
> 前端代碼位於`ui/flutter`目錄內。
|
> 前端代碼位於`ui/flutter`目錄內。
|
||||||
|
|
||||||
### 🌍 開發環境
|
### 開發環境
|
||||||
|
|
||||||
1. Golang 1.25+
|
1. Golang 1.21+
|
||||||
2. Flutter 3.41+
|
2. Flutter 3.16+
|
||||||
|
|
||||||
### 📋 克隆項目
|
### 克隆項目
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
git clone [email protected]:GopeedLab/gopeed.git
|
git clone [email protected]:GopeedLab/gopeed.git
|
||||||
```
|
```
|
||||||
|
|
||||||
### 🤝 協助開發
|
### 協助開發
|
||||||
|
|
||||||
請參考[協助指南](CONTRIBUTING_zh-TW.md)
|
請參考[協助指南](CONTRIBUTING_zh-TW.md)
|
||||||
|
|
||||||
### 🏗️ 編譯
|
### 編譯
|
||||||
|
|
||||||
#### 桌面端
|
#### 桌面端
|
||||||
|
|
||||||
@@ -158,7 +168,7 @@ gomobile init
|
|||||||
- android
|
- android
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
gomobile bind -tags nosqlite -ldflags="-w -s -checklinkname=0" -o ui/flutter/android/app/libs/libgopeed.aar -target=android -androidapi 21 -javapkg="com.gopeed" github.com/GopeedLab/gopeed/bind/mobile
|
gomobile bind -tags nosqlite -ldflags="-w -s" -o ui/flutter/android/app/libs/libgopeed.aar -target=android -androidapi 19 -javapkg="com.gopeed" github.com/GopeedLab/gopeed/bind/mobile
|
||||||
cd ui/flutter
|
cd ui/flutter
|
||||||
flutter build apk
|
flutter build apk
|
||||||
```
|
```
|
||||||
@@ -184,7 +194,7 @@ cp -r ui/flutter/build/web cmd/web/dist
|
|||||||
go build -tags nosqlite,web -ldflags="-s -w" -o bin/ github.com/GopeedLab/gopeed/cmd/web
|
go build -tags nosqlite,web -ldflags="-s -w" -o bin/ github.com/GopeedLab/gopeed/cmd/web
|
||||||
```
|
```
|
||||||
|
|
||||||
## ❤️ 感謝
|
## 感謝
|
||||||
|
|
||||||
### 貢獻者
|
### 貢獻者
|
||||||
|
|
||||||
|
|||||||
|
After Width: | Height: | Size: 26 KiB |
@@ -1,54 +0,0 @@
|
|||||||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 1200 400" width="1200" height="400">
|
|
||||||
<rect width="1200" height="400" fill="#040605"/>
|
|
||||||
<g fill="#5dc76c">
|
|
||||||
<rect x="-100" y="50" width="150" height="2" opacity="0.6">
|
|
||||||
<animate attributeName="x" from="-200" to="1300" dur="0.8s" repeatCount="indefinite"/>
|
|
||||||
</rect>
|
|
||||||
<rect x="-200" y="150" width="200" height="1" opacity="0.4">
|
|
||||||
<animate attributeName="x" from="-300" to="1300" dur="1.2s" repeatCount="indefinite"/>
|
|
||||||
</rect>
|
|
||||||
<rect x="-50" y="250" width="180" height="3" opacity="0.8">
|
|
||||||
<animate attributeName="x" from="-200" to="1300" dur="0.6s" repeatCount="indefinite"/>
|
|
||||||
</rect>
|
|
||||||
<rect x="-150" y="350" width="220" height="2" opacity="0.5">
|
|
||||||
<animate attributeName="x" from="-300" to="1300" dur="1s" repeatCount="indefinite"/>
|
|
||||||
</rect>
|
|
||||||
<rect x="-100" y="100" width="100" height="1" opacity="0.7">
|
|
||||||
<animate attributeName="x" from="-150" to="1300" dur="0.9s" repeatCount="indefinite"/>
|
|
||||||
</rect>
|
|
||||||
<rect x="-250" y="300" width="250" height="2" opacity="0.3">
|
|
||||||
<animate attributeName="x" from="-400" to="1300" dur="1.5s" repeatCount="indefinite"/>
|
|
||||||
</rect>
|
|
||||||
</g>
|
|
||||||
<defs>
|
|
||||||
<filter id="logoGlow" x="-50%" y="-50%" width="200%" height="200%">
|
|
||||||
<feGaussianBlur stdDeviation="12" result="blur" />
|
|
||||||
<feComposite in="SourceGraphic" in2="blur" operator="over" />
|
|
||||||
</filter>
|
|
||||||
</defs>
|
|
||||||
<g transform="translate(600, 160)">
|
|
||||||
<!-- Floating animation -->
|
|
||||||
<g>
|
|
||||||
<animateTransform attributeName="transform" type="translate" values="0,-8; 0,8; 0,-8" dur="3s" repeatCount="indefinite" />
|
|
||||||
|
|
||||||
<!-- Pulsing glow behind logo -->
|
|
||||||
<circle cx="0" cy="0" r="45" fill="#5dc76c" filter="url(#logoGlow)" opacity="0.3">
|
|
||||||
<animate attributeName="opacity" values="0.1; 0.6; 0.1" dur="2s" repeatCount="indefinite" />
|
|
||||||
<animate attributeName="r" values="40; 65; 40" dur="2s" repeatCount="indefinite" />
|
|
||||||
</circle>
|
|
||||||
|
|
||||||
<!-- Breathing scale animation -->
|
|
||||||
<g>
|
|
||||||
<animateTransform attributeName="transform" type="scale" values="0.95; 1.05; 0.95" dur="2s" repeatCount="indefinite" />
|
|
||||||
<g transform="scale(0.12) translate(-512, -512)">
|
|
||||||
<path style="opacity:0.998" fill="#5dc76c" d="M 951.5,763.5 C 940.739,755.276 929.405,747.776 917.5,741C 908.477,736.319 899.143,732.485 889.5,729.5C 885.844,625.779 840.844,545.613 754.5,489C 674.802,441.453 590.135,430.12 500.5,455C 419.755,480.605 359.588,530.771 320,605.5C 278.585,565.816 230.752,555.649 176.5,575C 131.948,594.935 106.448,629.102 100,677.5C 99.3333,685.833 99.3333,694.167 100,702.5C 102.081,714.824 105.081,726.824 109,738.5C 95.4882,745.492 82.9882,753.826 71.5,763.5C 37.7121,705.639 16.5454,643.306 8,576.5C -9.41698,416.759 35.9164,278.093 144,160.5C 244.661,58.2827 367.161,5.94939 511.5,3.5C 678.975,7.48829 814.142,74.8216 917,205.5C 1011.33,336.426 1040,480.759 1003,638.5C 991.564,682.671 974.397,724.337 951.5,763.5 Z"/>
|
|
||||||
<path style="opacity:0.999" fill="#fefefe" d="M 951.5,763.5 C 949.28,768.272 946.78,772.938 944,777.5C 938.305,786.913 932.138,795.913 925.5,804.5C 915.954,796.888 905.288,791.055 893.5,787C 878.282,782.958 863.115,778.791 848,774.5C 854.013,716.891 841.346,663.891 810,615.5C 768.341,553.228 711.174,512.395 638.5,493C 560.311,479.566 490.311,497.566 428.5,547C 411.963,560.536 397.13,575.703 384,592.5C 366.745,616.923 348.911,640.923 330.5,664.5C 310.135,651.302 290.469,637.135 271.5,622C 238.448,602.307 206.782,604.307 176.5,628C 159.375,642.752 148.875,661.252 145,683.5C 143.968,692.716 144.968,701.716 148,710.5C 154.334,725.339 160.167,740.339 165.5,755.5C 138.118,764.688 114.451,779.688 94.5,800.5C 85.653,788.968 77.9863,776.635 71.5,763.5C 82.9882,753.826 95.4882,745.492 109,738.5C 105.081,726.824 102.081,714.824 100,702.5C 99.3333,694.167 99.3333,685.833 100,677.5C 106.448,629.102 131.948,594.935 176.5,575C 230.752,555.649 278.585,565.816 320,605.5C 359.588,530.771 419.755,480.605 500.5,455C 590.135,430.12 674.802,441.453 754.5,489C 840.844,545.613 885.844,625.779 889.5,729.5C 899.143,732.485 908.477,736.319 917.5,741C 929.405,747.776 940.739,755.276 951.5,763.5 Z"/>
|
|
||||||
<path style="opacity:0.998" fill="#5dc76c" d="M 925.5,804.5 C 850.791,908.608 750.791,975.775 625.5,1006C 609.308,1009.7 592.975,1012.53 576.5,1014.5C 615.708,969.938 655.042,925.438 694.5,881C 664.168,880.5 633.835,880.333 603.5,880.5C 603.5,811.167 603.5,741.833 603.5,672.5C 542.167,672.5 480.833,672.5 419.5,672.5C 419.5,741.5 419.5,810.5 419.5,879.5C 388.493,879.167 357.493,879.5 326.5,880.5C 326.624,881.107 326.957,881.44 327.5,881.5C 366.805,925.808 406.138,970.141 445.5,1014.5C 318.63,996.317 212.796,939.651 128,844.5C 115.85,830.525 104.683,815.859 94.5,800.5C 114.451,779.688 138.118,764.688 165.5,755.5C 160.167,740.339 154.334,725.339 148,710.5C 144.968,701.716 143.968,692.716 145,683.5C 148.875,661.252 159.375,642.752 176.5,628C 206.782,604.307 238.448,602.307 271.5,622C 290.469,637.135 310.135,651.302 330.5,664.5C 348.911,640.923 366.745,616.923 384,592.5C 397.13,575.703 411.963,560.536 428.5,547C 490.311,497.566 560.311,479.566 638.5,493C 711.174,512.395 768.341,553.228 810,615.5C 841.346,663.891 854.013,716.891 848,774.5C 863.115,778.791 878.282,782.958 893.5,787C 905.288,791.055 915.954,796.888 925.5,804.5 Z"/>
|
|
||||||
<path style="opacity:0.999" fill="#fefefe" d="M 576.5,1014.5 C 532.836,1019.82 489.169,1019.82 445.5,1014.5C 406.138,970.141 366.805,925.808 327.5,881.5C 358.167,880.833 388.833,880.167 419.5,879.5C 419.5,810.5 419.5,741.5 419.5,672.5C 480.833,672.5 542.167,672.5 603.5,672.5C 603.5,741.833 603.5,811.167 603.5,880.5C 633.835,880.333 664.168,880.5 694.5,881C 655.042,925.438 615.708,969.938 576.5,1014.5 Z"/>
|
|
||||||
<path style="opacity:1" fill="#ade2b5" d="M 419.5,879.5 C 388.833,880.167 358.167,880.833 327.5,881.5C 326.957,881.44 326.624,881.107 326.5,880.5C 357.493,879.5 388.493,879.167 419.5,879.5 Z"/>
|
|
||||||
</g>
|
|
||||||
</g>
|
|
||||||
</g>
|
|
||||||
</g>
|
|
||||||
<text x="600" y="300" font-family="sans-serif" font-size="72" font-style="italic" font-weight="900" text-anchor="middle" fill="#fff" letter-spacing="8">GOPEED</text>
|
|
||||||
</svg>
|
|
||||||
|
Before Width: | Height: | Size: 6.0 KiB |
|
Before Width: | Height: | Size: 792 KiB |
|
Before Width: | Height: | Size: 787 KiB |
|
Before Width: | Height: | Size: 1.2 MiB After Width: | Height: | Size: 309 KiB |
|
Before Width: | Height: | Size: 55 KiB |
@@ -1,52 +0,0 @@
|
|||||||
# Batch task creation
|
|
||||||
|
|
||||||
`POST /api/v1/tasks/batch` accepts ordinary and grouped batches:
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"group": true,
|
|
||||||
"groupName": "album",
|
|
||||||
"opts": {"path": "/downloads"},
|
|
||||||
"reqs": [
|
|
||||||
{"rid": "resolved-resource-id", "opts": {"selectFiles": [1]}},
|
|
||||||
{"req": {"url": "https://example.com/cover.jpg"}}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
Each member accepts `rid` or `req`. A nonempty `rid` takes priority and creates a
|
|
||||||
task from the cached resolve result. A member's `opts`, when present, replaces
|
|
||||||
batch `opts` as a whole; omitted fields do not inherit individual batch values.
|
|
||||||
Members without `opts` use batch `opts`. BT file selections use the indexes from
|
|
||||||
the resolve result. Requests without `rid` continue to work.
|
|
||||||
|
|
||||||
For groups, batch `opts.path` selects the parent directory. Members inheriting
|
|
||||||
batch options, or with an empty path, use the reserved group directory. Explicit
|
|
||||||
relative member paths are resolved under that directory; do not prefix them with
|
|
||||||
the batch parent path. For example, parent `downloads` and member path `nested`
|
|
||||||
place the member in `downloads/<group name>/nested`. Absolute paths under the
|
|
||||||
parent directory preserve their relative structure within the group. In ordinary
|
|
||||||
batches, an empty member path uses the configured default download directory.
|
|
||||||
|
|
||||||
All members are attempted. If at least one task is persisted, the response is
|
|
||||||
successful and contains the created task IDs. A task that fails to start after
|
|
||||||
being persisted is included; use the existing task controls to retry it.
|
|
||||||
Successful tasks are kept when other members fail. If every member fails, the
|
|
||||||
API returns an error and removes any newly reserved empty group and directory.
|
|
||||||
|
|
||||||
Both ordinary and grouped batches now return an object in `data`:
|
|
||||||
|
|
||||||
```json
|
|
||||||
{"code": 0, "msg": "", "data": {"taskIds": ["task-1", "task-2"], "parentId": "group-id"}}
|
|
||||||
```
|
|
||||||
|
|
||||||
Ordinary batches omit `parentId`. **This replaces the former array response for
|
|
||||||
ordinary batches.** The outer `code`/`msg`/`data` envelope is unchanged. Clients
|
|
||||||
must read `data.taskIds`; there are no per-member results. A successful response
|
|
||||||
completes the batch, including partial success, so clients must not automatically
|
|
||||||
resubmit missing members.
|
|
||||||
|
|
||||||
With `group: true`, the server generates the group ID and reserves its directory.
|
|
||||||
Creation does not accept a client group ID or append to an existing group. Group
|
|
||||||
metadata remains available through the existing task list and detail responses;
|
|
||||||
`parentId` identifies membership for group operations.
|
|
||||||
@@ -2,7 +2,6 @@ package main
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
|
|
||||||
"github.com/GopeedLab/gopeed/pkg/base"
|
"github.com/GopeedLab/gopeed/pkg/base"
|
||||||
"github.com/GopeedLab/gopeed/pkg/download"
|
"github.com/GopeedLab/gopeed/pkg/download"
|
||||||
"github.com/GopeedLab/gopeed/pkg/protocol/http"
|
"github.com/GopeedLab/gopeed/pkg/protocol/http"
|
||||||
|
|||||||
@@ -1,28 +1,8 @@
|
|||||||
package main
|
package main
|
||||||
|
|
||||||
/*
|
|
||||||
#include <stdlib.h>
|
|
||||||
#include <stdint.h>
|
|
||||||
|
|
||||||
typedef void (*TaskEventCallback)(char* payload);
|
|
||||||
|
|
||||||
typedef void (*InvokeResultCallback)(uint64_t request_id, int success, char* payload);
|
|
||||||
|
|
||||||
static void callTaskEventCallback(uintptr_t callback, char* payload) {
|
|
||||||
((TaskEventCallback)callback)(payload);
|
|
||||||
}
|
|
||||||
|
|
||||||
static void callInvokeResultCallback(uintptr_t callback, uint64_t request_id, int success, char* payload) {
|
|
||||||
((InvokeResultCallback)callback)(request_id, success, payload);
|
|
||||||
}
|
|
||||||
*/
|
|
||||||
import "C"
|
import "C"
|
||||||
import (
|
import (
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
|
||||||
"unsafe"
|
|
||||||
|
|
||||||
nativebridge "github.com/GopeedLab/gopeed/bind/native"
|
|
||||||
"github.com/GopeedLab/gopeed/pkg/rest"
|
"github.com/GopeedLab/gopeed/pkg/rest"
|
||||||
"github.com/GopeedLab/gopeed/pkg/rest/model"
|
"github.com/GopeedLab/gopeed/pkg/rest/model"
|
||||||
)
|
)
|
||||||
@@ -36,104 +16,14 @@ func Start(cfg *C.char) (int, *C.char) {
|
|||||||
return 0, C.CString(err.Error())
|
return 0, C.CString(err.Error())
|
||||||
}
|
}
|
||||||
config.ProductionMode = true
|
config.ProductionMode = true
|
||||||
config.NativeMode = true
|
|
||||||
applyWebViewProvider(&config)
|
|
||||||
realPort, err := rest.Start(&config)
|
realPort, err := rest.Start(&config)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return 0, C.CString(err.Error())
|
return 0, C.CString(err.Error())
|
||||||
}
|
}
|
||||||
nativebridge.ResumeInvokes()
|
|
||||||
return realPort, nil
|
return realPort, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
//export Stop
|
//export Stop
|
||||||
func Stop() {
|
func Stop() {
|
||||||
nativebridge.Stop()
|
rest.Stop()
|
||||||
}
|
|
||||||
|
|
||||||
//export GetAPIServerState
|
|
||||||
func GetAPIServerState() *C.char {
|
|
||||||
return C.CString(apiServerResult(rest.GetAPIServerState()))
|
|
||||||
}
|
|
||||||
|
|
||||||
//export StartAPIServer
|
|
||||||
func StartAPIServer() *C.char {
|
|
||||||
return C.CString(apiServerResult(rest.StartAPIServer()))
|
|
||||||
}
|
|
||||||
|
|
||||||
//export StopAPIServer
|
|
||||||
func StopAPIServer() *C.char {
|
|
||||||
return C.CString(apiServerResult(rest.StopAPIServer()))
|
|
||||||
}
|
|
||||||
|
|
||||||
//export RestartAPIServer
|
|
||||||
func RestartAPIServer() *C.char {
|
|
||||||
return C.CString(apiServerResult(rest.RestartAPIServer()))
|
|
||||||
}
|
|
||||||
|
|
||||||
func apiServerResult(state *model.APIServerState, err error) string {
|
|
||||||
result := &model.APIServerOperationResult{State: state}
|
|
||||||
if err != nil {
|
|
||||||
result.Error = err.Error()
|
|
||||||
}
|
|
||||||
data, marshalErr := json.Marshal(result)
|
|
||||||
if marshalErr != nil {
|
|
||||||
return fmt.Sprintf(`{"state":null,"error":%q}`, marshalErr.Error())
|
|
||||||
}
|
|
||||||
return string(data)
|
|
||||||
}
|
|
||||||
|
|
||||||
//export InvokeAsync
|
|
||||||
func InvokeAsync(
|
|
||||||
method *C.char,
|
|
||||||
path *C.char,
|
|
||||||
query *C.char,
|
|
||||||
body *C.char,
|
|
||||||
requestID C.ulonglong,
|
|
||||||
callback C.uintptr_t,
|
|
||||||
) {
|
|
||||||
if callback == 0 {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
complete := func(result string, err error) {
|
|
||||||
success := C.int(1)
|
|
||||||
payload := result
|
|
||||||
if err != nil {
|
|
||||||
success = 0
|
|
||||||
payload = err.Error()
|
|
||||||
}
|
|
||||||
C.callInvokeResultCallback(
|
|
||||||
callback,
|
|
||||||
C.uint64_t(requestID),
|
|
||||||
success,
|
|
||||||
C.CString(payload),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
nativebridge.InvokeAsync(
|
|
||||||
C.GoString(method),
|
|
||||||
C.GoString(path),
|
|
||||||
C.GoString(query),
|
|
||||||
C.GoString(body),
|
|
||||||
complete,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
//export SubscribeTaskEvents
|
|
||||||
func SubscribeTaskEvents(mask C.ulonglong, callback C.uintptr_t) {
|
|
||||||
if mask == 0 || callback == 0 {
|
|
||||||
rest.SubscribeTaskEvents(0, nil)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
rest.SubscribeTaskEvents(uint64(mask), func(payload string) {
|
|
||||||
// Ownership crosses the asynchronous Dart callback boundary. Dart frees
|
|
||||||
// this string with FreeCString after decoding it.
|
|
||||||
C.callTaskEventCallback(callback, C.CString(payload))
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
//export FreeCString
|
|
||||||
func FreeCString(value *C.char) {
|
|
||||||
if value != nil {
|
|
||||||
C.free(unsafe.Pointer(value))
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
@@ -1,23 +0,0 @@
|
|||||||
//go:build darwin
|
|
||||||
|
|
||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"github.com/GopeedLab/gopeed/internal/webview/rpcprovider"
|
|
||||||
enginewebview "github.com/GopeedLab/gopeed/pkg/download/engine/webview"
|
|
||||||
"github.com/GopeedLab/gopeed/pkg/rest/model"
|
|
||||||
)
|
|
||||||
|
|
||||||
func applyWebViewProvider(config *model.StartConfig) {
|
|
||||||
if config == nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
config.WebViewProvider = newDesktopWebViewProvider(config.WebViewRPCConfig)
|
|
||||||
}
|
|
||||||
|
|
||||||
func newDesktopWebViewProvider(rpcCfg *enginewebview.RPCConfig) enginewebview.Provider {
|
|
||||||
if rpcCfg != nil && rpcCfg.Enabled() {
|
|
||||||
return rpcprovider.New(*rpcCfg)
|
|
||||||
}
|
|
||||||
return enginewebview.NewUnavailableProvider()
|
|
||||||
}
|
|
||||||
@@ -1,15 +0,0 @@
|
|||||||
//go:build !darwin
|
|
||||||
|
|
||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"github.com/GopeedLab/gopeed/internal/webview/goprovider"
|
|
||||||
"github.com/GopeedLab/gopeed/pkg/rest/model"
|
|
||||||
)
|
|
||||||
|
|
||||||
func applyWebViewProvider(config *model.StartConfig) {
|
|
||||||
if config == nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
config.WebViewProvider = goprovider.New()
|
|
||||||
}
|
|
||||||
@@ -4,10 +4,6 @@ package libgopeed
|
|||||||
import "C"
|
import "C"
|
||||||
import (
|
import (
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
|
|
||||||
nativebridge "github.com/GopeedLab/gopeed/bind/native"
|
|
||||||
"github.com/GopeedLab/gopeed/internal/webview/rpcprovider"
|
|
||||||
goapi "github.com/GopeedLab/gopeed/pkg/api"
|
|
||||||
"github.com/GopeedLab/gopeed/pkg/rest"
|
"github.com/GopeedLab/gopeed/pkg/rest"
|
||||||
"github.com/GopeedLab/gopeed/pkg/rest/model"
|
"github.com/GopeedLab/gopeed/pkg/rest/model"
|
||||||
)
|
)
|
||||||
@@ -18,87 +14,9 @@ func Start(cfg string) (int, error) {
|
|||||||
return 0, err
|
return 0, err
|
||||||
}
|
}
|
||||||
config.ProductionMode = true
|
config.ProductionMode = true
|
||||||
config.NativeMode = true
|
return rest.Start(&config)
|
||||||
applyWebViewProvider(&config)
|
|
||||||
port, err := rest.Start(&config)
|
|
||||||
if err == nil {
|
|
||||||
nativebridge.ResumeInvokes()
|
|
||||||
}
|
|
||||||
return port, err
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func Stop() {
|
func Stop() {
|
||||||
nativebridge.Stop()
|
rest.Stop()
|
||||||
}
|
|
||||||
|
|
||||||
func GetAPIServerState() string {
|
|
||||||
return apiServerResult(rest.GetAPIServerState())
|
|
||||||
}
|
|
||||||
|
|
||||||
func StartAPIServer() string {
|
|
||||||
return apiServerResult(rest.StartAPIServer())
|
|
||||||
}
|
|
||||||
|
|
||||||
func StopAPIServer() string {
|
|
||||||
return apiServerResult(rest.StopAPIServer())
|
|
||||||
}
|
|
||||||
|
|
||||||
func RestartAPIServer() string {
|
|
||||||
return apiServerResult(rest.RestartAPIServer())
|
|
||||||
}
|
|
||||||
|
|
||||||
func apiServerResult(state *model.APIServerState, err error) string {
|
|
||||||
result := &model.APIServerOperationResult{State: state}
|
|
||||||
if err != nil {
|
|
||||||
result.Error = err.Error()
|
|
||||||
}
|
|
||||||
data, _ := json.Marshal(result)
|
|
||||||
return string(data)
|
|
||||||
}
|
|
||||||
|
|
||||||
type InvokeResultListener interface {
|
|
||||||
OnResult(requestID int64, success bool, payload string)
|
|
||||||
}
|
|
||||||
|
|
||||||
func InvokeAsync(method, path, query, body string, requestID int64, listener InvokeResultListener) {
|
|
||||||
if listener == nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
nativebridge.InvokeAsync(method, path, query, body, func(result string, err error) {
|
|
||||||
if err != nil {
|
|
||||||
listener.OnResult(requestID, false, err.Error())
|
|
||||||
return
|
|
||||||
}
|
|
||||||
listener.OnResult(requestID, true, result)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
type TaskEventListener interface {
|
|
||||||
OnTaskEvent(payload string)
|
|
||||||
}
|
|
||||||
|
|
||||||
func SubscribeTaskEvents(mask int64, listener TaskEventListener) {
|
|
||||||
if mask == 0 || listener == nil {
|
|
||||||
rest.SubscribeTaskEvents(0, nil)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
rest.SubscribeTaskEvents(uint64(mask), listener.OnTaskEvent)
|
|
||||||
}
|
|
||||||
|
|
||||||
func applyWebViewProvider(config *model.StartConfig) {
|
|
||||||
if config == nil || config.WebViewRPCConfig == nil || !config.WebViewRPCConfig.Enabled() {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
config.WebViewProvider = rpcprovider.New(*config.WebViewRPCConfig)
|
|
||||||
}
|
|
||||||
|
|
||||||
func LiveActivityTaskEventMask() int64 {
|
|
||||||
return int64(
|
|
||||||
goapi.TaskEventDone |
|
|
||||||
goapi.TaskEventError |
|
|
||||||
goapi.TaskEventStart |
|
|
||||||
goapi.TaskEventProgress |
|
|
||||||
goapi.TaskEventPause |
|
|
||||||
goapi.TaskEventDelete,
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
@@ -1,240 +0,0 @@
|
|||||||
// Package nativebridge contains transport-neutral behavior shared by the
|
|
||||||
// Desktop C FFI and gomobile bridge adapters.
|
|
||||||
package nativebridge
|
|
||||||
|
|
||||||
import (
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"sync"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/GopeedLab/gopeed/pkg/rest"
|
|
||||||
)
|
|
||||||
|
|
||||||
// StopTimeout bounds native shutdown even when an in-process request never returns.
|
|
||||||
const StopTimeout = 3 * time.Second
|
|
||||||
|
|
||||||
var errNativeBridgeStopping = errors.New("native bridge is stopping")
|
|
||||||
|
|
||||||
type invokeRequest struct {
|
|
||||||
method string
|
|
||||||
path string
|
|
||||||
query string
|
|
||||||
body string
|
|
||||||
}
|
|
||||||
|
|
||||||
type invokeTask struct {
|
|
||||||
id uint64
|
|
||||||
request invokeRequest
|
|
||||||
complete func(result string, err error)
|
|
||||||
once sync.Once
|
|
||||||
}
|
|
||||||
|
|
||||||
func (t *invokeTask) finish(result string, err error) {
|
|
||||||
t.once.Do(func() {
|
|
||||||
completeCallback(t.complete, result, err)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// InvokeAsync starts an in-process API request and completes it through the
|
|
||||||
// supplied callback. Desktop and mobile adapters share this executor.
|
|
||||||
func InvokeAsync(method, path, query, body string, callback func(result string, err error)) {
|
|
||||||
if callback == nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
task := invokeTask{
|
|
||||||
request: invokeRequest{
|
|
||||||
method: method,
|
|
||||||
path: path,
|
|
||||||
query: query,
|
|
||||||
body: body,
|
|
||||||
},
|
|
||||||
complete: callback,
|
|
||||||
}
|
|
||||||
if !getInvokeExecutor().submit(task) {
|
|
||||||
completeCallback(callback, "", errNativeBridgeStopping)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// ResumeInvokes enables request submission after the native runtime starts.
|
|
||||||
func ResumeInvokes() {
|
|
||||||
getInvokeExecutor().resume()
|
|
||||||
}
|
|
||||||
|
|
||||||
// PauseInvokesAndWait prevents new submissions and gives running requests a
|
|
||||||
// bounded opportunity to finish before a native runtime is stopped.
|
|
||||||
// Requests still pending after StopTimeout are failed and detached so shutdown
|
|
||||||
// can continue without invoking their callbacks a second time.
|
|
||||||
func PauseInvokesAndWait() bool {
|
|
||||||
return getInvokeExecutor().pauseAndWait(StopTimeout)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Stop pauses native requests and shuts down the shared runtime, but never
|
|
||||||
// keeps the host application waiting longer than StopTimeout. Cleanup that
|
|
||||||
// cannot finish within the deadline is allowed to continue in the background.
|
|
||||||
func Stop() {
|
|
||||||
stopWithin(getInvokeExecutor(), rest.Stop, StopTimeout)
|
|
||||||
}
|
|
||||||
|
|
||||||
func stopWithin(executor *invokeExecutor, cleanup func(), timeout time.Duration) {
|
|
||||||
deadline := time.Now().Add(timeout)
|
|
||||||
executor.pauseAndWait(time.Until(deadline))
|
|
||||||
|
|
||||||
stopped := make(chan struct{})
|
|
||||||
go func() {
|
|
||||||
defer close(stopped)
|
|
||||||
cleanup()
|
|
||||||
}()
|
|
||||||
|
|
||||||
remaining := time.Until(deadline)
|
|
||||||
if remaining <= 0 {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
timer := time.NewTimer(remaining)
|
|
||||||
defer timer.Stop()
|
|
||||||
select {
|
|
||||||
case <-stopped:
|
|
||||||
case <-timer.C:
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
var (
|
|
||||||
invokeExecutorOnce sync.Once
|
|
||||||
sharedInvoker *invokeExecutor
|
|
||||||
)
|
|
||||||
|
|
||||||
func getInvokeExecutor() *invokeExecutor {
|
|
||||||
invokeExecutorOnce.Do(func() {
|
|
||||||
sharedInvoker = newInvokeExecutor(func(request invokeRequest) string {
|
|
||||||
return rest.Dispatch(request.method, request.path, request.query, request.body)
|
|
||||||
})
|
|
||||||
})
|
|
||||||
return sharedInvoker
|
|
||||||
}
|
|
||||||
|
|
||||||
// invokeExecutor starts every native API request in its own goroutine, matching
|
|
||||||
// net/http's request concurrency. It only tracks them to support bounded drain
|
|
||||||
// during shutdown; one stuck request must never prevent another from starting.
|
|
||||||
type invokeExecutor struct {
|
|
||||||
mu sync.Mutex
|
|
||||||
closed bool
|
|
||||||
accepting bool
|
|
||||||
nextID uint64
|
|
||||||
pending map[uint64]*invokeTask
|
|
||||||
drained chan struct{}
|
|
||||||
running sync.WaitGroup
|
|
||||||
handler func(invokeRequest) string
|
|
||||||
}
|
|
||||||
|
|
||||||
func newInvokeExecutor(handler func(invokeRequest) string) *invokeExecutor {
|
|
||||||
drained := make(chan struct{})
|
|
||||||
close(drained)
|
|
||||||
return &invokeExecutor{
|
|
||||||
handler: handler,
|
|
||||||
accepting: true,
|
|
||||||
pending: make(map[uint64]*invokeTask),
|
|
||||||
drained: drained,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (e *invokeExecutor) submit(task invokeTask) bool {
|
|
||||||
e.mu.Lock()
|
|
||||||
if e.closed || !e.accepting {
|
|
||||||
e.mu.Unlock()
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
if len(e.pending) == 0 {
|
|
||||||
e.drained = make(chan struct{})
|
|
||||||
}
|
|
||||||
taskID := e.nextID
|
|
||||||
e.nextID++
|
|
||||||
task.id = taskID
|
|
||||||
taskPtr := &task
|
|
||||||
e.pending[taskID] = taskPtr
|
|
||||||
e.running.Add(1)
|
|
||||||
e.mu.Unlock()
|
|
||||||
|
|
||||||
go e.run(taskPtr)
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
|
|
||||||
func (e *invokeExecutor) resume() {
|
|
||||||
e.mu.Lock()
|
|
||||||
e.accepting = true
|
|
||||||
e.mu.Unlock()
|
|
||||||
}
|
|
||||||
|
|
||||||
func (e *invokeExecutor) pauseAndWait(timeout time.Duration) bool {
|
|
||||||
e.mu.Lock()
|
|
||||||
e.accepting = false
|
|
||||||
if len(e.pending) == 0 {
|
|
||||||
e.mu.Unlock()
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
drained := e.drained
|
|
||||||
e.mu.Unlock()
|
|
||||||
|
|
||||||
timer := time.NewTimer(timeout)
|
|
||||||
defer timer.Stop()
|
|
||||||
select {
|
|
||||||
case <-drained:
|
|
||||||
return true
|
|
||||||
case <-timer.C:
|
|
||||||
}
|
|
||||||
|
|
||||||
e.mu.Lock()
|
|
||||||
if len(e.pending) == 0 {
|
|
||||||
e.mu.Unlock()
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
pending := make([]*invokeTask, 0, len(e.pending))
|
|
||||||
for _, task := range e.pending {
|
|
||||||
pending = append(pending, task)
|
|
||||||
}
|
|
||||||
e.pending = make(map[uint64]*invokeTask)
|
|
||||||
close(e.drained)
|
|
||||||
e.mu.Unlock()
|
|
||||||
|
|
||||||
for _, task := range pending {
|
|
||||||
task.finish("", errNativeBridgeStopping)
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
func (e *invokeExecutor) close() {
|
|
||||||
e.mu.Lock()
|
|
||||||
e.closed = true
|
|
||||||
e.accepting = false
|
|
||||||
e.mu.Unlock()
|
|
||||||
e.running.Wait()
|
|
||||||
}
|
|
||||||
|
|
||||||
func (e *invokeExecutor) run(task *invokeTask) {
|
|
||||||
defer e.running.Done()
|
|
||||||
result, err := e.execute(task.request)
|
|
||||||
task.finish(result, err)
|
|
||||||
e.mu.Lock()
|
|
||||||
if _, exists := e.pending[task.id]; exists {
|
|
||||||
delete(e.pending, task.id)
|
|
||||||
if len(e.pending) == 0 {
|
|
||||||
close(e.drained)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
e.mu.Unlock()
|
|
||||||
}
|
|
||||||
|
|
||||||
func (e *invokeExecutor) execute(request invokeRequest) (result string, err error) {
|
|
||||||
defer func() {
|
|
||||||
if recovered := recover(); recovered != nil {
|
|
||||||
err = fmt.Errorf("invoke panic: %v", recovered)
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
return e.handler(request), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func completeCallback(callback func(string, error), result string, err error) {
|
|
||||||
defer func() {
|
|
||||||
_ = recover()
|
|
||||||
}()
|
|
||||||
callback(result, err)
|
|
||||||
}
|
|
||||||
@@ -1,258 +0,0 @@
|
|||||||
package nativebridge
|
|
||||||
|
|
||||||
import (
|
|
||||||
"runtime"
|
|
||||||
"strconv"
|
|
||||||
"sync"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestInvokeExecutorStartsEveryRequestIndependently(t *testing.T) {
|
|
||||||
const taskCount = 64
|
|
||||||
release := make(chan struct{})
|
|
||||||
started := make(chan struct{}, taskCount)
|
|
||||||
executor := newInvokeExecutor(func(request invokeRequest) string {
|
|
||||||
started <- struct{}{}
|
|
||||||
<-release
|
|
||||||
return request.path
|
|
||||||
})
|
|
||||||
|
|
||||||
var completed sync.WaitGroup
|
|
||||||
completed.Add(taskCount)
|
|
||||||
for i := range taskCount {
|
|
||||||
accepted := executor.submit(invokeTask{
|
|
||||||
request: invokeRequest{path: strconv.Itoa(i)},
|
|
||||||
complete: func(string, error) {
|
|
||||||
completed.Done()
|
|
||||||
},
|
|
||||||
})
|
|
||||||
if !accepted {
|
|
||||||
t.Fatalf("request %d was rejected", i)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Every handler must start before any of them is released. A worker pool
|
|
||||||
// would stall here once all of its workers were occupied.
|
|
||||||
for range taskCount {
|
|
||||||
select {
|
|
||||||
case <-started:
|
|
||||||
case <-time.After(time.Second):
|
|
||||||
t.Fatal("a request waited behind other blocked requests")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
close(release)
|
|
||||||
completed.Wait()
|
|
||||||
executor.close()
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestInvokeExecutorConvertsPanicsToErrors(t *testing.T) {
|
|
||||||
executor := newInvokeExecutor(func(invokeRequest) string {
|
|
||||||
panic("boom")
|
|
||||||
})
|
|
||||||
completed := make(chan error, 1)
|
|
||||||
executor.submit(invokeTask{complete: func(_ string, err error) {
|
|
||||||
completed <- err
|
|
||||||
}})
|
|
||||||
select {
|
|
||||||
case err := <-completed:
|
|
||||||
if err == nil || err.Error() != "invoke panic: boom" {
|
|
||||||
t.Fatalf("unexpected error: %v", err)
|
|
||||||
}
|
|
||||||
case <-time.After(time.Second):
|
|
||||||
t.Fatal("panic result was not completed")
|
|
||||||
}
|
|
||||||
executor.close()
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestInvokeExecutorSurvivesCallbackPanic(t *testing.T) {
|
|
||||||
executor := newInvokeExecutor(func(request invokeRequest) string {
|
|
||||||
return request.path
|
|
||||||
})
|
|
||||||
executor.submit(invokeTask{complete: func(string, error) {
|
|
||||||
panic("callback failed")
|
|
||||||
}})
|
|
||||||
|
|
||||||
completed := make(chan string, 1)
|
|
||||||
executor.submit(invokeTask{
|
|
||||||
request: invokeRequest{path: "next"},
|
|
||||||
complete: func(result string, _ error) {
|
|
||||||
completed <- result
|
|
||||||
},
|
|
||||||
})
|
|
||||||
select {
|
|
||||||
case result := <-completed:
|
|
||||||
if result != "next" {
|
|
||||||
t.Fatalf("unexpected result: %q", result)
|
|
||||||
}
|
|
||||||
case <-time.After(time.Second):
|
|
||||||
t.Fatal("worker stopped after callback panic")
|
|
||||||
}
|
|
||||||
executor.close()
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestInvokeExecutorPausesAndDrains(t *testing.T) {
|
|
||||||
release := make(chan struct{})
|
|
||||||
started := make(chan struct{})
|
|
||||||
executor := newInvokeExecutor(func(request invokeRequest) string {
|
|
||||||
if request.path == "running" {
|
|
||||||
close(started)
|
|
||||||
<-release
|
|
||||||
}
|
|
||||||
return request.path
|
|
||||||
})
|
|
||||||
completed := make(chan struct{})
|
|
||||||
if !executor.submit(invokeTask{
|
|
||||||
request: invokeRequest{path: "running"},
|
|
||||||
complete: func(string, error) {
|
|
||||||
close(completed)
|
|
||||||
},
|
|
||||||
}) {
|
|
||||||
t.Fatal("initial task was rejected")
|
|
||||||
}
|
|
||||||
<-started
|
|
||||||
|
|
||||||
drained := make(chan struct{})
|
|
||||||
go func() {
|
|
||||||
executor.pauseAndWait(time.Second)
|
|
||||||
close(drained)
|
|
||||||
}()
|
|
||||||
deadline := time.Now().Add(time.Second)
|
|
||||||
for {
|
|
||||||
executor.mu.Lock()
|
|
||||||
paused := !executor.accepting
|
|
||||||
executor.mu.Unlock()
|
|
||||||
if paused {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
if time.Now().After(deadline) {
|
|
||||||
t.Fatal("executor did not pause")
|
|
||||||
}
|
|
||||||
runtime.Gosched()
|
|
||||||
}
|
|
||||||
select {
|
|
||||||
case <-drained:
|
|
||||||
t.Fatal("pause returned before the running request completed")
|
|
||||||
default:
|
|
||||||
}
|
|
||||||
if executor.submit(invokeTask{}) {
|
|
||||||
t.Fatal("paused executor accepted a new request")
|
|
||||||
}
|
|
||||||
|
|
||||||
close(release)
|
|
||||||
<-completed
|
|
||||||
select {
|
|
||||||
case <-drained:
|
|
||||||
case <-time.After(time.Second):
|
|
||||||
t.Fatal("pause did not return after pending requests drained")
|
|
||||||
}
|
|
||||||
|
|
||||||
executor.resume()
|
|
||||||
resumed := make(chan struct{})
|
|
||||||
if !executor.submit(invokeTask{complete: func(string, error) {
|
|
||||||
close(resumed)
|
|
||||||
}}) {
|
|
||||||
t.Fatal("resumed executor rejected a request")
|
|
||||||
}
|
|
||||||
<-resumed
|
|
||||||
executor.close()
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestInvokeExecutorPauseTimesOutAndFailsPendingRequests(t *testing.T) {
|
|
||||||
release := make(chan struct{})
|
|
||||||
started := make(chan struct{})
|
|
||||||
executor := newInvokeExecutor(func(request invokeRequest) string {
|
|
||||||
started <- struct{}{}
|
|
||||||
<-release
|
|
||||||
return request.path
|
|
||||||
})
|
|
||||||
|
|
||||||
errors := make(chan error, 2)
|
|
||||||
for _, path := range []string{"first", "second"} {
|
|
||||||
if !executor.submit(invokeTask{
|
|
||||||
request: invokeRequest{path: path},
|
|
||||||
complete: func(_ string, err error) {
|
|
||||||
errors <- err
|
|
||||||
},
|
|
||||||
}) {
|
|
||||||
t.Fatalf("request %q was rejected", path)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for range 2 {
|
|
||||||
<-started
|
|
||||||
}
|
|
||||||
|
|
||||||
const timeout = 30 * time.Millisecond
|
|
||||||
startedAt := time.Now()
|
|
||||||
if executor.pauseAndWait(timeout) {
|
|
||||||
t.Fatal("pause unexpectedly reported a complete drain")
|
|
||||||
}
|
|
||||||
if elapsed := time.Since(startedAt); elapsed < timeout || elapsed > 10*timeout {
|
|
||||||
t.Fatalf("pause returned after %s, want close to %s", elapsed, timeout)
|
|
||||||
}
|
|
||||||
for range 2 {
|
|
||||||
select {
|
|
||||||
case err := <-errors:
|
|
||||||
if err == nil || err.Error() != errNativeBridgeStopping.Error() {
|
|
||||||
t.Fatalf("unexpected timeout error: %v", err)
|
|
||||||
}
|
|
||||||
case <-time.After(time.Second):
|
|
||||||
t.Fatal("pending request was not failed after the timeout")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
close(release)
|
|
||||||
executor.close()
|
|
||||||
select {
|
|
||||||
case err := <-errors:
|
|
||||||
t.Fatalf("request callback ran more than once: %v", err)
|
|
||||||
default:
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestStopWithinUsesOneDeadlineForRequestsAndCleanup(t *testing.T) {
|
|
||||||
releaseRequest := make(chan struct{})
|
|
||||||
requestStarted := make(chan struct{})
|
|
||||||
executor := newInvokeExecutor(func(invokeRequest) string {
|
|
||||||
close(requestStarted)
|
|
||||||
<-releaseRequest
|
|
||||||
return "done"
|
|
||||||
})
|
|
||||||
requestResult := make(chan error, 1)
|
|
||||||
executor.submit(invokeTask{complete: func(_ string, err error) {
|
|
||||||
requestResult <- err
|
|
||||||
}})
|
|
||||||
<-requestStarted
|
|
||||||
|
|
||||||
releaseCleanup := make(chan struct{})
|
|
||||||
cleanupStarted := make(chan struct{})
|
|
||||||
cleanup := func() {
|
|
||||||
close(cleanupStarted)
|
|
||||||
<-releaseCleanup
|
|
||||||
}
|
|
||||||
|
|
||||||
const timeout = 40 * time.Millisecond
|
|
||||||
startedAt := time.Now()
|
|
||||||
stopWithin(executor, cleanup, timeout)
|
|
||||||
if elapsed := time.Since(startedAt); elapsed < timeout || elapsed > 10*timeout {
|
|
||||||
t.Fatalf("stop returned after %s, want close to %s", elapsed, timeout)
|
|
||||||
}
|
|
||||||
select {
|
|
||||||
case <-cleanupStarted:
|
|
||||||
case <-time.After(time.Second):
|
|
||||||
t.Fatal("cleanup was not started after the request deadline")
|
|
||||||
}
|
|
||||||
select {
|
|
||||||
case err := <-requestResult:
|
|
||||||
if err == nil || err.Error() != errNativeBridgeStopping.Error() {
|
|
||||||
t.Fatalf("unexpected request result: %v", err)
|
|
||||||
}
|
|
||||||
case <-time.After(time.Second):
|
|
||||||
t.Fatal("timed-out request was not failed")
|
|
||||||
}
|
|
||||||
|
|
||||||
close(releaseRequest)
|
|
||||||
close(releaseCleanup)
|
|
||||||
executor.close()
|
|
||||||
}
|
|
||||||
@@ -1,9 +1,6 @@
|
|||||||
package main
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
|
||||||
"os"
|
|
||||||
|
|
||||||
"github.com/GopeedLab/gopeed/cmd"
|
"github.com/GopeedLab/gopeed/cmd"
|
||||||
"github.com/GopeedLab/gopeed/pkg/rest/model"
|
"github.com/GopeedLab/gopeed/pkg/rest/model"
|
||||||
)
|
)
|
||||||
@@ -16,8 +13,5 @@ func main() {
|
|||||||
Storage: model.StorageBolt,
|
Storage: model.StorageBolt,
|
||||||
WebEnable: true,
|
WebEnable: true,
|
||||||
}
|
}
|
||||||
if err := cmd.Start(cfg); err != nil {
|
cmd.Start(cfg)
|
||||||
fmt.Fprintln(os.Stderr, err)
|
|
||||||
os.Exit(1)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
@@ -1,723 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"encoding/json"
|
|
||||||
"flag"
|
|
||||||
"fmt"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/GopeedLab/gopeed/internal/webview/goprovider"
|
|
||||||
"github.com/GopeedLab/gopeed/internal/webview/rpcprovider"
|
|
||||||
"github.com/GopeedLab/gopeed/pkg/base"
|
|
||||||
"github.com/GopeedLab/gopeed/pkg/download"
|
|
||||||
enginewebview "github.com/GopeedLab/gopeed/pkg/download/engine/webview"
|
|
||||||
)
|
|
||||||
|
|
||||||
type kvFlag map[string]string
|
|
||||||
|
|
||||||
type webViewConfig struct {
|
|
||||||
provider string
|
|
||||||
network string
|
|
||||||
address string
|
|
||||||
token string
|
|
||||||
}
|
|
||||||
|
|
||||||
func (l *kvFlag) String() string {
|
|
||||||
if l == nil {
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
data, _ := json.Marshal(map[string]string(*l))
|
|
||||||
return string(data)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (l *kvFlag) Set(value string) error {
|
|
||||||
if *l == nil {
|
|
||||||
*l = make(map[string]string)
|
|
||||||
}
|
|
||||||
idx := -1
|
|
||||||
for i := 0; i < len(value); i++ {
|
|
||||||
if value[i] == '=' {
|
|
||||||
idx = i
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if idx <= 0 {
|
|
||||||
return fmt.Errorf("invalid label %q, expected key=value", value)
|
|
||||||
}
|
|
||||||
(*l)[value[:idx]] = value[idx+1:]
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func main() {
|
|
||||||
os.Exit(goprovider.RunMainThreadLoop(run))
|
|
||||||
}
|
|
||||||
|
|
||||||
func run() int {
|
|
||||||
if len(os.Args) < 2 {
|
|
||||||
usage()
|
|
||||||
return 2
|
|
||||||
}
|
|
||||||
|
|
||||||
switch os.Args[1] {
|
|
||||||
case "resolve":
|
|
||||||
if err := runResolve(os.Args[2:]); err != nil {
|
|
||||||
fmt.Fprintln(os.Stderr, err)
|
|
||||||
return 1
|
|
||||||
}
|
|
||||||
case "download":
|
|
||||||
if err := runDownload(os.Args[2:]); err != nil {
|
|
||||||
fmt.Fprintln(os.Stderr, err)
|
|
||||||
return 1
|
|
||||||
}
|
|
||||||
case "eval":
|
|
||||||
if err := runEval(os.Args[2:]); err != nil {
|
|
||||||
fmt.Fprintln(os.Stderr, err)
|
|
||||||
return 1
|
|
||||||
}
|
|
||||||
case "-h", "--help", "help":
|
|
||||||
usage()
|
|
||||||
default:
|
|
||||||
fmt.Fprintf(os.Stderr, "unknown subcommand %q\n", os.Args[1])
|
|
||||||
usage()
|
|
||||||
return 2
|
|
||||||
}
|
|
||||||
return 0
|
|
||||||
}
|
|
||||||
|
|
||||||
func usage() {
|
|
||||||
fmt.Fprintf(os.Stderr, `gopeed extmock CLI
|
|
||||||
|
|
||||||
Usage:
|
|
||||||
extmock resolve [flags] <extension-dir> <url>
|
|
||||||
extmock download [flags] <extension-dir> <url> <output-dir>
|
|
||||||
extmock eval [flags] [script-file]
|
|
||||||
|
|
||||||
Flags:
|
|
||||||
-label key=value Request label, repeatable
|
|
||||||
-setting key=value Extension setting override, repeatable
|
|
||||||
-webview-provider WebView provider: go|rpc
|
|
||||||
-webview-network RPC network when provider=rpc: unix|tcp
|
|
||||||
-webview-address RPC address when provider=rpc
|
|
||||||
-webview-token RPC bearer token when provider=rpc
|
|
||||||
-verb Print download progress while running
|
|
||||||
-pretty Pretty-print JSON output
|
|
||||||
|
|
||||||
Examples:
|
|
||||||
extmock resolve ./ https://www.google.com
|
|
||||||
extmock download ./ https://example.com/video /tmp/extmock-download
|
|
||||||
extmock resolve ./ https://example.com/video -label env=test -setting token=abc
|
|
||||||
extmock eval ./test.js
|
|
||||||
extmock eval -e "new Blob(['hello']).size"
|
|
||||||
`)
|
|
||||||
}
|
|
||||||
|
|
||||||
func runResolve(args []string) error {
|
|
||||||
fs := flag.NewFlagSet("resolve", flag.ContinueOnError)
|
|
||||||
fs.SetOutput(os.Stderr)
|
|
||||||
var labels kvFlag
|
|
||||||
var settings kvFlag
|
|
||||||
var pretty bool
|
|
||||||
cfg := webViewConfig{}
|
|
||||||
fs.Var(&labels, "label", "Request label in key=value form (repeatable)")
|
|
||||||
fs.Var(&settings, "setting", "Extension setting override in key=value form (repeatable)")
|
|
||||||
fs.StringVar(&cfg.provider, "webview-provider", "", "WebView provider: go|rpc")
|
|
||||||
fs.StringVar(&cfg.network, "webview-network", "", "WebView RPC network: unix|tcp")
|
|
||||||
fs.StringVar(&cfg.address, "webview-address", "", "WebView RPC address")
|
|
||||||
fs.StringVar(&cfg.token, "webview-token", "", "WebView RPC bearer token")
|
|
||||||
fs.BoolVar(&pretty, "pretty", false, "Pretty-print JSON output")
|
|
||||||
|
|
||||||
flagArgs, positionalArgs, err := splitResolveArgs(args)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if err := fs.Parse(flagArgs); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if fs.NArg() != 0 {
|
|
||||||
return fmt.Errorf("unexpected trailing args: %v", fs.Args())
|
|
||||||
}
|
|
||||||
if len(positionalArgs) != 2 {
|
|
||||||
return fmt.Errorf("resolve expects exactly 2 args: <extension-dir> <url>")
|
|
||||||
}
|
|
||||||
|
|
||||||
extDir, err := filepath.Abs(positionalArgs[0])
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
url := positionalArgs[1]
|
|
||||||
|
|
||||||
d, cleanup, err := newExtMockDownloader(cfg)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
defer cleanup()
|
|
||||||
|
|
||||||
ext, err := d.InstallExtensionByFolder(extDir, true)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if len(settings) > 0 {
|
|
||||||
if err := d.UpdateExtensionSettings(ext.Identity, toAnyMap(settings)); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
result, err := d.Resolve(&base.Request{
|
|
||||||
URL: url,
|
|
||||||
Labels: map[string]string(labels),
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
return printJSON(result, pretty)
|
|
||||||
}
|
|
||||||
|
|
||||||
type downloadResult struct {
|
|
||||||
Resolve *download.ResolveResult `json:"resolve"`
|
|
||||||
OutputDir string `json:"outputDir"`
|
|
||||||
Files []*downloadedFileResult `json:"files"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type downloadedFileResult struct {
|
|
||||||
Index int `json:"index"`
|
|
||||||
Name string `json:"name"`
|
|
||||||
Path string `json:"path"`
|
|
||||||
TaskID string `json:"taskId"`
|
|
||||||
Status base.Status `json:"status"`
|
|
||||||
Size int64 `json:"size"`
|
|
||||||
Req *base.Request `json:"req,omitempty"`
|
|
||||||
}
|
|
||||||
|
|
||||||
func runDownload(args []string) error {
|
|
||||||
fs := flag.NewFlagSet("download", flag.ContinueOnError)
|
|
||||||
fs.SetOutput(os.Stderr)
|
|
||||||
var labels kvFlag
|
|
||||||
var settings kvFlag
|
|
||||||
var pretty bool
|
|
||||||
var verbose bool
|
|
||||||
cfg := webViewConfig{}
|
|
||||||
fs.Var(&labels, "label", "Request label in key=value form (repeatable)")
|
|
||||||
fs.Var(&settings, "setting", "Extension setting override in key=value form (repeatable)")
|
|
||||||
fs.StringVar(&cfg.provider, "webview-provider", "", "WebView provider: go|rpc")
|
|
||||||
fs.StringVar(&cfg.network, "webview-network", "", "WebView RPC network: unix|tcp")
|
|
||||||
fs.StringVar(&cfg.address, "webview-address", "", "WebView RPC address")
|
|
||||||
fs.StringVar(&cfg.token, "webview-token", "", "WebView RPC bearer token")
|
|
||||||
fs.BoolVar(&verbose, "verb", false, "Print download progress while running")
|
|
||||||
fs.BoolVar(&verbose, "verbose", false, "Print download progress while running")
|
|
||||||
fs.BoolVar(&pretty, "pretty", false, "Pretty-print JSON output")
|
|
||||||
|
|
||||||
flagArgs, positionalArgs, err := splitDownloadArgs(args)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if err := fs.Parse(flagArgs); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if fs.NArg() != 0 {
|
|
||||||
return fmt.Errorf("unexpected trailing args: %v", fs.Args())
|
|
||||||
}
|
|
||||||
if len(positionalArgs) != 3 {
|
|
||||||
return fmt.Errorf("download expects exactly 3 args: <extension-dir> <url> <output-dir>")
|
|
||||||
}
|
|
||||||
|
|
||||||
extDir, err := filepath.Abs(positionalArgs[0])
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
url := positionalArgs[1]
|
|
||||||
outputDir, err := filepath.Abs(positionalArgs[2])
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
d, cleanup, err := newExtMockDownloader(cfg)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
defer cleanup()
|
|
||||||
|
|
||||||
ext, err := d.InstallExtensionByFolder(extDir, true)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if len(settings) > 0 {
|
|
||||||
if err := d.UpdateExtensionSettings(ext.Identity, toAnyMap(settings)); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
result, err := d.Resolve(&base.Request{
|
|
||||||
URL: url,
|
|
||||||
Labels: map[string]string(labels),
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if result == nil || result.Res == nil {
|
|
||||||
return fmt.Errorf("resolve returned empty resource")
|
|
||||||
}
|
|
||||||
if len(result.Res.Files) == 0 {
|
|
||||||
return fmt.Errorf("resolve returned no files")
|
|
||||||
}
|
|
||||||
|
|
||||||
resourceOutputDir := outputDir
|
|
||||||
if result.Res.Name != "" {
|
|
||||||
resourceOutputDir = filepath.Join(outputDir, result.Res.Name)
|
|
||||||
}
|
|
||||||
if err := os.MkdirAll(resourceOutputDir, 0o755); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
waiter := newTaskTerminalWaiter(d)
|
|
||||||
summary := &downloadResult{
|
|
||||||
Resolve: result,
|
|
||||||
OutputDir: resourceOutputDir,
|
|
||||||
Files: make([]*downloadedFileResult, 0, len(result.Res.Files)),
|
|
||||||
}
|
|
||||||
|
|
||||||
for idx, file := range result.Res.Files {
|
|
||||||
if file == nil {
|
|
||||||
return fmt.Errorf("file %d is nil", idx)
|
|
||||||
}
|
|
||||||
if file.Req == nil {
|
|
||||||
return fmt.Errorf("file %d request is nil", idx)
|
|
||||||
}
|
|
||||||
|
|
||||||
targetDir := resourceOutputDir
|
|
||||||
if file.Path != "" {
|
|
||||||
targetDir = filepath.Join(resourceOutputDir, file.Path)
|
|
||||||
}
|
|
||||||
if err := os.MkdirAll(targetDir, 0o755); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
taskID, err := d.CreateDirect(file.Req, &base.Options{
|
|
||||||
Path: targetDir,
|
|
||||||
Name: file.Name,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("create task for %q failed: %w", file.Name, err)
|
|
||||||
}
|
|
||||||
waiter.Register(taskID)
|
|
||||||
|
|
||||||
summary.Files = append(summary.Files, &downloadedFileResult{
|
|
||||||
Index: idx,
|
|
||||||
Name: file.Name,
|
|
||||||
Path: filepath.Join(targetDir, file.Name),
|
|
||||||
TaskID: taskID,
|
|
||||||
Req: file.Req,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
var stopVerbose func()
|
|
||||||
if verbose {
|
|
||||||
stopVerbose = startVerboseTaskListener(d, summary.Files)
|
|
||||||
defer stopVerbose()
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, file := range summary.Files {
|
|
||||||
task, err := waiter.Wait(file.TaskID, 5*time.Minute)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("download %q failed: %w", file.Name, err)
|
|
||||||
}
|
|
||||||
file.Status = task.Status
|
|
||||||
if info, statErr := os.Stat(file.Path); statErr == nil {
|
|
||||||
file.Size = info.Size()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return printJSON(summary, pretty)
|
|
||||||
}
|
|
||||||
|
|
||||||
func runEval(args []string) error {
|
|
||||||
fs := flag.NewFlagSet("eval", flag.ContinueOnError)
|
|
||||||
fs.SetOutput(os.Stderr)
|
|
||||||
var (
|
|
||||||
script string
|
|
||||||
pretty bool
|
|
||||||
)
|
|
||||||
cfg := webViewConfig{}
|
|
||||||
fs.StringVar(&script, "e", "", "Inline JavaScript to execute")
|
|
||||||
fs.StringVar(&script, "eval", "", "Inline JavaScript to execute")
|
|
||||||
fs.StringVar(&cfg.provider, "webview-provider", "", "WebView provider: go|rpc")
|
|
||||||
fs.StringVar(&cfg.network, "webview-network", "", "WebView RPC network: unix|tcp")
|
|
||||||
fs.StringVar(&cfg.address, "webview-address", "", "WebView RPC address")
|
|
||||||
fs.StringVar(&cfg.token, "webview-token", "", "WebView RPC bearer token")
|
|
||||||
fs.BoolVar(&pretty, "pretty", false, "Pretty-print JSON output")
|
|
||||||
|
|
||||||
flagArgs, positionalArgs, err := splitEvalArgs(args)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if err := fs.Parse(flagArgs); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if fs.NArg() != 0 {
|
|
||||||
return fmt.Errorf("unexpected trailing args: %v", fs.Args())
|
|
||||||
}
|
|
||||||
if script == "" && len(positionalArgs) == 0 {
|
|
||||||
return fmt.Errorf("eval expects inline code via -e/--eval or a script file path")
|
|
||||||
}
|
|
||||||
if script != "" && len(positionalArgs) > 0 {
|
|
||||||
return fmt.Errorf("eval accepts either inline code or a script file path, not both")
|
|
||||||
}
|
|
||||||
if len(positionalArgs) > 1 {
|
|
||||||
return fmt.Errorf("eval expects at most 1 script file path")
|
|
||||||
}
|
|
||||||
|
|
||||||
d, cleanup, err := newExtMockDownloader(cfg)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
defer cleanup()
|
|
||||||
|
|
||||||
runtime, err := d.NewExtensionEngine(newExtMockExtension(), map[string]any{})
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
defer runtime.Close()
|
|
||||||
|
|
||||||
var value any
|
|
||||||
if script != "" {
|
|
||||||
value, err = runtime.Eval(script)
|
|
||||||
} else {
|
|
||||||
scriptPath, pathErr := filepath.Abs(positionalArgs[0])
|
|
||||||
if pathErr != nil {
|
|
||||||
return pathErr
|
|
||||||
}
|
|
||||||
value, err = runtime.EvalFile(scriptPath)
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
return printJSON(value, pretty)
|
|
||||||
}
|
|
||||||
|
|
||||||
func splitResolveArgs(args []string) ([]string, []string, error) {
|
|
||||||
var flagArgs []string
|
|
||||||
var positionalArgs []string
|
|
||||||
for i := 0; i < len(args); i++ {
|
|
||||||
arg := args[i]
|
|
||||||
switch {
|
|
||||||
case arg == "-pretty":
|
|
||||||
flagArgs = append(flagArgs, arg)
|
|
||||||
case arg == "-webview-provider", arg == "-webview-network", arg == "-webview-address", arg == "-webview-token":
|
|
||||||
if i+1 >= len(args) {
|
|
||||||
return nil, nil, fmt.Errorf("flag %q requires a value", arg)
|
|
||||||
}
|
|
||||||
flagArgs = append(flagArgs, arg, args[i+1])
|
|
||||||
i++
|
|
||||||
case arg == "-label":
|
|
||||||
if i+1 >= len(args) {
|
|
||||||
return nil, nil, fmt.Errorf("flag %q requires a value", arg)
|
|
||||||
}
|
|
||||||
flagArgs = append(flagArgs, arg, args[i+1])
|
|
||||||
i++
|
|
||||||
case arg == "-setting":
|
|
||||||
if i+1 >= len(args) {
|
|
||||||
return nil, nil, fmt.Errorf("flag %q requires a value", arg)
|
|
||||||
}
|
|
||||||
flagArgs = append(flagArgs, arg, args[i+1])
|
|
||||||
i++
|
|
||||||
case len(arg) > len("-label=") && arg[:len("-label=")] == "-label=":
|
|
||||||
flagArgs = append(flagArgs, arg)
|
|
||||||
case len(arg) > len("-setting=") && arg[:len("-setting=")] == "-setting=":
|
|
||||||
flagArgs = append(flagArgs, arg)
|
|
||||||
case hasFlagValue(arg, "-webview-provider"), hasFlagValue(arg, "-webview-network"),
|
|
||||||
hasFlagValue(arg, "-webview-address"), hasFlagValue(arg, "-webview-token"):
|
|
||||||
flagArgs = append(flagArgs, arg)
|
|
||||||
case len(arg) > 0 && arg[0] == '-':
|
|
||||||
return nil, nil, fmt.Errorf("unknown flag %q", arg)
|
|
||||||
default:
|
|
||||||
positionalArgs = append(positionalArgs, arg)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return flagArgs, positionalArgs, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func splitDownloadArgs(args []string) ([]string, []string, error) {
|
|
||||||
var flagArgs []string
|
|
||||||
var positionalArgs []string
|
|
||||||
for i := 0; i < len(args); i++ {
|
|
||||||
arg := args[i]
|
|
||||||
switch {
|
|
||||||
case arg == "-pretty", arg == "-verb", arg == "-verbose":
|
|
||||||
flagArgs = append(flagArgs, arg)
|
|
||||||
case arg == "-webview-provider", arg == "-webview-network", arg == "-webview-address", arg == "-webview-token":
|
|
||||||
if i+1 >= len(args) {
|
|
||||||
return nil, nil, fmt.Errorf("flag %q requires a value", arg)
|
|
||||||
}
|
|
||||||
flagArgs = append(flagArgs, arg, args[i+1])
|
|
||||||
i++
|
|
||||||
case arg == "-label":
|
|
||||||
if i+1 >= len(args) {
|
|
||||||
return nil, nil, fmt.Errorf("flag %q requires a value", arg)
|
|
||||||
}
|
|
||||||
flagArgs = append(flagArgs, arg, args[i+1])
|
|
||||||
i++
|
|
||||||
case arg == "-setting":
|
|
||||||
if i+1 >= len(args) {
|
|
||||||
return nil, nil, fmt.Errorf("flag %q requires a value", arg)
|
|
||||||
}
|
|
||||||
flagArgs = append(flagArgs, arg, args[i+1])
|
|
||||||
i++
|
|
||||||
case len(arg) > len("-label=") && arg[:len("-label=")] == "-label=":
|
|
||||||
flagArgs = append(flagArgs, arg)
|
|
||||||
case len(arg) > len("-setting=") && arg[:len("-setting=")] == "-setting=":
|
|
||||||
flagArgs = append(flagArgs, arg)
|
|
||||||
case hasFlagValue(arg, "-webview-provider"), hasFlagValue(arg, "-webview-network"),
|
|
||||||
hasFlagValue(arg, "-webview-address"), hasFlagValue(arg, "-webview-token"):
|
|
||||||
flagArgs = append(flagArgs, arg)
|
|
||||||
case len(arg) > 0 && arg[0] == '-':
|
|
||||||
return nil, nil, fmt.Errorf("unknown flag %q", arg)
|
|
||||||
default:
|
|
||||||
positionalArgs = append(positionalArgs, arg)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return flagArgs, positionalArgs, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func splitEvalArgs(args []string) ([]string, []string, error) {
|
|
||||||
var flagArgs []string
|
|
||||||
var positionalArgs []string
|
|
||||||
for i := 0; i < len(args); i++ {
|
|
||||||
arg := args[i]
|
|
||||||
switch {
|
|
||||||
case arg == "-pretty":
|
|
||||||
flagArgs = append(flagArgs, arg)
|
|
||||||
case arg == "-webview-provider", arg == "-webview-network", arg == "-webview-address", arg == "-webview-token":
|
|
||||||
if i+1 >= len(args) {
|
|
||||||
return nil, nil, fmt.Errorf("flag %q requires a value", arg)
|
|
||||||
}
|
|
||||||
flagArgs = append(flagArgs, arg, args[i+1])
|
|
||||||
i++
|
|
||||||
case arg == "-e", arg == "--eval":
|
|
||||||
if i+1 >= len(args) {
|
|
||||||
return nil, nil, fmt.Errorf("flag %q requires a value", arg)
|
|
||||||
}
|
|
||||||
flagArgs = append(flagArgs, arg, args[i+1])
|
|
||||||
i++
|
|
||||||
case hasFlagValue(arg, "-e"), hasFlagValue(arg, "--eval"),
|
|
||||||
hasFlagValue(arg, "-webview-provider"), hasFlagValue(arg, "-webview-network"),
|
|
||||||
hasFlagValue(arg, "-webview-address"), hasFlagValue(arg, "-webview-token"):
|
|
||||||
flagArgs = append(flagArgs, arg)
|
|
||||||
case len(arg) > 0 && arg[0] == '-':
|
|
||||||
return nil, nil, fmt.Errorf("unknown flag %q", arg)
|
|
||||||
default:
|
|
||||||
positionalArgs = append(positionalArgs, arg)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return flagArgs, positionalArgs, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func hasFlagValue(arg string, prefix string) bool {
|
|
||||||
return len(arg) > len(prefix)+1 && arg[:len(prefix)+1] == prefix+"="
|
|
||||||
}
|
|
||||||
|
|
||||||
func toAnyMap(values kvFlag) map[string]any {
|
|
||||||
result := make(map[string]any, len(values))
|
|
||||||
for key, value := range values {
|
|
||||||
result[key] = value
|
|
||||||
}
|
|
||||||
return result
|
|
||||||
}
|
|
||||||
|
|
||||||
func newExtMockDownloader(cfg webViewConfig) (*download.Downloader, func(), error) {
|
|
||||||
webViewProvider, err := newExtMockWebViewProvider(cfg)
|
|
||||||
if err != nil {
|
|
||||||
return nil, nil, err
|
|
||||||
}
|
|
||||||
d := download.NewDownloader(&download.DownloaderConfig{
|
|
||||||
Storage: download.NewMemStorage(),
|
|
||||||
WebViewProvider: webViewProvider,
|
|
||||||
})
|
|
||||||
if err := d.Setup(); err != nil {
|
|
||||||
return nil, nil, err
|
|
||||||
}
|
|
||||||
cleanup := func() {
|
|
||||||
_ = d.Clear()
|
|
||||||
}
|
|
||||||
return d, cleanup, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func newExtMockExtension() *download.Extension {
|
|
||||||
return &download.Extension{
|
|
||||||
Name: "extmock",
|
|
||||||
Author: "gopeed",
|
|
||||||
Title: "Gopeed ExtMock Runtime",
|
|
||||||
Version: "0.0.0",
|
|
||||||
DevMode: true,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func newExtMockWebViewProvider(cfg webViewConfig) (enginewebview.Provider, error) {
|
|
||||||
switch cfg.provider {
|
|
||||||
case "", "go":
|
|
||||||
return goprovider.New(), nil
|
|
||||||
case "rpc":
|
|
||||||
rpcCfg, err := resolveRPCConfig(cfg)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return rpcprovider.New(rpcCfg), nil
|
|
||||||
default:
|
|
||||||
return nil, fmt.Errorf("unsupported webview provider %q", cfg.provider)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func resolveRPCConfig(cfg webViewConfig) (enginewebview.RPCConfig, error) {
|
|
||||||
network := cfg.network
|
|
||||||
address := cfg.address
|
|
||||||
if network == "" && address == "" {
|
|
||||||
defaultAddress, err := defaultWebViewRPCUnixSocket()
|
|
||||||
if err != nil {
|
|
||||||
return enginewebview.RPCConfig{}, err
|
|
||||||
}
|
|
||||||
network = "unix"
|
|
||||||
address = defaultAddress
|
|
||||||
}
|
|
||||||
if network == "" || address == "" {
|
|
||||||
return enginewebview.RPCConfig{}, fmt.Errorf("webview rpc requires both network and address")
|
|
||||||
}
|
|
||||||
return enginewebview.RPCConfig{
|
|
||||||
Network: network,
|
|
||||||
Address: address,
|
|
||||||
Token: cfg.token,
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func defaultWebViewRPCUnixSocket() (string, error) {
|
|
||||||
homeDir, err := os.UserHomeDir()
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
return filepath.Join(homeDir, "gopeed_webview.sock"), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
type taskTerminalWaiter struct {
|
|
||||||
downloader *download.Downloader
|
|
||||||
}
|
|
||||||
|
|
||||||
func newTaskTerminalWaiter(d *download.Downloader) *taskTerminalWaiter {
|
|
||||||
return &taskTerminalWaiter{downloader: d}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (w *taskTerminalWaiter) Register(id string) {
|
|
||||||
}
|
|
||||||
|
|
||||||
func (w *taskTerminalWaiter) Wait(id string, timeout time.Duration) (*download.Task, error) {
|
|
||||||
deadline := time.Now().Add(timeout)
|
|
||||||
for time.Now().Before(deadline) {
|
|
||||||
if task, ok := w.currentTerminalTask(id); ok {
|
|
||||||
return task, w.taskError(task)
|
|
||||||
}
|
|
||||||
time.Sleep(50 * time.Millisecond)
|
|
||||||
}
|
|
||||||
return nil, fmt.Errorf("timeout waiting for task %s", id)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (w *taskTerminalWaiter) currentTerminalTask(id string) (*download.Task, bool) {
|
|
||||||
task := w.downloader.GetTask(id)
|
|
||||||
if task == nil {
|
|
||||||
return nil, false
|
|
||||||
}
|
|
||||||
if task.Status != base.DownloadStatusDone && task.Status != base.DownloadStatusError {
|
|
||||||
return nil, false
|
|
||||||
}
|
|
||||||
return task, true
|
|
||||||
}
|
|
||||||
|
|
||||||
func (w *taskTerminalWaiter) taskError(task *download.Task) error {
|
|
||||||
if task == nil {
|
|
||||||
return fmt.Errorf("task is nil")
|
|
||||||
}
|
|
||||||
if task.Status == base.DownloadStatusError {
|
|
||||||
return fmt.Errorf("task %s finished with status %s", task.ID, task.Status)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func startVerboseTaskListener(d *download.Downloader, files []*downloadedFileResult) func() {
|
|
||||||
tracked := make(map[string]string, len(files))
|
|
||||||
for _, file := range files {
|
|
||||||
tracked[file.TaskID] = file.Name
|
|
||||||
}
|
|
||||||
d.Listener(func(event *download.Event) {
|
|
||||||
if event == nil || event.Task == nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
name, ok := tracked[event.Task.ID]
|
|
||||||
if !ok {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
switch event.Key {
|
|
||||||
case download.EventKeyStart, download.EventKeyProgress, download.EventKeyDone, download.EventKeyError:
|
|
||||||
printVerboseTaskEvent(event, name)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
return func() {}
|
|
||||||
}
|
|
||||||
|
|
||||||
func printVerboseTaskEvent(event *download.Event, name string) {
|
|
||||||
task := event.Task
|
|
||||||
ts := time.Now().Format("15:04:05.000")
|
|
||||||
if task == nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
total := int64(0)
|
|
||||||
if task.Meta != nil && task.Meta.Res != nil {
|
|
||||||
total = task.Meta.Res.Size
|
|
||||||
}
|
|
||||||
downloaded := int64(0)
|
|
||||||
speed := int64(0)
|
|
||||||
if task.Progress != nil {
|
|
||||||
downloaded = task.Progress.Downloaded
|
|
||||||
speed = task.Progress.Speed
|
|
||||||
}
|
|
||||||
if event.Key == download.EventKeyError && event.Err != nil {
|
|
||||||
fmt.Fprintf(
|
|
||||||
os.Stderr,
|
|
||||||
"[%s] event=%s task=%s status=%s downloaded=%d total=%d speed=%d name=%q err=%v\n",
|
|
||||||
ts,
|
|
||||||
event.Key,
|
|
||||||
task.ID,
|
|
||||||
task.Status,
|
|
||||||
downloaded,
|
|
||||||
total,
|
|
||||||
speed,
|
|
||||||
name,
|
|
||||||
event.Err,
|
|
||||||
)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
fmt.Fprintf(
|
|
||||||
os.Stderr,
|
|
||||||
"[%s] event=%s task=%s status=%s downloaded=%d total=%d speed=%d name=%q\n",
|
|
||||||
ts,
|
|
||||||
event.Key,
|
|
||||||
task.ID,
|
|
||||||
task.Status,
|
|
||||||
downloaded,
|
|
||||||
total,
|
|
||||||
speed,
|
|
||||||
name,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
func printJSON(v any, pretty bool) error {
|
|
||||||
var (
|
|
||||||
data []byte
|
|
||||||
err error
|
|
||||||
)
|
|
||||||
if pretty {
|
|
||||||
data, err = json.MarshalIndent(v, "", " ")
|
|
||||||
} else {
|
|
||||||
data, err = json.Marshal(v)
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
fmt.Println(string(data))
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
@@ -7,11 +7,9 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
type args struct {
|
type args struct {
|
||||||
url string
|
url string
|
||||||
connections *int
|
connections *int
|
||||||
dir *string
|
dir *string
|
||||||
checksum *string
|
|
||||||
checksumAlgo *string
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func parse() *args {
|
func parse() *args {
|
||||||
@@ -22,8 +20,6 @@ func parse() *args {
|
|||||||
var args args
|
var args args
|
||||||
args.connections = flag.Int("C", 16, "Concurrent connections.")
|
args.connections = flag.Int("C", 16, "Concurrent connections.")
|
||||||
args.dir = flag.String("D", dir, "Store directory.")
|
args.dir = flag.String("D", dir, "Store directory.")
|
||||||
args.checksum = flag.String("checksum", "", "Expected checksum hash (hex encoded).")
|
|
||||||
args.checksumAlgo = flag.String("checksum-algo", "md5", "Checksum algorithm: md5, sha1, or sha256.")
|
|
||||||
flag.Parse()
|
flag.Parse()
|
||||||
t := flag.Args()
|
t := flag.Args()
|
||||||
if len(t) > 0 {
|
if len(t) > 0 {
|
||||||
|
|||||||
@@ -2,14 +2,13 @@ package main
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"strings"
|
|
||||||
"sync"
|
|
||||||
|
|
||||||
"github.com/GopeedLab/gopeed/internal/fetcher"
|
"github.com/GopeedLab/gopeed/internal/fetcher"
|
||||||
"github.com/GopeedLab/gopeed/pkg/base"
|
"github.com/GopeedLab/gopeed/pkg/base"
|
||||||
"github.com/GopeedLab/gopeed/pkg/download"
|
"github.com/GopeedLab/gopeed/pkg/download"
|
||||||
"github.com/GopeedLab/gopeed/pkg/protocol/http"
|
"github.com/GopeedLab/gopeed/pkg/protocol/http"
|
||||||
"github.com/GopeedLab/gopeed/pkg/util"
|
"github.com/GopeedLab/gopeed/pkg/util"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
)
|
)
|
||||||
|
|
||||||
const progressWidth = 20
|
const progressWidth = 20
|
||||||
@@ -19,18 +18,6 @@ func main() {
|
|||||||
|
|
||||||
var wg sync.WaitGroup
|
var wg sync.WaitGroup
|
||||||
wg.Add(1)
|
wg.Add(1)
|
||||||
var checksumOpt *http.ChecksumOption
|
|
||||||
if args.checksum != nil && *args.checksum != "" {
|
|
||||||
algo := "md5"
|
|
||||||
if args.checksumAlgo != nil && *args.checksumAlgo != "" {
|
|
||||||
algo = *args.checksumAlgo
|
|
||||||
}
|
|
||||||
checksumOpt = &http.ChecksumOption{
|
|
||||||
Algorithm: algo,
|
|
||||||
Expected: *args.checksum,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err := download.Boot().
|
_, err := download.Boot().
|
||||||
URL(args.url).
|
URL(args.url).
|
||||||
Listener(func(event *download.Event) {
|
Listener(func(event *download.Event) {
|
||||||
@@ -55,11 +42,8 @@ func main() {
|
|||||||
}
|
}
|
||||||
}).
|
}).
|
||||||
Create(&base.Options{
|
Create(&base.Options{
|
||||||
Path: *args.dir,
|
Path: *args.dir,
|
||||||
Extra: http.OptsExtra{
|
Extra: http.OptsExtra{Connections: *args.connections},
|
||||||
Connections: *args.connections,
|
|
||||||
Checksum: checksumOpt,
|
|
||||||
},
|
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
panic(err)
|
panic(err)
|
||||||
|
|||||||
@@ -1,19 +0,0 @@
|
|||||||
//go:build !windows
|
|
||||||
// +build !windows
|
|
||||||
|
|
||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"net"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
)
|
|
||||||
|
|
||||||
func Dial() (net.Conn, error) {
|
|
||||||
// Get binary path
|
|
||||||
exe, err := os.Executable()
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return net.Dial("unix", filepath.Join(filepath.Dir(exe), "gopeed_host.sock"))
|
|
||||||
}
|
|
||||||
@@ -1,11 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"net"
|
|
||||||
|
|
||||||
"github.com/Microsoft/go-winio"
|
|
||||||
)
|
|
||||||
|
|
||||||
func Dial() (net.Conn, error) {
|
|
||||||
return winio.DialPipe(`\\.\pipe\gopeed_host`, nil)
|
|
||||||
}
|
|
||||||
@@ -1,208 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"context"
|
|
||||||
"encoding/binary"
|
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
"net"
|
|
||||||
"net/http"
|
|
||||||
"os"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/pkg/browser"
|
|
||||||
)
|
|
||||||
|
|
||||||
type Message struct {
|
|
||||||
Method string `json:"method"`
|
|
||||||
Meta map[string]any `json:"meta"`
|
|
||||||
Params json.RawMessage `json:"params"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type Response struct {
|
|
||||||
Code int `json:"code"`
|
|
||||||
Data any `json:"data,omitempty"`
|
|
||||||
Message string `json:"message,omitempty"`
|
|
||||||
}
|
|
||||||
|
|
||||||
func check() (data bool, err error) {
|
|
||||||
conn, err := Dial()
|
|
||||||
if err != nil {
|
|
||||||
return false, err
|
|
||||||
}
|
|
||||||
defer conn.Close()
|
|
||||||
return true, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func wakeup(hidden bool) error {
|
|
||||||
running, _ := check()
|
|
||||||
if running {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
uri := "gopeed:"
|
|
||||||
if hidden {
|
|
||||||
uri = uri + "?hidden=true"
|
|
||||||
}
|
|
||||||
if err := browser.OpenURL(uri); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
for i := 0; i < 10; i++ {
|
|
||||||
if ok, _ := check(); ok {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
time.Sleep(1 * time.Second)
|
|
||||||
}
|
|
||||||
return fmt.Errorf("start gopeed failed")
|
|
||||||
}
|
|
||||||
|
|
||||||
// postToFlutter sends a POST request to Flutter RPC server
|
|
||||||
func postToFlutter(path string, body []byte, headers map[string]string, timeout time.Duration) (*http.Response, error) {
|
|
||||||
client := &http.Client{
|
|
||||||
Transport: &http.Transport{
|
|
||||||
DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
|
|
||||||
return Dial()
|
|
||||||
},
|
|
||||||
},
|
|
||||||
Timeout: timeout,
|
|
||||||
}
|
|
||||||
req, err := http.NewRequest("POST", "http://127.0.0.1"+path, bytes.NewBuffer(body))
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
req.Header.Set("Content-Type", "application/json")
|
|
||||||
for k, v := range headers {
|
|
||||||
req.Header.Set(k, v)
|
|
||||||
}
|
|
||||||
return client.Do(req)
|
|
||||||
}
|
|
||||||
|
|
||||||
var apiMap = map[string]func(message *Message) (data any, err error){
|
|
||||||
"ping": func(message *Message) (data any, err error) {
|
|
||||||
return check()
|
|
||||||
},
|
|
||||||
"wakeup": func(message *Message) (data any, err error) {
|
|
||||||
silent := false
|
|
||||||
if v, ok := message.Meta["silent"]; ok {
|
|
||||||
silent, _ = v.(bool)
|
|
||||||
}
|
|
||||||
err = wakeup(silent)
|
|
||||||
return
|
|
||||||
},
|
|
||||||
"create": func(message *Message) (data any, err error) {
|
|
||||||
buf, err := message.Params.MarshalJSON()
|
|
||||||
if err != nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// Task creation owns its presentation: silent requests stay in the
|
|
||||||
// background, while non-silent requests open a dedicated create window.
|
|
||||||
// Always bootstrap the main process hidden so it never flashes first.
|
|
||||||
if err := wakeup(true); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
headers := make(map[string]string)
|
|
||||||
if message.Meta != nil {
|
|
||||||
metaJson, _ := json.Marshal(message.Meta)
|
|
||||||
headers["X-Gopeed-Host-Meta"] = string(metaJson)
|
|
||||||
}
|
|
||||||
_, err = postToFlutter("/create", buf, headers, 10*time.Second)
|
|
||||||
return
|
|
||||||
},
|
|
||||||
"forward": func(message *Message) (data any, err error) {
|
|
||||||
buf, err := message.Params.MarshalJSON()
|
|
||||||
if err != nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
resp, err := postToFlutter("/forward", buf, nil, 60*time.Second)
|
|
||||||
if err != nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
defer resp.Body.Close()
|
|
||||||
|
|
||||||
respBody, err := io.ReadAll(resp.Body)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
var respData map[string]json.RawMessage
|
|
||||||
if err := json.Unmarshal(respBody, &respData); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return respData, nil
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
// go build -ldflags="-s -w" -o ui/flutter/assets/exec/ github.com/GopeedLab/gopeed/cmd/host
|
|
||||||
|
|
||||||
func main() {
|
|
||||||
for {
|
|
||||||
// Read message length (first 4 bytes)
|
|
||||||
var length uint32
|
|
||||||
if err := binary.Read(os.Stdin, binary.NativeEndian, &length); err != nil {
|
|
||||||
if err == io.EOF {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
sendError("Failed to read message length: " + err.Error())
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// Read the message
|
|
||||||
input := make([]byte, length)
|
|
||||||
if _, err := io.ReadFull(os.Stdin, input); err != nil {
|
|
||||||
sendError("Failed to read message: " + err.Error())
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// Parse message
|
|
||||||
var message Message
|
|
||||||
if err := json.Unmarshal(input, &message); err != nil {
|
|
||||||
sendError("Failed to parse message: " + err.Error())
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// Handle request
|
|
||||||
var data any
|
|
||||||
var err error
|
|
||||||
if handler, ok := apiMap[message.Method]; ok {
|
|
||||||
data, err = handler(&message)
|
|
||||||
} else {
|
|
||||||
err = errors.New("Unknown method: " + message.Method)
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
sendError(err.Error())
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
sendResponse(0, data, "")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func sendResponse(code int, data interface{}, message string) {
|
|
||||||
response := Response{
|
|
||||||
Code: code,
|
|
||||||
Data: data,
|
|
||||||
Message: message,
|
|
||||||
}
|
|
||||||
|
|
||||||
// Encode response
|
|
||||||
responseBytes, err := json.Marshal(response)
|
|
||||||
if err != nil {
|
|
||||||
sendError("Failed to encode response: " + err.Error())
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// Write message length
|
|
||||||
binary.Write(os.Stdout, binary.NativeEndian, uint32(len(responseBytes)))
|
|
||||||
// Write message
|
|
||||||
os.Stdout.Write(responseBytes)
|
|
||||||
}
|
|
||||||
|
|
||||||
func sendError(msg string) {
|
|
||||||
sendResponse(1, nil, msg)
|
|
||||||
}
|
|
||||||
@@ -8,73 +8,40 @@ import (
|
|||||||
"github.com/GopeedLab/gopeed/pkg/rest/model"
|
"github.com/GopeedLab/gopeed/pkg/rest/model"
|
||||||
"net/http"
|
"net/http"
|
||||||
"os"
|
"os"
|
||||||
"os/signal"
|
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"syscall"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
//go:embed banner.txt
|
//go:embed banner.txt
|
||||||
var banner string
|
var banner string
|
||||||
|
|
||||||
func Start(cfg *model.StartConfig) error {
|
func Start(cfg *model.StartConfig) {
|
||||||
|
fmt.Println(banner)
|
||||||
srv, listener, err := rest.BuildServer(cfg)
|
srv, listener, err := rest.BuildServer(cfg)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
panic(err)
|
||||||
}
|
}
|
||||||
fmt.Println(banner)
|
|
||||||
downloadCfg, err := rest.Downloader.GetConfig()
|
downloadCfg, err := rest.Downloader.GetConfig()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
listener.Close()
|
panic(err)
|
||||||
return err
|
|
||||||
}
|
}
|
||||||
if rest.Downloader.IsFirstLoad() {
|
if downloadCfg.FirstLoad {
|
||||||
// Set default download config
|
|
||||||
if cfg.DownloadConfig != nil {
|
|
||||||
cfg.DownloadConfig.Merge(downloadCfg)
|
|
||||||
// TODO Use PatchConfig
|
|
||||||
rest.Downloader.PutConfig(cfg.DownloadConfig)
|
|
||||||
downloadCfg = cfg.DownloadConfig
|
|
||||||
}
|
|
||||||
|
|
||||||
downloadDir := downloadCfg.DownloadDir
|
|
||||||
// Set default download dir, in docker, it will be ${exe}/Downloads, else it will be ${user}/Downloads
|
// Set default download dir, in docker, it will be ${exe}/Downloads, else it will be ${user}/Downloads
|
||||||
if downloadDir == "" {
|
var downloadDir string
|
||||||
if base.InDocker == "true" {
|
if base.InDocker == "true" {
|
||||||
downloadDir = filepath.Join(filepath.Dir(cfg.StorageDir), "Downloads")
|
downloadDir = filepath.Join(filepath.Dir(cfg.StorageDir), "Downloads")
|
||||||
} else {
|
} else {
|
||||||
userDir, err := os.UserHomeDir()
|
userDir, err := os.UserHomeDir()
|
||||||
if err == nil {
|
if err == nil {
|
||||||
downloadDir = filepath.Join(userDir, "Downloads")
|
downloadDir = filepath.Join(userDir, "Downloads")
|
||||||
}
|
|
||||||
}
|
|
||||||
if downloadDir != "" {
|
|
||||||
downloadCfg.DownloadDir = downloadDir
|
|
||||||
rest.Downloader.PutConfig(downloadCfg)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if downloadDir != "" {
|
||||||
// The default download dir is resolved only after the core starts, so
|
downloadCfg.DownloadDir = downloadDir
|
||||||
// seed the built-in categories now that the dir is known. Init is
|
rest.Downloader.PutConfig(downloadCfg)
|
||||||
// idempotent and leaves an existing category list untouched.
|
}
|
||||||
downloadCfg.Init()
|
|
||||||
rest.Downloader.PutConfig(downloadCfg)
|
|
||||||
}
|
}
|
||||||
watchExit()
|
|
||||||
|
|
||||||
fmt.Printf("Server start success on http://%s\n", listener.Addr().String())
|
fmt.Printf("Server start success on http://%s\n", listener.Addr().String())
|
||||||
if err := srv.Serve(listener); err != nil && err != http.ErrServerClosed {
|
if err := srv.Serve(listener); err != nil && err != http.ErrServerClosed {
|
||||||
return err
|
panic(err)
|
||||||
}
|
}
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func watchExit() {
|
|
||||||
sigs := make(chan os.Signal, 1)
|
|
||||||
signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
|
|
||||||
go func() {
|
|
||||||
sig := <-sigs
|
|
||||||
fmt.Printf("Server is shutting down due to signal: %s\n", sig)
|
|
||||||
rest.Downloader.Close()
|
|
||||||
os.Exit(0)
|
|
||||||
}()
|
|
||||||
}
|
}
|
||||||
@@ -1,35 +0,0 @@
|
|||||||
package cmd
|
|
||||||
|
|
||||||
import (
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/GopeedLab/gopeed/pkg/rest/model"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestStartInvalidWebAPITokenReturnsError(t *testing.T) {
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
webAuth *model.WebAuth
|
|
||||||
}{
|
|
||||||
{name: "no credentials"},
|
|
||||||
{name: "missing username", webAuth: &model.WebAuth{Password: "secret"}},
|
|
||||||
{name: "missing password", webAuth: &model.WebAuth{Username: "gopeed"}},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tt := range tests {
|
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
|
||||||
err := Start(&model.StartConfig{
|
|
||||||
WebEnable: true,
|
|
||||||
ApiToken: "test-token",
|
|
||||||
WebAuth: tt.webAuth,
|
|
||||||
})
|
|
||||||
if err == nil {
|
|
||||||
t.Fatal("Start() error = nil, want web authentication validation error")
|
|
||||||
}
|
|
||||||
if !strings.Contains(err.Error(), "web authentication username and password are required") {
|
|
||||||
t.Fatalf("Start() error = %q, want missing web authentication credentials", err)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,115 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"flag"
|
|
||||||
"fmt"
|
|
||||||
"log"
|
|
||||||
"os"
|
|
||||||
"syscall"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/pkg/browser"
|
|
||||||
"github.com/pkg/errors"
|
|
||||||
)
|
|
||||||
|
|
||||||
// go build -ldflags="-s -w" -o ui/flutter/assets/exec/ github.com/GopeedLab/gopeed/cmd/updater
|
|
||||||
|
|
||||||
func main() {
|
|
||||||
pid := flag.Int("pid", 0, "PID of the process to update")
|
|
||||||
updateChannel := flag.String("channel", "", "Update channel")
|
|
||||||
packagePath := flag.String("asset", "", "Path to the package asset")
|
|
||||||
exeDir := flag.String("exeDir", "", "Directory of the entry executable")
|
|
||||||
logPath := flag.String("log", "", "Log file path")
|
|
||||||
flag.Parse()
|
|
||||||
|
|
||||||
if *pid == 0 {
|
|
||||||
log.Println("Invalid PID")
|
|
||||||
os.Exit(1)
|
|
||||||
}
|
|
||||||
if *updateChannel == "" {
|
|
||||||
log.Println("Invalid update channel")
|
|
||||||
os.Exit(1)
|
|
||||||
}
|
|
||||||
|
|
||||||
if *logPath != "" {
|
|
||||||
logFile, err := os.OpenFile(*logPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0666)
|
|
||||||
if err == nil {
|
|
||||||
defer logFile.Close()
|
|
||||||
log.SetOutput(logFile)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
var (
|
|
||||||
restart bool
|
|
||||||
err error
|
|
||||||
)
|
|
||||||
if restart, err = update(*pid, *updateChannel, *packagePath, *exeDir); err != nil {
|
|
||||||
log.Printf("Update failed: %v\n", err)
|
|
||||||
os.Exit(1)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Restart the application
|
|
||||||
if restart {
|
|
||||||
browser.OpenURL("gopeed:///")
|
|
||||||
}
|
|
||||||
|
|
||||||
// Delete package asset
|
|
||||||
if *packagePath != "" {
|
|
||||||
os.Remove(*packagePath)
|
|
||||||
}
|
|
||||||
|
|
||||||
os.Exit(0)
|
|
||||||
}
|
|
||||||
|
|
||||||
func update(pid int, updateChannel, packagePath, exeDir string) (restart bool, err error) {
|
|
||||||
killSignalChan := make(chan any, 1)
|
|
||||||
|
|
||||||
go func() {
|
|
||||||
<-killSignalChan
|
|
||||||
|
|
||||||
if err = killProcess(pid); err != nil {
|
|
||||||
log.Printf("Failed to kill process: %v\n", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if err = waitForProcessExit(pid); err != nil {
|
|
||||||
log.Printf("Failed to wait for process exit: %v\n", err)
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
|
|
||||||
log.Printf("Updating process updateChannel=%s packagePath=%s exeDir=%s\n", updateChannel, packagePath, exeDir)
|
|
||||||
if restart, err = install(killSignalChan, updateChannel, packagePath, exeDir); err != nil {
|
|
||||||
return false, errors.Wrap(err, "failed to install package")
|
|
||||||
}
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
func waitForProcessExit(pid int) error {
|
|
||||||
deadline := time.Now().Add(10 * time.Second)
|
|
||||||
for time.Now().Before(deadline) {
|
|
||||||
process, err := os.FindProcess(pid)
|
|
||||||
if err != nil {
|
|
||||||
// On some systems, error is returned if process doesn't exist
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Send null signal to test if process exists
|
|
||||||
err = process.Signal(syscall.Signal(0))
|
|
||||||
if err != nil {
|
|
||||||
// If error occurs, the process no longer exists
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
time.Sleep(100 * time.Millisecond)
|
|
||||||
}
|
|
||||||
return fmt.Errorf("process %d still running after timeout", pid)
|
|
||||||
}
|
|
||||||
|
|
||||||
func killProcess(pid int) error {
|
|
||||||
process, err := os.FindProcess(pid)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
return process.Kill()
|
|
||||||
}
|
|
||||||
@@ -1,72 +0,0 @@
|
|||||||
//go:build darwin
|
|
||||||
|
|
||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
"os/exec"
|
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
|
||||||
)
|
|
||||||
|
|
||||||
func install(killSignalChan chan<- any, updateChannel, packagePath, destDir string) (bool, error) {
|
|
||||||
return true, installByDmg(killSignalChan, packagePath, destDir)
|
|
||||||
}
|
|
||||||
|
|
||||||
// installByDmg handles macOS dmg package installation
|
|
||||||
func installByDmg(killSignalChan chan<- any, packagePath, destDir string) error {
|
|
||||||
// /Applications/Gopeed.app/Contents/MacOS -> /Applications
|
|
||||||
appPath := getParentDir(getParentDir(getParentDir(destDir)))
|
|
||||||
output, err := exec.Command("hdiutil", "attach", packagePath, "-nobrowse").Output()
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
mountPoint := ""
|
|
||||||
for _, line := range strings.Split(string(output), "\n") {
|
|
||||||
if strings.Contains(line, "/Volumes/") {
|
|
||||||
// Find the /Volumes/ path in the line
|
|
||||||
// hdiutil output format: /dev/disk4s1 Apple_HFS /Volumes/Gopeed
|
|
||||||
// or with sequence number: /dev/disk4s1 Apple_HFS /Volumes/Gopeed 1
|
|
||||||
idx := strings.Index(line, "/Volumes/")
|
|
||||||
if idx != -1 {
|
|
||||||
// Extract everything from /Volumes/ onwards and trim whitespace
|
|
||||||
mountPoint = strings.TrimSpace(line[idx:])
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if mountPoint == "" {
|
|
||||||
return fmt.Errorf("failed to get mount point from hdiutil output: %s", string(output))
|
|
||||||
}
|
|
||||||
|
|
||||||
// Detach the mounted DMG
|
|
||||||
defer exec.Command("hdiutil", "detach", mountPoint, "-quiet").Run()
|
|
||||||
|
|
||||||
matches, err := filepath.Glob(filepath.Join(mountPoint, "*.app"))
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if len(matches) == 0 {
|
|
||||||
return fmt.Errorf("no .app found in dmg, mountPoint: %s", mountPoint)
|
|
||||||
}
|
|
||||||
|
|
||||||
killSignalChan <- nil
|
|
||||||
|
|
||||||
// Copy the new app to the destination
|
|
||||||
// cp -Rf /Volumes/GoPeed/GoPeed.app /Applications
|
|
||||||
if err := exec.Command("cp", "-Rf", matches[0], appPath).Run(); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Get parent directory safely, handling trailing separators
|
|
||||||
func getParentDir(path string) string {
|
|
||||||
// Remove trailing separators if they exist
|
|
||||||
path = strings.TrimRight(path, string(filepath.Separator))
|
|
||||||
// Now get the parent directory
|
|
||||||
return filepath.Dir(path)
|
|
||||||
}
|
|
||||||
@@ -1,72 +0,0 @@
|
|||||||
//go:build linux
|
|
||||||
|
|
||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
"os/exec"
|
|
||||||
)
|
|
||||||
|
|
||||||
func install(killSignalChan chan<- any, updateChannel, packagePath, destDir string) (bool, error) {
|
|
||||||
killSignalChan <- nil
|
|
||||||
switch updateChannel {
|
|
||||||
case "linuxDeb":
|
|
||||||
return true, installByDeb(packagePath)
|
|
||||||
case "linuxFlathub":
|
|
||||||
return true, installByFlathub()
|
|
||||||
case "linuxSnap":
|
|
||||||
return true, installBySnap()
|
|
||||||
default:
|
|
||||||
return false, fmt.Errorf("unsupported update channel for Linux: %s", updateChannel)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// executeInTerminal tries to execute a command in one of several common terminal emulators
|
|
||||||
func executeInTerminal(command string) error {
|
|
||||||
terminals := []string{
|
|
||||||
"gnome-terminal", // GNOME
|
|
||||||
"konsole", // KDE
|
|
||||||
"xfce4-terminal", // XFCE
|
|
||||||
"xterm", // X11
|
|
||||||
}
|
|
||||||
|
|
||||||
command = fmt.Sprintf(`echo "Starting update..." && echo "[CMD] %s" && %s`, command, command)
|
|
||||||
|
|
||||||
for _, term := range terminals {
|
|
||||||
if _, err := exec.LookPath(term); err == nil {
|
|
||||||
var cmd *exec.Cmd
|
|
||||||
switch term {
|
|
||||||
case "gnome-terminal", "xfce4-terminal":
|
|
||||||
cmd = exec.Command(term, "--", "bash", "-c", command)
|
|
||||||
case "konsole":
|
|
||||||
cmd = exec.Command(term, "-e", "bash", "-c", command)
|
|
||||||
case "xterm":
|
|
||||||
cmd = exec.Command(term, "-e", command)
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := cmd.Start(); err == nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return fmt.Errorf("no suitable terminal emulator found. Please install gnome-terminal, konsole, xfce4-terminal or xterm")
|
|
||||||
}
|
|
||||||
|
|
||||||
// installByDeb installs the .deb package
|
|
||||||
func installByDeb(packagePath string) error {
|
|
||||||
command := fmt.Sprintf(`sudo dpkg -i "%s"`, packagePath)
|
|
||||||
return executeInTerminal(command)
|
|
||||||
}
|
|
||||||
|
|
||||||
// installByFlathub updates the application via Flathub
|
|
||||||
func installByFlathub() error {
|
|
||||||
command := "flatpak update com.gopeed.Gopeed -y"
|
|
||||||
return executeInTerminal(command)
|
|
||||||
}
|
|
||||||
|
|
||||||
// installBySnap updates the application via Snap
|
|
||||||
func installBySnap() error {
|
|
||||||
command := "sudo snap refresh gopeed"
|
|
||||||
return executeInTerminal(command)
|
|
||||||
}
|
|
||||||
@@ -1,140 +0,0 @@
|
|||||||
//go:build windows
|
|
||||||
|
|
||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"archive/zip"
|
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
"os"
|
|
||||||
"os/exec"
|
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
|
||||||
)
|
|
||||||
|
|
||||||
func install(killSignalChan chan<- any, updateChannel, packagePath, destDir string) (bool, error) {
|
|
||||||
switch updateChannel {
|
|
||||||
case "windowsInstaller":
|
|
||||||
return false, installByInstaller(killSignalChan, packagePath, destDir)
|
|
||||||
default:
|
|
||||||
return true, installByPortable(killSignalChan, packagePath, destDir)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// installByInstaller extracts the installer from the zip file and runs it
|
|
||||||
func installByInstaller(killSignalChan chan<- any, packagePath, destDir string) error {
|
|
||||||
// Create a temp directory for extraction
|
|
||||||
tempDir, err := os.MkdirTemp("", "gopeed_update")
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
defer os.RemoveAll(tempDir)
|
|
||||||
|
|
||||||
// Extract the zip file
|
|
||||||
reader, err := zip.OpenReader(packagePath)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
defer reader.Close()
|
|
||||||
|
|
||||||
var installerPath string
|
|
||||||
for _, file := range reader.File {
|
|
||||||
cleanName := filepath.Clean(file.Name)
|
|
||||||
if strings.HasPrefix(cleanName, "..") || filepath.IsAbs(cleanName) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if file.FileInfo().IsDir() {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
path := filepath.Join(tempDir, cleanName)
|
|
||||||
|
|
||||||
if err := os.MkdirAll(filepath.Dir(path), os.ModePerm); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
dstFile, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, file.Mode())
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
srcFile, err := file.Open()
|
|
||||||
if err != nil {
|
|
||||||
dstFile.Close()
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err = io.Copy(dstFile, srcFile)
|
|
||||||
srcFile.Close()
|
|
||||||
dstFile.Close()
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
// If this is likely an installer (.exe, .msi), save its path
|
|
||||||
ext := strings.ToLower(filepath.Ext(file.Name))
|
|
||||||
if ext == ".exe" || ext == ".msi" {
|
|
||||||
installerPath = path
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if installerPath == "" {
|
|
||||||
return fmt.Errorf("no installer found in the update package")
|
|
||||||
}
|
|
||||||
|
|
||||||
// Run the installer
|
|
||||||
cmd := exec.Command(installerPath)
|
|
||||||
if err := cmd.Start(); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
killSignalChan <- nil
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// installByPortable extracts the portable version to the destination directory
|
|
||||||
func installByPortable(killSignalChan chan<- any, packagePath, destDir string) error {
|
|
||||||
killSignalChan <- nil
|
|
||||||
|
|
||||||
reader, err := zip.OpenReader(packagePath)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
defer reader.Close()
|
|
||||||
|
|
||||||
for _, file := range reader.File {
|
|
||||||
cleanName := filepath.Clean(file.Name)
|
|
||||||
if strings.HasPrefix(cleanName, "..") || filepath.IsAbs(cleanName) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
path := filepath.Join(destDir, cleanName)
|
|
||||||
|
|
||||||
if file.FileInfo().IsDir() {
|
|
||||||
os.MkdirAll(path, file.Mode())
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := os.MkdirAll(filepath.Dir(path), os.ModePerm); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
dstFile, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, file.Mode())
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
srcFile, err := file.Open()
|
|
||||||
if err != nil {
|
|
||||||
dstFile.Close()
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err = io.Copy(dstFile, srcFile)
|
|
||||||
srcFile.Close()
|
|
||||||
dstFile.Close()
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
@@ -3,219 +3,78 @@ package main
|
|||||||
import (
|
import (
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"flag"
|
"flag"
|
||||||
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"reflect"
|
|
||||||
"strconv"
|
|
||||||
"strings"
|
|
||||||
|
|
||||||
"github.com/GopeedLab/gopeed/pkg/base"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
type args struct {
|
type args struct {
|
||||||
TempDir *string `json:"tempDir"`
|
Address *string `json:"address"`
|
||||||
Address *string `json:"address"`
|
Port *int `json:"port"`
|
||||||
Port *int `json:"port"`
|
Username *string `json:"username"`
|
||||||
Username *string `json:"username"`
|
Password *string `json:"password"`
|
||||||
Password *string `json:"password"`
|
ApiToken *string `json:"apiToken"`
|
||||||
ApiToken *string `json:"apiToken"`
|
StorageDir *string `json:"storageDir"`
|
||||||
MCPEnable bool `json:"mcpEnable"`
|
|
||||||
StorageDir *string `json:"storageDir"`
|
|
||||||
WhiteDownloadDirs []string `json:"whiteDownloadDirs"`
|
|
||||||
// DownloadConfig when the first time to start the server, it will be configured as initial value
|
|
||||||
DownloadConfig *base.DownloaderStoreConfig `json:"downloadConfig"`
|
|
||||||
|
|
||||||
configPath *string
|
configPath *string
|
||||||
}
|
}
|
||||||
|
|
||||||
func parse() *args {
|
func parse() *args {
|
||||||
cfg := &args{}
|
var cliArgs args
|
||||||
|
cliArgs.Address = flag.String("A", "0.0.0.0", "Bind Address")
|
||||||
cliConfig := loadCliArgs()
|
cliArgs.Port = flag.Int("P", 9999, "Bind Port")
|
||||||
loadConfigFile(cfg, *cliConfig.configPath)
|
cliArgs.Username = flag.String("u", "gopeed", "HTTP Basic Auth Username")
|
||||||
loadEnvVars(cfg)
|
cliArgs.Password = flag.String("p", "", "HTTP Basic Auth Pwd")
|
||||||
// override with non-default command line arguments
|
cliArgs.ApiToken = flag.String("T", "", "API token, that can only be used when basic authentication is enabled.")
|
||||||
overrideWithCliArgs(cfg, cliConfig)
|
cliArgs.StorageDir = flag.String("d", "", "Storage directory")
|
||||||
// set default values
|
cliArgs.configPath = flag.String("c", "./config.json", "Config file path")
|
||||||
setDefaults(cfg, cliConfig)
|
|
||||||
return cfg
|
|
||||||
}
|
|
||||||
|
|
||||||
// loadCliArgs parses command line arguments and returns initial config
|
|
||||||
func loadCliArgs() *args {
|
|
||||||
cfg := &args{}
|
|
||||||
cfg.Address = flag.String("A", "0.0.0.0", "Bind Address")
|
|
||||||
flag.StringVar(cfg.Address, "address", "0.0.0.0", "Bind Address")
|
|
||||||
cfg.Port = flag.Int("P", 9999, "Bind Port")
|
|
||||||
flag.IntVar(cfg.Port, "port", 9999, "Bind Port")
|
|
||||||
cfg.Username = flag.String("u", "gopeed", "Web Authentication Username")
|
|
||||||
flag.StringVar(cfg.Username, "username", "gopeed", "Web Authentication Username")
|
|
||||||
cfg.Password = flag.String("p", "", "Web Authentication Password, if no password is set, web authentication will not be enabled")
|
|
||||||
flag.StringVar(cfg.Password, "password", "", "Web Authentication Password, if no password is set, web authentication will not be enabled")
|
|
||||||
cfg.ApiToken = flag.String("T", "", "API token (requires web authentication with username and password)")
|
|
||||||
flag.StringVar(cfg.ApiToken, "api-token", "", "API token (requires web authentication with username and password)")
|
|
||||||
flag.BoolVar(&cfg.MCPEnable, "mcp-enable", false, "Enable MCP endpoint")
|
|
||||||
cfg.TempDir = flag.String("temp-dir", "", "Temporary directory (defaults to the OS temporary directory)")
|
|
||||||
cfg.StorageDir = flag.String("d", "", "Storage directory")
|
|
||||||
flag.StringVar(cfg.StorageDir, "storage-dir", "", "Storage directory")
|
|
||||||
whiteDownloadDirs := flag.String("white-download-dirs", "", "White download directories, comma-separated")
|
|
||||||
cfg.configPath = flag.String("c", "./config.json", "Config file path")
|
|
||||||
flag.StringVar(cfg.configPath, "config", "./config.json", "Config file path")
|
|
||||||
flag.Parse()
|
flag.Parse()
|
||||||
|
|
||||||
// Parse white download directories from comma-separated string
|
// args priority: config file > cli args
|
||||||
if whiteDownloadDirs != nil && *whiteDownloadDirs != "" {
|
cfgArgs := loadConfig(*cliArgs.configPath)
|
||||||
dirs := strings.Split(*whiteDownloadDirs, ",")
|
if cfgArgs.Address == nil {
|
||||||
for i := range dirs {
|
cfgArgs.Address = cliArgs.Address
|
||||||
dirs[i] = strings.TrimSpace(dirs[i])
|
|
||||||
}
|
|
||||||
cfg.WhiteDownloadDirs = dirs
|
|
||||||
}
|
}
|
||||||
|
if cfgArgs.Port == nil {
|
||||||
return cfg
|
cfgArgs.Port = cliArgs.Port
|
||||||
|
}
|
||||||
|
if cfgArgs.Username == nil {
|
||||||
|
cfgArgs.Username = cliArgs.Username
|
||||||
|
}
|
||||||
|
if cfgArgs.Password == nil {
|
||||||
|
cfgArgs.Password = cliArgs.Password
|
||||||
|
}
|
||||||
|
if cfgArgs.ApiToken == nil {
|
||||||
|
cfgArgs.ApiToken = cliArgs.ApiToken
|
||||||
|
}
|
||||||
|
if cfgArgs.StorageDir == nil {
|
||||||
|
cfgArgs.StorageDir = cliArgs.StorageDir
|
||||||
|
}
|
||||||
|
return cfgArgs
|
||||||
}
|
}
|
||||||
|
|
||||||
// overrideWithCliArgs overrides config with non-empty command line arguments
|
func loadConfig(path string) *args {
|
||||||
func overrideWithCliArgs(cfg *args, cliConfig *args) {
|
var args args
|
||||||
flag.Visit(func(f *flag.Flag) {
|
|
||||||
switch f.Name {
|
|
||||||
case "A", "address":
|
|
||||||
cfg.Address = cliConfig.Address
|
|
||||||
case "P", "port":
|
|
||||||
cfg.Port = cliConfig.Port
|
|
||||||
case "u", "username":
|
|
||||||
cfg.Username = cliConfig.Username
|
|
||||||
case "p", "password":
|
|
||||||
cfg.Password = cliConfig.Password
|
|
||||||
case "T", "api-token":
|
|
||||||
cfg.ApiToken = cliConfig.ApiToken
|
|
||||||
case "mcp-enable":
|
|
||||||
cfg.MCPEnable = cliConfig.MCPEnable
|
|
||||||
case "temp-dir":
|
|
||||||
cfg.TempDir = cliConfig.TempDir
|
|
||||||
case "d", "storage-dir":
|
|
||||||
cfg.StorageDir = cliConfig.StorageDir
|
|
||||||
case "white-download-dirs":
|
|
||||||
cfg.WhiteDownloadDirs = cliConfig.WhiteDownloadDirs
|
|
||||||
case "c", "config":
|
|
||||||
cfg.configPath = cliConfig.configPath
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// setDefaults sets default values for any unset configuration fields
|
if !filepath.IsAbs(path) {
|
||||||
func setDefaults(cfg *args, cliConfig *args) {
|
dir, err := filepath.Abs(filepath.Dir(os.Args[0]))
|
||||||
if cfg.TempDir == nil {
|
|
||||||
cfg.TempDir = cliConfig.TempDir
|
|
||||||
}
|
|
||||||
if cfg.Address == nil {
|
|
||||||
cfg.Address = cliConfig.Address
|
|
||||||
}
|
|
||||||
if cfg.Port == nil {
|
|
||||||
cfg.Port = cliConfig.Port
|
|
||||||
}
|
|
||||||
if cfg.Username == nil {
|
|
||||||
cfg.Username = cliConfig.Username
|
|
||||||
}
|
|
||||||
if cfg.Password == nil {
|
|
||||||
cfg.Password = cliConfig.Password
|
|
||||||
}
|
|
||||||
if cfg.ApiToken == nil {
|
|
||||||
cfg.ApiToken = cliConfig.ApiToken
|
|
||||||
}
|
|
||||||
if cfg.StorageDir == nil {
|
|
||||||
cfg.StorageDir = cliConfig.StorageDir
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// loadConfigFile loads configuration from file
|
|
||||||
func loadConfigFile(cfg *args, configPath string) {
|
|
||||||
if !filepath.IsAbs(configPath) {
|
|
||||||
dir, err := os.Getwd()
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return
|
return &args
|
||||||
}
|
}
|
||||||
configPath = filepath.Join(dir, configPath)
|
path = filepath.Join(dir, path)
|
||||||
}
|
}
|
||||||
|
file, err := os.ReadFile(path)
|
||||||
file, err := os.ReadFile(configPath)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if os.IsNotExist(err) {
|
if os.IsNotExist(err) {
|
||||||
return
|
return &args
|
||||||
}
|
}
|
||||||
return
|
fmt.Println("config file read failed, reason:" + err.Error())
|
||||||
|
return &args
|
||||||
}
|
}
|
||||||
|
if err = json.Unmarshal(file, &args); err != nil {
|
||||||
if err = json.Unmarshal(file, cfg); err != nil {
|
fmt.Println("config file parse failed, reason:" + err.Error())
|
||||||
return
|
return &args
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// loadEnvVars loads configuration from environment variables with prefix GOPEED_
|
|
||||||
func loadEnvVars(cfg *args) {
|
|
||||||
v := reflect.ValueOf(cfg).Elem()
|
|
||||||
t := reflect.TypeOf(cfg).Elem()
|
|
||||||
|
|
||||||
for i := 0; i < v.NumField(); i++ {
|
|
||||||
field := v.Field(i)
|
|
||||||
fieldType := t.Field(i)
|
|
||||||
|
|
||||||
// Get json tag as environment variable suffix
|
|
||||||
jsonTag := fieldType.Tag.Get("json")
|
|
||||||
if jsonTag == "" || jsonTag == "-" {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
// Remove options like omitempty
|
|
||||||
if commaIdx := strings.Index(jsonTag, ","); commaIdx != -1 {
|
|
||||||
jsonTag = jsonTag[:commaIdx]
|
|
||||||
}
|
|
||||||
|
|
||||||
// Convert to uppercase and add GOPEED_ prefix
|
|
||||||
envKey := "GOPEED_" + strings.ToUpper(jsonTag)
|
|
||||||
envValue := os.Getenv(envKey)
|
|
||||||
|
|
||||||
if envValue == "" {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
// Set value based on field type
|
|
||||||
if field.Kind() == reflect.Ptr {
|
|
||||||
if field.IsNil() {
|
|
||||||
// Create new pointer instance
|
|
||||||
newVal := reflect.New(field.Type().Elem())
|
|
||||||
field.Set(newVal)
|
|
||||||
}
|
|
||||||
|
|
||||||
switch field.Type().Elem().Kind() {
|
|
||||||
case reflect.String:
|
|
||||||
field.Elem().SetString(envValue)
|
|
||||||
case reflect.Int:
|
|
||||||
if intVal, err := strconv.Atoi(envValue); err == nil {
|
|
||||||
field.Elem().SetInt(int64(intVal))
|
|
||||||
}
|
|
||||||
default:
|
|
||||||
// For complex types like DownloadConfig, try JSON unmarshaling
|
|
||||||
if field.Type().Elem() == reflect.TypeOf(base.DownloaderStoreConfig{}) {
|
|
||||||
var config base.DownloaderStoreConfig
|
|
||||||
if err := json.Unmarshal([]byte(envValue), &config); err == nil {
|
|
||||||
field.Set(reflect.ValueOf(&config))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} else if field.Kind() == reflect.Slice {
|
|
||||||
// Handle non-pointer slice types (like []string for WhiteDownloadDirs)
|
|
||||||
if field.Type().Elem().Kind() == reflect.String {
|
|
||||||
dirs := strings.Split(envValue, ",")
|
|
||||||
for i := range dirs {
|
|
||||||
dirs[i] = strings.TrimSpace(dirs[i])
|
|
||||||
}
|
|
||||||
field.Set(reflect.ValueOf(dirs))
|
|
||||||
}
|
|
||||||
} else if field.Kind() == reflect.Bool {
|
|
||||||
if boolVal, err := strconv.ParseBool(envValue); err == nil {
|
|
||||||
field.SetBool(boolVal)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
fmt.Printf("config file loaded: %s\n", path)
|
||||||
|
return &args
|
||||||
}
|
}
|
||||||
@@ -6,12 +6,11 @@ package main
|
|||||||
import (
|
import (
|
||||||
"embed"
|
"embed"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"github.com/GopeedLab/gopeed/cmd"
|
||||||
|
"github.com/GopeedLab/gopeed/pkg/rest/model"
|
||||||
"io/fs"
|
"io/fs"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
|
||||||
"github.com/GopeedLab/gopeed/cmd"
|
|
||||||
"github.com/GopeedLab/gopeed/pkg/rest/model"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
//go:embed dist/*
|
//go:embed dist/*
|
||||||
@@ -24,48 +23,37 @@ func main() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
args := parse()
|
args := parse()
|
||||||
var webAuth *model.WebAuth
|
var webBasicAuth *model.WebBasicAuth
|
||||||
if isNotBlank(args.Username) && isNotBlank(args.Password) {
|
if isNotBlank(args.Username) && isNotBlank(args.Password) {
|
||||||
webAuth = &model.WebAuth{
|
webBasicAuth = &model.WebBasicAuth{
|
||||||
Username: *args.Username,
|
Username: *args.Username,
|
||||||
Password: *args.Password,
|
Password: *args.Password,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
var storageDir string
|
var dir string
|
||||||
if args.StorageDir != nil && *args.StorageDir != "" {
|
if args.StorageDir != nil && *args.StorageDir != "" {
|
||||||
storageDir = *args.StorageDir
|
dir = *args.StorageDir
|
||||||
} else {
|
} else {
|
||||||
exe, err := os.Executable()
|
exe, err := os.Executable()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
panic(err)
|
panic(err)
|
||||||
}
|
}
|
||||||
storageDir = filepath.Join(filepath.Dir(exe), "storage")
|
dir = filepath.Dir(exe)
|
||||||
}
|
}
|
||||||
|
|
||||||
tempDir := ""
|
|
||||||
if args.TempDir != nil {
|
|
||||||
tempDir = *args.TempDir
|
|
||||||
}
|
|
||||||
cfg := &model.StartConfig{
|
cfg := &model.StartConfig{
|
||||||
Network: "tcp",
|
Network: "tcp",
|
||||||
Address: fmt.Sprintf("%s:%d", *args.Address, *args.Port),
|
Address: fmt.Sprintf("%s:%d", *args.Address, *args.Port),
|
||||||
Storage: model.StorageBolt,
|
Storage: model.StorageBolt,
|
||||||
StorageDir: storageDir,
|
StorageDir: filepath.Join(dir, "storage"),
|
||||||
TempDir: tempDir,
|
ApiToken: *args.ApiToken,
|
||||||
WhiteDownloadDirs: args.WhiteDownloadDirs,
|
ProductionMode: true,
|
||||||
ApiToken: *args.ApiToken,
|
WebEnable: true,
|
||||||
MCPEnable: args.MCPEnable,
|
WebFS: sub,
|
||||||
DownloadConfig: args.DownloadConfig,
|
WebBasicAuth: webBasicAuth,
|
||||||
ProductionMode: true,
|
|
||||||
WebEnable: true,
|
|
||||||
WebFS: sub,
|
|
||||||
WebAuth: webAuth,
|
|
||||||
}
|
|
||||||
if err := cmd.Start(cfg); err != nil {
|
|
||||||
fmt.Fprintln(os.Stderr, err)
|
|
||||||
os.Exit(1)
|
|
||||||
}
|
}
|
||||||
|
cmd.Start(cfg)
|
||||||
}
|
}
|
||||||
|
|
||||||
func isNotBlank(str *string) bool {
|
func isNotBlank(str *string) bool {
|
||||||
|
|||||||
@@ -1,15 +1,12 @@
|
|||||||
|
version: '3'
|
||||||
|
|
||||||
services:
|
services:
|
||||||
gopeed:
|
gopeed:
|
||||||
container_name: gopeed
|
container_name: gopeed
|
||||||
ports:
|
ports:
|
||||||
- 9999:9999 # HTTP port (host:container)
|
- "9999:9999" # HTTP port (host:container)
|
||||||
environment:
|
image: liwei2633/gopeed
|
||||||
- PUID=0
|
volumes:
|
||||||
- PGID=0
|
- ~/gopeed/Downloads:/root/Downloads # mount download path
|
||||||
- UMASK=022
|
#- ~/gopeed/storage:/app/storage # if you need to mount storage path, uncomment this line
|
||||||
volumes:
|
restart: unless-stopped
|
||||||
- ~/gopeed/Downloads:/app/Downloads # mount download path
|
|
||||||
#- ~/gopeed/storage:/app/storage # if you need to mount storage path, uncomment this line
|
|
||||||
restart: unless-stopped
|
|
||||||
image: liwei2633/gopeed
|
|
||||||
# command: -u Username -p Password # optional authentication
|
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
#!/bin/sh
|
#!/bin/bash
|
||||||
|
|
||||||
chown -R ${PUID}:${PGID} /app
|
chown -R ${PUID}:${PGID} /app
|
||||||
|
|
||||||
|
|||||||
@@ -1,171 +1,118 @@
|
|||||||
module github.com/GopeedLab/gopeed
|
module github.com/GopeedLab/gopeed
|
||||||
|
|
||||||
go 1.25.4
|
go 1.21
|
||||||
|
|
||||||
require (
|
require (
|
||||||
codeberg.org/gruf/go-ffmpreg v0.6.23-0.20260827095416-df1836cb5be0
|
github.com/anacrolix/missinggo/v2 v2.7.3
|
||||||
github.com/GopeedLab/webview_go v0.0.0-20260918010912-0d4c27bb7914
|
github.com/anacrolix/torrent v1.53.3
|
||||||
github.com/anacrolix/torrent v1.60.1-0.20251217073903-486bcbe758e0
|
|
||||||
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5
|
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5
|
||||||
github.com/bodgit/sevenzip v1.6.1
|
github.com/dop251/goja v0.0.0-20231027120936-b396bb4c349d
|
||||||
github.com/dop251/goja v0.0.0-20260106131823-651366fbe6e3
|
github.com/dop251/goja_nodejs v0.0.0-20231122114759-e84d9a924c5c
|
||||||
github.com/dop251/goja_nodejs v0.0.0-20240728170619-29b559befffc
|
github.com/go-git/go-git/v5 v5.8.1
|
||||||
github.com/fclairamb/ftpserverlib v0.31.0
|
|
||||||
github.com/go-git/go-git/v5 v5.19.2
|
|
||||||
github.com/google/jsonschema-go v0.4.3
|
|
||||||
github.com/gorilla/handlers v1.5.1
|
github.com/gorilla/handlers v1.5.1
|
||||||
github.com/gorilla/mux v1.8.0
|
github.com/gorilla/mux v1.8.0
|
||||||
github.com/imroc/req/v3 v3.58.0
|
|
||||||
github.com/jlaffaye/ftp v0.2.4
|
|
||||||
github.com/matoous/go-nanoid/v2 v2.0.0
|
github.com/matoous/go-nanoid/v2 v2.0.0
|
||||||
github.com/mattn/go-ieproxy v0.0.12
|
|
||||||
github.com/mholt/archives v0.1.5
|
|
||||||
github.com/modelcontextprotocol/go-sdk v1.7.0
|
|
||||||
github.com/monkeyWie/goed2k v0.0.0-20260317100435-7a7575cf2447
|
|
||||||
github.com/pkg/errors v0.9.1
|
github.com/pkg/errors v0.9.1
|
||||||
github.com/rs/dnscache v0.0.0-20230804202142-fc85eb664529
|
|
||||||
github.com/rs/zerolog v1.31.0
|
github.com/rs/zerolog v1.31.0
|
||||||
github.com/spf13/afero v1.15.0
|
github.com/virtuald/go-paniclog v0.0.0-20190812204905-43a7fa316459
|
||||||
github.com/tetratelabs/wazero v1.12.0
|
go.etcd.io/bbolt v1.3.8
|
||||||
github.com/xiaoqidun/setft v0.0.0-20220310121541-be86327699ad
|
golang.org/x/exp v0.0.0-20240119083558-1b970713d09a
|
||||||
go.etcd.io/bbolt v1.4.3
|
golang.org/x/sync v0.6.0
|
||||||
golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f
|
|
||||||
)
|
|
||||||
|
|
||||||
require (
|
|
||||||
github.com/cyphar/filepath-securejoin v0.6.1 // indirect
|
|
||||||
github.com/google/go-querystring v1.1.0 // indirect
|
|
||||||
github.com/onsi/gomega v1.36.3 // indirect
|
|
||||||
)
|
|
||||||
|
|
||||||
require (
|
|
||||||
github.com/STARRY-S/zip v0.2.3 // indirect
|
|
||||||
github.com/anacrolix/btree v0.1.1 // indirect
|
|
||||||
github.com/anacrolix/missinggo/v2 v2.10.0 // indirect
|
|
||||||
github.com/andybalholm/brotli v1.2.0 // indirect
|
|
||||||
github.com/bodgit/plumbing v1.3.0 // indirect
|
|
||||||
github.com/bodgit/windows v1.0.1 // indirect
|
|
||||||
github.com/cespare/xxhash/v2 v2.3.0 // indirect
|
|
||||||
github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707 // indirect
|
|
||||||
github.com/felixge/fgprof v0.9.5 // indirect
|
|
||||||
github.com/google/go-cmp v0.7.0 // indirect
|
|
||||||
github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
|
|
||||||
github.com/icholy/digest v1.1.0 // indirect
|
|
||||||
github.com/klauspost/compress v1.18.2 // indirect
|
|
||||||
github.com/klauspost/pgzip v1.2.6 // indirect
|
|
||||||
github.com/mikelolasagasti/xz v1.0.1 // indirect
|
|
||||||
github.com/minio/minlz v1.0.1 // indirect
|
|
||||||
github.com/nwaples/rardecode/v2 v2.2.0 // indirect
|
|
||||||
github.com/pierrec/lz4/v4 v4.1.22 // indirect
|
|
||||||
github.com/pion/transport/v4 v4.0.2 // indirect
|
|
||||||
github.com/quic-go/qpack v0.6.0 // indirect
|
|
||||||
github.com/quic-go/quic-go v0.59.0 // indirect
|
|
||||||
github.com/refraction-networking/utls v1.8.2 // indirect
|
|
||||||
github.com/segmentio/asm v1.1.3 // indirect
|
|
||||||
github.com/segmentio/encoding v0.5.4 // indirect
|
|
||||||
github.com/sorairolake/lzip-go v0.3.8 // indirect
|
|
||||||
github.com/ulikunitz/xz v0.5.15 // indirect
|
|
||||||
github.com/yosida95/uritemplate/v3 v3.0.2 // indirect
|
|
||||||
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
|
|
||||||
go4.org v0.0.0-20230225012048-214862532bf5 // indirect
|
|
||||||
golang.org/x/oauth2 v0.35.0 // indirect
|
|
||||||
golang.org/x/sync v0.22.0 // indirect
|
|
||||||
)
|
)
|
||||||
|
|
||||||
require (
|
require (
|
||||||
dario.cat/mergo v1.0.0 // indirect
|
dario.cat/mergo v1.0.0 // indirect
|
||||||
github.com/Microsoft/go-winio v0.6.2
|
github.com/Microsoft/go-winio v0.6.1 // indirect
|
||||||
github.com/ProtonMail/go-crypto v1.1.6 // indirect
|
github.com/ProtonMail/go-crypto v0.0.0-20230717121422-5aa5874ade95 // indirect
|
||||||
github.com/RoaringBitmap/roaring v1.9.4
|
github.com/RoaringBitmap/roaring v1.7.0 // indirect
|
||||||
|
github.com/acomagu/bufpipe v1.0.4 // indirect
|
||||||
|
github.com/ajwerner/btree v0.0.0-20211221152037-f427b3e689c0 // indirect
|
||||||
github.com/alecthomas/atomic v0.1.0-alpha2 // indirect
|
github.com/alecthomas/atomic v0.1.0-alpha2 // indirect
|
||||||
github.com/anacrolix/chansync v0.7.0 // indirect
|
github.com/anacrolix/chansync v0.4.0 // indirect
|
||||||
github.com/anacrolix/dht/v2 v2.23.0 // indirect
|
github.com/anacrolix/dht/v2 v2.21.0 // indirect
|
||||||
github.com/anacrolix/envpprof v1.5.0 // indirect
|
github.com/anacrolix/envpprof v1.3.0 // indirect
|
||||||
github.com/anacrolix/generics v0.2.0
|
github.com/anacrolix/generics v0.0.0-20230911070922-5dd7545c6b13 // indirect
|
||||||
github.com/anacrolix/go-libutp v1.3.2 // indirect
|
github.com/anacrolix/go-libutp v1.3.1 // indirect
|
||||||
github.com/anacrolix/log v0.17.1-0.20251118025802-918f1157b7bb // indirect
|
github.com/anacrolix/log v0.14.6-0.20231202035202-ed7a02cad0b4 // indirect
|
||||||
github.com/anacrolix/missinggo v1.3.0 // indirect
|
github.com/anacrolix/missinggo v1.3.0 // indirect
|
||||||
github.com/anacrolix/missinggo/perf v1.0.0 // indirect
|
github.com/anacrolix/missinggo/perf v1.0.0 // indirect
|
||||||
github.com/anacrolix/mmsg v1.1.1 // indirect
|
github.com/anacrolix/mmsg v1.0.0 // indirect
|
||||||
github.com/anacrolix/multiless v0.4.0 // indirect
|
github.com/anacrolix/multiless v0.3.1-0.20221221005021-2d12701f83f7 // indirect
|
||||||
github.com/anacrolix/stm v0.5.0 // indirect
|
github.com/anacrolix/stm v0.5.0 // indirect
|
||||||
github.com/anacrolix/sync v0.6.0 // indirect
|
github.com/anacrolix/sync v0.5.1 // indirect
|
||||||
github.com/anacrolix/upnp v0.1.4 // indirect
|
github.com/anacrolix/upnp v0.1.3 // indirect
|
||||||
github.com/anacrolix/utp v0.2.0 // indirect
|
github.com/anacrolix/utp v0.2.0 // indirect
|
||||||
github.com/bahlo/generic-list-go v0.2.0 // indirect
|
github.com/bahlo/generic-list-go v0.2.0 // indirect
|
||||||
github.com/benbjohnson/immutable v0.4.3 // indirect
|
github.com/benbjohnson/immutable v0.4.3 // indirect
|
||||||
github.com/bits-and-blooms/bitset v1.24.4 // indirect
|
github.com/bits-and-blooms/bitset v1.13.0 // indirect
|
||||||
github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8 // indirect
|
github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8 // indirect
|
||||||
github.com/cespare/xxhash v1.1.0 // indirect
|
github.com/cloudflare/circl v1.3.3 // indirect
|
||||||
github.com/cloudflare/circl v1.6.3 // indirect
|
|
||||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||||
github.com/dlclark/regexp2 v1.11.5 // indirect
|
github.com/dlclark/regexp2 v1.10.0 // indirect
|
||||||
github.com/dustin/go-humanize v1.0.1 // indirect
|
github.com/dustin/go-humanize v1.0.1 // indirect
|
||||||
github.com/edsrzf/mmap-go v1.2.0 // indirect
|
github.com/edsrzf/mmap-go v1.1.0 // indirect
|
||||||
github.com/emirpasic/gods v1.18.1 // indirect
|
github.com/emirpasic/gods v1.18.1 // indirect
|
||||||
github.com/felixge/httpsnoop v1.0.1 // indirect
|
github.com/felixge/httpsnoop v1.0.1 // indirect
|
||||||
github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
|
github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
|
||||||
github.com/go-git/go-billy/v5 v5.9.2 // indirect
|
github.com/go-git/go-billy/v5 v5.4.1 // indirect
|
||||||
github.com/go-llsqlite/adapter v0.2.0 // indirect
|
github.com/go-llsqlite/adapter v0.1.0 // indirect
|
||||||
github.com/go-llsqlite/crawshaw v0.6.0 // indirect
|
github.com/go-llsqlite/crawshaw v0.5.0 // indirect
|
||||||
github.com/go-logr/logr v1.4.3 // indirect
|
github.com/go-logr/logr v1.4.1 // indirect
|
||||||
github.com/go-logr/stdr v1.2.2 // indirect
|
github.com/go-logr/stdr v1.2.2 // indirect
|
||||||
github.com/go-sourcemap/sourcemap v2.1.4+incompatible // indirect
|
github.com/go-sourcemap/sourcemap v2.1.3+incompatible // indirect
|
||||||
github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect
|
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
|
||||||
github.com/google/btree v1.1.3 // indirect
|
github.com/google/btree v1.1.2 // indirect
|
||||||
github.com/google/pprof v0.0.0-20260115054156-294ebfa9ad83 // indirect
|
github.com/google/pprof v0.0.0-20231101202521-4ca4178f5c7a // indirect
|
||||||
github.com/google/uuid v1.6.0 // indirect
|
github.com/google/uuid v1.5.0 // indirect
|
||||||
github.com/gorilla/websocket v1.5.3 // indirect
|
github.com/gorilla/websocket v1.5.1 // indirect
|
||||||
github.com/huandu/xstrings v1.5.0 // indirect
|
github.com/huandu/xstrings v1.4.0 // indirect
|
||||||
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
|
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
|
||||||
github.com/kevinburke/ssh_config v1.2.0 // indirect
|
github.com/kevinburke/ssh_config v1.2.0 // indirect
|
||||||
github.com/klauspost/cpuid/v2 v2.3.0 // indirect
|
|
||||||
github.com/mattn/go-colorable v0.1.13 // indirect
|
github.com/mattn/go-colorable v0.1.13 // indirect
|
||||||
|
github.com/mattn/go-ieproxy v0.0.11 // indirect
|
||||||
github.com/mattn/go-isatty v0.0.20 // indirect
|
github.com/mattn/go-isatty v0.0.20 // indirect
|
||||||
github.com/minio/sha256-simd v1.0.1 // indirect
|
|
||||||
github.com/mr-tron/base58 v1.2.0 // indirect
|
|
||||||
github.com/mschoch/smat v0.2.0 // indirect
|
github.com/mschoch/smat v0.2.0 // indirect
|
||||||
github.com/multiformats/go-multihash v0.2.3 // indirect
|
github.com/pion/datachannel v1.5.5 // indirect
|
||||||
github.com/multiformats/go-varint v0.1.0 // indirect
|
github.com/pion/dtls/v2 v2.2.9 // indirect
|
||||||
github.com/ncruces/go-strftime v1.0.0 // indirect
|
github.com/pion/ice/v2 v2.3.12 // indirect
|
||||||
github.com/pion/datachannel v1.6.0 // indirect
|
github.com/pion/interceptor v0.1.25 // indirect
|
||||||
github.com/pion/dtls/v3 v3.1.4 // indirect
|
github.com/pion/logging v0.2.2 // indirect
|
||||||
github.com/pion/ice/v4 v4.2.0 // indirect
|
github.com/pion/mdns v0.0.9 // indirect
|
||||||
github.com/pion/interceptor v0.1.42 // indirect
|
|
||||||
github.com/pion/logging v0.2.4 // indirect
|
|
||||||
github.com/pion/mdns/v2 v2.1.0 // indirect
|
|
||||||
github.com/pion/randutil v0.1.0 // indirect
|
github.com/pion/randutil v0.1.0 // indirect
|
||||||
github.com/pion/rtcp v1.2.16 // indirect
|
github.com/pion/rtcp v1.2.13 // indirect
|
||||||
github.com/pion/rtp v1.10.0 // indirect
|
github.com/pion/rtp v1.8.3 // indirect
|
||||||
github.com/pion/sctp v1.9.1 // indirect
|
github.com/pion/sctp v1.8.9 // indirect
|
||||||
github.com/pion/sdp/v3 v3.0.17 // indirect
|
github.com/pion/sdp/v3 v3.0.6 // indirect
|
||||||
github.com/pion/srtp/v3 v3.0.10 // indirect
|
github.com/pion/srtp/v2 v2.0.18 // indirect
|
||||||
github.com/pion/stun/v3 v3.1.5 // indirect
|
github.com/pion/stun v0.6.1 // indirect
|
||||||
github.com/pion/turn/v4 v4.1.4 // indirect
|
github.com/pion/transport/v2 v2.2.4 // indirect
|
||||||
github.com/pion/webrtc/v4 v4.2.2-0.20260109001657-a5962f314db7 // indirect
|
github.com/pion/turn/v2 v2.1.4 // indirect
|
||||||
github.com/pjbgf/sha1cd v0.6.0 // indirect
|
github.com/pion/webrtc/v3 v3.2.24 // indirect
|
||||||
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c
|
github.com/pjbgf/sha1cd v0.3.0 // indirect
|
||||||
github.com/protolambda/ctxlock v0.1.0 // indirect
|
github.com/pmezard/go-difflib v1.0.0 // indirect
|
||||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
|
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
|
||||||
github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect
|
github.com/rs/dnscache v0.0.0-20230804202142-fc85eb664529 // indirect
|
||||||
github.com/skeema/knownhosts v1.3.1 // indirect
|
github.com/sergi/go-diff v1.1.0 // indirect
|
||||||
github.com/spaolacci/murmur3 v1.1.0 // indirect
|
github.com/skeema/knownhosts v1.2.0 // indirect
|
||||||
github.com/tidwall/btree v1.8.1 // indirect
|
github.com/stretchr/testify v1.8.4 // indirect
|
||||||
github.com/wlynxg/anet v0.0.5 // indirect
|
github.com/tidwall/btree v1.7.0 // indirect
|
||||||
github.com/xanzy/ssh-agent v0.3.3 // indirect
|
github.com/xanzy/ssh-agent v0.3.3 // indirect
|
||||||
go.opentelemetry.io/otel v1.39.0 // indirect
|
github.com/xiaoqidun/setft v0.0.0-20220310121541-be86327699ad // indirect
|
||||||
go.opentelemetry.io/otel/metric v1.39.0 // indirect
|
go.opentelemetry.io/otel v1.22.0 // indirect
|
||||||
go.opentelemetry.io/otel/trace v1.39.0 // indirect
|
go.opentelemetry.io/otel/metric v1.22.0 // indirect
|
||||||
golang.org/x/crypto v0.55.0 // indirect
|
go.opentelemetry.io/otel/trace v1.22.0 // indirect
|
||||||
golang.org/x/net v0.58.0
|
golang.org/x/crypto v0.18.0 // indirect
|
||||||
golang.org/x/sys v0.47.0 // indirect
|
golang.org/x/mod v0.14.0 // indirect
|
||||||
golang.org/x/text v0.41.0
|
golang.org/x/net v0.20.0 // indirect
|
||||||
golang.org/x/time v0.15.0
|
golang.org/x/sys v0.16.0 // indirect
|
||||||
|
golang.org/x/text v0.14.0 // indirect
|
||||||
|
golang.org/x/time v0.5.0 // indirect
|
||||||
|
golang.org/x/tools v0.17.0 // indirect
|
||||||
gopkg.in/warnings.v0 v0.1.2 // indirect
|
gopkg.in/warnings.v0 v0.1.2 // indirect
|
||||||
lukechampine.com/blake3 v1.4.1 // indirect
|
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||||
modernc.org/libc v1.67.4 // indirect
|
modernc.org/libc v1.40.6 // indirect
|
||||||
modernc.org/mathutil v1.7.1 // indirect
|
modernc.org/mathutil v1.6.0 // indirect
|
||||||
modernc.org/memory v1.11.0 // indirect
|
modernc.org/memory v1.7.2 // indirect
|
||||||
modernc.org/sqlite v1.43.0 // indirect
|
modernc.org/sqlite v1.28.0 // indirect
|
||||||
zombiezen.com/go/sqlite v1.4.2 // indirect
|
zombiezen.com/go/sqlite v1.1.0 // indirect
|
||||||
)
|
)
|
||||||
@@ -1,721 +0,0 @@
|
|||||||
package blob
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"context"
|
|
||||||
"crypto/rand"
|
|
||||||
"encoding/hex"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"github.com/GopeedLab/gopeed/internal/production"
|
|
||||||
"io"
|
|
||||||
"net"
|
|
||||||
"net/http"
|
|
||||||
"net/url"
|
|
||||||
"path"
|
|
||||||
"strconv"
|
|
||||||
"strings"
|
|
||||||
"sync"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
const urlPathPrefix = "/__blob/"
|
|
||||||
|
|
||||||
const rangeSourceFailureLimit = 2
|
|
||||||
|
|
||||||
// unclaimedSourceTTL bounds how long a session-backed source may keep its
|
|
||||||
// engine alive without ever being claimed by a download task.
|
|
||||||
var unclaimedSourceTTL = 10 * time.Minute
|
|
||||||
|
|
||||||
var (
|
|
||||||
ErrInvalidURL = errors.New("invalid blob url")
|
|
||||||
ErrInvalidOptions = errors.New("invalid blob options")
|
|
||||||
ErrSourceNotFound = errors.New("blob source not found")
|
|
||||||
ErrSourceRevoked = errors.New("blob source revoked")
|
|
||||||
ErrSourceClosed = errors.New("blob source closed")
|
|
||||||
ErrRangeNotAllowed = errors.New("blob range not allowed")
|
|
||||||
)
|
|
||||||
|
|
||||||
type SessionRef interface {
|
|
||||||
Retain()
|
|
||||||
Release()
|
|
||||||
}
|
|
||||||
|
|
||||||
type OpenRequest struct {
|
|
||||||
Offset int64
|
|
||||||
End int64
|
|
||||||
}
|
|
||||||
|
|
||||||
type OpenFunc func(ctx context.Context, req OpenRequest) (io.ReadCloser, error)
|
|
||||||
|
|
||||||
type CreateOptions struct {
|
|
||||||
ContentType string
|
|
||||||
Size int64
|
|
||||||
Range bool
|
|
||||||
Session SessionRef
|
|
||||||
}
|
|
||||||
|
|
||||||
type Metadata struct {
|
|
||||||
ContentType string
|
|
||||||
Size int64
|
|
||||||
Range bool
|
|
||||||
}
|
|
||||||
|
|
||||||
type Source struct {
|
|
||||||
producer production.Reader
|
|
||||||
ID string
|
|
||||||
URL string
|
|
||||||
ContentType string
|
|
||||||
|
|
||||||
mu sync.Mutex
|
|
||||||
size int64
|
|
||||||
rangeEnabled bool
|
|
||||||
revoked bool
|
|
||||||
taskRefs int
|
|
||||||
readErr error
|
|
||||||
rangeFailures int
|
|
||||||
session SessionRef
|
|
||||||
open OpenFunc
|
|
||||||
unclaimed *time.Timer
|
|
||||||
}
|
|
||||||
|
|
||||||
type Registry struct {
|
|
||||||
dir string
|
|
||||||
|
|
||||||
mu sync.RWMutex
|
|
||||||
sources map[string]*Source
|
|
||||||
|
|
||||||
serverMu sync.Mutex
|
|
||||||
listener net.Listener
|
|
||||||
server *http.Server
|
|
||||||
baseURL string
|
|
||||||
}
|
|
||||||
|
|
||||||
func NewRegistry(dir string) *Registry {
|
|
||||||
return &Registry{
|
|
||||||
dir: dir,
|
|
||||||
sources: make(map[string]*Source),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *Registry) Dir() string {
|
|
||||||
if r == nil {
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
return r.dir
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *Registry) IsURL(raw string) bool {
|
|
||||||
if r == nil {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
src, err := r.get(raw)
|
|
||||||
return err == nil && src != nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *Registry) CreateBlob(data []byte, contentType string) (string, error) {
|
|
||||||
buf := append([]byte(nil), data...)
|
|
||||||
return r.CreateOpener(func(ctx context.Context, req OpenRequest) (io.ReadCloser, error) {
|
|
||||||
if req.Offset < 0 || req.Offset > int64(len(buf)) {
|
|
||||||
return nil, ErrSourceNotFound
|
|
||||||
}
|
|
||||||
end := int64(len(buf))
|
|
||||||
if req.End >= req.Offset && req.End+1 < end {
|
|
||||||
end = req.End + 1
|
|
||||||
}
|
|
||||||
return io.NopCloser(bytes.NewReader(buf[req.Offset:end])), nil
|
|
||||||
}, &CreateOptions{
|
|
||||||
ContentType: contentType,
|
|
||||||
Size: int64(len(buf)),
|
|
||||||
Range: len(buf) > 0,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *Registry) CreateOpener(open OpenFunc, opts *CreateOptions) (string, error) {
|
|
||||||
if r == nil {
|
|
||||||
return "", ErrSourceNotFound
|
|
||||||
}
|
|
||||||
if open == nil {
|
|
||||||
return "", ErrInvalidOptions
|
|
||||||
}
|
|
||||||
if opts == nil {
|
|
||||||
opts = &CreateOptions{}
|
|
||||||
}
|
|
||||||
if opts.Size < 0 {
|
|
||||||
opts.Size = 0
|
|
||||||
}
|
|
||||||
if opts.Range && opts.Size <= 0 {
|
|
||||||
return "", fmt.Errorf("%w: range requires positive size", ErrInvalidOptions)
|
|
||||||
}
|
|
||||||
// Keep server generation selection and source insertion atomic with Close.
|
|
||||||
r.serverMu.Lock()
|
|
||||||
baseURL, err := r.ensureServerLocked()
|
|
||||||
if err != nil {
|
|
||||||
r.serverMu.Unlock()
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
id, err := randomID(18)
|
|
||||||
if err != nil {
|
|
||||||
r.serverMu.Unlock()
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
if opts.Session != nil {
|
|
||||||
opts.Session.Retain()
|
|
||||||
}
|
|
||||||
srcURL := fmt.Sprintf("%s%s", baseURL, id)
|
|
||||||
src := &Source{
|
|
||||||
ID: id,
|
|
||||||
URL: srcURL,
|
|
||||||
ContentType: opts.ContentType,
|
|
||||||
size: opts.Size,
|
|
||||||
rangeEnabled: opts.Range,
|
|
||||||
session: opts.Session,
|
|
||||||
open: open,
|
|
||||||
}
|
|
||||||
r.mu.Lock()
|
|
||||||
r.sources[id] = src
|
|
||||||
r.mu.Unlock()
|
|
||||||
r.serverMu.Unlock()
|
|
||||||
if opts.Session != nil && unclaimedSourceTTL > 0 {
|
|
||||||
src.mu.Lock()
|
|
||||||
if !src.revoked {
|
|
||||||
src.unclaimed = time.AfterFunc(unclaimedSourceTTL, func() {
|
|
||||||
r.expireUnclaimed(src)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
src.mu.Unlock()
|
|
||||||
}
|
|
||||||
return src.URL, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *Registry) Metadata(raw string) (Metadata, error) {
|
|
||||||
src, err := r.get(raw)
|
|
||||||
if err != nil {
|
|
||||||
return Metadata{}, err
|
|
||||||
}
|
|
||||||
src.mu.Lock()
|
|
||||||
defer src.mu.Unlock()
|
|
||||||
return Metadata{
|
|
||||||
ContentType: src.ContentType,
|
|
||||||
Size: src.size,
|
|
||||||
Range: src.rangeEnabled,
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Acquire retains a source for a task. A source remains available until every
|
|
||||||
// task that acquired it calls Release, unless it is explicitly revoked.
|
|
||||||
func (r *Registry) Acquire(raw string) error {
|
|
||||||
src, err := r.get(raw)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
return src.acquireTask()
|
|
||||||
}
|
|
||||||
|
|
||||||
// Release releases a task's reference to a source. Releasing the final task
|
|
||||||
// reference revokes the source. Sources that have never been acquired remain
|
|
||||||
// registered until Revoke or Registry.Close is called; session-backed sources
|
|
||||||
// are also bounded by the unclaimed-source TTL.
|
|
||||||
func (r *Registry) Release(raw string) error {
|
|
||||||
src, err := r.get(raw)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
session, last, err := src.releaseTask()
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if !last {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
r.deleteSource(src)
|
|
||||||
if session != nil {
|
|
||||||
session.Release()
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// SourceError returns the first error encountered while opening or reading a
|
|
||||||
// source. Invalid and missing sources have no source error. HTTP response write
|
|
||||||
// errors are deliberately not recorded here.
|
|
||||||
func (r *Registry) SourceError(raw string) error {
|
|
||||||
src, err := r.get(raw)
|
|
||||||
if err != nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
return src.sourceError()
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *Registry) Revoke(raw string) error {
|
|
||||||
src, err := r.get(raw)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
r.removeSource(src)
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *Registry) Close() error {
|
|
||||||
if r == nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
r.serverMu.Lock()
|
|
||||||
server := r.server
|
|
||||||
listener := r.listener
|
|
||||||
r.server = nil
|
|
||||||
r.listener = nil
|
|
||||||
r.baseURL = ""
|
|
||||||
r.mu.Lock()
|
|
||||||
sources := make([]*Source, 0, len(r.sources))
|
|
||||||
for _, src := range r.sources {
|
|
||||||
sources = append(sources, src)
|
|
||||||
}
|
|
||||||
r.sources = make(map[string]*Source)
|
|
||||||
r.mu.Unlock()
|
|
||||||
r.serverMu.Unlock()
|
|
||||||
if server != nil {
|
|
||||||
_ = server.Close()
|
|
||||||
}
|
|
||||||
if listener != nil {
|
|
||||||
_ = listener.Close()
|
|
||||||
}
|
|
||||||
for _, src := range sources {
|
|
||||||
src.close()
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// ensureServerLocked returns the current server URL or starts a new server.
|
|
||||||
// The caller must hold serverMu.
|
|
||||||
func (r *Registry) ensureServerLocked() (string, error) {
|
|
||||||
if r.baseURL != "" {
|
|
||||||
return r.baseURL, nil
|
|
||||||
}
|
|
||||||
listener, err := net.Listen("tcp", "127.0.0.1:0")
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
r.listener = listener
|
|
||||||
r.baseURL = "http://" + listener.Addr().String() + urlPathPrefix
|
|
||||||
server := &http.Server{Handler: r}
|
|
||||||
r.server = server
|
|
||||||
go func() {
|
|
||||||
if err := server.Serve(listener); err != nil && !errors.Is(err, http.ErrServerClosed) {
|
|
||||||
_ = r.Close()
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
return r.baseURL, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *Registry) ServeHTTP(w http.ResponseWriter, req *http.Request) {
|
|
||||||
if req.Method != http.MethodGet {
|
|
||||||
w.Header().Set("Allow", "GET")
|
|
||||||
http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
id := parseRequest(req)
|
|
||||||
if id == "" {
|
|
||||||
http.NotFound(w, req)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
src, err := r.getByID(id)
|
|
||||||
if err != nil {
|
|
||||||
http.NotFound(w, req)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
meta, open, session := src.acquireOpen()
|
|
||||||
if open == nil {
|
|
||||||
http.NotFound(w, req)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if session != nil {
|
|
||||||
defer session.Release()
|
|
||||||
}
|
|
||||||
if src.sourceError() != nil {
|
|
||||||
http.Error(w, "blob source unavailable", http.StatusGone)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
start, end, ranged, err := parseRange(req.Header.Get("Range"), meta.Size, meta.Range)
|
|
||||||
if err != nil {
|
|
||||||
http.Error(w, err.Error(), http.StatusRequestedRangeNotSatisfiable)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if !meta.Range {
|
|
||||||
start, end, ranged = 0, -1, false
|
|
||||||
}
|
|
||||||
reader, err := open(req.Context(), OpenRequest{
|
|
||||||
Offset: start,
|
|
||||||
End: end,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
if req.Context().Err() == nil || !errors.Is(err, req.Context().Err()) {
|
|
||||||
src.recordSourceError(err, meta.Range)
|
|
||||||
}
|
|
||||||
http.Error(w, "blob source unavailable", http.StatusGone)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if reader == nil {
|
|
||||||
err = fmt.Errorf("%w: opener returned a nil reader", ErrSourceClosed)
|
|
||||||
src.recordSourceError(err, meta.Range)
|
|
||||||
http.Error(w, "blob source unavailable", http.StatusGone)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
defer reader.Close()
|
|
||||||
if producer, ok := reader.(production.Reader); ok {
|
|
||||||
src.mu.Lock()
|
|
||||||
src.producer = producer
|
|
||||||
src.mu.Unlock()
|
|
||||||
if err := producer.WaitProduction(req.Context()); err != nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if producer.Production() != nil {
|
|
||||||
meta.Size = 0
|
|
||||||
meta.Range = false
|
|
||||||
start = 0
|
|
||||||
end = -1
|
|
||||||
ranged = false
|
|
||||||
src.mu.Lock()
|
|
||||||
src.size = 0
|
|
||||||
src.rangeEnabled = false
|
|
||||||
src.mu.Unlock()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
writeHeaders(w, meta, start, end, ranged)
|
|
||||||
if ranged {
|
|
||||||
w.WriteHeader(http.StatusPartialContent)
|
|
||||||
}
|
|
||||||
if producer, ok := reader.(production.Reader); ok && producer.Production() != nil {
|
|
||||||
if flusher, ok := w.(http.Flusher); ok {
|
|
||||||
flusher.Flush()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
limit := int64(-1)
|
|
||||||
if end >= start {
|
|
||||||
limit = end - start + 1
|
|
||||||
} else if meta.Size > 0 {
|
|
||||||
limit = meta.Size
|
|
||||||
}
|
|
||||||
_, readErr, writeErr := copyWithFlush(w, reader, limit)
|
|
||||||
if readErr != nil {
|
|
||||||
if ctxErr := req.Context().Err(); ctxErr == nil || !errors.Is(readErr, ctxErr) {
|
|
||||||
src.recordSourceError(readErr, meta.Range)
|
|
||||||
}
|
|
||||||
} else if writeErr == nil && req.Context().Err() == nil {
|
|
||||||
src.recordSourceSuccess(meta.Range)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *Source) acquireOpen() (Metadata, OpenFunc, SessionRef) {
|
|
||||||
s.mu.Lock()
|
|
||||||
defer s.mu.Unlock()
|
|
||||||
if s.revoked {
|
|
||||||
return Metadata{}, nil, nil
|
|
||||||
}
|
|
||||||
if s.session != nil {
|
|
||||||
// The source's own retain guarantees that the session cannot become
|
|
||||||
// closed while this additional active-reader retain is acquired.
|
|
||||||
s.session.Retain()
|
|
||||||
}
|
|
||||||
return Metadata{
|
|
||||||
ContentType: s.ContentType,
|
|
||||||
Size: s.size,
|
|
||||||
Range: s.rangeEnabled,
|
|
||||||
}, s.open, s.session
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *Source) acquireTask() error {
|
|
||||||
s.mu.Lock()
|
|
||||||
if s.revoked {
|
|
||||||
s.mu.Unlock()
|
|
||||||
return ErrSourceRevoked
|
|
||||||
}
|
|
||||||
s.taskRefs++
|
|
||||||
timer := s.unclaimed
|
|
||||||
s.unclaimed = nil
|
|
||||||
s.mu.Unlock()
|
|
||||||
if timer != nil {
|
|
||||||
timer.Stop()
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *Source) releaseTask() (session SessionRef, last bool, err error) {
|
|
||||||
s.mu.Lock()
|
|
||||||
defer s.mu.Unlock()
|
|
||||||
if s.revoked {
|
|
||||||
return nil, false, ErrSourceRevoked
|
|
||||||
}
|
|
||||||
if s.taskRefs == 0 {
|
|
||||||
return nil, false, ErrSourceClosed
|
|
||||||
}
|
|
||||||
s.taskRefs--
|
|
||||||
if s.taskRefs > 0 {
|
|
||||||
return nil, false, nil
|
|
||||||
}
|
|
||||||
s.revoked = true
|
|
||||||
s.open = nil
|
|
||||||
session = s.session
|
|
||||||
s.session = nil
|
|
||||||
return session, true, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *Source) sourceError() error {
|
|
||||||
s.mu.Lock()
|
|
||||||
defer s.mu.Unlock()
|
|
||||||
return s.readErr
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *Source) recordSourceError(err error, ranged bool) {
|
|
||||||
if err == nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
s.mu.Lock()
|
|
||||||
if s.readErr == nil && ranged {
|
|
||||||
s.rangeFailures++
|
|
||||||
if s.rangeFailures < rangeSourceFailureLimit {
|
|
||||||
s.mu.Unlock()
|
|
||||||
return
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if s.readErr == nil {
|
|
||||||
s.readErr = err
|
|
||||||
}
|
|
||||||
s.mu.Unlock()
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *Source) recordSourceSuccess(ranged bool) {
|
|
||||||
if !ranged {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
s.mu.Lock()
|
|
||||||
if s.readErr == nil {
|
|
||||||
s.rangeFailures = 0
|
|
||||||
}
|
|
||||||
s.mu.Unlock()
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *Source) close() {
|
|
||||||
s.mu.Lock()
|
|
||||||
alreadyRevoked := s.revoked
|
|
||||||
s.revoked = true
|
|
||||||
s.open = nil
|
|
||||||
session := s.session
|
|
||||||
s.session = nil
|
|
||||||
timer := s.unclaimed
|
|
||||||
s.unclaimed = nil
|
|
||||||
s.mu.Unlock()
|
|
||||||
if timer != nil {
|
|
||||||
timer.Stop()
|
|
||||||
}
|
|
||||||
if !alreadyRevoked && session != nil {
|
|
||||||
session.Release()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *Source) expireUnclaimed() (SessionRef, bool) {
|
|
||||||
s.mu.Lock()
|
|
||||||
defer s.mu.Unlock()
|
|
||||||
s.unclaimed = nil
|
|
||||||
if s.revoked || s.taskRefs > 0 {
|
|
||||||
return nil, false
|
|
||||||
}
|
|
||||||
s.revoked = true
|
|
||||||
s.open = nil
|
|
||||||
session := s.session
|
|
||||||
s.session = nil
|
|
||||||
return session, true
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *Registry) expireUnclaimed(src *Source) {
|
|
||||||
session, expired := src.expireUnclaimed()
|
|
||||||
if !expired {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
r.deleteSource(src)
|
|
||||||
if session != nil {
|
|
||||||
session.Release()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *Registry) removeSource(src *Source) {
|
|
||||||
src.close()
|
|
||||||
r.deleteSource(src)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *Registry) deleteSource(src *Source) {
|
|
||||||
r.mu.Lock()
|
|
||||||
if current := r.sources[src.ID]; current == src {
|
|
||||||
delete(r.sources, src.ID)
|
|
||||||
}
|
|
||||||
r.mu.Unlock()
|
|
||||||
}
|
|
||||||
|
|
||||||
func writeHeaders(w http.ResponseWriter, meta Metadata, start, end int64, ranged bool) {
|
|
||||||
if meta.ContentType != "" {
|
|
||||||
w.Header().Set("Content-Type", meta.ContentType)
|
|
||||||
}
|
|
||||||
if meta.Range {
|
|
||||||
w.Header().Set("Accept-Ranges", "bytes")
|
|
||||||
}
|
|
||||||
if meta.Size > 0 {
|
|
||||||
if ranged {
|
|
||||||
w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, meta.Size))
|
|
||||||
w.Header().Set("Content-Length", strconv.FormatInt(end-start+1, 10))
|
|
||||||
return
|
|
||||||
}
|
|
||||||
w.Header().Set("Content-Length", strconv.FormatInt(meta.Size, 10))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func copyWithFlush(w http.ResponseWriter, reader io.Reader, limit int64) (written int64, readErr error, writeErr error) {
|
|
||||||
if limit >= 0 {
|
|
||||||
reader = io.LimitReader(reader, limit)
|
|
||||||
}
|
|
||||||
buf := make([]byte, 32*1024)
|
|
||||||
flusher, canFlush := w.(http.Flusher)
|
|
||||||
for {
|
|
||||||
nr, er := reader.Read(buf)
|
|
||||||
if er != nil && !errors.Is(er, io.EOF) {
|
|
||||||
readErr = er
|
|
||||||
}
|
|
||||||
if nr > 0 {
|
|
||||||
nw, ew := w.Write(buf[:nr])
|
|
||||||
if nw > 0 {
|
|
||||||
written += int64(nw)
|
|
||||||
if canFlush {
|
|
||||||
flusher.Flush()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if ew != nil {
|
|
||||||
return written, readErr, ew
|
|
||||||
}
|
|
||||||
if nr != nw {
|
|
||||||
return written, readErr, io.ErrShortWrite
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if er != nil {
|
|
||||||
if errors.Is(er, io.EOF) {
|
|
||||||
if limit >= 0 && written < limit {
|
|
||||||
return written, io.ErrUnexpectedEOF, nil
|
|
||||||
}
|
|
||||||
return written, nil, nil
|
|
||||||
}
|
|
||||||
return written, readErr, nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func parseRange(header string, size int64, rangeEnabled bool) (start int64, end int64, ranged bool, err error) {
|
|
||||||
if header == "" || !rangeEnabled {
|
|
||||||
return 0, -1, false, nil
|
|
||||||
}
|
|
||||||
if size <= 0 {
|
|
||||||
return 0, 0, false, ErrRangeNotAllowed
|
|
||||||
}
|
|
||||||
if !strings.HasPrefix(header, "bytes=") {
|
|
||||||
return 0, 0, false, fmt.Errorf("unsupported range")
|
|
||||||
}
|
|
||||||
parts := strings.SplitN(strings.TrimPrefix(header, "bytes="), "-", 2)
|
|
||||||
if len(parts) != 2 || parts[0] == "" {
|
|
||||||
return 0, 0, false, fmt.Errorf("unsupported range")
|
|
||||||
}
|
|
||||||
start, err = strconv.ParseInt(parts[0], 10, 64)
|
|
||||||
if err != nil || start < 0 {
|
|
||||||
return 0, 0, false, fmt.Errorf("invalid range")
|
|
||||||
}
|
|
||||||
if start >= size {
|
|
||||||
return 0, 0, false, fmt.Errorf("range out of bounds")
|
|
||||||
}
|
|
||||||
end = size - 1
|
|
||||||
if parts[1] != "" {
|
|
||||||
end, err = strconv.ParseInt(parts[1], 10, 64)
|
|
||||||
if err != nil || end < start {
|
|
||||||
return 0, 0, false, fmt.Errorf("invalid range")
|
|
||||||
}
|
|
||||||
if end >= size {
|
|
||||||
end = size - 1
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return start, end, true, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *Registry) get(raw string) (*Source, error) {
|
|
||||||
id, ok := r.parseURL(raw)
|
|
||||||
if !ok {
|
|
||||||
return nil, ErrInvalidURL
|
|
||||||
}
|
|
||||||
src, err := r.getByID(id)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
src.mu.Lock()
|
|
||||||
revoked := src.revoked
|
|
||||||
src.mu.Unlock()
|
|
||||||
if revoked {
|
|
||||||
return nil, ErrSourceRevoked
|
|
||||||
}
|
|
||||||
return src, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *Registry) getByID(id string) (*Source, error) {
|
|
||||||
r.mu.RLock()
|
|
||||||
defer r.mu.RUnlock()
|
|
||||||
src := r.sources[id]
|
|
||||||
if src == nil {
|
|
||||||
return nil, ErrSourceNotFound
|
|
||||||
}
|
|
||||||
return src, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *Registry) parseURL(raw string) (id string, ok bool) {
|
|
||||||
u, err := url.Parse(raw)
|
|
||||||
if err != nil || u.Scheme != "http" || !strings.HasPrefix(u.Path, urlPathPrefix) {
|
|
||||||
return "", false
|
|
||||||
}
|
|
||||||
r.serverMu.Lock()
|
|
||||||
baseURL := r.baseURL
|
|
||||||
r.serverMu.Unlock()
|
|
||||||
if baseURL == "" || !strings.HasPrefix(raw, baseURL) {
|
|
||||||
return "", false
|
|
||||||
}
|
|
||||||
id = strings.TrimPrefix(u.Path, urlPathPrefix)
|
|
||||||
if id == "" || path.Base(id) != id {
|
|
||||||
return "", false
|
|
||||||
}
|
|
||||||
return id, true
|
|
||||||
}
|
|
||||||
|
|
||||||
func parseRequest(req *http.Request) string {
|
|
||||||
id := strings.TrimPrefix(req.URL.Path, urlPathPrefix)
|
|
||||||
if id == "" || id == req.URL.Path || path.Base(id) != id {
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
return id
|
|
||||||
}
|
|
||||||
|
|
||||||
func randomID(size int) (string, error) {
|
|
||||||
buf := make([]byte, size)
|
|
||||||
if _, err := rand.Read(buf); err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
return hex.EncodeToString(buf), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Production returns the active opener's input counter, if it generates output.
|
|
||||||
func (r *Registry) Production(raw string) production.Source {
|
|
||||||
if r == nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
src, err := r.get(raw)
|
|
||||||
if err != nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
src.mu.Lock()
|
|
||||||
reader := src.producer
|
|
||||||
src.mu.Unlock()
|
|
||||||
if reader == nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
return reader.Production()
|
|
||||||
}
|
|
||||||
@@ -1,898 +0,0 @@
|
|||||||
package blob
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"io"
|
|
||||||
"net/http"
|
|
||||||
"strings"
|
|
||||||
"sync"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
type terminalErrorReader struct {
|
|
||||||
err error
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *terminalErrorReader) Read([]byte) (int, error) {
|
|
||||||
return 0, r.err
|
|
||||||
}
|
|
||||||
|
|
||||||
type contextErrorReader struct {
|
|
||||||
ctx context.Context
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *contextErrorReader) Read([]byte) (int, error) {
|
|
||||||
<-r.ctx.Done()
|
|
||||||
return 0, r.ctx.Err()
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *contextErrorReader) Close() error { return nil }
|
|
||||||
|
|
||||||
type failingResponseWriter struct {
|
|
||||||
header http.Header
|
|
||||||
err error
|
|
||||||
}
|
|
||||||
|
|
||||||
func (w *failingResponseWriter) Header() http.Header {
|
|
||||||
return w.header
|
|
||||||
}
|
|
||||||
|
|
||||||
func (w *failingResponseWriter) Write([]byte) (int, error) {
|
|
||||||
return 0, w.err
|
|
||||||
}
|
|
||||||
|
|
||||||
func (w *failingResponseWriter) WriteHeader(int) {}
|
|
||||||
|
|
||||||
type countingSession struct {
|
|
||||||
mu sync.Mutex
|
|
||||||
refs int
|
|
||||||
zero chan struct{}
|
|
||||||
zeroOnce sync.Once
|
|
||||||
}
|
|
||||||
|
|
||||||
func newCountingSession() *countingSession {
|
|
||||||
return &countingSession{zero: make(chan struct{})}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *countingSession) Retain() {
|
|
||||||
s.mu.Lock()
|
|
||||||
s.refs++
|
|
||||||
s.mu.Unlock()
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *countingSession) Release() {
|
|
||||||
s.mu.Lock()
|
|
||||||
s.refs--
|
|
||||||
refs := s.refs
|
|
||||||
s.mu.Unlock()
|
|
||||||
if refs == 0 {
|
|
||||||
s.zeroOnce.Do(func() { close(s.zero) })
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *countingSession) RefCount() int {
|
|
||||||
s.mu.Lock()
|
|
||||||
defer s.mu.Unlock()
|
|
||||||
return s.refs
|
|
||||||
}
|
|
||||||
|
|
||||||
func setUnclaimedSourceTTLForTest(t *testing.T, ttl time.Duration) {
|
|
||||||
t.Helper()
|
|
||||||
previous := unclaimedSourceTTL
|
|
||||||
unclaimedSourceTTL = ttl
|
|
||||||
t.Cleanup(func() {
|
|
||||||
unclaimedSourceTTL = previous
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRegistryBlobHTTPGetAndRange(t *testing.T) {
|
|
||||||
registry := NewRegistry(t.TempDir())
|
|
||||||
defer registry.Close()
|
|
||||||
|
|
||||||
url, err := registry.CreateBlob([]byte("hello world"), "text/plain")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if !registry.IsURL(url) {
|
|
||||||
t.Fatal("expected registry to recognize created url")
|
|
||||||
}
|
|
||||||
|
|
||||||
resp, err := http.Get(url)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
body, err := io.ReadAll(resp.Body)
|
|
||||||
_ = resp.Body.Close()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if resp.StatusCode != http.StatusOK {
|
|
||||||
t.Fatalf("unexpected GET status: %d", resp.StatusCode)
|
|
||||||
}
|
|
||||||
if got := resp.Header.Get("Content-Length"); got != "11" {
|
|
||||||
t.Fatalf("unexpected content length: %q", got)
|
|
||||||
}
|
|
||||||
if got := resp.Header.Get("Accept-Ranges"); got != "bytes" {
|
|
||||||
t.Fatalf("unexpected accept ranges: %q", got)
|
|
||||||
}
|
|
||||||
if got := resp.Header.Get("Content-Type"); !strings.HasPrefix(got, "text/plain") {
|
|
||||||
t.Fatalf("unexpected content type: %q", got)
|
|
||||||
}
|
|
||||||
if string(body) != "hello world" {
|
|
||||||
t.Fatalf("unexpected GET body: %q", string(body))
|
|
||||||
}
|
|
||||||
|
|
||||||
req, err := http.NewRequest(http.MethodGet, url, nil)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
req.Header.Set("Range", "bytes=6-10")
|
|
||||||
resp, err = http.DefaultClient.Do(req)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
body, err = io.ReadAll(resp.Body)
|
|
||||||
_ = resp.Body.Close()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if resp.StatusCode != http.StatusPartialContent {
|
|
||||||
t.Fatalf("unexpected range status: %d", resp.StatusCode)
|
|
||||||
}
|
|
||||||
if got := resp.Header.Get("Content-Range"); got != "bytes 6-10/11" {
|
|
||||||
t.Fatalf("unexpected content range: %q", got)
|
|
||||||
}
|
|
||||||
if string(body) != "world" {
|
|
||||||
t.Fatalf("unexpected range body: %q", string(body))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRegistryEmptyBlob(t *testing.T) {
|
|
||||||
registry := NewRegistry(t.TempDir())
|
|
||||||
defer registry.Close()
|
|
||||||
|
|
||||||
url, err := registry.CreateBlob(nil, "application/octet-stream")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
meta, err := registry.Metadata(url)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if meta.Size != 0 || meta.Range {
|
|
||||||
t.Fatalf("unexpected empty blob metadata: %#v", meta)
|
|
||||||
}
|
|
||||||
|
|
||||||
resp, err := http.Get(url)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
body, readErr := io.ReadAll(resp.Body)
|
|
||||||
_ = resp.Body.Close()
|
|
||||||
if readErr != nil {
|
|
||||||
t.Fatal(readErr)
|
|
||||||
}
|
|
||||||
if resp.StatusCode != http.StatusOK {
|
|
||||||
t.Fatalf("unexpected status: %d", resp.StatusCode)
|
|
||||||
}
|
|
||||||
if len(body) != 0 {
|
|
||||||
t.Fatalf("unexpected body: %q", body)
|
|
||||||
}
|
|
||||||
if got := resp.Header.Get("Content-Length"); got != "0" {
|
|
||||||
t.Fatalf("unexpected content length: %q", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRegistryOpenerSizeWithoutRange(t *testing.T) {
|
|
||||||
registry := NewRegistry(t.TempDir())
|
|
||||||
defer registry.Close()
|
|
||||||
|
|
||||||
payload := []byte("hello opener")
|
|
||||||
url, err := registry.CreateOpener(func(ctx context.Context, req OpenRequest) (io.ReadCloser, error) {
|
|
||||||
if req.Offset != 0 || req.End != -1 {
|
|
||||||
t.Fatalf("unexpected non-range request: %#v", req)
|
|
||||||
}
|
|
||||||
return io.NopCloser(bytes.NewReader(payload)), nil
|
|
||||||
}, &CreateOptions{
|
|
||||||
ContentType: "text/plain",
|
|
||||||
Size: int64(len(payload)),
|
|
||||||
Range: false,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
req, err := http.NewRequest(http.MethodGet, url, nil)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
req.Header.Set("Range", "bytes=6-11")
|
|
||||||
resp, err := http.DefaultClient.Do(req)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
body, err := io.ReadAll(resp.Body)
|
|
||||||
_ = resp.Body.Close()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if resp.StatusCode != http.StatusOK {
|
|
||||||
t.Fatalf("unexpected status: %d", resp.StatusCode)
|
|
||||||
}
|
|
||||||
if got := resp.Header.Get("Content-Length"); got != "12" {
|
|
||||||
t.Fatalf("unexpected content length: %q", got)
|
|
||||||
}
|
|
||||||
if got := resp.Header.Get("Accept-Ranges"); got != "" {
|
|
||||||
t.Fatalf("unexpected accept ranges: %q", got)
|
|
||||||
}
|
|
||||||
if string(body) != string(payload) {
|
|
||||||
t.Fatalf("unexpected body: %q", string(body))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRegistryOpenerRangeConcurrentRequests(t *testing.T) {
|
|
||||||
registry := NewRegistry(t.TempDir())
|
|
||||||
defer registry.Close()
|
|
||||||
|
|
||||||
payload := []byte("abcdefghijklmnopqrstuvwxyz")
|
|
||||||
var mu sync.Mutex
|
|
||||||
var calls []OpenRequest
|
|
||||||
url, err := registry.CreateOpener(func(ctx context.Context, req OpenRequest) (io.ReadCloser, error) {
|
|
||||||
mu.Lock()
|
|
||||||
calls = append(calls, req)
|
|
||||||
mu.Unlock()
|
|
||||||
|
|
||||||
start := int(req.Offset)
|
|
||||||
end := len(payload)
|
|
||||||
if req.End >= 0 {
|
|
||||||
end = int(req.End) + 1
|
|
||||||
}
|
|
||||||
if start < 0 || start > end || end > len(payload) {
|
|
||||||
return nil, errors.New("bad test range")
|
|
||||||
}
|
|
||||||
return io.NopCloser(bytes.NewReader(payload[start:end])), nil
|
|
||||||
}, &CreateOptions{
|
|
||||||
Size: int64(len(payload)),
|
|
||||||
Range: true,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
ranges := []struct {
|
|
||||||
header string
|
|
||||||
want string
|
|
||||||
}{
|
|
||||||
{header: "bytes=0-4", want: "abcde"},
|
|
||||||
{header: "bytes=10-15", want: "klmnop"},
|
|
||||||
{header: "bytes=20-25", want: "uvwxyz"},
|
|
||||||
}
|
|
||||||
var wg sync.WaitGroup
|
|
||||||
for _, item := range ranges {
|
|
||||||
item := item
|
|
||||||
wg.Add(1)
|
|
||||||
go func() {
|
|
||||||
defer wg.Done()
|
|
||||||
req, err := http.NewRequest(http.MethodGet, url, nil)
|
|
||||||
if err != nil {
|
|
||||||
t.Error(err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
req.Header.Set("Range", item.header)
|
|
||||||
resp, err := http.DefaultClient.Do(req)
|
|
||||||
if err != nil {
|
|
||||||
t.Error(err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
body, err := io.ReadAll(resp.Body)
|
|
||||||
_ = resp.Body.Close()
|
|
||||||
if err != nil {
|
|
||||||
t.Error(err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if resp.StatusCode != http.StatusPartialContent {
|
|
||||||
t.Errorf("unexpected status for %s: %d", item.header, resp.StatusCode)
|
|
||||||
}
|
|
||||||
if string(body) != item.want {
|
|
||||||
t.Errorf("unexpected body for %s: %q", item.header, string(body))
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
}
|
|
||||||
wg.Wait()
|
|
||||||
|
|
||||||
mu.Lock()
|
|
||||||
defer mu.Unlock()
|
|
||||||
if len(calls) != len(ranges) {
|
|
||||||
t.Fatalf("unexpected opener call count: %d", len(calls))
|
|
||||||
}
|
|
||||||
seen := map[OpenRequest]bool{}
|
|
||||||
for _, call := range calls {
|
|
||||||
seen[call] = true
|
|
||||||
}
|
|
||||||
for _, want := range []OpenRequest{{Offset: 0, End: 4}, {Offset: 10, End: 15}, {Offset: 20, End: 25}} {
|
|
||||||
if !seen[want] {
|
|
||||||
t.Fatalf("missing opener request %#v, got %#v", want, calls)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRegistryRejectsRangeWithoutSize(t *testing.T) {
|
|
||||||
registry := NewRegistry(t.TempDir())
|
|
||||||
defer registry.Close()
|
|
||||||
|
|
||||||
_, err := registry.CreateOpener(func(ctx context.Context, req OpenRequest) (io.ReadCloser, error) {
|
|
||||||
return io.NopCloser(strings.NewReader("")), nil
|
|
||||||
}, &CreateOptions{Range: true})
|
|
||||||
if !errors.Is(err, ErrInvalidOptions) {
|
|
||||||
t.Fatalf("expected invalid options error, got %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRegistryTracksSourceErrors(t *testing.T) {
|
|
||||||
t.Run("reader error", func(t *testing.T) {
|
|
||||||
registry := NewRegistry(t.TempDir())
|
|
||||||
defer registry.Close()
|
|
||||||
|
|
||||||
boom := errors.New("reader failed")
|
|
||||||
url, err := registry.CreateOpener(func(context.Context, OpenRequest) (io.ReadCloser, error) {
|
|
||||||
return io.NopCloser(io.MultiReader(strings.NewReader("partial"), &terminalErrorReader{err: boom})), nil
|
|
||||||
}, &CreateOptions{})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
resp, err := http.Get(url)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
body, readErr := io.ReadAll(resp.Body)
|
|
||||||
_ = resp.Body.Close()
|
|
||||||
if readErr != nil {
|
|
||||||
t.Fatalf("the HTTP transport should end normally, got %v", readErr)
|
|
||||||
}
|
|
||||||
if string(body) != "partial" {
|
|
||||||
t.Fatalf("unexpected partial body: %q", body)
|
|
||||||
}
|
|
||||||
if got := registry.SourceError(url); !errors.Is(got, boom) {
|
|
||||||
t.Fatalf("expected source error %v, got %v", boom, got)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("short known-size reader", func(t *testing.T) {
|
|
||||||
registry := NewRegistry(t.TempDir())
|
|
||||||
defer registry.Close()
|
|
||||||
|
|
||||||
url, err := registry.CreateOpener(func(context.Context, OpenRequest) (io.ReadCloser, error) {
|
|
||||||
return io.NopCloser(strings.NewReader("short")), nil
|
|
||||||
}, &CreateOptions{Size: 10})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
resp, err := http.Get(url)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
_, _ = io.ReadAll(resp.Body)
|
|
||||||
_ = resp.Body.Close()
|
|
||||||
if got := registry.SourceError(url); !errors.Is(got, io.ErrUnexpectedEOF) {
|
|
||||||
t.Fatalf("expected unexpected EOF, got %v", got)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("range reader error becomes terminal", func(t *testing.T) {
|
|
||||||
registry := NewRegistry(t.TempDir())
|
|
||||||
defer registry.Close()
|
|
||||||
|
|
||||||
boom := errors.New("range reader failed")
|
|
||||||
url, err := registry.CreateOpener(func(context.Context, OpenRequest) (io.ReadCloser, error) {
|
|
||||||
return io.NopCloser(io.MultiReader(strings.NewReader("partial"), &terminalErrorReader{err: boom})), nil
|
|
||||||
}, &CreateOptions{Size: 10, Range: true})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
for attempt := 0; attempt < rangeSourceFailureLimit; attempt++ {
|
|
||||||
req, err := http.NewRequest(http.MethodGet, url, nil)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
req.Header.Set("Range", "bytes=0-9")
|
|
||||||
resp, err := http.DefaultClient.Do(req)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
_, _ = io.ReadAll(resp.Body)
|
|
||||||
_ = resp.Body.Close()
|
|
||||||
}
|
|
||||||
if got := registry.SourceError(url); !errors.Is(got, boom) {
|
|
||||||
t.Fatalf("expected range source error %v, got %v", boom, got)
|
|
||||||
}
|
|
||||||
|
|
||||||
resp, err := http.Get(url)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
_ = resp.Body.Close()
|
|
||||||
if resp.StatusCode != http.StatusGone {
|
|
||||||
t.Fatalf("expected terminal source status %d, got %d", http.StatusGone, resp.StatusCode)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("short range reader", func(t *testing.T) {
|
|
||||||
registry := NewRegistry(t.TempDir())
|
|
||||||
defer registry.Close()
|
|
||||||
|
|
||||||
url, err := registry.CreateOpener(func(context.Context, OpenRequest) (io.ReadCloser, error) {
|
|
||||||
return io.NopCloser(strings.NewReader("short")), nil
|
|
||||||
}, &CreateOptions{Size: 10, Range: true})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
for attempt := 0; attempt < rangeSourceFailureLimit; attempt++ {
|
|
||||||
req, err := http.NewRequest(http.MethodGet, url, nil)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
req.Header.Set("Range", "bytes=0-9")
|
|
||||||
resp, err := http.DefaultClient.Do(req)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
_, _ = io.ReadAll(resp.Body)
|
|
||||||
_ = resp.Body.Close()
|
|
||||||
}
|
|
||||||
if got := registry.SourceError(url); !errors.Is(got, io.ErrUnexpectedEOF) {
|
|
||||||
t.Fatalf("expected range unexpected EOF, got %v", got)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("successful range retry clears transient failure", func(t *testing.T) {
|
|
||||||
registry := NewRegistry(t.TempDir())
|
|
||||||
defer registry.Close()
|
|
||||||
|
|
||||||
attempt := 0
|
|
||||||
boom := errors.New("transient range failure")
|
|
||||||
url, err := registry.CreateOpener(func(context.Context, OpenRequest) (io.ReadCloser, error) {
|
|
||||||
attempt++
|
|
||||||
if attempt == 1 {
|
|
||||||
return io.NopCloser(&terminalErrorReader{err: boom}), nil
|
|
||||||
}
|
|
||||||
return io.NopCloser(strings.NewReader("0123456789")), nil
|
|
||||||
}, &CreateOptions{Size: 10, Range: true})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
for i := 0; i < 2; i++ {
|
|
||||||
req, err := http.NewRequest(http.MethodGet, url, nil)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
req.Header.Set("Range", "bytes=0-9")
|
|
||||||
resp, err := http.DefaultClient.Do(req)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
_, _ = io.ReadAll(resp.Body)
|
|
||||||
_ = resp.Body.Close()
|
|
||||||
}
|
|
||||||
if got := registry.SourceError(url); got != nil {
|
|
||||||
t.Fatalf("successful retry left transient source error: %v", got)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("request cancellation is not a source error", func(t *testing.T) {
|
|
||||||
registry := NewRegistry(t.TempDir())
|
|
||||||
defer registry.Close()
|
|
||||||
|
|
||||||
opened := make(chan struct{})
|
|
||||||
url, err := registry.CreateOpener(func(ctx context.Context, _ OpenRequest) (io.ReadCloser, error) {
|
|
||||||
close(opened)
|
|
||||||
return &contextErrorReader{ctx: ctx}, nil
|
|
||||||
}, &CreateOptions{})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
|
||||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
done := make(chan struct{})
|
|
||||||
go func() {
|
|
||||||
resp, _ := http.DefaultClient.Do(req)
|
|
||||||
if resp != nil {
|
|
||||||
_ = resp.Body.Close()
|
|
||||||
}
|
|
||||||
close(done)
|
|
||||||
}()
|
|
||||||
<-opened
|
|
||||||
cancel()
|
|
||||||
select {
|
|
||||||
case <-done:
|
|
||||||
case <-time.After(2 * time.Second):
|
|
||||||
t.Fatal("cancelled request did not finish")
|
|
||||||
}
|
|
||||||
if got := registry.SourceError(url); got != nil {
|
|
||||||
t.Fatalf("request cancellation was recorded as source error: %v", got)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("nil reader", func(t *testing.T) {
|
|
||||||
registry := NewRegistry(t.TempDir())
|
|
||||||
defer registry.Close()
|
|
||||||
|
|
||||||
url, err := registry.CreateOpener(func(context.Context, OpenRequest) (io.ReadCloser, error) {
|
|
||||||
return nil, nil
|
|
||||||
}, &CreateOptions{})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
resp, err := http.Get(url)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
_ = resp.Body.Close()
|
|
||||||
if resp.StatusCode != http.StatusGone {
|
|
||||||
t.Fatalf("expected nil reader status %d, got %d", http.StatusGone, resp.StatusCode)
|
|
||||||
}
|
|
||||||
if got := registry.SourceError(url); !errors.Is(got, ErrSourceClosed) {
|
|
||||||
t.Fatalf("expected closed source error, got %v", got)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRegistryDoesNotRecordResponseWriteError(t *testing.T) {
|
|
||||||
registry := NewRegistry(t.TempDir())
|
|
||||||
defer registry.Close()
|
|
||||||
|
|
||||||
url, err := registry.CreateOpener(func(context.Context, OpenRequest) (io.ReadCloser, error) {
|
|
||||||
return io.NopCloser(strings.NewReader("payload")), nil
|
|
||||||
}, &CreateOptions{Size: 7})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
req, err := http.NewRequest(http.MethodGet, url, nil)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
writerErr := errors.New("client connection closed")
|
|
||||||
registry.ServeHTTP(&failingResponseWriter{header: make(http.Header), err: writerErr}, req)
|
|
||||||
if got := registry.SourceError(url); got != nil {
|
|
||||||
t.Fatalf("response write error was recorded as a source error: %v", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRegistryTaskReferencesShareSource(t *testing.T) {
|
|
||||||
registry := NewRegistry(t.TempDir())
|
|
||||||
defer registry.Close()
|
|
||||||
|
|
||||||
url, err := registry.CreateBlob([]byte("shared"), "text/plain")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := registry.Acquire(url); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := registry.Acquire(url); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := registry.Release(url); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if !registry.IsURL(url) {
|
|
||||||
t.Fatal("the first task release revoked a source still used by another task")
|
|
||||||
}
|
|
||||||
|
|
||||||
resp, err := http.Get(url)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
body, readErr := io.ReadAll(resp.Body)
|
|
||||||
_ = resp.Body.Close()
|
|
||||||
if readErr != nil || string(body) != "shared" {
|
|
||||||
t.Fatalf("unexpected shared response body %q, error %v", body, readErr)
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := registry.Release(url); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if registry.IsURL(url) {
|
|
||||||
t.Fatal("the final task release did not revoke the source")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRegistryUnclaimedSessionSourceExpires(t *testing.T) {
|
|
||||||
setUnclaimedSourceTTLForTest(t, 25*time.Millisecond)
|
|
||||||
registry := NewRegistry(t.TempDir())
|
|
||||||
defer registry.Close()
|
|
||||||
|
|
||||||
session := newCountingSession()
|
|
||||||
url, err := registry.CreateOpener(func(context.Context, OpenRequest) (io.ReadCloser, error) {
|
|
||||||
return io.NopCloser(strings.NewReader("unused")), nil
|
|
||||||
}, &CreateOptions{Size: 6, Session: session})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if got := session.RefCount(); got != 1 {
|
|
||||||
t.Fatalf("unexpected initial session ref count: %d", got)
|
|
||||||
}
|
|
||||||
|
|
||||||
select {
|
|
||||||
case <-session.zero:
|
|
||||||
case <-time.After(2 * time.Second):
|
|
||||||
t.Fatal("unclaimed source did not expire")
|
|
||||||
}
|
|
||||||
if registry.IsURL(url) {
|
|
||||||
t.Fatal("expired source is still registered")
|
|
||||||
}
|
|
||||||
if got := session.RefCount(); got != 0 {
|
|
||||||
t.Fatalf("expired source retained its session: %d", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRegistryAcquireStopsUnclaimedSourceExpiration(t *testing.T) {
|
|
||||||
const ttl = 25 * time.Millisecond
|
|
||||||
setUnclaimedSourceTTLForTest(t, ttl)
|
|
||||||
registry := NewRegistry(t.TempDir())
|
|
||||||
defer registry.Close()
|
|
||||||
|
|
||||||
session := newCountingSession()
|
|
||||||
url, err := registry.CreateOpener(func(context.Context, OpenRequest) (io.ReadCloser, error) {
|
|
||||||
return io.NopCloser(strings.NewReader("claimed")), nil
|
|
||||||
}, &CreateOptions{Size: 7, Session: session})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := registry.Acquire(url); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
select {
|
|
||||||
case <-session.zero:
|
|
||||||
t.Fatal("claimed source expired")
|
|
||||||
case <-time.After(4 * ttl):
|
|
||||||
}
|
|
||||||
if !registry.IsURL(url) {
|
|
||||||
t.Fatal("claimed source was removed")
|
|
||||||
}
|
|
||||||
if err := registry.Release(url); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
select {
|
|
||||||
case <-session.zero:
|
|
||||||
case <-time.After(2 * time.Second):
|
|
||||||
t.Fatal("final task release did not release the session")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRegistryUnclaimedExpirationKeepsActiveReaderSession(t *testing.T) {
|
|
||||||
setUnclaimedSourceTTLForTest(t, 100*time.Millisecond)
|
|
||||||
registry := NewRegistry(t.TempDir())
|
|
||||||
defer registry.Close()
|
|
||||||
|
|
||||||
session := newCountingSession()
|
|
||||||
reader, writer := io.Pipe()
|
|
||||||
opened := make(chan struct{})
|
|
||||||
url, err := registry.CreateOpener(func(context.Context, OpenRequest) (io.ReadCloser, error) {
|
|
||||||
close(opened)
|
|
||||||
return reader, nil
|
|
||||||
}, &CreateOptions{Size: 7, Session: session})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
type result struct {
|
|
||||||
body []byte
|
|
||||||
err error
|
|
||||||
}
|
|
||||||
resultCh := make(chan result, 1)
|
|
||||||
go func() {
|
|
||||||
resp, err := http.Get(url)
|
|
||||||
if err != nil {
|
|
||||||
resultCh <- result{err: err}
|
|
||||||
return
|
|
||||||
}
|
|
||||||
body, err := io.ReadAll(resp.Body)
|
|
||||||
_ = resp.Body.Close()
|
|
||||||
resultCh <- result{body: body, err: err}
|
|
||||||
}()
|
|
||||||
|
|
||||||
select {
|
|
||||||
case <-opened:
|
|
||||||
case got := <-resultCh:
|
|
||||||
t.Fatalf("request ended before the opener ran: body %q, error %v", got.body, got.err)
|
|
||||||
case <-time.After(2 * time.Second):
|
|
||||||
t.Fatal("active request did not open")
|
|
||||||
}
|
|
||||||
deadline := time.NewTimer(2 * time.Second)
|
|
||||||
ticker := time.NewTicker(5 * time.Millisecond)
|
|
||||||
defer deadline.Stop()
|
|
||||||
defer ticker.Stop()
|
|
||||||
for registry.IsURL(url) {
|
|
||||||
select {
|
|
||||||
case <-ticker.C:
|
|
||||||
case <-deadline.C:
|
|
||||||
t.Fatal("unclaimed source did not expire while its reader was active")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if got := session.RefCount(); got != 1 {
|
|
||||||
t.Fatalf("expiration released the active-reader session ref: %d", got)
|
|
||||||
}
|
|
||||||
if _, err := writer.Write([]byte("payload")); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
_ = writer.Close()
|
|
||||||
|
|
||||||
select {
|
|
||||||
case got := <-resultCh:
|
|
||||||
if got.err != nil || string(got.body) != "payload" {
|
|
||||||
t.Fatalf("active response was interrupted: body %q, error %v", got.body, got.err)
|
|
||||||
}
|
|
||||||
case <-time.After(2 * time.Second):
|
|
||||||
t.Fatal("active response did not finish after source expiration")
|
|
||||||
}
|
|
||||||
select {
|
|
||||||
case <-session.zero:
|
|
||||||
if got := session.RefCount(); got != 0 {
|
|
||||||
t.Fatalf("unexpected final session ref count: %d", got)
|
|
||||||
}
|
|
||||||
case <-time.After(2 * time.Second):
|
|
||||||
t.Fatal("active-reader session ref was not released")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRegistryCloseCancelsUnclaimedExpiration(t *testing.T) {
|
|
||||||
const ttl = 25 * time.Millisecond
|
|
||||||
setUnclaimedSourceTTLForTest(t, ttl)
|
|
||||||
registry := NewRegistry(t.TempDir())
|
|
||||||
|
|
||||||
session := newCountingSession()
|
|
||||||
_, err := registry.CreateOpener(func(context.Context, OpenRequest) (io.ReadCloser, error) {
|
|
||||||
return io.NopCloser(strings.NewReader("unused")), nil
|
|
||||||
}, &CreateOptions{Size: 6, Session: session})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := registry.Close(); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
select {
|
|
||||||
case <-session.zero:
|
|
||||||
case <-time.After(2 * time.Second):
|
|
||||||
t.Fatal("registry close did not release the session")
|
|
||||||
}
|
|
||||||
<-time.After(4 * ttl)
|
|
||||||
if got := session.RefCount(); got != 0 {
|
|
||||||
t.Fatalf("expiration released the session again after close: %d", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRegistryRevokeKeepsActiveReaderSession(t *testing.T) {
|
|
||||||
registry := NewRegistry(t.TempDir())
|
|
||||||
defer registry.Close()
|
|
||||||
|
|
||||||
session := newCountingSession()
|
|
||||||
reader, writer := io.Pipe()
|
|
||||||
opened := make(chan struct{})
|
|
||||||
url, err := registry.CreateOpener(func(context.Context, OpenRequest) (io.ReadCloser, error) {
|
|
||||||
close(opened)
|
|
||||||
return reader, nil
|
|
||||||
}, &CreateOptions{Size: 7, Session: session})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
type result struct {
|
|
||||||
body []byte
|
|
||||||
err error
|
|
||||||
}
|
|
||||||
resultCh := make(chan result, 1)
|
|
||||||
go func() {
|
|
||||||
resp, err := http.Get(url)
|
|
||||||
if err != nil {
|
|
||||||
resultCh <- result{err: err}
|
|
||||||
return
|
|
||||||
}
|
|
||||||
body, err := io.ReadAll(resp.Body)
|
|
||||||
_ = resp.Body.Close()
|
|
||||||
resultCh <- result{body: body, err: err}
|
|
||||||
}()
|
|
||||||
|
|
||||||
<-opened
|
|
||||||
if got := session.RefCount(); got != 2 {
|
|
||||||
t.Fatalf("expected source and active-reader session refs, got %d", got)
|
|
||||||
}
|
|
||||||
if err := registry.Revoke(url); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if got := session.RefCount(); got != 1 {
|
|
||||||
t.Fatalf("revoke released the active-reader session ref: %d", got)
|
|
||||||
}
|
|
||||||
if _, err := writer.Write([]byte("payload")); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
_ = writer.Close()
|
|
||||||
|
|
||||||
select {
|
|
||||||
case got := <-resultCh:
|
|
||||||
if got.err != nil || string(got.body) != "payload" {
|
|
||||||
t.Fatalf("active response was interrupted: body %q, error %v", got.body, got.err)
|
|
||||||
}
|
|
||||||
case <-time.After(2 * time.Second):
|
|
||||||
t.Fatal("active response did not finish after revoke")
|
|
||||||
}
|
|
||||||
select {
|
|
||||||
case <-session.zero:
|
|
||||||
if got := session.RefCount(); got != 0 {
|
|
||||||
t.Fatalf("unexpected final session ref count: %d", got)
|
|
||||||
}
|
|
||||||
case <-time.After(2 * time.Second):
|
|
||||||
t.Fatal("active-reader session ref was not released")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRegistryHTTPErrorStatuses(t *testing.T) {
|
|
||||||
registry := NewRegistry(t.TempDir())
|
|
||||||
defer registry.Close()
|
|
||||||
|
|
||||||
url, err := registry.CreateBlob([]byte("secret"), "text/plain")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
badURL := url + "-missing"
|
|
||||||
resp, err := http.Get(badURL)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
_ = resp.Body.Close()
|
|
||||||
if resp.StatusCode != http.StatusNotFound {
|
|
||||||
t.Fatalf("unexpected missing source status: %d", resp.StatusCode)
|
|
||||||
}
|
|
||||||
|
|
||||||
req, err := http.NewRequest(http.MethodHead, url, nil)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
resp, err = http.DefaultClient.Do(req)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
_ = resp.Body.Close()
|
|
||||||
if resp.StatusCode != http.StatusMethodNotAllowed {
|
|
||||||
t.Fatalf("unexpected method status: %d", resp.StatusCode)
|
|
||||||
}
|
|
||||||
|
|
||||||
req, err = http.NewRequest(http.MethodGet, url, nil)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
req.Header.Set("Range", "bytes=99-100")
|
|
||||||
resp, err = http.DefaultClient.Do(req)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
_ = resp.Body.Close()
|
|
||||||
if resp.StatusCode != http.StatusRequestedRangeNotSatisfiable {
|
|
||||||
t.Fatalf("unexpected invalid range status: %d", resp.StatusCode)
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := registry.Revoke(url); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
resp, err = http.Get(url)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
_ = resp.Body.Close()
|
|
||||||
if resp.StatusCode != http.StatusNotFound {
|
|
||||||
t.Fatalf("unexpected revoked source status: %d", resp.StatusCode)
|
|
||||||
}
|
|
||||||
if registry.IsURL(url) {
|
|
||||||
t.Fatal("revoked source should not be usable")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,24 +1,14 @@
|
|||||||
package controller
|
package controller
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"github.com/GopeedLab/gopeed/internal/tempfiles"
|
|
||||||
"github.com/GopeedLab/gopeed/pkg/base"
|
"github.com/GopeedLab/gopeed/pkg/base"
|
||||||
"github.com/rs/zerolog"
|
|
||||||
"net/http"
|
|
||||||
"net/url"
|
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
)
|
)
|
||||||
|
|
||||||
type Controller struct {
|
type Controller struct {
|
||||||
TempFiles *tempfiles.Scope
|
GetConfig func(v any) bool
|
||||||
ManagedProduction func(string) bool
|
ProxyConfig *base.DownloaderProxyConfig
|
||||||
TempDir string
|
|
||||||
GetConfig func(v any)
|
|
||||||
GetProxy func(requestProxy *base.RequestProxy) func(*http.Request) (*url.URL, error)
|
|
||||||
// Logger is the downloader's core logger. It may be nil, in which case
|
|
||||||
// fetchers log nothing.
|
|
||||||
Logger *zerolog.Logger
|
|
||||||
FileController
|
FileController
|
||||||
//ContextDialer() (proxy.Dialer, error)
|
//ContextDialer() (proxy.Dialer, error)
|
||||||
}
|
}
|
||||||
@@ -32,11 +22,10 @@ type DefaultFileController struct {
|
|||||||
|
|
||||||
func NewController() *Controller {
|
func NewController() *Controller {
|
||||||
return &Controller{
|
return &Controller{
|
||||||
GetConfig: func(v any) {},
|
|
||||||
GetProxy: func(requestProxy *base.RequestProxy) func(*http.Request) (*url.URL, error) {
|
|
||||||
return requestProxy.ToHandler()
|
|
||||||
},
|
|
||||||
FileController: &DefaultFileController{},
|
FileController: &DefaultFileController{},
|
||||||
|
GetConfig: func(v any) bool {
|
||||||
|
return false
|
||||||
|
},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1,29 +1,28 @@
|
|||||||
package fetcher
|
package fetcher
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"path"
|
|
||||||
"strings"
|
|
||||||
|
|
||||||
"github.com/GopeedLab/gopeed/internal/controller"
|
"github.com/GopeedLab/gopeed/internal/controller"
|
||||||
"github.com/GopeedLab/gopeed/pkg/base"
|
"github.com/GopeedLab/gopeed/pkg/base"
|
||||||
|
"path"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Fetcher defines the interface for a download protocol.
|
// Fetcher defines the interface for a download protocol.
|
||||||
// Each download task will have a corresponding Fetcher instance for the management of the download task
|
// One fetcher for each download task
|
||||||
type Fetcher interface {
|
type Fetcher interface {
|
||||||
|
// Name return the name of the protocol.
|
||||||
|
Name() string
|
||||||
|
|
||||||
Setup(ctl *controller.Controller)
|
Setup(ctl *controller.Controller)
|
||||||
|
// Resolve resource info from request
|
||||||
Resolve(req *base.Request) error
|
Resolve(req *base.Request) error
|
||||||
|
// Create ready to download, but not started
|
||||||
|
Create(opts *base.Options) error
|
||||||
Start() error
|
Start() error
|
||||||
// Patch modifies task-specific data based on the protocol.
|
|
||||||
// For HTTP: can modify Request info (URL, headers, etc.)
|
|
||||||
// For BT: can modify SelectFiles (via opts.SelectFiles)
|
|
||||||
Patch(req *base.Request, opts *base.Options) error
|
|
||||||
Pause() error
|
Pause() error
|
||||||
Close() error
|
Close() error
|
||||||
|
|
||||||
// Stats returns both the durable protocol snapshot and transient runtime
|
// Stats refreshes health statistics and returns the latest information
|
||||||
// telemetry. Downloader persists Snapshot and keeps Runtime in memory only.
|
Stats() any
|
||||||
Stats() *Stats
|
|
||||||
// Meta returns the meta information of the download.
|
// Meta returns the meta information of the download.
|
||||||
Meta() *FetcherMeta
|
Meta() *FetcherMeta
|
||||||
// Progress returns the progress of the download.
|
// Progress returns the progress of the download.
|
||||||
@@ -32,20 +31,6 @@ type Fetcher interface {
|
|||||||
Wait() error
|
Wait() error
|
||||||
}
|
}
|
||||||
|
|
||||||
// Stats separates protocol information that remains meaningful after a task
|
|
||||||
// stops from telemetry that is valid only while the fetcher is active.
|
|
||||||
// Protocols own both payload schemas; Downloader treats them as opaque values.
|
|
||||||
type Stats struct {
|
|
||||||
Snapshot any `json:"snapshot"`
|
|
||||||
Runtime any `json:"runtime"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type Uploader interface {
|
|
||||||
Upload() error
|
|
||||||
UploadedBytes() int64
|
|
||||||
WaitUpload() error
|
|
||||||
}
|
|
||||||
|
|
||||||
// FetcherMeta defines the meta information of a fetcher.
|
// FetcherMeta defines the meta information of a fetcher.
|
||||||
type FetcherMeta struct {
|
type FetcherMeta struct {
|
||||||
Req *base.Request `json:"req"`
|
Req *base.Request `json:"req"`
|
||||||
@@ -83,78 +68,17 @@ func (m *FetcherMeta) RootDirPath() string {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
type FilterType int
|
// FetcherBuilder defines the interface for a fetcher builder.
|
||||||
|
type FetcherBuilder interface {
|
||||||
const (
|
// Schemes returns the schemes supported by the fetcher.
|
||||||
// FilterTypeUrl url type, pattern is the scheme, e.g. http://github.com -> http
|
Schemes() []string
|
||||||
FilterTypeUrl FilterType = iota
|
|
||||||
// FilterTypeFile file type, pattern is the file extension name, e.g. test.torrent -> torrent
|
|
||||||
FilterTypeFile
|
|
||||||
// FilterTypeBase64 base64 data type, pattern is the data mime type, e.g. data:application/x-bittorrent;base64 -> application/x-bittorrent
|
|
||||||
FilterTypeBase64
|
|
||||||
)
|
|
||||||
|
|
||||||
type SchemeFilter struct {
|
|
||||||
Type FilterType
|
|
||||||
Pattern string
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *SchemeFilter) Match(uri string) bool {
|
|
||||||
uriUpper := strings.ToUpper(uri)
|
|
||||||
patternUpper := strings.ToUpper(s.Pattern)
|
|
||||||
switch s.Type {
|
|
||||||
case FilterTypeUrl:
|
|
||||||
return strings.HasPrefix(uriUpper, patternUpper+":")
|
|
||||||
case FilterTypeFile:
|
|
||||||
// Strip query and fragment before matching so URLs like
|
|
||||||
// https://example.com/video.m3u8?sign=abc still route by extension.
|
|
||||||
clean := uri
|
|
||||||
if idx := strings.IndexAny(clean, "?#"); idx >= 0 {
|
|
||||||
clean = clean[:idx]
|
|
||||||
}
|
|
||||||
return strings.HasSuffix(strings.ToUpper(clean), "."+patternUpper)
|
|
||||||
case FilterTypeBase64:
|
|
||||||
return strings.HasPrefix(uriUpper, "DATA:"+patternUpper+";BASE64,")
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
// FetcherManager manage and control the fetcher
|
|
||||||
type FetcherManager interface {
|
|
||||||
// Name return the name of the protocol.
|
|
||||||
Name() string
|
|
||||||
// Filters registers the supported schemes.
|
|
||||||
Filters() []*SchemeFilter
|
|
||||||
// Build returns a new fetcher.
|
// Build returns a new fetcher.
|
||||||
Build() Fetcher
|
Build() Fetcher
|
||||||
// ParseName name displayed when the task is not yet resolved, parsed from the request URL
|
|
||||||
ParseName(u string) string
|
|
||||||
// AutoRename returns whether the fetcher need renaming the download file when has the same name file.
|
|
||||||
AutoRename() bool
|
|
||||||
|
|
||||||
// DefaultConfig returns the default configuration of the protocol.
|
|
||||||
DefaultConfig() any
|
|
||||||
// Store fetcher
|
// Store fetcher
|
||||||
Store(fetcher Fetcher) (any, error)
|
Store(fetcher Fetcher) (any, error)
|
||||||
// Restore fetcher
|
// Restore fetcher
|
||||||
Restore() (v any, f func(meta *FetcherMeta, v any) Fetcher)
|
Restore() (v any, f func(meta *FetcherMeta, v any) Fetcher)
|
||||||
// Close the fetcher manager, release resources.
|
|
||||||
Close() error
|
|
||||||
}
|
|
||||||
|
|
||||||
// StatefulFetcherManager is an optional extension for protocols that keep
|
|
||||||
// shared client state outside individual task fetchers.
|
|
||||||
type StatefulFetcherManager interface {
|
|
||||||
SetStateStore(store ProtocolStateStore)
|
|
||||||
}
|
|
||||||
|
|
||||||
// ProtocolStateStore persists shared protocol state for a fetcher manager.
|
|
||||||
// Downloader provides the concrete storage backend, while the protocol decides
|
|
||||||
// when state should be loaded or flushed.
|
|
||||||
type ProtocolStateStore interface {
|
|
||||||
Load(v any) (bool, error)
|
|
||||||
Save(v any) error
|
|
||||||
Delete() error
|
|
||||||
}
|
}
|
||||||
|
|
||||||
type DefaultFetcher struct {
|
type DefaultFetcher struct {
|
||||||
|
|||||||
@@ -1,114 +0,0 @@
|
|||||||
package fetcher
|
|
||||||
|
|
||||||
import (
|
|
||||||
"encoding/json"
|
|
||||||
"testing"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestStatsMarshalRawSnapshotWithoutRuntime(t *testing.T) {
|
|
||||||
stats := &Stats{
|
|
||||||
Snapshot: json.RawMessage(`{"value":1}`),
|
|
||||||
}
|
|
||||||
|
|
||||||
data, err := json.Marshal(stats)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Marshal() error = %v", err)
|
|
||||||
}
|
|
||||||
if got, want := string(data), `{"snapshot":{"value":1},"runtime":null}`; got != want {
|
|
||||||
t.Fatalf("Marshal() = %s, want %s", got, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestSchemeFilter_Match(t *testing.T) {
|
|
||||||
type fields struct {
|
|
||||||
Type FilterType
|
|
||||||
Pattern string
|
|
||||||
}
|
|
||||||
type args struct {
|
|
||||||
uri string
|
|
||||||
}
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
fields fields
|
|
||||||
args args
|
|
||||||
want bool
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
name: "url match",
|
|
||||||
fields: fields{
|
|
||||||
Type: FilterTypeUrl,
|
|
||||||
Pattern: "https",
|
|
||||||
},
|
|
||||||
args: args{
|
|
||||||
uri: "https://github.com",
|
|
||||||
},
|
|
||||||
want: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "url not match",
|
|
||||||
fields: fields{
|
|
||||||
Type: FilterTypeUrl,
|
|
||||||
Pattern: "https",
|
|
||||||
},
|
|
||||||
args: args{
|
|
||||||
uri: "ftp://github.com",
|
|
||||||
},
|
|
||||||
want: false,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "file match",
|
|
||||||
fields: fields{
|
|
||||||
Type: FilterTypeFile,
|
|
||||||
Pattern: "torrent",
|
|
||||||
},
|
|
||||||
args: args{
|
|
||||||
uri: "d:/temp/test.torrent",
|
|
||||||
},
|
|
||||||
want: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "file not match",
|
|
||||||
fields: fields{
|
|
||||||
Type: FilterTypeFile,
|
|
||||||
Pattern: "torrent",
|
|
||||||
},
|
|
||||||
args: args{
|
|
||||||
uri: "d:/temp/test.txt",
|
|
||||||
},
|
|
||||||
want: false,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "base64 match",
|
|
||||||
fields: fields{
|
|
||||||
Type: FilterTypeBase64,
|
|
||||||
Pattern: "application/x-bittorrent",
|
|
||||||
},
|
|
||||||
args: args{
|
|
||||||
uri: "data:application/x-bittorrent;base64,xxx",
|
|
||||||
},
|
|
||||||
want: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "base64 not match",
|
|
||||||
fields: fields{
|
|
||||||
Type: FilterTypeBase64,
|
|
||||||
Pattern: "application/x-bittorrent",
|
|
||||||
},
|
|
||||||
args: args{
|
|
||||||
uri: "data:application/javascript;base64,xxx",
|
|
||||||
},
|
|
||||||
want: false,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
for _, tt := range tests {
|
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
|
||||||
s := &SchemeFilter{
|
|
||||||
Type: tt.fields.Type,
|
|
||||||
Pattern: tt.fields.Pattern,
|
|
||||||
}
|
|
||||||
if got := s.Match(tt.args.uri); got != tt.want {
|
|
||||||
t.Errorf("Match() = %v, want %v", got, tt.want)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,72 +0,0 @@
|
|||||||
package fetcher
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
type TriggerSource string
|
|
||||||
|
|
||||||
const (
|
|
||||||
TriggerScheduled TriggerSource = "scheduled"
|
|
||||||
TriggerManual TriggerSource = "manual"
|
|
||||||
)
|
|
||||||
|
|
||||||
var ErrJobSkipped = errors.New("protocol job skipped")
|
|
||||||
var ErrConfigChanged = errors.New("protocol configuration changed during execution")
|
|
||||||
|
|
||||||
// ConfigState embeds the common backend-owned state section in a protocol config.
|
|
||||||
// Public config PUTs always retain this section from the latest stored config.
|
|
||||||
type ConfigState[T any] struct {
|
|
||||||
State T `json:"state"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// ProtocolHost is scoped to one manager. UpdateState runs against the latest
|
|
||||||
// config under Downloader's lock; callbacks must not perform network I/O.
|
|
||||||
type ProtocolHost struct {
|
|
||||||
Read func(any) error
|
|
||||||
UpdateState func(func(config, state json.RawMessage) (any, error)) error
|
|
||||||
Fetch func(context.Context, string, int64, func([]byte) error) ([]byte, error)
|
|
||||||
}
|
|
||||||
|
|
||||||
func ReadConfig[T any](host ProtocolHost) (*T, error) {
|
|
||||||
var cfg T
|
|
||||||
if err := host.Read(&cfg); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return &cfg, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// UpdateState gives the callback a detached config snapshot and mutable state.
|
|
||||||
// Only the state is committed; edits to the config snapshot are discarded.
|
|
||||||
func UpdateState[C, S any](host ProtocolHost, update func(*C, *S) error) error {
|
|
||||||
return host.UpdateState(func(rawConfig, rawState json.RawMessage) (any, error) {
|
|
||||||
var cfg C
|
|
||||||
var state S
|
|
||||||
if err := json.Unmarshal(rawConfig, &cfg); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
if err := json.Unmarshal(rawState, &state); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
if err := update(&cfg, &state); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return &state, nil
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
type CronJob struct {
|
|
||||||
Name string
|
|
||||||
Every time.Duration
|
|
||||||
Timeout time.Duration
|
|
||||||
Run func(context.Context, TriggerSource) error
|
|
||||||
}
|
|
||||||
type CronProvider interface{ CronJobs(ProtocolHost) []CronJob }
|
|
||||||
|
|
||||||
// ConfigNormalizer is optional. It owns protocol validation and derived fields.
|
|
||||||
type ConfigNormalizer interface {
|
|
||||||
NormalizeConfig(json.RawMessage) (any, error)
|
|
||||||
}
|
|
||||||
@@ -1,73 +0,0 @@
|
|||||||
# GitHub mirror execution
|
|
||||||
|
|
||||||
This package owns GitHub URL rewriting and execution strategies. It depends only
|
|
||||||
on the standard library and does not depend on Downloader, HTTP clients, Git, or
|
|
||||||
storage. Downloader persists `Config` at its top-level `githubMirror` field;
|
|
||||||
`pkg/base` preserves the existing configuration names through aliases.
|
|
||||||
|
|
||||||
## Race: resource reads
|
|
||||||
|
|
||||||
```go
|
|
||||||
body, err := githubmirror.Race(ctx, cfg.GithubMirror, rawURL,
|
|
||||||
githubmirror.Options[[]byte]{
|
|
||||||
Kind: githubmirror.Resource,
|
|
||||||
Timeout: 30 * time.Second,
|
|
||||||
},
|
|
||||||
func(ctx context.Context, candidate string) ([]byte, error) {
|
|
||||||
return fetchAndValidate(ctx, client, candidate)
|
|
||||||
},
|
|
||||||
)
|
|
||||||
```
|
|
||||||
|
|
||||||
The callback owns HTTP requests, size limits, and content validation. The first
|
|
||||||
completed valid result wins. Other operations are canceled and drained. There is
|
|
||||||
no HEAD-probe assumption or client-specific HTTP behavior inside the executor.
|
|
||||||
|
|
||||||
## Fallback: repository operations
|
|
||||||
|
|
||||||
```go
|
|
||||||
repo, err := githubmirror.Fallback(ctx, cfg.GithubMirror, officialURL,
|
|
||||||
githubmirror.Options[*git.Repository]{
|
|
||||||
Kind: githubmirror.Repository,
|
|
||||||
},
|
|
||||||
func(ctx context.Context, candidate string) (*git.Repository, error) {
|
|
||||||
// Use a clean temporary checkout for each attempt.
|
|
||||||
return clone(ctx, candidate)
|
|
||||||
},
|
|
||||||
)
|
|
||||||
```
|
|
||||||
|
|
||||||
The official URL is attempted first; mirrors follow their configured order.
|
|
||||||
jsDelivr is excluded for repositories. Stop on the first success. Perform
|
|
||||||
persistent installation or other business state changes after success, rather
|
|
||||||
than inside a potentially repeated attempt.
|
|
||||||
|
|
||||||
## Results with resources
|
|
||||||
|
|
||||||
For temporary files, set `Options[*Download].Discard` to remove the file. Return
|
|
||||||
the partial `*Download` alongside errors, so cleanup also covers failed reads or
|
|
||||||
validation. The executor discards every failure and every race loser exactly
|
|
||||||
once. The caller owns and must track/release the winning result.
|
|
||||||
|
|
||||||
- `Timeout == 0` uses 30 seconds per resource attempt and no overall deadline
|
|
||||||
for repository attempts; a negative timeout is invalid. Extension Git HTTP
|
|
||||||
transport limits connections/TLS handshakes to 10 seconds and resets a 60-second
|
|
||||||
read deadline as data arrives. Slow continuous transfers can finish, while
|
|
||||||
stalled sources fail and allow ordered fallback. Parent cancellation still applies.
|
|
||||||
- Attempts must honor the supplied context; draining waits for them to return.
|
|
||||||
- Attempt contexts are canceled after return, so callbacks must finish reading
|
|
||||||
HTTP bodies before returning. Returning a still-streaming response is unsupported.
|
|
||||||
- Discard callbacks must accept zero values, must not panic, and run serially.
|
|
||||||
- Parent cancellation wins over a result selected while the parent is canceled;
|
|
||||||
the selected result is then discarded as well.
|
|
||||||
- Errors preserve their causes through wrapping/joining for `errors.Is`.
|
|
||||||
- `github.com` and `raw.githubusercontent.com` expand into eligible mirrors.
|
|
||||||
jsDelivr is only eligible for raw source resources. `api.github.com` is never
|
|
||||||
mirrored and is rejected by the public mirror candidates API.
|
|
||||||
- Client application update checks request the official GitHub API directly,
|
|
||||||
falling back to the independent Gopeed website API on request or validation
|
|
||||||
failure. Only package downloads obtain URLs from
|
|
||||||
`POST /api/v1/mirrors/github/candidates`. Candidate generation never performs
|
|
||||||
HTTP requests.
|
|
||||||
- Disabled mirrors and other URLs execute only the original URL.
|
|
||||||
- Config is a caller-owned immutable snapshot for the duration of a call.
|
|
||||||
@@ -1,90 +0,0 @@
|
|||||||
// Package githubmirror rewrites GitHub URLs and executes mirrored operations.
|
|
||||||
package githubmirror
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
"net/url"
|
|
||||||
"strings"
|
|
||||||
)
|
|
||||||
|
|
||||||
type Mirror struct {
|
|
||||||
Type string `json:"type"`
|
|
||||||
URL string `json:"url"`
|
|
||||||
IsBuiltIn bool `json:"isBuiltIn"`
|
|
||||||
IsDeleted bool `json:"isDeleted"`
|
|
||||||
}
|
|
||||||
type Config struct {
|
|
||||||
Enabled bool `json:"enabled"`
|
|
||||||
Mirrors []Mirror `json:"mirrors"`
|
|
||||||
}
|
|
||||||
|
|
||||||
func DefaultConfig() *Config {
|
|
||||||
return &Config{Enabled: true, Mirrors: []Mirror{
|
|
||||||
{Type: "jsdelivr", URL: "https://fastly.jsdelivr.net/gh", IsBuiltIn: true},
|
|
||||||
{Type: "ghProxy", URL: "https://fastgit.cc", IsBuiltIn: true},
|
|
||||||
{Type: "ghProxy", URL: "https://gh-proxy.com", IsBuiltIn: true},
|
|
||||||
{Type: "ghProxy", URL: "https://ghfast.top", IsBuiltIn: true},
|
|
||||||
}}
|
|
||||||
}
|
|
||||||
func (c *Config) Validate() error {
|
|
||||||
if len(c.Mirrors) > 32 {
|
|
||||||
return fmt.Errorf("too many GitHub mirrors")
|
|
||||||
}
|
|
||||||
for _, mirror := range c.Mirrors {
|
|
||||||
u, err := url.Parse(mirror.URL)
|
|
||||||
if err != nil || u == nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" || u.User != nil || u.RawQuery != "" || u.Fragment != "" {
|
|
||||||
return fmt.Errorf("invalid GitHub mirror URL")
|
|
||||||
}
|
|
||||||
if mirror.Type != "jsdelivr" && mirror.Type != "ghProxy" {
|
|
||||||
return fmt.Errorf("invalid GitHub mirror type: %s", mirror.Type)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Kind describes what a candidate endpoint must support.
|
|
||||||
type Kind uint8
|
|
||||||
|
|
||||||
const (
|
|
||||||
Resource Kind = iota
|
|
||||||
Repository
|
|
||||||
)
|
|
||||||
|
|
||||||
// Candidates preserves the official URL first. jsDelivr is only eligible for
|
|
||||||
// raw source resources; Git operations use mirrors supporting Git Smart HTTP.
|
|
||||||
// GitHub API URLs are never mirrored.
|
|
||||||
func Candidates(c *Config, raw string, kind Kind) []string {
|
|
||||||
result := []string{raw}
|
|
||||||
u, err := url.Parse(raw)
|
|
||||||
if err != nil || c == nil || !c.Enabled || u.User != nil || (u.Scheme != "https" && u.Scheme != "http") {
|
|
||||||
return result
|
|
||||||
}
|
|
||||||
if u.Host != "github.com" && u.Host != "raw.githubusercontent.com" {
|
|
||||||
return result
|
|
||||||
}
|
|
||||||
seen := map[string]bool{raw: true}
|
|
||||||
for _, mirror := range c.Mirrors {
|
|
||||||
if mirror.IsDeleted {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
var candidate string
|
|
||||||
switch mirror.Type {
|
|
||||||
case "ghProxy":
|
|
||||||
candidate = strings.TrimRight(mirror.URL, "/") + "/" + raw
|
|
||||||
case "jsdelivr":
|
|
||||||
if kind == Repository || u.Host != "raw.githubusercontent.com" {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
parts := strings.SplitN(strings.TrimPrefix(u.EscapedPath(), "/"), "/", 4)
|
|
||||||
if len(parts) != 4 {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
candidate = strings.TrimRight(mirror.URL, "/") + "/" + parts[0] + "/" + parts[1] + "@" + parts[2] + "/" + parts[3]
|
|
||||||
}
|
|
||||||
if candidate != "" && !seen[candidate] {
|
|
||||||
result = append(result, candidate)
|
|
||||||
seen[candidate] = true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return result
|
|
||||||
}
|
|
||||||
@@ -1,38 +0,0 @@
|
|||||||
package githubmirror
|
|
||||||
|
|
||||||
import (
|
|
||||||
"reflect"
|
|
||||||
"testing"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestMirrorCandidates(t *testing.T) {
|
|
||||||
cfg := &Config{Enabled: true, Mirrors: []Mirror{
|
|
||||||
{Type: "jsdelivr", URL: "https://cdn.example/gh/"},
|
|
||||||
{Type: "ghProxy", URL: "https://mirror.example/"},
|
|
||||||
{Type: "ghProxy", URL: "https://deleted.example", IsDeleted: true},
|
|
||||||
}}
|
|
||||||
source := "https://raw.githubusercontent.com/owner/repo/main/a.txt"
|
|
||||||
want := []string{source, "https://cdn.example/gh/owner/repo@main/a.txt", "https://mirror.example/" + source}
|
|
||||||
if got := Candidates(cfg, source, Resource); !reflect.DeepEqual(got, want) {
|
|
||||||
t.Fatalf("got %v", got)
|
|
||||||
}
|
|
||||||
repo := "https://github.com/owner/repo.git"
|
|
||||||
if got := Candidates(cfg, repo, Repository); !reflect.DeepEqual(got, []string{repo, "https://mirror.example/" + repo}) {
|
|
||||||
t.Fatalf("git candidates %v", got)
|
|
||||||
}
|
|
||||||
for _, raw := range []string{"https://github.com.evil.example/a", "https://example.com/a", "https://user:[email protected]/a"} {
|
|
||||||
if got := Candidates(cfg, raw, Resource); !reflect.DeepEqual(got, []string{raw}) {
|
|
||||||
t.Fatalf("unexpected mirror: %v", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
apiURL := "https://api.github.com/repos/GopeedLab/gopeed/releases?per_page=10"
|
|
||||||
for _, kind := range []Kind{Resource, Repository} {
|
|
||||||
if got := Candidates(cfg, apiURL, kind); !reflect.DeepEqual(got, []string{apiURL}) {
|
|
||||||
t.Fatalf("GitHub API must never use mirrors: %v", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
cfg.Enabled = false
|
|
||||||
if got := Candidates(cfg, source, Resource); !reflect.DeepEqual(got, []string{source}) {
|
|
||||||
t.Fatalf("disabled mirror: %v", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,160 +0,0 @@
|
|||||||
package githubmirror
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Options configures execution, not URL persistence. Discard releases results
|
|
||||||
// from failed attempts and successful race losers, including partial results
|
|
||||||
// returned together with an error. The caller owns the winning result.
|
|
||||||
type Options[T any] struct {
|
|
||||||
Kind Kind
|
|
||||||
// Timeout limits each attempt. Zero uses 30 seconds for resources and no
|
|
||||||
// overall deadline for repositories, whose transport owns idle timeouts.
|
|
||||||
Timeout time.Duration
|
|
||||||
Discard func(T)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Attempt must honor ctx and return a completed, validated result. Its context
|
|
||||||
// is canceled after it returns, so live response streams must not be returned.
|
|
||||||
// Return any partially acquired resources along with errors so Discard can
|
|
||||||
// release them. Failed attempts must not mutate persistent business state.
|
|
||||||
type Attempt[T any] func(context.Context, string) (T, error)
|
|
||||||
|
|
||||||
const defaultTimeout = 30 * time.Second
|
|
||||||
|
|
||||||
func prepare[T any](ctx context.Context, config *Config, raw string, options Options[T], attempt Attempt[T]) ([]string, time.Duration, error) {
|
|
||||||
if err := ctx.Err(); err != nil {
|
|
||||||
return nil, 0, err
|
|
||||||
}
|
|
||||||
if raw == "" {
|
|
||||||
return nil, 0, errors.New("empty GitHub source URL")
|
|
||||||
}
|
|
||||||
if options.Kind != Resource && options.Kind != Repository {
|
|
||||||
return nil, 0, errors.New("invalid GitHub resource kind")
|
|
||||||
}
|
|
||||||
if options.Timeout < 0 {
|
|
||||||
return nil, 0, errors.New("invalid GitHub attempt timeout")
|
|
||||||
}
|
|
||||||
if attempt == nil {
|
|
||||||
return nil, 0, errors.New("GitHub mirror attempt is required")
|
|
||||||
}
|
|
||||||
if config != nil {
|
|
||||||
if err := config.Validate(); err != nil {
|
|
||||||
return nil, 0, err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
timeout := options.Timeout
|
|
||||||
if timeout == 0 && options.Kind == Resource {
|
|
||||||
timeout = defaultTimeout
|
|
||||||
}
|
|
||||||
return Candidates(config, raw, options.Kind), timeout, nil
|
|
||||||
}
|
|
||||||
func invoke[T any](ctx context.Context, timeout time.Duration, candidate string, attempt Attempt[T]) (value T, err error) {
|
|
||||||
var cancel context.CancelFunc
|
|
||||||
if timeout > 0 {
|
|
||||||
ctx, cancel = context.WithTimeout(ctx, timeout)
|
|
||||||
} else {
|
|
||||||
ctx, cancel = context.WithCancel(ctx)
|
|
||||||
}
|
|
||||||
defer cancel()
|
|
||||||
defer func() {
|
|
||||||
if recovered := recover(); recovered != nil {
|
|
||||||
err = fmt.Errorf("GitHub mirror attempt panicked: %v", recovered)
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
value, err = attempt(ctx, candidate)
|
|
||||||
if err == nil {
|
|
||||||
err = ctx.Err()
|
|
||||||
}
|
|
||||||
return
|
|
||||||
}
|
|
||||||
func discard[T any](options Options[T], value T) {
|
|
||||||
if options.Discard != nil {
|
|
||||||
options.Discard(value)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Race executes the official URL and eligible mirrors concurrently. The first
|
|
||||||
// completed successful result wins; losing attempts are canceled and drained
|
|
||||||
// before return so no temporary files or other resources outlive the call.
|
|
||||||
// HTTP status/content validation is the attempt's responsibility.
|
|
||||||
func Race[T any](ctx context.Context, config *Config, raw string, options Options[T], attempt Attempt[T]) (T, error) {
|
|
||||||
var zero T
|
|
||||||
candidates, timeout, err := prepare(ctx, config, raw, options, attempt)
|
|
||||||
if err != nil {
|
|
||||||
return zero, err
|
|
||||||
}
|
|
||||||
raceCtx, cancel := context.WithCancel(ctx)
|
|
||||||
defer cancel()
|
|
||||||
type outcome struct {
|
|
||||||
index int
|
|
||||||
value T
|
|
||||||
err error
|
|
||||||
}
|
|
||||||
outcomes := make(chan outcome, len(candidates))
|
|
||||||
for index, candidate := range candidates {
|
|
||||||
go func(index int, candidate string) {
|
|
||||||
value, err := invoke(raceCtx, timeout, candidate, attempt)
|
|
||||||
outcomes <- outcome{index, value, err}
|
|
||||||
}(index, candidate)
|
|
||||||
}
|
|
||||||
var winner T
|
|
||||||
won := false
|
|
||||||
failures := make([]error, len(candidates))
|
|
||||||
for range candidates {
|
|
||||||
outcome := <-outcomes
|
|
||||||
if !won && outcome.err == nil && ctx.Err() == nil {
|
|
||||||
winner = outcome.value
|
|
||||||
won = true
|
|
||||||
cancel()
|
|
||||||
} else {
|
|
||||||
discard(options, outcome.value)
|
|
||||||
if outcome.err != nil {
|
|
||||||
failures[outcome.index] = fmt.Errorf("GitHub source %d: %w", outcome.index+1, outcome.err)
|
|
||||||
}
|
|
||||||
if ctx.Err() != nil {
|
|
||||||
cancel()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if err := ctx.Err(); err != nil {
|
|
||||||
if won {
|
|
||||||
discard(options, winner)
|
|
||||||
}
|
|
||||||
return zero, err
|
|
||||||
}
|
|
||||||
if won {
|
|
||||||
return winner, nil
|
|
||||||
}
|
|
||||||
return zero, fmt.Errorf("all GitHub sources failed: %w", errors.Join(failures...))
|
|
||||||
}
|
|
||||||
|
|
||||||
// Fallback executes the official URL first, then eligible mirrors in their
|
|
||||||
// configured order. It stops immediately on success and cleans every failure.
|
|
||||||
func Fallback[T any](ctx context.Context, config *Config, raw string, options Options[T], attempt Attempt[T]) (T, error) {
|
|
||||||
var zero T
|
|
||||||
candidates, timeout, err := prepare(ctx, config, raw, options, attempt)
|
|
||||||
if err != nil {
|
|
||||||
return zero, err
|
|
||||||
}
|
|
||||||
failures := make([]error, 0, len(candidates))
|
|
||||||
for index, candidate := range candidates {
|
|
||||||
if err := ctx.Err(); err != nil {
|
|
||||||
return zero, err
|
|
||||||
}
|
|
||||||
value, err := invoke(ctx, timeout, candidate, attempt)
|
|
||||||
if err == nil && ctx.Err() == nil {
|
|
||||||
return value, nil
|
|
||||||
}
|
|
||||||
discard(options, value)
|
|
||||||
if parentErr := ctx.Err(); parentErr != nil {
|
|
||||||
return zero, parentErr
|
|
||||||
}
|
|
||||||
failures = append(failures, fmt.Errorf("GitHub source %d: %w", index+1, err))
|
|
||||||
}
|
|
||||||
return zero, fmt.Errorf("all GitHub sources failed: %w", errors.Join(failures...))
|
|
||||||
}
|
|
||||||
@@ -1,214 +0,0 @@
|
|||||||
package githubmirror
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"reflect"
|
|
||||||
"strings"
|
|
||||||
"sync"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
const testSource = "https://github.com/owner/repo.git"
|
|
||||||
|
|
||||||
func testConfig() *Config {
|
|
||||||
return &Config{Enabled: true, Mirrors: []Mirror{
|
|
||||||
{Type: "ghProxy", URL: "https://first.example"},
|
|
||||||
{Type: "ghProxy", URL: "https://second.example"},
|
|
||||||
}}
|
|
||||||
}
|
|
||||||
func TestRaceWinsAndDrainsOwnedResults(t *testing.T) {
|
|
||||||
config := testConfig()
|
|
||||||
candidates := Candidates(config, testSource, Resource)
|
|
||||||
var started sync.WaitGroup
|
|
||||||
started.Add(len(candidates))
|
|
||||||
discarded := []string{}
|
|
||||||
winner, err := Race(context.Background(), config, testSource, Options[string]{
|
|
||||||
Discard: func(value string) { discarded = append(discarded, value) },
|
|
||||||
}, func(ctx context.Context, address string) (string, error) {
|
|
||||||
started.Done()
|
|
||||||
started.Wait()
|
|
||||||
switch address {
|
|
||||||
case candidates[0]:
|
|
||||||
return address, errors.New("official offline")
|
|
||||||
case candidates[1]:
|
|
||||||
return address, nil
|
|
||||||
default:
|
|
||||||
<-ctx.Done()
|
|
||||||
return address, ctx.Err()
|
|
||||||
}
|
|
||||||
})
|
|
||||||
if err != nil || winner != candidates[1] {
|
|
||||||
t.Fatalf("winner=%s err=%v", winner, err)
|
|
||||||
}
|
|
||||||
if len(discarded) != 2 || !contains(discarded, candidates[0]) || !contains(discarded, candidates[2]) {
|
|
||||||
t.Fatalf("discarded %v", discarded)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
func contains(values []string, value string) bool {
|
|
||||||
for _, item := range values {
|
|
||||||
if item == value {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
func TestRaceCleansSimultaneousSuccessfulLosers(t *testing.T) {
|
|
||||||
config := testConfig()
|
|
||||||
var started sync.WaitGroup
|
|
||||||
started.Add(3)
|
|
||||||
cleaned := []string{}
|
|
||||||
winner, err := Race(context.Background(), config, testSource, Options[string]{Discard: func(value string) { cleaned = append(cleaned, value) }}, func(_ context.Context, address string) (string, error) {
|
|
||||||
started.Done()
|
|
||||||
started.Wait()
|
|
||||||
return address, nil
|
|
||||||
})
|
|
||||||
if err != nil || winner == "" || len(cleaned) != 2 || contains(cleaned, winner) {
|
|
||||||
t.Fatalf("winner=%s discarded=%v err=%v", winner, cleaned, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
func TestRaceParentCancellationCleansWinnerToo(t *testing.T) {
|
|
||||||
config := testConfig()
|
|
||||||
config.Mirrors = config.Mirrors[:1]
|
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
|
||||||
defer cancel()
|
|
||||||
cleaned := []string{}
|
|
||||||
winner, err := Race(ctx, config, testSource, Options[string]{Discard: func(value string) { cleaned = append(cleaned, value) }}, func(ctx context.Context, address string) (string, error) {
|
|
||||||
if address == testSource {
|
|
||||||
return address, nil
|
|
||||||
}
|
|
||||||
<-ctx.Done()
|
|
||||||
cancel()
|
|
||||||
return address, ctx.Err()
|
|
||||||
})
|
|
||||||
if !errors.Is(err, context.Canceled) || winner != "" || len(cleaned) != 2 || !contains(cleaned, testSource) {
|
|
||||||
t.Fatalf("winner=%s cleanup=%v err=%v", winner, cleaned, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
func TestRaceTimeoutAndErrorAggregation(t *testing.T) {
|
|
||||||
expected := errors.New("offline")
|
|
||||||
config := testConfig()
|
|
||||||
config.Mirrors = config.Mirrors[:1]
|
|
||||||
discarded := 0
|
|
||||||
_, err := Race(context.Background(), config, testSource, Options[string]{Timeout: 10 * time.Millisecond, Discard: func(string) { discarded++ }}, func(ctx context.Context, address string) (string, error) {
|
|
||||||
if address == testSource {
|
|
||||||
return address, expected
|
|
||||||
}
|
|
||||||
<-ctx.Done()
|
|
||||||
return address, ctx.Err()
|
|
||||||
})
|
|
||||||
if !errors.Is(err, expected) || !errors.Is(err, context.DeadlineExceeded) || discarded != 2 {
|
|
||||||
t.Fatalf("discarded=%d err=%v", discarded, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
func TestFallbackPreservesOrderAndStopsOnSuccess(t *testing.T) {
|
|
||||||
config := testConfig()
|
|
||||||
candidates := Candidates(config, testSource, Repository)
|
|
||||||
tried := []string{}
|
|
||||||
cleaned := []string{}
|
|
||||||
winner, err := Fallback(context.Background(), config, testSource, Options[string]{Kind: Repository, Discard: func(value string) { cleaned = append(cleaned, value) }}, func(_ context.Context, address string) (string, error) {
|
|
||||||
tried = append(tried, address)
|
|
||||||
if address == candidates[1] {
|
|
||||||
return address, nil
|
|
||||||
}
|
|
||||||
return address, errors.New("offline")
|
|
||||||
})
|
|
||||||
if err != nil || winner != candidates[1] || !reflect.DeepEqual(tried, candidates[:2]) || !reflect.DeepEqual(cleaned, candidates[:1]) {
|
|
||||||
t.Fatalf("winner=%s tried=%v discarded=%v err=%v", winner, tried, cleaned, err)
|
|
||||||
}
|
|
||||||
tried = nil
|
|
||||||
_, err = Fallback(context.Background(), config, testSource, Options[string]{Kind: Repository}, func(_ context.Context, address string) (string, error) {
|
|
||||||
tried = append(tried, address)
|
|
||||||
return address, nil
|
|
||||||
})
|
|
||||||
if err != nil || !reflect.DeepEqual(tried, candidates[:1]) {
|
|
||||||
t.Fatalf("tried=%v err=%v", tried, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
func TestFallbackTimeoutContinuesWithFreshContext(t *testing.T) {
|
|
||||||
discarded := []string{}
|
|
||||||
winner, err := Fallback(context.Background(), testConfig(), testSource, Options[string]{Kind: Repository, Timeout: 10 * time.Millisecond, Discard: func(value string) { discarded = append(discarded, value) }}, func(ctx context.Context, address string) (string, error) {
|
|
||||||
if address == testSource {
|
|
||||||
<-ctx.Done()
|
|
||||||
return address, ctx.Err()
|
|
||||||
}
|
|
||||||
if err := ctx.Err(); err != nil {
|
|
||||||
return address, err
|
|
||||||
}
|
|
||||||
return address, nil
|
|
||||||
})
|
|
||||||
if err != nil || !strings.HasPrefix(winner, "https://first.example/") || !reflect.DeepEqual(discarded, []string{testSource}) {
|
|
||||||
t.Fatalf("winner=%s cleanup=%v err=%v", winner, discarded, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
func TestFallbackCancellationStopsBeforeNextMirror(t *testing.T) {
|
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
|
||||||
defer cancel()
|
|
||||||
calls := 0
|
|
||||||
cleaned := 0
|
|
||||||
_, err := Fallback(ctx, testConfig(), testSource, Options[string]{Kind: Repository, Discard: func(string) { cleaned++ }}, func(ctx context.Context, address string) (string, error) {
|
|
||||||
calls++
|
|
||||||
cancel()
|
|
||||||
return address, ctx.Err()
|
|
||||||
})
|
|
||||||
if !errors.Is(err, context.Canceled) || calls != 1 || cleaned != 1 {
|
|
||||||
t.Fatalf("calls=%d discarded=%d err=%v", calls, cleaned, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
func TestExecutorsDoNotStartInvalidOrCanceledRequests(t *testing.T) {
|
|
||||||
canceled, cancel := context.WithCancel(context.Background())
|
|
||||||
cancel()
|
|
||||||
for _, test := range []struct {
|
|
||||||
ctx context.Context
|
|
||||||
config *Config
|
|
||||||
options Options[string]
|
|
||||||
attempt Attempt[string]
|
|
||||||
}{
|
|
||||||
{ctx: canceled, options: Options[string]{}, attempt: func(context.Context, string) (string, error) { t.Fatal("called after cancellation"); return "", nil }},
|
|
||||||
{ctx: context.Background(), options: Options[string]{Kind: 99}, attempt: func(context.Context, string) (string, error) { t.Fatal("invalid kind"); return "", nil }},
|
|
||||||
{ctx: context.Background(), options: Options[string]{Timeout: -time.Second}, attempt: func(context.Context, string) (string, error) { t.Fatal("invalid timeout"); return "", nil }},
|
|
||||||
{ctx: context.Background(), options: Options[string]{}, attempt: nil},
|
|
||||||
{ctx: context.Background(), config: &Config{Mirrors: []Mirror{{Type: "invalid", URL: "https://mirror.example"}}}, attempt: func(context.Context, string) (string, error) { t.Fatal("invalid config"); return "", nil }},
|
|
||||||
} {
|
|
||||||
if _, err := Race(test.ctx, test.config, testSource, test.options, test.attempt); err == nil {
|
|
||||||
t.Fatal("invalid race accepted")
|
|
||||||
}
|
|
||||||
if _, err := Fallback(test.ctx, test.config, testSource, test.options, test.attempt); err == nil {
|
|
||||||
t.Fatal("invalid fallback accepted")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
func TestAttemptPanicBecomesError(t *testing.T) {
|
|
||||||
attempt := func(context.Context, string) (string, error) { panic("broken operation") }
|
|
||||||
if _, err := Race(context.Background(), nil, testSource, Options[string]{}, attempt); err == nil || !strings.Contains(err.Error(), "broken operation") {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, err := Fallback(context.Background(), nil, testSource, Options[string]{}, attempt); err == nil || !strings.Contains(err.Error(), "broken operation") {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
func TestNonGithubResourcesAndDisabledMirrorsUseOnlyOriginal(t *testing.T) {
|
|
||||||
for _, test := range []struct {
|
|
||||||
config *Config
|
|
||||||
raw string
|
|
||||||
}{{testConfig(), "https://example.com/resource"}, {&Config{Enabled: false, Mirrors: testConfig().Mirrors}, testSource}} {
|
|
||||||
calls := 0
|
|
||||||
got, err := Race(context.Background(), test.config, test.raw, Options[string]{}, func(_ context.Context, address string) (string, error) { calls++; return address, nil })
|
|
||||||
if err != nil || got != test.raw || calls != 1 {
|
|
||||||
t.Fatalf("got=%s calls=%d err=%v", got, calls, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRepositoryAttemptsHaveNoDefaultDeadline(t *testing.T) {
|
|
||||||
value, err := Fallback(context.Background(), nil, testSource, Options[string]{Kind: Repository}, func(ctx context.Context, raw string) (string, error) {
|
|
||||||
if _, ok := ctx.Deadline(); ok {
|
|
||||||
t.Fatal("repository attempt has a total deadline")
|
|
||||||
}
|
|
||||||
return raw, nil
|
|
||||||
})
|
|
||||||
if err != nil || value != testSource {
|
|
||||||
t.Fatalf("value=%s error=%v", value, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,346 +0,0 @@
|
|||||||
package httpclient
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"crypto/tls"
|
|
||||||
"fmt"
|
|
||||||
"net"
|
|
||||||
"net/http"
|
|
||||||
"net/url"
|
|
||||||
"strings"
|
|
||||||
"sync"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/imroc/req/v3"
|
|
||||||
)
|
|
||||||
|
|
||||||
type ImpersonationMode string
|
|
||||||
|
|
||||||
const (
|
|
||||||
ImpersonationNative ImpersonationMode = "native"
|
|
||||||
ImpersonationFixed ImpersonationMode = "fixed"
|
|
||||||
ImpersonationAuto ImpersonationMode = "auto"
|
|
||||||
)
|
|
||||||
|
|
||||||
type Browser string
|
|
||||||
|
|
||||||
const (
|
|
||||||
BrowserChrome Browser = "chrome"
|
|
||||||
BrowserFirefox Browser = "firefox"
|
|
||||||
BrowserSafari Browser = "safari"
|
|
||||||
)
|
|
||||||
|
|
||||||
type Options struct {
|
|
||||||
Client ClientOptions
|
|
||||||
Transport TransportOptions
|
|
||||||
Impersonation ImpersonationOptions
|
|
||||||
}
|
|
||||||
|
|
||||||
type ClientOptions struct {
|
|
||||||
Jar http.CookieJar
|
|
||||||
CheckRedirect func(req *http.Request, via []*http.Request) error
|
|
||||||
Timeout time.Duration
|
|
||||||
}
|
|
||||||
|
|
||||||
type TransportOptions struct {
|
|
||||||
Proxy func(*http.Request) (*url.URL, error)
|
|
||||||
DialContext func(ctx context.Context, network, address string) (net.Conn, error)
|
|
||||||
TLSClientConfig *tls.Config
|
|
||||||
TLSHandshakeTimeout time.Duration
|
|
||||||
}
|
|
||||||
|
|
||||||
type ImpersonationOptions struct {
|
|
||||||
Mode ImpersonationMode
|
|
||||||
Browser Browser
|
|
||||||
Session *ImpersonationSession
|
|
||||||
}
|
|
||||||
|
|
||||||
type ImpersonationSession struct {
|
|
||||||
mu sync.RWMutex
|
|
||||||
selections map[string]impersonationSelection
|
|
||||||
}
|
|
||||||
|
|
||||||
type impersonationSelection struct {
|
|
||||||
browser Browser
|
|
||||||
fallbackDisabled bool
|
|
||||||
}
|
|
||||||
|
|
||||||
func NewImpersonationSession() *ImpersonationSession {
|
|
||||||
return &ImpersonationSession{
|
|
||||||
selections: make(map[string]impersonationSelection),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *ImpersonationSession) load(url string) (impersonationSelection, bool) {
|
|
||||||
s.mu.RLock()
|
|
||||||
defer s.mu.RUnlock()
|
|
||||||
selection, ok := s.selections[url]
|
|
||||||
return selection, ok
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *ImpersonationSession) store(url string, browser Browser) {
|
|
||||||
s.mu.Lock()
|
|
||||||
defer s.mu.Unlock()
|
|
||||||
s.selections[url] = impersonationSelection{browser: browser}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *ImpersonationSession) disableFallback(url string) {
|
|
||||||
s.mu.Lock()
|
|
||||||
defer s.mu.Unlock()
|
|
||||||
s.selections[url] = impersonationSelection{fallbackDisabled: true}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *ImpersonationSession) Clear() {
|
|
||||||
s.mu.Lock()
|
|
||||||
defer s.mu.Unlock()
|
|
||||||
clear(s.selections)
|
|
||||||
}
|
|
||||||
|
|
||||||
func NewClient(options Options) (*http.Client, error) {
|
|
||||||
var transport http.RoundTripper
|
|
||||||
switch options.Impersonation.Mode {
|
|
||||||
case "", ImpersonationNative:
|
|
||||||
transport = newNativeTransport(options.Transport)
|
|
||||||
case ImpersonationFixed:
|
|
||||||
browserTransport, err := newBrowserTransport(options.Transport, options.Impersonation.Browser)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
transport = browserTransport
|
|
||||||
case ImpersonationAuto:
|
|
||||||
browserTransports := make(map[Browser]http.RoundTripper, 3)
|
|
||||||
for _, browser := range []Browser{BrowserChrome, BrowserFirefox, BrowserSafari} {
|
|
||||||
browserTransport, err := newBrowserTransport(options.Transport, browser)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
browserTransports[browser] = browserTransport
|
|
||||||
}
|
|
||||||
transport = &adaptiveTransport{
|
|
||||||
native: newNativeTransport(options.Transport),
|
|
||||||
browsers: browserTransports,
|
|
||||||
jar: options.Client.Jar,
|
|
||||||
session: options.Impersonation.Session,
|
|
||||||
}
|
|
||||||
default:
|
|
||||||
return nil, fmt.Errorf("unsupported impersonation mode %q", options.Impersonation.Mode)
|
|
||||||
}
|
|
||||||
|
|
||||||
return &http.Client{
|
|
||||||
Transport: transport,
|
|
||||||
Jar: options.Client.Jar,
|
|
||||||
CheckRedirect: options.Client.CheckRedirect,
|
|
||||||
Timeout: options.Client.Timeout,
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func newNativeTransport(options TransportOptions) *http.Transport {
|
|
||||||
var tlsConfig *tls.Config
|
|
||||||
if options.TLSClientConfig != nil {
|
|
||||||
tlsConfig = options.TLSClientConfig.Clone()
|
|
||||||
}
|
|
||||||
return &http.Transport{
|
|
||||||
Proxy: options.Proxy,
|
|
||||||
DialContext: options.DialContext,
|
|
||||||
TLSClientConfig: tlsConfig,
|
|
||||||
TLSHandshakeTimeout: options.TLSHandshakeTimeout,
|
|
||||||
ForceAttemptHTTP2: true,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func newBrowserTransport(options TransportOptions, browser Browser) (http.RoundTripper, error) {
|
|
||||||
client := req.NewClient()
|
|
||||||
client.SetProxy(options.Proxy)
|
|
||||||
if options.DialContext != nil {
|
|
||||||
client.SetDial(options.DialContext)
|
|
||||||
}
|
|
||||||
client.SetTLSHandshakeTimeout(options.TLSHandshakeTimeout)
|
|
||||||
if options.TLSClientConfig != nil {
|
|
||||||
tlsConfig := options.TLSClientConfig.Clone()
|
|
||||||
if len(tlsConfig.NextProtos) == 0 {
|
|
||||||
tlsConfig.NextProtos = []string{"h2", "http/1.1"}
|
|
||||||
}
|
|
||||||
client.SetTLSClientConfig(tlsConfig)
|
|
||||||
}
|
|
||||||
|
|
||||||
switch browser {
|
|
||||||
case BrowserChrome:
|
|
||||||
client.ImpersonateChrome()
|
|
||||||
case BrowserFirefox:
|
|
||||||
client.ImpersonateFirefox()
|
|
||||||
case BrowserSafari:
|
|
||||||
client.ImpersonateSafari()
|
|
||||||
default:
|
|
||||||
return nil, fmt.Errorf("unsupported browser %q", browser)
|
|
||||||
}
|
|
||||||
|
|
||||||
return &defaultHeaderTransport{
|
|
||||||
next: client.GetClient().Transport,
|
|
||||||
defaults: client.Headers.Clone(),
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
type defaultHeaderTransport struct {
|
|
||||||
next http.RoundTripper
|
|
||||||
defaults http.Header
|
|
||||||
}
|
|
||||||
|
|
||||||
func (t *defaultHeaderTransport) RoundTrip(request *http.Request) (*http.Response, error) {
|
|
||||||
request = request.Clone(request.Context())
|
|
||||||
request.Header = request.Header.Clone()
|
|
||||||
for key, values := range t.defaults {
|
|
||||||
if len(request.Header.Values(key)) == 0 {
|
|
||||||
request.Header[key] = append([]string(nil), values...)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return t.next.RoundTrip(request)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (t *defaultHeaderTransport) CloseIdleConnections() {
|
|
||||||
closeIdleConnections(t.next)
|
|
||||||
}
|
|
||||||
|
|
||||||
type adaptiveTransport struct {
|
|
||||||
native http.RoundTripper
|
|
||||||
browsers map[Browser]http.RoundTripper
|
|
||||||
jar http.CookieJar
|
|
||||||
session *ImpersonationSession
|
|
||||||
}
|
|
||||||
|
|
||||||
func (t *adaptiveTransport) RoundTrip(request *http.Request) (*http.Response, error) {
|
|
||||||
cacheKey := requestCacheKey(request.URL)
|
|
||||||
if t.session != nil {
|
|
||||||
if selected, ok := t.session.load(cacheKey); ok {
|
|
||||||
if selected.fallbackDisabled {
|
|
||||||
return t.native.RoundTrip(request)
|
|
||||||
}
|
|
||||||
return t.browsers[selected.browser].RoundTrip(request)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
response, err := t.native.RoundTrip(request)
|
|
||||||
if !shouldFallback(response, err) {
|
|
||||||
return response, err
|
|
||||||
}
|
|
||||||
|
|
||||||
retry, ok := cloneRequestForRetry(request)
|
|
||||||
if !ok {
|
|
||||||
return response, err
|
|
||||||
}
|
|
||||||
if response != nil {
|
|
||||||
applyResponseCookies(t.jar, request, retry, response)
|
|
||||||
_ = response.Body.Close()
|
|
||||||
}
|
|
||||||
|
|
||||||
browser := browserFromUserAgent(request.UserAgent())
|
|
||||||
response, err = t.browsers[browser].RoundTrip(retry)
|
|
||||||
if t.session != nil {
|
|
||||||
if isSuccessfulFallback(response, err) {
|
|
||||||
t.session.store(cacheKey, browser)
|
|
||||||
} else if err == nil && response != nil {
|
|
||||||
t.session.disableFallback(cacheKey)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return response, err
|
|
||||||
}
|
|
||||||
|
|
||||||
func requestCacheKey(requestURL *url.URL) string {
|
|
||||||
cacheURL := *requestURL
|
|
||||||
cacheURL.Fragment = ""
|
|
||||||
cacheURL.RawFragment = ""
|
|
||||||
return cacheURL.String()
|
|
||||||
}
|
|
||||||
|
|
||||||
func applyResponseCookies(jar http.CookieJar, request, retry *http.Request, response *http.Response) {
|
|
||||||
if jar == nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
jar.SetCookies(request.URL, response.Cookies())
|
|
||||||
jarCookies := jar.Cookies(request.URL)
|
|
||||||
jarCookieNames := make(map[string]struct{}, len(jarCookies))
|
|
||||||
for _, cookie := range jarCookies {
|
|
||||||
jarCookieNames[cookie.Name] = struct{}{}
|
|
||||||
}
|
|
||||||
existingCookies := retry.Cookies()
|
|
||||||
retry.Header.Del("Cookie")
|
|
||||||
for _, cookie := range existingCookies {
|
|
||||||
if _, replaced := jarCookieNames[cookie.Name]; !replaced {
|
|
||||||
retry.AddCookie(cookie)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for _, cookie := range jarCookies {
|
|
||||||
retry.AddCookie(cookie)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (t *adaptiveTransport) CloseIdleConnections() {
|
|
||||||
closeIdleConnections(t.native)
|
|
||||||
for _, transport := range t.browsers {
|
|
||||||
closeIdleConnections(transport)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func closeIdleConnections(transport http.RoundTripper) {
|
|
||||||
if closer, ok := transport.(interface{ CloseIdleConnections() }); ok {
|
|
||||||
closer.CloseIdleConnections()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func shouldFallback(response *http.Response, err error) bool {
|
|
||||||
if err != nil {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
if response == nil {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
return response.StatusCode == http.StatusForbidden ||
|
|
||||||
strings.EqualFold(response.Header.Get("Cf-Mitigated"), "challenge")
|
|
||||||
}
|
|
||||||
|
|
||||||
func isSuccessfulFallback(response *http.Response, err error) bool {
|
|
||||||
return err == nil && response != nil &&
|
|
||||||
response.StatusCode >= http.StatusOK &&
|
|
||||||
response.StatusCode < http.StatusBadRequest
|
|
||||||
}
|
|
||||||
|
|
||||||
func cloneRequestForRetry(request *http.Request) (*http.Request, bool) {
|
|
||||||
if !isIdempotent(request) {
|
|
||||||
return nil, false
|
|
||||||
}
|
|
||||||
retry := request.Clone(request.Context())
|
|
||||||
if request.Body == nil || request.Body == http.NoBody {
|
|
||||||
return retry, true
|
|
||||||
}
|
|
||||||
if request.GetBody == nil {
|
|
||||||
return nil, false
|
|
||||||
}
|
|
||||||
body, err := request.GetBody()
|
|
||||||
if err != nil {
|
|
||||||
return nil, false
|
|
||||||
}
|
|
||||||
retry.Body = body
|
|
||||||
return retry, true
|
|
||||||
}
|
|
||||||
|
|
||||||
func isIdempotent(request *http.Request) bool {
|
|
||||||
switch request.Method {
|
|
||||||
case http.MethodGet, http.MethodHead, http.MethodOptions, http.MethodTrace:
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
return request.Header.Get("Idempotency-Key") != "" ||
|
|
||||||
request.Header.Get("X-Idempotency-Key") != ""
|
|
||||||
}
|
|
||||||
|
|
||||||
func browserFromUserAgent(userAgent string) Browser {
|
|
||||||
userAgent = strings.ToLower(userAgent)
|
|
||||||
switch {
|
|
||||||
case strings.Contains(userAgent, "firefox"):
|
|
||||||
return BrowserFirefox
|
|
||||||
case strings.Contains(userAgent, "safari") &&
|
|
||||||
!strings.Contains(userAgent, "chrome") &&
|
|
||||||
!strings.Contains(userAgent, "chromium"):
|
|
||||||
return BrowserSafari
|
|
||||||
default:
|
|
||||||
return BrowserChrome
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,962 +0,0 @@
|
|||||||
package httpclient_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"crypto/tls"
|
|
||||||
"errors"
|
|
||||||
"io"
|
|
||||||
"net"
|
|
||||||
"net/http"
|
|
||||||
"net/http/cookiejar"
|
|
||||||
"net/http/httptest"
|
|
||||||
"net/url"
|
|
||||||
"slices"
|
|
||||||
"strings"
|
|
||||||
"sync/atomic"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/GopeedLab/gopeed/internal/httpclient"
|
|
||||||
"github.com/GopeedLab/gopeed/internal/httpclient/httpclienttest"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestFixedChromeUsesBrowserTLSFingerprintAndHTTP2(t *testing.T) {
|
|
||||||
server := httpclienttest.NewFingerprintServer(t, httpclienttest.FingerprintServerOptions{
|
|
||||||
RequiredProfile: httpclienttest.ProfileAnyBrowser,
|
|
||||||
})
|
|
||||||
|
|
||||||
client, err := httpclient.NewClient(httpclient.Options{
|
|
||||||
Transport: httpclient.TransportOptions{
|
|
||||||
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
|
|
||||||
},
|
|
||||||
Impersonation: httpclient.ImpersonationOptions{
|
|
||||||
Mode: httpclient.ImpersonationFixed,
|
|
||||||
Browser: httpclient.BrowserChrome,
|
|
||||||
},
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("NewClient() error = %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
response, err := client.Get(server.URL)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Get() error = %v", err)
|
|
||||||
}
|
|
||||||
defer response.Body.Close()
|
|
||||||
|
|
||||||
if response.StatusCode != http.StatusOK {
|
|
||||||
t.Fatalf("status = %d, want %d", response.StatusCode, http.StatusOK)
|
|
||||||
}
|
|
||||||
if response.ProtoMajor != 2 {
|
|
||||||
t.Fatalf("protocol = %q, want HTTP/2", response.Proto)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAutoFallsBackFromNativeToBrowserFingerprint(t *testing.T) {
|
|
||||||
server := httpclienttest.NewFingerprintServer(t, httpclienttest.FingerprintServerOptions{
|
|
||||||
RequiredProfile: httpclienttest.ProfileAnyBrowser,
|
|
||||||
})
|
|
||||||
|
|
||||||
client, err := httpclient.NewClient(httpclient.Options{
|
|
||||||
Transport: httpclient.TransportOptions{
|
|
||||||
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
|
|
||||||
},
|
|
||||||
Impersonation: httpclient.ImpersonationOptions{
|
|
||||||
Mode: httpclient.ImpersonationAuto,
|
|
||||||
},
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("NewClient() error = %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
request, err := http.NewRequest(http.MethodGet, server.URL, nil)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("NewRequest() error = %v", err)
|
|
||||||
}
|
|
||||||
request.Header.Set("User-Agent", "Mozilla/5.0 Chrome/120.0.0.0 Safari/537.36")
|
|
||||||
|
|
||||||
response, err := client.Do(request)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Do() error = %v", err)
|
|
||||||
}
|
|
||||||
defer response.Body.Close()
|
|
||||||
|
|
||||||
if response.ProtoMajor != 2 {
|
|
||||||
t.Fatalf("protocol = %q, want HTTP/2", response.Proto)
|
|
||||||
}
|
|
||||||
if got, want := server.Profiles(), []httpclienttest.Profile{
|
|
||||||
httpclienttest.ProfileNative,
|
|
||||||
httpclienttest.ProfileChrome,
|
|
||||||
}; !slices.Equal(got, want) {
|
|
||||||
t.Fatalf("ClientHello profiles = %v, want %v", got, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAutoChoosesBrowserFingerprintFromUserAgent(t *testing.T) {
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
userAgent string
|
|
||||||
wantBrowser httpclient.Browser
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
name: "chrome",
|
|
||||||
userAgent: "Mozilla/5.0 Chrome/120.0.0.0 Safari/537.36",
|
|
||||||
wantBrowser: httpclient.BrowserChrome,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "firefox",
|
|
||||||
userAgent: "Mozilla/5.0 Firefox/120.0",
|
|
||||||
wantBrowser: httpclient.BrowserFirefox,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "safari",
|
|
||||||
userAgent: "Mozilla/5.0 Version/16.6 Safari/605.1.15",
|
|
||||||
wantBrowser: httpclient.BrowserSafari,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "unknown defaults to chrome",
|
|
||||||
userAgent: "Gopeed/1.0",
|
|
||||||
wantBrowser: httpclient.BrowserChrome,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
for _, test := range tests {
|
|
||||||
t.Run(test.name, func(t *testing.T) {
|
|
||||||
server := httpclienttest.NewFingerprintServer(t, httpclienttest.FingerprintServerOptions{
|
|
||||||
RequiredProfile: httpclienttest.Profile(test.wantBrowser),
|
|
||||||
})
|
|
||||||
client, err := httpclient.NewClient(httpclient.Options{
|
|
||||||
Transport: httpclient.TransportOptions{
|
|
||||||
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
|
|
||||||
},
|
|
||||||
Impersonation: httpclient.ImpersonationOptions{Mode: httpclient.ImpersonationAuto},
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("NewClient() error = %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
request, err := http.NewRequest(http.MethodGet, server.URL, nil)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("NewRequest() error = %v", err)
|
|
||||||
}
|
|
||||||
request.Header.Set("User-Agent", test.userAgent)
|
|
||||||
response, err := client.Do(request)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Do() error = %v", err)
|
|
||||||
}
|
|
||||||
response.Body.Close()
|
|
||||||
|
|
||||||
if response.ProtoMajor != 2 {
|
|
||||||
t.Fatalf("protocol = %q, want HTTP/2", response.Proto)
|
|
||||||
}
|
|
||||||
if got, want := server.Profiles(), []httpclienttest.Profile{
|
|
||||||
httpclienttest.ProfileNative,
|
|
||||||
httpclienttest.Profile(test.wantBrowser),
|
|
||||||
}; !slices.Equal(got, want) {
|
|
||||||
t.Fatalf("profiles = %v, want %v", got, want)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAutoFallsBackOnForbiddenAndPinsBrowserPerURL(t *testing.T) {
|
|
||||||
var requests atomic.Int32
|
|
||||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) {
|
|
||||||
requests.Add(1)
|
|
||||||
if request.Header.Get("Sec-Ch-Ua") == "" {
|
|
||||||
http.Error(w, "browser required", http.StatusForbidden)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
w.WriteHeader(http.StatusOK)
|
|
||||||
}))
|
|
||||||
t.Cleanup(server.Close)
|
|
||||||
|
|
||||||
client, err := httpclient.NewClient(httpclient.Options{
|
|
||||||
Impersonation: httpclient.ImpersonationOptions{
|
|
||||||
Mode: httpclient.ImpersonationAuto,
|
|
||||||
Session: httpclient.NewImpersonationSession(),
|
|
||||||
},
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("NewClient() error = %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
for range 2 {
|
|
||||||
request, err := http.NewRequest(http.MethodGet, server.URL, nil)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("NewRequest() error = %v", err)
|
|
||||||
}
|
|
||||||
request.Header.Set("User-Agent", "Mozilla/5.0 Chrome/120.0.0.0 Safari/537.36")
|
|
||||||
response, err := client.Do(request)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Do() error = %v", err)
|
|
||||||
}
|
|
||||||
response.Body.Close()
|
|
||||||
if response.StatusCode != http.StatusOK {
|
|
||||||
t.Fatalf("status = %d, want %d", response.StatusCode, http.StatusOK)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if requests.Load() != 3 {
|
|
||||||
t.Fatalf("requests = %d, want 3 (native + fallback + pinned)", requests.Load())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAutoDisablesFallbackWhenBrowserReturnsHTTPFailure(t *testing.T) {
|
|
||||||
var requests atomic.Int32
|
|
||||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) {
|
|
||||||
requests.Add(1)
|
|
||||||
if request.Header.Get("Sec-Ch-Ua") == "" {
|
|
||||||
w.WriteHeader(http.StatusForbidden)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
w.WriteHeader(http.StatusNotFound)
|
|
||||||
}))
|
|
||||||
t.Cleanup(server.Close)
|
|
||||||
|
|
||||||
client, err := httpclient.NewClient(httpclient.Options{
|
|
||||||
Impersonation: httpclient.ImpersonationOptions{
|
|
||||||
Mode: httpclient.ImpersonationAuto,
|
|
||||||
Session: httpclient.NewImpersonationSession(),
|
|
||||||
},
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("NewClient() error = %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
wantStatuses := []int{http.StatusNotFound, http.StatusForbidden}
|
|
||||||
for _, wantStatus := range wantStatuses {
|
|
||||||
response, err := client.Get(server.URL)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Get() error = %v", err)
|
|
||||||
}
|
|
||||||
response.Body.Close()
|
|
||||||
if response.StatusCode != wantStatus {
|
|
||||||
t.Fatalf("status = %d, want %d", response.StatusCode, wantStatus)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if requests.Load() != 3 {
|
|
||||||
t.Fatalf("requests = %d, want 3 (native + one fallback + native)", requests.Load())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAutoDoesNotDisableFallbackAfterTransportFailure(t *testing.T) {
|
|
||||||
server := httpclienttest.NewFingerprintServer(t, httpclienttest.FingerprintServerOptions{
|
|
||||||
RequiredProfile: httpclienttest.Profile("unsupported"),
|
|
||||||
})
|
|
||||||
client, err := httpclient.NewClient(httpclient.Options{
|
|
||||||
Transport: httpclient.TransportOptions{
|
|
||||||
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
|
|
||||||
},
|
|
||||||
Impersonation: httpclient.ImpersonationOptions{
|
|
||||||
Mode: httpclient.ImpersonationAuto,
|
|
||||||
Session: httpclient.NewImpersonationSession(),
|
|
||||||
},
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("NewClient() error = %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
for range 2 {
|
|
||||||
if _, err := client.Get(server.URL); err == nil {
|
|
||||||
t.Fatal("Get() error = nil, want TLS failure")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if got, want := server.Profiles(), []httpclienttest.Profile{
|
|
||||||
httpclienttest.ProfileNative,
|
|
||||||
httpclienttest.ProfileChrome,
|
|
||||||
httpclienttest.ProfileNative,
|
|
||||||
httpclienttest.ProfileChrome,
|
|
||||||
}; !slices.Equal(got, want) {
|
|
||||||
t.Fatalf("ClientHello profiles = %v, want %v", got, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAutoWithoutSessionDoesNotCacheSelectedBrowser(t *testing.T) {
|
|
||||||
var requests atomic.Int32
|
|
||||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) {
|
|
||||||
requests.Add(1)
|
|
||||||
if request.Header.Get("Sec-Ch-Ua") == "" {
|
|
||||||
w.WriteHeader(http.StatusForbidden)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
w.WriteHeader(http.StatusOK)
|
|
||||||
}))
|
|
||||||
t.Cleanup(server.Close)
|
|
||||||
|
|
||||||
client, err := httpclient.NewClient(httpclient.Options{
|
|
||||||
Impersonation: httpclient.ImpersonationOptions{Mode: httpclient.ImpersonationAuto},
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("NewClient() error = %v", err)
|
|
||||||
}
|
|
||||||
for range 2 {
|
|
||||||
response, err := client.Get(server.URL)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Get() error = %v", err)
|
|
||||||
}
|
|
||||||
response.Body.Close()
|
|
||||||
}
|
|
||||||
|
|
||||||
if requests.Load() != 4 {
|
|
||||||
t.Fatalf("requests = %d, want 4 (native + fallback without caching)", requests.Load())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAutoSharesSelectedBrowserAcrossClientsInSession(t *testing.T) {
|
|
||||||
var requests atomic.Int32
|
|
||||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) {
|
|
||||||
requests.Add(1)
|
|
||||||
if request.Header.Get("Sec-Ch-Ua") == "" {
|
|
||||||
w.WriteHeader(http.StatusForbidden)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
w.WriteHeader(http.StatusOK)
|
|
||||||
}))
|
|
||||||
t.Cleanup(server.Close)
|
|
||||||
session := httpclient.NewImpersonationSession()
|
|
||||||
|
|
||||||
for range 2 {
|
|
||||||
client, err := httpclient.NewClient(httpclient.Options{
|
|
||||||
Impersonation: httpclient.ImpersonationOptions{
|
|
||||||
Mode: httpclient.ImpersonationAuto,
|
|
||||||
Session: session,
|
|
||||||
},
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("NewClient() error = %v", err)
|
|
||||||
}
|
|
||||||
request, err := http.NewRequest(http.MethodGet, server.URL, nil)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("NewRequest() error = %v", err)
|
|
||||||
}
|
|
||||||
request.Header.Set("User-Agent", "Mozilla/5.0 Chrome/120.0.0.0 Safari/537.36")
|
|
||||||
response, err := client.Do(request)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Do() error = %v", err)
|
|
||||||
}
|
|
||||||
response.Body.Close()
|
|
||||||
}
|
|
||||||
|
|
||||||
if requests.Load() != 3 {
|
|
||||||
t.Fatalf("requests = %d, want 3 (native + fallback + session pinned)", requests.Load())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAutoDoesNotShareSelectedBrowserAcrossSessions(t *testing.T) {
|
|
||||||
var requests atomic.Int32
|
|
||||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) {
|
|
||||||
requests.Add(1)
|
|
||||||
if request.Header.Get("Sec-Ch-Ua") == "" {
|
|
||||||
w.WriteHeader(http.StatusForbidden)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
w.WriteHeader(http.StatusOK)
|
|
||||||
}))
|
|
||||||
t.Cleanup(server.Close)
|
|
||||||
|
|
||||||
for range 2 {
|
|
||||||
client, err := httpclient.NewClient(httpclient.Options{
|
|
||||||
Impersonation: httpclient.ImpersonationOptions{
|
|
||||||
Mode: httpclient.ImpersonationAuto,
|
|
||||||
Session: httpclient.NewImpersonationSession(),
|
|
||||||
},
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("NewClient() error = %v", err)
|
|
||||||
}
|
|
||||||
response, err := client.Get(server.URL)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Get() error = %v", err)
|
|
||||||
}
|
|
||||||
response.Body.Close()
|
|
||||||
}
|
|
||||||
|
|
||||||
if requests.Load() != 4 {
|
|
||||||
t.Fatalf("requests = %d, want 4 (native + fallback per session)", requests.Load())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAutoCachesSelectedBrowserByFullURL(t *testing.T) {
|
|
||||||
var publicBrowserRequests atomic.Int32
|
|
||||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) {
|
|
||||||
browserRequest := request.Header.Get("Sec-Ch-Ua") != ""
|
|
||||||
switch request.URL.Query().Get("access") {
|
|
||||||
case "protected":
|
|
||||||
if !browserRequest {
|
|
||||||
w.WriteHeader(http.StatusForbidden)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
case "public":
|
|
||||||
if browserRequest {
|
|
||||||
publicBrowserRequests.Add(1)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
w.WriteHeader(http.StatusOK)
|
|
||||||
}))
|
|
||||||
t.Cleanup(server.Close)
|
|
||||||
|
|
||||||
client, err := httpclient.NewClient(httpclient.Options{
|
|
||||||
Impersonation: httpclient.ImpersonationOptions{
|
|
||||||
Mode: httpclient.ImpersonationAuto,
|
|
||||||
Session: httpclient.NewImpersonationSession(),
|
|
||||||
},
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("NewClient() error = %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
for range 2 {
|
|
||||||
response, err := client.Get(server.URL + "/file?access=protected")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("protected Get() error = %v", err)
|
|
||||||
}
|
|
||||||
response.Body.Close()
|
|
||||||
}
|
|
||||||
response, err := client.Get(server.URL + "/file?access=public")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("public Get() error = %v", err)
|
|
||||||
}
|
|
||||||
response.Body.Close()
|
|
||||||
|
|
||||||
if publicBrowserRequests.Load() != 0 {
|
|
||||||
t.Fatalf("public browser requests = %d, want 0", publicBrowserRequests.Load())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestImpersonationSessionClearRemovesSelections(t *testing.T) {
|
|
||||||
var requests atomic.Int32
|
|
||||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) {
|
|
||||||
requests.Add(1)
|
|
||||||
if request.Header.Get("Sec-Ch-Ua") == "" {
|
|
||||||
w.WriteHeader(http.StatusForbidden)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
w.WriteHeader(http.StatusOK)
|
|
||||||
}))
|
|
||||||
t.Cleanup(server.Close)
|
|
||||||
|
|
||||||
session := httpclient.NewImpersonationSession()
|
|
||||||
client, err := httpclient.NewClient(httpclient.Options{
|
|
||||||
Impersonation: httpclient.ImpersonationOptions{
|
|
||||||
Mode: httpclient.ImpersonationAuto,
|
|
||||||
Session: session,
|
|
||||||
},
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("NewClient() error = %v", err)
|
|
||||||
}
|
|
||||||
for range 2 {
|
|
||||||
response, err := client.Get(server.URL)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Get() error = %v", err)
|
|
||||||
}
|
|
||||||
response.Body.Close()
|
|
||||||
session.Clear()
|
|
||||||
}
|
|
||||||
|
|
||||||
if requests.Load() != 4 {
|
|
||||||
t.Fatalf("requests = %d, want 4 (native + fallback after each clear)", requests.Load())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestGlobalAutoSelectionDoesNotAffectNativeMode(t *testing.T) {
|
|
||||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) {
|
|
||||||
if request.Header.Get("Sec-Ch-Ua") == "" {
|
|
||||||
w.WriteHeader(http.StatusForbidden)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
w.WriteHeader(http.StatusOK)
|
|
||||||
}))
|
|
||||||
t.Cleanup(server.Close)
|
|
||||||
|
|
||||||
autoClient, err := httpclient.NewClient(httpclient.Options{
|
|
||||||
Impersonation: httpclient.ImpersonationOptions{Mode: httpclient.ImpersonationAuto},
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("NewClient(auto) error = %v", err)
|
|
||||||
}
|
|
||||||
autoResponse, err := autoClient.Get(server.URL)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("auto Get() error = %v", err)
|
|
||||||
}
|
|
||||||
autoResponse.Body.Close()
|
|
||||||
if autoResponse.StatusCode != http.StatusOK {
|
|
||||||
t.Fatalf("auto status = %d, want %d", autoResponse.StatusCode, http.StatusOK)
|
|
||||||
}
|
|
||||||
|
|
||||||
nativeClient, err := httpclient.NewClient(httpclient.Options{
|
|
||||||
Impersonation: httpclient.ImpersonationOptions{Mode: httpclient.ImpersonationNative},
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("NewClient(native) error = %v", err)
|
|
||||||
}
|
|
||||||
nativeResponse, err := nativeClient.Get(server.URL)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("native Get() error = %v", err)
|
|
||||||
}
|
|
||||||
nativeResponse.Body.Close()
|
|
||||||
if nativeResponse.StatusCode != http.StatusForbidden {
|
|
||||||
t.Fatalf("native status = %d, want %d", nativeResponse.StatusCode, http.StatusForbidden)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAutoAppliesFallbackResponseCookiesBeforeRetry(t *testing.T) {
|
|
||||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) {
|
|
||||||
if request.Header.Get("Sec-Ch-Ua") == "" {
|
|
||||||
http.SetCookie(w, &http.Cookie{Name: "challenge", Value: "passed", Path: "/"})
|
|
||||||
w.WriteHeader(http.StatusForbidden)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
cookie, err := request.Cookie("challenge")
|
|
||||||
if err != nil || cookie.Value != "passed" {
|
|
||||||
w.WriteHeader(http.StatusForbidden)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
w.WriteHeader(http.StatusOK)
|
|
||||||
}))
|
|
||||||
t.Cleanup(server.Close)
|
|
||||||
|
|
||||||
jar, err := cookiejar.New(nil)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("cookiejar.New() error = %v", err)
|
|
||||||
}
|
|
||||||
client, err := httpclient.NewClient(httpclient.Options{
|
|
||||||
Client: httpclient.ClientOptions{Jar: jar},
|
|
||||||
Impersonation: httpclient.ImpersonationOptions{Mode: httpclient.ImpersonationAuto},
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("NewClient() error = %v", err)
|
|
||||||
}
|
|
||||||
response, err := client.Get(server.URL)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Get() error = %v", err)
|
|
||||||
}
|
|
||||||
response.Body.Close()
|
|
||||||
if response.StatusCode != http.StatusOK {
|
|
||||||
t.Fatalf("status = %d, want %d", response.StatusCode, http.StatusOK)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAutoDoesNotRetryRequestWithNonReplayableBody(t *testing.T) {
|
|
||||||
var requests atomic.Int32
|
|
||||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
|
||||||
requests.Add(1)
|
|
||||||
w.WriteHeader(http.StatusForbidden)
|
|
||||||
}))
|
|
||||||
t.Cleanup(server.Close)
|
|
||||||
|
|
||||||
client, err := httpclient.NewClient(httpclient.Options{
|
|
||||||
Impersonation: httpclient.ImpersonationOptions{Mode: httpclient.ImpersonationAuto},
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("NewClient() error = %v", err)
|
|
||||||
}
|
|
||||||
bodyReader, bodyWriter := io.Pipe()
|
|
||||||
t.Cleanup(func() {
|
|
||||||
bodyReader.Close()
|
|
||||||
bodyWriter.Close()
|
|
||||||
})
|
|
||||||
go func() {
|
|
||||||
_, _ = bodyWriter.Write([]byte("payload"))
|
|
||||||
_ = bodyWriter.Close()
|
|
||||||
}()
|
|
||||||
request, err := http.NewRequest(http.MethodPost, server.URL, bodyReader)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("NewRequest() error = %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
response, err := client.Do(request)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Do() error = %v", err)
|
|
||||||
}
|
|
||||||
response.Body.Close()
|
|
||||||
if requests.Load() != 1 {
|
|
||||||
t.Fatalf("requests = %d, want 1", requests.Load())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAutoDoesNotRetryReplayablePost(t *testing.T) {
|
|
||||||
var requests atomic.Int32
|
|
||||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
|
||||||
requests.Add(1)
|
|
||||||
w.WriteHeader(http.StatusForbidden)
|
|
||||||
}))
|
|
||||||
t.Cleanup(server.Close)
|
|
||||||
|
|
||||||
client, err := httpclient.NewClient(httpclient.Options{
|
|
||||||
Impersonation: httpclient.ImpersonationOptions{Mode: httpclient.ImpersonationAuto},
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("NewClient() error = %v", err)
|
|
||||||
}
|
|
||||||
request, err := http.NewRequest(http.MethodPost, server.URL, strings.NewReader("payload"))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("NewRequest() error = %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
response, err := client.Do(request)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Do() error = %v", err)
|
|
||||||
}
|
|
||||||
response.Body.Close()
|
|
||||||
if requests.Load() != 1 {
|
|
||||||
t.Fatalf("requests = %d, want 1", requests.Load())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestNativeUsesStandardTransportWithHTTP2(t *testing.T) {
|
|
||||||
server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
|
||||||
w.WriteHeader(http.StatusOK)
|
|
||||||
}))
|
|
||||||
server.EnableHTTP2 = true
|
|
||||||
server.StartTLS()
|
|
||||||
t.Cleanup(server.Close)
|
|
||||||
|
|
||||||
client, err := httpclient.NewClient(httpclient.Options{
|
|
||||||
Transport: httpclient.TransportOptions{
|
|
||||||
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
|
|
||||||
},
|
|
||||||
Impersonation: httpclient.ImpersonationOptions{
|
|
||||||
Mode: httpclient.ImpersonationNative,
|
|
||||||
},
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("NewClient() error = %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
response, err := client.Get(server.URL)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Get() error = %v", err)
|
|
||||||
}
|
|
||||||
defer response.Body.Close()
|
|
||||||
|
|
||||||
if response.ProtoMajor != 2 {
|
|
||||||
t.Fatalf("protocol = %q, want HTTP/2", response.Proto)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestClientOptionsPreserveCookiesAcrossRedirects(t *testing.T) {
|
|
||||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) {
|
|
||||||
switch request.URL.Path {
|
|
||||||
case "/start":
|
|
||||||
http.SetCookie(w, &http.Cookie{Name: "session", Value: "gopeed", Path: "/"})
|
|
||||||
http.Redirect(w, request, "/download", http.StatusFound)
|
|
||||||
case "/download":
|
|
||||||
cookie, err := request.Cookie("session")
|
|
||||||
if err != nil || cookie.Value != "gopeed" {
|
|
||||||
http.Error(w, "cookie missing", http.StatusBadRequest)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
w.WriteHeader(http.StatusOK)
|
|
||||||
default:
|
|
||||||
http.NotFound(w, request)
|
|
||||||
}
|
|
||||||
}))
|
|
||||||
t.Cleanup(server.Close)
|
|
||||||
|
|
||||||
jar, err := cookiejar.New(nil)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("cookiejar.New() error = %v", err)
|
|
||||||
}
|
|
||||||
var redirects atomic.Int32
|
|
||||||
client, err := httpclient.NewClient(httpclient.Options{
|
|
||||||
Client: httpclient.ClientOptions{
|
|
||||||
Jar: jar,
|
|
||||||
CheckRedirect: func(_ *http.Request, _ []*http.Request) error {
|
|
||||||
redirects.Add(1)
|
|
||||||
return nil
|
|
||||||
},
|
|
||||||
},
|
|
||||||
Impersonation: httpclient.ImpersonationOptions{Mode: httpclient.ImpersonationNative},
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("NewClient() error = %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
response, err := client.Get(server.URL + "/start")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Get() error = %v", err)
|
|
||||||
}
|
|
||||||
defer response.Body.Close()
|
|
||||||
|
|
||||||
if response.StatusCode != http.StatusOK {
|
|
||||||
t.Fatalf("status = %d, want %d", response.StatusCode, http.StatusOK)
|
|
||||||
}
|
|
||||||
if redirects.Load() != 1 {
|
|
||||||
t.Fatalf("redirect calls = %d, want 1", redirects.Load())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestClientTimeoutCancelsRequest(t *testing.T) {
|
|
||||||
server := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, request *http.Request) {
|
|
||||||
<-request.Context().Done()
|
|
||||||
}))
|
|
||||||
t.Cleanup(server.Close)
|
|
||||||
|
|
||||||
client, err := httpclient.NewClient(httpclient.Options{
|
|
||||||
Client: httpclient.ClientOptions{Timeout: 50 * time.Millisecond},
|
|
||||||
Impersonation: httpclient.ImpersonationOptions{
|
|
||||||
Mode: httpclient.ImpersonationNative,
|
|
||||||
},
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("NewClient() error = %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err = client.Get(server.URL)
|
|
||||||
if err == nil {
|
|
||||||
t.Fatal("Get() error = nil, want timeout")
|
|
||||||
}
|
|
||||||
var networkError net.Error
|
|
||||||
if !errors.As(err, &networkError) || !networkError.Timeout() {
|
|
||||||
t.Fatalf("Get() error = %v, want net.Error timeout", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestTransportOptionsUseProxyInNativeAndFixedModes(t *testing.T) {
|
|
||||||
proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) {
|
|
||||||
if request.URL.Host != "download.example" {
|
|
||||||
http.Error(w, "unexpected target", http.StatusBadRequest)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
w.WriteHeader(http.StatusOK)
|
|
||||||
}))
|
|
||||||
t.Cleanup(proxy.Close)
|
|
||||||
proxyURL, err := url.Parse(proxy.URL)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("url.Parse() error = %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
impersonation httpclient.ImpersonationOptions
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
name: "native",
|
|
||||||
impersonation: httpclient.ImpersonationOptions{Mode: httpclient.ImpersonationNative},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "fixed chrome",
|
|
||||||
impersonation: httpclient.ImpersonationOptions{
|
|
||||||
Mode: httpclient.ImpersonationFixed,
|
|
||||||
Browser: httpclient.BrowserChrome,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
for _, test := range tests {
|
|
||||||
t.Run(test.name, func(t *testing.T) {
|
|
||||||
client, err := httpclient.NewClient(httpclient.Options{
|
|
||||||
Transport: httpclient.TransportOptions{Proxy: http.ProxyURL(proxyURL)},
|
|
||||||
Impersonation: test.impersonation,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("NewClient() error = %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
response, err := client.Get("http://download.example/file")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Get() error = %v", err)
|
|
||||||
}
|
|
||||||
defer response.Body.Close()
|
|
||||||
if response.StatusCode != http.StatusOK {
|
|
||||||
t.Fatalf("status = %d, want %d", response.StatusCode, http.StatusOK)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestTransportOptionsUseDialContextInNativeAndFixedModes(t *testing.T) {
|
|
||||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
|
||||||
w.WriteHeader(http.StatusOK)
|
|
||||||
}))
|
|
||||||
t.Cleanup(server.Close)
|
|
||||||
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
impersonation httpclient.ImpersonationOptions
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
name: "native",
|
|
||||||
impersonation: httpclient.ImpersonationOptions{Mode: httpclient.ImpersonationNative},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "fixed chrome",
|
|
||||||
impersonation: httpclient.ImpersonationOptions{
|
|
||||||
Mode: httpclient.ImpersonationFixed,
|
|
||||||
Browser: httpclient.BrowserChrome,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
for _, test := range tests {
|
|
||||||
t.Run(test.name, func(t *testing.T) {
|
|
||||||
var calls atomic.Int32
|
|
||||||
dialer := &net.Dialer{}
|
|
||||||
client, err := httpclient.NewClient(httpclient.Options{
|
|
||||||
Transport: httpclient.TransportOptions{
|
|
||||||
DialContext: func(ctx context.Context, network, address string) (net.Conn, error) {
|
|
||||||
calls.Add(1)
|
|
||||||
return dialer.DialContext(ctx, network, address)
|
|
||||||
},
|
|
||||||
},
|
|
||||||
Impersonation: test.impersonation,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("NewClient() error = %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
response, err := client.Get(server.URL)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Get() error = %v", err)
|
|
||||||
}
|
|
||||||
response.Body.Close()
|
|
||||||
if calls.Load() == 0 {
|
|
||||||
t.Fatal("DialContext() was not called")
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestTransportOptionsApplyTLSHandshakeTimeoutInNativeAndFixedModes(t *testing.T) {
|
|
||||||
listener, err := net.Listen("tcp", "127.0.0.1:0")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("net.Listen() error = %v", err)
|
|
||||||
}
|
|
||||||
closed := make(chan struct{})
|
|
||||||
t.Cleanup(func() {
|
|
||||||
close(closed)
|
|
||||||
listener.Close()
|
|
||||||
})
|
|
||||||
go func() {
|
|
||||||
for {
|
|
||||||
connection, err := listener.Accept()
|
|
||||||
if err != nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
go func() {
|
|
||||||
<-closed
|
|
||||||
connection.Close()
|
|
||||||
}()
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
impersonation httpclient.ImpersonationOptions
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
name: "native",
|
|
||||||
impersonation: httpclient.ImpersonationOptions{Mode: httpclient.ImpersonationNative},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "fixed chrome",
|
|
||||||
impersonation: httpclient.ImpersonationOptions{
|
|
||||||
Mode: httpclient.ImpersonationFixed,
|
|
||||||
Browser: httpclient.BrowserChrome,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
for _, test := range tests {
|
|
||||||
t.Run(test.name, func(t *testing.T) {
|
|
||||||
client, err := httpclient.NewClient(httpclient.Options{
|
|
||||||
Client: httpclient.ClientOptions{Timeout: time.Second},
|
|
||||||
Transport: httpclient.TransportOptions{
|
|
||||||
TLSHandshakeTimeout: 50 * time.Millisecond,
|
|
||||||
},
|
|
||||||
Impersonation: test.impersonation,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("NewClient() error = %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
started := time.Now()
|
|
||||||
_, err = client.Get("https://" + listener.Addr().String())
|
|
||||||
if err == nil {
|
|
||||||
t.Fatal("Get() error = nil, want TLS handshake timeout")
|
|
||||||
}
|
|
||||||
if elapsed := time.Since(started); elapsed >= 500*time.Millisecond {
|
|
||||||
t.Fatalf("TLS handshake took %s, want less than 500ms", elapsed)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestFixedBrowserAddsProfileHeadersWithoutOverwritingExplicitHeaders(t *testing.T) {
|
|
||||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) {
|
|
||||||
w.Header().Set("Seen-User-Agent", request.UserAgent())
|
|
||||||
w.WriteHeader(http.StatusOK)
|
|
||||||
}))
|
|
||||||
t.Cleanup(server.Close)
|
|
||||||
|
|
||||||
client, err := httpclient.NewClient(httpclient.Options{
|
|
||||||
Impersonation: httpclient.ImpersonationOptions{
|
|
||||||
Mode: httpclient.ImpersonationFixed,
|
|
||||||
Browser: httpclient.BrowserChrome,
|
|
||||||
},
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("NewClient() error = %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
response, err := client.Get(server.URL)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Get() error = %v", err)
|
|
||||||
}
|
|
||||||
response.Body.Close()
|
|
||||||
if userAgent := response.Header.Get("Seen-User-Agent"); !strings.Contains(userAgent, "Chrome/120") {
|
|
||||||
t.Fatalf("profile User-Agent = %q, want Chrome/120", userAgent)
|
|
||||||
}
|
|
||||||
|
|
||||||
request, err := http.NewRequest(http.MethodGet, server.URL, nil)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("NewRequest() error = %v", err)
|
|
||||||
}
|
|
||||||
request.Header.Set("User-Agent", "Gopeed-Custom/1.0")
|
|
||||||
response, err = client.Do(request)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Do() error = %v", err)
|
|
||||||
}
|
|
||||||
response.Body.Close()
|
|
||||||
if userAgent := response.Header.Get("Seen-User-Agent"); userAgent != "Gopeed-Custom/1.0" {
|
|
||||||
t.Fatalf("explicit User-Agent = %q, want Gopeed-Custom/1.0", userAgent)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestFixedClientCloseIdleConnectionsReachesBrowserTransport(t *testing.T) {
|
|
||||||
var connections atomic.Int32
|
|
||||||
server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
|
||||||
w.WriteHeader(http.StatusOK)
|
|
||||||
}))
|
|
||||||
server.Config.ConnState = func(_ net.Conn, state http.ConnState) {
|
|
||||||
if state == http.StateNew {
|
|
||||||
connections.Add(1)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
server.Start()
|
|
||||||
t.Cleanup(server.Close)
|
|
||||||
|
|
||||||
client, err := httpclient.NewClient(httpclient.Options{
|
|
||||||
Impersonation: httpclient.ImpersonationOptions{
|
|
||||||
Mode: httpclient.ImpersonationFixed,
|
|
||||||
Browser: httpclient.BrowserChrome,
|
|
||||||
},
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("NewClient() error = %v", err)
|
|
||||||
}
|
|
||||||
for attempt := range 2 {
|
|
||||||
response, err := client.Get(server.URL)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Get() error = %v", err)
|
|
||||||
}
|
|
||||||
_, _ = io.Copy(io.Discard, response.Body)
|
|
||||||
response.Body.Close()
|
|
||||||
if attempt == 0 {
|
|
||||||
client.CloseIdleConnections()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if connections.Load() != 2 {
|
|
||||||
t.Fatalf("connections = %d, want 2", connections.Load())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,100 +0,0 @@
|
|||||||
package httpclienttest
|
|
||||||
|
|
||||||
import (
|
|
||||||
"crypto/tls"
|
|
||||||
"errors"
|
|
||||||
"io"
|
|
||||||
"log"
|
|
||||||
"net/http"
|
|
||||||
"net/http/httptest"
|
|
||||||
"sync"
|
|
||||||
"testing"
|
|
||||||
)
|
|
||||||
|
|
||||||
type Profile string
|
|
||||||
|
|
||||||
const (
|
|
||||||
ProfileNative Profile = "native"
|
|
||||||
ProfileChrome Profile = "chrome"
|
|
||||||
ProfileFirefox Profile = "firefox"
|
|
||||||
ProfileSafari Profile = "safari"
|
|
||||||
ProfileAnyBrowser Profile = "browser"
|
|
||||||
)
|
|
||||||
|
|
||||||
type FingerprintServerOptions struct {
|
|
||||||
RequiredProfile Profile
|
|
||||||
Handler http.Handler
|
|
||||||
}
|
|
||||||
|
|
||||||
type FingerprintServer struct {
|
|
||||||
*httptest.Server
|
|
||||||
|
|
||||||
mu sync.Mutex
|
|
||||||
profiles []Profile
|
|
||||||
}
|
|
||||||
|
|
||||||
func NewFingerprintServer(t testing.TB, options FingerprintServerOptions) *FingerprintServer {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
handler := options.Handler
|
|
||||||
if handler == nil {
|
|
||||||
handler = http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
|
||||||
w.WriteHeader(http.StatusOK)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
server := &FingerprintServer{
|
|
||||||
Server: httptest.NewUnstartedServer(handler),
|
|
||||||
}
|
|
||||||
server.EnableHTTP2 = true
|
|
||||||
server.Config.ErrorLog = log.New(io.Discard, "", 0)
|
|
||||||
server.TLS = &tls.Config{
|
|
||||||
GetConfigForClient: func(hello *tls.ClientHelloInfo) (*tls.Config, error) {
|
|
||||||
profile := classifyProfile(hello.CipherSuites)
|
|
||||||
server.mu.Lock()
|
|
||||||
server.profiles = append(server.profiles, profile)
|
|
||||||
server.mu.Unlock()
|
|
||||||
if profileMatches(profile, options.RequiredProfile) {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
return nil, errors.New("required TLS fingerprint was not presented")
|
|
||||||
},
|
|
||||||
}
|
|
||||||
server.StartTLS()
|
|
||||||
t.Cleanup(server.Close)
|
|
||||||
return server
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *FingerprintServer) Profiles() []Profile {
|
|
||||||
s.mu.Lock()
|
|
||||||
defer s.mu.Unlock()
|
|
||||||
return append([]Profile(nil), s.profiles...)
|
|
||||||
}
|
|
||||||
|
|
||||||
func profileMatches(actual, required Profile) bool {
|
|
||||||
if required == ProfileAnyBrowser {
|
|
||||||
return actual != ProfileNative
|
|
||||||
}
|
|
||||||
return actual == required
|
|
||||||
}
|
|
||||||
|
|
||||||
func classifyProfile(cipherSuites []uint16) Profile {
|
|
||||||
if len(cipherSuites) >= 5 && isGREASE(cipherSuites[0]) {
|
|
||||||
switch cipherSuites[4] {
|
|
||||||
case tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256:
|
|
||||||
return ProfileChrome
|
|
||||||
case tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384:
|
|
||||||
return ProfileSafari
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if len(cipherSuites) >= 3 &&
|
|
||||||
cipherSuites[0] == tls.TLS_AES_128_GCM_SHA256 &&
|
|
||||||
cipherSuites[1] == tls.TLS_CHACHA20_POLY1305_SHA256 &&
|
|
||||||
cipherSuites[2] == tls.TLS_AES_256_GCM_SHA384 {
|
|
||||||
return ProfileFirefox
|
|
||||||
}
|
|
||||||
return ProfileNative
|
|
||||||
}
|
|
||||||
|
|
||||||
func isGREASE(value uint16) bool {
|
|
||||||
return value&0x0f0f == 0x0a0a && byte(value) == byte(value>>8)
|
|
||||||
}
|
|
||||||
@@ -1,35 +0,0 @@
|
|||||||
// Package production associates generated resources with their input progress.
|
|
||||||
package production
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"sync"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Source counts unique input bytes and total received bytes (including refetches).
|
|
||||||
// Implementations must support concurrent snapshots.
|
|
||||||
type Source interface {
|
|
||||||
Progress() (downloaded, received int64)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Reader resolves its optional producer before response headers are sent.
|
|
||||||
// A nil producer preserves ordinary Blob behavior.
|
|
||||||
type Reader interface {
|
|
||||||
Production() Source
|
|
||||||
WaitProduction(context.Context) error
|
|
||||||
}
|
|
||||||
type Registry struct {
|
|
||||||
mu sync.Mutex
|
|
||||||
sources map[string]Source
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *Registry) Set(id string, source Source) {
|
|
||||||
r.mu.Lock()
|
|
||||||
defer r.mu.Unlock()
|
|
||||||
if r.sources == nil {
|
|
||||||
r.sources = make(map[string]Source)
|
|
||||||
}
|
|
||||||
r.sources[id] = source
|
|
||||||
}
|
|
||||||
func (r *Registry) Get(id string) Source { r.mu.Lock(); defer r.mu.Unlock(); return r.sources[id] }
|
|
||||||
func (r *Registry) Remove(id string) { r.mu.Lock(); defer r.mu.Unlock(); delete(r.sources, id) }
|
|
||||||
@@ -1,37 +1,6 @@
|
|||||||
package bt
|
package bt
|
||||||
|
|
||||||
import (
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/GopeedLab/gopeed/internal/fetcher"
|
|
||||||
)
|
|
||||||
|
|
||||||
type trackerSubscriptionStatus struct {
|
|
||||||
Success bool `json:"success"`
|
|
||||||
TrackerCount int `json:"trackerCount"`
|
|
||||||
LastFetchTime time.Time `json:"lastFetchTime"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type state struct {
|
|
||||||
TrackerSubscriptionStatuses map[string]trackerSubscriptionStatus `json:"trackerSubscriptionStatuses"`
|
|
||||||
TrackerSources map[string][]string `json:"trackerSources"`
|
|
||||||
LastTrackerUpdateTime time.Time `json:"lastTrackerUpdateTime"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type config struct {
|
type config struct {
|
||||||
fetcher.ConfigState[state]
|
ListenPort int `json:"listenPort"`
|
||||||
TrackerSubscribeURLs []string `json:"trackerSubscribeUrls"`
|
Trackers []string `json:"trackers"`
|
||||||
AutoUpdateTrackers bool `json:"autoUpdateTrackers"`
|
|
||||||
CustomTrackers []string `json:"customTrackers"`
|
|
||||||
ListenPort int `json:"listenPort"`
|
|
||||||
// SeedKeep is always keep seeding after downloading is complete, unless manually stopped.
|
|
||||||
SeedKeep bool `json:"seedKeep"`
|
|
||||||
// SeedRatio is the ratio of uploaded data to downloaded data to seed.
|
|
||||||
SeedRatio float64 `json:"seedRatio"`
|
|
||||||
// SeedTime is the time in seconds to seed after downloading is complete.
|
|
||||||
SeedTime int64 `json:"seedTime"`
|
|
||||||
// DisableDHT and DisablePEX turn off DHT and PEX peer discovery. Like ListenPort,
|
|
||||||
// they are read when the shared client is built.
|
|
||||||
DisableDHT bool `json:"disableDht"`
|
|
||||||
DisablePEX bool `json:"disablePex"`
|
|
||||||
}
|
}
|
||||||
@@ -1,49 +0,0 @@
|
|||||||
package bt
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"net"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/rs/dnscache"
|
|
||||||
)
|
|
||||||
|
|
||||||
// DnsCacheResolver resolves DNS requests for an HTTP client using an in-memory cache.
|
|
||||||
type DnsCacheResolver struct {
|
|
||||||
RefreshTimeout time.Duration
|
|
||||||
|
|
||||||
resolver dnscache.Resolver
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *DnsCacheResolver) DialContext(ctx context.Context, network, address string) (net.Conn, error) {
|
|
||||||
host, port, err := net.SplitHostPort(address)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
ips, err := r.resolver.LookupHost(ctx, host)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
var conn net.Conn
|
|
||||||
for _, ip := range ips {
|
|
||||||
var dialer net.Dialer
|
|
||||||
conn, err = dialer.DialContext(ctx, network, net.JoinHostPort(ip, port))
|
|
||||||
if err == nil {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return conn, err
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *DnsCacheResolver) Run(ctx context.Context) {
|
|
||||||
ticker := time.NewTicker(r.RefreshTimeout)
|
|
||||||
defer ticker.Stop()
|
|
||||||
for {
|
|
||||||
select {
|
|
||||||
case <-ctx.Done():
|
|
||||||
return
|
|
||||||
case <-ticker.C:
|
|
||||||
r.resolver.Refresh(true)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,464 +0,0 @@
|
|||||||
package bt
|
|
||||||
|
|
||||||
import (
|
|
||||||
"encoding/json"
|
|
||||||
"net/url"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
|
||||||
"sync/atomic"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/GopeedLab/gopeed/internal/controller"
|
|
||||||
"github.com/GopeedLab/gopeed/internal/fetcher"
|
|
||||||
"github.com/GopeedLab/gopeed/pkg/base"
|
|
||||||
btprotocol "github.com/GopeedLab/gopeed/pkg/protocol/bt"
|
|
||||||
"github.com/anacrolix/torrent"
|
|
||||||
pp "github.com/anacrolix/torrent/peer_protocol"
|
|
||||||
"golang.org/x/time/rate"
|
|
||||||
)
|
|
||||||
|
|
||||||
type btDownloadHarness struct {
|
|
||||||
*Fetcher
|
|
||||||
pendingRequests atomic.Int64
|
|
||||||
}
|
|
||||||
|
|
||||||
// These tests exercise the BT fetcher, native storage and real peer protocol.
|
|
||||||
// All content is generated locally and served by a loopback seeder.
|
|
||||||
func btDownloadFetcher(t *testing.T, f fetcher.Fetcher) *btDownloadHarness {
|
|
||||||
t.Helper()
|
|
||||||
if f == nil {
|
|
||||||
f = new(FetcherManager).Build()
|
|
||||||
}
|
|
||||||
ctl := controller.NewController()
|
|
||||||
ctl.GetConfig = func(v any) {
|
|
||||||
encoded, err := json.Marshal(new(FetcherManager).DefaultConfig())
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := json.Unmarshal(encoded, v); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
f.Setup(ctl)
|
|
||||||
t.Cleanup(func() {
|
|
||||||
if err := f.Close(); err != nil {
|
|
||||||
t.Error(err)
|
|
||||||
}
|
|
||||||
if err := closeClient(); err != nil {
|
|
||||||
t.Error(err)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
h := &btDownloadHarness{Fetcher: f.(*Fetcher)}
|
|
||||||
// Keep the test client on loopback and observe real cancellation replies.
|
|
||||||
// Waiting for them avoids the pinned library's immediate cancel/resume
|
|
||||||
// request scheduling bug, without relying on arbitrary sleep durations.
|
|
||||||
cfg = torrent.TestingConfig(t)
|
|
||||||
cfg.DisableTrackers = false
|
|
||||||
cfg.Seed = true
|
|
||||||
cfg.DisableUTP = true
|
|
||||||
cfg.DisableIPv6 = true
|
|
||||||
cfg.DisableWebtorrent = true
|
|
||||||
cfg.KeepAliveTimeout = time.Minute
|
|
||||||
cfg.MaxAllocPeerRequestDataPerConn = 1 << 20
|
|
||||||
cfg.Callbacks.SentRequest = []func(torrent.PeerRequestEvent){func(torrent.PeerRequestEvent) {
|
|
||||||
h.pendingRequests.Add(1)
|
|
||||||
}}
|
|
||||||
cfg.Callbacks.ReadMessage = func(_ *torrent.PeerConn, msg *pp.Message) {
|
|
||||||
if !msg.Keepalive && (msg.Type == pp.Piece || msg.Type == pp.Reject) {
|
|
||||||
h.pendingRequests.Add(-1)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
var err error
|
|
||||||
client, err = torrent.NewClient(cfg)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
return h
|
|
||||||
}
|
|
||||||
|
|
||||||
func btResolve(t *testing.T, f *btDownloadHarness, source string) {
|
|
||||||
t.Helper()
|
|
||||||
if err := f.Resolve(&base.Request{URL: source}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func btStart(t *testing.T, f *btDownloadHarness) {
|
|
||||||
t.Helper()
|
|
||||||
if err := f.Start(); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func btAwaitDone(t *testing.T, f *btDownloadHarness) {
|
|
||||||
t.Helper()
|
|
||||||
done := make(chan error, 1)
|
|
||||||
go func() { done <- f.Wait() }()
|
|
||||||
timeout := time.NewTimer(20 * time.Second)
|
|
||||||
defer timeout.Stop()
|
|
||||||
select {
|
|
||||||
case err := <-done:
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("BT download failed: %v", err)
|
|
||||||
}
|
|
||||||
if !f.isDone() {
|
|
||||||
t.Fatal("Wait returned before selected files were verified")
|
|
||||||
}
|
|
||||||
case <-timeout.C:
|
|
||||||
var status strings.Builder
|
|
||||||
client.WriteStatus(&status)
|
|
||||||
t.Log(status.String())
|
|
||||||
t.Fatalf("BT download did not finish: progress=%v stats=%+v pieces=%+v peers=%+v", f.Progress(), f.torrent.Stats(), f.torrent.PieceStateRuns(), f.Stats().Runtime)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func btAwaitPartial(t *testing.T, f *btDownloadHarness) {
|
|
||||||
t.Helper()
|
|
||||||
timeout := time.NewTimer(15 * time.Second)
|
|
||||||
defer timeout.Stop()
|
|
||||||
ticker := time.NewTicker(20 * time.Millisecond)
|
|
||||||
defer ticker.Stop()
|
|
||||||
for {
|
|
||||||
select {
|
|
||||||
case <-ticker.C:
|
|
||||||
progress := f.Progress().TotalDownloaded()
|
|
||||||
if progress >= 32*1024 && progress < f.Meta().Res.Size {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
case <-timeout.C:
|
|
||||||
t.Fatal("BT transfer did not reach partial progress")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func btPause(t *testing.T, f *btDownloadHarness) {
|
|
||||||
t.Helper()
|
|
||||||
if err := f.Pause(); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
timeout := time.NewTimer(5 * time.Second)
|
|
||||||
defer timeout.Stop()
|
|
||||||
ticker := time.NewTicker(10 * time.Millisecond)
|
|
||||||
defer ticker.Stop()
|
|
||||||
for {
|
|
||||||
if f.pendingRequests.Load() == 0 {
|
|
||||||
// The message callback runs under the client lock. Acquiring that
|
|
||||||
// lock ensures the final cancellation reply has finished processing.
|
|
||||||
client.Stats()
|
|
||||||
return
|
|
||||||
}
|
|
||||||
select {
|
|
||||||
case <-ticker.C:
|
|
||||||
case <-timeout.C:
|
|
||||||
t.Fatal("peer did not acknowledge cancelled requests")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestBTDownloadSources(t *testing.T) {
|
|
||||||
for _, kind := range []string{"magnet", "torrent", "file URI", "data URI"} {
|
|
||||||
t.Run(kind, func(t *testing.T) {
|
|
||||||
fixture := newBTFixture(t, map[string][]byte{"": btPayload(96*1024, kind)}, 0)
|
|
||||||
source := fixture.Magnet
|
|
||||||
switch kind {
|
|
||||||
case "torrent":
|
|
||||||
source = fixture.TorrentFile
|
|
||||||
case "file URI":
|
|
||||||
source = fixture.FileURI()
|
|
||||||
case "data URI":
|
|
||||||
source = fixture.DataURI
|
|
||||||
}
|
|
||||||
f := btDownloadFetcher(t, nil)
|
|
||||||
btResolve(t, f, source)
|
|
||||||
if f.Meta().Opts != nil {
|
|
||||||
t.Fatal("Resolve assigned download options")
|
|
||||||
}
|
|
||||||
f.storage.mu.RLock()
|
|
||||||
bound := f.storage.owner != nil
|
|
||||||
f.storage.mu.RUnlock()
|
|
||||||
if bound {
|
|
||||||
t.Fatal("Resolve bound download storage")
|
|
||||||
}
|
|
||||||
resolvedTorrent := f.torrent
|
|
||||||
dir := filepath.Join(t.TempDir(), "chosen-after-resolve")
|
|
||||||
f.Meta().Opts = &base.Options{Path: dir}
|
|
||||||
if _, err := os.Stat(dir); !os.IsNotExist(err) {
|
|
||||||
t.Fatalf("output directory exists before Start: %v", err)
|
|
||||||
}
|
|
||||||
btStart(t, f)
|
|
||||||
if f.torrent != resolvedTorrent {
|
|
||||||
t.Fatal("Start replaced the resolved torrent")
|
|
||||||
}
|
|
||||||
btAwaitDone(t, f)
|
|
||||||
fixture.AssertFiles(t, dir)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestBTDirectDownloadWithExtraTrackers(t *testing.T) {
|
|
||||||
for _, source := range []string{"request", "config", "subscription"} {
|
|
||||||
t.Run(source, func(t *testing.T) {
|
|
||||||
fixture := makeBTFixture(t, map[string][]byte{
|
|
||||||
"a.bin": btPayload(20*1024, "direct-a"),
|
|
||||||
"nested/b.bin": btPayload(48*1024, "direct-b"),
|
|
||||||
}, 0, false, false)
|
|
||||||
magnet, err := url.Parse(fixture.Magnet)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
query := magnet.Query()
|
|
||||||
query.Del("tr")
|
|
||||||
query.Del("x.pe")
|
|
||||||
magnet.RawQuery = query.Encode()
|
|
||||||
req := &base.Request{URL: magnet.String()}
|
|
||||||
f := btDownloadFetcher(t, nil)
|
|
||||||
if source == "request" {
|
|
||||||
req.Extra = &btprotocol.ReqExtra{Trackers: []string{fixture.TrackerURL}}
|
|
||||||
} else {
|
|
||||||
backendConfig := new(FetcherManager).DefaultConfig().(*config)
|
|
||||||
if source == "config" {
|
|
||||||
backendConfig.CustomTrackers = []string{fixture.TrackerURL}
|
|
||||||
} else {
|
|
||||||
backendConfig.TrackerSubscribeURLs = []string{"https://subscription.example/list"}
|
|
||||||
backendConfig.State.TrackerSources[backendConfig.TrackerSubscribeURLs[0]] = []string{fixture.TrackerURL}
|
|
||||||
}
|
|
||||||
f.ctl.GetConfig = func(dst any) {
|
|
||||||
raw, err := json.Marshal(backendConfig)
|
|
||||||
if err != nil {
|
|
||||||
t.Error(err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if err := json.Unmarshal(raw, dst); err != nil {
|
|
||||||
t.Error(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// CreateDirect supplies options before Downloader resolves the task.
|
|
||||||
dir := filepath.Join(t.TempDir(), "direct-download")
|
|
||||||
opts := &base.Options{Path: dir, SelectFiles: []int{}}
|
|
||||||
f.Meta().Opts = opts
|
|
||||||
resolved := make(chan error, 1)
|
|
||||||
go func() { resolved <- f.Resolve(req) }()
|
|
||||||
select {
|
|
||||||
case err := <-resolved:
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
case <-time.After(5 * time.Second):
|
|
||||||
f.torrentDropFunc()
|
|
||||||
<-resolved
|
|
||||||
t.Fatal("direct magnet download did not obtain metadata from its extra tracker")
|
|
||||||
}
|
|
||||||
if f.Meta().Opts != opts || len(opts.SelectFiles) != len(fixture.Files) {
|
|
||||||
t.Fatal("resolution lost direct download options or did not select all files")
|
|
||||||
}
|
|
||||||
btStart(t, f)
|
|
||||||
btAwaitDone(t, f)
|
|
||||||
fixture.AssertFiles(t, dir)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestBTDownloadSelectedFiles(t *testing.T) {
|
|
||||||
fixture := newBTFixture(t, map[string][]byte{
|
|
||||||
"a.bin": btPayload(20*1024, "a"), "nested/b.bin": btPayload(48*1024, "b"),
|
|
||||||
}, 0)
|
|
||||||
f := btDownloadFetcher(t, nil)
|
|
||||||
dir := t.TempDir()
|
|
||||||
btResolve(t, f, fixture.Magnet)
|
|
||||||
selected := -1
|
|
||||||
for i, file := range f.Meta().Res.Files {
|
|
||||||
if file.Name == "b.bin" {
|
|
||||||
selected = i
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if selected < 0 {
|
|
||||||
t.Fatal("selected file absent from metadata")
|
|
||||||
}
|
|
||||||
f.Meta().Opts = &base.Options{Path: dir}
|
|
||||||
if err := f.Patch(nil, &base.Options{SelectFiles: []int{selected}}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
btStart(t, f)
|
|
||||||
btAwaitDone(t, f)
|
|
||||||
fixture.AssertFiles(t, dir, filepath.Join("bundle", "nested", "b.bin"))
|
|
||||||
if _, err := os.Stat(filepath.Join(dir, "bundle", "a.bin")); !os.IsNotExist(err) {
|
|
||||||
t.Fatalf("unselected output remains: %v", err)
|
|
||||||
}
|
|
||||||
if f.Meta().Res.Size != 48*1024 {
|
|
||||||
t.Fatal("selected resource size is incorrect")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestBTPauseAndResume(t *testing.T) {
|
|
||||||
fixture := newBTFixture(t, map[string][]byte{"": btPayload(512*1024, "pause")}, 64*1024)
|
|
||||||
f := btDownloadFetcher(t, nil)
|
|
||||||
dir := t.TempDir()
|
|
||||||
btResolve(t, f, fixture.Magnet)
|
|
||||||
f.Meta().Opts = &base.Options{Path: dir}
|
|
||||||
btStart(t, f)
|
|
||||||
btAwaitPartial(t, f)
|
|
||||||
btPause(t, f)
|
|
||||||
resolvedTorrent := f.torrent
|
|
||||||
fixture.Limiter.SetLimit(rate.Inf)
|
|
||||||
// Once cancelled requests have settled, no more data should arrive.
|
|
||||||
stats := f.torrent.Stats()
|
|
||||||
readWhilePaused := stats.BytesReadData.Int64()
|
|
||||||
time.Sleep(200 * time.Millisecond)
|
|
||||||
stats = f.torrent.Stats()
|
|
||||||
if stats.BytesReadData.Int64() != readWhilePaused || f.isDone() {
|
|
||||||
t.Fatal("data download continued while paused")
|
|
||||||
}
|
|
||||||
btStart(t, f)
|
|
||||||
if f.torrent != resolvedTorrent {
|
|
||||||
t.Fatal("resume replaced the resolved torrent")
|
|
||||||
}
|
|
||||||
btAwaitDone(t, f)
|
|
||||||
fixture.AssertFiles(t, dir)
|
|
||||||
}
|
|
||||||
|
|
||||||
type btFetcherCheckpoint struct {
|
|
||||||
Meta *fetcher.FetcherMeta `json:"meta"`
|
|
||||||
Data json.RawMessage `json:"data"`
|
|
||||||
}
|
|
||||||
|
|
||||||
func btStore(t *testing.T, f *btDownloadHarness) []byte {
|
|
||||||
t.Helper()
|
|
||||||
data, err := new(FetcherManager).Store(f.Fetcher)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
encodedData, err := json.Marshal(data)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
encoded, err := json.Marshal(btFetcherCheckpoint{Meta: f.Meta(), Data: encodedData})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
return encoded
|
|
||||||
}
|
|
||||||
|
|
||||||
func btRestore(t *testing.T, encoded []byte) *btDownloadHarness {
|
|
||||||
t.Helper()
|
|
||||||
var checkpoint btFetcherCheckpoint
|
|
||||||
if err := json.Unmarshal(encoded, &checkpoint); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
data, restore := new(FetcherManager).Restore()
|
|
||||||
if err := json.Unmarshal(checkpoint.Data, data); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
return btDownloadFetcher(t, restore(checkpoint.Meta, data))
|
|
||||||
}
|
|
||||||
|
|
||||||
func btCloseForRestart(t *testing.T, f *btDownloadHarness) {
|
|
||||||
t.Helper()
|
|
||||||
if err := f.Close(); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := new(FetcherManager).Close(); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestBTColdStartResume(t *testing.T) {
|
|
||||||
for _, kind := range []string{"v1", "hybrid"} {
|
|
||||||
t.Run(kind, func(t *testing.T) {
|
|
||||||
var fixture *btFixture
|
|
||||||
if kind == "hybrid" {
|
|
||||||
fixture = newBTHybridFixture(t, btPayload(512*1024, "restart-hybrid"))
|
|
||||||
fixture.Limiter.SetLimit(64 * 1024)
|
|
||||||
} else {
|
|
||||||
fixture = newBTFixture(t, map[string][]byte{"": btPayload(512*1024, "restart")}, 64*1024)
|
|
||||||
}
|
|
||||||
f := btDownloadFetcher(t, nil)
|
|
||||||
dir := t.TempDir()
|
|
||||||
btResolve(t, f, fixture.Magnet)
|
|
||||||
f.Meta().Opts = &base.Options{Path: dir}
|
|
||||||
btStart(t, f)
|
|
||||||
btAwaitPartial(t, f)
|
|
||||||
btPause(t, f)
|
|
||||||
checkpoint := btStore(t, f)
|
|
||||||
hash := f.Meta().Res.Hash
|
|
||||||
btCloseForRestart(t, f)
|
|
||||||
seedStats := fixture.Torrent.Stats()
|
|
||||||
sentBeforeRestart := seedStats.BytesWrittenData.Int64()
|
|
||||||
restored := btRestore(t, checkpoint)
|
|
||||||
if restored.Meta().Opts.Path != dir || restored.Meta().Res.Hash != hash {
|
|
||||||
t.Fatal("download directory or hash was lost")
|
|
||||||
}
|
|
||||||
fixture.Limiter.SetLimit(rate.Inf)
|
|
||||||
btStart(t, restored)
|
|
||||||
btAwaitDone(t, restored)
|
|
||||||
fixture.AssertFiles(t, dir)
|
|
||||||
seedStats = fixture.Torrent.Stats()
|
|
||||||
if sent := seedStats.BytesWrittenData.Int64() - sentBeforeRestart; sent >= 512*1024 {
|
|
||||||
t.Fatalf("restart redownloaded the entire file: %d bytes", sent)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestBTDownloadHybrid(t *testing.T) {
|
|
||||||
fixture := newBTHybridFixture(t, btPayload(80*1024, "hybrid-v2"))
|
|
||||||
f := btDownloadFetcher(t, nil)
|
|
||||||
dir := t.TempDir()
|
|
||||||
btResolve(t, f, fixture.Magnet)
|
|
||||||
f.Meta().Opts = &base.Options{Path: dir}
|
|
||||||
btStart(t, f)
|
|
||||||
btAwaitDone(t, f)
|
|
||||||
fixture.AssertFiles(t, dir)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestBTRepairCorruptedExistingFile(t *testing.T) {
|
|
||||||
fixture := newBTFixture(t, map[string][]byte{"": btPayload(128*1024, "repair")}, 0)
|
|
||||||
dir := t.TempDir()
|
|
||||||
damaged := append([]byte(nil), fixture.Files["payload.bin"]...)
|
|
||||||
damaged[20000] ^= 0xff
|
|
||||||
if err := os.WriteFile(filepath.Join(dir, "payload.bin"), damaged, 0600); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
f := btDownloadFetcher(t, nil)
|
|
||||||
btResolve(t, f, fixture.Magnet)
|
|
||||||
f.Meta().Opts = &base.Options{Path: dir}
|
|
||||||
btStart(t, f)
|
|
||||||
btAwaitDone(t, f)
|
|
||||||
fixture.AssertFiles(t, dir)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestBTPatchSelectionWhilePaused(t *testing.T) {
|
|
||||||
fixture := newBTFixture(t, map[string][]byte{"a.bin": btPayload(256*1024, "patch-a"), "b.bin": btPayload(256*1024, "patch-b")}, 64*1024)
|
|
||||||
f := btDownloadFetcher(t, nil)
|
|
||||||
dir := t.TempDir()
|
|
||||||
btResolve(t, f, fixture.Magnet)
|
|
||||||
f.Meta().Opts = &base.Options{Path: dir, SelectFiles: []int{0}}
|
|
||||||
btStart(t, f)
|
|
||||||
btAwaitPartial(t, f)
|
|
||||||
btPause(t, f)
|
|
||||||
if err := f.Patch(nil, &base.Options{SelectFiles: []int{1}}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
fixture.Limiter.SetLimit(rate.Inf)
|
|
||||||
btStart(t, f)
|
|
||||||
btAwaitDone(t, f)
|
|
||||||
fixture.AssertFiles(t, dir, filepath.Join("bundle", "b.bin"))
|
|
||||||
for _, suffix := range []string{"", ".part"} {
|
|
||||||
if _, err := os.Stat(filepath.Join(dir, "bundle", "a.bin") + suffix); !os.IsNotExist(err) {
|
|
||||||
t.Fatalf("unselected output remains: %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestBTDownloadMultiFileWithEmptyFile(t *testing.T) {
|
|
||||||
fixture := newBTFixture(t, map[string][]byte{"empty.txt": {}, "nested/data.bin": btPayload(48*1024, "empty-file")}, 0)
|
|
||||||
f := btDownloadFetcher(t, nil)
|
|
||||||
dir := t.TempDir()
|
|
||||||
btResolve(t, f, fixture.Magnet)
|
|
||||||
f.Meta().Opts = &base.Options{Path: dir}
|
|
||||||
btStart(t, f)
|
|
||||||
btAwaitDone(t, f)
|
|
||||||
fixture.AssertFiles(t, dir)
|
|
||||||
}
|
|
||||||
@@ -3,27 +3,22 @@ package bt
|
|||||||
import (
|
import (
|
||||||
"encoding/base64"
|
"encoding/base64"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
gohttp "net/http"
|
|
||||||
"net/url"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"reflect"
|
|
||||||
"sync"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/GopeedLab/gopeed/internal/controller"
|
"github.com/GopeedLab/gopeed/internal/controller"
|
||||||
"github.com/GopeedLab/gopeed/internal/fetcher"
|
"github.com/GopeedLab/gopeed/internal/fetcher"
|
||||||
"github.com/GopeedLab/gopeed/internal/test"
|
"github.com/GopeedLab/gopeed/internal/test"
|
||||||
"github.com/GopeedLab/gopeed/pkg/base"
|
"github.com/GopeedLab/gopeed/pkg/base"
|
||||||
"github.com/GopeedLab/gopeed/pkg/protocol/bt"
|
"github.com/GopeedLab/gopeed/pkg/protocol/bt"
|
||||||
|
"os"
|
||||||
|
"reflect"
|
||||||
|
"testing"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestFetcher_Resolve_Torrent(t *testing.T) {
|
func TestFetcher_Resolve_Torrent(t *testing.T) {
|
||||||
doResolve(t, buildFetcher(t))
|
doResolve(t, buildFetcher())
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestFetcher_Resolve_DataUri_Torrent(t *testing.T) {
|
func TestFetcher_Resolve_DataUri_Torrent(t *testing.T) {
|
||||||
fetcher := buildFetcher(t)
|
fetcher := buildFetcher()
|
||||||
buf, err := os.ReadFile("./testdata/ubuntu-22.04-live-server-amd64.iso.torrent")
|
buf, err := os.ReadFile("./testdata/ubuntu-22.04-live-server-amd64.iso.torrent")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
@@ -38,6 +33,7 @@ func TestFetcher_Resolve_DataUri_Torrent(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
want := &base.Resource{
|
want := &base.Resource{
|
||||||
|
Name: "ubuntu-22.04-live-server-amd64.iso",
|
||||||
Size: 1466714112,
|
Size: 1466714112,
|
||||||
Range: true,
|
Range: true,
|
||||||
Files: []*base.FileInfo{
|
Files: []*base.FileInfo{
|
||||||
@@ -54,7 +50,7 @@ func TestFetcher_Resolve_DataUri_Torrent(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestFetcher_Config(t *testing.T) {
|
func TestFetcher_Config(t *testing.T) {
|
||||||
doResolve(t, buildConfigFetcher(t, nil))
|
doResolve(t, buildConfigFetcher(nil))
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestFetcher_ResolveWithProxy(t *testing.T) {
|
func TestFetcher_ResolveWithProxy(t *testing.T) {
|
||||||
@@ -62,7 +58,7 @@ func TestFetcher_ResolveWithProxy(t *testing.T) {
|
|||||||
proxyListener := test.StartSocks5Server(usr, pwd)
|
proxyListener := test.StartSocks5Server(usr, pwd)
|
||||||
defer proxyListener.Close()
|
defer proxyListener.Close()
|
||||||
|
|
||||||
doResolve(t, buildConfigFetcher(t, &base.DownloaderProxyConfig{
|
doResolve(t, buildConfigFetcher(&base.DownloaderProxyConfig{
|
||||||
Enable: true,
|
Enable: true,
|
||||||
System: false,
|
System: false,
|
||||||
Scheme: "socks5",
|
Scheme: "socks5",
|
||||||
@@ -72,425 +68,58 @@ func TestFetcher_ResolveWithProxy(t *testing.T) {
|
|||||||
}))
|
}))
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestFetcher_CloseAfterInit(t *testing.T) {
|
|
||||||
if err := closeClient(); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
for i := 0; i < 100; i++ {
|
|
||||||
f := buildFetcher(t).(*Fetcher)
|
|
||||||
f.meta.Req = &base.Request{}
|
|
||||||
lock.Lock()
|
|
||||||
err := f.initClient()
|
|
||||||
lock.Unlock()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := closeClient(); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestFetcher_CloseWithoutClient(t *testing.T) {
|
|
||||||
if err := closeClient(); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
t.Run("restored task", func(t *testing.T) {
|
|
||||||
_, build := new(FetcherManager).Restore()
|
|
||||||
f := build(&fetcher.FetcherMeta{
|
|
||||||
Req: &base.Request{URL: "./testdata/test.torrent"},
|
|
||||||
Opts: &base.Options{},
|
|
||||||
}, &fetcherData{})
|
|
||||||
f.Setup(controller.NewController())
|
|
||||||
if err := f.Close(); err != nil {
|
|
||||||
t.Errorf("Close() got = %v, want nil", err)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("failed resolve", func(t *testing.T) {
|
|
||||||
f := buildFetcher(t)
|
|
||||||
if err := f.Resolve(&base.Request{URL: "./testdata/not-exist.torrent"}); err == nil {
|
|
||||||
t.Fatal("Resolve() got = nil, want error")
|
|
||||||
}
|
|
||||||
if err := f.Close(); err != nil {
|
|
||||||
t.Errorf("Close() got = %v, want nil", err)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestFetcher_ResolveWhileClientCloses(t *testing.T) {
|
|
||||||
if err := closeClient(); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
t.Cleanup(func() { closeClient() })
|
|
||||||
|
|
||||||
var wg sync.WaitGroup
|
|
||||||
for _, u := range []string{"./testdata/test.torrent", "./testdata/ubuntu-22.04-live-server-amd64.iso.torrent"} {
|
|
||||||
wg.Add(1)
|
|
||||||
go func() {
|
|
||||||
defer wg.Done()
|
|
||||||
for i := 0; i < 10; i++ {
|
|
||||||
f := buildFetcher(t)
|
|
||||||
// The other goroutine may close the client after the torrent was added.
|
|
||||||
if err := f.Resolve(&base.Request{URL: u}); err != nil && err.Error() != "torrent closed during resolution" {
|
|
||||||
t.Errorf("Resolve(%s) got = %v", u, err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
f.Close()
|
|
||||||
if err := closeClient(); err != nil {
|
|
||||||
t.Errorf("closeClient() got = %v", err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
}
|
|
||||||
wg.Wait()
|
|
||||||
}
|
|
||||||
|
|
||||||
func doResolve(t *testing.T, fetcher fetcher.Fetcher) {
|
func doResolve(t *testing.T, fetcher fetcher.Fetcher) {
|
||||||
t.Run("Resolve Single File", func(t *testing.T) {
|
err := fetcher.Resolve(&base.Request{
|
||||||
err := fetcher.Resolve(&base.Request{
|
URL: "./testdata/ubuntu-22.04-live-server-amd64.iso.torrent",
|
||||||
URL: "./testdata/ubuntu-22.04-live-server-amd64.iso.torrent",
|
Extra: bt.ReqExtra{
|
||||||
Extra: bt.ReqExtra{
|
Trackers: []string{
|
||||||
Trackers: []string{
|
"udp://tracker.birkenwald.de:6969/announce",
|
||||||
"udp://tracker.birkenwald.de:6969/announce",
|
"udp://tracker.bitsearch.to:1337/announce",
|
||||||
"udp://tracker.bitsearch.to:1337/announce",
|
|
||||||
},
|
|
||||||
},
|
},
|
||||||
})
|
},
|
||||||
if err != nil {
|
|
||||||
panic(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
want := &base.Resource{
|
|
||||||
Size: 1466714112,
|
|
||||||
Range: true,
|
|
||||||
Files: []*base.FileInfo{
|
|
||||||
{
|
|
||||||
Name: "ubuntu-22.04-live-server-amd64.iso",
|
|
||||||
Size: 1466714112,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
Hash: "8a55cfbd5ca5d11507364765936c4f9e55b253ed",
|
|
||||||
}
|
|
||||||
if !reflect.DeepEqual(want, fetcher.Meta().Res) {
|
|
||||||
t.Errorf("Resolve Single File Resolve() got = %v, want %v", fetcher.Meta().Res, want)
|
|
||||||
}
|
|
||||||
})
|
})
|
||||||
|
if err != nil {
|
||||||
t.Run("Resolve Multi Files", func(t *testing.T) {
|
panic(err)
|
||||||
err := fetcher.Resolve(&base.Request{
|
|
||||||
URL: "./testdata/test.torrent",
|
|
||||||
Extra: bt.ReqExtra{
|
|
||||||
Trackers: []string{
|
|
||||||
"udp://tracker.birkenwald.de:6969/announce",
|
|
||||||
"udp://tracker.bitsearch.to:1337/announce",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
panic(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
want := &base.Resource{
|
|
||||||
Name: "test",
|
|
||||||
Size: 107484864,
|
|
||||||
Range: true,
|
|
||||||
Files: []*base.FileInfo{
|
|
||||||
{
|
|
||||||
Name: "c.txt",
|
|
||||||
Path: "path",
|
|
||||||
Size: 98501754,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
Name: "b.txt",
|
|
||||||
Size: 8904996,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
Name: "a.txt",
|
|
||||||
Size: 78114,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
Hash: "ccbc92b0cd8deec16a2ef4be242a8c9243b1cedb",
|
|
||||||
}
|
|
||||||
if !reflect.DeepEqual(want, fetcher.Meta().Res) {
|
|
||||||
t.Errorf("Resolve Multi Files Resolve() got = %v, want %v", fetcher.Meta().Res, want)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("Resolve Unclean Torrent", func(t *testing.T) {
|
|
||||||
err := fetcher.Resolve(&base.Request{
|
|
||||||
URL: "./testdata/test.unclean.torrent",
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Errorf("Resolve Unclean Torrent Resolve() got = %v, want nil", err)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("Resolve file scheme Torrent", func(t *testing.T) {
|
|
||||||
file, _ := filepath.Abs("./testdata/test.unclean.torrent")
|
|
||||||
uri := "file:///" + file
|
|
||||||
err := fetcher.Resolve(&base.Request{
|
|
||||||
URL: uri,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Errorf("Resolve file scheme Torrent Resolve() got = %v, want nil", err)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestFetcherManager_ParseName(t *testing.T) {
|
|
||||||
type args struct {
|
|
||||||
u string
|
|
||||||
}
|
}
|
||||||
tests := []struct {
|
|
||||||
name string
|
want := &base.Resource{
|
||||||
args args
|
Name: "ubuntu-22.04-live-server-amd64.iso",
|
||||||
want string
|
Size: 1466714112,
|
||||||
}{
|
Range: true,
|
||||||
{
|
Files: []*base.FileInfo{
|
||||||
name: "broken url",
|
{
|
||||||
args: args{
|
Name: "ubuntu-22.04-live-server-amd64.iso",
|
||||||
u: "magnet://!@#%github.com",
|
Size: 1466714112,
|
||||||
},
|
},
|
||||||
want: "",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "dn",
|
|
||||||
args: args{
|
|
||||||
u: "magnet:?xt=urn:btih:8a55cfbd5ca5d11507364765936c4f9e55b253ed&dn=ubuntu-22.04-live-server-amd64.iso",
|
|
||||||
},
|
|
||||||
want: "ubuntu-22.04-live-server-amd64.iso",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "no dn",
|
|
||||||
args: args{
|
|
||||||
u: "magnet:?xt=urn:btih:8a55cfbd5ca5d11507364765936c4f9e55b253ed",
|
|
||||||
},
|
|
||||||
want: "8a55cfbd5ca5d11507364765936c4f9e55b253ed",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "non standard magnet",
|
|
||||||
args: args{
|
|
||||||
u: "magnet:?xxt=abcd",
|
|
||||||
},
|
|
||||||
want: "",
|
|
||||||
},
|
},
|
||||||
|
Hash: "8a55cfbd5ca5d11507364765936c4f9e55b253ed",
|
||||||
}
|
}
|
||||||
for _, tt := range tests {
|
if !reflect.DeepEqual(want, fetcher.Meta().Res) {
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
t.Errorf("Resolve() got = %v, want %v", fetcher.Meta().Res, want)
|
||||||
fm := &FetcherManager{}
|
|
||||||
if got := fm.ParseName(tt.args.u); got != tt.want {
|
|
||||||
t.Errorf("ParseName() = %v, want %v", got, tt.want)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func buildFetcher(t testing.TB) fetcher.Fetcher {
|
func buildFetcher() fetcher.Fetcher {
|
||||||
fb := new(FetcherManager)
|
fetcher := new(FetcherBuilder).Build()
|
||||||
fetcher := fb.Build()
|
fetcher.Setup(controller.NewController())
|
||||||
newController := controller.NewController()
|
|
||||||
newController.GetConfig = func(v any) {
|
|
||||||
json.Unmarshal([]byte(test.ToJson(fb.DefaultConfig())), v)
|
|
||||||
}
|
|
||||||
fetcher.Setup(newController)
|
|
||||||
t.Cleanup(func() { fetcher.Close(); closeClient() })
|
|
||||||
return fetcher
|
return fetcher
|
||||||
}
|
}
|
||||||
|
|
||||||
func buildConfigFetcher(t testing.TB, proxyConfig *base.DownloaderProxyConfig) fetcher.Fetcher {
|
func buildConfigFetcher(proxyConfig *base.DownloaderProxyConfig) fetcher.Fetcher {
|
||||||
fetcher := new(FetcherManager).Build()
|
fetcher := new(FetcherBuilder).Build()
|
||||||
newController := controller.NewController()
|
newController := controller.NewController()
|
||||||
mockCfg := config{
|
mockCfg := config{
|
||||||
CustomTrackers: []string{
|
Trackers: []string{
|
||||||
"udp://tracker.birkenwald.de:6969/announce",
|
"udp://tracker.birkenwald.de:6969/announce",
|
||||||
"udp://tracker.bitsearch.to:1337/announce",
|
"udp://tracker.bitsearch.to:1337/announce",
|
||||||
}}
|
}}
|
||||||
newController.GetConfig = func(v any) {
|
newController.GetConfig = func(v any) bool {
|
||||||
json.Unmarshal([]byte(test.ToJson(mockCfg)), v)
|
if err := json.Unmarshal([]byte(test.ToJson(mockCfg)), v); err != nil {
|
||||||
}
|
return false
|
||||||
newController.GetProxy = func(requestProxy *base.RequestProxy) func(*gohttp.Request) (*url.URL, error) {
|
|
||||||
return proxyConfig.ToHandler()
|
|
||||||
}
|
|
||||||
fetcher.Setup(newController)
|
|
||||||
t.Cleanup(func() { fetcher.Close(); closeClient() })
|
|
||||||
return fetcher
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestFetcher_Patch tests the Patch functionality for BT fetcher.
|
|
||||||
// It tests modifying selected files after Resolve (without downloading).
|
|
||||||
func TestFetcher_Patch(t *testing.T) {
|
|
||||||
f := buildFetcher(t)
|
|
||||||
|
|
||||||
// Resolve a multi-file torrent
|
|
||||||
err := f.Resolve(&base.Request{
|
|
||||||
URL: "./testdata/test.torrent",
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Resolve failed: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Resolution describes files without choosing download options.
|
|
||||||
meta := f.Meta()
|
|
||||||
if len(meta.Res.Files) != 3 {
|
|
||||||
t.Fatalf("Expected 3 files, got %d", len(meta.Res.Files))
|
|
||||||
}
|
|
||||||
if meta.Opts != nil {
|
|
||||||
t.Fatal("Resolve should not assign download options")
|
|
||||||
}
|
|
||||||
|
|
||||||
// Total size of all files: 107484864
|
|
||||||
totalSize := int64(107484864)
|
|
||||||
if meta.Res.Size != totalSize {
|
|
||||||
t.Fatalf("Expected total size %d, got %d", totalSize, meta.Res.Size)
|
|
||||||
}
|
|
||||||
|
|
||||||
t.Run("Patch with valid indices", func(t *testing.T) {
|
|
||||||
// Select only file 0 and 2 (c.txt and a.txt)
|
|
||||||
err := f.Patch(nil, &base.Options{
|
|
||||||
SelectFiles: []int{0, 2},
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Patch failed: %v", err)
|
|
||||||
}
|
}
|
||||||
|
return true
|
||||||
meta := f.Meta()
|
|
||||||
if !reflect.DeepEqual(meta.Opts.SelectFiles, []int{0, 2}) {
|
|
||||||
t.Errorf("Expected SelectFiles [0, 2], got %v", meta.Opts.SelectFiles)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Size should be recalculated: c.txt (98501754) + a.txt (78114) = 98579868
|
|
||||||
expectedSize := int64(98501754 + 78114)
|
|
||||||
if meta.Res.Size != expectedSize {
|
|
||||||
t.Errorf("Expected size %d, got %d", expectedSize, meta.Res.Size)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("Patch with invalid indices are silently ignored", func(t *testing.T) {
|
|
||||||
// Mix of valid (0, 1) and invalid (-1, 5, 100) indices
|
|
||||||
err := f.Patch(nil, &base.Options{
|
|
||||||
SelectFiles: []int{-1, 0, 5, 1, 100},
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Patch should not return error for invalid indices: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
meta := f.Meta()
|
|
||||||
// Only valid indices 0 and 1 should remain
|
|
||||||
if !reflect.DeepEqual(meta.Opts.SelectFiles, []int{0, 1}) {
|
|
||||||
t.Errorf("Expected SelectFiles [0, 1], got %v", meta.Opts.SelectFiles)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Size should be: c.txt (98501754) + b.txt (8904996) = 107406750
|
|
||||||
expectedSize := int64(98501754 + 8904996)
|
|
||||||
if meta.Res.Size != expectedSize {
|
|
||||||
t.Errorf("Expected size %d, got %d", expectedSize, meta.Res.Size)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("Patch with all invalid indices results in empty selection", func(t *testing.T) {
|
|
||||||
err := f.Patch(nil, &base.Options{
|
|
||||||
SelectFiles: []int{-5, 10, 999},
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Patch should not return error: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
meta := f.Meta()
|
|
||||||
if len(meta.Opts.SelectFiles) != 0 {
|
|
||||||
t.Errorf("Expected empty SelectFiles, got %v", meta.Opts.SelectFiles)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Note: CalcSize with empty selectFiles calculates total size of all files
|
|
||||||
// This is by design - empty selection in CalcSize means "all files"
|
|
||||||
// But SelectFiles being empty means no files are selected for download
|
|
||||||
if meta.Res.Size != totalSize {
|
|
||||||
t.Errorf("Expected size %d (CalcSize with empty slice = all files), got %d", totalSize, meta.Res.Size)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("Patch with nil opts does nothing", func(t *testing.T) {
|
|
||||||
// First set a known state
|
|
||||||
f.Patch(nil, &base.Options{SelectFiles: []int{1}})
|
|
||||||
prevSelectFiles := f.Meta().Opts.SelectFiles
|
|
||||||
|
|
||||||
// Patch with nil opts
|
|
||||||
err := f.Patch(nil, nil)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Patch with nil opts should not fail: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Should remain unchanged
|
|
||||||
if !reflect.DeepEqual(f.Meta().Opts.SelectFiles, prevSelectFiles) {
|
|
||||||
t.Errorf("SelectFiles should remain unchanged after nil opts Patch")
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("Patch progress array is resized", func(t *testing.T) {
|
|
||||||
btFetcher := f.(*Fetcher)
|
|
||||||
// Initialize progress array
|
|
||||||
btFetcher.data.Progress = make(fetcher.Progress, 3)
|
|
||||||
|
|
||||||
err := f.Patch(nil, &base.Options{
|
|
||||||
SelectFiles: []int{0, 2},
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Patch failed: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(btFetcher.data.Progress) != 2 {
|
|
||||||
t.Errorf("Expected Progress length 2, got %d", len(btFetcher.data.Progress))
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestFetcher_PeerDiscoveryConfig(t *testing.T) {
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
stored any
|
|
||||||
wantDisableDHT bool
|
|
||||||
wantDisablePEX bool
|
|
||||||
}{
|
|
||||||
{"default config", new(FetcherManager).DefaultConfig(), false, false},
|
|
||||||
{"stored config without the fields", map[string]any{"listenPort": 0, "seedRatio": 1.0}, false, false},
|
|
||||||
{"both disabled", map[string]any{"disableDht": true, "disablePex": true}, true, true},
|
|
||||||
{"only DHT disabled", map[string]any{"disableDht": true}, true, false},
|
|
||||||
{"only PEX disabled", map[string]any{"disablePex": true}, false, true},
|
|
||||||
}
|
|
||||||
for _, tt := range tests {
|
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
|
||||||
f := buildFetcherWithConfig(tt.stored).(*Fetcher)
|
|
||||||
if f.config.DisableDHT != tt.wantDisableDHT || f.config.DisablePEX != tt.wantDisablePEX {
|
|
||||||
t.Errorf("config got DisableDHT = %v, DisablePEX = %v, want %v, %v", f.config.DisableDHT, f.config.DisablePEX, tt.wantDisableDHT, tt.wantDisablePEX)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestFetcher_PeerDiscoveryDisabledOnClient(t *testing.T) {
|
|
||||||
if err := closeClient(); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
f := buildFetcherWithConfig(map[string]any{"disableDht": true, "disablePex": true})
|
|
||||||
if err := f.Resolve(&base.Request{URL: "./testdata/test.torrent"}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if !cfg.NoDHT || !cfg.DisablePEX {
|
|
||||||
t.Errorf("client config got NoDHT = %v, DisablePEX = %v, want true", cfg.NoDHT, cfg.DisablePEX)
|
|
||||||
}
|
|
||||||
if n := len(client.DhtServers()); n != 0 {
|
|
||||||
t.Errorf("client got %d DHT servers, want 0", n)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func buildFetcherWithConfig(stored any) fetcher.Fetcher {
|
|
||||||
fetcher := new(FetcherManager).Build()
|
|
||||||
newController := controller.NewController()
|
|
||||||
newController.GetConfig = func(v any) {
|
|
||||||
json.Unmarshal([]byte(test.ToJson(stored)), v)
|
|
||||||
}
|
}
|
||||||
|
newController.ProxyConfig = proxyConfig
|
||||||
fetcher.Setup(newController)
|
fetcher.Setup(newController)
|
||||||
return fetcher
|
return fetcher
|
||||||
}
|
}
|
||||||
@@ -0,0 +1,214 @@
|
|||||||
|
// modify from github.com/anacrolix/torrent/storage/file.go
|
||||||
|
|
||||||
|
package bt
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"github.com/anacrolix/torrent/storage"
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
|
||||||
|
"github.com/anacrolix/missinggo/v2"
|
||||||
|
|
||||||
|
"github.com/anacrolix/torrent/common"
|
||||||
|
"github.com/anacrolix/torrent/metainfo"
|
||||||
|
"github.com/anacrolix/torrent/segments"
|
||||||
|
)
|
||||||
|
|
||||||
|
// File-based storage for torrents, that isn't yet bound to a particular torrent.
|
||||||
|
type fileClientImpl struct {
|
||||||
|
opts newFileClientOpts
|
||||||
|
}
|
||||||
|
|
||||||
|
type newFileClientOpts struct {
|
||||||
|
// The base directory for all downloads.
|
||||||
|
ClientBaseDir string
|
||||||
|
HandleFileTorrent func(infoHash metainfo.Hash, ft *fileTorrentImpl)
|
||||||
|
FilePathMaker storage.FilePathMaker
|
||||||
|
PieceCompletion storage.PieceCompletion
|
||||||
|
}
|
||||||
|
|
||||||
|
func newFileOpts(opts newFileClientOpts) storage.ClientImplCloser {
|
||||||
|
if opts.FilePathMaker == nil {
|
||||||
|
opts.FilePathMaker = func(opts storage.FilePathMakerOpts) string {
|
||||||
|
var parts []string
|
||||||
|
if opts.Info.Name != metainfo.NoName {
|
||||||
|
parts = append(parts, opts.Info.Name)
|
||||||
|
}
|
||||||
|
return filepath.Join(append(parts, opts.File.Path...)...)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if opts.PieceCompletion == nil {
|
||||||
|
ret, err := storage.NewDefaultPieceCompletionForDir(opts.ClientBaseDir)
|
||||||
|
if err != nil {
|
||||||
|
ret = storage.NewMapPieceCompletion()
|
||||||
|
}
|
||||||
|
opts.PieceCompletion = ret
|
||||||
|
}
|
||||||
|
return fileClientImpl{opts}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (fs fileClientImpl) Close() error {
|
||||||
|
return fs.opts.PieceCompletion.Close()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (fs fileClientImpl) OpenTorrent(info *metainfo.Info, infoHash metainfo.Hash) (_ storage.TorrentImpl, err error) {
|
||||||
|
upvertedFiles := info.UpvertedFiles()
|
||||||
|
files := make([]file, 0, len(upvertedFiles))
|
||||||
|
for _, fileInfo := range upvertedFiles {
|
||||||
|
filePath := filepath.Join("", fs.opts.FilePathMaker(storage.FilePathMakerOpts{
|
||||||
|
Info: info,
|
||||||
|
File: &fileInfo,
|
||||||
|
}))
|
||||||
|
f := file{
|
||||||
|
rawPath: filePath,
|
||||||
|
path: filePath,
|
||||||
|
length: fileInfo.Length,
|
||||||
|
}
|
||||||
|
if f.length == 0 {
|
||||||
|
err = CreateNativeZeroLengthFile(f.path)
|
||||||
|
if err != nil {
|
||||||
|
err = fmt.Errorf("creating zero length file: %w", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
files = append(files, f)
|
||||||
|
}
|
||||||
|
t := &fileTorrentImpl{
|
||||||
|
files,
|
||||||
|
segments.NewIndex(common.LengthIterFromUpvertedFiles(upvertedFiles)),
|
||||||
|
infoHash,
|
||||||
|
fs.opts.PieceCompletion,
|
||||||
|
}
|
||||||
|
fs.opts.HandleFileTorrent(infoHash, t)
|
||||||
|
return storage.TorrentImpl{
|
||||||
|
Piece: t.Piece,
|
||||||
|
Close: t.Close,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type file struct {
|
||||||
|
// The safe, OS-local file path.
|
||||||
|
rawPath string
|
||||||
|
path string
|
||||||
|
length int64
|
||||||
|
}
|
||||||
|
|
||||||
|
type fileTorrentImpl struct {
|
||||||
|
files []file
|
||||||
|
segmentLocater segments.Index
|
||||||
|
infoHash metainfo.Hash
|
||||||
|
completion storage.PieceCompletion
|
||||||
|
}
|
||||||
|
|
||||||
|
func (fts *fileTorrentImpl) Piece(p metainfo.Piece) storage.PieceImpl {
|
||||||
|
// Create a view onto the file-based torrent storage.
|
||||||
|
_io := fileTorrentImplIO{fts}
|
||||||
|
// Return the appropriate segments of this.
|
||||||
|
return &filePieceImpl{
|
||||||
|
fts,
|
||||||
|
p,
|
||||||
|
missinggo.NewSectionWriter(_io, p.Offset(), p.Length()),
|
||||||
|
io.NewSectionReader(_io, p.Offset(), p.Length()),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (fts *fileTorrentImpl) Close() error {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (fts *fileTorrentImpl) setTorrentDir(dir string) {
|
||||||
|
for i, f := range fts.files {
|
||||||
|
fts.files[i].path = filepath.Join(dir, f.rawPath)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A helper to create zero-length files which won't appear for file-orientated storage since no
|
||||||
|
// writes will ever occur to them (no torrent data is associated with a zero-length file). The
|
||||||
|
// caller should make sure the file name provided is safe/sanitized.
|
||||||
|
func CreateNativeZeroLengthFile(name string) error {
|
||||||
|
os.MkdirAll(filepath.Dir(name), 0o777)
|
||||||
|
var f io.Closer
|
||||||
|
f, err := os.Create(name)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return f.Close()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Exposes file-based storage of a torrent, as one big ReadWriterAt.
|
||||||
|
type fileTorrentImplIO struct {
|
||||||
|
fts *fileTorrentImpl
|
||||||
|
}
|
||||||
|
|
||||||
|
// Returns EOF on short or missing file.
|
||||||
|
func (fst *fileTorrentImplIO) readFileAt(file file, b []byte, off int64) (n int, err error) {
|
||||||
|
f, err := os.Open(file.path)
|
||||||
|
if os.IsNotExist(err) {
|
||||||
|
// File missing is treated the same as a short file.
|
||||||
|
err = io.EOF
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
defer f.Close()
|
||||||
|
// Limit the read to within the expected bounds of this file.
|
||||||
|
if int64(len(b)) > file.length-off {
|
||||||
|
b = b[:file.length-off]
|
||||||
|
}
|
||||||
|
for off < file.length && len(b) != 0 {
|
||||||
|
n1, err1 := f.ReadAt(b, off)
|
||||||
|
b = b[n1:]
|
||||||
|
n += n1
|
||||||
|
off += int64(n1)
|
||||||
|
if n1 == 0 {
|
||||||
|
err = err1
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Only returns EOF at the end of the torrent. Premature EOF is ErrUnexpectedEOF.
|
||||||
|
func (fst fileTorrentImplIO) ReadAt(b []byte, off int64) (n int, err error) {
|
||||||
|
fst.fts.segmentLocater.Locate(segments.Extent{off, int64(len(b))}, func(i int, e segments.Extent) bool {
|
||||||
|
n1, err1 := fst.readFileAt(fst.fts.files[i], b[:e.Length], e.Start)
|
||||||
|
n += n1
|
||||||
|
b = b[n1:]
|
||||||
|
err = err1
|
||||||
|
return err == nil // && int64(n1) == e.Length
|
||||||
|
})
|
||||||
|
if len(b) != 0 && err == nil {
|
||||||
|
err = io.EOF
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
func (fst fileTorrentImplIO) WriteAt(p []byte, off int64) (n int, err error) {
|
||||||
|
// log.Printf("write at %v: %v bytes", off, len(p))
|
||||||
|
fst.fts.segmentLocater.Locate(segments.Extent{off, int64(len(p))}, func(i int, e segments.Extent) bool {
|
||||||
|
name := fst.fts.files[i].path
|
||||||
|
os.MkdirAll(filepath.Dir(name), 0o777)
|
||||||
|
var f *os.File
|
||||||
|
f, err = os.OpenFile(name, os.O_WRONLY|os.O_CREATE, 0o666)
|
||||||
|
if err != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
var n1 int
|
||||||
|
n1, err = f.WriteAt(p[:e.Length], e.Start)
|
||||||
|
// log.Printf("%v %v wrote %v: %v", i, e, n1, err)
|
||||||
|
closeErr := f.Close()
|
||||||
|
n += n1
|
||||||
|
p = p[n1:]
|
||||||
|
if err == nil {
|
||||||
|
err = closeErr
|
||||||
|
}
|
||||||
|
if err == nil && int64(n1) != e.Length {
|
||||||
|
err = io.ErrShortWrite
|
||||||
|
}
|
||||||
|
return err == nil
|
||||||
|
})
|
||||||
|
return
|
||||||
|
}
|
||||||
@@ -0,0 +1,36 @@
|
|||||||
|
// modify from github.com/anacrolix/torrent/storage/file_misc.go
|
||||||
|
|
||||||
|
package bt
|
||||||
|
|
||||||
|
import "github.com/anacrolix/torrent/metainfo"
|
||||||
|
|
||||||
|
type requiredLength struct {
|
||||||
|
fileIndex int
|
||||||
|
length int64
|
||||||
|
}
|
||||||
|
|
||||||
|
func extentCompleteRequiredLengths(info *metainfo.Info, off, n int64) (ret []requiredLength) {
|
||||||
|
if n == 0 {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
for i, fi := range info.UpvertedFiles() {
|
||||||
|
if off >= fi.Length {
|
||||||
|
off -= fi.Length
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
n1 := n
|
||||||
|
if off+n1 > fi.Length {
|
||||||
|
n1 = fi.Length - off
|
||||||
|
}
|
||||||
|
ret = append(ret, requiredLength{
|
||||||
|
fileIndex: i,
|
||||||
|
length: off + n1,
|
||||||
|
})
|
||||||
|
n -= n1
|
||||||
|
if n == 0 {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
off = 0
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
@@ -0,0 +1,61 @@
|
|||||||
|
// modify from github.com/anacrolix/torrent/storage/file_piece.go
|
||||||
|
|
||||||
|
package bt
|
||||||
|
|
||||||
|
import (
|
||||||
|
"github.com/anacrolix/torrent/metainfo"
|
||||||
|
"github.com/anacrolix/torrent/storage"
|
||||||
|
"io"
|
||||||
|
"log"
|
||||||
|
"os"
|
||||||
|
)
|
||||||
|
|
||||||
|
type filePieceImpl struct {
|
||||||
|
*fileTorrentImpl
|
||||||
|
p metainfo.Piece
|
||||||
|
io.WriterAt
|
||||||
|
io.ReaderAt
|
||||||
|
}
|
||||||
|
|
||||||
|
var _ storage.PieceImpl = (*filePieceImpl)(nil)
|
||||||
|
|
||||||
|
func (fs *filePieceImpl) pieceKey() metainfo.PieceKey {
|
||||||
|
return metainfo.PieceKey{InfoHash: fs.infoHash, Index: fs.p.Index()}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (fs *filePieceImpl) Completion() storage.Completion {
|
||||||
|
c, err := fs.completion.Get(fs.pieceKey())
|
||||||
|
if err != nil {
|
||||||
|
log.Printf("error getting piece completion: %s", err)
|
||||||
|
c.Ok = false
|
||||||
|
return c
|
||||||
|
}
|
||||||
|
|
||||||
|
verified := true
|
||||||
|
if c.Complete {
|
||||||
|
// If it's allegedly complete, check that its constituent files have the necessary length.
|
||||||
|
for _, fi := range extentCompleteRequiredLengths(fs.p.Info, fs.p.Offset(), fs.p.Length()) {
|
||||||
|
s, err := os.Stat(fs.files[fi.fileIndex].path)
|
||||||
|
if err != nil || s.Size() < fi.length {
|
||||||
|
verified = false
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if !verified {
|
||||||
|
// The completion was wrong, fix it.
|
||||||
|
c.Complete = false
|
||||||
|
fs.completion.Set(fs.pieceKey(), false)
|
||||||
|
}
|
||||||
|
|
||||||
|
return c
|
||||||
|
}
|
||||||
|
|
||||||
|
func (fs *filePieceImpl) MarkComplete() error {
|
||||||
|
return fs.completion.Set(fs.pieceKey(), true)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (fs *filePieceImpl) MarkNotComplete() error {
|
||||||
|
return fs.completion.Set(fs.pieceKey(), false)
|
||||||
|
}
|
||||||
@@ -1,212 +0,0 @@
|
|||||||
package bt
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"context"
|
|
||||||
"encoding/base64"
|
|
||||||
"encoding/binary"
|
|
||||||
"net"
|
|
||||||
"net/http"
|
|
||||||
"net/http/httptest"
|
|
||||||
"net/url"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/anacrolix/torrent"
|
|
||||||
"github.com/anacrolix/torrent/bencode"
|
|
||||||
"github.com/anacrolix/torrent/merkle"
|
|
||||||
"github.com/anacrolix/torrent/metainfo"
|
|
||||||
"golang.org/x/time/rate"
|
|
||||||
)
|
|
||||||
|
|
||||||
// btFixture serves generated, trackerless content over a real loopback TCP peer.
|
|
||||||
// Its HTTP tracker exercises .torrent downloads; magnet links also carry the
|
|
||||||
// explicit x.pe address.
|
|
||||||
type btFixture struct {
|
|
||||||
MetaInfo *metainfo.MetaInfo
|
|
||||||
Magnet string
|
|
||||||
TorrentFile string
|
|
||||||
DataURI string
|
|
||||||
TrackerURL string
|
|
||||||
Files map[string][]byte // Paths relative to the selected download directory.
|
|
||||||
Seeder *torrent.Client
|
|
||||||
Torrent *torrent.Torrent
|
|
||||||
Limiter *rate.Limiter
|
|
||||||
}
|
|
||||||
|
|
||||||
func newBTFixture(t testing.TB, files map[string][]byte, bytesPerSecond int) *btFixture {
|
|
||||||
return makeBTFixture(t, files, bytesPerSecond, false, true)
|
|
||||||
}
|
|
||||||
|
|
||||||
func newBTHybridFixture(t testing.TB, payload []byte) *btFixture {
|
|
||||||
return makeBTFixture(t, map[string][]byte{"": payload}, 0, true, true)
|
|
||||||
}
|
|
||||||
|
|
||||||
func makeBTFixture(t testing.TB, files map[string][]byte, bytesPerSecond int, v2, private bool) *btFixture {
|
|
||||||
t.Helper()
|
|
||||||
source := t.TempDir()
|
|
||||||
name := "payload.bin"
|
|
||||||
if _, single := files[""]; !single {
|
|
||||||
name = "bundle"
|
|
||||||
}
|
|
||||||
root := filepath.Join(source, name)
|
|
||||||
expected := make(map[string][]byte, len(files))
|
|
||||||
for path, data := range files {
|
|
||||||
target := filepath.Join(root, filepath.FromSlash(path))
|
|
||||||
if err := os.MkdirAll(filepath.Dir(target), 0700); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := os.WriteFile(target, data, 0600); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
expected[filepath.Join(name, filepath.FromSlash(path))] = append([]byte(nil), data...)
|
|
||||||
}
|
|
||||||
info := metainfo.Info{PieceLength: 16 * 1024}
|
|
||||||
if err := info.BuildFromFilePath(root); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
var layers map[string]string
|
|
||||||
if v2 {
|
|
||||||
data := files[""]
|
|
||||||
h := merkle.NewHash()
|
|
||||||
h.Write(data)
|
|
||||||
root := h.Sum(nil)
|
|
||||||
var pieceHashes []byte
|
|
||||||
for off := 0; off < len(data); off += int(info.PieceLength) {
|
|
||||||
end := min(off+int(info.PieceLength), len(data))
|
|
||||||
ph := merkle.NewHash()
|
|
||||||
ph.Write(data[off:end])
|
|
||||||
pieceHashes = append(pieceHashes, ph.SumMinLength(nil, int(info.PieceLength))...)
|
|
||||||
}
|
|
||||||
info = metainfo.Info{Name: name, PieceLength: info.PieceLength, Length: int64(len(data)), Pieces: info.Pieces, MetaVersion: 2, FileTree: metainfo.FileTree{File: metainfo.FileTreeFile{Length: int64(len(data)), PiecesRoot: string(root)}}}
|
|
||||||
layers = map[string]string{string(root): string(pieceHashes)}
|
|
||||||
}
|
|
||||||
// Private torrents avoid public DHT discovery once metadata is available.
|
|
||||||
info.Private = &private
|
|
||||||
raw, err := bencode.Marshal(&info)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
mi := &metainfo.MetaInfo{InfoBytes: raw, PieceLayers: layers}
|
|
||||||
cfg := torrent.TestingConfig(t)
|
|
||||||
cfg.DataDir = source
|
|
||||||
cfg.Seed = true
|
|
||||||
cfg.DisableUTP = true
|
|
||||||
cfg.DisableIPv6 = true
|
|
||||||
cfg.DisableWebtorrent = true
|
|
||||||
cfg.KeepAliveTimeout = time.Minute
|
|
||||||
cfg.MaxAllocPeerRequestDataPerConn = 1 << 20
|
|
||||||
limiter := rate.NewLimiter(rate.Inf, 16*1024)
|
|
||||||
if bytesPerSecond > 0 {
|
|
||||||
limiter.SetLimit(rate.Limit(bytesPerSecond))
|
|
||||||
}
|
|
||||||
cfg.UploadRateLimiter = limiter
|
|
||||||
seed, err := torrent.NewClient(cfg)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
t.Cleanup(func() { seed.Close() })
|
|
||||||
seeded, _, err := seed.AddTorrentSpec(torrent.TorrentSpecFromMetaInfo(mi))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
|
||||||
defer cancel()
|
|
||||||
if err := seeded.VerifyDataContext(ctx); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
select {
|
|
||||||
case <-seeded.Complete().On():
|
|
||||||
case <-ctx.Done():
|
|
||||||
t.Fatal("local seeder did not finish verification")
|
|
||||||
}
|
|
||||||
addr := seed.ListenAddrs()[0].String()
|
|
||||||
host, portString, err := net.SplitHostPort(addr)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
resolved, err := net.ResolveTCPAddr("tcp4", net.JoinHostPort(host, portString))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
compact := make([]byte, 6)
|
|
||||||
copy(compact, resolved.IP.To4())
|
|
||||||
binary.BigEndian.PutUint16(compact[4:], uint16(resolved.Port))
|
|
||||||
tracker := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
response, err := bencode.Marshal(map[string]any{"interval": 3600, "peers": string(compact)})
|
|
||||||
if err != nil {
|
|
||||||
http.Error(w, err.Error(), 500)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
w.Write(response)
|
|
||||||
}))
|
|
||||||
t.Cleanup(tracker.Close)
|
|
||||||
// The pinned library has a hybrid tracker-dispatch assertion bug.
|
|
||||||
// Hybrid fixtures use their explicit loopback peer rather than a tracker.
|
|
||||||
if !v2 {
|
|
||||||
mi.Announce = tracker.URL
|
|
||||||
}
|
|
||||||
var encoded bytes.Buffer
|
|
||||||
if err := mi.Write(&encoded); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
torrentFile := filepath.Join(t.TempDir(), "fixture.torrent")
|
|
||||||
if err := os.WriteFile(torrentFile, encoded.Bytes(), 0600); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
m, err := mi.MagnetV2()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
magnet := m.String()
|
|
||||||
u, err := url.Parse(magnet)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
q := u.Query()
|
|
||||||
q.Add("x.pe", addr)
|
|
||||||
u.RawQuery = q.Encode()
|
|
||||||
return &btFixture{MetaInfo: mi, Magnet: u.String(), TorrentFile: torrentFile, DataURI: "data:application/x-bittorrent;base64," + base64.StdEncoding.EncodeToString(encoded.Bytes()), TrackerURL: tracker.URL, Files: expected, Seeder: seed, Torrent: seeded, Limiter: limiter}
|
|
||||||
}
|
|
||||||
|
|
||||||
func btPayload(size int, label string) []byte {
|
|
||||||
data := make([]byte, size)
|
|
||||||
for i := range data {
|
|
||||||
data[i] = byte((i*31 + i/251) % 251)
|
|
||||||
}
|
|
||||||
copy(data, label)
|
|
||||||
return data
|
|
||||||
}
|
|
||||||
|
|
||||||
func (f *btFixture) AssertFiles(t testing.TB, dir string, paths ...string) {
|
|
||||||
t.Helper()
|
|
||||||
if len(paths) == 0 {
|
|
||||||
for path := range f.Files {
|
|
||||||
paths = append(paths, path)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for _, path := range paths {
|
|
||||||
got, err := os.ReadFile(filepath.Join(dir, path))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
want, ok := f.Files[path]
|
|
||||||
if !ok {
|
|
||||||
t.Fatalf("unknown fixture file %q", path)
|
|
||||||
}
|
|
||||||
if !bytes.Equal(got, want) {
|
|
||||||
t.Fatalf("BT file %s differs: received %d bytes, want %d", path, len(got), len(want))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (f *btFixture) FileURI() string {
|
|
||||||
path := filepath.ToSlash(f.TorrentFile)
|
|
||||||
if !strings.HasPrefix(path, "/") {
|
|
||||||
path = "/" + path
|
|
||||||
}
|
|
||||||
return (&url.URL{Scheme: "file", Path: path}).String()
|
|
||||||
}
|
|
||||||
@@ -1,367 +0,0 @@
|
|||||||
package bt
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"crypto/sha1"
|
|
||||||
"fmt"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/GopeedLab/gopeed/internal/fetcher"
|
|
||||||
"github.com/GopeedLab/gopeed/pkg/base"
|
|
||||||
"github.com/anacrolix/torrent"
|
|
||||||
"github.com/anacrolix/torrent/bencode"
|
|
||||||
"github.com/anacrolix/torrent/metainfo"
|
|
||||||
"github.com/anacrolix/torrent/storage"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestStartDoesNotPublishUnverifiedCompletion(t *testing.T) {
|
|
||||||
cfg := torrent.TestingConfig(t)
|
|
||||||
cfg.DisableTCP = true
|
|
||||||
cfg.DisableUTP = true
|
|
||||||
c, err := torrent.NewClient(cfg)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
defer c.Close()
|
|
||||||
payload := btPayload(32*1024, "unverified")
|
|
||||||
info := metainfo.Info{Name: "payload.bin", Length: int64(len(payload)), PieceLength: 16 * 1024}
|
|
||||||
for offset := 0; offset < len(payload); offset += int(info.PieceLength) {
|
|
||||||
hash := sha1.Sum(payload[offset : offset+int(info.PieceLength)])
|
|
||||||
info.Pieces = append(info.Pieces, hash[:]...)
|
|
||||||
}
|
|
||||||
raw, err := bencode.Marshal(info)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
store := &deferredFileStorage{}
|
|
||||||
spec := torrent.TorrentSpecFromMetaInfo(&metainfo.MetaInfo{InfoBytes: raw})
|
|
||||||
spec.Storage = store
|
|
||||||
tor, _, err := c.AddTorrentSpec(spec)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
tor.DisallowDataDownload()
|
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
|
||||||
defer cancel()
|
|
||||||
f := &Fetcher{torrent: tor, storage: store, meta: &fetcher.FetcherMeta{}, data: &fetcherData{}, torrentDropCtx: ctx, torrentDropFunc: cancel, config: &config{SeedKeep: true}}
|
|
||||||
f.torrentReady.Store(true)
|
|
||||||
f.updateRes()
|
|
||||||
dir := t.TempDir()
|
|
||||||
payload[20000] ^= 0xff
|
|
||||||
if err := os.WriteFile(filepath.Join(dir, info.Name), payload, 0600); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
f.meta.Opts = &base.Options{Path: dir}
|
|
||||||
// Retain the original event channel to detect even a transient completion.
|
|
||||||
// With no network transports, the damaged piece cannot have been repaired.
|
|
||||||
completed := tor.Complete().On()
|
|
||||||
if err := f.Start(); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
select {
|
|
||||||
case <-completed:
|
|
||||||
t.Fatal("Start published completion before verifying the existing file")
|
|
||||||
default:
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestStartUsesSelectionSubmittedAfterResolve(t *testing.T) {
|
|
||||||
cfg := torrent.TestingConfig(t)
|
|
||||||
cfg.DataDir = t.TempDir()
|
|
||||||
cfg.DisableTCP = true
|
|
||||||
cfg.DisableUTP = true
|
|
||||||
c, err := torrent.NewClient(cfg)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
defer c.Close()
|
|
||||||
store := &deferredFileStorage{}
|
|
||||||
mi, err := metainfo.LoadFromFile("testdata/test.torrent")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
spec := torrent.TorrentSpecFromMetaInfo(mi)
|
|
||||||
spec.Storage = store
|
|
||||||
tor, _, err := c.AddTorrentSpec(spec)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
tor.DisallowDataDownload()
|
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
|
||||||
defer cancel()
|
|
||||||
f := &Fetcher{torrent: tor, storage: store, meta: &fetcher.FetcherMeta{}, data: &fetcherData{}, torrentDropCtx: ctx, torrentDropFunc: cancel, config: &config{SeedKeep: true}}
|
|
||||||
f.torrentReady.Store(true)
|
|
||||||
f.updateRes()
|
|
||||||
f.meta.Opts = &base.Options{Path: t.TempDir(), SelectFiles: []int{2}}
|
|
||||||
if err := f.Start(); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
for i, file := range tor.Files() {
|
|
||||||
want := torrent.PiecePriorityNone
|
|
||||||
if i == 2 {
|
|
||||||
want = torrent.PiecePriorityNormal
|
|
||||||
}
|
|
||||||
if file.Priority() != want {
|
|
||||||
t.Errorf("file %d priority = %v, want %v", i, file.Priority(), want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if len(f.data.Progress) != 1 {
|
|
||||||
t.Fatalf("progress tracks %d files, want 1", len(f.data.Progress))
|
|
||||||
}
|
|
||||||
if err := f.Pause(); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := f.Start(); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if tor.Files()[0].Priority() != torrent.PiecePriorityNone {
|
|
||||||
t.Fatal("resume selected an unselected file")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestWaitCleansUnselectedFilesFromSharedPiece(t *testing.T) {
|
|
||||||
for _, mode := range []string{"partial", "part only", "both paths", "all", "already removed", "remove error"} {
|
|
||||||
t.Run(mode, func(t *testing.T) {
|
|
||||||
dir := t.TempDir()
|
|
||||||
keepPath := filepath.Join(dir, "bundle", "keep.txt")
|
|
||||||
skipPath := filepath.Join(dir, "bundle", "nested", "skip.txt")
|
|
||||||
if err := os.MkdirAll(filepath.Dir(skipPath), 0755); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := os.WriteFile(keepPath, []byte("keep"), 0644); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := os.WriteFile(skipPath, []byte("skip"), 0644); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
// Both files occupy the same piece, so downloading keep.txt also writes skip.txt.
|
|
||||||
hash := sha1.Sum([]byte("keepskip"))
|
|
||||||
info := metainfo.Info{Name: "bundle", PieceLength: 16, Pieces: hash[:], Files: []metainfo.FileInfo{
|
|
||||||
{Length: 4, Path: []string{"keep.txt"}},
|
|
||||||
{Length: 4, Path: []string{"nested", "skip.txt"}},
|
|
||||||
}}
|
|
||||||
storageOpts := storage.NewFileClientOpts{
|
|
||||||
ClientBaseDir: dir,
|
|
||||||
TorrentDirMaker: func(baseDir string, _ *metainfo.Info, _ metainfo.Hash) string { return baseDir },
|
|
||||||
}
|
|
||||||
// VerifyDataContext can return before storage finishes marking the piece
|
|
||||||
// complete. Disable automatic part-file promotion so it cannot rename
|
|
||||||
// the .part fixtures that this test creates to exercise cleanup.
|
|
||||||
storageOpts.UsePartFiles.Set(false)
|
|
||||||
store := storage.NewFileOpts(storageOpts)
|
|
||||||
defer store.Close()
|
|
||||||
config := torrent.TestingConfig(t)
|
|
||||||
config.DisableTCP = true
|
|
||||||
config.DisableUTP = true
|
|
||||||
c, err := torrent.NewClient(config)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
defer c.Close()
|
|
||||||
spec, err := torrent.TorrentSpecFromMetaInfoErr(&metainfo.MetaInfo{InfoBytes: bencode.MustMarshal(&info)})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
spec.Storage = store
|
|
||||||
tor, _, err := c.AddTorrentSpec(spec)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
|
||||||
defer cancel()
|
|
||||||
if err := tor.VerifyDataContext(ctx); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
f := &Fetcher{torrent: tor, meta: &fetcher.FetcherMeta{Opts: &base.Options{Path: dir, SelectFiles: []int{0}}}, data: &fetcherData{}, torrentDropCtx: ctx}
|
|
||||||
f.torrentReady.Store(true)
|
|
||||||
f.updateRes()
|
|
||||||
if mode == "all" {
|
|
||||||
f.meta.Opts.SelectFiles = []int{0, 1}
|
|
||||||
}
|
|
||||||
if !f.isDone() {
|
|
||||||
t.Fatal("selected file is not complete")
|
|
||||||
}
|
|
||||||
if mode == "part only" || mode == "both paths" || mode == "all" {
|
|
||||||
if err := os.WriteFile(skipPath+".part", []byte("skip"), 0644); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if mode == "part only" || mode == "already removed" || mode == "remove error" {
|
|
||||||
if err := os.Remove(skipPath); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if mode == "remove error" {
|
|
||||||
if err := os.Mkdir(skipPath, 0755); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := os.WriteFile(filepath.Join(skipPath, "child"), []byte("preserve"), 0644); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
err = f.Wait()
|
|
||||||
if ctx.Err() != nil {
|
|
||||||
t.Fatal(ctx.Err())
|
|
||||||
}
|
|
||||||
if mode == "remove error" {
|
|
||||||
if err == nil {
|
|
||||||
t.Fatal("cleanup failure was silently ignored")
|
|
||||||
}
|
|
||||||
} else if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
data, err := os.ReadFile(keepPath)
|
|
||||||
if err != nil || string(data) != "keep" {
|
|
||||||
t.Fatalf("selected file changed: %q, %v", data, err)
|
|
||||||
}
|
|
||||||
if mode == "all" {
|
|
||||||
if _, err := os.Stat(skipPath + ".part"); err != nil {
|
|
||||||
t.Fatalf("selected part file was removed: %v", err)
|
|
||||||
}
|
|
||||||
data, err := os.ReadFile(skipPath)
|
|
||||||
if err != nil || string(data) != "skip" {
|
|
||||||
t.Fatalf("selected file changed: %q, %v", data, err)
|
|
||||||
}
|
|
||||||
} else if mode != "remove error" {
|
|
||||||
for _, removed := range []string{skipPath, skipPath + ".part", filepath.Dir(skipPath)} {
|
|
||||||
if _, err := os.Stat(removed); !os.IsNotExist(err) {
|
|
||||||
t.Fatalf("unselected file or empty directory remains: %s: %v", removed, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRemoveUnselectedFilePrunesOnlyEmptyParents(t *testing.T) {
|
|
||||||
for _, preserveSibling := range []bool{false, true} {
|
|
||||||
t.Run(fmt.Sprint(preserveSibling), func(t *testing.T) {
|
|
||||||
root := t.TempDir()
|
|
||||||
relative := filepath.Join("bundle", "nested", "deep", "skip.txt")
|
|
||||||
name := filepath.Join(root, relative)
|
|
||||||
if err := os.MkdirAll(filepath.Dir(name), 0755); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := os.WriteFile(name+".part", []byte("partial"), 0644); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
sibling := filepath.Join(root, "bundle", "keep.txt")
|
|
||||||
if preserveSibling {
|
|
||||||
if err := os.WriteFile(sibling, []byte("keep"), 0644); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if err := removeUnselectedFile(root, "bundle", relative); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, err := os.Stat(filepath.Join(root, "bundle", "nested")); !os.IsNotExist(err) {
|
|
||||||
t.Fatalf("empty ancestors remain: %v", err)
|
|
||||||
}
|
|
||||||
if preserveSibling {
|
|
||||||
data, err := os.ReadFile(sibling)
|
|
||||||
if err != nil || string(data) != "keep" {
|
|
||||||
t.Fatalf("sibling changed: %q, %v", data, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if info, err := os.Stat(filepath.Join(root, "bundle")); err != nil || !info.IsDir() {
|
|
||||||
t.Fatalf("torrent root directory was removed: %v", err)
|
|
||||||
}
|
|
||||||
if _, err := os.Stat(root); err != nil {
|
|
||||||
t.Fatalf("download directory was removed: %v", err)
|
|
||||||
}
|
|
||||||
// A file directly inside the torrent root must not cause that root to be pruned.
|
|
||||||
rootFile := filepath.Join("bundle", "root.txt")
|
|
||||||
if err := os.WriteFile(filepath.Join(root, rootFile)+".part", []byte("partial"), 0644); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := removeUnselectedFile(root, "bundle", rootFile); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, err := os.Stat(filepath.Join(root, "bundle")); err != nil {
|
|
||||||
t.Fatalf("torrent root directory was removed after root-level cleanup: %v", err)
|
|
||||||
}
|
|
||||||
// Repeating cleanup after the file and its subdirectories are gone is harmless.
|
|
||||||
if err := removeUnselectedFile(root, "bundle", relative); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestCleanupMatchesNamedAndRootlessStorage(t *testing.T) {
|
|
||||||
for _, version := range []int64{1, 2} {
|
|
||||||
for _, layout := range []string{"rootless", "wrapped", "file-tree-root"} {
|
|
||||||
if version == 1 && layout == "file-tree-root" {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
t.Run(fmt.Sprintf("v%d/%s", version, layout), func(t *testing.T) {
|
|
||||||
dir := t.TempDir()
|
|
||||||
info := metainfo.Info{Name: metainfo.NoName, PieceLength: 16384}
|
|
||||||
root := dir
|
|
||||||
if layout == "wrapped" {
|
|
||||||
info.Name = "bundle"
|
|
||||||
root = filepath.Join(dir, "bundle")
|
|
||||||
}
|
|
||||||
if version == 1 {
|
|
||||||
info.Files = []metainfo.FileInfo{{Path: []string{"keep.txt"}}, {Path: []string{"nested", "deep", "skip.txt"}}}
|
|
||||||
} else {
|
|
||||||
info.MetaVersion = 2
|
|
||||||
info.FileTree = metainfo.FileTree{Dir: map[string]metainfo.FileTree{
|
|
||||||
"keep.txt": {},
|
|
||||||
"nested": {Dir: map[string]metainfo.FileTree{"deep": {Dir: map[string]metainfo.FileTree{"skip.txt": {}}}}},
|
|
||||||
}}
|
|
||||||
}
|
|
||||||
if layout == "file-tree-root" {
|
|
||||||
info.FileTree = metainfo.FileTree{Dir: map[string]metainfo.FileTree{"tree-root": info.FileTree}}
|
|
||||||
root = filepath.Join(dir, "tree-root")
|
|
||||||
}
|
|
||||||
// Keep the library's default FilePathMaker here to independently verify
|
|
||||||
// that cleanup agrees with its real disk layout, including NoName.
|
|
||||||
store := storage.NewFileOpts(storage.NewFileClientOpts{
|
|
||||||
ClientBaseDir: dir,
|
|
||||||
TorrentDirMaker: func(baseDir string, _ *metainfo.Info, _ metainfo.Hash) string { return baseDir },
|
|
||||||
})
|
|
||||||
defer store.Close()
|
|
||||||
torrentStorage, err := store.OpenTorrent(context.Background(), &info, metainfo.Hash{1})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
defer torrentStorage.Close()
|
|
||||||
skip := filepath.Join(root, "nested", "deep", "skip.txt")
|
|
||||||
if _, err := os.Stat(skip); err != nil {
|
|
||||||
t.Fatalf("unexpected storage layout: %v", err)
|
|
||||||
}
|
|
||||||
if err := os.WriteFile(skip+".part", []byte("partial"), 0644); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
files := info.UpvertedFiles()
|
|
||||||
boundary, name := torrentFileLayout(&info, files[1])
|
|
||||||
if err := removeUnselectedFile(dir, boundary, name); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, err := os.Stat(filepath.Join(root, "nested")); !os.IsNotExist(err) {
|
|
||||||
t.Fatalf("empty subtree remains: %v", err)
|
|
||||||
}
|
|
||||||
if _, err := os.Stat(filepath.Join(root, "keep.txt")); err != nil {
|
|
||||||
t.Fatalf("selected file removed: %v", err)
|
|
||||||
}
|
|
||||||
// Also verify the boundary when the protected directory becomes empty.
|
|
||||||
if err := os.Remove(filepath.Join(root, "keep.txt")); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
for _, file := range files {
|
|
||||||
boundary, name := torrentFileLayout(&info, file)
|
|
||||||
if err := removeUnselectedFile(dir, boundary, name); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if stat, err := os.Stat(root); err != nil || !stat.IsDir() {
|
|
||||||
t.Fatalf("cleanup boundary was removed: %v", err)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,199 +0,0 @@
|
|||||||
package bt
|
|
||||||
|
|
||||||
import (
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/GopeedLab/gopeed/internal/fetcher"
|
|
||||||
"github.com/GopeedLab/gopeed/pkg/base"
|
|
||||||
protocolbt "github.com/GopeedLab/gopeed/pkg/protocol/bt"
|
|
||||||
"github.com/RoaringBitmap/roaring"
|
|
||||||
"github.com/anacrolix/torrent"
|
|
||||||
"github.com/anacrolix/torrent/storage"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestStatsSeparatesSnapshotAndRuntime(t *testing.T) {
|
|
||||||
frame := (&Fetcher{
|
|
||||||
meta: &fetcher.FetcherMeta{Res: &base.Resource{Size: 256}},
|
|
||||||
data: &fetcherData{SeedBytes: 128, SeedTime: 64},
|
|
||||||
}).Stats()
|
|
||||||
if frame.Snapshot != nil {
|
|
||||||
t.Fatalf("snapshot = %T, want nil until piece completion is known", frame.Snapshot)
|
|
||||||
}
|
|
||||||
runtime, ok := frame.Runtime.(*protocolbt.StatsRuntime)
|
|
||||||
if !ok {
|
|
||||||
t.Fatalf("runtime type = %T", frame.Runtime)
|
|
||||||
}
|
|
||||||
if len(runtime.Peers) != 0 || runtime.TotalPeers != 0 {
|
|
||||||
t.Fatalf("unexpected runtime: %+v", runtime)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestStatsBeforeTorrentMetadataIsAvailable(t *testing.T) {
|
|
||||||
config := torrent.TestingConfig(t)
|
|
||||||
config.DisableTCP = true
|
|
||||||
config.DisableUTP = true
|
|
||||||
client, err := torrent.NewClient(config)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
defer client.Close()
|
|
||||||
|
|
||||||
torrentWithoutMetadata, err := client.AddMagnet(
|
|
||||||
"magnet:?xt=urn:btih:0123456789abcdef0123456789abcdef01234567",
|
|
||||||
)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if info := torrentWithoutMetadata.Info(); info != nil {
|
|
||||||
t.Fatalf("torrent info = %+v, want nil before metadata is available", info)
|
|
||||||
}
|
|
||||||
|
|
||||||
frame := (&Fetcher{
|
|
||||||
torrent: torrentWithoutMetadata,
|
|
||||||
meta: &fetcher.FetcherMeta{},
|
|
||||||
data: &fetcherData{},
|
|
||||||
}).Stats()
|
|
||||||
if frame.Snapshot != nil {
|
|
||||||
t.Fatalf("snapshot = %T, want nil before metadata is available", frame.Snapshot)
|
|
||||||
}
|
|
||||||
runtime, ok := frame.Runtime.(*protocolbt.StatsRuntime)
|
|
||||||
if !ok {
|
|
||||||
t.Fatalf("runtime type = %T", frame.Runtime)
|
|
||||||
}
|
|
||||||
if len(runtime.Peers) != 0 || runtime.TotalPeers != 0 {
|
|
||||||
t.Fatalf("unexpected runtime before metadata is available: %+v", runtime)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestApplyBTPieceRunsPublishesVerifiedCompletionOnly(t *testing.T) {
|
|
||||||
pieceMap := base.NewPieceMap(4, 1024)
|
|
||||||
ready := applyBTPieceRuns(pieceMap, []torrent.PieceStateRun{
|
|
||||||
{PieceState: torrent.PieceState{Completion: torrentCompletion(true, true)}, Length: 1},
|
|
||||||
{PieceState: torrent.PieceState{Completion: torrentCompletion(true, false), Partial: true}, Length: 2},
|
|
||||||
{PieceState: torrent.PieceState{Completion: torrentCompletion(false, false), Hashing: true}, Length: 1},
|
|
||||||
})
|
|
||||||
if ready {
|
|
||||||
t.Fatal("piece map became ready while one completion is unknown")
|
|
||||||
}
|
|
||||||
want := []bool{true, false, false, false}
|
|
||||||
for index, expected := range want {
|
|
||||||
got, err := pieceMap.Get(index)
|
|
||||||
if err != nil || got != expected {
|
|
||||||
t.Fatalf("piece %d = %v, %v; want %v", index, got, err, expected)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
knownMap := base.NewPieceMap(2, 1024)
|
|
||||||
if !applyBTPieceRuns(knownMap, []torrent.PieceStateRun{
|
|
||||||
{PieceState: torrent.PieceState{Completion: torrentCompletion(true, true)}, Length: 1},
|
|
||||||
{PieceState: torrent.PieceState{Completion: torrentCompletion(true, false)}, Length: 1},
|
|
||||||
}) {
|
|
||||||
t.Fatal("fully known completion map was not ready")
|
|
||||||
}
|
|
||||||
if applyBTPieceRuns(base.NewPieceMap(2, 1024), []torrent.PieceStateRun{
|
|
||||||
{PieceState: torrent.PieceState{Completion: torrentCompletion(true, true)}, Length: 1},
|
|
||||||
}) {
|
|
||||||
t.Fatal("incomplete piece runs were published")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func torrentCompletion(ok, complete bool) storage.Completion {
|
|
||||||
return storage.Completion{Ok: ok, Complete: complete}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestNormalizeBTPeerMetadata(t *testing.T) {
|
|
||||||
if got := normalizeBTPeerSource(torrent.PeerSourceDhtGetPeers); got != "dht" {
|
|
||||||
t.Fatalf("DHT source = %q", got)
|
|
||||||
}
|
|
||||||
if got := normalizeBTPeerSource(torrent.PeerSourcePex); got != "pex" {
|
|
||||||
t.Fatalf("PEX source = %q", got)
|
|
||||||
}
|
|
||||||
transports := map[string]string{
|
|
||||||
"udp4": "utp",
|
|
||||||
"udp6": "utp",
|
|
||||||
"tcp4": "tcp",
|
|
||||||
"tcp6": "tcp",
|
|
||||||
"webrtc": "webrtc",
|
|
||||||
"unix": "unknown",
|
|
||||||
}
|
|
||||||
for network, want := range transports {
|
|
||||||
if got := normalizeBTTransport(network); got != want {
|
|
||||||
t.Errorf("normalizeBTTransport(%q) = %q, want %q", network, got, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestBTPeerCompletion(t *testing.T) {
|
|
||||||
if got := btPeerCompletion(1, 4); got == nil || *got != 0.25 {
|
|
||||||
t.Fatalf("completion = %v, want 0.25", got)
|
|
||||||
}
|
|
||||||
if got := btPeerCompletion(5, 4); got == nil || *got != 1 {
|
|
||||||
t.Fatalf("clamped completion = %v, want 1", got)
|
|
||||||
}
|
|
||||||
if got := btPeerCompletion(-1, 4); got == nil || *got != 0 {
|
|
||||||
t.Fatalf("negative completion = %v, want 0", got)
|
|
||||||
}
|
|
||||||
if got := btPeerCompletion(0, 0); got != nil {
|
|
||||||
t.Fatalf("completion without metadata = %v, want nil", *got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestBTPeerRelevanceUsesWantedMissingPieces(t *testing.T) {
|
|
||||||
missing := btMissingWantedPieces([]torrent.PieceStateRun{
|
|
||||||
{
|
|
||||||
PieceState: torrent.PieceState{
|
|
||||||
Completion: torrentCompletion(true, true),
|
|
||||||
Priority: torrent.PiecePriorityNormal,
|
|
||||||
},
|
|
||||||
Length: 2,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
PieceState: torrent.PieceState{
|
|
||||||
Completion: torrentCompletion(true, false),
|
|
||||||
Priority: torrent.PiecePriorityNormal,
|
|
||||||
},
|
|
||||||
Length: 2,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
PieceState: torrent.PieceState{
|
|
||||||
Completion: torrentCompletion(true, false),
|
|
||||||
Priority: torrent.PiecePriorityNone,
|
|
||||||
},
|
|
||||||
Length: 2,
|
|
||||||
},
|
|
||||||
}, 6)
|
|
||||||
if missing == nil || !missing.Equals(roaring.BitmapOf(2, 3)) {
|
|
||||||
t.Fatalf("missing wanted pieces = %v, want {2, 3}", missing)
|
|
||||||
}
|
|
||||||
|
|
||||||
if got := btPeerRelevance(roaring.BitmapOf(0, 2, 4), missing); got == nil || *got != 0.5 {
|
|
||||||
t.Fatalf("relevance = %v, want 0.5", got)
|
|
||||||
}
|
|
||||||
if got := btPeerRelevance(roaring.BitmapOf(0, 1, 4), missing); got == nil || *got != 0 {
|
|
||||||
t.Fatalf("zero relevance = %v, want 0", got)
|
|
||||||
}
|
|
||||||
if got := btPeerRelevance(roaring.BitmapOf(2, 3), roaring.New()); got != nil {
|
|
||||||
t.Fatalf("relevance with no missing pieces = %v, want nil", *got)
|
|
||||||
}
|
|
||||||
if got := btMissingWantedPieces(nil, 6); got != nil {
|
|
||||||
t.Fatalf("incomplete piece states = %v, want nil", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestBTPeerRelevanceTracksRemainingDownload(t *testing.T) {
|
|
||||||
missing := roaring.BitmapOf(90, 91, 92, 93, 94, 95, 96, 97, 98, 99)
|
|
||||||
if got := btPeerRelevance(missing.Clone(), missing); got == nil || *got != 1 {
|
|
||||||
t.Fatalf("peer covering every missing piece = %v, want 1", got)
|
|
||||||
}
|
|
||||||
|
|
||||||
oneRelevantPiece := roaring.BitmapOf(90)
|
|
||||||
if got := btPeerRelevance(oneRelevantPiece, missing); got == nil || *got != 0.1 {
|
|
||||||
t.Fatalf("peer covering one of ten missing pieces = %v, want 0.1", got)
|
|
||||||
}
|
|
||||||
|
|
||||||
missingAfterTransfer := missing.Clone()
|
|
||||||
missingAfterTransfer.Remove(90)
|
|
||||||
if got := btPeerRelevance(oneRelevantPiece, missingAfterTransfer); got == nil || *got != 0 {
|
|
||||||
t.Fatalf("peer after transferring its only relevant piece = %v, want 0", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,116 +0,0 @@
|
|||||||
package bt
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"os"
|
|
||||||
"sync"
|
|
||||||
|
|
||||||
g "github.com/anacrolix/generics"
|
|
||||||
"github.com/anacrolix/torrent/metainfo"
|
|
||||||
"github.com/anacrolix/torrent/storage"
|
|
||||||
)
|
|
||||||
|
|
||||||
// deferredFileStorage lets metadata resolve without opening the user's files.
|
|
||||||
// It binds once. After binding, pieces are supplied directly by file storage.
|
|
||||||
type deferredFileStorage struct {
|
|
||||||
mu sync.RWMutex
|
|
||||||
ctx context.Context
|
|
||||||
info *metainfo.Info
|
|
||||||
hash metainfo.Hash
|
|
||||||
knownHashes []bool
|
|
||||||
owner storage.ClientImplCloser
|
|
||||||
torrent storage.TorrentImpl
|
|
||||||
closed bool
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *deferredFileStorage) OpenTorrent(ctx context.Context, info *metainfo.Info, hash metainfo.Hash) (storage.TorrentImpl, error) {
|
|
||||||
s.mu.Lock()
|
|
||||||
defer s.mu.Unlock()
|
|
||||||
if s.closed || s.info != nil {
|
|
||||||
return storage.TorrentImpl{}, errors.New("torrent storage already opened or closed")
|
|
||||||
}
|
|
||||||
s.ctx, s.info, s.hash = ctx, info, hash
|
|
||||||
s.knownHashes = make([]bool, info.NumPieces())
|
|
||||||
return storage.TorrentImpl{
|
|
||||||
PieceWithHash: func(p metainfo.Piece, hash g.Option[[]byte]) storage.PieceImpl {
|
|
||||||
s.mu.Lock()
|
|
||||||
defer s.mu.Unlock()
|
|
||||||
if s.closed {
|
|
||||||
return metadataPiece{err: os.ErrClosed}
|
|
||||||
}
|
|
||||||
if s.owner == nil {
|
|
||||||
s.knownHashes[p.Index()] = hash.Ok
|
|
||||||
// A missing v2 hash must remain unknown until its arrival changes
|
|
||||||
// completion and updates the library's piece request priorities.
|
|
||||||
return metadataPiece{known: hash.Ok}
|
|
||||||
}
|
|
||||||
if s.torrent.PieceWithHash != nil {
|
|
||||||
return s.torrent.PieceWithHash(p, hash)
|
|
||||||
}
|
|
||||||
return s.torrent.Piece(p)
|
|
||||||
},
|
|
||||||
Close: s.Close,
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *deferredFileStorage) bind(opts storage.NewFileClientOpts) ([]bool, error) {
|
|
||||||
s.mu.Lock()
|
|
||||||
defer s.mu.Unlock()
|
|
||||||
if s.closed {
|
|
||||||
return nil, os.ErrClosed
|
|
||||||
}
|
|
||||||
if s.info == nil {
|
|
||||||
return nil, errors.New("torrent metadata is not ready")
|
|
||||||
}
|
|
||||||
if s.owner != nil {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
if opts.FilePathMaker == nil {
|
|
||||||
opts.FilePathMaker = func(o storage.FilePathMakerOpts) string { _, name := torrentFileLayout(o.Info, *o.File); return name }
|
|
||||||
}
|
|
||||||
owner := storage.NewFileOpts(opts)
|
|
||||||
t, err := owner.OpenTorrent(s.ctx, s.info, s.hash)
|
|
||||||
if err != nil {
|
|
||||||
owner.Close()
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
s.owner, s.torrent = owner, t
|
|
||||||
return append([]bool(nil), s.knownHashes...), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *deferredFileStorage) Close() error {
|
|
||||||
s.mu.Lock()
|
|
||||||
defer s.mu.Unlock()
|
|
||||||
if s.closed {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
s.closed = true
|
|
||||||
if s.owner == nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
var err error
|
|
||||||
if s.torrent.Close != nil {
|
|
||||||
err = s.torrent.Close()
|
|
||||||
}
|
|
||||||
return errors.Join(err, s.owner.Close())
|
|
||||||
}
|
|
||||||
|
|
||||||
type metadataPiece struct {
|
|
||||||
known bool
|
|
||||||
err error
|
|
||||||
}
|
|
||||||
|
|
||||||
func (p metadataPiece) Completion() storage.Completion {
|
|
||||||
return storage.Completion{Ok: p.known, Err: p.err}
|
|
||||||
}
|
|
||||||
func (p metadataPiece) ReadAt([]byte, int64) (int, error) {
|
|
||||||
return 0, errors.New("download storage is not initialized")
|
|
||||||
}
|
|
||||||
func (p metadataPiece) WriteAt([]byte, int64) (int, error) {
|
|
||||||
return 0, errors.New("download storage is not initialized")
|
|
||||||
}
|
|
||||||
func (p metadataPiece) MarkComplete() error { return errors.New("download storage is not initialized") }
|
|
||||||
func (p metadataPiece) MarkNotComplete() error {
|
|
||||||
return errors.New("download storage is not initialized")
|
|
||||||
}
|
|
||||||
@@ -1,234 +0,0 @@
|
|||||||
package bt
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bufio"
|
|
||||||
"bytes"
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"net/url"
|
|
||||||
"slices"
|
|
||||||
"strings"
|
|
||||||
"sync"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/GopeedLab/gopeed/internal/fetcher"
|
|
||||||
)
|
|
||||||
|
|
||||||
var defaultTrackerSubscribeURLs = []string{
|
|
||||||
"https://raw.githubusercontent.com/ngosang/trackerslist/master/trackers_all.txt",
|
|
||||||
"https://raw.githubusercontent.com/ngosang/trackerslist/master/trackers_all_http.txt",
|
|
||||||
"https://raw.githubusercontent.com/ngosang/trackerslist/master/trackers_all_https.txt",
|
|
||||||
"https://raw.githubusercontent.com/ngosang/trackerslist/master/trackers_all_ip.txt",
|
|
||||||
"https://raw.githubusercontent.com/ngosang/trackerslist/master/trackers_all_udp.txt",
|
|
||||||
"https://raw.githubusercontent.com/ngosang/trackerslist/master/trackers_all_ws.txt",
|
|
||||||
"https://raw.githubusercontent.com/ngosang/trackerslist/master/trackers_best.txt",
|
|
||||||
"https://raw.githubusercontent.com/ngosang/trackerslist/master/trackers_best_ip.txt",
|
|
||||||
"https://raw.githubusercontent.com/XIU2/TrackersListCollection/master/all.txt",
|
|
||||||
"https://raw.githubusercontent.com/XIU2/TrackersListCollection/master/best.txt",
|
|
||||||
"https://raw.githubusercontent.com/XIU2/TrackersListCollection/master/http.txt",
|
|
||||||
}
|
|
||||||
|
|
||||||
func validTracker(raw string) bool {
|
|
||||||
u, err := url.Parse(raw)
|
|
||||||
if err != nil || u.Hostname() == "" || u.User != nil {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
switch u.Scheme {
|
|
||||||
case "udp", "http", "https", "ws", "wss":
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
func uniqueTrackers(groups ...[]string) []string {
|
|
||||||
result := []string{}
|
|
||||||
seen := map[string]bool{}
|
|
||||||
for _, group := range groups {
|
|
||||||
for _, raw := range group {
|
|
||||||
value := strings.TrimSpace(raw)
|
|
||||||
if value != "" && !seen[value] {
|
|
||||||
seen[value] = true
|
|
||||||
result = append(result, value)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return result
|
|
||||||
}
|
|
||||||
|
|
||||||
// effectiveTrackers derives the client list without persisting duplicate lists.
|
|
||||||
func (c *config) effectiveTrackers() []string {
|
|
||||||
groups := [][]string{c.CustomTrackers}
|
|
||||||
for _, source := range c.TrackerSubscribeURLs {
|
|
||||||
groups = append(groups, c.State.TrackerSources[source])
|
|
||||||
}
|
|
||||||
return uniqueTrackers(groups...)
|
|
||||||
}
|
|
||||||
func (fm *FetcherManager) NormalizeConfig(raw json.RawMessage) (any, error) {
|
|
||||||
c := fm.DefaultConfig().(*config)
|
|
||||||
if err := json.Unmarshal(raw, c); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
if c.ListenPort < 0 || c.ListenPort > 65535 || c.SeedRatio < 0 || c.SeedTime < 0 {
|
|
||||||
return nil, errors.New("invalid BT configuration")
|
|
||||||
}
|
|
||||||
if len(c.TrackerSubscribeURLs) > 64 {
|
|
||||||
return nil, errors.New("too many tracker subscriptions")
|
|
||||||
}
|
|
||||||
c.TrackerSubscribeURLs = uniqueTrackers(c.TrackerSubscribeURLs)
|
|
||||||
for _, source := range c.TrackerSubscribeURLs {
|
|
||||||
u, err := url.Parse(source)
|
|
||||||
if err != nil || u == nil || (u.Scheme != "https" && u.Scheme != "http") || u.Hostname() == "" || u.User != nil {
|
|
||||||
return nil, errors.New("invalid tracker subscription URL")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
c.CustomTrackers = uniqueTrackers(c.CustomTrackers)
|
|
||||||
for _, tracker := range c.CustomTrackers {
|
|
||||||
if !validTracker(tracker) {
|
|
||||||
return nil, fmt.Errorf("invalid tracker URL: %s", tracker)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if c.State.TrackerSources == nil {
|
|
||||||
c.State.TrackerSources = map[string][]string{}
|
|
||||||
}
|
|
||||||
for source := range c.State.TrackerSources {
|
|
||||||
if !slices.Contains(c.TrackerSubscribeURLs, source) {
|
|
||||||
delete(c.State.TrackerSources, source)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if c.State.TrackerSubscriptionStatuses == nil {
|
|
||||||
c.State.TrackerSubscriptionStatuses = map[string]trackerSubscriptionStatus{}
|
|
||||||
}
|
|
||||||
for source := range c.State.TrackerSubscriptionStatuses {
|
|
||||||
if !slices.Contains(c.TrackerSubscribeURLs, source) {
|
|
||||||
delete(c.State.TrackerSubscriptionStatuses, source)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return c, nil
|
|
||||||
}
|
|
||||||
func (fm *FetcherManager) CronJobs(host fetcher.ProtocolHost) []fetcher.CronJob {
|
|
||||||
return []fetcher.CronJob{{
|
|
||||||
Name: "update-trackers",
|
|
||||||
Every: 24 * time.Hour,
|
|
||||||
Timeout: 90 * time.Second,
|
|
||||||
Run: func(ctx context.Context, source fetcher.TriggerSource) error {
|
|
||||||
return fm.updateTrackers(ctx, host, source)
|
|
||||||
},
|
|
||||||
}}
|
|
||||||
}
|
|
||||||
func parseTrackers(data []byte) ([]string, error) {
|
|
||||||
scanner := bufio.NewScanner(bytes.NewReader(data))
|
|
||||||
scanner.Buffer(make([]byte, 4096), 64*1024)
|
|
||||||
result := []string{}
|
|
||||||
for scanner.Scan() {
|
|
||||||
line := strings.TrimSpace(strings.TrimPrefix(scanner.Text(), "\ufeff"))
|
|
||||||
if validTracker(line) {
|
|
||||||
result = append(result, line)
|
|
||||||
}
|
|
||||||
if len(result) > 20000 {
|
|
||||||
return nil, errors.New("too many trackers")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if err := scanner.Err(); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
if len(result) == 0 {
|
|
||||||
return nil, errors.New("subscription contains no valid trackers")
|
|
||||||
}
|
|
||||||
return uniqueTrackers(result), nil
|
|
||||||
}
|
|
||||||
func (fm *FetcherManager) updateTrackers(ctx context.Context, host fetcher.ProtocolHost, source fetcher.TriggerSource) error {
|
|
||||||
snapshot, err := fetcher.ReadConfig[config](host)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if (source == fetcher.TriggerScheduled && !snapshot.AutoUpdateTrackers) || len(snapshot.TrackerSubscribeURLs) == 0 {
|
|
||||||
return fetcher.ErrJobSkipped
|
|
||||||
}
|
|
||||||
fetched := map[string][]string{}
|
|
||||||
statuses := map[string]trackerSubscriptionStatus{}
|
|
||||||
var failures []error
|
|
||||||
var mu sync.Mutex
|
|
||||||
var wg sync.WaitGroup
|
|
||||||
slots := make(chan struct{}, 4)
|
|
||||||
for _, address := range snapshot.TrackerSubscribeURLs {
|
|
||||||
wg.Add(1)
|
|
||||||
go func(address string) {
|
|
||||||
defer wg.Done()
|
|
||||||
select {
|
|
||||||
case slots <- struct{}{}:
|
|
||||||
defer func() { <-slots }()
|
|
||||||
case <-ctx.Done():
|
|
||||||
return
|
|
||||||
}
|
|
||||||
data, err := host.Fetch(ctx, address, 2<<20, func(body []byte) error { _, err := parseTrackers(body); return err })
|
|
||||||
var trackers []string
|
|
||||||
if err == nil {
|
|
||||||
trackers, err = parseTrackers(data)
|
|
||||||
}
|
|
||||||
mu.Lock()
|
|
||||||
defer mu.Unlock()
|
|
||||||
statuses[address] = trackerSubscriptionStatus{Success: err == nil, TrackerCount: len(trackers), LastFetchTime: time.Now().UTC()}
|
|
||||||
if err != nil {
|
|
||||||
failures = append(failures, fmt.Errorf("subscription %s: %w", address, err))
|
|
||||||
} else {
|
|
||||||
fetched[address] = trackers
|
|
||||||
}
|
|
||||||
}(address)
|
|
||||||
}
|
|
||||||
wg.Wait()
|
|
||||||
if err := ctx.Err(); err != nil {
|
|
||||||
failures = append(failures, err)
|
|
||||||
}
|
|
||||||
if len(statuses) == 0 {
|
|
||||||
return errors.Join(failures...)
|
|
||||||
}
|
|
||||||
err = fetcher.UpdateState[config, state](host, func(current *config, state *state) error {
|
|
||||||
if !slices.Equal(current.TrackerSubscribeURLs, snapshot.TrackerSubscribeURLs) {
|
|
||||||
return fetcher.ErrConfigChanged
|
|
||||||
}
|
|
||||||
if state.TrackerSources == nil {
|
|
||||||
state.TrackerSources = map[string][]string{}
|
|
||||||
}
|
|
||||||
if state.TrackerSubscriptionStatuses == nil {
|
|
||||||
state.TrackerSubscriptionStatuses = map[string]trackerSubscriptionStatus{}
|
|
||||||
}
|
|
||||||
for address, status := range statuses {
|
|
||||||
state.TrackerSubscriptionStatuses[address] = status
|
|
||||||
}
|
|
||||||
for address, trackers := range fetched {
|
|
||||||
state.TrackerSources[address] = trackers
|
|
||||||
}
|
|
||||||
if len(failures) == 0 {
|
|
||||||
state.LastTrackerUpdateTime = time.Now().UTC()
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
current, err := fetcher.ReadConfig[config](host)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
fm.applyTrackers(current.effectiveTrackers())
|
|
||||||
return errors.Join(failures...)
|
|
||||||
}
|
|
||||||
func (fm *FetcherManager) applyTrackers(trackers []string) {
|
|
||||||
lock.Lock()
|
|
||||||
defer lock.Unlock()
|
|
||||||
if client == nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
announce := make([][]string, 0, len(trackers))
|
|
||||||
for _, tracker := range trackers {
|
|
||||||
announce = append(announce, []string{tracker})
|
|
||||||
}
|
|
||||||
for _, t := range client.Torrents() {
|
|
||||||
info := t.Info()
|
|
||||||
if info == nil || (info.Private != nil && *info.Private) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
t.AddTrackers(announce)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,217 +0,0 @@
|
|||||||
package bt
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
|
||||||
"reflect"
|
|
||||||
"sync"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/GopeedLab/gopeed/internal/fetcher"
|
|
||||||
)
|
|
||||||
|
|
||||||
func trackerTestHost(t *testing.T, c *config, fetch func(context.Context, string, int64, func([]byte) error) ([]byte, error)) (fetcher.ProtocolHost, *sync.Mutex) {
|
|
||||||
t.Helper()
|
|
||||||
mu := &sync.Mutex{}
|
|
||||||
return fetcher.ProtocolHost{
|
|
||||||
Read: func(dst any) error {
|
|
||||||
mu.Lock()
|
|
||||||
defer mu.Unlock()
|
|
||||||
raw, err := json.Marshal(c)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
return json.Unmarshal(raw, dst)
|
|
||||||
},
|
|
||||||
UpdateState: func(change func(config, state json.RawMessage) (any, error)) error {
|
|
||||||
mu.Lock()
|
|
||||||
defer mu.Unlock()
|
|
||||||
raw, _ := json.Marshal(c)
|
|
||||||
currentState, _ := json.Marshal(c.State)
|
|
||||||
next, err := change(raw, currentState)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
raw, err = json.Marshal(next)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
return json.Unmarshal(raw, &c.State)
|
|
||||||
},
|
|
||||||
Fetch: fetch,
|
|
||||||
}, mu
|
|
||||||
}
|
|
||||||
func TestTrackerRefreshPartialSuccessUsesLatestConfig(t *testing.T) {
|
|
||||||
fm := &FetcherManager{}
|
|
||||||
cfg := fm.DefaultConfig().(*config)
|
|
||||||
cfg.TrackerSubscribeURLs = []string{"https://a.example/list", "https://b.example/list"}
|
|
||||||
cfg.CustomTrackers = []string{"udp://custom.example:80/announce"}
|
|
||||||
cfg.State.TrackerSources = map[string][]string{cfg.TrackerSubscribeURLs[0]: {"udp://old.example:80/announce"}, cfg.TrackerSubscribeURLs[1]: {"udp://cached.example:80/announce"}}
|
|
||||||
host, mu := trackerTestHost(t, cfg, nil)
|
|
||||||
host.Fetch = func(ctx context.Context, address string, _ int64, validate func([]byte) error) ([]byte, error) {
|
|
||||||
if address == "https://b.example/list" {
|
|
||||||
return nil, errors.New("offline")
|
|
||||||
}
|
|
||||||
// Simulate a frontend update while the network request is in flight.
|
|
||||||
mu.Lock()
|
|
||||||
cfg.CustomTrackers = []string{"udp://new-custom.example:80/announce"}
|
|
||||||
mu.Unlock()
|
|
||||||
data := []byte("udp://fresh.example:80/announce\n# ignored\nudp://fresh.example:80/announce\n")
|
|
||||||
return data, validate(data)
|
|
||||||
}
|
|
||||||
err := fm.updateTrackers(context.Background(), host, fetcher.TriggerManual)
|
|
||||||
if err == nil {
|
|
||||||
t.Fatal("partial failure must be reported")
|
|
||||||
}
|
|
||||||
want := []string{"udp://new-custom.example:80/announce", "udp://fresh.example:80/announce", "udp://cached.example:80/announce"}
|
|
||||||
if !reflect.DeepEqual(cfg.effectiveTrackers(), want) {
|
|
||||||
t.Fatalf("got %v", cfg.effectiveTrackers())
|
|
||||||
}
|
|
||||||
success := cfg.State.TrackerSubscriptionStatuses["https://a.example/list"]
|
|
||||||
failure := cfg.State.TrackerSubscriptionStatuses["https://b.example/list"]
|
|
||||||
if !success.Success || success.TrackerCount != 1 || success.LastFetchTime.IsZero() || failure.Success || failure.TrackerCount != 0 || failure.LastFetchTime.IsZero() {
|
|
||||||
t.Fatalf("unexpected subscription statuses: %+v", cfg.State.TrackerSubscriptionStatuses)
|
|
||||||
}
|
|
||||||
if !cfg.State.LastTrackerUpdateTime.IsZero() {
|
|
||||||
t.Fatal("partial refresh marked fully successful")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
func TestTrackerRefreshDiscardsChangedSubscriptions(t *testing.T) {
|
|
||||||
fm := &FetcherManager{}
|
|
||||||
cfg := fm.DefaultConfig().(*config)
|
|
||||||
cfg.TrackerSubscribeURLs = []string{"https://a.example/list"}
|
|
||||||
host, mu := trackerTestHost(t, cfg, nil)
|
|
||||||
host.Fetch = func(context.Context, string, int64, func([]byte) error) ([]byte, error) {
|
|
||||||
mu.Lock()
|
|
||||||
cfg.TrackerSubscribeURLs = []string{}
|
|
||||||
mu.Unlock()
|
|
||||||
return []byte("udp://fresh.example:80/announce"), nil
|
|
||||||
}
|
|
||||||
if err := fm.updateTrackers(context.Background(), host, fetcher.TriggerManual); !errors.Is(err, fetcher.ErrConfigChanged) {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if len(cfg.State.TrackerSources) != 0 {
|
|
||||||
t.Fatal("outdated source was committed")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
func TestTrackerRefreshManualWorksWhenAutomaticDisabled(t *testing.T) {
|
|
||||||
fm := &FetcherManager{}
|
|
||||||
cfg := fm.DefaultConfig().(*config)
|
|
||||||
cfg.AutoUpdateTrackers = false
|
|
||||||
cfg.TrackerSubscribeURLs = []string{"https://a.example/list"}
|
|
||||||
calls := 0
|
|
||||||
host, _ := trackerTestHost(t, cfg, func(context.Context, string, int64, func([]byte) error) ([]byte, error) {
|
|
||||||
calls++
|
|
||||||
return []byte("udp://fresh.example:80/announce"), nil
|
|
||||||
})
|
|
||||||
if err := fm.updateTrackers(context.Background(), host, fetcher.TriggerScheduled); !errors.Is(err, fetcher.ErrJobSkipped) {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if calls != 0 {
|
|
||||||
t.Fatal("scheduled job fetched while disabled")
|
|
||||||
}
|
|
||||||
if err := fm.updateTrackers(context.Background(), host, fetcher.TriggerManual); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if calls != 1 || cfg.State.LastTrackerUpdateTime.IsZero() {
|
|
||||||
t.Fatal("manual refresh did not succeed")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
func TestTrackerConfigPreservesExplicitSettings(t *testing.T) {
|
|
||||||
value, err := (&FetcherManager{}).NormalizeConfig(json.RawMessage(`{
|
|
||||||
"listenPort":6881, "trackerSubscribeUrls":[], "autoUpdateTrackers":false,
|
|
||||||
"customTrackers":["udp://custom.example:80/announce"]
|
|
||||||
}`))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
cfg := value.(*config)
|
|
||||||
if cfg.AutoUpdateTrackers || len(cfg.TrackerSubscribeURLs) != 0 || cfg.ListenPort != 6881 {
|
|
||||||
t.Fatalf("%+v", cfg)
|
|
||||||
}
|
|
||||||
if !reflect.DeepEqual(cfg.effectiveTrackers(), cfg.CustomTrackers) {
|
|
||||||
t.Fatalf("derived trackers: %v", cfg.effectiveTrackers())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestParseTrackerList(t *testing.T) {
|
|
||||||
got, err := parseTrackers([]byte("\ufeffudp://tracker.example:80/announce\r\n\n<html>error</html>\nhttps://tracker.example/announce\n"))
|
|
||||||
if err != nil || len(got) != 2 {
|
|
||||||
t.Fatalf("%v %v", got, err)
|
|
||||||
}
|
|
||||||
if _, err := parseTrackers([]byte("<html>mirror error</html>")); err == nil {
|
|
||||||
t.Fatal("invalid mirror body accepted")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
func TestTrackerDefaultsRegisterDailyJob(t *testing.T) {
|
|
||||||
jobs := (&FetcherManager{}).CronJobs(fetcher.ProtocolHost{})
|
|
||||||
if len(jobs) != 1 || jobs[0].Name != "update-trackers" || jobs[0].Every != 24*time.Hour {
|
|
||||||
t.Fatal(jobs)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestTrackerRefreshAllFailuresPersistStatusesAndKeepCache(t *testing.T) {
|
|
||||||
fm := &FetcherManager{}
|
|
||||||
cfg := fm.DefaultConfig().(*config)
|
|
||||||
address := "https://custom.example/trackers.txt"
|
|
||||||
cfg.TrackerSubscribeURLs = []string{address}
|
|
||||||
cfg.State.TrackerSources[address] = []string{"udp://cached.example:80/announce"}
|
|
||||||
cfg.State.TrackerSubscriptionStatuses[address] = trackerSubscriptionStatus{Success: true, TrackerCount: 1, LastFetchTime: time.Now().Add(-time.Hour)}
|
|
||||||
host, _ := trackerTestHost(t, cfg, func(context.Context, string, int64, func([]byte) error) ([]byte, error) {
|
|
||||||
return nil, errors.New("offline")
|
|
||||||
})
|
|
||||||
before := time.Now()
|
|
||||||
if err := fm.updateTrackers(context.Background(), host, fetcher.TriggerManual); err == nil {
|
|
||||||
t.Fatal("failure was not reported")
|
|
||||||
}
|
|
||||||
status, ok := cfg.State.TrackerSubscriptionStatuses[address]
|
|
||||||
if !ok || status.Success || status.TrackerCount != 0 || status.LastFetchTime.Before(before) {
|
|
||||||
t.Fatalf("failed fetch status not stored: %+v", status)
|
|
||||||
}
|
|
||||||
if !reflect.DeepEqual(cfg.State.TrackerSources[address], []string{"udp://cached.example:80/announce"}) {
|
|
||||||
t.Fatal("last good cache lost")
|
|
||||||
}
|
|
||||||
if !reflect.DeepEqual(cfg.effectiveTrackers(), cfg.State.TrackerSources[address]) {
|
|
||||||
t.Fatal("cached trackers not retained")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestTrackerConfigPrunesRemovedSubscriptions(t *testing.T) {
|
|
||||||
fm := &FetcherManager{}
|
|
||||||
cfg := fm.DefaultConfig().(*config)
|
|
||||||
address := "https://custom.example/trackers.txt"
|
|
||||||
cfg.TrackerSubscribeURLs = []string{}
|
|
||||||
cfg.State.TrackerSources[address] = []string{"udp://cached.example:80/announce"}
|
|
||||||
cfg.State.TrackerSubscriptionStatuses[address] = trackerSubscriptionStatus{Success: true, TrackerCount: 1, LastFetchTime: time.Now()}
|
|
||||||
raw, _ := json.Marshal(cfg)
|
|
||||||
value, err := fm.NormalizeConfig(raw)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
next := value.(*config)
|
|
||||||
if len(next.State.TrackerSources) != 0 || len(next.State.TrackerSubscriptionStatuses) != 0 || len(next.effectiveTrackers()) != 0 {
|
|
||||||
t.Fatal("removed subscription still owns cached results")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestTrackerRefreshTimeoutPersistsCompletedAttemptStatus(t *testing.T) {
|
|
||||||
fm := &FetcherManager{}
|
|
||||||
cfg := fm.DefaultConfig().(*config)
|
|
||||||
address := "https://custom.example/trackers.txt"
|
|
||||||
cfg.TrackerSubscribeURLs = []string{address}
|
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
|
||||||
defer cancel()
|
|
||||||
host, _ := trackerTestHost(t, cfg, func(context.Context, string, int64, func([]byte) error) ([]byte, error) {
|
|
||||||
cancel()
|
|
||||||
return nil, context.DeadlineExceeded
|
|
||||||
})
|
|
||||||
if err := fm.updateTrackers(ctx, host, fetcher.TriggerManual); !errors.Is(err, context.DeadlineExceeded) {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
status, ok := cfg.State.TrackerSubscriptionStatuses[address]
|
|
||||||
if !ok || status.Success || status.TrackerCount != 0 || status.LastFetchTime.IsZero() {
|
|
||||||
t.Fatalf("timeout status lost: %+v", status)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,15 +0,0 @@
|
|||||||
package ed2k
|
|
||||||
|
|
||||||
const (
|
|
||||||
defaultServerList = "45.82.80.155:5687,176.123.5.89:4725,85.121.5.137:4232,176.123.2.239:4232,145.239.2.134:4661,91.208.162.87:4232,37.15.61.236:4232"
|
|
||||||
defaultServerMet = "ed2k://|serverlist|http://upd.emule-security.org/server.met|/"
|
|
||||||
defaultNodesDat = "https://upd.emule-security.org/nodes.dat"
|
|
||||||
)
|
|
||||||
|
|
||||||
type config struct {
|
|
||||||
ListenPort int `json:"listenPort"`
|
|
||||||
UDPPort int `json:"udpPort"`
|
|
||||||
ServerAddr string `json:"serverAddr"`
|
|
||||||
ServerMet string `json:"serverMet"`
|
|
||||||
NodesDat string `json:"nodesDat"`
|
|
||||||
}
|
|
||||||
@@ -1,555 +0,0 @@
|
|||||||
package ed2k
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
|
||||||
"sync"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/GopeedLab/gopeed/internal/controller"
|
|
||||||
"github.com/GopeedLab/gopeed/internal/fetcher"
|
|
||||||
"github.com/GopeedLab/gopeed/pkg/base"
|
|
||||||
ped2k "github.com/GopeedLab/gopeed/pkg/protocol/ed2k"
|
|
||||||
"github.com/monkeyWie/goed2k"
|
|
||||||
gprotocol "github.com/monkeyWie/goed2k/protocol"
|
|
||||||
)
|
|
||||||
|
|
||||||
type clientStateStore struct {
|
|
||||||
store fetcher.ProtocolStateStore
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *clientStateStore) Load() (*goed2k.ClientState, error) {
|
|
||||||
if s == nil || s.store == nil {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
var state goed2k.ClientState
|
|
||||||
exist, err := s.store.Load(&state)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
if !exist {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
return &state, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *clientStateStore) Save(state *goed2k.ClientState) error {
|
|
||||||
if s == nil || s.store == nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
if state == nil {
|
|
||||||
return s.store.Delete()
|
|
||||||
}
|
|
||||||
return s.store.Save(state)
|
|
||||||
}
|
|
||||||
|
|
||||||
type Fetcher struct {
|
|
||||||
ctl *controller.Controller
|
|
||||||
config *config
|
|
||||||
|
|
||||||
manager *FetcherManager
|
|
||||||
meta *fetcher.FetcherMeta
|
|
||||||
handle goed2k.TransferHandle
|
|
||||||
|
|
||||||
waitCtx context.Context
|
|
||||||
waitCancel context.CancelFunc
|
|
||||||
}
|
|
||||||
|
|
||||||
func (f *Fetcher) Setup(ctl *controller.Controller) {
|
|
||||||
f.ctl = ctl
|
|
||||||
if f.meta == nil {
|
|
||||||
f.meta = &fetcher.FetcherMeta{}
|
|
||||||
}
|
|
||||||
f.waitCtx, f.waitCancel = context.WithCancel(context.Background())
|
|
||||||
f.ctl.GetConfig(&f.config)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (f *Fetcher) Resolve(req *base.Request) error {
|
|
||||||
link, err := parseLink(req.URL)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
f.meta.Req = req
|
|
||||||
|
|
||||||
f.meta.Res = buildResource(link)
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (f *Fetcher) Start() error {
|
|
||||||
link, err := parseLink(f.meta.Req.URL)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if f.meta.Res == nil {
|
|
||||||
f.meta.Res = buildResource(link)
|
|
||||||
}
|
|
||||||
|
|
||||||
client, err := f.getClient()
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
targetPath := f.meta.SingleFilepath()
|
|
||||||
if err := os.MkdirAll(filepath.Dir(targetPath), 0o755); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
handle := client.FindTransfer(link.Hash)
|
|
||||||
if handle.IsValid() {
|
|
||||||
f.handle = handle
|
|
||||||
if handle.IsPaused() {
|
|
||||||
if err := client.ResumeTransfer(handle.GetHash()); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
atp := goed2k.AddTransferParams{
|
|
||||||
Hash: link.Hash,
|
|
||||||
CreateTime: time.Now().UnixMilli(),
|
|
||||||
Size: link.NumberValue,
|
|
||||||
FilePath: targetPath,
|
|
||||||
}
|
|
||||||
handle, err = client.AddTransfer(atp)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
f.handle = handle
|
|
||||||
if handle.IsValid() && handle.IsPaused() {
|
|
||||||
if err := client.ResumeTransfer(handle.GetHash()); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (f *Fetcher) Patch(req *base.Request, opts *base.Options) error {
|
|
||||||
handle := f.currentHandle()
|
|
||||||
|
|
||||||
if opts != nil && (opts.Name != "" || opts.Path != "") && handle.IsValid() {
|
|
||||||
return errors.New("cannot change ed2k target path after transfer started")
|
|
||||||
}
|
|
||||||
if req != nil && req.URL != "" && handle.IsValid() {
|
|
||||||
return errors.New("cannot change ed2k link after transfer started")
|
|
||||||
}
|
|
||||||
|
|
||||||
if req != nil {
|
|
||||||
if req.URL != "" {
|
|
||||||
link, err := parseLink(req.URL)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
f.meta.Req.URL = req.URL
|
|
||||||
f.meta.Res = buildResource(link)
|
|
||||||
}
|
|
||||||
if req.Labels != nil {
|
|
||||||
if f.meta.Req.Labels == nil {
|
|
||||||
f.meta.Req.Labels = make(map[string]string)
|
|
||||||
}
|
|
||||||
for k, v := range req.Labels {
|
|
||||||
f.meta.Req.Labels[k] = v
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if req.Proxy != nil {
|
|
||||||
f.meta.Req.Proxy = req.Proxy
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if opts != nil {
|
|
||||||
if opts.Name != "" {
|
|
||||||
f.meta.Opts.Name = opts.Name
|
|
||||||
}
|
|
||||||
if opts.Path != "" {
|
|
||||||
f.meta.Opts.Path = opts.Path
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (f *Fetcher) Pause() error {
|
|
||||||
handle := f.currentHandle()
|
|
||||||
if !handle.IsValid() {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
client, err := f.getClient()
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if err := client.PauseTransfer(handle.GetHash()); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
f.handle = handle
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (f *Fetcher) Close() error {
|
|
||||||
if f.waitCancel != nil {
|
|
||||||
f.waitCancel()
|
|
||||||
}
|
|
||||||
handle := f.currentHandle()
|
|
||||||
if !handle.IsValid() {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
client, err := f.getClient()
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
f.handle = handle
|
|
||||||
return client.RemoveTransfer(handle.GetHash(), false)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (f *Fetcher) Meta() *fetcher.FetcherMeta {
|
|
||||||
return f.meta
|
|
||||||
}
|
|
||||||
|
|
||||||
func (f *Fetcher) Stats() *fetcher.Stats {
|
|
||||||
handle := f.currentHandle()
|
|
||||||
if !handle.IsValid() {
|
|
||||||
return &fetcher.Stats{
|
|
||||||
Runtime: &ped2k.StatsRuntime{
|
|
||||||
Peers: make([]*base.PeerStats, 0),
|
|
||||||
},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
status := handle.GetStatus()
|
|
||||||
pieceMap := buildED2KPieceMap(handle.PieceSnapshots())
|
|
||||||
mapReady := pieceMap != nil && status.State != goed2k.LoadingResumeData && status.State != goed2k.Verifying
|
|
||||||
var snapshot any
|
|
||||||
if mapReady {
|
|
||||||
snapshot = &ped2k.StatsSnapshot{
|
|
||||||
Upload: status.Upload,
|
|
||||||
TotalDone: status.TotalDone,
|
|
||||||
TotalReceived: status.TotalReceived,
|
|
||||||
TotalWanted: status.TotalWanted,
|
|
||||||
PieceMap: pieceMap.Clone(),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// Resume loading and verification are transitional. A nil Snapshot retains
|
|
||||||
// the last trusted completion bitset until goed2k publishes a stable map.
|
|
||||||
return &fetcher.Stats{
|
|
||||||
Snapshot: snapshot,
|
|
||||||
Runtime: &ped2k.StatsRuntime{
|
|
||||||
State: string(status.State),
|
|
||||||
Paused: status.Paused,
|
|
||||||
ActivePeers: handle.ActiveConnections(),
|
|
||||||
TotalPeers: status.NumPeers,
|
|
||||||
DownloadRate: status.DownloadRate,
|
|
||||||
UploadRate: status.UploadRate,
|
|
||||||
Peers: buildED2KPeers(handle.GetPeersInfo()),
|
|
||||||
},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func buildED2KPieceMap(snapshots []goed2k.PieceSnapshot) *base.PieceMap {
|
|
||||||
if len(snapshots) == 0 {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
pieceMap := base.NewPieceMap(len(snapshots), goed2k.PieceSize)
|
|
||||||
for _, snapshot := range snapshots {
|
|
||||||
if snapshot.Index < 0 || snapshot.Index >= pieceMap.PieceCount() {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
_ = pieceMap.Update(snapshot.Index, snapshot.State == goed2k.PieceSnapshotFinished)
|
|
||||||
}
|
|
||||||
return pieceMap
|
|
||||||
}
|
|
||||||
|
|
||||||
func buildED2KPeers(peerInfos []goed2k.PeerInfo) []*base.PeerStats {
|
|
||||||
peers := make([]*base.PeerStats, 0, len(peerInfos))
|
|
||||||
for _, peerInfo := range peerInfos {
|
|
||||||
peers = append(peers, &base.PeerStats{
|
|
||||||
Address: peerInfo.Endpoint.String(),
|
|
||||||
Client: ed2kPeerClient(peerInfo),
|
|
||||||
DownloadSpeed: int64(peerInfo.PayloadDownloadSpeed),
|
|
||||||
UploadSpeed: int64(peerInfo.PayloadUploadSpeed),
|
|
||||||
PieceCount: peerInfo.RemotePieces.Count(),
|
|
||||||
Source: peerInfo.SourceString(),
|
|
||||||
Transport: "tcp",
|
|
||||||
})
|
|
||||||
}
|
|
||||||
return peers
|
|
||||||
}
|
|
||||||
|
|
||||||
func ed2kPeerClient(peerInfo goed2k.PeerInfo) string {
|
|
||||||
return strings.TrimSpace(peerInfo.ModName + " " + peerInfo.StrModVersion)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (f *Fetcher) Progress() fetcher.Progress {
|
|
||||||
handle := f.currentHandle()
|
|
||||||
if !handle.IsValid() {
|
|
||||||
return fetcher.Progress{0}
|
|
||||||
}
|
|
||||||
return fetcher.Progress{handle.GetStatus().TotalReceived}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (f *Fetcher) Wait() error {
|
|
||||||
client, err := f.getClient()
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
hash, err := f.hash()
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
handle := f.currentHandle()
|
|
||||||
if handle.IsValid() && handle.IsFinished() {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
progressCh, cancel := client.SubscribeTransferProgress()
|
|
||||||
defer cancel()
|
|
||||||
|
|
||||||
for {
|
|
||||||
select {
|
|
||||||
case <-f.waitCtx.Done():
|
|
||||||
return nil
|
|
||||||
case event, ok := <-progressCh:
|
|
||||||
if !ok {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
for _, transfer := range event.Transfers {
|
|
||||||
if transfer.Hash.Compare(hash) != 0 {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
// Removal can happen during task deletion or client shutdown, both of
|
|
||||||
// which should unblock Wait without treating it as a download failure.
|
|
||||||
if transfer.Removed || transfer.State == goed2k.Finished {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (f *Fetcher) getClient() (*goed2k.Client, error) {
|
|
||||||
if f.manager == nil {
|
|
||||||
f.manager = &FetcherManager{}
|
|
||||||
}
|
|
||||||
return f.manager.initClient(f.config)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (f *Fetcher) currentHandle() goed2k.TransferHandle {
|
|
||||||
if f.handle.IsValid() {
|
|
||||||
return f.handle
|
|
||||||
}
|
|
||||||
if f.manager == nil {
|
|
||||||
return f.handle
|
|
||||||
}
|
|
||||||
client := f.manager.currentClient()
|
|
||||||
if client == nil {
|
|
||||||
return f.handle
|
|
||||||
}
|
|
||||||
hash, err := f.hash()
|
|
||||||
if err != nil {
|
|
||||||
return f.handle
|
|
||||||
}
|
|
||||||
handle := client.FindTransfer(hash)
|
|
||||||
if handle.IsValid() {
|
|
||||||
f.handle = handle
|
|
||||||
}
|
|
||||||
return f.handle
|
|
||||||
}
|
|
||||||
|
|
||||||
func (f *Fetcher) hash() (gprotocol.Hash, error) {
|
|
||||||
if f.meta == nil || f.meta.Req == nil {
|
|
||||||
return gprotocol.Invalid, errors.New("ed2k link is empty")
|
|
||||||
}
|
|
||||||
link, err := parseLink(f.meta.Req.URL)
|
|
||||||
if err != nil {
|
|
||||||
return gprotocol.Invalid, err
|
|
||||||
}
|
|
||||||
return link.Hash, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
type FetcherManager struct {
|
|
||||||
mu sync.Mutex
|
|
||||||
client *goed2k.Client
|
|
||||||
stateStore *clientStateStore
|
|
||||||
}
|
|
||||||
|
|
||||||
func (fm *FetcherManager) SetStateStore(store fetcher.ProtocolStateStore) {
|
|
||||||
fm.mu.Lock()
|
|
||||||
defer fm.mu.Unlock()
|
|
||||||
|
|
||||||
if fm.stateStore == nil {
|
|
||||||
fm.stateStore = &clientStateStore{}
|
|
||||||
}
|
|
||||||
fm.stateStore.store = store
|
|
||||||
if fm.client != nil {
|
|
||||||
fm.client.SetStateStore(fm.stateStore)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (fm *FetcherManager) Name() string {
|
|
||||||
return "ed2k"
|
|
||||||
}
|
|
||||||
|
|
||||||
func (fm *FetcherManager) Filters() []*fetcher.SchemeFilter {
|
|
||||||
return []*fetcher.SchemeFilter{
|
|
||||||
{
|
|
||||||
Type: fetcher.FilterTypeUrl,
|
|
||||||
Pattern: "ED2K",
|
|
||||||
},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (fm *FetcherManager) Build() fetcher.Fetcher {
|
|
||||||
return &Fetcher{manager: fm}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (fm *FetcherManager) ParseName(u string) string {
|
|
||||||
link, err := parseLink(u)
|
|
||||||
if err != nil {
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
return link.StringValue
|
|
||||||
}
|
|
||||||
|
|
||||||
func (fm *FetcherManager) AutoRename() bool {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
|
|
||||||
func (fm *FetcherManager) DefaultConfig() any {
|
|
||||||
return &config{
|
|
||||||
ListenPort: 0,
|
|
||||||
UDPPort: 0,
|
|
||||||
ServerAddr: defaultServerList,
|
|
||||||
ServerMet: defaultServerMet,
|
|
||||||
NodesDat: defaultNodesDat,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (fm *FetcherManager) Store(f fetcher.Fetcher) (any, error) {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (fm *FetcherManager) Restore() (v any, f func(meta *fetcher.FetcherMeta, v any) fetcher.Fetcher) {
|
|
||||||
return nil, func(meta *fetcher.FetcherMeta, v any) fetcher.Fetcher {
|
|
||||||
return &Fetcher{
|
|
||||||
manager: fm,
|
|
||||||
meta: meta,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (fm *FetcherManager) Close() error {
|
|
||||||
fm.mu.Lock()
|
|
||||||
defer fm.mu.Unlock()
|
|
||||||
|
|
||||||
if fm.client != nil {
|
|
||||||
fm.client.Close()
|
|
||||||
fm.client = nil
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func parseLink(raw string) (goed2k.EMuleLink, error) {
|
|
||||||
link, err := goed2k.ParseEMuleLink(raw)
|
|
||||||
if err != nil {
|
|
||||||
return goed2k.EMuleLink{}, err
|
|
||||||
}
|
|
||||||
if link.Type != goed2k.LinkFile {
|
|
||||||
return goed2k.EMuleLink{}, errors.New("unsupported ed2k link type")
|
|
||||||
}
|
|
||||||
return link, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func buildResource(link goed2k.EMuleLink) *base.Resource {
|
|
||||||
return &base.Resource{
|
|
||||||
Size: link.NumberValue,
|
|
||||||
Range: false,
|
|
||||||
Hash: link.Hash.String(),
|
|
||||||
Files: []*base.FileInfo{
|
|
||||||
{
|
|
||||||
Name: link.StringValue,
|
|
||||||
Size: link.NumberValue,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func splitCommaList(value string) []string {
|
|
||||||
if strings.TrimSpace(value) == "" {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
parts := strings.Split(value, ",")
|
|
||||||
out := make([]string, 0, len(parts))
|
|
||||||
for _, part := range parts {
|
|
||||||
part = strings.TrimSpace(part)
|
|
||||||
if part != "" {
|
|
||||||
out = append(out, part)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
func (fm *FetcherManager) getStateStoreLocked() *clientStateStore {
|
|
||||||
if fm.stateStore == nil {
|
|
||||||
fm.stateStore = &clientStateStore{}
|
|
||||||
}
|
|
||||||
return fm.stateStore
|
|
||||||
}
|
|
||||||
|
|
||||||
func (fm *FetcherManager) currentClient() *goed2k.Client {
|
|
||||||
fm.mu.Lock()
|
|
||||||
defer fm.mu.Unlock()
|
|
||||||
return fm.client
|
|
||||||
}
|
|
||||||
|
|
||||||
func (fm *FetcherManager) initClient(cfg *config) (*goed2k.Client, error) {
|
|
||||||
fm.mu.Lock()
|
|
||||||
defer fm.mu.Unlock()
|
|
||||||
|
|
||||||
if fm.client != nil {
|
|
||||||
return fm.client, nil
|
|
||||||
}
|
|
||||||
if cfg == nil {
|
|
||||||
cfg = fm.DefaultConfig().(*config)
|
|
||||||
}
|
|
||||||
|
|
||||||
settings := goed2k.NewSettings()
|
|
||||||
settings.ListenPort = cfg.ListenPort
|
|
||||||
settings.UDPPort = cfg.UDPPort
|
|
||||||
settings.EnableDHT = true
|
|
||||||
settings.EnableUPnP = true
|
|
||||||
settings.ReconnectToServer = true
|
|
||||||
|
|
||||||
client := goed2k.NewClient(settings)
|
|
||||||
client.SetStateStore(fm.getStateStoreLocked())
|
|
||||||
if err := client.LoadState(""); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
if err := client.Start(); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
fm.client = client
|
|
||||||
// Bootstrap is best-effort: downloads can still proceed later even if
|
|
||||||
// server list or DHT initialization fails during startup.
|
|
||||||
bootstrapClient(client, cfg)
|
|
||||||
return fm.client, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func bootstrapClient(client *goed2k.Client, cfg *config) {
|
|
||||||
for _, serverAddr := range splitCommaList(cfg.ServerAddr) {
|
|
||||||
go func(serverAddr string) {
|
|
||||||
_ = client.Connect(serverAddr)
|
|
||||||
}(serverAddr)
|
|
||||||
}
|
|
||||||
for _, source := range splitCommaList(cfg.ServerMet) {
|
|
||||||
go func(source string) {
|
|
||||||
_ = client.ConnectServerMet(source)
|
|
||||||
}(source)
|
|
||||||
}
|
|
||||||
if cfg.NodesDat != "" {
|
|
||||||
go func() {
|
|
||||||
_ = client.LoadDHTNodesDat(cfg.NodesDat)
|
|
||||||
}()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,43 +0,0 @@
|
|||||||
package ed2k
|
|
||||||
|
|
||||||
import (
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/GopeedLab/gopeed/internal/controller"
|
|
||||||
"github.com/GopeedLab/gopeed/pkg/base"
|
|
||||||
)
|
|
||||||
|
|
||||||
const testLink = "ed2k://|file|cn_windows_10_multi-edition_vl_version_1709_updated_sept_2017_x64_dvd_100090774.iso|4630972416|8867C5E54405FF9452225B66EFEE690A|/"
|
|
||||||
|
|
||||||
func TestFetcher_Resolve(t *testing.T) {
|
|
||||||
f := (&FetcherManager{}).Build()
|
|
||||||
f.Setup(controller.NewController())
|
|
||||||
|
|
||||||
f.Meta().Opts = &base.Options{Path: t.TempDir()}
|
|
||||||
err := f.Resolve(&base.Request{URL: testLink})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Resolve() error = %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
meta := f.Meta()
|
|
||||||
if meta.Res == nil {
|
|
||||||
t.Fatal("Resolve() resource is nil")
|
|
||||||
}
|
|
||||||
if got, want := meta.Res.Hash, "8867C5E54405FF9452225B66EFEE690A"; got != want {
|
|
||||||
t.Fatalf("Resolve() hash = %s, want %s", got, want)
|
|
||||||
}
|
|
||||||
if got, want := meta.Res.Size, int64(4630972416); got != want {
|
|
||||||
t.Fatalf("Resolve() size = %d, want %d", got, want)
|
|
||||||
}
|
|
||||||
if got, want := meta.Res.Files[0].Name, "cn_windows_10_multi-edition_vl_version_1709_updated_sept_2017_x64_dvd_100090774.iso"; got != want {
|
|
||||||
t.Fatalf("Resolve() name = %q, want %q", got, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestFetcherManager_ParseName(t *testing.T) {
|
|
||||||
got := (&FetcherManager{}).ParseName(testLink)
|
|
||||||
want := "cn_windows_10_multi-edition_vl_version_1709_updated_sept_2017_x64_dvd_100090774.iso"
|
|
||||||
if got != want {
|
|
||||||
t.Fatalf("ParseName() = %q, want %q", got, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,72 +0,0 @@
|
|||||||
package ed2k
|
|
||||||
|
|
||||||
import (
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
ped2k "github.com/GopeedLab/gopeed/pkg/protocol/ed2k"
|
|
||||||
"github.com/monkeyWie/goed2k"
|
|
||||||
"github.com/monkeyWie/goed2k/protocol"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestStatsSeparatesSnapshotAndRuntimeWithoutHandle(t *testing.T) {
|
|
||||||
frame := (&Fetcher{}).Stats()
|
|
||||||
if frame.Snapshot != nil {
|
|
||||||
t.Fatalf("snapshot = %T, want nil without a valid handle", frame.Snapshot)
|
|
||||||
}
|
|
||||||
runtime, ok := frame.Runtime.(*ped2k.StatsRuntime)
|
|
||||||
if !ok {
|
|
||||||
t.Fatalf("runtime type = %T", frame.Runtime)
|
|
||||||
}
|
|
||||||
if runtime.Peers == nil || len(runtime.Peers) != 0 {
|
|
||||||
t.Fatalf("unexpected runtime peers: %+v", runtime.Peers)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestBuildED2KPieceMapPreservesIndexes(t *testing.T) {
|
|
||||||
snapshots := []goed2k.PieceSnapshot{
|
|
||||||
{Index: 0, State: goed2k.PieceSnapshotFinished},
|
|
||||||
{Index: 1, State: goed2k.PieceSnapshotMissing},
|
|
||||||
{Index: 2, State: goed2k.PieceSnapshotDownloading},
|
|
||||||
{Index: 3, State: goed2k.PieceSnapshotFinished},
|
|
||||||
}
|
|
||||||
pieceMap := buildED2KPieceMap(snapshots)
|
|
||||||
if pieceMap == nil {
|
|
||||||
t.Fatal("buildED2KPieceMap returned nil")
|
|
||||||
}
|
|
||||||
want := []bool{true, false, false, true}
|
|
||||||
for index, expected := range want {
|
|
||||||
got, err := pieceMap.Get(index)
|
|
||||||
if err != nil || got != expected {
|
|
||||||
t.Fatalf("piece %d = %v, %v; want %v", index, got, err, expected)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if pieceMap.CompletedPieces() != 2 {
|
|
||||||
t.Fatalf("CompletedPieces() = %d, want 2", pieceMap.CompletedPieces())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestBuildED2KPeers(t *testing.T) {
|
|
||||||
remotePieces := protocol.NewBitField(4)
|
|
||||||
remotePieces.SetBit(0)
|
|
||||||
remotePieces.SetBit(2)
|
|
||||||
endpoint, err := protocol.EndpointFromString("127.0.0.1", 4662)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("EndpointFromString: %v", err)
|
|
||||||
}
|
|
||||||
peers := buildED2KPeers([]goed2k.PeerInfo{{
|
|
||||||
PayloadDownloadSpeed: 2048,
|
|
||||||
PayloadUploadSpeed: 1024,
|
|
||||||
RemotePieces: remotePieces,
|
|
||||||
Endpoint: endpoint,
|
|
||||||
ModName: "aMule",
|
|
||||||
StrModVersion: "2.3.3",
|
|
||||||
SourceFlag: int(goed2k.PeerServer | goed2k.PeerDHT),
|
|
||||||
}})
|
|
||||||
if len(peers) != 1 {
|
|
||||||
t.Fatalf("len(peers) = %d, want 1", len(peers))
|
|
||||||
}
|
|
||||||
peer := peers[0]
|
|
||||||
if peer.Client != "aMule 2.3.3" || peer.PieceCount != 2 || peer.Source != "server|kad" || peer.Transport != "tcp" {
|
|
||||||
t.Fatalf("unexpected peer: %+v", peer)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,404 +0,0 @@
|
|||||||
package ftp
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"crypto/tls"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
"net"
|
|
||||||
"net/http"
|
|
||||||
"net/textproto"
|
|
||||||
"net/url"
|
|
||||||
"strings"
|
|
||||||
"sync"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/jlaffaye/ftp"
|
|
||||||
"golang.org/x/net/proxy"
|
|
||||||
|
|
||||||
pftp "github.com/GopeedLab/gopeed/pkg/protocol/ftp"
|
|
||||||
)
|
|
||||||
|
|
||||||
const (
|
|
||||||
connectTimeout = 15 * time.Second
|
|
||||||
readTimeout = 15 * time.Second
|
|
||||||
quitTimeout = 2 * time.Second
|
|
||||||
)
|
|
||||||
|
|
||||||
// target is a parsed FTP URL. It holds the password, so it must never be
|
|
||||||
// formatted into an error or a log line.
|
|
||||||
type target struct {
|
|
||||||
host string // host name or IP, the TLS server name
|
|
||||||
addr string // host:port
|
|
||||||
user string
|
|
||||||
pass string
|
|
||||||
path string // path relative to the login directory, "" for the directory itself
|
|
||||||
tlsMode string
|
|
||||||
}
|
|
||||||
|
|
||||||
// parseTarget reads an ftp, ftps or ftpes URL. The path follows RFC 1738: it
|
|
||||||
// is relative to the login directory, and an encoded leading slash
|
|
||||||
// (ftp://host/%2Fpub) makes it absolute.
|
|
||||||
func parseTarget(rawURL, optTLS string) (*target, error) {
|
|
||||||
u, err := url.Parse(rawURL)
|
|
||||||
if err != nil {
|
|
||||||
// url.Error repeats the whole URL, password included.
|
|
||||||
return nil, errors.New("ftp: invalid URL")
|
|
||||||
}
|
|
||||||
t := &target{}
|
|
||||||
switch strings.ToLower(u.Scheme) {
|
|
||||||
case "ftps":
|
|
||||||
t.tlsMode = pftp.TLSImplicit
|
|
||||||
case "ftpes":
|
|
||||||
t.tlsMode = pftp.TLSExplicit
|
|
||||||
case "ftp":
|
|
||||||
switch strings.ToLower(optTLS) {
|
|
||||||
case "", pftp.TLSNone:
|
|
||||||
t.tlsMode = pftp.TLSNone
|
|
||||||
case pftp.TLSExplicit:
|
|
||||||
t.tlsMode = pftp.TLSExplicit
|
|
||||||
case pftp.TLSImplicit:
|
|
||||||
t.tlsMode = pftp.TLSImplicit
|
|
||||||
default:
|
|
||||||
return nil, fmt.Errorf("ftp: unknown tls mode %q", optTLS)
|
|
||||||
}
|
|
||||||
default:
|
|
||||||
return nil, fmt.Errorf("ftp: unsupported scheme %q", u.Scheme)
|
|
||||||
}
|
|
||||||
t.host = u.Hostname()
|
|
||||||
if t.host == "" {
|
|
||||||
return nil, errors.New("ftp: URL has no host")
|
|
||||||
}
|
|
||||||
port := u.Port()
|
|
||||||
if port == "" {
|
|
||||||
port = "21"
|
|
||||||
if t.tlsMode == pftp.TLSImplicit {
|
|
||||||
port = "990"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
t.addr = net.JoinHostPort(t.host, port)
|
|
||||||
if u.User != nil {
|
|
||||||
// Username and Password are already percent-decoded.
|
|
||||||
t.user = u.User.Username()
|
|
||||||
t.pass, _ = u.User.Password()
|
|
||||||
}
|
|
||||||
if t.user == "" {
|
|
||||||
t.user, t.pass = "anonymous", "anonymous@"
|
|
||||||
}
|
|
||||||
t.path = strings.TrimPrefix(u.Path, "/")
|
|
||||||
return t, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
type dialContextFunc func(ctx context.Context, network, addr string) (net.Conn, error)
|
|
||||||
|
|
||||||
// session is one logged-in control connection. It owns the network
|
|
||||||
// connections it opens, so abort can unblock any read or write in progress.
|
|
||||||
type session struct {
|
|
||||||
*ftp.ServerConn
|
|
||||||
|
|
||||||
ctrl net.Conn // control connection as dialed, under any TLS layer
|
|
||||||
dataHost string // where data connections go
|
|
||||||
|
|
||||||
mu sync.Mutex
|
|
||||||
data net.Conn // current data connection as dialed
|
|
||||||
closed bool
|
|
||||||
stop func() bool
|
|
||||||
}
|
|
||||||
|
|
||||||
// deadline bounds the next control-connection exchange. The control
|
|
||||||
// connection is idle during a transfer, so every exchange sets a new one.
|
|
||||||
func (s *session) deadline() {
|
|
||||||
_ = s.ctrl.SetDeadline(time.Now().Add(readTimeout))
|
|
||||||
}
|
|
||||||
|
|
||||||
// clearDeadline lets the control connection stay idle through a transfer.
|
|
||||||
func (s *session) clearDeadline() {
|
|
||||||
_ = s.ctrl.SetDeadline(time.Time{})
|
|
||||||
}
|
|
||||||
|
|
||||||
// abort closes every connection at once. Pending calls fail immediately.
|
|
||||||
func (s *session) abort() {
|
|
||||||
s.mu.Lock()
|
|
||||||
defer s.mu.Unlock()
|
|
||||||
s.closed = true
|
|
||||||
_ = s.ctrl.Close()
|
|
||||||
if s.data != nil {
|
|
||||||
_ = s.data.Close()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// quit logs out and waits, briefly, for the server to hang up. Servers that
|
|
||||||
// cap logins count a session until they have seen it end, so the next login
|
|
||||||
// would otherwise be refused.
|
|
||||||
func (s *session) quit() {
|
|
||||||
if s.stop != nil {
|
|
||||||
s.stop()
|
|
||||||
}
|
|
||||||
_ = s.ctrl.SetDeadline(time.Now().Add(quitTimeout))
|
|
||||||
if s.ServerConn != nil {
|
|
||||||
_ = s.ServerConn.Quit()
|
|
||||||
}
|
|
||||||
s.abort()
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *session) setData(c net.Conn) error {
|
|
||||||
s.mu.Lock()
|
|
||||||
defer s.mu.Unlock()
|
|
||||||
if s.closed {
|
|
||||||
_ = c.Close()
|
|
||||||
return net.ErrClosed
|
|
||||||
}
|
|
||||||
s.data = c
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// ctrlConn is the control connection handed to the ftp library.
|
|
||||||
type ctrlConn struct {
|
|
||||||
net.Conn
|
|
||||||
}
|
|
||||||
|
|
||||||
// RemoteAddr must be a *net.TCPAddr: the library asserts it. A connection
|
|
||||||
// through a proxy may report something else, and its address is not used
|
|
||||||
// for data connections anyway (see dialData).
|
|
||||||
func (c ctrlConn) RemoteAddr() net.Addr {
|
|
||||||
if a, ok := c.Conn.RemoteAddr().(*net.TCPAddr); ok {
|
|
||||||
return a
|
|
||||||
}
|
|
||||||
return &net.TCPAddr{}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Close is called by the library's Quit right after it sends QUIT. Reading
|
|
||||||
// until the server hangs up (bounded by the deadline quit set) tells us the
|
|
||||||
// server has ended the session.
|
|
||||||
func (c ctrlConn) Close() error {
|
|
||||||
_, _ = io.Copy(io.Discard, c.Conn)
|
|
||||||
return c.Conn.Close()
|
|
||||||
}
|
|
||||||
|
|
||||||
// dial opens a control connection, upgrades it to TLS when asked, and logs in.
|
|
||||||
//
|
|
||||||
// The library is given a dial function for every connection. That is how
|
|
||||||
// data connections go through the same proxy as the control connection, but
|
|
||||||
// it also means the library does no TLS on them: with a dial function it
|
|
||||||
// neither wraps an implicit-TLS control connection nor any data connection.
|
|
||||||
// We do both here. Explicit TLS (AUTH TLS) is still done by the library, and
|
|
||||||
// so are PBSZ and PROT P after login. For the same reason the library's
|
|
||||||
// DialWithTimeout and DialWithContext would have no effect: connectTimeout
|
|
||||||
// and ctx are applied to our own dials instead.
|
|
||||||
func (f *Fetcher) dial(ctx context.Context, t *target) (*session, error) {
|
|
||||||
netDial, proxied, err := f.netDialer(t)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
var tlsConf *tls.Config
|
|
||||||
if t.tlsMode != pftp.TLSNone {
|
|
||||||
tlsConf = f.tlsConfig(t.host)
|
|
||||||
}
|
|
||||||
|
|
||||||
dialCtx, cancel := context.WithTimeout(ctx, connectTimeout)
|
|
||||||
defer cancel()
|
|
||||||
raw, err := netDial(dialCtx, "tcp", t.addr)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
ctrl := raw
|
|
||||||
if t.tlsMode == pftp.TLSImplicit {
|
|
||||||
tc := tls.Client(raw, tlsConf)
|
|
||||||
if err := tc.HandshakeContext(dialCtx); err != nil {
|
|
||||||
_ = raw.Close()
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
ctrl = tc
|
|
||||||
}
|
|
||||||
|
|
||||||
s := &session{ctrl: raw, dataHost: t.host}
|
|
||||||
if a, ok := raw.RemoteAddr().(*net.TCPAddr); ok && !proxied {
|
|
||||||
// Same IP as the control connection, as the library itself does,
|
|
||||||
// so a name with several addresses cannot send us elsewhere.
|
|
||||||
s.dataHost = a.IP.String()
|
|
||||||
}
|
|
||||||
s.stop = context.AfterFunc(ctx, s.abort)
|
|
||||||
s.deadline()
|
|
||||||
|
|
||||||
handedOut := false
|
|
||||||
opts := []ftp.DialOption{
|
|
||||||
ftp.DialWithShutTimeout(readTimeout),
|
|
||||||
ftp.DialWithDialFunc(func(network, addr string) (net.Conn, error) {
|
|
||||||
if !handedOut {
|
|
||||||
// Dial asks for the control connection first, once.
|
|
||||||
handedOut = true
|
|
||||||
return ctrlConn{ctrl}, nil
|
|
||||||
}
|
|
||||||
return s.dialData(ctx, netDial, tlsConf, addr)
|
|
||||||
}),
|
|
||||||
}
|
|
||||||
switch t.tlsMode {
|
|
||||||
case pftp.TLSImplicit:
|
|
||||||
opts = append(opts, ftp.DialWithTLS(tlsConf))
|
|
||||||
case pftp.TLSExplicit:
|
|
||||||
opts = append(opts, ftp.DialWithExplicitTLS(tlsConf))
|
|
||||||
}
|
|
||||||
c, err := ftp.Dial(t.addr, opts...)
|
|
||||||
if err != nil {
|
|
||||||
s.stop()
|
|
||||||
s.abort()
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
s.ServerConn = c
|
|
||||||
s.deadline()
|
|
||||||
if err := c.Login(t.user, t.pass); err != nil {
|
|
||||||
s.stop()
|
|
||||||
s.abort()
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return s, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// dialData opens a passive data connection. The library takes the host from
|
|
||||||
// the control connection's peer, which through a proxy is the proxy, so only
|
|
||||||
// the port is used here.
|
|
||||||
func (s *session) dialData(ctx context.Context, netDial dialContextFunc, tlsConf *tls.Config, addr string) (net.Conn, error) {
|
|
||||||
_, port, err := net.SplitHostPort(addr)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
dialCtx, cancel := context.WithTimeout(ctx, connectTimeout)
|
|
||||||
defer cancel()
|
|
||||||
raw, err := netDial(dialCtx, "tcp", net.JoinHostPort(s.dataHost, port))
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
if err := s.setData(raw); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
if tlsConf != nil {
|
|
||||||
// The handshake runs on the first read, as in the library, because
|
|
||||||
// some servers only start it once the transfer command is answered.
|
|
||||||
return tls.Client(raw, tlsConf), nil
|
|
||||||
}
|
|
||||||
return raw, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// tlsConfig returns the TLS settings for one control connection and its data
|
|
||||||
// connections. Certificates are verified unless the request sets
|
|
||||||
// skipVerifyCert, as for HTTP.
|
|
||||||
//
|
|
||||||
// Each call gets its own one-entry session cache, so a data connection
|
|
||||||
// resumes the TLS session of its own control connection. vsftpd
|
|
||||||
// (require_ssl_reuse) only checks that some session was resumed, but ProFTPD's
|
|
||||||
// mod_tls also checks that it is the control connection's; a cache shared by
|
|
||||||
// several workers would hand a data connection another worker's session.
|
|
||||||
// A cache in the manager's base config is kept as given.
|
|
||||||
func (f *Fetcher) tlsConfig(host string) *tls.Config {
|
|
||||||
cfg := &tls.Config{}
|
|
||||||
if f.manager != nil && f.manager.baseTLS != nil {
|
|
||||||
cfg = f.manager.baseTLS.Clone()
|
|
||||||
}
|
|
||||||
cfg.ServerName = host
|
|
||||||
if f.meta.Req != nil && f.meta.Req.SkipVerifyCert {
|
|
||||||
cfg.InsecureSkipVerify = true
|
|
||||||
}
|
|
||||||
if cfg.ClientSessionCache == nil {
|
|
||||||
cfg.ClientSessionCache = tls.NewLRUClientSessionCache(1)
|
|
||||||
}
|
|
||||||
return cfg
|
|
||||||
}
|
|
||||||
|
|
||||||
// netDialer picks how to reach the server, from the same proxy settings the
|
|
||||||
// HTTP protocol reads (the task's proxy, otherwise the global one).
|
|
||||||
func (f *Fetcher) netDialer(t *target) (dialContextFunc, bool, error) {
|
|
||||||
direct := (&net.Dialer{Timeout: connectTimeout}).DialContext
|
|
||||||
pu, err := f.proxyURL(t)
|
|
||||||
if err != nil {
|
|
||||||
return nil, false, err
|
|
||||||
}
|
|
||||||
if pu == nil {
|
|
||||||
return direct, false, nil
|
|
||||||
}
|
|
||||||
switch strings.ToLower(pu.Scheme) {
|
|
||||||
case "socks5", "socks5h":
|
|
||||||
d, err := proxy.FromURL(pu, &net.Dialer{Timeout: connectTimeout})
|
|
||||||
if err != nil {
|
|
||||||
return nil, false, fmt.Errorf("ftp: socks5 proxy: %w", err)
|
|
||||||
}
|
|
||||||
if cd, ok := d.(proxy.ContextDialer); ok {
|
|
||||||
return cd.DialContext, true, nil
|
|
||||||
}
|
|
||||||
return func(_ context.Context, network, addr string) (net.Conn, error) {
|
|
||||||
return d.Dial(network, addr)
|
|
||||||
}, true, nil
|
|
||||||
default:
|
|
||||||
// An HTTP proxy is not used. Every passive data connection goes to
|
|
||||||
// a port the server picks, and HTTP proxies usually allow CONNECT to
|
|
||||||
// port 443 only, so the transfer would fail after the login worked.
|
|
||||||
f.warnOnce.Do(func() {
|
|
||||||
if l := f.logger(); l != nil {
|
|
||||||
l.Warn().Msgf("ftp: the %s proxy at %s cannot carry FTP data connections (HTTP proxy), connecting directly; use a SOCKS5 proxy for FTP", strings.ToLower(pu.Scheme), pu.Host)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
return direct, false, nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (f *Fetcher) proxyURL(t *target) (*url.URL, error) {
|
|
||||||
if f.ctl == nil || f.ctl.GetProxy == nil {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
handler := f.ctl.GetProxy(f.meta.Req.Proxy)
|
|
||||||
if handler == nil {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
// The proxy chooser reads only the URL; the credentials stay out.
|
|
||||||
req := &http.Request{URL: &url.URL{Scheme: "ftp", Host: t.addr, Path: "/" + t.path}, Header: http.Header{}}
|
|
||||||
pu, err := handler(req)
|
|
||||||
if err != nil {
|
|
||||||
return nil, errors.New("ftp: cannot read the proxy settings")
|
|
||||||
}
|
|
||||||
return pu, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// isLoginRefusal tells a server that is full apart from bad credentials.
|
|
||||||
// 421 always means "not now". 530 means a refused login; it is only a
|
|
||||||
// capacity limit when these credentials have worked before. The library
|
|
||||||
// drops the code when USER itself is refused, so a "too many" message counts
|
|
||||||
// as well.
|
|
||||||
func isLoginRefusal(err error, credentialsWorked bool) bool {
|
|
||||||
var te *textproto.Error
|
|
||||||
if errors.As(err, &te) {
|
|
||||||
switch te.Code {
|
|
||||||
case ftp.StatusNotAvailable:
|
|
||||||
return true
|
|
||||||
case ftp.StatusNotLoggedIn:
|
|
||||||
return credentialsWorked || strings.Contains(strings.ToLower(te.Msg), "too many")
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
return strings.Contains(strings.ToLower(err.Error()), "too many")
|
|
||||||
}
|
|
||||||
|
|
||||||
// isPermanent reports errors a retry cannot fix: a 5xx reply and a
|
|
||||||
// certificate that does not verify.
|
|
||||||
func isPermanent(err error) bool {
|
|
||||||
var te *textproto.Error
|
|
||||||
if errors.As(err, &te) {
|
|
||||||
return te.Code >= 500
|
|
||||||
}
|
|
||||||
var verifyErr *tls.CertificateVerificationError
|
|
||||||
if errors.As(err, &verifyErr) {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
var recordErr tls.RecordHeaderError
|
|
||||||
if errors.As(err, &recordErr) {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
return errors.Is(err, errSizeChanged)
|
|
||||||
}
|
|
||||||
|
|
||||||
func replyCode(err error) int {
|
|
||||||
var te *textproto.Error
|
|
||||||
if errors.As(err, &te) {
|
|
||||||
return te.Code
|
|
||||||
}
|
|
||||||
return 0
|
|
||||||
}
|
|
||||||
@@ -1,146 +0,0 @@
|
|||||||
package ftp_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"strings"
|
|
||||||
"sync"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/rs/zerolog"
|
|
||||||
|
|
||||||
"github.com/GopeedLab/gopeed/internal/logger"
|
|
||||||
iftp "github.com/GopeedLab/gopeed/internal/protocol/ftp"
|
|
||||||
"github.com/GopeedLab/gopeed/pkg/base"
|
|
||||||
"github.com/GopeedLab/gopeed/pkg/download"
|
|
||||||
pftp "github.com/GopeedLab/gopeed/pkg/protocol/ftp"
|
|
||||||
)
|
|
||||||
|
|
||||||
// syncBuffer is a log sink that the downloader's goroutines can share.
|
|
||||||
type syncBuffer struct {
|
|
||||||
mu sync.Mutex
|
|
||||||
buf bytes.Buffer
|
|
||||||
}
|
|
||||||
|
|
||||||
func (b *syncBuffer) Write(p []byte) (int, error) {
|
|
||||||
b.mu.Lock()
|
|
||||||
defer b.mu.Unlock()
|
|
||||||
return b.buf.Write(p)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (b *syncBuffer) String() string {
|
|
||||||
b.mu.Lock()
|
|
||||||
defer b.mu.Unlock()
|
|
||||||
return b.buf.String()
|
|
||||||
}
|
|
||||||
|
|
||||||
// taskEvents records the first done or error event of every task, so the
|
|
||||||
// test never reads a task's status while the downloader writes it.
|
|
||||||
type taskEvents struct {
|
|
||||||
mu sync.Mutex
|
|
||||||
final map[string]download.EventKey
|
|
||||||
}
|
|
||||||
|
|
||||||
func (e *taskEvents) listen(ev *download.Event) {
|
|
||||||
if ev.Key != download.EventKeyDone && ev.Key != download.EventKeyError {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
e.mu.Lock()
|
|
||||||
defer e.mu.Unlock()
|
|
||||||
if _, seen := e.final[ev.Task.ID]; !seen {
|
|
||||||
e.final[ev.Task.ID] = ev.Key
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (e *taskEvents) wait(t *testing.T, id string) download.EventKey {
|
|
||||||
t.Helper()
|
|
||||||
deadline := time.Now().Add(60 * time.Second)
|
|
||||||
for time.Now().Before(deadline) {
|
|
||||||
e.mu.Lock()
|
|
||||||
key, ok := e.final[id]
|
|
||||||
e.mu.Unlock()
|
|
||||||
if ok {
|
|
||||||
return key
|
|
||||||
}
|
|
||||||
time.Sleep(20 * time.Millisecond)
|
|
||||||
}
|
|
||||||
t.Fatalf("task %s did not finish", id)
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestPasswordNeverLogged runs FTP tasks through the downloader with its core
|
|
||||||
// logger captured: a success, a missing file and a wrong password, all behind
|
|
||||||
// an HTTP proxy that FTP must skip with a warning.
|
|
||||||
func TestPasswordNeverLogged(t *testing.T) {
|
|
||||||
srv := iftp.StartFTPServer(t, iftp.ServerOptions{User: "u", Pass: "p@ss"})
|
|
||||||
want := srv.WriteFile(t, "secret.bin", iftp.RandomBytes(300000, 21))
|
|
||||||
|
|
||||||
d := download.NewDownloader(&download.DownloaderConfig{Storage: download.NewMemStorage(), StorageDir: t.TempDir()})
|
|
||||||
logs := &syncBuffer{}
|
|
||||||
d.Logger = &logger.Logger{Logger: zerolog.New(logs)}
|
|
||||||
events := &taskEvents{final: map[string]download.EventKey{}}
|
|
||||||
d.Listener(events.listen)
|
|
||||||
if err := d.Setup(); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
// Close, not Clear: Clear resets the extension list without a lock while
|
|
||||||
// the last task's done hooks may still read it.
|
|
||||||
defer d.Close()
|
|
||||||
cfg, err := d.GetConfig()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
cfg.Proxy = &base.DownloaderProxyConfig{Enable: true, Scheme: "http", Host: "127.0.0.1:1", Usr: "proxyuser", Pwd: "proxysecret"}
|
|
||||||
if err := d.PutConfig(cfg); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
dir := t.TempDir()
|
|
||||||
create := func(rawURL string) string {
|
|
||||||
id, err := d.CreateDirect(&base.Request{URL: rawURL}, &base.Options{Path: dir, Extra: &pftp.OptsExtra{Connections: 2}})
|
|
||||||
if err != nil {
|
|
||||||
if strings.Contains(err.Error(), "p@ss") || strings.Contains(err.Error(), "p%40ss") {
|
|
||||||
t.Fatalf("the password leaked into an error: %v", err)
|
|
||||||
}
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
return id
|
|
||||||
}
|
|
||||||
root := "ftp://u:p%40ss@" + srv.Addr + "/"
|
|
||||||
|
|
||||||
okID := create(root + "secret.bin")
|
|
||||||
if key := events.wait(t, okID); key != download.EventKeyDone {
|
|
||||||
t.Fatalf("download ended with %q, logs:\n%s", key, logs.String())
|
|
||||||
}
|
|
||||||
ok := d.GetTask(okID)
|
|
||||||
if ok.Protocol != "ftp" {
|
|
||||||
t.Fatalf("protocol %q, want ftp", ok.Protocol)
|
|
||||||
}
|
|
||||||
if got := iftp.FileSHA256(t, ok.Meta.SingleFilepath()); got != want {
|
|
||||||
t.Fatalf("sha256 %s, want %s", got, want)
|
|
||||||
}
|
|
||||||
missingID := create(root + "missing.bin")
|
|
||||||
if key := events.wait(t, missingID); key != download.EventKeyError {
|
|
||||||
t.Fatalf("a missing file must fail, got %q", key)
|
|
||||||
}
|
|
||||||
wrongID := create("ftp://u:wr0ng%40pw@" + srv.Addr + "/secret.bin")
|
|
||||||
if key := events.wait(t, wrongID); key != download.EventKeyError {
|
|
||||||
t.Fatalf("a wrong password must fail, got %q", key)
|
|
||||||
}
|
|
||||||
missing, wrong := d.GetTask(missingID), d.GetTask(wrongID)
|
|
||||||
for _, task := range []*download.Task{ok, missing, wrong} {
|
|
||||||
if name := task.Name(); strings.Contains(name, "p@ss") || strings.Contains(name, "p%40ss") || strings.Contains(name, "wr0ng") {
|
|
||||||
t.Fatalf("task name %q shows the password", name)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
out := logs.String()
|
|
||||||
if !strings.Contains(out, "task id") || !strings.Contains(out, "HTTP proxy") {
|
|
||||||
t.Fatalf("the captured log misses the expected lines, so it proves nothing:\n%s", out)
|
|
||||||
}
|
|
||||||
for _, secret := range []string{"p@ss", "p%40ss", "wr0ng", "proxysecret"} {
|
|
||||||
if strings.Contains(out, secret) {
|
|
||||||
t.Fatalf("%q appears in the log:\n%s", secret, out)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,810 +0,0 @@
|
|||||||
// Package ftp downloads files and directory trees over FTP, FTPS (implicit
|
|
||||||
// TLS) and FTPES (explicit TLS, AUTH TLS).
|
|
||||||
//
|
|
||||||
// A file is split into segments of at least 1 MiB. Each segment is fetched on
|
|
||||||
// its own login with REST and RETR, and its data connection is closed once
|
|
||||||
// the segment's last byte has arrived. A server that refuses a login lowers
|
|
||||||
// the task's connection ceiling to the logins that worked, and a server that
|
|
||||||
// refuses REST gets one sequential stream per file.
|
|
||||||
package ftp
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
"os"
|
|
||||||
"path"
|
|
||||||
"sync"
|
|
||||||
"sync/atomic"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/jlaffaye/ftp"
|
|
||||||
"github.com/rs/zerolog"
|
|
||||||
|
|
||||||
"github.com/GopeedLab/gopeed/internal/controller"
|
|
||||||
"github.com/GopeedLab/gopeed/internal/fetcher"
|
|
||||||
"github.com/GopeedLab/gopeed/pkg/base"
|
|
||||||
pftp "github.com/GopeedLab/gopeed/pkg/protocol/ftp"
|
|
||||||
)
|
|
||||||
|
|
||||||
const (
|
|
||||||
// Many servers cap logins per user, so FTP starts lower than HTTP.
|
|
||||||
defaultConnections = 4
|
|
||||||
maxConnections = 256
|
|
||||||
minSegmentSize = 1 << 20
|
|
||||||
maxRetries = 5
|
|
||||||
copyBufferSize = 256 << 10
|
|
||||||
)
|
|
||||||
|
|
||||||
// retryDelay is the first wait before a retry. Later retries wait longer.
|
|
||||||
var retryDelay = 500 * time.Millisecond
|
|
||||||
|
|
||||||
var (
|
|
||||||
errSizeChanged = errors.New("ftp: the remote file changed size since the task was resolved")
|
|
||||||
errShortTransfer = errors.New("ftp: the transfer ended before the segment was complete")
|
|
||||||
errRestUnsupported = errors.New("ftp: the server does not support REST")
|
|
||||||
errDiskWrite = errors.New("ftp: cannot write the downloaded data")
|
|
||||||
)
|
|
||||||
|
|
||||||
type config struct {
|
|
||||||
Connections int `json:"connections"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// segment is a byte range of one file. End is inclusive, so an empty file
|
|
||||||
// has End == Begin-1 and nothing to fetch.
|
|
||||||
type segment struct {
|
|
||||||
File int `json:",omitempty"` // index in Res.Files
|
|
||||||
Begin int64
|
|
||||||
End int64
|
|
||||||
Downloaded int64
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *segment) remain() int64 {
|
|
||||||
return s.End - s.Begin + 1 - s.Downloaded
|
|
||||||
}
|
|
||||||
|
|
||||||
// lane is one worker that logged in, shown as one connection in the stats.
|
|
||||||
type lane struct {
|
|
||||||
downloaded int64
|
|
||||||
retryTimes int
|
|
||||||
failed bool
|
|
||||||
completed bool
|
|
||||||
}
|
|
||||||
|
|
||||||
// loginError marks a failure to connect or log in, as opposed to a failure
|
|
||||||
// during a transfer.
|
|
||||||
type loginError struct{ err error }
|
|
||||||
|
|
||||||
func (e *loginError) Error() string { return e.err.Error() }
|
|
||||||
func (e *loginError) Unwrap() error { return e.err }
|
|
||||||
|
|
||||||
type Fetcher struct {
|
|
||||||
manager *FetcherManager
|
|
||||||
ctl *controller.Controller
|
|
||||||
config *config
|
|
||||||
meta *fetcher.FetcherMeta
|
|
||||||
doneCh chan error
|
|
||||||
|
|
||||||
warnOnce sync.Once
|
|
||||||
// loginOK is set once these credentials have logged in, after which a
|
|
||||||
// 530 on another login means the server is full.
|
|
||||||
loginOK atomic.Bool
|
|
||||||
|
|
||||||
// mu guards the download state below.
|
|
||||||
mu sync.Mutex
|
|
||||||
segments []*segment
|
|
||||||
maxConns int
|
|
||||||
noRest bool
|
|
||||||
lanes []*lane
|
|
||||||
done bool
|
|
||||||
|
|
||||||
// runMu serializes Start and Pause.
|
|
||||||
runMu sync.Mutex
|
|
||||||
cancel context.CancelFunc
|
|
||||||
runDone chan struct{}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (f *Fetcher) Setup(ctl *controller.Controller) {
|
|
||||||
f.ctl = ctl
|
|
||||||
f.doneCh = make(chan error, 1)
|
|
||||||
if f.meta == nil {
|
|
||||||
f.meta = &fetcher.FetcherMeta{}
|
|
||||||
}
|
|
||||||
if ctl != nil && ctl.GetConfig != nil {
|
|
||||||
ctl.GetConfig(&f.config)
|
|
||||||
}
|
|
||||||
if f.config == nil {
|
|
||||||
f.config = &config{Connections: defaultConnections}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (f *Fetcher) logger() *zerolog.Logger {
|
|
||||||
if f.ctl == nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
return f.ctl.Logger
|
|
||||||
}
|
|
||||||
|
|
||||||
func (f *Fetcher) warn(format string, args ...any) {
|
|
||||||
if l := f.logger(); l != nil {
|
|
||||||
l.Warn().Msgf(format, args...)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (f *Fetcher) initOptions() error {
|
|
||||||
if f.meta.Opts == nil {
|
|
||||||
f.meta.Opts = &base.Options{}
|
|
||||||
}
|
|
||||||
opts := f.meta.Opts
|
|
||||||
if err := base.ParseOptExtra[pftp.OptsExtra](opts); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if opts.Extra == nil {
|
|
||||||
opts.Extra = &pftp.OptsExtra{}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (f *Fetcher) extra() *pftp.OptsExtra {
|
|
||||||
if f.meta.Opts != nil {
|
|
||||||
if e, ok := f.meta.Opts.Extra.(*pftp.OptsExtra); ok && e != nil {
|
|
||||||
return e
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return &pftp.OptsExtra{}
|
|
||||||
}
|
|
||||||
|
|
||||||
// tlsMode is the request's "tls" field, empty when the request has none.
|
|
||||||
func (f *Fetcher) tlsMode() string {
|
|
||||||
if f.meta.Req != nil {
|
|
||||||
if e, ok := f.meta.Req.Extra.(*pftp.ReqExtra); ok && e != nil {
|
|
||||||
return e.TLS
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
|
|
||||||
// connections is the number of parallel logins to use. The caller holds mu.
|
|
||||||
func (f *Fetcher) connections() int {
|
|
||||||
if f.noRest {
|
|
||||||
return 1
|
|
||||||
}
|
|
||||||
n := f.extra().Connections
|
|
||||||
if n <= 0 && f.config != nil {
|
|
||||||
n = f.config.Connections
|
|
||||||
}
|
|
||||||
if n <= 0 {
|
|
||||||
n = defaultConnections
|
|
||||||
}
|
|
||||||
n = min(n, maxConnections)
|
|
||||||
if f.maxConns > 0 {
|
|
||||||
n = min(n, f.maxConns)
|
|
||||||
}
|
|
||||||
return n
|
|
||||||
}
|
|
||||||
|
|
||||||
func (f *Fetcher) Resolve(req *base.Request) error {
|
|
||||||
if err := base.ParseReqExtra[pftp.ReqExtra](req); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
f.meta.Req = req
|
|
||||||
t, err := parseTarget(req.URL, f.tlsMode())
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
s, err := f.dial(context.Background(), t)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
// quit, not abort: a server that caps logins must see this session end
|
|
||||||
// before the download's own logins arrive.
|
|
||||||
defer s.quit()
|
|
||||||
f.loginOK.Store(true)
|
|
||||||
res, err := f.resolveTarget(s, t)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
f.meta.Res = res
|
|
||||||
f.mu.Lock()
|
|
||||||
f.segments = nil
|
|
||||||
f.done = false
|
|
||||||
f.mu.Unlock()
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// selected returns the indexes of the files to download, in SelectFiles
|
|
||||||
// order. The caller holds mu or owns the fetcher.
|
|
||||||
func (f *Fetcher) selected() []int {
|
|
||||||
if f.meta == nil || f.meta.Res == nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
if f.meta.Opts != nil && len(f.meta.Opts.SelectFiles) > 0 {
|
|
||||||
return f.meta.Opts.SelectFiles
|
|
||||||
}
|
|
||||||
all := make([]int, len(f.meta.Res.Files))
|
|
||||||
for i := range all {
|
|
||||||
all[i] = i
|
|
||||||
}
|
|
||||||
return all
|
|
||||||
}
|
|
||||||
|
|
||||||
// localPath is where a file lands: under the task path, with the file's
|
|
||||||
// directories below the URL kept.
|
|
||||||
func (f *Fetcher) localPath(index int) string {
|
|
||||||
if f.meta.Res.Name == "" {
|
|
||||||
return f.meta.SingleFilepath()
|
|
||||||
}
|
|
||||||
file := f.meta.Res.Files[index]
|
|
||||||
return path.Join(f.meta.FolderPath(), file.Path, file.Name)
|
|
||||||
}
|
|
||||||
|
|
||||||
// remotePath is the server path of a file. A single file uses the URL's own
|
|
||||||
// path, which a local rename never touches.
|
|
||||||
func (f *Fetcher) remotePath(t *target, index int) string {
|
|
||||||
if f.meta.Res.Name == "" {
|
|
||||||
return t.path
|
|
||||||
}
|
|
||||||
file := f.meta.Res.Files[index]
|
|
||||||
return path.Join(t.path, file.Path, file.Name)
|
|
||||||
}
|
|
||||||
|
|
||||||
// plan splits every selected file into segments of at least minSegmentSize,
|
|
||||||
// one per connection at most. The caller holds mu.
|
|
||||||
func (f *Fetcher) plan() {
|
|
||||||
conns := f.connections()
|
|
||||||
f.segments = nil
|
|
||||||
for _, index := range f.selected() {
|
|
||||||
size := f.meta.Res.Files[index].Size
|
|
||||||
n := int64(1)
|
|
||||||
if !f.noRest {
|
|
||||||
n = max(1, min(int64(conns), size/minSegmentSize))
|
|
||||||
}
|
|
||||||
for i := int64(0); i < n; i++ {
|
|
||||||
f.segments = append(f.segments, &segment{
|
|
||||||
File: index,
|
|
||||||
Begin: i * size / n,
|
|
||||||
End: (i+1)*size/n - 1,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// planSequential gives every unfinished file one segment from byte 0,
|
|
||||||
// because without REST a transfer cannot start in the middle. The caller
|
|
||||||
// holds mu.
|
|
||||||
func (f *Fetcher) planSequential() {
|
|
||||||
complete := map[int]bool{}
|
|
||||||
for _, s := range f.segments {
|
|
||||||
if _, seen := complete[s.File]; !seen {
|
|
||||||
complete[s.File] = true
|
|
||||||
}
|
|
||||||
if s.remain() > 0 {
|
|
||||||
complete[s.File] = false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
var out []*segment
|
|
||||||
for _, index := range f.selected() {
|
|
||||||
size := f.meta.Res.Files[index].Size
|
|
||||||
if complete[index] {
|
|
||||||
out = append(out, &segment{File: index, Begin: 0, End: size - 1, Downloaded: size})
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
out = append(out, &segment{File: index, Begin: 0, End: size - 1})
|
|
||||||
}
|
|
||||||
f.segments = out
|
|
||||||
// Bytes counted by the old segments are fetched again.
|
|
||||||
f.lanes = nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// openFiles opens every selected file for writing. A fresh plan creates and
|
|
||||||
// sizes them; a resumed one keeps their content, and a file that has gone
|
|
||||||
// missing starts over.
|
|
||||||
func (f *Fetcher) openFiles(fresh bool) (map[int]*os.File, error) {
|
|
||||||
files := map[int]*os.File{}
|
|
||||||
closeAll := func() {
|
|
||||||
for _, file := range files {
|
|
||||||
_ = file.Close()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for _, index := range f.selected() {
|
|
||||||
name := f.localPath(index)
|
|
||||||
var (
|
|
||||||
file *os.File
|
|
||||||
err error
|
|
||||||
)
|
|
||||||
_, statErr := os.Stat(name)
|
|
||||||
switch {
|
|
||||||
case fresh || os.IsNotExist(statErr):
|
|
||||||
file, err = f.ctl.Touch(name, f.meta.Res.Files[index].Size)
|
|
||||||
if err == nil && !fresh {
|
|
||||||
f.mu.Lock()
|
|
||||||
for _, s := range f.segments {
|
|
||||||
if s.File == index {
|
|
||||||
s.Downloaded = 0
|
|
||||||
}
|
|
||||||
}
|
|
||||||
f.mu.Unlock()
|
|
||||||
}
|
|
||||||
case statErr != nil:
|
|
||||||
err = statErr
|
|
||||||
default:
|
|
||||||
file, err = os.OpenFile(name, os.O_RDWR, 0)
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
closeAll()
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
files[index] = file
|
|
||||||
}
|
|
||||||
return files, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (f *Fetcher) Start() error {
|
|
||||||
f.runMu.Lock()
|
|
||||||
defer f.runMu.Unlock()
|
|
||||||
if f.runDone != nil {
|
|
||||||
select {
|
|
||||||
case <-f.runDone:
|
|
||||||
default:
|
|
||||||
return nil // already running
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if err := f.initOptions(); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if err := base.ParseReqExtra[pftp.ReqExtra](f.meta.Req); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
t, err := parseTarget(f.meta.Req.URL, f.tlsMode())
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
// A paused or failed run may have left a result nobody read.
|
|
||||||
select {
|
|
||||||
case <-f.doneCh:
|
|
||||||
default:
|
|
||||||
}
|
|
||||||
|
|
||||||
f.mu.Lock()
|
|
||||||
if f.done {
|
|
||||||
f.mu.Unlock()
|
|
||||||
f.doneCh <- nil
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
fresh := len(f.segments) == 0
|
|
||||||
if fresh {
|
|
||||||
f.plan()
|
|
||||||
} else if f.noRest {
|
|
||||||
f.planSequential()
|
|
||||||
}
|
|
||||||
f.mu.Unlock()
|
|
||||||
|
|
||||||
files, err := f.openFiles(fresh)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
|
||||||
f.cancel = cancel
|
|
||||||
f.runDone = make(chan struct{})
|
|
||||||
go f.run(ctx, t, files, f.runDone)
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (f *Fetcher) run(ctx context.Context, t *target, files map[int]*os.File, done chan struct{}) {
|
|
||||||
defer close(done)
|
|
||||||
err := f.download(ctx, t, files)
|
|
||||||
for _, file := range files {
|
|
||||||
if cerr := file.Close(); cerr != nil && err == nil {
|
|
||||||
err = cerr
|
|
||||||
}
|
|
||||||
}
|
|
||||||
f.mu.Lock()
|
|
||||||
if err == nil {
|
|
||||||
f.done = true
|
|
||||||
for _, l := range f.lanes {
|
|
||||||
l.completed = true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
f.mu.Unlock()
|
|
||||||
if err != nil && ctx.Err() != nil {
|
|
||||||
return // paused
|
|
||||||
}
|
|
||||||
select {
|
|
||||||
case f.doneCh <- err:
|
|
||||||
default:
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (f *Fetcher) download(ctx context.Context, t *target, files map[int]*os.File) error {
|
|
||||||
for {
|
|
||||||
err := f.runWorkers(ctx, t, files)
|
|
||||||
if !errors.Is(err, errRestUnsupported) {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
f.mu.Lock()
|
|
||||||
f.noRest = true
|
|
||||||
f.planSequential()
|
|
||||||
f.mu.Unlock()
|
|
||||||
f.warn("ftp: the server refused REST, so each file is downloaded in one stream")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// pool hands out the segments of one run to its workers. Every decision to
|
|
||||||
// leave the run is taken under mu together with the queue it depends on, so
|
|
||||||
// a segment put back can never be left without a worker.
|
|
||||||
type pool struct {
|
|
||||||
mu sync.Mutex
|
|
||||||
queue []*segment
|
|
||||||
running int // workers still in the run
|
|
||||||
}
|
|
||||||
|
|
||||||
// take returns the next segment. A worker that finds the queue empty leaves
|
|
||||||
// the run.
|
|
||||||
func (p *pool) take() *segment {
|
|
||||||
p.mu.Lock()
|
|
||||||
defer p.mu.Unlock()
|
|
||||||
if len(p.queue) == 0 {
|
|
||||||
p.running--
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
s := p.queue[0]
|
|
||||||
p.queue = p.queue[1:]
|
|
||||||
return s
|
|
||||||
}
|
|
||||||
|
|
||||||
func (p *pool) putBack(s *segment) {
|
|
||||||
p.mu.Lock()
|
|
||||||
defer p.mu.Unlock()
|
|
||||||
p.queue = append([]*segment{s}, p.queue...)
|
|
||||||
}
|
|
||||||
|
|
||||||
// refused puts back the segment of a worker whose login the server turned
|
|
||||||
// away. The worker leaves when another worker is still in the run to take
|
|
||||||
// the segment, and left is the number of workers that remain; the last
|
|
||||||
// worker stays and retries.
|
|
||||||
func (p *pool) refused(s *segment) (leave bool, left int) {
|
|
||||||
p.mu.Lock()
|
|
||||||
defer p.mu.Unlock()
|
|
||||||
p.queue = append([]*segment{s}, p.queue...)
|
|
||||||
if p.running > 1 {
|
|
||||||
p.running--
|
|
||||||
return true, p.running
|
|
||||||
}
|
|
||||||
return false, p.running
|
|
||||||
}
|
|
||||||
|
|
||||||
// leave takes a worker out of the run when it stops for any other reason.
|
|
||||||
func (p *pool) leave() {
|
|
||||||
p.mu.Lock()
|
|
||||||
defer p.mu.Unlock()
|
|
||||||
p.running--
|
|
||||||
}
|
|
||||||
|
|
||||||
func (f *Fetcher) runWorkers(ctx context.Context, t *target, files map[int]*os.File) error {
|
|
||||||
runCtx, cancel := context.WithCancel(ctx)
|
|
||||||
defer cancel()
|
|
||||||
|
|
||||||
f.mu.Lock()
|
|
||||||
p := &pool{}
|
|
||||||
for _, s := range f.segments {
|
|
||||||
if s.remain() > 0 {
|
|
||||||
p.queue = append(p.queue, s)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
workers := min(f.connections(), len(p.queue))
|
|
||||||
f.mu.Unlock()
|
|
||||||
if workers == 0 {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
p.running = workers
|
|
||||||
|
|
||||||
var (
|
|
||||||
wg sync.WaitGroup
|
|
||||||
errMu sync.Mutex
|
|
||||||
firstErr error
|
|
||||||
)
|
|
||||||
for id := 0; id < workers; id++ {
|
|
||||||
wg.Add(1)
|
|
||||||
go func(id int) {
|
|
||||||
defer wg.Done()
|
|
||||||
if err := f.worker(runCtx, t, files, p, id); err != nil {
|
|
||||||
errMu.Lock()
|
|
||||||
if firstErr == nil {
|
|
||||||
firstErr = err
|
|
||||||
}
|
|
||||||
errMu.Unlock()
|
|
||||||
cancel()
|
|
||||||
}
|
|
||||||
}(id)
|
|
||||||
}
|
|
||||||
wg.Wait()
|
|
||||||
|
|
||||||
if err := ctx.Err(); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if firstErr != nil {
|
|
||||||
return firstErr
|
|
||||||
}
|
|
||||||
f.mu.Lock()
|
|
||||||
defer f.mu.Unlock()
|
|
||||||
for _, s := range f.segments {
|
|
||||||
if s.remain() > 0 {
|
|
||||||
return errors.New("ftp: the download stopped with data missing")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// worker fetches segments from the pool until it is empty. A worker whose
|
|
||||||
// login the server refuses (421, or 530 once these credentials have worked)
|
|
||||||
// hands its segment back and leaves while other workers remain, lowering the
|
|
||||||
// task's ceiling to the workers left. This holds whether or not the worker
|
|
||||||
// had logged in before: a slot can also go to another client mid-download.
|
|
||||||
func (f *Fetcher) worker(ctx context.Context, t *target, files map[int]*os.File, p *pool, id int) error {
|
|
||||||
var ln *lane
|
|
||||||
f.mu.Lock()
|
|
||||||
if id < len(f.lanes) {
|
|
||||||
ln = f.lanes[id]
|
|
||||||
}
|
|
||||||
f.mu.Unlock()
|
|
||||||
onLogin := func() {
|
|
||||||
f.mu.Lock()
|
|
||||||
if ln == nil {
|
|
||||||
ln = &lane{}
|
|
||||||
f.lanes = append(f.lanes, ln)
|
|
||||||
}
|
|
||||||
f.mu.Unlock()
|
|
||||||
}
|
|
||||||
|
|
||||||
failures := 0
|
|
||||||
pendingRetries := 0
|
|
||||||
for {
|
|
||||||
seg := p.take()
|
|
||||||
if seg == nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
err := f.fetchSegment(ctx, t, files, seg, &ln, onLogin)
|
|
||||||
if err == nil {
|
|
||||||
failures = 0
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if ctx.Err() != nil {
|
|
||||||
p.putBack(seg)
|
|
||||||
p.leave()
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
var le *loginError
|
|
||||||
if errors.As(err, &le) && isLoginRefusal(le.err, f.loginOK.Load()) {
|
|
||||||
if leave, left := p.refused(seg); leave {
|
|
||||||
f.lowerCeiling(left)
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
// The last worker: wait for a slot.
|
|
||||||
} else {
|
|
||||||
p.putBack(seg)
|
|
||||||
if errors.Is(err, errRestUnsupported) {
|
|
||||||
p.leave()
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if isPermanent(err) || errors.Is(err, errDiskWrite) {
|
|
||||||
f.markFailed(ln)
|
|
||||||
p.leave()
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
failures++
|
|
||||||
f.mu.Lock()
|
|
||||||
if ln != nil {
|
|
||||||
ln.retryTimes += pendingRetries + 1
|
|
||||||
pendingRetries = 0
|
|
||||||
} else {
|
|
||||||
pendingRetries++
|
|
||||||
}
|
|
||||||
f.mu.Unlock()
|
|
||||||
if failures > maxRetries {
|
|
||||||
f.markFailed(ln)
|
|
||||||
p.leave()
|
|
||||||
return fmt.Errorf("ftp: giving up after %d retries: %w", maxRetries, err)
|
|
||||||
}
|
|
||||||
select {
|
|
||||||
case <-ctx.Done():
|
|
||||||
p.leave()
|
|
||||||
return nil
|
|
||||||
case <-time.After(retryDelay * time.Duration(failures)):
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (f *Fetcher) lowerCeiling(n int) {
|
|
||||||
f.mu.Lock()
|
|
||||||
defer f.mu.Unlock()
|
|
||||||
if f.maxConns == 0 || n < f.maxConns {
|
|
||||||
f.maxConns = n
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (f *Fetcher) markFailed(ln *lane) {
|
|
||||||
if ln == nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
f.mu.Lock()
|
|
||||||
ln.failed = true
|
|
||||||
f.mu.Unlock()
|
|
||||||
}
|
|
||||||
|
|
||||||
// isRestRefused tells a REST the server does not implement. RETR itself
|
|
||||||
// fails with 550 and the like, never with these codes.
|
|
||||||
func isRestRefused(err error) bool {
|
|
||||||
switch replyCode(err) {
|
|
||||||
case ftp.StatusBadCommand, ftp.StatusBadArguments, ftp.StatusNotImplemented, ftp.StatusNotImplementedParameter:
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
// fetchSegment logs in, retrieves the segment from its first missing byte
|
|
||||||
// and writes it in place. It stops reading at the segment's end and closes
|
|
||||||
// the data connection; the server then reports the transfer aborted, which
|
|
||||||
// is expected.
|
|
||||||
func (f *Fetcher) fetchSegment(ctx context.Context, t *target, files map[int]*os.File, seg *segment, ln **lane, onLogin func()) error {
|
|
||||||
s, err := f.dial(ctx, t)
|
|
||||||
if err != nil {
|
|
||||||
return &loginError{err}
|
|
||||||
}
|
|
||||||
defer s.quit()
|
|
||||||
f.loginOK.Store(true)
|
|
||||||
onLogin()
|
|
||||||
|
|
||||||
remote := f.remotePath(t, seg.File)
|
|
||||||
f.mu.Lock()
|
|
||||||
noRest := f.noRest
|
|
||||||
wantSize := f.meta.Res.Files[seg.File].Size
|
|
||||||
if noRest {
|
|
||||||
seg.Downloaded = 0
|
|
||||||
}
|
|
||||||
pos := seg.Begin + seg.Downloaded
|
|
||||||
end := seg.End
|
|
||||||
f.mu.Unlock()
|
|
||||||
|
|
||||||
// A file that changed on the server would mix old and new bytes.
|
|
||||||
s.deadline()
|
|
||||||
if size, err := s.FileSize(remote); err == nil && size != wantSize {
|
|
||||||
return errSizeChanged
|
|
||||||
} else if err != nil && replyCode(err) == 0 {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
s.deadline()
|
|
||||||
var resp *ftp.Response
|
|
||||||
if noRest {
|
|
||||||
resp, err = s.Retr(remote)
|
|
||||||
} else {
|
|
||||||
resp, err = s.RetrFrom(remote, uint64(pos))
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
if !noRest && pos > 0 && isRestRefused(err) {
|
|
||||||
return errRestUnsupported
|
|
||||||
}
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
defer resp.Close()
|
|
||||||
s.clearDeadline()
|
|
||||||
|
|
||||||
file := files[seg.File]
|
|
||||||
buf := make([]byte, copyBufferSize)
|
|
||||||
for pos <= end {
|
|
||||||
n := int(min(int64(len(buf)), end-pos+1))
|
|
||||||
_ = resp.SetDeadline(time.Now().Add(readTimeout))
|
|
||||||
nr, rerr := resp.Read(buf[:n])
|
|
||||||
if nr > 0 {
|
|
||||||
if _, werr := file.WriteAt(buf[:nr], pos); werr != nil {
|
|
||||||
return fmt.Errorf("%w: %w", errDiskWrite, werr)
|
|
||||||
}
|
|
||||||
pos += int64(nr)
|
|
||||||
f.mu.Lock()
|
|
||||||
seg.Downloaded += int64(nr)
|
|
||||||
if *ln != nil {
|
|
||||||
(*ln).downloaded += int64(nr)
|
|
||||||
}
|
|
||||||
f.mu.Unlock()
|
|
||||||
}
|
|
||||||
if rerr != nil {
|
|
||||||
if pos > end {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
if errors.Is(rerr, io.EOF) {
|
|
||||||
return errShortTransfer
|
|
||||||
}
|
|
||||||
return rerr
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (f *Fetcher) Patch(req *base.Request, opts *base.Options) error {
|
|
||||||
if req == nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
if req.URL != "" {
|
|
||||||
f.meta.Req.URL = req.URL
|
|
||||||
}
|
|
||||||
if req.Labels != nil {
|
|
||||||
if f.meta.Req.Labels == nil {
|
|
||||||
f.meta.Req.Labels = make(map[string]string)
|
|
||||||
}
|
|
||||||
for k, v := range req.Labels {
|
|
||||||
f.meta.Req.Labels[k] = v
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if req.Proxy != nil {
|
|
||||||
f.meta.Req.Proxy = req.Proxy
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (f *Fetcher) Pause() error {
|
|
||||||
f.runMu.Lock()
|
|
||||||
defer f.runMu.Unlock()
|
|
||||||
if f.cancel != nil {
|
|
||||||
f.cancel()
|
|
||||||
}
|
|
||||||
if f.runDone != nil {
|
|
||||||
<-f.runDone
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (f *Fetcher) Close() error {
|
|
||||||
return f.Pause()
|
|
||||||
}
|
|
||||||
|
|
||||||
func (f *Fetcher) Meta() *fetcher.FetcherMeta {
|
|
||||||
return f.meta
|
|
||||||
}
|
|
||||||
|
|
||||||
// Stats reports one connection per worker that logged in, in the HTTP
|
|
||||||
// protocol's shape.
|
|
||||||
func (f *Fetcher) Stats() *fetcher.Stats {
|
|
||||||
f.mu.Lock()
|
|
||||||
defer f.mu.Unlock()
|
|
||||||
var total int64
|
|
||||||
if n := int64(len(f.lanes)); n > 0 && f.meta != nil && f.meta.Res != nil && f.meta.Res.Size > 0 {
|
|
||||||
total = (f.meta.Res.Size + n - 1) / n
|
|
||||||
}
|
|
||||||
conns := make([]*pftp.StatsConnection, 0, len(f.lanes))
|
|
||||||
for _, l := range f.lanes {
|
|
||||||
conns = append(conns, &pftp.StatsConnection{
|
|
||||||
Downloaded: l.downloaded,
|
|
||||||
Total: total,
|
|
||||||
Completed: l.completed || (f.done && !l.failed),
|
|
||||||
Failed: l.failed,
|
|
||||||
RetryTimes: l.retryTimes,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
return &fetcher.Stats{Snapshot: &pftp.Stats{Connections: conns}}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Progress returns the bytes downloaded of each selected file, in
|
|
||||||
// SelectFiles order.
|
|
||||||
func (f *Fetcher) Progress() fetcher.Progress {
|
|
||||||
f.mu.Lock()
|
|
||||||
defer f.mu.Unlock()
|
|
||||||
perFile := map[int]int64{}
|
|
||||||
for _, s := range f.segments {
|
|
||||||
perFile[s.File] += s.Downloaded
|
|
||||||
}
|
|
||||||
sel := f.selected()
|
|
||||||
p := make(fetcher.Progress, len(sel))
|
|
||||||
for i, index := range sel {
|
|
||||||
p[i] = perFile[index]
|
|
||||||
}
|
|
||||||
return p
|
|
||||||
}
|
|
||||||
|
|
||||||
func (f *Fetcher) Wait() error {
|
|
||||||
return <-f.doneCh
|
|
||||||
}
|
|
||||||
@@ -1,109 +0,0 @@
|
|||||||
package ftp
|
|
||||||
|
|
||||||
import (
|
|
||||||
"crypto/tls"
|
|
||||||
"net/url"
|
|
||||||
"path"
|
|
||||||
"strings"
|
|
||||||
|
|
||||||
"github.com/GopeedLab/gopeed/internal/fetcher"
|
|
||||||
"github.com/GopeedLab/gopeed/pkg/base"
|
|
||||||
pftp "github.com/GopeedLab/gopeed/pkg/protocol/ftp"
|
|
||||||
)
|
|
||||||
|
|
||||||
// fetcherData is what a task saves to resume later. Segments has the same
|
|
||||||
// Begin, End and Downloaded fields every version writes; the other fields are
|
|
||||||
// optional, so a save without them still restores.
|
|
||||||
type fetcherData struct {
|
|
||||||
Segments []*segment
|
|
||||||
// MaxConns is the login ceiling the server enforced, 0 when none was hit.
|
|
||||||
MaxConns int `json:",omitempty"`
|
|
||||||
// NoRest records that the server refused REST, so the task downloads each
|
|
||||||
// file in one sequential stream.
|
|
||||||
NoRest bool `json:",omitempty"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// FetcherManager handles ftp://, ftps:// (implicit TLS) and ftpes:// (explicit
|
|
||||||
// TLS) URLs.
|
|
||||||
type FetcherManager struct {
|
|
||||||
// baseTLS is the base for every TLS connection. Nil means the system
|
|
||||||
// roots; the package's tests set it to trust their own certificate. The
|
|
||||||
// task's skipVerifyCert still turns verification off.
|
|
||||||
baseTLS *tls.Config
|
|
||||||
}
|
|
||||||
|
|
||||||
func (fm *FetcherManager) Name() string {
|
|
||||||
return "ftp"
|
|
||||||
}
|
|
||||||
|
|
||||||
func (fm *FetcherManager) Filters() []*fetcher.SchemeFilter {
|
|
||||||
return []*fetcher.SchemeFilter{
|
|
||||||
{Type: fetcher.FilterTypeUrl, Pattern: "FTP"},
|
|
||||||
{Type: fetcher.FilterTypeUrl, Pattern: "FTPS"},
|
|
||||||
{Type: fetcher.FilterTypeUrl, Pattern: "FTPES"},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (fm *FetcherManager) Build() fetcher.Fetcher {
|
|
||||||
return &Fetcher{manager: fm}
|
|
||||||
}
|
|
||||||
|
|
||||||
// ParseName names a task before it is resolved. It never includes the
|
|
||||||
// userinfo of the URL.
|
|
||||||
func (fm *FetcherManager) ParseName(u string) string {
|
|
||||||
parsed, err := url.Parse(u)
|
|
||||||
if err != nil {
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
name := path.Base(strings.TrimSuffix(parsed.Path, "/"))
|
|
||||||
if name == "" || name == "/" || name == "." {
|
|
||||||
name = parsed.Hostname()
|
|
||||||
}
|
|
||||||
return name
|
|
||||||
}
|
|
||||||
|
|
||||||
func (fm *FetcherManager) AutoRename() bool {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
|
|
||||||
func (fm *FetcherManager) DefaultConfig() any {
|
|
||||||
return &config{Connections: defaultConnections}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (fm *FetcherManager) Store(f fetcher.Fetcher) (any, error) {
|
|
||||||
ff := f.(*Fetcher)
|
|
||||||
ff.mu.Lock()
|
|
||||||
defer ff.mu.Unlock()
|
|
||||||
segments := make([]*segment, len(ff.segments))
|
|
||||||
for i, s := range ff.segments {
|
|
||||||
c := *s
|
|
||||||
segments[i] = &c
|
|
||||||
}
|
|
||||||
return &fetcherData{
|
|
||||||
Segments: segments,
|
|
||||||
MaxConns: ff.maxConns,
|
|
||||||
NoRest: ff.noRest,
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (fm *FetcherManager) Restore() (v any, f func(meta *fetcher.FetcherMeta, v any) fetcher.Fetcher) {
|
|
||||||
return &fetcherData{}, func(meta *fetcher.FetcherMeta, v any) fetcher.Fetcher {
|
|
||||||
fd := v.(*fetcherData)
|
|
||||||
ff := fm.Build().(*Fetcher)
|
|
||||||
ff.meta = meta
|
|
||||||
if meta != nil && meta.Req != nil {
|
|
||||||
_ = base.ParseReqExtra[pftp.ReqExtra](meta.Req)
|
|
||||||
}
|
|
||||||
if meta != nil && meta.Opts != nil {
|
|
||||||
_ = base.ParseOptExtra[pftp.OptsExtra](meta.Opts)
|
|
||||||
}
|
|
||||||
ff.segments = fd.Segments
|
|
||||||
ff.maxConns = fd.MaxConns
|
|
||||||
ff.noRest = fd.NoRest
|
|
||||||
return ff
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (fm *FetcherManager) Close() error {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
@@ -1,818 +0,0 @@
|
|||||||
package ftp
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"context"
|
|
||||||
"crypto/tls"
|
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"net"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"slices"
|
|
||||||
"strings"
|
|
||||||
"sync/atomic"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/armon/go-socks5"
|
|
||||||
"github.com/rs/zerolog"
|
|
||||||
|
|
||||||
"github.com/GopeedLab/gopeed/internal/controller"
|
|
||||||
"github.com/GopeedLab/gopeed/internal/fetcher"
|
|
||||||
"github.com/GopeedLab/gopeed/pkg/base"
|
|
||||||
pftp "github.com/GopeedLab/gopeed/pkg/protocol/ftp"
|
|
||||||
)
|
|
||||||
|
|
||||||
const mib = 1 << 20
|
|
||||||
|
|
||||||
// testManager trusts the test server's certificate, the way an embedding
|
|
||||||
// application could add private roots.
|
|
||||||
func testManager(srv *FTPServer) *FetcherManager {
|
|
||||||
fm := &FetcherManager{}
|
|
||||||
if srv != nil {
|
|
||||||
fm.baseTLS = &tls.Config{RootCAs: srv.CA}
|
|
||||||
}
|
|
||||||
return fm
|
|
||||||
}
|
|
||||||
|
|
||||||
func testController(fm *FetcherManager) *controller.Controller {
|
|
||||||
ctl := controller.NewController()
|
|
||||||
ctl.GetConfig = func(v any) {
|
|
||||||
data, _ := json.Marshal(fm.DefaultConfig())
|
|
||||||
_ = json.Unmarshal(data, v)
|
|
||||||
}
|
|
||||||
return ctl
|
|
||||||
}
|
|
||||||
|
|
||||||
func buildFetcher(fm *FetcherManager) *Fetcher {
|
|
||||||
f := fm.Build().(*Fetcher)
|
|
||||||
f.Setup(testController(fm))
|
|
||||||
return f
|
|
||||||
}
|
|
||||||
|
|
||||||
func ftpURL(srv *FTPServer, scheme, userinfo, p string) string {
|
|
||||||
if userinfo != "" {
|
|
||||||
userinfo += "@"
|
|
||||||
}
|
|
||||||
return scheme + "://" + userinfo + srv.Addr + "/" + strings.TrimPrefix(p, "/")
|
|
||||||
}
|
|
||||||
|
|
||||||
// resolveOpts resolves req and then gives the fetcher its options, in the
|
|
||||||
// order the downloader uses.
|
|
||||||
func resolveOpts(f *Fetcher, req *base.Request, opts *base.Options) error {
|
|
||||||
if err := f.Resolve(req); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
f.Meta().Opts = opts
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func resolveTask(t *testing.T, f *Fetcher, rawURL string, extra *pftp.OptsExtra) {
|
|
||||||
t.Helper()
|
|
||||||
resolveRequest(t, f, &base.Request{URL: rawURL}, extra)
|
|
||||||
}
|
|
||||||
|
|
||||||
func resolveRequest(t *testing.T, f *Fetcher, req *base.Request, extra *pftp.OptsExtra) {
|
|
||||||
t.Helper()
|
|
||||||
opts := &base.Options{Path: t.TempDir()}
|
|
||||||
if extra != nil {
|
|
||||||
opts.Extra = extra
|
|
||||||
}
|
|
||||||
if err := resolveOpts(f, req, opts); err != nil {
|
|
||||||
t.Fatalf("resolve %s: %v", req.URL, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func waitFetcher(t *testing.T, f fetcher.Fetcher) error {
|
|
||||||
t.Helper()
|
|
||||||
done := make(chan error, 1)
|
|
||||||
go func() { done <- f.Wait() }()
|
|
||||||
select {
|
|
||||||
case err := <-done:
|
|
||||||
return err
|
|
||||||
case <-time.After(60 * time.Second):
|
|
||||||
t.Fatal("download did not finish in 60s")
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func startAndWait(t *testing.T, f *Fetcher) {
|
|
||||||
t.Helper()
|
|
||||||
if err := f.Start(); err != nil {
|
|
||||||
t.Fatalf("start: %v", err)
|
|
||||||
}
|
|
||||||
if err := waitFetcher(t, f); err != nil {
|
|
||||||
t.Fatalf("download: %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func checkFile(t *testing.T, name, want string) {
|
|
||||||
t.Helper()
|
|
||||||
if got := FileSHA256(t, name); got != want {
|
|
||||||
t.Fatalf("%s: sha256 %s, want %s", name, got, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func ftpStats(t *testing.T, f *Fetcher) *pftp.Stats {
|
|
||||||
t.Helper()
|
|
||||||
stats, ok := f.Stats().Snapshot.(*pftp.Stats)
|
|
||||||
if !ok {
|
|
||||||
t.Fatalf("stats snapshot is %T, want *ftp.Stats", f.Stats().Snapshot)
|
|
||||||
}
|
|
||||||
return stats
|
|
||||||
}
|
|
||||||
|
|
||||||
func storeData(t *testing.T, fm *FetcherManager, f *Fetcher) []byte {
|
|
||||||
t.Helper()
|
|
||||||
data, err := fm.Store(f)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
blob, err := json.Marshal(data)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
return blob
|
|
||||||
}
|
|
||||||
|
|
||||||
// restoreFetcher rebuilds a fetcher the way the downloader does after a
|
|
||||||
// restart: the meta and the save blob both come back from JSON.
|
|
||||||
func restoreFetcher(t *testing.T, fm *FetcherManager, meta *fetcher.FetcherMeta, blob []byte) *Fetcher {
|
|
||||||
t.Helper()
|
|
||||||
metaJSON, err := json.Marshal(meta)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
var restoredMeta fetcher.FetcherMeta
|
|
||||||
if err := json.Unmarshal(metaJSON, &restoredMeta); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
v, build := fm.Restore()
|
|
||||||
if err := json.Unmarshal(blob, v); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
f := build(&restoredMeta, v).(*Fetcher)
|
|
||||||
f.Setup(testController(fm))
|
|
||||||
return f
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestResolveFile(t *testing.T) {
|
|
||||||
for _, noMLST := range []bool{false, true} {
|
|
||||||
t.Run(fmt.Sprintf("noMLST=%v", noMLST), func(t *testing.T) {
|
|
||||||
srv := StartFTPServer(t, ServerOptions{NoMLST: noMLST})
|
|
||||||
data := RandomBytes(100*1024, 1)
|
|
||||||
srv.WriteFile(t, "pub/file name.bin", data)
|
|
||||||
|
|
||||||
f := buildFetcher(testManager(srv))
|
|
||||||
resolveTask(t, f, ftpURL(srv, "ftp", "", "pub/file%20name.bin"), nil)
|
|
||||||
res := f.Meta().Res
|
|
||||||
if res.Name != "" {
|
|
||||||
t.Fatalf("a file must not be a folder resource, got name %q", res.Name)
|
|
||||||
}
|
|
||||||
if len(res.Files) != 1 {
|
|
||||||
t.Fatalf("got %d files, want 1", len(res.Files))
|
|
||||||
}
|
|
||||||
file := res.Files[0]
|
|
||||||
if file.Name != "file name.bin" || file.Path != "" || file.Size != int64(len(data)) {
|
|
||||||
t.Fatalf("got file %+v, want name %q, empty path, size %d", file, "file name.bin", len(data))
|
|
||||||
}
|
|
||||||
if res.Size != int64(len(data)) {
|
|
||||||
t.Fatalf("resource size %d, want %d", res.Size, len(data))
|
|
||||||
}
|
|
||||||
if file.Ctime == nil || time.Since(*file.Ctime) > time.Hour {
|
|
||||||
t.Fatalf("modification time %v is missing or wrong", file.Ctime)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func writeTree(t *testing.T, srv *FTPServer) map[string]string {
|
|
||||||
t.Helper()
|
|
||||||
return map[string]string{
|
|
||||||
"a.txt": srv.WriteFile(t, "pub/tree/a.txt", []byte("hello")),
|
|
||||||
"sub/b.bin": srv.WriteFile(t, "pub/tree/sub/b.bin", RandomBytes(3*mib+17, 2)),
|
|
||||||
"sub/deeper/c.dat": srv.WriteFile(t, "pub/tree/sub/deeper/c.dat", RandomBytes(2000, 3)),
|
|
||||||
"sub/empty.txt": srv.WriteFile(t, "pub/tree/sub/empty.txt", nil),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestResolveDir(t *testing.T) {
|
|
||||||
for _, noMLST := range []bool{false, true} {
|
|
||||||
for _, p := range []string{"pub/tree", "pub/tree/"} {
|
|
||||||
t.Run(fmt.Sprintf("noMLST=%v/%s", noMLST, p), func(t *testing.T) {
|
|
||||||
srv := StartFTPServer(t, ServerOptions{NoMLST: noMLST})
|
|
||||||
writeTree(t, srv)
|
|
||||||
if err := os.MkdirAll(filepath.Join(srv.Root, "pub/tree/void"), 0o755); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
f := buildFetcher(testManager(srv))
|
|
||||||
resolveTask(t, f, ftpURL(srv, "ftp", "", p), nil)
|
|
||||||
res := f.Meta().Res
|
|
||||||
if res.Name != "tree" {
|
|
||||||
t.Fatalf("folder name %q, want tree", res.Name)
|
|
||||||
}
|
|
||||||
var got []string
|
|
||||||
for _, file := range res.Files {
|
|
||||||
got = append(got, fmt.Sprintf("%s|%s|%d", file.Path, file.Name, file.Size))
|
|
||||||
}
|
|
||||||
want := []string{
|
|
||||||
"|a.txt|5",
|
|
||||||
"sub|b.bin|3145745",
|
|
||||||
"sub|empty.txt|0",
|
|
||||||
"sub/deeper|c.dat|2000",
|
|
||||||
}
|
|
||||||
if !slices.Equal(got, want) {
|
|
||||||
t.Fatalf("files\n got %q\nwant %q", got, want)
|
|
||||||
}
|
|
||||||
if res.Size != 5+3145745+2000 {
|
|
||||||
t.Fatalf("resource size %d", res.Size)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestResolveDirLimits(t *testing.T) {
|
|
||||||
srv := StartFTPServer(t, ServerOptions{})
|
|
||||||
srv.WriteFile(t, "deep/1/2/3/4/5/6/7/8/f.txt", []byte("ok"))
|
|
||||||
srv.WriteFile(t, "deep9/1/2/3/4/5/6/7/8/9/f.txt", []byte("too deep"))
|
|
||||||
for i := 0; i < 4; i++ {
|
|
||||||
srv.WriteFile(t, fmt.Sprintf("many/f%d", i), []byte("x"))
|
|
||||||
}
|
|
||||||
|
|
||||||
f := buildFetcher(testManager(srv))
|
|
||||||
resolveTask(t, f, ftpURL(srv, "ftp", "", "deep"), nil)
|
|
||||||
if got := f.Meta().Res.Files[0].Path; got != "1/2/3/4/5/6/7/8" {
|
|
||||||
t.Fatalf("depth 8 path %q", got)
|
|
||||||
}
|
|
||||||
|
|
||||||
f = buildFetcher(testManager(srv))
|
|
||||||
err := resolveOpts(f, &base.Request{URL: ftpURL(srv, "ftp", "", "deep9")}, &base.Options{Path: t.TempDir()})
|
|
||||||
if err == nil || !strings.Contains(err.Error(), "8 levels") {
|
|
||||||
t.Fatalf("a tree deeper than 8 levels must fail and name the limit, got %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
saved := maxWalkFiles
|
|
||||||
maxWalkFiles = 3
|
|
||||||
defer func() { maxWalkFiles = saved }()
|
|
||||||
f = buildFetcher(testManager(srv))
|
|
||||||
err = resolveOpts(f, &base.Request{URL: ftpURL(srv, "ftp", "", "many")}, &base.Options{Path: t.TempDir()})
|
|
||||||
if err == nil || !strings.Contains(err.Error(), "3 files") {
|
|
||||||
t.Fatalf("a tree over the file cap must fail and name the limit, got %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestDownloadPlain(t *testing.T) {
|
|
||||||
srv := StartFTPServer(t, ServerOptions{})
|
|
||||||
data := RandomBytes(3*mib+12345, 4)
|
|
||||||
want := srv.WriteFile(t, "pub/plain.bin", data)
|
|
||||||
|
|
||||||
f := buildFetcher(testManager(srv))
|
|
||||||
resolveTask(t, f, ftpURL(srv, "ftp", "", "pub/plain.bin"), &pftp.OptsExtra{Connections: 4})
|
|
||||||
startAndWait(t, f)
|
|
||||||
checkFile(t, f.Meta().SingleFilepath(), want)
|
|
||||||
if got := f.Progress().TotalDownloaded(); got != int64(len(data)) {
|
|
||||||
t.Fatalf("progress %d, want %d", got, len(data))
|
|
||||||
}
|
|
||||||
stats := ftpStats(t, f)
|
|
||||||
if len(stats.Connections) == 0 || len(stats.Connections) > 4 {
|
|
||||||
t.Fatalf("got %d connections in stats", len(stats.Connections))
|
|
||||||
}
|
|
||||||
var sum int64
|
|
||||||
for _, c := range stats.Connections {
|
|
||||||
sum += c.Downloaded
|
|
||||||
if !c.Completed || c.Failed {
|
|
||||||
t.Fatalf("connection stats %+v after success", c)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if sum != int64(len(data)) {
|
|
||||||
t.Fatalf("connections downloaded %d bytes, want %d", sum, len(data))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestDownloadDir(t *testing.T) {
|
|
||||||
srv := StartFTPServer(t, ServerOptions{})
|
|
||||||
want := writeTree(t, srv)
|
|
||||||
|
|
||||||
f := buildFetcher(testManager(srv))
|
|
||||||
resolveTask(t, f, ftpURL(srv, "ftp", "", "pub/tree"), &pftp.OptsExtra{Connections: 2})
|
|
||||||
startAndWait(t, f)
|
|
||||||
root := f.Meta().FolderPath()
|
|
||||||
if root != filepath.Join(f.Meta().Opts.Path, "tree") {
|
|
||||||
t.Fatalf("folder path %q", root)
|
|
||||||
}
|
|
||||||
for rel, sum := range want {
|
|
||||||
checkFile(t, filepath.Join(root, filepath.FromSlash(rel)), sum)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func testDownloadTLS(t *testing.T, mode string, schemes map[string]string) {
|
|
||||||
srv := StartFTPServer(t, ServerOptions{TLS: mode})
|
|
||||||
data := RandomBytes(2*mib+999, 5)
|
|
||||||
want := srv.WriteFile(t, "secure.bin", data)
|
|
||||||
for scheme, reqTLS := range schemes {
|
|
||||||
t.Run(scheme+"+"+reqTLS, func(t *testing.T) {
|
|
||||||
f := buildFetcher(testManager(srv))
|
|
||||||
req := &base.Request{URL: ftpURL(srv, scheme, "", "secure.bin"), Extra: &pftp.ReqExtra{TLS: reqTLS}}
|
|
||||||
resolveRequest(t, f, req, &pftp.OptsExtra{Connections: 2})
|
|
||||||
startAndWait(t, f)
|
|
||||||
checkFile(t, f.Meta().SingleFilepath(), want)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestDownloadExplicitTLS(t *testing.T) {
|
|
||||||
testDownloadTLS(t, pftp.TLSExplicit, map[string]string{
|
|
||||||
"ftpes": "",
|
|
||||||
"ftp": pftp.TLSExplicit,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestDownloadImplicitTLS(t *testing.T) {
|
|
||||||
testDownloadTLS(t, pftp.TLSImplicit, map[string]string{
|
|
||||||
"ftps": "",
|
|
||||||
"ftp": pftp.TLSImplicit,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// The TLS mode of a plain ftp:// link lives on the request, so a task that is
|
|
||||||
// saved and restored from JSON still logs in over TLS.
|
|
||||||
func TestRestoredRequestKeepsTLSMode(t *testing.T) {
|
|
||||||
srv := StartFTPServer(t, ServerOptions{TLS: pftp.TLSExplicit})
|
|
||||||
want := srv.WriteFile(t, "keep.bin", RandomBytes(2*mib+5, 12))
|
|
||||||
fm := testManager(srv)
|
|
||||||
f := buildFetcher(fm)
|
|
||||||
req := &base.Request{URL: ftpURL(srv, "ftp", "", "keep.bin"), Extra: &pftp.ReqExtra{TLS: pftp.TLSExplicit}}
|
|
||||||
resolveRequest(t, f, req, &pftp.OptsExtra{Connections: 2})
|
|
||||||
|
|
||||||
restored := restoreFetcher(t, fm, f.Meta(), storeData(t, fm, f))
|
|
||||||
if _, ok := restored.Meta().Req.Extra.(*pftp.ReqExtra); !ok {
|
|
||||||
t.Fatalf("restored request extra is %T, want *ftp.ReqExtra", restored.Meta().Req.Extra)
|
|
||||||
}
|
|
||||||
// Explicit mode refuses plain logins, so the download only finishes over TLS.
|
|
||||||
startAndWait(t, restored)
|
|
||||||
checkFile(t, restored.Meta().SingleFilepath(), want)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestBadCertRejectedByDefault(t *testing.T) {
|
|
||||||
for mode, scheme := range map[string]string{pftp.TLSExplicit: "ftpes", pftp.TLSImplicit: "ftps"} {
|
|
||||||
t.Run(mode, func(t *testing.T) {
|
|
||||||
srv := StartFTPServer(t, ServerOptions{TLS: mode})
|
|
||||||
want := srv.WriteFile(t, "f.bin", RandomBytes(4096, 6))
|
|
||||||
rawURL := ftpURL(srv, scheme, "", "f.bin")
|
|
||||||
|
|
||||||
// No injected roots: the self-signed certificate is unknown.
|
|
||||||
f := buildFetcher(testManager(nil))
|
|
||||||
err := resolveOpts(f, &base.Request{URL: rawURL}, &base.Options{Path: t.TempDir()})
|
|
||||||
var verifyErr *tls.CertificateVerificationError
|
|
||||||
if !errors.As(err, &verifyErr) {
|
|
||||||
t.Fatalf("want a certificate verification error, got %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// The request's existing skipVerifyCert switch accepts it.
|
|
||||||
f = buildFetcher(testManager(nil))
|
|
||||||
if err := resolveOpts(f, &base.Request{URL: rawURL, SkipVerifyCert: true}, &base.Options{Path: t.TempDir()}); err != nil {
|
|
||||||
t.Fatalf("skipVerifyCert resolve: %v", err)
|
|
||||||
}
|
|
||||||
startAndWait(t, f)
|
|
||||||
checkFile(t, f.Meta().SingleFilepath(), want)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestSegmentsUseRest(t *testing.T) {
|
|
||||||
srv := StartFTPServer(t, ServerOptions{})
|
|
||||||
srv.ReadDelay.Store(int64(time.Millisecond))
|
|
||||||
size := 8 * mib
|
|
||||||
want := srv.WriteFile(t, "big.bin", RandomBytes(size, 7))
|
|
||||||
|
|
||||||
f := buildFetcher(testManager(srv))
|
|
||||||
resolveTask(t, f, ftpURL(srv, "ftp", "", "big.bin"), &pftp.OptsExtra{Connections: 4})
|
|
||||||
startAndWait(t, f)
|
|
||||||
checkFile(t, f.Meta().SingleFilepath(), want)
|
|
||||||
|
|
||||||
rests := srv.RestOffsets()
|
|
||||||
for _, off := range []int64{2 * mib, 4 * mib, 6 * mib} {
|
|
||||||
if !slices.Contains(rests, off) {
|
|
||||||
t.Fatalf("REST offsets %v miss the segment start %d", rests, off)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for _, off := range rests {
|
|
||||||
if off <= 0 || off >= int64(size) {
|
|
||||||
t.Fatalf("REST offset %d out of range", off)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if got := len(ftpStats(t, f).Connections); got != 4 {
|
|
||||||
t.Fatalf("stats report %d connections, want 4", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestLoginCapFallsBack(t *testing.T) {
|
|
||||||
srv := StartFTPServer(t, ServerOptions{MaxLogins: 1})
|
|
||||||
want := srv.WriteFile(t, "capped.bin", RandomBytes(4*mib+5, 8))
|
|
||||||
|
|
||||||
fm := testManager(srv)
|
|
||||||
f := buildFetcher(fm)
|
|
||||||
resolveTask(t, f, ftpURL(srv, "ftp", "", "capped.bin"), &pftp.OptsExtra{Connections: 4})
|
|
||||||
startAndWait(t, f)
|
|
||||||
checkFile(t, f.Meta().SingleFilepath(), want)
|
|
||||||
|
|
||||||
if srv.Refused() == 0 {
|
|
||||||
t.Fatal("the server never refused a login, so the cap was not exercised")
|
|
||||||
}
|
|
||||||
if got := len(ftpStats(t, f).Connections); got != 1 {
|
|
||||||
t.Fatalf("stats report %d connections, want 1", got)
|
|
||||||
}
|
|
||||||
var saved fetcherData
|
|
||||||
if err := json.Unmarshal(storeData(t, fm, f), &saved); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if saved.MaxConns != 1 {
|
|
||||||
t.Fatalf("saved login ceiling %d, want 1", saved.MaxConns)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestNoRestSequential(t *testing.T) {
|
|
||||||
srv := StartFTPServer(t, ServerOptions{NoRest: true})
|
|
||||||
srv.ReadDelay.Store(int64(time.Millisecond))
|
|
||||||
want := srv.WriteFile(t, "norest.bin", RandomBytes(4*mib+3, 9))
|
|
||||||
|
|
||||||
fm := testManager(srv)
|
|
||||||
f := buildFetcher(fm)
|
|
||||||
resolveTask(t, f, ftpURL(srv, "ftp", "", "norest.bin"), &pftp.OptsExtra{Connections: 4})
|
|
||||||
startAndWait(t, f)
|
|
||||||
checkFile(t, f.Meta().SingleFilepath(), want)
|
|
||||||
|
|
||||||
if rests := srv.RestOffsets(); len(rests) != 0 {
|
|
||||||
t.Fatalf("the server cannot have seen a REST, got %v", rests)
|
|
||||||
}
|
|
||||||
var saved fetcherData
|
|
||||||
if err := json.Unmarshal(storeData(t, fm, f), &saved); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if !saved.NoRest {
|
|
||||||
t.Fatal("the missing REST support was not recorded")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestResumeAfterKill stops the server's transfers at a fixed byte count,
|
|
||||||
// waits until the client has written exactly those bytes, then pauses, saves,
|
|
||||||
// restores and finishes. No timing decides where the pause lands.
|
|
||||||
func TestResumeAfterKill(t *testing.T) {
|
|
||||||
srv := StartFTPServer(t, ServerOptions{})
|
|
||||||
size := int64(4*mib + 777)
|
|
||||||
want := srv.WriteFile(t, "resume.bin", RandomBytes(int(size), 10))
|
|
||||||
stallAt := size / 2
|
|
||||||
srv.StallAfter(stallAt)
|
|
||||||
|
|
||||||
fm := testManager(srv)
|
|
||||||
f := buildFetcher(fm)
|
|
||||||
resolveTask(t, f, ftpURL(srv, "ftp", "", "resume.bin"), &pftp.OptsExtra{Connections: 2})
|
|
||||||
if err := f.Start(); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
select {
|
|
||||||
case <-srv.Stalled():
|
|
||||||
case <-time.After(30 * time.Second):
|
|
||||||
t.Fatal("the server never reached the stall point")
|
|
||||||
}
|
|
||||||
deadline := time.Now().Add(30 * time.Second)
|
|
||||||
for f.Progress().TotalDownloaded() != stallAt {
|
|
||||||
if time.Now().After(deadline) {
|
|
||||||
t.Fatalf("progress %d, want the %d bytes the server sent", f.Progress().TotalDownloaded(), stallAt)
|
|
||||||
}
|
|
||||||
time.Sleep(5 * time.Millisecond)
|
|
||||||
}
|
|
||||||
if err := f.Pause(); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
srv.Release()
|
|
||||||
blob := storeData(t, fm, f)
|
|
||||||
|
|
||||||
var saved fetcherData
|
|
||||||
if err := json.Unmarshal(blob, &saved); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
var savedTotal int64
|
|
||||||
resumeAt := map[int64]bool{}
|
|
||||||
for _, seg := range saved.Segments {
|
|
||||||
savedTotal += seg.Downloaded
|
|
||||||
if seg.Begin+seg.Downloaded <= seg.End {
|
|
||||||
resumeAt[seg.Begin+seg.Downloaded] = true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if savedTotal != stallAt {
|
|
||||||
t.Fatalf("saved progress %d, want %d", savedTotal, stallAt)
|
|
||||||
}
|
|
||||||
|
|
||||||
before := len(srv.RetrOffsets())
|
|
||||||
restored := restoreFetcher(t, fm, f.Meta(), blob)
|
|
||||||
if got := restored.Progress().TotalDownloaded(); got != savedTotal {
|
|
||||||
t.Fatalf("restored progress %d, want %d", got, savedTotal)
|
|
||||||
}
|
|
||||||
startAndWait(t, restored)
|
|
||||||
checkFile(t, restored.Meta().SingleFilepath(), want)
|
|
||||||
|
|
||||||
resumed := srv.RetrOffsets()[before:]
|
|
||||||
if len(resumed) == 0 {
|
|
||||||
t.Fatal("no transfer after resume")
|
|
||||||
}
|
|
||||||
for _, off := range resumed {
|
|
||||||
if !resumeAt[off] {
|
|
||||||
t.Fatalf("resumed RETR at %d, want one of the saved offsets %v", off, resumeAt)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestLateRefusalAfterQueueDrained: worker B holds the second file's segment
|
|
||||||
// while the server takes its time to refuse B's login. Worker A finishes the
|
|
||||||
// first file meanwhile and finds the queue empty. B must not leave too, or
|
|
||||||
// nobody fetches the second file.
|
|
||||||
func TestLateRefusalAfterQueueDrained(t *testing.T) {
|
|
||||||
srv := StartFTPServer(t, ServerOptions{MaxLogins: 1, RefuseDelay: 500 * time.Millisecond})
|
|
||||||
want := map[string]string{
|
|
||||||
"a.bin": srv.WriteFile(t, "dir/a.bin", RandomBytes(1000, 16)),
|
|
||||||
"b.bin": srv.WriteFile(t, "dir/b.bin", RandomBytes(1000, 17)),
|
|
||||||
}
|
|
||||||
|
|
||||||
f := buildFetcher(testManager(srv))
|
|
||||||
resolveTask(t, f, ftpURL(srv, "ftp", "", "dir"), &pftp.OptsExtra{Connections: 2})
|
|
||||||
startAndWait(t, f)
|
|
||||||
for name, sum := range want {
|
|
||||||
checkFile(t, filepath.Join(f.Meta().FolderPath(), name), sum)
|
|
||||||
}
|
|
||||||
if srv.Refused() == 0 {
|
|
||||||
t.Fatal("the server never refused a login, so the late refusal was not exercised")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestLoginCapLoweredMidway: both workers have logged in when the server's
|
|
||||||
// cap drops to one, as when another client takes a slot. The refused worker
|
|
||||||
// leaves, the ceiling drops to one, and the task finishes.
|
|
||||||
func TestLoginCapLoweredMidway(t *testing.T) {
|
|
||||||
srv := StartFTPServer(t, ServerOptions{})
|
|
||||||
srv.ReadDelay.Store(int64(time.Millisecond))
|
|
||||||
want := map[string]string{}
|
|
||||||
for i := 0; i < 8; i++ {
|
|
||||||
name := fmt.Sprintf("f%d.bin", i)
|
|
||||||
want[name] = srv.WriteFile(t, "dir/"+name, RandomBytes(512<<10, byte(30+i)))
|
|
||||||
}
|
|
||||||
|
|
||||||
fm := testManager(srv)
|
|
||||||
f := buildFetcher(fm)
|
|
||||||
resolveTask(t, f, ftpURL(srv, "ftp", "", "dir"), &pftp.OptsExtra{Connections: 2})
|
|
||||||
if err := f.Start(); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
deadline := time.Now().Add(30 * time.Second)
|
|
||||||
for len(ftpStats(t, f).Connections) < 2 || f.Progress().TotalDownloaded() < 512<<10 {
|
|
||||||
if time.Now().After(deadline) {
|
|
||||||
t.Fatal("the two workers never got going")
|
|
||||||
}
|
|
||||||
time.Sleep(2 * time.Millisecond)
|
|
||||||
}
|
|
||||||
srv.SetMaxLogins(1)
|
|
||||||
if err := waitFetcher(t, f); err != nil {
|
|
||||||
t.Fatalf("download: %v", err)
|
|
||||||
}
|
|
||||||
for name, sum := range want {
|
|
||||||
checkFile(t, filepath.Join(f.Meta().FolderPath(), name), sum)
|
|
||||||
}
|
|
||||||
if srv.Refused() == 0 {
|
|
||||||
t.Fatal("the server never refused a login, so the lowered cap was not exercised")
|
|
||||||
}
|
|
||||||
for _, c := range ftpStats(t, f).Connections {
|
|
||||||
if c.Failed {
|
|
||||||
t.Fatalf("a refused worker failed: %+v", c)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
var saved fetcherData
|
|
||||||
if err := json.Unmarshal(storeData(t, fm, f), &saved); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if saved.MaxConns != 1 {
|
|
||||||
t.Fatalf("saved login ceiling %d, want 1", saved.MaxConns)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestDataConnectionsResumeOwnSession runs four FTPS workers against a server
|
|
||||||
// that, like ProFTPD, accepts a data connection only when it resumes the TLS
|
|
||||||
// session of its own control connection.
|
|
||||||
func TestDataConnectionsResumeOwnSession(t *testing.T) {
|
|
||||||
srv := StartFTPServer(t, ServerOptions{TLS: pftp.TLSImplicit, RequireSessionReuse: true})
|
|
||||||
srv.ReadDelay.Store(int64(200 * time.Microsecond))
|
|
||||||
want := srv.WriteFile(t, "reuse.bin", RandomBytes(8*mib, 18))
|
|
||||||
|
|
||||||
f := buildFetcher(testManager(srv))
|
|
||||||
resolveTask(t, f, ftpURL(srv, "ftps", "", "reuse.bin"), &pftp.OptsExtra{Connections: 4})
|
|
||||||
startAndWait(t, f)
|
|
||||||
checkFile(t, f.Meta().SingleFilepath(), want)
|
|
||||||
ok, bad := srv.SessionReuse()
|
|
||||||
if ok < 4 || bad != 0 {
|
|
||||||
t.Fatalf("data connections: %d resumed their own session, %d did not", ok, bad)
|
|
||||||
}
|
|
||||||
for _, c := range ftpStats(t, f).Connections {
|
|
||||||
if c.RetryTimes != 0 {
|
|
||||||
t.Fatalf("a worker needed retries: %+v", c)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRetryAfterDroppedConnection(t *testing.T) {
|
|
||||||
srv := StartFTPServer(t, ServerOptions{})
|
|
||||||
want := srv.WriteFile(t, "drop.bin", RandomBytes(3*mib, 11))
|
|
||||||
srv.DropAfter.Store(mib + 100)
|
|
||||||
|
|
||||||
f := buildFetcher(testManager(srv))
|
|
||||||
resolveTask(t, f, ftpURL(srv, "ftp", "", "drop.bin"), &pftp.OptsExtra{Connections: 1})
|
|
||||||
startAndWait(t, f)
|
|
||||||
checkFile(t, f.Meta().SingleFilepath(), want)
|
|
||||||
|
|
||||||
offsets := srv.RetrOffsets()
|
|
||||||
if len(offsets) != 2 || offsets[0] != 0 || offsets[1] <= 0 || offsets[1] > mib+100 {
|
|
||||||
t.Fatalf("RETR offsets %v, want a restart at the dropped offset", offsets)
|
|
||||||
}
|
|
||||||
if got := ftpStats(t, f).Connections[0].RetryTimes; got != 1 {
|
|
||||||
t.Fatalf("retry count %d, want 1", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestPercentEncodedPassword(t *testing.T) {
|
|
||||||
srv := StartFTPServer(t, ServerOptions{User: "u", Pass: "p@ss"})
|
|
||||||
want := srv.WriteFile(t, "secret.bin", RandomBytes(5000, 12))
|
|
||||||
|
|
||||||
f := buildFetcher(testManager(srv))
|
|
||||||
resolveTask(t, f, ftpURL(srv, "ftp", "u:p%40ss", "secret.bin"), nil)
|
|
||||||
startAndWait(t, f)
|
|
||||||
checkFile(t, f.Meta().SingleFilepath(), want)
|
|
||||||
|
|
||||||
f = buildFetcher(testManager(srv))
|
|
||||||
err := resolveOpts(f, &base.Request{URL: ftpURL(srv, "ftp", "u:wrong", "secret.bin")}, &base.Options{Path: t.TempDir()})
|
|
||||||
if err == nil || !strings.Contains(err.Error(), "530") {
|
|
||||||
t.Fatalf("a wrong password must fail with 530, got %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestOldSaveFormatResumes(t *testing.T) {
|
|
||||||
srv := StartFTPServer(t, ServerOptions{})
|
|
||||||
data := RandomBytes(2*mib+1, 13)
|
|
||||||
want := srv.WriteFile(t, "old.bin", data)
|
|
||||||
|
|
||||||
fm := testManager(srv)
|
|
||||||
f := buildFetcher(fm)
|
|
||||||
resolveTask(t, f, ftpURL(srv, "ftp", "", "old.bin"), &pftp.OptsExtra{Connections: 1})
|
|
||||||
|
|
||||||
// A save blob that has only the segment fields every version writes.
|
|
||||||
const done = 700000
|
|
||||||
blob := []byte(fmt.Sprintf(`{"Segments":[{"Begin":0,"End":%d,"Downloaded":%d}]}`, len(data)-1, done))
|
|
||||||
name := f.Meta().SingleFilepath()
|
|
||||||
if err := os.WriteFile(name, append(data[:done:done], make([]byte, len(data)-done)...), 0o644); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
restored := restoreFetcher(t, fm, f.Meta(), blob)
|
|
||||||
if got := restored.Progress().TotalDownloaded(); got != done {
|
|
||||||
t.Fatalf("restored progress %d, want %d", got, done)
|
|
||||||
}
|
|
||||||
startAndWait(t, restored)
|
|
||||||
checkFile(t, name, want)
|
|
||||||
if got := srv.RetrOffsets(); !slices.Equal(got, []int64{done}) {
|
|
||||||
t.Fatalf("RETR offsets %v, want [%d]", got, done)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestDownloadThroughSOCKS5(t *testing.T) {
|
|
||||||
srv := StartFTPServer(t, ServerOptions{TLS: pftp.TLSExplicit})
|
|
||||||
want := srv.WriteFile(t, "proxied.bin", RandomBytes(2*mib+1, 14))
|
|
||||||
|
|
||||||
var dials atomic.Int32
|
|
||||||
proxySrv, err := socks5.New(&socks5.Config{
|
|
||||||
Dial: func(ctx context.Context, network, addr string) (net.Conn, error) {
|
|
||||||
dials.Add(1)
|
|
||||||
var d net.Dialer
|
|
||||||
return d.DialContext(ctx, network, addr)
|
|
||||||
},
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
ln, err := net.Listen("tcp", "127.0.0.1:0")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
defer ln.Close()
|
|
||||||
go proxySrv.Serve(ln)
|
|
||||||
|
|
||||||
f := buildFetcher(testManager(srv))
|
|
||||||
req := &base.Request{
|
|
||||||
URL: ftpURL(srv, "ftpes", "", "proxied.bin"),
|
|
||||||
Proxy: &base.RequestProxy{
|
|
||||||
Mode: base.RequestProxyModeCustom,
|
|
||||||
Scheme: "socks5",
|
|
||||||
Host: ln.Addr().String(),
|
|
||||||
},
|
|
||||||
}
|
|
||||||
if err := resolveOpts(f, req, &base.Options{Path: t.TempDir(), Extra: &pftp.OptsExtra{Connections: 2}}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
startAndWait(t, f)
|
|
||||||
checkFile(t, f.Meta().SingleFilepath(), want)
|
|
||||||
// Resolve's control and listing, then per worker one control and one
|
|
||||||
// data connection at least: every one of them went through the proxy.
|
|
||||||
if got := dials.Load(); got < 4 {
|
|
||||||
t.Fatalf("the proxy saw %d connections, want every control and data connection", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestHTTPProxyFallsBackToDirect(t *testing.T) {
|
|
||||||
srv := StartFTPServer(t, ServerOptions{})
|
|
||||||
want := srv.WriteFile(t, "direct.bin", RandomBytes(100000, 15))
|
|
||||||
|
|
||||||
var logs bytes.Buffer
|
|
||||||
logger := zerolog.New(&logs)
|
|
||||||
fm := testManager(srv)
|
|
||||||
f := fm.Build().(*Fetcher)
|
|
||||||
ctl := testController(fm)
|
|
||||||
ctl.Logger = &logger
|
|
||||||
f.Setup(ctl)
|
|
||||||
req := &base.Request{
|
|
||||||
URL: ftpURL(srv, "ftp", "", "direct.bin"),
|
|
||||||
Proxy: &base.RequestProxy{
|
|
||||||
Mode: base.RequestProxyModeCustom,
|
|
||||||
Scheme: "http",
|
|
||||||
Host: "127.0.0.1:1",
|
|
||||||
Usr: "proxyuser",
|
|
||||||
Pwd: "proxysecret",
|
|
||||||
},
|
|
||||||
}
|
|
||||||
if err := resolveOpts(f, req, &base.Options{Path: t.TempDir(), Extra: &pftp.OptsExtra{Connections: 2}}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
startAndWait(t, f)
|
|
||||||
checkFile(t, f.Meta().SingleFilepath(), want)
|
|
||||||
|
|
||||||
out := logs.String()
|
|
||||||
if n := strings.Count(out, "HTTP proxy"); n != 1 {
|
|
||||||
t.Fatalf("want exactly one HTTP proxy warning, got %d in %q", n, out)
|
|
||||||
}
|
|
||||||
if strings.Contains(out, "proxysecret") {
|
|
||||||
t.Fatalf("the proxy password leaked into the log: %q", out)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestTorrentNameIsFile(t *testing.T) {
|
|
||||||
fm := testManager(nil)
|
|
||||||
for _, u := range []string{"ftp://h/x.torrent", "ftps://h/x.torrent", "ftpes://h/x.torrent", "FTP://h/v.m3u8"} {
|
|
||||||
matched := false
|
|
||||||
for _, filter := range fm.Filters() {
|
|
||||||
if filter.Match(u) {
|
|
||||||
matched = true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if !matched {
|
|
||||||
t.Fatalf("%s is not matched by the FTP filters", u)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if fm.Name() != "ftp" {
|
|
||||||
t.Fatalf("manager name %q", fm.Name())
|
|
||||||
}
|
|
||||||
|
|
||||||
srv := StartFTPServer(t, ServerOptions{})
|
|
||||||
want := srv.WriteFile(t, "x.torrent", []byte("d8:announce0:e"))
|
|
||||||
f := buildFetcher(testManager(srv))
|
|
||||||
resolveTask(t, f, ftpURL(srv, "ftp", "", "x.torrent"), nil)
|
|
||||||
res := f.Meta().Res
|
|
||||||
if res.Name != "" || len(res.Files) != 1 || res.Files[0].Name != "x.torrent" {
|
|
||||||
t.Fatalf("x.torrent must resolve to one FTP file, got name %q and %d files", res.Name, len(res.Files))
|
|
||||||
}
|
|
||||||
startAndWait(t, f)
|
|
||||||
checkFile(t, f.Meta().SingleFilepath(), want)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestParseNameHidesCredentials(t *testing.T) {
|
|
||||||
fm := testManager(nil)
|
|
||||||
for u, want := range map[string]string{
|
|
||||||
"ftp://u:p%[email protected]/pub/a.iso": "a.iso",
|
|
||||||
"ftp://u:p%[email protected]/": "example.com",
|
|
||||||
"ftps://u:p%[email protected]": "example.com",
|
|
||||||
"ftp://u:p%[email protected]/pub/dir/": "dir",
|
|
||||||
} {
|
|
||||||
if got := fm.ParseName(u); got != want {
|
|
||||||
t.Fatalf("ParseName(%q) = %q, want %q", u, got, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestUnsafeEntryNames(t *testing.T) {
|
|
||||||
for name, ok := range map[string]bool{
|
|
||||||
"a.txt": true, "..a": true, "a b": true,
|
|
||||||
"": false, ".": false, "..": false, "a/b": false, `a\b`: false, "../x": false, "x\x00y": false,
|
|
||||||
} {
|
|
||||||
if got := safeEntryName(name); got != ok {
|
|
||||||
t.Fatalf("safeEntryName(%q) = %v, want %v", name, got, ok)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,587 +0,0 @@
|
|||||||
package ftp
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bufio"
|
|
||||||
"crypto/ecdsa"
|
|
||||||
"crypto/elliptic"
|
|
||||||
"crypto/rand"
|
|
||||||
"crypto/sha256"
|
|
||||||
"crypto/tls"
|
|
||||||
"crypto/x509"
|
|
||||||
"crypto/x509/pkix"
|
|
||||||
"encoding/hex"
|
|
||||||
"errors"
|
|
||||||
"io"
|
|
||||||
"math/big"
|
|
||||||
"net"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"regexp"
|
|
||||||
"strconv"
|
|
||||||
"strings"
|
|
||||||
"sync"
|
|
||||||
"sync/atomic"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
ftpserver "github.com/fclairamb/ftpserverlib"
|
|
||||||
"github.com/spf13/afero"
|
|
||||||
|
|
||||||
pftp "github.com/GopeedLab/gopeed/pkg/protocol/ftp"
|
|
||||||
)
|
|
||||||
|
|
||||||
// ServerOptions are the knobs of the in-process FTP test server.
|
|
||||||
type ServerOptions struct {
|
|
||||||
// TLS is pftp.TLSNone (or empty), pftp.TLSExplicit or pftp.TLSImplicit.
|
|
||||||
// Explicit mode refuses plain logins, so a passing test proves TLS was used.
|
|
||||||
TLS string
|
|
||||||
// MaxLogins caps concurrent control connections. Beyond it the server
|
|
||||||
// greets with "421 Too many connections" and hangs up. Zero means no cap.
|
|
||||||
MaxLogins int
|
|
||||||
// NoRest answers "502" to REST. It filters the plain control stream, so it
|
|
||||||
// works with TLSNone and TLSImplicit only.
|
|
||||||
NoRest bool
|
|
||||||
// NoMLST disables MLST and MLSD, so clients fall back to SIZE and LIST.
|
|
||||||
NoMLST bool
|
|
||||||
// User and Pass are the only accepted credentials. An empty User accepts
|
|
||||||
// any login, anonymous included.
|
|
||||||
User string
|
|
||||||
Pass string
|
|
||||||
// RefuseDelay holds every 421 refusal back this long, like a server
|
|
||||||
// that answers slowly when it is full.
|
|
||||||
RefuseDelay time.Duration
|
|
||||||
// RequireSessionReuse makes every data connection resume the TLS
|
|
||||||
// session of the control connection that opened it, as ProFTPD's
|
|
||||||
// mod_tls does by default. A data connection that does not is refused.
|
|
||||||
// It needs TLSImplicit, where this test's wrapper sees the plain
|
|
||||||
// control stream and so learns which passive port belongs to whom.
|
|
||||||
RequireSessionReuse bool
|
|
||||||
}
|
|
||||||
|
|
||||||
// FTPServer is an ftpserverlib server over a temp directory.
|
|
||||||
type FTPServer struct {
|
|
||||||
Addr string // host:port
|
|
||||||
Root string // directory served as "/"
|
|
||||||
// CA trusts the server's self-signed certificate.
|
|
||||||
CA *x509.CertPool
|
|
||||||
|
|
||||||
opts ServerOptions
|
|
||||||
tlsConf *tls.Config
|
|
||||||
server *ftpserver.FtpServer
|
|
||||||
|
|
||||||
// ReadDelay is slept before every read of a transferred file, in
|
|
||||||
// nanoseconds, to slow downloads down. Tests may change it at any time.
|
|
||||||
ReadDelay atomic.Int64
|
|
||||||
|
|
||||||
// DropAfter, when positive, makes the next transfer fail once after it
|
|
||||||
// has sent that many bytes, as if the data connection were killed.
|
|
||||||
DropAfter atomic.Int64
|
|
||||||
|
|
||||||
mu sync.Mutex
|
|
||||||
active int
|
|
||||||
retrs []int64 // offset of every RETR, 0 when no REST came first
|
|
||||||
refused int
|
|
||||||
logins int
|
|
||||||
|
|
||||||
// Passive port to the control connection that announced it, and the
|
|
||||||
// outcome of every data connection's session check.
|
|
||||||
pasvOwner map[int]string
|
|
||||||
reuseOK int
|
|
||||||
reuseBad int
|
|
||||||
|
|
||||||
// Stall state, see StallAfter. stallMu is held across file reads.
|
|
||||||
stallMu sync.Mutex
|
|
||||||
stallAt int64
|
|
||||||
sent int64
|
|
||||||
stallCh chan struct{}
|
|
||||||
stalled chan struct{}
|
|
||||||
stalledOnce *sync.Once
|
|
||||||
}
|
|
||||||
|
|
||||||
// StartFTPServer starts a server and stops it when the test ends.
|
|
||||||
func StartFTPServer(t *testing.T, opts ServerOptions) *FTPServer {
|
|
||||||
t.Helper()
|
|
||||||
s := &FTPServer{Root: t.TempDir(), opts: opts}
|
|
||||||
s.tlsConf, s.CA = selfSignedTLS(t)
|
|
||||||
|
|
||||||
base, err := net.Listen("tcp", "127.0.0.1:0")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
var ln net.Listener = base
|
|
||||||
if opts.RequireSessionReuse {
|
|
||||||
if opts.TLS != pftp.TLSImplicit {
|
|
||||||
t.Fatal("RequireSessionReuse needs TLSImplicit")
|
|
||||||
}
|
|
||||||
s.requireSessionReuse(t, base.Addr().(*net.TCPAddr).Port)
|
|
||||||
}
|
|
||||||
if opts.TLS == pftp.TLSImplicit {
|
|
||||||
ln = tls.NewListener(ln, s.tlsConf)
|
|
||||||
}
|
|
||||||
ln = &limitListener{Listener: ln, srv: s}
|
|
||||||
|
|
||||||
s.Addr = base.Addr().String()
|
|
||||||
s.server = ftpserver.NewFtpServer(&mainDriver{srv: s, listener: ln})
|
|
||||||
if err := s.server.Listen(); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
go s.server.Serve()
|
|
||||||
t.Cleanup(func() { _ = s.server.Stop() })
|
|
||||||
return s
|
|
||||||
}
|
|
||||||
|
|
||||||
// WriteFile creates a file under the served root and returns its SHA-256.
|
|
||||||
func (s *FTPServer) WriteFile(t *testing.T, rel string, data []byte) string {
|
|
||||||
t.Helper()
|
|
||||||
name := filepath.Join(s.Root, filepath.FromSlash(rel))
|
|
||||||
if err := os.MkdirAll(filepath.Dir(name), 0o755); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := os.WriteFile(name, data, 0o644); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
return SHA256Hex(data)
|
|
||||||
}
|
|
||||||
|
|
||||||
// RetrOffsets returns the start offset of every RETR the server received.
|
|
||||||
func (s *FTPServer) RetrOffsets() []int64 {
|
|
||||||
s.mu.Lock()
|
|
||||||
defer s.mu.Unlock()
|
|
||||||
return append([]int64(nil), s.retrs...)
|
|
||||||
}
|
|
||||||
|
|
||||||
// RestOffsets returns the non-zero RETR offsets, which only a REST sets.
|
|
||||||
func (s *FTPServer) RestOffsets() []int64 {
|
|
||||||
var out []int64
|
|
||||||
for _, off := range s.RetrOffsets() {
|
|
||||||
if off != 0 {
|
|
||||||
out = append(out, off)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// Refused returns how many connections were turned away with 421.
|
|
||||||
func (s *FTPServer) Refused() int {
|
|
||||||
s.mu.Lock()
|
|
||||||
defer s.mu.Unlock()
|
|
||||||
return s.refused
|
|
||||||
}
|
|
||||||
|
|
||||||
// Logins returns the number of successful logins.
|
|
||||||
func (s *FTPServer) Logins() int {
|
|
||||||
s.mu.Lock()
|
|
||||||
defer s.mu.Unlock()
|
|
||||||
return s.logins
|
|
||||||
}
|
|
||||||
|
|
||||||
// SetMaxLogins changes the login cap while the server runs, as when other
|
|
||||||
// clients take slots.
|
|
||||||
func (s *FTPServer) SetMaxLogins(n int) {
|
|
||||||
s.mu.Lock()
|
|
||||||
s.opts.MaxLogins = n
|
|
||||||
s.mu.Unlock()
|
|
||||||
}
|
|
||||||
|
|
||||||
// SessionReuse returns how many data connections resumed their own control
|
|
||||||
// connection's TLS session, and how many did not.
|
|
||||||
func (s *FTPServer) SessionReuse() (ok, bad int) {
|
|
||||||
s.mu.Lock()
|
|
||||||
defer s.mu.Unlock()
|
|
||||||
return s.reuseOK, s.reuseBad
|
|
||||||
}
|
|
||||||
|
|
||||||
// StallAfter makes every transfer stop sending once the server has sent n
|
|
||||||
// bytes in total, until Release. Stalled is closed when that happens.
|
|
||||||
func (s *FTPServer) StallAfter(n int64) {
|
|
||||||
s.stallMu.Lock()
|
|
||||||
defer s.stallMu.Unlock()
|
|
||||||
s.stallAt, s.sent = n, 0
|
|
||||||
s.stallCh = make(chan struct{})
|
|
||||||
s.stalled = make(chan struct{})
|
|
||||||
s.stalledOnce = &sync.Once{}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Stalled is closed once a transfer has stopped at the StallAfter limit.
|
|
||||||
func (s *FTPServer) Stalled() <-chan struct{} {
|
|
||||||
s.stallMu.Lock()
|
|
||||||
defer s.stallMu.Unlock()
|
|
||||||
return s.stalled
|
|
||||||
}
|
|
||||||
|
|
||||||
// Sent returns the bytes sent since StallAfter.
|
|
||||||
func (s *FTPServer) Sent() int64 {
|
|
||||||
s.stallMu.Lock()
|
|
||||||
defer s.stallMu.Unlock()
|
|
||||||
return s.sent
|
|
||||||
}
|
|
||||||
|
|
||||||
// Release ends the stall. The stalled transfers fail.
|
|
||||||
func (s *FTPServer) Release() {
|
|
||||||
s.stallMu.Lock()
|
|
||||||
defer s.stallMu.Unlock()
|
|
||||||
if s.stallAt > 0 {
|
|
||||||
s.stallAt = 0
|
|
||||||
close(s.stallCh)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
var errStalled = errors.New("test server released a stalled transfer")
|
|
||||||
|
|
||||||
// stall limits a read to the StallAfter budget, and blocks once the budget
|
|
||||||
// is spent.
|
|
||||||
func (s *FTPServer) stall(read func([]byte) (int, error), p []byte) (int, error) {
|
|
||||||
s.stallMu.Lock()
|
|
||||||
if s.stallAt <= 0 {
|
|
||||||
s.stallMu.Unlock()
|
|
||||||
return read(p)
|
|
||||||
}
|
|
||||||
allowed := s.stallAt - s.sent
|
|
||||||
if allowed <= 0 {
|
|
||||||
ch := s.stallCh
|
|
||||||
s.stalledOnce.Do(func() { close(s.stalled) })
|
|
||||||
s.stallMu.Unlock()
|
|
||||||
<-ch
|
|
||||||
return 0, errStalled
|
|
||||||
}
|
|
||||||
if int64(len(p)) > allowed {
|
|
||||||
p = p[:allowed]
|
|
||||||
}
|
|
||||||
n, err := read(p)
|
|
||||||
s.sent += int64(n)
|
|
||||||
s.stallMu.Unlock()
|
|
||||||
return n, err
|
|
||||||
}
|
|
||||||
|
|
||||||
var pasvReply = regexp.MustCompile(`^(?:229 [^(]*\(\|\|\|(\d+)\|\)|227 [^(]*\(\d+,\d+,\d+,\d+,(\d+),(\d+)\))`)
|
|
||||||
|
|
||||||
// notePassive records the passive port a control connection announced.
|
|
||||||
func (s *FTPServer) notePassive(owner string, reply []byte) {
|
|
||||||
m := pasvReply.FindSubmatch(reply)
|
|
||||||
if m == nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
var port int
|
|
||||||
if len(m[1]) > 0 {
|
|
||||||
port, _ = strconv.Atoi(string(m[1]))
|
|
||||||
} else {
|
|
||||||
hi, _ := strconv.Atoi(string(m[2]))
|
|
||||||
lo, _ := strconv.Atoi(string(m[3]))
|
|
||||||
port = hi<<8 | lo
|
|
||||||
}
|
|
||||||
s.mu.Lock()
|
|
||||||
s.pasvOwner[port] = owner
|
|
||||||
s.mu.Unlock()
|
|
||||||
}
|
|
||||||
|
|
||||||
// requireSessionReuse stamps each control connection's session tickets with
|
|
||||||
// its address, and checks that each data connection resumed a ticket of the
|
|
||||||
// control connection that announced its passive port.
|
|
||||||
func (s *FTPServer) requireSessionReuse(t *testing.T, controlPort int) {
|
|
||||||
base := s.tlsConf
|
|
||||||
var key [32]byte
|
|
||||||
if _, err := rand.Read(key[:]); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
base.SetSessionTicketKeys([][32]byte{key})
|
|
||||||
s.pasvOwner = map[int]string{}
|
|
||||||
stamp := func(owner string) func(tls.ConnectionState, *tls.SessionState) ([]byte, error) {
|
|
||||||
return func(cs tls.ConnectionState, ss *tls.SessionState) ([]byte, error) {
|
|
||||||
ss.Extra = [][]byte{[]byte(owner)}
|
|
||||||
return base.EncryptTicket(cs, ss)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
base.GetConfigForClient = func(hello *tls.ClientHelloInfo) (*tls.Config, error) {
|
|
||||||
cfg := base.Clone()
|
|
||||||
cfg.GetConfigForClient = nil
|
|
||||||
local, _ := hello.Conn.LocalAddr().(*net.TCPAddr)
|
|
||||||
if local == nil || local.Port == controlPort {
|
|
||||||
cfg.WrapSession = stamp(hello.Conn.RemoteAddr().String())
|
|
||||||
return cfg, nil
|
|
||||||
}
|
|
||||||
s.mu.Lock()
|
|
||||||
owner := s.pasvOwner[local.Port]
|
|
||||||
s.mu.Unlock()
|
|
||||||
var resumedFrom string
|
|
||||||
cfg.UnwrapSession = func(identity []byte, cs tls.ConnectionState) (*tls.SessionState, error) {
|
|
||||||
ss, err := base.DecryptTicket(identity, cs)
|
|
||||||
if err != nil || ss == nil {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
if len(ss.Extra) > 0 {
|
|
||||||
resumedFrom = string(ss.Extra[0])
|
|
||||||
}
|
|
||||||
return ss, nil
|
|
||||||
}
|
|
||||||
cfg.WrapSession = stamp(owner)
|
|
||||||
cfg.VerifyConnection = func(cs tls.ConnectionState) error {
|
|
||||||
ok := owner != "" && cs.DidResume && resumedFrom == owner
|
|
||||||
s.mu.Lock()
|
|
||||||
if ok {
|
|
||||||
s.reuseOK++
|
|
||||||
} else {
|
|
||||||
s.reuseBad++
|
|
||||||
}
|
|
||||||
s.mu.Unlock()
|
|
||||||
if !ok {
|
|
||||||
return errors.New("data connection did not resume its control connection's TLS session")
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
return cfg, nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *FTPServer) acquire() bool {
|
|
||||||
s.mu.Lock()
|
|
||||||
defer s.mu.Unlock()
|
|
||||||
if s.opts.MaxLogins > 0 && s.active >= s.opts.MaxLogins {
|
|
||||||
s.refused++
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
s.active++
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *FTPServer) release() {
|
|
||||||
s.mu.Lock()
|
|
||||||
s.active--
|
|
||||||
s.mu.Unlock()
|
|
||||||
}
|
|
||||||
|
|
||||||
// limitListener counts control connections and refuses those beyond
|
|
||||||
// MaxLogins with a 421 greeting, like vsftpd's max_clients and max_per_ip.
|
|
||||||
type limitListener struct {
|
|
||||||
net.Listener
|
|
||||||
srv *FTPServer
|
|
||||||
}
|
|
||||||
|
|
||||||
func (l *limitListener) Accept() (net.Conn, error) {
|
|
||||||
for {
|
|
||||||
c, err := l.Listener.Accept()
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
if !l.srv.acquire() {
|
|
||||||
go func() {
|
|
||||||
time.Sleep(l.srv.opts.RefuseDelay)
|
|
||||||
_ = c.SetDeadline(time.Now().Add(5 * time.Second))
|
|
||||||
_, _ = io.WriteString(c, "421 Too many connections\r\n")
|
|
||||||
_ = c.Close()
|
|
||||||
}()
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
cc := &countedConn{Conn: c, release: l.srv.release}
|
|
||||||
if l.srv.opts.RequireSessionReuse {
|
|
||||||
owner := c.RemoteAddr().String()
|
|
||||||
cc.sniff = func(b []byte) { l.srv.notePassive(owner, b) }
|
|
||||||
}
|
|
||||||
var conn net.Conn = cc
|
|
||||||
if l.srv.opts.NoRest {
|
|
||||||
conn = &noRestConn{countedConn: conn.(*countedConn), in: bufio.NewReader(c)}
|
|
||||||
}
|
|
||||||
return conn, nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
type countedConn struct {
|
|
||||||
net.Conn
|
|
||||||
once sync.Once
|
|
||||||
release func()
|
|
||||||
sniff func([]byte) // sees every reply the server writes
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *countedConn) Write(b []byte) (int, error) {
|
|
||||||
if c.sniff != nil {
|
|
||||||
c.sniff(b)
|
|
||||||
}
|
|
||||||
return c.Conn.Write(b)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *countedConn) Close() error {
|
|
||||||
c.once.Do(c.release)
|
|
||||||
return c.Conn.Close()
|
|
||||||
}
|
|
||||||
|
|
||||||
// noRestConn answers REST itself with 502 and hides it from the server.
|
|
||||||
type noRestConn struct {
|
|
||||||
*countedConn
|
|
||||||
in *bufio.Reader
|
|
||||||
pending []byte
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *noRestConn) Read(p []byte) (int, error) {
|
|
||||||
for len(c.pending) == 0 {
|
|
||||||
line, err := c.in.ReadBytes('\n')
|
|
||||||
if len(line) > 0 {
|
|
||||||
if strings.HasPrefix(strings.ToUpper(string(line)), "REST ") {
|
|
||||||
if _, werr := io.WriteString(c.Conn, "502 REST not implemented\r\n"); werr != nil {
|
|
||||||
return 0, werr
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
c.pending = line
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
if len(c.pending) > 0 {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
return 0, err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
n := copy(p, c.pending)
|
|
||||||
c.pending = c.pending[n:]
|
|
||||||
return n, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
type mainDriver struct {
|
|
||||||
srv *FTPServer
|
|
||||||
listener net.Listener
|
|
||||||
}
|
|
||||||
|
|
||||||
func (d *mainDriver) GetSettings() (*ftpserver.Settings, error) {
|
|
||||||
settings := &ftpserver.Settings{
|
|
||||||
Listener: d.listener,
|
|
||||||
DisableMLSD: d.srv.opts.NoMLST,
|
|
||||||
DisableMLST: d.srv.opts.NoMLST,
|
|
||||||
}
|
|
||||||
switch d.srv.opts.TLS {
|
|
||||||
case pftp.TLSExplicit:
|
|
||||||
settings.TLSRequired = ftpserver.MandatoryEncryption
|
|
||||||
case pftp.TLSImplicit:
|
|
||||||
settings.TLSRequired = ftpserver.ImplicitEncryption
|
|
||||||
}
|
|
||||||
return settings, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (d *mainDriver) ClientConnected(ftpserver.ClientContext) (string, error) {
|
|
||||||
return "gopeed test server", nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (d *mainDriver) ClientDisconnected(ftpserver.ClientContext) {}
|
|
||||||
|
|
||||||
func (d *mainDriver) AuthUser(_ ftpserver.ClientContext, user, pass string) (ftpserver.ClientDriver, error) {
|
|
||||||
if d.srv.opts.User != "" && (user != d.srv.opts.User || pass != d.srv.opts.Pass) {
|
|
||||||
return nil, errors.New("bad credentials")
|
|
||||||
}
|
|
||||||
d.srv.mu.Lock()
|
|
||||||
d.srv.logins++
|
|
||||||
d.srv.mu.Unlock()
|
|
||||||
return &clientDriver{Fs: afero.NewBasePathFs(afero.NewOsFs(), d.srv.Root), srv: d.srv}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (d *mainDriver) GetTLSConfig() (*tls.Config, error) {
|
|
||||||
return d.srv.tlsConf, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
type clientDriver struct {
|
|
||||||
afero.Fs
|
|
||||||
srv *FTPServer
|
|
||||||
}
|
|
||||||
|
|
||||||
// GetHandle sees the REST offset of every transfer. The server seeks the
|
|
||||||
// returned file to that offset itself.
|
|
||||||
func (d *clientDriver) GetHandle(name string, flags int, offset int64) (ftpserver.FileTransfer, error) {
|
|
||||||
f, err := d.Fs.OpenFile(name, flags, 0o644)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
if flags == os.O_RDONLY {
|
|
||||||
d.srv.mu.Lock()
|
|
||||||
d.srv.retrs = append(d.srv.retrs, offset)
|
|
||||||
d.srv.mu.Unlock()
|
|
||||||
}
|
|
||||||
return &slowFile{File: f, srv: d.srv, dropAfter: d.srv.DropAfter.Swap(0)}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
var errDropped = errors.New("test server dropped the transfer")
|
|
||||||
|
|
||||||
type slowFile struct {
|
|
||||||
afero.File
|
|
||||||
srv *FTPServer
|
|
||||||
sent int64
|
|
||||||
dropAfter int64
|
|
||||||
}
|
|
||||||
|
|
||||||
func (f *slowFile) Read(p []byte) (int, error) {
|
|
||||||
if delay := time.Duration(f.srv.ReadDelay.Load()); delay > 0 {
|
|
||||||
time.Sleep(delay)
|
|
||||||
if len(p) > 16<<10 {
|
|
||||||
p = p[:16<<10]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if f.dropAfter > 0 {
|
|
||||||
if f.sent >= f.dropAfter {
|
|
||||||
return 0, errDropped
|
|
||||||
}
|
|
||||||
if rest := f.dropAfter - f.sent; int64(len(p)) > rest {
|
|
||||||
p = p[:rest]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
n, err := f.srv.stall(f.File.Read, p)
|
|
||||||
f.sent += int64(n)
|
|
||||||
return n, err
|
|
||||||
}
|
|
||||||
|
|
||||||
// selfSignedTLS returns a server config with a fresh certificate for
|
|
||||||
// 127.0.0.1 and localhost, and a pool that trusts it.
|
|
||||||
func selfSignedTLS(t *testing.T) (*tls.Config, *x509.CertPool) {
|
|
||||||
t.Helper()
|
|
||||||
key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
tmpl := &x509.Certificate{
|
|
||||||
SerialNumber: big.NewInt(time.Now().UnixNano()),
|
|
||||||
Subject: pkix.Name{CommonName: "gopeed ftp test"},
|
|
||||||
NotBefore: time.Now().Add(-time.Hour),
|
|
||||||
NotAfter: time.Now().Add(time.Hour),
|
|
||||||
IPAddresses: []net.IP{net.ParseIP("127.0.0.1")},
|
|
||||||
DNSNames: []string{"localhost"},
|
|
||||||
KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign,
|
|
||||||
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
|
|
||||||
IsCA: true,
|
|
||||||
BasicConstraintsValid: true,
|
|
||||||
}
|
|
||||||
der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
cert, err := x509.ParseCertificate(der)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
pool := x509.NewCertPool()
|
|
||||||
pool.AddCert(cert)
|
|
||||||
return &tls.Config{
|
|
||||||
Certificates: []tls.Certificate{{Certificate: [][]byte{der}, PrivateKey: key, Leaf: cert}},
|
|
||||||
MinVersion: tls.VersionTLS12,
|
|
||||||
}, pool
|
|
||||||
}
|
|
||||||
|
|
||||||
func SHA256Hex(data []byte) string {
|
|
||||||
sum := sha256.Sum256(data)
|
|
||||||
return hex.EncodeToString(sum[:])
|
|
||||||
}
|
|
||||||
|
|
||||||
func FileSHA256(t *testing.T, name string) string {
|
|
||||||
t.Helper()
|
|
||||||
data, err := os.ReadFile(name)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
return SHA256Hex(data)
|
|
||||||
}
|
|
||||||
|
|
||||||
// RandomBytes returns n reproducible pseudo-random bytes.
|
|
||||||
func RandomBytes(n int, seed byte) []byte {
|
|
||||||
out := make([]byte, n)
|
|
||||||
x := uint32(seed) + 1
|
|
||||||
for i := range out {
|
|
||||||
x ^= x << 13
|
|
||||||
x ^= x >> 17
|
|
||||||
x ^= x << 5
|
|
||||||
out[i] = byte(x)
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
@@ -1,187 +0,0 @@
|
|||||||
package ftp
|
|
||||||
|
|
||||||
import (
|
|
||||||
"cmp"
|
|
||||||
"fmt"
|
|
||||||
"path"
|
|
||||||
"slices"
|
|
||||||
"strings"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/jlaffaye/ftp"
|
|
||||||
|
|
||||||
"github.com/GopeedLab/gopeed/pkg/base"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Limits of a directory download. A tree past either one is refused with an
|
|
||||||
// error that names the limit; it is never cut short.
|
|
||||||
const maxWalkDepth = 8
|
|
||||||
|
|
||||||
// maxWalkFiles is a variable so tests can lower it.
|
|
||||||
var maxWalkFiles = 10000
|
|
||||||
|
|
||||||
// resolveTarget describes the URL's path: one file, or a directory walked
|
|
||||||
// into a multi-file resource whose paths are relative to it.
|
|
||||||
func (f *Fetcher) resolveTarget(s *session, t *target) (*base.Resource, error) {
|
|
||||||
isDir, size, mtime, err := f.stat(s, t.path)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
if !isDir {
|
|
||||||
return &base.Resource{
|
|
||||||
Size: size,
|
|
||||||
Range: true,
|
|
||||||
Files: []*base.FileInfo{{
|
|
||||||
Name: path.Base(t.path),
|
|
||||||
Size: size,
|
|
||||||
Ctime: mtime,
|
|
||||||
}},
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
files, err := f.walk(s, t.path)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
name := path.Base(strings.TrimSuffix(t.path, "/"))
|
|
||||||
if name == "" || name == "." || name == "/" {
|
|
||||||
name = t.host
|
|
||||||
}
|
|
||||||
res := &base.Resource{Name: name, Range: true, Files: files}
|
|
||||||
res.CalcSize(nil)
|
|
||||||
return res, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// stat tells a file from a directory. MLST answers both at once. Without
|
|
||||||
// it, SIZE answers for a file, and a directory is one we can change into.
|
|
||||||
func (f *Fetcher) stat(s *session, p string) (isDir bool, size int64, mtime *time.Time, err error) {
|
|
||||||
if p == "" || strings.HasSuffix(p, "/") {
|
|
||||||
return true, 0, nil, nil
|
|
||||||
}
|
|
||||||
s.deadline()
|
|
||||||
if entry, mlstErr := s.GetEntry(p); mlstErr == nil {
|
|
||||||
switch entry.Type {
|
|
||||||
case ftp.EntryTypeFolder:
|
|
||||||
return true, 0, nil, nil
|
|
||||||
case ftp.EntryTypeFile:
|
|
||||||
if !entry.Time.IsZero() {
|
|
||||||
t := entry.Time
|
|
||||||
mtime = &t
|
|
||||||
}
|
|
||||||
return false, int64(entry.Size), mtime, nil
|
|
||||||
}
|
|
||||||
// A link: SIZE below follows it on the server.
|
|
||||||
}
|
|
||||||
// MLST is missing or refused the path: the older commands decide. A
|
|
||||||
// broken connection fails them too, with its own error.
|
|
||||||
|
|
||||||
s.deadline()
|
|
||||||
size, sizeErr := s.FileSize(p)
|
|
||||||
if sizeErr == nil {
|
|
||||||
if s.IsGetTimeSupported() {
|
|
||||||
s.deadline()
|
|
||||||
if t, err := s.GetTime(p); err == nil {
|
|
||||||
mtime = &t
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return false, size, mtime, nil
|
|
||||||
}
|
|
||||||
if replyCode(sizeErr) == 0 {
|
|
||||||
return false, 0, nil, sizeErr
|
|
||||||
}
|
|
||||||
|
|
||||||
// SIZE failed: a directory, or a missing path.
|
|
||||||
s.deadline()
|
|
||||||
cwd, err := s.CurrentDir()
|
|
||||||
if err != nil {
|
|
||||||
return false, 0, nil, err
|
|
||||||
}
|
|
||||||
s.deadline()
|
|
||||||
if err := s.ChangeDir(p); err != nil {
|
|
||||||
return false, 0, nil, sizeErr
|
|
||||||
}
|
|
||||||
s.deadline()
|
|
||||||
if err := s.ChangeDir(cwd); err != nil {
|
|
||||||
return false, 0, nil, err
|
|
||||||
}
|
|
||||||
return true, 0, nil, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// safeEntryName reports whether a name from a listing can become a local
|
|
||||||
// path element. A hostile server could otherwise list "../x" and write
|
|
||||||
// outside the task directory.
|
|
||||||
func safeEntryName(name string) bool {
|
|
||||||
if name == "" || name == "." || name == ".." {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
return !strings.ContainsAny(name, "/\\\x00")
|
|
||||||
}
|
|
||||||
|
|
||||||
// walk lists a directory tree, at most maxWalkDepth levels below root and
|
|
||||||
// maxWalkFiles files. Files come back sorted by path, then name.
|
|
||||||
func (f *Fetcher) walk(s *session, root string) ([]*base.FileInfo, error) {
|
|
||||||
type dir struct {
|
|
||||||
remote string // path on the server
|
|
||||||
rel string // path below root, "" for root itself
|
|
||||||
depth int
|
|
||||||
}
|
|
||||||
var files []*base.FileInfo
|
|
||||||
stack := []dir{{remote: strings.TrimSuffix(root, "/"), rel: "", depth: 0}}
|
|
||||||
for len(stack) > 0 {
|
|
||||||
d := stack[len(stack)-1]
|
|
||||||
stack = stack[:len(stack)-1]
|
|
||||||
|
|
||||||
s.deadline()
|
|
||||||
entries, err := s.List(d.remote)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
for _, e := range entries {
|
|
||||||
if e.Name == "." || e.Name == ".." {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if !safeEntryName(e.Name) {
|
|
||||||
f.warn("ftp: skipping a listed name that is not a safe file name: %q", e.Name)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
remote := path.Join(d.remote, e.Name)
|
|
||||||
rel := path.Join(d.rel, e.Name)
|
|
||||||
if d.remote == "" {
|
|
||||||
remote = e.Name
|
|
||||||
}
|
|
||||||
entryType, size := e.Type, int64(e.Size)
|
|
||||||
if entryType == ftp.EntryTypeLink {
|
|
||||||
// A link to a file has a size; a link to a directory is
|
|
||||||
// not followed, so a loop cannot make the walk endless.
|
|
||||||
s.deadline()
|
|
||||||
linkSize, err := s.FileSize(remote)
|
|
||||||
if err != nil {
|
|
||||||
f.warn("ftp: skipping link %q: %v", rel, err)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
entryType, size = ftp.EntryTypeFile, linkSize
|
|
||||||
}
|
|
||||||
switch entryType {
|
|
||||||
case ftp.EntryTypeFolder:
|
|
||||||
if d.depth+1 > maxWalkDepth {
|
|
||||||
return nil, fmt.Errorf("ftp: the folder is deeper than %d levels, the limit for a folder download", maxWalkDepth)
|
|
||||||
}
|
|
||||||
stack = append(stack, dir{remote: remote, rel: rel, depth: d.depth + 1})
|
|
||||||
case ftp.EntryTypeFile:
|
|
||||||
if len(files) >= maxWalkFiles {
|
|
||||||
return nil, fmt.Errorf("ftp: the folder has more than %d files, the limit for a folder download", maxWalkFiles)
|
|
||||||
}
|
|
||||||
fi := &base.FileInfo{Name: e.Name, Path: d.rel, Size: size}
|
|
||||||
if !e.Time.IsZero() {
|
|
||||||
t := e.Time
|
|
||||||
fi.Ctime = &t
|
|
||||||
}
|
|
||||||
files = append(files, fi)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
slices.SortFunc(files, func(a, b *base.FileInfo) int {
|
|
||||||
return cmp.Or(strings.Compare(a.Path, b.Path), strings.Compare(a.Name, b.Name))
|
|
||||||
})
|
|
||||||
return files, nil
|
|
||||||
}
|
|
||||||
@@ -1,30 +0,0 @@
|
|||||||
package hls
|
|
||||||
|
|
||||||
const (
|
|
||||||
defaultSegmentConnections = 8
|
|
||||||
defaultMaxRetries = 5
|
|
||||||
defaultTimeoutSeconds = 30
|
|
||||||
|
|
||||||
// maxPrefetchSizeSegments caps the HEAD probing done at resolve time.
|
|
||||||
// Playlists larger than this skip size prefetch entirely.
|
|
||||||
maxPrefetchSizeSegments = 500
|
|
||||||
)
|
|
||||||
|
|
||||||
type config struct {
|
|
||||||
SegmentConnections int `json:"segmentConnections"`
|
|
||||||
MaxRetries int `json:"maxRetries"`
|
|
||||||
TimeoutSeconds int `json:"timeoutSeconds"`
|
|
||||||
PrefetchContentLength bool `json:"prefetchContentLength"`
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *config) normalize() {
|
|
||||||
if c.SegmentConnections <= 0 {
|
|
||||||
c.SegmentConnections = defaultSegmentConnections
|
|
||||||
}
|
|
||||||
if c.MaxRetries < 0 {
|
|
||||||
c.MaxRetries = defaultMaxRetries
|
|
||||||
}
|
|
||||||
if c.TimeoutSeconds <= 0 {
|
|
||||||
c.TimeoutSeconds = defaultTimeoutSeconds
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,59 +0,0 @@
|
|||||||
package hls
|
|
||||||
|
|
||||||
import (
|
|
||||||
"crypto/aes"
|
|
||||||
"crypto/cipher"
|
|
||||||
"fmt"
|
|
||||||
)
|
|
||||||
|
|
||||||
// decryptAES128 decrypts an AES-128-CBC encrypted segment and strips PKCS7
|
|
||||||
// padding. data length must be a multiple of the AES block size.
|
|
||||||
func decryptAES128(data, key, iv []byte) ([]byte, error) {
|
|
||||||
if len(key) != 16 {
|
|
||||||
return nil, fmt.Errorf("invalid AES-128 key size %d", len(key))
|
|
||||||
}
|
|
||||||
if len(iv) != 16 {
|
|
||||||
return nil, fmt.Errorf("invalid AES-128 IV size %d", len(iv))
|
|
||||||
}
|
|
||||||
if len(data) == 0 || len(data)%aes.BlockSize != 0 {
|
|
||||||
return nil, fmt.Errorf("invalid AES-128 data size %d", len(data))
|
|
||||||
}
|
|
||||||
block, err := aes.NewCipher(key)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
plain := make([]byte, len(data))
|
|
||||||
cipher.NewCBCDecrypter(block, iv).CryptBlocks(plain, data)
|
|
||||||
return pkcs7Unpad(plain, aes.BlockSize)
|
|
||||||
}
|
|
||||||
|
|
||||||
func pkcs7Unpad(data []byte, blockSize int) ([]byte, error) {
|
|
||||||
if len(data) == 0 {
|
|
||||||
return nil, fmt.Errorf("empty plaintext")
|
|
||||||
}
|
|
||||||
pad := int(data[len(data)-1])
|
|
||||||
if pad == 0 || pad > blockSize || pad > len(data) {
|
|
||||||
return nil, fmt.Errorf("invalid PKCS7 padding")
|
|
||||||
}
|
|
||||||
for _, b := range data[len(data)-pad:] {
|
|
||||||
if int(b) != pad {
|
|
||||||
return nil, fmt.Errorf("invalid PKCS7 padding")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return data[:len(data)-pad], nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// sequenceIV derives the default per-segment IV mandated by RFC 8216: the
|
|
||||||
// segment media sequence number as a 16-byte big-endian integer.
|
|
||||||
func sequenceIV(sequence int64) []byte {
|
|
||||||
iv := make([]byte, 16)
|
|
||||||
iv[8] = byte(sequence >> 56)
|
|
||||||
iv[9] = byte(sequence >> 48)
|
|
||||||
iv[10] = byte(sequence >> 40)
|
|
||||||
iv[11] = byte(sequence >> 32)
|
|
||||||
iv[12] = byte(sequence >> 24)
|
|
||||||
iv[13] = byte(sequence >> 16)
|
|
||||||
iv[14] = byte(sequence >> 8)
|
|
||||||
iv[15] = byte(sequence)
|
|
||||||
return iv
|
|
||||||
}
|
|
||||||