Compare commits

..
71 Commits
Author SHA1 Message Date
Levi 31228e3edf release: v1.5.3 2023-12-15 19:23:16 +08:00
Levi 31a32d7e14 fix: handle api panic (#302) 2023-12-15 19:20:56 +08:00
Levi 65fd6fb489 fix: http pause handle (#303) 2023-12-15 19:15:38 +08:00
Levi 953b878a89 fix: adapt mobile proxy settings ui (#301) 2023-12-15 19:09:48 +08:00
Elliot ae4ca3da58 feat: improve zh-TW translation (#300) 2023-12-14 18:06:49 +08:00
Levi d09bd180af release: v1.5.2 2023-12-13 09:38:34 +08:00
liwei 3d24e353c5 docs: docker data path mounted 2023-12-13 08:57:57 +08:00
Levi ce7489b0c0 feat: support volume storage dir on docker (#291) 2023-12-13 08:51:22 +08:00
Levi 2b5091ba21 feat: improve extension engine (#298)
1. support onError event
2. add built-in MessageError
2023-12-12 19:00:42 +08:00
Levi ff32971aca fix: repair invalid filename (#296) 2023-12-12 18:40:36 +08:00
Levi f2af61f9aa fix: sort of completed tasks (#297) 2023-12-12 18:36:40 +08:00
Elliot d935b9d631 feat: improve zh_TW translate (#287) 2023-12-04 23:01:49 +08:00
liwei ae66d1d83d docs: fix contributing 2023-12-04 13:41:27 +08:00
madoka773 aa91ca1d50 ci: linux build use libappindicator (#286) 2023-12-04 13:36:27 +08:00
liwei 2d4bdea5cb fix: android file open 2023-12-02 22:51:44 +08:00
liwei 2f2393ac20 fix: i18n word 2023-12-02 19:22:37 +08:00
Levi 5a761e244a release: v1.5.1 2023-12-02 18:47:07 +08:00
Levi 6538766780 fix: incorrect file name in task files view (#285) 2023-12-02 18:45:16 +08:00
Levi 16b7a3d5b3 feat: support proxy (#284) 2023-12-02 18:36:52 +08:00
Levi ef97528114 ci: fix window 2023-11-29 21:12:06 +08:00
Levi b58669bb46 ci: fix windows build 2023-11-29 20:35:20 +08:00
Levi 51d28c5e53 release: v1.5.0 2023-11-29 20:18:08 +08:00
Levi db15650664 feat: support browse files on web (#283) 2023-11-29 18:24:01 +08:00
Levi ee7da179c1 chore: upgrade flutter to v3.16.1 (#282) 2023-11-29 13:32:07 +08:00
Levi 6bb32372ab feat: support more configurations (#281) 2023-11-29 12:52:51 +08:00
Levi 7d6428c967 fix: task can not completed (#280) 2023-11-28 18:27:36 +08:00
Levi bba5107dd5 fix: crash on android arm-32 (#279) 2023-11-27 11:10:50 +08:00
liwei ddfe76b635 fix: script engine event loop stop 2023-11-25 00:43:40 +08:00
Levi aad7395dde feat: enhanced extension script engine (#277)
1. update goja version
2. add node vm polyfill
3. non-stop event loop handle
4 improve logger print
2023-11-24 22:41:16 +08:00
Levi 6f541715df fix: extension download path (#274) 2023-11-24 18:16:03 +08:00
neovali 848797b2fd fix: .deb build (#269) 2023-11-15 21:01:28 +08:00
Levi 0a05bf41e7 fix: incorrect storage path in linux (#270) 2023-11-15 16:26:29 +08:00
liwei 485656d1d4 ci: fix linux desktop entry 2023-11-13 16:23:26 +08:00
liwei c074e0c2bc chore: generate locales 2023-11-13 15:55:02 +08:00
yuloury 0db0a2c987 ci: fix deb desktop entry (#268) 2023-11-11 09:23:17 +08:00
Elliot b06a79b5c7 feat: improve zh_TW translate 2023-11-10 09:11:50 +08:00
liwei 4840c4b8a4 ci: fix build env 2023-11-09 18:49:35 +08:00
Levi e72b097aa2 release: v1.4.4 2023-11-09 17:09:15 +08:00
Levi af52c4ce7e feat: support create task directly (#262) 2023-11-09 16:52:02 +08:00
Levi aae79c2449 feat: running foreground service on mobile (#261) 2023-11-07 17:52:10 +08:00
Levi 83838afce9 feat: support more linux distributions (#254) 2023-11-01 14:54:24 +08:00
Levi 4690107e32 fix: gopeed cli BT download (#258) 2023-10-31 10:16:28 +08:00
liwei 74b08b3631 chore: build locales.g.dart 2023-10-31 09:24:55 +08:00
Kiber_kotan 953d348c0c feat: improve Russian translation (#257) 2023-10-30 11:27:43 +08:00
Levi 00cea68b6e feat: add extension intro links (#252) 2023-10-26 18:35:22 +08:00
Levi daa9d4c707 release: v1.4.3 2023-10-24 16:17:48 +08:00
liwei 6bc587752a chore: update introduction 2023-10-24 16:16:16 +08:00
Levi 11befc09bd refractor: downloader task action dispatch (#249) 2023-10-24 16:02:39 +08:00
liwei ff6c0da262 chore: locales build 2023-10-23 13:55:13 +08:00
Levi aab4a37a7e feat: adapt mobile extension list (#242) 2023-10-21 16:36:22 +08:00
Levi 74c9491893 fix: linux system tray tooltip (#243) 2023-10-21 16:29:15 +08:00
Levi 256faf2cbd feat: add extension setting tooltip (#247) 2023-10-21 16:16:46 +08:00
Om Jogani 32e3620f5e feat: Implemented History Panel for reusing previously used Magnet & Torrent Links (#245) 2023-10-21 15:52:34 +08:00
Levi 39202b44a1 fix: http retry strategy (#240) 2023-10-20 12:15:21 +08:00
Levi 457d9e85d6 release: v1.4.2 2023-10-19 18:36:46 +08:00
Levi 858ad74339 feat: extension on start event (#239) 2023-10-19 18:08:27 +08:00
Levi 755d7ce814 fix: add system tray tooltip (#237) 2023-10-19 14:00:15 +08:00
Levi 854f600d3f test: add extension script error case (#238) 2023-10-19 13:59:59 +08:00
Levi 28da89f08c release: v1.4.1 2023-10-18 11:10:04 +08:00
Levi dddefb68a2 feat: support storage state on extension (#235) 2023-10-18 10:59:51 +08:00
liwei e882b663a8 ci: docker build free disk 2023-10-17 11:58:24 +08:00
liwei 6f8e9fc68f ci: fix go version on Windows 2023-10-17 10:52:19 +08:00
liwei 7799a4032c ci: fix windows build 2023-10-17 10:41:49 +08:00
liwei 6316e9986a ci: fix go version 2023-10-17 10:26:21 +08:00
liwei 39d1028ee9 fix: web start 2023-10-17 09:45:19 +08:00
liwei 107f90649b release: v1.4.0 2023-10-17 09:19:47 +08:00
liwei c8b66501b7 chore: generate zh_TW 2023-10-17 09:19:08 +08:00
Elliot 398e3f7adc feat: update zh_TW.json (#225) 2023-10-16 23:20:20 +08:00
Levi 158018c248 ci: support arm/v6 and arm/v7 on docker (#234) 2023-10-16 16:19:12 +08:00
liwei c56447ad5f chore: update donate link 2023-10-16 12:49:59 +08:00
Levi c073fc0a67 feat: extension support (#78) 2023-10-16 09:47:31 +08:00
184 changed files with 10971 additions and 2505 deletions

No files matched your search

-27
View File
@@ -1,27 +0,0 @@
## Donate
🙏 Thank you for considering a donation to Gopeed project! Your support helps us continue to develop and maintain our software.
<div style="text-align: center;">
<table>
<tr>
<th>微信</th>
<th>支付宝</th>
<th>Ko-fi</th>
<th>PayPal</th>
</tr>
<tr>
<td><img src="/.donate/qr/wechatpay.jpg" alt="wechatpay" width="300" /></td>
<td><img src="/.donate/qr/alipay.jpg" alt="alipay" width="300" /></td>
<td><a href="https://ko-fi.com/gopeed" target="_blank">Click Me</a></td>
<td><a href="https://paypal.me/monkeyWie" target="_blank">Click Me</a></td>
</table>
</div>
### Crypto
- **BTC**: 35jYkPN8bgkh7AWjt3B1VgMQHqtGYcvF6p
- **ETH:** 0x928a7fd2398c941Ac6713704e606Fa414595e3A0
- **Matic:** 0x928a7fd2398c941Ac6713704e606Fa414595e3A0
- **BNB:** 0x928a7fd2398c941Ac6713704e606Fa414595e3A0
- **USDT(ERC20)**: 0x928a7fd2398c941Ac6713704e606Fa414595e3A0
Binary file not shown.

Before

Width:  |  Height:  |  Size: 81 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 76 KiB

+1 -1
View File
@@ -12,4 +12,4 @@
#custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2']
# github: monkeyWie
ko_fi: gopeed
custom: https://github.com/GopeedLab/gopeed/blob/main/.donate/index.md#donate
custom: https://docs.gopeed.com/donate.html
+9 -1
View File
@@ -11,12 +11,16 @@ categories:
- 'fix'
- 'bugfix'
- 'bug'
- title: '🧪 Tests'
label: 'test'
- title: '🧰 Maintenance'
label: 'chore'
- title: '📖 Document'
label: 'doc'
- title: '🚀 CI/CD'
label: 'ci'
- title: '🌎 Internationalization'
label: 'translation'
change-template: '- $TITLE @$AUTHOR (#$NUMBER)'
version-resolver:
major:
@@ -30,6 +34,10 @@ version-resolver:
- 'patch'
default: patch
template: |
## Changes
# Changes
$CHANGES
# 更新日志
$CHANGES
+222 -51
View File
@@ -2,10 +2,19 @@ name: build
on:
workflow_dispatch:
inputs:
platform:
description: 'Build platform'
required: true
default: 'all'
channel:
description: 'Snap channel'
required: true
default: 'stable'
env:
GO_VERSION: "1.20"
FLUTTER_VERSION: 3.10.6
GO_VERSION: "1.19"
FLUTTER_VERSION: "3.16.1"
permissions:
contents: write
@@ -24,13 +33,14 @@ jobs:
myToken: ${{ github.token }}
build-windows:
if: ${{ github.event.inputs.platform == 'all' || github.event.inputs.platform == 'windows' }}
runs-on: windows-latest
needs: [get-release]
needs: [ get-release ]
steps:
- uses: actions/checkout@v3
- uses: actions/setup-go@v4
with:
go-version: ${{ env.GO_VERSION }}
go-version: "1.20"
- uses: subosito/flutter-action@v2
with:
flutter-version: ${{ env.FLUTTER_VERSION }}
@@ -44,9 +54,9 @@ jobs:
go build -tags nosqlite -ldflags="-w -s" -buildmode=c-shared -o ui/flutter/windows/libgopeed.dll github.com/GopeedLab/gopeed/bind/desktop
cd ui/flutter
flutter build windows
$mingw = "C:\ProgramData\Chocolatey\lib\mingw\tools\install\mingw64\bin"
$mingw = "C:\Program Files\Git\mingw64\bin"
$system = "C:\Windows\System32"
$release = "build\windows\runner\Release\"
$release = "build\windows\x64\runner\Release\"
cp $mingw\libstdc++-6.dll $release
cp $mingw\libgcc_s_seh-1.dll $release
cp $mingw\libwinpthread-1.dll $release
@@ -100,7 +110,7 @@ jobs:
Name: "desktopicon"; Description: "{cm:CreateDesktopIcon}"; GroupDescription: "{cm:AdditionalIcons}"; Flags: unchecked
[Files]
Source: ".\runner\Release\*"; DestDir: "{app}"; Flags: ignoreversion recursesubdirs
Source: ".\x64\runner\Release\*"; DestDir: "{app}"; Flags: ignoreversion recursesubdirs
; NOTE: Don't use "Flags: ignoreversion" on any shared system files
[UninstallDelete]
@@ -124,8 +134,9 @@ jobs:
asset_path: ui/flutter/build/windows/Output/*
overwrite: true
build-macos-arm64-lib:
if: ${{ github.event.inputs.platform == 'all' || github.event.inputs.platform == 'macos' }}
runs-on: ubuntu-latest
needs: [get-release]
needs: [ get-release ]
steps:
- uses: actions/checkout@v3
- uses: actions/setup-go@v4
@@ -141,8 +152,9 @@ jobs:
name: macos-arm64-lib
path: libgopeed.dylib
build-macos:
if: ${{ github.event.inputs.platform == 'all' || github.event.inputs.platform == 'macos' }}
runs-on: macos-latest
needs: [get-release, build-macos-arm64-lib]
needs: [ get-release, build-macos-arm64-lib ]
steps:
- uses: actions/checkout@v3
- uses: actions/setup-go@v4
@@ -155,7 +167,9 @@ jobs:
with:
flutter-version: ${{ env.FLUTTER_VERSION }}
- name: Install appdmg
run: npm install -g appdmg
run: |
python3 -m pip install setuptools
npm install -g appdmg
- uses: actions/download-artifact@v3
with:
name: macos-arm64-lib
@@ -197,8 +211,9 @@ jobs:
asset_path: ui/flutter/build/macos/Build/Products/Release/dist/*
overwrite: true
build-linux:
runs-on: ubuntu-latest
needs: [get-release]
if: ${{ github.event.inputs.platform == 'all' || github.event.inputs.platform == 'linux' }}
runs-on: ubuntu-20.04
needs: [ get-release ]
steps:
- uses: actions/checkout@v3
- uses: actions/setup-go@v4
@@ -209,7 +224,7 @@ jobs:
flutter-version: ${{ env.FLUTTER_VERSION }}
- run: |
sudo apt-get update -y
sudo apt-get install -y ninja-build libgtk-3-dev libayatana-appindicator3-dev
sudo apt-get install -y ninja-build libgtk-3-dev libappindicator3-dev
- name: Build
env:
VERSION: ${{ needs.get-release.outputs.tag_name }}
@@ -217,48 +232,195 @@ jobs:
go build -tags nosqlite -ldflags="-w -s" -buildmode=c-shared -o ui/flutter/linux/bundle/lib/libgopeed.so github.com/GopeedLab/gopeed/bind/desktop
cd ui/flutter
flutter build linux
mkdir -p debian/gui
cp assets/icon/icon_1024.png debian/gui/gopeed.png
cat>debian/debian.yaml<<EOF
flutter_app:
command: gopeed
arch: x64
parent: /usr/local/lib
control:
Package: gopeed
Version: ${VERSION:1}
Architecture: amd64
Essential: no
Priority: optional
Depends:
Maintainer: monkeyWie
Description: High speed downloader that supports all platforms.
mkdir -p build/dist
# Build portable
tar -C build/linux/x64/release/bundle -czaf build/dist/Gopeed-$VERSION-linux-amd64-portable.tar.gz ./
# Build deb package
mkdir -p debian/gopeed/opt/gopeed
mkdir -p debian/gopeed/usr/share/applications
mkdir -p debian/gopeed/usr/share/icons/hicolor/scalable/apps
mkdir -p debian/gopeed/DEBIAN
cp -r build/linux/x64/release/bundle/* debian/gopeed/opt/gopeed/
cp assets/icon/icon.svg debian/gopeed/usr/share/icons/hicolor/scalable/apps/com.gopeed.Gopeed.svg
cat>debian/gopeed/DEBIAN/postinst<<EOF
#!/bin/bash
ln -sf /opt/gopeed/gopeed /usr/bin/gopeed
EOF
cat>debian/gui/gopeed.desktop<<EOF
[Desktop Entry]
Version=${VERSION:1}
Name=Gopeed
GenericName=Gopeed
Comment=High speed downloader that supports all platforms.
Terminal=false
Type=Application
Categories=Utility
Keywords=Flutter;Downloader;
cat>debian/gopeed/DEBIAN/postrm<<EOF
#!/bin/bash
rm -rf /usr/bin/gopeed
EOF
dart pub global activate flutter_to_debian
flutter_to_debian
mv debian/packages/gopeed*.deb debian/packages/Gopeed-$VERSION-linux-amd64.deb
chmod 0755 debian/gopeed/DEBIAN/postinst
chmod 0755 debian/gopeed/DEBIAN/postrm
cat>debian/gopeed/DEBIAN/control<<EOF
Package: gopeed
Version: ${VERSION:1}
Architecture: amd64
Essential: no
Priority: optional
Depends:
Maintainer: monkeyWie
Description: High speed downloader that supports all platforms.
EOF
cp linux/assets/com.gopeed.Gopeed.desktop debian/gopeed/usr/share/applications/gopeed.desktop
dpkg-deb --build debian/gopeed
mv debian/gopeed.deb build/dist/Gopeed-$VERSION-linux-amd64.deb
# Build AppImage
sudo apt-get -y install libfuse2
wget -O appimage-builder https://github.com/AppImageCrafters/appimage-builder/releases/download/v1.1.0/appimage-builder-1.1.0-x86_64.AppImage
chmod +x appimage-builder
mkdir AppDir
cp -r build/linux/x64/release/bundle/* AppDir
mkdir -p AppDir/usr/share/icons/hicolor/scalable/apps/
cp assets/icon/icon.svg AppDir/usr/share/icons/hicolor/scalable/apps/gopeed.svg
cat>AppImageBuilder.yml<<EOF
version: 1
AppDir:
path: ${PWD}/AppDir
app_info:
id: com.gopeed.gopeed
name: Gopeed
icon: gopeed
version: ${VERSION:1}
exec: gopeed
exec_args: \$@
apt:
arch: amd64
allow_unauthenticated: true
sources:
- sourceline: deb https://mirrors.tuna.tsinghua.edu.cn/ubuntu/ jammy main restricted
universe multiverse
- sourceline: deb https://mirrors.tuna.tsinghua.edu.cn/ubuntu/ jammy-updates main
restricted universe multiverse
- sourceline: deb https://mirrors.tuna.tsinghua.edu.cn/ubuntu/ jammy-backports
main restricted universe multiverse
- sourceline: deb http://security.ubuntu.com/ubuntu/ jammy-security main restricted
universe multiverse
include: []
files:
include: []
exclude:
- usr/share/man
- usr/share/doc/*/README.*
- usr/share/doc/*/changelog.*
- usr/share/doc/*/NEWS.*
- usr/share/doc/*/TODO.*
test:
fedora-30:
image: appimagecrafters/tests-env:fedora-30
command: ./AppRun
debian-stable:
image: appimagecrafters/tests-env:debian-stable
command: ./AppRun
archlinux-latest:
image: appimagecrafters/tests-env:archlinux-latest
command: ./AppRun
centos-7:
image: appimagecrafters/tests-env:centos-7
command: ./AppRun
ubuntu-xenial:
image: appimagecrafters/tests-env:ubuntu-xenial
command: ./AppRun
AppImage:
arch: x86_64
update-information: guess
EOF
./appimage-builder --skip-tests --recipe AppImageBuilder.yml
mv Gopeed-*-x86_64.AppImage build/dist/Gopeed-$VERSION-linux-amd64.AppImage
- name: Upload
uses: shogo82148/actions-upload-release-asset@v1
with:
upload_url: ${{ needs.get-release.outputs.upload_url }}
asset_path: ui/flutter/debian/packages/*
asset_path: ui/flutter/build/dist/*
overwrite: true
build-android:
build-snap:
if: ${{ github.event.inputs.platform == 'all' || github.event.inputs.platform == 'snap' }}
runs-on: ubuntu-latest
needs: [get-release]
needs: [ get-release ]
steps:
- uses: actions/checkout@v3
- uses: actions/setup-go@v4
with:
go-version: ${{ env.GO_VERSION }}
- uses: subosito/flutter-action@v2
with:
flutter-version: ${{ env.FLUTTER_VERSION }}
- name: Setup LXD
uses: canonical/[email protected]
with:
channel: latest/stable
- name: Build
env:
VERSION: ${{ needs.get-release.outputs.tag_name }}
run: |
go build -tags nosqlite -ldflags="-w -s" -buildmode=c-shared -o ui/flutter/linux/bundle/lib/libgopeed.so github.com/GopeedLab/gopeed/bind/desktop
cd ui/flutter
sudo snap install snapcraft --classic
mkdir -p snap/gui
cp assets/icon/icon.svg snap/gui/gopeed.svg
cat>snap/snapcraft.yaml<<EOF
name: gopeed
version: ${VERSION:1}
summary: A modern download manager
description: High speed downloader that supports all platforms.
confinement: strict
base: core22
grade: stable
architectures:
- build-on: amd64
build-for: amd64
slots:
dbus-gopeed:
interface: dbus
bus: session
name: com.gopeed.gopeed
apps:
gopeed:
command: gopeed
extensions: [gnome]
plugs:
- network
slots:
- dbus-gopeed
parts:
gopeed:
source: .
plugin: flutter
stage-packages:
- libgtk-3-dev
- libappindicator3-dev
flutter-target: lib/main.dart
EOF
cp linux/assets/com.gopeed.Gopeed.desktop snap/gui/gopeed.desktop
sed -i 's/Icon=com.gopeed.Gopeed/Icon=\${SNAP}\/meta\/gui\/gopeed.svg/g' snap/gui/gopeed.desktop
snapcraft --use-lxd
# Snapcraft login
export SNAPCRAFT_STORE_CREDENTIALS=${{ secrets.SNAP_STORE_LOGIN }}
snapcraft upload --release=${{ github.event.inputs.channel }} gopeed_${VERSION:1}_amd64.snap
build-android:
if: ${{ github.event.inputs.platform == 'all' || github.event.inputs.platform == 'android' }}
runs-on: ubuntu-latest
needs: [ get-release ]
steps:
- uses: actions/checkout@v3
- uses: actions/setup-go@v4
@@ -274,9 +436,9 @@ jobs:
- name: Build
env:
VERSION: ${{ needs.get-release.outputs.tag_name }}
APK_KEYSTORE : ${{ secrets.APK_KEYSTORE }}
APK_KEY_PASSWORD : ${{ secrets.APK_KEY_PASSWORD }}
APK_STORE_PASSWORD : ${{ secrets.APK_STORE_PASSWORD }}
APK_KEYSTORE: ${{ secrets.APK_KEYSTORE }}
APK_KEY_PASSWORD: ${{ secrets.APK_KEY_PASSWORD }}
APK_STORE_PASSWORD: ${{ secrets.APK_STORE_PASSWORD }}
run: |
go install golang.org/x/mobile/cmd/gomobile@latest
go get golang.org/x/mobile/bind
@@ -294,8 +456,9 @@ jobs:
asset_path: ui/flutter/dist/*
overwrite: true
build-ios:
if: ${{ github.event.inputs.platform == 'all' || github.event.inputs.platform == 'ios' }}
runs-on: macos-latest
needs: [get-release]
needs: [ get-release ]
steps:
- uses: actions/checkout@v3
- uses: actions/setup-go@v4
@@ -326,8 +489,9 @@ jobs:
asset_path: ui/flutter/dist/*
overwrite: true
build-web:
if: ${{ github.event.inputs.platform == 'all' || github.event.inputs.platform == 'web' }}
runs-on: ubuntu-latest
needs: [get-release]
needs: [ get-release ]
steps:
- uses: actions/checkout@v3
- uses: actions/setup-go@v4
@@ -373,9 +537,14 @@ jobs:
asset_path: dist/zip/*
overwrite: true
build-docker:
if: ${{ github.event.inputs.platform == 'all' || github.event.inputs.platform == 'docker' }}
runs-on: ubuntu-latest
needs: [get-release]
needs: [ get-release ]
steps:
- name: Remove unnecessary files
run: |
sudo rm -rf /usr/share/dotnet
sudo rm -rf "$AGENT_TOOLSDIRECTORY"
- uses: actions/checkout@v3
- name: Set up QEMU
uses: docker/setup-qemu-action@v2
@@ -396,6 +565,8 @@ jobs:
linux/386
linux/amd64
linux/arm64
linux/arm/v6
linux/arm/v7
tags: |
liwei2633/gopeed:latest
liwei2633/gopeed:${{ env.VERSION }}
+1 -1
View File
@@ -17,7 +17,7 @@ on:
- "Dockerfile"
env:
GO_VERSION: "1.20"
GO_VERSION: "1.19"
permissions:
contents: write
+2 -2
View File
@@ -27,8 +27,8 @@ on:
- "go.sum"
env:
GO_VERSION: "1.20"
FLUTTER_VERSION: 3.10.6
GO_VERSION: "1.19"
FLUTTER_VERSION: "3.16.1"
jobs:
# lint:
+5 -1
View File
@@ -25,4 +25,8 @@ bin/
.DS_Store
dist/
node_modules/
cmd/web/dist/
.test_storage/
.test_download/
/extensions
+2 -23
View File
@@ -17,17 +17,9 @@ start the front-end flutter project in `debug` mode to run.
## Translation
The internationalization files of Gopeed are located in the `ui/flutter/assets/locales` directory.
The internationalization files of Gopeed are located in the `ui/flutter/lib/i18n/langs` directory.
You only need to add the corresponding language file in this directory.
Generate locales after you edit locales:
```
get generate locales
```
Please refer to `en_us.dart` for translation.
## flutter development
@@ -38,17 +30,4 @@ Turn on build_runner watcher if you want to edit api/models:
```
flutter pub run build_runner watch
```
get-cli commands usages:
```
create:
controller: Generate controller
page: Use to generate pages
view: Generate view
generate:
locales: Generate translation file from json files
model: generate Class model from json
```
```
+1 -22
View File
@@ -17,17 +17,9 @@
## 翻訳
Gopeed の国際化ファイルは `ui/flutter/assets/locales` ディレクトリにあります。
Gopeed の国際化ファイルは `ui/flutter/lib/i18n/langs` ディレクトリにあります。
このディレクトリに対応する言語ファイルを追加するだけでよいです。
ロケール編集後にロケールを生成:
```
get generate locales
```
翻訳については `en_us.dart` を参照してください。
## flutter での開発
@@ -39,16 +31,3 @@ api/models を編集したい場合は build_runner watcher をオンにしま
```
flutter pub run build_runner watch
```
get-cli コマンドの使用法:
```
create:
controller: Generate controller
page: Use to generate pages
view: Generate view
generate:
locales: Generate translation file from json files
model: generate Class model from json
```
+2 -20
View File
@@ -14,15 +14,9 @@ flutter 项目即可运行。
## 翻译
Gopeed 的国际化文件位于 `ui/flutter/assets/locales` 目录下,只需要在该目录下添加对应的语言文件即可。
Gopeed 的国际化文件位于 `ui/flutter/lib/i18n/langs` 目录下,只需要在该目录下添加对应的语言文件即可。
编辑locales后请运行以下命令:
```
get generate locales
```
请注意以 `en_US.json` 为参照进行翻译。
请注意以 `en_us.dart` 为参照进行翻译。
## flutter开发
@@ -34,15 +28,3 @@ get generate locales
flutter pub run build_runner watch
```
适用get-cli 命令:
```
create:
controller: Generate controller
page: Use to generate pages
view: Generate view
generate:
locales: Generate translation file from json files
model: generate Class model from json
```
+28
View File
@@ -0,0 +1,28 @@
# Gopeed 協助指南
首先感謝您願意幫助我們改進並優化該項目,這份指南將會幫助您更好的參與 Gopeed 的開發。
## 分支說明
本項目只有一個分支,即 `main` 分支,如果您想要參與 Gopeed 的開發,請先 fork 該項目,再在您自己的 fork 中進行開發,開發完成後再開啟PR,以合併至 `main` 分支。
## 離線開發
建議使用 web 端進行開發與調試,首先啟動服務,使用指令 `go run cmd/api/main.go` 啟動 ,該服務默認連接埠為 `9999`,接著以 `debug` 模式啟動前端 flutter 項目即可。
## 翻譯
Gopeed 的翻譯文件位於 `ui/flutter/lib/i18n/langs` 目錄中,只需要修改或新建翻譯文件即可。
請以 `en_us.dart` 作為參照。
## flutter開發
每次提交PR前請務必執行 `dart format ./ui/flutter`
如果需要編輯 api/models,請打開build_runner watcher:
```
flutter pub run build_runner watch
```
+2 -1
View File
@@ -1,4 +1,4 @@
FROM instrumentisto/flutter:3.10.6 AS flutter
FROM instrumentisto/flutter:3.16.1 AS flutter
WORKDIR /app
COPY ./ui/flutter/pubspec.yaml ./ui/flutter/pubspec.lock ./
RUN flutter pub get
@@ -17,5 +17,6 @@ FROM alpine:3.14.2
LABEL maintainer="monkeyWie"
WORKDIR /app
COPY --from=go /app/dist/gopeed ./
VOLUME ["/app/storage"]
EXPOSE 9999
ENTRYPOINT ["./gopeed"]
+9 -7
View File
@@ -2,20 +2,21 @@
[![Codecov](https://codecov.io/gh/GopeedLab/gopeed/branch/main/graph/badge.svg)](https://codecov.io/gh/GopeedLab/gopeed)
[![Release](https://img.shields.io/github/release/GopeedLab/gopeed.svg)](https://github.com/GopeedLab/gopeed/releases)
[![Download](https://img.shields.io/github/downloads/GopeedLab/gopeed/total.svg)](https://github.com/GopeedLab/gopeed/releases)
[![Donate](https://img.shields.io/badge/%24-donate-ff69b4.svg)](https://github.com/GopeedLab/gopeed/blob/main/.donate/index.md#donate)
[![Donate](https://img.shields.io/badge/%24-donate-ff69b4.svg)](https://docs.gopeed.com/donate.html)
[![Discord](https://img.shields.io/discord/1037992631881449472?label=Discord&logo=discord&style=social)](https://discord.gg/ZUJqJrwCGB)
![](_docs/img/banner.png)
[![ko-fi](https://ko-fi.com/img/githubbutton_sm.svg)](https://ko-fi.com/R6R6IJGN6)
[English](/README.md) | [中文](/README_zh-CN.md) | [日本語](/README_ja-JP.md)
[English](/README.md) | [中文](/README_zh-CN.md) | [日本語](/README_ja-JP.md) | [正體中文](/README_zh-TW.md)
## Introduction
Gopeed (full name Go Speed), a high-speed downloader developed by `Golang` + `Flutter`, supports (HTTP, BitTorrent, Magnet) protocol, and supports all platforms.
Gopeed (full name Go Speed), a high-speed downloader developed by `Golang` + `Flutter`, supports (HTTP, BitTorrent, Magnet) protocol, and supports all platforms. In addition to basic download functions, Gopeed is also a highly customizable downloader that supports implementing more features through integration with [APIs](https://docs.gopeed.com/dev-api.html) or installation and development of [extensions](https://docs.gopeed.com/dev-extension.html).
Visit ✈ [Official Website](https://gopeed.com) | 📖 [Develop Docs](https://docs.gopeed.com)
Visit ✈ [Official Website](https://gopeed.com) | 📖 [Official Docs](https://docs.gopeed.com)
## Install
@@ -46,9 +47,11 @@ go install github.com/GopeedLab/gopeed/cmd/gopeed@latest
#### Start directly
```bash
docker run -d -p 9999:9999 -v /path/to/download:/root/Downloads liwei2633/gopeed
docker run -d -p 9999:9999 -v /path/to/download:/root/Downloads -v /path/to/storage:/app/storage liwei2633/gopeed
```
[More details for docker](https://docs.gopeed.com/install.html#docker-installation)
#### Using Docker Compose
```bash
@@ -61,7 +64,7 @@ When the docker container is running, you can access the web page through `http:
## Donate
If you like this project, please consider [donating](/.donate/index.md#donate) to support the development of this project, thank you!
If you like this project, please consider [donating](https://docs.gopeed.com/donate.html) to support the development of this project, thank you!
## Showcase
@@ -159,7 +162,6 @@ cd ../../
rm -rf cmd/web/dist
cp -r ui/flutter/build/web cmd/web/dist
go build -tags nosqlite,web -ldflags="-s -w" -o bin/ github.com/GopeedLab/gopeed/cmd/web
go run cmd/web/main.go
```
## Credits
+6 -5
View File
@@ -2,18 +2,18 @@
[![Codecov](https://codecov.io/gh/GopeedLab/gopeed/branch/main/graph/badge.svg)](https://codecov.io/gh/GopeedLab/gopeed)
[![Release](https://img.shields.io/github/release/GopeedLab/gopeed.svg)](https://github.com/GopeedLab/gopeed/releases)
[![Download](https://img.shields.io/github/downloads/GopeedLab/gopeed/total.svg)](https://github.com/GopeedLab/gopeed/releases)
[![Donate](https://img.shields.io/badge/%24-donate-ff69b4.svg)](https://github.com/GopeedLab/gopeed/blob/main/.donate/index.md#donate)
[![Donate](https://img.shields.io/badge/%24-donate-ff69b4.svg)](https://docs.gopeed.com/donate.html)
[![Discord](https://img.shields.io/discord/1037992631881449472?label=Discord&logo=discord&style=social)](https://discord.gg/ZUJqJrwCGB)
![](_docs/img/banner.png)
[![ko-fi](https://ko-fi.com/img/githubbutton_sm.svg)](https://ko-fi.com/R6R6IJGN6)
[English](/README.md) | [中文](/README_zh-CN.md) | [日本語](/README_ja-JP.md)
[English](/README.md) | [中文](/README_zh-CN.md) | [日本語](/README_ja-JP.md) | [正體中文](/README_zh-TW.md)
## はじめに
Gopeed (正式名 Go Speed) は `Golang` + `Flutter` によって開発された高速ダウンローダーで、(HTTP、BitTorrent、Magnet) プロトコルをサポートし、すべてのプラットフォームをサポートします。
Gopeed (正式名 Go Speed) は `Golang` + `Flutter` によって開発された高速ダウンローダーで、(HTTP、BitTorrent、Magnet) プロトコルをサポートし、すべてのプラットフォームをサポートします。基本的なダウンロード機能に加え、[APIs](https://docs.gopeed.com/dev-api.html)との連動や[拡張機能](https://docs.gopeed.com/dev-extension.html)のインストール・開発による追加機能にも対応した、カスタマイズ性の高いダウンローダーです。
見て下さい ✈ [公式ウェブサイト](https://gopeed.com) | 📖 [開発ドキュメント](https://docs.gopeed.com)
@@ -46,9 +46,11 @@ go install github.com/GopeedLab/gopeed/cmd/gopeed@latest
#### 直接始める
```bash
docker run -d -p 9999:9999 -v /path/to/download:/root/Downloads liwei2633/gopeed
docker run -d -p 9999:9999 -v /path/to/download:/root/Downloads -v /path/to/storage:/app/storage liwei2633/gopeed
```
[Dockerの使用については](https://docs.gopeed.com/install.html#docker-installation)
#### Docker Compose の使用
```bash
@@ -159,7 +161,6 @@ cd ../../
rm -rf cmd/web/dist
cp -r ui/flutter/build/web cmd/web/dist
go build -tags nosqlite,web -ldflags="-s -w" -o bin/ github.com/GopeedLab/gopeed/cmd/web
go run cmd/web/main.go
```
## 感謝
+9 -8
View File
@@ -2,20 +2,20 @@
[![Codecov](https://codecov.io/gh/GopeedLab/gopeed/branch/main/graph/badge.svg)](https://codecov.io/gh/GopeedLab/gopeed)
[![Release](https://img.shields.io/github/release/GopeedLab/gopeed.svg)](https://github.com/GopeedLab/gopeed/releases)
[![Download](https://img.shields.io/github/downloads/GopeedLab/gopeed/total.svg)](https://github.com/GopeedLab/gopeed/releases)
[![Donate](https://img.shields.io/badge/%24-donate-ff69b4.svg)](https://github.com/GopeedLab/gopeed/blob/main/.donate/index.md#donate)
[![Donate](https://img.shields.io/badge/%24-donate-ff69b4.svg)](https://docs.gopeed.com/zh/donate.html)
[![Discord](https://img.shields.io/discord/1037992631881449472?label=Discord&logo=discord&style=social)](https://discord.gg/ZUJqJrwCGB)
![](_docs/img/banner.png)
[![ko-fi](https://ko-fi.com/img/githubbutton_sm.svg)](https://ko-fi.com/R6R6IJGN6)
[English](/README.md) | [中文](/README_zh-CN.md) | [日本語](/README_ja-JP.md)
[English](/README.md) | [中文](/README_zh-CN.md) | [日本語](/README_ja-JP.md) | [正體中文](/README_zh-TW.md)
## 介绍
Gopeed(全称 Go Speed),是一款由`Golang`+`Flutter`开发的高速下载器,支持(HTTP、BitTorrent、Magnet)协议下载,并且支持全平台使用。
Gopeed(全称 Go Speed),直译过来中文名叫做`够快下载器`(不是狗屁下载器!),是一款由`Golang`+`Flutter`开发的高速下载器,支持(HTTP、BitTorrent、Magnet)协议下载,并且支持全平台使用。除了基本的下载功能外,Gopeed 还是一款高度可定制化的下载器,支持通过对接[APIs](https://docs.gopeed.com/zh/dev-api.html)或者安装和开发[扩展](https://docs.gopeed.com/zh/dev-extension.html)来实现更多的功能。
访问 ✈ [官方网站](https://gopeed.com/zh-CN) | 📖 [开发文档](https://docs.gopeed.com/zh/)
访问 ✈ [官方网站](https://gopeed.com/zh-CN) | 📖 [官方文档](https://docs.gopeed.com/zh/)
## 安装
@@ -46,9 +46,11 @@ go install github.com/GopeedLab/gopeed/cmd/gopeed@latest
#### 直接运行
```bash
docker run -d -p 9999:9999 -v /path/to/download:/root/Downloads liwei2633/gopeed
docker run -d -p 9999:9999 -v /path/to/download:/root/Downloads -v /path/to/storage:/app/storage liwei2633/gopeed
```
[更多关于 Docker 的使用](https://docs.gopeed.com/zh/install.html#docker-%E5%AE%89%E8%A3%85)
#### 使用 Docker Compose
```bash
@@ -59,9 +61,9 @@ docker-compose up -d
当 docker 容器运行时,可以通过 `http://localhost:9999` 访问 web 页面。
## 打赏
## 赞助
如果觉得项目对你有帮助,请考虑[打赏](/.donate/index.md#donate)以支持这个项目的发展,非常感谢!
如果觉得项目对你有帮助,请考虑[赞助](https://docs.gopeed.com/zh/donate)以支持这个项目的发展,非常感谢!
## 界面展示
@@ -159,7 +161,6 @@ cd ../../
rm -rf cmd/web/dist
cp -r ui/flutter/build/web cmd/web/dist
go build -tags nosqlite,web -ldflags="-s -w" -o bin/ github.com/GopeedLab/gopeed/cmd/web
go run cmd/web/main.go
```
## 感谢
+180
View File
@@ -0,0 +1,180 @@
[![Test Status](https://github.com/GopeedLab/gopeed/workflows/test/badge.svg)](https://github.com/GopeedLab/gopeed/actions?query=workflow%3Atest)
[![Codecov](https://codecov.io/gh/GopeedLab/gopeed/branch/main/graph/badge.svg)](https://codecov.io/gh/GopeedLab/gopeed)
[![Release](https://img.shields.io/github/release/GopeedLab/gopeed.svg)](https://github.com/GopeedLab/gopeed/releases)
[![Download](https://img.shields.io/github/downloads/GopeedLab/gopeed/total.svg)](https://github.com/GopeedLab/gopeed/releases)
[![Donate](https://img.shields.io/badge/%24-donate-ff69b4.svg)](https://docs.gopeed.com/donate.html)
[![Discord](https://img.shields.io/discord/1037992631881449472?label=Discord&logo=discord&style=social)](https://discord.gg/ZUJqJrwCGB)
![](_docs/img/banner.png)
[![ko-fi](https://ko-fi.com/img/githubbutton_sm.svg)](https://ko-fi.com/R6R6IJGN6)
[English](/README.md) | [中文](/README_zh-CN.md) | [日本語](/README_ja-JP.md) | [正體中文](/README_zh-TW.md)
## 簡介
Gopeed(全稱 Go Speed),是一款使用`Golang`+`Flutter`編寫的高速下載軟體,支援(HTTP、BitTorrent、Magnet)協定,同時支援所有的平台。
前往 ✈ [主頁](https://gopeed.com/zh-CN) | 📖 [文檔](https://docs.gopeed.com/zh/)
## 安裝
**已支援的平台**
- [x] Windows
- [x] MacOS
- [x] Linux
- [x] Android
- [x] iOS
- [x] Web
- [x] Docker
[下載](https://github.com/GopeedLab/gopeed/releases/latest)
> 註:MacOS 版運行時若提示損壞,請在終端中執行 `xattr -d com.apple.quarantine /Applications/Gopeed.app`
### 使用CLI安裝
使用`go install`安裝:
```bash
go install github.com/GopeedLab/gopeed/cmd/gopeed@latest
```
### Docker
#### 直接執行
```bash
docker run -d -p 9999:9999 -v /path/to/download:/root/Downloads -v /path/to/storage:/app/storage liwei2633/gopeed
```
[更多關於 Docker 的使用](https://docs.gopeed.com/zh/install.html#docker-%E5%AE%89%E8%A3%85)
#### 使用 Docker Compose
```bash
docker-compose up -d
```
#### 訪問服務
當 docker 容器運作時,可以通過 `http://localhost:9999` 訪問 web 頁面。
## 贊助
如果你認為該項目對你有所幫助,請考慮[贊助](https://docs.gopeed.com/zh/donate)以支持該項目的持續發展,謝謝!
## 軟體介面
![](_docs/img/ui-demo.png)
## 開發
該項目分為前端與後端,前端使用`flutter`編寫,後端使用`Golang`編寫,兩邊通過`http`協定進行通訊,在 unix 系統下,則使用`unix socket`,在 windows 系統下,則使用`tcp`協定。
> 前端代碼位於`ui/flutter`目錄內。
### 開發環境
1. Golang 1.19+
2. Flutter 3.10+
### 克隆項目
```bash
git clone [email protected]:GopeedLab/gopeed.git
```
### 協助開發
請參考[協助指南](CONTRIBUTING_zh-TW.md)
### 編譯
#### 桌面端
首先需要按照[flutter desktop 官方文檔](https://docs.flutter.dev/development/platform-integration/desktop)配置開發環境,並準備好`cgo`環境,具體方法可以自行搜索。
組建指令:
- windows
```bash
go build -tags nosqlite -ldflags="-w -s" -buildmode=c-shared -o ui/flutter/windows/libgopeed.dll github.com/GopeedLab/gopeed/bind/desktop
cd ui/flutter
flutter build windows
```
- macos
```bash
go build -tags nosqlite -ldflags="-w -s" -buildmode=c-shared -o ui/flutter/macos/Frameworks/libgopeed.dylib github.com/GopeedLab/gopeed/bind/desktop
cd ui/flutter
flutter build macos
```
- linux
```bash
go build -tags nosqlite -ldflags="-w -s" -buildmode=c-shared -o ui/flutter/linux/bundle/lib/libgopeed.so github.com/GopeedLab/gopeed/bind/desktop
cd ui/flutter
flutter build linux
```
#### 移動設備
需要`cgo`環境,並安裝`gomobile`:
```bash
go install golang.org/x/mobile/cmd/gomobile@latest
go get golang.org/x/mobile/bind
gomobile init
```
組建指令:
- android
```bash
gomobile bind -tags nosqlite -ldflags="-w -s" -o ui/flutter/android/app/libs/libgopeed.aar -target=android -androidapi 19 -javapkg=com.gopeed github.com/GopeedLab/gopeed/bind/mobile
cd ui/flutter
flutter build apk
```
- ios
```bash
gomobile bind -tags nosqlite -ldflags="-w -s" -o ui/flutter/ios/Frameworks/Libgopeed.xcframework -target=ios github.com/GopeedLab/gopeed/bind/mobile
cd ui/flutter
flutter build ios --no-codesign
```
#### 網頁端
組建指令:
```bash
cd ui/flutter
flutter build web
cd ../../
rm -rf cmd/web/dist
cp -r ui/flutter/build/web cmd/web/dist
go build -tags nosqlite,web -ldflags="-s -w" -o bin/ github.com/GopeedLab/gopeed/cmd/web
```
## 感謝
### 貢獻者
<a href="https://github.com/GopeedLab/gopeed/graphs/contributors">
<img src="https://contrib.rocks/image?repo=GopeedLab/gopeed" />
</a>
### JetBrains
[![goland](_docs/img/goland.svg)](https://www.jetbrains.com/?from=gopeed)
## 軟體許可
該軟體遵循 [GPLv3](LICENSE) 。
+1
View File
@@ -15,6 +15,7 @@ func Start(cfg *C.char) (int, *C.char) {
if err := json.Unmarshal([]byte(C.GoString(cfg)), &config); err != nil {
return 0, C.CString(err.Error())
}
config.ProductionMode = true
realPort, err := rest.Start(&config)
if err != nil {
return 0, C.CString(err.Error())
+1
View File
@@ -13,6 +13,7 @@ func Start(cfg string) (int, error) {
if err := json.Unmarshal([]byte(cfg), &config); err != nil {
return 0, err
}
config.ProductionMode = true
return rest.Start(&config)
}
+5 -4
View File
@@ -1,16 +1,17 @@
package main
import (
_ "embed"
"github.com/GopeedLab/gopeed/cmd"
"github.com/GopeedLab/gopeed/pkg/rest/model"
)
// only for local development
func main() {
cfg := &model.StartConfig{
Network: "tcp",
Address: "127.0.0.1:9999",
Storage: model.StorageBolt,
Network: "tcp",
Address: "127.0.0.1:9999",
Storage: model.StorageBolt,
WebEnable: true,
}
cmd.Start(cfg)
}
+14 -1
View File
@@ -2,6 +2,7 @@ package main
import (
"fmt"
"github.com/GopeedLab/gopeed/internal/fetcher"
"github.com/GopeedLab/gopeed/pkg/base"
"github.com/GopeedLab/gopeed/pkg/download"
"github.com/GopeedLab/gopeed/pkg/protocol/http"
@@ -47,16 +48,28 @@ func main() {
if err != nil {
panic(err)
}
printProgress(emptyTask, "downloading...")
wg.Wait()
}
var (
lastLineLen = 0
sb = new(strings.Builder)
emptyTask = &download.Task{
Progress: &download.Progress{},
Meta: &fetcher.FetcherMeta{
Res: &base.Resource{},
},
}
)
func printProgress(task *download.Task, title string) {
rate := float64(task.Progress.Downloaded) / float64(task.Meta.Res.Size)
var rate float64
if task.Meta.Res.Size <= 0 {
rate = 0
} else {
rate = float64(task.Progress.Downloaded) / float64(task.Meta.Res.Size)
}
completeWidth := int(progressWidth * rate)
speed := util.ByteFmt(task.Progress.Speed)
totalSize := util.ByteFmt(task.Meta.Res.Size)
+1 -3
View File
@@ -23,13 +23,11 @@ func parse() *args {
cliArgs.Address = flag.String("A", "0.0.0.0", "Bind Address")
cliArgs.Port = flag.Int("P", 9999, "Bind Port")
cliArgs.Username = flag.String("u", "gopeed", "HTTP Basic Auth Username")
cliArgs.Password = flag.String("p", "", "HTTP Basic Auth Password")
cliArgs.Password = flag.String("p", "", "HTTP Basic Auth Pwd")
cliArgs.ApiToken = flag.String("T", "", "API token, that can only be used when basic authentication is enabled.")
cliArgs.configPath = flag.String("c", "./config.json", "Config file path")
flag.Parse()
fmt.Println("parse")
// args priority: config file > cli args
cfgArgs := loadConfig(*cliArgs.configPath)
if cfgArgs.Address == nil {
+17 -7
View File
@@ -9,6 +9,8 @@ import (
"github.com/GopeedLab/gopeed/cmd"
"github.com/GopeedLab/gopeed/pkg/rest/model"
"io/fs"
"os"
"path/filepath"
)
//go:embed dist/*
@@ -29,14 +31,22 @@ func main() {
}
}
exe, err := os.Executable()
if err != nil {
panic(err)
}
dir := filepath.Dir(exe)
cfg := &model.StartConfig{
Network: "tcp",
Address: fmt.Sprintf("%s:%d", *args.Address, *args.Port),
Storage: model.StorageBolt,
ApiToken: *args.ApiToken,
WebEnable: true,
WebFS: sub,
WebBasicAuth: webBasicAuth,
Network: "tcp",
Address: fmt.Sprintf("%s:%d", *args.Address, *args.Port),
Storage: model.StorageBolt,
StorageDir: filepath.Join(dir, "storage"),
ApiToken: *args.ApiToken,
ProductionMode: true,
WebEnable: true,
WebFS: sub,
WebBasicAuth: webBasicAuth,
}
cmd.Start(cfg)
}
+3 -2
View File
@@ -4,8 +4,9 @@ services:
gopeed:
container_name: gopeed
ports:
- 9999:9999 # HTTP port (host:container)
- "9999:9999" # HTTP port (host:container)
image: liwei2633/gopeed
volumes:
- ~/Downloads:/root/Downloads # replace with your prefer path
- ~/gopeed/Downloads:/root/Downloads # mount download path
#- ~/gopeed/storage:/app/storage # if you need to mount storage path, uncomment this line
restart: unless-stopped
+35 -6
View File
@@ -1,21 +1,31 @@
module github.com/GopeedLab/gopeed
go 1.20
go 1.19
require (
github.com/anacrolix/missinggo/v2 v2.7.2-0.20230527121029-a582b4f397b9
github.com/anacrolix/torrent v1.52.5
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5
github.com/dop251/goja v0.0.0-20231027120936-b396bb4c349d
github.com/dop251/goja_nodejs v0.0.0-20231122114759-e84d9a924c5c
github.com/go-git/go-git/v5 v5.8.1
github.com/gorilla/handlers v1.5.1
github.com/gorilla/mux v1.8.0
github.com/matoous/go-nanoid/v2 v2.0.0
github.com/rs/zerolog v1.31.0
github.com/virtuald/go-paniclog v0.0.0-20190812204905-43a7fa316459
go.etcd.io/bbolt v1.3.7-0.20231005155708-db45d0d6b1f6
go.etcd.io/bbolt v1.3.8
golang.org/x/exp v0.0.0-20220613132600-b0d781184e0d
golang.org/x/sync v0.3.0
)
require (
crawshaw.io/sqlite v0.3.3-0.20220618202545-d1964889ea3c // indirect
dario.cat/mergo v1.0.0 // indirect
github.com/Microsoft/go-winio v0.6.1 // indirect
github.com/ProtonMail/go-crypto v0.0.0-20230717121422-5aa5874ade95 // indirect
github.com/RoaringBitmap/roaring v1.2.3 // indirect
github.com/acomagu/bufpipe v1.0.4 // indirect
github.com/ajwerner/btree v0.0.0-20211221152037-f427b3e689c0 // indirect
github.com/alecthomas/atomic v0.1.0-alpha2 // indirect
github.com/anacrolix/chansync v0.3.0 // indirect
@@ -36,17 +46,29 @@ require (
github.com/benbjohnson/immutable v0.3.0 // indirect
github.com/bits-and-blooms/bitset v1.2.2 // indirect
github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8 // indirect
github.com/cloudflare/circl v1.3.3 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/dlclark/regexp2 v1.10.0 // indirect
github.com/dustin/go-humanize v1.0.0 // indirect
github.com/edsrzf/mmap-go v1.1.0 // indirect
github.com/emirpasic/gods v1.18.1 // indirect
github.com/felixge/httpsnoop v1.0.1 // indirect
github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
github.com/go-git/go-billy/v5 v5.4.1 // indirect
github.com/go-logr/logr v1.2.3 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-sourcemap/sourcemap v2.1.3+incompatible // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/google/btree v1.1.2 // indirect
github.com/google/pprof v0.0.0-20231101202521-4ca4178f5c7a // indirect
github.com/google/uuid v1.3.0 // indirect
github.com/gorilla/websocket v1.5.0 // indirect
github.com/huandu/xstrings v1.3.2 // indirect
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
github.com/kevinburke/ssh_config v1.2.0 // indirect
github.com/lispad/go-generics-tools v1.1.0 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.19 // indirect
github.com/mschoch/smat v0.2.0 // indirect
github.com/pion/datachannel v1.5.2 // indirect
github.com/pion/dtls/v2 v2.2.4 // indirect
@@ -66,14 +88,21 @@ require (
github.com/pion/turn/v2 v2.0.8 // indirect
github.com/pion/udp v0.1.4 // indirect
github.com/pion/webrtc/v3 v3.1.42 // indirect
github.com/pjbgf/sha1cd v0.3.0 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/rs/dnscache v0.0.0-20211102005908-e0241e321417 // indirect
github.com/sergi/go-diff v1.1.0 // indirect
github.com/skeema/knownhosts v1.2.0 // indirect
github.com/tidwall/btree v1.6.0 // indirect
github.com/xanzy/ssh-agent v0.3.3 // indirect
go.opentelemetry.io/otel v1.8.0 // indirect
go.opentelemetry.io/otel/trace v1.8.0 // indirect
golang.org/x/crypto v0.5.0 // indirect
golang.org/x/exp v0.0.0-20220613132600-b0d781184e0d // indirect
golang.org/x/net v0.7.0 // indirect
golang.org/x/sys v0.13.0 // indirect
golang.org/x/crypto v0.14.0 // indirect
golang.org/x/mod v0.8.0 // indirect
golang.org/x/net v0.17.0 // indirect
golang.org/x/sys v0.14.0 // indirect
golang.org/x/text v0.14.0 // indirect
golang.org/x/time v0.0.0-20220609170525-579cf78fd858 // indirect
golang.org/x/tools v0.6.0 // indirect
gopkg.in/warnings.v0 v0.1.2 // indirect
)
+115 -7
View File
@@ -5,8 +5,15 @@ crawshaw.io/iox v0.0.0-20181124134642-c51c3df30797/go.mod h1:sXBiorCo8c46JlQV3oX
crawshaw.io/sqlite v0.3.2/go.mod h1:igAO5JulrQ1DbdZdtVq48mnZUBAPOeFzer7VhDWNtW4=
crawshaw.io/sqlite v0.3.3-0.20220618202545-d1964889ea3c h1:wvzox0eLO6CKQAMcOqz7oH3UFqMpMmK7kwmwV+22HIs=
crawshaw.io/sqlite v0.3.3-0.20220618202545-d1964889ea3c/go.mod h1:igAO5JulrQ1DbdZdtVq48mnZUBAPOeFzer7VhDWNtW4=
dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk=
dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk=
filippo.io/edwards25519 v1.0.0-rc.1 h1:m0VOOB23frXZvAOK44usCgLWvtsxIoMCTBGJZlpmGfU=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY=
github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow=
github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM=
github.com/ProtonMail/go-crypto v0.0.0-20230717121422-5aa5874ade95 h1:KLq8BE0KwCL+mmXnjLWEAOYO+2l2AE4YMmqG1ZpZHBs=
github.com/ProtonMail/go-crypto v0.0.0-20230717121422-5aa5874ade95/go.mod h1:EjAoLdwvbIOoOQr3ihjnSoLZRtE8azugULFRteWMNc0=
github.com/RoaringBitmap/roaring v0.4.7/go.mod h1:8khRDP4HmeXns4xIj9oGrKSz7XTQiJx2zgh7AcNke4w=
github.com/RoaringBitmap/roaring v0.4.17/go.mod h1:D3qVegWTmfCaX4Bl5CrBE9hfrSrrXIr8KVNvRsDi1NI=
github.com/RoaringBitmap/roaring v0.4.23/go.mod h1:D0gp8kJQgE1A4LQ5wFLggQEyvDi06Mq5mKs52e1TwOo=
@@ -14,6 +21,8 @@ github.com/RoaringBitmap/roaring v1.2.3 h1:yqreLINqIrX22ErkKI0vY47/ivtJr6n+kMhVO
github.com/RoaringBitmap/roaring v1.2.3/go.mod h1:plvDsJQpxOC5bw8LRteu/MLWHsHez/3y6cubLI4/1yE=
github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo=
github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI=
github.com/acomagu/bufpipe v1.0.4 h1:e3H4WUzM3npvo5uv95QuJM3cQspFNtFBzvJ2oNjKIDQ=
github.com/acomagu/bufpipe v1.0.4/go.mod h1:mxdxdup/WdsKVreO5GpW4+M/1CE2sMG4jeGJ2sYmHc4=
github.com/ajwerner/btree v0.0.0-20211221152037-f427b3e689c0 h1:byYvvbfSo3+9efR4IeReh77gVs4PnNDR3AMOE9NJ7a0=
github.com/ajwerner/btree v0.0.0-20211221152037-f427b3e689c0/go.mod h1:q37NoqncT41qKc048STsifIt69LfUJ8SrWWcz/yam5k=
github.com/alecthomas/assert/v2 v2.0.0-alpha3 h1:pcHeMvQ3OMstAWgaeaXIAL8uzB9xMm2zlxt+/4ml8lk=
@@ -79,7 +88,10 @@ github.com/anacrolix/upnp v0.1.3-0.20220123035249-922794e51c96 h1:QAVZ3pN/J4/Uzi
github.com/anacrolix/upnp v0.1.3-0.20220123035249-922794e51c96/go.mod h1:Wa6n8cYIdaG35x15aH3Zy6d03f7P728QfdcDeD/IEOs=
github.com/anacrolix/utp v0.1.0 h1:FOpQOmIwYsnENnz7tAGohA+r6iXpRjrq8ssKSre2Cp4=
github.com/anacrolix/utp v0.1.0/go.mod h1:MDwc+vsGEq7RMw6lr2GKOEqjWny5hO5OZXRVNaBJ2Dk=
github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8=
github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk=
github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg=
github.com/benbjohnson/immutable v0.2.0/go.mod h1:uc6OHo6PN2++n98KHLxW8ef4W42ylHiQSENghE1ezxI=
@@ -95,13 +107,31 @@ github.com/bradfitz/iter v0.0.0-20140124041915-454541ec3da2/go.mod h1:PyRFw1Lt2w
github.com/bradfitz/iter v0.0.0-20190303215204-33e6a9893b0c/go.mod h1:PyRFw1Lt2wKX4ZVSQ2mk+PeDa1rxyObEDlApuIsUKuo=
github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8 h1:GKTyiRCL6zVf5wWaqKnf+7Qs6GbEPfd4iMOitWzXJx8=
github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8/go.mod h1:spo1JLcs67NmW1aVLEgtA8Yy1elc+X8y5SRW1sFW4Og=
github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0=
github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/chzyer/logex v1.2.0/go.mod h1:9+9sk7u7pGNWYMkh0hdiL++6OeibzJccyQU4p4MedaY=
github.com/chzyer/readline v1.5.0/go.mod h1:x22KAscuvRqlLoK9CsoYsmxoXZMMFVyOl86cAH8qUic=
github.com/chzyer/test v0.0.0-20210722231415-061457976a23/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
github.com/cloudflare/circl v1.3.3 h1:fE/Qz0QdIGqeWfnwq0RE0R7MI51s0M2E4Ga9kq5AEMs=
github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA=
github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dlclark/regexp2 v1.4.1-0.20201116162257-a2a8dda75c91/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc=
github.com/dlclark/regexp2 v1.7.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
github.com/dlclark/regexp2 v1.10.0 h1:+/GIL799phkJqYW+3YbOd8LCcbHzT0Pbo8zl70MHsq0=
github.com/dlclark/regexp2 v1.10.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
github.com/dop251/goja v0.0.0-20211022113120-dc8c55024d06/go.mod h1:R9ET47fwRVRPZnOGvHxxhuZcbrMCuiqOz3Rlrh4KSnk=
github.com/dop251/goja v0.0.0-20231027120936-b396bb4c349d h1:wi6jN5LVt/ljaBG4ue79Ekzb12QfJ52L9Q98tl8SWhw=
github.com/dop251/goja v0.0.0-20231027120936-b396bb4c349d/go.mod h1:QMWlm50DNe14hD7t24KEqZuUdC9sOTy8W6XbCU1mlw4=
github.com/dop251/goja_nodejs v0.0.0-20210225215109-d91c329300e7/go.mod h1:hn7BA7c8pLvoGndExHudxTDKZ84Pyvv+90pbBjbTz0Y=
github.com/dop251/goja_nodejs v0.0.0-20211022123610-8dd9abb0616d/go.mod h1:DngW8aVqWbuLRMHItjPUyqdj+HWPvnQe8V8y1nDpIbM=
github.com/dop251/goja_nodejs v0.0.0-20231122114759-e84d9a924c5c h1:hLoodLRD4KLWIH8eyAQCLcH8EqIrjac7fCkp/fHnvuQ=
github.com/dop251/goja_nodejs v0.0.0-20231122114759-e84d9a924c5c/go.mod h1:bhGPmCgCCTSRfiMYWjpS46IDo9EUZXlsuUaPXSWGbv0=
github.com/dustin/go-humanize v0.0.0-20180421182945-02af3965c54e/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo=
github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
@@ -110,18 +140,29 @@ github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1
github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I=
github.com/edsrzf/mmap-go v1.1.0 h1:6EUwBLQ/Mcr1EYLE4Tn1VdW1A4ckqCQWZBw8Hr0kjpQ=
github.com/edsrzf/mmap-go v1.1.0/go.mod h1:19H/e8pUPLicwkyNgOykDXkJ9F0MHE+Z52B8EIth78Q=
github.com/elazarl/goproxy v0.0.0-20221015165544-a0805db90819 h1:RIB4cRk+lBqKK3Oy0r2gRX4ui7tuhiZq2SuTtTCi0/0=
github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc=
github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ=
github.com/felixge/httpsnoop v1.0.1 h1:lvB5Jl89CsZtGIWuTcDM1E/vkVs49/Ml7JJe07l8SPQ=
github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
github.com/frankban/quicktest v1.9.0/go.mod h1:ui7WezCLWMWxVWr1GETZY3smRy0G4KWq9vcPtJmFl7Y=
github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
github.com/gliderlabs/ssh v0.3.5 h1:OcaySEmAQJgyYcArR+gGGTHCyE7nvhEMTlYY+Dp8CpY=
github.com/glycerine/go-unsnap-stream v0.0.0-20180323001048-9f0cb55181dd/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE=
github.com/glycerine/go-unsnap-stream v0.0.0-20181221182339-f9677308dec2/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE=
github.com/glycerine/go-unsnap-stream v0.0.0-20190901134440-81cf024a9e0a/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE=
github.com/glycerine/goconvey v0.0.0-20180728074245-46e3a41ad493/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24=
github.com/glycerine/goconvey v0.0.0-20190315024820-982ee783a72e/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24=
github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24=
github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI=
github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmSxCcxctByoQdvwPiA7DTK7jaaFDBTtu0ic=
github.com/go-git/go-billy/v5 v5.4.1 h1:Uwp5tDRkPr+l/TnbHOQzp+tmJfLceOlbVucgpTz8ix4=
github.com/go-git/go-billy/v5 v5.4.1/go.mod h1:vjbugF6Fz7JIflbVpl1hJsGjSHNltrSw45YK/ukIvQg=
github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20230305113008-0c11038e723f h1:Pz0DHeFij3XFhoBRGUDPzSJ+w2UcK5/0JvF8DRI58r8=
github.com/go-git/go-git/v5 v5.8.1 h1:Zo79E4p7TRk0xoRgMq0RShiTHGKcKI4+DI6BfJc/Q+A=
github.com/go-git/go-git/v5 v5.8.1/go.mod h1:FHFuoD6yGz5OSKEBK+aWN9Oah0q54Jxl0abmj6GnqAo=
github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
@@ -131,13 +172,18 @@ github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0=
github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
github.com/go-sourcemap/sourcemap v2.1.3+incompatible h1:W1iEw64niKVGogNgBN3ePyLFfuisuzeidWPMPWmECqU=
github.com/go-sourcemap/sourcemap v2.1.3+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg=
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE=
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
@@ -163,6 +209,9 @@ github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/pprof v0.0.0-20230207041349-798e818bf904/go.mod h1:uglQLonpP8qtYCYyzA+8c/9qtqgA3qsXGYqCPKARAFg=
github.com/google/pprof v0.0.0-20231101202521-4ca4178f5c7a h1:fEBsGL/sjAuJrgah5XqmmYsTLzJp/TO9Lhy39gkverk=
github.com/google/pprof v0.0.0-20231101202521-4ca4178f5c7a/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik=
github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
@@ -186,16 +235,22 @@ github.com/huandu/xstrings v1.3.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq
github.com/huandu/xstrings v1.3.1/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE=
github.com/huandu/xstrings v1.3.2 h1:L18LIDzqlW6xN2rEkpdV8+oL/IXWJ1APd+vsdYy4Wdw=
github.com/huandu/xstrings v1.3.2/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE=
github.com/ianlancetaylor/demangle v0.0.0-20220319035150-800ac71e25c2/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w=
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A=
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo=
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4=
github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
@@ -207,6 +262,13 @@ github.com/lispad/go-generics-tools v1.1.0/go.mod h1:2csd1EJljo/gy5qG4khXol7ivCP
github.com/matoous/go-nanoid v1.5.0/go.mod h1:zyD2a71IubI24efhpvkJz+ZwfwagzgSO6UNiFsZKN7U=
github.com/matoous/go-nanoid/v2 v2.0.0 h1:d19kur2QuLeHmJBkvYkFdhFBzLoo1XVm2GgTpL+9Tj0=
github.com/matoous/go-nanoid/v2 v2.0.0/go.mod h1:FtS4aGPVfEkxKxhdWPAspZpZSh1cOjtM7Ej/So3hR0g=
github.com/matryer/is v1.2.0 h1:92UTHpy8CDwaJ08GqLDzhhuixiBUUD1p3AU6PHddz4A=
github.com/matryer/is v1.2.0/go.mod h1:2fLPjFQM9rhQ15aVEtbuwhJinnOqrmgXPNdZsdwlWXA=
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA=
github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
@@ -216,6 +278,7 @@ github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE
github.com/mschoch/smat v0.2.0 h1:8imxQsjDm8yFEAVBe7azKmKSgzSkZXDuKkSq9374khM=
github.com/mschoch/smat v0.2.0/go.mod h1:kc9mz7DoBKqDyiRL7VZN8KvXQMWeTaVnttLRXOlotKw=
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
@@ -273,6 +336,8 @@ github.com/pion/udp v0.1.4 h1:OowsTmu1Od3sD6i3fQUJxJn2fEvJO6L1TidgadtbTI8=
github.com/pion/udp v0.1.4/go.mod h1:G8LDo56HsFwC24LIcnT4YIDU5qcB6NepqqjP0keL2us=
github.com/pion/webrtc/v3 v3.1.42 h1:wJEQFIXVanptnQcHOLTuIo4AtGB2+mG2x4OhIhnITOA=
github.com/pion/webrtc/v3 v3.1.42/go.mod h1:ffD9DulDrPxyWvDPUIPAOSAWx9GUlOExiJPf7cCcMLA=
github.com/pjbgf/sha1cd v0.3.0 h1:4D5XXmUUBUl/xQ6IjCkEAbqXskkq/4O7LmGn0AqMDs4=
github.com/pjbgf/sha1cd v0.3.0/go.mod h1:nZ1rrWOcGJ5uZgEEVL1VUM9iRQiZvWdbZjkKyFzPPsI=
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@@ -302,11 +367,19 @@ github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6po
github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
github.com/rs/dnscache v0.0.0-20211102005908-e0241e321417 h1:Lt9DzQALzHoDwMBGJ6v8ObDPR0dzr2a6sXTB1Fq7IHs=
github.com/rs/dnscache v0.0.0-20211102005908-e0241e321417/go.mod h1:qe5TWALJ8/a1Lqznoc5BDHpYX/8HU60Hm2AwRmqzxqA=
github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
github.com/rs/zerolog v1.31.0 h1:FcTR3NnLWW+NnTwwhFWiJSZr4ECLpqCm6QsEnyvbV4A=
github.com/rs/zerolog v1.31.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss=
github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 h1:GHRpF1pTW19a8tTFrMLUcfWwyC0pnifVo2ClaLq+hP8=
github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46/go.mod h1:uAQ5PCi+MFsC7HjREoAz1BU+Mq60+05gifQSsHSDG/8=
github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZKJ48R1S7H23Ji7oFO5Bw=
github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0=
github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
github.com/skeema/knownhosts v1.2.0 h1:h9r9cf0+u7wSE+M183ZtMGgOJKiL96brpaz5ekfJCpM=
github.com/skeema/knownhosts v1.2.0/go.mod h1:g4fPeYpque7P0xefxtGzV81ihjC8sX2IqpAoNkjxbMo=
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
github.com/smartystreets/assertions v0.0.0-20190215210624-980c5ac6f3ac/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s=
@@ -324,8 +397,8 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
github.com/tidwall/btree v1.6.0 h1:LDZfKfQIBHGHWSwckhXI0RPSXzlo+KYdjK7FWSqOzzg=
github.com/tidwall/btree v1.6.0/go.mod h1:twD9XRA5jj9VUQGELzDO4HPQTNJsoWWfYEL+EUQ2cKY=
github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE=
@@ -335,10 +408,12 @@ github.com/virtuald/go-paniclog v0.0.0-20190812204905-43a7fa316459 h1:x9pIfbdIjn
github.com/virtuald/go-paniclog v0.0.0-20190812204905-43a7fa316459/go.mod h1:nFvuG3SWu3VWqobG3cX8nt57wXU0OOFapeCs/8axIuM=
github.com/willf/bitset v1.1.9/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4=
github.com/willf/bitset v1.1.10/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4=
github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM=
github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw=
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
go.etcd.io/bbolt v1.3.7-0.20231005155708-db45d0d6b1f6 h1:NkHVT1GBVV9hUj1iWglFxAuNlvLHELFn71U2H7aOIog=
go.etcd.io/bbolt v1.3.7-0.20231005155708-db45d0d6b1f6/go.mod h1:tDI2vlr15+D9cyN3odhFrZJi3oJj8HlHaOEFPrUiGSY=
go.etcd.io/bbolt v1.3.8 h1:xs88BrvEv273UsB79e0hcVrlUWmS0a8upikMFhSyAtA=
go.etcd.io/bbolt v1.3.8/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw=
go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=
go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=
go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
@@ -354,8 +429,12 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y
golang.org/x/crypto v0.0.0-20220131195533-30dcbda58838/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.0.0-20220427172511-eb4f295cb31f/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.0.0-20220516162934-403b01795ae8/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.5.0 h1:U/0M97KRkSFvyD/3FSmdP5W5swImpNgle/EHFhOsQPE=
golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4=
golang.org/x/crypto v0.5.0/go.mod h1:NK/OQwhpMQP3MwtdjgLlYHnH9ebylxKWv3e0fK+mkQU=
golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU=
golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc=
golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20220613132600-b0d781184e0d h1:vtUKgx8dahOomfFzLREU8nSv25YHnTgLBn4rDnWZdU0=
golang.org/x/exp v0.0.0-20220613132600-b0d781184e0d/go.mod h1:Kr81I6Kryrl9sr8s2FK3vxD90NdsKWRuOIl2O4CvYbA=
@@ -365,6 +444,8 @@ golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTk
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/mod v0.8.0 h1:LUYupSeNrTNCGzR/hVBk2NHZO4hXcVaW1k4Qx7rjPx8=
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -390,9 +471,13 @@ golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su
golang.org/x/net v0.0.0-20220531201128-c960675eff93/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY=
golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws=
golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g=
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM=
golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -403,6 +488,7 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E=
golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -416,6 +502,7 @@ golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -424,31 +511,45 @@ golang.org/x/sys v0.0.0-20200413165638-669c56c373c4/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220608164250-635b8c9b7f68/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE=
golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q=
golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc=
golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ=
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U=
golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/time v0.0.0-20220609170525-579cf78fd858 h1:Dpdu/EMxGMFgq0CeYMh4fazTD2vtlZRYE7wyynxJb9U=
golang.org/x/time v0.0.0-20220609170525-579cf78fd858/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -460,6 +561,8 @@ golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM=
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -485,14 +588,19 @@ gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLks
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME=
gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI=
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+6 -5
View File
@@ -1,13 +1,14 @@
package controller
import (
"errors"
"net/url"
"os"
"path/filepath"
)
type Controller struct {
GetConfig func(v any) (bool, error)
GetConfig func(v any) bool
ProxyUrl *url.URL
FileController
//ContextDialer() (proxy.Dialer, error)
}
@@ -22,8 +23,8 @@ type DefaultFileController struct {
func NewController() *Controller {
return &Controller{
FileController: &DefaultFileController{},
GetConfig: func(v any) (bool, error) {
return false, nil
GetConfig: func(v any) bool {
return false
},
}
}
@@ -34,7 +35,7 @@ func (c *DefaultFileController) Touch(name string, size int64) (file *os.File, e
return
}
file, err = os.Create(name)
if err != nil && !errors.Is(err, os.ErrExist) {
if err != nil {
return
}
if size > 0 {
+28 -7
View File
@@ -12,14 +12,13 @@ type Fetcher interface {
// Name return the name of the protocol.
Name() string
Setup(ctl *controller.Controller) error
Setup(ctl *controller.Controller)
// Resolve resource info from request
Resolve(req *base.Request) error
// Create ready to download, but not started
Create(opts *base.Options) error
Start() error
Pause() error
Continue() error
Close() error
// Meta returns the meta information of the download.
@@ -37,12 +36,34 @@ type FetcherMeta struct {
Opts *base.Options `json:"opts"`
}
func (m *FetcherMeta) Filepath(file *base.FileInfo) string {
finalName := m.Opts.Name
if finalName == "" {
finalName = file.Name
// FolderPath return the folder path of the meta info.
func (m *FetcherMeta) FolderPath() string {
// check if rename folder
folder := m.Res.Name
if m.Opts.Name != "" {
folder = m.Opts.Name
}
return path.Join(m.Opts.Path, folder)
}
// SingleFilepath return the single file path of the meta info.
func (m *FetcherMeta) SingleFilepath() string {
// check if rename file
file := m.Res.Files[0]
fileName := file.Name
if m.Opts.Name != "" {
fileName = m.Opts.Name
}
return path.Join(m.Opts.Path, file.Path, fileName)
}
// RootDirPath return the root dir path of the task file.
func (m *FetcherMeta) RootDirPath() string {
if m.Res.Name != "" {
return m.FolderPath()
} else {
return m.Opts.Path
}
return path.Join(m.Opts.Path, m.Res.RootDir, finalName)
}
// FetcherBuilder defines the interface for a fetcher builder.
+52
View File
@@ -0,0 +1,52 @@
package logger
import (
"github.com/GopeedLab/gopeed/pkg/util"
"github.com/rs/zerolog"
"io"
"os"
"path/filepath"
)
type Logger struct {
zerolog.Logger
logFile *os.File
}
func (l *Logger) CLose() {
l.logFile.Close()
}
// NewLogger create a new logger
func NewLogger(logFile bool, logPath string) *Logger {
var out io.Writer
if logFile {
// log to file
logDir := filepath.Dir(logPath)
if err := util.CreateDirIfNotExist(logDir); err != nil {
panic(err)
}
var (
logfile *os.File
err error
)
logfile, err = os.OpenFile(logPath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666)
if err != nil {
panic(err)
}
out = logfile
} else {
out = os.Stdout
}
logger := &Logger{}
if logFile {
logger.logFile = out.(*os.File)
}
logger.Logger = zerolog.New(zerolog.ConsoleWriter{
NoColor: true,
Out: out,
TimeFormat: "2006-01-02 15:04:05",
}).With().Timestamp().Logger()
return logger
}
+35
View File
@@ -0,0 +1,35 @@
package logger
import (
"os"
"testing"
)
func TestNewLogger(t *testing.T) {
logger := NewLogger(false, "")
logger.Info().Msg("test")
}
func TestNewLoggerFile(t *testing.T) {
logPath := "./testdata/test.log"
logger := NewLogger(true, logPath)
defer func() {
logger.CLose()
os.Remove(logPath)
}()
logger.Info().Msg("test")
// read log file, if not exist or empty, test fail.
file, err := os.Open(logPath)
if err != nil {
t.Fatal(err)
}
defer file.Close()
stat, err := file.Stat()
if err != nil {
t.Fatal(err)
}
if stat.Size() == 0 {
t.Fatal("log file is empty")
}
}
+2 -1
View File
@@ -1,5 +1,6 @@
package bt
type config struct {
Trackers []string `json:"trackers"`
ListenPort int `json:"listenPort"`
Trackers []string `json:"trackers"`
}
+54 -55
View File
@@ -9,7 +9,7 @@ import (
"github.com/GopeedLab/gopeed/pkg/util"
"github.com/anacrolix/torrent"
"github.com/anacrolix/torrent/metainfo"
"path"
"net/http"
"path/filepath"
"sync"
"sync/atomic"
@@ -24,7 +24,8 @@ var (
)
type Fetcher struct {
ctl *controller.Controller
ctl *controller.Controller
config *config
torrent *torrent.Torrent
meta *fetcher.FetcherMeta
@@ -38,11 +39,18 @@ func (f *Fetcher) Name() string {
return "bt"
}
func (f *Fetcher) Setup(ctl *controller.Controller) (err error) {
func (f *Fetcher) Setup(ctl *controller.Controller) {
f.ctl = ctl
if f.meta == nil {
f.meta = &fetcher.FetcherMeta{}
}
exist := f.ctl.GetConfig(&f.config)
if !exist {
f.config = &config{
ListenPort: 0,
Trackers: []string{},
}
}
return
}
@@ -55,7 +63,10 @@ func (f *Fetcher) initClient() (err error) {
}
cfg := torrent.NewDefaultClientConfig()
cfg.ListenPort = 0
cfg.ListenPort = f.config.ListenPort
if f.ctl.ProxyUrl != nil {
cfg.HTTPProxy = http.ProxyURL(f.ctl.ProxyUrl)
}
cfg.DefaultStorage = newFileOpts(newFileClientOpts{
ClientBaseDir: cfg.DataDir,
HandleFileTorrent: func(infoHash metainfo.Hash, ft *fileTorrentImpl) {
@@ -77,52 +88,17 @@ func (f *Fetcher) Resolve(req *base.Request) error {
if err := f.addTorrent(req); err != nil {
return err
}
go func() {
// recycle unused torrent resource
time.Sleep(time.Minute * 3)
if !f.create.Load() {
f.torrentReady.Store(false)
f.safeDrop()
}
}()
res := &base.Resource{
Name: f.torrent.Name(),
Range: true,
RootDir: f.torrent.Name(),
Files: make([]*base.FileInfo, len(f.torrent.Files())),
Hash: f.torrent.InfoHash().String(),
}
for i, file := range f.torrent.Files() {
res.Files[i] = &base.FileInfo{
Name: filepath.Base(file.DisplayPath()),
Path: util.Dir(file.Path()),
Size: file.Length(),
}
res.Size += file.Length()
}
f.updateRes()
f.meta.Req = req
f.meta.Res = res
return nil
}
func (f *Fetcher) Create(opts *base.Options) (err error) {
f.create.Store(true)
f.meta.Opts = opts
if len(opts.SelectFiles) == 0 {
opts.SelectFiles = make([]int, 0)
for i := range f.torrent.Files() {
opts.SelectFiles = append(opts.SelectFiles, i)
}
if f.meta.Res != nil {
torrentDirMap[f.meta.Res.Hash] = f.meta.FolderPath()
}
if opts.Path != "" {
torrentDirMap[f.meta.Res.Hash] = path.Join(f.meta.Opts.Path, f.meta.Res.RootDir)
if ft, ok := ftMap[f.meta.Res.Hash]; ok {
// reuse resolve fetcher
ft.setTorrentDir(torrentDirMap[f.meta.Res.Hash])
}
}
f.progress = make(fetcher.Progress, len(f.meta.Opts.SelectFiles))
return nil
}
@@ -132,7 +108,18 @@ func (f *Fetcher) Start() (err error) {
return
}
}
if ft, ok := ftMap[f.meta.Res.Hash]; ok {
ft.setTorrentDir(f.meta.FolderPath())
}
files := f.torrent.Files()
// If the user does not specify the file to download, all files will be downloaded by default
if len(f.meta.Opts.SelectFiles) == 0 {
f.meta.Opts.SelectFiles = make([]int, len(files))
for i := range files {
f.meta.Opts.SelectFiles[i] = i
}
}
f.progress = make(fetcher.Progress, len(f.meta.Opts.SelectFiles))
if len(f.meta.Opts.SelectFiles) == len(files) {
f.torrent.DownloadAll()
} else {
@@ -150,10 +137,6 @@ func (f *Fetcher) Pause() (err error) {
return
}
func (f *Fetcher) Continue() (err error) {
return f.Start()
}
func (f *Fetcher) Close() (err error) {
f.safeDrop()
return nil
@@ -170,7 +153,7 @@ func (f *Fetcher) safeDrop() {
func (f *Fetcher) Wait() (err error) {
for {
if f.torrentReady.Load() {
if f.torrentReady.Load() && len(f.meta.Opts.SelectFiles) > 0 {
done := true
for _, selectIndex := range f.meta.Opts.SelectFiles {
file := f.torrent.Files()[selectIndex]
@@ -190,7 +173,7 @@ func (f *Fetcher) Wait() (err error) {
}
}
if !selected {
util.SafeRemove(filepath.Join(f.meta.Opts.Path, file.Path()))
util.SafeRemove(filepath.Join(f.meta.Opts.Path, f.meta.Res.Name, file.Path()))
}
}
break
@@ -217,6 +200,27 @@ func (f *Fetcher) Progress() fetcher.Progress {
return f.progress
}
func (f *Fetcher) updateRes() {
res := &base.Resource{
Name: f.torrent.Name(),
Range: true,
Files: make([]*base.FileInfo, len(f.torrent.Files())),
Hash: f.torrent.InfoHash().String(),
}
for i, file := range f.torrent.Files() {
res.Files[i] = &base.FileInfo{
Name: filepath.Base(file.DisplayPath()),
Path: util.Dir(file.Path()),
Size: file.Length(),
}
}
res.CalcSize(nil)
f.meta.Res = res
if f.meta.Opts != nil {
f.meta.Opts.InitSelectFiles(len(res.Files))
}
}
func (f *Fetcher) addTorrent(req *base.Request) (err error) {
if err = f.initClient(); err != nil {
return
@@ -239,11 +243,6 @@ func (f *Fetcher) addTorrent(req *base.Request) (err error) {
if err != nil {
return
}
var cfg config
exist, err := f.ctl.GetConfig(&cfg)
if err != nil {
return
}
// use map to deduplicate
trackers := make(map[string]bool)
@@ -255,8 +254,8 @@ func (f *Fetcher) addTorrent(req *base.Request) (err error) {
}
}
}
if exist && len(cfg.Trackers) > 0 {
for _, tracker := range cfg.Trackers {
if len(f.config.Trackers) > 0 {
for _, tracker := range f.config.Trackers {
trackers[tracker] = true
}
}
+22 -13
View File
@@ -8,6 +8,8 @@ import (
"github.com/GopeedLab/gopeed/internal/test"
"github.com/GopeedLab/gopeed/pkg/base"
"github.com/GopeedLab/gopeed/pkg/protocol/bt"
"github.com/GopeedLab/gopeed/pkg/util"
"net/url"
"os"
"reflect"
"testing"
@@ -33,10 +35,9 @@ func TestFetcher_Resolve_DataUri_Torrent(t *testing.T) {
}
want := &base.Resource{
Name: "ubuntu-22.04-live-server-amd64.iso",
Size: 1466714112,
Range: true,
RootDir: "ubuntu-22.04-live-server-amd64.iso",
Name: "ubuntu-22.04-live-server-amd64.iso",
Size: 1466714112,
Range: true,
Files: []*base.FileInfo{
{
Name: "ubuntu-22.04-live-server-amd64.iso",
@@ -51,7 +52,15 @@ func TestFetcher_Resolve_DataUri_Torrent(t *testing.T) {
}
func TestFetcher_Config(t *testing.T) {
doResolve(t, buildConfigFetcher())
doResolve(t, buildConfigFetcher(nil))
}
func TestFetcher_ResolveWithProxy(t *testing.T) {
usr, pwd := "admin", "123"
proxyListener := test.StartSocks5Server(usr, pwd)
defer proxyListener.Close()
doResolve(t, buildConfigFetcher(util.BuildProxyUrl("socks5", proxyListener.Addr().String(), usr, pwd)))
}
func doResolve(t *testing.T, fetcher fetcher.Fetcher) {
@@ -69,10 +78,9 @@ func doResolve(t *testing.T, fetcher fetcher.Fetcher) {
}
want := &base.Resource{
Name: "ubuntu-22.04-live-server-amd64.iso",
Size: 1466714112,
Range: true,
RootDir: "ubuntu-22.04-live-server-amd64.iso",
Name: "ubuntu-22.04-live-server-amd64.iso",
Size: 1466714112,
Range: true,
Files: []*base.FileInfo{
{
Name: "ubuntu-22.04-live-server-amd64.iso",
@@ -92,7 +100,7 @@ func buildFetcher() fetcher.Fetcher {
return fetcher
}
func buildConfigFetcher() fetcher.Fetcher {
func buildConfigFetcher(proxyUrl *url.URL) fetcher.Fetcher {
fetcher := new(FetcherBuilder).Build()
newController := controller.NewController()
mockCfg := config{
@@ -100,12 +108,13 @@ func buildConfigFetcher() fetcher.Fetcher {
"udp://tracker.birkenwald.de:6969/announce",
"udp://tracker.bitsearch.to:1337/announce",
}}
newController.GetConfig = func(v any) (bool, error) {
newController.GetConfig = func(v any) bool {
if err := json.Unmarshal([]byte(test.ToJson(mockCfg)), v); err != nil {
return false, err
return false
}
return true, nil
return true
}
newController.ProxyUrl = proxyUrl
fetcher.Setup(newController)
return fetcher
}
+2 -1
View File
@@ -1,5 +1,6 @@
package http
type config struct {
Connections int `json:"connections"`
UserAgent string `json:"userAgent"`
Connections int `json:"connections"`
}
+156 -162
View File
@@ -3,6 +3,7 @@ package http
import (
"bytes"
"context"
"errors"
"fmt"
"github.com/GopeedLab/gopeed/internal/controller"
"github.com/GopeedLab/gopeed/internal/fetcher"
@@ -17,6 +18,7 @@ import (
"os"
"path"
"strconv"
"time"
)
type RequestError struct {
@@ -33,45 +35,50 @@ func (re *RequestError) Error() string {
}
type chunk struct {
Status base.Status
Begin int64
End int64
Downloaded int64
retryTimes int
}
func newChunk(begin int64, end int64) *chunk {
return &chunk{
Status: base.DownloadStatusReady,
Begin: begin,
End: end,
Begin: begin,
End: end,
}
}
type Fetcher struct {
ctl *controller.Controller
config *config
doneCh chan error
meta *fetcher.FetcherMeta
status base.Status
chunks []*chunk
file *os.File
ctx context.Context
cancel context.CancelFunc
pauseCh chan interface{}
file *os.File
cancel context.CancelFunc
eg *errgroup.Group
}
func (f *Fetcher) Name() string {
return "http"
}
func (f *Fetcher) Setup(ctl *controller.Controller) (err error) {
func (f *Fetcher) Setup(ctl *controller.Controller) {
f.ctl = ctl
f.doneCh = make(chan error, 1)
f.pauseCh = make(chan interface{}, 1)
if f.meta == nil {
f.meta = &fetcher.FetcherMeta{}
}
exist := f.ctl.GetConfig(&f.config)
if !exist {
f.config = &config{
UserAgent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36",
Connections: 1,
}
}
return
}
@@ -79,11 +86,11 @@ func (f *Fetcher) Resolve(req *base.Request) error {
if err := base.ParseReqExtra[fhttp.ReqExtra](req); err != nil {
return err
}
httpReq, err := buildRequest(nil, req)
httpReq, err := f.buildRequest(nil, req)
if err != nil {
return err
}
client := buildClient()
client := f.buildClient()
// send Range request to check whether the server supports breakpoint continuation
// just test one byte, Range: bytes=0-0
httpReq.Header.Set(base.HttpHeaderRange, fmt.Sprintf(base.HttpHeaderRangeFormat, 0, 0))
@@ -139,10 +146,9 @@ func (f *Fetcher) Resolve(req *base.Request) error {
}
// unknown file filePath
if file.Name == "" || file.Name == "/" || file.Name == "." {
file.Name = "unknown"
file.Name = httpReq.URL.Hostname()
}
res.Files = append(res.Files, file)
res.Name = file.Name
f.meta.Req = req
f.meta.Res = res
return nil
@@ -150,7 +156,6 @@ func (f *Fetcher) Resolve(req *base.Request) error {
func (f *Fetcher) Create(opts *base.Options) error {
f.meta.Opts = opts
f.status = base.DownloadStatusReady
if err := base.ParseReqExtra[fhttp.ReqExtra](f.meta.Req); err != nil {
return err
@@ -163,59 +168,44 @@ func (f *Fetcher) Create(opts *base.Options) error {
}
extra := opts.Extra.(*fhttp.OptsExtra)
if extra.Connections == 0 {
var cfg config
exist, err := f.ctl.GetConfig(&cfg)
if err != nil {
return err
}
if exist {
extra.Connections = cfg.Connections
} else {
extra.Connections = 1
}
extra.Connections = f.config.Connections
}
return nil
}
func (f *Fetcher) Start() (err error) {
// 创建文件
name := f.filepath()
f.file, err = f.ctl.Touch(name, f.meta.Res.Size)
name := f.meta.SingleFilepath()
// if file not exist, create it, else open it
_, err = os.Stat(name)
if err != nil {
if os.IsNotExist(err) {
f.file, err = f.ctl.Touch(name, f.meta.Res.Size)
} else {
return
}
} else {
f.file, err = os.OpenFile(name, os.O_RDWR, os.ModeAppend)
}
if err != nil {
return err
}
f.status = base.DownloadStatusRunning
f.chunks = f.splitChunk()
if f.chunks == nil {
f.chunks = f.splitChunk()
}
f.fetch()
return
}
func (f *Fetcher) Pause() (err error) {
if base.DownloadStatusRunning != f.status {
return
}
f.status = base.DownloadStatusPause
if f.cancel != nil {
f.cancel()
<-f.pauseCh
// wait for pause handle complete
f.eg.Wait()
f.file.Close()
}
return
}
func (f *Fetcher) Continue() (err error) {
if base.DownloadStatusRunning == f.status || base.DownloadStatusDone == f.status {
return
}
f.status = base.DownloadStatusRunning
f.file, err = os.OpenFile(f.filepath(), os.O_RDWR, os.ModeAppend)
if err != nil {
return err
}
f.fetch()
return
}
func (f *Fetcher) Close() (err error) {
if err = f.Pause(); err != nil {
return
@@ -243,61 +233,67 @@ func (f *Fetcher) Wait() (err error) {
return <-f.doneCh
}
func (f *Fetcher) filepath() string {
return f.meta.Filepath(f.meta.Res.Files[0])
}
func (f *Fetcher) fetch() {
f.ctx, f.cancel = context.WithCancel(context.Background())
eg, _ := errgroup.WithContext(f.ctx)
var ctx context.Context
ctx, f.cancel = context.WithCancel(context.Background())
f.eg, _ = errgroup.WithContext(ctx)
for i := 0; i < len(f.chunks); i++ {
i := i
eg.Go(func() error {
return f.fetchChunk(i)
f.eg.Go(func() error {
return f.fetchChunk(i, ctx)
})
}
go func() {
err := eg.Wait()
// 下载停止,关闭文件句柄
f.file.Close()
if f.status == base.DownloadStatusPause {
f.pauseCh <- nil
} else {
if err != nil {
f.status = base.DownloadStatusError
} else {
f.status = base.DownloadStatusDone
}
f.doneCh <- err
err := f.eg.Wait()
// check if canceled
if errors.Is(err, context.Canceled) {
return
}
f.file.Close()
f.doneCh <- err
}()
}
func (f *Fetcher) fetchChunk(index int) (err error) {
func (f *Fetcher) fetchChunk(index int, ctx context.Context) (err error) {
chunk := f.chunks[index]
chunk.retryTimes = 0
httpReq, err := buildRequest(f.ctx, f.meta.Req)
httpReq, err := f.buildRequest(ctx, f.meta.Req)
if err != nil {
return err
}
var (
client = buildClient()
buf = make([]byte, 8192)
client = f.buildClient()
buf = make([]byte, 8192)
maxRetries = 3
)
// 重试5次
for i := 0; i < 5; i++ {
// 如果下载完成直接返回
if chunk.Status == base.DownloadStatusDone {
// retry until all remain chunks failed
for true {
// if chunk is completed, return
if f.meta.Res.Range && chunk.Downloaded >= chunk.End-chunk.Begin+1 {
return
}
// 如果已暂停直接跳出
if f.status == base.DownloadStatusPause {
break
if chunk.retryTimes >= maxRetries {
if !f.meta.Res.Range {
return
}
// check if all failed
allFailed := true
for _, c := range f.chunks {
if chunk.Downloaded < chunk.End-chunk.Begin+1 && c.retryTimes < maxRetries {
allFailed = false
break
}
}
if allFailed {
return
}
}
var (
resp *http.Response
retry bool
resp *http.Response
)
if f.meta.Res.Range {
httpReq.Header.Set(base.HttpHeaderRange,
@@ -310,57 +306,98 @@ func (f *Fetcher) fetchChunk(index int) (err error) {
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode != base.HttpCodeOK && resp.StatusCode != base.HttpCodePartialContent {
err = NewRequestError(resp.StatusCode, resp.Status)
return err
}
return nil
}()
if err != nil {
// 请求失败重试
continue
}
// 请求成功就重置错误次数,连续失败5次才终止
i = 0
retry, err = func() (bool, error) {
defer resp.Body.Close()
// Http request success, reset retry times
chunk.retryTimes = 0
for {
n, err := resp.Body.Read(buf)
if n > 0 {
_, err := f.file.WriteAt(buf[:n], chunk.Begin+chunk.Downloaded)
if err != nil {
return false, err
return err
}
chunk.Downloaded += int64(n)
}
if err != nil {
if err != io.EOF {
return true, err
if err == io.EOF {
return nil
}
break
return err
}
}
return false, nil
return nil
}()
if !retry {
// 下载成功,跳出重试
break
if err != nil {
// If canceled, do not retry
if errors.Is(err, context.Canceled) {
return
}
// retry request after 1 second
chunk.retryTimes = chunk.retryTimes + 1
time.Sleep(time.Second)
continue
}
// if chunk is completed, reset other not complete chunks retry times
if err == nil {
for _, c := range f.chunks {
if c != chunk && c.retryTimes > 0 {
c.retryTimes = 0
}
}
}
break
}
return
}
func (f *Fetcher) buildRequest(ctx context.Context, req *base.Request) (httpReq *http.Request, err error) {
url, err := url.Parse(req.URL)
if err != nil {
return
}
var (
method string
body io.Reader
)
headers := make(map[string][]string)
if req.Extra == nil {
method = http.MethodGet
} else {
extra := req.Extra.(*fhttp.ReqExtra)
if extra.Method != "" {
method = extra.Method
} else {
method = http.MethodGet
}
if len(extra.Header) > 0 {
for k, v := range extra.Header {
headers[k] = []string{v}
}
}
if extra.Body != "" {
body = bytes.NewBufferString(extra.Body)
}
}
if _, ok := headers[base.HttpHeaderUserAgent]; !ok {
// load user agent from config
headers[base.HttpHeaderUserAgent] = []string{f.config.UserAgent}
}
if ctx != nil {
httpReq, err = http.NewRequestWithContext(ctx, method, url.String(), body)
} else {
httpReq, err = http.NewRequest(method, url.String(), body)
}
if err != nil {
chunk.Status = base.DownloadStatusError
return
}
isComplete := f.meta.Res.Range && chunk.Downloaded >= chunk.End-chunk.Begin+1
if f.status == base.DownloadStatusPause && !isComplete {
chunk.Status = base.DownloadStatusPause
return
}
chunk.Status = base.DownloadStatusDone
return
httpReq.Header = headers
return httpReq, nil
}
func (f *Fetcher) splitChunk() (chunks []*chunk) {
@@ -391,59 +428,19 @@ func (f *Fetcher) splitChunk() (chunks []*chunk) {
return
}
func buildClient() *http.Client {
func (f *Fetcher) buildClient() *http.Client {
transport := &http.Transport{}
// Cookie handle
jar, _ := cookiejar.New(nil)
if f.ctl.ProxyUrl != nil {
transport.Proxy = http.ProxyURL(f.ctl.ProxyUrl)
}
return &http.Client{
Jar: jar,
Transport: transport,
Jar: jar,
}
}
func buildRequest(ctx context.Context, req *base.Request) (httpReq *http.Request, err error) {
url, err := url.Parse(req.URL)
if err != nil {
return
}
var (
method string
body io.Reader
)
headers := make(map[string][]string)
if req.Extra == nil {
method = http.MethodGet
} else {
extra := req.Extra.(*fhttp.ReqExtra)
if extra.Method != "" {
method = extra.Method
} else {
method = http.MethodGet
}
if len(extra.Header) > 0 {
for k, v := range extra.Header {
headers[k] = []string{v}
}
}
if extra.Body != "" {
body = bytes.NewBufferString(extra.Body)
}
}
if _, ok := headers[base.HttpHeaderUserAgent]; !ok {
headers[base.HttpHeaderUserAgent] = []string{base.AgentName}
}
if ctx != nil {
httpReq, err = http.NewRequestWithContext(ctx, method, url.String(), body)
} else {
httpReq, err = http.NewRequest(method, url.String(), body)
}
if err != nil {
return
}
httpReq.Header = headers
return httpReq, nil
}
type fetcherData struct {
Chunks []*chunk
}
@@ -474,12 +471,9 @@ func (fb *FetcherBuilder) Restore() (v any, f func(meta *fetcher.FetcherMeta, v
fb := &FetcherBuilder{}
fetcher := fb.Build().(*Fetcher)
fetcher.meta = meta
fetcher.status = base.DownloadStatusPause
base.ParseReqExtra[fhttp.ReqExtra](fetcher.meta.Req)
base.ParseOptsExtra[fhttp.OptsExtra](fetcher.meta.Opts)
if len(fd.Chunks) == 0 {
fetcher.chunks = fetcher.splitChunk()
} else {
if len(fd.Chunks) > 0 {
fetcher.chunks = fd.Chunks
}
return fetcher
+64 -15
View File
@@ -8,6 +8,7 @@ import (
"github.com/GopeedLab/gopeed/internal/test"
"github.com/GopeedLab/gopeed/pkg/base"
"github.com/GopeedLab/gopeed/pkg/protocol/http"
"github.com/GopeedLab/gopeed/pkg/util"
"net"
"reflect"
"testing"
@@ -16,7 +17,6 @@ import (
func TestFetcher_Resolve(t *testing.T) {
testResolve(test.StartTestFileServer, test.BuildName, &base.Resource{
Name: test.BuildName,
Size: test.BuildSize,
Range: true,
Files: []*base.FileInfo{
@@ -27,7 +27,6 @@ func TestFetcher_Resolve(t *testing.T) {
},
}, t)
testResolve(test.StartTestCustomServer, "disposition", &base.Resource{
Name: test.BuildName,
Size: test.BuildSize,
Range: false,
Files: []*base.FileInfo{
@@ -38,7 +37,6 @@ func TestFetcher_Resolve(t *testing.T) {
},
}, t)
testResolve(test.StartTestCustomServer, test.BuildName, &base.Resource{
Name: test.BuildName,
Size: 0,
Range: false,
Files: []*base.FileInfo{
@@ -48,6 +46,17 @@ func TestFetcher_Resolve(t *testing.T) {
},
},
}, t)
fetcher := buildFetcher()
err := fetcher.Resolve(&base.Request{
URL: "http://github.com",
})
if err != nil {
t.Fatal(err)
}
if fetcher.Meta().Res.Files[0].Name != "github.com" {
t.Errorf("Resolve() got = %v, want %v", fetcher.Meta().Res, "github.com")
}
}
func testResolve(startTestServer func() net.Listener, path string, want *base.Resource, t *testing.T) {
@@ -68,7 +77,7 @@ func testResolve(startTestServer func() net.Listener, path string, want *base.Re
func TestFetcher_DownloadNormal(t *testing.T) {
listener := test.StartTestFileServer()
defer listener.Close()
// 正常下载
downloadNormal(listener, 1, t)
downloadNormal(listener, 5, t)
downloadNormal(listener, 8, t)
@@ -78,7 +87,7 @@ func TestFetcher_DownloadNormal(t *testing.T) {
func TestFetcher_DownloadContinue(t *testing.T) {
listener := test.StartTestFileServer()
defer listener.Close()
// 暂停继续
downloadContinue(listener, 1, t)
downloadContinue(listener, 5, t)
downloadContinue(listener, 8, t)
@@ -88,7 +97,7 @@ func TestFetcher_DownloadContinue(t *testing.T) {
func TestFetcher_DownloadChunked(t *testing.T) {
listener := test.StartTestCustomServer()
defer listener.Close()
// chunked编码下载
downloadNormal(listener, 1, t)
downloadNormal(listener, 2, t)
}
@@ -96,24 +105,33 @@ func TestFetcher_DownloadChunked(t *testing.T) {
func TestFetcher_DownloadPost(t *testing.T) {
listener := test.StartTestPostServer()
defer listener.Close()
// post下载
downloadPost(listener, 1, t)
}
func TestFetcher_DownloadRetry(t *testing.T) {
listener := test.StartTestRetryServer()
defer listener.Close()
// chunked编码下载
downloadNormal(listener, 1, t)
}
func TestFetcher_DownloadError(t *testing.T) {
listener := test.StartTestErrorServer()
defer listener.Close()
// chunked编码下载
downloadError(listener, 1, t)
}
func TestFetcher_DownloadLimit(t *testing.T) {
listener := test.StartTestLimitServer(4, 0)
defer listener.Close()
downloadNormal(listener, 1, t)
downloadNormal(listener, 2, t)
downloadNormal(listener, 8, t)
}
func TestFetcher_DownloadResume(t *testing.T) {
listener := test.StartTestFileServer()
defer listener.Close()
@@ -124,8 +142,19 @@ func TestFetcher_DownloadResume(t *testing.T) {
downloadResume(listener, 16, t)
}
func TestFetcher_DownloadWithProxy(t *testing.T) {
httpListener := test.StartTestFileServer()
defer httpListener.Close()
proxyListener := test.StartSocks5Server("", "")
defer proxyListener.Close()
downloadWithProxy(httpListener, proxyListener, t)
}
func TestFetcher_Config(t *testing.T) {
fetcher := doDownloadReady(buildConfigFetcher(), test.StartTestFileServer(), 0, t)
listener := test.StartTestFileServer()
defer listener.Close()
fetcher := doDownloadReady(buildConfigFetcher(), listener, 0, t)
err := fetcher.Start()
if err != nil {
t.Fatal(err)
@@ -222,7 +251,7 @@ func downloadContinue(listener net.Listener, connections int, t *testing.T) {
t.Fatal(err)
}
time.Sleep(time.Millisecond * 50)
if err := fetcher.Continue(); err != nil {
if err := fetcher.Start(); err != nil {
t.Fatal(err)
}
err = fetcher.Wait()
@@ -270,7 +299,7 @@ func downloadResume(listener net.Listener, connections int, t *testing.T) {
t.Fatal(err)
}
fetcher.Setup(controller.NewController())
fetcher.Continue()
fetcher.Start()
err = fetcher.Wait()
if err != nil {
@@ -283,6 +312,26 @@ func downloadResume(listener net.Listener, connections int, t *testing.T) {
}
}
func downloadWithProxy(httpListener net.Listener, proxyListener net.Listener, t *testing.T) {
fetcher := downloadReady(httpListener, 4, t)
ctl := controller.NewController()
ctl.ProxyUrl = util.BuildProxyUrl("socks5", proxyListener.Addr().String(), "", "")
fetcher.Setup(ctl)
err := fetcher.Start()
if err != nil {
t.Fatal(err)
}
err = fetcher.Wait()
if err != nil {
t.Fatal(err)
}
want := test.FileMd5(test.BuildFile)
got := test.FileMd5(test.DownloadFile)
if want != got {
t.Errorf("Download() got = %v, want %v", got, want)
}
}
func buildFetcher() *Fetcher {
fetcher := new(FetcherBuilder).Build()
fetcher.Setup(controller.NewController())
@@ -295,11 +344,11 @@ func buildConfigFetcher() fetcher.Fetcher {
mockCfg := config{
Connections: 16,
}
newController.GetConfig = func(v any) (bool, error) {
newController.GetConfig = func(v any) bool {
if err := json.Unmarshal([]byte(test.ToJson(mockCfg)), v); err != nil {
return false, err
return false
}
return true, nil
return true
}
fetcher.Setup(newController)
return fetcher
+165 -18
View File
@@ -1,13 +1,18 @@
package test
import (
"context"
"encoding/json"
"fmt"
"github.com/armon/go-socks5"
"io"
"math/rand"
"net"
"net/http"
"os"
"strconv"
"strings"
"sync/atomic"
"time"
)
@@ -17,8 +22,10 @@ const (
Dir = "./"
BuildFile = Dir + BuildName
DownloadName = "download.data"
DownloadFile = Dir + DownloadName
DownloadName = "download.data"
DownloadRename = "download (1).data"
DownloadFile = Dir + DownloadName
DownloadRenameFile = Dir + DownloadRename
)
func StartTestFileServer() net.Listener {
@@ -78,7 +85,7 @@ func StartTestRetryServer() net.Listener {
mux := http.NewServeMux()
mux.HandleFunc("/"+BuildName, func(writer http.ResponseWriter, request *http.Request) {
counter++
if counter != 1 && counter < 5 {
if counter != 1 && counter < 2 {
writer.WriteHeader(500)
return
}
@@ -131,6 +138,123 @@ func StartTestErrorServer() net.Listener {
})
}
// StartTestLimitServer connections limit server
func StartTestLimitServer(maxConnections int32, delay int64) net.Listener {
var connections atomic.Int32
return startTestServer(func() http.Handler {
mux := http.NewServeMux()
mux.HandleFunc("/"+BuildName, func(writer http.ResponseWriter, request *http.Request) {
defer func() {
connections.Add(-1)
}()
connections.Add(1)
if maxConnections != 0 && connections.Load() > maxConnections {
writer.WriteHeader(403)
return
}
r := request.Header.Get("Range")
if r == "" {
writer.Header().Set("Content-Length", fmt.Sprintf("%d", BuildSize))
writer.WriteHeader(200)
file, err := os.Open(BuildFile)
if err != nil {
panic(err)
}
defer file.Close()
slowCopy(writer, file, delay)
} else {
// split range
s := strings.Split(r, "=")
if len(s) != 2 {
writer.WriteHeader(400)
return
}
s = strings.Split(s[1], "-")
if len(s) != 2 {
writer.WriteHeader(400)
return
}
start, err := strconv.ParseInt(s[0], 10, 64)
if err != nil {
writer.WriteHeader(400)
return
}
end, err := strconv.ParseInt(s[1], 10, 64)
if err != nil {
writer.WriteHeader(400)
return
}
if start < 0 || end < 0 || start > end {
writer.WriteHeader(400)
return
}
if end >= BuildSize {
end = BuildSize - 1
}
writer.Header().Set("Content-Length", fmt.Sprintf("%d", end-start+1))
writer.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, BuildSize))
writer.Header().Set("Accept-Ranges", "bytes")
writer.WriteHeader(206)
file, err := os.Open(BuildFile)
if err != nil {
writer.WriteHeader(500)
return
}
defer file.Close()
file.Seek(start, 0)
slowCopyN(writer, file, end-start+1, delay)
}
})
return mux
})
}
// slowCopyN copies n bytes from src to dst, speed limit is bytes per second
func slowCopy(dst io.Writer, src io.Reader, delay int64) (written int64, err error) {
buf := make([]byte, 32*1024)
for {
nr, er := src.Read(buf)
if nr > 0 {
nw, ew := dst.Write(buf[0:nr])
if nw > 0 {
written += int64(nw)
}
if ew != nil {
err = ew
break
}
if nr != nw {
err = io.ErrShortWrite
break
}
}
if er != nil {
if er != io.EOF {
err = er
}
break
}
if delay > 0 {
time.Sleep(time.Millisecond * time.Duration(delay))
}
}
return written, err
}
func slowCopyN(dst io.Writer, src io.Reader, n int64, delay int64) (written int64, err error) {
written, err = slowCopy(dst, io.LimitReader(src, n), delay)
if written == n {
return n, nil
}
if written < n && err == nil {
// src stopped early; must have been EOF.
err = io.EOF
}
return
}
func startTestServer(serverHandle func() http.Handler) net.Listener {
listener, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
@@ -140,6 +264,7 @@ func startTestServer(serverHandle func() http.Handler) net.Listener {
if err != nil {
panic(err)
}
defer file.Close()
// 随机生成一个文件
l := int64(8192)
buf := make([]byte, l)
@@ -160,28 +285,29 @@ func startTestServer(serverHandle func() http.Handler) net.Listener {
server.Handler = serverHandle()
go server.Serve(listener)
return &delFileListener{
File: file,
return &shutdownListener{
server: server,
Listener: listener,
}
}
type delFileListener struct {
*os.File
type shutdownListener struct {
server *http.Server
net.Listener
}
func (c *delFileListener) Close() error {
defer func() {
c.File.Close()
if err := ifExistAndRemove(c.File.Name()); err != nil {
fmt.Println(err)
}
if err := ifExistAndRemove(DownloadFile); err != nil {
fmt.Println(err)
}
}()
return c.Listener.Close()
func (c *shutdownListener) Close() error {
closeErr := c.server.Shutdown(context.Background())
if err := ifExistAndRemove(BuildFile); err != nil {
fmt.Println(err)
}
if err := ifExistAndRemove(DownloadFile); err != nil {
fmt.Println(err)
}
if err := ifExistAndRemove(DownloadRenameFile); err != nil {
fmt.Println(err)
}
return closeErr
}
func ifExistAndRemove(name string) error {
@@ -190,3 +316,24 @@ func ifExistAndRemove(name string) error {
}
return nil
}
func StartSocks5Server(usr, pwd string) net.Listener {
conf := &socks5.Config{}
if usr != "" && pwd != "" {
conf.Credentials = socks5.StaticCredentials{
usr: pwd,
}
}
server, err := socks5.New(conf)
if err != nil {
panic(err)
}
listener, err := net.Listen("tcp", "127.0.0.1:0") // 你可以根据需要更改监听地址
if err != nil {
panic(err)
}
go server.Serve(listener)
return listener
}
+1 -5
View File
@@ -19,11 +19,7 @@ const (
HttpHeaderContentLength = "Content-Length"
HttpHeaderContentRange = "Content-Range"
HttpHeaderContentDisposition = "Content-Disposition"
HttpHeaderUserAgent = "User-Agent"
HttpHeaderUserAgent = "Usr-Agent"
HttpHeaderRangeFormat = "bytes=%d-%d"
)
const (
AgentName = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36"
)
+59 -2
View File
@@ -1,31 +1,69 @@
package base
import (
"fmt"
"github.com/GopeedLab/gopeed/pkg/util"
"golang.org/x/exp/slices"
)
// Request download request
type Request struct {
URL string `json:"url"`
Extra any `json:"extra"`
// Labels is used to mark the download task
Labels map[string]string `json:"labels"`
}
func (r *Request) Validate() error {
if r.URL == "" {
return fmt.Errorf("invalid request url")
}
return nil
}
// Resource download resource
type Resource struct {
// if name is not empty, the resource is a folder and the name is the folder name
Name string `json:"name"`
Size int64 `json:"size"`
// is support range download
Range bool `json:"range"`
RootDir string `json:"rootDir"`
Range bool `json:"range"`
// file list
Files []*FileInfo `json:"files"`
Hash string `json:"hash"`
}
func (r *Resource) Validate() error {
if r.Name == "" {
return fmt.Errorf("invalid resource name")
}
if r.Files == nil || len(r.Files) == 0 {
return fmt.Errorf("invalid resource files")
}
for _, file := range r.Files {
if file.Name == "" {
return fmt.Errorf("invalid resource file name")
}
}
return nil
}
func (r *Resource) CalcSize(selectFiles []int) {
var size int64
for i, file := range r.Files {
if len(selectFiles) == 0 || slices.Contains(selectFiles, i) {
size += file.Size
}
}
r.Size = size
}
type FileInfo struct {
Name string `json:"name"`
Path string `json:"path"`
Size int64 `json:"size"`
Req *Request `json:"req"`
}
// Options for download
@@ -40,6 +78,25 @@ type Options struct {
Extra any `json:"extra"`
}
func (o *Options) InitSelectFiles(fileSize int) {
// if selectFiles is empty, select all files
if len(o.SelectFiles) == 0 {
o.SelectFiles = make([]int, fileSize)
for i := 0; i < fileSize; i++ {
o.SelectFiles[i] = i
}
}
}
func (o *Options) Clone() *Options {
return &Options{
Name: o.Name,
Path: o.Path,
SelectFiles: o.SelectFiles,
Extra: o.Extra,
}
}
func ParseReqExtra[E any](req *Request) error {
if req.Extra == nil {
return nil
+462 -302
View File
@@ -4,13 +4,14 @@ import (
"errors"
"github.com/GopeedLab/gopeed/internal/controller"
"github.com/GopeedLab/gopeed/internal/fetcher"
"github.com/GopeedLab/gopeed/internal/logger"
"github.com/GopeedLab/gopeed/pkg/base"
"github.com/GopeedLab/gopeed/pkg/util"
gonanoid "github.com/matoous/go-nanoid/v2"
"github.com/rs/zerolog"
"github.com/rs/zerolog/pkgerrors"
"github.com/virtuald/go-paniclog"
"math"
"os"
"path"
"path/filepath"
"sort"
"strings"
@@ -26,6 +27,14 @@ const (
bucketSave = "save"
// downloader config bucket
bucketConfig = "config"
// downloader extension bucket
bucketExtension = "extension"
// downloader extension storage bucket
bucketExtensionStorage = "extension_storage"
)
var (
ErrTaskNotFound = errors.New("task not found")
)
type Listener func(event *Event)
@@ -40,16 +49,23 @@ type Progress struct {
}
type Downloader struct {
cfg *DownloaderConfig
fetchBuilders map[string]fetcher.FetcherBuilder
fetcherMap map[string]fetcher.Fetcher
storage Storage
tasks []*Task
listener Listener
Logger *logger.Logger
ExtensionLogger *logger.Logger
lock *sync.Mutex
fetcherMapLock *sync.RWMutex
closed atomic.Bool
cfg *DownloaderConfig
fetcherBuilders map[string]fetcher.FetcherBuilder
fetcherCache map[string]fetcher.Fetcher
storage Storage
tasks []*Task
waitTasks []*Task
listener Listener
lock *sync.Mutex
fetcherMapLock *sync.RWMutex
checkDuplicateLock *sync.Mutex
closed atomic.Bool
extensions []*Extension
}
func NewDownloader(cfg *DownloaderConfig) *Downloader {
@@ -59,28 +75,36 @@ func NewDownloader(cfg *DownloaderConfig) *Downloader {
cfg.Init()
d := &Downloader{
cfg: cfg,
cfg: cfg,
fetcherBuilders: make(map[string]fetcher.FetcherBuilder),
fetcherCache: make(map[string]fetcher.Fetcher),
waitTasks: make([]*Task, 0),
storage: cfg.Storage,
fetchBuilders: make(map[string]fetcher.FetcherBuilder),
fetcherMap: make(map[string]fetcher.Fetcher),
lock: &sync.Mutex{},
fetcherMapLock: &sync.RWMutex{},
checkDuplicateLock: &sync.Mutex{},
storage: cfg.Storage,
lock: &sync.Mutex{},
fetcherMapLock: &sync.RWMutex{},
extensions: make([]*Extension, 0),
}
for _, f := range cfg.FetchBuilders {
for _, p := range f.Schemes() {
d.fetchBuilders[strings.ToUpper(p)] = f
d.fetcherBuilders[strings.ToUpper(p)] = f
}
}
logPanic(filepath.Join(cfg.StorageDir, "logs"))
zerolog.ErrorStackMarshaler = pkgerrors.MarshalStack
d.Logger = logger.NewLogger(cfg.ProductionMode, filepath.Join(cfg.StorageDir, "logs", "core.log"))
d.ExtensionLogger = logger.NewLogger(cfg.ProductionMode, filepath.Join(cfg.StorageDir, "logs", "extension.log"))
if cfg.ProductionMode {
logPanic(filepath.Join(cfg.StorageDir, "logs"))
}
return d
}
func (d *Downloader) Setup() error {
// setup storage
if err := d.storage.Setup([]string{bucketTask, bucketSave, bucketConfig}); err != nil {
if err := d.storage.Setup([]string{bucketTask, bucketSave, bucketConfig, bucketExtension, bucketExtensionStorage}); err != nil {
return err
}
// load config from storage
@@ -100,7 +124,7 @@ func (d *Downloader) Setup() error {
d.cfg.DownloaderStoreConfig.Init()
// load tasks from storage
var tasks []*Task
if err := d.storage.List(bucketTask, &tasks); err != nil {
if err = d.storage.List(bucketTask, &tasks); err != nil {
return err
}
if tasks == nil {
@@ -118,16 +142,25 @@ func (d *Downloader) Setup() error {
task.Status = base.DownloadStatusPause
}
}
// sort by create time
sort.Slice(tasks, func(i, j int) bool {
return tasks[i].CreatedAt.Before(tasks[j].CreatedAt)
})
}
d.tasks = tasks
// sort by create time
sort.Slice(d.tasks, func(i, j int) bool {
return d.tasks[i].CreatedAt.Before(d.tasks[j].CreatedAt)
})
// 每个tick统计一次下载速度
// load extensions from storage
var extensions []*Extension
if err = d.storage.List(bucketExtension, &extensions); err != nil {
return err
}
if extensions == nil {
extensions = make([]*Extension, 0)
}
d.extensions = extensions
// calculate download speed every tick
go func() {
for !d.closed.Load() {
if len(d.tasks) > 0 {
for _, task := range d.tasks {
@@ -151,13 +184,16 @@ func (d *Downloader) Setup() error {
// store fetcher progress
data, err := task.fetcherBuilder.Store(task.fetcher)
if err != nil {
return // TODO log
d.Logger.Error().Stack().Err(err).Msgf("serialize fetcher failed: %s", task.ID)
return
}
if err := d.storage.Put(bucketSave, task.ID, data); err != nil {
return // TODO log
d.Logger.Error().Stack().Err(err).Msgf("persist fetcher failed: %s", task.ID)
return
}
if err := d.storage.Put(bucketTask, task.ID, task); err != nil {
return // TODO log
d.Logger.Error().Stack().Err(err).Msgf("persist task failed: %s", task.ID)
return
}
}()
}
@@ -170,7 +206,7 @@ func (d *Downloader) Setup() error {
func (d *Downloader) parseFb(url string) (fetcher.FetcherBuilder, error) {
schema := util.ParseSchema(url)
fetchBuilder, ok := d.fetchBuilders[schema]
fetchBuilder, ok := d.fetcherBuilders[schema]
if ok {
return fetchBuilder, nil
}
@@ -179,29 +215,40 @@ func (d *Downloader) parseFb(url string) (fetcher.FetcherBuilder, error) {
func (d *Downloader) setupFetcher(fetcher fetcher.Fetcher) {
ctl := controller.NewController()
ctl.GetConfig = func(v any) (bool, error) {
ctl.GetConfig = func(v any) bool {
return d.getProtocolConfig(fetcher.Name(), v)
}
ctl.ProxyUrl = d.cfg.ProxyUrl()
fetcher.Setup(ctl)
}
func (d *Downloader) Resolve(req *base.Request) (rr *ResolveResult, err error) {
fb, err := d.parseFb(req.URL)
if err != nil {
return
}
fetcher := fb.Build()
d.setupFetcher(fetcher)
err = fetcher.Resolve(req)
if err != nil {
return
}
rrId, err := gonanoid.New()
if err != nil {
return
}
res, err := d.triggerOnResolve(req)
if err != nil {
return
}
if res != nil && len(res.Files) > 0 {
rr = &ResolveResult{
Res: res,
}
return
}
fetcher, err := d.buildFetcher(req.URL)
if err != nil {
return
}
err = fetcher.Resolve(req)
if err != nil {
return
}
d.fetcherMapLock.Lock()
d.fetcherMap[rrId] = fetcher
d.fetcherCache[rrId] = fetcher
d.fetcherMapLock.Unlock()
rr = &ResolveResult{
ID: rrId,
@@ -210,12 +257,41 @@ func (d *Downloader) Resolve(req *base.Request) (rr *ResolveResult, err error) {
return
}
func (d *Downloader) notifyRunning() {
go func() {
d.lock.Lock()
defer d.lock.Unlock()
remainRunningCount := d.remainRunningCount()
if remainRunningCount == 0 {
return
}
if len(d.waitTasks) > 0 {
wt := d.waitTasks[0]
d.waitTasks = d.waitTasks[1:]
d.doStart(wt)
}
}()
}
func (d *Downloader) remainRunningCount() int {
runningCount := 0
for _, t := range d.tasks {
if t.Status == base.DownloadStatusRunning {
runningCount++
}
}
return d.cfg.MaxRunning - runningCount
}
func (d *Downloader) CreateDirect(req *base.Request, opts *base.Options) (taskId string, err error) {
rr, err := d.Resolve(req)
var fetcher fetcher.Fetcher
fetcher, err = d.buildFetcher(req.URL)
if err != nil {
return
}
return d.Create(rr.ID, opts)
fetcher.Meta().Req = req
return d.doCreate(fetcher, opts)
}
func (d *Downloader) Create(rrId string, opts *base.Options) (taskId string, err error) {
@@ -223,228 +299,217 @@ func (d *Downloader) Create(rrId string, opts *base.Options) (taskId string, err
opts = &base.Options{}
}
d.fetcherMapLock.RLock()
fetcher, ok := d.fetcherMap[rrId]
fetcher, ok := d.fetcherCache[rrId]
d.fetcherMapLock.RUnlock()
if !ok {
return "", errors.New("invalid resource id")
}
defer func() {
d.fetcherMapLock.Lock()
delete(d.fetcherMap, rrId)
delete(d.fetcherCache, rrId)
d.fetcherMapLock.Unlock()
}()
meta := fetcher.Meta()
meta.Opts = opts
res := meta.Res
if len(opts.SelectFiles) == 0 {
opts.SelectFiles = make([]int, 0)
for i := range res.Files {
opts.SelectFiles = append(opts.SelectFiles, i)
}
}
if opts.Path == "" {
storeConfig, err := d.GetConfig()
if err != nil {
return "", err
}
opts.Path = storeConfig.DownloadDir
}
fb, err := d.parseFb(fetcher.Meta().Req.URL)
if err != nil {
return
}
// check if the download file is duplicated and rename it automatically.
files := res.Files
if res.RootDir != "" {
fullDirPath := path.Join(opts.Path, res.RootDir)
newName, err := util.CheckDuplicateAndRename(fullDirPath)
if err != nil {
return "", err
}
res.RootDir = newName
} else if len(files) == 1 {
fullPath := meta.Filepath(files[0])
newName, err := util.CheckDuplicateAndRename(fullPath)
if err != nil {
return "", err
}
opts.Name = newName
}
err = fetcher.Create(opts)
if err != nil {
return
}
task := NewTask()
task.fetcherBuilder = fb
task.fetcher = fetcher
task.Meta = fetcher.Meta()
task.Progress = &Progress{}
task.Status = base.DownloadStatusReady
// calculate select files size
for _, selectIndex := range opts.SelectFiles {
task.Size += res.Files[selectIndex].Size
}
initTask(task)
err = func() error {
d.lock.Lock()
defer d.lock.Unlock()
d.tasks = append(d.tasks, task)
return d.storage.Put(bucketTask, task.ID, task.clone())
}()
if err != nil {
return
}
taskId = task.ID
d.tryRunning(func(remain int) {
if remain > 0 {
err = d.start(task)
}
})
return
return d.doCreate(fetcher, opts)
}
func (d *Downloader) Pause(id string) (err error) {
task := d.GetTask(id)
if task == nil {
return
return ErrTaskNotFound
}
if err = d.doPause(task, base.DownloadStatusPause); err != nil {
return err
}
return d.notifyRunning()
func() {
task.lock.Lock()
defer task.lock.Unlock()
if task.Status == base.DownloadStatusPause {
return
}
err = d.doPause(task)
if err != nil {
return
}
d.notifyRunning()
}()
return
}
func (d *Downloader) Continue(id string) (err error) {
task := d.GetTask(id)
if task == nil {
return
return ErrTaskNotFound
}
d.tryRunning(func(remain int) {
if remain == 0 {
// no more task can be running, need to doPause a running task
func() {
d.lock.Lock()
defer d.lock.Unlock()
remainRunningCount := d.remainRunningCount()
if remainRunningCount == 0 {
for _, t := range d.tasks {
if t.Status == base.DownloadStatusRunning {
err = d.doPause(t, base.DownloadStatusWait)
if err = d.doPause(t); err != nil {
return
}
t.Status = base.DownloadStatusWait
d.waitTasks = append(d.waitTasks, t)
break
}
}
}
if err != nil {
}()
func() {
task.lock.Lock()
defer task.lock.Unlock()
if task.Status == base.DownloadStatusRunning {
return
}
})
if err != nil {
return
}
return d.doContinue(task)
if err = d.doStart(task); err != nil {
return
}
}()
return
}
func (d *Downloader) PauseAll() (err error) {
d.lock.Lock()
defer d.lock.Unlock()
func() {
d.lock.Lock()
defer d.lock.Unlock()
// Clear wait tasks
d.waitTasks = d.waitTasks[:0]
}()
for _, task := range d.tasks {
if task.Status == base.DownloadStatusRunning {
if err = d.doPause(task, base.DownloadStatusPause); err != nil {
return
}
err = func() error {
task.lock.Lock()
defer task.lock.Unlock()
return d.doPause(task)
}()
if err != nil {
return
}
}
return
}
func (d *Downloader) ContinueAll() (err error) {
d.lock.Lock()
defer d.lock.Unlock()
runningCount := 0
continueTasks := make([]*Task, 0)
for _, task := range d.tasks {
if task.Status == base.DownloadStatusRunning {
runningCount++
} else if task.Status != base.DownloadStatusDone {
continueTasks = append(continueTasks, task)
}
}
// calculate how many tasks can be continued, can't exceed maxRunning
continueCount := int(math.Min(float64(d.cfg.MaxRunning-runningCount), float64(len(continueTasks))))
for i := 0; i < continueCount; i++ {
if err = d.doContinue(continueTasks[i]); err != nil {
return
}
}
return
}
continuedTasks := make([]*Task, 0)
func (d *Downloader) Delete(id string, force bool) (err error) {
d.lock.Lock()
defer d.lock.Unlock()
task := d.GetTask(id)
if task == nil {
return
}
task.lock.Lock()
defer task.lock.Unlock()
if task.fetcher != nil {
err = task.fetcher.Close()
func() {
d.lock.Lock()
defer d.lock.Unlock()
// calculate how many tasks can be continued, can't exceed maxRunning
remainCount := d.remainRunningCount()
for _, task := range d.tasks {
if task.Status != base.DownloadStatusRunning && task.Status != base.DownloadStatusDone {
if len(continuedTasks) < remainCount {
continuedTasks = append(continuedTasks, task)
} else {
task.Status = base.DownloadStatusWait
d.waitTasks = append(d.waitTasks, task)
}
}
}
}()
for _, task := range continuedTasks {
tt := task
err = func() error {
tt.lock.Lock()
defer tt.lock.Unlock()
return d.doStart(tt)
}()
if err != nil {
return
}
}
for i, t := range d.tasks {
if t.ID == id {
d.tasks = append(d.tasks[:i], d.tasks[i+1:]...)
break
}
return
}
func (d *Downloader) Delete(id string, force bool) (err error) {
task := d.GetTask(id)
if task == nil {
return ErrTaskNotFound
}
err = d.storage.Delete(bucketTask, id)
if err != nil {
return
}
err = d.storage.Delete(bucketSave, id)
if err != nil {
return
}
if force {
if task.Meta.Res.RootDir != "" {
if err = os.RemoveAll(path.Join(task.Meta.Opts.Path, task.Meta.Res.RootDir)); err != nil {
return
func() {
d.lock.Lock()
defer d.lock.Unlock()
for i, t := range d.tasks {
if t.ID == id {
d.tasks = append(d.tasks[:i], d.tasks[i+1:]...)
}
} else {
for _, file := range task.Meta.Res.Files {
if err = util.SafeRemove(task.Meta.Filepath(file)); err != nil {
}
}()
err = d.doDelete(task, force)
if err != nil {
return
}
d.notifyRunning()
return
}
func (d *Downloader) doDelete(task *Task, force bool) (err error) {
err = func() error {
if task.fetcher != nil {
if err := task.fetcher.Close(); err != nil {
return err
}
}
if err := d.storage.Delete(bucketTask, task.ID); err != nil {
return err
}
if err := d.storage.Delete(bucketSave, task.ID); err != nil {
return err
}
if force && task.Meta.Res != nil {
if task.Meta.Res.Name != "" {
if err := os.RemoveAll(task.Meta.FolderPath()); err != nil {
return err
}
} else {
if err := util.SafeRemove(task.Meta.SingleFilepath()); err != nil {
return err
}
}
}
d.emit(EventKeyDelete, task)
task = nil
return nil
}()
if err != nil {
d.Logger.Error().Stack().Err(err).Msgf("delete task failed, task id: %s", task.ID)
}
d.emit(EventKeyDelete, task)
if task.Status == base.DownloadStatusRunning {
if err = d.doNotifyRunning(); err != nil {
return err
}
}
task = nil
return
}
func (d *Downloader) Close() error {
d.lock.Lock()
defer d.lock.Unlock()
d.closed.Store(true)
for _, task := range d.tasks {
d.doPause(task, base.DownloadStatusPause)
if err := d.PauseAll(); err != nil {
return err
}
return d.storage.Close()
}
func (d *Downloader) Clear() error {
if err := d.Close(); err != nil {
return err
if !d.closed.Load() {
if err := d.Close(); err != nil {
return err
}
}
d.tasks = make([]*Task, 0)
d.extensions = make([]*Extension, 0)
if err := d.storage.Clear(); err != nil {
return err
}
@@ -491,44 +556,53 @@ func (d *Downloader) PutConfig(v *DownloaderStoreConfig) error {
return d.storage.Put(bucketConfig, "config", v)
}
func (d *Downloader) getProtocolConfig(name string, v any) (bool, error) {
func (d *Downloader) getProtocolConfig(name string, v any) bool {
cfg, err := d.GetConfig()
if err != nil {
return false, err
return false
}
if cfg.ProtocolConfig == nil || cfg.ProtocolConfig[name] == nil {
return false, nil
return false
}
if err := util.MapToStruct(cfg.ProtocolConfig[name], v); err != nil {
return false, err
d.Logger.Warn().Err(err).Msgf("get protocol config failed")
return false
}
return true, nil
return true
}
// wait task done
func (d *Downloader) watch(task *Task) {
err := task.fetcher.Wait()
if err != nil {
task.Status = base.DownloadStatusError
d.emit(EventKeyError, task, err)
} else {
task.Progress.Used = task.timer.Used()
if task.Size == 0 {
task.Size = task.fetcher.Progress().TotalDownloaded()
task.Meta.Res.Size = task.Size
}
used := task.Progress.Used / int64(time.Second)
if used == 0 {
used = 1
}
task.Progress.Speed = task.Size / used
task.Progress.Downloaded = task.Size
task.Status = base.DownloadStatusDone
d.storage.Put(bucketTask, task.ID, task.clone())
d.emit(EventKeyDone, task)
d.doOnError(task, err)
return
}
task.Progress.Used = task.timer.Used()
if task.Meta.Res.Size == 0 {
task.Meta.Res.Size = task.fetcher.Progress().TotalDownloaded()
}
used := task.Progress.Used / int64(time.Second)
if used == 0 {
used = 1
}
totalSize := task.Meta.Res.Size
task.Progress.Speed = totalSize / used
task.Progress.Downloaded = totalSize
task.updateStatus(base.DownloadStatusDone)
d.storage.Put(bucketTask, task.ID, task.clone())
d.emit(EventKeyDone, task)
d.emit(EventKeyFinally, task, err)
if task.Status == EventKeyDone {
d.notifyRunning()
}
func (d *Downloader) doOnError(task *Task, err error) {
d.Logger.Warn().Err(err).Msgf("task download failed, task id: %s", task.ID)
task.updateStatus(base.DownloadStatusError)
d.triggerOnError(task, err)
if task.Status == base.DownloadStatusError {
d.emit(EventKeyError, task, err)
d.emit(EventKeyFinally, task, err)
d.notifyRunning()
}
}
@@ -552,98 +626,175 @@ func (d *Downloader) restoreFetcher(task *Task) error {
return nil
}()
if err != nil {
// TODO log
d.Logger.Error().Stack().Err(err).Msgf("deserialize fetcher failed, task id: %s", task.ID)
}
if task.fetcher == nil {
task.fetcher = fb.Build()
}
d.setupFetcher(task.fetcher)
task.fetcher.Create(task.Meta.Opts)
if task.fetcher.Meta().Req == nil {
task.fetcher.Meta().Req = task.Meta.Req
}
if task.fetcher.Meta().Res == nil {
task.fetcher.Meta().Res = task.Meta.Res
}
go d.watch(task)
} else if task.Status == base.DownloadStatusError {
go d.watch(task)
}
task.fetcher.Create(task.Meta.Opts)
return nil
}
func (d *Downloader) tryRunning(cb func(remain int)) {
d.lock.Lock()
defer d.lock.Unlock()
func (d *Downloader) doCreate(fetcher fetcher.Fetcher, opts *base.Options) (taskId string, err error) {
if opts == nil {
opts = &base.Options{}
}
runningCount := 0
for _, task := range d.tasks {
if task.Status == base.DownloadStatusRunning {
runningCount++
meta := fetcher.Meta()
meta.Opts = opts
if opts.Path == "" {
storeConfig, err := d.GetConfig()
if err != nil {
return "", err
}
}
cb(d.cfg.MaxRunning - runningCount)
}
func (d *Downloader) notifyRunning() error {
d.lock.Lock()
defer d.lock.Unlock()
return d.doNotifyRunning()
}
func (d *Downloader) doNotifyRunning() error {
for _, task := range d.tasks {
if task.Status == base.DownloadStatusReady || task.Status == base.DownloadStatusWait {
return d.doContinue(task)
}
}
return nil
}
func (d *Downloader) start(task *Task) error {
var event EventKey
if task.Status == base.DownloadStatusReady {
event = EventKeyStart
} else {
event = EventKeyContinue
opts.Path = storeConfig.DownloadDir
}
task.Status = base.DownloadStatusRunning
task.Progress.Speed = 0
task.timer.Start()
if err := d.storage.Put(bucketTask, task.ID, task.clone()); err != nil {
return err
}
var err error
if event == EventKeyStart {
err = task.fetcher.Start()
}
if event == EventKeyContinue {
err = task.fetcher.Continue()
}
fb, err := d.parseFb(fetcher.Meta().Req.URL)
if err != nil {
return err
return
}
go d.watch(task)
d.emit(event, task)
return nil
}
func (d *Downloader) doPause(task *Task, status base.Status) (err error) {
task.lock.Lock()
defer task.lock.Unlock()
if task.Status == base.DownloadStatusRunning {
task.Status = status
task.timer.Pause()
task.fetcher.Pause()
d.storage.Put(bucketTask, task.ID, task.clone())
d.emit(EventKeyPause, task)
task := NewTask()
task.fetcherBuilder = fb
task.fetcher = fetcher
task.Meta = fetcher.Meta()
task.Progress = &Progress{}
initTask(task)
if err = fetcher.Create(opts); err != nil {
return
}
if err = d.storage.Put(bucketTask, task.ID, task.clone()); err != nil {
return
}
taskId = task.ID
func() {
d.lock.Lock()
defer d.lock.Unlock()
d.tasks = append(d.tasks, task)
remainRunningCount := d.remainRunningCount()
if remainRunningCount == 0 {
task.Status = base.DownloadStatusWait
d.waitTasks = append(d.waitTasks, task)
return
}
d.doStart(task)
}()
go d.watch(task)
return
}
func (d *Downloader) doContinue(task *Task) (err error) {
task.lock.Lock()
defer task.lock.Unlock()
func (d *Downloader) doStart(task *Task) (err error) {
if task.Status != base.DownloadStatusRunning && task.Status != base.DownloadStatusDone {
err = d.restoreFetcher(task)
err := d.restoreFetcher(task)
if err != nil {
return
d.Logger.Error().Stack().Err(err).Msgf("restore fetcher failed, task id: %s", task.ID)
return err
}
err = d.start(task)
}
isCreate := task.Status == base.DownloadStatusReady
doStart := func() error {
task.lock.Lock()
defer task.lock.Unlock()
d.triggerOnStart(task)
task.updateStatus(base.DownloadStatusRunning)
if task.Meta.Res == nil {
err := task.fetcher.Resolve(task.Meta.Req)
if err != nil {
return err
}
task.Meta.Res = task.fetcher.Meta().Res
}
if isCreate {
d.checkDuplicateLock.Lock()
defer d.checkDuplicateLock.Unlock()
task.Meta.Opts.Name = util.ReplaceInvalidFilename(task.Meta.Opts.Name)
// check if the download file is duplicated and rename it automatically.
if task.Meta.Res.Name != "" {
task.Meta.Res.Name = util.ReplaceInvalidFilename(task.Meta.Res.Name)
fullDirPath := task.Meta.FolderPath()
newName, err := util.CheckDuplicateAndRename(fullDirPath)
if err != nil {
return err
}
task.Meta.Opts.Name = newName
} else {
task.Meta.Res.Files[0].Name = util.ReplaceInvalidFilename(task.Meta.Res.Files[0].Name)
fullFilePath := task.Meta.SingleFilepath()
newName, err := util.CheckDuplicateAndRename(fullFilePath)
if err != nil {
return err
}
task.Meta.Opts.Name = newName
}
task.Meta.Res.CalcSize(task.Meta.Opts.SelectFiles)
}
task.Progress.Speed = 0
task.timer.Start()
if err := d.storage.Put(bucketTask, task.ID, task.clone()); err != nil {
return err
}
if err := task.fetcher.Start(); err != nil {
return err
}
d.emit(EventKeyStart, task)
return nil
}
go func() {
err := doStart()
if err != nil {
d.doOnError(task, err)
}
}()
return
}
func (d *Downloader) doPause(task *Task) (err error) {
err = func() error {
if task.Status != base.DownloadStatusDone {
task.updateStatus(base.DownloadStatusPause)
task.timer.Pause()
if task.fetcher != nil {
if err := task.fetcher.Pause(); err != nil {
return err
}
}
if err := d.storage.Put(bucketTask, task.ID, task.clone()); err != nil {
return err
}
d.emit(EventKeyPause, task)
}
return nil
}()
if err != nil {
d.Logger.Error().Stack().Err(err).Msgf("pause task failed, task id: %s", task.ID)
return
}
return
}
@@ -659,6 +810,16 @@ func logPanic(logDir string) {
paniclog.RedirectStderr(f)
}
func (d *Downloader) buildFetcher(url string) (fetcher.Fetcher, error) {
fb, err := d.parseFb(url)
if err != nil {
return nil, err
}
fetcher := fb.Build()
d.setupFetcher(fetcher)
return fetcher, nil
}
func initTask(task *Task) {
task.timer = util.NewTimer(task.Progress.Used)
task.lock = &sync.Mutex{}
@@ -696,12 +857,11 @@ func (b *boot) Listener(listener Listener) *boot {
}
func (b *boot) Create(opts *base.Options) (string, error) {
rr, err := b.Resolve()
if err != nil {
return "", err
}
defaultDownloader.Listener(b.listener)
return defaultDownloader.Create(rr.ID, opts)
return defaultDownloader.CreateDirect(&base.Request{
URL: b.url,
Extra: b.extra,
}, opts)
}
func Boot() *boot {
+123 -6
View File
@@ -27,7 +27,6 @@ func TestDownloader_Resolve(t *testing.T) {
t.Fatal(err)
}
want := &base.Resource{
Name: test.BuildName,
Size: test.BuildSize,
Range: true,
Files: []*base.FileInfo{
@@ -85,6 +84,127 @@ func TestDownloader_Create(t *testing.T) {
}
}
func TestDownloader_CreateWithProxy(t *testing.T) {
// No proxy
doTestDownloaderCreateWithProxy(t, false, func(proxyCfg *DownloaderProxyConfig) *DownloaderProxyConfig {
return nil
})
// Disable proxy
doTestDownloaderCreateWithProxy(t, false, func(proxyCfg *DownloaderProxyConfig) *DownloaderProxyConfig {
proxyCfg.Enable = false
return proxyCfg
})
// Invalid proxy scheme
doTestDownloaderCreateWithProxy(t, false, func(proxyCfg *DownloaderProxyConfig) *DownloaderProxyConfig {
proxyCfg.Scheme = ""
return proxyCfg
})
// Invalid proxy host
doTestDownloaderCreateWithProxy(t, false, func(proxyCfg *DownloaderProxyConfig) *DownloaderProxyConfig {
proxyCfg.Host = ""
return proxyCfg
})
// Use proxy without auth
doTestDownloaderCreateWithProxy(t, false, func(proxyCfg *DownloaderProxyConfig) *DownloaderProxyConfig {
return proxyCfg
})
// Use proxy with auth
doTestDownloaderCreateWithProxy(t, true, func(proxyCfg *DownloaderProxyConfig) *DownloaderProxyConfig {
return proxyCfg
})
}
func doTestDownloaderCreateWithProxy(t *testing.T, auth bool, buildProxyConfig func(proxyCfg *DownloaderProxyConfig) *DownloaderProxyConfig) {
httpListener := test.StartTestFileServer()
defer httpListener.Close()
usr, pwd := "", ""
if auth {
usr, pwd = "admin", "123"
}
proxyListener := test.StartSocks5Server(usr, pwd)
defer proxyListener.Close()
downloader := NewDownloader(nil)
if err := downloader.Setup(); err != nil {
t.Fatal(err)
}
defer downloader.Clear()
downloader.cfg.DownloaderStoreConfig.Proxy = buildProxyConfig(&DownloaderProxyConfig{
Enable: true,
Scheme: "socks5",
Host: proxyListener.Addr().String(),
Usr: usr,
Pwd: pwd,
})
req := &base.Request{
URL: "http://" + httpListener.Addr().String() + "/" + test.BuildName,
}
rr, err := downloader.Resolve(req)
if err != nil {
t.Fatal(err)
}
want := &base.Resource{
Size: test.BuildSize,
Range: true,
Files: []*base.FileInfo{
{
Name: test.BuildName,
Path: "",
Size: test.BuildSize,
},
},
}
if !reflect.DeepEqual(want, rr.Res) {
t.Errorf("Resolve() got = %v, want %v", rr.Res, want)
}
}
func TestDownloader_CreateRename(t *testing.T) {
listener := test.StartTestFileServer()
defer listener.Close()
downloader := NewDownloader(nil)
if err := downloader.Setup(); err != nil {
t.Fatal(err)
}
defer downloader.Clear()
req := &base.Request{
URL: "http://" + listener.Addr().String() + "/" + test.BuildName,
}
var wg sync.WaitGroup
wg.Add(2)
downloader.Listener(func(event *Event) {
if event.Key == EventKeyDone {
wg.Done()
}
})
for i := 0; i < 2; i++ {
_, err := downloader.CreateDirect(req, &base.Options{
Path: test.Dir,
Name: test.DownloadName,
Extra: http.OptsExtra{
Connections: 4,
},
})
if err != nil {
t.Fatal(err)
}
}
wg.Wait()
want := test.FileMd5(test.BuildFile)
got := test.FileMd5(test.DownloadFile)
if want != got {
t.Errorf("Downloader_CreateRename() got = %v, want %v", got, want)
}
got = test.FileMd5(test.DownloadRenameFile)
if want != got {
t.Errorf("Downloader_CreateRename() got = %v, want %v", got, want)
}
}
func TestDownloader_StoreAndRestore(t *testing.T) {
listener := test.StartTestSlowFileServer(time.Millisecond * 2000)
defer listener.Close()
@@ -161,10 +281,7 @@ func TestDownloader_Protocol_Config(t *testing.T) {
defer downloader.Clear()
var httpCfg map[string]any
exits, err := downloader.getProtocolConfig("http", &httpCfg)
if err != nil {
t.Fatal(err)
}
exits := downloader.getProtocolConfig("http", &httpCfg)
if exits {
t.Errorf("getProtocolConfig() got = %v, want %v", exits, false)
}
@@ -190,7 +307,7 @@ func TestDownloader_Protocol_Config(t *testing.T) {
},
}
if err = downloader.PutConfig(storeCfg); err != nil {
if err := downloader.PutConfig(storeCfg); err != nil {
t.Fatal(err)
}
+182
View File
@@ -0,0 +1,182 @@
package engine
import (
_ "embed"
"errors"
gojaerror "github.com/GopeedLab/gopeed/pkg/download/engine/inject/error"
"github.com/GopeedLab/gopeed/pkg/download/engine/inject/file"
"github.com/GopeedLab/gopeed/pkg/download/engine/inject/formdata"
"github.com/GopeedLab/gopeed/pkg/download/engine/inject/vm"
"github.com/GopeedLab/gopeed/pkg/download/engine/inject/xhr"
"github.com/dop251/goja"
"github.com/dop251/goja_nodejs/eventloop"
gojaurl "github.com/dop251/goja_nodejs/url"
"net/url"
"time"
)
//go:embed polyfill/out/index.js
var polyfillScript string
type Engine struct {
loop *eventloop.EventLoop
Runtime *goja.Runtime
}
// RunString executes the script and returns the go type value
// if script result is promise, it will be resolved
func (e *Engine) RunString(script string) (value any, err error) {
defer func() {
if r := recover(); r != nil {
err = r.(error)
}
}()
var result goja.Value
e.loop.Run(func(runtime *goja.Runtime) {
result, err = runtime.RunString(script)
if err == nil {
go e.await(result)
}
})
if err != nil {
return
}
return resolveResult(result)
}
// CallFunction calls the function and returns the go type value
// if function result is promise, it will be resolved
func (e *Engine) CallFunction(fn goja.Callable, args ...any) (value any, err error) {
defer func() {
if r := recover(); r != nil {
err = r.(error)
}
}()
var result goja.Value
e.loop.Run(func(runtime *goja.Runtime) {
if args == nil {
result, err = fn(nil)
} else {
var jsArgs []goja.Value
for _, arg := range args {
jsArgs = append(jsArgs, runtime.ToValue(arg))
}
result, err = fn(nil, jsArgs...)
}
if err == nil {
go e.await(result)
}
})
if err != nil {
return
}
return resolveResult(result)
}
// loop.Run will hang if the script result has a non-stop code, such as setInterval.
// This method will stop the event loop when the promise result is resolved.
func (e *Engine) await(value any) {
if value == nil {
return
}
if v, ok := value.(goja.Value); ok {
// if result is promise, wait for it to be resolved
if p, ok := v.Export().(*goja.Promise); ok {
if p.State() != goja.PromiseStatePending {
return
}
// check promise state every 100 milliseconds, until it is resolved
for {
time.Sleep(time.Millisecond * 100)
if p.State() == goja.PromiseStatePending {
continue
}
break
}
// stop the event loop
e.loop.StopNoWait()
}
}
}
func (e *Engine) Close() {
e.loop.Stop()
}
type Config struct {
ProxyURL *url.URL
}
func NewEngine(cfg *Config) *Engine {
if cfg == nil {
cfg = &Config{}
}
loop := eventloop.NewEventLoop()
engine := &Engine{
loop: loop,
}
loop.Run(func(runtime *goja.Runtime) {
engine.Runtime = runtime
runtime.SetFieldNameMapper(goja.TagFieldNameMapper("json", true))
vm.Enable(runtime)
gojaurl.Enable(runtime)
if err := gojaerror.Enable(runtime); err != nil {
return
}
if err := file.Enable(runtime); err != nil {
return
}
if err := formdata.Enable(runtime); err != nil {
return
}
if err := xhr.Enable(runtime, cfg.ProxyURL); err != nil {
return
}
if _, err := runtime.RunString(polyfillScript); err != nil {
return
}
// polyfill global
if err := runtime.Set("global", runtime.GlobalObject()); err != nil {
return
}
// polyfill window.location
if _, err := runtime.RunString("global.location = new URL('http://localhost');"); err != nil {
return
}
return
})
return engine
}
func Run(script string) (value any, err error) {
engine := NewEngine(nil)
return engine.RunString(script)
}
// if the value is Promise, it will be resolved and return the result.
func resolveResult(value goja.Value) (any, error) {
export := value.Export()
switch export.(type) {
case *goja.Promise:
p := export.(*goja.Promise)
switch p.State() {
case goja.PromiseStatePending:
return nil, nil
case goja.PromiseStateFulfilled:
return p.Result().Export(), nil
case goja.PromiseStateRejected:
if err, ok := p.Result().Export().(error); ok {
return nil, err
} else {
return nil, errors.New(p.Result().String())
}
}
}
return export, nil
}
+397
View File
@@ -0,0 +1,397 @@
package engine
import (
"crypto/md5"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"github.com/GopeedLab/gopeed/internal/test"
gojaerror "github.com/GopeedLab/gopeed/pkg/download/engine/inject/error"
"github.com/GopeedLab/gopeed/pkg/download/engine/inject/file"
gojautil "github.com/GopeedLab/gopeed/pkg/download/engine/util"
"github.com/GopeedLab/gopeed/pkg/util"
"github.com/dop251/goja"
"io"
"net"
"net/http"
"strconv"
"strings"
"testing"
"time"
)
func TestPolyfill(t *testing.T) {
doTestPolyfill(t, "MessageError")
doTestPolyfill(t, "XMLHttpRequest")
doTestPolyfill(t, "Blob")
doTestPolyfill(t, "FormData")
doTestPolyfill(t, "fetch")
doTestPolyfill(t, "__gopeed_create_vm")
}
func TestError(t *testing.T) {
engine := NewEngine(nil)
_, err := engine.RunString(`
throw new MessageError('test');
`)
if me, ok := gojautil.AssertError[*gojaerror.MessageError](err); !ok {
t.Fatalf("expect MessageError, but got %v", me)
}
}
func TestFetch(t *testing.T) {
server := startServer()
defer server.Close()
engine := NewEngine(nil)
if _, err := engine.RunString(fmt.Sprintf("var host = 'http://%s';", server.Addr().String())); err != nil {
t.Fatal(err)
}
_, err := engine.RunString(`
async function testGet(){
const resp = await fetch(host+'/get');
return resp.status;
}
async function testText(){
const resp = await fetch(host+'/text',{
method: 'POST',
body: 'test'
});
return await resp.text();
}
async function testOctetStream(file){
const resp = await fetch(host+'/octetStream',{
method: 'POST',
body: file
});
return await resp.text();
}
async function testFormData(file){
const formData = new FormData();
formData.append('name', 'test');
formData.append('f', file);
const resp = await fetch(host+'/formData',{
method: 'POST',
body: formData
});
return await resp.json();
}
function testProgress(){
return new Promise((resolve, reject) => {
const xhr = new XMLHttpRequest();
xhr.open('GET', host+'/get');
const xhrUploadPromise = new Promise((resolve, reject) => {
xhr.upload.onprogress = function(e){
if(e.loaded === e.total){
resolve();
}
}
});
const xhrPromise = new Promise((resolve, reject) => {
xhr.onprogress = function(e){
if(e.loaded === e.total){
resolve();
}
}
});
Promise.all([xhrUploadPromise, xhrPromise]).then(() => {
resolve();
});
xhr.send();
setTimeout(() => {
reject('timeout');
}, 1000);
});
}
function testAbort(){
return new Promise((resolve, reject) => {
const xhr = new XMLHttpRequest();
xhr.open('GET', host+'/timeout?duration=500');
xhr.onabort = function() {
resolve();
};
xhr.send();
setTimeout(() => {
xhr.abort();
}, 200);
setTimeout(() => {
reject('timeout');
}, 1000);
});
}
function testTimeout(){
return new Promise((resolve, reject) => {
const xhr = new XMLHttpRequest();
const t = 500;
xhr.open('GET', host+'/timeout?duration='+t);
xhr.timeout = t - 200;
xhr.onload = function() {
resolve();
};
xhr.ontimeout = function() {
reject('timeout');
};
xhr.send();
});
}
`)
if err != nil {
t.Fatal(err)
}
result, err := callTestFun(engine, "testGet")
if err != nil {
t.Fatal(err)
}
if result != int64(200) {
t.Fatalf("testGet failed, want %d, got %d", 200, result)
}
result, err = callTestFun(engine, "testText")
if err != nil {
t.Fatal(err)
}
if result != "test" {
t.Fatalf("testText failed, want %s, got %s", "test", result)
}
func() {
jsFile, _, md5 := buildFile(t, engine.Runtime)
result, err = callTestFun(engine, "testOctetStream", jsFile)
if err != nil {
t.Fatal(err)
}
if result != md5 {
t.Fatalf("testOctetStream failed, want %s, got %s", md5, result)
}
}()
func() {
jsFile, goFile, md5 := buildFile(t, engine.Runtime)
result, err = callTestFun(engine, "testFormData", jsFile)
if err != nil {
t.Fatal(err)
}
want := map[string]any{
"name": "test",
"f": map[string]string{
"filename": goFile.Name,
"md5": md5,
},
}
if !test.JsonEqual(result, want) {
t.Fatalf("testFormData failed, want %v, got %v", want, result)
}
}()
_, err = callTestFun(engine, "testProgress")
if err != nil {
t.Fatal("progress test failed")
}
_, err = callTestFun(engine, "testAbort")
if err != nil {
t.Fatal("abort test failed")
}
_, err = callTestFun(engine, "testTimeout")
if err == nil || err.Error() != "timeout" {
t.Fatalf("timeout test failed, want %s, got %s", "timeout", err)
}
}
func TestFetchWithProxy(t *testing.T) {
doTestFetchWithProxy(t, "", "")
doTestFetchWithProxy(t, "admin", "123")
}
func doTestFetchWithProxy(t *testing.T, usr, pwd string) {
httpListener := startServer()
defer httpListener.Close()
proxyListener := test.StartSocks5Server(usr, pwd)
defer proxyListener.Close()
engine := NewEngine(&Config{ProxyURL: util.BuildProxyUrl("socks5", proxyListener.Addr().String(), usr, pwd)})
if _, err := engine.RunString(fmt.Sprintf("var host = 'http://%s';", httpListener.Addr().String())); err != nil {
t.Fatal(err)
}
respCode, err := engine.RunString(`
(async function(){
const resp = await fetch(host+'/get');
return resp.status;
})()
`)
if err != nil {
t.Fatal(err)
}
if respCode != int64(200) {
t.Fatalf("fetch with proxy failed, want %d, got %d", 200, respCode)
}
}
func TestVm(t *testing.T) {
engine := NewEngine(nil)
value, err := engine.RunString(`
const vm = __gopeed_create_vm()
vm.set('a', 1)
vm.set('b', 2)
const result = vm.runString('a=a+1;b=b+1;a+b;')
const out = {
"a": vm.get('a'),
"b": vm.get('b'),
"result": result
}
out
`)
if err != nil {
t.Fatal(err)
}
want := map[string]any{
"a": 2,
"b": 3,
"result": 5,
}
if !test.JsonEqual(value, want) {
t.Fatalf("vm test failed, want %v, got %v", want, value)
}
}
func TestNonStopLoop(t *testing.T) {
engine := NewEngine(nil)
_, err := engine.RunString(`
function leak(){
setInterval(() => {
},500)
}
function test(){
leak()
return new Promise((resolve, reject) => {
setTimeout(() => {
resolve('done')
}, 1000)
})
}
`)
if err != nil {
t.Fatal(err)
}
val, err := callTestFun(engine, "test")
if err != nil {
panic(err)
}
if val != "done" {
t.Fatalf("infinite loop test failed, want %s, got %s", "done", val)
}
}
func doTestPolyfill(t *testing.T, module string) {
value, err := Run(fmt.Sprintf(`
!!globalThis['%s']
`, module))
if err != nil {
t.Fatal(err)
}
if !value.(bool) {
t.Fatalf("module %s not polyfilled", module)
}
}
func startServer() net.Listener {
listener, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
panic(err)
}
server := &http.Server{}
mux := http.NewServeMux()
mux.HandleFunc("/get", func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
w.Write([]byte("ok"))
})
mux.HandleFunc("/text", func(w http.ResponseWriter, r *http.Request) {
buf, _ := io.ReadAll(r.Body)
w.WriteHeader(http.StatusOK)
w.Write(buf)
})
mux.HandleFunc("/octetStream", func(w http.ResponseWriter, r *http.Request) {
md5 := calcMd5(r.Body)
w.WriteHeader(http.StatusOK)
w.Write([]byte(md5))
})
mux.HandleFunc("/formData", func(w http.ResponseWriter, r *http.Request) {
err := r.ParseMultipartForm(1024 * 1024 * 30)
if err != nil {
w.WriteHeader(http.StatusBadRequest)
w.Write([]byte(err.Error()))
return
}
result := make(map[string]any)
for k, v := range r.MultipartForm.Value {
result[k] = v[0]
}
for k, v := range r.MultipartForm.File {
f, _ := v[0].Open()
result[k] = map[string]string{
"filename": v[0].Filename,
"md5": calcMd5(f),
}
}
w.WriteHeader(http.StatusOK)
buf, _ := json.Marshal(result)
w.Write(buf)
})
mux.HandleFunc("/timeout", func(w http.ResponseWriter, r *http.Request) {
duration := r.URL.Query().Get("duration")
t, _ := strconv.Atoi(duration)
time.Sleep(time.Duration(t) * time.Millisecond)
w.WriteHeader(http.StatusOK)
w.Write([]byte("ok"))
})
server.Handler = mux
go server.Serve(listener)
return listener
}
func buildFile(t *testing.T, runtime *goja.Runtime) (goja.Value, *file.File, string) {
jsFile, err := file.NewJsFile(runtime)
if err != nil {
t.Fatal(err)
}
f := jsFile.Export().(*file.File)
data := "test"
f.Reader = strings.NewReader(data)
f.Name = "test.txt"
f.Size = int64(len(data))
return jsFile, f, calcMd5(strings.NewReader(data))
}
func callTestFun(engine *Engine, fun string, args ...any) (any, error) {
test, ok := goja.AssertFunction(engine.Runtime.Get(fun))
if !ok {
return nil, errors.New("function not found:" + fun)
}
return engine.CallFunction(test, args...)
}
func calcMd5(reader io.Reader) string {
// Open a new hash interface to write to
hash := md5.New()
// Copy the file in the hash interface and check for any error
if _, err := io.Copy(hash, reader); err != nil {
return ""
}
return hex.EncodeToString(hash.Sum(nil))
}
@@ -0,0 +1,29 @@
package error
import (
"github.com/dop251/goja"
)
type MessageError struct {
Message string `json:"message"`
}
func (e *MessageError) Error() string {
return e.Message
}
func Enable(runtime *goja.Runtime) error {
messageError := runtime.ToValue(func(call goja.ConstructorCall) *goja.Object {
var message string
if len(call.Arguments) > 0 {
message = call.Arguments[0].String()
}
instance := &MessageError{
Message: message,
}
instanceValue := runtime.ToValue(instance).(*goja.Object)
instanceValue.SetPrototype(call.This.Prototype())
return instanceValue
})
return runtime.Set("MessageError", messageError)
}
+32
View File
@@ -0,0 +1,32 @@
package file
import (
"errors"
"github.com/dop251/goja"
"io"
)
type File struct {
io.Reader `json:""`
io.Closer `json:""`
Name string `json:"name"`
Size int64 `json:"size"`
}
func NewJsFile(runtime *goja.Runtime) (goja.Value, error) {
fileCtor, ok := goja.AssertConstructor(runtime.Get("File"))
if !ok {
return nil, errors.New("file is not defined")
}
return fileCtor(nil)
}
func Enable(runtime *goja.Runtime) error {
file := runtime.ToValue(func(call goja.ConstructorCall) *goja.Object {
instance := &File{}
instanceValue := runtime.ToValue(instance).(*goja.Object)
instanceValue.SetPrototype(call.This.Prototype())
return instanceValue
})
return runtime.Set("File", file)
}
@@ -0,0 +1,68 @@
package formdata
import "github.com/dop251/goja"
type FormData struct {
data map[string]any
}
func (fd *FormData) Append(name string, value any) {
fd.data[name] = value
}
func (fd *FormData) Delete(name string) {
delete(fd.data, name)
}
func (fd *FormData) Entries() []any {
var entries []any
for k, v := range fd.data {
entries = append(entries, []any{k, v})
}
return entries
}
func (fd *FormData) Get(name string) any {
return fd.data[name]
}
func (fd *FormData) GetAll(name string) []any {
return []any{fd.data[name]}
}
func (fd *FormData) Has(name string) bool {
_, ok := fd.data[name]
return ok
}
func (fd *FormData) Keys() []string {
var keys []string
for k := range fd.data {
keys = append(keys, k)
}
return keys
}
func (fd *FormData) Set(name string, value any) {
fd.data[name] = value
}
func (fd *FormData) Values() []any {
var values []any
for _, v := range fd.data {
values = append(values, v)
}
return values
}
func Enable(runtime *goja.Runtime) error {
file := runtime.ToValue(func(call goja.ConstructorCall) *goja.Object {
instance := &FormData{
data: make(map[string]any),
}
instanceValue := runtime.ToValue(instance).(*goja.Object)
instanceValue.SetPrototype(call.This.Prototype())
return instanceValue
})
return runtime.Set("FormData", file)
}
+44
View File
@@ -0,0 +1,44 @@
package vm
import (
"github.com/dop251/goja"
"github.com/dop251/goja_nodejs/eventloop"
)
type Vm struct {
loop *eventloop.EventLoop
}
func (vm *Vm) Set(name string, value any) {
vm.loop.Run(func(runtime *goja.Runtime) {
runtime.Set(name, value)
})
}
func (vm *Vm) Get(name string) (value any) {
vm.loop.Run(func(runtime *goja.Runtime) {
value = runtime.Get(name)
})
return
}
func (vm *Vm) RunString(script string) (value any, err error) {
defer func() {
if r := recover(); r != nil {
err = r.(error)
}
}()
vm.loop.Run(func(runtime *goja.Runtime) {
value, err = runtime.RunString(script)
})
return
}
func Enable(runtime *goja.Runtime) error {
return runtime.Set("__gopeed_create_vm", func(call goja.FunctionCall) goja.Value {
return runtime.ToValue(&Vm{
loop: eventloop.NewEventLoop(),
})
})
}
+425
View File
@@ -0,0 +1,425 @@
package xhr
import (
"bytes"
"github.com/GopeedLab/gopeed/pkg/download/engine/inject/file"
"github.com/GopeedLab/gopeed/pkg/download/engine/inject/formdata"
"github.com/GopeedLab/gopeed/pkg/download/engine/util"
"github.com/dop251/goja"
"io"
"mime/multipart"
"net"
"net/http"
"net/url"
"time"
)
const (
eventLoad = "load"
eventReadystatechange = "readystatechange"
eventProgress = "progress"
eventAbort = "abort"
eventError = "error"
eventTimeout = "timeout"
)
type ProgressEvent struct {
Type string `json:"type"`
LengthComputable bool `json:"lengthComputable"`
Loaded int64 `json:"loaded"`
Total int64 `json:"total"`
}
type EventProp struct {
eventListeners map[string]func(event *ProgressEvent)
Onload func(event *ProgressEvent) `json:"onload"`
Onprogress func(event *ProgressEvent) `json:"onprogress"`
Onabort func(event *ProgressEvent) `json:"onabort"`
Onerror func(event *ProgressEvent) `json:"onerror"`
Ontimeout func(event *ProgressEvent) `json:"ontimeout"`
}
func (ep *EventProp) AddEventListener(event string, cb func(event *ProgressEvent)) {
ep.eventListeners[event] = cb
}
func (ep *EventProp) RemoveEventListener(event string) {
delete(ep.eventListeners, event)
}
func (ep *EventProp) callOnload() {
event := &ProgressEvent{
Type: eventLoad,
LengthComputable: false,
}
if ep.Onload != nil {
ep.Onload(event)
}
ep.callEventListener(event)
}
func (ep *EventProp) callOnprogress(loaded, total int64) {
event := &ProgressEvent{
Type: eventProgress,
LengthComputable: true,
Loaded: loaded,
Total: total,
}
if ep.Onprogress != nil {
ep.Onprogress(event)
}
ep.callEventListener(event)
}
func (ep *EventProp) callOnabort() {
event := &ProgressEvent{
Type: eventAbort,
LengthComputable: false,
}
if ep.Onabort != nil {
ep.Onabort(event)
}
ep.callEventListener(event)
}
func (ep *EventProp) callOnerror() {
event := &ProgressEvent{
Type: eventError,
LengthComputable: false,
}
if ep.Onerror != nil {
ep.Onerror(event)
}
ep.callEventListener(event)
}
func (ep *EventProp) callOntimeout() {
event := &ProgressEvent{
Type: eventTimeout,
LengthComputable: false,
}
if ep.Ontimeout != nil {
ep.Ontimeout(event)
}
ep.callEventListener(event)
}
func (ep *EventProp) callEventListener(event *ProgressEvent) {
if cb, ok := ep.eventListeners[event.Type]; ok {
cb(event)
}
}
type XMLHttpRequestUpload struct {
*EventProp
}
type XMLHttpRequest struct {
method string
url string
requestHeaders map[string]string
responseHeaders map[string]string
aborted bool
proxyUrl *url.URL
Upload *XMLHttpRequestUpload `json:"upload"`
Timeout int `json:"timeout"`
ReadyState int `json:"readyState"`
Status int `json:"status"`
StatusText string `json:"statusText"`
Response string `json:"response"`
ResponseText string `json:"responseText"`
*EventProp
Onreadystatechange func(event *ProgressEvent) `json:"onreadystatechange"`
}
func (xhr *XMLHttpRequest) Open(method, url string) {
xhr.method = method
xhr.url = url
xhr.requestHeaders = make(map[string]string)
xhr.responseHeaders = make(map[string]string)
xhr.doReadystatechange(1)
}
func (xhr *XMLHttpRequest) SetRequestHeader(key, value string) {
xhr.requestHeaders[key] = value
}
func (xhr *XMLHttpRequest) Send(data goja.Value) {
var req *http.Request
var err error
d := xhr.parseData(data)
var (
contentType string
contentLength int64
)
if d == nil || xhr.method == "GET" || xhr.method == "HEAD" {
req, err = http.NewRequest(xhr.method, xhr.url, nil)
} else {
switch d.(type) {
case string:
req, err = http.NewRequest(xhr.method, xhr.url, bytes.NewBufferString(d.(string)))
contentType = "text/plain;charset=UTF-8"
contentLength = int64(len(d.(string)))
case *file.File:
req, err = http.NewRequest(xhr.method, xhr.url, d.(*file.File).Reader)
contentType = "application/octet-stream"
contentLength = d.(*file.File).Size
case *formdata.FormData:
pr, pw := io.Pipe()
mw := NewMultipart(pw)
for _, e := range d.(*formdata.FormData).Entries() {
arr := e.([]any)
k := arr[0].(string)
v := arr[1]
switch v.(type) {
case string:
mw.WriteField(k, v.(string))
case *file.File:
mw.WriteFile(k, v.(*file.File))
}
}
go func() {
defer pw.Close()
defer mw.Close()
mw.Send()
}()
req, err = http.NewRequest(xhr.method, xhr.url, pr)
contentType = mw.FormDataContentType()
contentLength = mw.Size()
}
}
if err != nil {
xhr.callOnerror()
return
}
if contentType != "" {
req.Header.Set("Content-Type", contentType)
}
if contentLength > 0 {
req.ContentLength = contentLength
}
for k, v := range xhr.requestHeaders {
req.Header.Set(k, v)
}
transport := &http.Transport{}
if xhr.proxyUrl != nil {
transport.Proxy = http.ProxyURL(xhr.proxyUrl)
}
client := &http.Client{
Transport: transport,
Timeout: time.Duration(xhr.Timeout) * time.Millisecond,
}
resp, err := client.Do(req)
if err != nil {
// handle timeout error
if err, ok := err.(net.Error); ok && err.Timeout() {
if xhr.Timeout > 0 {
xhr.Upload.callOntimeout()
xhr.callOntimeout()
}
return
}
xhr.Upload.callOnerror()
xhr.callOnerror()
return
}
defer resp.Body.Close()
xhr.Upload.callOnprogress(contentLength, contentLength)
if !xhr.aborted {
xhr.Upload.callOnload()
}
for k, v := range resp.Header {
xhr.responseHeaders[k] = v[0]
}
xhr.Status = resp.StatusCode
xhr.StatusText = resp.Status
xhr.doReadystatechange(2)
buf, err := io.ReadAll(resp.Body)
if err != nil {
xhr.callOnerror()
return
}
xhr.doReadystatechange(3)
xhr.Response = string(buf)
xhr.ResponseText = xhr.Response
xhr.doReadystatechange(4)
respBodyLen := int64(len(buf))
xhr.callOnprogress(respBodyLen, respBodyLen)
if !xhr.aborted {
xhr.callOnload()
}
return
}
func (xhr *XMLHttpRequest) Abort() {
xhr.doReadystatechange(0)
xhr.aborted = true
xhr.Upload.callOnabort()
xhr.callOnabort()
}
func (xhr *XMLHttpRequest) GetResponseHeader(key string) string {
return xhr.responseHeaders[key]
}
func (xhr *XMLHttpRequest) GetAllResponseHeaders() string {
var buf bytes.Buffer
for k, v := range xhr.responseHeaders {
buf.WriteString(k)
buf.WriteString(": ")
buf.WriteString(v)
buf.WriteString("\r\n")
}
return buf.String()
}
func (xhr *XMLHttpRequest) callOnreadystatechange() {
event := &ProgressEvent{
Type: eventReadystatechange,
LengthComputable: false,
}
if xhr.Onreadystatechange != nil {
xhr.Onreadystatechange(event)
}
xhr.callEventListener(event)
}
func (xhr *XMLHttpRequest) doReadystatechange(state int) {
if xhr.aborted {
return
}
xhr.ReadyState = state
xhr.callOnreadystatechange()
}
// parse js data to go struct
func (xhr *XMLHttpRequest) parseData(data goja.Value) any {
// check if data is null or undefined
if data == nil || goja.IsNull(data) || goja.IsUndefined(data) || goja.IsNaN(data) {
return nil
}
// check if data is File
f, ok := data.Export().(*file.File)
if ok {
return f
}
// check if data is FormData
fd, ok := data.Export().(*formdata.FormData)
if ok {
return fd
}
// otherwise, return data as string
return data.String()
}
func Enable(runtime *goja.Runtime, proxyUrl *url.URL) error {
progressEvent := runtime.ToValue(func(call goja.ConstructorCall) *goja.Object {
if len(call.Arguments) < 1 {
util.ThrowTypeError(runtime, "Failed to construct 'ProgressEvent': 1 argument required, but only 0 present.")
}
instance := &ProgressEvent{
Type: call.Argument(0).String(),
}
instanceValue := runtime.ToValue(instance).(*goja.Object)
instanceValue.SetPrototype(call.This.Prototype())
return instanceValue
})
xhr := runtime.ToValue(func(call goja.ConstructorCall) *goja.Object {
instance := &XMLHttpRequest{
proxyUrl: proxyUrl,
Upload: &XMLHttpRequestUpload{
EventProp: &EventProp{
eventListeners: make(map[string]func(event *ProgressEvent)),
},
},
EventProp: &EventProp{
eventListeners: make(map[string]func(event *ProgressEvent)),
},
}
instanceValue := runtime.ToValue(instance).(*goja.Object)
instanceValue.SetPrototype(call.This.Prototype())
return instanceValue
})
if err := runtime.Set("ProgressEvent", progressEvent); err != nil {
return err
}
if err := runtime.Set("XMLHttpRequest", xhr); err != nil {
return err
}
return nil
}
// Wrap multipart.Writer and stat content length
type multipartWrapper struct {
statBuffer *bytes.Buffer
statWriter *multipart.Writer
writer *multipart.Writer
fields map[string]any
}
func NewMultipart(w io.Writer) *multipartWrapper {
var buf bytes.Buffer
return &multipartWrapper{
statBuffer: &buf,
statWriter: multipart.NewWriter(&buf),
writer: multipart.NewWriter(w),
fields: make(map[string]any),
}
}
func (w *multipartWrapper) WriteField(fieldname string, value string) error {
w.fields[fieldname] = value
return w.statWriter.WriteField(fieldname, value)
}
func (w *multipartWrapper) WriteFile(fieldname string, file *file.File) error {
w.fields[fieldname] = file
_, err := w.statWriter.CreateFormFile(fieldname, file.Name)
if err != nil {
return err
}
return nil
}
func (w *multipartWrapper) Size() int64 {
w.statWriter.Close()
size := int64(w.statBuffer.Len())
for _, v := range w.fields {
switch v.(type) {
case *file.File:
f := v.(*file.File)
size += f.Size
}
}
return size
}
func (w *multipartWrapper) Send() error {
for k, v := range w.fields {
switch v.(type) {
case string:
if err := w.writer.WriteField(k, v.(string)); err != nil {
return err
}
case *file.File:
f := v.(*file.File)
fw, err := w.writer.CreateFormFile(k, f.Name)
if err != nil {
return err
}
if _, err = io.Copy(fw, f); err != nil {
return err
}
}
}
return nil
}
func (w *multipartWrapper) FormDataContentType() string {
return w.writer.FormDataContentType()
}
func (w *multipartWrapper) Close() error {
return w.writer.Close()
}
@@ -0,0 +1 @@
(()=>{var t={725:function(t,e,r){var o,n,i;i="undefined"!=typeof self&&self||"undefined"!=typeof window&&window||void 0!==r.g&&r.g||this,o=function(t){"use strict";var e=i.BlobBuilder||i.WebKitBlobBuilder||i.MSBlobBuilder||i.MozBlobBuilder,r=i.URL||i.webkitURL||function(t,e){return(e=document.createElement("a")).href=t,e},o=i.Blob,n=r.createObjectURL,a=r.revokeObjectURL,s=i.Symbol&&i.Symbol.toStringTag,u=!1,f=!1,l=e&&e.prototype.append&&e.prototype.getBlob;try{u=2===new Blob(["ä"]).size,f=2===new Blob([new Uint8Array([1,2])]).size}catch(t){}function c(t){return t.map((function(t){if(t.buffer instanceof ArrayBuffer){var e=t.buffer;if(t.byteLength!==e.byteLength){var r=new Uint8Array(t.byteLength);r.set(new Uint8Array(e,t.byteOffset,t.byteLength)),e=r.buffer}return e}return t}))}function h(t,r){r=r||{};var o=new e;return c(t).forEach((function(t){o.append(t)})),r.type?o.getBlob(r.type):o.getBlob()}function p(t,e){return new o(c(t),e||{})}i.Blob&&(h.prototype=Blob.prototype,p.prototype=Blob.prototype);var y="function"==typeof TextEncoder?TextEncoder.prototype.encode.bind(new TextEncoder):function(t){for(var e=0,r=t.length,o=i.Uint8Array||Array,n=0,a=Math.max(32,r+(r>>1)+7),s=new o(a>>3<<3);e<r;){var u=t.charCodeAt(e++);if(u>=55296&&u<=56319){if(e<r){var f=t.charCodeAt(e);56320==(64512&f)&&(++e,u=((1023&u)<<10)+(1023&f)+65536)}if(u>=55296&&u<=56319)continue}if(n+4>s.length){a+=8,a=(a*=1+e/t.length*2)>>3<<3;var l=new Uint8Array(a);l.set(s),s=l}if(0!=(4294967168&u)){if(0==(4294965248&u))s[n++]=u>>6&31|192;else if(0==(4294901760&u))s[n++]=u>>12&15|224,s[n++]=u>>6&63|128;else{if(0!=(4292870144&u))continue;s[n++]=u>>18&7|240,s[n++]=u>>12&63|128,s[n++]=u>>6&63|128}s[n++]=63&u|128}else s[n++]=u}return s.slice(0,n)},d="function"==typeof TextDecoder?TextDecoder.prototype.decode.bind(new TextDecoder):function(t){for(var e=t.length,r=[],o=0;o<e;){var n,i,a,s,u=t[o],f=null,l=u>239?4:u>223?3:u>191?2:1;if(o+l<=e)switch(l){case 1:u<128&&(f=u);break;case 2:128==(192&(n=t[o+1]))&&(s=(31&u)<<6|63&n)>127&&(f=s);break;case 3:n=t[o+1],i=t[o+2],128==(192&n)&&128==(192&i)&&(s=(15&u)<<12|(63&n)<<6|63&i)>2047&&(s<55296||s>57343)&&(f=s);break;case 4:n=t[o+1],i=t[o+2],a=t[o+3],128==(192&n)&&128==(192&i)&&128==(192&a)&&(s=(15&u)<<18|(63&n)<<12|(63&i)<<6|63&a)>65535&&s<1114112&&(f=s)}null===f?(f=65533,l=1):f>65535&&(f-=65536,r.push(f>>>10&1023|55296),f=56320|1023&f),r.push(f),o+=l}for(var c=r.length,h="",p=0;p<c;)h+=String.fromCharCode.apply(String,r.slice(p,p+=4096));return h};function b(){var e=!!i.ActiveXObject||"-ms-scroll-limit"in document.documentElement.style&&"-ms-ime-align"in document.documentElement.style,r=i.XMLHttpRequest&&i.XMLHttpRequest.prototype.send;e&&r&&(XMLHttpRequest.prototype.send=function(t){t instanceof Blob?(this.setRequestHeader("Content-Type",t.type),r.call(this,t)):r.call(this,t)});try{new File([],""),t.File=i.File,t.FileReader=i.FileReader}catch(e){try{t.File=new Function('class File extends Blob {constructor(chunks, name, opts) {opts = opts || {};super(chunks, opts || {});this.name = name.replace(/\\//g, ":");this.lastModifiedDate = opts.lastModified ? new Date(opts.lastModified) : new Date();this.lastModified = +this.lastModifiedDate;}};return new File([], ""), File')()}catch(e){t.File=function(t,e,r){var o=new Blob(t,r),n=r&&void 0!==r.lastModified?new Date(r.lastModified):new Date;return o.name=e.replace(/\//g,":"),o.lastModifiedDate=n,o.lastModified=+n,o.toString=function(){return"[object File]"},s&&(o[s]="File"),o}}}}u?(b(),t.Blob=f?i.Blob:p):l?(b(),t.Blob=h):function(){function e(t){for(var e=new Array(t.byteLength),r=new Uint8Array(t),o=e.length;o--;)e[o]=r[o];return e}function o(t){for(var e="ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=",r=[],o=0;o<t.length;o+=3){var n=t[o],i=o+1<t.length,a=i?t[o+1]:0,s=o+2<t.length,u=s?t[o+2]:0,f=n>>2,l=(3&n)<<4|a>>4,c=(15&a)<<2|u>>6,h=63&u;s||(h=64,i||(c=64)),r.push(e[f],e[l],e[c],e[h])}return r.join("")}var s=Object.create||function(t){function e(){}return e.prototype=t,new e};function u(t){return Object.prototype.toString.call(t).slice(8,-1)}function f(t,e){return"object"==typeof t&&Object.prototype.isPrototypeOf.call(t.prototype,e)}var l=["Int8Array","Uint8Array","Uint8ClampedArray","Int16Array","Uint16Array","Int32Array","Uint32Array","Float32Array","Float64Array","ArrayBuffer"];function c(t){return e=l,r=u(t),-1!==e.indexOf(r)||f(i.ArrayBuffer,t);var e,r}function h(t,r){r=null==r?{}:r;for(var o=0,n=(t=t?t.slice():[]).length;o<n;o++){var a=t[o];a instanceof h?t[o]=a._buffer:"string"==typeof a?t[o]=y(a):"DataView"===u(s=a)||f(i.DataView,s)?t[o]=e(a.buffer):c(a)?t[o]=e(a):t[o]=y(String(a))}var s;this._buffer=i.Uint8Array?function(t){for(var e=0,r=t.length;r--;)e+=t[r].length;for(var o=new Uint8Array(e),n=0,i=0;i<t.length;i++){var a=t[i];o.set(a,n),n+=a.byteLength||a.length}return o}(t):[].concat.apply([],t),this.size=this._buffer.length,this.type=r.type||"",/[^\u0020-\u007E]/.test(this.type)?this.type="":this.type=this.type.toLowerCase()}funcLine truncated
File diff suppressed because it is too large. Load diff
+22
View File
@@ -0,0 +1,22 @@
{
"name": "polyfill",
"version": "1.0.0",
"description": "",
"main": "index.js",
"type": "module",
"scripts": {
"test": "echo \"Error: no test specified\" && exit 1",
"dev": "webpack --mode production --watch",
"build": "webpack --mode production"
},
"author": "",
"license": "ISC",
"devDependencies": {
"webpack": "^5.75.0",
"webpack-cli": "^5.0.1"
},
"dependencies": {
"blob-polyfill": "^7.0.20220408",
"whatwg-fetch": "^3.6.2"
}
}
@@ -0,0 +1,3 @@
import { Blob } from "blob-polyfill";
globalThis.Blob = Blob;
@@ -0,0 +1,13 @@
globalThis.crypto = {
getRandomValues(arr) {
for (let i = 0, len = arr.length; i < len; i++) {
arr[i] = Math.floor(Math.random() * 256);
}
return arr;
},
randomUUID() {
return '10000000-1000-4000-8000-100000000000'.replace(/[018]/g, (c) => {
return (c ^ (this.getRandomValues(new Uint8Array(1))[0] & (15 >> (c / 4)))).toString(16)
})
}
}
@@ -0,0 +1 @@
import 'whatwg-fetch'
@@ -0,0 +1,3 @@
import "./blob/index.js"
import "./crypto/index.js"
import "./fetch/index.js"
@@ -0,0 +1,12 @@
import path from "path";
import { fileURLToPath } from "url";
const __dirname = fileURLToPath(import.meta.url);
export default {
entry: "./src/index.js",
output: {
filename: "index.js",
path: path.resolve(__dirname, "../out"),
},
};
+24
View File
@@ -0,0 +1,24 @@
package util
import (
"github.com/dop251/goja"
)
func ThrowTypeError(vm *goja.Runtime, msg string) {
panic(vm.NewTypeError(msg))
}
func AssertError[T error](err error) (t T, r bool) {
if err == nil {
return
}
if e, ok := err.(T); ok {
return e, true
}
if e, ok := err.(*goja.Exception); ok {
if ee, okk := e.Value().Export().(T); okk {
return ee, true
}
}
return
}
-1
View File
@@ -5,7 +5,6 @@ type EventKey string
const (
EventKeyStart = "start"
EventKeyPause = "pause"
EventKeyContinue = "continue"
EventKeyProgress = "progress"
EventKeyError = "error"
EventKeyDelete = "delete"
+786
View File
@@ -0,0 +1,786 @@
package download
import (
"encoding/json"
"fmt"
"github.com/GopeedLab/gopeed/internal/logger"
"github.com/GopeedLab/gopeed/pkg/base"
"github.com/GopeedLab/gopeed/pkg/download/engine"
gojaerror "github.com/GopeedLab/gopeed/pkg/download/engine/inject/error"
gojautil "github.com/GopeedLab/gopeed/pkg/download/engine/util"
"github.com/GopeedLab/gopeed/pkg/util"
"github.com/dop251/goja"
"github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/plumbing/transport"
"io"
"os"
"path"
"path/filepath"
"strconv"
"strings"
"time"
)
var (
gitSuffix = ".git"
tempExtensionsDir = ".extensions"
extensionsDir = "extensions"
extensionIgnoreDirs = []string{gitSuffix, "node_modules"}
ErrExtensionNoManifest = fmt.Errorf("manifest.json not found")
ErrExtensionNotFound = fmt.Errorf("extension not found")
)
type ActivationEvent string
const (
EventOnResolve ActivationEvent = "onResolve"
EventOnStart ActivationEvent = "onStart"
EventOnError ActivationEvent = "onError"
//EventOnDone ActivationEvent = "onDone"
)
func (d *Downloader) InstallExtensionByGit(url string) (*Extension, error) {
return d.fetchExtensionByGit(url, d.InstallExtensionByFolder)
}
func (d *Downloader) InstallExtensionByFolder(path string, devMode bool) (*Extension, error) {
ext, err := d.parseExtensionByPath(path)
if err != nil {
return nil, err
}
// if dev mode, don't copy to the extensions' directory
if devMode {
ext.DevMode = true
ext.DevPath, _ = filepath.Abs(path)
} else {
if err = util.CopyDir(path, d.ExtensionPath(ext), extensionIgnoreDirs...); err != nil {
return nil, err
}
}
// if extension is not installed, add it to the list, otherwise update it
installedExt := d.getExtension(ext.Identity)
if installedExt == nil {
ext.CreatedAt = time.Now()
ext.UpdatedAt = ext.CreatedAt
installedExt = ext
d.extensions = append(d.extensions, installedExt)
} else {
installedExt.update(ext)
}
if err = d.storage.Put(bucketExtension, installedExt.Identity, installedExt); err != nil {
return nil, err
}
return installedExt, nil
}
// UpgradeCheckExtension Check if there is a new version for the extension.
func (d *Downloader) UpgradeCheckExtension(identity string) (newVersion string, err error) {
ext, err := d.GetExtension(identity)
if err != nil {
return
}
installUrl := ext.buildInstallUrl()
if installUrl == "" {
return
}
_, err = d.fetchExtensionByGit(installUrl, func(tempExtPath string, devMode bool) (*Extension, error) {
tempExt, err := d.parseExtensionByPath(tempExtPath)
if err != nil {
return nil, err
}
if tempExt.Version != ext.Version {
newVersion = tempExt.Version
}
return tempExt, nil
})
return
}
func (d *Downloader) UpgradeExtension(identity string) error {
ext, err := d.GetExtension(identity)
if err != nil {
return err
}
installUrl := ext.buildInstallUrl()
if installUrl == "" {
return nil
}
if _, err := d.InstallExtensionByGit(installUrl); err != nil {
return err
}
return nil
}
func (d *Downloader) UpdateExtensionSettings(identity string, settings map[string]any) error {
ext, err := d.GetExtension(identity)
if err != nil {
return err
}
for _, setting := range ext.Settings {
if value, ok := settings[setting.Name]; ok {
setting.Value = tryParse(value, setting.Type)
}
}
return d.storage.Put(bucketExtension, ext.Identity, ext)
}
func (d *Downloader) SwitchExtension(identity string, status bool) error {
ext, err := d.GetExtension(identity)
if err != nil {
return err
}
ext.Disabled = !status
return d.storage.Put(bucketExtension, ext.Identity, ext)
}
func (d *Downloader) DeleteExtension(identity string) error {
ext, err := d.GetExtension(identity)
if err != nil {
return err
}
// remove from disk
if !ext.DevMode {
if err := os.RemoveAll(d.ExtensionPath(ext)); err != nil {
return err
}
}
// remove from extensions
for i, e := range d.extensions {
if e.Identity == identity {
d.extensions = append(d.extensions[:i], d.extensions[i+1:]...)
break
}
}
return d.storage.Delete(bucketExtension, identity)
}
func (d *Downloader) GetExtensions() []*Extension {
return d.extensions
}
func (d *Downloader) GetExtension(identity string) (*Extension, error) {
extension := d.getExtension(identity)
if extension == nil {
return nil, ErrExtensionNotFound
}
return extension, nil
}
func (d *Downloader) getExtension(identity string) *Extension {
for _, ext := range d.extensions {
if ext.Identity == identity {
return ext
}
}
return nil
}
func (d *Downloader) fetchExtensionByGit(url string, handler func(tempExtPath string, devMode bool) (*Extension, error)) (*Extension, error) {
if !strings.HasPrefix(url, "https://") && !strings.HasPrefix(url, "http://") {
url = "https://" + url
}
// resolve project path
parentPath, projectPath := path.Split(url)
// resolve project name and sub path
pathArr := strings.SplitN(projectPath, "#", 2)
projectPath = strings.TrimSuffix(pathArr[0], gitSuffix)
subPath := ""
if len(pathArr) > 1 {
subPath = pathArr[1]
}
tempExtDir := filepath.Join(d.cfg.StorageDir, tempExtensionsDir, projectPath)
if err := util.RmAndMkDirAll(tempExtDir); err != nil {
return nil, err
}
proxyOptions := transport.ProxyOptions{}
proxyUrl := d.cfg.ProxyUrl()
if proxyUrl != nil {
proxyOptions.URL = proxyUrl.Scheme + "://" + proxyUrl.Host
proxyOptions.Username = proxyUrl.User.Username()
proxyOptions.Password, _ = proxyUrl.User.Password()
}
// clone project to extension temp dir
gitUrl := parentPath + projectPath + gitSuffix
if _, err := git.PlainClone(tempExtDir, false, &git.CloneOptions{
URL: gitUrl,
Depth: 1,
ProxyOptions: proxyOptions,
}); err != nil {
return nil, err
}
defer os.RemoveAll(tempExtDir)
return handler(filepath.Join(tempExtDir, subPath), false)
}
func (d *Downloader) parseExtensionByPath(path string) (*Extension, error) {
// resolve extension manifest
manifestTempPath := filepath.Join(path, "manifest.json")
if _, err := os.Stat(manifestTempPath); os.IsNotExist(err) {
return nil, ErrExtensionNoManifest
}
file, err := os.ReadFile(manifestTempPath)
if err != nil {
return nil, err
}
var ext Extension
if err = json.Unmarshal(file, &ext); err != nil {
return nil, err
}
if err = ext.validate(); err != nil {
return nil, err
}
ext.Identity = ext.buildIdentity()
return &ext, nil
}
func (d *Downloader) triggerOnResolve(req *base.Request) (res *base.Resource, err error) {
err = doTrigger(d,
EventOnResolve,
req,
&OnResolveContext{
Req: req,
},
func(ext *Extension, gopeed *Instance, ctx *OnResolveContext) {
// Validate resource structure
if ctx.Res != nil && len(ctx.Res.Files) > 0 {
if err := ctx.Res.Validate(); err != nil {
gopeed.Logger.logger.Warn().Err(err).Msgf("[%s] resource invalid", ext.buildIdentity())
return
}
ctx.Res.Name = util.ReplaceInvalidFilename(ctx.Res.Name)
for _, file := range ctx.Res.Files {
file.Name = util.ReplaceInvalidFilename(file.Name)
}
ctx.Res.CalcSize(nil)
}
res = ctx.Res
},
)
return
}
func (d *Downloader) triggerOnStart(task *Task) {
doTrigger(d,
EventOnStart,
task.Meta.Req,
&OnStartContext{
Task: NewExtensionTask(d, task),
},
func(ext *Extension, gopeed *Instance, ctx *OnStartContext) {
// Validate request structure
if ctx.Task.Meta.Req != nil {
if err := ctx.Task.Meta.Req.Validate(); err != nil {
gopeed.Logger.logger.Warn().Err(err).Msgf("[%s] request invalid", ext.buildIdentity())
return
}
}
},
)
return
}
func (d *Downloader) triggerOnError(task *Task, err error) {
doTrigger(d,
EventOnError,
task.Meta.Req,
&OnErrorContext{
Task: NewExtensionTask(d, task),
Error: err,
},
nil,
)
}
func doTrigger[T any](d *Downloader, event ActivationEvent, req *base.Request, ctx T, handler func(ext *Extension, gopeed *Instance, ctx T)) error {
// init extension global object
gopeed := &Instance{
Events: make(InstanceEvents),
}
var err error
for _, ext := range d.extensions {
if ext.Disabled {
continue
}
for _, script := range ext.Scripts {
if script.match(event, req) {
gopeed.Info = NewExtensionInfo(ext)
gopeed.Logger = newInstanceLogger(ext, d.ExtensionLogger)
gopeed.Settings = parseSettings(ext.Settings)
gopeed.Storage = &ContextStorage{
storage: d.storage,
identity: ext.buildIdentity(),
}
scriptFilePath := filepath.Join(d.ExtensionPath(ext), script.Entry)
if _, err = os.Stat(scriptFilePath); os.IsNotExist(err) {
gopeed.Logger.logger.Error().Err(err).Msgf("[%s] script file not exist", ext.buildIdentity())
continue
}
func() {
var scriptFile *os.File
scriptFile, err = os.Open(scriptFilePath)
if err != nil {
gopeed.Logger.logger.Error().Err(err).Msgf("[%s] open script file failed", ext.buildIdentity())
return
}
defer scriptFile.Close()
var scriptBuf []byte
scriptBuf, err = io.ReadAll(scriptFile)
if err != nil {
gopeed.Logger.logger.Error().Err(err).Msgf("[%s] read script file failed", ext.buildIdentity())
return
}
// Init request labels
if req.Labels == nil {
req.Labels = make(map[string]string)
}
engine := engine.NewEngine(&engine.Config{
ProxyURL: d.cfg.ProxyUrl(),
})
defer engine.Close()
err = engine.Runtime.Set("gopeed", gopeed)
if err != nil {
gopeed.Logger.logger.Error().Err(err).Msgf("[%s] engine inject failed", ext.buildIdentity())
return
}
_, err = engine.RunString(string(scriptBuf))
if err != nil {
gopeed.Logger.logger.Error().Err(err).Msgf("[%s] run script failed", ext.buildIdentity())
return
}
if fn, ok := gopeed.Events[event]; ok {
_, err = engine.CallFunction(fn, ctx)
if err != nil {
gopeed.Logger.logger.Error().Err(err).Msgf("[%s] call function failed: %s", ext.buildIdentity(), event)
return
}
if handler != nil {
handler(ext, gopeed, ctx)
}
}
}()
}
}
}
// Only return MessageError
if me, ok := gojautil.AssertError[*gojaerror.MessageError](err); ok {
return me
}
return nil
}
func (d *Downloader) ExtensionPath(ext *Extension) string {
if ext.DevMode {
return ext.DevPath
}
return filepath.Join(d.cfg.StorageDir, extensionsDir, ext.Identity)
}
type Extension struct {
// Identity is global unique for an extension, it's a combination of author and name
Identity string `json:"identity"`
Name string `json:"name"`
Author string `json:"author"`
Title string `json:"title"`
Description string `json:"description"`
Icon string `json:"icon"`
// Version semantic version string, like: 1.0.0
Version string `json:"version"`
// Homepage homepage url
Homepage string `json:"homepage"`
// Repository git repository info
Repository *Repository `json:"repository"`
Scripts []*Script `json:"scripts"`
Settings []*Setting `json:"settings"`
// Disabled if true, this extension will be ignored
Disabled bool `json:"disabled"`
DevMode bool `json:"devMode"`
// DevPath is the local path of extension source code
DevPath string `json:"devPath"`
CreatedAt time.Time `json:"createdAt"`
UpdatedAt time.Time `json:"updatedAt"`
}
func (e *Extension) validate() error {
if e.Name == "" {
return fmt.Errorf("extension name is required")
}
if e.Title == "" {
return fmt.Errorf("extension title is required")
}
if e.Version == "" {
return fmt.Errorf("extension version is required")
}
return nil
}
func (e *Extension) buildIdentity() string {
if e.Author == "" {
return e.Name
}
return e.Author + "@" + e.Name
}
func (e *Extension) buildInstallUrl() string {
if e.Repository == nil || e.Repository.Url == "" {
return ""
}
repoUrl := e.Repository.Url
if e.Repository.Directory != "" {
if strings.HasSuffix(repoUrl, "/") {
repoUrl = repoUrl[:len(repoUrl)-1]
}
dir := e.Repository.Directory
if strings.HasPrefix(dir, "/") {
dir = dir[1:]
}
repoUrl = repoUrl + "#" + dir
}
return repoUrl
}
func (e *Extension) update(newExt *Extension) error {
e.Title = newExt.Title
e.Description = newExt.Description
e.Icon = newExt.Icon
e.Version = newExt.Version
e.Homepage = newExt.Homepage
e.Repository = newExt.Repository
e.Scripts = newExt.Scripts
// merge settings
// if new setting not exist in old settings, append it
for _, newSetting := range newExt.Settings {
var exist bool
for _, setting := range e.Settings {
if setting.Name == newSetting.Name {
exist = true
break
}
}
if !exist {
e.Settings = append(e.Settings, newSetting)
}
}
// if old setting not exist in new settings, remove it
for i := 0; i < len(e.Settings); i++ {
var exist bool
for _, setting := range newExt.Settings {
if setting.Name == e.Settings[i].Name {
exist = true
break
}
}
if !exist {
e.Settings = append(e.Settings[:i], e.Settings[i+1:]...)
i--
}
}
// if new setting exist in old settings, update it
for _, newSetting := range newExt.Settings {
for _, setting := range e.Settings {
if setting.Name == newSetting.Name {
setting.Title = newSetting.Title
setting.Description = newSetting.Description
setting.Options = newSetting.Options
// if type changed, reset value
if setting.Type != newSetting.Type {
setting.Type = newSetting.Type
setting.Value = newSetting.Value
}
break
}
}
}
e.UpdatedAt = time.Now()
return nil
}
type Repository struct {
Url string `json:"url"`
Directory string `json:"directory"`
}
type Script struct {
// Event active event name
Event string `json:"event"`
// Match rules
Match *Match `json:"match"`
// Entry js script file path
Entry string `json:"entry"`
}
func (s *Script) match(event ActivationEvent, req *base.Request) bool {
if s.Event == "" {
return false
}
if s.Event != string(event) {
return false
}
if s.Match == nil || (len(s.Match.Urls) == 0 && len(s.Match.Labels) == 0) {
return false
}
// match url
for _, url := range s.Match.Urls {
if util.Match(url, req.URL) {
return true
}
}
// match label
for _, label := range s.Match.Labels {
if _, ok := req.Labels[label]; ok {
return true
}
}
return false
}
type Match struct {
// Urls match expression, refer to https://developer.chrome.com/docs/extensions/mv3/match_patterns/
Urls []string `json:"urls"`
// Labels match request labels
Labels []string `json:"labels"`
}
type SettingType string
const (
SettingTypeString SettingType = "string"
SettingTypeNumber SettingType = "number"
SettingTypeBoolean SettingType = "boolean"
)
type Setting struct {
Name string `json:"name"`
Title string `json:"title"`
Description string `json:"description"`
Required bool `json:"required"`
// setting type
Type SettingType `json:"type"`
// setting value
Value any `json:"value"`
//Multiple bool `json:"multiple"`
Options []*Option `json:"options"`
}
type Option struct {
Label string `json:"label"`
Value any `json:"value"`
}
// Instance inject to js context when extension script is activated
type Instance struct {
Events InstanceEvents `json:"events"`
Info *ExtensionInfo `json:"info"`
Logger *InstanceLogger `json:"logger"`
Settings map[string]any `json:"settings"`
Storage *ContextStorage `json:"storage"`
}
type InstanceEvents map[ActivationEvent]goja.Callable
func (h InstanceEvents) register(name ActivationEvent, fn goja.Callable) {
h[name] = fn
}
func (h InstanceEvents) OnResolve(fn goja.Callable) {
h.register(EventOnResolve, fn)
}
func (h InstanceEvents) OnStart(fn goja.Callable) {
h.register(EventOnStart, fn)
}
func (h InstanceEvents) OnError(fn goja.Callable) {
h.register(EventOnError, fn)
}
//func (h InstanceEvents) OnDone(fn goja.Callable) {
// h.register(HookEventOnDone, fn)
//}
type ExtensionInfo struct {
Identity string `json:"identity"`
Name string `json:"name"`
Author string `json:"author"`
Title string `json:"title"`
Version string `json:"version"`
}
func NewExtensionInfo(ext *Extension) *ExtensionInfo {
return &ExtensionInfo{
Identity: ext.buildIdentity(),
Name: ext.Name,
Author: ext.Author,
Title: ext.Title,
Version: ext.Version,
}
}
type InstanceLogger struct {
identity string
devMode bool
logger *logger.Logger
}
func (l *InstanceLogger) Debug(msg ...goja.Value) {
if l.devMode {
l.logger.Debug().Msg(l.append(msg...))
}
}
func (l *InstanceLogger) Info(msg ...goja.Value) {
l.logger.Info().Msg(l.append(msg...))
}
func (l *InstanceLogger) Warn(msg ...goja.Value) {
l.logger.Warn().Msg(l.append(msg...))
}
func (l *InstanceLogger) Error(msg ...goja.Value) {
l.logger.Error().Msg(l.append(msg...))
}
func (l *InstanceLogger) append(msg ...goja.Value) string {
strMsg := make([]string, len(msg))
for i, m := range msg {
strMsg[i] = m.String()
}
return fmt.Sprintf("[%s] %s", l.identity, strings.Join(strMsg, " "))
}
func newInstanceLogger(extension *Extension, logger *logger.Logger) *InstanceLogger {
return &InstanceLogger{
identity: extension.buildIdentity(),
devMode: extension.DevMode,
logger: logger,
}
}
type OnResolveContext struct {
Req *base.Request `json:"req"`
Res *base.Resource `json:"res"`
}
type OnStartContext struct {
Task *ExtensionTask `json:"task"`
}
type OnErrorContext struct {
Task *ExtensionTask `json:"task"`
Error error `json:"error"`
}
type ExtensionTask struct {
*Task
download *Downloader
}
func NewExtensionTask(download *Downloader, task *Task) *ExtensionTask {
return &ExtensionTask{
Task: task.clone(),
download: download,
}
}
func (t *ExtensionTask) Continue() error {
return t.download.Continue(t.ID)
}
func (t *ExtensionTask) Pause() error {
return t.download.Pause(t.ID)
}
func parseSettings(settings []*Setting) map[string]any {
m := make(map[string]any)
for _, s := range settings {
var val any
if s.Value != nil {
val = s.Value
}
m[s.Name] = tryParse(val, s.Type)
}
return m
}
func tryParse(val any, settingType SettingType) any {
if val == nil {
return nil
}
switch settingType {
case SettingTypeString:
return fmt.Sprint(val)
case SettingTypeNumber:
vv, err := strconv.ParseFloat(fmt.Sprint(val), 64)
if err != nil {
return nil
}
return vv
case SettingTypeBoolean:
vv, err := strconv.ParseBool(fmt.Sprint(val))
if err != nil {
return nil
}
return vv
default:
return nil
}
}
type ContextStorage struct {
storage Storage
identity string
}
func (s *ContextStorage) Get(key string) any {
raw := s.getRawData()
if v, ok := raw[key]; ok {
return v
}
return nil
}
func (s *ContextStorage) Set(key string, value string) {
raw := s.getRawData()
raw[key] = value
s.storage.Put(bucketExtensionStorage, s.identity, raw)
}
func (s *ContextStorage) Remove(key string) {
raw := s.getRawData()
delete(raw, key)
s.storage.Put(bucketExtensionStorage, s.identity, raw)
}
func (s *ContextStorage) Keys() []string {
raw := s.getRawData()
keys := make([]string, 0)
for k := range raw {
keys = append(keys, k)
}
return keys
}
func (s *ContextStorage) Clear() {
s.storage.Delete(bucketExtensionStorage, s.identity)
}
func (s *ContextStorage) getRawData() map[string]string {
var data map[string]string
s.storage.Get(bucketExtensionStorage, s.identity, &data)
if data == nil {
data = make(map[string]string)
}
return data
}
+422
View File
@@ -0,0 +1,422 @@
package download
import (
"errors"
"github.com/GopeedLab/gopeed/internal/logger"
"github.com/GopeedLab/gopeed/pkg/base"
gojaerror "github.com/GopeedLab/gopeed/pkg/download/engine/inject/error"
"github.com/dop251/goja"
"os"
"testing"
"time"
)
func TestDownloader_InstallExtensionByFolder(t *testing.T) {
setupDownloader(func(downloader *Downloader) {
if _, err := downloader.InstallExtensionByFolder("./testdata/extensions/basic", false); err != nil {
t.Fatal(err)
}
rr, err := downloader.Resolve(&base.Request{
URL: "https://github.com/test",
})
if err != nil {
t.Fatal(err)
}
if len(rr.Res.Files) == 1 {
t.Fatal("resolve error")
}
})
}
func TestDownloader_InstallExtensionByFolderDevMode(t *testing.T) {
setupDownloader(func(downloader *Downloader) {
if _, err := downloader.InstallExtensionByFolder("./testdata/extensions/basic", true); err != nil {
t.Fatal(err)
}
rr, err := downloader.Resolve(&base.Request{
URL: "https://github.com/test",
})
if err != nil {
t.Fatal(err)
}
if len(rr.Res.Files) == 1 {
t.Fatal("resolve error")
}
})
}
func TestDownloader_InstallExtensionByGit(t *testing.T) {
setupDownloader(func(downloader *Downloader) {
if _, err := downloader.InstallExtensionByGit("https://github.com/GopeedLab/gopeed-extension-samples#github-release-sample"); err != nil {
t.Fatal(err)
}
rr, err := downloader.Resolve(&base.Request{
URL: "https://github.com/GopeedLab/gopeed/releases",
})
if err != nil {
t.Fatal(err)
}
if len(rr.Res.Files) == 1 {
t.Fatal("resolve error")
}
})
}
func TestDownloader_InstallExtensionByGitSimple(t *testing.T) {
setupDownloader(func(downloader *Downloader) {
if _, err := downloader.InstallExtensionByGit("github.com/GopeedLab/gopeed-extension-samples#github-release-sample"); err != nil {
t.Fatal(err)
}
rr, err := downloader.Resolve(&base.Request{
URL: "https://github.com/GopeedLab/gopeed/releases",
})
if err != nil {
t.Fatal(err)
}
if len(rr.Res.Files) == 1 {
t.Fatal("resolve error")
}
})
}
func TestDownloader_InstallExtensionByGitFull(t *testing.T) {
setupDownloader(func(downloader *Downloader) {
if _, err := downloader.InstallExtensionByGit("https://github.com/GopeedLab/gopeed-extension-samples.git#github-release-sample"); err != nil {
t.Fatal(err)
}
rr, err := downloader.Resolve(&base.Request{
URL: "https://github.com/GopeedLab/gopeed/releases",
})
if err != nil {
t.Fatal(err)
}
if len(rr.Res.Files) == 1 {
t.Fatal("resolve error")
}
})
}
func TestDownloader_UpgradeExtension(t *testing.T) {
getSetting := func(settings []*Setting, name string) *Setting {
for _, setting := range settings {
if setting.Name == name {
return setting
}
}
return nil
}
setupDownloader(func(downloader *Downloader) {
installedExt, err := downloader.InstallExtensionByFolder("./testdata/extensions/update", false)
if err != nil {
t.Fatal(err)
}
extensions := downloader.GetExtensions()
if len(extensions) == 0 {
t.Fatal("extension not installed")
}
oldVersion := installedExt.Version
// fetch new version from git
newVersion, err := downloader.UpgradeCheckExtension(installedExt.Identity)
if err != nil {
t.Fatal(err)
}
if newVersion == "" {
t.Fatal("new version not found")
}
// update extension
if err = downloader.UpgradeExtension(installedExt.Identity); err != nil {
t.Fatal(err)
}
upgradeExt := downloader.getExtension(installedExt.Identity)
if upgradeExt.Version == oldVersion {
t.Fatal("extension update fail")
}
// check setting update
s1 := getSetting(upgradeExt.Settings, "s1")
if s1.Title == "S1 old" {
t.Fatal("setting update fail")
}
// check setting type update
s2 := getSetting(upgradeExt.Settings, "s2")
if s2.Type == "number" {
t.Fatal("setting type update fail")
}
// check setting remove
d1 := getSetting(upgradeExt.Settings, "d1")
if d1 != nil {
t.Fatal("setting remove fail")
}
// check setting add
s3 := getSetting(upgradeExt.Settings, "s3")
if s3 == nil {
t.Fatal("setting add fail")
}
rr, err := downloader.Resolve(&base.Request{
URL: "https://test.com",
})
if err != nil {
t.Fatal(err)
}
if rr.Res.Name != "test" {
t.Fatal("script update fail")
}
})
}
func TestDownloader_Extension_OnStart(t *testing.T) {
downloadAndCheck := func(req *base.Request) {
setupDownloader(func(downloader *Downloader) {
if _, err := downloader.InstallExtensionByFolder("./testdata/extensions/on_start", false); err != nil {
t.Fatal(err)
}
errCh := make(chan error, 1)
downloader.Listener(func(event *Event) {
if event.Key == EventKeyFinally {
errCh <- event.Err
}
})
id, err := downloader.CreateDirect(req, nil)
if err != nil {
t.Fatal(err)
}
select {
case err = <-errCh:
break
case <-time.After(time.Second * 10):
err = errors.New("timeout")
}
if err != nil {
panic("extension on start download error: " + err.Error())
}
task := downloader.GetTask(id)
if task.Meta.Req.URL != "https://github.com" {
t.Fatalf("except url: https://github.com, actual: %s", task.Meta.Req.URL)
}
if task.Meta.Req.Labels["modified"] != "true" {
t.Fatalf("except label: modified=true, actual: %s", task.Meta.Req.Labels["modified"])
}
})
}
// url match
downloadAndCheck(&base.Request{
URL: "https://github.com/gopeed/test/404",
})
// label match
downloadAndCheck(&base.Request{
URL: "https://test.com",
Labels: map[string]string{
"test": "true",
},
})
}
func TestDownloader_Extension_OnError(t *testing.T) {
setupDownloader(func(downloader *Downloader) {
if _, err := downloader.InstallExtensionByFolder("./testdata/extensions/on_error", false); err != nil {
t.Fatal(err)
}
errCh := make(chan error, 1)
downloader.Listener(func(event *Event) {
if event.Key == EventKeyFinally {
errCh <- event.Err
}
})
id, err := downloader.CreateDirect(&base.Request{
URL: "https://github.com/gopeed/test/404",
Labels: map[string]string{
"test": "true",
},
}, nil)
if err != nil {
t.Fatal(err)
}
select {
case err = <-errCh:
break
case <-time.After(time.Second * 10):
err = errors.New("timeout")
}
if err != nil {
panic("extension on error download error: " + err.Error())
}
// extension on error modify url and continue download
task := downloader.GetTask(id)
if task.Meta.Req.URL != "https://github.com" {
t.Fatalf("except url: https://github.com, actual: %s", task.Meta.Req.URL)
}
if task.Status != base.DownloadStatusDone {
t.Fatalf("except status is done, actual: %s", task.Status)
}
})
}
func TestDownloader_Extension_Errors(t *testing.T) {
setupDownloader(func(downloader *Downloader) {
if _, err := downloader.InstallExtensionByFolder("./testdata/extensions/script_error", false); err != nil {
t.Fatal(err)
}
rr, err := downloader.Resolve(&base.Request{
URL: "https://github.com/test",
})
if err != nil {
t.Fatal(err)
}
if len(rr.Res.Files) == 2 {
t.Fatal("script error catch failed")
}
})
setupDownloader(func(downloader *Downloader) {
if _, err := downloader.InstallExtensionByFolder("./testdata/extensions/function_error", false); err != nil {
t.Fatal(err)
}
rr, err := downloader.Resolve(&base.Request{
URL: "https://github.com/test",
})
if err != nil {
t.Fatal(err)
}
if len(rr.Res.Files) == 2 {
t.Fatal("function error catch failed")
}
})
setupDownloader(func(downloader *Downloader) {
if _, err := downloader.InstallExtensionByFolder("./testdata/extensions/message_error", false); err != nil {
t.Fatal(err)
}
_, err := downloader.Resolve(&base.Request{
URL: "https://github.com/test",
})
if err == nil {
t.Fatalf("except error, but got nil")
}
me, ok := err.(*gojaerror.MessageError)
if !ok {
t.Fatalf("except MessageError type, but got %s", err)
}
want := "test"
if me.Error() != want {
t.Fatalf("except MessageError message %s, but got %s", want, me.Message)
}
})
}
func TestDownloader_Extension_Settings(t *testing.T) {
setupDownloader(func(downloader *Downloader) {
if _, err := downloader.InstallExtensionByFolder("./testdata/extensions/settings_empty", false); err != nil {
t.Fatal(err)
}
rr, err := downloader.Resolve(&base.Request{
URL: "https://github.com/test",
})
if err != nil {
t.Fatal(err)
}
if len(rr.Res.Files) == 1 {
t.Fatal("settings parse error")
}
})
setupDownloader(func(downloader *Downloader) {
installedExt, err := downloader.InstallExtensionByFolder("./testdata/extensions/settings_all", false)
if err != nil {
t.Fatal(err)
}
downloader.UpdateExtensionSettings(installedExt.Identity, map[string]any{
"stringValued": "valued",
"numberValued": 1.1,
"booleanValued": true,
})
rr, err := downloader.Resolve(&base.Request{
URL: "https://github.com/test",
})
if err != nil {
t.Fatal(err)
}
if len(rr.Res.Files) == 1 {
t.Fatal("settings parse error")
}
})
}
func TestDownloader_ExtensionStorage(t *testing.T) {
setupDownloader(func(downloader *Downloader) {
if _, err := downloader.InstallExtensionByFolder("./testdata/extensions/storage", false); err != nil {
t.Fatal(err)
}
rr, err := downloader.Resolve(&base.Request{
URL: "https://github.com/test",
})
if err != nil {
t.Fatal(err)
}
if len(rr.Res.Files) == 1 {
t.Fatal("resolve error")
}
})
}
func TestDownloader_SwitchExtension(t *testing.T) {
setupDownloader(func(downloader *Downloader) {
installedExt, err := downloader.InstallExtensionByFolder("./testdata/extensions/basic", false)
if err != nil {
t.Fatal(err)
}
if installedExt.Disabled == true {
t.Fatal("extension disabled")
}
if err = downloader.SwitchExtension(installedExt.Identity, false); err != nil {
t.Fatal(err)
}
if installedExt.Disabled == false {
t.Fatal("extension enabled")
}
})
}
func TestDownloader_DeleteExtension(t *testing.T) {
setupDownloader(func(downloader *Downloader) {
installedExt, err := downloader.InstallExtensionByFolder("./testdata/extensions/settings_all", false)
if err != nil {
t.Fatal(err)
}
extensions := downloader.GetExtensions()
if err := downloader.DeleteExtension(installedExt.Identity); err != nil {
t.Fatal(err)
}
extensions = downloader.GetExtensions()
if len(extensions) != 0 {
t.Fatal("extension delete fail")
}
})
}
func TestDownloader_Extension_Logger(t *testing.T) {
logger := logger.NewLogger(false, "")
il := newInstanceLogger(&Extension{
Name: "test",
}, logger)
il.Debug(goja.NaN(), goja.Undefined())
il.Info(goja.NaN(), goja.Undefined())
il.Warn(goja.NaN(), goja.Undefined())
il.Error(goja.NaN(), goja.Undefined())
}
func setupDownloader(fn func(downloader *Downloader)) {
defaultDownloader.Setup()
defaultDownloader.cfg.StorageDir = ".test_storage"
defaultDownloader.cfg.DownloadDir = ".test_download"
defer func() {
defaultDownloader.Clear()
os.RemoveAll(defaultDownloader.cfg.StorageDir)
os.RemoveAll(defaultDownloader.cfg.DownloadDir)
}()
fn(defaultDownloader)
}
+39 -8
View File
@@ -8,6 +8,7 @@ import (
"github.com/GopeedLab/gopeed/pkg/base"
"github.com/GopeedLab/gopeed/pkg/util"
gonanoid "github.com/matoous/go-nanoid/v2"
"net/url"
"sync"
"time"
)
@@ -22,8 +23,8 @@ type Task struct {
Meta *fetcher.FetcherMeta `json:"meta"`
Status base.Status `json:"status"`
Progress *Progress `json:"progress"`
Size int64 `json:"size"`
CreatedAt time.Time `json:"createdAt"`
UpdatedAt time.Time `json:"updatedAt"`
fetcherBuilder fetcher.FetcherBuilder
fetcher fetcher.Fetcher
@@ -41,17 +42,23 @@ func NewTask() *Task {
ID: id,
Status: base.DownloadStatusReady,
CreatedAt: time.Now(),
UpdatedAt: time.Now(),
}
}
func (t *Task) updateStatus(status base.Status) {
t.UpdatedAt = time.Now()
t.Status = status
}
func (t *Task) clone() *Task {
return &Task{
ID: t.ID,
Meta: t.Meta,
Status: t.Status,
Progress: t.Progress,
Size: t.Size,
CreatedAt: t.CreatedAt,
UpdatedAt: t.UpdatedAt,
}
}
@@ -75,6 +82,8 @@ type DownloaderConfig struct {
Storage Storage
StorageDir string
ProductionMode bool
*DownloaderStoreConfig
}
@@ -99,17 +108,39 @@ func (cfg *DownloaderConfig) Init() *DownloaderConfig {
// DownloaderStoreConfig is the config that can restore the downloader.
type DownloaderStoreConfig struct {
FirstLoad bool `json:"-"` // fromNoStore is the flag that the config is first time init and not from store
FirstLoad bool `json:"-"` // FirstLoad is the flag that the config is first time init and not from store
DownloadDir string `json:"downloadDir"` // DownloadDir is the default directory to save the downloaded files
MaxRunning int `json:"maxRunning"` // MaxRunning is the max running download count
ProtocolConfig map[string]any `json:"protocolConfig"` // ProtocolConfig is special config for each protocol
Extra map[string]any `json:"extra"` // Extra is the extra config
DownloadDir string `json:"downloadDir"` // DownloadDir is the default directory to save the downloaded files
MaxRunning int `json:"maxRunning"` // MaxRunning is the max running download count
ProtocolConfig map[string]any `json:"protocolConfig"` // ProtocolConfig is special config for each protocol
Extra map[string]any `json:"extra"`
Proxy *DownloaderProxyConfig `json:"proxy"`
}
func (cfg *DownloaderStoreConfig) Init() *DownloaderStoreConfig {
if cfg.MaxRunning == 0 {
cfg.MaxRunning = 3
cfg.MaxRunning = 5
}
if cfg.Proxy == nil {
cfg.Proxy = &DownloaderProxyConfig{}
}
return cfg
}
func (cfg *DownloaderStoreConfig) ProxyUrl() *url.URL {
if cfg.Proxy == nil {
return nil
}
if cfg.Proxy.Enable == false || cfg.Proxy.Scheme == "" || cfg.Proxy.Host == "" {
return nil
}
return util.BuildProxyUrl(cfg.Proxy.Scheme, cfg.Proxy.Host, cfg.Proxy.Usr, cfg.Proxy.Pwd)
}
type DownloaderProxyConfig struct {
Enable bool `json:"enable"`
Scheme string `json:"scheme"`
Host string `json:"host"`
Usr string `json:"usr"`
Pwd string `json:"pwd"`
}
+23 -1
View File
@@ -6,6 +6,7 @@ import (
"os"
"path/filepath"
"reflect"
"sync"
)
type Storage interface {
@@ -47,10 +48,13 @@ func changeValue(p any, v any) {
var memData = make(map[string]map[string]any)
type MemStorage struct {
lock *sync.RWMutex
}
func NewMemStorage() *MemStorage {
return &MemStorage{}
return &MemStorage{
lock: &sync.RWMutex{},
}
}
func (n *MemStorage) Setup(buckets []string) error {
@@ -63,11 +67,15 @@ func (n *MemStorage) Setup(buckets []string) error {
}
func (n *MemStorage) Put(bucket string, key string, v any) error {
n.lock.Lock()
defer n.lock.Unlock()
memData[bucket][key] = v
return nil
}
func (n *MemStorage) Get(bucket string, key string, v any) (bool, error) {
n.lock.RLock()
defer n.lock.RUnlock()
if dv, ok := memData[bucket][key]; ok {
changeValue(v, dv)
return true, nil
@@ -76,6 +84,8 @@ func (n *MemStorage) Get(bucket string, key string, v any) (bool, error) {
}
func (n *MemStorage) List(bucket string, v any) error {
n.lock.RLock()
defer n.lock.RUnlock()
data := memData[bucket]
list := make([]any, 0)
for _, v := range data {
@@ -86,6 +96,8 @@ func (n *MemStorage) List(bucket string, v any) error {
}
func (n *MemStorage) Pop(bucket string, key string, v any) error {
n.lock.Lock()
defer n.lock.Unlock()
data := memData[bucket]
changeValue(v, data[key])
delete(data, key)
@@ -93,6 +105,8 @@ func (n *MemStorage) Pop(bucket string, key string, v any) error {
}
func (n *MemStorage) Delete(bucket string, key string) error {
n.lock.Lock()
defer n.lock.Unlock()
delete(memData[bucket], key)
return nil
}
@@ -102,6 +116,8 @@ func (n *MemStorage) Close() error {
}
func (n *MemStorage) Clear() error {
n.lock.Lock()
defer n.lock.Unlock()
memData = make(map[string]map[string]any)
return nil
}
@@ -116,6 +132,9 @@ type BoltStorage struct {
}
func NewBoltStorage(dir string) *BoltStorage {
if err := os.MkdirAll(dir, 0755); err != nil {
panic(err)
}
path := filepath.Join(dir, dbFile)
db, err := bbolt.Open(path, 0600, nil)
if err != nil {
@@ -203,6 +222,9 @@ func (b *BoltStorage) Pop(bucket string, key string, v any) error {
if err != nil {
return err
}
if len(data) == 0 {
return nil
}
return json.Unmarshal(data, v)
}
+16
View File
@@ -0,0 +1,16 @@
gopeed.events.onResolve(async function (ctx) {
ctx.res = {
name: "test",
files: Array(2).fill(true).map((_, i) => ({
name: `test-${i}.txt`,
size: 1024,
req: {
url: ctx.req.url + "/" + i,
labels:{
"from": gopeed.info.name,
}
}
}),
),
};
});
+23
View File
@@ -0,0 +1,23 @@
{
"name": "basic",
"title": "gopeed extension basic test",
"version": "0.0.1",
"scripts": [
{
"event": "onResolve",
"match": {
"urls": [
"*://github.com/*"
]
},
"entry": "index.js"
}
],
"settings": [
{
"name": "ua",
"title": "User-Agent",
"type": "string"
}
]
}
+18
View File
@@ -0,0 +1,18 @@
gopeed.events.onResolve(async function (ctx) {
ctx.res = {
name: "test",
files: Array(2).fill(true).map((_, i) => ({
name: `test-${i}.txt`,
size: 1024,
req: {
url: ctx.req.url + "/" + i,
extra: {
headers: {
'User-Agent': ctx.settings.ua,
},
}
}
}),
),
};
});
+22
View File
@@ -0,0 +1,22 @@
{
"name": "extra",
"title": "gopeed extension extra test",
"version": "0.0.1",
"scripts": [
{
"event": "onResolve",
"matches": [
"*://github.com/*"
],
"entry": "index.js"
}
],
"settings": [
{
"name": "ua",
"title": "User-Agent",
"type": "string",
"value": "gopeed"
}
]
}
@@ -0,0 +1,17 @@
gopeed.events.onResolve(async function (ctx) {
const aaa = {};
// access undefined property
gopeed.logger.info(aaa.bbb.ccc);
ctx.res = {
name: "test",
files: Array(2).fill(true).map((_, i) => ({
name: `test-${i}.txt`,
size: 1024,
req: {
url: ctx.req.url + "/" + i,
}
}),
),
};
});
@@ -0,0 +1,23 @@
{
"name": "function-error",
"title": "gopeed extension function error test",
"version": "0.0.1",
"scripts": [
{
"event": "onResolve",
"match": {
"urls": [
"*://github.com/*"
]
},
"entry": "index.js"
}
],
"settings": [
{
"name": "ua",
"title": "User-Agent",
"type": "string"
}
]
}
@@ -0,0 +1,3 @@
gopeed.events.onResolve(async function (ctx) {
throw new MessageError("test");
});
@@ -0,0 +1,16 @@
{
"name": "message-error",
"title": "gopeed extension message error test",
"version": "0.0.1",
"scripts": [
{
"event": "onResolve",
"match": {
"urls": [
"*://github.com/*"
]
},
"entry": "index.js"
}
]
}
+8
View File
@@ -0,0 +1,8 @@
gopeed.events.onError(async function (ctx) {
gopeed.logger.info("url", ctx.task.meta.req.url);
gopeed.logger.info("error", ctx.error);
ctx.task.meta.req.url = "https://github.com";
ctx.task.pause();
ctx.task.continue();
});
+16
View File
@@ -0,0 +1,16 @@
{
"name": "on-error",
"title": "gopeed extension on error event test",
"version": "0.0.1",
"scripts": [
{
"event": "onError",
"match": {
"labels": [
"test"
]
},
"entry": "index.js"
}
]
}
+6
View File
@@ -0,0 +1,6 @@
gopeed.events.onStart(async function (ctx) {
gopeed.logger.info("url", ctx.task.meta.req.url);
ctx.task.meta.req.url = "https://github.com";
ctx.task.meta.req.labels['modified'] = 'true';
});
+19
View File
@@ -0,0 +1,19 @@
{
"name": "on-start",
"title": "gopeed extension on start event test",
"version": "0.0.1",
"scripts": [
{
"event": "onStart",
"match": {
"urls": [
"*://github.com/*"
],
"labels": [
"test"
]
},
"entry": "index.js"
}
]
}
+16
View File
@@ -0,0 +1,16 @@
const aaa = {};
gopeed.logger.info(aaa.bbb.ccc);
gopeed.events.onResolve(async function (ctx) {
ctx.res = {
name: "test",
files: Array(2).fill(true).map((_, i) => ({
name: `test-${i}.txt`,
size: 1024,
req: {
url: ctx.req.url + "/" + i,
}
}),
),
};
});
@@ -0,0 +1,23 @@
{
"name": "script-error",
"title": "gopeed extension script error test",
"version": "0.0.1",
"scripts": [
{
"event": "onResolve",
"match": {
"urls": [
"*://github.com/*"
]
},
"entry": "index.js"
}
],
"settings": [
{
"name": "ua",
"title": "User-Agent",
"type": "string"
}
]
}
+43
View File
@@ -0,0 +1,43 @@
gopeed.events.onResolve(async function (ctx) {
if (gopeed.settings.string != null) {
throw new Error("string is not null");
}
if (gopeed.settings.number != null) {
throw new Error("number is not null");
}
if (gopeed.settings.boolean != null) {
throw new Error("boolean is not null");
}
if (gopeed.settings.stringDefault !== "default") {
throw new Error("string default value is incorrect");
}
if (gopeed.settings.numberDefault !== 1) {
throw new Error("number default value is incorrect");
}
if (gopeed.settings.booleanDefault !== true) {
throw new Error("boolean default value is incorrect");
}
if (gopeed.settings.stringValued !== "valued") {
throw new Error("string value is incorrect");
}
if (gopeed.settings.numberValued !== 1.1) {
throw new Error("number value is incorrect");
}
if (gopeed.settings.booleanValued !== true) {
throw new Error("boolean value is incorrect");
}
ctx.res = {
name: "test",
files: Array(2).fill(true).map((_, i) => ({
name: `test-${i}.txt`,
size: 1024,
req: {
url: ctx.req.url + "/" + i,
}
}),
),
};
});
@@ -0,0 +1,66 @@
{
"name": "settings-all",
"title": "gopeed extension settings all type test",
"version": "0.0.1",
"scripts": [
{
"event": "onResolve",
"match": {
"urls": [
"*://*/*"
]
},
"entry": "index.js"
}
],
"settings": [
{
"name": "string",
"title": "string null test",
"type": "string"
},
{
"name": "number",
"title": "number null test",
"type": "number"
},
{
"name": "boolean",
"title": "boolean null test",
"type": "boolean"
},
{
"name": "stringDefault",
"title": "string default test",
"type": "string",
"value": "default"
},
{
"name": "numberDefault",
"title": "number default test",
"type": "number",
"value": 1
},
{
"name": "booleanDefault",
"title": "boolean default test",
"type": "boolean",
"value": true
},
{
"name": "stringValued",
"title": "string valued test",
"type": "string"
},
{
"name": "numberValued",
"title": "number valued test",
"type": "number"
},
{
"name": "booleanValued",
"title": "boolean valued test",
"type": "boolean"
}
]
}
@@ -0,0 +1,17 @@
gopeed.events.onResolve(async function (ctx) {
if (Object.keys(gopeed.settings).length > 0){
throw new Error("settings is not empty");
}
ctx.res = {
name: "test",
files: Array(2).fill(true).map((_, i) => ({
name: `test-${i}.txt`,
size: 1024,
req: {
url: ctx.req.url + "/" + i,
}
}),
),
};
});
@@ -0,0 +1,16 @@
{
"name": "settings-empty",
"title": "gopeed extension settings empty test",
"version": "0.0.1",
"scripts": [
{
"event": "onResolve",
"match": {
"urls": [
"*://*/*"
]
},
"entry": "index.js"
}
]
}
+49
View File
@@ -0,0 +1,49 @@
gopeed.events.onResolve(async function (ctx) {
const key = "key"
const value1 = "value1", value2 = JSON.stringify({a: 1, b: "2"})
if (gopeed.storage.get(key) !== null) {
throw new Error("storage get null error")
}
gopeed.storage.remove(key)
if(gopeed.storage.keys().length !== 0) {
throw new Error("storage keys null error")
}
gopeed.storage.set(key, value1)
if (gopeed.storage.get(key) !== value1) {
throw new Error("storage put1 error")
}
gopeed.storage.set(key, value2)
if (gopeed.storage.get(key) !== value2) {
throw new Error("storage put2 error")
}
if(gopeed.storage.keys().length !== 1) {
throw new Error("storage keys error")
}
gopeed.storage.remove(key)
if (gopeed.storage.get(key) !== null) {
throw new Error("storage delete error")
}
gopeed.storage.set(key, value1)
gopeed.storage.clear()
if (gopeed.storage.get(key) !== null) {
throw new Error("storage clear error")
}
ctx.res = {
name: "test",
files: Array(2).fill(true).map((_, i) => ({
name: `test-${i}.txt`,
size: 1024,
req: {
url: ctx.req.url + "/" + i,
}
}),
),
};
});
+23
View File
@@ -0,0 +1,23 @@
{
"name": "storage",
"title": "gopeed extension storage test",
"version": "0.0.1",
"scripts": [
{
"event": "onResolve",
"match": {
"urls": [
"*://*/*"
]
},
"entry": "index.js"
}
],
"settings": [
{
"name": "ua",
"title": "User-Agent",
"type": "string"
}
]
}
+3
View File
@@ -0,0 +1,3 @@
gopeed.events.onResolve(async function (ctx) {
// do nothing for test
});
+47
View File
@@ -0,0 +1,47 @@
{
"name": "extension-test",
"author": "gopeed",
"title": "Gopeed Extension Test",
"description": "Test extension settings and upgrade",
"version": "0.0.1",
"homepage": "https://gopeed.com",
"repository": {
"url": "https://github.com/GopeedLab/gopeed-extension-samples",
"directory": "extension-test"
},
"scripts": [
{
"event": "onResolve",
"match": {
"urls": [
"*://*/*"
]
},
"entry": "index.js"
}
],
"settings": [
{
"name": "s1",
"title": "S1 old",
"description": "Test setting update",
"type": "string",
"required": true
},
{
"name": "s2",
"title": "s2 number old",
"description": "Test setting type update",
"type": "number",
"required": true,
"value": 1
},
{
"name": "d1",
"title": "Delete test",
"description": "Test setting delete",
"type": "string",
"required": true
}
]
}
+104 -27
View File
@@ -8,7 +8,6 @@ import (
"io"
"net/http"
"net/url"
"strings"
)
func Resolve(w http.ResponseWriter, r *http.Request) {
@@ -31,9 +30,9 @@ func CreateTask(w http.ResponseWriter, r *http.Request) {
err error
)
if req.Rid != "" {
taskId, err = Downloader.Create(req.Rid, req.Opts)
taskId, err = Downloader.Create(req.Rid, req.Opt)
} else if req.Req != nil {
taskId, err = Downloader.CreateDirect(req.Req, req.Opts)
taskId, err = Downloader.CreateDirect(req.Req, req.Opt)
} else {
WriteJson(w, model.NewErrorResult("param invalid: rid or req", model.CodeInvalidParam))
return
@@ -121,21 +120,16 @@ func GetTask(w http.ResponseWriter, r *http.Request) {
}
func GetTasks(w http.ResponseWriter, r *http.Request) {
status := r.FormValue("status")
if status == "" {
status = strings.Join([]string{
string(base.DownloadStatusReady),
string(base.DownloadStatusRunning),
string(base.DownloadStatusPause),
string(base.DownloadStatusError),
string(base.DownloadStatusDone),
}, ",")
}
statusArr := strings.Split(status, ",")
r.ParseForm()
status := r.Form["status"]
tasks := Downloader.GetTasks()
if len(status) == 0 {
WriteJson(w, model.NewOkResult(tasks))
return
}
result := make([]*download.Task, 0)
for _, task := range tasks {
for _, s := range statusArr {
for _, s := range status {
if task.Status == base.Status(s) {
result = append(result, task)
}
@@ -159,6 +153,101 @@ func PutConfig(w http.ResponseWriter, r *http.Request) {
WriteJson(w, model.NewNilResult())
}
func InstallExtension(w http.ResponseWriter, r *http.Request) {
var req model.InstallExtension
if ReadJson(r, w, &req) {
var (
installedExt *download.Extension
err error
)
if req.DevMode {
installedExt, err = Downloader.InstallExtensionByFolder(req.URL, true)
} else {
installedExt, err = Downloader.InstallExtensionByGit(req.URL)
}
if err != nil {
WriteJson(w, model.NewErrorResult(err.Error()))
return
}
WriteJson(w, model.NewOkResult(installedExt.Identity))
}
}
func GetExtensions(w http.ResponseWriter, r *http.Request) {
list := Downloader.GetExtensions()
WriteJson(w, model.NewOkResult(list))
}
func GetExtension(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
identity := vars["identity"]
ext, err := Downloader.GetExtension(identity)
if err != nil {
WriteJson(w, model.NewErrorResult(err.Error()))
return
}
WriteJson(w, model.NewOkResult(ext))
}
func UpdateExtensionSettings(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
identity := vars["identity"]
var req model.UpdateExtensionSettings
if ReadJson(r, w, &req) {
if err := Downloader.UpdateExtensionSettings(identity, req.Settings); err != nil {
WriteJson(w, model.NewErrorResult(err.Error()))
return
}
}
WriteJson(w, model.NewNilResult())
}
func SwitchExtension(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
identity := vars["identity"]
var switchExtension model.SwitchExtension
if ReadJson(r, w, &switchExtension) {
if err := Downloader.SwitchExtension(identity, switchExtension.Status); err != nil {
WriteJson(w, model.NewErrorResult(err.Error()))
return
}
}
WriteJson(w, model.NewNilResult())
}
func DeleteExtension(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
identity := vars["identity"]
if err := Downloader.DeleteExtension(identity); err != nil {
WriteJson(w, model.NewErrorResult(err.Error()))
return
}
WriteJson(w, model.NewNilResult())
}
func UpdateCheckExtension(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
identity := vars["identity"]
newVersion, err := Downloader.UpgradeCheckExtension(identity)
if err != nil {
WriteJson(w, model.NewErrorResult(err.Error()))
return
}
WriteJson(w, model.NewOkResult(&model.UpdateCheckExtensionResp{
NewVersion: newVersion,
}))
}
func UpdateExtension(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
identity := vars["identity"]
if err := Downloader.UpgradeExtension(identity); err != nil {
WriteJson(w, model.NewErrorResult(err.Error()))
return
}
WriteJson(w, model.NewNilResult())
}
func DoProxy(w http.ResponseWriter, r *http.Request) {
target := r.Header.Get("X-Target-Uri")
if target == "" {
@@ -191,18 +280,6 @@ func DoProxy(w http.ResponseWriter, r *http.Request) {
return
}
w.Write(buf)
//var reader io.ReadCloser
//switch resp.Header.Get("Content-Encoding") {
//case "gzip":
// reader, err = gzip.NewReader(resp.Body)
// defer reader.Close()
//default:
// reader = resp.Body
//}
//if _, err := io.Copy(w, resp.Body); err != nil {
// writeError(w, err.Error())
// return
//}
}
func writeError(w http.ResponseWriter, msg string) {
+18
View File
@@ -0,0 +1,18 @@
package model
type InstallExtension struct {
DevMode bool `json:"devMode"`
URL string `json:"url"`
}
type UpdateExtensionSettings struct {
Settings map[string]any `json:"settings"`
}
type SwitchExtension struct {
Status bool `json:"status"`
}
type UpdateCheckExtensionResp struct {
NewVersion string `json:"newVersion"`
}
+2
View File
@@ -20,6 +20,8 @@ type StartConfig struct {
StorageDir string `json:"storageDir"`
ApiToken string `json:"apiToken"`
ProductionMode bool
WebEnable bool
WebFS fs.FS
WebBasicAuth *WebBasicAuth
+3 -3
View File
@@ -3,7 +3,7 @@ package model
import "github.com/GopeedLab/gopeed/pkg/base"
type CreateTask struct {
Rid string `json:"rid"`
Req *base.Request `json:"req"`
Opts *base.Options `json:"opts"`
Rid string `json:"rid"`
Req *base.Request `json:"req"`
Opt *base.Options `json:"opt"`
}
+85 -3
View File
@@ -3,13 +3,19 @@ package rest
import (
"context"
"encoding/json"
"fmt"
"github.com/GopeedLab/gopeed/pkg/download"
"github.com/GopeedLab/gopeed/pkg/rest/model"
"github.com/GopeedLab/gopeed/pkg/util"
"github.com/gorilla/handlers"
"github.com/gorilla/mux"
"github.com/pkg/errors"
"net"
"net/http"
"os"
"path/filepath"
"regexp"
"strings"
)
var (
@@ -26,7 +32,6 @@ func Start(startCfg *model.StartConfig) (int, error) {
go func() {
if err := srv.Serve(listener); err != nil && err != http.ErrServerClosed {
// TODO log
panic(err)
}
}()
@@ -41,12 +46,12 @@ func Start(startCfg *model.StartConfig) (int, error) {
func Stop() {
if srv != nil {
if err := srv.Shutdown(context.TODO()); err != nil {
// TODO log
Downloader.Logger.Warn().Err(err).Msg("shutdown server failed")
}
}
if Downloader != nil {
if err := Downloader.Close(); err != nil {
// TODO log
Downloader.Logger.Warn().Err(err).Msg("close downloader failed")
}
}
}
@@ -58,6 +63,7 @@ func BuildServer(startCfg *model.StartConfig) (*http.Server, net.Listener, error
startCfg.Init()
downloadCfg := &download.DownloaderConfig{
ProductionMode: startCfg.ProductionMode,
RefreshInterval: startCfg.RefreshInterval,
}
if startCfg.Storage == model.StorageBolt {
@@ -93,8 +99,18 @@ func BuildServer(startCfg *model.StartConfig) (*http.Server, net.Listener, error
r.Methods(http.MethodGet).Path("/api/v1/tasks").HandlerFunc(GetTasks)
r.Methods(http.MethodGet).Path("/api/v1/config").HandlerFunc(GetConfig)
r.Methods(http.MethodPut).Path("/api/v1/config").HandlerFunc(PutConfig)
r.Methods(http.MethodPost).Path("/api/v1/extensions").HandlerFunc(InstallExtension)
r.Methods(http.MethodGet).Path("/api/v1/extensions").HandlerFunc(GetExtensions)
r.Methods(http.MethodGet).Path("/api/v1/extensions/{identity}").HandlerFunc(GetExtension)
r.Methods(http.MethodPut).Path("/api/v1/extensions/{identity}/settings").HandlerFunc(UpdateExtensionSettings)
r.Methods(http.MethodPut).Path("/api/v1/extensions/{identity}/switch").HandlerFunc(SwitchExtension)
r.Methods(http.MethodDelete).Path("/api/v1/extensions/{identity}").HandlerFunc(DeleteExtension)
r.Methods(http.MethodGet).Path("/api/v1/extensions/{identity}/update").HandlerFunc(UpdateCheckExtension)
r.Methods(http.MethodPost).Path("/api/v1/extensions/{identity}/update").HandlerFunc(UpdateExtension)
r.Path("/api/v1/proxy").HandlerFunc(DoProxy)
if startCfg.WebEnable {
r.PathPrefix("/fs/tasks").Handler(http.FileServer(new(taskFileSystem)))
r.PathPrefix("/fs/extensions").Handler(http.FileServer(new(extensionFileSystem)))
r.PathPrefix("/").Handler(http.FileServer(http.FS(startCfg.WebFS)))
}
@@ -117,6 +133,20 @@ func BuildServer(startCfg *model.StartConfig) (*http.Server, net.Listener, error
})
}
// recover panic
r.Use(func(h http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
defer func() {
if v := recover(); v != nil {
err := errors.WithStack(fmt.Errorf("%v", v))
Downloader.Logger.Error().Stack().Err(err).Msg("http server panic")
WriteJson(w, model.NewErrorResult(err.Error(), model.CodeError))
}
}()
h.ServeHTTP(w, r)
})
})
srv = &http.Server{Handler: handlers.CORS(
handlers.AllowedHeaders([]string{"Content-Type", "X-Api-Token", "X-Target-Uri"}),
handlers.AllowedMethods([]string{"GET", "POST", "PUT", "PATCH", "DELETE", "OPTIONS"}),
@@ -125,6 +155,58 @@ func BuildServer(startCfg *model.StartConfig) (*http.Server, net.Listener, error
return srv, listener, nil
}
func resolvePath(urlPath string, prefix string) (identity string, path string, err error) {
// remove prefix
clearPath := strings.TrimPrefix(urlPath, prefix)
// match extension identity, eg: /fs/extensions/identity/xxx
reg := regexp.MustCompile(`^/([^/]+)/(.*)$`)
if !reg.MatchString(clearPath) {
err = os.ErrNotExist
return
}
matched := reg.FindStringSubmatch(clearPath)
if len(matched) != 3 {
err = os.ErrNotExist
return
}
return matched[1], matched[2], nil
}
// handle task file resource
type taskFileSystem struct {
}
func (e *taskFileSystem) Open(name string) (http.File, error) {
// get extension identity
identity, path, err := resolvePath(name, "/fs/tasks")
if err != nil {
return nil, err
}
task := Downloader.GetTask(identity)
if task == nil {
return nil, os.ErrNotExist
}
return os.Open(filepath.Join(task.Meta.RootDirPath(), path))
}
// handle extension file resource
type extensionFileSystem struct {
}
func (e *extensionFileSystem) Open(name string) (http.File, error) {
// get extension identity
identity, path, err := resolvePath(name, "/fs/extensions")
if err != nil {
return nil, err
}
extension, err := Downloader.GetExtension(identity)
if err != nil {
return nil, os.ErrNotExist
}
extensionPath := Downloader.ExtensionPath(extension)
return os.Open(filepath.Join(extensionPath, path))
}
func ReadJson(r *http.Request, w http.ResponseWriter, v any) bool {
if err := json.NewDecoder(r.Body).Decode(v); err != nil {
WriteJson(w, model.NewErrorResult(err.Error()))
+191 -35
View File
@@ -27,13 +27,12 @@ var (
Extra: map[string]any{
"method": "",
"header": map[string]string{
"User-Agent": "gopeed",
"Usr-Agent": "gopeed",
},
"body": "",
},
}
taskRes = &base.Resource{
Name: test.BuildName,
Size: test.BuildSize,
Range: true,
Files: []*base.FileInfo{
@@ -44,15 +43,23 @@ var (
},
},
}
createOpts = &base.Options{
Path: test.Dir,
Name: test.DownloadName,
Extra: map[string]any{
"connections": 2,
},
}
createReq = &model.CreateTask{
Req: taskReq,
Opts: &base.Options{
Path: test.Dir,
Name: test.DownloadName,
Extra: map[string]any{
"connections": 2,
},
},
Opt: createOpts,
}
createResoledReq = &model.CreateTask{
Req: taskReq,
Opt: createOpts,
}
installExtensionReq = &model.InstallExtension{
URL: "https://github.com/GopeedLab/gopeed-extension-samples#github-contributor-avatars-sample",
}
)
@@ -66,6 +73,35 @@ func TestResolve(t *testing.T) {
}
func TestCreateTask(t *testing.T) {
doTest(func() {
resp := httpRequestCheckOk[*download.ResolveResult](http.MethodPost, "/api/v1/resolve", taskReq)
var wg sync.WaitGroup
wg.Add(1)
Downloader.Listener(func(event *download.Event) {
if event.Key == download.EventKeyFinally {
wg.Done()
}
})
taskId := httpRequestCheckOk[string](http.MethodPost, "/api/v1/tasks", &model.CreateTask{
Rid: resp.ID,
Opt: createOpts,
})
if taskId == "" {
t.Fatal("create task failed")
}
wg.Wait()
want := test.FileMd5(test.BuildFile)
got := test.FileMd5(test.DownloadFile)
if want != got {
t.Errorf("CreateTask() got = %v, want %v", got, want)
}
})
}
func TestCreateDirectTask(t *testing.T) {
doTest(func() {
var wg sync.WaitGroup
wg.Add(1)
@@ -84,32 +120,41 @@ func TestCreateTask(t *testing.T) {
want := test.FileMd5(test.BuildFile)
got := test.FileMd5(test.DownloadFile)
if want != got {
t.Errorf("CreateTask() got = %v, want %v", got, want)
t.Errorf("CreateDirectTask() got = %v, want %v", got, want)
}
})
}
func TestCreateDirectTaskWithResoled(t *testing.T) {
doTest(func() {
var wg sync.WaitGroup
wg.Add(1)
Downloader.Listener(func(event *download.Event) {
if event.Key == download.EventKeyFinally {
wg.Done()
}
})
taskId := httpRequestCheckOk[string](http.MethodPost, "/api/v1/tasks", createResoledReq)
if taskId == "" {
t.Fatal("create task failed")
}
wg.Wait()
want := test.FileMd5(test.BuildFile)
got := test.FileMd5(test.DownloadFile)
if want != got {
t.Errorf("CreateDirectTaskWithResoled() got = %v, want %v", got, want)
}
})
}
func TestPauseAndContinueTask(t *testing.T) {
doTest(func() {
type result struct {
pauseCount int
continueCount int
md5 string
}
var got = result{
pauseCount: 0,
continueCount: 0,
md5: "",
}
var wg sync.WaitGroup
wg.Add(1)
Downloader.Listener(func(event *download.Event) {
switch event.Key {
case download.EventKeyPause:
got.pauseCount++
case download.EventKeyContinue:
got.continueCount++
case download.EventKeyFinally:
wg.Done()
}
@@ -132,12 +177,8 @@ func TestPauseAndContinueTask(t *testing.T) {
}
wg.Wait()
want := result{
pauseCount: 1,
continueCount: 1,
md5: test.FileMd5(test.BuildFile),
}
got.md5 = test.FileMd5(test.DownloadFile)
want := test.FileMd5(test.BuildFile)
got := test.FileMd5(test.DownloadFile)
if !reflect.DeepEqual(got, want) {
t.Errorf("PauseAndContinueTask() got = %v, want %v", got, want)
}
@@ -163,17 +204,18 @@ func TestPauseAllAndContinueALLTasks(t *testing.T) {
// continue all
httpRequestCheckOk[any](http.MethodPut, "/api/v1/tasks/continue", nil)
time.Sleep(time.Millisecond * 100)
tasks := httpRequestCheckOk[[]*download.Task](http.MethodGet, fmt.Sprintf("/api/v1/tasks?status=%s", base.DownloadStatusRunning), nil)
if len(tasks) != cfg.MaxRunning {
t.Errorf("ContinueAllTasks() got = %v, want %v", len(tasks), cfg.MaxRunning)
}
// pause all
httpRequestCheckOk[any](http.MethodPut, "/api/v1/tasks/pause", nil)
tasks = httpRequestCheckOk[[]*download.Task](http.MethodGet, fmt.Sprintf("/api/v1/tasks?status=%s,%s", base.DownloadStatusReady, base.DownloadStatusPause), nil)
time.Sleep(time.Millisecond * 100)
tasks = httpRequestCheckOk[[]*download.Task](http.MethodGet, fmt.Sprintf("/api/v1/tasks?status=%s", base.DownloadStatusPause), nil)
if len(tasks) != total {
t.Errorf("PauseAllTasks() got = %v, want %v", len(tasks), total)
}
})
}
@@ -210,7 +252,7 @@ func TestGetTasks(t *testing.T) {
}
})
httpRequestCheckOk[string](http.MethodPost, fmt.Sprintf("/api/v1/tasks?status=%s,%s",
httpRequestCheckOk[string](http.MethodPost, fmt.Sprintf("/api/v1/tasks?status=%s&status=%s",
base.DownloadStatusReady, base.DownloadStatusRunning), createReq)
httpRequestCheckOk[[]*download.Task](http.MethodGet, "/api/v1/tasks", nil)
@@ -248,6 +290,115 @@ func TestGetAndPutConfig(t *testing.T) {
})
}
func TestInstallExtension(t *testing.T) {
doTest(func() {
identity := httpRequestCheckOk[string](http.MethodPost, "/api/v1/extensions", installExtensionReq)
if identity == "" {
t.Errorf("InstallExtension() got = %v, want %v", identity, "not empty")
}
// not a valid extension repository
code, _ := httpRequest[string](http.MethodPost, "/api/v1/extensions", &model.InstallExtension{
URL: "https://github.com/GopeedLab/gopeed",
})
checkCode(code, model.CodeError)
// not a git repository
code, _ = httpRequest[string](http.MethodPost, "/api/v1/extensions", &model.InstallExtension{
URL: "https://github.com",
})
checkCode(code, model.CodeError)
})
}
func TestGetExtensions(t *testing.T) {
doTest(func() {
httpRequestCheckOk[string](http.MethodPost, "/api/v1/extensions", installExtensionReq)
extensions := httpRequestCheckOk[[]*download.Extension](http.MethodGet, "/api/v1/extensions", nil)
if len(extensions) == 0 {
t.Errorf("GetExtensions() got = %v, want %v", len(extensions), "not empty")
}
})
}
func TestUpdateExtensionSettings(t *testing.T) {
doTest(func() {
identity := httpRequestCheckOk[string](http.MethodPost, "/api/v1/extensions", installExtensionReq)
httpRequestCheckOk[any](http.MethodPut, "/api/v1/extensions/"+identity+"/settings", &model.UpdateExtensionSettings{
Settings: map[string]any{
"undefined": "test",
"ua": "test",
},
})
settings := httpRequestCheckOk[*download.Extension](http.MethodGet, "/api/v1/extensions/"+identity, nil).Settings
if len(settings) != 1 {
t.Errorf("UpdateExtensionSettings() got = %v, want %v", len(settings), 1)
}
if settings[0].Name != "ua" || settings[0].Value != "test" {
t.Errorf("UpdateExtensionSettings() got = %v, want %v", settings[0].Value, "test")
}
})
}
func TestSwitchExtension(t *testing.T) {
doTest(func() {
identity := httpRequestCheckOk[string](http.MethodPost, "/api/v1/extensions", installExtensionReq)
httpRequestCheckOk[any](http.MethodPut, "/api/v1/extensions/"+identity+"/switch", &model.SwitchExtension{
Status: false,
})
extensions := httpRequestCheckOk[[]*download.Extension](http.MethodGet, "/api/v1/extensions", nil)
if !extensions[0].Disabled {
t.Errorf("TestSwitchExtension() got = %v, want %v", extensions[0].Disabled, true)
}
})
}
func TestDeleteExtension(t *testing.T) {
doTest(func() {
identity := httpRequestCheckOk[string](http.MethodPost, "/api/v1/extensions", installExtensionReq)
httpRequestCheckOk[any](http.MethodDelete, "/api/v1/extensions/"+identity, nil)
extensions := httpRequestCheckOk[[]*download.Extension](http.MethodGet, "/api/v1/extensions", nil)
if len(extensions) != 0 {
t.Errorf("TestDeleteExtension() got = %v, want %v", len(extensions), 0)
}
})
}
func TestUpdateCheckExtension(t *testing.T) {
doTest(func() {
identity := httpRequestCheckOk[string](http.MethodPost, "/api/v1/extensions", installExtensionReq)
resp := httpRequestCheckOk[*model.UpdateCheckExtensionResp](http.MethodGet, "/api/v1/extensions/"+identity+"/update", nil)
// no new version
if resp.NewVersion != "" {
t.Errorf("UpdateCheckExtension() got = %v, want %v", resp.NewVersion, "")
}
// force update
httpRequestCheckOk[any](http.MethodPost, "/api/v1/extensions/"+identity+"/update", nil)
})
}
func TestFsExtension(t *testing.T) {
doTest(func() {
identity := httpRequestCheckOk[string](http.MethodPost, "/api/v1/extensions", installExtensionReq)
statusCode, _ := doHttpRequest0(http.MethodGet, "/fs/extensions/"+identity+"/icon.png", nil, nil)
if statusCode != http.StatusOK {
t.Errorf("FsExtension() got = %v, want %v", statusCode, http.StatusOK)
}
})
}
func TestFsExtensionFail(t *testing.T) {
doTest(func() {
statusCode, _ := doHttpRequest0(http.MethodGet, "/fs/extensions/not_exist/icon.png", nil, nil)
if statusCode != http.StatusNotFound {
t.Errorf("TestFsExtensionFail() got = %v, want %v", statusCode, http.StatusNotFound)
}
})
}
func TestDoProxy(t *testing.T) {
doTest(func() {
code, respBody := doHttpRequest0(http.MethodGet, "/api/v1/proxy", map[string]string{
@@ -341,15 +492,19 @@ func doTest(handler func()) {
var cfg = &model.StartConfig{}
cfg.Init()
cfg.Storage = storage
cfg.StorageDir = ".test_storage"
cfg.WebEnable = true
fileListener := doStart(cfg)
defer func() {
if err := fileListener.Close(); err != nil {
panic(err)
}
Stop()
Downloader.Clear()
}()
defer Downloader.Clear()
defer func() {
Downloader.PauseAll()
tasks := Downloader.GetTasks()
taskIds := make([]string, len(tasks))
for i, task := range tasks {
@@ -358,6 +513,7 @@ func doTest(handler func()) {
for _, id := range taskIds {
Downloader.Delete(id, true)
}
os.RemoveAll(cfg.StorageDir)
}()
taskReq.URL = "http://" + fileListener.Addr().String() + "/" + test.BuildName
handler()
+75
View File
@@ -0,0 +1,75 @@
package util
import (
"net/url"
"regexp"
"strings"
)
// Match url with pattern by chrome extension match pattern style
// https://developer.chrome.com/docs/extensions/mv3/match_patterns/
func Match(pattern string, u string) bool {
scheme, host, path := parsePattern(pattern)
url, err := url.Parse(u)
if err != nil {
return false
}
if scheme != "*" && scheme != url.Scheme {
return false
}
if !matchHost(host, url.Hostname()) {
return false
}
if !matchPath(path, url.Path) {
return false
}
return true
}
func parsePattern(pattern string) (scheme string, host string, path string) {
parts := strings.Split(pattern, "://")
if len(parts) == 2 {
scheme = parts[0]
pattern = parts[1]
} else {
scheme = ""
}
parts = strings.SplitN(pattern, "/", 2)
if len(parts) == 2 {
host = parts[0]
path = "/" + parts[1]
} else {
host = pattern
path = "/"
}
return
}
func matchHost(pattern string, host string) bool {
if pattern == "*" {
return true
}
if strings.HasPrefix(pattern, "*.") {
return strings.HasSuffix(host, pattern[1:])
}
return pattern == host
}
func matchPath(pattern string, path string) bool {
if pattern == "*" {
return true
}
if !strings.HasSuffix(pattern, "*") && !strings.HasSuffix(pattern, "/") {
pattern += "/"
}
if !strings.HasSuffix(path, "/") {
path += "/"
}
if strings.Contains(pattern, "*") {
pattern = strings.Replace(pattern, "*", ".*", -1)
matched, _ := regexp.MatchString("^"+pattern+"$", path)
return matched
}
return pattern == path
}
+36
View File
@@ -0,0 +1,36 @@
package util
import "testing"
func TestMatch(t *testing.T) {
tests := []struct {
pattern string
urls []string
want bool
}{
{"*://*/*", []string{"https://www.google.com/", "http://example.org/foo/bar.html"}, true},
{"https://*/*", []string{"https://www.google.com", "https://example.org/foo/bar.html"}, true},
{"*://www.google.com", []string{"https://www.google.com/", "https://www.google.com"}, true},
{"*://*.google.com/", []string{"https://a.www.google.com/", "https://c.www.google.com/", "https://www.google.com/"}, true},
{"https://*/foo*", []string{"https://www.google.com/foo", "https://example.com/foo/bar.html"}, true},
{"https://www.google.com/*/b/*", []string{"https://www.google.com/a/b", "https://www.google.com/a/b/c"}, true},
{"https://*.google.com/foo*bar", []string{"https://www.google.com/foo/baz/bar", "https://docs.google.com/foobar"}, true},
{"https://www.google.com/*abc*", []string{"https://www.google.com/abc", "https://www.google.com/123abc", "https://www.google.com/abc456", "https://www.google.com/123abc456"}, true},
{"https://example.org/foo/bar.html", []string{"https://example.org/foo/bar.html"}, true},
{"http://127.0.0.1/*", []string{"http://127.0.0.1/", "http://127.0.0.1/foo/bar.html"}, true},
{"*://mail.google.com/*", []string{"http://mail.google.com/foo/baz/bar", "https://mail.google.com/foobar"}, true},
{"https://www.google.com/", []string{"http://www.google.com/"}, false},
{"www.google.com/", []string{"http://www.google.com/", "https://www.google.com/"}, false},
{"www.google.com/*c", []string{"https://www.google.com/a", "https://www.google.com/b"}, false},
{"https://*.example.org/*", []string{"https://www.google.com", "https://docs.google.com"}, false},
}
for _, tt := range tests {
t.Run(tt.pattern, func(t *testing.T) {
for _, url := range tt.urls {
if got := Match(tt.pattern, url); got != tt.want {
t.Errorf("Match() = %v, want %v", got, tt.want)
}
}
})
}
}
+110 -14
View File
@@ -3,8 +3,10 @@ package util
import (
"errors"
"fmt"
"io"
"os"
syspath "path"
"path/filepath"
"strings"
)
@@ -64,22 +66,105 @@ func CheckDuplicateAndRename(path string) (string, error) {
}
}
// GetSingleDir get the top level single folder name,if not exist, return empty string
func GetSingleDir(paths []string) string {
if len(paths) == 0 {
return ""
// CopyDir Copy all files to the target directory, if the file already exists, it will be overwritten.
// Remove target file if the source file is not exist.
func CopyDir(source string, target string, excludeDir ...string) error {
if err := os.MkdirAll(target, 0755); err != nil {
return err
}
split := strings.Split(paths[0], "/")
if len(split) == 0 || split[0] == "" {
return ""
}
dir := split[0]
for i := 1; i < len(paths); i++ {
if !strings.HasPrefix(paths[i], dir) {
return ""
if err := filepath.Walk(source, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if info.IsDir() {
if len(excludeDir) > 0 {
for _, dir := range excludeDir {
if info.IsDir() && info.Name() == dir {
return filepath.SkipDir
}
}
}
return nil
}
relPath, err := filepath.Rel(source, path)
if err != nil {
return err
}
targetPath := filepath.Join(target, relPath)
if err := os.MkdirAll(filepath.Dir(targetPath), 0755); err != nil {
return err
}
if err := copyForce(path, targetPath); err != nil {
return err
}
return nil
}); err != nil {
return err
}
return dir
if err := filepath.Walk(target, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if info.IsDir() {
if len(excludeDir) > 0 {
for _, dir := range excludeDir {
if info.IsDir() && info.Name() == dir {
return filepath.SkipDir
}
}
}
return nil
}
relPath, err := filepath.Rel(target, path)
if err != nil {
return err
}
targetPath := filepath.Join(target, relPath)
sourcePath := filepath.Join(source, relPath)
// if source file is not exist, remove target file
if _, err := os.Stat(sourcePath); os.IsNotExist(err) {
if err := SafeRemove(targetPath); err != nil {
return err
}
}
return nil
}); err != nil {
return err
}
return nil
}
// RmAndMkDirAll remove and create directory, if the directory already exists, it will be overwritten.
func RmAndMkDirAll(path string) error {
if err := os.RemoveAll(path); err != nil {
return err
}
if err := os.MkdirAll(path, 0755); err != nil {
return err
}
return nil
}
// copy file, if the target file already exists, it will be overwritten.
func copyForce(source string, target string) error {
sourceFile, err := os.Open(source)
if err != nil {
return err
}
defer sourceFile.Close()
targetFile, err := os.Create(target)
if err != nil {
return err
}
defer targetFile.Close()
_, err = io.Copy(targetFile, sourceFile)
if err != nil {
return err
}
return nil
}
func CreateDirIfNotExist(dir string) error {
@@ -92,9 +177,20 @@ func CreateDirIfNotExist(dir string) error {
// IsExistsFile check file exists and is a file
func IsExistsFile(path string) bool {
info, err := os.Stat(path)
// 判断路径是否存在,并且是一个文件
// if file exists and is a file
if err == nil && !info.IsDir() {
return true
}
return false
}
// ReplaceInvalidFilename replace invalid path characters
func ReplaceInvalidFilename(path string) string {
if path == "" {
return ""
}
for _, char := range invalidPathChars {
path = strings.ReplaceAll(path, char, "_")
}
return path
}
+6
View File
@@ -0,0 +1,6 @@
//go:build !windows
// +build !windows
package util
var invalidPathChars = []string{`/`, `:`}
+48 -62
View File
@@ -78,7 +78,7 @@ func TestFilepath(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := Filepath(tt.args.path, tt.args.originName, tt.args.customName); got != tt.want {
t.Errorf("Filepath() = %v, want %v", got, tt.want)
t.Errorf("SingleFilepath() = %v, want %v", got, tt.want)
}
})
}
@@ -105,67 +105,6 @@ func TestSafeRemove(t *testing.T) {
}
}
func TestGetSingleDir(t *testing.T) {
type args struct {
paths []string
}
tests := []struct {
name string
args args
want string
}{
{
name: "empty",
args: args{
paths: []string{},
},
want: "",
},
{
name: "blank",
args: args{
paths: []string{""},
},
want: "",
},
{
name: "blank multi",
args: args{
paths: []string{"", "a"},
},
want: "",
},
{
name: "multi dir not single",
args: args{
paths: []string{"a", "b"},
},
want: "",
},
{
name: "single dir",
args: args{
paths: []string{"a"},
},
want: "a",
},
{
name: "multi dir single",
args: args{
paths: []string{"a/b", "a/c"},
},
want: "a",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := GetSingleDir(tt.args.paths); got != tt.want {
t.Errorf("IsSingleDir() = %v, want %v", got, tt.want)
}
})
}
}
func TestCheckDuplicateAndRename(t *testing.T) {
doCheckDuplicateAndRename(t, []string{}, "a.txt", "a.txt")
doCheckDuplicateAndRename(t, []string{"a.txt"}, "a.txt", "a (1).txt")
@@ -239,3 +178,50 @@ func TestIsExistsFile(t *testing.T) {
})
}
}
func TestReplaceInvalidFilename(t *testing.T) {
type args struct {
path string
}
tests := []struct {
name string
args args
want string
}{
{
name: "blank",
args: args{
path: "",
},
want: "",
},
{
name: "normal",
args: args{
path: "test.txt",
},
want: "test.txt",
},
{
name: "case1",
args: args{
path: "te/st.txt",
},
want: "te_st.txt",
},
{
name: "case2",
args: args{
path: "te/st:.txt",
},
want: "te_st_.txt",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := ReplaceInvalidFilename(tt.args.path); got != tt.want {
t.Errorf("ReplaceInvalidFilename() = %v, want %v", got, tt.want)
}
})
}
}
+3
View File
@@ -0,0 +1,3 @@
package util
var invalidPathChars = []string{`\`, `/`, `:`, `*`, `?`, `"`, `<`, `>`, `|`}
+14
View File
@@ -2,6 +2,7 @@ package util
import (
"encoding/base64"
"net/url"
"regexp"
"strings"
)
@@ -37,3 +38,16 @@ func ParseDataUri(uri string) (string, []byte) {
}
return mime, data
}
// BuildProxyUrl builds a proxy url with given host, username and password.
func BuildProxyUrl(scheme, host, usr, pwd string) *url.URL {
var user *url.Userinfo
if usr != "" && pwd != "" {
user = url.UserPassword(usr, pwd)
}
return &url.URL{
Scheme: scheme,
User: user,
Host: host,
}
}
+2
View File
@@ -62,3 +62,5 @@ macos/Flutter/GeneratedPluginRegistrant.swift
windows/flutter/generated_plugin_registrant.cc
windows/flutter/generated_plugin_registrant.h
windows/flutter/generated_plugins.cmake
/extensions
+1 -1
View File
@@ -47,7 +47,7 @@ android {
applicationId "com.gopeed"
// You can update the following values to match your application needs.
// For more information, see: https://docs.flutter.dev/deployment/android#reviewing-the-build-configuration.
minSdkVersion 19
minSdkVersion 21
targetSdkVersion 29
versionCode flutterVersionCode.toInteger()
versionName flutterVersionName
Loaded 100 of 184 files, more files were not shown because too many files have changed in this diff. Show more